diff options
Diffstat (limited to 'erts/emulator')
117 files changed, 18669 insertions, 10644 deletions
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 9c79bf3da0..5dfa60ee74 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2017. All Rights Reserved. +# Copyright Ericsson AB 1996-2018. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -645,7 +645,7 @@ PRELOAD_BEAM = $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ $(ERL_TOP)/erts/preloaded/ebin/erts_internal.beam \ $(ERL_TOP)/erts/preloaded/ebin/erl_tracer.beam \ $(ERL_TOP)/erts/preloaded/ebin/erts_literal_area_collector.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erts_dirty_process_code_checker.beam + $(ERL_TOP)/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam ifeq ($(TARGET),win32) # On windows the preloaded objects are in a resource object. @@ -832,7 +832,7 @@ RUN_OBJS += \ $(OBJDIR)/erl_alloc.o $(OBJDIR)/erl_mtrace.o \ $(OBJDIR)/erl_alloc_util.o $(OBJDIR)/erl_goodfit_alloc.o \ $(OBJDIR)/erl_bestfit_alloc.o $(OBJDIR)/erl_afit_alloc.o \ - $(OBJDIR)/erl_instrument.o $(OBJDIR)/erl_init.o \ + $(OBJDIR)/erl_init.o \ $(OBJDIR)/erl_atom_table.o $(OBJDIR)/erl_bif_table.o \ $(OBJDIR)/erl_bif_ddll.o $(OBJDIR)/erl_bif_guard.o \ $(OBJDIR)/erl_bif_info.o $(OBJDIR)/erl_bif_op.o \ @@ -844,7 +844,7 @@ RUN_OBJS += \ $(OBJDIR)/utils.o $(OBJDIR)/bif.o \ $(OBJDIR)/io.o $(OBJDIR)/erl_printf_term.o\ $(OBJDIR)/erl_debug.o $(OBJDIR)/erl_md5.o \ - $(OBJDIR)/erl_message.o \ + $(OBJDIR)/erl_message.o $(OBJDIR)/erl_proc_sig_queue.o \ $(OBJDIR)/erl_process_dict.o $(OBJDIR)/erl_process_lock.o \ $(OBJDIR)/erl_port_task.o $(OBJDIR)/erl_arith.o \ $(OBJDIR)/time.o $(OBJDIR)/erl_time_sup.o \ @@ -858,11 +858,11 @@ RUN_OBJS += \ $(OBJDIR)/register.o $(OBJDIR)/break.o \ $(OBJDIR)/erl_async.o $(OBJDIR)/erl_lock_check.o \ $(OBJDIR)/erl_gc.o $(OBJDIR)/erl_lock_count.o \ - $(OBJDIR)/erl_posix_str.o \ + $(OBJDIR)/erl_posix_str.o \ $(OBJDIR)/erl_bits.o $(OBJDIR)/erl_math.o \ $(OBJDIR)/erl_fun.o $(OBJDIR)/erl_bif_port.o \ $(OBJDIR)/erl_term.o $(OBJDIR)/erl_node_tables.o \ - $(OBJDIR)/erl_monitors.o $(OBJDIR)/erl_process_dump.o \ + $(OBJDIR)/erl_monitor_link.o $(OBJDIR)/erl_process_dump.o \ $(OBJDIR)/erl_hl_timer.o $(OBJDIR)/erl_cpu_topology.o \ $(OBJDIR)/erl_drv_thread.o $(OBJDIR)/erl_bif_chksum.o \ $(OBJDIR)/erl_bif_re.o $(OBJDIR)/erl_unicode.o \ diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 38b5f0c5e3..fba0611042 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2017. All Rights Reserved. +# Copyright Ericsson AB 1996-2018. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -106,6 +106,7 @@ atom atom atom atom_used atom attributes atom auto_connect +atom await_exit atom await_microstate_accounting_modifications atom await_port_send_result atom await_proc_exit @@ -214,7 +215,6 @@ atom dist_data atom Div='/' atom div atom dmonitor_node -atom dmonitor_p atom DollarDollar='$$' atom DollarUnderscore='$_' atom dollar_endonly @@ -358,6 +358,7 @@ atom loaded atom load_cancelled atom load_failure atom local +atom logger atom long_gc atom long_schedule atom low @@ -673,3 +674,4 @@ atom xor atom x86 atom yes atom yield +atom nifs diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 87367b44ab..d9312f4df8 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2017. All Rights Reserved. + * Copyright Ericsson AB 1999-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,6 +37,7 @@ #include "erl_bits.h" #include "erl_thr_progress.h" #include "erl_nfunc_sched.h" +#include "erl_proc_sig_queue.h" #ifdef HIPE # include "hipe_bif0.h" # define IF_HIPE(X) (X) @@ -65,7 +66,6 @@ static struct { Process *erts_code_purger = NULL; -Process *erts_dirty_process_code_checker; erts_atomic_t erts_copy_literal_area__; #define ERTS_SET_COPY_LITERAL_AREA(LA) \ erts_atomic_set_nob(&erts_copy_literal_area__, \ @@ -607,7 +607,9 @@ BIF_RETTYPE erts_internal_check_dirty_process_code_2(BIF_ALIST_2) int reds = 0; Eterm res; - if (BIF_P != erts_dirty_process_code_checker) + if (BIF_P != erts_dirty_process_signal_handler + && BIF_P != erts_dirty_process_signal_handler_high + && BIF_P != erts_dirty_process_signal_handler_max) BIF_ERROR(BIF_P, EXC_NOTSUP); if (is_not_internal_pid(BIF_ARG_1)) @@ -901,15 +903,59 @@ static void hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, Eterm *start, Eterm *end, char *lit_start, Uint lit_size); +static ERTS_INLINE void +msg_copy_literal_area(ErtsMessage *msgp, int *redsp, + char *literals, Uint lit_bsize) +{ + ErlHeapFragment *hfrag, *hf; + Uint lit_sz = 0; + + *redsp += 1; + + if (!ERTS_SIG_IS_INTERNAL_MSG(msgp) || !msgp->data.attached) + return; + + if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) + hfrag = &msgp->hfrag; + else + hfrag = msgp->data.heap_frag; + + for (hf = hfrag; hf; hf = hf->next) { + lit_sz += hfrag_literal_size(&hf->mem[0], + &hf->mem[hf->used_size], + literals, lit_bsize); + *redsp += 1; + } + + *redsp += lit_sz / 16; /* Better value needed... */ + if (lit_sz > 0) { + ErlHeapFragment *bp = new_message_buffer(lit_sz); + Eterm *hp = bp->mem; + + for (hf = hfrag; hf; hf = hf->next) { + hfrag_literal_copy(&hp, &bp->off_heap, + &hf->mem[0], + &hf->mem[hf->used_size], + literals, lit_bsize); + hfrag = hf; + } + + /* link new hfrag last */ + ASSERT(hfrag->next == NULL); + hfrag->next = bp; + bp->next = NULL; + } +} + Eterm erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed) { ErtsLiteralArea *la; - ErtsMessage *msgp; struct erl_off_heap_header* oh; char *literals; Uint lit_bsize; ErlHeapFragment *hfrag; + ErtsMessage *mfp; la = ERTS_COPY_LITERAL_AREA(); if (!la) @@ -927,46 +973,13 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed */ erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ); - ERTS_MSGQ_MV_INQ2PRIVQ(c_p); + erts_proc_sig_fetch(c_p); erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); - for (msgp = c_p->msg.first; msgp; msgp = msgp->next) { - ErlHeapFragment *hf; - Uint lit_sz = 0; - - *redsp += 1; - - if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) - hfrag = &msgp->hfrag; - else if (is_value(ERL_MESSAGE_TERM(msgp)) && msgp->data.heap_frag) - hfrag = msgp->data.heap_frag; - else - continue; /* Content on heap or in external term format... */ - - for (hf = hfrag; hf; hf = hf->next) { - lit_sz += hfrag_literal_size(&hf->mem[0], &hf->mem[hf->used_size], - literals, lit_bsize); - *redsp += 1; - } - - *redsp += lit_sz / 16; /* Better value needed... */ - if (lit_sz > 0) { - ErlHeapFragment *bp = new_message_buffer(lit_sz); - Eterm *hp = bp->mem; - - for (hf = hfrag; hf; hf = hf->next) { - hfrag_literal_copy(&hp, &bp->off_heap, - &hf->mem[0], &hf->mem[hf->used_size], - literals, lit_bsize); - hfrag = hf; - } - - /* link new hfrag last */ - ASSERT(hfrag->next == NULL); - hfrag->next = bp; - bp->next = NULL; - } - } + ERTS_FOREACH_SIG_PRIVQS(c_p, msgp, msg_copy_literal_area(msgp, + redsp, + literals, + lit_bsize)); if (gc_allowed) { /* @@ -1041,8 +1054,8 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed * process off heap structure. * - Check for literals */ - for (msgp = c_p->msg_frag; msgp; msgp = msgp->next) { - hfrag = erts_message_to_heap_frag(msgp); + for (mfp = c_p->msg_frag; mfp; mfp = mfp->next) { + hfrag = erts_message_to_heap_frag(mfp); for (; hfrag; hfrag = hfrag->next) { Eterm *hp, *hp_end; @@ -1744,11 +1757,11 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) release_literal_areas.last = ref; } erts_mtx_unlock(&release_literal_areas.mtx); - erts_queue_message(erts_literal_area_collector, + erts_queue_proc_message(BIF_P, + erts_literal_area_collector, 0, erts_alloc_message(0, NULL), - am_copy_literals, - BIF_P->common.id); + am_copy_literals); } return ret; diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 5eb68b817e..b8a8d06315 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -1191,7 +1191,7 @@ dirty_send_message(Process *c_p, Eterm to, Eterm tag) mp = erts_alloc_message_heap(rp, &rp_locks, 3, &hp, &ohp); msg = TUPLE2(hp, tag, c_p->common.id); - erts_queue_message(rp, rp_locks, mp, msg, c_p->common.id); + erts_queue_proc_message(c_p, rp, rp_locks, mp, msg); if (rp == real_c_p) rp_locks &= ~c_p_locks; diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index fbd0e38735..ee287243a4 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2017. All Rights Reserved. + * Copyright Ericsson AB 1996-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,6 +44,7 @@ #include "hipe_bif1.h" #endif #include "dtrace-wrapper.h" +#include "erl_proc_sig_queue.h" /* #define HARDDEBUG 1 */ @@ -1454,7 +1455,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, ErtsCodeMFA *bif_mfa) reg[3] = c_p->ftrace; if ((new_pc = next_catch(c_p, reg))) { c_p->cp = 0; /* To avoid keeping stale references. */ - c_p->msg.saved_last = 0; /* No longer safe to use this position */ + ERTS_RECV_MARK_CLEAR(c_p); /* No longer safe to use this position */ return new_pc; } if (c_p->catches > 0) erts_exit(ERTS_ERROR_EXIT, "Catch not found"); @@ -2419,8 +2420,7 @@ erts_hibernate(Process* c_p, Eterm* reg) * shrink the heap. */ erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); - ERTS_MSGQ_MV_INQ2PRIVQ(c_p); - if (!c_p->msg.len) { + if (!erts_proc_sig_fetch(c_p)) { erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); c_p->fvalue = NIL; PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2428,8 +2428,7 @@ erts_hibernate(Process* c_p, Eterm* reg) ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); - ERTS_MSGQ_MV_INQ2PRIVQ(c_p); - if (!c_p->msg.len) + if (!erts_proc_sig_fetch(c_p)) erts_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 0184c567f1..e61199a8fd 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -554,6 +554,7 @@ static Eterm get_module_info(Process* p, ErtsCodeIndex code_ix, static Eterm exported_from_module(Process* p, ErtsCodeIndex code_ix, Eterm mod); static Eterm functions_in_module(Process* p, BeamCodeHeader*); +static Eterm nifs_in_module(Process* p, Eterm module); static Eterm attributes_for_module(Process* p, BeamCodeHeader*); static Eterm compilation_info_for_module(Process* p, BeamCodeHeader*); static Eterm md5_of_module(Process* p, BeamCodeHeader*); @@ -4521,6 +4522,19 @@ is_empty_map(LoaderState* stp, GenOpArg Lit) } /* + * Predicate to test whether the given literal is an export. + */ +static int +literal_is_export(LoaderState* stp, GenOpArg Lit) +{ + Eterm term; + + ASSERT(Lit.type == TAG_q); + term = stp->literals[Lit.val].term; + return is_export(term); +} + +/* * Pseudo predicate map_key_sort that will sort the Rest operand for * map instructions as a side effect. */ @@ -5954,6 +5968,8 @@ get_module_info(Process* p, ErtsCodeIndex code_ix, BeamCodeHeader* code_hdr, return exported_from_module(p, code_ix, module); } else if (what == am_functions) { return functions_in_module(p, code_hdr); + } else if (what == am_nifs) { + return nifs_in_module(p, module); } else if (what == am_attributes) { return attributes_for_module(p, code_hdr); } else if (what == am_compile) { @@ -6007,6 +6023,46 @@ functions_in_module(Process* p, /* Process whose heap to use. */ } /* + * Builds a list of all NIFs in the given module: + * [{Name, Arity},...] + */ +Eterm +nifs_in_module(Process* p, Eterm module) +{ + Eterm nif_list, *hp; + Module *mod; + + mod = erts_get_module(module, erts_active_code_ix()); + nif_list = NIL; + + if (mod->curr.nif != NULL) { + int func_count, func_ix; + ErlNifFunc *funcs; + + func_count = erts_nif_get_funcs(mod->curr.nif, &funcs); + hp = HAlloc(p, func_count * 5); + + for (func_ix = func_count - 1; func_ix >= 0; func_ix--) { + Eterm name, arity, pair; + ErlNifFunc *func; + + func = &funcs[func_ix]; + + name = am_atom_put(func->name, sys_strlen(func->name)); + arity = make_small(func->arity); + + pair = TUPLE2(hp, name, arity); + hp += 3; + + nif_list = CONS(hp, pair, nif_list); + hp += 2; + } + } + + return nif_list; +} + +/* * Returns 'true' if mod has any native compiled functions, otherwise 'false' */ diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 652b95105f..79244b8544 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2017. All Rights Reserved. + * Copyright Ericsson AB 1996-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,11 +46,12 @@ #include "erl_bif_unique.h" #include "erl_map.h" #include "erl_msacc.h" +#include "erl_proc_sig_queue.h" Export *erts_await_result; +static Export await_exit_trap; static Export* flush_monitor_messages_trap = NULL; static Export* set_cpu_topology_trap = NULL; -static Export* await_proc_exit_trap = NULL; static Export* await_port_send_result_trap = NULL; Export* erts_format_cpu_topology_trap = NULL; static Export dsend_continue_trap_export; @@ -92,83 +93,51 @@ BIF_RETTYPE spawn_3(BIF_ALIST_3) /* Utility to add a new link between processes p and another internal * process (rpid). Process p must be the currently executing process. */ -static int insert_internal_link(Process* p, Eterm rpid) -{ - Process *rp; - ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK; - - ASSERT(is_internal_pid(rpid)); - - if (IS_TRACED(p) - && (ERTS_TRACE_FLAGS(p) & (F_TRACE_SOL|F_TRACE_SOL1))) { - rp_locks = ERTS_PROC_LOCKS_ALL; - } - - erts_proc_lock(p, ERTS_PROC_LOCK_LINK); - - /* get a pointer to the process struct of the linked process */ - rp = erts_pid2proc_opt(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, - rpid, rp_locks, - ERTS_P2P_FLG_ALLOW_OTHER_X); - - if (!rp) { - erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); - return 0; - } - - if (p != rp) { - erts_add_link(&ERTS_P_LINKS(p), LINK_PID, rp->common.id); - erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, p->common.id); - - ASSERT(IS_TRACER_VALID(ERTS_TRACER(p))); - - if (IS_TRACED(p)) { - if (ERTS_TRACE_FLAGS(p) & (F_TRACE_SOL|F_TRACE_SOL1)) { - ERTS_TRACE_FLAGS(rp) |= (ERTS_TRACE_FLAGS(p) & TRACEE_FLAGS); - erts_tracer_replace(&rp->common, ERTS_TRACER(p)); - if (ERTS_TRACE_FLAGS(p) & F_TRACE_SOL1) { /* maybe override */ - ERTS_TRACE_FLAGS(rp) &= ~(F_TRACE_SOL1 | F_TRACE_SOL); - ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOL1 | F_TRACE_SOL); - } - } - } - } - if (IS_TRACED_FL(rp, F_TRACE_PROCS)) - trace_proc(p, p == rp ? rp_locks : ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, - rp, am_getting_linked, p->common.id); - - if (p == rp) - erts_proc_unlock(p, rp_locks & ~ERTS_PROC_LOCK_MAIN); - else { - erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); - erts_proc_unlock(rp, rp_locks); - } - - return 1; -} - /* create a link to the process */ BIF_RETTYPE link_1(BIF_ALIST_1) { - DistEntry *dep; - if (IS_TRACED_FL(BIF_P, F_TRACE_PROCS)) { trace_proc(BIF_P, ERTS_PROC_LOCK_MAIN, BIF_P, am_link, BIF_ARG_1); } /* check that the pid or port which is our argument is OK */ if (is_internal_pid(BIF_ARG_1)) { - if (insert_internal_link(BIF_P, BIF_ARG_1)) { - BIF_RET(am_true); - } - else { - goto res_no_proc; - } + int created; + ErtsLinkData *ldp; + ErtsLink *lnk; + + if (BIF_P->common.id == BIF_ARG_1) + BIF_RET(am_true); + + if (!erts_proc_lookup(BIF_ARG_1)) + goto res_no_proc; + + lnk = erts_link_tree_lookup_create(&ERTS_P_LINKS(BIF_P), + &created, + ERTS_LNK_TYPE_PROC, + BIF_P->common.id, + BIF_ARG_1); + if (!created) + BIF_RET(am_true); + + ldp = erts_link_to_data(lnk); + + + if (erts_proc_sig_send_link(BIF_P, BIF_ARG_1, &ldp->b)) + BIF_RET(am_true); + + erts_link_tree_delete(&ERTS_P_LINKS(BIF_P), lnk); + erts_link_release_both(ldp); + goto res_no_proc; } if (is_internal_port(BIF_ARG_1)) { - int send_link_signal = 0; + int created; + ErtsLinkData *ldp; + ErtsLink *lnk; + Eterm ref; + Eterm *refp; Port *prt = erts_port_lookup(BIF_ARG_1, (erts_port_synchronous_ops ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP @@ -177,31 +146,31 @@ BIF_RETTYPE link_1(BIF_ALIST_1) goto res_no_proc; } - erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); - - if (erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, BIF_ARG_1) >= 0) - send_link_signal = 1; - /* else: already linked */ + lnk = erts_link_tree_lookup_create(&ERTS_P_LINKS(BIF_P), + &created, + ERTS_LNK_TYPE_PORT, + BIF_P->common.id, + BIF_ARG_1); + if (!created) + BIF_RET(am_true); - erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + ldp = erts_link_to_data(lnk); + refp = erts_port_synchronous_ops ? &ref : NULL; - if (send_link_signal) { - Eterm ref; - Eterm *refp = erts_port_synchronous_ops ? &ref : NULL; - - switch (erts_port_link(BIF_P, prt, BIF_P->common.id, refp)) { - case ERTS_PORT_OP_DROPPED: - case ERTS_PORT_OP_BADARG: - goto res_no_proc; - case ERTS_PORT_OP_SCHEDULED: - if (refp) { - ASSERT(is_internal_ordinary_ref(ref)); - BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true); - } - default: - break; - } - } + switch (erts_port_link(BIF_P, prt, &ldp->b, refp)) { + case ERTS_PORT_OP_DROPPED: + case ERTS_PORT_OP_BADARG: + erts_link_tree_delete(&ERTS_P_LINKS(BIF_P), lnk); + erts_link_release_both(ldp); + goto res_no_proc; + case ERTS_PORT_OP_SCHEDULED: + if (refp) { + ASSERT(is_internal_ordinary_ref(ref)); + BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true); + } + default: + break; + } BIF_RET(am_true); } else if (is_external_port(BIF_ARG_1) @@ -210,317 +179,203 @@ BIF_RETTYPE link_1(BIF_ALIST_1) } if (is_external_pid(BIF_ARG_1)) { + ErtsLinkData *ldp; + int created; + DistEntry *dep; + ErtsLink *lnk; + int code; + ErtsDSigData dsd; + + dep = external_pid_dist_entry(BIF_ARG_1); + if (dep == erts_this_dist_entry) + goto res_no_proc; - erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); - - /* We may earn time by checking first that we're not linked already */ - if (erts_lookup_link(ERTS_P_LINKS(BIF_P), BIF_ARG_1) != NULL) { - erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); - BIF_RET(am_true); - } - else { - ErtsLink *lnk; - int code; - ErtsDSigData dsd; - dep = external_pid_dist_entry(BIF_ARG_1); - if (dep == erts_this_dist_entry) { - erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); - goto res_no_proc; - } - - code = erts_dsig_prepare(&dsd, dep, BIF_P, - (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK), - ERTS_DSP_RLOCK, 0, 1); - switch (code) { - case ERTS_DSIG_PREP_NOT_ALIVE: - case ERTS_DSIG_PREP_NOT_CONNECTED: { - ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK; - erts_aint32_t state; - erts_proc_lock(BIF_P, (ERTS_PROC_LOCKS_ALL & ~locks)); - locks = ERTS_PROC_LOCKS_ALL; - erts_send_exit_signal(BIF_P, BIF_ARG_1, BIF_P, &locks, - am_noconnection, NIL, NULL, 0); - erts_proc_unlock(BIF_P, locks & ERTS_PROC_LOCKS_ALL_MINOR); - - /* - * Copy-paste from old dist_exit_3, not sure if we really - * need erts_handle_pending_exit when exit_2 does not. - */ - state = erts_atomic32_read_acqb(&BIF_P->state); - if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) { - if (state & ERTS_PSFLG_PENDING_EXIT) - erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN); - ERTS_BIF_EXITED(BIF_P); - } - BIF_RET(am_true); - } - case ERTS_DSIG_PREP_PENDING: - case ERTS_DSIG_PREP_CONNECTED: - /* - * We have (pending) connection. - * Setup link and enqueue link signal. - */ - erts_de_links_lock(dep); - - erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, BIF_ARG_1); - lnk = erts_add_or_lookup_link(&(dep->nlinks), - LINK_PID, - BIF_P->common.id); - ASSERT(lnk != NULL); - erts_add_link(&ERTS_LINK_ROOT(lnk), LINK_PID, BIF_ARG_1); - - erts_de_links_unlock(dep); - erts_de_runlock(dep); - erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); - - code = erts_dsig_send_link(&dsd, BIF_P->common.id, BIF_ARG_1); - if (code == ERTS_DSIG_SEND_YIELD) - ERTS_BIF_YIELD_RETURN(BIF_P, am_true); - BIF_RET(am_true); - default: - ERTS_ASSERT(! "Invalid dsig prepare result"); - } - } + lnk = erts_link_tree_lookup_create(&ERTS_P_LINKS(BIF_P), + &created, + ERTS_LNK_TYPE_DIST_PROC, + BIF_P->common.id, + BIF_ARG_1); + + if (!created) + BIF_RET(am_true); /* Already present... */ + + ldp = erts_link_to_data(lnk); + + code = erts_dsig_prepare(&dsd, dep, BIF_P, + ERTS_PROC_LOCK_MAIN, + ERTS_DSP_RLOCK, 0, 1); + switch (code) { + case ERTS_DSIG_PREP_NOT_ALIVE: + case ERTS_DSIG_PREP_NOT_CONNECTED: + erts_link_set_dead_dist(&ldp->b, dep->sysname); + erts_proc_sig_send_link_exit(NULL, BIF_ARG_1, &ldp->b, + am_noconnection, NIL); + BIF_RET(am_true); + + case ERTS_DSIG_PREP_PENDING: + case ERTS_DSIG_PREP_CONNECTED: { + /* + * We have (pending) connection. + * Setup link and enqueue link signal. + */ +#ifdef DEBUG + int inserted = +#endif + erts_link_dist_insert(&ldp->b, dep->mld); + ASSERT(inserted); + erts_de_runlock(dep); + + code = erts_dsig_send_link(&dsd, BIF_P->common.id, BIF_ARG_1); + if (code == ERTS_DSIG_SEND_YIELD) + ERTS_BIF_YIELD_RETURN(BIF_P, am_true); + BIF_RET(am_true); + break; + } + default: + ERTS_ASSERT(! "Invalid dsig prepare result"); + } } BIF_ERROR(BIF_P, BADARG); -res_no_proc: { - erts_aint32_t state = erts_atomic32_read_nob(&BIF_P->state); - if (state & ERTS_PSFLG_TRAP_EXIT) { - ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN; - erts_deliver_exit_message(BIF_ARG_1, BIF_P, &locks, am_noproc, NIL); - erts_proc_unlock(BIF_P, ~ERTS_PROC_LOCK_MAIN & locks); - BIF_RET(am_true); - } - else - BIF_ERROR(BIF_P, EXC_NOPROC); +res_no_proc: + if (BIF_P->flags & F_TRAP_EXIT) { + ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN; + erts_deliver_exit_message(BIF_ARG_1, BIF_P, &locks, am_noproc, NIL); + erts_proc_unlock(BIF_P, ~ERTS_PROC_LOCK_MAIN & locks); + BIF_RET(am_true); + } + else { + /* + * This behaviour is *really* sad but link/1 has + * behaved like this for ages (and this behaviour is + * actually documented)... :'-( + * + * The proper behavior would have been to + * send calling process an exit signal.. + */ + BIF_ERROR(BIF_P, EXC_NOPROC); } } -/* This function is allowed to return range of values handled by demonitor/1-2 - * Namely: atoms true, false, yield, internal_error, badarg or THE_NON_VALUE - */ static Eterm -remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to) +demonitor(Process *c_p, Eterm ref, Eterm *multip) { - ErtsDSigData dsd; - ErtsMonitor *dmon; - ErtsMonitor *mon; - int code; - Eterm res = am_false; - - ERTS_LC_ASSERT((ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK) - == erts_proc_lc_my_proc_locks(c_p)); - - code = erts_dsig_prepare(&dsd, dep, c_p, ERTS_PROC_LOCK_MAIN, - ERTS_DSP_RLOCK, 0, 0); - switch (code) { - case ERTS_DSIG_PREP_NOT_ALIVE: - case ERTS_DSIG_PREP_NOT_CONNECTED: - /* - * In the smp case this is possible if the node goes - * down just before the call to demonitor. - */ - if (dep) { - erts_de_links_lock(dep); - dmon = erts_remove_monitor(&dep->monitors, ref); - erts_de_links_unlock(dep); - if (dmon) - erts_destroy_monitor(dmon); - } - mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref); - erts_proc_unlock(c_p, ERTS_PROC_LOCK_LINK); - - res = am_true; - break; - - case ERTS_DSIG_PREP_PENDING: - case ERTS_DSIG_PREP_CONNECTED: + ErtsMonitor *mon; /* The monitor entry to delete */ - erts_de_links_lock(dep); - mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref); - dmon = erts_remove_monitor(&dep->monitors, ref); - erts_de_links_unlock(dep); - erts_de_runlock(dep); - erts_proc_unlock(c_p, ERTS_PROC_LOCK_LINK); + *multip = am_false; - if (!dmon) { - /* - * This is possible when smp support is enabled. - * 'DOWN' message just arrived. - */ - res = am_true; - } - else { - /* - * Soft (no force) send, use ->data in dist slot - * monitor list since in case of monitor name - * the atom is stored there. Yield if necessary. - */ - code = erts_dsig_send_demonitor(&dsd, - c_p->common.id, - (mon->name != NIL - ? mon->name - : mon->u.pid), - ref, - 0); - res = (code == ERTS_DSIG_SEND_YIELD ? am_yield : am_true); - erts_destroy_monitor(dmon); - } - break; - default: - ERTS_ASSERT(! "Invalid dsig prepare result"); - } + if (is_not_internal_ref(ref)) { + if (is_external_ref(ref) + && (erts_this_dist_entry + == external_ref_dist_entry(ref))) { + return am_false; + } + return am_badarg; /* Not monitored by this monitor's ref */ + } + mon = erts_monitor_tree_lookup(ERTS_P_MONITORS(c_p), ref); + if (!mon) + return am_false; - /* - * We aren't allowed to destroy 'mon' until now, since 'to' - * may refer into 'mon' (external pid). - */ - ASSERT(mon); /* Since link lock wasn't released between - lookup and remove */ - erts_destroy_monitor(mon); + if (!erts_monitor_is_origin(mon)) + return am_badarg; - ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); - return res; -} + erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p), mon); -static ERTS_INLINE void -demonitor_local_process(Process *c_p, Eterm ref, Eterm to, Eterm *res) -{ - Process *rp = erts_pid2proc_opt(c_p, - ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, - to, - ERTS_PROC_LOCK_LINK, - ERTS_P2P_FLG_ALLOW_OTHER_X); - ErtsMonitor *mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref); - - if (!mon) - *res = am_false; - else - { - *res = am_true; - erts_destroy_monitor(mon); - } - if (rp) { - ErtsMonitor *rmon; - rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); - if (rp != c_p) - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - if (rmon != NULL) - erts_destroy_monitor(rmon); - } - else { - ERTS_ASSERT_IS_NOT_EXITING(c_p); - } -} - -static ERTS_INLINE BIF_RETTYPE -demonitor_local_port(Process *origin, Eterm ref, Eterm target) -{ - BIF_RETTYPE res = am_false; - Port *port = erts_port_lookup_raw(target); - - if (!port) { - BIF_ERROR(origin, BADARG); - } - erts_proc_unlock(origin, ERTS_PROC_LOCK_LINK); - - if (port) { - Eterm trap_ref; - switch (erts_port_demonitor(origin, ERTS_PORT_DEMONITOR_NORMAL, - port, ref, &trap_ref)) { - case ERTS_PORT_OP_DROPPED: - case ERTS_PORT_OP_BADARG: - break; - case ERTS_PORT_OP_SCHEDULED: - BIF_TRAP3(await_port_send_result_trap, origin, trap_ref, - am_busy_port, am_true); - /* the busy_port atom will never be returned, because it cannot be - * returned from erts_port_(de)monitor, but just in case if in future - * internal API changes - you may see this atom */ - default: - break; - } - } - else { - ERTS_ASSERT_IS_NOT_EXITING(origin); - } - BIF_RET(res); -} + switch (mon->type) { -/* Can return atom true, false, yield, internal_error, badarg or - * THE_NON_VALUE if error occurred or trap has been set up - */ -static -BIF_RETTYPE demonitor(Process *c_p, Eterm ref, Eterm *multip) -{ - ErtsMonitor *mon = NULL; /* The monitor entry to delete */ - Eterm to = NIL; /* Monitor link traget */ - DistEntry *dep = NULL; /* Target's distribution entry */ - BIF_RETTYPE res = am_false; - int unlock_link = 1; + case ERTS_MON_TYPE_TIME_OFFSET: + *multip = am_true; + erts_demonitor_time_offset(mon); + return am_true; + + case ERTS_MON_TYPE_PORT: { + Port *prt; + ASSERT(is_internal_port(mon->other.item)); + prt = erts_port_lookup(mon->other.item, ERTS_PORT_SFLGS_DEAD); + if (!prt || erts_port_demonitor(c_p, prt, mon) == ERTS_PORT_OP_DROPPED) + erts_monitor_release(mon); + return am_true; + } - erts_proc_lock(c_p, ERTS_PROC_LOCK_LINK); + case ERTS_MON_TYPE_PROC: + erts_proc_sig_send_demonitor(mon); + return am_true; - if (is_not_internal_ref(ref)) { - res = am_badarg; - goto done; /* Cannot be this monitor's ref */ - } + case ERTS_MON_TYPE_DIST_PROC: { + ErtsMonitorData *mdp = erts_monitor_to_data(mon); + Eterm to = mon->other.item; + DistEntry *dep; + int code = ERTS_DSIG_SEND_OK; + int deleted; + ErtsDSigData dsd; - mon = erts_lookup_monitor(ERTS_P_MONITORS(c_p), ref); - if (!mon) { - goto done; - } + ASSERT(is_external_pid(to) || is_node_name_atom(to)); - switch (mon->type) { - case MON_TIME_OFFSET: - *multip = am_true; - erts_demonitor_time_offset(ref); - res = am_true; - break; - case MON_ORIGIN: - to = mon->u.pid; - *multip = am_false; - if (is_atom(to)) { + if (is_external_pid(to)) + dep = external_pid_dist_entry(to); + else { /* Monitoring a name at node to */ - ASSERT(is_node_name_atom(to)); dep = erts_sysname_to_connected_dist_entry(to); ASSERT(dep != erts_this_dist_entry); - } else if (is_port(to)) { - if (port_dist_entry(to) != erts_this_dist_entry) { - goto badarg; + if (!dep) { + erts_monitor_release(mon); + return am_false; } - res = demonitor_local_port(c_p, ref, to); - unlock_link = 0; - goto done; - } else { - ASSERT(is_pid(to)); - dep = pid_dist_entry(to); } - if (dep != erts_this_dist_entry) { - res = remote_demonitor(c_p, dep, ref, to); - /* remote_demonitor() unlocks link lock on c_p */ - unlock_link = 0; + + code = erts_dsig_prepare(&dsd, dep, c_p, ERTS_PROC_LOCK_MAIN, + ERTS_DSP_RLOCK, 0, 0); + + deleted = erts_monitor_dist_delete(&mdp->target); + + switch (code) { + case ERTS_DSIG_PREP_NOT_ALIVE: + case ERTS_DSIG_PREP_NOT_CONNECTED: + /* + * In the smp case this is possible if the node goes + * down just before the call to demonitor. + */ + break; + + case ERTS_DSIG_PREP_PENDING: + case ERTS_DSIG_PREP_CONNECTED: { + Eterm watched; + + erts_de_runlock(dep); + + if (mon->flags & ERTS_ML_FLG_NAME) + watched = ((ErtsMonitorDataExtended *) mdp)->u.name; + else + watched = to; + + /* + * Soft (no force) send, use ->data in dist slot + * monitor list since in case of monitor name + * the atom is stored there. Yield if necessary. + */ + code = erts_dsig_send_demonitor(&dsd, c_p->common.id, + watched, mdp->ref, 0); + break; } - else { /* Local monitor */ - demonitor_local_process(c_p, ref, to, &res); + + default: + ERTS_INTERNAL_ERROR("invalid result from erts_dsig_prepare()"); + break; } - break; - default /* case */ : -badarg: - res = am_badarg; /* will be converted to error by caller */ - *multip = am_false; - break; - } -done: - if (unlock_link) - erts_proc_unlock(c_p, ERTS_PROC_LOCK_LINK); + if (deleted) + erts_monitor_release(&mdp->target); + + erts_monitor_release(mon); + return code == ERTS_DSIG_SEND_YIELD ? am_yield : am_true; + } - ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); - BIF_RET(res); + default: + ERTS_INTERNAL_ERROR("Unexpected monitor type"); + return am_false; + } } BIF_RETTYPE demonitor_1(BIF_ALIST_1) @@ -528,25 +383,23 @@ BIF_RETTYPE demonitor_1(BIF_ALIST_1) Eterm multi; switch (demonitor(BIF_P, BIF_ARG_1, &multi)) { case am_false: - case am_true: BIF_RET(am_true); - case THE_NON_VALUE: BIF_RET(THE_NON_VALUE); - case am_yield: ERTS_BIF_YIELD_RETURN(BIF_P, am_true); - case am_badarg: BIF_ERROR(BIF_P, BADARG); - - case am_internal_error: + case am_true: + BIF_RET(am_true); + case am_yield: + ERTS_BIF_YIELD_RETURN(BIF_P, am_true); + case am_badarg: default: - ASSERT(! "demonitor(): internal error"); - BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); + BIF_ERROR(BIF_P, BADARG); } } BIF_RETTYPE demonitor_2(BIF_ALIST_2) { - BIF_RETTYPE res = am_true; - Eterm multi = am_false; - int info = 0; - int flush = 0; - Eterm list = BIF_ARG_2; + BIF_RETTYPE res; + Eterm multi = am_false; + int info = 0; + int flush = 0; + Eterm list = BIF_ARG_2; while (is_list(list)) { Eterm* consp = list_val(list); @@ -566,10 +419,9 @@ BIF_RETTYPE demonitor_2(BIF_ALIST_2) if (is_not_nil(list)) goto badarg; + res = am_true; switch (demonitor(BIF_P, BIF_ARG_1, &multi)) { - case THE_NON_VALUE: - /* If other error occurred or trap has been set up - pass through */ - BIF_RET(THE_NON_VALUE); + case am_false: if (info) res = am_false; @@ -578,20 +430,29 @@ flush_messages: BIF_TRAP3(flush_monitor_messages_trap, BIF_P, BIF_ARG_1, multi, res); } + /* Fall through... */ + case am_true: if (multi == am_true && flush) goto flush_messages; BIF_RET(res); + case am_yield: - ERTS_BIF_YIELD_RETURN(BIF_P, am_true); + /* return true after yield... */ + if (flush) { + ERTS_VBUMP_ALL_REDS(BIF_P); + goto flush_messages; + } + ERTS_BIF_YIELD_RETURN(BIF_P, am_true); + case am_badarg: -badarg: - BIF_ERROR(BIF_P, BADARG); - case am_internal_error: default: - ASSERT(! "demonitor(): internal error"); - BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); + break; + } + +badarg: + BIF_ERROR(BIF_P, BADARG); } /* Type must be atomic object! */ @@ -631,292 +492,212 @@ erts_queue_monitor_message(Process *p, erts_queue_message(p, *p_locksp, msgp, tup, am_system); } -static Eterm -local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int boolean) +BIF_RETTYPE monitor_2(BIF_ALIST_2) { - Eterm ret = mon_ref; - Process *rp; - ErtsProcLocks p_locks = ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK; - - if (target == p->common.id) { - return ret; - } + Eterm target = BIF_ARG_2; + Eterm tmp_heap[3]; + Eterm ref, id, name; + ErtsMonitorData *mdp; + + if (BIF_ARG_1 == am_process) { + DistEntry *dep; + int byname; + + if (is_internal_pid(target)) { + name = NIL; + id = target; + + local_process: + + ref = erts_make_ref(BIF_P); + if (id != BIF_P->common.id) { + mdp = erts_monitor_create(ERTS_MON_TYPE_PROC, + ref, BIF_P->common.id, + id, name); + erts_monitor_tree_insert(&ERTS_P_MONITORS(BIF_P), + &mdp->origin); + + if (!erts_proc_sig_send_monitor(&mdp->target, id)) + erts_proc_sig_send_monitor_down(&mdp->target, + am_noproc); + } + BIF_RET(ref); + } - erts_proc_lock(p, ERTS_PROC_LOCK_LINK); - rp = erts_pid2proc_opt(p, p_locks, - target, ERTS_PROC_LOCK_LINK, - ERTS_P2P_FLG_ALLOW_OTHER_X); - if (!rp) { - erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); - p_locks &= ~ERTS_PROC_LOCK_LINK; - if (boolean) - ret = am_false; - else - erts_queue_monitor_message(p, &p_locks, - mon_ref, am_process, target, am_noproc); - } - else { - ASSERT(rp != p); + if (is_atom(target)) { + local_named_process: + name = target; + id = erts_whereis_name_to_id(BIF_P, target); + if (is_internal_pid(id)) + goto local_process; + target = TUPLE2(&tmp_heap[0], name, + erts_this_dist_entry->sysname); + goto noproc; + } - if (boolean) - ret = am_true; + if (is_external_pid(target)) { + ErtsDSigData dsd; + int code; + + dep = external_pid_dist_entry(target); + if (dep == erts_this_dist_entry) + goto noproc; + + id = target; + name = NIL; + byname = 0; + + remote_process: + + ref = erts_make_ref(BIF_P); + mdp = erts_monitor_create(ERTS_MON_TYPE_DIST_PROC, ref, + BIF_P->common.id, id, name); + erts_monitor_tree_insert(&ERTS_P_MONITORS(BIF_P), &mdp->origin); + + code = erts_dsig_prepare(&dsd, dep, + BIF_P, ERTS_PROC_LOCK_MAIN, + ERTS_DSP_RLOCK, 0, 1); + switch (code) { + case ERTS_DSIG_PREP_NOT_ALIVE: + case ERTS_DSIG_PREP_NOT_CONNECTED: + erts_monitor_set_dead_dist(&mdp->target, dep->sysname); + erts_proc_sig_send_monitor_down(&mdp->target, am_noconnection); + code = ERTS_DSIG_SEND_OK; + break; - erts_add_monitor(&ERTS_P_MONITORS(p), MON_ORIGIN, mon_ref, target, NIL); - erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, mon_ref, p->common.id, NIL); + case ERTS_DSIG_PREP_PENDING: + case ERTS_DSIG_PREP_CONNECTED: { +#ifdef DEBUG + int inserted = +#endif - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - } + erts_monitor_dist_insert(&mdp->target, dep->mld); + ASSERT(inserted); + erts_de_runlock(dep); - erts_proc_unlock(p, p_locks & ~ERTS_PROC_LOCK_MAIN); + code = erts_dsig_send_monitor(&dsd, BIF_P->common.id, target, ref); + break; + } - return ret; -} + default: + ERTS_ASSERT(! "Invalid dsig prepare result"); + code = ERTS_DSIG_SEND_OK; + break; + } -static BIF_RETTYPE -local_port_monitor(Process *origin, Eterm target) -{ - BIF_RETTYPE ref = erts_make_ref(origin); - Port *port = erts_sig_lookup_port(origin, target); - ErtsProcLocks p_locks = ERTS_PROC_LOCK_MAIN; + if (byname) + erts_deref_dist_entry(dep); - if (!port) { -res_no_proc: - /* Send the DOWN message immediately. Ref is made on the fly because - * caller has never seen it yet. */ - erts_queue_monitor_message(origin, &p_locks, ref, - am_port, target, am_noproc); - } - else { - switch (erts_port_monitor(origin, port, target, &ref)) { - case ERTS_PORT_OP_DROPPED: - case ERTS_PORT_OP_BADARG: - goto res_no_proc; - case ERTS_PORT_OP_SCHEDULED: - BIF_TRAP3(await_port_send_result_trap, origin, ref, - am_busy_port, ref); - /* the busy_port atom will never be returned, because it cannot be - * returned from erts_port_monitor, but just in case if in future - * internal API changes - you may see this atom */ - default: - break; + if (code == ERTS_DSIG_SEND_YIELD) + ERTS_BIF_YIELD_RETURN(BIF_P, ref); + BIF_RET(ref); } - } - erts_proc_unlock(origin, p_locks & ~ERTS_PROC_LOCK_MAIN); - BIF_RET(ref); -} -/* Type = process | port :: atom(), 1st argument passed to erlang:monitor/2 - */ -static BIF_RETTYPE -local_name_monitor(Process *self, Eterm type, Eterm target_name) -{ - BIF_RETTYPE ret = erts_make_ref(self); + if (is_tuple(target)) { + Eterm *tpl = tuple_val(target); + if (arityval(tpl[0]) != 2) + goto badarg; + if (is_not_atom(tpl[1]) || is_not_atom(tpl[2])) + goto badarg; + if (!erts_is_alive && tpl[2] != am_Noname) + goto badarg; + target = tpl[1]; + dep = erts_find_or_insert_dist_entry(tpl[2]); + if (dep == erts_this_dist_entry) { + erts_deref_dist_entry(dep); + goto local_named_process; + } - ErtsProcLocks p_locks = ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK; - Process *proc = NULL; - Port *port = NULL; + id = dep->sysname; + name = target; + byname = 1; + goto remote_process; + } - erts_proc_lock(self, ERTS_PROC_LOCK_LINK); + /* badarg... */ + } + else if (BIF_ARG_1 == am_port) { + + if (is_internal_port(target)) { + Port *prt; + name = NIL; + id = target; + local_port: + ref = erts_make_ref(BIF_P); + mdp = erts_monitor_create(ERTS_MON_TYPE_PORT, ref, + BIF_P->common.id, id, name); + erts_monitor_tree_insert(&ERTS_P_MONITORS(BIF_P), &mdp->origin); + prt = erts_port_lookup(id, ERTS_PORT_SFLGS_INVALID_LOOKUP); + if (!prt || erts_port_monitor(BIF_P, prt, &mdp->target) == ERTS_PORT_OP_DROPPED) + erts_proc_sig_send_monitor_down(&mdp->target, am_noproc); + BIF_RET(ref); + } - erts_whereis_name(self, p_locks, target_name, - &proc, ERTS_PROC_LOCK_LINK, - ERTS_P2P_FLG_ALLOW_OTHER_X, - &port, 0); + if (is_atom(target)) { + local_named_port: + name = target; + id = erts_whereis_name_to_id(BIF_P, target); + if (is_internal_port(id)) + goto local_port; + target = TUPLE2(&tmp_heap[0], name, + erts_this_dist_entry->sysname); + goto noproc; + } - /* If the name is not registered, - * or if we asked for proc and got a port, - * or if we asked for port and got a proc, - * we just send the 'DOWN' message. - */ - if ((!proc && !port) || - (type == am_process && port) || - (type == am_port && proc)) { - DeclareTmpHeap(lhp,3,self); - Eterm item; - UseTmpHeap(3,self); - - erts_proc_unlock(self, ERTS_PROC_LOCK_LINK); - p_locks &= ~ERTS_PROC_LOCK_LINK; - - item = TUPLE2(lhp, target_name, erts_this_dist_entry->sysname); - erts_queue_monitor_message(self, &p_locks, - ret, - type, /* = process|port :: atom() */ - item, am_noproc); - UnUseTmpHeap(3,self); - } - else if (port) { - erts_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN); - p_locks &= ~ERTS_PROC_LOCK_MAIN; - - switch (erts_port_monitor(self, port, target_name, &ret)) { - case ERTS_PORT_OP_DONE: - return ret; - case ERTS_PORT_OP_SCHEDULED: { /* Scheduled a signal */ - ASSERT(is_internal_ordinary_ref(ret)); - BIF_TRAP3(await_port_send_result_trap, self, - ret, am_true, ret); - /* bif_trap returns */ - } break; - default: + if (is_external_port(target)) { + if (erts_this_dist_entry == external_port_dist_entry(target)) + goto noproc; goto badarg; } - } - else if (proc != self) { - erts_add_monitor(&ERTS_P_MONITORS(self), MON_ORIGIN, ret, - proc->common.id, target_name); - erts_add_monitor(&ERTS_P_MONITORS(proc), MON_TARGET, ret, - self->common.id, target_name); - erts_proc_unlock(proc, ERTS_PROC_LOCK_LINK); - } - - if (p_locks) { - erts_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN); - } - BIF_RET(ret); -badarg: - if (p_locks) { - erts_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN); - } - BIF_ERROR(self, BADARG); -} - -static BIF_RETTYPE -remote_monitor(Process *p, Eterm bifarg1, Eterm bifarg2, - DistEntry *dep, Eterm target, int byname) -{ - ErtsDSigData dsd; - BIF_RETTYPE ret; - int code; - ASSERT(dep); - erts_proc_lock(p, ERTS_PROC_LOCK_LINK); - code = erts_dsig_prepare(&dsd, dep, - p, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK), - ERTS_DSP_RLOCK, 0, 1); - switch (code) { - case ERTS_DSIG_PREP_NOT_ALIVE: - case ERTS_DSIG_PREP_NOT_CONNECTED: - erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); - ERTS_BIF_PREP_TRAP2(ret, dmonitor_p_trap, p, bifarg1, bifarg2); - break; - case ERTS_DSIG_PREP_PENDING: - case ERTS_DSIG_PREP_CONNECTED: - { - Eterm p_trgt, p_name, d_name, mon_ref; + if (is_tuple(target)) { + Eterm *tpl = tuple_val(target); + if (arityval(tpl[0]) != 2) + goto badarg; + if (is_not_atom(tpl[1]) || is_not_atom(tpl[2])) + goto badarg; + if (tpl[2] == erts_this_dist_entry->sysname) { + target = tpl[1]; + goto local_named_port; + } + } - mon_ref = erts_make_ref(p); + /* badarg... */ + } + else if (BIF_ARG_1 == am_time_offset) { - if (byname) { - p_trgt = dep->sysname; - p_name = target; - d_name = target; - } - else { - p_trgt = target; - p_name = NIL; - d_name = NIL; - } + if (target != am_clock_service) + goto badarg; + ref = erts_make_ref(BIF_P); + mdp = erts_monitor_create(ERTS_MON_TYPE_TIME_OFFSET, + ref, BIF_P->common.id, + am_clock_service, NIL); + erts_monitor_tree_insert(&ERTS_P_MONITORS(BIF_P), &mdp->origin); - erts_de_links_lock(dep); + erts_monitor_time_offset(&mdp->target); - erts_add_monitor(&ERTS_P_MONITORS(p), MON_ORIGIN, mon_ref, p_trgt, - p_name); - erts_add_monitor(&(dep->monitors), MON_TARGET, mon_ref, p->common.id, - d_name); + BIF_RET(ref); + } - erts_de_links_unlock(dep); - erts_de_runlock(dep); - erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); +badarg: - code = erts_dsig_send_monitor(&dsd, p->common.id, target, mon_ref); - if (code == ERTS_DSIG_SEND_YIELD) - ERTS_BIF_PREP_YIELD_RETURN(ret, p, mon_ref); - else - ERTS_BIF_PREP_RET(ret, mon_ref); - } - break; - default: - ERTS_ASSERT(! "Invalid dsig prepare result"); - } + BIF_ERROR(BIF_P, BADARG); - BIF_RET(ret); -} - -BIF_RETTYPE monitor_2(BIF_ALIST_2) -{ - Eterm target = BIF_ARG_2; - BIF_RETTYPE ret; - DistEntry *dep = NULL; +noproc: { + ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN; - /* Only process monitors are implemented */ - switch (BIF_ARG_1) { - case am_time_offset: { - Eterm ref; - if (BIF_ARG_2 != am_clock_service) { - goto badarg; - } - ref = erts_make_ref(BIF_P); - erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); - erts_add_monitor(&ERTS_P_MONITORS(BIF_P), MON_TIME_OFFSET, - ref, am_clock_service, NIL); - erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); - erts_monitor_time_offset(BIF_P->common.id, ref); - BIF_RET(ref); - } - case am_process: - case am_port: - break; - default: - goto badarg; - } + ref = erts_make_ref(BIF_P); + erts_queue_monitor_message(BIF_P, + &locks, + ref, + BIF_ARG_1, + target, + am_noproc); + if (locks != ERTS_PROC_LOCK_MAIN) + erts_proc_unlock(BIF_P, locks & ~ERTS_PROC_LOCK_MAIN); - if (is_internal_pid(target) && BIF_ARG_1 == am_process) { -local_pid: - ret = local_pid_monitor(BIF_P, target, erts_make_ref(BIF_P), 0); - } else if (is_external_pid(target) && BIF_ARG_1 == am_process) { - dep = external_pid_dist_entry(target); - if (dep == erts_this_dist_entry) - goto local_pid; - ret = remote_monitor(BIF_P, BIF_ARG_1, BIF_ARG_2, dep, target, 0); - } else if (is_internal_port(target) && BIF_ARG_1 == am_port) { -local_port: - ret = local_port_monitor(BIF_P, target); - } else if (is_external_port(target) && BIF_ARG_1 == am_port) { - dep = external_port_dist_entry(target); - if (dep == erts_this_dist_entry) { - goto local_port; - } - goto badarg; /* No want remote port */ - } else if (is_atom(target)) { - ret = local_name_monitor(BIF_P, BIF_ARG_1, target); - } else if (is_tuple(target)) { - Eterm *tp = tuple_val(target); - Eterm remote_node; - Eterm name; - if (arityval(*tp) != 2) { - goto badarg; - } - remote_node = tp[2]; - name = tp[1]; - if (!is_atom(remote_node) || !is_atom(name)) { - goto badarg; - } - if (!erts_is_alive && remote_node != am_Noname) { - goto badarg; /* Remote monitor from (this) undistributed node */ - } - dep = erts_find_or_insert_dist_entry(remote_node); - if (dep == erts_this_dist_entry) { - ret = local_name_monitor(BIF_P, BIF_ARG_1, name); - } else { - ret = remote_monitor(BIF_P, BIF_ARG_1, BIF_ARG_2, dep, name, 1); - } - erts_deref_dist_entry(dep); - } else { -badarg: - ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + BIF_RET(ref); } - return ret; } /**********************************************************************/ @@ -1089,91 +870,74 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) /* remove a link from a process */ BIF_RETTYPE unlink_1(BIF_ALIST_1) { - Process *rp; - DistEntry *dep; - ErtsLink *l = NULL, *rl = NULL; - ErtsProcLocks cp_locks = ERTS_PROC_LOCK_MAIN; - - /* - * SMP specific note concerning incoming exit signals: - * We have to have at least the status lock during removal of - * the link half on current process, and check for and handle - * a present pending exit while the status lock is held. This - * in order to ensure that we wont be exited by a link after - * it has been removed. - * - * (We also have to have the link lock, of course, in order to - * be allowed to remove the link...) - */ - if (IS_TRACED_FL(BIF_P, F_TRACE_PROCS)) { - trace_proc(BIF_P, cp_locks, BIF_P, am_unlink, BIF_ARG_1); + trace_proc(BIF_P, ERTS_PROC_LOCK_MAIN, + BIF_P, am_unlink, BIF_ARG_1); } - if (is_internal_port(BIF_ARG_1)) { - erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); - if (ERTS_PROC_PENDING_EXIT(BIF_P)) - goto handle_pending_exit; - - l = erts_remove_link(&ERTS_P_LINKS(BIF_P), BIF_ARG_1); + if (is_internal_pid(BIF_ARG_1)) { + ErtsLink *lnk = erts_link_tree_lookup(ERTS_P_LINKS(BIF_P), BIF_ARG_1); + if (lnk) { + erts_link_tree_delete(&ERTS_P_LINKS(BIF_P), lnk); + erts_proc_sig_send_unlink(BIF_P, lnk); + } + BIF_RET(am_true); + } - erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); + if (is_internal_port(BIF_ARG_1)) { + ErtsLink *lnk = erts_link_tree_lookup(ERTS_P_LINKS(BIF_P), BIF_ARG_1); - if (l) { + if (lnk) { + Eterm ref; + Eterm *refp = erts_port_synchronous_ops ? &ref : NULL; + ErtsPortOpResult res = ERTS_PORT_OP_DROPPED; Port *prt; - erts_destroy_link(l); + erts_link_tree_delete(&ERTS_P_LINKS(BIF_P), lnk); /* Send unlink signal */ prt = erts_port_lookup(BIF_ARG_1, ERTS_PORT_SFLGS_DEAD); if (prt) { - ErtsPortOpResult res; - Eterm ref; - Eterm *refp = erts_port_synchronous_ops ? &ref : NULL; #ifdef DEBUG ref = NIL; #endif - res = erts_port_unlink(BIF_P, prt, BIF_P->common.id, refp); + res = erts_port_unlink(BIF_P, prt, lnk, refp); - if (refp && res == ERTS_PORT_OP_SCHEDULED) { - ASSERT(is_internal_ordinary_ref(ref)); - BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true); - } } + + if (res == ERTS_PORT_OP_DROPPED) + erts_link_release(lnk); + else if (refp && res == ERTS_PORT_OP_SCHEDULED) { + ASSERT(is_internal_ordinary_ref(ref)); + BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true); + } } BIF_RET(am_true); } - else if (is_external_port(BIF_ARG_1) - && external_port_dist_entry(BIF_ARG_1) == erts_this_dist_entry) { - BIF_RET(am_true); - } - - if (is_not_pid(BIF_ARG_1)) - BIF_ERROR(BIF_P, BADARG); if (is_external_pid(BIF_ARG_1)) { - ErtsDistLinkData dld; + ErtsLink *lnk, *dlnk; + ErtsLinkData *ldp; + DistEntry *dep; int code; ErtsDSigData dsd; - /* Blind removal, we might have trapped or anything, this leaves - us in a state where monitors might be inconsistent, but the dist - code should take care of it. */ - erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); - if (ERTS_PROC_PENDING_EXIT(BIF_P)) - goto handle_pending_exit; - l = erts_remove_link(&ERTS_P_LINKS(BIF_P), BIF_ARG_1); - - erts_proc_unlock(BIF_P, - ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); - - if (l) - erts_destroy_link(l); dep = external_pid_dist_entry(BIF_ARG_1); - if (dep == erts_this_dist_entry) { + if (dep == erts_this_dist_entry) BIF_RET(am_true); - } + + lnk = erts_link_tree_lookup(ERTS_P_LINKS(BIF_P), BIF_ARG_1); + if (!lnk) + BIF_RET(am_true); + + erts_link_tree_delete(&ERTS_P_LINKS(BIF_P), lnk); + dlnk = erts_link_to_other(lnk, &ldp); + + if (erts_link_dist_delete(dlnk)) + erts_link_release_both(ldp); + else + erts_link_release(lnk); code = erts_dsig_prepare(&dsd, dep, BIF_P, ERTS_PROC_LOCK_MAIN, ERTS_DSP_NO_LOCK, 0, 0); @@ -1181,74 +945,27 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) case ERTS_DSIG_PREP_NOT_ALIVE: case ERTS_DSIG_PREP_NOT_CONNECTED: BIF_RET(am_true); - case ERTS_DSIG_PREP_PENDING: case ERTS_DSIG_PREP_CONNECTED: - erts_remove_dist_link(&dld, BIF_P->common.id, BIF_ARG_1, dep); code = erts_dsig_send_unlink(&dsd, BIF_P->common.id, BIF_ARG_1); - erts_destroy_dist_link(&dld); if (code == ERTS_DSIG_SEND_YIELD) ERTS_BIF_YIELD_RETURN(BIF_P, am_true); - BIF_RET(am_true); - + break; default: ASSERT(! "Invalid dsig prepare result"); BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); } - } - - /* Internal pid... */ - - erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); - cp_locks |= ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS; - - /* get process struct */ - rp = erts_pid2proc_opt(BIF_P, cp_locks, - BIF_ARG_1, ERTS_PROC_LOCK_LINK, - ERTS_P2P_FLG_ALLOW_OTHER_X); - - if (ERTS_PROC_PENDING_EXIT(BIF_P)) { - if (rp && rp != BIF_P) - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - goto handle_pending_exit; - } - - /* unlink and ignore errors */ - l = erts_remove_link(&ERTS_P_LINKS(BIF_P), BIF_ARG_1); - if (l != NULL) - erts_destroy_link(l); - - if (!rp) { - ERTS_ASSERT_IS_NOT_EXITING(BIF_P); + BIF_RET(am_true); } - else { - rl = erts_remove_link(&ERTS_P_LINKS(rp), BIF_P->common.id); - if (rl != NULL) - erts_destroy_link(rl); - - if (IS_TRACED_FL(rp, F_TRACE_PROCS) && rl != NULL) { - erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS); - cp_locks &= ~ERTS_PROC_LOCK_STATUS; - trace_proc(BIF_P, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK), - rp, am_getting_unlinked, BIF_P->common.id); - } - if (rp != BIF_P) - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + if (is_external_port(BIF_ARG_1)) { + if (external_port_dist_entry(BIF_ARG_1) == erts_this_dist_entry) + BIF_RET(am_true); + /* Links to Remote ports not supported... */ } - - erts_proc_unlock(BIF_P, cp_locks & ~ERTS_PROC_LOCK_MAIN); - BIF_RET(am_true); - - handle_pending_exit: - erts_handle_pending_exit(BIF_P, (ERTS_PROC_LOCK_MAIN - | ERTS_PROC_LOCK_LINK - | ERTS_PROC_LOCK_STATUS)); - ASSERT(ERTS_PROC_IS_EXITING(BIF_P)); - erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); - ERTS_BIF_EXITED(BIF_P); + BIF_ERROR(BIF_P, BADARG); } BIF_RETTYPE hibernate_3(BIF_ALIST_3) @@ -1433,6 +1150,13 @@ BIF_RETTYPE raise_3(BIF_ALIST_3) /* Create stacktrace and store */ if (erts_backtrace_depth < depth) { depth = erts_backtrace_depth; + if (depth == 0) { + /* + * For consistency with stacktraces generated + * automatically, always include one element. + */ + depth = 1; + } must_copy = 1; } if (must_copy) { @@ -1493,22 +1217,69 @@ BIF_RETTYPE raise_3(BIF_ALIST_3) return am_badarg; } +static BIF_RETTYPE +erts_internal_await_exit_trap(BIF_ALIST_0) +{ + /* + * We have sent ourselves an exit signal which will + * terminate ourselves. Handle all signals until + * terminated in order to ensure that signal order + * is preserved. Yield if necessary. + */ + erts_aint32_t state; + int reds = ERTS_BIF_REDS_LEFT(BIF_P); + (void) erts_proc_sig_handle_incoming(BIF_P, &state, &reds, + reds, !0); + BUMP_REDS(BIF_P, reds); + if (state & ERTS_PSFLG_EXITING) + ERTS_BIF_EXITED(BIF_P); + + ERTS_BIF_YIELD0(&await_exit_trap, BIF_P); +} + /**********************************************************************/ -/* send an exit message to another process (if trapping exits) or - exit the other process */ +/* send an exit signal to another process */ -BIF_RETTYPE exit_2(BIF_ALIST_2) +static BIF_RETTYPE send_exit_signal_bif(Process *c_p, Eterm id, Eterm reason, int exit2) { - Process *rp; + BIF_RETTYPE ret_val; - /* - * If the first argument is not a pid, or a local port it is an error. - */ + /* + * 'id' not a process id, nor a local port id is a 'badarg' error. + */ - if (is_internal_port(BIF_ARG_1)) { + if (is_internal_pid(id)) { + /* + * Preserve the very old and *very strange* behaviour + * of erlang:exit/2... + * + * - terminate ourselves even though exit reason + * is normal (unless we trap exit) + * - terminate ourselves before exit/2 return + */ + int exit2_suicide = (exit2 + && c_p->common.id == id + && (reason == am_kill + || !(c_p->flags & F_TRAP_EXIT))); + erts_proc_sig_send_exit(c_p, c_p->common.id, id, + reason, NIL, exit2_suicide); + if (!exit2_suicide) + ERTS_BIF_PREP_RET(ret_val, am_true); + else { + erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ); + erts_proc_sig_fetch(c_p); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); + ERTS_BIF_PREP_TRAP0(ret_val, &await_exit_trap, c_p); + } + } + else if (is_internal_port(id)) { Eterm ref, *refp; Uint32 invalid_flags; Port *prt; + ErtsPortOpResult res = ERTS_PORT_OP_DONE; +#ifdef DEBUG + ref = NIL; +#endif if (erts_port_synchronous_ops) { refp = &ref; @@ -1519,108 +1290,75 @@ BIF_RETTYPE exit_2(BIF_ALIST_2) invalid_flags = ERTS_PORT_SFLGS_INVALID_LOOKUP; } - prt = erts_port_lookup(BIF_ARG_1, invalid_flags); - - if (prt) { - ErtsPortOpResult res; - -#ifdef DEBUG - ref = NIL; -#endif - - res = erts_port_exit(BIF_P, 0, prt, BIF_P->common.id, BIF_ARG_2, refp); - - ERTS_BIF_CHK_EXITED(BIF_P); - - if (refp && res == ERTS_PORT_OP_SCHEDULED) { - ASSERT(is_internal_ordinary_ref(ref)); - BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true); - } - - } - - BIF_RET(am_true); + prt = erts_port_lookup(id, invalid_flags); + if (prt) + res = erts_port_exit(c_p, 0, prt, c_p->common.id, reason, refp); + + if (!refp || res != ERTS_PORT_OP_SCHEDULED) + ERTS_BIF_PREP_RET(ret_val, am_true); + else { + ASSERT(is_internal_ordinary_ref(ref)); + ERTS_BIF_PREP_TRAP3(ret_val, await_port_send_result_trap, + c_p, ref, am_true, am_true); + } } - else if(is_external_port(BIF_ARG_1) - && external_port_dist_entry(BIF_ARG_1) == erts_this_dist_entry) - BIF_RET(am_true); - - /* - * If it is a remote pid, send a signal to the remote node. - */ - - if (is_external_pid(BIF_ARG_1)) { - int code; - ErtsDSigData dsd; - DistEntry *dep; - - dep = external_pid_dist_entry(BIF_ARG_1); - ERTS_ASSERT(dep); - if(dep == erts_this_dist_entry) - BIF_RET(am_true); - - code = erts_dsig_prepare(&dsd, dep, BIF_P, ERTS_PROC_LOCK_MAIN, - ERTS_DSP_NO_LOCK, 0, 1); - switch (code) { - case ERTS_DSIG_PREP_NOT_ALIVE: - case ERTS_DSIG_PREP_NOT_CONNECTED: - BIF_RET(am_true); - case ERTS_DSIG_PREP_PENDING: - case ERTS_DSIG_PREP_CONNECTED: - code = erts_dsig_send_exit2(&dsd, BIF_P->common.id, BIF_ARG_1, BIF_ARG_2); - if (code == ERTS_DSIG_SEND_YIELD) - ERTS_BIF_YIELD_RETURN(BIF_P, am_true); - BIF_RET(am_true); - default: - ERTS_ASSERT(! "Invalid dsig prepare result"); - } + else if (is_external_pid(id)) { + DistEntry *dep = external_pid_dist_entry(id); + if (dep == erts_this_dist_entry) + ERTS_BIF_PREP_RET(ret_val, am_true); /* Old incarnation of this node... */ + else { + int code; + ErtsDSigData dsd; + + code = erts_dsig_prepare(&dsd, dep, c_p, ERTS_PROC_LOCK_MAIN, + ERTS_DSP_NO_LOCK, 0, 1); + switch (code) { + case ERTS_DSIG_PREP_NOT_ALIVE: + case ERTS_DSIG_PREP_NOT_CONNECTED: + ERTS_BIF_PREP_RET(ret_val, am_true); + break; + case ERTS_DSIG_PREP_PENDING: + case ERTS_DSIG_PREP_CONNECTED: + code = erts_dsig_send_exit2(&dsd, c_p->common.id, id, reason); + if (code == ERTS_DSIG_SEND_YIELD) + ERTS_BIF_PREP_YIELD_RETURN(ret_val, c_p, am_true); + else + ERTS_BIF_PREP_RET(ret_val, am_true); + break; + default: + ASSERT(! "Invalid dsig prepare result"); + ERTS_BIF_PREP_ERROR(ret_val, c_p, EXC_INTERNAL_ERROR); + break; + } + } } - else if (is_not_internal_pid(BIF_ARG_1)) { - BIF_ERROR(BIF_P, BADARG); + else if (is_external_port(id)) { + DistEntry *dep = external_port_dist_entry(id); + if(dep == erts_this_dist_entry) + ERTS_BIF_PREP_RET(ret_val, am_true); /* Old incarnation of this node... */ + else + ERTS_BIF_PREP_ERROR(ret_val, c_p, BADARG); } else { - /* - * The pid is internal. Verify that it refers to an existing process. - */ - ErtsProcLocks rp_locks; - - if (BIF_ARG_1 == BIF_P->common.id) { - rp_locks = ERTS_PROC_LOCKS_ALL; - rp = BIF_P; - erts_proc_lock(rp, ERTS_PROC_LOCKS_ALL_MINOR); - } - else { - rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; - rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, - BIF_ARG_1, rp_locks); - if (!rp) { - BIF_RET(am_true); - } - } + /* Not an id of a process or a port... */ - /* - * Send an exit signal. - */ - erts_send_exit_signal(BIF_P, - BIF_P->common.id, - rp, - &rp_locks, - BIF_ARG_2, - NIL, - NULL, - BIF_P == rp ? ERTS_XSIG_FLG_NO_IGN_NORMAL : 0); - if (rp == BIF_P) - rp_locks &= ~ERTS_PROC_LOCK_MAIN; - if (rp_locks) - erts_proc_unlock(rp, rp_locks); - /* - * We may have exited ourselves and may have to take action. - */ - ERTS_BIF_CHK_EXITED(BIF_P); - BIF_RET(am_true); + ERTS_BIF_PREP_ERROR(ret_val, c_p, BADARG); } + + return ret_val; +} + +BIF_RETTYPE exit_2(BIF_ALIST_2) +{ + return send_exit_signal_bif(BIF_P, BIF_ARG_1, BIF_ARG_2, !0); } +BIF_RETTYPE exit_signal_2(BIF_ALIST_2) +{ + return send_exit_signal_bif(BIF_P, BIF_ARG_1, BIF_ARG_2, 0); +} + + /**********************************************************************/ /* this sets some process info- trapping exits or the error handler */ @@ -1709,36 +1447,13 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) BIF_RET(old_value); } else if (BIF_ARG_1 == am_trap_exit) { - erts_aint32_t state; - Uint trap_exit; - if (BIF_ARG_2 == am_true) { - trap_exit = 1; - } else if (BIF_ARG_2 == am_false) { - trap_exit = 0; - } else { - goto error; - } - /* - * NOTE: It is important that we check for pending exit signals - * and handle them before returning if trap_exit is set to - * true. For more info, see implementation of - * erts_send_exit_signal(). - */ - erts_proc_lock(BIF_P, ERTS_PROC_LOCKS_XSIG_SEND); - if (trap_exit) - state = erts_atomic32_read_bor_mb(&BIF_P->state, - ERTS_PSFLG_TRAP_EXIT); + old_value = (BIF_P->flags & F_TRAP_EXIT) ? am_true : am_false; + if (BIF_ARG_2 == am_true) + BIF_P->flags |= F_TRAP_EXIT; + else if (BIF_ARG_2 == am_false) + BIF_P->flags &= ~F_TRAP_EXIT; else - state = erts_atomic32_read_band_mb(&BIF_P->state, - ~ERTS_PSFLG_TRAP_EXIT); - erts_proc_unlock(BIF_P, ERTS_PROC_LOCKS_XSIG_SEND); - - if (state & ERTS_PSFLG_PENDING_EXIT) { - erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN); - ERTS_BIF_EXITED(BIF_P); - } - - old_value = (state & ERTS_PSFLG_TRAP_EXIT) ? am_true : am_false; + goto error; BIF_RET(old_value); } else if (BIF_ARG_1 == am_scheduler) { @@ -2142,9 +1857,6 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx) } switch (erts_port_command(p, ps_flags, pt, msg, refp)) { - case ERTS_PORT_OP_CALLER_EXIT: - /* We are exiting... */ - return SEND_USER_ERROR; case ERTS_PORT_OP_BUSY: /* Nothing has been sent */ if (ctx->suspend) @@ -4373,14 +4085,89 @@ BIF_RETTYPE group_leader_0(BIF_ALIST_0) } /**********************************************************************/ -/* arg1 == leader, arg2 == new member */ +/* set group leader */ -BIF_RETTYPE group_leader_2(BIF_ALIST_2) +int +erts_set_group_leader(Process *proc, Eterm new_gl) { - Process* new_member; + + erts_aint32_t state; - if (is_not_pid(BIF_ARG_1)) { - BIF_ERROR(BIF_P, BADARG); + ASSERT(is_pid(new_gl)); + + state = erts_atomic32_read_nob(&proc->state); + + if (state & ERTS_PSFLG_EXITING) + return 0; + + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(proc)); + + if (!(state & ERTS_PSFLG_DIRTY_RUNNING)) + proc->group_leader = STORE_NC_IN_PROC(proc, new_gl); + else { + ErlHeapFragment *bp; + Eterm *hp; + /* + * Currently executing on a dirty scheduler, + * so we are not allowed to write to its heap. + * Store group leader pid in heap fragment. + */ + bp = new_message_buffer(NC_HEAP_SIZE(new_gl)); + hp = bp->mem; + proc->group_leader = STORE_NC(&hp, + &proc->off_heap, + new_gl); + bp->next = proc->mbuf; + proc->mbuf = bp; + proc->mbuf_sz += bp->used_size; + } + + return !0; +} + +BIF_RETTYPE erts_internal_group_leader_3(BIF_ALIST_3) +{ + if (is_not_pid(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + if (is_not_internal_pid(BIF_ARG_2)) + BIF_ERROR(BIF_P, BADARG); + if (is_not_internal_ref(BIF_ARG_3)) + BIF_ERROR(BIF_P, BADARG); + + erts_proc_sig_send_group_leader(BIF_P, + BIF_ARG_2, + BIF_ARG_1, + BIF_ARG_3); + BIF_RET(am_ok); +} + +BIF_RETTYPE erts_internal_group_leader_2(BIF_ALIST_2) +{ + if (is_not_pid(BIF_ARG_1)) + BIF_RET(am_badarg); + + if (is_internal_pid(BIF_ARG_2)) { + Process *rp; + int res; + + if (BIF_ARG_2 == BIF_P->common.id) + rp = BIF_P; + else { + rp = erts_try_lock_sig_free_proc(BIF_ARG_2, + ERTS_PROC_LOCK_MAIN, + NULL); + if (!rp) + BIF_RET(am_badarg); + if (rp == ERTS_PROC_LOCK_BUSY) + BIF_RET(am_false); + } + + res = erts_set_group_leader(rp, BIF_ARG_1); + + if (rp != BIF_P) + erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); + + BIF_RET(res ? am_true : am_badarg); } if (is_external_pid(BIF_ARG_2)) { @@ -4408,74 +4195,8 @@ BIF_RETTYPE group_leader_2(BIF_ALIST_2) ERTS_ASSERT(! "Invalid dsig prepare result"); } } - else if (is_internal_pid(BIF_ARG_2)) { - int await_x; - ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS; - new_member = erts_pid2proc_nropt(BIF_P, ERTS_PROC_LOCK_MAIN, - BIF_ARG_2, locks); - if (!new_member) - BIF_ERROR(BIF_P, BADARG); - - if (new_member == ERTS_PROC_LOCK_BUSY) - ERTS_BIF_YIELD2(bif_export[BIF_group_leader_2], BIF_P, - BIF_ARG_1, BIF_ARG_2); - - await_x = (new_member != BIF_P - && ERTS_PROC_PENDING_EXIT(new_member)); - if (!await_x) { - if (is_immed(BIF_ARG_1)) - new_member->group_leader = BIF_ARG_1; - else { - locks &= ~ERTS_PROC_LOCK_STATUS; - erts_proc_unlock(new_member, ERTS_PROC_LOCK_STATUS); - if (new_member == BIF_P - || !(erts_atomic32_read_nob(&new_member->state) - & ERTS_PSFLG_DIRTY_RUNNING)) { - new_member->group_leader = STORE_NC_IN_PROC(new_member, - BIF_ARG_1); - } - else { - ErlHeapFragment *bp; - Eterm *hp; - /* - * Other process executing on a dirty scheduler, - * so we are not allowed to write to its heap. - * Store in heap fragment. - */ - - bp = new_message_buffer(NC_HEAP_SIZE(BIF_ARG_1)); - hp = bp->mem; - new_member->group_leader = STORE_NC(&hp, - &new_member->off_heap, - BIF_ARG_1); - bp->next = new_member->mbuf; - new_member->mbuf = bp; - new_member->mbuf_sz += bp->used_size; - } - } - } - - if (new_member == BIF_P) - locks &= ~ERTS_PROC_LOCK_MAIN; - if (locks) - erts_proc_unlock(new_member, locks); - - if (await_x) { - /* Wait for new_member to terminate; then badarg */ - Eterm args[2] = {BIF_ARG_1, BIF_ARG_2}; - ERTS_BIF_AWAIT_X_APPLY_TRAP(BIF_P, - BIF_ARG_2, - am_erlang, - am_group_leader, - args, - 2); - } - BIF_RET(am_true); - } - else { - BIF_ERROR(BIF_P, BADARG); - } + BIF_RET(am_badarg); } BIF_RETTYPE system_flag_2(BIF_ALIST_2) @@ -4662,7 +4383,6 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) BIF_RET(ret); } else if (BIF_ARG_1 == make_small(1)) { int i, max; - ErtsMessage* mp; erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_thr_progress_block(); @@ -4677,16 +4397,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) #endif p->seq_trace_clock = 0; p->seq_trace_lastcnt = 0; - ERTS_MSGQ_MV_INQ2PRIVQ(p); - mp = p->msg.first; - while(mp != NULL) { -#ifdef USE_VM_PROBES - ERL_MESSAGE_TOKEN(mp) = (ERL_MESSAGE_DT_UTAG(mp) != NIL) ? am_have_dt_utag : NIL; -#else - ERL_MESSAGE_TOKEN(mp) = NIL; -#endif - mp = mp->next; - } + + erts_proc_sig_clear_seq_trace_tokens(p); } } @@ -4949,85 +4661,6 @@ static BIF_RETTYPE bif_return_trap(BIF_ALIST_2) BIF_RET(res); } -/* - * NOTE: The erts_bif_prep_await_proc_exit_*() functions are - * tightly coupled with the implementation of erlang:await_proc_exit/3. - * The erts_bif_prep_await_proc_exit_*() functions can safely call - * skip_current_msgq() since they know that erlang:await_proc_exit/3 - * unconditionally will do a monitor and then unconditionally will - * wait for the corresponding 'DOWN' message in a receive, and no other - * receive is done before this receive. This optimization removes an - * unnecessary scan of the currently existing message queue (which - * can be large). If the erlang:await_proc_exit/3 implementation - * is changed so that the above isn't true, nasty bugs in later - * receives, etc, may appear. - */ - -static ERTS_INLINE int -skip_current_msgq(Process *c_p) -{ - int res; -#if defined(ERTS_ENABLE_LOCK_CHECK) - erts_proc_lc_chk_only_proc_main(c_p); -#endif - - erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - if (ERTS_PROC_PENDING_EXIT(c_p)) { - KILL_CATCHES(c_p); - c_p->freason = EXC_EXIT; - res = 0; - } - else { - ERTS_MSGQ_MV_INQ2PRIVQ(c_p); - c_p->msg.save = c_p->msg.last; - res = 1; - } - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - return res; -} - -void -erts_bif_prep_await_proc_exit_data_trap(Process *c_p, Eterm pid, Eterm ret) -{ - if (skip_current_msgq(c_p)) { - ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p, pid, am_data, ret); - } -} - -void -erts_bif_prep_await_proc_exit_reason_trap(Process *c_p, Eterm pid) -{ - if (skip_current_msgq(c_p)) { - ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p, - pid, am_reason, am_undefined); - } -} - -void -erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, - Eterm pid, - Eterm module, - Eterm function, - Eterm args[], - int nargs) -{ - ASSERT(is_atom(module) && is_atom(function)); - if (skip_current_msgq(c_p)) { - Eterm term; - Eterm *hp; - int i; - - hp = HAlloc(c_p, 4+2*nargs); - term = NIL; - for (i = nargs-1; i >= 0; i--) { - term = CONS(hp, args[i], term); - hp += 2; - } - term = TUPLE3(hp, module, function, term); - ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p, pid, am_apply, term); - } -} - Export bif_return_trap_export; void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, @@ -5063,6 +4696,9 @@ void erts_init_bif(void) am_erts_internal, am_dsend_continue_trap, 1, dsend_continue_trap_1); + erts_init_trap_export(&await_exit_trap, am_erts_internal, + am_await_exit, 0, erts_internal_await_exit_trap); + flush_monitor_messages_trap = erts_export_put(am_erts_internal, am_flush_monitor_messages, 3); @@ -5077,7 +4713,6 @@ void erts_init_bif(void) erts_format_cpu_topology_trap = erts_export_put(am_erlang, am_format_cpu_topology, 1); - await_proc_exit_trap = erts_export_put(am_erlang,am_await_proc_exit,3); await_port_send_result_trap = erts_export_put(am_erts_internal, am_await_port_send_result, 3); system_flag_scheduler_wall_time_trap diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index a2bc883dbe..a47339253e 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -306,7 +306,7 @@ do { \ (Proc)->freason = TRAP; \ } while (0) -#define BIF_TRAP0(p, Trap_) do { \ +#define BIF_TRAP0(Trap_, p) do { \ (p)->arity = 0; \ (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]); \ (p)->freason = TRAP; \ @@ -404,7 +404,7 @@ do { \ #define ERTS_BIF_YIELD0(TRP, P) \ do { \ ERTS_VBUMP_ALL_REDS((P)); \ - BIF_TRAP0((TRP), (P)); \ + BIF_TRAP0((TRP), (P)); \ } while (0) #define ERTS_BIF_YIELD1(TRP, P, A0) \ @@ -425,10 +425,16 @@ do { \ BIF_TRAP3((TRP), (P), (A0), (A1), (A2)); \ } while (0) +#define ERTS_BIF_PREP_EXITED(RET, PROC) \ +do { \ + KILL_CATCHES((PROC)); \ + ERTS_BIF_PREP_ERROR((RET), (PROC), EXTAG_EXIT); \ +} while (0) + #define ERTS_BIF_EXITED(PROC) \ do { \ KILL_CATCHES((PROC)); \ - BIF_ERROR((PROC), EXC_EXIT); \ + BIF_ERROR((PROC), EXTAG_EXIT); \ } while (0) #define ERTS_BIF_CHK_EXITED(PROC) \ @@ -437,67 +443,6 @@ do { \ ERTS_BIF_EXITED((PROC)); \ } while (0) -/* - * The ERTS_BIF_*_AWAIT_X_*_TRAP makros either exits the caller, or - * sets up a trap to erlang:await_proc_exit/3. - * - * The caller is acquired to hold the 'main' lock on C_P. No other locks - * are allowed to be held. - */ - -#define ERTS_BIF_PREP_AWAIT_X_DATA_TRAP(RET, C_P, PID, DATA) \ -do { \ - erts_bif_prep_await_proc_exit_data_trap((C_P), (PID), (DATA)); \ - (RET) = THE_NON_VALUE; \ -} while (0) - -#define ERTS_BIF_PREP_AWAIT_X_REASON_TRAP(RET, C_P, PID) \ -do { \ - erts_bif_prep_await_proc_exit_reason_trap((C_P), (PID)); \ - (RET) = THE_NON_VALUE; \ -} while (0) - -#define ERTS_BIF_PREP_AWAIT_X_APPLY_TRAP(RET, C_P, PID, M, F, A, AN) \ -do { \ - erts_bif_prep_await_proc_exit_apply_trap((C_P), (PID), \ - (M), (F), (A), (AN)); \ - (RET) = THE_NON_VALUE; \ -} while (0) - -#define ERTS_BIF_AWAIT_X_DATA_TRAP(C_P, PID, DATA) \ -do { \ - erts_bif_prep_await_proc_exit_data_trap((C_P), (PID), (DATA)); \ - return THE_NON_VALUE; \ -} while (0) - -#define ERTS_BIF_AWAIT_X_REASON_TRAP(C_P, PID) \ -do { \ - erts_bif_prep_await_proc_exit_reason_trap((C_P), (PID)); \ - return THE_NON_VALUE; \ -} while (0) - -#define ERTS_BIF_AWAIT_X_APPLY_TRAP(C_P, PID, M, F, A, AN) \ -do { \ - erts_bif_prep_await_proc_exit_apply_trap((C_P), (PID), \ - (M), (F), (A), (AN)); \ - return THE_NON_VALUE; \ -} while (0) - -void -erts_bif_prep_await_proc_exit_data_trap(Process *c_p, - Eterm pid, - Eterm data); -void -erts_bif_prep_await_proc_exit_reason_trap(Process *c_p, - Eterm pid); -void -erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, - Eterm pid, - Eterm module, - Eterm function, - Eterm args[], - int nargs); - int erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *reg); diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index be653ee2a0..276bef2bbb 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2017. All Rights Reserved. +# Copyright Ericsson AB 1996-2018. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -62,6 +62,7 @@ bif erlang:erase/0 bif erlang:erase/1 bif erlang:exit/1 bif erlang:exit/2 +bif erlang:exit_signal/2 bif erlang:external_size/1 bif erlang:external_size/2 gcbif erlang:float/1 @@ -73,7 +74,8 @@ bif erlang:get/0 bif erlang:get/1 bif erlang:get_keys/1 bif erlang:group_leader/0 -bif erlang:group_leader/2 +bif erts_internal:group_leader/2 +bif erts_internal:group_leader/3 bif erlang:halt/2 bif erlang:phash/2 bif erlang:phash2/1 @@ -187,6 +189,8 @@ bif erts_internal:release_literal_area_switch/0 bif erts_internal:scheduler_wall_time/1 +bif erts_internal:dirty_process_handle_signals/1 + # inet_db support bif erlang:port_set_data/2 bif erlang:port_get_data/1 @@ -262,6 +266,7 @@ bif erlang:demonitor/1 bif erlang:demonitor/2 bif erlang:is_process_alive/1 +bif erts_internal:is_process_alive/2 bif erlang:error/1 error_1 bif erlang:error/2 error_2 @@ -435,13 +440,6 @@ bif erts_debug:dirty_io/2 bif erts_debug:dirty/3 # -# Monitor testing bif's... -# -bif erts_debug:dump_monitors/1 -bif erts_debug:dump_links/1 - - -# # Lock counter bif's # bif erts_debug:lcnt_control/2 @@ -695,3 +693,6 @@ bif erts_internal:new_connection/1 bif erts_internal:abort_connection/2 bif erts_internal:map_next/3 bif ets:whereis/1 +bif erts_internal:gather_alloc_histograms/1 +bif erts_internal:gather_carrier_info/1 +ubif erlang:map_get/2 diff --git a/erts/emulator/beam/bif_instrs.tab b/erts/emulator/beam/bif_instrs.tab index 0932b8b985..0f074280db 100644 --- a/erts/emulator/beam/bif_instrs.tab +++ b/erts/emulator/beam/bif_instrs.tab @@ -432,9 +432,17 @@ nif_bif.call_nif() { live_hf_end = c_p->mbuf; ERTS_CHK_MBUF_SZ(c_p); erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL); + + ASSERT((c_p->scheduler_data)->current_nif == NULL); + (c_p->scheduler_data)->current_nif = &env; + nif_bif_result = (*fp)(&env, bif_nif_arity, reg); if (env.exception_thrown) nif_bif_result = THE_NON_VALUE; + + ASSERT((c_p->scheduler_data)->current_nif == &env); + (c_p->scheduler_data)->current_nif = NULL; + erts_post_nif(&env); ERTS_CHK_MBUF_SZ(c_p); diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index 95d324d2c1..d53f75c279 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -113,6 +113,40 @@ new_binary(Process *p, byte *buf, Uint len) return build_proc_bin(&MSO(p), HAlloc(p, PROC_BIN_SIZE), bptr); } +Eterm +erts_heap_factory_new_binary(ErtsHeapFactory *hfact, byte *buf, Uint len, + Uint reserve_size) +{ + Eterm *hp; + Binary* bptr; + + if (len <= ERL_ONHEAP_BIN_LIMIT) { + ErlHeapBin* hb; + hp = erts_produce_heap(hfact, heap_bin_size(len), reserve_size); + hb = (ErlHeapBin *) hp; + hb->thing_word = header_heap_bin(len); + hb->size = len; + if (buf != NULL) { + sys_memcpy(hb->data, buf, len); + } + return make_binary(hb); + } + + /* + * Allocate the binary struct itself. + */ + bptr = erts_bin_nrml_alloc(len); + if (buf != NULL) { + sys_memcpy(bptr->orig_bytes, buf, len); + } + + hp = erts_produce_heap(hfact, PROC_BIN_SIZE, reserve_size); + + return build_proc_bin(hfact->off_heap, hp, bptr); +} + + + /* * When heap binary is not desired... */ diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 3967f7f7fc..9ff52c92b8 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2017. All Rights Reserved. + * Copyright Ericsson AB 1996-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,9 +36,9 @@ #include "hash.h" #include "atom.h" #include "beam_load.h" -#include "erl_instrument.h" #include "erl_hl_timer.h" #include "erl_thr_progress.h" +#include "erl_proc_sig_queue.h" /* Forward declarations -- should really appear somewhere else */ static void process_killer(void); @@ -107,36 +107,11 @@ process_killer(void) if ((j = sys_get_key(0)) <= 0) erts_exit(0, ""); switch(j) { - case 'k': { - ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; - erts_aint32_t state; - erts_proc_inc_refc(rp); - erts_proc_lock(rp, rp_locks); - state = erts_atomic32_read_acqb(&rp->state); - if (state & (ERTS_PSFLG_FREE - | ERTS_PSFLG_EXITING - | ERTS_PSFLG_ACTIVE - | ERTS_PSFLG_ACTIVE_SYS - | ERTS_PSFLG_IN_RUNQ - | ERTS_PSFLG_RUNNING - | ERTS_PSFLG_RUNNING_SYS - | ERTS_PSFLG_DIRTY_RUNNING - | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { - erts_printf("Can only kill WAITING processes this way\n"); - } - else { - (void) erts_send_exit_signal(NULL, - NIL, - rp, - &rp_locks, - am_kill, - NIL, - NULL, - 0); - } - erts_proc_unlock(rp, rp_locks); - erts_proc_dec_refc(rp); - } + case 'k': + /* Send a 'kill' exit signal from init process */ + erts_proc_sig_send_exit(NULL, erts_init_process_id, + rp->common.id, am_kill, NIL, + 0); case 'n': br = 1; break; case 'r': return; default: return; @@ -161,47 +136,64 @@ static void doit_print_link(ErtsLink *lnk, void *vpcontext) if (pcontext->is_first) { pcontext->is_first = 0; - erts_print(to, to_arg, "%T", lnk->pid); + erts_print(to, to_arg, "%T", lnk->other.item); } else { - erts_print(to, to_arg, ", %T", lnk->pid); + erts_print(to, to_arg, ", %T", lnk->other.item); } } static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext) { + ErtsMonitorData *mdp; PrintMonitorContext *pcontext = vpcontext; fmtfn_t to = pcontext->to; void *to_arg = pcontext->to_arg; char *prefix = ", "; - if (pcontext->is_first) { - pcontext->is_first = 0; - prefix = ""; - } - + mdp = erts_monitor_to_data(mon); switch (mon->type) { - case MON_ORIGIN: - if (is_atom(mon->u.pid)) { /* dist by name */ - ASSERT(is_node_name_atom(mon->u.pid)); - erts_print(to, to_arg, "%s{to,{%T,%T},%T}", prefix, mon->name, - mon->u.pid, mon->ref); - } else if (is_atom(mon->name)){ /* local by name */ - erts_print(to, to_arg, "%s{to,{%T,%T},%T}", prefix, mon->name, - erts_this_dist_entry->sysname, mon->ref); - } else { /* local and distributed by pid */ - erts_print(to, to_arg, "%s{to,%T,%T}", prefix, mon->u.pid, mon->ref); - } - break; - case MON_TARGET: - erts_print(to, to_arg, "%s{from,%T,%T}", prefix, mon->u.pid, mon->ref); - break; - case MON_NIF_TARGET: { - ErtsResource* rsrc = mon->u.resource; - erts_print(to, to_arg, "%s{from,{%T,%T},%T}", prefix, rsrc->type->module, - rsrc->type->name, mon->ref); - break; - } + case ERTS_MON_TYPE_PROC: + case ERTS_MON_TYPE_PORT: + case ERTS_MON_TYPE_TIME_OFFSET: + case ERTS_MON_TYPE_DIST_PROC: + case ERTS_MON_TYPE_RESOURCE: + case ERTS_MON_TYPE_NODE: + + if (pcontext->is_first) { + pcontext->is_first = 0; + prefix = ""; + } + + if (erts_monitor_is_target(mon)) { + if (mon->type != ERTS_MON_TYPE_RESOURCE) + erts_print(to, to_arg, "%s{from,%T,%T}", prefix, mon->other.item, mdp->ref); + else { + ErtsResource* rsrc = mon->other.ptr; + erts_print(to, to_arg, "%s{from,{%T,%T},%T}", prefix, rsrc->type->module, + rsrc->type->name, mdp->ref); + } + } + else { + if (!(mon->flags & ERTS_ML_FLG_NAME)) + erts_print(to, to_arg, "%s{to,%T,%T}", prefix, mon->other.item, mdp->ref); + else { + ErtsMonitorDataExtended *mdep = (ErtsMonitorDataExtended *) mdp; + Eterm node; + if (mdep->dist) + node = mdep->dist->nodename; + else + node = erts_this_dist_entry->sysname; + erts_print(to, to_arg, "%s{to,{%T,%T},%T}", prefix, mdep->u.name, + node, mdp->ref); + } + } + + break; + + default: + /* ignore other monitors... */ + break; } } @@ -211,6 +203,7 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p) { int garbing = 0; int running = 0; + Sint len; struct saved_calls *scb; erts_aint32_t state; @@ -257,15 +250,22 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p) } erts_print(to, to_arg, "Spawned by: %T\n", p->parent); - ERTS_MSGQ_MV_INQ2PRIVQ(p); - erts_print(to, to_arg, "Message queue length: %d\n", p->msg.len); + + erts_proc_lock(p, ERTS_PROC_LOCK_MSGQ); + len = erts_proc_sig_fetch(p); + erts_proc_unlock(p, ERTS_PROC_LOCK_MSGQ); + erts_print(to, to_arg, "Message queue length: %d\n", len); /* display the message queue only if there is anything in it */ - if (!ERTS_IS_CRASH_DUMPING && p->msg.first != NULL && !garbing) { - ErtsMessage* mp; + if (!ERTS_IS_CRASH_DUMPING && p->sig_qs.first != NULL && !garbing) { erts_print(to, to_arg, "Message queue: ["); - for (mp = p->msg.first; mp; mp = mp->next) - erts_print(to, to_arg, mp->next ? "%T," : "%T", ERL_MESSAGE_TERM(mp)); + ERTS_FOREACH_SIG_PRIVQS( + p, mp, + { + if (ERTS_SIG_IS_NON_MSG((ErtsSignal *) mp)) + erts_print(to, to_arg, mp->next ? "%T," : "%T", + ERL_MESSAGE_TERM(mp)); + }); erts_print(to, to_arg, "]\n"); } @@ -306,11 +306,12 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p) } /* display the links only if there are any*/ - if (ERTS_P_LINKS(p) || ERTS_P_MONITORS(p)) { + if (ERTS_P_LINKS(p) || ERTS_P_MONITORS(p) || ERTS_P_LT_MONITORS(p)) { PrintMonitorContext context = {1, to, to_arg}; erts_print(to, to_arg,"Link list: ["); - erts_doforall_links(ERTS_P_LINKS(p), &doit_print_link, &context); - erts_doforall_monitors(ERTS_P_MONITORS(p), &doit_print_monitor, &context); + erts_link_tree_foreach(ERTS_P_LINKS(p), doit_print_link, &context); + erts_monitor_tree_foreach(ERTS_P_MONITORS(p), doit_print_monitor, &context); + erts_monitor_list_foreach(ERTS_P_LT_MONITORS(p), doit_print_monitor, &context); erts_print(to, to_arg,"]\n"); } @@ -953,20 +954,6 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) erts_cbprintf(to, to_arg, "=atoms\n"); dump_atoms(to, to_arg); - /* Keep the instrumentation data at the end of the dump */ - if (erts_instr_memory_map || erts_instr_stat) { - erts_cbprintf(to, to_arg, "=instr_data\n"); - - if (erts_instr_stat) { - erts_cbprintf(to, to_arg, "=memory_status\n"); - erts_instr_dump_stat_to(to, to_arg, 0); - } - if (erts_instr_memory_map) { - erts_cbprintf(to, to_arg, "=memory_map\n"); - erts_instr_dump_memory_map_to(to, to_arg); - } - } - erts_cbprintf(to, to_arg, "=end\n"); if (fp) { fclose(fp); diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index cd799e04b8..026f0a62d4 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2017. All Rights Reserved. + * Copyright Ericsson AB 1996-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,6 +45,7 @@ #include "erl_binary.h" #include "erl_thr_progress.h" #include "dtrace-wrapper.h" +#include "erl_proc_sig_queue.h" #define DIST_CTL_DEFAULT_SIZE 64 @@ -109,7 +110,6 @@ int erts_dist_buf_busy_limit; /* distribution trap functions */ Export* dmonitor_node_trap = NULL; -Export* dmonitor_p_trap = NULL; /* local variables */ static Export *dist_ctrl_put_data_trap; @@ -184,6 +184,161 @@ get_suspended_on_de(DistEntry *dep, erts_aint32_t unset_qflgs) } } +#define ERTS_MON_LNK_FIRE_LIMIT 100 + +static void monitor_connection_down(ErtsMonitor *mon, void *unused) +{ + if (erts_monitor_is_origin(mon)) + erts_proc_sig_send_demonitor(mon); + else + erts_proc_sig_send_monitor_down(mon, am_noconnection); +} + +static void link_connection_down(ErtsLink *lnk, void *vdist) +{ + erts_proc_sig_send_link_exit(NULL, THE_NON_VALUE, lnk, + am_noconnection, NIL); +} + +typedef enum { + ERTS_CML_CLEANUP_STATE_LINKS, + ERTS_CML_CLEANUP_STATE_MONITORS, + ERTS_CML_CLEANUP_STATE_ONAME_MONITORS, + ERTS_CML_CLEANUP_STATE_NODE_MONITORS +} ErtsConMonLnkCleaupState; + +typedef struct { + ErtsConMonLnkCleaupState state; + ErtsMonLnkDist *dist; + void *yield_state; + int trigger_node_monitors; + Eterm nodename; + Eterm visability; + Eterm reason; + ErlOffHeap oh; + Eterm heap[1]; +} ErtsConMonLnkCleanup; + +static void +con_monitor_link_cleanup(void *vcmlcp) +{ + ErtsConMonLnkCleanup *cmlcp = vcmlcp; + ErtsMonLnkDist *dist = cmlcp->dist; + ErtsSchedulerData *esdp; + int yield; + + switch (cmlcp->state) { + case ERTS_CML_CLEANUP_STATE_LINKS: + yield = erts_link_list_foreach_delete_yielding(&dist->links, + link_connection_down, + NULL, &cmlcp->yield_state, + ERTS_MON_LNK_FIRE_LIMIT); + if (yield) + break; + + ASSERT(!cmlcp->yield_state); + cmlcp->state = ERTS_CML_CLEANUP_STATE_MONITORS; + case ERTS_CML_CLEANUP_STATE_MONITORS: + yield = erts_monitor_list_foreach_delete_yielding(&dist->monitors, + monitor_connection_down, + NULL, &cmlcp->yield_state, + ERTS_MON_LNK_FIRE_LIMIT); + if (yield) + break; + + ASSERT(!cmlcp->yield_state); + cmlcp->state = ERTS_CML_CLEANUP_STATE_ONAME_MONITORS; + case ERTS_CML_CLEANUP_STATE_ONAME_MONITORS: + yield = erts_monitor_tree_foreach_delete_yielding(&dist->orig_name_monitors, + monitor_connection_down, + NULL, &cmlcp->yield_state, + ERTS_MON_LNK_FIRE_LIMIT/2); + if (yield) + break; + + cmlcp->dist = NULL; + erts_mon_link_dist_dec_refc(dist); + + ASSERT(!cmlcp->yield_state); + cmlcp->state = ERTS_CML_CLEANUP_STATE_NODE_MONITORS; + case ERTS_CML_CLEANUP_STATE_NODE_MONITORS: + if (cmlcp->trigger_node_monitors) { + send_nodes_mon_msgs(NULL, + am_nodedown, + cmlcp->nodename, + cmlcp->visability, + cmlcp->reason); + } + erts_cleanup_offheap(&cmlcp->oh); + erts_free(ERTS_ALC_T_CML_CLEANUP, vcmlcp); + return; /* done */ + } + + /* yield... */ + + esdp = erts_get_scheduler_data(); + ASSERT(esdp && esdp->type == ERTS_SCHED_NORMAL); + erts_schedule_misc_aux_work((int) esdp->no, + con_monitor_link_cleanup, + (void *) cmlcp); +} + +static void +schedule_con_monitor_link_cleanup(ErtsMonLnkDist *dist, + Eterm nodename, + Eterm visability, + Eterm reason) +{ + if (dist || is_value(nodename)) { + ErtsSchedulerData *esdp; + ErtsConMonLnkCleanup *cmlcp; + Uint rsz, size; + + size = sizeof(ErtsConMonLnkCleanup); + + if (is_non_value(reason) || is_immed(reason)) { + rsz = 0; + size -= sizeof(Eterm); + } + else { + rsz = size_object(reason); + size += sizeof(Eterm) * (rsz - 1); + } + + cmlcp = erts_alloc(ERTS_ALC_T_CML_CLEANUP, size); + + ERTS_INIT_OFF_HEAP(&cmlcp->oh); + + cmlcp->yield_state = NULL; + cmlcp->dist = dist; + if (!dist) + cmlcp->state = ERTS_CML_CLEANUP_STATE_NODE_MONITORS; + else { + cmlcp->state = ERTS_CML_CLEANUP_STATE_LINKS; + erts_mtx_lock(&dist->mtx); + ASSERT(dist->alive); + dist->alive = 0; + erts_mtx_unlock(&dist->mtx); + } + + cmlcp->trigger_node_monitors = is_value(nodename); + cmlcp->nodename = nodename; + cmlcp->visability = visability; + if (rsz == 0) + cmlcp->reason = reason; + else { + Eterm *hp = &cmlcp->heap[0]; + cmlcp->reason = copy_struct(reason, rsz, &hp, &cmlcp->oh); + } + + esdp = erts_get_scheduler_data(); + ASSERT(esdp && esdp->type == ERTS_SCHED_NORMAL); + erts_schedule_misc_aux_work((int) esdp->no, + con_monitor_link_cleanup, + (void *) cmlcp); + } +} + /* ** A full node name constists of a "n@h" ** @@ -235,170 +390,6 @@ int is_node_name_atom(Eterm a) return is_node_name((char*)atom_tab(i)->name, atom_tab(i)->len); } -typedef struct { - DistEntry *dep; - Eterm *lhp; -} NetExitsContext; - -/* -** This function is called when a distribution -** port or process terminates -*/ -static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp) -{ - Process *rp; - ErtsMonitor *rmon; - DistEntry *dep = ((NetExitsContext *) vnecp)->dep; - ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK; - - rp = erts_pid2proc(NULL, 0, mon->u.pid, rp_locks); - if (!rp) - goto done; - - if (mon->type == MON_ORIGIN) { - /* local pid is being monitored */ - rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref); - /* ASSERT(rmon != NULL); nope, can happen during process exit */ - if (rmon != NULL) { - erts_destroy_monitor(rmon); - } - } else { - DeclareTmpHeapNoproc(lhp,3); - Eterm watched; - UseTmpHeapNoproc(3); - ASSERT(mon->type == MON_TARGET); - rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref); - /* ASSERT(rmon != NULL); can happen during process exit */ - if (rmon != NULL) { - ASSERT(rmon->type == MON_ORIGIN); - ASSERT(is_atom(rmon->name) || is_nil(rmon->name)); - watched = (is_atom(rmon->name) - ? TUPLE2(lhp, rmon->name, dep->sysname) - : rmon->u.pid); - rp_locks |= ERTS_PROC_LOCKS_MSG_SEND; - erts_proc_lock(rp, ERTS_PROC_LOCKS_MSG_SEND); - erts_queue_monitor_message(rp, &rp_locks, mon->ref, am_process, - watched, am_noconnection); - erts_destroy_monitor(rmon); - } - UnUseTmpHeapNoproc(3); - } - erts_proc_unlock(rp, rp_locks); - done: - erts_destroy_monitor(mon); -} - -typedef struct { - NetExitsContext *necp; - ErtsLink *lnk; -} LinkNetExitsContext; - -/* -** This is the function actually doing the job of sending exit messages -** for links in a dist entry upon net_exit (the node goes down), NB, -** only process links, not node monitors are handled here, -** they reside in a separate tree.... -*/ -static void doit_link_net_exits_sub(ErtsLink *sublnk, void *vlnecp) -{ - ErtsLink *lnk = ((LinkNetExitsContext *) vlnecp)->lnk; /* the local pid */ - ErtsLink *rlnk; - Process *rp; - - ASSERT(lnk->type == LINK_PID); - if (is_internal_pid(lnk->pid)) { - int xres; - ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCKS_XSIG_SEND; - - rp = erts_pid2proc(NULL, 0, lnk->pid, rp_locks); - if (!rp) { - goto done; - } - - rlnk = erts_remove_link(&ERTS_P_LINKS(rp), sublnk->pid); - xres = erts_send_exit_signal(NULL, - sublnk->pid, - rp, - &rp_locks, - am_noconnection, - NIL, - NULL, - 0); - - if (rlnk) { - erts_destroy_link(rlnk); - if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) { - /* We didn't exit the process and it is traced */ - trace_proc(NULL, 0, rp, am_getting_unlinked, sublnk->pid); - } - } - erts_proc_unlock(rp, rp_locks); - } - done: - erts_destroy_link(sublnk); - -} - - - - - -/* -** This function is called when a distribution -** port or process terminates, once for each link on the high level, -** it in turn traverses the link subtree for the specific link node... -*/ -static void doit_link_net_exits(ErtsLink *lnk, void *vnecp) -{ - LinkNetExitsContext lnec = {(NetExitsContext *) vnecp, lnk}; - ASSERT(lnk->type == LINK_PID); - erts_sweep_links(ERTS_LINK_ROOT(lnk), &doit_link_net_exits_sub, (void *) &lnec); -#ifdef DEBUG - ERTS_LINK_ROOT(lnk) = NULL; -#endif - erts_destroy_link(lnk); -} - - -static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp) -{ - DistEntry *dep = ((NetExitsContext *) vnecp)->dep; - Eterm name = dep->sysname; - Process *rp; - ErtsLink *rlnk; - Uint i,n; - ASSERT(lnk->type == LINK_NODE); - if (is_internal_pid(lnk->pid)) { - ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK; - ErlOffHeap *ohp; - rp = erts_proc_lookup(lnk->pid); - if (!rp) - goto done; - erts_proc_lock(rp, rp_locks); - if (!ERTS_PROC_IS_EXITING(rp)) { - rlnk = erts_remove_link(&ERTS_P_LINKS(rp), name); - if (rlnk != NULL) { - ASSERT(is_atom(rlnk->pid) && (rlnk->type == LINK_NODE)); - erts_destroy_link(rlnk); - } - n = ERTS_LINK_REFC(lnk); - for (i = 0; i < n; ++i) { - Eterm tup; - Eterm *hp; - ErtsMessage *msgp; - - msgp = erts_alloc_message_heap(rp, &rp_locks, - 3, &hp, &ohp); - tup = TUPLE2(hp, am_nodedown, name); - erts_queue_message(rp, rp_locks, msgp, tup, am_system); - } - } - erts_proc_unlock(rp, rp_locks); - } - done: - erts_destroy_link(lnk); -} - static void set_node_not_alive(void *unused) { @@ -444,15 +435,8 @@ inc_no_nodes(void) static void kill_dist_ctrl_proc(void *vpid) { - Eterm pid = (Eterm) vpid; - ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; - Process *rp = erts_pid2proc(NULL, 0, pid, rp_locks); - if (rp) { - erts_send_exit_signal(NULL, rp->common.id, rp, &rp_locks, - am_kill, NIL, NULL, 0); - if (rp_locks) - erts_proc_unlock(rp, rp_locks); - } + erts_proc_sig_send_exit(NULL, (Eterm) vpid, (Eterm) vpid, + am_kill, NIL, 0); } static void @@ -572,10 +556,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) } } else { /* Call from distribution controller (port/process) */ - NetExitsContext nec = {dep}; - ErtsLink *nlinks; - ErtsLink *node_links; - ErtsMonitor *monitors; + ErtsMonLnkDist *mld; Uint32 flags; erts_atomic_set_mb(&dep->dist_cmd_scheduled, 1); @@ -601,14 +582,8 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) erts_mtx_unlock(&dep->qlock); } - erts_de_links_lock(dep); - monitors = dep->monitors; - nlinks = dep->nlinks; - node_links = dep->node_links; - dep->monitors = NULL; - dep->nlinks = NULL; - dep->node_links = NULL; - erts_de_links_unlock(dep); + mld = dep->mld; + dep->mld = NULL; nodename = dep->sysname; flags = dep->flags; @@ -617,15 +592,14 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) erts_de_rwunlock(dep); - erts_sweep_monitors(monitors, &doit_monitor_net_exits, (void *) &nec); - erts_sweep_links(nlinks, &doit_link_net_exits, (void *) &nec); - erts_sweep_links(node_links, &doit_node_link_net_exits, (void *) &nec); - - send_nodes_mon_msgs(NULL, - am_nodedown, - nodename, - flags & DFLAG_PUBLISHED ? am_visible : am_hidden, - reason == am_normal ? am_connection_closed : reason); + schedule_con_monitor_link_cleanup(mld, + nodename, + (flags & DFLAG_PUBLISHED + ? am_visible + : am_hidden), + (reason == am_normal + ? am_connection_closed + : reason)); clear_dist_entry(dep); } @@ -661,7 +635,6 @@ void init_dist(void) /* Lookup/Install all references to trap functions */ dmonitor_node_trap = trap_function(am_dmonitor_node,3); - dmonitor_p_trap = trap_function(am_dmonitor_p, 2); dist_ctrl_put_data_trap = erts_export_put(am_erts_internal, am_dist_ctrl_put_data, 2); @@ -771,14 +744,6 @@ static void clear_dist_entry(DistEntry *dep) cache = dep->cache; dep->cache = NULL; -#ifdef DEBUG - erts_de_links_lock(dep); - ASSERT(!dep->nlinks); - ASSERT(!dep->node_links); - ASSERT(!dep->monitors); - erts_de_links_unlock(dep); -#endif - erts_mtx_lock(&dep->qlock); erts_atomic64_set_nob(&dep->in, 0); @@ -887,8 +852,7 @@ erts_dsig_send_unlink(ErtsDSigData *dsdp, Eterm local, Eterm remote) /* A local process that's being monitored by a remote one exits. We send: - {DOP_MONITOR_P_EXIT, Local pid or name, Remote pid, ref, reason}, - which is rather sad as only the ref is needed, no pid's... */ + {DOP_MONITOR_P_EXIT, Local pid or name, Remote pid, ref, reason} */ int erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, Eterm ref, Eterm reason) @@ -909,12 +873,6 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, ctl = TUPLE5(&ctl_heap[0], make_small(DOP_MONITOR_P_EXIT), watched, watcher, ref, reason); -#ifdef DEBUG - erts_de_links_lock(dsdp->dep); - ASSERT(!erts_lookup_monitor(dsdp->dep->monitors, ref)); - erts_de_links_unlock(dsdp->dep); -#endif - res = dsig_send_ctl(dsdp, ctl, 1); UnUseTmpHeapNoproc(6); return res; @@ -953,8 +911,7 @@ erts_dsig_send_monitor(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, /* A local process monitoring a remote one wants to stop monitoring, either because of a demonitor bif call or because the local process died. We send - {DOP_DEMONITOR_P, Local pid, Remote pid or name, ref}, which is once again - rather redundant as only the ref will be needed on the other side... */ + {DOP_DEMONITOR_P, Local pid, Remote pid or name, ref} */ int erts_dsig_send_demonitor(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, Eterm ref, int force) @@ -980,6 +937,24 @@ erts_dsig_send_demonitor(ErtsDSigData *dsdp, Eterm watcher, return res; } +static int can_send_seqtrace_token(ErtsSendContext* ctx, Eterm token) { + Eterm label; + + if (ctx->dep->flags & DFLAG_BIG_SEQTRACE_LABELS) { + /* The other end is capable of handling arbitrary seq_trace labels. */ + return 1; + } + + /* The other end only tolerates smalls, but since we could potentially be + * talking to an old 32-bit emulator from a 64-bit one, we have to check + * whether the label is small on any emulator. */ + label = SEQ_TRACE_T_LABEL(token); + + return is_small(label) && + signed_val(label) <= (ERTS_SINT32_MAX >> _TAG_IMMED1_SIZE) && + signed_val(label) >= (ERTS_SINT32_MIN >> _TAG_IMMED1_SIZE); +} + int erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx) { @@ -1013,37 +988,38 @@ erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx) "%T", remote); msize = size_object(message); if (have_seqtrace(token)) { - tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); + tok_label = SEQ_TRACE_T_DTRACE_LABEL(token); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); } } #endif - if (token != NIL) { - Eterm el1, el2; - if (ctx->dep->flags & DFLAG_SEND_SENDER) { - el1 = make_small(DOP_SEND_SENDER_TT); - el2 = sender->common.id; - } - else { - el1 = make_small(DOP_SEND_TT); - el2 = am_Empty; - } - ctl = TUPLE4(&ctx->ctl_heap[0], el1, el2, remote, token); - } - else { - Eterm el1, el2; + { + Eterm dist_op, sender_id; + int send_token; + + send_token = (token != NIL && can_send_seqtrace_token(ctx, token)); + if (ctx->dep->flags & DFLAG_SEND_SENDER) { - el1 = make_small(DOP_SEND_SENDER); - el2 = sender->common.id; + dist_op = make_small(send_token ? + DOP_SEND_SENDER_TT : + DOP_SEND_SENDER); + sender_id = sender->common.id; + } else { + dist_op = make_small(send_token ? + DOP_SEND_TT : + DOP_SEND); + sender_id = am_Empty; } - else { - el1 = make_small(DOP_SEND); - el2 = am_Empty; + + if (send_token) { + ctl = TUPLE4(&ctx->ctl_heap[0], dist_op, sender_id, remote, token); + } else { + ctl = TUPLE3(&ctx->ctl_heap[0], dist_op, sender_id, remote); } - ctl = TUPLE3(&ctx->ctl_heap[0], el1, el2, remote); } + DTRACE6(message_send, sender_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); DTRACE7(message_send_remote, sender_name, node_name, receiver_name, @@ -1089,19 +1065,20 @@ erts_dsig_send_reg_msg(Eterm remote_name, Eterm message, "{%T,%s}", remote_name, node_name); msize = size_object(message); if (have_seqtrace(token)) { - tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); + tok_label = SEQ_TRACE_T_DTRACE_LABEL(token); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); } } #endif - if (token != NIL) + if (token != NIL && can_send_seqtrace_token(ctx, token)) ctl = TUPLE5(&ctx->ctl_heap[0], make_small(DOP_REG_SEND_TT), sender->common.id, am_Empty, remote_name, token); else ctl = TUPLE4(&ctx->ctl_heap[0], make_small(DOP_REG_SEND), sender->common.id, am_Empty, remote_name); + DTRACE6(message_send, sender_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); DTRACE7(message_send_remote, sender_name, node_name, receiver_name, @@ -1153,7 +1130,7 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, erts_snprintf(reason_str, sizeof(DTRACE_CHARBUF_NAME(reason_str)), "%T", reason); if (have_seqtrace(token)) { - tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); + tok_label = SEQ_TRACE_T_DTRACE_LABEL(token); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); } @@ -1276,8 +1253,6 @@ int erts_net_message(Port *prt, Sint type; Eterm token; Eterm token_size; - ErtsMonitor *mon; - ErtsLink *lnk; Uint tuple_arity; int res; Uint32 connection_id; @@ -1375,88 +1350,89 @@ int erts_net_message(Port *prt, token = NIL; switch (type = unsigned_val(tuple[1])) { - case DOP_LINK: + case DOP_LINK: { + ErtsDSigData dsd; + int code; + if (tuple_arity != 3) { goto invalid_message; } from = tuple[2]; to = tuple[3]; /* local proc to link to */ - if (is_not_pid(from) || is_not_pid(to)) { - goto invalid_message; - } + if (is_not_external_pid(from)) + goto invalid_message; - rp = erts_pid2proc_opt(NULL, 0, - to, ERTS_PROC_LOCK_LINK, - ERTS_P2P_FLG_ALLOW_OTHER_X); - if (!rp) { - /* This is tricky (we MUST force a distributed send) */ - ErtsDSigData dsd; - int code; - code = erts_dsig_prepare(&dsd, dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0); - if (code == ERTS_DSIG_PREP_CONNECTED) { - code = erts_dsig_send_exit(&dsd, to, from, am_noproc); - ASSERT(code == ERTS_DSIG_SEND_OK); - } - break; - } + if (dep != external_pid_dist_entry(from)) + goto invalid_message; - erts_de_links_lock(dep); - res = erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, from); + if (is_external_pid(to)) { + if (external_pid_dist_entry(to) != erts_this_dist_entry) + goto invalid_message; + /* old incarnation of node; reply noproc... */ + } + else if (is_internal_pid(to)) { + ErtsLinkData *ldp = erts_link_create(ERTS_LNK_TYPE_DIST_PROC, + from, to); + ASSERT(ldp->a.other.item == to); + ASSERT(eq(ldp->b.other.item, from)); +#ifdef DEBUG + code = +#endif + erts_link_dist_insert(&ldp->a, dep->mld); + ASSERT(code); - if (res < 0) { - /* It was already there! Lets skip the rest... */ - erts_de_links_unlock(dep); - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - break; - } - lnk = erts_add_or_lookup_link(&(dep->nlinks), LINK_PID, rp->common.id); - erts_add_link(&(ERTS_LINK_ROOT(lnk)), LINK_PID, from); - erts_de_links_unlock(dep); + if (erts_proc_sig_send_link(NULL, to, &ldp->b)) + break; /* done */ - if (IS_TRACED_FL(rp, F_TRACE_PROCS)) - trace_proc(NULL, 0, rp, am_getting_linked, from); + /* Failed to send signal; cleanup and reply noproc... */ + +#ifdef DEBUG + code = +#endif + erts_link_dist_delete(&ldp->a); + ASSERT(code); + erts_link_release_both(ldp); + } + + code = erts_dsig_prepare(&dsd, dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0); + if (code == ERTS_DSIG_PREP_CONNECTED) { + code = erts_dsig_send_exit(&dsd, to, from, am_noproc); + ASSERT(code == ERTS_DSIG_SEND_OK); + } - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); break; + } case DOP_UNLINK: { - ErtsDistLinkData dld; if (tuple_arity != 3) { goto invalid_message; } from = tuple[2]; to = tuple[3]; - if (is_not_pid(from) || is_not_pid(to)) { + if (is_not_external_pid(from)) + goto invalid_message; + if (dep != external_pid_dist_entry(from)) goto invalid_message; - } - - rp = erts_pid2proc_opt(NULL, 0, - to, ERTS_PROC_LOCK_LINK, - ERTS_P2P_FLG_ALLOW_OTHER_X); - if (!rp) - break; - - lnk = erts_remove_link(&ERTS_P_LINKS(rp), from); - if (IS_TRACED_FL(rp, F_TRACE_PROCS) && lnk != NULL) { - trace_proc(NULL, 0, rp, am_getting_unlinked, from); - } + if (is_external_pid(to) + && erts_this_dist_entry == external_pid_dist_entry(from)) + break; - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + if (is_not_internal_pid(to)) + goto invalid_message; - erts_remove_dist_link(&dld, to, from, dep); - erts_destroy_dist_link(&dld); - if (lnk) - erts_destroy_link(lnk); + erts_proc_sig_send_dist_unlink(dep, from, to); break; } case DOP_MONITOR_P: { /* A remote process wants to monitor us, we get: {DOP_MONITOR_P, Remote pid, local pid or name, ref} */ - Eterm name; - + Eterm pid, name; + ErtsDSigData dsd; + int code; + if (tuple_arity != 4) { goto invalid_message; } @@ -1465,44 +1441,58 @@ int erts_net_message(Port *prt, watched = tuple[3]; /* local proc to monitor */ ref = tuple[4]; - if (is_not_ref(ref)) { + if (is_not_external_pid(watcher)) + goto invalid_message; + else if (external_pid_dist_entry(watcher) != dep) + goto invalid_message; + + if (is_not_ref(ref)) goto invalid_message; - } - if (is_atom(watched)) { - name = watched; - rp = erts_whereis_process(NULL, 0, - watched, ERTS_PROC_LOCK_LINK, - ERTS_P2P_FLG_ALLOW_OTHER_X); - } - else { - name = NIL; - rp = erts_pid2proc_opt(NULL, 0, - watched, ERTS_PROC_LOCK_LINK, - ERTS_P2P_FLG_ALLOW_OTHER_X); - } + if (is_internal_pid(watched)) { + name = NIL; + pid = watched; + } + else if (is_atom(watched)) { + name = watched; + pid = erts_whereis_name_to_id(NULL, watched); + /* if port or undefined; reply noproc... */ + } + else if (is_external_pid(watched) + && external_pid_dist_entry(watched) == erts_this_dist_entry) { + name = NIL; + pid = am_undefined; /* old incarnation; reply noproc... */ + } + else + goto invalid_message; - if (!rp) { - ErtsDSigData dsd; - int code; - code = erts_dsig_prepare(&dsd, dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0); - if (code == ERTS_DSIG_PREP_CONNECTED) { - code = erts_dsig_send_m_exit(&dsd, watcher, watched, ref, - am_noproc); - ASSERT(code == ERTS_DSIG_SEND_OK); - } - } - else { - if (is_atom(watched)) - watched = rp->common.id; - erts_de_links_lock(dep); - erts_add_monitor(&(dep->monitors), MON_ORIGIN, ref, watched, name); - erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, ref, watcher, name); - erts_de_links_unlock(dep); - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - } + if (is_internal_pid(pid)) { + ErtsMonitorData *mdp; + mdp = erts_monitor_create(ERTS_MON_TYPE_DIST_PROC, + ref, watcher, pid, name); - break; + code = erts_monitor_dist_insert(&mdp->origin, dep->mld); + ASSERT(code); (void)code; + + if (erts_proc_sig_send_monitor(&mdp->target, pid)) + break; /* done */ + + /* Failed to send to local proc; cleanup reply noproc... */ + + code = erts_monitor_dist_delete(&mdp->origin); + ASSERT(code); (void)code; + erts_monitor_release_both(mdp); + + } + + code = erts_dsig_prepare(&dsd, dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0); + if (code == ERTS_DSIG_PREP_CONNECTED) { + code = erts_dsig_send_m_exit(&dsd, watcher, watched, ref, + am_noproc); + ASSERT(code == ERTS_DSIG_SEND_OK); + } + + break; } case DOP_DEMONITOR_P: @@ -1513,36 +1503,43 @@ int erts_net_message(Port *prt, if (tuple_arity != 4) { goto invalid_message; } - /* watcher = tuple[2]; */ - /* watched = tuple[3]; May be an atom in case of monitor name */ + + watcher = tuple[2]; + watched = tuple[3]; ref = tuple[4]; - if(is_not_ref(ref)) { + if (is_not_ref(ref)) { goto invalid_message; } - erts_de_links_lock(dep); - mon = erts_remove_monitor(&(dep->monitors),ref); - erts_de_links_unlock(dep); - /* ASSERT(mon != NULL); can happen in case of broken dist message */ - if (mon == NULL) { - break; - } - watched = mon->u.pid; - erts_destroy_monitor(mon); - rp = erts_pid2proc_opt(NULL, 0, - watched, ERTS_PROC_LOCK_LINK, - ERTS_P2P_FLG_ALLOW_OTHER_X); - if (!rp) { - break; - } - mon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - ASSERT(mon != NULL); - if (mon == NULL) { - break; - } - erts_destroy_monitor(mon); + if (is_not_external_pid(watcher) || external_pid_dist_entry(watcher) != dep) + goto invalid_message; + + if (is_internal_pid(watched)) + erts_proc_sig_send_dist_demonitor(watched, ref); + else if (is_external_pid(watched) + && external_pid_dist_entry(watched) == erts_this_dist_entry) { + /* old incarnation; ignore it */ + ; + } + else if (is_atom(watched)) { + ErtsMonLnkDist *mld = dep->mld; + ErtsMonitor *mon; + + erts_mtx_lock(&mld->mtx); + + mon = erts_monitor_tree_lookup(mld->orig_name_monitors, ref); + if (mon) + erts_monitor_tree_delete(&mld->orig_name_monitors, mon); + + erts_mtx_unlock(&mld->mtx); + + if (mon) + erts_proc_sig_send_demonitor(mon); + } + else + goto invalid_message; + break; case DOP_REG_SEND_TT: @@ -1669,68 +1666,36 @@ int erts_net_message(Port *prt, /* We are monitoring a process on the remote node which dies, we get {DOP_MONITOR_P_EXIT, Remote pid or name, Local pid, ref, reason} */ - - DeclareTmpHeapNoproc(lhp,3); - Eterm sysname; - ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_MSG_SEND|ERTS_PROC_LOCK_LINK; - if (tuple_arity != 5) { goto invalid_message; } - /* watched = tuple[2]; */ /* remote proc which died */ - /* watcher = tuple[3]; */ + watched = tuple[2]; /* remote proc or name which died */ + watcher = tuple[3]; ref = tuple[4]; reason = tuple[5]; - if(is_not_ref(ref)) { + if (is_not_ref(ref)) goto invalid_message; - } - erts_de_links_lock(dep); - sysname = dep->sysname; - mon = erts_remove_monitor(&(dep->monitors), ref); - /* - * If demonitor was performed at the same time as the - * monitored process exits, monitoring side will have - * removed info about monitor. In this case, do nothing - * and everything will be as it should. - */ - erts_de_links_unlock(dep); - if (mon == NULL) { - break; - } - rp = erts_pid2proc(NULL, 0, mon->u.pid, rp_locks); - - erts_destroy_monitor(mon); - if (rp == NULL) { - break; - } + if (is_not_external_pid(watched) && is_not_atom(watched)) + goto invalid_message; - mon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); + if (is_not_internal_pid(watcher)) { + if (!is_external_pid(watcher)) + goto invalid_message; + if (erts_this_dist_entry == external_pid_dist_entry(watcher)) + break; + goto invalid_message; + } - if (mon == NULL) { - erts_proc_unlock(rp, rp_locks); - break; - } - UseTmpHeapNoproc(3); - - watched = (is_not_nil(mon->name) - ? TUPLE2(&lhp[0], mon->name, sysname) - : mon->u.pid); - - erts_queue_monitor_message(rp, &rp_locks, - ref, am_process, watched, reason); - erts_proc_unlock(rp, rp_locks); - erts_destroy_monitor(mon); - UnUseTmpHeapNoproc(3); + erts_proc_sig_send_dist_monitor_down(dep, ref, watched, + watcher, reason); break; } case DOP_EXIT_TT: case DOP_EXIT: { - ErtsDistLinkData dld; - ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCKS_XSIG_SEND; /* 'from', which 'to' is linked to, died */ if (type == DOP_EXIT) { if (tuple_arity != 4) { @@ -1750,56 +1715,19 @@ int erts_net_message(Port *prt, token = tuple[4]; reason = tuple[5]; } - if (is_not_pid(from) || is_not_internal_pid(to)) { + if (is_not_external_pid(from) + || dep != external_pid_dist_entry(from) + || is_not_internal_pid(to)) { goto invalid_message; } - rp = erts_pid2proc(NULL, 0, to, rp_locks); - if (!rp) - lnk = NULL; - else { - lnk = erts_remove_link(&ERTS_P_LINKS(rp), from); - - /* If lnk == NULL, we have unlinked on this side, i.e. - * ignore exit. - */ - if (lnk) { - int xres; -#if 0 - /* Arndt: Maybe it should never be 'kill', but it can be, - namely when a linked process does exit(kill). Until we know - whether that is incorrect and what should happen instead, - we leave the assertion out. */ - ASSERT(reason != am_kill); /* should never be kill (killed) */ -#endif - xres = erts_send_exit_signal(NULL, - from, - rp, - &rp_locks, - reason, - token, - NULL, - ERTS_XSIG_FLG_IGN_KILL); - if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) { - /* We didn't exit the process and it is traced */ - if (rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) { - erts_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND); - rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND; - } - trace_proc(NULL, 0, rp, am_getting_unlinked, from); - } - } - erts_proc_unlock(rp, rp_locks); - } - erts_remove_dist_link(&dld, to, from, dep); - if (lnk) - erts_destroy_link(lnk); - erts_destroy_dist_link(&dld); + erts_proc_sig_send_dist_link_exit(dep, + from, to, + reason, token); break; } case DOP_EXIT2_TT: - case DOP_EXIT2: { - ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; + case DOP_EXIT2: /* 'from' is send an exit signal to 'to' */ if (type == DOP_EXIT2) { if (tuple_arity != 4) { @@ -1821,20 +1749,10 @@ int erts_net_message(Port *prt, if (is_not_pid(from) || is_not_internal_pid(to)) { goto invalid_message; } - rp = erts_pid2proc(NULL, 0, to, rp_locks); - if (rp) { - (void) erts_send_exit_signal(NULL, - from, - rp, - &rp_locks, - reason, - token, - NULL, - 0); - erts_proc_unlock(rp, rp_locks); - } + + erts_proc_sig_send_exit(NULL, from, to, reason, token, 0); break; - } + case DOP_GROUP_LEADER: if (tuple_arity != 3) { goto invalid_message; @@ -1845,11 +1763,7 @@ int erts_net_message(Port *prt, goto invalid_message; } - rp = erts_pid2proc(NULL, 0, to, ERTS_PROC_LOCK_MAIN); - if (!rp) - break; - rp->group_leader = STORE_NC_IN_PROC(rp, from); - erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); + (void) erts_proc_sig_send_group_leader(NULL, to, from, NIL); break; default: @@ -2989,57 +2903,55 @@ static void doit_print_monitor_info(ErtsMonitor *mon, void *vptdp) { fmtfn_t to = ((struct print_to_data *) vptdp)->to; void *arg = ((struct print_to_data *) vptdp)->arg; - Process *rp; - ErtsMonitor *rmon; - rp = erts_proc_lookup(mon->u.pid); - if (!rp || (rmon = erts_lookup_monitor(ERTS_P_MONITORS(rp), mon->ref)) == NULL) { - erts_print(to, arg, "Warning, stray monitor for: %T\n", mon->u.pid); - } else if (mon->type == MON_ORIGIN) { - /* Local pid is being monitored */ + ErtsMonitorDataExtended *mdep; + + ASSERT(mon->flags & ERTS_ML_FLG_EXTENDED); + + mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon); + + ASSERT(mdep->dist); + + if (erts_monitor_is_origin(mon)) { erts_print(to, arg, "Remotely monitored by: %T %T\n", - mon->u.pid, rmon->u.pid); - } else { - erts_print(to, arg, "Remote monitoring: %T ", mon->u.pid); - if (is_not_atom(rmon->u.pid)) - erts_print(to, arg, "%T\n", rmon->u.pid); - else - erts_print(to, arg, "{%T, %T}\n", - rmon->name, - rmon->u.pid); /* which in this case is the - remote system name... */ + mon->other.item, mdep->md.target.other.item); + } + else { + erts_print(to, arg, "Remote monitoring: %T ", mon->other.item); + if (mon->flags & ERTS_ML_FLG_NAME) + erts_print(to, arg, "{%T, %T}\n", mdep->u.name, mdep->dist->nodename); + else + erts_print(to, arg, "%T\n", mdep->md.origin.other.item); } } -static void print_monitor_info(fmtfn_t to, void *arg, ErtsMonitor *mon) +static void print_monitor_info(fmtfn_t to, void *arg, DistEntry *dep) { struct print_to_data ptd = {to, arg}; - erts_doforall_monitors(mon,&doit_print_monitor_info,&ptd); -} - -typedef struct { - struct print_to_data *ptdp; - Eterm from; -} PrintLinkContext; - -static void doit_print_link_info2(ErtsLink *lnk, void *vpplc) -{ - PrintLinkContext *pplc = (PrintLinkContext *) vpplc; - erts_print(pplc->ptdp->to, pplc->ptdp->arg, "Remote link: %T %T\n", - pplc->from, lnk->pid); + if (dep->mld) { + erts_monitor_list_foreach(dep->mld->monitors, + doit_print_monitor_info, + (void *) &ptd); + erts_monitor_tree_foreach(dep->mld->orig_name_monitors, + doit_print_monitor_info, + (void *) &ptd); + } } static void doit_print_link_info(ErtsLink *lnk, void *vptdp) { - if (is_internal_pid(lnk->pid) && erts_proc_lookup(lnk->pid)) { - PrintLinkContext plc = {(struct print_to_data *) vptdp, lnk->pid}; - erts_doforall_links(ERTS_LINK_ROOT(lnk), &doit_print_link_info2, &plc); - } + struct print_to_data *ptdp = vptdp; + ErtsLink *lnk2 = erts_link_to_other(lnk, NULL); + erts_print(ptdp->to, ptdp->arg, "Remote link: %T %T\n", + lnk2->other.item, lnk->other.item); } -static void print_link_info(fmtfn_t to, void *arg, ErtsLink *lnk) +static void print_link_info(fmtfn_t to, void *arg, DistEntry *dep) { struct print_to_data ptd = {to, arg}; - erts_doforall_links(lnk, &doit_print_link_info, (void *) &ptd); + if (dep->mld) + erts_link_list_foreach(dep->mld->links, + doit_print_link_info, + (void *) &ptd); } typedef struct { @@ -3047,23 +2959,6 @@ typedef struct { Eterm sysname; } PrintNodeLinkContext; - -static void doit_print_nodelink_info(ErtsLink *lnk, void *vpcontext) -{ - PrintNodeLinkContext *pcontext = vpcontext; - - if (is_internal_pid(lnk->pid) && erts_proc_lookup(lnk->pid)) - erts_print(pcontext->ptd.to, pcontext->ptd.arg, - "Remote monitoring: %T %T\n", lnk->pid, pcontext->sysname); -} - -static void print_nodelink_info(fmtfn_t to, void *arg, ErtsLink *lnk, Eterm sysname) -{ - PrintNodeLinkContext context = {{to, arg}, sysname}; - erts_doforall_links(lnk, &doit_print_nodelink_info, &context); -} - - static int info_dist_entry(fmtfn_t to, void *arg, DistEntry *dep, int visible, int connected) { @@ -3094,8 +2989,8 @@ info_dist_entry(fmtfn_t to, void *arg, DistEntry *dep, int visible, int connecte erts_print(to, arg, "Name: %T", dep->sysname); erts_print(to, arg, "\n"); if (!connected && is_nil(dep->cid)) { - if (dep->nlinks) { - erts_print(to, arg, "Error: Got links to not connected node:%T\n", + if (dep->mld) { + erts_print(to, arg, "Error: Got links/monitors to not connected node:%T\n", dep->sysname); } return 0; @@ -3104,9 +2999,8 @@ info_dist_entry(fmtfn_t to, void *arg, DistEntry *dep, int visible, int connecte erts_print(to, arg, "Controller: %T\n", dep->cid, to); erts_print_node_info(to, arg, dep->sysname, NULL, NULL); - print_monitor_info(to, arg, dep->monitors); - print_link_info(to, arg, dep->nlinks); - print_nodelink_info(to, arg, dep->node_links, dep->sysname); + print_monitor_info(to, arg, dep); + print_link_info(to, arg, dep); return 0; @@ -3193,8 +3087,7 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2) goto error; /* Check that all trap functions are defined !! */ - if (dmonitor_node_trap->addressv[0] == NULL || - dmonitor_p_trap->addressv[0] == NULL) { + if (dmonitor_node_trap->addressv[0] == NULL) { goto error; } @@ -3582,25 +3475,16 @@ static Sint abort_connection(DistEntry* dep, Uint32 conn_id) kill_connection(dep); } else if (dep->state == ERTS_DE_STATE_PENDING) { - NetExitsContext nec = {dep}; - ErtsLink *nlinks; - ErtsLink *node_links; - ErtsMonitor *monitors; ErtsAtomCache *cache; ErtsDistOutputBuf *obuf; ErtsProcList *resume_procs; Sint reds = 0; + ErtsMonLnkDist *mld; ASSERT(is_nil(dep->cid)); - erts_de_links_lock(dep); - monitors = dep->monitors; - nlinks = dep->nlinks; - node_links = dep->node_links; - dep->monitors = NULL; - dep->nlinks = NULL; - dep->node_links = NULL; - erts_de_links_unlock(dep); + mld = dep->mld; + dep->mld = NULL; cache = dep->cache; dep->cache = NULL; @@ -3619,9 +3503,8 @@ static Sint abort_connection(DistEntry* dep, Uint32 conn_id) erts_de_rwunlock(dep); - erts_sweep_monitors(monitors, &doit_monitor_net_exits, &nec); - erts_sweep_links(nlinks, &doit_link_net_exits, &nec); - erts_sweep_links(node_links, &doit_node_link_net_exits, &nec); + schedule_con_monitor_link_cleanup(mld, THE_NON_VALUE, + THE_NON_VALUE, THE_NON_VALUE); if (resume_procs) { int resumed = erts_resume_processes(resume_procs); @@ -3704,7 +3587,7 @@ int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks) dhandle = erts_build_dhandle(&hp, ohp, dep); msg = TUPLE4(hp, am_auto_connect, dep->sysname, make_small(conn_id), dhandle); - erts_queue_message(net_kernel, nk_locks, mp, msg, proc->common.id); + erts_queue_proc_message(proc, net_kernel, nk_locks, mp, msg); erts_proc_unlock(net_kernel, nk_locks); } @@ -3861,8 +3744,8 @@ BIF_RETTYPE is_alive_0(BIF_ALIST_0) static BIF_RETTYPE monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options) { - DistEntry *dep; - ErtsLink *lnk; + BIF_RETTYPE ret; + DistEntry *dep = NULL; Eterm l; int async_connect = 1; @@ -3881,33 +3764,71 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options) if (l != NIL) { BIF_ERROR(p, BADARG); } + if (l != NIL) + goto badarg; - if (is_not_atom(Node) || - ((Bool != am_true) && (Bool != am_false)) || - ((erts_this_node->sysname == am_Noname) - && (Node != erts_this_node->sysname))) { - BIF_ERROR(p, BADARG); + if (is_not_atom(Node)) + goto badarg; + + if (erts_this_node->sysname == am_Noname && Node != am_Noname) + goto badarg; + + switch (Bool) { + + case am_false: { + ErtsMonitor *mon; + /* + * Before OTP-21, monitor_node(Node, false) triggered + * auto-connect and a 'nodedown' message if that failed. + * Now it's a simple no-op which feels more reasonable. + */ + mon = erts_monitor_tree_lookup(ERTS_P_MONITORS(p), Node); + if (mon) { + ErtsMonitorDataExtended *mdep; + ASSERT(erts_monitor_is_origin(mon)); + + mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon); + + ASSERT((mdep->u.refc > 0)); + if (--mdep->u.refc == 0) { + if (!mdep->uptr.node_monitors) + erts_monitor_tree_delete(&ERTS_P_MONITORS(p), mon); + else { + ErtsMonitor *sub_mon; + ErtsMonitorDataExtended *sub_mdep; + sub_mon = erts_monitor_list_last(mdep->uptr.node_monitors); + erts_monitor_list_delete(&mdep->uptr.node_monitors, sub_mon); + sub_mon->flags &= ~ERTS_ML_FLG_IN_SUBTABLE; + sub_mdep = ((ErtsMonitorDataExtended *) + erts_monitor_to_data(sub_mon)); + sub_mdep->uptr.node_monitors = mdep->uptr.node_monitors; + mdep->uptr.node_monitors = NULL; + erts_monitor_tree_replace(&ERTS_P_MONITORS(p), mon, sub_mon); + } + if (erts_monitor_dist_delete(&mdep->md.target)) + erts_monitor_release_both((ErtsMonitorData *) mdep); + else + erts_monitor_release(mon); + } + } + break; } - if (Bool == am_true) { + case am_true: { ErtsDSigData dsd; dsd.node = Node; + dep = erts_find_or_insert_dist_entry(Node); if (dep == erts_this_dist_entry) - goto done; - - erts_proc_lock(p, ERTS_PROC_LOCK_LINK); + break; switch (erts_dsig_prepare(&dsd, dep, p, - (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK), + ERTS_PROC_LOCK_MAIN, ERTS_DSP_RLOCK, 0, async_connect)) { case ERTS_DSIG_PREP_NOT_ALIVE: case ERTS_DSIG_PREP_NOT_CONNECTED: /* Trap to either send 'nodedown' or do passive connection attempt */ - trap: - erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); - erts_deref_dist_entry(dep); - BIF_TRAP3(dmonitor_node_trap, p, Node, Bool, Options); + goto do_trap; case ERTS_DSIG_PREP_PENDING: if (!async_connect) { /* @@ -3915,68 +3836,87 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options) * to ensure passive connection attempt */ erts_de_runlock(dep); - goto trap; + goto do_trap; } /*fall through*/ - case ERTS_DSIG_PREP_CONNECTED: - erts_de_links_lock(dep); - erts_de_runlock(dep); - lnk = erts_add_or_lookup_link(&(dep->node_links), LINK_NODE, - p->common.id); - ++ERTS_LINK_REFC(lnk); - lnk = erts_add_or_lookup_link(&ERTS_P_LINKS(p), LINK_NODE, Node); - ++ERTS_LINK_REFC(lnk); - erts_de_links_unlock(dep); + case ERTS_DSIG_PREP_CONNECTED: { + ErtsMonitor *mon; + ErtsMonitorDataExtended *mdep; + int created; + + mon = erts_monitor_tree_lookup_create(&ERTS_P_MONITORS(p), + &created, + ERTS_MON_TYPE_NODE, + p->common.id, + Node); + mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon); + if (created) { +#ifdef DEBUG + int inserted = +#endif + erts_monitor_dist_insert(&mdep->md.target, dep->mld); + ASSERT(inserted); + ASSERT(mdep->dist->connection_id == dep->connection_id); + } + else if (mdep->dist->connection_id != dep->connection_id) { + ErtsMonitorDataExtended *mdep2; + ErtsMonitor *mon2; +#ifdef DEBUG + int inserted; +#endif + mdep2 = ((ErtsMonitorDataExtended *) + erts_monitor_create(ERTS_MON_TYPE_NODE, NIL, + p->common.id, Node, NIL)); + mon2 = &mdep2->md.origin; +#ifdef DEBUG + inserted = +#endif + erts_monitor_dist_insert(&mdep->md.target, dep->mld); + ASSERT(inserted); + ASSERT(mdep2->dist->connection_id == dep->connection_id); + + mdep2->uptr.node_monitors = mdep->uptr.node_monitors; + mdep->uptr.node_monitors = NULL; + erts_monitor_tree_replace(&ERTS_P_MONITORS(p), mon, mon2); + erts_monitor_list_insert(&mdep2->uptr.node_monitors, mon); + mon->flags |= ERTS_ML_FLG_IN_SUBTABLE; + mdep = mdep2; + } + + mdep->u.refc++; + break; + } + default: ERTS_ASSERT(! "Invalid dsig prepare result"); } - erts_deref_dist_entry(dep); - } - else { /* Bool == false */ - dep = erts_sysname_to_connected_dist_entry(Node); - if (!dep) { - /* - * Before OTP-21 this case triggered auto-connect - * and a 'nodedown' message if that failed. - * Now it's a simple no-op which feels more reasonable. - */ - BIF_RET(am_true); - } - if (dep == erts_this_dist_entry) - goto done; - erts_proc_lock(p, ERTS_PROC_LOCK_LINK); - erts_de_rlock(dep); - if (dep->state == ERTS_DE_STATE_IDLE) { - ASSERT(!dep->node_links); - erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); - erts_de_runlock(dep); - goto done; - } - erts_de_links_lock(dep); erts_de_runlock(dep); - lnk = erts_lookup_link(dep->node_links, p->common.id); - if (lnk != NULL) { - if ((--ERTS_LINK_REFC(lnk)) == 0) { - erts_destroy_link(erts_remove_link(&(dep->node_links), - p->common.id)); - } - } - lnk = erts_lookup_link(ERTS_P_LINKS(p), Node); - if (lnk != NULL) { - if ((--ERTS_LINK_REFC(lnk)) == 0) { - erts_destroy_link(erts_remove_link(&ERTS_P_LINKS(p), - Node)); - } - } - erts_de_links_unlock(dep); + + break; } - erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); + default: + goto badarg; + } - done: - BIF_RET(am_true); + ERTS_BIF_PREP_RET(ret, am_true); + +do_return: + + if (dep) + erts_deref_dist_entry(dep); + + return ret; + +do_trap: + ERTS_BIF_PREP_TRAP3(ret, dmonitor_node_trap, p, Node, Bool, Options); + goto do_return; + +badarg: + ERTS_BIF_PREP_ERROR(ret, p, BADARG); + goto do_return; } BIF_RETTYPE monitor_node_3(BIF_ALIST_3) @@ -4027,18 +3967,9 @@ BIF_RETTYPE net_kernel_dflag_unicode_io_1(BIF_ALIST_1) #define ERTS_NODES_MON_OPT_TYPES \ (ERTS_NODES_MON_OPT_TYPE_VISIBLE|ERTS_NODES_MON_OPT_TYPE_HIDDEN) -typedef struct ErtsNodesMonitor_ ErtsNodesMonitor; -struct ErtsNodesMonitor_ { - ErtsNodesMonitor *prev; - ErtsNodesMonitor *next; - Process *proc; - Uint16 opts; - Uint16 no; -}; - static erts_mtx_t nodes_monitors_mtx; -static ErtsNodesMonitor *nodes_monitors; -static ErtsNodesMonitor *nodes_monitors_end; +static ErtsMonitor *nodes_monitors; +static Uint no_nodes_monitors; /* * Nodes monitors are stored in a double linked list. 'nodes_monitors' @@ -4057,109 +3988,169 @@ init_nodes_monitors(void) erts_mtx_init(&nodes_monitors_mtx, "nodes_monitors", NIL, ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); nodes_monitors = NULL; - nodes_monitors_end = NULL; + no_nodes_monitors = 0; } -static ERTS_INLINE Uint -nodes_mon_msg_sz(ErtsNodesMonitor *nmp, Eterm what, Eterm reason) +Eterm +erts_monitor_nodes(Process *c_p, Eterm on, Eterm olist) { - Uint sz; - if (!nmp->opts) { - sz = 3; - } - else { - sz = 0; + Eterm key, old_value, opts_list = olist; + Uint opts = (Uint) 0; + + ASSERT(c_p); + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN); - if (nmp->opts & ERTS_NODES_MON_OPT_TYPES) - sz += 2 + 3; + if (on != am_true && on != am_false) + return THE_NON_VALUE; + + if (is_not_nil(opts_list)) { + int all = 0, visible = 0, hidden = 0; - if (what == am_nodedown - && (nmp->opts & ERTS_NODES_MON_OPT_DOWN_REASON)) { - if (is_not_immed(reason)) - sz += size_object(reason); - sz += 2 + 3; + while (is_list(opts_list)) { + Eterm *cp = list_val(opts_list); + Eterm opt = CAR(cp); + opts_list = CDR(cp); + if (opt == am_nodedown_reason) + opts |= ERTS_NODES_MON_OPT_DOWN_REASON; + else if (is_tuple(opt)) { + Eterm* tp = tuple_val(opt); + if (arityval(tp[0]) != 2) + return THE_NON_VALUE; + switch (tp[1]) { + case am_node_type: + switch (tp[2]) { + case am_visible: + if (hidden || all) + return THE_NON_VALUE; + opts |= ERTS_NODES_MON_OPT_TYPE_VISIBLE; + visible = 1; + break; + case am_hidden: + if (visible || all) + return THE_NON_VALUE; + opts |= ERTS_NODES_MON_OPT_TYPE_HIDDEN; + hidden = 1; + break; + case am_all: + if (visible || hidden) + return THE_NON_VALUE; + opts |= ERTS_NODES_MON_OPT_TYPES; + all = 1; + break; + default: + return THE_NON_VALUE; + } + break; + default: + return THE_NON_VALUE; + } + } + else { + return THE_NON_VALUE; + } } - sz += 4; + if (is_not_nil(opts_list)) + return THE_NON_VALUE; } - return sz; + + key = make_small(opts); + + if (on == am_true) { + ErtsMonitorDataExtended *mdep; + ErtsMonitor *omon; + int created; + omon = erts_monitor_tree_lookup_create(&ERTS_P_MONITORS(c_p), + &created, + ERTS_MON_TYPE_NODES, + c_p->common.id, + key); + mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(omon); + if (created) { + erts_mtx_lock(&nodes_monitors_mtx); + no_nodes_monitors++; + erts_monitor_list_insert(&nodes_monitors, &mdep->md.target); + erts_mtx_unlock(&nodes_monitors_mtx); + } + old_value = mdep->u.refc; + mdep->u.refc++; + } + else { + ErtsMonitorDataExtended *mdep; + ErtsMonitor *omon; + omon = erts_monitor_tree_lookup(ERTS_P_MONITORS(c_p), key); + if (!omon) + old_value = 0; + else { + mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(omon); + old_value = mdep->u.refc; + ASSERT(mdep->u.refc > 0); + erts_mtx_lock(&nodes_monitors_mtx); + ASSERT(no_nodes_monitors > 0); + no_nodes_monitors--; + ASSERT(erts_monitor_is_in_table(&mdep->md.target)); + erts_monitor_list_delete(&nodes_monitors, &mdep->md.target); + erts_mtx_unlock(&nodes_monitors_mtx); + erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p), omon); + erts_monitor_release_both((ErtsMonitorData *) mdep); + } + } + + return erts_make_integer(old_value, c_p); } -static ERTS_INLINE void -send_nodes_mon_msg(Process *rp, - ErtsProcLocks *rp_locksp, - ErtsNodesMonitor *nmp, - Eterm node, - Eterm what, - Eterm type, - Eterm reason, - Uint sz) +void +erts_monitor_nodes_delete(ErtsMonitor *omon) { - Eterm msg; - Eterm *hp; - ErtsMessage *mp; - ErlOffHeap *ohp; -#ifdef DEBUG - Eterm *hend; -#endif + ErtsMonitorData *mdp; - mp = erts_alloc_message_heap(rp, rp_locksp, sz, &hp, &ohp); -#ifdef DEBUG - hend = hp + sz; -#endif + ASSERT(omon->type == ERTS_MON_TYPE_NODES); + ASSERT(erts_monitor_is_origin(omon)); - if (!nmp->opts) { - msg = TUPLE2(hp, what, node); -#ifdef DEBUG - hp += 3; -#endif - } - else { - Eterm tup; - Eterm info = NIL; + mdp = erts_monitor_to_data(omon); - if (nmp->opts & (ERTS_NODES_MON_OPT_TYPE_VISIBLE - | ERTS_NODES_MON_OPT_TYPE_HIDDEN)) { + erts_mtx_lock(&nodes_monitors_mtx); + ASSERT(erts_monitor_is_in_table(&mdp->target)); + ASSERT(no_nodes_monitors > 0); + no_nodes_monitors--; + erts_monitor_list_delete(&nodes_monitors, &mdp->target); + erts_mtx_unlock(&nodes_monitors_mtx); + erts_monitor_release_both(mdp); +} - tup = TUPLE2(hp, am_node_type, type); - hp += 3; - info = CONS(hp, tup, info); - hp += 2; - } +typedef struct { + Eterm pid; + Eterm options; +} ErtsNodesMonitorData; - if (what == am_nodedown - && (nmp->opts & ERTS_NODES_MON_OPT_DOWN_REASON)) { - Eterm rsn_cpy; - - if (is_immed(reason)) - rsn_cpy = reason; - else { - Eterm rsn_sz = size_object(reason); - rsn_cpy = copy_struct(reason, rsn_sz, &hp, ohp); - } +typedef struct { + ErtsNodesMonitorData *nmdp; + Uint i; +} ErtsNodesMonitorContext; - tup = TUPLE2(hp, am_nodedown_reason, rsn_cpy); - hp += 3; - info = CONS(hp, tup, info); - hp += 2; - } +static void +save_nodes_monitor(ErtsMonitor *mon, void *vctxt) +{ + ErtsNodesMonitorContext *ctxt = vctxt; + ErtsMonitorData *mdp = erts_monitor_to_data(mon); - msg = TUPLE3(hp, what, node, info); -#ifdef DEBUG - hp += 4; -#endif - } + ASSERT(erts_monitor_is_target(mon)); + ASSERT(mon->type == ERTS_MON_TYPE_NODES); - ASSERT(hend == hp); - erts_queue_message(rp, *rp_locksp, mp, msg, am_system); + ctxt->nmdp[ctxt->i].pid = mon->other.item; + ctxt->nmdp[ctxt->i].options = mdp->origin.other.item; + + ctxt->i++; } static void send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reason) { - ErtsNodesMonitor *nmp; - ErtsProcLocks rp_locks = 0; /* Init to shut up false warning */ - Process *rp = NULL; + Uint opts; + Uint i, no, reason_size; + ErtsNodesMonitorData def_buf[100]; + ErtsNodesMonitorData *nmdp = &def_buf[0]; + ErtsNodesMonitorContext ctxt; ASSERT(is_immed(what)); ASSERT(is_immed(node)); @@ -4180,31 +4171,44 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas } #endif - ERTS_LC_ASSERT(!c_p - || (erts_proc_lc_my_proc_locks(c_p) - == ERTS_PROC_LOCK_MAIN)); + ctxt.i = 0; + + reason_size = is_immed(reason) ? 0 : size_object(reason); + erts_mtx_lock(&nodes_monitors_mtx); + if (no_nodes_monitors > sizeof(def_buf)/sizeof(def_buf[0])) + nmdp = erts_alloc(ERTS_ALC_T_TMP, + no_nodes_monitors*sizeof(ErtsNodesMonitorData)); + ctxt.nmdp = nmdp; + erts_monitor_list_foreach(nodes_monitors, + save_nodes_monitor, + (void *) &ctxt); + + ASSERT(ctxt.i == no_nodes_monitors); + no = no_nodes_monitors; - for (nmp = nodes_monitors; nmp; nmp = nmp->next) { - int i; - Uint16 no; - Uint sz; + erts_mtx_unlock(&nodes_monitors_mtx); - ASSERT(nmp->proc != NULL); + for (i = 0; i < no; i++) { + Eterm tmp_heap[3+2+3+2+4 /* max need */]; + Eterm *hp, msg; + Uint hsz; - if (!nmp->opts) { + ASSERT(is_small(nmdp[i].options)); + opts = (Uint) signed_val(nmdp[i].options); + if (!opts) { if (type != am_visible) continue; } else { switch (type) { case am_hidden: - if (!(nmp->opts & ERTS_NODES_MON_OPT_TYPE_HIDDEN)) + if (!(opts & ERTS_NODES_MON_OPT_TYPE_HIDDEN)) continue; break; case am_visible: - if ((nmp->opts & ERTS_NODES_MON_OPT_TYPES) - && !(nmp->opts & ERTS_NODES_MON_OPT_TYPE_VISIBLE)) + if ((opts & ERTS_NODES_MON_OPT_TYPES) + && !(opts & ERTS_NODES_MON_OPT_TYPE_VISIBLE)) continue; break; default: @@ -4212,342 +4216,162 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas } } - if (rp != nmp->proc) { - if (rp) { - if (rp == c_p) - rp_locks &= ~ERTS_PROC_LOCK_MAIN; - erts_proc_unlock(rp, rp_locks); - } + hsz = 0; + hp = &tmp_heap[0]; - rp = nmp->proc; - rp_locks = 0; - if (rp == c_p) - rp_locks |= ERTS_PROC_LOCK_MAIN; - } - - ASSERT(rp); - - sz = nodes_mon_msg_sz(nmp, what, reason); - - for (i = 0, no = nmp->no; i < no; i++) - send_nodes_mon_msg(rp, - &rp_locks, - nmp, - node, - what, - type, - reason, - sz); - } + if (!opts) { + msg = TUPLE2(hp, what, node); + hp += 3; + } + else { + Eterm tup; + Eterm info = NIL; - if (rp) { - if (rp == c_p) - rp_locks &= ~ERTS_PROC_LOCK_MAIN; - erts_proc_unlock(rp, rp_locks); - } + if (opts & (ERTS_NODES_MON_OPT_TYPE_VISIBLE + | ERTS_NODES_MON_OPT_TYPE_HIDDEN)) { - erts_mtx_unlock(&nodes_monitors_mtx); -} + tup = TUPLE2(hp, am_node_type, type); + hp += 3; + info = CONS(hp, tup, info); + hp += 2; + } -static Eterm -insert_nodes_monitor(Process *c_p, Uint32 opts) -{ - Uint16 no = 1; - Eterm res = am_false; - ErtsNodesMonitor *xnmp, *nmp; + if (what == am_nodedown + && (opts & ERTS_NODES_MON_OPT_DOWN_REASON)) { + hsz += reason_size; + tup = TUPLE2(hp, am_nodedown_reason, reason); + hp += 3; + info = CONS(hp, tup, info); + hp += 2; + } - ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&nodes_monitors_mtx)); - ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); + msg = TUPLE3(hp, what, node, info); + hp += 4; + } - xnmp = c_p->nodes_monitors; - if (xnmp) { - ASSERT(!xnmp->prev || xnmp->prev->proc != c_p); + ASSERT(hp - &tmp_heap[0] <= sizeof(tmp_heap)/sizeof(tmp_heap[0])); - while (1) { - ASSERT(xnmp->proc == c_p); - if (xnmp->opts == opts) - break; - if (!xnmp->next || xnmp->next->proc != c_p) - break; - xnmp = xnmp->next; - } - ASSERT(xnmp); - ASSERT(xnmp->proc == c_p); - ASSERT(xnmp->opts == opts - || !xnmp->next - || xnmp->next->proc != c_p); - - if (xnmp->opts != opts) - goto alloc_new; - else { - res = am_true; - no = xnmp->no++; - if (!xnmp->no) { - /* - * 'no' wrapped; transfer all prevous monitors to new - * element (which will be the next element in the list) - * and set this to one... - */ - xnmp->no = 1; - goto alloc_new; - } - } - } - else { - alloc_new: - nmp = erts_alloc(ERTS_ALC_T_NODES_MON, sizeof(ErtsNodesMonitor)); - nmp->proc = c_p; - nmp->opts = opts; - nmp->no = no; - - if (xnmp) { - ASSERT(nodes_monitors); - ASSERT(c_p->nodes_monitors); - nmp->next = xnmp->next; - nmp->prev = xnmp; - xnmp->next = nmp; - if (nmp->next) { - ASSERT(nodes_monitors_end != xnmp); - ASSERT(nmp->next->prev == xnmp); - nmp->next->prev = nmp; - } - else { - ASSERT(nodes_monitors_end == xnmp); - nodes_monitors_end = nmp; - } - } - else { - ASSERT(!c_p->nodes_monitors); - c_p->nodes_monitors = nmp; - nmp->next = NULL; - nmp->prev = nodes_monitors_end; - if (nodes_monitors_end) { - ASSERT(nodes_monitors); - nodes_monitors_end->next = nmp; - } - else { - ASSERT(!nodes_monitors); - nodes_monitors = nmp; - } - nodes_monitors_end = nmp; - } - } - return res; -} + hsz += hp - &tmp_heap[0]; -static Eterm -remove_nodes_monitors(Process *c_p, Uint32 opts, int all) -{ - Eterm res = am_false; - ErtsNodesMonitor *nmp; - - ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&nodes_monitors_mtx)); - ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); - - nmp = c_p->nodes_monitors; - ASSERT(!nmp || !nmp->prev || nmp->prev->proc != c_p); - - while (nmp && nmp->proc == c_p) { - if (!all && nmp->opts != opts) - nmp = nmp->next; - else { /* if (all || nmp->opts == opts) */ - ErtsNodesMonitor *free_nmp; - res = am_true; - if (nmp->prev) { - ASSERT(nodes_monitors != nmp); - nmp->prev->next = nmp->next; - } - else { - ASSERT(nodes_monitors == nmp); - nodes_monitors = nmp->next; - } - if (nmp->next) { - ASSERT(nodes_monitors_end != nmp); - nmp->next->prev = nmp->prev; - } - else { - ASSERT(nodes_monitors_end == nmp); - nodes_monitors_end = nmp->prev; - } - free_nmp = nmp; - nmp = nmp->next; - if (c_p->nodes_monitors == free_nmp) - c_p->nodes_monitors = nmp && nmp->proc == c_p ? nmp : NULL; - erts_free(ERTS_ALC_T_NODES_MON, free_nmp); - } + erts_proc_sig_send_persistent_monitor_msg(ERTS_MON_TYPE_NODES, + nmdp[i].options, + am_system, + nmdp[i].pid, + msg, + hsz); } - - ASSERT(!all || !c_p->nodes_monitors); - return res; -} -void -erts_delete_nodes_monitors(Process *c_p, ErtsProcLocks locks) -{ -#if defined(ERTS_ENABLE_LOCK_CHECK) - if (c_p) { - ErtsProcLocks might_unlock = locks & ~ERTS_PROC_LOCK_MAIN; - if (might_unlock) - erts_proc_lc_might_unlock(c_p, might_unlock); - } -#endif - if (erts_mtx_trylock(&nodes_monitors_mtx) == EBUSY) { - ErtsProcLocks unlock_locks = locks & ~ERTS_PROC_LOCK_MAIN; - if (c_p && unlock_locks) - erts_proc_unlock(c_p, unlock_locks); - erts_mtx_lock(&nodes_monitors_mtx); - if (c_p && unlock_locks) - erts_proc_lock(c_p, unlock_locks); - } - remove_nodes_monitors(c_p, 0, 1); - erts_mtx_unlock(&nodes_monitors_mtx); + if (nmdp != &def_buf[0]) + erts_free(ERTS_ALC_T_TMP, nmdp); } + -Eterm -erts_monitor_nodes(Process *c_p, Eterm on, Eterm olist) -{ +typedef struct { + Eterm **hpp; + Uint *szp; Eterm res; - Eterm opts_list = olist; - Uint16 opts = (Uint16) 0; - - ASSERT(c_p); - ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN); - - if (on != am_true && on != am_false) - return THE_NON_VALUE; - - if (is_not_nil(opts_list)) { - int all = 0, visible = 0, hidden = 0; - - while (is_list(opts_list)) { - Eterm *cp = list_val(opts_list); - Eterm opt = CAR(cp); - opts_list = CDR(cp); - if (opt == am_nodedown_reason) - opts |= ERTS_NODES_MON_OPT_DOWN_REASON; - else if (is_tuple(opt)) { - Eterm* tp = tuple_val(opt); - if (arityval(tp[0]) != 2) - return THE_NON_VALUE; - switch (tp[1]) { - case am_node_type: - switch (tp[2]) { - case am_visible: - if (hidden || all) - return THE_NON_VALUE; - opts |= ERTS_NODES_MON_OPT_TYPE_VISIBLE; - visible = 1; - break; - case am_hidden: - if (visible || all) - return THE_NON_VALUE; - opts |= ERTS_NODES_MON_OPT_TYPE_HIDDEN; - hidden = 1; - break; - case am_all: - if (visible || hidden) - return THE_NON_VALUE; - opts |= ERTS_NODES_MON_OPT_TYPES; - all = 1; - break; - default: - return THE_NON_VALUE; - } - break; - default: - return THE_NON_VALUE; - } - } - else { - return THE_NON_VALUE; - } - } +} ErtsNodesMonitorInfoContext; - if (is_not_nil(opts_list)) - return THE_NON_VALUE; - } - erts_mtx_lock(&nodes_monitors_mtx); - - if (on == am_true) - res = insert_nodes_monitor(c_p, opts); - else - res = remove_nodes_monitors(c_p, opts, 0); - - erts_mtx_unlock(&nodes_monitors_mtx); - - return res; +static void +nodes_monitor_info(ErtsMonitor *mon, void *vctxt) +{ + ErtsMonitorDataExtended *mdep; + ErtsNodesMonitorInfoContext *ctxt = vctxt; + Uint no, i, opts, *szp; + Eterm **hpp, res; + + hpp = ctxt->hpp; + szp = ctxt->szp; + res = ctxt->res; + + ASSERT(erts_monitor_is_target(mon)); + ASSERT(mon->type == ERTS_MON_TYPE_NODES); + mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon); + no = mdep->u.refc; + + ASSERT(is_small(mdep->md.origin.other.item)); + opts = (Uint) signed_val(mdep->md.origin.other.item); + + for (i = 0; i < no; i++) { + Eterm olist = NIL; + if (opts & ERTS_NODES_MON_OPT_TYPES) { + Eterm type; + switch (opts & ERTS_NODES_MON_OPT_TYPES) { + case ERTS_NODES_MON_OPT_TYPES: type = am_all; break; + case ERTS_NODES_MON_OPT_TYPE_VISIBLE: type = am_visible; break; + case ERTS_NODES_MON_OPT_TYPE_HIDDEN: type = am_hidden; break; + default: erts_exit(ERTS_ABORT_EXIT, "Bad node type found\n"); + } + olist = erts_bld_cons(hpp, szp, + erts_bld_tuple(hpp, szp, 2, + am_node_type, + type), + olist); + } + if (opts & ERTS_NODES_MON_OPT_DOWN_REASON) + olist = erts_bld_cons(hpp, szp, am_nodedown_reason, olist); + res = erts_bld_cons(hpp, szp, + erts_bld_tuple(hpp, szp, 2, + mon->other.item, + olist), + res); + } + + ctxt->hpp = hpp; + ctxt->szp = szp; + ctxt->res = res; } -/* - * Note, this function is only used for debuging. - */ - Eterm erts_processes_monitoring_nodes(Process *c_p) { - ErtsNodesMonitor *nmp; - Eterm res; + /* + * Note, this function is only used for debugging. + */ + ErtsNodesMonitorInfoContext ctxt; Eterm *hp; - Eterm **hpp; Uint sz; - Uint *szp; #ifdef DEBUG Eterm *hend; #endif ASSERT(c_p); ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN); + + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); + erts_mtx_lock(&nodes_monitors_mtx); sz = 0; - szp = &sz; - hpp = NULL; + ctxt.szp = &sz; + ctxt.hpp = NULL; - bld_result: - res = NIL; - - for (nmp = nodes_monitors_end; nmp; nmp = nmp->prev) { - Uint16 i; - for (i = 0; i < nmp->no; i++) { - Eterm olist = NIL; - if (nmp->opts & ERTS_NODES_MON_OPT_TYPES) { - Eterm type; - switch (nmp->opts & ERTS_NODES_MON_OPT_TYPES) { - case ERTS_NODES_MON_OPT_TYPES: type = am_all; break; - case ERTS_NODES_MON_OPT_TYPE_VISIBLE: type = am_visible; break; - case ERTS_NODES_MON_OPT_TYPE_HIDDEN: type = am_hidden; break; - default: erts_exit(ERTS_ABORT_EXIT, "Bad node type found\n"); - } - olist = erts_bld_cons(hpp, szp, - erts_bld_tuple(hpp, szp, 2, - am_node_type, - type), - olist); - } - if (nmp->opts & ERTS_NODES_MON_OPT_DOWN_REASON) - olist = erts_bld_cons(hpp, szp, am_nodedown_reason, olist); - res = erts_bld_cons(hpp, szp, - erts_bld_tuple(hpp, szp, 2, - nmp->proc->common.id, - olist), - res); - } - } + while (1) { + ctxt.res = NIL; + + erts_monitor_list_foreach(nodes_monitors, + nodes_monitor_info, + (void *) &ctxt); + + if (ctxt.hpp) + break; - if (!hpp) { hp = HAlloc(c_p, sz); #ifdef DEBUG hend = hp + sz; #endif - hpp = &hp; - szp = NULL; - goto bld_result; + ctxt.hpp = &hp; + ctxt.szp = NULL; } ASSERT(hp == hend); erts_mtx_unlock(&nodes_monitors_mtx); - return res; + erts_thr_progress_unblock(); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + + return ctxt.res; } diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index b1b7ce9c78..dda2029a4c 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2017. All Rights Reserved. + * Copyright Ericsson AB 1996-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,7 +45,8 @@ #define DFLAG_MAP_TAG 0x20000 #define DFLAG_BIG_CREATION 0x40000 #define DFLAG_SEND_SENDER 0x80000 -#define DFLAG_NO_MAGIC 0x100000 /* internal for pending connection */ +#define DFLAG_BIG_SEQTRACE_LABELS 0x100000 +#define DFLAG_NO_MAGIC 0x200000 /* internal for pending connection */ /* Mandatory flags for distribution */ #define DFLAG_DIST_MANDATORY (DFLAG_EXTENDED_REFERENCES \ @@ -73,7 +74,8 @@ | DFLAG_UTF8_ATOMS \ | DFLAG_MAP_TAG \ | DFLAG_BIG_CREATION \ - | DFLAG_SEND_SENDER) + | DFLAG_SEND_SENDER \ + | DFLAG_BIG_SEQTRACE_LABELS) /* Flags addable by local distr implementations */ #define DFLAG_DIST_ADDABLE DFLAG_DIST_DEFAULT @@ -120,7 +122,6 @@ /* distribution trap functions */ extern Export* dmonitor_node_trap; -extern Export* dmonitor_p_trap; typedef enum { ERTS_DSP_NO_LOCK, @@ -274,58 +275,13 @@ void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry) #endif -typedef struct { - ErtsLink *d_lnk; - ErtsLink *d_sub_lnk; -} ErtsDistLinkData; - -ERTS_GLB_INLINE void erts_remove_dist_link(ErtsDistLinkData *, - Eterm, - Eterm, - DistEntry *); -ERTS_GLB_INLINE int erts_was_dist_link_removed(ErtsDistLinkData *); -ERTS_GLB_INLINE void erts_destroy_dist_link(ErtsDistLinkData *); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -ERTS_GLB_INLINE void -erts_remove_dist_link(ErtsDistLinkData *dldp, - Eterm lid, - Eterm rid, - DistEntry *dep) -{ - erts_de_links_lock(dep); - dldp->d_lnk = erts_lookup_link(dep->nlinks, lid); - if (!dldp->d_lnk) - dldp->d_sub_lnk = NULL; - else { - dldp->d_sub_lnk = erts_remove_link(&ERTS_LINK_ROOT(dldp->d_lnk), rid); - dldp->d_lnk = (ERTS_LINK_ROOT(dldp->d_lnk) - ? NULL - : erts_remove_link(&dep->nlinks, lid)); - } - erts_de_links_unlock(dep); -} - -ERTS_GLB_INLINE int -erts_was_dist_link_removed(ErtsDistLinkData *dldp) -{ - return dldp->d_sub_lnk != NULL; -} - -ERTS_GLB_INLINE void -erts_destroy_dist_link(ErtsDistLinkData *dldp) -{ - if (dldp->d_lnk) - erts_destroy_link(dldp->d_lnk); - if (dldp->d_sub_lnk) - erts_destroy_link(dldp->d_sub_lnk); -} - +#ifdef DEBUG +#define ERTS_DBG_CHK_NO_DIST_LNK(D, R, L) \ + erts_dbg_chk_no_dist_proc_link((D), (R), (L)) +#else +#define ERTS_DBG_CHK_NO_DIST_LNK(D, R, L) #endif - - /* Define for testing */ /* #define EXTREME_TTB_TRAPPING 1 */ diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index f7f5506a72..d99d2ea57b 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2017. All Rights Reserved. + * Copyright Ericsson AB 2002-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,9 +38,9 @@ #include "erl_db.h" #include "erl_binary.h" #include "erl_bits.h" -#include "erl_instrument.h" +#include "erl_mtrace.h" #include "erl_mseg.h" -#include "erl_monitors.h" +#include "erl_monitor_link.h" #include "erl_hl_timer.h" #include "erl_cpu_topology.h" #include "erl_thr_queue.h" @@ -202,8 +202,6 @@ typedef struct { int top_pad; AlcUInit_t alloc_util; struct { - int stat; - int map; char *mtrace; char *nodename; } instr; @@ -428,6 +426,7 @@ set_default_binary_alloc_opts(struct au_init *ip) #endif ip->init.util.ts = ERTS_ALC_MTA_BINARY; ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; + ip->init.util.atags = 1; } static void @@ -464,6 +463,7 @@ set_default_driver_alloc_opts(struct au_init *ip) #endif ip->init.util.ts = ERTS_ALC_MTA_DRIVER; ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; + ip->init.util.atags = 1; } static void @@ -501,6 +501,7 @@ set_default_test_alloc_opts(struct au_init *ip) ip->init.util.mmbcs = 0; /* Main carrier size */ ip->init.util.ts = ERTS_ALC_MTA_TEST; ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; + ip->init.util.atags = 1; /* Use a constant minimal MBC size */ #if ERTS_SA_MB_CARRIERS @@ -636,10 +637,10 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_PROC)] = sizeof(Process); - fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MONITOR_SH)] - = ERTS_MONITOR_SH_SIZE * sizeof(Uint); - fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NLINK_SH)] - = ERTS_LINK_SH_SIZE * sizeof(Uint); + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MONITOR)] + = sizeof(ErtsMonitorDataHeap); + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_LINK)] + = sizeof(ErtsLinkData); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)] = sizeof(ErtsDrvSelectDataState); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_SEL_D_STATE)] @@ -906,7 +907,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) &test_alloc_state); erts_mtrace_install_wrapper_functions(); - extra_block_size += erts_instr_init(init.instr.stat, init.instr.map); init_aireq_alloc(); @@ -1411,7 +1411,9 @@ handle_au_arg(struct au_init *auip, } if (!strategy_support_carrier_migration(auip)) auip->init.util.acul = 0; - } + } else if (has_prefix("atags", sub_param)) { + auip->init.util.atags = get_bool_value(sub_param + 5, argv, ip); + } else goto bad_switch; break; @@ -1741,24 +1743,6 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) break; case 'i': switch (argv[i][3]) { - case 's': - arg = get_value(argv[i]+4, argv, &i); - if (sys_strcmp("true", arg) == 0) - init->instr.stat = 1; - else if (sys_strcmp("false", arg) == 0) - init->instr.stat = 0; - else - bad_value(param, param+3, arg); - break; - case 'm': - arg = get_value(argv[i]+4, argv, &i); - if (sys_strcmp("true", arg) == 0) - init->instr.map = 1; - else if (sys_strcmp("false", arg) == 0) - init->instr.map = 0; - else - bad_value(param, param+3, arg); - break; case 't': init->instr.mtrace = get_value(argv[i]+4, argv, &i); break; @@ -1817,9 +1801,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) case '-': if (argv[i][2] == '\0') { /* End of system flags reached */ - if (init->instr.mtrace - /* || init->instr.stat - || init->instr.map */) { + if (init->instr.mtrace) { while (i < *argc) { if(sys_strcmp(argv[i], "-sname") == 0 || sys_strcmp(argv[i], "-name") == 0) { @@ -2097,7 +2079,7 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) * NOTE! When updating this function, make sure to also update * erlang:memory/[0,1] in $ERL_TOP/erts/preloaded/src/erlang.erl */ -#define ERTS_MEM_NEED_ALL_ALCU (!erts_instr_stat && want_tot_or_sys) +#define ERTS_MEM_NEED_ALL_ALCU (want_tot_or_sys) struct { int total; int processes; @@ -2108,7 +2090,6 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) int binary; int code; int ets; - int maximum; } want = {0}; struct { UWord total; @@ -2120,7 +2101,6 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) UWord binary; UWord code; UWord ets; - UWord maximum; } size = {0}; Eterm atoms[sizeof(size)/sizeof(UWord)]; UWord *uintps[sizeof(size)/sizeof(UWord)]; @@ -2173,12 +2153,6 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) want.ets = 1; atoms[length] = am_ets; uintps[length++] = &size.ets; - - want.maximum = erts_instr_stat; - if (want.maximum) { - atoms[length] = am_maximum; - uintps[length++] = &size.maximum; - } } else { DeclareTmpHeapNoproc(tmp_heap,2); @@ -2260,18 +2234,6 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) uintps[length++] = &size.ets; } break; - case am_maximum: - if (erts_instr_stat) { - if (!want.maximum) { - want.maximum = 1; - atoms[length] = am_maximum; - uintps[length++] = &size.maximum; - } - } else { - UnUseTmpHeapNoproc(2); - return am_badarg; - } - break; default: UnUseTmpHeapNoproc(2); return am_badarg; @@ -2370,7 +2332,6 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) } tmp += erts_ptab_mem_size(&erts_proc); tmp += erts_bif_timer_memory_size(); - tmp += erts_tot_link_lh_size(); size.processes = size.processes_used = tmp; @@ -2381,12 +2342,11 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) add_fix_values(&size.processes, &size.processes_used, fi, - ERTS_ALC_T_MONITOR_SH); - + ERTS_ALC_T_MONITOR); add_fix_values(&size.processes, &size.processes_used, fi, - ERTS_ALC_T_NLINK_SH); + ERTS_ALC_T_LINK); add_fix_values(&size.processes, &size.processes_used, fi, @@ -2439,14 +2399,7 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) size.ets += erts_get_ets_misc_mem_size(); } - if (erts_instr_stat && (want_tot_or_sys || want.maximum)) { - if (want_tot_or_sys) { - size.total = erts_instr_get_total(); - size.system = size.total - size.processes; - } - size.maximum = erts_instr_get_max_total(); - } - else if (want_tot_or_sys) { + if (want_tot_or_sys) { size.system = size.total - size.processes; } @@ -2524,18 +2477,6 @@ erts_allocated_areas(fmtfn_t *print_to_p, void *print_to_arg, void *proc) i = 0; - if (erts_instr_stat) { - values[i].arity = 2; - values[i].name = "total"; - values[i].ui[0] = erts_instr_get_total(); - i++; - - values[i].arity = 2; - values[i].name = "maximum"; - values[i].ui[0] = erts_instr_get_max_total(); - i++; - } - values[i].arity = 2; values[i].name = "sys_misc"; values[i].ui[0] = erts_sys_misc_mem_sz(); @@ -2617,11 +2558,6 @@ erts_allocated_areas(fmtfn_t *print_to_p, void *print_to_arg, void *proc) i++; values[i].arity = 2; - values[i].name = "link_lh"; - values[i].ui[0] = erts_tot_link_lh_size(); - i++; - - values[i].arity = 2; values[i].name = "process_table"; values[i].ui[0] = erts_ptab_mem_size(&erts_proc); i++; @@ -2831,10 +2767,7 @@ erts_allocator_info(fmtfn_t to, void *arg) erts_alcu_au_info_options(&to, arg, NULL, NULL); erts_print(to, arg, "=allocator:instr\n"); - erts_print(to, arg, "option m: %s\n", - erts_instr_memory_map ? "true" : "false"); - erts_print(to, arg, "option s: %s\n", - erts_instr_stat ? "true" : "false"); + erts_print(to, arg, "option t: %s\n", erts_mtrace_enabled ? "true" : "false"); @@ -2940,16 +2873,12 @@ erts_allocator_options(void *proc) NULL, hpp, szp); #endif { - Eterm o[3], v[3]; - o[0] = am_atom_put("m", 1); - v[0] = erts_instr_memory_map ? am_true : am_false; - o[1] = am_atom_put("s", 1); - v[1] = erts_instr_stat ? am_true : am_false; - o[2] = am_atom_put("t", 1); - v[2] = erts_mtrace_enabled ? am_true : am_false; + Eterm o[1], v[1]; + o[0] = am_atom_put("t", 1); + v[0] = erts_mtrace_enabled ? am_true : am_false; atoms[length] = am_atom_put("instr", 5); - terms[length++] = erts_bld_2tup_list(hpp, szp, 3, o, v); + terms[length++] = erts_bld_2tup_list(hpp, szp, 1, o, v); } atoms[length] = am_atom_put("lock_physical_memory", 20); @@ -3465,8 +3394,8 @@ badarg: /* * The allocator wrapper prelocking stuff below is about the locking order. - * It only affects wrappers (erl_mtrace.c and erl_instrument.c) that keep locks - * during alloc/realloc/free. + * It only affects wrappers (erl_mtrace.c) that keep locks during + * alloc/realloc/free. * * Some query functions in erl_alloc_util.c lock the allocator mutex and then * use erts_printf that in turn may call the sys allocator through the wrappers. diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 8107f133aa..4a6a19b210 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2003-2017. All Rights Reserved. +# Copyright Ericsson AB 2003-2018. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -282,6 +282,11 @@ type THR_PRGR_IDATA LONG_LIVED SYSTEM thr_prgr_internal_data type THR_PRGR_DATA LONG_LIVED SYSTEM thr_prgr_data type T_THR_PRGR_DATA SHORT_LIVED SYSTEM temp_thr_prgr_data type RELEASE_LAREA SHORT_LIVED SYSTEM release_literal_area +type SIG_DATA SHORT_LIVED PROCESSES signal_data +type DIST_DEMONITOR SHORT_LIVED PROCESSES dist_demonitor +type CML_CLEANUP SHORT_LIVED SYSTEM connection_ml_cleanup +type ML_YIELD_STATE SHORT_LIVED SYSTEM monitor_link_yield_state +type ML_DIST STANDARD SYSTEM monitor_link_dist type ENVIRONMENT SYSTEM SYSTEM environment @@ -326,8 +331,8 @@ type LCNT_VECTOR SHORT_LIVED SYSTEM lcnt_sample_vector type DEBUG SHORT_LIVED SYSTEM debugging type DDLL_PROCESS STANDARD SYSTEM ddll_processes -type MONITOR_LH STANDARD PROCESSES monitor_lh -type NLINK_LH STANDARD PROCESSES nlink_lh +type MONITOR_EXT STANDARD PROCESSES monitor_extended +type LINK_EXT STANDARD PROCESSES link_extended type CODE LONG_LIVED CODE code type LITERAL LITERAL CODE literal type LITERAL_REF SHORT_LIVED CODE literal_area_ref @@ -340,8 +345,8 @@ type LL_TEMP_TERM LONG_LIVED SYSTEM ll_temp_term type NIF_TRAP_EXPORT STANDARD PROCESSES nif_trap_export_entry type NIF_EXP_TRACE FIXED_SIZE PROCESSES nif_export_trace type EXPORT LONG_LIVED CODE export_entry -type MONITOR_SH FIXED_SIZE PROCESSES monitor_sh -type NLINK_SH FIXED_SIZE PROCESSES nlink_sh +type MONITOR FIXED_SIZE PROCESSES monitor +type LINK FIXED_SIZE PROCESSES link type AINFO_REQ SHORT_LIVED SYSTEM alloc_info_request type SCHED_WTIME_REQ SHORT_LIVED SYSTEM sched_wall_time_request type GC_INFO_REQ SHORT_LIVED SYSTEM gc_info_request diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index e148be7af6..fdf355d503 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -48,6 +48,8 @@ #include "erl_mseg.h" #include "erl_threads.h" #include "erl_thr_progress.h" +#include "erl_bif_unique.h" +#include "erl_nif.h" #ifdef ERTS_ENABLE_LOCK_COUNT #include "erl_lock_count.h" @@ -139,6 +141,33 @@ MBC after deallocating first block: [Carrier_t|pad|Block_t 111| udata... ] */ +/* Allocation tags ... + * + * These are added to the footer of every block when enabled. Currently they + * consist of the allocation type and an atom identifying the allocating + * driver/nif (or 'system' if that can't be determined), but the format is not + * supposed to be set in stone. + * + * The packing scheme requires that the atom values are small enough to fit + * into a word with ERTS_ALC_N_BITS to spare. Users must check for overflow + * before MAKE_ATAG(). */ + +typedef UWord alcu_atag_t; + +#define MAKE_ATAG(IdAtom, Type) \ + (ASSERT((Type) >= ERTS_ALC_N_MIN && (Type) <= ERTS_ALC_N_MAX), \ + ASSERT(atom_val(IdAtom) <= MAX_ATAG_ATOM_ID), \ + (atom_val(IdAtom) << ERTS_ALC_N_BITS) | (Type)) + +#define ATAG_ID(AT) (make_atom((AT) >> ERTS_ALC_N_BITS)) +#define ATAG_TYPE(AT) ((AT) & ERTS_ALC_N_MASK) + +#define MAX_ATAG_ATOM_ID (ERTS_UWORD_MAX >> ERTS_ALC_N_BITS) + +#define DBG_IS_VALID_ATAG(Allocator, AT) \ + (ATAG_TYPE(AT) >= ERTS_ALC_N_MIN && \ + ATAG_TYPE(AT) <= ERTS_ALC_N_MAX && \ + (Allocator)->alloc_no == ERTS_ALC_T2A(ERTS_ALC_N2T(ATAG_TYPE(AT)))) /* Blocks ... */ @@ -153,10 +182,17 @@ MBC after deallocating first block: #endif #define FBLK_FTR_SZ (sizeof(FreeBlkFtr_t)) +#define GET_BLK_ATAG(B) \ + (((alcu_atag_t *) (((char *) (B)) + (BLK_SZ(B))))[-1]) +#define SET_BLK_ATAG(B, T) \ + (((alcu_atag_t *) (((char *) (B)) + (BLK_SZ(B))))[-1] = (T)) + +#define BLK_ATAG_SZ(AP) ((AP)->atags ? sizeof(alcu_atag_t) : 0) + #define UMEMSZ2BLKSZ(AP, SZ) \ - (ABLK_HDR_SZ + (SZ) <= (AP)->min_block_size \ + (ABLK_HDR_SZ + BLK_ATAG_SZ(AP) + (SZ) <= (AP)->min_block_size \ ? (AP)->min_block_size \ - : UNIT_CEILING(ABLK_HDR_SZ + (SZ))) + : UNIT_CEILING(ABLK_HDR_SZ + BLK_ATAG_SZ(AP) + (SZ))) #define UMEM2BLK(P) ((Block_t *) (((char *) (P)) - ABLK_HDR_SZ)) #define BLK2UMEM(P) ((void *) (((char *) (P)) + ABLK_HDR_SZ)) @@ -688,6 +724,62 @@ static void destroy_carrier(Allctr_t *, Block_t *, Carrier_t **); static void mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp); static void dealloc_block(Allctr_t *, void *, ErtsAlcFixList_t *, int); +static alcu_atag_t determine_alloc_tag(Allctr_t *allocator, ErtsAlcType_t type) +{ + ErtsSchedulerData *esdp; + Eterm id; + + ERTS_CT_ASSERT(_unchecked_atom_val(am_system) <= MAX_ATAG_ATOM_ID); + ASSERT(allocator->atags); + + esdp = erts_get_scheduler_data(); + id = am_system; + + if (esdp) { + if (esdp->current_nif) { + Module *mod = erts_nif_get_module((esdp->current_nif)->mod_nif); + + /* Mod can be NULL if a resource destructor allocates memory after + * the module has been unloaded. */ + if (mod) { + id = make_atom(mod->module); + } + } else if (esdp->current_port) { + Port *p = esdp->current_port; + id = (p->drv_ptr)->name_atom; + } + + /* We fall back to 'system' if we can't pack the driver/NIF name into + * the tag. This may be a bit misleading but we've made no promises + * that the information is complete. + * + * This can only happen on 32-bit emulators when a new driver/NIF has + * been loaded *after* 16 million atoms have been used, and supporting + * that fringe case is not worth an extra word. 64-bit emulators are + * unaffected since the atom cache limits atom indexes to 32 bits. */ + if(MAX_ATOM_TABLE_SIZE > MAX_ATAG_ATOM_ID) { + if (atom_val(id) > MAX_ATAG_ATOM_ID) { + id = am_system; + } + } + } + + return MAKE_ATAG(id, type); +} + +static void set_alloc_tag(Allctr_t *allocator, void *p, alcu_atag_t tag) +{ + Block_t *block; + + ASSERT(DBG_IS_VALID_ATAG(allocator, tag)); + ASSERT(allocator->atags && p); + (void)allocator; + + block = UMEM2BLK(p); + + SET_BLK_ATAG(block, tag); +} + /* internal data... */ #if 0 @@ -4242,6 +4334,7 @@ static struct { Eterm e; Eterm t; Eterm ramv; + Eterm atags; #if HAVE_ERTS_MSEG Eterm asbcst; Eterm rsbcst; @@ -4311,7 +4404,7 @@ static struct { #endif } am; -static Eterm fix_type_atoms[ERTS_ALC_NO_FIXED_SIZES]; +static Eterm alloc_type_atoms[ERTS_ALC_N_MAX + 1]; static ERTS_INLINE void atom_init(Eterm *atom, char *name) { @@ -4342,6 +4435,7 @@ init_atoms(Allctr_t *allctr) AM_INIT(e); AM_INIT(t); AM_INIT(ramv); + AM_INIT(atags); #if HAVE_ERTS_MSEG AM_INIT(asbcst); AM_INIT(rsbcst); @@ -4413,12 +4507,12 @@ init_atoms(Allctr_t *allctr) } #endif - for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) { - ErtsAlcType_t n = ERTS_ALC_N_MIN_A_FIXED_SIZE + ix; - char *name = (char *) ERTS_ALC_N2TD(n); - size_t len = sys_strlen(name); - fix_type_atoms[ix] = am_atom_put(name, len); - } + for (ix = ERTS_ALC_N_MIN; ix <= ERTS_ALC_N_MAX; ix++) { + const char *name = ERTS_ALC_N2TD(ix); + size_t len = sys_strlen(name); + + alloc_type_atoms[ix] = am_atom_put(name, len); + } } if (allctr && !allctr->atoms_initialized) { @@ -4531,6 +4625,7 @@ sz_info_fix(Allctr_t *allctr, ErtsAlcFixList_t *fix = &allctr->fix[ix]; UWord alloced = fix->type_size * fix->u.cpool.allocated; UWord used = fix->type_size * fix->u.cpool.used; + ErtsAlcType_t n = ERTS_ALC_N_MIN_A_FIXED_SIZE + ix; if (print_to_p) { fmtfn_t to = *print_to_p; @@ -4538,15 +4633,14 @@ sz_info_fix(Allctr_t *allctr, erts_print(to, arg, "fix type internal: %s %bpu %bpu\n", - (char *) ERTS_ALC_N2TD(ERTS_ALC_N_MIN_A_FIXED_SIZE - + ix), + (char *) ERTS_ALC_N2TD(n), alloced, used); } if (hpp || szp) { add_3tup(hpp, szp, &res, - fix_type_atoms[ix], + alloc_type_atoms[n], bld_unstable_uint(hpp, szp, alloced), bld_unstable_uint(hpp, szp, used)); } @@ -4559,6 +4653,7 @@ sz_info_fix(Allctr_t *allctr, ErtsAlcFixList_t *fix = &allctr->fix[ix]; UWord alloced = fix->type_size * fix->u.nocpool.allocated; UWord used = fix->type_size*fix->u.nocpool.used; + ErtsAlcType_t n = ERTS_ALC_N_MIN_A_FIXED_SIZE + ix; if (print_to_p) { fmtfn_t to = *print_to_p; @@ -4566,15 +4661,14 @@ sz_info_fix(Allctr_t *allctr, erts_print(to, arg, "fix type: %s %bpu %bpu\n", - (char *) ERTS_ALC_N2TD(ERTS_ALC_N_MIN_A_FIXED_SIZE - + ix), + (char *) ERTS_ALC_N2TD(n), alloced, used); } if (hpp || szp) { add_3tup(hpp, szp, &res, - fix_type_atoms[ix], + alloc_type_atoms[n], bld_unstable_uint(hpp, szp, alloced), bld_unstable_uint(hpp, szp, used)); } @@ -5000,6 +5094,7 @@ info_options(Allctr_t *allctr, "option e: true\n" "option t: %s\n" "option ramv: %s\n" + "option atags: %s\n" "option sbct: %beu\n" #if HAVE_ERTS_MSEG "option asbcst: %bpu\n" @@ -5018,6 +5113,7 @@ info_options(Allctr_t *allctr, "option acul: %bpu\n", topt, allctr->ramv ? "true" : "false", + allctr->atags ? "true" : "false", allctr->sbc_threshold, #if HAVE_ERTS_MSEG allctr->mseg_opt.abs_shrink_th, @@ -5087,6 +5183,7 @@ info_options(Allctr_t *allctr, am_sbct, bld_uint(hpp, szp, allctr->sbc_threshold)); add_2tup(hpp, szp, &res, am.ramv, allctr->ramv ? am_true : am_false); + add_2tup(hpp, szp, &res, am.atags, allctr->atags ? am_true : am_false); add_2tup(hpp, szp, &res, am.t, (allctr->t ? am_true : am_false)); add_2tup(hpp, szp, &res, am.e, am_true); } @@ -5408,9 +5505,8 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t * /* ----------------------------------------------------------------------- */ static ERTS_INLINE void * -do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) +do_erts_alcu_alloc(ErtsAlcType_t type, Allctr_t *allctr, Uint size) { - Allctr_t *allctr = (Allctr_t *) extra; void *res; ASSERT(initialized); @@ -5449,10 +5545,19 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) void *erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) { + Allctr_t *allctr = (Allctr_t *) extra; void *res; + ASSERT(!"This is not thread safe"); - res = do_erts_alcu_alloc(type, extra, size); + + res = do_erts_alcu_alloc(type, allctr, size); + + if (allctr->atags && res) { + set_alloc_tag(allctr, res, determine_alloc_tag(allctr, type)); + } + DEBUG_CHECK_ALIGNMENT(res); + return res; } @@ -5462,13 +5567,25 @@ void * erts_alcu_alloc_ts(ErtsAlcType_t type, void *extra, Uint size) { Allctr_t *allctr = (Allctr_t *) extra; + alcu_atag_t tag = 0; void *res; + + if (allctr->atags) { + tag = determine_alloc_tag(allctr, type); + } + erts_mtx_lock(&allctr->mutex); - res = do_erts_alcu_alloc(type, extra, size); - DEBUG_CHECK_ALIGNMENT(res); + res = do_erts_alcu_alloc(type, allctr, size); + + if (allctr->atags && res) { + set_alloc_tag(allctr, res, tag); + } erts_mtx_unlock(&allctr->mutex); + + DEBUG_CHECK_ALIGNMENT(res); + return res; } @@ -5478,6 +5595,7 @@ erts_alcu_alloc_thr_spec(ErtsAlcType_t type, void *extra, Uint size) { ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; int ix; + alcu_atag_t tag = 0; Allctr_t *allctr; void *res; @@ -5487,11 +5605,19 @@ erts_alcu_alloc_thr_spec(ErtsAlcType_t type, void *extra, Uint size) allctr = tspec->allctr[ix]; + if (allctr->atags) { + tag = determine_alloc_tag(allctr, type); + } + if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); res = do_erts_alcu_alloc(type, allctr, size); + if (allctr->atags && res) { + set_alloc_tag(allctr, res, tag); + } + if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); @@ -5504,10 +5630,15 @@ void * erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size) { Allctr_t *pref_allctr; + alcu_atag_t tag = 0; void *res; pref_allctr = get_pref_allctr(extra); + if (pref_allctr->atags) { + tag = determine_alloc_tag(pref_allctr, type); + } + if (pref_allctr->thread_safe) erts_mtx_lock(&pref_allctr->mutex); @@ -5523,12 +5654,15 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size) res = do_erts_alcu_alloc(type, pref_allctr, size); } + if (pref_allctr->atags && res) { + set_alloc_tag(pref_allctr, res, tag); + } + if (pref_allctr->thread_safe) erts_mtx_unlock(&pref_allctr->mutex); DEBUG_CHECK_ALIGNMENT(res); - return res; } @@ -5537,10 +5671,9 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size) /* ------------------------------------------------------------------------- */ static ERTS_INLINE void -do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p, +do_erts_alcu_free(ErtsAlcType_t type, Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp) { - Allctr_t *allctr = (Allctr_t *) extra; ASSERT(initialized); ASSERT(allctr); @@ -5572,7 +5705,8 @@ do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p, void erts_alcu_free(ErtsAlcType_t type, void *extra, void *p) { - do_erts_alcu_free(type, extra, p, NULL); + Allctr_t *allctr = (Allctr_t *) extra; + do_erts_alcu_free(type, allctr, p, NULL); } @@ -5581,7 +5715,7 @@ erts_alcu_free_ts(ErtsAlcType_t type, void *extra, void *p) { Allctr_t *allctr = (Allctr_t *) extra; erts_mtx_lock(&allctr->mutex); - do_erts_alcu_free(type, extra, p, NULL); + do_erts_alcu_free(type, allctr, p, NULL); erts_mtx_unlock(&allctr->mutex); } @@ -5641,13 +5775,12 @@ erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p) static ERTS_INLINE void * do_erts_alcu_realloc(ErtsAlcType_t type, - void *extra, + Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, Carrier_t **busy_pcrr_pp) { - Allctr_t *allctr = (Allctr_t *) extra; Block_t *blk; void *res; @@ -5661,7 +5794,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); if (!p) { - res = do_erts_alcu_alloc(type, extra, size); + res = do_erts_alcu_alloc(type, allctr, size); INC_CC(allctr->calls.this_realloc); DEC_CC(allctr->calls.this_alloc); return res; @@ -5670,7 +5803,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, #if ALLOC_ZERO_EQ_NULL if (!size) { ASSERT(p); - do_erts_alcu_free(type, extra, p, busy_pcrr_pp); + do_erts_alcu_free(type, allctr, p, busy_pcrr_pp); INC_CC(allctr->calls.this_realloc); DEC_CC(allctr->calls.this_free); return NULL; @@ -5755,19 +5888,29 @@ do_erts_alcu_realloc(ErtsAlcType_t type, void * erts_alcu_realloc(ErtsAlcType_t type, void *extra, void *p, Uint size) { + Allctr_t *allctr = (Allctr_t *)extra; void *res; - res = do_erts_alcu_realloc(type, extra, p, size, 0, NULL); + + res = do_erts_alcu_realloc(type, allctr, p, size, 0, NULL); + DEBUG_CHECK_ALIGNMENT(res); + + if (allctr->atags && res) { + set_alloc_tag(allctr, res, determine_alloc_tag(allctr, type)); + } + return res; } void * erts_alcu_realloc_mv(ErtsAlcType_t type, void *extra, void *p, Uint size) { + Allctr_t *allctr = (Allctr_t *)extra; void *res; - res = do_erts_alcu_alloc(type, extra, size); + + res = do_erts_alcu_alloc(type, allctr, size); if (!res) - res = erts_alcu_realloc(type, extra, p, size); + res = do_erts_alcu_realloc(type, allctr, p, size, 0, NULL); else { Block_t *blk; size_t cpy_size; @@ -5777,23 +5920,42 @@ erts_alcu_realloc_mv(ErtsAlcType_t type, void *extra, void *p, Uint size) if (cpy_size > size) cpy_size = size; sys_memcpy(res, p, cpy_size); - do_erts_alcu_free(type, extra, p, NULL); + do_erts_alcu_free(type, allctr, p, NULL); } + DEBUG_CHECK_ALIGNMENT(res); + + if (allctr->atags && res) { + set_alloc_tag(allctr, res, determine_alloc_tag(allctr, type)); + } + return res; } - void * erts_alcu_realloc_ts(ErtsAlcType_t type, void *extra, void *ptr, Uint size) { Allctr_t *allctr = (Allctr_t *) extra; + alcu_atag_t tag = 0; void *res; + + if (allctr->atags) { + tag = determine_alloc_tag(allctr, type); + } + erts_mtx_lock(&allctr->mutex); - res = do_erts_alcu_realloc(type, extra, ptr, size, 0, NULL); + + res = do_erts_alcu_realloc(type, allctr, ptr, size, 0, NULL); + + if (allctr->atags && res) { + set_alloc_tag(allctr, res, tag); + } + erts_mtx_unlock(&allctr->mutex); + DEBUG_CHECK_ALIGNMENT(res); + return res; } @@ -5801,11 +5963,17 @@ void * erts_alcu_realloc_mv_ts(ErtsAlcType_t type, void *extra, void *p, Uint size) { Allctr_t *allctr = (Allctr_t *) extra; + alcu_atag_t tag = 0; void *res; + + if (allctr->atags) { + tag = determine_alloc_tag(allctr, type); + } + erts_mtx_lock(&allctr->mutex); - res = do_erts_alcu_alloc(type, extra, size); + res = do_erts_alcu_alloc(type, allctr, size); if (!res) - res = erts_alcu_realloc_ts(type, extra, p, size); + res = do_erts_alcu_realloc(type, allctr, p, size, 0, NULL); else { Block_t *blk; size_t cpy_size; @@ -5815,10 +5983,17 @@ erts_alcu_realloc_mv_ts(ErtsAlcType_t type, void *extra, void *p, Uint size) if (cpy_size > size) cpy_size = size; sys_memcpy(res, p, cpy_size); - do_erts_alcu_free(type, extra, p, NULL); + do_erts_alcu_free(type, allctr, p, NULL); } + + if (allctr->atags && res) { + set_alloc_tag(allctr, res, tag); + } + erts_mtx_unlock(&allctr->mutex); + DEBUG_CHECK_ALIGNMENT(res); + return res; } @@ -5829,6 +6004,7 @@ erts_alcu_realloc_thr_spec(ErtsAlcType_t type, void *extra, { ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; int ix; + alcu_atag_t tag = 0; Allctr_t *allctr; void *res; @@ -5838,11 +6014,19 @@ erts_alcu_realloc_thr_spec(ErtsAlcType_t type, void *extra, allctr = tspec->allctr[ix]; + if (allctr->atags) { + tag = determine_alloc_tag(allctr, type); + } + if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); res = do_erts_alcu_realloc(type, allctr, ptr, size, 0, NULL); + if (allctr->atags && res) { + set_alloc_tag(allctr, res, tag); + } + if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); @@ -5857,6 +6041,7 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra, { ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; int ix; + alcu_atag_t tag = 0; Allctr_t *allctr; void *res; @@ -5866,14 +6051,16 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra, allctr = tspec->allctr[ix]; + if (allctr->atags) { + tag = determine_alloc_tag(allctr, type); + } + if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); res = do_erts_alcu_alloc(type, allctr, size); if (!res) { - if (allctr->thread_safe) - erts_mtx_unlock(&allctr->mutex); - res = erts_alcu_realloc_thr_spec(type, allctr, ptr, size); + res = do_erts_alcu_realloc(type, allctr, ptr, size, 0, NULL); } else { Block_t *blk; @@ -5885,29 +6072,34 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra, cpy_size = size; sys_memcpy(res, ptr, cpy_size); do_erts_alcu_free(type, allctr, ptr, NULL); - if (allctr->thread_safe) - erts_mtx_unlock(&allctr->mutex); } + if (allctr->atags && res) { + set_alloc_tag(allctr, res, tag); + } + + if (allctr->thread_safe) + erts_mtx_unlock(&allctr->mutex); + DEBUG_CHECK_ALIGNMENT(res); return res; } static ERTS_INLINE void * -realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size, +realloc_thr_pref(ErtsAlcType_t type, Allctr_t *pref_allctr, void *p, Uint size, int force_move) { void *res; - Allctr_t *pref_allctr, *used_allctr; + Allctr_t *used_allctr; UWord old_user_size; Carrier_t *busy_pcrr_p; + alcu_atag_t tag = 0; int retried; - if (!p) - return erts_alcu_alloc_thr_pref(type, extra, size); - - pref_allctr = get_pref_allctr(extra); + if (pref_allctr->atags) { + tag = determine_alloc_tag(pref_allctr, type); + } if (pref_allctr->thread_safe) erts_mtx_lock(&pref_allctr->mutex); @@ -5936,6 +6128,11 @@ restart: retried = 1; goto restart; } + + if (pref_allctr->atags && res) { + set_alloc_tag(pref_allctr, res, tag); + } + if (pref_allctr->thread_safe) erts_mtx_unlock(&pref_allctr->mutex); } @@ -5944,6 +6141,9 @@ restart: if (!res) goto unlock_ts_return; else { + if (pref_allctr->atags) { + set_alloc_tag(pref_allctr, res, tag); + } DEBUG_CHECK_ALIGNMENT(res); @@ -5974,20 +6174,34 @@ restart: } } + DEBUG_CHECK_ALIGNMENT(res); + return res; } void * erts_alcu_realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size) { - return realloc_thr_pref(type, extra, p, size, 0); + if (p) { + Allctr_t *pref_allctr = get_pref_allctr(extra); + + return realloc_thr_pref(type, pref_allctr, p, size, 0); + } + + return erts_alcu_alloc_thr_pref(type, extra, size); } void * erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size) { - return realloc_thr_pref(type, extra, p, size, 1); + if (p) { + Allctr_t *pref_allctr = get_pref_allctr(extra); + + return realloc_thr_pref(type, pref_allctr, p, size, 1); + } + + return erts_alcu_alloc_thr_pref(type, extra, size); } @@ -6071,6 +6285,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->t = 0; allctr->ramv = init->ramv; + allctr->atags = init->atags; allctr->main_carrier_size = init->mmbcs; #if HAVE_ERTS_MSEG @@ -6320,6 +6535,1072 @@ erts_alcu_init(AlcUInit_t *init) initialized = 1; } +/* ------------------------------------------------------------------------- */ + +/* Allocation histograms and carrier information is gathered by walking through + * all carriers associated with each allocator instance. This is done as + * aux_yield_work on the scheduler that owns each instance. + * + * Yielding is implemented by temporarily inserting a "dummy carrier" at the + * last position. It's permanently "busy" so it won't get picked up by someone + * else when in the carrier pool, and we never make the employer aware of it + * through callbacks so we can't accidentally allocate on it. + * + * Plain malloc/free is used to guarantee we won't allocate with the allocator + * we're scanning. */ + +/* Yield between carriers once this many blocks have been processed. Note that + * a single carrier scan may exceed this figure. */ +#ifndef DEBUG + #define BLOCKSCAN_REDUCTIONS (8000) +#else + #define BLOCKSCAN_REDUCTIONS (400) +#endif + +/* Abort a single carrier scan after this many blocks to prevent really large + * MBCs from blocking forever. */ +#define BLOCKSCAN_BAILOUT_THRESHOLD (16000) + +typedef struct alcu_blockscan { + /* A per-scheduler list used when multiple scans have been queued. The + * current scanner will always run until completion/abort before moving on + * to the next. */ + struct alcu_blockscan *scanner_queue; + + Allctr_t *allocator; + Process *process; + + int (*current_op)(struct alcu_blockscan *scanner); + int (*next_op)(struct alcu_blockscan *scanner); + int reductions; + + ErtsAlcCPoolData_t *cpool_cursor; + CarrierList_t *current_clist; + Carrier_t *clist_cursor; + Carrier_t dummy_carrier; + + /* Called if the process that started this job dies before we're done. */ + void (*abort)(void *user_data); + + /* Called on each carrier. The callback must return the number of blocks + * scanned to yield properly between carriers. + * + * Note that it's not possible to "yield back" into a carrier. */ + int (*scan)(Allctr_t *, void *user_data, Carrier_t *); + + /* Called when all carriers have been scanned. The callback may return + * non-zero to yield. */ + int (*finish)(void *user_data); + + void *user_data; +} blockscan_t; + +static Carrier_t *blockscan_restore_clist_cursor(blockscan_t *state) +{ + Carrier_t *cursor = state->clist_cursor; + + ASSERT(state->clist_cursor == (state->current_clist)->first || + state->clist_cursor == &state->dummy_carrier); + + if (cursor == &state->dummy_carrier) { + cursor = cursor->next; + + unlink_carrier(state->current_clist, state->clist_cursor); + } + + return cursor; +} + +static void blockscan_save_clist_cursor(blockscan_t *state, Carrier_t *after) +{ + ASSERT(state->clist_cursor == (state->current_clist)->first || + state->clist_cursor == &state->dummy_carrier); + + state->clist_cursor = &state->dummy_carrier; + + (state->clist_cursor)->next = after->next; + (state->clist_cursor)->prev = after; + + relink_carrier(state->current_clist, state->clist_cursor); +} + +static int blockscan_clist_yielding(blockscan_t *state) +{ + Carrier_t *cursor = blockscan_restore_clist_cursor(state); + + if (ERTS_PROC_IS_EXITING(state->process)) { + return 0; + } + + while (cursor) { + /* Skip dummy carriers inserted by another (concurrent) block scan. + * This can happen when scanning thread-safe allocators from multiple + * schedulers. */ + if (CARRIER_SZ(cursor) > 0) { + int blocks_scanned = state->scan(state->allocator, + state->user_data, + cursor); + + state->reductions -= blocks_scanned; + + if (state->reductions <= 0) { + blockscan_save_clist_cursor(state, cursor); + return 1; + } + } + + cursor = cursor->next; + } + + return 0; +} + +static ErtsAlcCPoolData_t *blockscan_restore_cpool_cursor(blockscan_t *state) +{ + ErtsAlcCPoolData_t *cursor; + + cursor = cpool_aint2cpd(cpool_read(&(state->cpool_cursor)->next)); + + if (state->cpool_cursor == &state->dummy_carrier.cpool) { + cpool_delete(state->allocator, state->allocator, &state->dummy_carrier); + } + + return cursor; +} + +static void blockscan_save_cpool_cursor(blockscan_t *state, + ErtsAlcCPoolData_t *after) +{ + ErtsAlcCPoolData_t *dummy_carrier, *prev_carrier, *next_carrier; + + dummy_carrier = &state->dummy_carrier.cpool; + + next_carrier = cpool_aint2cpd(cpool_mod_mark(&after->next)); + prev_carrier = cpool_aint2cpd(cpool_mod_mark(&next_carrier->prev)); + + cpool_init(&dummy_carrier->next, (erts_aint_t)next_carrier); + cpool_init(&dummy_carrier->prev, (erts_aint_t)prev_carrier); + + cpool_set_mod_marked(&prev_carrier->next, + (erts_aint_t)dummy_carrier, + (erts_aint_t)next_carrier); + cpool_set_mod_marked(&next_carrier->prev, + (erts_aint_t)dummy_carrier, + (erts_aint_t)prev_carrier); + + state->cpool_cursor = dummy_carrier; +} + +static int blockscan_cpool_yielding(blockscan_t *state) +{ + ErtsAlcCPoolData_t *sentinel, *cursor; + + sentinel = &carrier_pool[(state->allocator)->alloc_no].sentinel; + cursor = blockscan_restore_cpool_cursor(state); + + if (ERTS_PROC_IS_EXITING(state->process)) { + return 0; + } + + while (cursor != sentinel) { + Carrier_t *carrier; + erts_aint_t exp; + + /* When a deallocation happens on a pooled carrier it will be routed to + * its owner, so the only way to be sure that it isn't modified while + * scanning is to skip all carriers that aren't ours. The deallocations + * deferred to us will get handled when we're done. */ + while (cursor->orig_allctr != state->allocator) { + cursor = cpool_aint2cpd(cpool_read(&cursor->next)); + + if (cursor == sentinel) { + return 0; + } + } + + carrier = ErtsContainerStruct(cursor, Carrier_t, cpool); + exp = erts_atomic_read_rb(&carrier->allctr); + + if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { + ASSERT(state->allocator == (Allctr_t*)(exp & ~ERTS_CRR_ALCTR_FLG_MASK)); + ASSERT(!(exp & ERTS_CRR_ALCTR_FLG_BUSY)); + + if (erts_atomic_cmpxchg_acqb(&carrier->allctr, + exp | ERTS_CRR_ALCTR_FLG_BUSY, + exp) == exp) { + /* Skip dummy carriers inserted by another (concurrent) block + * scan. This can happen when scanning thread-safe allocators + * from multiple schedulers. */ + if (CARRIER_SZ(carrier) > 0) { + int blocks_scanned = state->scan(state->allocator, + state->user_data, + carrier); + + state->reductions -= blocks_scanned; + + if (state->reductions <= 0) { + blockscan_save_cpool_cursor(state, cursor); + erts_atomic_set_relb(&carrier->allctr, exp); + + return 1; + } + } + + erts_atomic_set_relb(&carrier->allctr, exp); + } + } + + cursor = cpool_aint2cpd(cpool_read(&cursor->next)); + } + + return 0; +} + +static int blockscan_yield_helper(blockscan_t *state, + int (*yielding_op)(blockscan_t*)) +{ + /* Note that we don't check whether to abort here; only yielding_op knows + * whether the carrier is still in the list/pool. */ + + if ((state->allocator)->thread_safe) { + /* Locked scans have to be as short as possible. */ + state->reductions = 1; + + erts_mtx_lock(&(state->allocator)->mutex); + } else { + state->reductions = BLOCKSCAN_REDUCTIONS; + } + + if (yielding_op(state)) { + state->next_op = state->current_op; + } + + if ((state->allocator)->thread_safe) { + erts_mtx_unlock(&(state->allocator)->mutex); + } + + return 1; +} + +/* */ + +static int blockscan_finish(blockscan_t *state) +{ + if (ERTS_PROC_IS_EXITING(state->process)) { + state->abort(state->user_data); + return 0; + } + + state->current_op = blockscan_finish; + + return state->finish(state->user_data); +} + +static int blockscan_sweep_sbcs(blockscan_t *state) +{ + if (state->current_op != blockscan_sweep_sbcs) { + SET_CARRIER_HDR(&state->dummy_carrier, 0, SCH_SBC, state->allocator); + state->current_clist = &(state->allocator)->sbc_list; + state->clist_cursor = (state->current_clist)->first; + } + + state->current_op = blockscan_sweep_sbcs; + state->next_op = blockscan_finish; + + return blockscan_yield_helper(state, blockscan_clist_yielding); +} + +static int blockscan_sweep_mbcs(blockscan_t *state) +{ + if (state->current_op != blockscan_sweep_mbcs) { + SET_CARRIER_HDR(&state->dummy_carrier, 0, SCH_MBC, state->allocator); + state->current_clist = &(state->allocator)->mbc_list; + state->clist_cursor = (state->current_clist)->first; + } + + state->current_op = blockscan_sweep_mbcs; + state->next_op = blockscan_sweep_sbcs; + + return blockscan_yield_helper(state, blockscan_clist_yielding); +} + +static int blockscan_sweep_cpool(blockscan_t *state) +{ + if (state->current_op != blockscan_sweep_cpool) { + ErtsAlcCPoolData_t *sentinel; + + SET_CARRIER_HDR(&state->dummy_carrier, 0, SCH_MBC, state->allocator); + sentinel = &carrier_pool[(state->allocator)->alloc_no].sentinel; + state->cpool_cursor = sentinel; + } + + state->current_op = blockscan_sweep_cpool; + state->next_op = blockscan_sweep_mbcs; + + return blockscan_yield_helper(state, blockscan_cpool_yielding); +} + +static int blockscan_get_specific_allocator(int allocator_num, + int sched_id, + Allctr_t **out) +{ + ErtsAllocatorInfo_t *ai; + Allctr_t *allocator; + + ASSERT(allocator_num >= ERTS_ALC_A_MIN && + allocator_num <= ERTS_ALC_A_MAX); + ASSERT(sched_id >= 0 && sched_id <= erts_no_schedulers); + + ai = &erts_allctrs_info[allocator_num]; + + if (!ai->enabled || !ai->alloc_util) { + return 0; + } + + if (!ai->thr_spec) { + if (sched_id != 0) { + /* Only thread-specific allocators can be scanned on a specific + * scheduler. */ + return 0; + } + + allocator = (Allctr_t*)ai->extra; + ASSERT(allocator->thread_safe); + } else { + ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t*)ai->extra; + + ASSERT(sched_id < tspec->size); + + allocator = tspec->allctr[sched_id]; + } + + *out = allocator; + + return 1; +} + +static void blockscan_sched_trampoline(void *arg) +{ + ErtsAlcuBlockscanYieldData *yield; + ErtsSchedulerData *esdp; + blockscan_t *scanner; + + esdp = erts_get_scheduler_data(); + scanner = (blockscan_t*)arg; + + yield = ERTS_SCHED_AUX_YIELD_DATA(esdp, alcu_blockscan); + + ASSERT((yield->last == NULL) == (yield->current == NULL)); + + if (yield->last != NULL) { + blockscan_t *prev_scanner = yield->last; + + ASSERT(prev_scanner->scanner_queue == NULL); + + prev_scanner->scanner_queue = scanner; + } else { + yield->current = scanner; + } + + scanner->scanner_queue = NULL; + yield->last = scanner; + + erts_notify_new_aux_yield_work(esdp); +} + +static void blockscan_dispatch(blockscan_t *scanner, Process *owner, + Allctr_t *allocator, int sched_id) +{ + ASSERT(erts_get_scheduler_id() != 0); + + if (sched_id == 0) { + /* Global instances are always handled on the current scheduler. */ + sched_id = ERTS_ALC_GET_THR_IX(); + ASSERT(allocator->thread_safe); + } + + scanner->allocator = allocator; + scanner->process = owner; + + erts_proc_inc_refc(scanner->process); + + cpool_init_carrier_data(scanner->allocator, &scanner->dummy_carrier); + erts_atomic_init_nob(&(scanner->dummy_carrier).allctr, + (erts_aint_t)allocator | ERTS_CRR_ALCTR_FLG_BUSY); + + if (ERTS_ALC_IS_CPOOL_ENABLED(scanner->allocator)) { + scanner->next_op = blockscan_sweep_cpool; + } else { + scanner->next_op = blockscan_sweep_mbcs; + } + + /* Aux yield jobs can only be set up while running on the scheduler that + * services them, so we move there before continuing. + * + * We can't drive the scan itself through this since the scheduler will + * always finish *all* misc aux work in one go which makes it impossible to + * yield. */ + erts_schedule_misc_aux_work(sched_id, blockscan_sched_trampoline, scanner); +} + +int erts_handle_yielded_alcu_blockscan(ErtsSchedulerData *esdp, + ErtsAlcuBlockscanYieldData *yield) +{ + blockscan_t *scanner = yield->current; + + (void)esdp; + + ASSERT((yield->last == NULL) == (yield->current == NULL)); + + if (scanner) { + if (scanner->next_op(scanner)) { + return 1; + } + + ASSERT(ERTS_PROC_IS_EXITING(scanner->process) || + scanner->current_op == blockscan_finish); + + yield->current = scanner->scanner_queue; + + if (yield->current == NULL) { + ASSERT(scanner == yield->last); + yield->last = NULL; + } + + erts_proc_dec_refc(scanner->process); + + /* Plain free is intentional. */ + free(scanner); + + return yield->current != NULL; + } + + return 0; +} + +void erts_alcu_sched_spec_data_init(ErtsSchedulerData *esdp) +{ + ErtsAlcuBlockscanYieldData *yield; + + yield = ERTS_SCHED_AUX_YIELD_DATA(esdp, alcu_blockscan); + + yield->current = NULL; + yield->last = NULL; +} + +/* ------------------------------------------------------------------------- */ + +static ERTS_INLINE int u64_log2(Uint64 v) +{ + static const int log2_tab64[64] = { + 63, 0, 58, 1, 59, 47, 53, 2, + 60, 39, 48, 27, 54, 33, 42, 3, + 61, 51, 37, 40, 49, 18, 28, 20, + 55, 30, 34, 11, 43, 14, 22, 4, + 62, 57, 46, 52, 38, 26, 32, 41, + 50, 36, 17, 19, 29, 10, 13, 21, + 56, 45, 25, 31, 35, 16, 9, 12, + 44, 24, 15, 8, 23, 7, 6, 5}; + + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v |= v >> 32; + + return log2_tab64[((Uint64)((v - (v >> 1))*0x07EDD5E59A4E28C2)) >> 58]; +} + +/* ------------------------------------------------------------------------- */ + +typedef struct hist_tree__ { + struct hist_tree__ *parent; + struct hist_tree__ *left; + struct hist_tree__ *right; + + int is_red; + + alcu_atag_t tag; + UWord histogram[1]; +} hist_tree_t; + +#define ERTS_RBT_PREFIX hist_tree +#define ERTS_RBT_T hist_tree_t +#define ERTS_RBT_KEY_T UWord +#define ERTS_RBT_FLAGS_T int +#define ERTS_RBT_INIT_EMPTY_TNODE(T) ((void)0) +#define ERTS_RBT_IS_RED(T) ((T)->is_red) +#define ERTS_RBT_SET_RED(T) ((T)->is_red = 1) +#define ERTS_RBT_IS_BLACK(T) (!ERTS_RBT_IS_RED(T)) +#define ERTS_RBT_SET_BLACK(T) ((T)->is_red = 0) +#define ERTS_RBT_GET_FLAGS(T) ((T)->is_red) +#define ERTS_RBT_SET_FLAGS(T, F) ((T)->is_red = F) +#define ERTS_RBT_GET_PARENT(T) ((T)->parent) +#define ERTS_RBT_SET_PARENT(T, P) ((T)->parent = P) +#define ERTS_RBT_GET_RIGHT(T) ((T)->right) +#define ERTS_RBT_SET_RIGHT(T, R) ((T)->right = (R)) +#define ERTS_RBT_GET_LEFT(T) ((T)->left) +#define ERTS_RBT_SET_LEFT(T, L) ((T)->left = (L)) +#define ERTS_RBT_GET_KEY(T) ((T)->tag) +#define ERTS_RBT_IS_LT(KX, KY) (KX < KY) +#define ERTS_RBT_IS_EQ(KX, KY) (KX == KY) +#define ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING +#define ERTS_RBT_WANT_FOREACH_DESTROY +#define ERTS_RBT_WANT_INSERT +#define ERTS_RBT_WANT_LOOKUP +#define ERTS_RBT_UNDEF + +#include "erl_rbtree.h" + +typedef struct { + blockscan_t common; + + ErtsIRefStorage iref; + Process *process; + + hist_tree_rbt_yield_state_t hist_tree_yield; + hist_tree_t *hist_tree; + UWord hist_count; + + UWord hist_slot_start; + int hist_slot_count; + + UWord unscanned_size; + + ErtsHeapFactory msg_factory; + int building_result; + Eterm result_list; +} gather_ahist_t; + +static void gather_ahist_update(gather_ahist_t *state, UWord tag, UWord size) +{ + hist_tree_t *hist_node; + UWord size_interval; + int hist_slot; + + hist_node = hist_tree_rbt_lookup(state->hist_tree, tag); + + if (hist_node == NULL) { + /* Plain calloc is intentional. */ + hist_node = (hist_tree_t*)calloc(1, sizeof(hist_tree_t) + + (state->hist_slot_count - 1) * + sizeof(hist_node->histogram[0])); + hist_node->tag = tag; + + hist_tree_rbt_insert(&state->hist_tree, hist_node); + state->hist_count++; + } + + size_interval = (size / state->hist_slot_start); + size_interval = u64_log2(size_interval + 1); + + hist_slot = MIN(size_interval, state->hist_slot_count - 1); + + hist_node->histogram[hist_slot]++; +} + +static int gather_ahist_scan(Allctr_t *allocator, + void *user_data, + Carrier_t *carrier) +{ + gather_ahist_t *state; + int blocks_scanned; + Block_t *block; + + state = (gather_ahist_t*)user_data; + blocks_scanned = 1; + + if (IS_SB_CARRIER(carrier)) { + alcu_atag_t tag; + + block = SBC2BLK(allocator, carrier); + tag = GET_BLK_ATAG(block); + + ASSERT(DBG_IS_VALID_ATAG(allocator, tag)); + + gather_ahist_update(state, tag, SBC_BLK_SZ(block)); + } else { + UWord scanned_bytes = MBC_HEADER_SIZE(allocator); + + ASSERT(IS_MB_CARRIER(carrier)); + + block = MBC_TO_FIRST_BLK(allocator, carrier); + + while (1) { + UWord block_size = MBC_BLK_SZ(block); + + if (IS_ALLOCED_BLK(block)) { + alcu_atag_t tag = GET_BLK_ATAG(block); + + ASSERT(DBG_IS_VALID_ATAG(allocator, tag)); + + gather_ahist_update(state, tag, block_size); + } + + scanned_bytes += block_size; + + if (blocks_scanned >= BLOCKSCAN_BAILOUT_THRESHOLD) { + state->unscanned_size += CARRIER_SZ(carrier) - scanned_bytes; + break; + } else if (IS_LAST_BLK(block)) { + break; + } + + block = NXT_BLK(block); + blocks_scanned++; + } + } + + return blocks_scanned; +} + +static void gather_ahist_append_result(hist_tree_t *node, void *arg) +{ + gather_ahist_t *state = (gather_ahist_t*)arg; + + Eterm histogram_tuple, tag_tuple; + + Eterm *hp; + int ix; + + ASSERT(state->building_result); + + hp = erts_produce_heap(&state->msg_factory, 7 + state->hist_slot_count, 0); + + hp[0] = make_arityval(state->hist_slot_count); + + for (ix = 0; ix < state->hist_slot_count; ix++) { + hp[1 + ix] = make_small(node->histogram[ix]); + } + + histogram_tuple = make_tuple(hp); + hp += 1 + state->hist_slot_count; + + hp[0] = make_arityval(3); + hp[1] = ATAG_ID(node->tag); + hp[2] = alloc_type_atoms[ATAG_TYPE(node->tag)]; + hp[3] = histogram_tuple; + + tag_tuple = make_tuple(hp); + hp += 4; + + state->result_list = CONS(hp, tag_tuple, state->result_list); + + /* Plain free is intentional. */ + free(node); +} + +static void gather_ahist_send(gather_ahist_t *state) +{ + Eterm result_tuple, unscanned_size, task_ref; + + Uint term_size; + Eterm *hp; + + ASSERT((state->result_list == NIL) ^ (state->hist_count > 0)); + ASSERT(state->building_result); + + term_size = 4 + erts_iref_storage_heap_size(&state->iref); + term_size += IS_USMALL(0, state->unscanned_size) ? 0 : BIG_UINT_HEAP_SIZE; + + hp = erts_produce_heap(&state->msg_factory, term_size, 0); + + task_ref = erts_iref_storage_make_ref(&state->iref, &hp, + &(state->msg_factory.message)->hfrag.off_heap, 0); + + unscanned_size = bld_unstable_uint(&hp, NULL, state->unscanned_size); + + hp[0] = make_arityval(3); + hp[1] = task_ref; + hp[2] = unscanned_size; + hp[3] = state->result_list; + + result_tuple = make_tuple(hp); + + erts_factory_trim_and_close(&state->msg_factory, &result_tuple, 1); + + erts_queue_message(state->process, 0, state->msg_factory.message, + result_tuple, am_system); +} + +static int gather_ahist_finish(void *arg) +{ + gather_ahist_t *state = (gather_ahist_t*)arg; + + if (!state->building_result) { + ErtsMessage *message; + Uint minimum_size; + Eterm *hp; + + /* {Ref, unscanned size, [{Tag, {Histogram}} | Rest]} */ + minimum_size = 4 + erts_iref_storage_heap_size(&state->iref) + + state->hist_count * (7 + state->hist_slot_count); + + message = erts_alloc_message(minimum_size, &hp); + erts_factory_selfcontained_message_init(&state->msg_factory, + message, hp); + + ERTS_RBT_YIELD_STAT_INIT(&state->hist_tree_yield); + + state->result_list = NIL; + state->building_result = 1; + } + + if (hist_tree_rbt_foreach_destroy_yielding(&state->hist_tree, + &gather_ahist_append_result, + state, + &state->hist_tree_yield, + BLOCKSCAN_REDUCTIONS)) { + return 1; + } + + gather_ahist_send(state); + + return 0; +} + +static void gather_ahist_destroy_result(hist_tree_t *node, void *arg) +{ + (void)arg; + free(node); +} + +static void gather_ahist_abort(void *arg) +{ + gather_ahist_t *state = (gather_ahist_t*)arg; + + if (state->building_result) { + erts_factory_undo(&state->msg_factory); + } + + hist_tree_rbt_foreach_destroy(&state->hist_tree, + &gather_ahist_destroy_result, + NULL); +} + +int erts_alcu_gather_alloc_histograms(Process *p, int allocator_num, + int sched_id, int hist_width, + UWord hist_start, Eterm ref) +{ + gather_ahist_t *gather_state; + blockscan_t *scanner; + Allctr_t *allocator; + + ASSERT(is_internal_ref(ref)); + + if (!blockscan_get_specific_allocator(allocator_num, + sched_id, + &allocator)) { + return 0; + } else if (!allocator->atags) { + return 0; + } + + ensure_atoms_initialized(allocator); + + /* Plain calloc is intentional. */ + gather_state = (gather_ahist_t*)calloc(1, sizeof(gather_ahist_t)); + scanner = &gather_state->common; + + scanner->abort = gather_ahist_abort; + scanner->scan = gather_ahist_scan; + scanner->finish = gather_ahist_finish; + scanner->user_data = gather_state; + + erts_iref_storage_save(&gather_state->iref, ref); + gather_state->hist_slot_start = hist_start; + gather_state->hist_slot_count = hist_width; + gather_state->process = p; + + blockscan_dispatch(scanner, p, allocator, sched_id); + + return 1; +} + +/* ------------------------------------------------------------------------- */ + +typedef struct chist_node__ { + struct chist_node__ *next; + + UWord carrier_size; + UWord unscanned_size; + UWord allocated_size; + + /* BLOCKSCAN_BAILOUT_THRESHOLD guarantees we won't overflow this or the + * counters in the free block histogram. */ + int allocated_count; + int flags; + + int histogram[1]; +} chist_node_t; + +typedef struct { + blockscan_t common; + + ErtsIRefStorage iref; + Process *process; + + Eterm allocator_desc; + + chist_node_t *info_list; + UWord info_count; + + UWord hist_slot_start; + int hist_slot_count; + + ErtsHeapFactory msg_factory; + int building_result; + Eterm result_list; +} gather_cinfo_t; + +static int gather_cinfo_scan(Allctr_t *allocator, + void *user_data, + Carrier_t *carrier) +{ + gather_cinfo_t *state; + chist_node_t *node; + int blocks_scanned; + Block_t *block; + + state = (gather_cinfo_t*)user_data; + node = calloc(1, sizeof(chist_node_t) + + (state->hist_slot_count - 1) * + sizeof(node->histogram[0])); + blocks_scanned = 1; + + /* ERTS_CRR_ALCTR_FLG_BUSY is ignored since we've set it ourselves and it + * would be misleading to include it. */ + node->flags = erts_atomic_read_rb(&carrier->allctr) & + (ERTS_CRR_ALCTR_FLG_MASK & ~ERTS_CRR_ALCTR_FLG_BUSY); + node->carrier_size = CARRIER_SZ(carrier); + + if (IS_SB_CARRIER(carrier)) { + UWord block_size; + + block = SBC2BLK(allocator, carrier); + block_size = SBC_BLK_SZ(block); + + node->allocated_size = block_size; + node->allocated_count = 1; + } else { + UWord scanned_bytes = MBC_HEADER_SIZE(allocator); + + block = MBC_TO_FIRST_BLK(allocator, carrier); + + while (1) { + UWord block_size = MBC_BLK_SZ(block); + + scanned_bytes += block_size; + + if (IS_ALLOCED_BLK(block)) { + node->allocated_size += block_size; + node->allocated_count++; + } else { + UWord size_interval; + int hist_slot; + + size_interval = (block_size / state->hist_slot_start); + size_interval = u64_log2(size_interval + 1); + + hist_slot = MIN(size_interval, state->hist_slot_count - 1); + + node->histogram[hist_slot]++; + } + + if (blocks_scanned >= BLOCKSCAN_BAILOUT_THRESHOLD) { + node->unscanned_size += CARRIER_SZ(carrier) - scanned_bytes; + break; + } else if (IS_LAST_BLK(block)) { + break; + } + + block = NXT_BLK(block); + blocks_scanned++; + } + } + + node->next = state->info_list; + state->info_list = node; + state->info_count++; + + return blocks_scanned; +} + +static void gather_cinfo_append_result(gather_cinfo_t *state, + chist_node_t *info) +{ + Eterm carrier_size, unscanned_size, allocated_size; + Eterm histogram_tuple, carrier_tuple; + + Uint term_size; + Eterm *hp; + int ix; + + ASSERT(state->building_result); + + term_size = 11 + state->hist_slot_count; + term_size += IS_USMALL(0, info->carrier_size) ? 0 : BIG_UINT_HEAP_SIZE; + term_size += IS_USMALL(0, info->unscanned_size) ? 0 : BIG_UINT_HEAP_SIZE; + term_size += IS_USMALL(0, info->allocated_size) ? 0 : BIG_UINT_HEAP_SIZE; + + hp = erts_produce_heap(&state->msg_factory, term_size, 0); + + hp[0] = make_arityval(state->hist_slot_count); + + for (ix = 0; ix < state->hist_slot_count; ix++) { + hp[1 + ix] = make_small(info->histogram[ix]); + } + + histogram_tuple = make_tuple(hp); + hp += 1 + state->hist_slot_count; + + carrier_size = bld_unstable_uint(&hp, NULL, info->carrier_size); + unscanned_size = bld_unstable_uint(&hp, NULL, info->unscanned_size); + allocated_size = bld_unstable_uint(&hp, NULL, info->allocated_size); + + hp[0] = make_arityval(7); + hp[1] = state->allocator_desc; + hp[2] = carrier_size; + hp[3] = unscanned_size; + hp[4] = allocated_size; + hp[5] = make_small(info->allocated_count); + hp[6] = (info->flags & ERTS_CRR_ALCTR_FLG_IN_POOL) ? am_true : am_false; + hp[7] = histogram_tuple; + + carrier_tuple = make_tuple(hp); + hp += 8; + + state->result_list = CONS(hp, carrier_tuple, state->result_list); + + free(info); +} + +static void gather_cinfo_send(gather_cinfo_t *state) +{ + Eterm result_tuple, task_ref; + + int term_size; + Eterm *hp; + + ASSERT((state->result_list == NIL) ^ (state->info_count > 0)); + ASSERT(state->building_result); + + term_size = 3 + erts_iref_storage_heap_size(&state->iref); + hp = erts_produce_heap(&state->msg_factory, term_size, 0); + + task_ref = erts_iref_storage_make_ref(&state->iref, &hp, + &(state->msg_factory.message)->hfrag.off_heap, 0); + + hp[0] = make_arityval(2); + hp[1] = task_ref; + hp[2] = state->result_list; + + result_tuple = make_tuple(hp); + + erts_factory_trim_and_close(&state->msg_factory, &result_tuple, 1); + + erts_queue_message(state->process, 0, state->msg_factory.message, + result_tuple, am_system); +} + +static int gather_cinfo_finish(void *arg) +{ + gather_cinfo_t *state = (gather_cinfo_t*)arg; + int reductions = BLOCKSCAN_REDUCTIONS; + + if (!state->building_result) { + ErtsMessage *message; + Uint minimum_size; + Eterm *hp; + + /* {Ref, [{Carrier size, unscanned size, allocated size, + * allocated block count, {Free block histogram}} | Rest]} */ + minimum_size = 3 + erts_iref_storage_heap_size(&state->iref) + + state->info_count * (11 + state->hist_slot_count); + + message = erts_alloc_message(minimum_size, &hp); + erts_factory_selfcontained_message_init(&state->msg_factory, + message, hp); + + state->result_list = NIL; + state->building_result = 1; + } + + while (state->info_list) { + chist_node_t *current = state->info_list; + state->info_list = current->next; + + gather_cinfo_append_result(state, current); + + if (reductions-- <= 0) { + return 1; + } + } + + gather_cinfo_send(state); + + return 0; +} + +static void gather_cinfo_abort(void *arg) +{ + gather_cinfo_t *state = (gather_cinfo_t*)arg; + + if (state->building_result) { + erts_factory_undo(&state->msg_factory); + } + + while (state->info_list) { + chist_node_t *current = state->info_list; + state->info_list = current->next; + + free(current); + } +} + +int erts_alcu_gather_carrier_info(struct process *p, int allocator_num, + int sched_id, int hist_width, + UWord hist_start, Eterm ref) +{ + gather_cinfo_t *gather_state; + blockscan_t *scanner; + + const char *allocator_desc; + Allctr_t *allocator; + + ASSERT(is_internal_ref(ref)); + + if (!blockscan_get_specific_allocator(allocator_num, + sched_id, + &allocator)) { + return 0; + } + + allocator_desc = ERTS_ALC_A2AD(allocator_num); + + /* Plain calloc is intentional. */ + gather_state = (gather_cinfo_t*)calloc(1, sizeof(gather_cinfo_t)); + scanner = &gather_state->common; + + scanner->abort = gather_cinfo_abort; + scanner->scan = gather_cinfo_scan; + scanner->finish = gather_cinfo_finish; + scanner->user_data = gather_state; + + gather_state->allocator_desc = erts_atom_put((byte *)allocator_desc, + sys_strlen(allocator_desc), + ERTS_ATOM_ENC_LATIN1, 1); + erts_iref_storage_save(&gather_state->iref, ref); + gather_state->hist_slot_start = hist_start * 2; + gather_state->hist_slot_count = hist_width; + gather_state->process = p; + + blockscan_dispatch(scanner, p, allocator, sched_id); + + return 1; +} + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * NOTE: erts_alcu_test() is only supposed to be used for testing. * @@ -6441,6 +7722,7 @@ erts_alcu_verify_unused_ts(Allctr_t *allctr) erts_mtx_unlock(&allctr->mutex); } + #ifdef DEBUG int is_sbc_blk(Block_t* blk) { diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index 05c8a0db3b..ff4d10b206 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -50,6 +50,7 @@ typedef struct { int tspec; int tpref; int ramv; + int atags; UWord sbct; UWord asbcst; UWord rsbcst; @@ -106,6 +107,7 @@ typedef struct { 0, /* (bool) tspec: thread specific */\ 0, /* (bool) tpref: thread preferred */\ 0, /* (bool) ramv: realloc always moves */\ + 0, /* (bool) atags: tagged allocations */\ 512*1024, /* (bytes) sbct: sbc threshold */\ 2*1024*2024, /* (amount) asbcst: abs sbc shrink threshold */\ 20, /* (%) rsbcst: rel sbc shrink threshold */\ @@ -142,6 +144,7 @@ typedef struct { 0, /* (bool) tspec: thread specific */\ 0, /* (bool) tpref: thread preferred */\ 0, /* (bool) ramv: realloc always moves */\ + 0, /* (bool) atags: tagged allocations */\ 64*1024, /* (bytes) sbct: sbc threshold */\ 2*1024*2024, /* (amount) asbcst: abs sbc shrink threshold */\ 20, /* (%) rsbcst: rel sbc shrink threshold */\ @@ -224,6 +227,36 @@ void erts_lcnt_update_allocator_locks(int enable); int erts_alcu_try_set_dyn_param(Allctr_t*, Eterm param, Uint value); +/* Gathers per-tag allocation histograms from the given allocator number + * (ERTS_ALC_A_*) and scheduler id. An id of 0 means the global instance will + * be used. + * + * The results are sent to `p`, and it returns the number of messages to wait + * for. */ +int erts_alcu_gather_alloc_histograms(struct process *p, int allocator_num, + int sched_id, int hist_width, + UWord hist_start, Eterm ref); + +/* Gathers per-carrier info from the given allocator number (ERTS_ALC_A_*) and + * scheduler id. An id of 0 means the global instance will be used. + * + * The results are sent to `p`, and it returns the number of messages to wait + * for. */ +int erts_alcu_gather_carrier_info(struct process *p, int allocator_num, + int sched_id, int hist_width, + UWord hist_start, Eterm ref); + +struct alcu_blockscan; + +typedef struct { + struct alcu_blockscan *current; + struct alcu_blockscan *last; +} ErtsAlcuBlockscanYieldData; + +int erts_handle_yielded_alcu_blockscan(struct ErtsSchedulerData_ *esdp, + ErtsAlcuBlockscanYieldData *yield); +void erts_alcu_sched_spec_data_init(struct ErtsSchedulerData_ *esdp); + #endif /* !ERL_ALLOC_UTIL__ */ #if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__) @@ -548,6 +581,7 @@ struct Allctr_t_ { /* Options */ int t; int ramv; + int atags; Uint sbc_threshold; Uint sbc_move_threshold; Uint mbc_move_threshold; @@ -684,6 +718,7 @@ struct Allctr_t_ { #endif }; + int erts_alcu_start(Allctr_t *, AllctrInit_t *); void erts_alcu_stop(Allctr_t *); diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index 579e9b12f4..294bce115f 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1505,6 +1505,7 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name) res = ERL_DE_LOAD_ERROR_BAD_NAME; goto error; } + erts_atomic_init_nob(&(dh->refc), (erts_aint_t) 0); erts_atomic32_init_nob(&dh->port_count, 0); dh->full_path = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(path) + 1); @@ -1512,7 +1513,7 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name) dh->flags = 0; dh->status = ERL_DE_OK; - if (erts_add_driver_entry(dp, dh, 1) != 0 /* io.c */) { + if (erts_add_driver_entry(dp, dh, 1, 1) != 0 /* io.c */) { /* * The init in the driver struct did not return 0 */ diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 3f9b584c2e..8687aefd78 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2017. All Rights Reserved. + * Copyright Ericsson AB 1999-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,7 +38,7 @@ #include "erl_message.h" #include "erl_binary.h" #include "erl_db.h" -#include "erl_instrument.h" +#include "erl_mtrace.h" #include "dist.h" #include "erl_gc.h" #include "erl_cpu_topology.h" @@ -50,6 +50,8 @@ #define ERTS_PTAB_WANT_DEBUG_FUNCS__ #include "erl_ptab.h" #include "erl_time.h" +#include "erl_proc_sig_queue.h" +#include "erl_alloc_util.h" #ifdef HIPE #include "hipe_arch.h" #endif @@ -72,6 +74,9 @@ static Export *gather_msacc_res_trap; static Export *gather_gc_info_res_trap; static Export *gather_system_check_res_trap; +static Export *is_process_alive_trap; + + #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) static char otp_version[] = ERLANG_OTP_VERSION; @@ -151,8 +156,10 @@ static Eterm os_type_tuple; static Eterm os_version_tuple; static Eterm -current_function(Process* p, Process* rp, Eterm** hpp, int full_info); -static Eterm current_stacktrace(Process* p, Process* rp, Eterm** hpp); +current_function(Process* p, ErtsHeapFactory *hfact, Process* rp, + int full_info, Uint reserve_size, int flags); +static Eterm current_stacktrace(ErtsHeapFactory *hfact, Process* rp, + Uint reserve_size); static Eterm bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh) @@ -211,21 +218,27 @@ bld_magic_ref_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh) make_monitor_list: returns a list of records.. -record(erl_monitor, { - type, % MON_ORIGIN or MON_TARGET (1 or 3) - ref, + type, % process | port | time_offset | dist_process | resource + % | node | nodes | suspend + dir, % origin | target + ref, % reference or [] pid, % Process or nodename - name % registered name or [] + extra % registered name, integer or [] }). */ static void do_calc_mon_size(ErtsMonitor *mon, void *vpsz) { + ErtsMonitorData *mdp = erts_monitor_to_data(mon); Uint *psz = vpsz; - *psz += NC_HEAP_SIZE(mon->ref); - *psz += (mon->type == MON_NIF_TARGET ? - erts_resource_ref_size(mon->u.resource) : - (is_immed(mon->u.pid) ? 0 : NC_HEAP_SIZE(mon->u.pid))); - *psz += 8; /* CONS + 5-tuple */ + *psz += is_immed(mdp->ref) ? 0 : NC_HEAP_SIZE(mdp->ref); + + if (mon->type == ERTS_MON_TYPE_RESOURCE && erts_monitor_is_target(mon)) + *psz += erts_resource_ref_size(mon->other.ptr); + else + *psz += is_immed(mon->other.item) ? 0 : NC_HEAP_SIZE(mon->other.item); + + *psz += 9; /* CONS + 6-tuple */ } typedef struct { @@ -237,35 +250,97 @@ typedef struct { static void do_make_one_mon_element(ErtsMonitor *mon, void * vpmlc) { + ErtsMonitorData *mdp = erts_monitor_to_data(mon); MonListContext *pmlc = vpmlc; - Eterm tup; - Eterm r = STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->ref); - Eterm p = (mon->type == MON_NIF_TARGET ? - erts_bld_resource_ref(&(pmlc->hp), &MSO(pmlc->p), mon->u.resource) - : (is_immed(mon->u.pid) ? mon->u.pid - : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->u.pid))); - tup = TUPLE5(pmlc->hp, pmlc->tag, make_small(mon->type), r, p, mon->name); - pmlc->hp += 6; + Eterm tup, t, d, r, p, x; + + r = is_immed(mdp->ref) ? mdp->ref : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mdp->ref); + if (mon->type == ERTS_MON_TYPE_RESOURCE && erts_monitor_is_target(mon)) + p = erts_bld_resource_ref(&(pmlc->hp), &MSO(pmlc->p), mon->other.ptr); + else + p = (is_immed(mon->other.item) + ? mon->other.item + : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->other.item)); + + if (mon->flags & ERTS_ML_FLG_NAME) + x = ((ErtsMonitorDataExtended *) mdp)->u.name; + else if (erts_monitor_is_target(mon)) + x = NIL; + else if (mon->type == ERTS_MON_TYPE_NODE || mon->type == ERTS_MON_TYPE_NODES) + x = make_small(((ErtsMonitorDataExtended *) mdp)->u.refc); + else + x = NIL; + + switch (mon->type) { + case ERTS_MON_TYPE_PROC: + t = am_process; + break; + case ERTS_MON_TYPE_PORT: + t = am_port; + break; + case ERTS_MON_TYPE_TIME_OFFSET: + t = am_time_offset; + break; + case ERTS_MON_TYPE_DIST_PROC: { + ERTS_DECL_AM(dist_process); + t = AM_dist_process; + break; + } + case ERTS_MON_TYPE_RESOURCE: { + ERTS_DECL_AM(resource); + t = AM_resource; + break; + } + case ERTS_MON_TYPE_NODE: + t = am_node; + break; + case ERTS_MON_TYPE_NODES: { + ERTS_DECL_AM(nodes); + t = AM_nodes; + break; + } + case ERTS_MON_TYPE_SUSPEND: + t = am_suspend; + break; + default: + ERTS_INTERNAL_ERROR("Unknown monitor type"); + t = am_error; + break; + } + if (erts_monitor_is_target(mon)) { + ERTS_DECL_AM(target); + d = AM_target; + } + else { + ERTS_DECL_AM(origin); + d = AM_origin; + } + tup = TUPLE6(pmlc->hp, pmlc->tag, t, d, r, p, x); + pmlc->hp += 7; pmlc->res = CONS(pmlc->hp, tup, pmlc->res); pmlc->hp += 2; } static Eterm -make_monitor_list(Process *p, ErtsMonitor *root) +make_monitor_list(Process *p, int tree, ErtsMonitor *root, Eterm tail) { DECL_AM(erl_monitor); Uint sz = 0; MonListContext mlc; + void (*foreach)(ErtsMonitor *, + void (*)(ErtsMonitor *, void *), + void *); - erts_doforall_monitors(root, &do_calc_mon_size, &sz); - if (sz == 0) { - return NIL; - } + foreach = tree ? erts_monitor_tree_foreach : erts_monitor_list_foreach; + + (*foreach)(root, do_calc_mon_size, &sz); + if (sz == 0) + return tail; mlc.p = p; mlc.hp = HAlloc(p,sz); - mlc.res = NIL; + mlc.res = tail; mlc.tag = AM_erl_monitor; - erts_doforall_monitors(root, &do_make_one_mon_element, &mlc); + (*foreach)(root, do_make_one_mon_element, &mlc); return mlc.res; } @@ -273,20 +348,22 @@ make_monitor_list(Process *p, ErtsMonitor *root) make_link_list: returns a list of records.. -record(erl_link, { - type, % LINK_NODE or LINK_PID (1 or 3) - pid, % Process or nodename - targets % List of erl_link's or nil + type, % process | port | dist_process + pid, % Process or port + id % (address) }). */ -static void do_calc_lnk_size(ErtsLink *lnk, void *vpsz) +static void calc_lnk_size(ErtsLink *lnk, void *vpsz) { Uint *psz = vpsz; - *psz += is_immed(lnk->pid) ? 0 : NC_HEAP_SIZE(lnk->pid); - if (lnk->type != LINK_NODE && ERTS_LINK_ROOT(lnk) != NULL) { - /* Node links use this pointer as ref counter... */ - erts_doforall_links(ERTS_LINK_ROOT(lnk),&do_calc_lnk_size,vpsz); - } + Uint sz = 0; + ErtsLinkData *ldp = erts_link_to_data(lnk); + + (void) erts_bld_uword(NULL, &sz, (UWord) ldp); + + *psz += sz; + *psz += is_immed(lnk->other.item) ? 0 : size_object(lnk->other.item); *psz += 7; /* CONS + 4-tuple */ } @@ -297,37 +374,58 @@ typedef struct { Eterm tag; } LnkListContext; -static void do_make_one_lnk_element(ErtsLink *lnk, void * vpllc) +static void make_one_lnk_element(ErtsLink *lnk, void * vpllc) { LnkListContext *pllc = vpllc; - Eterm tup; - Eterm old_res, targets = NIL; - Eterm p = (is_immed(lnk->pid) - ? lnk->pid - : STORE_NC(&(pllc->hp), &MSO(pllc->p), lnk->pid)); - if (lnk->type == LINK_NODE) { - targets = make_small(ERTS_LINK_REFC(lnk)); - } else if (ERTS_LINK_ROOT(lnk) != NULL) { - old_res = pllc->res; - pllc->res = NIL; - erts_doforall_links(ERTS_LINK_ROOT(lnk),&do_make_one_lnk_element, vpllc); - targets = pllc->res; - pllc->res = old_res; - } - tup = TUPLE4(pllc->hp, pllc->tag, make_small(lnk->type), p, targets); + Eterm tup, t, pid, id; + ErtsLinkData *ldp = erts_link_to_data(lnk); + + id = erts_bld_uword(&pllc->hp, NULL, (UWord) ldp); + + if (is_immed(lnk->other.item)) + pid = lnk->other.item; + else { + Uint sz = size_object(lnk->other.item); + pid = copy_struct(lnk->other.item, sz, &(pllc->hp), &MSO(pllc->p)); + } + + switch (lnk->type) { + case ERTS_LNK_TYPE_PROC: + t = am_process; + break; + case ERTS_LNK_TYPE_PORT: + t = am_port; + break; + case ERTS_LNK_TYPE_DIST_PROC: { + ERTS_DECL_AM(dist_process); + t = AM_dist_process; + break; + } + default: + ERTS_INTERNAL_ERROR("Unkown link type"); + t = am_undefined; + break; + } + + tup = TUPLE4(pllc->hp, pllc->tag, t, pid, id); pllc->hp += 5; pllc->res = CONS(pllc->hp, tup, pllc->res); pllc->hp += 2; } static Eterm -make_link_list(Process *p, ErtsLink *root, Eterm tail) +make_link_list(Process *p, int tree, ErtsLink *root, Eterm tail) { DECL_AM(erl_link); Uint sz = 0; LnkListContext llc; + void (*foreach)(ErtsLink *, + void (*)(ErtsLink *, void *), + void *); - erts_doforall_links(root, &do_calc_lnk_size, &sz); + foreach = tree ? erts_link_tree_foreach : erts_link_list_foreach; + + (*foreach)(root, calc_lnk_size, (void *) &sz); if (sz == 0) { return tail; } @@ -335,7 +433,7 @@ make_link_list(Process *p, ErtsLink *root, Eterm tail) llc.hp = HAlloc(p,sz); llc.res = tail; llc.tag = AM_erl_link; - erts_doforall_links(root, &do_make_one_lnk_element, &llc); + (*foreach)(root, make_one_lnk_element, (void *) &llc); return llc.res; } @@ -381,6 +479,8 @@ typedef struct { Eterm term; ErtsResource* resource; }entity; + int named; + Uint16 type; Eterm node; /* pid is actual target being monitored, no matter pid/port or name */ Eterm pid; @@ -423,78 +523,103 @@ static void collect_one_link(ErtsLink *lnk, void *vmicp) { MonitorInfoCollection *micp = vmicp; EXTEND_MONITOR_INFOS(micp); - if (!(lnk->type == LINK_PID)) { - return; - } - micp->mi[micp->mi_i].entity.term = lnk->pid; - micp->sz += 2 + NC_HEAP_SIZE(lnk->pid); + micp->mi[micp->mi_i].entity.term = lnk->other.item; + micp->sz += 2 + NC_HEAP_SIZE(lnk->other.item); micp->mi_i++; } static void collect_one_origin_monitor(ErtsMonitor *mon, void *vmicp) { - MonitorInfoCollection *micp = vmicp; + if (erts_monitor_is_origin(mon)) { + MonitorInfoCollection *micp = vmicp; - if (mon->type != MON_ORIGIN) { - return; - } - EXTEND_MONITOR_INFOS(micp); - if (is_atom(mon->u.pid)) { /* external by name */ - micp->mi[micp->mi_i].entity.term = mon->name; - micp->mi[micp->mi_i].node = mon->u.pid; - micp->sz += 3; /* need one 2-tuple */ - } else if (is_external_pid(mon->u.pid)) { /* external by pid */ - micp->mi[micp->mi_i].entity.term = mon->u.pid; - micp->mi[micp->mi_i].node = NIL; - micp->sz += NC_HEAP_SIZE(mon->u.pid); - } else if (!is_nil(mon->name)) { /* internal by name */ - micp->mi[micp->mi_i].entity.term = mon->name; - micp->mi[micp->mi_i].node = erts_this_dist_entry->sysname; - micp->sz += 3; /* need one 2-tuple */ - } else { /* internal by pid */ - micp->mi[micp->mi_i].entity.term = mon->u.pid; - micp->mi[micp->mi_i].node = NIL; - /* no additional heap space needed */ - } - - /* have always pid at hand, to assist with figuring out if its a port or - * a process, when we monitored by name and process_info is requested. - * See: erl_bif_info.c:process_info_aux section for am_monitors */ - micp->mi[micp->mi_i].pid = mon->u.pid; + EXTEND_MONITOR_INFOS(micp); + + micp->mi[micp->mi_i].type = mon->type; + + switch (mon->type) { + case ERTS_MON_TYPE_PROC: + case ERTS_MON_TYPE_PORT: + case ERTS_MON_TYPE_DIST_PROC: + case ERTS_MON_TYPE_TIME_OFFSET: + if (!(mon->flags & ERTS_ML_FLG_NAME)) { + micp->mi[micp->mi_i].named = 0; + micp->mi[micp->mi_i].entity.term = mon->other.item; + micp->mi[micp->mi_i].node = NIL; + if (is_not_atom(mon->other.item)) + micp->sz += NC_HEAP_SIZE(mon->other.item); + } + else { + ErtsMonitorDataExtended *mdep; + micp->mi[micp->mi_i].named = !0; + mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon); + micp->mi[micp->mi_i].entity.term = mdep->u.name; + if (mdep->dist) + micp->mi[micp->mi_i].node = mdep->dist->nodename; + else + micp->mi[micp->mi_i].node = erts_this_dist_entry->sysname; + micp->sz += 3; /* need one 2-tuple */ + } - micp->mi_i++; - micp->sz += 2 + 3; /* For a cons cell and a 2-tuple */ + /* have always pid at hand, to assist with figuring out if its a port or + * a process, when we monitored by name and process_info is requested. + * See: erl_bif_info.c:process_info_aux section for am_monitors */ + micp->mi[micp->mi_i].pid = mon->other.item; + + micp->mi_i++; + micp->sz += 2 + 3; /* For a cons cell and a 2-tuple */ + break; + default: + break; + } + } } static void collect_one_target_monitor(ErtsMonitor *mon, void *vmicp) { MonitorInfoCollection *micp = vmicp; - if (mon->type != MON_TARGET && mon->type != MON_NIF_TARGET) { - return; - } + if (erts_monitor_is_target(mon)) { - EXTEND_MONITOR_INFOS(micp); + EXTEND_MONITOR_INFOS(micp); + micp->mi[micp->mi_i].type = mon->type; + micp->mi[micp->mi_i].named = !!(mon->flags & ERTS_ML_FLG_NAME); + switch (mon->type) { + + case ERTS_MON_TYPE_PROC: + case ERTS_MON_TYPE_PORT: + case ERTS_MON_TYPE_DIST_PROC: + + micp->mi[micp->mi_i].entity.term = mon->other.item; + micp->mi[micp->mi_i].node = NIL; + micp->sz += NC_HEAP_SIZE(mon->other.item); + + micp->sz += 2; /* cons */; + micp->mi_i++; + break; + + case ERTS_MON_TYPE_RESOURCE: + + micp->mi[micp->mi_i].entity.resource = mon->other.ptr; + micp->mi[micp->mi_i].node = NIL; + micp->sz += erts_resource_ref_size(mon->other.ptr); + + micp->sz += 2; /* cons */; + micp->mi_i++; + break; + + default: + break; + } - if (mon->type == MON_NIF_TARGET) { - micp->mi[micp->mi_i].entity.resource = mon->u.resource; - micp->mi[micp->mi_i].node = make_small(MON_NIF_TARGET); - micp->sz += erts_resource_ref_size(mon->u.resource); - } - else { - micp->mi[micp->mi_i].entity.term = mon->u.pid; - micp->mi[micp->mi_i].node = NIL; - micp->sz += NC_HEAP_SIZE(mon->u.pid); } - micp->sz += 2; /* cons */; - micp->mi_i++; } typedef struct { Process *c_p; ErtsProcLocks c_p_locks; - ErtsSuspendMonitor **smi; + ErtsMonitorSuspend **smi; Uint smi_i; Uint smi_max; int sz; @@ -517,10 +642,10 @@ do { \ (SMICP)->smi, \ ((SMICP)->smi_max \ + ERTS_SMI_INC) \ - * sizeof(ErtsSuspendMonitor *)) \ + * sizeof(ErtsMonitorSuspend *)) \ : erts_alloc(ERTS_ALC_T_TMP, \ ERTS_SMI_INC \ - * sizeof(ErtsSuspendMonitor *))); \ + * sizeof(ErtsMonitorSuspend *))); \ (SMICP)->smi_max += ERTS_SMI_INC; \ } \ } while (0) @@ -533,12 +658,13 @@ do { \ } while (0) static void -collect_one_suspend_monitor(ErtsSuspendMonitor *smon, void *vsmicp) +collect_one_suspend_monitor(ErtsMonitor *mon, void *vsmicp) { + ErtsMonitorSuspend *smon = erts_monitor_suspend(mon); ErtsSuspendMonitorInfoCollection *smicp = vsmicp; Process *suspendee = erts_pid2proc(smicp->c_p, smicp->c_p_locks, - smon->pid, + mon->other.item, 0); if (suspendee) { /* suspendee is alive */ Sint a, p; @@ -570,125 +696,222 @@ collect_one_suspend_monitor(ErtsSuspendMonitor *smon, void *vsmicp) * process_info/[1,2] */ -#define ERTS_PI_FAIL_TYPE_BADARG 0 -#define ERTS_PI_FAIL_TYPE_YIELD 1 -#define ERTS_PI_FAIL_TYPE_AWAIT_EXIT 2 - -static ERTS_INLINE ErtsProcLocks -pi_locks(Eterm info) -{ - switch (info) { - case am_status: - case am_priority: - case am_trap_exit: - return ERTS_PROC_LOCK_STATUS; - case am_links: - case am_monitors: - case am_monitored_by: - case am_suspending: - return ERTS_PROC_LOCK_LINK; - case am_messages: - case am_message_queue_len: - case am_total_heap_size: - return ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_MSGQ; - case am_memory: - return ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_MSGQ; - default: - return ERTS_PROC_LOCK_MAIN; - } -} - /* * All valid process_info arguments. */ -static Eterm pi_args[] = { - am_registered_name, - am_current_function, - am_initial_call, - am_status, - am_messages, - am_message_queue_len, - am_links, - am_monitors, - am_monitored_by, - am_dictionary, - am_trap_exit, - am_error_handler, - am_heap_size, - am_stack_size, - am_memory, - am_garbage_collection, - am_group_leader, - am_reductions, - am_priority, - am_trace, - am_binary, - am_sequential_trace_token, - am_catchlevel, - am_backtrace, - am_last_calls, - am_total_heap_size, - am_suspending, - am_min_heap_size, - am_min_bin_vheap_size, - am_max_heap_size, - am_current_location, - am_current_stacktrace, - am_message_queue_data, - am_garbage_collection_info, - am_magic_ref + +#define ERTS_PI_IX_REGISTERED_NAME 0 +#define ERTS_PI_IX_CURRENT_FUNCTION 1 +#define ERTS_PI_IX_INITIAL_CALL 2 +#define ERTS_PI_IX_STATUS 3 +#define ERTS_PI_IX_MESSAGES 4 +#define ERTS_PI_IX_MESSAGE_QUEUE_LEN 5 +#define ERTS_PI_IX_LINKS 6 +#define ERTS_PI_IX_MONITORS 7 +#define ERTS_PI_IX_MONITORED_BY 8 +#define ERTS_PI_IX_DICTIONARY 9 +#define ERTS_PI_IX_TRAP_EXIT 10 +#define ERTS_PI_IX_ERROR_HANDLER 11 +#define ERTS_PI_IX_HEAP_SIZE 12 +#define ERTS_PI_IX_STACK_SIZE 13 +#define ERTS_PI_IX_MEMORY 14 +#define ERTS_PI_IX_GARBAGE_COLLECTION 15 +#define ERTS_PI_IX_GROUP_LEADER 16 +#define ERTS_PI_IX_REDUCTIONS 17 +#define ERTS_PI_IX_PRIORITY 18 +#define ERTS_PI_IX_TRACE 19 +#define ERTS_PI_IX_BINARY 20 +#define ERTS_PI_IX_SEQUENTIAL_TRACE_TOKEN 21 +#define ERTS_PI_IX_CATCHLEVEL 22 +#define ERTS_PI_IX_BACKTRACE 23 +#define ERTS_PI_IX_LAST_CALLS 24 +#define ERTS_PI_IX_TOTAL_HEAP_SIZE 25 +#define ERTS_PI_IX_SUSPENDING 26 +#define ERTS_PI_IX_MIN_HEAP_SIZE 27 +#define ERTS_PI_IX_MIN_BIN_VHEAP_SIZE 28 +#define ERTS_PI_IX_MAX_HEAP_SIZE 29 +#define ERTS_PI_IX_CURRENT_LOCATION 30 +#define ERTS_PI_IX_CURRENT_STACKTRACE 31 +#define ERTS_PI_IX_MESSAGE_QUEUE_DATA 32 +#define ERTS_PI_IX_GARBAGE_COLLECTION_INFO 33 +#define ERTS_PI_IX_MAGIC_REF 34 +#define ERTS_PI_IX_FULLSWEEP_AFTER 35 + +#define ERTS_PI_FLAG_SINGELTON (1 << 0) +#define ERTS_PI_FLAG_ALWAYS_WRAP (1 << 1) +#define ERTS_PI_FLAG_WANT_MSGS (1 << 2) +#define ERTS_PI_FLAG_NEED_MSGQ_LEN (1 << 3) +#define ERTS_PI_FLAG_FORCE_SIG_SEND (1 << 4) +#define ERTS_PI_FLAG_REQUEST_FOR_OTHER (1 << 5) + +#define ERTS_PI_UNRESERVE(RS, SZ) \ + (ASSERT((RS) >= (SZ)), (RS) -= (SZ)) + + +typedef struct { + Eterm name; + Uint reserve_size; + int flags; + ErtsProcLocks locks; +} ErtsProcessInfoArgs; + +static ErtsProcessInfoArgs pi_args[] = { + {am_registered_name, 0, 0, ERTS_PROC_LOCK_MAIN}, + {am_current_function, 4, 0, ERTS_PROC_LOCK_MAIN}, + {am_initial_call, 4, 0, ERTS_PROC_LOCK_MAIN}, + {am_status, 0, 0, 0}, + {am_messages, 0, ERTS_PI_FLAG_WANT_MSGS|ERTS_PI_FLAG_NEED_MSGQ_LEN|ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN}, + {am_message_queue_len, 0, ERTS_PI_FLAG_NEED_MSGQ_LEN, ERTS_PROC_LOCK_MAIN}, + {am_links, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN}, + {am_monitors, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN}, + {am_monitored_by, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN}, + {am_dictionary, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN}, + {am_trap_exit, 0, 0, ERTS_PROC_LOCK_MAIN}, + {am_error_handler, 0, 0, ERTS_PROC_LOCK_MAIN}, + {am_heap_size, 0, 0, ERTS_PROC_LOCK_MAIN}, + {am_stack_size, 0, 0, ERTS_PROC_LOCK_MAIN}, + {am_memory, 0, ERTS_PI_FLAG_NEED_MSGQ_LEN|ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN}, + {am_garbage_collection, 3+2 + 3+2 + 3+2 + 3+2 + 3+2 + ERTS_MAX_HEAP_SIZE_MAP_SZ, 0, ERTS_PROC_LOCK_MAIN}, + {am_group_leader, 0, 0, ERTS_PROC_LOCK_MAIN}, + {am_reductions, 0, 0, ERTS_PROC_LOCK_MAIN}, + {am_priority, 0, 0, 0}, + {am_trace, 0, 0, ERTS_PROC_LOCK_MAIN}, + {am_binary, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN}, + {am_sequential_trace_token, 0, 0, ERTS_PROC_LOCK_MAIN}, + {am_catchlevel, 0, 0, ERTS_PROC_LOCK_MAIN}, + {am_backtrace, 0, 0, ERTS_PROC_LOCK_MAIN}, + {am_last_calls, 0, 0, ERTS_PROC_LOCK_MAIN}, + {am_total_heap_size, 0, ERTS_PI_FLAG_NEED_MSGQ_LEN|ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN}, + {am_suspending, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, 0}, + {am_min_heap_size, 0, 0, ERTS_PROC_LOCK_MAIN}, + {am_min_bin_vheap_size, 0, 0, ERTS_PROC_LOCK_MAIN}, + {am_max_heap_size, 0, 0, ERTS_PROC_LOCK_MAIN}, + {am_current_location, 0, 0, ERTS_PROC_LOCK_MAIN}, + {am_current_stacktrace, 0, 0, ERTS_PROC_LOCK_MAIN}, + {am_message_queue_data, 0, 0, ERTS_PROC_LOCK_MAIN}, + {am_garbage_collection_info, ERTS_PROCESS_GC_INFO_MAX_SIZE, 0, ERTS_PROC_LOCK_MAIN}, + {am_magic_ref, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN}, + {am_fullsweep_after, 0, 0, ERTS_PROC_LOCK_MAIN} }; -#define ERTS_PI_ARGS ((int) (sizeof(pi_args)/sizeof(Eterm))) +#define ERTS_PI_ARGS ((int) (sizeof(pi_args)/sizeof(pi_args[0]))) + +#ifdef DEBUG +# define ERTS_PI_DEF_ARR_SZ 2 +#else +# define ERTS_PI_DEF_ARR_SZ ERTS_PI_ARGS +#endif static ERTS_INLINE Eterm pi_ix2arg(int ix) { if (ix < 0 || ERTS_PI_ARGS <= ix) return am_undefined; - return pi_args[ix]; + return pi_args[ix].name; +} + +static ERTS_INLINE int +pi_ix2flags(int ix) +{ + if (ix < 0 || ERTS_PI_ARGS <= ix) + return 0; + return pi_args[ix].flags; +} + +static ERTS_INLINE Uint +pi_ix2rsz(int ix) +{ + if (ix < 0 || ERTS_PI_ARGS <= ix) + return 0; + return pi_args[ix].reserve_size; +} + +static ERTS_INLINE ErtsProcLocks +pi_ix2locks(int ix) +{ + if (ix < 0 || ERTS_PI_ARGS <= ix) + return 0; + return pi_args[ix].locks; } static ERTS_INLINE int pi_arg2ix(Eterm arg) { switch (arg) { - case am_registered_name: return 0; - case am_current_function: return 1; - case am_initial_call: return 2; - case am_status: return 3; - case am_messages: return 4; - case am_message_queue_len: return 5; - case am_links: return 6; - case am_monitors: return 7; - case am_monitored_by: return 8; - case am_dictionary: return 9; - case am_trap_exit: return 10; - case am_error_handler: return 11; - case am_heap_size: return 12; - case am_stack_size: return 13; - case am_memory: return 14; - case am_garbage_collection: return 15; - case am_group_leader: return 16; - case am_reductions: return 17; - case am_priority: return 18; - case am_trace: return 19; - case am_binary: return 20; - case am_sequential_trace_token: return 21; - case am_catchlevel: return 22; - case am_backtrace: return 23; - case am_last_calls: return 24; - case am_total_heap_size: return 25; - case am_suspending: return 26; - case am_min_heap_size: return 27; - case am_min_bin_vheap_size: return 28; - case am_max_heap_size: return 29; - case am_current_location: return 30; - case am_current_stacktrace: return 31; - case am_message_queue_data: return 32; - case am_garbage_collection_info: return 33; - case am_magic_ref: return 34; - default: return -1; + case am_registered_name: + return ERTS_PI_IX_REGISTERED_NAME; + case am_current_function: + return ERTS_PI_IX_CURRENT_FUNCTION; + case am_initial_call: + return ERTS_PI_IX_INITIAL_CALL; + case am_status: + return ERTS_PI_IX_STATUS; + case am_messages: + return ERTS_PI_IX_MESSAGES; + case am_message_queue_len: + return ERTS_PI_IX_MESSAGE_QUEUE_LEN; + case am_links: + return ERTS_PI_IX_LINKS; + case am_monitors: + return ERTS_PI_IX_MONITORS; + case am_monitored_by: + return ERTS_PI_IX_MONITORED_BY; + case am_dictionary: + return ERTS_PI_IX_DICTIONARY; + case am_trap_exit: + return ERTS_PI_IX_TRAP_EXIT; + case am_error_handler: + return ERTS_PI_IX_ERROR_HANDLER; + case am_heap_size: + return ERTS_PI_IX_HEAP_SIZE; + case am_stack_size: + return ERTS_PI_IX_STACK_SIZE; + case am_memory: + return ERTS_PI_IX_MEMORY; + case am_garbage_collection: + return ERTS_PI_IX_GARBAGE_COLLECTION; + case am_group_leader: + return ERTS_PI_IX_GROUP_LEADER; + case am_reductions: + return ERTS_PI_IX_REDUCTIONS; + case am_priority: + return ERTS_PI_IX_PRIORITY; + case am_trace: + return ERTS_PI_IX_TRACE; + case am_binary: + return ERTS_PI_IX_BINARY; + case am_sequential_trace_token: + return ERTS_PI_IX_SEQUENTIAL_TRACE_TOKEN; + case am_catchlevel: + return ERTS_PI_IX_CATCHLEVEL; + case am_backtrace: + return ERTS_PI_IX_BACKTRACE; + case am_last_calls: + return ERTS_PI_IX_LAST_CALLS; + case am_total_heap_size: + return ERTS_PI_IX_TOTAL_HEAP_SIZE; + case am_suspending: + return ERTS_PI_IX_SUSPENDING; + case am_min_heap_size: + return ERTS_PI_IX_MIN_HEAP_SIZE; + case am_min_bin_vheap_size: + return ERTS_PI_IX_MIN_BIN_VHEAP_SIZE; + case am_max_heap_size: + return ERTS_PI_IX_MAX_HEAP_SIZE; + case am_current_location: + return ERTS_PI_IX_CURRENT_LOCATION; + case am_current_stacktrace: + return ERTS_PI_IX_CURRENT_STACKTRACE; + case am_message_queue_data: + return ERTS_PI_IX_MESSAGE_QUEUE_DATA; + case am_garbage_collection_info: + return ERTS_PI_IX_GARBAGE_COLLECTION_INFO; + case am_magic_ref: + return ERTS_PI_IX_MAGIC_REF; + case am_fullsweep_after: + return ERTS_PI_IX_FULLSWEEP_AFTER; + default: + return -1; } } @@ -741,153 +964,41 @@ process_info_init(void) } -static ERTS_INLINE Process * -pi_pid2proc(Process *c_p, Eterm pid, ErtsProcLocks info_locks) -{ - /* - * If the main lock is needed, we use erts_pid2proc_not_running() - * instead of erts_pid2proc() for two reasons: - * * Current function of pid and possibly other information will - * have been updated so that process_info() is consistent with an - * info-request/info-response signal model. - * * We avoid blocking the whole scheduler executing the - * process that is calling process_info() for a long time - * which will happen if pid is currently running. - * The caller of process_info() may have to yield if pid - * is currently running. - */ - - if (info_locks & ERTS_PROC_LOCK_MAIN) - return erts_pid2proc_not_running(c_p, ERTS_PROC_LOCK_MAIN, - pid, info_locks); - else - return erts_pid2proc(c_p, ERTS_PROC_LOCK_MAIN, - pid, info_locks); -} - - - static BIF_RETTYPE -process_info_aux(Process *BIF_P, +process_info_aux(Process *c_p, + ErtsHeapFactory *hfact, Process *rp, ErtsProcLocks rp_locks, - Eterm rpid, - Eterm item, - int always_wrap); - -#define ERTS_PI_RES_ELEM_IX_BUF_INC 1024 -#define ERTS_PI_DEF_RES_ELEM_IX_BUF_SZ ERTS_PI_ARGS + int item_ix, + int flags, + Uint *reserve_sizep, + Uint *reds); -static Eterm -process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap, - int *fail_type) +Eterm +erts_process_info(Process *c_p, + ErtsHeapFactory *hfact, + Process *rp, + ErtsProcLocks rp_locks, + int *item_ix, + int item_ix_len, + int flags, + Uint reserve_size, + Uint *reds) { - int want_messages = 0; - int def_res_elem_ix_buf[ERTS_PI_DEF_RES_ELEM_IX_BUF_SZ]; - int *res_elem_ix = &def_res_elem_ix_buf[0]; - int res_elem_ix_ix = -1; - int res_elem_ix_sz = ERTS_PI_DEF_RES_ELEM_IX_BUF_SZ; + Eterm res; Eterm part_res[ERTS_PI_ARGS]; - Eterm res, arg; - Uint *hp, *hp_end; - ErtsProcLocks locks = (ErtsProcLocks) 0; - int res_len, ix; - Process *rp = NULL; + int item_ix_ix, ix; - *fail_type = ERTS_PI_FAIL_TYPE_BADARG; + if (ERTS_PI_FLAG_SINGELTON & flags) { + ASSERT(item_ix_len == 1); + res = process_info_aux(c_p, hfact, rp, rp_locks, item_ix[0], + flags, &reserve_size, reds); + return res; + } for (ix = 0; ix < ERTS_PI_ARGS; ix++) part_res[ix] = THE_NON_VALUE; - ASSERT(is_list(list)); - - while (is_list(list)) { - Eterm* consp = list_val(list); - - arg = CAR(consp); - ix = pi_arg2ix(arg); - if (ix < 0) { - res = THE_NON_VALUE; - goto done; - } - if (arg == am_messages) - want_messages = 1; - locks |= pi_locks(arg); - res_elem_ix_ix++; - if (res_elem_ix_ix >= res_elem_ix_sz) { - if (res_elem_ix != &def_res_elem_ix_buf[0]) - res_elem_ix = - erts_realloc(ERTS_ALC_T_TMP, - res_elem_ix, - sizeof(int)*(res_elem_ix_sz - += ERTS_PI_RES_ELEM_IX_BUF_INC)); - else { - int new_res_elem_ix_sz = ERTS_PI_RES_ELEM_IX_BUF_INC; - int *new_res_elem_ix = erts_alloc(ERTS_ALC_T_TMP, - sizeof(int)*new_res_elem_ix_sz); - sys_memcpy((void *) new_res_elem_ix, - (void *) res_elem_ix, - sizeof(int)*res_elem_ix_sz); - res_elem_ix = new_res_elem_ix; - res_elem_ix_sz = new_res_elem_ix_sz; - } - } - res_elem_ix[res_elem_ix_ix] = ix; - list = CDR(consp); - } - if (is_not_nil(list)) { - res = THE_NON_VALUE; - goto done; - } - - res_len = res_elem_ix_ix+1; - - ASSERT(res_len > 0); - - rp = pi_pid2proc(c_p, pid, locks|ERTS_PROC_LOCK_STATUS); - if (!rp) { - res = am_undefined; - goto done; - } - else if (rp == ERTS_PROC_LOCK_BUSY) { - rp = NULL; - res = THE_NON_VALUE; - *fail_type = ERTS_PI_FAIL_TYPE_YIELD; - goto done; - } - else if (c_p != rp && ERTS_PROC_PENDING_EXIT(rp)) { - locks |= ERTS_PROC_LOCK_STATUS; - res = THE_NON_VALUE; - *fail_type = ERTS_PI_FAIL_TYPE_AWAIT_EXIT; - goto done; - } - else { - ErtsProcLocks unlock_locks = 0; - - if (c_p == rp) - locks |= ERTS_PROC_LOCK_MAIN; - - if (!(locks & ERTS_PROC_LOCK_STATUS)) - unlock_locks |= ERTS_PROC_LOCK_STATUS; - - if (locks & ERTS_PROC_LOCK_MSGQ) { - /* - * Move in queue into private queue and - * release msgq lock, enabling others to - * send messages to the process while it - * is being inspected... - */ - ASSERT(locks & ERTS_PROC_LOCK_MAIN); - ERTS_MSGQ_MV_INQ2PRIVQ(rp); - locks &= ~ERTS_PROC_LOCK_MSGQ; - unlock_locks |= ERTS_PROC_LOCK_MSGQ; - } - - if (unlock_locks) - erts_proc_unlock(rp, unlock_locks); - - } - /* * We always handle 'messages' first if it should be part * of the result. This since if both 'messages' and @@ -895,28 +1006,31 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap, * change the result of 'message_queue_len' (in case * the queue contain bad distribution messages). */ - if (want_messages) { + if (flags & ERTS_PI_FLAG_WANT_MSGS) { ix = pi_arg2ix(am_messages); ASSERT(part_res[ix] == THE_NON_VALUE); - part_res[ix] = process_info_aux(c_p, rp, locks, pid, am_messages, always_wrap); - ASSERT(part_res[ix] != THE_NON_VALUE); + res = process_info_aux(c_p, hfact, rp, rp_locks, ix, + flags, &reserve_size, reds); + ASSERT(res != am_undefined); + ASSERT(res != THE_NON_VALUE); + part_res[ix] = res; } - for (; res_elem_ix_ix >= 0; res_elem_ix_ix--) { - ix = res_elem_ix[res_elem_ix_ix]; + for (item_ix_ix = item_ix_len - 1; item_ix_ix >= 0; item_ix_ix--) { + ix = item_ix[item_ix_ix]; if (part_res[ix] == THE_NON_VALUE) { - arg = pi_ix2arg(ix); - part_res[ix] = process_info_aux(c_p, rp, locks, pid, arg, always_wrap); - ASSERT(part_res[ix] != THE_NON_VALUE); + res = process_info_aux(c_p, hfact, rp, rp_locks, ix, + flags, &reserve_size, reds); + ASSERT(res != am_undefined); + ASSERT(res != THE_NON_VALUE); + part_res[ix] = res; } } - hp = HAlloc(c_p, res_len*2); - hp_end = hp + res_len*2; res = NIL; - for (res_elem_ix_ix = res_len - 1; res_elem_ix_ix >= 0; res_elem_ix_ix--) { - ix = res_elem_ix[res_elem_ix_ix]; + for (item_ix_ix = item_ix_len - 1; item_ix_ix >= 0; item_ix_ix--) { + ix = item_ix[item_ix_ix]; ASSERT(part_res[ix] != THE_NON_VALUE); /* * If we should ignore the value of registered_name, @@ -924,171 +1038,310 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap, * beginning of process_info_aux(). */ if (is_nil(part_res[ix])) { - ASSERT(!always_wrap); + ASSERT(!(flags & ERTS_PI_FLAG_ALWAYS_WRAP)); ASSERT(pi_ix2arg(ix) == am_registered_name); } else { + Eterm *hp; + ERTS_PI_UNRESERVE(reserve_size, 2); + hp = erts_produce_heap(hfact, 2, reserve_size); res = CONS(hp, part_res[ix], res); - hp += 2; } } - if (!always_wrap) { - HRelease(c_p, hp_end, hp); - } - - done: - - if (c_p == rp) - locks &= ~ERTS_PROC_LOCK_MAIN; - if (locks && rp) - erts_proc_unlock(rp, locks); - - if (res_elem_ix != &def_res_elem_ix_buf[0]) - erts_free(ERTS_ALC_T_TMP, res_elem_ix); - return res; } -BIF_RETTYPE process_info_1(BIF_ALIST_1) +static void +pi_setup_grow(int **arr, int *def_arr, Uint *sz, int ix); + +static BIF_RETTYPE +process_info_bif(Process *c_p, Eterm pid, Eterm opt, int always_wrap, int pi2) { + ErtsHeapFactory hfact; + int def_arr[ERTS_PI_DEF_ARR_SZ]; + int *item_ix = &def_arr[0]; + Process *rp = NULL; + erts_aint32_t state; + BIF_RETTYPE ret; + Uint reds = 0; + ErtsProcLocks locks = 0; + int flags; + Uint reserve_size; + int len; Eterm res; - int fail_type; - if (is_external_pid(BIF_ARG_1) - && external_pid_dist_entry(BIF_ARG_1) == erts_this_dist_entry) - BIF_RET(am_undefined); - - if (is_not_internal_pid(BIF_ARG_1)) { - BIF_ERROR(BIF_P, BADARG); - } - - res = process_info_list(BIF_P, BIF_ARG_1, pi_1_keys_list, 0, &fail_type); - if (is_non_value(res)) { - switch (fail_type) { - case ERTS_PI_FAIL_TYPE_BADARG: - BIF_ERROR(BIF_P, BADARG); - case ERTS_PI_FAIL_TYPE_YIELD: - ERTS_BIF_YIELD1(bif_export[BIF_process_info_1], BIF_P, BIF_ARG_1); - case ERTS_PI_FAIL_TYPE_AWAIT_EXIT: - ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined); - default: - erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", __FILE__, __LINE__); - } + ERTS_CT_ASSERT(ERTS_PI_DEF_ARR_SZ > 0); + + if (c_p->common.id == pid) { + int local_only = c_p->flags & F_LOCAL_SIGS_ONLY; + int sreds = ERTS_BIF_REDS_LEFT(c_p); + int sres; + + if (!local_only) { + erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ); + erts_proc_sig_fetch(c_p); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); + } + + sres = erts_proc_sig_handle_incoming(c_p, &state, &sreds, sreds, !0); + if (state & ERTS_PSFLG_EXITING) { + c_p->flags &= ~F_LOCAL_SIGS_ONLY; + goto exited; + } + if (!sres) { + /* + * More signals to handle; need to yield and continue. + * Prevent fetching of more signals by setting + * local-sigs-only flag. + */ + c_p->flags |= F_LOCAL_SIGS_ONLY; + goto yield; + } + + c_p->flags &= ~F_LOCAL_SIGS_ONLY; } - ASSERT(!(BIF_P->flags & F_P2PNR_RESCHED)); - BIF_RET(res); -} + if (is_atom(opt)) { + int ix = pi_arg2ix(opt); + item_ix[0] = ix; + len = 1; + locks = pi_ix2locks(ix); + reserve_size = 3 + pi_ix2rsz(ix); + flags = ERTS_PI_FLAG_SINGELTON; + flags |= pi_ix2flags(ix); + if (ix < 0) + goto badarg; + } + else { + Eterm list = opt; + Uint size = ERTS_PI_DEF_ARR_SZ; + len = 0; + reserve_size = 0; + locks = 0; + flags = 0; -BIF_RETTYPE process_info_2(BIF_ALIST_2) -{ - Eterm res; - Process *rp; - Eterm pid = BIF_ARG_1; - ErtsProcLocks info_locks; - int fail_type; + while (is_list(list)) { + Eterm *consp = list_val(list); + Eterm arg = CAR(consp); + int ix = pi_arg2ix(arg); + if (ix < 0) + goto badarg; + + if (len >= size) + pi_setup_grow(&item_ix, def_arr, &size, len); + + item_ix[len++] = ix; + + locks |= pi_ix2locks(ix); + flags |= pi_ix2flags(ix); + reserve_size += pi_ix2rsz(ix); + reserve_size += 3; /* 2-tuple */ + reserve_size += 2; /* cons */ + + list = CDR(consp); + } + + if (is_not_nil(list)) + goto badarg; + } - if (is_external_pid(pid) - && external_pid_dist_entry(pid) == erts_this_dist_entry) - BIF_RET(am_undefined); - if (is_not_internal_pid(pid)) { - BIF_ERROR(BIF_P, BADARG); + if (is_external_pid(pid) + && external_pid_dist_entry(pid) == erts_this_dist_entry) + goto undefined; + goto badarg; } - if (is_nil(BIF_ARG_2)) - BIF_RET(NIL); + if (always_wrap) + flags |= ERTS_PI_FLAG_ALWAYS_WRAP; - if (is_list(BIF_ARG_2)) { - res = process_info_list(BIF_P, BIF_ARG_1, BIF_ARG_2, 1, &fail_type); - if (is_non_value(res)) { - switch (fail_type) { - case ERTS_PI_FAIL_TYPE_BADARG: - BIF_ERROR(BIF_P, BADARG); - case ERTS_PI_FAIL_TYPE_YIELD: - ERTS_BIF_YIELD2(bif_export[BIF_process_info_2], BIF_P, - BIF_ARG_1, BIF_ARG_2); - case ERTS_PI_FAIL_TYPE_AWAIT_EXIT: - ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined); - default: - erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", - __FILE__, __LINE__); - } - } - ASSERT(!(BIF_P->flags & F_P2PNR_RESCHED)); - BIF_RET(res); + if (c_p->common.id == pid) { + rp = c_p; + if (locks & ~ERTS_PROC_LOCK_MAIN) + erts_proc_lock(c_p, locks & ~ERTS_PROC_LOCK_MAIN); + locks |= ERTS_PROC_LOCK_MAIN; + } + else { + if (flags & ERTS_PI_FLAG_FORCE_SIG_SEND) + goto send_signal; + rp = erts_try_lock_sig_free_proc(pid, locks, &state); + if (!rp) + goto undefined; + if (rp == ERTS_PROC_LOCK_BUSY) { + rp = NULL; + goto send_signal; + } + if (state & ERTS_PSFLG_EXITING) { + if (locks) + erts_proc_unlock(rp, locks); + locks = 0; + /* wait for it to terminate properly... */ + goto send_signal; + } + if (flags & ERTS_PI_FLAG_NEED_MSGQ_LEN) { + ASSERT(locks & ERTS_PROC_LOCK_MAIN); + erts_proc_lock(rp, ERTS_PROC_LOCK_MSGQ); + erts_proc_sig_fetch(rp); + if (c_p->sig_qs.cont) { + erts_proc_unlock(rp, locks|ERTS_PROC_LOCK_MSGQ); + locks = 0; + goto send_signal; + } + erts_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ); + } } - if (pi_arg2ix(BIF_ARG_2) < 0) - BIF_ERROR(BIF_P, BADARG); + erts_factory_proc_init(&hfact, c_p); - info_locks = pi_locks(BIF_ARG_2); + res = erts_process_info(c_p, &hfact, rp, locks, item_ix, len, + flags, reserve_size, &reds); - rp = pi_pid2proc(BIF_P, pid, info_locks|ERTS_PROC_LOCK_STATUS); - if (!rp) - res = am_undefined; - else if (rp == ERTS_PROC_LOCK_BUSY) - ERTS_BIF_YIELD2(bif_export[BIF_process_info_2], BIF_P, - BIF_ARG_1, BIF_ARG_2); - else if (rp != BIF_P && ERTS_PROC_PENDING_EXIT(rp)) { - erts_proc_unlock(rp, info_locks|ERTS_PROC_LOCK_STATUS); - ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined); + erts_factory_close(&hfact); + + if (reds > INT_MAX/2) + reds = INT_MAX/2; + BUMP_REDS(c_p, (int) reds); + + state = erts_atomic32_read_acqb(&rp->state); + if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_FREE)) { + if (state & ERTS_PSFLG_FREE) { + ASSERT(!locks); + goto undefined; + } + if (locks) + erts_proc_unlock(rp, locks); + locks = 0; + /* wait for it to terminate properly... */ + goto send_signal; } - else { - ErtsProcLocks unlock_locks = 0; - if (BIF_P == rp) - info_locks |= ERTS_PROC_LOCK_MAIN; + ERTS_BIF_PREP_RET(ret, res); - if (!(info_locks & ERTS_PROC_LOCK_STATUS)) - unlock_locks |= ERTS_PROC_LOCK_STATUS; +done: - if (info_locks & ERTS_PROC_LOCK_MSGQ) { - /* - * Move in queue into private queue and - * release msgq lock, enabling others to - * send messages to the process while it - * is being inspected... - */ - ASSERT(info_locks & ERTS_PROC_LOCK_MAIN); - ERTS_MSGQ_MV_INQ2PRIVQ(rp); - info_locks &= ~ERTS_PROC_LOCK_MSGQ; - unlock_locks |= ERTS_PROC_LOCK_MSGQ; - } + if (c_p == rp) + locks &= ~ERTS_PROC_LOCK_MAIN; + + if (locks && rp) + erts_proc_unlock(rp, locks); - if (unlock_locks) - erts_proc_unlock(rp, unlock_locks); + if (item_ix != def_arr) + erts_free(ERTS_ALC_T_TMP, item_ix); - res = process_info_aux(BIF_P, rp, info_locks, pid, BIF_ARG_2, 0); + return ret; + +badarg: + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + goto done; + +undefined: + ERTS_BIF_PREP_RET(ret, am_undefined); + goto done; + +exited: + ERTS_BIF_PREP_EXITED(ret, c_p); + goto done; + +yield: + if (pi2) + ERTS_BIF_PREP_YIELD2(ret, bif_export[BIF_process_info_2], c_p, pid, opt); + else + ERTS_BIF_PREP_YIELD1(ret, bif_export[BIF_process_info_1], c_p, pid); + goto done; + +send_signal: { + Eterm ref = erts_make_ref(c_p); + int enqueued, need_msgq_len; + flags |= ERTS_PI_FLAG_REQUEST_FOR_OTHER; + need_msgq_len = (flags & ERTS_PI_FLAG_NEED_MSGQ_LEN); + /* + * Set receive mark so we wont have to scan the whole + * message queue for the result. Note caller unconditionally + * has to enter a receive only matching messages containing + * 'ref', or restore save pointer. + */ + ERTS_RECV_MARK_SAVE(c_p); + ERTS_RECV_MARK_SET(c_p); + enqueued = erts_proc_sig_send_process_info_request(c_p, pid, item_ix, + len, need_msgq_len, + flags, reserve_size, + ref); + if (!enqueued) { + /* Restore save pointer... */ + JOIN_MESSAGE(c_p); + goto undefined; + } + ERTS_BIF_PREP_TRAP1(ret, erts_await_result, c_p, ref); + goto done; } - ASSERT(is_value(res)); +} - if (BIF_P == rp) - info_locks &= ~ERTS_PROC_LOCK_MAIN; - if (rp && info_locks) - erts_proc_unlock(rp, info_locks); +static void +pi_setup_grow(int **arr, int *def_arr, Uint *sz, int ix) +{ + *sz = (ix+1) + ERTS_PI_DEF_ARR_SZ; + if (*arr != def_arr) + *arr = erts_realloc(ERTS_ALC_T_TMP, *arr, (*sz)*sizeof(int)); + else { + int *new_arr = erts_alloc(ERTS_ALC_T_TMP, (*sz)*sizeof(int)); + sys_memcpy((void *) new_arr, (void *) def_arr, + sizeof(int)*ERTS_PI_DEF_ARR_SZ); + *arr = new_arr; + } +} - ASSERT(!(BIF_P->flags & F_P2PNR_RESCHED)); - BIF_RET(res); + +BIF_RETTYPE process_info_2(BIF_ALIST_2) +{ + return process_info_bif(BIF_P, BIF_ARG_1, BIF_ARG_2, !is_atom(BIF_ARG_2), !0); +} + +BIF_RETTYPE process_info_1(BIF_ALIST_1) +{ + return process_info_bif(BIF_P, BIF_ARG_1, pi_1_keys_list, 0, 0); } Eterm -process_info_aux(Process *BIF_P, +process_info_aux(Process *c_p, + ErtsHeapFactory *hfact, Process *rp, ErtsProcLocks rp_locks, - Eterm rpid, - Eterm item, - int always_wrap) + int item_ix, + int flags, + Uint *reserve_sizep, + Uint *reds) { Eterm *hp; Eterm res = NIL; + Uint reserved; + Uint reserve_size = *reserve_sizep; + +#ifdef ERTS_ENABLE_LOCK_CHECK + ErtsProcLocks locks = erts_proc_lc_my_proc_locks(rp); + + switch (item_ix) { + case ERTS_PI_IX_STATUS: + case ERTS_PI_IX_PRIORITY: + case ERTS_PI_IX_SUSPENDING: + ERTS_LC_ASSERT((locks & ~ERTS_PROC_LOCK_MAIN) == 0); + break; + default: + ERTS_LC_ASSERT(locks == ERTS_PROC_LOCK_MAIN); + break; + } +#endif + + reserved = pi_ix2rsz(item_ix); + ERTS_PI_UNRESERVE(reserve_size, reserved); + + (*reds)++; ASSERT(rp); /* - * Q: Why this always_wrap argument? + * Q: Why this ERTS_PI_FLAG_ALWAYS_WRAP flag? * * A: registered_name is strange. If process has no registered name, * process_info(Pid, registered_name) returns [], and @@ -1100,41 +1353,39 @@ process_info_aux(Process *BIF_P, * registered_name behaves as it should, i.e. a * {registered_name, []} will appear in the resulting list. * - * If always_wrap != 0, process_info_aux() always wrap the result - * in a key two tuple. + * If ERTS_PI_FLAG_ALWAYS_WRAP is set, process_info_aux() always + * wrap the result in a key two tuple. */ - switch (item) { + switch (item_ix) { - case am_registered_name: - if (rp->common.u.alive.reg) { - hp = HAlloc(BIF_P, 3); + case ERTS_PI_IX_REGISTERED_NAME: + if (rp->common.u.alive.reg) res = rp->common.u.alive.reg->name; - } else { - if (always_wrap) { - hp = HAlloc(BIF_P, 3); + else { + if (flags & ERTS_PI_FLAG_ALWAYS_WRAP) res = NIL; - } - else { + else return NIL; - } } break; - case am_current_function: - res = current_function(BIF_P, rp, &hp, 0); + case ERTS_PI_IX_CURRENT_FUNCTION: + res = current_function(c_p, hfact, rp, 0, + reserve_size, flags); break; - case am_current_location: - res = current_function(BIF_P, rp, &hp, 1); + case ERTS_PI_IX_CURRENT_LOCATION: + res = current_function(c_p, hfact, rp, 1, + reserve_size, flags); break; - case am_current_stacktrace: - res = current_stacktrace(BIF_P, rp, &hp); + case ERTS_PI_IX_CURRENT_STACKTRACE: + res = current_stacktrace(hfact, rp, reserve_size); break; - case am_initial_call: - hp = HAlloc(BIF_P, 3+4); + case ERTS_PI_IX_INITIAL_CALL: + hp = erts_produce_heap(hfact, 4, reserve_size); res = TUPLE3(hp, rp->u.initial.module, rp->u.initial.function, @@ -1142,96 +1393,130 @@ process_info_aux(Process *BIF_P, hp += 4; break; - case am_status: - res = erts_process_status(rp, rpid); - ASSERT(res != am_undefined); - hp = HAlloc(BIF_P, 3); + case ERTS_PI_IX_STATUS: { + erts_aint32_t state = erts_atomic32_read_nob(&rp->state); + res = erts_process_state2status(state); + if (res == am_running && (state & ERTS_PSFLG_RUNNING_SYS)) { + ASSERT(c_p == rp); + ASSERT(flags & ERTS_PI_FLAG_REQUEST_FOR_OTHER); + if (!(state & (ERTS_PSFLG_SYS_TASKS + | ERTS_PSFLG_ACTIVE + | ERTS_PSFLG_SIG_Q + | ERTS_PSFLG_SIG_IN_Q))) { + /* + * We are servicing a process-info request from + * another process. If that other process could + * have inspected our state itself, we would have + * been in the 'waiting' state. + */ + res = am_waiting; + } + } break; + } - case am_messages: { - - if (rp->msg.len == 0 || ERTS_TRACE_FLAGS(rp) & F_SENSITIVE) { - hp = HAlloc(BIF_P, 3); - } else { + case ERTS_PI_IX_MESSAGES: { + ASSERT(flags & ERTS_PI_FLAG_NEED_MSGQ_LEN); + if (rp->sig_qs.len == 0 || (ERTS_TRACE_FLAGS(rp) & F_SENSITIVE)) + res = NIL; + else { + int info_on_self = !(flags & ERTS_PI_FLAG_REQUEST_FOR_OTHER); ErtsMessageInfo *mip; - Sint i; + Sint i, len; Uint heap_need; -#ifdef DEBUG - Eterm *hp_end; -#endif mip = erts_alloc(ERTS_ALC_T_TMP, - rp->msg.len*sizeof(ErtsMessageInfo)); + rp->sig_qs.len*sizeof(ErtsMessageInfo)); /* * Note that message queue may shrink when calling - * erts_prep_msgq_for_inspection() since it removes + * erts_proc_sig_prep_msgq_for_inspection() since it removes * corrupt distribution messages. */ - heap_need = erts_prep_msgq_for_inspection(BIF_P, rp, rp_locks, mip); - heap_need += 3; /* top 2-tuple */ - heap_need += rp->msg.len*2; /* Cons cells */ + heap_need = erts_proc_sig_prep_msgq_for_inspection(c_p, rp, + rp_locks, + info_on_self, + mip); + len = rp->sig_qs.len; - hp = HAlloc(BIF_P, heap_need); /* heap_need is exact */ -#ifdef DEBUG - hp_end = hp + heap_need; -#endif + heap_need += len*2; /* Cons cells */ + + reserve_size += heap_need; /* Build list of messages... */ - for (i = rp->msg.len - 1, res = NIL; i >= 0; i--) { + for (i = len - 1, res = NIL; i >= 0; i--) { Eterm msg = ERL_MESSAGE_TERM(mip[i].msgp); Uint sz = mip[i].size; + ERTS_PI_UNRESERVE(reserve_size, sz+2); + hp = erts_produce_heap(hfact, sz+2, reserve_size); + if (sz != 0) - msg = copy_struct(msg, sz, &hp, &BIF_P->off_heap); + msg = copy_struct(msg, sz, &hp, hfact->off_heap); res = CONS(hp, msg, res); hp += 2; } - ASSERT(hp_end == hp + 3); + *reds += (Uint) len / 4; erts_free(ERTS_ALC_T_TMP, mip); } break; } - case am_message_queue_len: - hp = HAlloc(BIF_P, 3); - res = make_small(rp->msg.len); + case ERTS_PI_IX_MESSAGE_QUEUE_LEN: { + Sint len = rp->sig_qs.len; + ASSERT(flags & ERTS_PI_FLAG_NEED_MSGQ_LEN); + ASSERT(len >= 0); + if (len <= MAX_SMALL) + res = make_small(len); + else { + hp = erts_produce_heap(hfact, BIG_UINT_HEAP_SIZE, reserve_size); + res = uint_to_big((Uint) len, hp); + } break; + } - case am_links: { + case ERTS_PI_IX_LINKS: { MonitorInfoCollection mic; int i; Eterm item; INIT_MONITOR_INFOS(mic); - erts_doforall_links(ERTS_P_LINKS(rp),&collect_one_link,&mic); + erts_link_tree_foreach(ERTS_P_LINKS(rp), collect_one_link, (void *) &mic); - hp = HAlloc(BIF_P, 3 + mic.sz); + reserve_size += mic.sz; res = NIL; for (i = 0; i < mic.mi_i; i++) { - item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity.term); + Eterm item_src = mic.mi[i].entity.term; + Uint sz = NC_HEAP_SIZE(item_src) + 2; + ERTS_PI_UNRESERVE(reserve_size, sz); + hp = erts_produce_heap(hfact, sz, reserve_size); + item = STORE_NC(&hp, hfact->off_heap, item_src); res = CONS(hp, item, res); - hp += 2; } + + *reds += (Uint) mic.mi_i / 4; + DESTROY_MONITOR_INFOS(mic); break; } - case am_monitors: { + case ERTS_PI_IX_MONITORS: { MonitorInfoCollection mic; int i; INIT_MONITOR_INFOS(mic); - erts_doforall_monitors(ERTS_P_MONITORS(rp), - &collect_one_origin_monitor, &mic); - hp = HAlloc(BIF_P, 3 + mic.sz); + erts_monitor_tree_foreach(ERTS_P_MONITORS(rp), + collect_one_origin_monitor, + (void *) &mic); + + reserve_size += mic.sz; res = NIL; for (i = 0; i < mic.mi_i; i++) { - if (is_atom(mic.mi[i].entity.term)) { + if (mic.mi[i].named) { /* Monitor by name. * Build {process|port, {Name, Node}} and cons it. */ @@ -1243,84 +1528,136 @@ process_info_aux(Process *BIF_P, || is_port(mic.mi[i].pid) || is_atom(mic.mi[i].pid)); + ERTS_PI_UNRESERVE(reserve_size, 3+3+2); + hp = erts_produce_heap(hfact, 3+3+2, reserve_size); + t1 = TUPLE2(hp, mic.mi[i].entity.term, mic.mi[i].node); hp += 3; t2 = TUPLE2(hp, m_type, t1); hp += 3; res = CONS(hp, t2, res); - hp += 2; } else { - /* Monitor by pid. Build {process|port, Pid} and cons it. */ + /* Build {process|port|time_offset, Pid|clock_service} and cons it. */ Eterm t; - Eterm pid = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity.term); + Eterm pid; + Eterm m_type; + Eterm pid_src = mic.mi[i].entity.term; + Uint sz = is_atom(pid_src) ? 0 : NC_HEAP_SIZE(pid_src); + sz += 3 + 2; + + ERTS_PI_UNRESERVE(reserve_size, sz); + hp = erts_produce_heap(hfact, sz, reserve_size); + + pid = (is_atom(pid_src) + ? pid_src + : STORE_NC(&hp, hfact->off_heap, pid_src)); + + switch (mic.mi[i].type) { + case ERTS_MON_TYPE_PORT: + m_type = am_port; + break; + case ERTS_MON_TYPE_TIME_OFFSET: + m_type = am_time_offset; + break; + default: + m_type = am_process; + break; + } - Eterm m_type = is_port(mic.mi[i].pid) ? am_port : am_process; ASSERT(is_pid(mic.mi[i].pid) || is_port(mic.mi[i].pid)); t = TUPLE2(hp, m_type, pid); hp += 3; res = CONS(hp, t, res); - hp += 2; } } + + *reds += (Uint) mic.mi_i / 4; + DESTROY_MONITOR_INFOS(mic); break; } - case am_monitored_by: { + case ERTS_PI_IX_MONITORED_BY: { MonitorInfoCollection mic; int i; Eterm item; INIT_MONITOR_INFOS(mic); - erts_doforall_monitors(ERTS_P_MONITORS(rp),&collect_one_target_monitor,&mic); - hp = HAlloc(BIF_P, 3 + mic.sz); + erts_monitor_list_foreach(ERTS_P_LT_MONITORS(rp), + collect_one_target_monitor, + (void *) &mic); + erts_monitor_tree_foreach(ERTS_P_MONITORS(rp), + collect_one_target_monitor, + (void *) &mic); + + reserve_size += mic.sz; res = NIL; for (i = 0; i < mic.mi_i; ++i) { - if (mic.mi[i].node == make_small(MON_NIF_TARGET)) { - item = erts_bld_resource_ref(&hp, &MSO(BIF_P), mic.mi[i].entity.resource); - } - else { - item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity.term); - } + Uint sz = 2; + + if (mic.mi[i].type == ERTS_MON_TYPE_RESOURCE) + sz += erts_resource_ref_size(mic.mi[i].entity.resource); + else + sz += NC_HEAP_SIZE(mic.mi[i].entity.term); + + ERTS_PI_UNRESERVE(reserve_size, sz); + hp = erts_produce_heap(hfact, sz, reserve_size); + + if (mic.mi[i].type == ERTS_MON_TYPE_RESOURCE) + item = erts_bld_resource_ref(&hp, + hfact->off_heap, + mic.mi[i].entity.resource); + else + item = STORE_NC(&hp, + hfact->off_heap, + mic.mi[i].entity.term); res = CONS(hp, item, res); - hp += 2; } + + *reds += (Uint) mic.mi_i / 4; + DESTROY_MONITOR_INFOS(mic); break; } - case am_suspending: { + case ERTS_PI_IX_SUSPENDING: { ErtsSuspendMonitorInfoCollection smic; int i; Eterm item; -#ifdef DEBUG - Eterm *hp_end; -#endif + + erts_proc_lock(rp, ERTS_PROC_LOCK_STATUS); ERTS_INIT_SUSPEND_MONITOR_INFOS(smic, - BIF_P, - (BIF_P == rp + c_p, + (c_p == rp ? ERTS_PROC_LOCK_MAIN - : 0) | ERTS_PROC_LOCK_LINK); + : 0) | ERTS_PROC_LOCK_STATUS); + + erts_monitor_tree_foreach(rp->suspend_monitors, + &collect_one_suspend_monitor, + &smic); + + reserve_size += smic.sz; - erts_doforall_suspend_monitors(rp->suspend_monitors, - &collect_one_suspend_monitor, - &smic); - hp = HAlloc(BIF_P, 3 + smic.sz); -#ifdef DEBUG - hp_end = hp + smic.sz; -#endif - res = NIL; for (i = 0; i < smic.smi_i; i++) { Sint a = (Sint) smic.smi[i]->active; /* quiet compiler warnings */ Sint p = (Sint) smic.smi[i]->pending; /* on 64-bit machines... */ Eterm active; Eterm pending; + Uint sz = 4 + 2; + if (!IS_SSMALL(a)) + sz += BIG_UINT_HEAP_SIZE; + if (!IS_SSMALL(p)) + sz += BIG_UINT_HEAP_SIZE; + + ERTS_PI_UNRESERVE(reserve_size, sz); + hp = erts_produce_heap(hfact, sz, reserve_size); + if (IS_SSMALL(a)) active = make_small(a); else { @@ -1333,90 +1670,86 @@ process_info_aux(Process *BIF_P, pending = small_to_big(p, hp); hp += BIG_UINT_HEAP_SIZE; } - item = TUPLE3(hp, smic.smi[i]->pid, active, pending); + item = TUPLE3(hp, smic.smi[i]->mon.other.item, active, pending); hp += 4; res = CONS(hp, item, res); - hp += 2; } + erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + + *reds += (Uint) smic.smi_i / 4; + ERTS_DESTROY_SUSPEND_MONITOR_INFOS(smic); - ASSERT(hp == hp_end); break; } - case am_dictionary: - if (ERTS_TRACE_FLAGS(rp) & F_SENSITIVE) { + case ERTS_PI_IX_DICTIONARY: + if (!rp->dictionary || (ERTS_TRACE_FLAGS(rp) & F_SENSITIVE)) { res = NIL; } else { - res = erts_dictionary_copy(BIF_P, rp->dictionary); + Uint num = rp->dictionary->numElements; + res = erts_dictionary_copy(hfact, rp->dictionary, reserve_size); + *reds += (Uint) num / 4; } - hp = HAlloc(BIF_P, 3); + break; - case am_trap_exit: { - erts_aint32_t state = erts_atomic32_read_nob(&rp->state); - hp = HAlloc(BIF_P, 3); - if (state & ERTS_PSFLG_TRAP_EXIT) - res = am_true; - else - res = am_false; + case ERTS_PI_IX_TRAP_EXIT: + res = (rp->flags & F_TRAP_EXIT) ? am_true : am_false; break; - } - case am_error_handler: - hp = HAlloc(BIF_P, 3); - res = erts_proc_get_error_handler(BIF_P); + case ERTS_PI_IX_ERROR_HANDLER: + res = erts_proc_get_error_handler(rp); break; - case am_heap_size: { - Uint hsz = 3; + case ERTS_PI_IX_HEAP_SIZE: { + Uint hsz = 0; (void) erts_bld_uint(NULL, &hsz, HEAP_SIZE(rp)); - hp = HAlloc(BIF_P, hsz); + hp = erts_produce_heap(hfact, hsz, reserve_size); res = erts_bld_uint(&hp, NULL, HEAP_SIZE(rp)); break; } - case am_fullsweep_after: { - Uint hsz = 3; + case ERTS_PI_IX_FULLSWEEP_AFTER: { + Uint hsz = 0; (void) erts_bld_uint(NULL, &hsz, MAX_GEN_GCS(rp)); - hp = HAlloc(BIF_P, hsz); + hp = erts_produce_heap(hfact, hsz, reserve_size); res = erts_bld_uint(&hp, NULL, MAX_GEN_GCS(rp)); break; } - case am_min_heap_size: { - Uint hsz = 3; + case ERTS_PI_IX_MIN_HEAP_SIZE: { + Uint hsz = 0; (void) erts_bld_uint(NULL, &hsz, MIN_HEAP_SIZE(rp)); - hp = HAlloc(BIF_P, hsz); + hp = erts_produce_heap(hfact, hsz, reserve_size); res = erts_bld_uint(&hp, NULL, MIN_HEAP_SIZE(rp)); break; } - case am_min_bin_vheap_size: { - Uint hsz = 3; + case ERTS_PI_IX_MIN_BIN_VHEAP_SIZE: { + Uint hsz = 0; (void) erts_bld_uint(NULL, &hsz, MIN_VHEAP_SIZE(rp)); - hp = HAlloc(BIF_P, hsz); + hp = erts_produce_heap(hfact, hsz, reserve_size); res = erts_bld_uint(&hp, NULL, MIN_VHEAP_SIZE(rp)); break; } - case am_max_heap_size: { - Uint hsz = 3; + case ERTS_PI_IX_MAX_HEAP_SIZE: { + Uint hsz = 0; (void) erts_max_heap_size_map(MAX_HEAP_SIZE_GET(rp), MAX_HEAP_SIZE_FLAGS_GET(rp), NULL, &hsz); - hp = HAlloc(BIF_P, hsz); + hp = erts_produce_heap(hfact, hsz, reserve_size); res = erts_max_heap_size_map(MAX_HEAP_SIZE_GET(rp), MAX_HEAP_SIZE_FLAGS_GET(rp), &hp, NULL); break; } - case am_total_heap_size: { - ErtsMessage *mp; + case ERTS_PI_IX_TOTAL_HEAP_SIZE: { Uint total_heap_size; - Uint hsz = 3; + Uint hsz = 0; total_heap_size = rp->heap_sz; if (rp->old_hend && rp->old_heap) @@ -1424,44 +1757,53 @@ process_info_aux(Process *BIF_P, total_heap_size += rp->mbuf_sz; - if (rp->flags & F_ON_HEAP_MSGQ) - for (mp = rp->msg.first; mp; mp = mp->next) + if (rp->flags & F_ON_HEAP_MSGQ) { + ErtsMessage *mp; + ASSERT(flags & ERTS_PI_FLAG_NEED_MSGQ_LEN); + for (mp = rp->sig_qs.first; mp; mp = mp->next) { + ASSERT(ERTS_SIG_IS_MSG(mp)); if (mp->data.attached) total_heap_size += erts_msg_attached_data_size(mp); + } + *reds += (Uint) rp->sig_qs.len / 4; + } (void) erts_bld_uint(NULL, &hsz, total_heap_size); - hp = HAlloc(BIF_P, hsz); + hp = erts_produce_heap(hfact, hsz, reserve_size); res = erts_bld_uint(&hp, NULL, total_heap_size); break; } - case am_stack_size: { + case ERTS_PI_IX_STACK_SIZE: { Uint stack_size = STACK_START(rp) - rp->stop; - Uint hsz = 3; + Uint hsz = 0; (void) erts_bld_uint(NULL, &hsz, stack_size); - hp = HAlloc(BIF_P, hsz); + hp = erts_produce_heap(hfact, hsz, reserve_size); res = erts_bld_uint(&hp, NULL, stack_size); break; } - case am_memory: { /* Memory consumed in bytes */ - Uint hsz = 3; + case ERTS_PI_IX_MEMORY: { /* Memory consumed in bytes */ + Uint hsz = 0; Uint size = erts_process_memory(rp, 0); (void) erts_bld_uint(NULL, &hsz, size); - hp = HAlloc(BIF_P, hsz); + hp = erts_produce_heap(hfact, hsz, reserve_size); res = erts_bld_uint(&hp, NULL, size); + + ASSERT(flags & ERTS_PI_FLAG_NEED_MSGQ_LEN); + *reds += (Uint) rp->sig_qs.len / 4; + break; } - case am_garbage_collection: { + case ERTS_PI_IX_GARBAGE_COLLECTION: { DECL_AM(minor_gcs); Eterm t; Uint map_sz = 0; erts_max_heap_size_map(MAX_HEAP_SIZE_GET(rp), MAX_HEAP_SIZE_FLAGS_GET(rp), NULL, &map_sz); - hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2 + 3+2 + 3+2 + map_sz + 3); - /* last "3" is for outside tuple */ + hp = erts_produce_heap(hfact, 3+2 + 3+2 + 3+2 + 3+2 + 3+2 + map_sz, reserve_size); t = TUPLE2(hp, AM_minor_gcs, make_small(GEN_GCS(rp))); hp += 3; res = CONS(hp, t, NIL); hp += 2; @@ -1480,89 +1822,76 @@ process_info_aux(Process *BIF_P, break; } - case am_garbage_collection_info: { + case ERTS_PI_IX_GARBAGE_COLLECTION_INFO: { Uint sz = 0, actual_sz = 0; - if (rp == BIF_P) { - sz += ERTS_PROCESS_GC_INFO_MAX_SIZE; - } else { - erts_process_gc_info(rp, &sz, NULL, 0, 0); - sz += 3; - } + erts_process_gc_info(rp, &sz, NULL, 0, 0); - hp = HAlloc(BIF_P, sz); + hp = erts_produce_heap(hfact, sz, reserve_size); res = erts_process_gc_info(rp, &actual_sz, &hp, 0, 0); - /* We may have some extra space, fill with 0 tuples */ - if (actual_sz <= sz - 3) { - for (; actual_sz < sz - 3; hp++, actual_sz++) - hp[0] = make_arityval(0); - } else { - for (; actual_sz < sz; hp++, actual_sz++) - hp[0] = make_arityval(0); - hp = HAlloc(BIF_P, 3); - } - break; } - case am_group_leader: { + case ERTS_PI_IX_GROUP_LEADER: { int sz = NC_HEAP_SIZE(rp->group_leader); - hp = HAlloc(BIF_P, 3 + sz); - res = STORE_NC(&hp, &MSO(BIF_P), rp->group_leader); + hp = erts_produce_heap(hfact, sz, reserve_size); + res = STORE_NC(&hp, hfact->off_heap, rp->group_leader); break; } - case am_reductions: { - Uint reds = rp->reds + erts_current_reductions(BIF_P, rp); - Uint hsz = 3; + case ERTS_PI_IX_REDUCTIONS: { + Uint reds = rp->reds + erts_current_reductions(c_p, rp); + Uint hsz = 0; (void) erts_bld_uint(NULL, &hsz, reds); - hp = HAlloc(BIF_P, hsz); + hp = erts_produce_heap(hfact, hsz, reserve_size); res = erts_bld_uint(&hp, NULL, reds); break; } - case am_priority: - hp = HAlloc(BIF_P, 3); - res = erts_get_process_priority(rp); + case ERTS_PI_IX_PRIORITY: { + erts_aint32_t state = erts_atomic32_read_nob(&rp->state); + if (ERTS_PSFLG_EXITING & state) + return am_undefined; + res = erts_get_process_priority(state); break; + } - case am_trace: - hp = HAlloc(BIF_P, 3); + case ERTS_PI_IX_TRACE: res = make_small(ERTS_TRACE_FLAGS(rp) & TRACEE_FLAGS); break; - case am_binary: { - Uint sz = 3; + case ERTS_PI_IX_BINARY: { + Uint sz = 0; (void) bld_bin_list(NULL, &sz, &MSO(rp)); - hp = HAlloc(BIF_P, sz); + hp = erts_produce_heap(hfact, sz, reserve_size); res = bld_bin_list(&hp, NULL, &MSO(rp)); break; } - case am_sequential_trace_token: - res = copy_object(rp->seq_trace_token, BIF_P); - hp = HAlloc(BIF_P, 3); + case ERTS_PI_IX_SEQUENTIAL_TRACE_TOKEN: { + Uint sz = size_object(rp->seq_trace_token); + hp = erts_produce_heap(hfact, sz, reserve_size); + res = copy_struct(rp->seq_trace_token, sz, &hp, hfact->off_heap); break; + } - case am_catchlevel: - hp = HAlloc(BIF_P, 3); - res = make_small(catchlevel(BIF_P)); + case ERTS_PI_IX_CATCHLEVEL: + res = make_small(catchlevel(rp)); break; - case am_backtrace: { + case ERTS_PI_IX_BACKTRACE: { erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0); erts_stack_dump(ERTS_PRINT_DSBUF, (void *) dsbufp, rp); - res = new_binary(BIF_P, (byte *) dsbufp->str, dsbufp->str_len); + res = erts_heap_factory_new_binary(hfact, (byte *) dsbufp->str, + dsbufp->str_len, reserve_size); erts_destroy_tmp_dsbuf(dsbufp); - hp = HAlloc(BIF_P, 3); break; } - case am_last_calls: { + case ERTS_PI_IX_LAST_CALLS: { struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(rp); if (!scb) { - hp = HAlloc(BIF_P, 3); res = am_false; } else { /* @@ -1570,23 +1899,34 @@ process_info_aux(Process *BIF_P, * Might be less than that, if there are sends, receives or timeouts, * so we must do a HRelease() to avoid creating holes. */ - Uint needed = scb->n*(2+4) + 3; - Eterm* limit; + Sint needed = scb->n*(2+4); Eterm term, list; int i, j; + Export *exp; + + reserve_size += needed; - hp = HAlloc(BIF_P, needed); - limit = hp + needed; list = NIL; for (i = 0; i < scb->n; i++) { + Uint sz; j = scb->cur - i - 1; if (j < 0) j += scb->len; - if (scb->ct[j] == &exp_send) + + sz = 2; + exp = scb->ct[j]; + if (exp != &exp_send && exp != &exp_receive && exp != &exp_timeout) + sz += 4; + + needed -= sz; + ERTS_PI_UNRESERVE(reserve_size, sz); + hp = erts_produce_heap(hfact, sz, reserve_size); + + if (exp == &exp_send) term = am_send; - else if (scb->ct[j] == &exp_receive) + else if (exp == &exp_receive) term = am_receive; - else if (scb->ct[j] == &exp_timeout) + else if (exp == &exp_timeout) term = am_timeout; else { term = TUPLE3(hp, @@ -1596,18 +1936,18 @@ process_info_aux(Process *BIF_P, hp += 4; } list = CONS(hp, term, list); - hp += 2; } + + ASSERT(needed >= 0); + if (needed > 0) + reserve_size -= needed; + res = list; - res = TUPLE2(hp, item, res); - hp += 3; - HRelease(BIF_P,limit,hp); - return res; } break; } - case am_message_queue_data: + case ERTS_PI_IX_MESSAGE_QUEUE_DATA: switch (rp->flags & (F_OFF_HEAP_MSGQ|F_ON_HEAP_MSGQ)) { case F_OFF_HEAP_MSGQ: res = am_off_heap; @@ -1620,14 +1960,15 @@ process_info_aux(Process *BIF_P, ERTS_INTERNAL_ERROR("Inconsistent message queue management state"); break; } - hp = HAlloc(BIF_P, 3); break; - case am_magic_ref: { - Uint sz = 3; + case ERTS_PI_IX_MAGIC_REF: { + Uint sz = 0; (void) bld_magic_ref_bin_list(NULL, &sz, &MSO(rp)); - hp = HAlloc(BIF_P, sz); + hp = erts_produce_heap(hfact, sz, 0); res = bld_magic_ref_bin_list(&hp, NULL, &MSO(rp)); + + *reds += (Uint) 10; break; } @@ -1636,12 +1977,17 @@ process_info_aux(Process *BIF_P, } - return TUPLE2(hp, item, res); + ERTS_PI_UNRESERVE(reserve_size, 3); + *reserve_sizep = reserve_size; + hp = erts_produce_heap(hfact, 3, reserve_size); + + return TUPLE2(hp, pi_ix2arg(item_ix), res); } #undef MI_INC static Eterm -current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info) +current_function(Process *c_p, ErtsHeapFactory *hfact, Process* rp, + int full_info, Uint reserve_size, int flags) { Eterm* hp; Eterm res; @@ -1658,7 +2004,7 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info) } } - if (BIF_P == rp) { + if (c_p == rp && !(flags & ERTS_PI_FLAG_REQUEST_FOR_OTHER)) { FunctionInfo fi2; /* @@ -1678,24 +2024,22 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info) * Return the result. */ if (rp->current == NULL) { - hp = HAlloc(BIF_P, 3); res = am_undefined; } else if (full_info) { - hp = HAlloc(BIF_P, 3+fi.needed); - hp = erts_build_mfa_item(&fi, hp, am_true, &res); + hp = erts_produce_heap(hfact, fi.needed, reserve_size); + erts_build_mfa_item(&fi, hp, am_true, &res); } else { - hp = HAlloc(BIF_P, 3+4); + hp = erts_produce_heap(hfact, 4, reserve_size); res = TUPLE3(hp, rp->current->module, rp->current->function, make_small(rp->current->arity)); - hp += 4; } - *hpp = hp; return res; } static Eterm -current_stacktrace(Process* p, Process* rp, Eterm** hpp) +current_stacktrace(ErtsHeapFactory *hfact, Process* rp, + Uint reserve_size) { Uint sz; struct StackTrace* s; @@ -1704,7 +2048,7 @@ current_stacktrace(Process* p, Process* rp, Eterm** hpp) FunctionInfo* stkp; Uint heap_size; int i; - Eterm* hp = *hpp; + Eterm* hp; Eterm mfa; Eterm res = NIL; @@ -1734,17 +2078,23 @@ current_stacktrace(Process* p, Process* rp, Eterm** hpp) } } - hp = HAlloc(p, heap_size); + reserve_size += heap_size; + + /* + * We intentionally produce heap in small chunks + * (for more info see process_info_aux()). + */ while (stkp > stk) { stkp--; + sz = stkp->needed + 2; + ERTS_PI_UNRESERVE(reserve_size, sz); + hp = erts_produce_heap(hfact, sz, reserve_size); hp = erts_build_mfa_item(stkp, hp, am_true, &mfa); res = CONS(hp, mfa, res); - hp += 2; } erts_free(ERTS_ALC_T_TMP, stk); erts_free(ERTS_ALC_T_TMP, s); - *hpp = hp; return res; } @@ -1802,45 +2152,6 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ return make_small(sizeof(UWord)); } goto badarg; - } else if (sel == am_allocated) { - if (arity == 2) { - Eterm res = THE_NON_VALUE; - char *buf; - Sint len = is_string(*tp); - if (len <= 0) - return res; - buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1); - if (intlist_to_buf(*tp, buf, len) != len) - erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); - buf[len] = '\0'; - res = erts_instr_dump_memory_map(buf) ? am_true : am_false; - erts_free(ERTS_ALC_T_TMP, (void *) buf); - if (is_non_value(res)) - goto badarg; - return res; - } - else if (arity == 3 && tp[0] == am_status) { - if (is_atom(tp[1])) - return erts_instr_get_stat(BIF_P, tp[1], 1); - else { - Eterm res = THE_NON_VALUE; - char *buf; - Sint len = is_string(tp[1]); - if (len <= 0) - return res; - buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1); - if (intlist_to_buf(tp[1], buf, len) != len) - erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); - buf[len] = '\0'; - res = erts_instr_dump_stat(buf, 1) ? am_true : am_false; - erts_free(ERTS_ALC_T_TMP, (void *) buf); - if (is_non_value(res)) - goto badarg; - return res; - } - } - else - goto badarg; } else if (sel == am_allocator) { switch (arity) { case 2: @@ -2208,8 +2519,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } else if (BIF_ARG_1 == am_allocated_areas) { res = erts_allocated_areas(NULL, NULL, BIF_P); BIF_RET(res); - } else if (BIF_ARG_1 == am_allocated) { - BIF_RET(erts_instr_get_memory_map(BIF_P)); } else if (BIF_ARG_1 == am_hipe_architecture) { #if defined(HIPE) BIF_RET(hipe_arch_name); @@ -2350,9 +2659,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) sizeof(ERLANG_ARCHITECTURE)-1, NIL)); } - else if (BIF_ARG_1 == am_memory_types) { - return erts_instr_get_type_info(BIF_P); - } else if (BIF_ARG_1 == am_os_type) { BIF_RET(os_type_tuple); } @@ -2862,6 +3168,16 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } +static void monitor_size(ErtsMonitor *mon, void *vsz) +{ + *((Uint *) vsz) = erts_monitor_size(mon); +} + +static void link_size(ErtsMonitor *lnk, void *vsz) +{ + *((Uint *) vsz) = erts_link_size(lnk); +} + /**********************************************************************/ /* Return information on ports */ /* Info: @@ -2897,7 +3213,7 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt, INIT_MONITOR_INFOS(mic); - erts_doforall_links(ERTS_P_LINKS(prt), &collect_one_link, &mic); + erts_link_tree_foreach(ERTS_P_LINKS(prt), collect_one_link, (void *) &mic); if (szp) *szp += mic.sz; @@ -2921,11 +3237,11 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt, else if (item == am_monitors) { MonitorInfoCollection mic; int i; - Eterm item; INIT_MONITOR_INFOS(mic); - erts_doforall_monitors(ERTS_P_MONITORS(prt), - &collect_one_origin_monitor, &mic); + erts_monitor_tree_foreach(ERTS_P_MONITORS(prt), + collect_one_origin_monitor, + (void *) &mic); if (szp) *szp += mic.sz; @@ -2934,11 +3250,10 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt, res = NIL; for (i = 0; i < mic.mi_i; i++) { Eterm t; - Eterm m_type; - item = STORE_NC(hpp, ohp, mic.mi[i].entity.term); - m_type = is_port(item) ? am_port : am_process; - t = TUPLE2(*hpp, m_type, item); + ASSERT(mic.mi[i].type == ERTS_MON_TYPE_PORT); + ASSERT(is_internal_pid(mic.mi[i].entity.term)); + t = TUPLE2(*hpp, am_process, mic.mi[i].entity.term); *hpp += 3; res = CONS(*hpp, t, res); *hpp += 2; @@ -2957,15 +3272,19 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt, Eterm item; INIT_MONITOR_INFOS(mic); - erts_doforall_monitors(ERTS_P_MONITORS(prt), - &collect_one_target_monitor, &mic); + erts_monitor_list_foreach(ERTS_P_LT_MONITORS(prt), + collect_one_target_monitor, + (void *) &mic); + erts_monitor_tree_foreach(ERTS_P_MONITORS(prt), + collect_one_target_monitor, + (void *) &mic); if (szp) *szp += mic.sz; if (hpp) { res = NIL; for (i = 0; i < mic.mi_i; ++i) { - ASSERT(mic.mi[i].node == NIL); + ASSERT(mic.mi[i].type != ERTS_MON_TYPE_RESOURCE); item = STORE_NC(hpp, ohp, mic.mi[i].entity.term); res = CONS(*hpp, item, res); *hpp += 2; @@ -3042,7 +3361,12 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt, */ Uint size = 0; - erts_doforall_links(ERTS_P_LINKS(prt), &erts_one_link_size, &size); + erts_link_tree_foreach(ERTS_P_LINKS(prt), + link_size, (void *) &size); + erts_monitor_tree_foreach(ERTS_P_MONITORS(prt), + monitor_size, (void *) &size); + erts_monitor_list_foreach(ERTS_P_LT_MONITORS(prt), + monitor_size, (void *) &size); size += erts_port_data_size(prt); @@ -3264,35 +3588,53 @@ fun_info_mfa_1(BIF_ALIST_1) BIF_ERROR(p, BADARG); } +BIF_RETTYPE erts_internal_is_process_alive_2(BIF_ALIST_2) +{ + if (!is_internal_pid(BIF_ARG_1) || !is_internal_ordinary_ref(BIF_ARG_2)) + BIF_ERROR(BIF_P, BADARG); + erts_proc_sig_send_is_alive_request(BIF_P, BIF_ARG_1, BIF_ARG_2); + BIF_RET(am_ok); +} + BIF_RETTYPE is_process_alive_1(BIF_ALIST_1) { - if(is_internal_pid(BIF_ARG_1)) { - Process *rp; - - if (BIF_ARG_1 == BIF_P->common.id) - BIF_RET(am_true); - - rp = erts_proc_lookup_raw(BIF_ARG_1); - if (!rp) { - BIF_RET(am_false); - } - else { - if (erts_atomic32_read_acqb(&rp->state) - & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) - ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_false); - else - BIF_RET(am_true); - } - } - else if(is_external_pid(BIF_ARG_1)) { - if(external_pid_dist_entry(BIF_ARG_1) == erts_this_dist_entry) + if (is_internal_pid(BIF_ARG_1)) { + erts_aint32_t state; + Process *rp; + + if (BIF_ARG_1 == BIF_P->common.id) + BIF_RET(am_true); + + rp = erts_proc_lookup_raw(BIF_ARG_1); + if (!rp) + BIF_RET(am_false); + + state = erts_atomic32_read_acqb(&rp->state); + if (state & (ERTS_PSFLG_EXITING + | ERTS_PSFLG_SIG_Q + | ERTS_PSFLG_SIG_IN_Q)) { + /* + * If in exiting state, trap out and send 'is alive' + * request and wait for it to complete termination. + * + * If process has signals enqueued, we need to + * send it an 'is alive' request via its signal + * queue in order to ensure that signal order is + * preserved (we may earlier have sent it an + * exit signal that has not been processed yet). + */ + BIF_TRAP1(is_process_alive_trap, BIF_P, BIF_ARG_1); + } + + BIF_RET(am_true); + } + + if (is_external_pid(BIF_ARG_1)) { + if (external_pid_dist_entry(BIF_ARG_1) == erts_this_dist_entry) BIF_RET(am_false); /* A pid from an old incarnation of this node */ - else - BIF_ERROR(BIF_P, BADARG); - } - else { - BIF_ERROR(BIF_P, BADARG); } + + BIF_ERROR(BIF_P, BADARG); } BIF_RETTYPE process_display_2(BIF_ALIST_2) @@ -3310,16 +3652,6 @@ BIF_RETTYPE process_display_2(BIF_ALIST_2) if (rp == ERTS_PROC_LOCK_BUSY) ERTS_BIF_YIELD2(bif_export[BIF_process_display_2], BIF_P, BIF_ARG_1, BIF_ARG_2); - if (rp != BIF_P && ERTS_PROC_PENDING_EXIT(rp)) { - Eterm args[2] = {BIF_ARG_1, BIF_ARG_2}; - erts_proc_unlock(rp, ERTS_PROC_LOCKS_ALL); - ERTS_BIF_AWAIT_X_APPLY_TRAP(BIF_P, - BIF_ARG_1, - am_erlang, - am_process_display, - args, - 2); - } erts_stack_dump(ERTS_PRINT_STDERR, NULL, rp); erts_proc_unlock(rp, (BIF_P == rp ? ERTS_PROC_LOCKS_ALL_MINOR @@ -3590,7 +3922,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) Eterm res = NIL; Uint *hp = HAlloc(BIF_P, 2*ERTS_PI_ARGS); for (i = ERTS_PI_ARGS-1; i >= 0; i--) { - res = CONS(hp, pi_args[i], res); + res = CONS(hp, pi_args[i].name, res); hp += 2; } BIF_RET(res); @@ -3680,6 +4012,15 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(am_false); #endif } + else if (ERTS_IS_ATOM_STR("lc_graph", BIF_ARG_1)) { +#ifdef ERTS_ENABLE_LOCK_CHECK + Eterm res = erts_lc_dump_graph(); + BIF_RET(res); +#else + BIF_RET(am_notsup); +#endif + } + } else if (is_tuple(BIF_ARG_1)) { Eterm* tp = tuple_val(BIF_ARG_1); @@ -3692,22 +4033,59 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(erts_process_status(NULL, tp[2])); } } + else if (ERTS_IS_ATOM_STR("connection_id", tp[1])) { + DistEntry *dep; + Eterm *hp, res; + Uint con_id, hsz = 0; + if (!is_atom(tp[2])) + BIF_ERROR(BIF_P, BADARG); + dep = erts_sysname_to_connected_dist_entry(tp[2]); + if (!dep) + BIF_ERROR(BIF_P, BADARG); + erts_de_rlock(dep); + con_id = (Uint) dep->connection_id; + erts_de_runlock(dep); + (void) erts_bld_uint(NULL, &hsz, con_id); + hp = hsz ? HAlloc(BIF_P, hsz) : NULL; + res = erts_bld_uint(&hp, NULL, con_id); + BIF_RET(res); + } else if (ERTS_IS_ATOM_STR("link_list", tp[1])) { /* Used by erl_link_SUITE (emulator) */ if(is_internal_pid(tp[2])) { + erts_aint32_t state; Eterm res; Process *p; + int sigs_done, local_only; p = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, tp[2], - ERTS_PROC_LOCK_LINK); + ERTS_PROC_LOCK_MAIN); if (!p) { ERTS_ASSERT_IS_NOT_EXITING(BIF_P); BIF_RET(am_undefined); } - res = make_link_list(BIF_P, ERTS_P_LINKS(p), NIL); - erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); + + local_only = 0; + do { + int reds = CONTEXT_REDS; + sigs_done = erts_proc_sig_handle_incoming(p, + &state, + &reds, + CONTEXT_REDS, + local_only); + local_only = !0; + } while (!sigs_done && !(state & ERTS_PSFLG_EXITING)); + + if (!(state & ERTS_PSFLG_EXITING)) + res = make_link_list(BIF_P, 1, ERTS_P_LINKS(p), NIL); + else if (BIF_P == p) + ERTS_BIF_EXITED(BIF_P); + else + res = am_undefined; + if (BIF_P != p) + erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN); BIF_RET(res); } else if(is_internal_port(tp[2])) { @@ -3718,19 +4096,20 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) ERTS_PORT_SFLGS_INVALID_LOOKUP); if(!p) BIF_RET(am_undefined); - res = make_link_list(BIF_P, ERTS_P_LINKS(p), NIL); + res = make_link_list(BIF_P, 1, ERTS_P_LINKS(p), NIL); erts_port_release(p); BIF_RET(res); } else if(is_node_name_atom(tp[2])) { DistEntry *dep = erts_find_dist_entry(tp[2]); if(dep) { - Eterm subres; - erts_de_links_lock(dep); - subres = make_link_list(BIF_P, dep->nlinks, NIL); - subres = make_link_list(BIF_P, dep->node_links, subres); - erts_de_links_unlock(dep); - BIF_RET(subres); + Eterm res = NIL; + if (dep->mld) { + erts_mtx_lock(&dep->mld->mtx); + res = make_link_list(BIF_P, 0, dep->mld->links, NIL); + erts_mtx_unlock(&dep->mld->mtx); + } + BIF_RET(res); } else { BIF_RET(am_undefined); } @@ -3739,27 +4118,54 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("monitor_list", tp[1])) { /* Used by erl_link_SUITE (emulator) */ if(is_internal_pid(tp[2])) { + erts_aint32_t state; Process *p; Eterm res; + int sigs_done, local_only; p = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, tp[2], - ERTS_PROC_LOCK_LINK); + ERTS_PROC_LOCK_MAIN); if (!p) { ERTS_ASSERT_IS_NOT_EXITING(BIF_P); BIF_RET(am_undefined); } - res = make_monitor_list(BIF_P, ERTS_P_MONITORS(p)); - erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); + + local_only = 0; + do { + int reds = CONTEXT_REDS; + sigs_done = erts_proc_sig_handle_incoming(p, + &state, + &reds, + CONTEXT_REDS, + local_only); + local_only = !0; + } while (!sigs_done && !(state & ERTS_PSFLG_EXITING)); + + if (!(state & ERTS_PSFLG_EXITING)) { + res = make_monitor_list(BIF_P, 1, ERTS_P_MONITORS(p), NIL); + res = make_monitor_list(BIF_P, 0, ERTS_P_LT_MONITORS(p), res); + } + else { + if (BIF_P == p) + ERTS_BIF_EXITED(BIF_P); + else + res = am_undefined; + } + if (BIF_P != p) + erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN); BIF_RET(res); } else if(is_node_name_atom(tp[2])) { DistEntry *dep = erts_find_dist_entry(tp[2]); if(dep) { - Eterm ml; - erts_de_links_lock(dep); - ml = make_monitor_list(BIF_P, dep->monitors); - erts_de_links_unlock(dep); + Eterm ml = NIL; + if (dep->mld) { + erts_mtx_lock(&dep->mld->mtx); + ml = make_monitor_list(BIF_P, 1, dep->mld->orig_name_monitors, NIL); + ml = make_monitor_list(BIF_P, 0, dep->mld->monitors, ml); + erts_mtx_unlock(&dep->mld->mtx); + } BIF_RET(ml); } else { BIF_RET(am_undefined); @@ -3777,18 +4183,6 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) } BIF_RET(res); } - else if (ERTS_IS_ATOM_STR("have_pending_exit", tp[1])) { - Process *rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, - tp[2], ERTS_PROC_LOCK_STATUS); - if (!rp) { - BIF_RET(am_undefined); - } - else { - Eterm res = ERTS_PROC_PENDING_EXIT(rp) ? am_true : am_false; - erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); - BIF_RET(res); - } - } else if (ERTS_IS_ATOM_STR("binary_info", tp[1])) { Eterm bin = tp[2]; if (is_binary(bin)) { @@ -4115,57 +4509,6 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) erts_set_gc_state(BIF_P, enable); BIF_RET(res); } - else if (ERTS_IS_ATOM_STR("send_fake_exit_signal", BIF_ARG_1)) { - /* Used by signal_SUITE (emulator) */ - - /* Testcases depend on the exit being received via - a pending exit when the receiver is the same as - the caller. */ - if (is_tuple(BIF_ARG_2)) { - Eterm* tp = tuple_val(BIF_ARG_2); - if (arityval(tp[0]) == 3 - && (is_pid(tp[1]) || is_port(tp[1])) - && is_internal_pid(tp[2])) { - int xres; - ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; - Process *rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, - tp[2], rp_locks); - if (!rp) { - DECL_AM(dead); - BIF_RET(AM_dead); - } - - if (BIF_P == rp) - rp_locks |= ERTS_PROC_LOCK_MAIN; - xres = erts_send_exit_signal(NULL, /* NULL in order to - force a pending exit - when we send to our - selves. */ - tp[1], - rp, - &rp_locks, - tp[3], - NIL, - NULL, - 0); - if (BIF_P == rp) - rp_locks &= ~ERTS_PROC_LOCK_MAIN; - erts_proc_unlock(rp, rp_locks); - if (xres > 1) { - DECL_AM(message); - BIF_RET(AM_message); - } - else if (xres == 0) { - DECL_AM(unaffected); - BIF_RET(AM_unaffected); - } - else { - DECL_AM(exit); - BIF_RET(AM_exit); - } - } - } - } else if (ERTS_IS_ATOM_STR("colliding_names", BIF_ARG_1)) { /* Used by ets_SUITE (stdlib) */ if (is_tuple(BIF_ARG_2)) { @@ -4351,6 +4694,55 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); } +static BIF_RETTYPE +gather_histograms_helper(Process * c_p, Eterm arg_tuple, + int gather(Process *, int, int, int, UWord, Eterm)) +{ + SWord hist_start, hist_width, sched_id; + int msg_count, alloc_num; + Eterm *args; + + /* This is an internal BIF, so the error checking is mostly left to erlang + * code. */ + + ASSERT(is_tuple_arity(arg_tuple, 5)); + args = tuple_val(arg_tuple); + + for (alloc_num = ERTS_ALC_A_MIN; alloc_num <= ERTS_ALC_A_MAX; alloc_num++) { + if(erts_is_atom_str(ERTS_ALC_A2AD(alloc_num), args[1], 0)) { + break; + } + } + + if (alloc_num > ERTS_ALC_A_MAX) { + BIF_ERROR(c_p, BADARG); + } + + sched_id = signed_val(args[2]); + hist_width = signed_val(args[3]); + hist_start = signed_val(args[4]); + + if (sched_id < 0 || sched_id > erts_no_schedulers) { + BIF_ERROR(c_p, BADARG); + } + + msg_count = gather(c_p, alloc_num, sched_id, hist_width, hist_start, args[5]); + + BIF_RET(make_small(msg_count)); +} + +BIF_RETTYPE erts_internal_gather_alloc_histograms_1(BIF_ALIST_1) +{ + return gather_histograms_helper(BIF_P, BIF_ARG_1, + erts_alcu_gather_alloc_histograms); +} + +BIF_RETTYPE erts_internal_gather_carrier_info_1(BIF_ALIST_1) +{ + return gather_histograms_helper(BIF_P, BIF_ARG_1, + erts_alcu_gather_carrier_info); +} + #ifdef ERTS_ENABLE_LOCK_COUNT typedef struct { @@ -4779,6 +5171,10 @@ erts_bif_info_init(void) = erts_export_put(am_erts_internal, am_gather_microstate_accounting_result, 2); gather_system_check_res_trap = erts_export_put(am_erts_internal, am_gather_system_check_result, 1); + + is_process_alive_trap = erts_export_put(am_erts_internal, am_is_process_alive, 1); + + process_info_init(); os_info_init(); } diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 1feb892b93..7fe4e02782 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2017. All Rights Reserved. + * Copyright Ericsson AB 2001-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,6 +43,7 @@ #include "erl_bits.h" #include "erl_bif_unique.h" #include "dtrace-wrapper.h" +#include "erl_proc_sig_queue.h" static Port *open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump); static int merge_global_environment(erts_osenv_t *env, Eterm key_value_pairs); @@ -53,10 +54,13 @@ char *erts_default_arg0 = "default"; BIF_RETTYPE erts_internal_open_port_2(BIF_ALIST_2) { + BIF_RETTYPE ret; Port *port; Eterm res; char *str; int err_type, err_num; + ErtsLinkData *ldp; + ErtsLink *lnk; port = open_port(BIF_P, BIF_ARG_1, BIF_ARG_2, &err_type, &err_num); if (!port) { @@ -78,6 +82,16 @@ BIF_RETTYPE erts_internal_open_port_2(BIF_ALIST_2) BIF_RET(res); } + ldp = erts_link_create(ERTS_LNK_TYPE_PORT, BIF_P->common.id, port->common.id); + ASSERT(ldp->a.other.item == port->common.id); + ASSERT(ldp->b.other.item == BIF_P->common.id); + /* + * This link should not already be present, but can potentially + * due to id wrapping... + */ + lnk = erts_link_tree_lookup_insert(&ERTS_P_LINKS(BIF_P), &ldp->a); + erts_link_tree_insert(&ERTS_P_LINKS(port), &ldp->b); + if (port->drv_ptr->flags & ERL_DRV_FLAG_USE_INIT_ACK) { /* Copied from erl_port_task.c */ @@ -86,39 +100,30 @@ BIF_RETTYPE erts_internal_open_port_2(BIF_ALIST_2) erts_make_ref_in_array(port->async_open_port->ref); port->async_open_port->to = BIF_P->common.id; - erts_proc_lock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE | ERTS_PROC_LOCK_LINK); - if (ERTS_PROC_PENDING_EXIT(BIF_P)) { - /* need to exit caller instead */ - erts_proc_unlock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE | ERTS_PROC_LOCK_LINK); - KILL_CATCHES(BIF_P); - BIF_P->freason = EXC_EXIT; - erts_port_release(port); - BIF_RET(am_badarg); - } - - ERTS_MSGQ_MV_INQ2PRIVQ(BIF_P); - BIF_P->msg.save = BIF_P->msg.last; - - erts_proc_unlock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE); + /* + * We unconditionaly *must* do a receive on a message + * containing the reference after this... + */ + ERTS_RECV_MARK_SAVE(BIF_P); + ERTS_RECV_MARK_SET(BIF_P); res = erts_proc_store_ref(BIF_P, port->async_open_port->ref); } else { res = port->common.id; - erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); } - erts_add_link(&ERTS_P_LINKS(port), LINK_PID, BIF_P->common.id); - erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, port->common.id); - if (IS_TRACED_FL(BIF_P, F_TRACE_PROCS)) - trace_proc(BIF_P, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, BIF_P, + trace_proc(BIF_P, ERTS_PROC_LOCK_MAIN, BIF_P, am_link, port->common.id); - erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + ERTS_BIF_PREP_RET(ret, res); erts_port_release(port); - BIF_RET(res); + if (lnk) + erts_link_release(lnk); + + return ret; } static ERTS_INLINE Port * @@ -195,7 +200,6 @@ BIF_RETTYPE erts_internal_port_command_3(BIF_ALIST_3) #endif switch (erts_port_output(BIF_P, flags, prt, prt->common.id, BIF_ARG_2, &ref)) { - case ERTS_PORT_OP_CALLER_EXIT: case ERTS_PORT_OP_BADARG: case ERTS_PORT_OP_DROPPED: ERTS_BIF_PREP_RET(res, am_badarg); @@ -254,7 +258,6 @@ BIF_RETTYPE erts_internal_port_call_3(BIF_ALIST_3) op = (unsigned int) uint_op; switch (erts_port_call(BIF_P, prt, op, BIF_ARG_3, &retval)) { - case ERTS_PORT_OP_CALLER_EXIT: case ERTS_PORT_OP_DROPPED: case ERTS_PORT_OP_BADARG: retval = am_badarg; @@ -272,11 +275,8 @@ BIF_RETTYPE erts_internal_port_call_3(BIF_ALIST_3) } state = erts_atomic32_read_acqb(&BIF_P->state); - if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) { - if (state & ERTS_PSFLG_PENDING_EXIT) - erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN); + if (state & ERTS_PSFLG_EXITING) ERTS_BIF_EXITED(BIF_P); - } BIF_RET(retval); } @@ -302,7 +302,6 @@ BIF_RETTYPE erts_internal_port_control_3(BIF_ALIST_3) op = (unsigned int) uint_op; switch (erts_port_control(BIF_P, prt, op, BIF_ARG_3, &retval)) { - case ERTS_PORT_OP_CALLER_EXIT: case ERTS_PORT_OP_BADARG: case ERTS_PORT_OP_DROPPED: retval = am_badarg; @@ -320,11 +319,8 @@ BIF_RETTYPE erts_internal_port_control_3(BIF_ALIST_3) } state = erts_atomic32_read_acqb(&BIF_P->state); - if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) { - if (state & ERTS_PSFLG_PENDING_EXIT) - erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN); + if (state & ERTS_PSFLG_EXITING) ERTS_BIF_EXITED(BIF_P); - } BIF_RET(retval); } @@ -347,7 +343,6 @@ BIF_RETTYPE erts_internal_port_close_1(BIF_ALIST_1) BIF_RET(am_badarg); switch (erts_port_exit(BIF_P, 0, prt, BIF_P->common.id, am_normal, &ref)) { - case ERTS_PORT_OP_CALLER_EXIT: case ERTS_PORT_OP_BADARG: case ERTS_PORT_OP_DROPPED: BIF_RET(am_badarg); @@ -380,7 +375,6 @@ BIF_RETTYPE erts_internal_port_connect_2(BIF_ALIST_2) #endif switch (erts_port_connect(BIF_P, 0, prt, BIF_P->common.id, BIF_ARG_2, &ref)) { - case ERTS_PORT_OP_CALLER_EXIT: case ERTS_PORT_OP_BADARG: case ERTS_PORT_OP_DROPPED: BIF_RET(am_badarg); @@ -418,7 +412,6 @@ BIF_RETTYPE erts_internal_port_info_1(BIF_ALIST_1) } switch (erts_port_info(BIF_P, prt, THE_NON_VALUE, &retval)) { - case ERTS_PORT_OP_CALLER_EXIT: case ERTS_PORT_OP_BADARG: BIF_RET(am_badarg); case ERTS_PORT_OP_DROPPED: @@ -457,7 +450,6 @@ BIF_RETTYPE erts_internal_port_info_2(BIF_ALIST_2) } switch (erts_port_info(BIF_P, prt, BIF_ARG_2, &retval)) { - case ERTS_PORT_OP_CALLER_EXIT: case ERTS_PORT_OP_BADARG: BIF_RET(am_badarg); case ERTS_PORT_OP_DROPPED: diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 22942b40c4..1953f79d79 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2017. All Rights Reserved. + * Copyright Ericsson AB 1999-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,6 +40,7 @@ #include "erl_binary.h" #include "erl_thr_progress.h" #include "erl_bif_unique.h" +#include "erl_proc_sig_queue.h" #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) @@ -1806,9 +1807,6 @@ Eterm erts_seq_trace(Process *p, Eterm arg1, Eterm arg2, return old_value; } else if (arg1 == am_label) { - if (! is_small(arg2)) { - return THE_NON_VALUE; - } new_seq_trace_token(p); if (build_result) { old_value = SEQ_TRACE_TOKEN_LABEL(p); diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h index b9d141d585..a3816fa820 100644 --- a/erts/emulator/beam/erl_bits.h +++ b/erts/emulator/beam/erl_bits.h @@ -111,7 +111,7 @@ typedef struct erl_bin_match_struct{ #define copy_binary_to_buffer(DstBuffer, DstBufOffset, SrcBuffer, SrcBufferOffset, NumBits) \ do { \ if (BIT_OFFSET(DstBufOffset) == 0 && (SrcBufferOffset == 0) && \ - (BIT_OFFSET(NumBits)==0)) { \ + (BIT_OFFSET(NumBits)==0) && (NumBits != 0)) { \ sys_memcpy(DstBuffer+BYTE_OFFSET(DstBufOffset), \ SrcBuffer, NBYTES(NumBits)); \ } else { \ diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index a76d769283..ca2ebb7c27 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -283,6 +283,13 @@ make_tid(Process *c_p, DbTable *tb) return erts_mk_magic_ref(&hp, &c_p->off_heap, tb->common.btid); } +Eterm +erts_db_make_tid(Process *c_p, DbTableCommon *tb) +{ + return make_tid(c_p, (DbTable*)tb); +} + + /* ** The meta hash table of all NAMED ets tables @@ -3540,7 +3547,7 @@ send_ets_transfer_message(Process *c_p, Process *proc, hd_copy = copy_struct(heir_data, hd_sz, &hp, ohp); sender = c_p->common.id; msg = TUPLE4(hp, am_ETS_TRANSFER, tid, sender, hd_copy); - erts_queue_message(proc, *locks, mp, msg, sender); + erts_queue_proc_message(c_p, proc, *locks, mp, msg); } diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index 318e90cb28..eb6da2c9fb 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -128,6 +128,7 @@ extern erts_atomic_t erts_ets_misc_mem_size; Eterm erts_ets_colliding_names(Process*, Eterm name, Uint cnt); Uint erts_db_get_max_tabs(void); +Eterm erts_db_make_tid(Process *c_p, DbTableCommon *tb); #ifdef ERTS_ENABLE_LOCK_COUNT void erts_lcnt_enable_db_lock_count(DbTable *tb, int enable); diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 5d49b2ea14..cb5c496e90 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -340,8 +340,8 @@ typedef int (*extra_match_validator_t)(int keypos, Eterm match, Eterm guard, Ete static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix); static void alloc_seg(DbTableHash *tb); static int free_seg(DbTableHash *tb, int free_records); -static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr, - HashDbTerm *list); +static HashDbTerm* next_live(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr, + HashDbTerm *list); static HashDbTerm* search_list(DbTableHash* tb, Eterm key, HashValue hval, HashDbTerm *list); static void shrink(DbTableHash* tb, int nitems); @@ -646,9 +646,9 @@ int db_create_hash(Process *p, DbTable *tbl) rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ; if (erts_ets_rwmtx_spin_count >= 0) rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count; - tb->locks = (DbTableHashFineLocks*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, /* Other type maybe? */ - (DbTable *) tb, - sizeof(DbTableHashFineLocks)); + tb->locks = (DbTableHashFineLocks*) erts_db_alloc(ERTS_ALC_T_DB_SEG, /* Other type maybe? */ + (DbTable *) tb, + sizeof(DbTableHashFineLocks)); for (i=0; i<DB_HASH_LOCK_CNT; ++i) { erts_rwmtx_init_opt(&tb->locks->lck_vec[i].lck, &rwmtx_opt, "db_hash_slot", tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB); @@ -672,19 +672,9 @@ static int db_first_hash(Process *p, DbTable *tbl, Eterm *ret) erts_rwmtx_t* lck = RLOCK_HASH(tb,ix); HashDbTerm* list; - for (;;) { - list = BUCKET(tb,ix); - if (list != NULL) { - if (list->hvalue == INVALID_HASH) { - list = next(tb,&ix,&lck,list); - } - break; - } - if ((ix=next_slot(tb,ix,&lck)) == 0) { - list = NULL; - break; - } - } + list = BUCKET(tb,ix); + list = next_live(tb, &ix, &lck, list); + if (list != NULL) { *ret = db_copy_key(p, tbl, &list->dbterm); RUNLOCK_HASH(lck); @@ -721,13 +711,13 @@ static int db_next_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) } /* Key found */ - b = next(tb, &ix, &lck, b); + b = next_live(tb, &ix, &lck, b->next); if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) { while (b != 0) { if (!has_live_key(tb, b, key, hval)) { break; } - b = next(tb, &ix, &lck, b); + b = next_live(tb, &ix, &lck, b->next); } } if (b == NULL) { @@ -1463,20 +1453,24 @@ static ERTS_INLINE int on_mtraversal_simple_trap(Export* trap_function, BUMP_ALL_REDS(p); if (IS_USMALL(0, got)) { - hp = HAlloc(p, base_halloc_sz + 5); + hp = HAllocX(p, base_halloc_sz + 5, ERTS_MAGIC_REF_THING_SIZE); egot = make_small(got); } else { - hp = HAlloc(p, base_halloc_sz + BIG_UINT_HEAP_SIZE + 5); + hp = HAllocX(p, base_halloc_sz + BIG_UINT_HEAP_SIZE + 5, + ERTS_MAGIC_REF_THING_SIZE); egot = uint_to_big(got, hp); hp += BIG_UINT_HEAP_SIZE; } if (is_first_trap) { + if (is_atom(tid)) + tid = erts_db_make_tid(p, &tb->common); mpb = erts_db_make_match_prog_ref(p, *mpp, &hp); *mpp = NULL; /* otherwise the caller will destroy it */ } else { + ASSERT(!is_atom(tid)); mpb = prev_continuation_tptr[3]; } @@ -1590,11 +1584,17 @@ static int mtraversal_select_chunk_on_loop_ended(void* context_ptr, Sint slot_ix been in 'user space' */ } if (rest != NIL || slot_ix >= 0) { /* Need more calls */ - sc_context_ptr->hp = HAlloc(sc_context_ptr->p, 3 + 7 + ERTS_MAGIC_REF_THING_SIZE); + Eterm tid = sc_context_ptr->tid; + sc_context_ptr->hp = HAllocX(sc_context_ptr->p, + 3 + 7 + ERTS_MAGIC_REF_THING_SIZE, + ERTS_MAGIC_REF_THING_SIZE); mpb = erts_db_make_match_prog_ref(sc_context_ptr->p, *mpp, &sc_context_ptr->hp); + if (is_atom(tid)) + tid = erts_db_make_tid(sc_context_ptr->p, + &sc_context_ptr->tb->common); continuation = TUPLE6( sc_context_ptr->hp, - sc_context_ptr->tid, + tid, make_small(slot_ix), make_small(sc_context_ptr->chunk_size), mpb, rest, @@ -1631,12 +1631,16 @@ static int mtraversal_select_chunk_on_trap(void* context_ptr, Sint slot_ix, Sint BUMP_ALL_REDS(sc_context_ptr->p); if (sc_context_ptr->prev_continuation_tptr == NULL) { + Eterm tid = sc_context_ptr->tid; /* First time we're trapping */ - hp = HAlloc(sc_context_ptr->p, 7 + ERTS_MAGIC_REF_THING_SIZE); + hp = HAllocX(sc_context_ptr->p, 7 + ERTS_MAGIC_REF_THING_SIZE, + ERTS_MAGIC_REF_THING_SIZE); + if (is_atom(tid)) + tid = erts_db_make_tid(sc_context_ptr->p, &sc_context_ptr->tb->common); mpb = erts_db_make_match_prog_ref(sc_context_ptr->p, *mpp, &hp); continuation = TUPLE6( hp, - sc_context_ptr->tid, + tid, make_small(slot_ix), make_small(sc_context_ptr->chunk_size), mpb, @@ -2905,14 +2909,14 @@ static HashDbTerm* search_list(DbTableHash* tb, Eterm key, /* It return the next live object in a table, NULL if no more */ /* In-bucket: RLOCKED */ /* Out-bucket: RLOCKED unless NULL */ -static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr, - HashDbTerm *list) +static HashDbTerm* next_live(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr, + HashDbTerm *list) { int i; ERTS_LC_ASSERT(IS_HASH_RLOCKED(tb,*iptr)); - for (list = list->next; list != NULL; list = list->next) { + for ( ; list != NULL; list = list->next) { if (list->hvalue != INVALID_HASH) return list; } diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 956662aefa..6354abfd1f 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2017. All Rights Reserved. + * Copyright Ericsson AB 1998-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,6 +38,7 @@ #include "erl_binary.h" #include "erl_map.h" #include "erl_thr_progress.h" +#include "erl_proc_sig_queue.h" #include "erl_db_util.h" @@ -74,12 +75,35 @@ DBIF_TABLE_GUARD | DBIF_TABLE_BODY | DBIF_TRACE_GUARD | DBIF_TRACE_BODY typedef struct DMC_STACK_TYPE(Type) { \ int pos; \ int siz; \ - Type def[DMC_DEFAULT_SIZE]; \ + int bytes; \ Type *data; \ + Type def[DMC_DEFAULT_SIZE]; \ } DMC_STACK_TYPE(Type) + + +typedef int Dummy; +DMC_DECLARE_STACK_TYPE(Dummy); + +static void dmc_stack_grow(DMC_Dummy_stack* s) +{ + int was_bytes = s->bytes; + s->siz *= 2; + s->bytes *= 2; + if (s->data == s->def) { + s->data = erts_alloc(ERTS_ALC_T_DB_MC_STK, s->bytes); + sys_memcpy(s->data, s->def, was_bytes); + } + else { + s->data = erts_realloc(ERTS_ALC_T_DB_MC_STK, s->data, s->bytes); + } +} -#define DMC_INIT_STACK(Name) \ - (Name).pos = 0; (Name).siz = DMC_DEFAULT_SIZE; (Name).data = (Name).def +#define DMC_INIT_STACK(Name) do { \ + (Name).pos = 0; \ + (Name).siz = DMC_DEFAULT_SIZE; \ + (Name).bytes = sizeof((Name).def); \ + (Name).data = (Name).def; \ +} while (0) #define DMC_STACK_DATA(Name) (Name).data @@ -87,21 +111,19 @@ typedef struct DMC_STACK_TYPE(Type) { \ #define DMC_PUSH(On, What) \ do { \ - if ((On).pos >= (On).siz) { \ - (On).siz *= 2; \ - (On).data \ - = (((On).def == (On).data) \ - ? sys_memcpy(erts_alloc(ERTS_ALC_T_DB_MC_STK, \ - (On).siz*sizeof(*((On).data))), \ - (On).def, \ - DMC_DEFAULT_SIZE*sizeof(*((On).data))) \ - : erts_realloc(ERTS_ALC_T_DB_MC_STK, \ - (void *) (On).data, \ - (On).siz*sizeof(*((On).data)))); \ - } \ + if ((On).pos >= (On).siz) \ + dmc_stack_grow((DMC_Dummy_stack*)&(On)); \ (On).data[(On).pos++] = What; \ } while (0) +#define DMC_PUSH2(On, A, B) \ +do { \ + if ((On).pos+1 >= (On).siz) \ + dmc_stack_grow((DMC_Dummy_stack*)&(On)); \ + (On).data[(On).pos++] = A; \ + (On).data[(On).pos++] = B; \ +} while (0) + #define DMC_POP(From) (From).data[--(From).pos] #define DMC_TOP(From) (From).data[(From).pos - 1] @@ -155,6 +177,7 @@ set_tracee_flags(Process *tracee_p, ErtsTracer tracer, : am_false); erts_tracer_replace(&tracee_p->common, tracer); ERTS_TRACE_FLAGS(tracee_p) = flags; + return ret; } /* @@ -615,6 +638,12 @@ static DMCGuardBif guard_tab[] = DBIF_ALL }, { + am_map_get, + &map_get_2, + 2, + DBIF_ALL + }, + { am_bit_size, &bit_size_1, 1, @@ -1537,8 +1566,7 @@ restart: if (is_flatmap(t)) { num_iters = flatmap_get_size(flatmap_val(t)); if (!structure_checked) { - DMC_PUSH(text, matchMap); - DMC_PUSH(text, num_iters); + DMC_PUSH2(text, matchMap, num_iters); } structure_checked = 0; for (i = 0; i < num_iters; ++i) { @@ -1558,8 +1586,7 @@ restart: } goto error; } - DMC_PUSH(text, matchKey); - DMC_PUSH(text, dmc_private_copy(&context, key)); + DMC_PUSH2(text, matchKey, dmc_private_copy(&context, key)); { int old_stack = ++(context.stack_used); Eterm value = flatmap_get_values(flatmap_val(t))[i]; @@ -1587,8 +1614,7 @@ restart: Eterm *kv; num_iters = hashmap_size(t); if (!structure_checked) { - DMC_PUSH(text, matchMap); - DMC_PUSH(text, num_iters); + DMC_PUSH2(text, matchMap, num_iters); } structure_checked = 0; @@ -1614,8 +1640,7 @@ restart: DESTROY_WSTACK(wstack); goto error; } - DMC_PUSH(text, matchKey); - DMC_PUSH(text, dmc_private_copy(&context, key)); + DMC_PUSH2(text, matchKey, dmc_private_copy(&context, key)); { int old_stack = ++(context.stack_used); res = dmc_one_term(&context, &heap, &stack, &text, @@ -1645,8 +1670,7 @@ restart: num_iters = arityval(*tuple_val(t)); if (!structure_checked) { /* i.e. we did not pop it */ - DMC_PUSH(text,matchTuple); - DMC_PUSH(text,num_iters); + DMC_PUSH2(text, matchTuple, num_iters); } structure_checked = 0; for (i = 1; i <= num_iters; ++i) { @@ -2490,25 +2514,20 @@ restart: if (have_no_seqtrace(SEQ_TRACE_TOKEN(c_p))) *esp++ = NIL; else { - Eterm sender = SEQ_TRACE_TOKEN_SENDER(c_p); - Uint sender_sz = is_immed(sender) ? 0 : size_object(sender); - ehp = HAllocX(build_proc, 6 + sender_sz, HEAP_XTRA); - if (sender_sz) { - sender = copy_struct(sender, sender_sz, &ehp, &MSO(build_proc)); - } - *esp++ = make_tuple(ehp); - ehp[0] = make_arityval(5); - ehp[1] = SEQ_TRACE_TOKEN_FLAGS(c_p); - ehp[2] = SEQ_TRACE_TOKEN_LABEL(c_p); - ehp[3] = SEQ_TRACE_TOKEN_SERIAL(c_p); - ehp[4] = sender; - ehp[5] = SEQ_TRACE_TOKEN_LASTCNT(c_p); - ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5); - ASSERT(is_immed(ehp[1])); - ASSERT(is_immed(ehp[2])); - ASSERT(is_immed(ehp[3])); - ASSERT(is_immed(ehp[5])); - } + Eterm token; + Uint token_sz; + + ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5); + ASSERT(is_immed(SEQ_TRACE_TOKEN_FLAGS(c_p))); + ASSERT(is_immed(SEQ_TRACE_TOKEN_SERIAL(c_p))); + ASSERT(is_immed(SEQ_TRACE_TOKEN_LASTCNT(c_p))); + + token = SEQ_TRACE_TOKEN(c_p); + token_sz = size_object(token); + + ehp = HAllocX(build_proc, token_sz, HEAP_XTRA); + *esp++ = copy_struct(token, token_sz, &ehp, &MSO(build_proc)); + } break; case matchEnableTrace: ASSERT(c_p == self); @@ -3535,20 +3554,17 @@ static DMCRet dmc_one_term(DMCContext *context, return retRestart; } if (heap->vars[n].is_bound) { - DMC_PUSH(*text,matchCmp); - DMC_PUSH(*text,n); + DMC_PUSH2(*text, matchCmp, n); } else { /* Not bound, bind! */ if (n >= heap->vars_used) heap->vars_used = n + 1; - DMC_PUSH(*text,matchBind); - DMC_PUSH(*text,n); + DMC_PUSH2(*text, matchBind, n); heap->vars[n].is_bound = 1; } } else if (c == am_Underscore) { DMC_PUSH(*text, matchSkip); } else { /* Any immediate value */ - DMC_PUSH(*text, matchEq); - DMC_PUSH(*text, (Uint) c); + DMC_PUSH2(*text, matchEq, (Uint) c); } break; case TAG_PRIMARY_LIST: @@ -3561,9 +3577,8 @@ static DMCRet dmc_one_term(DMCContext *context, switch ((hdr & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE) { case (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE): n = arityval(*tuple_val(c)); - DMC_PUSH(*text, matchPushT); + DMC_PUSH2(*text, matchPushT, n); ++(context->stack_used); - DMC_PUSH(*text, n); DMC_PUSH(*stack, c); break; case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE): @@ -3571,9 +3586,8 @@ static DMCRet dmc_one_term(DMCContext *context, n = flatmap_get_size(flatmap_val(c)); else n = hashmap_size(c); - DMC_PUSH(*text, matchPushM); + DMC_PUSH2(*text, matchPushM, n); ++(context->stack_used); - DMC_PUSH(*text, n); DMC_PUSH(*stack, c); break; case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE): @@ -3598,8 +3612,7 @@ static DMCRet dmc_one_term(DMCContext *context, break; } case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): - DMC_PUSH(*text,matchEqFloat); - DMC_PUSH(*text, (Uint) float_val(c)[1]); + DMC_PUSH2(*text, matchEqFloat, (Uint) float_val(c)[1]); #ifdef ARCH_64 DMC_PUSH(*text, (Uint) 0); #else @@ -3607,8 +3620,7 @@ static DMCRet dmc_one_term(DMCContext *context, #endif break; default: /* BINARY, FUN, VECTOR, or EXTERNAL */ - DMC_PUSH(*text, matchEqBin); - DMC_PUSH(*text, dmc_private_copy(context, c)); + DMC_PUSH2(*text, matchEqBin, dmc_private_copy(context, c)); break; } break; @@ -3658,8 +3670,7 @@ static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(UWord) *text, emb->next = context->save; context->save = emb; } - DMC_PUSH(*text,matchPushC); - DMC_PUSH(*text,(Uint) tmp); + DMC_PUSH2(*text, matchPushC, (Uint)tmp); if (++context->stack_used > context->stack_need) context->stack_need = context->stack_used; } @@ -3797,8 +3808,7 @@ dmc_tuple(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, *constant = 1; return retOk; } - DMC_PUSH(*text, matchMkTuple); - DMC_PUSH(*text, nelems); + DMC_PUSH2(*text, matchMkTuple, nelems); context->stack_used -= (nelems - 1); *constant = 0; return retOk; @@ -3825,13 +3835,11 @@ dmc_map(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, *constant = 1; return retOk; } - DMC_PUSH(*text, matchPushC); - DMC_PUSH(*text, dmc_private_copy(context, m->keys)); + DMC_PUSH2(*text, matchPushC, dmc_private_copy(context, m->keys)); if (++context->stack_used > context->stack_need) { context->stack_need = context->stack_used; } - DMC_PUSH(*text, matchMkFlatMap); - DMC_PUSH(*text, nelems); + DMC_PUSH2(*text, matchMkFlatMap, nelems); context->stack_used -= nelems; *constant = 0; return retOk; @@ -3883,8 +3891,7 @@ dmc_map(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, do_emit_constant(context, text, CDR(kv)); } } - DMC_PUSH(*text, matchMkHashMap); - DMC_PUSH(*text, nelems); + DMC_PUSH2(*text, matchMkHashMap, nelems); context->stack_used -= nelems; DESTROY_WSTACK(wstack); return retOk; @@ -5736,5 +5743,3 @@ void db_match_dis(Binary *bp) } #endif /* DMC_DEBUG */ - - diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c index bf8244564a..db78378257 100644 --- a/erts/emulator/beam/erl_debug.c +++ b/erts/emulator/beam/erl_debug.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2017. All Rights Reserved. + * Copyright Ericsson AB 1998-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -404,15 +404,16 @@ void verify_process(Process *p) erts_exit(ERTS_ERROR_EXIT,"Wild pointer found in " name " of %T!\n",p->common.id); } - ErtsMessage* mp = p->msg.first; - VERBOSE(DEBUG_MEMORY,("Verify process: %T...\n",p->common.id)); - while (mp != NULL) { - VERIFY_ETERM("message term",ERL_MESSAGE_TERM(mp)); - VERIFY_ETERM("message token",ERL_MESSAGE_TOKEN(mp)); - mp = mp->next; - } + ERTS_FOREACH_SIG_PRIVQS( + p, mp, + { + if (ERTS_SIG_IS_MSG(mp)) { + VERIFY_ETERM("message term",ERL_MESSAGE_TERM(mp)); + VERIFY_ETERM("message token",ERL_MESSAGE_TOKEN(mp)); + } + }); erts_check_stack(p); erts_check_heap(p); @@ -532,16 +533,15 @@ static void print_process_memory(Process *p) erts_printf("-- %-*s ---%s-%s-%s-%s--\n", PTR_SIZE, "PCB", dashes, dashes, dashes, dashes); - if (p->msg.first != NULL) { - ErtsMessage* mp; - erts_printf(" Message Queue:\n"); - mp = p->msg.first; - while (mp != NULL) { - erts_printf("| 0x%0*lx | 0x%0*lx |\n",PTR_SIZE, - ERL_MESSAGE_TERM(mp),PTR_SIZE,ERL_MESSAGE_TOKEN(mp)); - mp = mp->next; - } - } + + erts_printf(" Message Queue:\n"); + ERTS_FOREACH_SIG_PRIVQS( + p, mp, + { + if (ERTS_SIG_IS_MSG(mp)) + erts_printf("| 0x%0*lx | 0x%0*lx |\n",PTR_SIZE, + ERL_MESSAGE_TERM(mp),PTR_SIZE,ERL_MESSAGE_TOKEN(mp)); + }); if (p->dictionary != NULL) { int n = ERTS_PD_SIZE(p->dictionary); diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 1c64644efc..0692cea0ee 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2017. All Rights Reserved. + * Copyright Ericsson AB 2002-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,6 +43,7 @@ #include "erl_bif_unique.h" #include "dist.h" #include "erl_nfunc_sched.h" +#include "erl_proc_sig_queue.h" #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20 @@ -154,7 +155,6 @@ static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size, Eterm* objv, int nobj); static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size); static void offset_mqueue(Process *p, Sint offs, char* area, Uint area_size); -static void move_msgq_to_heap(Process *p); static int reached_max_heap_size(Process *p, Uint total_heap_size, Uint extra_heap_size, Uint extra_old_heap_size); static void init_gc_info(ErtsGCInfo *gcip); @@ -189,6 +189,21 @@ static struct { ErtsGCInfo info; } dirty_gc; +static void move_msgs_to_heap(Process *c_p) +{ + Uint64 pre_oh, post_oh; + + pre_oh = c_p->off_heap.overhead; + erts_proc_sig_move_msgs_to_heap(c_p); + post_oh = c_p->off_heap.overhead; + + if (pre_oh != post_oh) { + /* Got new binaries; update bin vheap size... */ + c_p->bin_vheap_sz = next_vheap_size(c_p, post_oh, + c_p->bin_vheap_sz); + } +} + static ERTS_INLINE int gc_cost(Uint gc_moved_live_words, Uint resize_moved_words) { @@ -207,6 +222,7 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(gcireq, ErtsGCInfoReq, 5, ERTS_ALC_T_GC_INFO_REQ) + /* * Initialize GC global data. */ @@ -405,6 +421,13 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end, return result; } +#ifdef HIPE + if (p->hipe_smp.have_receive_locks) { + /* Do not want to GC with message queue locked... */ + return result; + } +#endif + if (!p->mbuf) { /* Must have GC:d in BIF call... invalidate live_hf_end */ live_hf_end = ERTS_INVALID_HFRAG_PTR; @@ -565,20 +588,26 @@ young_gen_usage(Process *p) hsz = p->mbuf_sz; if (p->flags & F_ON_HEAP_MSGQ) { - ErtsMessage *mp; - for (mp = p->msg.first; mp; mp = mp->next) { - /* - * We leave not yet decoded distribution messages - * as they are in the queue since it is not - * possible to determine a maximum size until - * actual decoding. However, we use their estimated - * size when calculating need, and by this making - * it more likely that they will fit on the heap - * when actually decoded. - */ - if (mp->data.attached) - hsz += erts_msg_attached_data_size(mp); - } + ERTS_FOREACH_SIG_PRIVQS( + p, mp, + { + /* + * We leave not yet decoded distribution messages + * as they are in the queue since it is not + * possible to determine a maximum size until + * actual decoding. However, we use their estimated + * size when calculating need, and by this making + * it more likely that they will fit on the heap + * when actually decoded. + * + * We however ignore off heap messages... + */ + if (ERTS_SIG_IS_MSG(mp) + && mp->data.attached + && mp->data.attached != ERTS_MSG_COMBINED_HFRAG) { + hsz += erts_msg_attached_data_size(mp); + } + }); } hsz += p->htop - p->heap; @@ -621,7 +650,7 @@ check_for_possibly_long_gc(Process *p, Uint ygen_usage) sz = ygen_usage; sz += p->hend - p->stop; if (p->flags & F_ON_HEAP_MSGQ) - sz += p->msg.len; + sz += erts_proc_sig_privqs_len(p); if (major) sz += p->old_htop - p->old_heap; @@ -761,13 +790,9 @@ do_major_collection: long_gc/large_gc triggers when this happens as process was killed before a GC could be done. */ if (reds == -2) { - ErtsProcLocks locks = ERTS_PROC_LOCKS_ALL; int res; - erts_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); - erts_send_exit_signal(p, p->common.id, p, &locks, - am_kill, NIL, NULL, 0); - erts_proc_unlock(p, locks & ERTS_PROC_LOCKS_ALL_MINOR); + erts_set_self_exiting(p, am_killed); delay_gc_after_start: /* erts_send_exit_signal looks for ERTS_PSFLG_GC, so @@ -1375,7 +1400,7 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end, new_sz, objv, nobj); if (p->flags & F_ON_HEAP_MSGQ) - move_msgq_to_heap(p); + move_msgs_to_heap(p); new_mature = p->old_htop - prev_old_htop; @@ -1760,7 +1785,7 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, HIGH_WATER(p) = HEAP_TOP(p); if (p->flags & F_ON_HEAP_MSGQ) - move_msgq_to_heap(p); + move_msgs_to_heap(p); ErtsGcQuickSanityCheck(p); @@ -2332,9 +2357,9 @@ collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, Eterm* n_htop) return n_htop; } -static ERTS_INLINE void -copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap, - ErlHeapFragment *bp, Eterm *refs, int nrefs) +void +erts_copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap, + ErlHeapFragment *bp, Eterm *refs, int nrefs) { Uint sz; int i; @@ -2440,61 +2465,6 @@ copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap, bp->off_heap.first = NULL; } -static void -move_msgq_to_heap(Process *p) -{ - - ErtsMessage **mpp = &p->msg.first; - Uint64 pre_oh = MSO(p).overhead; - - while (*mpp) { - ErtsMessage *mp = *mpp; - - if (mp->data.attached) { - ErlHeapFragment *bp; - - /* - * We leave not yet decoded distribution messages - * as they are in the queue since it is not - * possible to determine a maximum size until - * actual decoding... - */ - if (is_value(ERL_MESSAGE_TERM(mp))) { - - bp = erts_message_to_heap_frag(mp); - - if (bp->next) - erts_move_multi_frags(&p->htop, &p->off_heap, bp, - mp->m, ERL_MESSAGE_REF_ARRAY_SZ, 0); - else - copy_one_frag(&p->htop, &p->off_heap, bp, - mp->m, ERL_MESSAGE_REF_ARRAY_SZ); - - if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) { - mp->data.heap_frag = NULL; - free_message_buffer(bp); - } - else { - ErtsMessage *new_mp = erts_alloc_message(0, NULL); - sys_memcpy((void *) new_mp->m, (void *) mp->m, - sizeof(Eterm)*ERL_MESSAGE_REF_ARRAY_SZ); - erts_msgq_replace_msg_ref(&p->msg, new_mp, mpp); - mp->next = NULL; - erts_cleanup_messages(mp); - mp = new_mp; - } - } - } - - mpp = &(*mpp)->next; - } - - if (pre_oh != MSO(p).overhead) { - /* Got new binaries; update vheap size... */ - BIN_VHEAP_SZ(p) = next_vheap_size(p, MSO(p).overhead, BIN_VHEAP_SZ(p)); - } -} - static Uint setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) { @@ -2579,15 +2549,17 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) break; case F_OFF_HEAP_MSGQ_CHNG: case 0: { + Sint len; /* * We do not have off heap message queue enabled, i.e. we - * need to add message queue to rootset... + * need to add signal queues to rootset... */ - ErtsMessage *mp; + + len = erts_proc_sig_privqs_len(p); /* Ensure large enough rootset... */ - if (n + p->msg.len > rootset->size) { - Uint new_size = n + p->msg.len; + if (n + len > rootset->size) { + Uint new_size = n + len; ERTS_GC_ASSERT(roots == rootset->def); roots = erts_alloc(ERTS_ALC_T_ROOTSET, new_size*sizeof(Roots)); @@ -2595,18 +2567,19 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) rootset->size = new_size; } - for (mp = p->msg.first; mp; mp = mp->next) { - - if (!mp->data.attached) { - /* - * Message may refer data on heap; - * add it to rootset... - */ - roots[n].v = mp->m; - roots[n].sz = ERL_MESSAGE_REF_ARRAY_SZ; - n++; - } - } + ERTS_FOREACH_SIG_PRIVQS( + p, mp, + { + if (ERTS_SIG_IS_INTERNAL_MSG(mp) && !mp->data.attached) { + /* + * Message may refer data on heap; + * add it to rootset... + */ + roots[n].v = mp->m; + roots[n].sz = ERL_MESSAGE_REF_ARRAY_SZ; + n++; + } + }); break; } } @@ -3117,51 +3090,59 @@ offset_off_heap(Process* p, Sint offs, char* area, Uint area_size) } } +#ifndef USE_VM_PROBES +#define ERTS_OFFSET_DT_UTAG(MP, A, ASZ, O) +#else +#define ERTS_OFFSET_DT_UTAG(MP, A, ASZ, O) \ + do { \ + Eterm utag__ = ERL_MESSAGE_DT_UTAG((MP)); \ + if (is_boxed(utag__) && ErtsInArea(ptr_val(utag__), (A), (ASZ))) { \ + ERL_MESSAGE_DT_UTAG((MP)) = offset_ptr(utag__, (O)); \ + } \ + } while (0) +#endif + +static ERTS_INLINE void +offset_message(ErtsMessage *mp, Sint offs, char* area, Uint area_size) +{ + Eterm mesg = ERL_MESSAGE_TERM(mp); + if (ERTS_SIG_IS_MSG_TAG(mesg)) { + if (ERTS_SIG_IS_INTERNAL_MSG_TAG(mesg)) { + switch (primary_tag(mesg)) { + case TAG_PRIMARY_LIST: + case TAG_PRIMARY_BOXED: + if (ErtsInArea(ptr_val(mesg), area, area_size)) { + ERL_MESSAGE_TERM(mp) = offset_ptr(mesg, offs); + } + break; + } + } + mesg = ERL_MESSAGE_TOKEN(mp); + if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) { + ERL_MESSAGE_TOKEN(mp) = offset_ptr(mesg, offs); + } + + ERTS_OFFSET_DT_UTAG(mp, area, area_size, offs); + + ASSERT((is_nil(ERL_MESSAGE_TOKEN(mp)) || + is_tuple(ERL_MESSAGE_TOKEN(mp)) || + is_atom(ERL_MESSAGE_TOKEN(mp)))); + } +} + /* * Offset pointers in message queue. */ static void offset_mqueue(Process *p, Sint offs, char* area, Uint area_size) { - ErtsMessage* mp = p->msg.first; - - if ((p->flags & (F_OFF_HEAP_MSGQ|F_OFF_HEAP_MSGQ_CHNG)) != F_OFF_HEAP_MSGQ) { - - while (mp != NULL) { - Eterm mesg = ERL_MESSAGE_TERM(mp); - if (is_value(mesg)) { - switch (primary_tag(mesg)) { - case TAG_PRIMARY_LIST: - case TAG_PRIMARY_BOXED: - if (ErtsInArea(ptr_val(mesg), area, area_size)) { - ERL_MESSAGE_TERM(mp) = offset_ptr(mesg, offs); - } - break; - } - } - mesg = ERL_MESSAGE_TOKEN(mp); - if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) { - ERL_MESSAGE_TOKEN(mp) = offset_ptr(mesg, offs); - } -#ifdef USE_VM_PROBES - mesg = ERL_MESSAGE_DT_UTAG(mp); - if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) { - ERL_MESSAGE_DT_UTAG(mp) = offset_ptr(mesg, offs); - } -#endif - - ASSERT((is_nil(ERL_MESSAGE_TOKEN(mp)) || - is_tuple(ERL_MESSAGE_TOKEN(mp)) || - is_atom(ERL_MESSAGE_TOKEN(mp)))); - mp = mp->next; - } - - } + if ((p->flags & (F_OFF_HEAP_MSGQ|F_OFF_HEAP_MSGQ_CHNG)) != F_OFF_HEAP_MSGQ) + ERTS_FOREACH_SIG_PRIVQS(p, mp, offset_message(mp, offs, area, area_size)); } static void ERTS_INLINE offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size, - Eterm* objv, int nobj) + Eterm* objv, int nobj) { Eterm *v; Uint sz; @@ -3378,7 +3359,6 @@ erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp, }; Eterm res = THE_NON_VALUE; - ErtsMessage *mp; ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(*tags)); ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == ERTS_PROCESS_GC_INFO_MAX_TERMS); @@ -3397,9 +3377,15 @@ erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp, be the same as adding old_heap_block_size + heap_block_size + mbuf_size. */ - for (mp = p->msg.first; mp; mp = mp->next) - if (mp->data.attached) - values[2] += erts_msg_attached_data_size(mp); + ERTS_FOREACH_SIG_PRIVQS( + p, mp, + { + if (ERTS_SIG_IS_MSG(mp) + && mp->data.attached + && mp->data.attached != ERTS_MSG_COMBINED_HFRAG) { + values[2] += erts_msg_attached_data_size(mp); + } + }); } res = erts_bld_atom_uword_2tup_list(hpp, @@ -3473,7 +3459,7 @@ erts_max_heap_size_map(Sint max_heap_size, Uint max_heap_flags, Eterm **hpp, Uint *sz) { if (!hpp) { - *sz += (2*3 + 1 + MAP_HEADER_FLATMAP_SZ); + *sz += ERTS_MAX_HEAP_SIZE_MAP_SZ; return THE_NON_VALUE; } else { Eterm *hp = *hpp; diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index dec0ab1143..b9b1ed728c 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2007-2017. All Rights Reserved. + * Copyright Ericsson AB 2007-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -154,6 +154,8 @@ typedef struct { Uint64 garbage_cols; } ErtsGCInfo; +#define ERTS_MAX_HEAP_SIZE_MAP_SZ (2*3 + 1 + MAP_HEADER_FLATMAP_SZ) + #define ERTS_PROCESS_GC_INFO_MAX_TERMS (11) /* number of elements in process_gc_info*/ #define ERTS_PROCESS_GC_INFO_MAX_SIZE \ (ERTS_PROCESS_GC_INFO_MAX_TERMS * (2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE)) @@ -181,6 +183,8 @@ void erts_free_heap_frags(struct process* p); Eterm erts_max_heap_size_map(Sint, Uint, Eterm **, Uint *); int erts_max_heap_size(Eterm, Uint *, Uint *); void erts_deallocate_young_generation(Process *c_p); +void erts_copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap, + ErlHeapFragment *bp, Eterm *refs, int nrefs); #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG) int erts_dbg_within_proc(Eterm *ptr, Process *p, Eterm* real_htop); #endif diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index bda2c9b94d..6ec6f8065e 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2015-2017. All Rights Reserved. + * Copyright Ericsson AB 2015-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,6 +38,7 @@ #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" #include "erl_hl_timer.h" +#include "erl_proc_sig_queue.h" #ifdef ERTS_MAGIC_REF_BIF_TIMERS #include "erl_binary.h" #endif @@ -2470,28 +2471,21 @@ access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info) req->rrefn[1] = rrefn[1]; req->rrefn[2] = rrefn[2]; - erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - - if (ERTS_PROC_PENDING_EXIT(c_p)) - ERTS_VBUMP_ALL_REDS(c_p); - else { - /* - * Caller needs to wait for a message containing - * the ref that we just created. No such message - * can exist in callers message queue at this time. - * We therefore move the save pointer of the - * callers message queue to the end of the queue. - * - * NOTE: It is of vital importance that the caller - * immediately do a receive unconditionaly - * waiting for the message with the reference; - * otherwise, next receive will *not* work - * as expected! - */ - ERTS_MSGQ_MV_INQ2PRIVQ(c_p); - c_p->msg.save = c_p->msg.last; - } - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + /* + * Caller needs to wait for a message containing + * the ref that we just created. No such message + * can exist in callers message queue at this time. + * We therefore move the save pointer of the + * callers message queue to the end of the queue. + * + * NOTE: It is of vital importance that the caller + * immediately do a receive unconditionaly + * waiting for the message with the reference; + * otherwise, next receive will *not* work + * as expected! + */ + ERTS_RECV_MARK_SAVE(c_p); + ERTS_RECV_MARK_SET(c_p); ERTS_BIF_PREP_TRAP1(ret, erts_await_result, c_p, rref); } diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 8430a5559b..57c6c10c7f 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2017. All Rights Reserved. + * Copyright Ericsson AB 1997-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,7 +37,7 @@ #include "erl_mseg.h" #include "erl_threads.h" #include "erl_hl_timer.h" -#include "erl_instrument.h" +#include "erl_mtrace.h" #include "erl_printf_term.h" #include "erl_misc_utils.h" #include "packet_parser.h" @@ -51,6 +51,8 @@ #include "erl_time.h" #include "erl_check_io.h" #include "erl_osenv.h" +#include "erl_proc_sig_queue.h" + #ifdef HIPE #include "hipe_mode_switch.h" /* for hipe_mode_switch_init() */ #include "hipe_signal.h" /* for hipe_signal_init() */ @@ -60,7 +62,7 @@ # include <sys/resource.h> #endif -#define ERTS_DEFAULT_NO_ASYNC_THREADS 10 +#define ERTS_DEFAULT_NO_ASYNC_THREADS 1 #define ERTS_DEFAULT_SCHED_STACK_SIZE 128 #define ERTS_DEFAULT_DCPU_SCHED_STACK_SIZE 40 @@ -126,6 +128,8 @@ const Eterm etp_hole_marker = 0; static int modified_sched_thread_suggested_stack_size = 0; +Eterm erts_init_process_id; + /* * Note about VxWorks: All variables must be initialized by executable code, * not by an initializer. Otherwise a new instance of the emulator will @@ -304,8 +308,9 @@ erl_init(int ncpu, int node_tab_delete_delay, ErtsDbSpinCount db_spin_count) { + erts_monitor_link_init(); + erts_proc_sig_queue_init(); erts_bif_unique_init(); - erts_init_monitors(); erts_init_time(time_correction, time_warp_mode); erts_init_sys_common_misc(); erts_init_process(ncpu, proc_tab_sz, legacy_proc_tab); @@ -413,7 +418,7 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** } static Eterm -erl_system_process_otp(Eterm parent_pid, char* modname, int off_heap_msgq) +erl_system_process_otp(Eterm parent_pid, char* modname, int off_heap_msgq, int prio) { Eterm start_mod; Process* parent; @@ -428,9 +433,16 @@ erl_system_process_otp(Eterm parent_pid, char* modname, int off_heap_msgq) parent = erts_pid2proc(NULL, 0, parent_pid, ERTS_PROC_LOCK_MAIN); - so.flags = erts_default_spo_flags|SPO_SYSTEM_PROC; + so.flags = erts_default_spo_flags|SPO_SYSTEM_PROC|SPO_USE_ARGS; if (off_heap_msgq) so.flags |= SPO_OFF_HEAP_MSGQ; + so.min_heap_size = H_MIN_SIZE; + so.min_vheap_size = BIN_VH_MIN_SIZE; + so.max_heap_size = H_MAX_SIZE; + so.max_heap_flags = H_MAX_FLAGS; + so.priority = prio; + so.max_gen_gcs = (Uint16) erts_atomic32_read_nob(&erts_max_gen_gcs); + so.scheduler = 0; res = erl_create_process(parent, start_mod, am_start, NIL, &so); erts_proc_unlock(parent, ERTS_PROC_LOCK_MAIN); return res; @@ -593,6 +605,10 @@ void erts_usage(void) erts_fprintf(stderr, "-stbt type u|ns|ts|ps|s|nnts|nnps|tnnps|db\n"); erts_fprintf(stderr, "-sbwt val set scheduler busy wait threshold, valid values are:\n"); erts_fprintf(stderr, " none|very_short|short|medium|long|very_long.\n"); + erts_fprintf(stderr, "-sbwtdcpu val set dirty CPU scheduler busy wait threshold, valid values are:\n"); + erts_fprintf(stderr, " none|very_short|short|medium|long|very_long.\n"); + erts_fprintf(stderr, "-sbwtdio val set dirty IO scheduler busy wait threshold, valid values are:\n"); + erts_fprintf(stderr, " none|very_short|short|medium|long|very_long.\n"); erts_fprintf(stderr, "-scl bool enable/disable compaction of scheduler load,\n"); erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); erts_fprintf(stderr, "-sct cput set cpu topology,\n"); @@ -611,6 +627,10 @@ void erts_usage(void) erts_fprintf(stderr, " very_lazy|lazy|medium|eager|very_eager.\n"); erts_fprintf(stderr, "-swt val set scheduler wakeup threshold, valid values are:\n"); erts_fprintf(stderr, " very_low|low|medium|high|very_high.\n"); + erts_fprintf(stderr, "-swtdcpu val set dirty CPU scheduler wakeup threshold, valid values are:\n"); + erts_fprintf(stderr, " very_low|low|medium|high|very_high.\n"); + erts_fprintf(stderr, "-swtdio val set dirty IO scheduler wakeup threshold, valid values are:\n"); + erts_fprintf(stderr, " very_low|low|medium|high|very_high.\n"); erts_fprintf(stderr, "-sss size suggested stack size in kilo words for scheduler threads,\n"); erts_fprintf(stderr, " valid range is [%d-%d] (default %d)\n", ERTS_SCHED_THREAD_MIN_STACK_SIZE, @@ -1200,7 +1220,6 @@ erl_start(int argc, char **argv) ErtsTimeWarpMode time_warp_mode; int node_tab_delete_delay = ERTS_NODE_TAB_DELAY_GC_DEFAULT; ErtsDbSpinCount db_spin_count = ERTS_DB_SPNCNT_NORMAL; - Eterm otp_ring0_pid; set_default_time_adj(&time_correction, &time_warp_mode); @@ -1676,15 +1695,41 @@ erl_start(int argc, char **argv) erts_usage(); } } + else if (has_prefix("bwtdcpu", sub_param)) { + arg = get_arg(sub_param + 7, argv[i+1], &i); + + if (erts_sched_set_busy_wait_threshold(ERTS_SCHED_DIRTY_CPU, arg) != 0) { + erts_fprintf(stderr, "bad dirty CPU scheduler busy wait threshold: %s\n", + arg); + erts_usage(); + } + + VERBOSE(DEBUG_SYSTEM, + ("dirty CPU scheduler wakeup threshold: %s\n", arg)); + } + else if (has_prefix("bwtdio", sub_param)) { + arg = get_arg(sub_param + 6, argv[i+1], &i); + + if (erts_sched_set_busy_wait_threshold(ERTS_SCHED_DIRTY_IO, arg) != 0) { + erts_fprintf(stderr, "bad dirty IO scheduler busy wait threshold: %s\n", + arg); + erts_usage(); + } + + VERBOSE(DEBUG_SYSTEM, + ("dirty IO scheduler wakeup threshold: %s\n", arg)); + } else if (has_prefix("bwt", sub_param)) { - arg = get_arg(sub_param+3, argv[i+1], &i); - if (erts_sched_set_busy_wait_threshold(arg) != 0) { + arg = get_arg(sub_param + 3, argv[i+1], &i); + + if (erts_sched_set_busy_wait_threshold(ERTS_SCHED_NORMAL, arg) != 0) { erts_fprintf(stderr, "bad scheduler busy wait threshold: %s\n", arg); erts_usage(); } + VERBOSE(DEBUG_SYSTEM, - ("scheduler wakup threshold: %s\n", arg)); + ("scheduler wakeup threshold: %s\n", arg)); } else if (has_prefix("cl", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); @@ -1801,9 +1846,29 @@ erl_start(int argc, char **argv) VERBOSE(DEBUG_SYSTEM, ("scheduler wake cleanup threshold: %s\n", arg)); } + else if (has_prefix("wtdcpu", sub_param)) { + arg = get_arg(sub_param+6, argv[i+1], &i); + if (erts_sched_set_wakeup_other_threshold(ERTS_SCHED_DIRTY_CPU, arg) != 0) { + erts_fprintf(stderr, "dirty CPU scheduler wakeup threshold: %s\n", + arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, + ("dirty CPU scheduler wakeup threshold: %s\n", arg)); + } + else if (has_prefix("wtdio", sub_param)) { + arg = get_arg(sub_param+5, argv[i+1], &i); + if (erts_sched_set_wakeup_other_threshold(ERTS_SCHED_DIRTY_IO, arg) != 0) { + erts_fprintf(stderr, "dirty IO scheduler wakeup threshold: %s\n", + arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, + ("dirty IO scheduler wakeup threshold: %s\n", arg)); + } else if (has_prefix("wt", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); - if (erts_sched_set_wakeup_other_thresold(arg) != 0) { + if (erts_sched_set_wakeup_other_threshold(ERTS_SCHED_NORMAL, arg) != 0) { erts_fprintf(stderr, "scheduler wakeup threshold: %s\n", arg); erts_usage(); @@ -1813,7 +1878,7 @@ erl_start(int argc, char **argv) } else if (has_prefix("ws", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); - if (erts_sched_set_wakeup_other_type(arg) != 0) { + if (erts_sched_set_wakeup_other_type(ERTS_SCHED_NORMAL, arg) != 0) { erts_fprintf(stderr, "scheduler wakeup strategy: %s\n", arg); erts_usage(); @@ -2191,8 +2256,8 @@ erl_start(int argc, char **argv) erts_initialized = 1; - otp_ring0_pid = erl_first_process_otp("otp_ring0", NULL, 0, - boot_argc, boot_argv); + erts_init_process_id = erl_first_process_otp("otp_ring0", NULL, 0, + boot_argc, boot_argv); { /* @@ -2202,14 +2267,18 @@ erl_start(int argc, char **argv) */ Eterm pid; - pid = erl_system_process_otp(otp_ring0_pid, "erts_code_purger", !0); + pid = erl_system_process_otp(erts_init_process_id, + "erts_code_purger", !0, + PRIORITY_NORMAL); erts_code_purger = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, internal_pid_index(pid)); ASSERT(erts_code_purger && erts_code_purger->common.id == pid); erts_proc_inc_refc(erts_code_purger); - pid = erl_system_process_otp(otp_ring0_pid, "erts_literal_area_collector", !0); + pid = erl_system_process_otp(erts_init_process_id, + "erts_literal_area_collector", + !0, PRIORITY_NORMAL); erts_literal_area_collector = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, internal_pid_index(pid)); @@ -2217,14 +2286,35 @@ erl_start(int argc, char **argv) && erts_literal_area_collector->common.id == pid); erts_proc_inc_refc(erts_literal_area_collector); - pid = erl_system_process_otp(otp_ring0_pid, "erts_dirty_process_code_checker", !0); - erts_dirty_process_code_checker + pid = erl_system_process_otp(erts_init_process_id, + "erts_dirty_process_signal_handler", + !0, PRIORITY_NORMAL); + erts_dirty_process_signal_handler = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, internal_pid_index(pid)); - ASSERT(erts_dirty_process_code_checker - && erts_dirty_process_code_checker->common.id == pid); - erts_proc_inc_refc(erts_dirty_process_code_checker); - + ASSERT(erts_dirty_process_signal_handler + && erts_dirty_process_signal_handler->common.id == pid); + erts_proc_inc_refc(erts_dirty_process_signal_handler); + + pid = erl_system_process_otp(erts_init_process_id, + "erts_dirty_process_signal_handler", + !0, PRIORITY_HIGH); + erts_dirty_process_signal_handler_high + = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, + internal_pid_index(pid)); + ASSERT(erts_dirty_process_signal_handler_high + && erts_dirty_process_signal_handler_high->common.id == pid); + erts_proc_inc_refc(erts_dirty_process_signal_handler_high); + + pid = erl_system_process_otp(erts_init_process_id, + "erts_dirty_process_signal_handler", + !0, PRIORITY_MAX); + erts_dirty_process_signal_handler_max + = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, + internal_pid_index(pid)); + ASSERT(erts_dirty_process_signal_handler_max + && erts_dirty_process_signal_handler_max->common.id == pid); + erts_proc_inc_refc(erts_dirty_process_signal_handler_max); } erts_start_schedulers(); diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c deleted file mode 100644 index 2f70e7996e..0000000000 --- a/erts/emulator/beam/erl_instrument.c +++ /dev/null @@ -1,1257 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2003-2016. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "global.h" -#include "big.h" -#include "erl_instrument.h" -#include "erl_threads.h" - -typedef union { long l; double d; } Align_t; - -typedef struct { - Uint size; -#ifdef VALGRIND - void* valgrind_leak_suppressor; -#endif - Align_t mem[1]; -} StatBlock_t; - -#define STAT_BLOCK_HEADER_SIZE (sizeof(StatBlock_t) - sizeof(Align_t)) - -typedef struct MapStatBlock_t_ MapStatBlock_t; -struct MapStatBlock_t_ { - Uint size; - ErtsAlcType_t type_no; - Eterm pid; - MapStatBlock_t *prev; - MapStatBlock_t *next; - Align_t mem[1]; -}; - -#define MAP_STAT_BLOCK_HEADER_SIZE (sizeof(MapStatBlock_t) - sizeof(Align_t)) - -typedef struct { - Uint size; - Uint max_size; - Uint max_size_ever; - - Uint blocks; - Uint max_blocks; - Uint max_blocks_ever; -} Stat_t; - -static erts_mtx_t instr_mutex; -static erts_mtx_t instr_x_mutex; - -int erts_instr_memory_map; -int erts_instr_stat; - -static ErtsAllocatorFunctions_t real_allctrs[ERTS_ALC_A_MAX+1]; - -struct stats_ { - Stat_t tot; - Stat_t a[ERTS_ALC_A_MAX+1]; - Stat_t *ap[ERTS_ALC_A_MAX+1]; - Stat_t c[ERTS_ALC_C_MAX+1]; - Stat_t n[ERTS_ALC_N_MAX+1]; -}; - -static struct stats_ *stats; - -static MapStatBlock_t *mem_anchor; - -static Eterm *am_tot; -static Eterm *am_n; -static Eterm *am_a; -static Eterm *am_c; - -static int atoms_initialized; - -static struct { - Eterm total; - Eterm allocators; - Eterm classes; - Eterm types; - Eterm sizes; - Eterm blocks; - Eterm instr_hdr; -#ifdef DEBUG - Eterm end_of_atoms; -#endif -} am; - -static void ERTS_INLINE atom_init(Eterm *atom, const char *name) -{ - *atom = am_atom_put((char *) name, sys_strlen(name)); -} -#define AM_INIT(AM) atom_init(&am.AM, #AM) - -static void -init_atoms(void) -{ -#ifdef DEBUG - Eterm *atom; - for (atom = (Eterm *) &am; atom <= &am.end_of_atoms; atom++) { - *atom = THE_NON_VALUE; - } -#endif - - AM_INIT(total); - AM_INIT(allocators); - AM_INIT(classes); - AM_INIT(types); - AM_INIT(sizes); - AM_INIT(blocks); - AM_INIT(instr_hdr); - -#ifdef DEBUG - for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) { - ASSERT(*atom != THE_NON_VALUE); - } -#endif - - atoms_initialized = 1; -} - -#undef AM_INIT - -static void -init_am_tot(void) -{ - am_tot = (Eterm *) erts_alloc(ERTS_ALC_T_INSTR_INFO, - sizeof(Eterm)); - atom_init(am_tot, "total"); -} - - -static void -init_am_n(void) -{ - int i; - am_n = (Eterm *) erts_alloc(ERTS_ALC_T_INSTR_INFO, - (ERTS_ALC_N_MAX+1)*sizeof(Eterm)); - - for (i = ERTS_ALC_N_MIN; i <= ERTS_ALC_N_MAX; i++) { - atom_init(&am_n[i], ERTS_ALC_N2TD(i)); - } - -} - -static void -init_am_c(void) -{ - int i; - am_c = (Eterm *) erts_alloc(ERTS_ALC_T_INSTR_INFO, - (ERTS_ALC_C_MAX+1)*sizeof(Eterm)); - - for (i = ERTS_ALC_C_MIN; i <= ERTS_ALC_C_MAX; i++) { - atom_init(&am_c[i], ERTS_ALC_C2CD(i)); - } - -} - -static void -init_am_a(void) -{ - int i; - am_a = (Eterm *) erts_alloc(ERTS_ALC_T_INSTR_INFO, - (ERTS_ALC_A_MAX+1)*sizeof(Eterm)); - - for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { - atom_init(&am_a[i], ERTS_ALC_A2AD(i)); - } - -} - -static ERTS_INLINE void -stat_upd_alloc(ErtsAlcType_t n, Uint size) -{ - ErtsAlcType_t t = ERTS_ALC_N2T(n); - ErtsAlcType_t a = ERTS_ALC_T2A(t); - ErtsAlcType_t c = ERTS_ALC_T2C(t); - - stats->ap[a]->size += size; - if (stats->ap[a]->max_size < stats->ap[a]->size) - stats->ap[a]->max_size = stats->ap[a]->size; - - stats->c[c].size += size; - if (stats->c[c].max_size < stats->c[c].size) - stats->c[c].max_size = stats->c[c].size; - - stats->n[n].size += size; - if (stats->n[n].max_size < stats->n[n].size) - stats->n[n].max_size = stats->n[n].size; - - stats->tot.size += size; - if (stats->tot.max_size < stats->tot.size) - stats->tot.max_size = stats->tot.size; - - stats->ap[a]->blocks++; - if (stats->ap[a]->max_blocks < stats->ap[a]->blocks) - stats->ap[a]->max_blocks = stats->ap[a]->blocks; - - stats->c[c].blocks++; - if (stats->c[c].max_blocks < stats->c[c].blocks) - stats->c[c].max_blocks = stats->c[c].blocks; - - stats->n[n].blocks++; - if (stats->n[n].max_blocks < stats->n[n].blocks) - stats->n[n].max_blocks = stats->n[n].blocks; - - stats->tot.blocks++; - if (stats->tot.max_blocks < stats->tot.blocks) - stats->tot.max_blocks = stats->tot.blocks; - -} - - -static ERTS_INLINE void -stat_upd_free(ErtsAlcType_t n, Uint size) -{ - ErtsAlcType_t t = ERTS_ALC_N2T(n); - ErtsAlcType_t a = ERTS_ALC_T2A(t); - ErtsAlcType_t c = ERTS_ALC_T2C(t); - - ASSERT(stats->ap[a]->size >= size); - stats->ap[a]->size -= size; - - ASSERT(stats->c[c].size >= size); - stats->c[c].size -= size; - - ASSERT(stats->n[n].size >= size); - stats->n[n].size -= size; - - ASSERT(stats->tot.size >= size); - stats->tot.size -= size; - - ASSERT(stats->ap[a]->blocks > 0); - stats->ap[a]->blocks--; - - ASSERT(stats->c[c].blocks > 0); - stats->c[c].blocks--; - - ASSERT(stats->n[n].blocks > 0); - stats->n[n].blocks--; - - ASSERT(stats->tot.blocks > 0); - stats->tot.blocks--; - -} - - -static ERTS_INLINE void -stat_upd_realloc(ErtsAlcType_t n, Uint size, Uint old_size) -{ - if (old_size) - stat_upd_free(n, old_size); - stat_upd_alloc(n, size); -} - -/* - * stat instrumentation callback functions - */ - -static void stat_pre_lock(void) -{ - erts_mtx_lock(&instr_mutex); -} - -static void stat_pre_unlock(void) -{ - erts_mtx_unlock(&instr_mutex); -} - -static ErtsAllocatorWrapper_t instr_wrapper; - -static void * -stat_alloc(ErtsAlcType_t n, void *extra, Uint size) -{ - ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; - Uint ssize; - void *res; - - if (!erts_is_allctr_wrapper_prelocked()) { - erts_mtx_lock(&instr_mutex); - } - - ssize = size + STAT_BLOCK_HEADER_SIZE; - res = (*real_af->alloc)(n, real_af->extra, ssize); - if (res) { - stat_upd_alloc(n, size); - ((StatBlock_t *) res)->size = size; -#ifdef VALGRIND - /* Suppress "possibly leaks" by storing an actual dummy pointer - to the _start_ of the allocated block.*/ - ((StatBlock_t *) res)->valgrind_leak_suppressor = res; -#endif - res = (void *) ((StatBlock_t *) res)->mem; - } - - if (!erts_is_allctr_wrapper_prelocked()) { - erts_mtx_unlock(&instr_mutex); - } - - return res; -} - -static void * -stat_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) -{ - ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; - Uint old_size; - Uint ssize; - void *sptr; - void *res; - - if (!erts_is_allctr_wrapper_prelocked()) { - erts_mtx_lock(&instr_mutex); - } - - if (ptr) { - sptr = (void *) (((char *) ptr) - STAT_BLOCK_HEADER_SIZE); - old_size = ((StatBlock_t *) sptr)->size; - } - else { - sptr = NULL; - old_size = 0; - } - - ssize = size + STAT_BLOCK_HEADER_SIZE; - res = (*real_af->realloc)(n, real_af->extra, sptr, ssize); - if (res) { - stat_upd_realloc(n, size, old_size); - ((StatBlock_t *) res)->size = size; -#ifdef VALGRIND - ((StatBlock_t *) res)->valgrind_leak_suppressor = res; -#endif - res = (void *) ((StatBlock_t *) res)->mem; - } - - if (!erts_is_allctr_wrapper_prelocked()) { - erts_mtx_unlock(&instr_mutex); - } - - return res; -} - -static void -stat_free(ErtsAlcType_t n, void *extra, void *ptr) -{ - ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; - void *sptr; - - if (!erts_is_allctr_wrapper_prelocked()) { - erts_mtx_lock(&instr_mutex); - } - - if (ptr) { - sptr = (void *) (((char *) ptr) - STAT_BLOCK_HEADER_SIZE); - stat_upd_free(n, ((StatBlock_t *) sptr)->size); - } - else { - sptr = NULL; - } - - (*real_af->free)(n, real_af->extra, sptr); - - if (!erts_is_allctr_wrapper_prelocked()) { - erts_mtx_unlock(&instr_mutex); - } - -} - -/* - * map stat instrumentation callback functions - */ - -static void map_stat_pre_lock(void) -{ - erts_mtx_lock(&instr_x_mutex); - erts_mtx_lock(&instr_mutex); -} - -static void map_stat_pre_unlock(void) -{ - erts_mtx_unlock(&instr_mutex); - erts_mtx_unlock(&instr_x_mutex); -} - -static void * -map_stat_alloc(ErtsAlcType_t n, void *extra, Uint size) -{ - ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; - Uint msize; - void *res; - - if (!erts_is_allctr_wrapper_prelocked()) { - erts_mtx_lock(&instr_mutex); - } - - msize = size + MAP_STAT_BLOCK_HEADER_SIZE; - res = (*real_af->alloc)(n, real_af->extra, msize); - if (res) { - MapStatBlock_t *mb = (MapStatBlock_t *) res; - stat_upd_alloc(n, size); - - mb->size = size; - mb->type_no = n; - mb->pid = erts_get_current_pid(); - - mb->prev = NULL; - mb->next = mem_anchor; - if (mem_anchor) - mem_anchor->prev = mb; - mem_anchor = mb; - - res = (void *) mb->mem; - } - - if (!erts_is_allctr_wrapper_prelocked()) { - erts_mtx_unlock(&instr_mutex); - } - - return res; -} - -static void * -map_stat_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) -{ - ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; - Uint old_size; - Uint msize; - void *mptr; - void *res; - - if (!erts_is_allctr_wrapper_prelocked()) { - erts_mtx_lock(&instr_x_mutex); - erts_mtx_lock(&instr_mutex); - } - - if (ptr) { - mptr = (void *) (((char *) ptr) - MAP_STAT_BLOCK_HEADER_SIZE); - old_size = ((MapStatBlock_t *) mptr)->size; - } - else { - mptr = NULL; - old_size = 0; - } - - msize = size + MAP_STAT_BLOCK_HEADER_SIZE; - res = (*real_af->realloc)(n, real_af->extra, mptr, msize); - if (res) { - MapStatBlock_t *mb = (MapStatBlock_t *) res; - - mb->size = size; - mb->type_no = n; - mb->pid = erts_get_current_pid(); - - stat_upd_realloc(n, size, old_size); - - if (mptr != res) { - - if (mptr) { - if (mb->prev) - mb->prev->next = mb; - else { - ASSERT(mem_anchor == (MapStatBlock_t *) mptr); - mem_anchor = mb; - } - if (mb->next) - mb->next->prev = mb; - } - else { - mb->prev = NULL; - mb->next = mem_anchor; - if (mem_anchor) - mem_anchor->prev = mb; - mem_anchor = mb; - } - - } - - res = (void *) mb->mem; - } - if (!erts_is_allctr_wrapper_prelocked()) { - erts_mtx_unlock(&instr_mutex); - erts_mtx_unlock(&instr_x_mutex); - } - - return res; -} - -static void -map_stat_free(ErtsAlcType_t n, void *extra, void *ptr) -{ - ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; - void *mptr; - - if (!erts_is_allctr_wrapper_prelocked()) { - erts_mtx_lock(&instr_x_mutex); - erts_mtx_lock(&instr_mutex); - } - - if (ptr) { - MapStatBlock_t *mb; - - mptr = (void *) (((char *) ptr) - MAP_STAT_BLOCK_HEADER_SIZE); - mb = (MapStatBlock_t *) mptr; - - stat_upd_free(n, mb->size); - - if (mb->prev) - mb->prev->next = mb->next; - else - mem_anchor = mb->next; - if (mb->next) - mb->next->prev = mb->prev; - } - else { - mptr = NULL; - } - - (*real_af->free)(n, real_af->extra, mptr); - - if (!erts_is_allctr_wrapper_prelocked()) { - erts_mtx_unlock(&instr_mutex); - erts_mtx_unlock(&instr_x_mutex); - } - -} - -static void dump_memory_map_to_stream(fmtfn_t to, void* to_arg) -{ - ErtsAlcType_t n; - MapStatBlock_t *bp; - int lock = !ERTS_IS_CRASH_DUMPING; - if (lock) { - ASSERT(!erts_is_allctr_wrapper_prelocked()); - erts_mtx_lock(&instr_mutex); - } - - /* Write header */ - - erts_cbprintf(to, to_arg, - "{instr_hdr,\n" - " %lu,\n" - " %lu,\n" - " {", - (unsigned long) ERTS_INSTR_VSN, - (unsigned long) MAP_STAT_BLOCK_HEADER_SIZE); - -#if ERTS_ALC_N_MIN != 1 -#error ERTS_ALC_N_MIN is not 1 -#endif - - for (n = ERTS_ALC_N_MIN; n <= ERTS_ALC_N_MAX; n++) { - ErtsAlcType_t t = ERTS_ALC_N2T(n); - ErtsAlcType_t a = ERTS_ALC_T2A(t); - ErtsAlcType_t c = ERTS_ALC_T2C(t); - const char *astr; - - if (erts_allctrs_info[a].enabled) - astr = ERTS_ALC_A2AD(a); - else - astr = ERTS_ALC_A2AD(ERTS_ALC_A_SYSTEM); - - erts_cbprintf(to, to_arg, - "%s{%s,%s,%s}%s", - (n == ERTS_ALC_N_MIN) ? "" : " ", - ERTS_ALC_N2TD(n), - astr, - ERTS_ALC_C2CD(c), - (n == ERTS_ALC_N_MAX) ? "" : ",\n"); - } - - erts_cbprintf(to, to_arg, "}}.\n"); - - /* Write memory data */ - for (bp = mem_anchor; bp; bp = bp->next) { - if (is_internal_pid(bp->pid)) - erts_cbprintf(to, to_arg, - "{%lu, %lu, %lu, {%lu,%lu,%lu}}.\n", - (UWord) bp->type_no, - (UWord) bp->mem, - (UWord) bp->size, - (UWord) pid_channel_no(bp->pid), - (UWord) pid_number(bp->pid), - (UWord) pid_serial(bp->pid)); - else - erts_cbprintf(to, to_arg, - "{%lu, %lu, %lu, undefined}.\n", - (UWord) bp->type_no, - (UWord) bp->mem, - (UWord) bp->size); - } - - if (lock) - erts_mtx_unlock(&instr_mutex); -} - -int erts_instr_dump_memory_map_to(fmtfn_t to, void* to_arg) -{ - if (!erts_instr_memory_map) - return 0; - - dump_memory_map_to_stream(to, to_arg); - return 1; -} - -int erts_instr_dump_memory_map(const char *name) -{ - int fd; - - if (!erts_instr_memory_map) - return 0; - - fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, 0640); - if (fd < 0) - return 0; - - dump_memory_map_to_stream(erts_write_fd, (void*)&fd); - - close(fd); - return 1; -} - -Eterm erts_instr_get_memory_map(Process *proc) -{ - MapStatBlock_t *org_mem_anchor; - Eterm hdr_tuple, md_list, res; - Eterm *hp; - Uint hsz; - MapStatBlock_t *bp; -#ifdef DEBUG - Eterm *end_hp; -#endif - - if (!erts_instr_memory_map) - return am_false; - - if (!atoms_initialized) - init_atoms(); - if (!am_n) - init_am_n(); - if (!am_c) - init_am_c(); - if (!am_a) - init_am_a(); - - erts_mtx_lock(&instr_x_mutex); - erts_mtx_lock(&instr_mutex); - - /* Header size */ - hsz = 5 + 1 + (ERTS_ALC_N_MAX+1-ERTS_ALC_N_MIN)*(1 + 4); - - /* Memory data list */ - for (bp = mem_anchor; bp; bp = bp->next) { - if (is_internal_pid(bp->pid)) { -#if (_PID_NUM_SIZE - 1 > MAX_SMALL) - if (internal_pid_number(bp->pid) > MAX_SMALL) - hsz += BIG_UINT_HEAP_SIZE; -#endif -#if (_PID_SER_SIZE - 1 > MAX_SMALL) - if (internal_pid_serial(bp->pid) > MAX_SMALL) - hsz += BIG_UINT_HEAP_SIZE; -#endif - hsz += 4; - } - - if ((UWord) bp->mem > MAX_SMALL) - hsz += BIG_UINT_HEAP_SIZE; - if (bp->size > MAX_SMALL) - hsz += BIG_UINT_HEAP_SIZE; - - hsz += 5 + 2; - } - - hsz += 3; /* Root tuple */ - - org_mem_anchor = mem_anchor; - mem_anchor = NULL; - - erts_mtx_unlock(&instr_mutex); - - hp = HAlloc(proc, hsz); /* May end up calling map_stat_alloc() */ - - erts_mtx_lock(&instr_mutex); - -#ifdef DEBUG - end_hp = hp + hsz; -#endif - - { /* Build header */ - ErtsAlcType_t n; - Eterm type_map; - Uint *hp2 = hp; -#ifdef DEBUG - Uint *hp2_end; -#endif - - hp += (ERTS_ALC_N_MAX + 1 - ERTS_ALC_N_MIN)*4; - -#ifdef DEBUG - hp2_end = hp; -#endif - - type_map = make_tuple(hp); - *(hp++) = make_arityval(ERTS_ALC_N_MAX + 1 - ERTS_ALC_N_MIN); - - for (n = ERTS_ALC_N_MIN; n <= ERTS_ALC_N_MAX; n++) { - ErtsAlcType_t t = ERTS_ALC_N2T(n); - ErtsAlcType_t a = ERTS_ALC_T2A(t); - ErtsAlcType_t c = ERTS_ALC_T2C(t); - - if (!erts_allctrs_info[a].enabled) - a = ERTS_ALC_A_SYSTEM; - - *(hp++) = TUPLE3(hp2, am_n[n], am_a[a], am_c[c]); - hp2 += 4; - } - - ASSERT(hp2 == hp2_end); - - hdr_tuple = TUPLE4(hp, - am.instr_hdr, - make_small(ERTS_INSTR_VSN), - make_small(MAP_STAT_BLOCK_HEADER_SIZE), - type_map); - - hp += 5; - } - - /* Build memory data list */ - - for (md_list = NIL, bp = org_mem_anchor; bp; bp = bp->next) { - Eterm tuple; - Eterm type; - Eterm ptr; - Eterm size; - Eterm pid; - - if (is_not_internal_pid(bp->pid)) - pid = am_undefined; - else { - Eterm c; - Eterm n; - Eterm s; - -#if (ERST_INTERNAL_CHANNEL_NO > MAX_SMALL) -#error Oversized internal channel number -#endif - c = make_small(ERST_INTERNAL_CHANNEL_NO); - -#if (_PID_NUM_SIZE - 1 > MAX_SMALL) - if (internal_pid_number(bp->pid) > MAX_SMALL) { - n = uint_to_big(internal_pid_number(bp->pid), hp); - hp += BIG_UINT_HEAP_SIZE; - } - else -#endif - n = make_small(internal_pid_number(bp->pid)); - -#if (_PID_SER_SIZE - 1 > MAX_SMALL) - if (internal_pid_serial(bp->pid) > MAX_SMALL) { - s = uint_to_big(internal_pid_serial(bp->pid), hp); - hp += BIG_UINT_HEAP_SIZE; - } - else -#endif - s = make_small(internal_pid_serial(bp->pid)); - pid = TUPLE3(hp, c, n, s); - hp += 4; - } - - -#if ERTS_ALC_N_MAX > MAX_SMALL -#error Oversized memory type number -#endif - type = make_small(bp->type_no); - - if ((UWord) bp->mem > MAX_SMALL) { - ptr = uint_to_big((UWord) bp->mem, hp); - hp += BIG_UINT_HEAP_SIZE; - } - else - ptr = make_small((UWord) bp->mem); - - if (bp->size > MAX_SMALL) { - size = uint_to_big(bp->size, hp); - hp += BIG_UINT_HEAP_SIZE; - } - else - size = make_small(bp->size); - - tuple = TUPLE4(hp, type, ptr, size, pid); - hp += 5; - - md_list = CONS(hp, tuple, md_list); - hp += 2; - } - - res = TUPLE2(hp, hdr_tuple, md_list); - - ASSERT(hp + 3 == end_hp); - - if (mem_anchor) { - for (bp = mem_anchor; bp->next; bp = bp->next) - ; - ASSERT(org_mem_anchor); - org_mem_anchor->prev = bp; - bp->next = org_mem_anchor; - } - else { - mem_anchor = org_mem_anchor; - } - - erts_mtx_unlock(&instr_mutex); - erts_mtx_unlock(&instr_x_mutex); - - return res; -} - -static ERTS_INLINE void -begin_new_max_period(Stat_t *stat, int min, int max) -{ - int i; - for (i = min; i <= max; i++) { - stat[i].max_size = stat[i].size; - stat[i].max_blocks = stat[i].blocks; - } -} - -static ERTS_INLINE void -update_max_ever_values(Stat_t *stat, int min, int max) -{ - int i; - for (i = min; i <= max; i++) { - if (stat[i].max_size_ever < stat[i].max_size) - stat[i].max_size_ever = stat[i].max_size; - if (stat[i].max_blocks_ever < stat[i].max_blocks) - stat[i].max_blocks_ever = stat[i].max_blocks; - } -} - -#define bld_string erts_bld_string -#define bld_tuple erts_bld_tuple -#define bld_tuplev erts_bld_tuplev -#define bld_list erts_bld_list -#define bld_2tup_list erts_bld_2tup_list -#define bld_uint erts_bld_uint - -Eterm -erts_instr_get_stat(Process *proc, Eterm what, int begin_max_period) -{ - int i, len, max, min, allctr; - Eterm *names, *values, res; - Uint arr_size, stat_size, hsz, *hszp, *hp, **hpp; - Stat_t *stat_src, *stat; - - if (!erts_instr_stat) - return am_false; - - if (!atoms_initialized) - init_atoms(); - - if (what == am.total) { - min = 0; - max = 0; - allctr = 0; - stat_size = sizeof(Stat_t); - stat_src = &stats->tot; - if (!am_tot) - init_am_tot(); - names = am_tot; - } - else if (what == am.allocators) { - min = ERTS_ALC_A_MIN; - max = ERTS_ALC_A_MAX; - allctr = 1; - stat_size = sizeof(Stat_t)*(ERTS_ALC_A_MAX+1); - stat_src = stats->a; - if (!am_a) - init_am_a(); - names = am_a; - } - else if (what == am.classes) { - min = ERTS_ALC_C_MIN; - max = ERTS_ALC_C_MAX; - allctr = 0; - stat_size = sizeof(Stat_t)*(ERTS_ALC_C_MAX+1); - stat_src = stats->c; - if (!am_c) - init_am_c(); - names = &am_c[ERTS_ALC_C_MIN]; - } - else if (what == am.types) { - min = ERTS_ALC_N_MIN; - max = ERTS_ALC_N_MAX; - allctr = 0; - stat_size = sizeof(Stat_t)*(ERTS_ALC_N_MAX+1); - stat_src = stats->n; - if (!am_n) - init_am_n(); - names = &am_n[ERTS_ALC_N_MIN]; - } - else { - return THE_NON_VALUE; - } - - stat = (Stat_t *) erts_alloc(ERTS_ALC_T_TMP, stat_size); - - arr_size = (max - min + 1)*sizeof(Eterm); - - if (allctr) - names = (Eterm *) erts_alloc(ERTS_ALC_T_TMP, arr_size); - - values = (Eterm *) erts_alloc(ERTS_ALC_T_TMP, arr_size); - - erts_mtx_lock(&instr_mutex); - - update_max_ever_values(stat_src, min, max); - - sys_memcpy((void *) stat, (void *) stat_src, stat_size); - - if (begin_max_period) - begin_new_max_period(stat_src, min, max); - - erts_mtx_unlock(&instr_mutex); - - hsz = 0; - hszp = &hsz; - hpp = NULL; - - restart_bld: - - len = 0; - for (i = min; i <= max; i++) { - if (!allctr || erts_allctrs_info[i].enabled) { - Eterm s[2]; - - if (allctr) - names[len] = am_a[i]; - - s[0] = bld_tuple(hpp, hszp, 4, - am.sizes, - bld_uint(hpp, hszp, stat[i].size), - bld_uint(hpp, hszp, stat[i].max_size), - bld_uint(hpp, hszp, stat[i].max_size_ever)); - - s[1] = bld_tuple(hpp, hszp, 4, - am.blocks, - bld_uint(hpp, hszp, stat[i].blocks), - bld_uint(hpp, hszp, stat[i].max_blocks), - bld_uint(hpp, hszp, stat[i].max_blocks_ever)); - - values[len] = bld_list(hpp, hszp, 2, s); - - len++; - } - } - - res = bld_2tup_list(hpp, hszp, len, names, values); - - if (!hpp) { - hp = HAlloc(proc, hsz); - hszp = NULL; - hpp = &hp; - goto restart_bld; - } - - erts_free(ERTS_ALC_T_TMP, (void *) stat); - erts_free(ERTS_ALC_T_TMP, (void *) values); - if (allctr) - erts_free(ERTS_ALC_T_TMP, (void *) names); - - return res; -} - -static void -dump_stat_to_stream(fmtfn_t to, void* to_arg, int begin_max_period) -{ - ErtsAlcType_t i, a_max, a_min; - - erts_mtx_lock(&instr_mutex); - - erts_cbprintf(to, to_arg, - "{instr_vsn,%lu}.\n", - (unsigned long) ERTS_INSTR_VSN); - - update_max_ever_values(&stats->tot, 0, 0); - - erts_cbprintf(to, to_arg, - "{total,[{total,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}]}.\n", - (UWord) stats->tot.size, - (UWord) stats->tot.max_size, - (UWord) stats->tot.max_size_ever, - (UWord) stats->tot.blocks, - (UWord) stats->tot.max_blocks, - (UWord) stats->tot.max_blocks_ever); - - a_max = 0; - a_min = ~0; - for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { - if (erts_allctrs_info[i].enabled) { - if (a_min > i) - a_min = i; - if (a_max < i) - a_max = i; - } - } - - ASSERT(ERTS_ALC_A_MIN <= a_min && a_min <= ERTS_ALC_A_MAX); - ASSERT(ERTS_ALC_A_MIN <= a_max && a_max <= ERTS_ALC_A_MAX); - ASSERT(a_min <= a_max); - - update_max_ever_values(stats->a, a_min, a_max); - - for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { - if (erts_allctrs_info[i].enabled) { - erts_cbprintf(to, to_arg, - "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s", - i == a_min ? "{allocators,\n [" : " ", - ERTS_ALC_A2AD(i), - (UWord) stats->a[i].size, - (UWord) stats->a[i].max_size, - (UWord) stats->a[i].max_size_ever, - (UWord) stats->a[i].blocks, - (UWord) stats->a[i].max_blocks, - (UWord) stats->a[i].max_blocks_ever, - i == a_max ? "]}.\n" : ",\n"); - } - } - - update_max_ever_values(stats->c, ERTS_ALC_C_MIN, ERTS_ALC_C_MAX); - - for (i = ERTS_ALC_C_MIN; i <= ERTS_ALC_C_MAX; i++) { - erts_cbprintf(to, to_arg, - "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s", - i == ERTS_ALC_C_MIN ? "{classes,\n [" : " ", - ERTS_ALC_C2CD(i), - (UWord) stats->c[i].size, - (UWord) stats->c[i].max_size, - (UWord) stats->c[i].max_size_ever, - (UWord) stats->c[i].blocks, - (UWord) stats->c[i].max_blocks, - (UWord) stats->c[i].max_blocks_ever, - i == ERTS_ALC_C_MAX ? "]}.\n" : ",\n" ); - } - - update_max_ever_values(stats->n, ERTS_ALC_N_MIN, ERTS_ALC_N_MAX); - - for (i = ERTS_ALC_N_MIN; i <= ERTS_ALC_N_MAX; i++) { - erts_cbprintf(to, to_arg, - "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s", - i == ERTS_ALC_N_MIN ? "{types,\n [" : " ", - ERTS_ALC_N2TD(i), - (UWord) stats->n[i].size, - (UWord) stats->n[i].max_size, - (UWord) stats->n[i].max_size_ever, - (UWord) stats->n[i].blocks, - (UWord) stats->n[i].max_blocks, - (UWord) stats->n[i].max_blocks_ever, - i == ERTS_ALC_N_MAX ? "]}.\n" : ",\n" ); - } - - if (begin_max_period) { - begin_new_max_period(&stats->tot, 0, 0); - begin_new_max_period(stats->a, a_min, a_max); - begin_new_max_period(stats->c, ERTS_ALC_C_MIN, ERTS_ALC_C_MAX); - begin_new_max_period(stats->n, ERTS_ALC_N_MIN, ERTS_ALC_N_MAX); - } - - erts_mtx_unlock(&instr_mutex); - -} - -int erts_instr_dump_stat_to(fmtfn_t to, void* to_arg, int begin_max_period) -{ - if (!erts_instr_stat) - return 0; - - dump_stat_to_stream(to, to_arg, begin_max_period); - return 1; -} - -int erts_instr_dump_stat(const char *name, int begin_max_period) -{ - int fd; - - if (!erts_instr_stat) - return 0; - - fd = open(name, O_WRONLY | O_CREAT | O_TRUNC,0640); - if (fd < 0) - return 0; - - dump_stat_to_stream(erts_write_fd, (void*)&fd, begin_max_period); - - close(fd); - return 1; -} - - -Uint -erts_instr_get_total(void) -{ - return erts_instr_stat ? stats->tot.size : 0; -} - -Uint -erts_instr_get_max_total(void) -{ - if (erts_instr_stat) { - update_max_ever_values(&stats->tot, 0, 0); - return stats->tot.max_size_ever; - } - return 0; -} - -Eterm -erts_instr_get_type_info(Process *proc) -{ - Eterm res, *tpls; - Uint hsz, *hszp, *hp, **hpp; - ErtsAlcType_t n; - - if (!am_n) - init_am_n(); - if (!am_a) - init_am_a(); - if (!am_c) - init_am_c(); - - tpls = (Eterm *) erts_alloc(ERTS_ALC_T_TMP, - (ERTS_ALC_N_MAX-ERTS_ALC_N_MIN+1) - * sizeof(Eterm)); - hsz = 0; - hszp = &hsz; - hpp = NULL; - - restart_bld: - -#if ERTS_ALC_N_MIN != 1 -#error ERTS_ALC_N_MIN is not 1 -#endif - - for (n = ERTS_ALC_N_MIN; n <= ERTS_ALC_N_MAX; n++) { - ErtsAlcType_t t = ERTS_ALC_N2T(n); - ErtsAlcType_t a = ERTS_ALC_T2A(t); - ErtsAlcType_t c = ERTS_ALC_T2C(t); - - if (!erts_allctrs_info[a].enabled) - a = ERTS_ALC_A_SYSTEM; - - tpls[n - ERTS_ALC_N_MIN] - = bld_tuple(hpp, hszp, 3, am_n[n], am_a[a], am_c[c]); - } - - res = bld_tuplev(hpp, hszp, ERTS_ALC_N_MAX-ERTS_ALC_N_MIN+1, tpls); - - if (!hpp) { - hp = HAlloc(proc, hsz); - hszp = NULL; - hpp = &hp; - goto restart_bld; - } - - erts_free(ERTS_ALC_T_TMP, tpls); - - return res; -} - -Uint -erts_instr_init(int stat, int map_stat) -{ - Uint extra_sz; - int i; - - am_tot = NULL; - am_n = NULL; - am_c = NULL; - am_a = NULL; - - erts_instr_memory_map = 0; - erts_instr_stat = 0; - atoms_initialized = 0; - - if (!stat && !map_stat) - return 0; - - stats = erts_alloc(ERTS_ALC_T_INSTR_INFO, sizeof(struct stats_)); - - erts_mtx_init(&instr_mutex, "instr", NIL, - ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG); - - mem_anchor = NULL; - - /* Install instrumentation functions */ - ERTS_CT_ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs)); - - sys_memcpy((void *)real_allctrs,(void *)erts_allctrs,sizeof(erts_allctrs)); - - sys_memzero((void *) &stats->tot, sizeof(Stat_t)); - sys_memzero((void *) stats->a, sizeof(Stat_t)*(ERTS_ALC_A_MAX+1)); - sys_memzero((void *) stats->c, sizeof(Stat_t)*(ERTS_ALC_C_MAX+1)); - sys_memzero((void *) stats->n, sizeof(Stat_t)*(ERTS_ALC_N_MAX+1)); - - for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { - if (erts_allctrs_info[i].enabled) - stats->ap[i] = &stats->a[i]; - else - stats->ap[i] = &stats->a[ERTS_ALC_A_SYSTEM]; - } - - if (map_stat) { - - erts_mtx_init(&instr_x_mutex, "instr_x", NIL, - ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG); - - erts_instr_memory_map = 1; - erts_instr_stat = 1; - for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { - erts_allctrs[i].alloc = map_stat_alloc; - erts_allctrs[i].realloc = map_stat_realloc; - erts_allctrs[i].free = map_stat_free; - erts_allctrs[i].extra = (void *) &real_allctrs[i]; - } - instr_wrapper.lock = map_stat_pre_lock; - instr_wrapper.unlock = map_stat_pre_unlock; - extra_sz = MAP_STAT_BLOCK_HEADER_SIZE; - } - else { - erts_instr_stat = 1; - for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { - erts_allctrs[i].alloc = stat_alloc; - erts_allctrs[i].realloc = stat_realloc; - erts_allctrs[i].free = stat_free; - erts_allctrs[i].extra = (void *) &real_allctrs[i]; - } - instr_wrapper.lock = stat_pre_lock; - instr_wrapper.unlock = stat_pre_unlock; - extra_sz = STAT_BLOCK_HEADER_SIZE; - } - erts_allctr_wrapper_prelock_init(&instr_wrapper); - return extra_sz; -} - diff --git a/erts/emulator/beam/erl_instrument.h b/erts/emulator/beam/erl_instrument.h deleted file mode 100644 index 351172b2fa..0000000000 --- a/erts/emulator/beam/erl_instrument.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2003-2016. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -#ifndef ERL_INSTRUMENT_H__ -#define ERL_INSTRUMENT_H__ - -#include "erl_mtrace.h" - -#define ERTS_INSTR_VSN 2 - -extern int erts_instr_memory_map; -extern int erts_instr_stat; - -Uint erts_instr_init(int stat, int map_stat); -int erts_instr_dump_memory_map_to(fmtfn_t to, void* to_arg); -int erts_instr_dump_memory_map(const char *name); -Eterm erts_instr_get_memory_map(Process *process); -int erts_instr_dump_stat_to(fmtfn_t to, void* to_arg, int begin_max_period); -int erts_instr_dump_stat(const char *name, int begin_max_period); -Eterm erts_instr_get_stat(Process *proc, Eterm what, int begin_max_period); -Eterm erts_instr_get_type_info(Process *proc); -Uint erts_instr_get_total(void); -Uint erts_instr_get_max_total(void); - -#endif diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 773b138d92..d66410367b 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2017. All Rights Reserved. + * Copyright Ericsson AB 2005-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,6 +41,7 @@ #include "erl_lock_check.h" #include "erl_term.h" #include "erl_threads.h" +#include "erl_atom_table.h" typedef struct { char *name; @@ -75,10 +76,10 @@ static erts_lc_lock_order_t erts_lock_order[] = { * if only one lock use * the lock name)" */ + { "NO LOCK", NULL }, { "driver_lock", "driver_name" }, { "port_lock", "port_id" }, { "port_data_lock", "address" }, - { "bif_timers", NULL }, { "reg_tab", NULL }, { "proc_main", "pid" }, { "old_code", "address" }, @@ -92,7 +93,6 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "db_hash_slot", "address" }, { "resource_monitors", "address" }, { "driver_list", NULL }, - { "proc_link", "pid" }, { "proc_msgq", "pid" }, { "proc_btm", "pid" }, { "dist_entry", "address" }, @@ -104,7 +104,6 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "node_table", NULL }, { "dist_table", NULL }, { "sys_tracers", NULL }, - { "module_tab", NULL }, { "export_tab", NULL }, { "fun_tab", NULL }, { "environ", NULL }, @@ -112,7 +111,6 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "drv_ev_state_grow", NULL, }, { "drv_ev_state", "address" }, { "safe_hash", "address" }, - { "removed_fd_pre_alloc_lock", "address" }, { "state_prealloc", NULL }, { "schdlr_sspnd", NULL }, { "migration_info_update", NULL }, @@ -135,10 +133,6 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "msacc_list_mutex", NULL }, { "msacc_unmanaged_mutex", NULL }, { "atom_tab", NULL }, - { "misc_op_list_pre_alloc_lock", "address" }, - { "message_pre_alloc_lock", "address" }, - { "ptimer_pre_alloc_lock", "address", }, - { "btm_pre_alloc_lock", NULL, }, { "dist_entry_out_queue", "address" }, { "port_sched_lock", "port_id" }, { "sys_msg_q", NULL }, @@ -148,20 +142,12 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "mtrace_op", NULL }, { "instr_x", NULL }, { "instr", NULL }, - { "pollsets_lock", NULL }, { "alcu_allocator", "index" }, { "mseg", NULL }, - { "port_task_pre_alloc_lock", "address" }, - { "proclist_pre_alloc_lock", "address" }, - { "xports_list_pre_alloc_lock", "address" }, - { "inet_buffer_stack_lock", NULL }, - { "system_block", NULL }, { "get_time", NULL }, { "get_corrected_time", NULL }, { "runtime", NULL }, - { "breakpoints", NULL }, { "pix_lock", "address" }, - { "run_queues_lists", NULL }, { "sched_stat", NULL }, { "async_init_mtx", NULL }, #ifdef __WIN32__ @@ -195,10 +181,10 @@ static const char *rw_op_str(erts_lock_options_t options) return erts_lock_options_get_short_desc(options); } -typedef struct erts_lc_locked_lock_t_ erts_lc_locked_lock_t; -struct erts_lc_locked_lock_t_ { - erts_lc_locked_lock_t *next; - erts_lc_locked_lock_t *prev; +typedef struct lc_locked_lock_t_ lc_locked_lock_t; +struct lc_locked_lock_t_ { + lc_locked_lock_t *next; + lc_locked_lock_t *prev; UWord extra; Sint16 id; char *file; @@ -208,32 +194,47 @@ struct erts_lc_locked_lock_t_ { }; typedef struct { - erts_lc_locked_lock_t *first; - erts_lc_locked_lock_t *last; -} erts_lc_locked_lock_list_t; + lc_locked_lock_t *first; + lc_locked_lock_t *last; +} lc_locked_lock_list_t; + +typedef union lc_free_block_t_ lc_free_block_t; +union lc_free_block_t_ { + lc_free_block_t *next; + lc_locked_lock_t lock; +}; + +typedef struct { + /* + * m[X][Y] & 1 if we locked X directly after Y was locked. + * m[X][Y] & 2 if we locked X indirectly after Y was locked. + * m[X][0] = 1 if we locked X when nothing else was locked. + * m[0][] is unused as it would represent locking "NO LOCK" + */ + char m[ERTS_LOCK_ORDER_SIZE][ERTS_LOCK_ORDER_SIZE]; + +} lc_matrix_t; -typedef struct erts_lc_locked_locks_t_ erts_lc_locked_locks_t; -struct erts_lc_locked_locks_t_ { +static lc_matrix_t tot_lc_matrix; + +typedef struct lc_thread_t_ lc_thread_t; +struct lc_thread_t_ { char *thread_name; int emu_thread; erts_tid_t tid; - erts_lc_locked_locks_t *next; - erts_lc_locked_locks_t *prev; - erts_lc_locked_lock_list_t locked; - erts_lc_locked_lock_list_t required; -}; - -typedef union erts_lc_free_block_t_ erts_lc_free_block_t; -union erts_lc_free_block_t_ { - erts_lc_free_block_t *next; - erts_lc_locked_lock_t lock; + lc_thread_t *next; + lc_thread_t *prev; + lc_locked_lock_list_t locked; + lc_locked_lock_list_t required; + lc_free_block_t *free_blocks; + lc_matrix_t matrix; }; static ethr_tsd_key locks_key; -static erts_lc_locked_locks_t *erts_locked_locks = NULL; +static lc_thread_t *lc_threads = NULL; +static ethr_spinlock_t lc_threads_lock; -static erts_lc_free_block_t *free_blocks = NULL; #ifdef ERTS_LC_STATIC_ALLOC #define ERTS_LC_FB_CHUNK_SIZE 10000 @@ -241,176 +242,165 @@ static erts_lc_free_block_t *free_blocks = NULL; #define ERTS_LC_FB_CHUNK_SIZE 10 #endif -static ethr_spinlock_t free_blocks_lock; static ERTS_INLINE void -lc_lock(void) +lc_lock_threads(void) { - ethr_spin_lock(&free_blocks_lock); + ethr_spin_lock(&lc_threads_lock); } static ERTS_INLINE void -lc_unlock(void) +lc_unlock_threads(void) { - ethr_spin_unlock(&free_blocks_lock); + ethr_spin_unlock(&lc_threads_lock); } -static ERTS_INLINE void lc_free(void *p) +static ERTS_INLINE void lc_free(lc_thread_t* thr, lc_locked_lock_t *p) { - erts_lc_free_block_t *fb = (erts_lc_free_block_t *) p; + lc_free_block_t *fb = (lc_free_block_t *) p; #ifdef DEBUG - sys_memset((void *) p, 0xdf, sizeof(erts_lc_free_block_t)); + sys_memset((void *) p, 0xdf, sizeof(lc_free_block_t)); #endif - lc_lock(); - fb->next = free_blocks; - free_blocks = fb; - lc_unlock(); + fb->next = thr->free_blocks; + thr->free_blocks = fb; } -#ifdef ERTS_LC_STATIC_ALLOC - -static void *lc_core_alloc(void) -{ - lc_unlock(); - ERTS_INTERNAL_ERROR("Lock checker out of memory!\n"); -} - -#else - -static void *lc_core_alloc(void) +static lc_locked_lock_t *lc_core_alloc(lc_thread_t* thr) { int i; - erts_lc_free_block_t *fbs; - lc_unlock(); - fbs = (erts_lc_free_block_t *) malloc(sizeof(erts_lc_free_block_t) + lc_free_block_t *fbs; + fbs = (lc_free_block_t *) malloc(sizeof(lc_free_block_t) * ERTS_LC_FB_CHUNK_SIZE); if (!fbs) { ERTS_INTERNAL_ERROR("Lock checker failed to allocate memory!"); } for (i = 1; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) { #ifdef DEBUG - sys_memset((void *) &fbs[i], 0xdf, sizeof(erts_lc_free_block_t)); + sys_memset((void *) &fbs[i], 0xdf, sizeof(lc_free_block_t)); #endif fbs[i].next = &fbs[i+1]; } #ifdef DEBUG sys_memset((void *) &fbs[ERTS_LC_FB_CHUNK_SIZE-1], - 0xdf, sizeof(erts_lc_free_block_t)); + 0xdf, sizeof(lc_free_block_t)); #endif - lc_lock(); - fbs[ERTS_LC_FB_CHUNK_SIZE-1].next = free_blocks; - free_blocks = &fbs[1]; - return (void *) &fbs[0]; + fbs[ERTS_LC_FB_CHUNK_SIZE-1].next = thr->free_blocks; + thr->free_blocks = &fbs[1]; + return &fbs[0].lock; } -#endif - -static ERTS_INLINE void *lc_alloc(void) +static ERTS_INLINE lc_locked_lock_t *lc_alloc(lc_thread_t* thr) { - void *res; - lc_lock(); - if (!free_blocks) - res = lc_core_alloc(); + lc_locked_lock_t *res; + if (!thr->free_blocks) + res = lc_core_alloc(thr); else { - res = (void *) free_blocks; - free_blocks = free_blocks->next; + res = &thr->free_blocks->lock; + thr->free_blocks = thr->free_blocks->next; } - lc_unlock(); return res; } -static erts_lc_locked_locks_t * -create_locked_locks(char *thread_name) +static lc_thread_t * +create_thread_data(char *thread_name) { - erts_lc_locked_locks_t *l_lcks = malloc(sizeof(erts_lc_locked_locks_t)); - if (!l_lcks) + lc_thread_t *thr = malloc(sizeof(lc_thread_t)); + if (!thr) ERTS_INTERNAL_ERROR("Lock checker failed to allocate memory!"); - l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown"); - if (!l_lcks->thread_name) + thr->thread_name = strdup(thread_name ? thread_name : "unknown"); + if (!thr->thread_name) ERTS_INTERNAL_ERROR("Lock checker failed to allocate memory!"); - l_lcks->emu_thread = 0; - l_lcks->tid = erts_thr_self(); - l_lcks->required.first = NULL; - l_lcks->required.last = NULL; - l_lcks->locked.first = NULL; - l_lcks->locked.last = NULL; - l_lcks->prev = NULL; - lc_lock(); - l_lcks->next = erts_locked_locks; - if (erts_locked_locks) - erts_locked_locks->prev = l_lcks; - erts_locked_locks = l_lcks; - lc_unlock(); - erts_tsd_set(locks_key, (void *) l_lcks); - return l_lcks; + thr->emu_thread = 0; + thr->tid = erts_thr_self(); + thr->required.first = NULL; + thr->required.last = NULL; + thr->locked.first = NULL; + thr->locked.last = NULL; + thr->prev = NULL; + thr->free_blocks = NULL; + sys_memzero(&thr->matrix, sizeof(thr->matrix)); + + lc_lock_threads(); + thr->next = lc_threads; + if (lc_threads) + lc_threads->prev = thr; + lc_threads = thr; + lc_unlock_threads(); + erts_tsd_set(locks_key, (void *) thr); + return thr; } +static void collect_matrix(lc_matrix_t*); + static void -destroy_locked_locks(erts_lc_locked_locks_t *l_lcks) -{ - ASSERT(l_lcks->thread_name); - free((void *) l_lcks->thread_name); - ASSERT(l_lcks->required.first == NULL); - ASSERT(l_lcks->required.last == NULL); - ASSERT(l_lcks->locked.first == NULL); - ASSERT(l_lcks->locked.last == NULL); - - lc_lock(); - if (l_lcks->prev) - l_lcks->prev->next = l_lcks->next; +destroy_locked_locks(lc_thread_t *thr) +{ + ASSERT(thr->thread_name); + free((void *) thr->thread_name); + ASSERT(thr->required.first == NULL); + ASSERT(thr->required.last == NULL); + ASSERT(thr->locked.first == NULL); + ASSERT(thr->locked.last == NULL); + + lc_lock_threads(); + if (thr->prev) + thr->prev->next = thr->next; else { - ASSERT(erts_locked_locks == l_lcks); - erts_locked_locks = l_lcks->next; + ASSERT(lc_threads == thr); + lc_threads = thr->next; } + if (thr->next) + thr->next->prev = thr->prev; + + collect_matrix(&thr->matrix); - if (l_lcks->next) - l_lcks->next->prev = l_lcks->prev; - lc_unlock(); + lc_unlock_threads(); - free((void *) l_lcks); + free((void *) thr); } -static ERTS_INLINE erts_lc_locked_locks_t * +static ERTS_INLINE lc_thread_t * get_my_locked_locks(void) { return erts_tsd_get(locks_key); } -static ERTS_INLINE erts_lc_locked_locks_t * +static ERTS_INLINE lc_thread_t * make_my_locked_locks(void) { - erts_lc_locked_locks_t *l_lcks = get_my_locked_locks(); - if (l_lcks) - return l_lcks; + lc_thread_t *thr = get_my_locked_locks(); + if (thr) + return thr; else - return create_locked_locks(NULL); + return create_thread_data(NULL); } -static ERTS_INLINE erts_lc_locked_lock_t * -new_locked_lock(erts_lc_lock_t *lck, erts_lock_options_t options, +static ERTS_INLINE lc_locked_lock_t * +new_locked_lock(lc_thread_t* thr, + erts_lc_lock_t *lck, erts_lock_options_t options, char *file, unsigned int line) { - erts_lc_locked_lock_t *l_lck = (erts_lc_locked_lock_t *) lc_alloc(); - l_lck->next = NULL; - l_lck->prev = NULL; - l_lck->id = lck->id; - l_lck->extra = lck->extra; - l_lck->file = file; - l_lck->line = line; - l_lck->flags = lck->flags; - l_lck->taken_options = options; - return l_lck; + lc_locked_lock_t *ll = lc_alloc(thr); + ll->next = NULL; + ll->prev = NULL; + ll->id = lck->id; + ll->extra = lck->extra; + ll->file = file; + ll->line = line; + ll->flags = lck->flags; + ll->taken_options = options; + return ll; } static void raw_print_lock(char *prefix, Sint16 id, Wterm extra, erts_lock_flags_t flags, char* file, unsigned int line, char *suffix) { - char *lname = (0 <= id && id < ERTS_LOCK_ORDER_SIZE + char *lname = (1 <= id && id < ERTS_LOCK_ORDER_SIZE ? erts_lock_order[id].name : "unknown"); erts_fprintf(stderr,"%s'%s:",prefix,lname); @@ -440,20 +430,20 @@ print_lock(char *prefix, erts_lc_lock_t *lck, char *suffix) } static void -print_curr_locks(erts_lc_locked_locks_t *l_lcks) +print_curr_locks(lc_thread_t *thr) { - erts_lc_locked_lock_t *l_lck; - if (!l_lcks || !l_lcks->locked.first) + lc_locked_lock_t *ll; + if (!thr || !thr->locked.first) erts_fprintf(stderr, "Currently no locks are locked by the %s thread.\n", - l_lcks->thread_name); + thr->thread_name); else { erts_fprintf(stderr, "Currently these locks are locked by the %s thread:\n", - l_lcks->thread_name); - for (l_lck = l_lcks->locked.first; l_lck; l_lck = l_lck->next) - raw_print_lock(" ", l_lck->id, l_lck->extra, l_lck->flags, - l_lck->file, l_lck->line, "\n"); + thr->thread_name); + for (ll = thr->locked.first; ll; ll = ll->next) + raw_print_lock(" ", ll->id, ll->extra, ll->flags, + ll->file, ll->line, "\n"); } } @@ -482,55 +472,55 @@ uninitialized_lock(void) } static void -lock_twice(char *prefix, erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck, +lock_twice(char *prefix, lc_thread_t *thr, erts_lc_lock_t *lck, erts_lock_options_t options) { erts_fprintf(stderr, "%s (%s)", prefix, rw_op_str(options)); print_lock(" ", lck, " lock which is already locked by thread!\n"); - print_curr_locks(l_lcks); + print_curr_locks(thr); lc_abort(); } static void -unlock_op_mismatch(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck, +unlock_op_mismatch(lc_thread_t *thr, erts_lc_lock_t *lck, erts_lock_options_t options) { erts_fprintf(stderr, "Unlocking (%s) ", rw_op_str(options)); print_lock("", lck, " lock which mismatch previous lock operation!\n"); - print_curr_locks(l_lcks); + print_curr_locks(thr); lc_abort(); } static void -unlock_of_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) +unlock_of_not_locked(lc_thread_t *thr, erts_lc_lock_t *lck) { print_lock("Unlocking ", lck, " lock which is not locked by thread!\n"); - print_curr_locks(l_lcks); + print_curr_locks(thr); lc_abort(); } static void -lock_order_violation(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) +lock_order_violation(lc_thread_t *thr, erts_lc_lock_t *lck) { print_lock("Lock order violation occured when locking ", lck, "!\n"); - print_curr_locks(l_lcks); + print_curr_locks(thr); print_lock_order(); lc_abort(); } static void -type_order_violation(char *op, erts_lc_locked_locks_t *l_lcks, +type_order_violation(char *op, lc_thread_t *thr, erts_lc_lock_t *lck) { erts_fprintf(stderr, "Lock type order violation occured when "); print_lock(op, lck, "!\n"); - ASSERT(l_lcks); - print_curr_locks(l_lcks); + ASSERT(thr); + print_curr_locks(thr); lc_abort(); } static void -lock_mismatch(erts_lc_locked_locks_t *l_lcks, int exact, +lock_mismatch(lc_thread_t *thr, int exact, int failed_have, erts_lc_lock_t *have, int have_len, int failed_have_not, erts_lc_lock_t *have_not, int have_not_len) { @@ -577,39 +567,39 @@ lock_mismatch(erts_lc_locked_locks_t *l_lcks, int exact, print_lock2(" ", have_not[i].id, have_not[i].extra, 0, "\n"); } } - print_curr_locks(l_lcks); + print_curr_locks(thr); lc_abort(); } static void -unlock_of_required_lock(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) +unlock_of_required_lock(lc_thread_t *thr, erts_lc_lock_t *lck) { print_lock("Unlocking required ", lck, " lock!\n"); - print_curr_locks(l_lcks); + print_curr_locks(thr); lc_abort(); } static void -unrequire_of_not_required_lock(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) +unrequire_of_not_required_lock(lc_thread_t *thr, erts_lc_lock_t *lck) { print_lock("Unrequire on ", lck, " lock not required!\n"); - print_curr_locks(l_lcks); + print_curr_locks(thr); lc_abort(); } static void -require_twice(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) +require_twice(lc_thread_t *thr, erts_lc_lock_t *lck) { print_lock("Require on ", lck, " lock already required!\n"); - print_curr_locks(l_lcks); + print_curr_locks(thr); lc_abort(); } static void -required_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) +required_not_locked(lc_thread_t *thr, erts_lc_lock_t *lck) { print_lock("Required ", lck, " lock not locked!\n"); - print_curr_locks(l_lcks); + print_curr_locks(thr); lc_abort(); } @@ -617,15 +607,15 @@ required_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) static void thread_exit_handler(void) { - erts_lc_locked_locks_t *l_lcks = get_my_locked_locks(); - if (l_lcks) { - if (l_lcks->locked.first) { + lc_thread_t *thr = get_my_locked_locks(); + if (thr) { + if (thr->locked.first) { erts_fprintf(stderr, "Thread exiting while having locked locks!\n"); - print_curr_locks(l_lcks); + print_curr_locks(thr); lc_abort(); } - destroy_locked_locks(l_lcks); + destroy_locked_locks(thr); /* erts_tsd_set(locks_key, NULL); */ } } @@ -643,24 +633,24 @@ lc_abort(void) void erts_lc_set_thread_name(char *thread_name) { - erts_lc_locked_locks_t *l_lcks = get_my_locked_locks(); - if (!l_lcks) - l_lcks = create_locked_locks(thread_name); + lc_thread_t *thr = get_my_locked_locks(); + if (!thr) + thr = create_thread_data(thread_name); else { - ASSERT(l_lcks->thread_name); - free((void *) l_lcks->thread_name); - l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown"); - if (!l_lcks->thread_name) + ASSERT(thr->thread_name); + free((void *) thr->thread_name); + thr->thread_name = strdup(thread_name ? thread_name : "unknown"); + if (!thr->thread_name) ERTS_INTERNAL_ERROR("strdup failed"); } - l_lcks->emu_thread = 1; + thr->emu_thread = 1; } int erts_lc_is_emu_thr(void) { - erts_lc_locked_locks_t *l_lcks = get_my_locked_locks(); - return l_lcks->emu_thread; + lc_thread_t *thr = get_my_locked_locks(); + return thr->emu_thread; } int @@ -706,7 +696,7 @@ erts_lc_get_lock_order_id(char *name) return (Sint16) -1; } -static int compare_locked_by_id(erts_lc_locked_lock_t *locked_lock, erts_lc_lock_t *comparand) +static int compare_locked_by_id(lc_locked_lock_t *locked_lock, erts_lc_lock_t *comparand) { if(locked_lock->id < comparand->id) { return -1; @@ -717,7 +707,7 @@ static int compare_locked_by_id(erts_lc_locked_lock_t *locked_lock, erts_lc_lock return 0; } -static int compare_locked_by_id_extra(erts_lc_locked_lock_t *locked_lock, erts_lc_lock_t *comparand) +static int compare_locked_by_id_extra(lc_locked_lock_t *locked_lock, erts_lc_lock_t *comparand) { int order = compare_locked_by_id(locked_lock, comparand); @@ -732,18 +722,18 @@ static int compare_locked_by_id_extra(erts_lc_locked_lock_t *locked_lock, erts_l return 0; } -typedef int (*locked_compare_func)(erts_lc_locked_lock_t *, erts_lc_lock_t *); +typedef int (*locked_compare_func)(lc_locked_lock_t *, erts_lc_lock_t *); /* Searches through a list of taken locks, bailing when it hits an entry whose * order relative to the search template is the opposite of the one at the * start of the search. (*closest_neighbor) is either set to the exact match, * or the one closest to it in the sort order. */ static int search_locked_list(locked_compare_func compare, - erts_lc_locked_lock_t *locked_locks, + lc_locked_lock_t *locked_locks, erts_lc_lock_t *search_template, - erts_lc_locked_lock_t **closest_neighbor) + lc_locked_lock_t **closest_neighbor) { - erts_lc_locked_lock_t *iterator = locked_locks; + lc_locked_lock_t *iterator = locked_locks; (*closest_neighbor) = iterator; @@ -779,9 +769,9 @@ static int search_locked_list(locked_compare_func compare, /* Searches for a lock in the given list that matches search_template, and sets * (*locked_locks) to the closest lock in the sort order. */ static int -find_lock(erts_lc_locked_lock_t **locked_locks, erts_lc_lock_t *search_template) +find_lock(lc_locked_lock_t **locked_locks, erts_lc_lock_t *search_template) { - erts_lc_locked_lock_t *closest_neighbor; + lc_locked_lock_t *closest_neighbor; int found_lock; found_lock = search_locked_list(compare_locked_by_id_extra, @@ -810,9 +800,9 @@ find_lock(erts_lc_locked_lock_t **locked_locks, erts_lc_lock_t *search_template) /* Searches for a lock in the given list by id, and sets (*locked_locks) to the * closest lock in the sort order. */ static int -find_id(erts_lc_locked_lock_t **locked_locks, Sint16 id) +find_id(lc_locked_lock_t **locked_locks, Sint16 id) { - erts_lc_locked_lock_t *closest_neighbor; + lc_locked_lock_t *closest_neighbor; erts_lc_lock_t search_template; int found_lock; @@ -831,34 +821,34 @@ find_id(erts_lc_locked_lock_t **locked_locks, Sint16 id) void erts_lc_have_locks(int *resv, erts_lc_lock_t *locks, int len) { - erts_lc_locked_locks_t *l_lcks = get_my_locked_locks(); + lc_thread_t *thr = get_my_locked_locks(); int i; - if (!l_lcks) { + if (!thr) { for (i = 0; i < len; i++) resv[i] = 0; } else { - erts_lc_locked_lock_t *l_lck = l_lcks->locked.first; + lc_locked_lock_t *ll = thr->locked.first; for (i = 0; i < len; i++) - resv[i] = find_lock(&l_lck, &locks[i]); + resv[i] = find_lock(&ll, &locks[i]); } } void erts_lc_have_lock_ids(int *resv, int *ids, int len) { - erts_lc_locked_locks_t *l_lcks = get_my_locked_locks(); + lc_thread_t *thr = get_my_locked_locks(); int i; - if (!l_lcks) { + if (!thr) { for (i = 0; i < len; i++) resv[i] = 0; } else { - erts_lc_locked_lock_t *l_lck = l_lcks->locked.first; + lc_locked_lock_t *ll = thr->locked.first; for (i = 0; i < len; i++) - resv[i] = find_id(&l_lck, ids[i]); + resv[i] = find_id(&ll, ids[i]); } } @@ -867,27 +857,27 @@ erts_lc_check(erts_lc_lock_t *have, int have_len, erts_lc_lock_t *have_not, int have_not_len) { int i; - erts_lc_locked_locks_t *l_lcks = get_my_locked_locks(); - erts_lc_locked_lock_t *l_lck; + lc_thread_t *thr = get_my_locked_locks(); + lc_locked_lock_t *ll; if (have && have_len > 0) { - if (!l_lcks) + if (!thr) lock_mismatch(NULL, 0, -1, have, have_len, -1, have_not, have_not_len); - l_lck = l_lcks->locked.first; + ll = thr->locked.first; for (i = 0; i < have_len; i++) { - if (!find_lock(&l_lck, &have[i])) - lock_mismatch(l_lcks, 0, + if (!find_lock(&ll, &have[i])) + lock_mismatch(thr, 0, i, have, have_len, -1, have_not, have_not_len); } } - if (have_not && have_not_len > 0 && l_lcks) { - l_lck = l_lcks->locked.first; + if (have_not && have_not_len > 0 && thr) { + ll = thr->locked.first; for (i = 0; i < have_not_len; i++) { - if (find_lock(&l_lck, &have_not[i])) - lock_mismatch(l_lcks, 0, + if (find_lock(&ll, &have_not[i])) + lock_mismatch(thr, 0, -1, have, have_len, i, have_not, have_not_len); } @@ -897,8 +887,8 @@ erts_lc_check(erts_lc_lock_t *have, int have_len, void erts_lc_check_exact(erts_lc_lock_t *have, int have_len) { - erts_lc_locked_locks_t *l_lcks = get_my_locked_locks(); - if (!l_lcks) { + lc_thread_t *thr = get_my_locked_locks(); + if (!thr) { if (have && have_len > 0) lock_mismatch(NULL, 1, -1, have, have_len, @@ -906,17 +896,17 @@ erts_lc_check_exact(erts_lc_lock_t *have, int have_len) } else { int i; - erts_lc_locked_lock_t *l_lck = l_lcks->locked.first; + lc_locked_lock_t *ll = thr->locked.first; for (i = 0; i < have_len; i++) { - if (!find_lock(&l_lck, &have[i])) - lock_mismatch(l_lcks, 1, + if (!find_lock(&ll, &have[i])) + lock_mismatch(thr, 1, i, have, have_len, -1, NULL, 0); } - for (i = 0, l_lck = l_lcks->locked.first; l_lck; l_lck = l_lck->next) + for (i = 0, ll = thr->locked.first; ll; ll = ll->next) i++; if (i != have_len) - lock_mismatch(l_lcks, 1, + lock_mismatch(thr, 1, -1, have, have_len, -1, NULL, 0); } @@ -925,16 +915,16 @@ erts_lc_check_exact(erts_lc_lock_t *have, int have_len) void erts_lc_check_no_locked_of_type(erts_lock_flags_t type) { - erts_lc_locked_locks_t *l_lcks = get_my_locked_locks(); - if (l_lcks) { - erts_lc_locked_lock_t *l_lck = l_lcks->locked.first; - for (l_lck = l_lcks->locked.first; l_lck; l_lck = l_lck->next) { - if ((l_lck->flags & ERTS_LOCK_FLAGS_MASK_TYPE) == type) { + lc_thread_t *thr = get_my_locked_locks(); + if (thr) { + lc_locked_lock_t *ll = thr->locked.first; + for (ll = thr->locked.first; ll; ll = ll->next) { + if ((ll->flags & ERTS_LOCK_FLAGS_MASK_TYPE) == type) { erts_fprintf(stderr, "Locked lock of type %s found which isn't " "allowed here!\n", - erts_lock_flags_get_type_name(l_lck->flags)); - print_curr_locks(l_lcks); + erts_lock_flags_get_type_name(ll->flags)); + print_curr_locks(thr); lc_abort(); } } @@ -952,7 +942,7 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, erts_lock_options_t options) * This in order to make sure that caller can handle * the situation without causing a lock order violation. */ - erts_lc_locked_locks_t *l_lcks; + lc_thread_t *thr; if (lck->inited != ERTS_LC_INITITALIZED) uninitialized_lock(); @@ -960,25 +950,25 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, erts_lock_options_t options) if (lck->id < 0) return 0; - l_lcks = get_my_locked_locks(); + thr = get_my_locked_locks(); - if (!l_lcks || !l_lcks->locked.first) { - ASSERT(!l_lcks || !l_lcks->locked.last); + if (!thr || !thr->locked.first) { + ASSERT(!thr || !thr->locked.last); return 0; } else { - erts_lc_locked_lock_t *tl_lck; + lc_locked_lock_t *tl_lck; - ASSERT(l_lcks->locked.last); + ASSERT(thr->locked.last); #if 0 /* Ok when trylocking I guess... */ - if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, l_lcks->locked.last->flags)) - type_order_violation("trylocking ", l_lcks, lck); + if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, thr->locked.last->flags)) + type_order_violation("trylocking ", thr, lck); #endif - if (l_lcks->locked.last->id < lck->id - || (l_lcks->locked.last->id == lck->id - && l_lcks->locked.last->extra < lck->extra)) + if (thr->locked.last->id < lck->id + || (thr->locked.last->id == lck->id + && thr->locked.last->extra < lck->extra)) return 0; /* @@ -987,11 +977,11 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, erts_lock_options_t options) /* Check that we are not trying to lock this lock twice */ - for (tl_lck = l_lcks->locked.last; tl_lck; tl_lck = tl_lck->prev) { + for (tl_lck = thr->locked.last; tl_lck; tl_lck = tl_lck->prev) { if (tl_lck->id < lck->id || (tl_lck->id == lck->id && tl_lck->extra <= lck->extra)) { if (tl_lck->id == lck->id && tl_lck->extra == lck->extra) - lock_twice("Trylocking", l_lcks, lck, options); + lock_twice("Trylocking", thr, lck, options); break; } } @@ -1016,8 +1006,8 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, erts_lock_options_t options) void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, erts_lock_options_t options, char *file, unsigned int line) { - erts_lc_locked_locks_t *l_lcks; - erts_lc_locked_lock_t *l_lck; + lc_thread_t *thr; + lc_locked_lock_t *ll; if (lck->inited != ERTS_LC_INITITALIZED) uninitialized_lock(); @@ -1025,43 +1015,43 @@ void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, erts_lock_options_t if (lck->id < 0) return; - l_lcks = make_my_locked_locks(); - l_lck = locked ? new_locked_lock(lck, options, file, line) : NULL; + thr = make_my_locked_locks(); + ll = locked ? new_locked_lock(thr, lck, options, file, line) : NULL; - if (!l_lcks->locked.last) { - ASSERT(!l_lcks->locked.first); + if (!thr->locked.last) { + ASSERT(!thr->locked.first); if (locked) - l_lcks->locked.first = l_lcks->locked.last = l_lck; + thr->locked.first = thr->locked.last = ll; } else { - erts_lc_locked_lock_t *tl_lck; + lc_locked_lock_t *tl_lck; #if 0 /* Ok when trylocking I guess... */ - if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, l_lcks->locked.last->flags)) - type_order_violation("trylocking ", l_lcks, lck); + if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, thr->locked.last->flags)) + type_order_violation("trylocking ", thr, lck); #endif - for (tl_lck = l_lcks->locked.last; tl_lck; tl_lck = tl_lck->prev) { + for (tl_lck = thr->locked.last; tl_lck; tl_lck = tl_lck->prev) { if (tl_lck->id < lck->id || (tl_lck->id == lck->id && tl_lck->extra <= lck->extra)) { if (tl_lck->id == lck->id && tl_lck->extra == lck->extra) - lock_twice("Trylocking", l_lcks, lck, options); + lock_twice("Trylocking", thr, lck, options); if (locked) { - l_lck->next = tl_lck->next; - l_lck->prev = tl_lck; + ll->next = tl_lck->next; + ll->prev = tl_lck; if (tl_lck->next) - tl_lck->next->prev = l_lck; + tl_lck->next->prev = ll; else - l_lcks->locked.last = l_lck; - tl_lck->next = l_lck; + thr->locked.last = ll; + tl_lck->next = ll; } return; } } if (locked) { - l_lck->next = l_lcks->locked.first; - l_lcks->locked.first->prev = l_lck; - l_lcks->locked.first = l_lck; + ll->next = thr->locked.first; + thr->locked.first->prev = ll; + thr->locked.first = ll; } } @@ -1070,83 +1060,83 @@ void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, erts_lock_options_t void erts_lc_require_lock_flg(erts_lc_lock_t *lck, erts_lock_options_t options, char *file, unsigned int line) { - erts_lc_locked_locks_t *l_lcks = make_my_locked_locks(); - erts_lc_locked_lock_t *l_lck = l_lcks->locked.first; - if (!find_lock(&l_lck, lck)) - required_not_locked(l_lcks, lck); - l_lck = new_locked_lock(lck, options, file, line); - if (!l_lcks->required.last) { - ASSERT(!l_lcks->required.first); - l_lck->next = l_lck->prev = NULL; - l_lcks->required.first = l_lcks->required.last = l_lck; + lc_thread_t *thr = make_my_locked_locks(); + lc_locked_lock_t *ll = thr->locked.first; + if (!find_lock(&ll, lck)) + required_not_locked(thr, lck); + ll = new_locked_lock(thr, lck, options, file, line); + if (!thr->required.last) { + ASSERT(!thr->required.first); + ll->next = ll->prev = NULL; + thr->required.first = thr->required.last = ll; } else { - erts_lc_locked_lock_t *l_lck2; - ASSERT(l_lcks->required.first); - for (l_lck2 = l_lcks->required.last; + lc_locked_lock_t *l_lck2; + ASSERT(thr->required.first); + for (l_lck2 = thr->required.last; l_lck2; l_lck2 = l_lck2->prev) { if (l_lck2->id < lck->id || (l_lck2->id == lck->id && l_lck2->extra < lck->extra)) break; else if (l_lck2->id == lck->id && l_lck2->extra == lck->extra) - require_twice(l_lcks, lck); + require_twice(thr, lck); } if (!l_lck2) { - l_lck->next = l_lcks->required.first; - l_lck->prev = NULL; - l_lcks->required.first->prev = l_lck; - l_lcks->required.first = l_lck; + ll->next = thr->required.first; + ll->prev = NULL; + thr->required.first->prev = ll; + thr->required.first = ll; } else { - l_lck->next = l_lck2->next; - if (l_lck->next) { - ASSERT(l_lcks->required.last != l_lck2); - l_lck->next->prev = l_lck; + ll->next = l_lck2->next; + if (ll->next) { + ASSERT(thr->required.last != l_lck2); + ll->next->prev = ll; } else { - ASSERT(l_lcks->required.last == l_lck2); - l_lcks->required.last = l_lck; + ASSERT(thr->required.last == l_lck2); + thr->required.last = ll; } - l_lck->prev = l_lck2; - l_lck2->next = l_lck; + ll->prev = l_lck2; + l_lck2->next = ll; } } } void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, erts_lock_options_t options) { - erts_lc_locked_locks_t *l_lcks = make_my_locked_locks(); - erts_lc_locked_lock_t *l_lck = l_lcks->locked.first; - if (!find_lock(&l_lck, lck)) - required_not_locked(l_lcks, lck); - l_lck = l_lcks->required.first; - if (!find_lock(&l_lck, lck)) - unrequire_of_not_required_lock(l_lcks, lck); - if (l_lck->prev) { - ASSERT(l_lcks->required.first != l_lck); - l_lck->prev->next = l_lck->next; + lc_thread_t *thr = make_my_locked_locks(); + lc_locked_lock_t *ll = thr->locked.first; + if (!find_lock(&ll, lck)) + required_not_locked(thr, lck); + ll = thr->required.first; + if (!find_lock(&ll, lck)) + unrequire_of_not_required_lock(thr, lck); + if (ll->prev) { + ASSERT(thr->required.first != ll); + ll->prev->next = ll->next; } else { - ASSERT(l_lcks->required.first == l_lck); - l_lcks->required.first = l_lck->next; + ASSERT(thr->required.first == ll); + thr->required.first = ll->next; } - if (l_lck->next) { - ASSERT(l_lcks->required.last != l_lck); - l_lck->next->prev = l_lck->prev; + if (ll->next) { + ASSERT(thr->required.last != ll); + ll->next->prev = ll->prev; } else { - ASSERT(l_lcks->required.last == l_lck); - l_lcks->required.last = l_lck->prev; + ASSERT(thr->required.last == ll); + thr->required.last = ll->prev; } - lc_free((void *) l_lck); + lc_free(thr, ll); } void erts_lc_lock_flg_x(erts_lc_lock_t *lck, erts_lock_options_t options, char *file, unsigned int line) { - erts_lc_locked_locks_t *l_lcks; - erts_lc_locked_lock_t *l_lck; + lc_thread_t *thr; + lc_locked_lock_t *new_ll; if (lck->inited != ERTS_LC_INITITALIZED) uninitialized_lock(); @@ -1154,32 +1144,45 @@ void erts_lc_lock_flg_x(erts_lc_lock_t *lck, erts_lock_options_t options, if (lck->id < 0) return; - l_lcks = make_my_locked_locks(); - l_lck = new_locked_lock(lck, options, file, line); + thr = make_my_locked_locks(); + new_ll = new_locked_lock(thr, lck, options, file, line); - if (!l_lcks->locked.last) { - ASSERT(!l_lcks->locked.first); - l_lcks->locked.last = l_lcks->locked.first = l_lck; + if (!thr->locked.last) { + ASSERT(!thr->locked.first); + thr->locked.last = thr->locked.first = new_ll; + ASSERT(0 < lck->id && lck->id < ERTS_LOCK_ORDER_SIZE); + thr->matrix.m[lck->id][0] = 1; } - else if (l_lcks->locked.last->id < lck->id - || (l_lcks->locked.last->id == lck->id - && l_lcks->locked.last->extra < lck->extra)) { - if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, l_lcks->locked.last->flags)) - type_order_violation("locking ", l_lcks, lck); - l_lck->prev = l_lcks->locked.last; - l_lcks->locked.last->next = l_lck; - l_lcks->locked.last = l_lck; + else if (thr->locked.last->id < lck->id + || (thr->locked.last->id == lck->id + && thr->locked.last->extra < lck->extra)) { + lc_locked_lock_t* ll; + if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, thr->locked.last->flags)) { + type_order_violation("locking ", thr, lck); + } + + ASSERT(0 < lck->id && lck->id < ERTS_LOCK_ORDER_SIZE); + ll = thr->locked.last; + thr->matrix.m[lck->id][ll->id] |= 1; + for (ll = ll->prev; ll; ll = ll->prev) { + ASSERT(0 < ll->id && ll->id < ERTS_LOCK_ORDER_SIZE); + thr->matrix.m[lck->id][ll->id] |= 2; + } + + new_ll->prev = thr->locked.last; + thr->locked.last->next = new_ll; + thr->locked.last = new_ll; } - else if (l_lcks->locked.last->id == lck->id && l_lcks->locked.last->extra == lck->extra) - lock_twice("Locking", l_lcks, lck, options); + else if (thr->locked.last->id == lck->id && thr->locked.last->extra == lck->extra) + lock_twice("Locking", thr, lck, options); else - lock_order_violation(l_lcks, lck); + lock_order_violation(thr, lck); } void erts_lc_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options) { - erts_lc_locked_locks_t *l_lcks; - erts_lc_locked_lock_t *l_lck; + lc_thread_t *thr; + lc_locked_lock_t *ll; if (lck->inited != ERTS_LC_INITITALIZED) uninitialized_lock(); @@ -1187,38 +1190,38 @@ void erts_lc_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options) if (lck->id < 0) return; - l_lcks = get_my_locked_locks(); + thr = get_my_locked_locks(); - if (l_lcks) { - l_lck = l_lcks->required.first; - if (find_lock(&l_lck, lck)) - unlock_of_required_lock(l_lcks, lck); + if (thr) { + ll = thr->required.first; + if (find_lock(&ll, lck)) + unlock_of_required_lock(thr, lck); } - for (l_lck = l_lcks ? l_lcks->locked.last : NULL; l_lck; l_lck = l_lck->prev) { - if (l_lck->id == lck->id && l_lck->extra == lck->extra) { - if ((l_lck->taken_options & ERTS_LOCK_OPTIONS_RDWR) != options) - unlock_op_mismatch(l_lcks, lck, options); - if (l_lck->prev) - l_lck->prev->next = l_lck->next; + for (ll = thr ? thr->locked.last : NULL; ll; ll = ll->prev) { + if (ll->id == lck->id && ll->extra == lck->extra) { + if ((ll->taken_options & ERTS_LOCK_OPTIONS_RDWR) != options) + unlock_op_mismatch(thr, lck, options); + if (ll->prev) + ll->prev->next = ll->next; else - l_lcks->locked.first = l_lck->next; - if (l_lck->next) - l_lck->next->prev = l_lck->prev; + thr->locked.first = ll->next; + if (ll->next) + ll->next->prev = ll->prev; else - l_lcks->locked.last = l_lck->prev; - lc_free((void *) l_lck); + thr->locked.last = ll->prev; + lc_free(thr, ll); return; } } - unlock_of_not_locked(l_lcks, lck); + unlock_of_not_locked(thr, lck); } void erts_lc_might_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options) { - erts_lc_locked_locks_t *l_lcks; - erts_lc_locked_lock_t *l_lck; + lc_thread_t *thr; + lc_locked_lock_t *ll; if (lck->inited != ERTS_LC_INITITALIZED) uninitialized_lock(); @@ -1226,17 +1229,17 @@ void erts_lc_might_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options) if (lck->id < 0) return; - l_lcks = get_my_locked_locks(); + thr = get_my_locked_locks(); - if (l_lcks) { - l_lck = l_lcks->required.first; - if (find_lock(&l_lck, lck)) - unlock_of_required_lock(l_lcks, lck); + if (thr) { + ll = thr->required.first; + if (find_lock(&ll, lck)) + unlock_of_required_lock(thr, lck); } - l_lck = l_lcks->locked.first; - if (!find_lock(&l_lck, lck)) - unlock_of_not_locked(l_lcks, lck); + ll = thr->locked.first; + if (!find_lock(&ll, lck)) + unlock_of_not_locked(thr, lck); } int @@ -1317,26 +1320,7 @@ erts_lc_destroy_lock(erts_lc_lock_t *lck) void erts_lc_init(void) { -#ifdef ERTS_LC_STATIC_ALLOC - int i; - static erts_lc_free_block_t fbs[ERTS_LC_FB_CHUNK_SIZE]; - for (i = 0; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) { -#ifdef DEBUG - sys_memset((void *) &fbs[i], 0xdf, sizeof(erts_lc_free_block_t)); -#endif - fbs[i].next = &fbs[i+1]; - } -#ifdef DEBUG - sys_memset((void *) &fbs[ERTS_LC_FB_CHUNK_SIZE-1], - 0xdf, sizeof(erts_lc_free_block_t)); -#endif - fbs[ERTS_LC_FB_CHUNK_SIZE-1].next = NULL; - free_blocks = &fbs[0]; -#else /* #ifdef ERTS_LC_STATIC_ALLOC */ - free_blocks = NULL; -#endif /* #ifdef ERTS_LC_STATIC_ALLOC */ - - if (ethr_spinlock_init(&free_blocks_lock) != 0) + if (ethr_spinlock_init(&lc_threads_lock) != 0) ERTS_INTERNAL_ERROR("spinlock_init failed"); erts_tsd_key_create(&locks_key,"erts_lock_check_key"); @@ -1358,5 +1342,76 @@ erts_lc_pll(void) print_curr_locks(get_my_locked_locks()); } +static void collect_matrix(lc_matrix_t* matrix) +{ + int i, j; + for (i = 1; i < ERTS_LOCK_ORDER_SIZE; i++) { + for (j = 0; j <= i; j++) { + tot_lc_matrix.m[i][j] |= matrix->m[i][j]; + } +#ifdef DEBUG + for ( ; j < ERTS_LOCK_ORDER_SIZE; j++) { + ASSERT(matrix->m[i][j] == 0); + } +#endif + } +} + +Eterm +erts_lc_dump_graph(void) +{ + const char* basename = "lc_graph."; + char filename[40]; + lc_matrix_t* tot = &tot_lc_matrix; + lc_thread_t* thr; + int i, j, name_max = 0; + FILE* ff; + + lc_lock_threads(); + for (thr = lc_threads; thr; thr = thr->next) { + collect_matrix(&thr->matrix); + } + lc_unlock_threads(); + + sys_strcpy(filename, basename); + sys_get_pid(filename + strlen(basename), + sizeof(filename) - strlen(basename)); + ff = fopen(filename, "w"); + if (!ff) + return am_error; + + for (i = 1; i < ERTS_LOCK_ORDER_SIZE; i++) { + int len = strlen(erts_lock_order[i].name); + if (name_max < len) + name_max = len; + } + fputs("%This file was generated by erts_debug:lc_graph()\n\n", ff); + fputs("%{ThisLockName, ThisLockId, LockedDirectlyBeforeThis, LockedIndirectlyBeforeThis}\n", ff); + fprintf(ff, "[{%*s, %2d}", name_max, "\"NO LOCK\"", 0); + for (i = 1; i < ERTS_LOCK_ORDER_SIZE; i++) { + char* delim = ""; + fprintf(ff, ",\n {%*s, %2d, [", name_max, erts_lock_order[i].name, i); + for (j = 0; j < ERTS_LOCK_ORDER_SIZE; j++) { + if (tot->m[i][j] & 1) { + fprintf(ff, "%s%d", delim, j); + delim = ","; + } + } + fprintf(ff, "], ["); + delim = ""; + for (j = 0; j < ERTS_LOCK_ORDER_SIZE; j++) { + if (tot->m[i][j] == 2) { + fprintf(ff, "%s%d", delim, j); + delim = ","; + } + } + fputs("]}", ff); + } + fputs("].", ff); + fclose(ff); + erts_fprintf(stderr, "Created file '%s' in current working directory\n", + filename); + return am_ok; +} #endif /* #ifdef ERTS_ENABLE_LOCK_CHECK */ diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h index 5c2c38e8f2..138bc810bd 100644 --- a/erts/emulator/beam/erl_lock_check.h +++ b/erts/emulator/beam/erl_lock_check.h @@ -94,6 +94,8 @@ void erts_lc_unrequire_lock(erts_lc_lock_t *lck); int erts_lc_is_emu_thr(void); +Eterm erts_lc_dump_graph(void); + #define ERTS_LC_ASSERT(A) \ ((void) (((A) || ERTS_SOMEONE_IS_CRASH_DUMPING) ? 1 : erts_lc_assert_failed(__FILE__, __LINE__, #A))) #else /* #ifdef ERTS_ENABLE_LOCK_CHECK */ diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index 8047a9567f..f577b017c3 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -44,6 +44,7 @@ * DONE: * - erlang:is_map/1 * - erlang:map_size/1 + * - erlang:map_get/2 * * - maps:find/2 * - maps:from_list/1 @@ -202,7 +203,7 @@ BIF_RETTYPE maps_find_2(BIF_ALIST_2) { BIF_ERROR(BIF_P, BADMAP); } -/* maps:get/2 +/* maps:get/2 and erlang:map_get/2 * return value if key *matches* a key in the map * exception badkey if none matches */ @@ -223,6 +224,10 @@ BIF_RETTYPE maps_get_2(BIF_ALIST_2) { BIF_ERROR(BIF_P, BADMAP); } +BIF_RETTYPE map_get_2(BIF_ALIST_2) { + BIF_RET(maps_get_2(BIF_CALL_ARGS)); +} + /* maps:from_list/1 * List may be unsorted [{K,V}] */ @@ -490,7 +495,9 @@ Eterm erts_map_from_ks_and_vs(ErtsHeapFactory *factory, Eterm *ks0, Eterm *vs0, sys_memcpy(ks, ks0, n * sizeof(Eterm)); sys_memcpy(vs, vs0, n * sizeof(Eterm)); - erts_validate_and_sort_flatmap(mp); + if (!erts_validate_and_sort_flatmap(mp)) { + return THE_NON_VALUE; + } return make_flatmap(mp); } else { diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 6f7c71ef98..bea7a0fe86 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2017. All Rights Reserved. + * Copyright Ericsson AB 1997-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,6 +33,7 @@ #include "erl_binary.h" #include "dtrace-wrapper.h" #include "beam_bp.h" +#include "erl_proc_sig_queue.h" ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message_ref, ErtsMessageRef, @@ -207,7 +208,7 @@ erts_cleanup_messages(ErtsMessage *msgp) while (mp) { ErtsMessage *fmp; ErlHeapFragment *bp; - if (is_non_value(ERL_MESSAGE_TERM(mp))) { + if (ERTS_SIG_IS_EXTERNAL_MSG(mp)) { if (is_not_immed(ERL_MESSAGE_TOKEN(mp))) { bp = (ErlHeapFragment *) mp->data.dist_ext->ext_endp; erts_cleanup_offheap(&bp->off_heap); @@ -215,10 +216,13 @@ erts_cleanup_messages(ErtsMessage *msgp) if (mp->data.dist_ext) erts_free_dist_ext_copy(mp->data.dist_ext); } - else { - if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) + else { + if (ERTS_SIG_IS_INTERNAL_MSG(mp) + && mp->data.attached != ERTS_MSG_COMBINED_HFRAG) { bp = mp->data.heap_frag; + } else { + mp->data.attached = ERTS_MSG_COMBINED_HFRAG; bp = mp->hfrag.next; erts_cleanup_offheap(&mp->hfrag.off_heap); } @@ -260,11 +264,6 @@ erts_queue_dist_message(Process *rcvr, Eterm from) { ErtsMessage* mp; -#ifdef USE_VM_PROBES - Sint tok_label = 0; - Sint tok_lastcnt = 0; - Sint tok_serial = 0; -#endif erts_aint_t state; ERTS_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr)); @@ -272,6 +271,7 @@ erts_queue_dist_message(Process *rcvr, mp = erts_alloc_message(0, NULL); mp->data.dist_ext = dist_ext; + ERL_MESSAGE_FROM(mp) = dist_ext->dep->sysname; ERL_MESSAGE_TERM(mp) = THE_NON_VALUE; #ifdef USE_VM_PROBES ERL_MESSAGE_DT_UTAG(mp) = NIL; @@ -294,220 +294,153 @@ erts_queue_dist_message(Process *rcvr, } } + state = erts_atomic32_read_acqb(&rcvr->state); - if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) { + if (state & ERTS_PSFLG_EXITING) { if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); /* Drop message if receiver is exiting or has a pending exit ... */ erts_cleanup_messages(mp); } - else - if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) { - if (from == am_Empty) - from = dist_ext->dep->sysname; - - /* Ahh... need to decode it in order to trace it... */ - if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) - erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); - if (!erts_decode_dist_message(rcvr, rcvr_locks, mp, 0)) - erts_free_message(mp); - else { - Eterm msg = ERL_MESSAGE_TERM(mp); - token = ERL_MESSAGE_TOKEN(mp); -#ifdef USE_VM_PROBES - if (DTRACE_ENABLED(message_queued)) { - DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); - - dtrace_proc_str(rcvr, receiver_name); - if (have_seqtrace(token)) { - tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); - tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); - tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); - } - DTRACE6(message_queued, - receiver_name, size_object(msg), rcvr->msg.len, - tok_label, tok_lastcnt, tok_serial); - } -#endif - erts_queue_message(rcvr, rcvr_locks, mp, msg, from); - } - } else { - /* Enqueue message on external format */ + LINK_MESSAGE(rcvr, mp); -#ifdef USE_VM_PROBES - if (DTRACE_ENABLED(message_queued)) { - DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); - - dtrace_proc_str(rcvr, receiver_name); - if (have_seqtrace(token)) { - tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); - tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); - tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); - } - /* - * TODO: We don't know the real size of the external message here. - * -1 will appear to a D script as 4294967295. - */ - DTRACE6(message_queued, receiver_name, -1, rcvr->msg.len + 1, - tok_label, tok_lastcnt, tok_serial); - } -#endif - - LINK_MESSAGE(rcvr, mp, &mp->next, 1); + if (rcvr_locks & ERTS_PROC_LOCK_MAIN) + erts_proc_sig_fetch(rcvr); if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); - erts_proc_notify_new_message(rcvr, - rcvr_locks - ); + erts_proc_notify_new_message(rcvr, rcvr_locks); } } /* Add messages last in message queue */ -static Sint +static void queue_messages(Process* receiver, - erts_aint32_t *receiver_state, ErtsProcLocks receiver_locks, ErtsMessage* first, ErtsMessage** last, - Uint len, - Eterm from) + Uint len) { - ErtsTracingEvent* te; - Sint res; int locked_msgq = 0; erts_aint32_t state; - ASSERT(is_value(ERL_MESSAGE_TERM(first))); - ASSERT(ERL_MESSAGE_TOKEN(first) == am_undefined || - ERL_MESSAGE_TOKEN(first) == NIL || - is_tuple(ERL_MESSAGE_TOKEN(first))); - -#ifdef ERTS_ENABLE_LOCK_CHECK - ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(receiver) < ERTS_PROC_LOCK_MSGQ || - receiver_locks == erts_proc_lc_my_proc_locks(receiver)); +#ifdef DEBUG + { + ErtsMessage* fmsg = ERTS_SIG_IS_MSG(first) ? first : first->next; + ASSERT(fmsg); + ASSERT(is_value(ERL_MESSAGE_TERM(fmsg))); + ASSERT(is_value(ERL_MESSAGE_FROM(fmsg))); + ASSERT(ERL_MESSAGE_TOKEN(fmsg) == am_undefined || + ERL_MESSAGE_TOKEN(fmsg) == NIL || + is_tuple(ERL_MESSAGE_TOKEN(fmsg))); + } #endif - if (!(receiver_locks & ERTS_PROC_LOCK_MSGQ)) { - if (erts_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) { - ErtsProcLocks need_locks; - - if (receiver_state) - state = *receiver_state; - else - state = erts_atomic32_read_nob(&receiver->state); - if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) - goto exiting; + ERTS_LC_ASSERT((erts_proc_lc_my_proc_locks(receiver) & ERTS_PROC_LOCK_MSGQ) + == (receiver_locks & ERTS_PROC_LOCK_MSGQ)); - need_locks = receiver_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ); - if (need_locks) { - erts_proc_unlock(receiver, need_locks); - } - need_locks |= ERTS_PROC_LOCK_MSGQ; - erts_proc_lock(receiver, need_locks); - } + if (!(receiver_locks & ERTS_PROC_LOCK_MSGQ)) { + erts_proc_lock(receiver, ERTS_PROC_LOCK_MSGQ); locked_msgq = 1; } - state = erts_atomic32_read_nob(&receiver->state); - if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) { - exiting: + if (state & ERTS_PSFLG_EXITING) { /* Drop message if receiver is exiting or has a pending exit... */ if (locked_msgq) - erts_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); + erts_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); + if (ERTS_SIG_IS_NON_MSG(first)) { + ErtsSchedulerData* esdp = erts_get_scheduler_data(); + ASSERT(esdp); + ASSERT(!esdp->pending_signal.sig); + esdp->pending_signal.sig = (ErtsSignal*) first; + esdp->pending_signal.to = receiver->common.id; + first = first->next; + } erts_cleanup_messages(first); - return 0; + return; } - res = receiver->msg.len; - if (receiver_locks & ERTS_PROC_LOCK_MAIN) { - /* - * We move 'in queue' to 'private queue' and place - * message at the end of 'private queue' in order - * to ensure that the 'in queue' doesn't contain - * references into the heap. By ensuring this, - * we don't need to include the 'in queue' in - * the root set when garbage collecting. - */ - res += receiver->msg_inq.len; - ERTS_MSGQ_MV_INQ2PRIVQ(receiver); - LINK_MESSAGE_PRIVQ(receiver, first, last, len); + if (last == &first->next) { + ASSERT(len == 1); + LINK_MESSAGE(receiver, first); } - else - { - LINK_MESSAGE(receiver, first, last, len); + else { + erts_enqueue_signals(receiver, first, last, NULL, len, state); } - if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE) - && (te = &erts_receive_tracing[erts_active_bp_ix()], - te->on)) { - - ErtsMessage *msg = first; + if (receiver_locks & ERTS_PROC_LOCK_MAIN) + erts_proc_sig_fetch(receiver); -#ifdef USE_VM_PROBES - if (DTRACE_ENABLED(message_queued)) { - DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); - Sint tok_label = 0; - Sint tok_lastcnt = 0; - Sint tok_serial = 0; - Eterm seq_trace_token = ERL_MESSAGE_TOKEN(msg); - - dtrace_proc_str(receiver, receiver_name); - if (seq_trace_token != NIL && is_tuple(seq_trace_token)) { - tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token)); - tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token)); - tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token)); - } - DTRACE6(message_queued, - receiver_name, size_object(ERL_MESSAGE_TERM(msg)), - receiver->msg.len, - tok_label, tok_lastcnt, tok_serial); - } -#endif - while (msg) { - trace_receive(receiver, from, ERL_MESSAGE_TERM(msg), te); - msg = msg->next; - } - - } if (locked_msgq) { erts_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); } erts_proc_notify_new_message(receiver, receiver_locks); - return res; } -static Sint -queue_message(Process* receiver, - erts_aint32_t *receiver_state, - ErtsProcLocks receiver_locks, - ErtsMessage* mp, Eterm msg, Eterm from) +static ERTS_INLINE +ErtsMessage* prepend_pending_sig_maybe(Process* sender, Process* receiver, + ErtsMessage* mp) { - ERL_MESSAGE_TERM(mp) = msg; - return queue_messages(receiver, receiver_state, receiver_locks, - mp, &mp->next, 1, from); + ErtsSchedulerData* esdp = sender->scheduler_data; + ErtsSignal* pend_sig; + + if (!esdp || esdp->pending_signal.to != receiver->common.id) + return mp; + + pend_sig = esdp->pending_signal.sig; + + ASSERT(esdp->pending_signal.dbg_from == sender); + esdp->pending_signal.sig = NULL; + esdp->pending_signal.to = THE_NON_VALUE; + pend_sig->common.next = mp; + pend_sig->common.specific.next = NULL; + return (ErtsMessage*) pend_sig; } -Sint +/** + * + * @brief Send one message from *NOT* a local process. + * + */ +void erts_queue_message(Process* receiver, ErtsProcLocks receiver_locks, ErtsMessage* mp, Eterm msg, Eterm from) { - return queue_message(receiver, NULL, receiver_locks, mp, msg, from); + ASSERT(is_not_internal_pid(from)); + ERL_MESSAGE_TERM(mp) = msg; + ERL_MESSAGE_FROM(mp) = from; + queue_messages(receiver, receiver_locks, mp, &mp->next, 1); } +/** + * @brief Send one message from a local process. + */ +void +erts_queue_proc_message(Process* sender, + Process* receiver, ErtsProcLocks receiver_locks, + ErtsMessage* mp, Eterm msg) +{ + ERL_MESSAGE_TERM(mp) = msg; + ERL_MESSAGE_FROM(mp) = sender->common.id; + queue_messages(receiver, receiver_locks, + prepend_pending_sig_maybe(sender, receiver, mp), + &mp->next, 1); +} -Sint -erts_queue_messages(Process* receiver, ErtsProcLocks receiver_locks, - ErtsMessage* first, ErtsMessage** last, Uint len, - Eterm from) + +void +erts_queue_proc_messages(Process* sender, + Process* receiver, ErtsProcLocks receiver_locks, + ErtsMessage* first, ErtsMessage** last, Uint len) { - return queue_messages(receiver, NULL, receiver_locks, - first, last, len, from); + queue_messages(receiver, receiver_locks, + prepend_pending_sig_maybe(sender, receiver, first), + last, len); } void @@ -644,7 +577,7 @@ erts_try_alloc_message_on_heap(Process *pp, * Send a local message when sender & receiver processes are known. */ -Sint +void erts_send_message(Process* sender, Process* receiver, ErtsProcLocks *receiver_locks, @@ -655,7 +588,6 @@ erts_send_message(Process* sender, ErtsMessage* mp; ErlOffHeap *ohp; Eterm token = NIL; - Sint res = 0; #ifdef USE_VM_PROBES DTRACE_CHARBUF(sender_name, 64); DTRACE_CHARBUF(receiver_name, 64); @@ -701,7 +633,8 @@ erts_send_message(Process* sender, seq_trace_update_send(sender); seq_trace_output(stoken, message, SEQ_TRACE_SEND, receiver->common.id, sender); - seq_trace_size = 6; /* TUPLE5 */ + + seq_trace_size = size_object(stoken); } #ifdef USE_VM_PROBES if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) { @@ -750,7 +683,7 @@ erts_send_message(Process* sender, } if (DTRACE_ENABLED(message_send)) { if (have_seqtrace(stoken)) { - tok_label = signed_val(SEQ_TRACE_T_LABEL(stoken)); + tok_label = SEQ_TRACE_T_DTRACE_LABEL(stoken); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(stoken)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(stoken)); } @@ -797,13 +730,8 @@ erts_send_message(Process* sender, #ifdef USE_VM_PROBES ERL_MESSAGE_DT_UTAG(mp) = utag; #endif - res = queue_message(receiver, - &receiver_state, - *receiver_locks, - mp, message, - sender->common.id); - return res; + erts_queue_proc_message(sender, receiver, *receiver_locks, mp, message); } @@ -922,70 +850,77 @@ Sint erts_move_messages_off_heap(Process *c_p) { int reds = 1; + int i; + ErtsMessage *msgq[] = {c_p->sig_qs.first, c_p->sig_qs.cont}; /* * Move all messages off heap. This *only* occurs when the * process had off heap message disabled and just enabled * it... */ - ErtsMessage *mp; - reds += c_p->msg.len / 10; + reds += erts_proc_sig_privqs_len(c_p) / 10; ASSERT(erts_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ); ASSERT(c_p->flags & F_OFF_HEAP_MSGQ_CHNG); - for (mp = c_p->msg.first; mp; mp = mp->next) { - Uint msg_sz, token_sz; + for (i = 0; i < sizeof(msgq)/sizeof(msgq[0]); i++) { + ErtsMessage *mp; + for (mp = msgq[i]; mp; mp = mp->next) { + Uint msg_sz, token_sz; #ifdef USE_VM_PROBES - Uint utag_sz; + Uint utag_sz; #endif - Eterm *hp; - ErlHeapFragment *hfrag; + Eterm *hp; + ErlHeapFragment *hfrag; - if (mp->data.attached) - continue; + if (!ERTS_SIG_IS_INTERNAL_MSG(mp)) + continue; - if (is_immed(ERL_MESSAGE_TERM(mp)) + if (mp->data.attached) + continue; + + if (is_immed(ERL_MESSAGE_TERM(mp)) #ifdef USE_VM_PROBES - && is_immed(ERL_MESSAGE_DT_UTAG(mp)) + && is_immed(ERL_MESSAGE_DT_UTAG(mp)) #endif - && is_not_immed(ERL_MESSAGE_TOKEN(mp))) - continue; + && is_not_immed(ERL_MESSAGE_TOKEN(mp))) + continue; - /* - * The message refers into the heap. Copy the message - * from the heap into a heap fragment and attach - * it to the message... - */ - msg_sz = size_object(ERL_MESSAGE_TERM(mp)); + /* + * The message refers into the heap. Copy the message + * from the heap into a heap fragment and attach + * it to the message... + */ + msg_sz = size_object(ERL_MESSAGE_TERM(mp)); #ifdef USE_VM_PROBES - utag_sz = size_object(ERL_MESSAGE_DT_UTAG(mp)); + utag_sz = size_object(ERL_MESSAGE_DT_UTAG(mp)); #endif - token_sz = size_object(ERL_MESSAGE_TOKEN(mp)); + token_sz = size_object(ERL_MESSAGE_TOKEN(mp)); - hfrag = new_message_buffer(msg_sz + hfrag = new_message_buffer(msg_sz #ifdef USE_VM_PROBES - + utag_sz + + utag_sz #endif - + token_sz); - hp = hfrag->mem; - if (is_not_immed(ERL_MESSAGE_TERM(mp))) - ERL_MESSAGE_TERM(mp) = copy_struct(ERL_MESSAGE_TERM(mp), - msg_sz, &hp, - &hfrag->off_heap); - if (is_not_immed(ERL_MESSAGE_TOKEN(mp))) - ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp), - token_sz, &hp, - &hfrag->off_heap); + + token_sz); + hp = hfrag->mem; + if (is_not_immed(ERL_MESSAGE_TERM(mp))) + ERL_MESSAGE_TERM(mp) = copy_struct(ERL_MESSAGE_TERM(mp), + msg_sz, &hp, + &hfrag->off_heap); + if (is_not_immed(ERL_MESSAGE_TOKEN(mp))) + ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp), + token_sz, &hp, + &hfrag->off_heap); #ifdef USE_VM_PROBES - if (is_not_immed(ERL_MESSAGE_DT_UTAG(mp))) - ERL_MESSAGE_DT_UTAG(mp) = copy_struct(ERL_MESSAGE_DT_UTAG(mp), - utag_sz, &hp, - &hfrag->off_heap); + if (is_not_immed(ERL_MESSAGE_DT_UTAG(mp))) + ERL_MESSAGE_DT_UTAG(mp) = copy_struct(ERL_MESSAGE_DT_UTAG(mp), + utag_sz, &hp, + &hfrag->off_heap); #endif - mp->data.heap_frag = hfrag; - reds += 1; + mp->data.heap_frag = hfrag; + reds += 1; + } } return reds; @@ -1015,7 +950,7 @@ erts_complete_off_heap_message_queue_change(Process *c_p) else { reds += 2; erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ); - ERTS_MSGQ_MV_INQ2PRIVQ(c_p); + erts_proc_sig_fetch(c_p); erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); reds += erts_move_messages_off_heap(c_p); } @@ -1225,139 +1160,6 @@ erts_decode_dist_message(Process *proc, ErtsProcLocks proc_locks, return 1; } -/* - * ERTS_INSPECT_MSGQ_KEEP_OH_MSGS == 0 will move off heap messages - * into the heap of the inspected process if off_heap_message_queue - * is false when process_info(_, messages) is called. That is, the - * following GC will have more data in the rootset compared to the - * scenario when process_info(_, messages) had not been called. - * - * ERTS_INSPECT_MSGQ_KEEP_OH_MSGS != 0 will keep off heap messages - * off heap when process_info(_, messages) is called regardless of - * the off_heap_message_queue setting of the process. That is, it - * will change the following execution of the process as little as - * possible. - */ -#define ERTS_INSPECT_MSGQ_KEEP_OH_MSGS 1 - -Uint -erts_prep_msgq_for_inspection(Process *c_p, Process *rp, - ErtsProcLocks rp_locks, ErtsMessageInfo *mip) -{ - Uint tot_heap_size; - ErtsMessage* mp; - Sint i; - int self_on_heap; - - /* - * Prepare the message queue for inspection - * by process_info(). - * - * - * - Decode all messages on external format - * - Remove all corrupt dist messages from queue - * - Save pointer to, and heap size need of each - * message in the mip array. - * - Return total heap size need for all messages - * that needs to be copied. - * - * If ERTS_INSPECT_MSGQ_KEEP_OH_MSGS == 0: - * - In case off heap messages is disabled and - * we are inspecting our own queue, move all - * off heap data into the heap. - */ - - self_on_heap = c_p == rp && !(c_p->flags & F_OFF_HEAP_MSGQ); - - tot_heap_size = 0; - i = 0; - mp = rp->msg.first; - while (mp) { - Eterm msg = ERL_MESSAGE_TERM(mp); - - mip[i].size = 0; - - if (is_non_value(msg)) { - /* Dist message on external format; decode it... */ - if (mp->data.attached) - erts_decode_dist_message(rp, rp_locks, mp, - ERTS_INSPECT_MSGQ_KEEP_OH_MSGS); - - msg = ERL_MESSAGE_TERM(mp); - - if (is_non_value(msg)) { - ErtsMessage **mpp; - ErtsMessage *bad_mp = mp; - /* - * Bad distribution message; remove - * it from the queue... - */ - ASSERT(!mp->data.attached); - - mpp = i == 0 ? &rp->msg.first : &mip[i-1].msgp->next; - - ASSERT(*mpp == bad_mp); - - erts_msgq_update_internal_pointers(&rp->msg, mpp, &bad_mp->next); - - mp = mp->next; - *mpp = mp; - rp->msg.len--; - bad_mp->next = NULL; - erts_cleanup_messages(bad_mp); - continue; - } - } - - ASSERT(is_value(msg)); - -#if ERTS_INSPECT_MSGQ_KEEP_OH_MSGS - if (is_not_immed(msg) && (!self_on_heap || mp->data.attached)) { - Uint sz = size_object(msg); - mip[i].size = sz; - tot_heap_size += sz; - } -#else - if (self_on_heap) { - if (mp->data.attached) { - ErtsMessage *tmp = NULL; - if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) { - erts_link_mbuf_to_proc(rp, mp->data.heap_frag); - mp->data.attached = NULL; - } - else { - /* - * Need to replace the message reference since - * we will get references to the message data - * from the heap... - */ - ErtsMessage **mpp; - tmp = erts_alloc_message(0, NULL); - sys_memcpy((void *) tmp->m, (void *) mp->m, - sizeof(Eterm)*ERL_MESSAGE_REF_ARRAY_SZ); - mpp = i == 0 ? &rp->msg.first : &mip[i-1].msgp->next; - erts_msgq_replace_msg_ref(&rp->msg, tmp, mpp); - erts_save_message_in_proc(rp, mp); - mp = tmp; - } - } - } - else if (is_not_immed(msg)) { - Uint sz = size_object(msg); - mip[i].size = sz; - tot_heap_size += sz; - } - -#endif - - mip[i].msgp = mp; - i++; - mp = mp->next; - } - - return tot_heap_size; -} - void erts_factory_proc_init(ErtsHeapFactory* factory, Process* p) { @@ -1547,32 +1349,18 @@ void erts_factory_dummy_init(ErtsHeapFactory* factory) factory->mode = FACTORY_CLOSED; } -static void reserve_heap(ErtsHeapFactory*, Uint need, Uint xtra); - -Eterm* erts_produce_heap(ErtsHeapFactory* factory, Uint need, Uint xtra) -{ - Eterm* res; - - ASSERT((unsigned int)factory->mode > (unsigned int)FACTORY_CLOSED); - if (factory->hp + need > factory->hp_end) { - reserve_heap(factory, need, xtra); - } - res = factory->hp; - factory->hp += need; - return res; -} - Eterm* erts_reserve_heap(ErtsHeapFactory* factory, Uint need) { ASSERT((unsigned int)factory->mode > (unsigned int)FACTORY_CLOSED); if (factory->hp + need > factory->hp_end) { - reserve_heap(factory, need, 200); + erts_reserve_heap__(factory, need, 200); } return factory->hp; } -static void reserve_heap(ErtsHeapFactory* factory, Uint need, Uint xtra) +void erts_reserve_heap__(ErtsHeapFactory* factory, Uint need, Uint xtra) { + /* internal... */ ErlHeapFragment* bp; switch (factory->mode) { @@ -1582,7 +1370,9 @@ static void reserve_heap(ErtsHeapFactory* factory, Uint need, Uint xtra) factory->hp_end = factory->hp + need; return; - case FACTORY_MESSAGE: + case FACTORY_MESSAGE: { + int replace_oh; + int replace_msg_hfrag; if (!factory->heap_frags) { ASSERT(factory->message->data.attached == ERTS_MSG_COMBINED_HFRAG); bp = &factory->message->hfrag; @@ -1594,25 +1384,45 @@ static void reserve_heap(ErtsHeapFactory* factory, Uint need, Uint xtra) bp = factory->heap_frags; } + replace_oh = 0; + replace_msg_hfrag = 0; + if (bp) { - ASSERT(factory->hp > bp->mem); + ASSERT(factory->hp >= bp->mem); ASSERT(factory->hp <= factory->hp_end); ASSERT(factory->hp_end == bp->mem + bp->alloc_size); bp->used_size = factory->hp - bp->mem; + if (!bp->used_size && factory->heap_frags) { + factory->heap_frags = bp->next; + bp->next = NULL; + ASSERT(!bp->off_heap.first); + if (factory->off_heap == &bp->off_heap) + replace_oh = !0; + if (factory->message && factory->message->data.heap_frag == bp) + replace_msg_hfrag = !0; + free_message_buffer(bp); + } } bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(factory->alloc_type, ERTS_HEAP_FRAG_SIZE(need+xtra)); bp->next = factory->heap_frags; factory->heap_frags = bp; bp->alloc_size = need + xtra; - bp->used_size = need; + bp->used_size = need + xtra; bp->off_heap.first = NULL; bp->off_heap.overhead = 0; - + if (replace_oh) { + factory->off_heap = &bp->off_heap; + factory->off_heap_saved.first = factory->off_heap->first; + factory->off_heap_saved.overhead = factory->off_heap->overhead; + } + if (replace_msg_hfrag) + factory->message->data.heap_frag = bp; factory->hp = bp->mem; factory->hp_end = bp->mem + bp->alloc_size; return; + } case FACTORY_STATIC: case FACTORY_CLOSED: @@ -1695,9 +1505,11 @@ void erts_factory_trim_and_close(ErtsHeapFactory* factory, if (bp->next == NULL) { Uint used_sz = factory->hp - bp->mem; ASSERT(used_sz <= bp->alloc_size); - if (used_sz > 0) - bp = erts_resize_message_buffer(bp, used_sz, - brefs, brefs_size); + if (used_sz > 0) { + if (used_sz != bp->alloc_size) + bp = erts_resize_message_buffer(bp, used_sz, + brefs, brefs_size); + } else { free_message_buffer(bp); bp = NULL; diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index a14f4f51d8..d120111634 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2016. All Rights Reserved. + * Copyright Ericsson AB 1997-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,11 @@ #ifndef __ERL_MESSAGE_H__ #define __ERL_MESSAGE_H__ +#include "sys.h" +#define ERTS_PROC_SIG_QUEUE_TYPE_ONLY +#include "erl_proc_sig_queue.h" +#undef ERTS_PROC_SIG_QUEUE_TYPE_ONLY + struct proc_bin; struct external_thing_; @@ -84,12 +89,33 @@ void erts_factory_static_init(ErtsHeapFactory*, Eterm* hp, Uint size, ErlOffHeap void erts_factory_tmp_init(ErtsHeapFactory*, Eterm* hp, Uint size, Uint32 atype); void erts_factory_dummy_init(ErtsHeapFactory*); -Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra); +ERTS_GLB_INLINE Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra); + Eterm* erts_reserve_heap(ErtsHeapFactory*, Uint need); void erts_factory_close(ErtsHeapFactory*); void erts_factory_trim_and_close(ErtsHeapFactory*,Eterm *brefs, Uint brefs_size); void erts_factory_undo(ErtsHeapFactory*); +void erts_reserve_heap__(ErtsHeapFactory*, Uint need, Uint xtra); /* internal */ + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE Eterm * +erts_produce_heap(ErtsHeapFactory* factory, Uint need, Uint xtra) +{ + Eterm* res; + + ASSERT((unsigned int)factory->mode > (unsigned int)FACTORY_CLOSED); + if (factory->hp + need > factory->hp_end) { + erts_reserve_heap__(factory, need, xtra); + } + res = factory->hp; + factory->hp += need; + return res; +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + #ifdef CHECK_FOR_HOLES # define ERTS_FACTORY_HOLE_CHECK(f) do { \ /*if ((f)->p) erts_check_for_holes((f)->p);*/ \ @@ -118,15 +144,16 @@ struct erl_heap_fragment { }; /* m[0] = message, m[1] = seq trace token */ -#define ERL_MESSAGE_REF_ARRAY_SZ 2 +#define ERL_MESSAGE_REF_ARRAY_SZ 3 #define ERL_MESSAGE_TERM(mp) ((mp)->m[0]) #define ERL_MESSAGE_TOKEN(mp) ((mp)->m[1]) +#define ERL_MESSAGE_FROM(mp) ((mp)->m[2]) #ifdef USE_VM_PROBES /* m[2] = dynamic trace user tag */ #undef ERL_MESSAGE_REF_ARRAY_SZ -#define ERL_MESSAGE_REF_ARRAY_SZ 3 -#define ERL_MESSAGE_DT_UTAG(mp) ((mp)->m[2]) +#define ERL_MESSAGE_REF_ARRAY_SZ 4 +#define ERL_MESSAGE_DT_UTAG(mp) ((mp)->m[3]) #else #endif @@ -157,28 +184,124 @@ struct erl_mesg { ErlHeapFragment hfrag; }; +/* + * The ErtsMessage struct is only one special type + * of signal. All signal structs have a common + * begining and can be differentiated by looking + * at the ErtsSignal 'common.tag' field. + * + * - An ordinary message will have a value + * - A distribution message that has not been + * decoded yet will have the non-value. + * - Other signals will have an external pid + * header tag. In order to differentiate + * between those signals one needs to look + * at the arity part of the header (see + * erts_proc_sig_queue.h). + */ + +#define ERTS_SIG_IS_NON_MSG_TAG(Tag) \ + (is_external_pid_header((Tag))) + +#define ERTS_SIG_IS_NON_MSG(Sig) \ + ERTS_SIG_IS_NON_MSG_TAG(((ErtsSignal *) (Sig))->common.tag) + +#define ERTS_SIG_IS_INTERNAL_MSG_TAG(Tag) \ + (!is_header((Tag))) +#define ERTS_SIG_IS_INTERNAL_MSG(Sig) \ + ERTS_SIG_IS_INTERNAL_MSG_TAG(((ErtsSignal *) (Sig))->common.tag) + +#define ERTS_SIG_IS_EXTERNAL_MSG_TAG(Tag) \ + ((Tag) == THE_NON_VALUE) +#define ERTS_SIG_IS_EXTERNAL_MSG(Sig) \ + ERTS_SIG_IS_EXTERNAL_MSG_TAG(((ErtsSignal *) (Sig))->common.tag) + +#define ERTS_SIG_IS_MSG_TAG(Tag) \ + (!ERTS_SIG_IS_NON_MSG_TAG(Tag)) +#define ERTS_SIG_IS_MSG(Sig) \ + ERTS_SIG_IS_MSG_TAG(((ErtsSignal *) (Sig))->common.tag) + +typedef union { + ErtsSignalCommon common; + ErtsMessageRef msg; +} ErtsSignal; + +typedef struct { + /* pointers to next pointers pointing to... */ + ErtsMessage **next; /* ... next (non-message) signal */ + ErtsMessage **last; /* ... last (non-message) signal */ +} ErtsMsgQNMSigs; + /* Size of default message buffer (erl_message.c) */ #define ERL_MESSAGE_BUF_SZ 500 typedef struct { - ErtsMessage* first; - ErtsMessage** last; /* point to the last next pointer */ - ErtsMessage** save; - Sint len; /* queue length */ - /* - * The following field is used by the recv_mark/1 and - * recv_set/1 instructions. + * ** The signal queues private to a process. ** + * + * These are: + * - an inner queue which only consists of + * message signals + * - a middle queue which contains a mixture + * of message and non-message signals + * + * When the process isn't processing signals in + * erts_proc_sig_handle_incoming(): + * - the message queue corresponds to the inner + * queue. Messages in the middle queue (and + * in the outer queue) are in transit and + * have NOT been received yet! + * + * When the process is processing signals in + * erts_proc_sig_handle_incoming(): + * - the message queue corresponds to the inner + * queue plus the head of the middle queue up + * to the signal currently being processed. + * Any messages further back in the middle queue + * (and in the outer queue) are still in transit + * and have NOT been received yet! + * + * In the general case the 'len' field of this + * structure does NOT correspond to the message + * queue length. When the process is inspected + * via process info it does however correspond + * to the message queue length, but this is a + * special case! + * + * When no process-info request is in transit to + * the process the 'len' field corresponds to + * the total amount of messages in inner and + * middle queues (which does NOT correspond to + * the message queue length). When process-info + * requests are in transit to the process, the + * usage of the 'len' field changes and is used + * as an offset which even might be negative. */ - ErtsMessage** saved_last; /* saved last pointer */ -} ErlMessageQueue; + /* inner queue */ + ErtsMessage *first; + ErtsMessage **last; /* point to the last next pointer */ + ErtsMessage **save; + + /* middle queue */ + ErtsMessage *cont; + ErtsMessage **cont_last; + ErtsMsgQNMSigs nmsigs; + + /* Common for inner and middle queue */ + ErtsMessage **saved_last; /* saved last pointer */ + Sint len; /* NOT message queue length (see above) */ +} ErtsSignalPrivQueues; typedef struct { ErtsMessage* first; ErtsMessage** last; /* point to the last next pointer */ - Sint len; /* queue length */ -} ErlMessageInQueue; + Sint len; /* number of messages in queue */ + ErtsMsgQNMSigs nmsigs; +#ifdef ERTS_PROC_SIG_HARD_DEBUG + int may_contain_heap_terms; +#endif +} ErtsSignalInQueue; typedef struct erl_trace_message_queue__ { struct erl_trace_message_queue__ *next; /* point to the next receiver */ @@ -188,9 +311,45 @@ typedef struct erl_trace_message_queue__ { Sint len; /* queue length */ } ErlTraceMessageQueue; +#define ERTS_RECV_MARK_SAVE(P) \ + do { \ + erts_proc_lock((P), ERTS_PROC_LOCK_MSGQ); \ + erts_proc_sig_fetch((P)); \ + erts_proc_unlock((P), ERTS_PROC_LOCK_MSGQ); \ + if ((P)->sig_qs.cont) { \ + (P)->sig_qs.saved_last = (P)->sig_qs.cont_last; \ + (P)->flags |= F_DEFERRED_SAVED_LAST; \ + } \ + else { \ + (P)->sig_qs.saved_last = (P)->sig_qs.last; \ + (P)->flags &= ~F_DEFERRED_SAVED_LAST; \ + } \ + } while (0) + +#define ERTS_RECV_MARK_SET(P) \ + do { \ + if ((P)->sig_qs.saved_last) { \ + if ((P)->flags & F_DEFERRED_SAVED_LAST) { \ + /* Points to middle queue; use end of inner */ \ + (P)->sig_qs.save = (P)->sig_qs.last; \ + ASSERT(!PEEK_MESSAGE((P))); \ + } \ + else { \ + /* Points to inner queue; safe to use */ \ + (P)->sig_qs.save = (P)->sig_qs.saved_last; \ + } \ + } \ + } while (0) + +#define ERTS_RECV_MARK_CLEAR(P) \ + do { \ + (P)->sig_qs.saved_last = NULL; \ + (P)->flags &= ~F_DEFERRED_SAVED_LAST; \ + } while (0) + /* Get "current" message */ -#define PEEK_MESSAGE(p) (*(p)->msg.save) +#define PEEK_MESSAGE(p) (*(p)->sig_qs.save) #ifdef USE_VM_PROBES #define LINK_MESSAGE_DTAG(mp, dt) ERL_MESSAGE_DT_UTAG(mp) = dt @@ -198,58 +357,53 @@ typedef struct erl_trace_message_queue__ { #define LINK_MESSAGE_DTAG(mp, dt) #endif -#define LINK_MESSAGE_IMPL(p, first_msg, last_msg, num_msgs, where) do { \ - *(p)->where.last = (first_msg); \ - (p)->where.last = (last_msg); \ - (p)->where.len += (num_msgs); \ - } while(0) +#ifdef USE_VM_PROBES +# define ERTS_MSG_RECV_TRACED(P) \ + ((ERTS_TRACE_FLAGS((P)) & F_TRACE_RECEIVE) \ + || DTRACE_ENABLED(message_queued)) +#else +# define ERTS_MSG_RECV_TRACED(P) \ + (ERTS_TRACE_FLAGS((P)) & F_TRACE_RECEIVE) +#endif -/* Add message last in private message queue */ -#define LINK_MESSAGE_PRIVQ(p, first_msg, last_msg, len) \ +/* Add one message last in message queue */ +#define LINK_MESSAGE(p, msg) \ do { \ - LINK_MESSAGE_IMPL(p, first_msg, last_msg, len, msg); \ - } while (0) - -/* Add message last_msg in message queue */ -#define LINK_MESSAGE(p, first_msg, last_msg, len) \ - LINK_MESSAGE_IMPL(p, first_msg, last_msg, len, msg_inq) - -#define ERTS_MSGQ_MV_INQ2PRIVQ(p) \ - do { \ - if (p->msg_inq.first) { \ - *p->msg.last = p->msg_inq.first; \ - p->msg.last = p->msg_inq.last; \ - p->msg.len += p->msg_inq.len; \ - p->msg_inq.first = NULL; \ - p->msg_inq.last = &p->msg_inq.first; \ - p->msg_inq.len = 0; \ - } \ - } while (0) - + ASSERT(ERTS_SIG_IS_MSG(msg)); \ + ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE__((p), "before"); \ + *(p)->sig_inq.last = (msg); \ + (p)->sig_inq.last = &(msg)->next; \ + (p)->sig_inq.len++; \ + ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE__((p), "before"); \ + } while(0) /* Unlink current message */ -#define UNLINK_MESSAGE(p,msgp) do { \ - ErtsMessage* __mp = (msgp)->next; \ - *(p)->msg.save = __mp; \ - (p)->msg.len--; \ - if (__mp == NULL) \ - (p)->msg.last = (p)->msg.save; \ -} while(0) +#define UNLINK_MESSAGE(p,msgp) \ + do { \ + ErtsMessage *mp__ = (msgp)->next; \ + ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__((p), 0, "before"); \ + *(p)->sig_qs.save = mp__; \ + (p)->sig_qs.len--; \ + if (mp__ == NULL) \ + (p)->sig_qs.last = (p)->sig_qs.save; \ + ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__((p), 0, "after"); \ + } while(0) /* * Reset message save point (after receive match). * Also invalidate the saved position since it may no * longer be safe to use. */ -#define JOIN_MESSAGE(p) do { \ - (p)->msg.save = &(p)->msg.first; \ - (p)->msg.saved_last = 0; \ -} while(0) +#define JOIN_MESSAGE(p) \ + do { \ + (p)->sig_qs.save = &(p)->sig_qs.first; \ + ERTS_RECV_MARK_CLEAR((p)); \ + } while(0) /* Save current message */ #define SAVE_MESSAGE(p) \ - (p)->msg.save = &(*(p)->msg.save)->next + (p)->sig_qs.save = &(*(p)->sig_qs.save)->next #define ERTS_SND_FLG_NO_SEQ_TRACE (((unsigned) 1) << 0) @@ -276,6 +430,7 @@ typedef struct erl_trace_message_queue__ { (MP)->next = NULL; \ ERL_MESSAGE_TERM(MP) = THE_NON_VALUE; \ ERL_MESSAGE_TOKEN(MP) = NIL; \ + ERL_MESSAGE_FROM(MP) = NIL; \ ERL_MESSAGE_DT_UTAG_INIT(MP); \ MP->data.attached = NULL; \ } while (0) @@ -286,11 +441,12 @@ ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint, Eterm *, Uint); void free_message_buffer(ErlHeapFragment *); void erts_queue_dist_message(Process*, ErtsProcLocks, ErtsDistExternal *, Eterm, Eterm); -Sint erts_queue_message(Process*, ErtsProcLocks,ErtsMessage*, Eterm, Eterm); -Sint erts_queue_messages(Process*, ErtsProcLocks, - ErtsMessage*, ErtsMessage**, Uint, Eterm); +void erts_queue_message(Process*, ErtsProcLocks,ErtsMessage*, Eterm, Eterm); +void erts_queue_proc_message(Process* from,Process* to, ErtsProcLocks,ErtsMessage*, Eterm); +void erts_queue_proc_messages(Process* from, Process* to, ErtsProcLocks, + ErtsMessage*, ErtsMessage**, Uint); void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm); -Sint erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned); +void erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned); void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp); Uint erts_msg_attached_data_size_aux(ErtsMessage *msg); @@ -305,16 +461,6 @@ int erts_decode_dist_message(Process *, ErtsProcLocks, ErtsMessage *, int); void erts_cleanup_messages(ErtsMessage *mp); -typedef struct { - Uint size; - ErtsMessage *msgp; -} ErtsMessageInfo; - -Uint erts_prep_msgq_for_inspection(Process *c_p, - Process *rp, - ErtsProcLocks rp_locks, - ErtsMessageInfo *mip); - void *erts_alloc_message_ref(void); void erts_free_message_ref(void *); @@ -356,12 +502,6 @@ ERTS_GLB_FORCE_INLINE ErtsMessage *erts_shrink_message(ErtsMessage *mp, Uint sz, ERTS_GLB_FORCE_INLINE void erts_free_message(ErtsMessage *mp); ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment*); ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg); -ERTS_GLB_INLINE void erts_msgq_update_internal_pointers(ErlMessageQueue *msgq, - ErtsMessage **newpp, - ErtsMessage **oldpp); -ERTS_GLB_INLINE void erts_msgq_replace_msg_ref(ErlMessageQueue *msgq, - ErtsMessage *newp, - ErtsMessage **oldpp); #define ERTS_MSG_COMBINED_HFRAG ((void *) 0x1) @@ -465,28 +605,6 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg) } } -ERTS_GLB_INLINE void -erts_msgq_update_internal_pointers(ErlMessageQueue *msgq, - ErtsMessage **newpp, - ErtsMessage **oldpp) -{ - if (msgq->save == oldpp) - msgq->save = newpp; - if (msgq->last == oldpp) - msgq->last = newpp; - if (msgq->saved_last == oldpp) - msgq->saved_last = newpp; -} - -ERTS_GLB_INLINE void -erts_msgq_replace_msg_ref(ErlMessageQueue *msgq, ErtsMessage *newp, ErtsMessage **oldpp) -{ - ErtsMessage *oldp = *oldpp; - newp->next = oldp->next; - erts_msgq_update_internal_pointers(msgq, &newp->next, &oldp->next); - *oldpp = newp; -} - #endif Uint erts_mbuf_size(Process *p); @@ -500,4 +618,17 @@ Uint erts_mbuf_size(Process *p); # define ERTS_CHK_MBUF_SZ(P) ((void) 1) #endif +#define ERTS_FOREACH_SIG_PRIVQS(PROC, MVAR, CODE) \ + do { \ + int i__; \ + ErtsMessage *msgs__[] = {(PROC)->sig_qs.first, \ + (PROC)->sig_qs.cont}; \ + for (i__ = 0; i__ < sizeof(msgs__)/sizeof(msgs__[0]); i__++) { \ + ErtsMessage *MVAR; \ + for (MVAR = msgs__[i__]; MVAR; MVAR = MVAR->next) { \ + CODE; \ + } \ + } \ + } while (0) + #endif diff --git a/erts/emulator/beam/erl_monitor_link.c b/erts/emulator/beam/erl_monitor_link.c new file mode 100644 index 0000000000..70f36fb6b7 --- /dev/null +++ b/erts/emulator/beam/erl_monitor_link.c @@ -0,0 +1,1341 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Monitor and link implementation. + * + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include <stddef.h> +#include "global.h" +#include "erl_node_tables.h" +#include "erl_monitor_link.h" + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Red-black tree implementation used for monitors and links. * +\* */ + +static ERTS_INLINE Eterm +ml_get_key(ErtsMonLnkNode *mln) +{ + char *ptr = (char *) mln; + ptr += mln->key_offset; + +#ifdef ERTS_ML_DEBUG + switch (mln->type) { + case ERTS_MON_TYPE_PROC: + case ERTS_MON_TYPE_PORT: + case ERTS_MON_TYPE_DIST_PROC: + case ERTS_MON_TYPE_TIME_OFFSET: + case ERTS_MON_TYPE_RESOURCE: { + ErtsMonitorData *mdp = erts_monitor_to_data(mln); + ERTS_ML_ASSERT(&mdp->ref == (Eterm *) ptr); + break; + } + case ERTS_LNK_TYPE_PROC: + case ERTS_LNK_TYPE_DIST_PROC: + case ERTS_LNK_TYPE_PORT: + case ERTS_MON_TYPE_NODE: + case ERTS_MON_TYPE_NODES: + case ERTS_MON_TYPE_SUSPEND: + ERTS_ML_ASSERT(&mln->other.item == (Eterm *) ptr); + break; + default: + ERTS_INTERNAL_ERROR("Invalid type"); + break; + } +#endif + + return *((Eterm *) ptr); +} + +/* + * Comparison functions for valid *and only valid* keys. That + * is, if the set of valid keys is changed, this function needs + * to be updated... + * + * Note that this function does *not* order terms according to + * term order, and that the order may vary between emulator + * restarts... + */ +static int +ml_cmp_keys(Eterm key1, Eterm key2) +{ + /* + * A monitor key is either an internal pid, a (nodename) atom, + * a small (monitor nodes bitmask), an ordinary internal ref, + * or an external ref. + * + * Order between monitors with keys of different types: + * internal pid < atom < small < internal ref < external ref + * + * A link key is either a pid, an internal port + * + * Order between links with keys of different types: + * internal pid < internal port < external pid + * + */ + ERTS_ML_ASSERT(is_internal_pid(key1) + || is_internal_port(key1) + || is_atom(key1) + || is_small(key1) + || is_external_pid(key1) + || is_internal_ordinary_ref(key1) + || is_external_ref(key1)); + + ERTS_ML_ASSERT(is_internal_pid(key2) + || is_internal_port(key2) + || is_atom(key2) + || is_small(key2) + || is_external_pid(key2) + || is_internal_ordinary_ref(key2) + || is_external_ref(key2)); + + if (is_immed(key1)) { + int key1_tag, key2_tag; + + ERTS_ML_ASSERT(is_internal_pid(key1) + || is_internal_port(key1) + || is_atom(key1) + || is_small(key1)); + + if (key1 == key2) + return 0; + + if (is_boxed(key2)) + return -1; + + ERTS_ML_ASSERT(is_internal_pid(key2) + || is_internal_port(key2) + || is_atom(key2) + || is_small(key2)); + + key1_tag = (int) (key1 & _TAG_IMMED1_MASK); + key2_tag = (int) (key2 & _TAG_IMMED1_MASK); + + if (key1_tag != key2_tag) + return key1_tag - key2_tag; + + ASSERT((is_atom(key1) && is_atom(key2)) + || (is_small(key1) && is_small(key2)) + || (is_internal_pid(key1) && is_internal_pid(key2)) + || (is_internal_port(key1) && is_internal_port(key2))); + + return key1 < key2 ? -1 : 1; + } + else { + Eterm *w1, hdr1; + + ERTS_ML_ASSERT(is_boxed(key1)); + + w1 = boxed_val(key1); + hdr1 = *w1; + + if ((hdr1 & _TAG_HEADER_MASK) == _TAG_HEADER_REF) { + Eterm *w2; + + if (!is_internal_ref(key2)) + return is_immed(key2) ? 1 : -1; + + w2 = internal_ref_val(key2); + + ERTS_ML_ASSERT(w1[0] == ERTS_REF_THING_HEADER); + ERTS_ML_ASSERT(w2[0] == ERTS_REF_THING_HEADER); + + return sys_memcmp((void *) &w1[1], (void *) &w2[1], + (ERTS_REF_THING_SIZE - 1)*sizeof(Eterm)); + } + + ERTS_ML_ASSERT(is_external(key1)); + + if (is_not_external(key2)) + return 1; + else { + Uint ndw1, ndw2; + ExternalThing *et1, *et2; + ErlNode *n1, *n2; + + ERTS_ML_ASSERT((is_external_ref(key1) && is_external_ref(key2)) + || (is_external_pid(key1) && is_external_pid(key2))); + + et1 = (ExternalThing *) w1; + et2 = (ExternalThing *) external_val(key2); + + n1 = et1->node; + n2 = et2->node; + + if (n1 != n2) { + if (n1->sysname != n2->sysname) + return n1->sysname < n2->sysname ? -1 : 1; + ASSERT(n1->creation != n2->creation); + return n1->creation < n2->creation ? -1 : 1; + } + + ndw1 = external_thing_data_words(et1); + ndw2 = external_thing_data_words(et1); + if (ndw1 != ndw2) + return ndw1 < ndw2 ? -1 : 1; + + return sys_memcmp((void *) &et1->data.ui[0], + (void *) &et2->data.ui[0], + ndw1*sizeof(Eterm)); + } + } +} + +#define ERTS_ML_TPFLG_RED (((UWord) 1) << 0) + +#define ERTS_ML_TPFLGS_MASK (ERTS_ML_TPFLG_RED) + +#define ERTS_RBT_PREFIX mon_lnk +#define ERTS_RBT_T ErtsMonLnkNode +#define ERTS_RBT_KEY_T Eterm +#define ERTS_RBT_FLAGS_T UWord +#define ERTS_RBT_INIT_EMPTY_TNODE(T) \ + do { \ + (T)->node.tree.parent = (UWord) NULL; \ + (T)->node.tree.right = NULL; \ + (T)->node.tree.left = NULL; \ + } while (0) +#define ERTS_RBT_IS_RED(T) \ + (!!((T)->node.tree.parent & ERTS_ML_TPFLG_RED)) +#define ERTS_RBT_SET_RED(T) \ + ((T)->node.tree.parent |= ERTS_ML_TPFLG_RED) +#define ERTS_RBT_IS_BLACK(T) \ + (!ERTS_RBT_IS_RED((T))) +#define ERTS_RBT_SET_BLACK(T) \ + ((T)->node.tree.parent &= ~ERTS_ML_TPFLG_RED) +#define ERTS_RBT_GET_FLAGS(T) \ + ((T)->node.tree.parent & ERTS_ML_TPFLGS_MASK) +#define ERTS_RBT_SET_FLAGS(T, F) \ + do { \ + ERTS_ML_ASSERT((((UWord) (F)) & ~ERTS_ML_TPFLGS_MASK) == 0); \ + (T)->node.tree.parent &= ~ERTS_ML_TPFLGS_MASK; \ + (T)->node.tree.parent |= (F); \ + } while (0) +#define ERTS_RBT_GET_PARENT(T) \ + ((ERTS_RBT_T *) ((T)->node.tree.parent & ~ERTS_ML_TPFLGS_MASK)) +#define ERTS_RBT_SET_PARENT(T, P) \ + do { \ + ERTS_ML_ASSERT((((UWord) (P)) & ERTS_ML_TPFLGS_MASK) == 0); \ + (T)->node.tree.parent &= ERTS_ML_TPFLGS_MASK; \ + (T)->node.tree.parent |= (UWord) (P); \ + } while (0) +#define ERTS_RBT_GET_RIGHT(T) ((T)->node.tree.right) +#define ERTS_RBT_SET_RIGHT(T, R) ((T)->node.tree.right = (R)) +#define ERTS_RBT_GET_LEFT(T) ((T)->node.tree.left) +#define ERTS_RBT_SET_LEFT(T, L) ((T)->node.tree.left = (L)) +#define ERTS_RBT_GET_KEY(T) (ml_get_key((T))) +#define ERTS_RBT_CMP_KEYS(KX, KY) (ml_cmp_keys((KX), (KY))) +#define ERTS_RBT_WANT_DELETE +#define ERTS_RBT_WANT_LOOKUP +#define ERTS_RBT_WANT_LOOKUP_INSERT +#define ERTS_RBT_WANT_LOOKUP_CREATE +#define ERTS_RBT_WANT_INSERT +#define ERTS_RBT_WANT_REPLACE +#define ERTS_RBT_WANT_FOREACH +#define ERTS_RBT_WANT_FOREACH_YIELDING +#define ERTS_RBT_WANT_FOREACH_DESTROY +#define ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING +#define ERTS_RBT_UNDEF + +#include "erl_rbtree.h" + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Tree Operations * +\* */ + +static ErtsMonLnkNode * +ml_rbt_lookup(ErtsMonLnkNode *root, Eterm ref) +{ + ErtsMonLnkNode *ml = mon_lnk_rbt_lookup(root, ref); + ASSERT(!ml || (ml->flags & ERTS_ML_FLG_IN_TABLE)); + return ml; +} + +static ErtsMonLnkNode * +ml_rbt_lookup_insert(ErtsMonLnkNode **root, ErtsMonLnkNode *ml) +{ + ErtsMonLnkNode *res; + ERTS_ML_ASSERT(!(ml->flags & ERTS_ML_FLG_IN_TABLE)); + res = mon_lnk_rbt_lookup_insert(root, ml); + if (!res) + ml->flags |= ERTS_ML_FLG_IN_TABLE; + return res; +} + +static ErtsMonLnkNode * +ml_rbt_lookup_create(ErtsMonLnkNode ** root, Eterm key, + ErtsMonLnkNode *(*create)(Eterm, void *), + void *carg, int *created) +{ + ErtsMonLnkNode *ml; + ml = mon_lnk_rbt_lookup_create(root, key, create, carg, created); + if (*created) + ml->flags |= ERTS_ML_FLG_IN_TABLE; + return ml; +} + +static void +ml_rbt_insert(ErtsMonLnkNode **root, ErtsMonLnkNode *ml) +{ +#ifdef ERTS_ML_DEBUG + ErtsMonLnkNode *res; + ERTS_ML_ASSERT(!(ml->flags & ERTS_ML_FLG_IN_TABLE)); + res = ml_rbt_lookup(*root, ml_get_key(ml)); + ERTS_ML_ASSERT(res == NULL); +#endif + + mon_lnk_rbt_insert(root, ml); + ml->flags |= ERTS_ML_FLG_IN_TABLE; +} + +static void +ml_rbt_replace(ErtsMonLnkNode **root, ErtsMonLnkNode *old, ErtsMonLnkNode *new) +{ + ERTS_ML_ASSERT(old->flags & ERTS_ML_FLG_IN_TABLE); + ERTS_ML_ASSERT(!(new->flags & ERTS_ML_FLG_IN_TABLE)); + + mon_lnk_rbt_replace(root, old, new); + old->flags &= ~ERTS_ML_FLG_IN_TABLE; + new->flags |= ERTS_ML_FLG_IN_TABLE; +} + +static void +ml_rbt_delete(ErtsMonLnkNode **root, ErtsMonLnkNode *ml) +{ + ERTS_ML_ASSERT(ml->flags & ERTS_ML_FLG_IN_TABLE); + mon_lnk_rbt_delete(root, ml); + ml->flags &= ~ERTS_ML_FLG_IN_TABLE; +} + + +static void +ml_rbt_foreach(ErtsMonLnkNode *root, + void (*func)(ErtsMonLnkNode *, void *), + void *arg) +{ + mon_lnk_rbt_foreach(root, func, arg); +} + +typedef struct { + ErtsMonLnkNode *root; + mon_lnk_rbt_yield_state_t rbt_ystate; +} ErtsMonLnkYieldState; + +static int +ml_rbt_foreach_yielding(ErtsMonLnkNode *root, + void (*func)(ErtsMonLnkNode *, void *), + void *arg, + void **vyspp, + Sint limit) +{ + int res; + ErtsMonLnkYieldState ys = {root, ERTS_RBT_YIELD_STAT_INITER}; + ErtsMonLnkYieldState *ysp; + + ysp = (ErtsMonLnkYieldState *) *vyspp; + if (!ysp) + ysp = &ys; + res = mon_lnk_rbt_foreach_yielding(ysp->root, func, arg, + &ysp->rbt_ystate, limit); + if (res == 0) { + if (ysp != &ys) + erts_free(ERTS_ALC_T_ML_YIELD_STATE, ysp); + *vyspp = NULL; + } + else { + + if (ysp == &ys) { + ysp = erts_alloc(ERTS_ALC_T_ML_YIELD_STATE, + sizeof(ErtsMonLnkYieldState)); + sys_memcpy((void *) ysp, (void *) &ys, + sizeof(ErtsMonLnkYieldState)); + } + + *vyspp = (void *) ysp; + } + + return res; +} + +typedef struct { + void (*func)(ErtsMonLnkNode *, void *); + void *arg; +} ErtsMonLnkForeachDeleteContext; + +static void +rbt_wrap_foreach_delete(ErtsMonLnkNode *ml, void *vctxt) +{ + ErtsMonLnkForeachDeleteContext *ctxt = vctxt; + ERTS_ML_ASSERT(ml->flags & ERTS_ML_FLG_IN_TABLE); + ml->flags &= ~ERTS_ML_FLG_IN_TABLE; + ctxt->func(ml, ctxt->arg); +} + +static void +ml_rbt_foreach_delete(ErtsMonLnkNode **root, + void (*func)(ErtsMonLnkNode *, void *), + void *arg) +{ + ErtsMonLnkForeachDeleteContext ctxt; + ctxt.func = func; + ctxt.arg = arg; + mon_lnk_rbt_foreach_destroy(root, + rbt_wrap_foreach_delete, + (void *) &ctxt); +} + +static int +ml_rbt_foreach_delete_yielding(ErtsMonLnkNode **root, + void (*func)(ErtsMonLnkNode *, void *), + void *arg, + void **vyspp, + Sint limit) +{ + int res; + ErtsMonLnkYieldState ys = {*root, ERTS_RBT_YIELD_STAT_INITER}; + ErtsMonLnkYieldState *ysp; + ErtsMonLnkForeachDeleteContext ctxt; + ctxt.func = func; + ctxt.arg = arg; + + ysp = (ErtsMonLnkYieldState *) *vyspp; + if (!ysp) { + *root = NULL; + ysp = &ys; + } + res = mon_lnk_rbt_foreach_destroy_yielding(&ysp->root, + rbt_wrap_foreach_delete, + (void *) &ctxt, + &ysp->rbt_ystate, + limit); + if (res == 0) { + if (ysp != &ys) + erts_free(ERTS_ALC_T_ML_YIELD_STATE, ysp); + *vyspp = NULL; + } + else { + + if (ysp == &ys) { + ysp = erts_alloc(ERTS_ALC_T_ML_YIELD_STATE, + sizeof(ErtsMonLnkYieldState)); + sys_memcpy((void *) ysp, (void *) &ys, + sizeof(ErtsMonLnkYieldState)); + } + + *vyspp = (void *) ysp; + } + + return res; +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * List Operations * +\* */ + +static int +ml_dl_list_foreach_yielding(ErtsMonLnkNode *list, + void (*func)(ErtsMonLnkNode *, void *), + void *arg, + void **vyspp, + Sint limit) +{ + Sint cnt = 0; + ErtsMonLnkNode *ml = (ErtsMonLnkNode *) *vyspp; + + ERTS_ML_ASSERT(!ml || list); + + if (!ml) + ml = list; + + if (ml) { + do { + ERTS_ML_ASSERT(ml->flags & ERTS_ML_FLG_IN_TABLE); + func(ml, arg); + ml = ml->node.list.next; + cnt++; + } while (ml != list && cnt < limit); + if (ml != list) { + *vyspp = (void *) ml; + return 1; /* yield */ + } + } + + *vyspp = NULL; + return 0; /* done */ +} + +static int +ml_dl_list_foreach_delete_yielding(ErtsMonLnkNode **list, + void (*func)(ErtsMonLnkNode *, void *), + void *arg, + void **vyspp, + Sint limit) +{ + Sint cnt = 0; + ErtsMonLnkNode *first = *list; + ErtsMonLnkNode *ml = (ErtsMonLnkNode *) *vyspp; + + ERTS_ML_ASSERT(!ml || first); + + if (!ml) + ml = first; + + if (ml) { + do { + ErtsMonLnkNode *next = ml->node.list.next; + ERTS_ML_ASSERT(ml->flags & ERTS_ML_FLG_IN_TABLE); + ml->flags &= ~ERTS_ML_FLG_IN_TABLE; + func(ml, arg); + ml = next; + cnt++; + } while (ml != first && cnt < limit); + if (ml != first) { + *vyspp = (void *) ml; + return 1; /* yield */ + } + } + + *vyspp = NULL; + *list = NULL; + return 0; /* done */ +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Misc * +\* */ + +ErtsMonLnkDist * +erts_mon_link_dist_create(Eterm nodename) +{ + ErtsMonLnkDist *mld = erts_alloc(ERTS_ALC_T_ML_DIST, + sizeof(ErtsMonLnkDist)); + mld->nodename = nodename; + mld->connection_id = ~((Uint32) 0); + erts_atomic_init_nob(&mld->refc, 1); + erts_mtx_init(&mld->mtx, "dist_entry_links", nodename, + ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); + mld->alive = !0; + mld->links = NULL; + mld->monitors = NULL; + mld->orig_name_monitors = NULL; + return mld; +} + +void +erts_mon_link_dist_destroy__(ErtsMonLnkDist *mld) +{ + ERTS_ML_ASSERT(erts_atomic_read_nob(&mld->refc) == 0); + ERTS_ML_ASSERT(!mld->alive); + ERTS_ML_ASSERT(!mld->links); + ERTS_ML_ASSERT(!mld->monitors); + ERTS_ML_ASSERT(!mld->orig_name_monitors); + + erts_mtx_destroy(&mld->mtx); + erts_free(ERTS_ALC_T_ML_DIST, mld); +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Monitor Operations * +\* */ + +#ifdef ERTS_ML_DEBUG +size_t erts_monitor_origin_offset; +size_t erts_monitor_origin_key_offset; +size_t erts_monitor_target_offset; +size_t erts_monitor_target_key_offset; +size_t erts_monitor_node_key_offset; +#endif + +static ERTS_INLINE void +monitor_init(void) +{ +#ifdef ERTS_ML_DEBUG + erts_monitor_origin_offset = offsetof(ErtsMonitorData, origin); + erts_monitor_origin_key_offset = offsetof(ErtsMonitorData, ref); + ASSERT(erts_monitor_origin_key_offset >= erts_monitor_origin_offset); + erts_monitor_origin_key_offset -= erts_monitor_origin_offset; + erts_monitor_target_offset = offsetof(ErtsMonitorData, target); + erts_monitor_target_key_offset = offsetof(ErtsMonitorData, ref); + ASSERT(erts_monitor_target_key_offset >= erts_monitor_target_offset); + erts_monitor_target_key_offset -= erts_monitor_target_offset; + erts_monitor_node_key_offset = offsetof(ErtsMonitor, other.item); +#endif +} + +ErtsMonitor * +erts_monitor_tree_lookup(ErtsMonitor *root, Eterm key) +{ + ERTS_ML_ASSERT(is_internal_ordinary_ref(key) + || is_external_ref(key) + || is_atom(key) + || is_small(key) + || is_internal_pid(key)); + return (ErtsMonitor *) ml_rbt_lookup((ErtsMonLnkNode *) root, key); +} + +ErtsMonitor * +erts_monotor_tree_lookup_insert(ErtsMonitor **root, ErtsMonitor *mon) +{ + return (ErtsMonitor *) ml_rbt_lookup_insert((ErtsMonLnkNode **) root, + (ErtsMonLnkNode *) mon); +} + +typedef struct { + Uint16 type; + Eterm origin; +} ErtsMonitorCreateCtxt; + +static ErtsMonLnkNode * +create_monitor(Eterm target, void *vcctxt) +{ + ErtsMonitorCreateCtxt *cctxt = vcctxt; + ErtsMonitorData *mdp = erts_monitor_create(cctxt->type, + NIL, + cctxt->origin, + target, + NIL); + ERTS_ML_ASSERT(ml_cmp_keys(ml_get_key(&mdp->origin), target) == 0); + return (ErtsMonLnkNode *) &mdp->origin; +} + +ErtsMonitor * +erts_monitor_tree_lookup_create(ErtsMonitor **root, int *created, Uint16 type, + Eterm origin, Eterm target) +{ + ErtsMonitor *res; + ErtsMonitorCreateCtxt cctxt = {type, origin}; + + ERTS_ML_ASSERT(type == ERTS_MON_TYPE_NODE || type == ERTS_MON_TYPE_NODES); + + res = (ErtsMonitor *) ml_rbt_lookup_create((ErtsMonLnkNode **) root, + target, create_monitor, + (void *) &cctxt, + created); + + ERTS_ML_ASSERT(res && erts_monitor_is_origin(res)); + + return res; +} + +void +erts_monitor_tree_insert(ErtsMonitor **root, ErtsMonitor *mon) +{ + ml_rbt_insert((ErtsMonLnkNode **) root, (ErtsMonLnkNode *) mon); +} + +void +erts_monitor_tree_replace(ErtsMonitor **root, ErtsMonitor *old, ErtsMonitor *new) +{ + ml_rbt_replace((ErtsMonLnkNode **) root, + (ErtsMonLnkNode *) old, + (ErtsMonLnkNode *) new); +} + +void +erts_monitor_tree_delete(ErtsMonitor **root, ErtsMonitor *mon) +{ + ml_rbt_delete((ErtsMonLnkNode **) root, (ErtsMonLnkNode *) mon); +} + +void +erts_monitor_tree_foreach(ErtsMonitor *root, + void (*func)(ErtsMonitor *, void *), + void *arg) +{ + ml_rbt_foreach((ErtsMonLnkNode *) root, + (void (*)(ErtsMonLnkNode*, void*)) func, + arg); +} + +int +erts_monitor_tree_foreach_yielding(ErtsMonitor *root, + void (*func)(ErtsMonitor *, void *), + void *arg, + void **vyspp, + Sint limit) +{ + return ml_rbt_foreach_yielding((ErtsMonLnkNode *) root, + (void (*)(ErtsMonLnkNode*, void*)) func, + arg, vyspp, limit); +} + +void +erts_monitor_tree_foreach_delete(ErtsMonitor **root, + void (*func)(ErtsMonitor *, void *), + void *arg) +{ + ml_rbt_foreach_delete((ErtsMonLnkNode **) root, + (void (*)(ErtsMonLnkNode*, void*)) func, + arg); +} + +int +erts_monitor_tree_foreach_delete_yielding(ErtsMonitor **root, + void (*func)(ErtsMonitor *, void *), + void *arg, + void **vyspp, + Sint limit) +{ + return ml_rbt_foreach_delete_yielding((ErtsMonLnkNode **) root, + (void (*)(ErtsMonLnkNode*, void*)) func, + arg, vyspp, limit); +} + +void +erts_monitor_list_foreach(ErtsMonitor *list, + void (*func)(ErtsMonitor *, void *), + void *arg) +{ + void *ystate = NULL; + while (ml_dl_list_foreach_yielding((ErtsMonLnkNode *) list, + (void (*)(ErtsMonLnkNode *, void *)) func, + arg, &ystate, (Sint) INT_MAX)); +} + +int +erts_monitor_list_foreach_yielding(ErtsMonitor *list, + void (*func)(ErtsMonitor *, void *), + void *arg, + void **vyspp, + Sint limit) +{ + return ml_dl_list_foreach_yielding((ErtsMonLnkNode *) list, + (void (*)(ErtsMonLnkNode *, void *)) func, + arg, vyspp, limit); +} + +void +erts_monitor_list_foreach_delete(ErtsMonitor **list, + void (*func)(ErtsMonitor *, void *), + void *arg) +{ + void *ystate = NULL; + while (ml_dl_list_foreach_delete_yielding((ErtsMonLnkNode **) list, + (void (*)(ErtsMonLnkNode*, void*)) func, + arg, &ystate, (Sint) INT_MAX)); +} + +int +erts_monitor_list_foreach_delete_yielding(ErtsMonitor **list, + void (*func)(ErtsMonitor *, void *), + void *arg, + void **vyspp, + Sint limit) +{ + return ml_dl_list_foreach_delete_yielding((ErtsMonLnkNode **) list, + (void (*)(ErtsMonLnkNode*, void*)) func, + arg, vyspp, limit); +} + +ErtsMonitorData * +erts_monitor_create(Uint16 type, Eterm ref, Eterm orgn, Eterm trgt, Eterm name) +{ + ErtsMonitorData *mdp; + + switch (type) { + case ERTS_MON_TYPE_PROC: + case ERTS_MON_TYPE_PORT: + case ERTS_MON_TYPE_TIME_OFFSET: + if (is_nil(name)) { + ErtsMonitorDataHeap *mdhp; + ErtsORefThing *ortp; + + ERTS_ML_ASSERT(is_immed(orgn) && is_immed(trgt)); + ERTS_ML_ASSERT(is_internal_ordinary_ref(ref)); + + mdhp = erts_alloc(ERTS_ALC_T_MONITOR, sizeof(ErtsMonitorDataHeap)); + mdp = &mdhp->md; + ERTS_ML_ASSERT(((void *) mdp) == ((void *) mdhp)); + + ortp = (ErtsORefThing *) (char *) internal_ref_val(ref); + mdhp->oref_thing = *ortp; + mdp->ref = make_internal_ref(&mdhp->oref_thing.header); + + mdp->origin.other.item = trgt; + mdp->origin.offset = (Uint16) offsetof(ErtsMonitorData, origin); + mdp->origin.key_offset = (Uint16) offsetof(ErtsMonitorData, ref); + ERTS_ML_ASSERT(mdp->origin.key_offset >= mdp->origin.offset); + mdp->origin.key_offset -= mdp->origin.offset; + mdp->origin.flags = (Uint16) 0; + mdp->origin.type = type; + + mdp->target.other.item = orgn; + mdp->target.offset = (Uint16) offsetof(ErtsMonitorData, target); + mdp->target.key_offset = (Uint16) offsetof(ErtsMonitorData, ref); + ERTS_ML_ASSERT(mdp->target.key_offset >= mdp->target.offset); + mdp->target.key_offset -= mdp->target.offset; + mdp->target.flags = ERTS_ML_FLG_TARGET; + mdp->target.type = type; + break; + } + case ERTS_MON_TYPE_DIST_PROC: + case ERTS_MON_TYPE_RESOURCE: + case ERTS_MON_TYPE_NODE: + case ERTS_MON_TYPE_NODES: { + ErtsMonitorDataExtended *mdep; + Uint size = sizeof(ErtsMonitorDataExtended) - sizeof(Eterm); + Uint rsz, osz, tsz; + Eterm *hp; + ErlOffHeap oh; + Uint16 name_flag = is_nil(name) ? ((Uint16) 0) : ERTS_ML_FLG_NAME; + + rsz = is_immed(ref) ? 0 : size_object(ref); + tsz = is_immed(trgt) ? 0 : size_object(trgt); + if (type == ERTS_MON_TYPE_RESOURCE) + osz = 0; + else + osz = is_immed(orgn) ? 0 : size_object(orgn); + + size += (rsz + osz + tsz) * sizeof(Eterm); + + mdep = erts_alloc(ERTS_ALC_T_MONITOR_EXT, size); + + ERTS_INIT_OFF_HEAP(&oh); + + hp = &mdep->heap[0]; + + mdp = &mdep->md; + ERTS_ML_ASSERT(((void *) mdp) == ((void *) mdep)); + + mdp->ref = rsz ? copy_struct(ref, rsz, &hp, &oh) : ref; + + mdp->origin.other.item = tsz ? copy_struct(trgt, tsz, &hp, &oh) : trgt; + mdp->origin.offset = (Uint16) offsetof(ErtsMonitorData, origin); + mdp->origin.flags = ERTS_ML_FLG_EXTENDED|name_flag; + mdp->origin.type = type; + + if (type == ERTS_MON_TYPE_RESOURCE) + mdp->target.other.ptr = (void *) orgn; + else + mdp->target.other.item = osz ? copy_struct(orgn, osz, &hp, &oh) : orgn; + mdp->target.offset = (Uint16) offsetof(ErtsMonitorData, target); + mdp->target.flags = ERTS_ML_FLG_TARGET|ERTS_ML_FLG_EXTENDED|name_flag; + mdp->target.type = type; + + if (type == ERTS_MON_TYPE_NODE || type == ERTS_MON_TYPE_NODES) { + mdep->u.refc = 0; + mdp->origin.key_offset = (Uint16) offsetof(ErtsMonitor, other.item); + mdp->target.key_offset = (Uint16) offsetof(ErtsMonitor, other.item); + ERTS_ML_ASSERT(!oh.first); + mdep->uptr.node_monitors = NULL; + } + else { + mdep->u.name = name; + + mdp->origin.key_offset = (Uint16) offsetof(ErtsMonitorData, ref); + ERTS_ML_ASSERT(mdp->origin.key_offset >= mdp->origin.offset); + mdp->origin.key_offset -= mdp->origin.offset; + + mdp->target.key_offset = (Uint16) offsetof(ErtsMonitorData, ref); + ERTS_ML_ASSERT(mdp->target.key_offset >= mdp->target.offset); + mdp->target.key_offset -= mdp->target.offset; + + mdep->uptr.ohhp = oh.first; + } + mdep->dist = NULL; + break; + } + case ERTS_MON_TYPE_SUSPEND: + ERTS_INTERNAL_ERROR("Use erts_monitor_suspend_create() instead..."); + mdp = NULL; + break; + default: + ERTS_INTERNAL_ERROR("Invalid monitor type"); + mdp = NULL; + break; + } + + erts_atomic32_init_nob(&mdp->refc, 2); + + return mdp; +} + +/* + * erts_monitor_destroy__() should only be called from + * erts_monitor_release() or erts_monitor_release_both(). + */ +void +erts_monitor_destroy__(ErtsMonitorData *mdp) +{ + ERTS_ML_ASSERT(erts_atomic32_read_nob(&mdp->refc) == 0); + ERTS_ML_ASSERT(!(mdp->origin.flags & ERTS_ML_FLG_IN_TABLE)); + ERTS_ML_ASSERT(!(mdp->target.flags & ERTS_ML_FLG_IN_TABLE)); + ERTS_ML_ASSERT((mdp->origin.flags & ERTS_ML_FLGS_SAME) + == (mdp->target.flags & ERTS_ML_FLGS_SAME)); + ERTS_ML_ASSERT(mdp->origin.type != ERTS_MON_TYPE_SUSPEND); + + if (!(mdp->origin.flags & ERTS_ML_FLG_EXTENDED)) + erts_free(ERTS_ALC_T_MONITOR, mdp); + else { + ErtsMonitorDataExtended *mdep = (ErtsMonitorDataExtended *) mdp; + ErlOffHeap oh; + if (mdp->origin.type == ERTS_MON_TYPE_NODE) + ERTS_ML_ASSERT(!mdep->uptr.node_monitors); + else if (mdep->uptr.ohhp) { + ERTS_INIT_OFF_HEAP(&oh); + oh.first = mdep->uptr.ohhp; + erts_cleanup_offheap(&oh); + } + if (mdep->dist) + erts_mon_link_dist_dec_refc(mdep->dist); + erts_free(ERTS_ALC_T_MONITOR_EXT, mdp); + } +} + +void +erts_monitor_set_dead_dist(ErtsMonitor *mon, Eterm nodename) +{ + ErtsMonitorDataExtended *mdep; + mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon); + + ERTS_ML_ASSERT(mon->flags & ERTS_ML_FLG_EXTENDED); + ERTS_ML_ASSERT(mon->type == ERTS_MON_TYPE_DIST_PROC); + ERTS_ML_ASSERT(!mdep->dist); + + mdep->dist = erts_mon_link_dist_create(nodename); + mdep->dist->alive = 0; +} + +Uint +erts_monitor_size(ErtsMonitor *mon) +{ + Uint size, refc; + ErtsMonitorData *mdp = erts_monitor_to_data(mon); + + ERTS_ML_ASSERT(mon->type != ERTS_MON_TYPE_SUSPEND); + + if (!(mon->flags & ERTS_ML_FLG_EXTENDED)) + size = sizeof(ErtsMonitorDataHeap); + else { + ErtsMonitorDataExtended *mdep; + Uint hsz = 0; + + mdep = (ErtsMonitorDataExtended *) mdp; + + if (mon->type != ERTS_MON_TYPE_NODE + && mon->type != ERTS_MON_TYPE_NODES) { + if (!is_immed(mdep->md.ref)) + hsz += NC_HEAP_SIZE(mdep->md.ref); + if (mon->type == ERTS_MON_TYPE_DIST_PROC) { + if (!is_immed(mdep->md.origin.other.item)) + hsz += NC_HEAP_SIZE(mdep->md.origin.other.item); + if (!is_immed(mdep->md.target.other.item)) + hsz += NC_HEAP_SIZE(mdep->md.target.other.item); + } + } + size = sizeof(ErtsMonitorDataExtended) + (hsz - 1)*sizeof(Eterm); + } + + refc = (Uint) erts_atomic32_read_nob(&mdp->refc); + ASSERT(refc > 0); + + return size / refc; +} + + +/* suspend monitors... */ + +ErtsMonitorSuspend * +erts_monitor_suspend_create(Eterm pid) +{ + ErtsMonitorSuspend *msp; + + ERTS_ML_ASSERT(is_internal_pid(pid)); + + msp = erts_alloc(ERTS_ALC_T_SUSPEND_MON, + sizeof(ErtsMonitorSuspend)); + msp->mon.offset = (Uint16) offsetof(ErtsMonitorSuspend, mon); + msp->mon.key_offset = (Uint16) offsetof(ErtsMonitor, other.item); + msp->mon.other.item = pid; + msp->mon.flags = 0; + msp->mon.type = ERTS_MON_TYPE_SUSPEND; + msp->pending = 0; + msp->active = 0; + return msp; +} + +static ErtsMonLnkNode * +create_monitor_suspend(Eterm pid, void *unused) +{ + ErtsMonitorSuspend *msp = erts_monitor_suspend_create(pid); + return (ErtsMonLnkNode *) &msp->mon; +} + +ErtsMonitorSuspend * +erts_monitor_suspend_tree_lookup_create(ErtsMonitor **root, int *created, + Eterm pid) +{ + ErtsMonitor *mon; + mon = (ErtsMonitor *) ml_rbt_lookup_create((ErtsMonLnkNode **) root, + pid, create_monitor_suspend, + NULL, + created); + return erts_monitor_suspend(mon); +} + +void +erts_monitor_suspend_destroy(ErtsMonitorSuspend *msp) +{ + ERTS_ML_ASSERT(!(msp->mon.flags & ERTS_ML_FLG_IN_TABLE)); + erts_free(ERTS_ALC_T_SUSPEND_MON, msp); +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Link Operations * + * * +\* */ + +#ifdef ERTS_ML_DEBUG +size_t erts_link_a_offset; +size_t erts_link_b_offset; +size_t erts_link_key_offset; +#endif + +static ERTS_INLINE void +link_init(void) +{ +#ifdef ERTS_ML_DEBUG + erts_link_a_offset = offsetof(ErtsLinkData, a); + erts_link_b_offset = offsetof(ErtsLinkData, b); + erts_link_key_offset = offsetof(ErtsLink, other.item); +#endif +} + +ErtsLink * +erts_link_tree_lookup(ErtsLink *root, Eterm key) +{ + ASSERT(is_pid(key) || is_internal_port(key)); + return (ErtsLink *) ml_rbt_lookup((ErtsMonLnkNode *) root, key); +} + +ErtsLink * +erts_link_tree_lookup_insert(ErtsLink **root, ErtsLink *lnk) +{ + return (ErtsLink *) ml_rbt_lookup_insert((ErtsMonLnkNode **) root, + (ErtsMonLnkNode *) lnk); +} + +typedef struct { + Uint16 type; + Eterm a; +} ErtsLinkCreateCtxt; + +static ErtsMonLnkNode * +create_link(Eterm b, void *vcctxt) +{ + ErtsLinkCreateCtxt *cctxt = vcctxt; + ErtsLinkData *ldp = erts_link_create(cctxt->type, cctxt->a, b); + ERTS_ML_ASSERT(ml_cmp_keys(ldp->a.other.item, b) == 0); + return (ErtsMonLnkNode *) &ldp->a; +} + +ErtsLink *erts_link_tree_lookup_create(ErtsLink **root, int *created, + Uint16 type, Eterm this, + Eterm other) +{ + ErtsLinkCreateCtxt cctxt; + cctxt.type = type; + cctxt.a = this; + return (ErtsLink *) ml_rbt_lookup_create((ErtsMonLnkNode **) root, + other, create_link, + (void *) &cctxt, + created); +} + +void +erts_link_tree_insert(ErtsLink **root, ErtsLink *lnk) +{ + ml_rbt_insert((ErtsMonLnkNode **) root, (ErtsMonLnkNode *) lnk); +} + +void +erts_link_tree_replace(ErtsLink **root, ErtsLink *old, ErtsLink *new) +{ + ml_rbt_replace((ErtsMonLnkNode **) root, + (ErtsMonLnkNode *) old, + (ErtsMonLnkNode *) new); +} + +void +erts_link_tree_delete(ErtsLink **root, ErtsLink *lnk) +{ + ml_rbt_delete((ErtsMonLnkNode **) root, (ErtsMonLnkNode *) lnk); +} + +void +erts_link_tree_foreach(ErtsLink *root, + void (*func)(ErtsLink *, void *), + void *arg) +{ + ml_rbt_foreach((ErtsMonLnkNode *) root, + (void (*)(ErtsMonLnkNode*, void*)) func, + arg); + +} + +int +erts_link_tree_foreach_yielding(ErtsLink *root, + void (*func)(ErtsLink *, void *), + void *arg, + void **vyspp, + Sint limit) +{ + return ml_rbt_foreach_yielding((ErtsMonLnkNode *) root, + (void (*)(ErtsMonLnkNode*, void*)) func, + arg, vyspp, limit); +} + +void +erts_link_tree_foreach_delete(ErtsLink **root, + void (*func)(ErtsLink *, void *), + void *arg) +{ + ml_rbt_foreach_delete((ErtsMonLnkNode **) root, + (void (*)(ErtsMonLnkNode*, void*)) func, + arg); +} + +int +erts_link_tree_foreach_delete_yielding(ErtsLink **root, + void (*func)(ErtsLink *, void *), + void *arg, + void **vyspp, + Sint limit) +{ + return ml_rbt_foreach_delete_yielding((ErtsMonLnkNode **) root, + (void (*)(ErtsMonLnkNode*, void*)) func, + arg, vyspp, limit); +} + +void +erts_link_list_foreach(ErtsLink *list, + void (*func)(ErtsLink *, void *), + void *arg) +{ + void *ystate = NULL; + while (ml_dl_list_foreach_yielding((ErtsMonLnkNode *) list, + (void (*)(ErtsMonLnkNode *, void *)) func, + arg, &ystate, (Sint) INT_MAX)); +} + +int +erts_link_list_foreach_yielding(ErtsLink *list, + void (*func)(ErtsLink *, void *), + void *arg, + void **vyspp, + Sint limit) +{ + return ml_dl_list_foreach_yielding((ErtsMonLnkNode *) list, + (void (*)(ErtsMonLnkNode *, void *)) func, + arg, vyspp, limit); +} + +void +erts_link_list_foreach_delete(ErtsLink **list, + void (*func)(ErtsLink *, void *), + void *arg) +{ + void *ystate = NULL; + while (ml_dl_list_foreach_delete_yielding((ErtsMonLnkNode **) list, + (void (*)(ErtsMonLnkNode*, void*)) func, + arg, &ystate, (Sint) INT_MAX)); +} + +int +erts_link_list_foreach_delete_yielding(ErtsLink **list, + void (*func)(ErtsLink *, void *), + void *arg, + void **vyspp, + Sint limit) +{ + return ml_dl_list_foreach_delete_yielding((ErtsMonLnkNode **) list, + (void (*)(ErtsMonLnkNode*, void*)) func, + arg, vyspp, limit); +} + +ErtsLinkData * +erts_link_create(Uint16 type, Eterm a, Eterm b) +{ + ErtsLinkData *ldp; + +#ifdef ERTS_ML_DEBUG + switch (type) { + case ERTS_LNK_TYPE_PROC: + ERTS_ML_ASSERT(is_internal_pid(a) && is_internal_pid(a)); + break; + case ERTS_LNK_TYPE_PORT: + ERTS_ML_ASSERT(is_internal_pid(a) || is_internal_pid(b)); + ERTS_ML_ASSERT(is_internal_port(a) || is_internal_port(b)); + break; + case ERTS_LNK_TYPE_DIST_PROC: + ERTS_ML_ASSERT(is_internal_pid(a) || is_internal_pid(b)); + ERTS_ML_ASSERT(is_external_pid(a) || is_external_pid(b)); + break; + default: + ERTS_INTERNAL_ERROR("Invalid link type"); + break; + } +#endif + + if (type != ERTS_LNK_TYPE_DIST_PROC) { + ldp = erts_alloc(ERTS_ALC_T_LINK, sizeof(ErtsLinkData)); + + ldp->a.other.item = b; + ldp->a.flags = (Uint16) 0; + + ldp->b.other.item = a; + ldp->b.flags = (Uint16) 0; + } + else { + ErtsLinkDataExtended *ldep; + Uint size, hsz; + Eterm *hp; + ErlOffHeap oh; + + if (is_internal_pid(a)) + hsz = NC_HEAP_SIZE(b); + else + hsz = NC_HEAP_SIZE(a); + ERTS_ML_ASSERT(hsz > 0); + + size = sizeof(ErtsLinkDataExtended) - sizeof(Eterm); + size += hsz*sizeof(Eterm); + + ldp = erts_alloc(ERTS_ALC_T_LINK_EXT, size); + + ldp->a.flags = ERTS_ML_FLG_EXTENDED; + ldp->b.flags = ERTS_ML_FLG_EXTENDED; + + ldep = (ErtsLinkDataExtended *) ldp; + hp = &ldep->heap[0]; + + ERTS_INIT_OFF_HEAP(&oh); + + if (is_internal_pid(a)) { + ldp->a.other.item = STORE_NC(&hp, &oh, b); + ldp->b.other.item = a; + } + else { + ldp->a.other.item = b; + ldp->b.other.item = STORE_NC(&hp, &oh, a); + } + + ldep->ohhp = oh.first; + ldep->dist = NULL; + } + + erts_atomic32_init_nob(&ldp->refc, 2); + + ldp->a.key_offset = (Uint16) offsetof(ErtsLink, other.item); + ldp->a.offset = (Uint16) offsetof(ErtsLinkData, a); + ldp->a.type = type; + + ldp->b.key_offset = (Uint16) offsetof(ErtsLink, other.item); + ldp->b.offset = (Uint16) offsetof(ErtsLinkData, b); + ldp->b.type = type; + + return ldp; +} + +/* + * erts_link_destroy__() should only be called from + * erts_link_release() or erts_link_release_both(). + */ +void +erts_link_destroy__(ErtsLinkData *ldp) +{ + ERTS_ML_ASSERT(erts_atomic32_read_nob(&ldp->refc) == 0); + ERTS_ML_ASSERT(!(ldp->a.flags & ERTS_ML_FLG_IN_TABLE)); + ERTS_ML_ASSERT(!(ldp->b.flags & ERTS_ML_FLG_IN_TABLE)); + ERTS_ML_ASSERT((ldp->a.flags & ERTS_ML_FLGS_SAME) + == (ldp->b.flags & ERTS_ML_FLGS_SAME)); + + if (!(ldp->a.flags & ERTS_ML_FLG_EXTENDED)) + erts_free(ERTS_ALC_T_LINK, ldp); + else { + ErtsLinkDataExtended *ldep = (ErtsLinkDataExtended *) ldp; + ErlOffHeap oh; + if (ldep->ohhp) { + ERTS_INIT_OFF_HEAP(&oh); + oh.first = ldep->ohhp; + erts_cleanup_offheap(&oh); + } + if (ldep->dist) + erts_mon_link_dist_dec_refc(ldep->dist); + erts_free(ERTS_ALC_T_LINK_EXT, ldep); + } +} + +void +erts_link_set_dead_dist(ErtsLink *lnk, Eterm nodename) +{ + ErtsLinkDataExtended *ldep; + ldep = (ErtsLinkDataExtended *) erts_link_to_data(lnk); + + ERTS_ML_ASSERT(lnk->flags & ERTS_ML_FLG_EXTENDED); + ERTS_ML_ASSERT(lnk->type == ERTS_LNK_TYPE_DIST_PROC); + ERTS_ML_ASSERT(!ldep->dist); + + ldep->dist = erts_mon_link_dist_create(nodename); + ldep->dist->alive = 0; +} + +Uint +erts_link_size(ErtsLink *lnk) +{ + Uint size, refc; + ErtsLinkData *ldp = erts_link_to_data(lnk); + + if (!(lnk->flags & ERTS_ML_FLG_EXTENDED)) + size = sizeof(ErtsLinkData); + else { + ErtsLinkDataExtended *ldep = (ErtsLinkDataExtended *) ldp; + + ASSERT(lnk->type == ERTS_LNK_TYPE_DIST_PROC); + ASSERT(is_external_pid_header(ldep->heap[0])); + + size = sizeof(ErtsLinkDataExtended); + size += thing_arityval(ldep->heap[0])*sizeof(Eterm); + } + + refc = (Uint) erts_atomic32_read_nob(&ldp->refc); + ASSERT(refc > 0); + + return size / refc; +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Misc * +\* */ + +void +erts_monitor_link_init(void) +{ + monitor_init(); + link_init(); +} diff --git a/erts/emulator/beam/erl_monitor_link.h b/erts/emulator/beam/erl_monitor_link.h new file mode 100644 index 0000000000..603aead8cc --- /dev/null +++ b/erts/emulator/beam/erl_monitor_link.h @@ -0,0 +1,2326 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Monitor and link implementation. + * + * === Monitors ================================================== + * + * The monitor data structure contains: + * - an 'origin' part that should be inserted in a data structure + * of the origin entity, i.e., the entity monitoring another + * entity + * - a 'target' part that should be inserted in a data structure + * of the target entity, i.e., the entity being monitored by + * another entity + * - a shared part that contains information shared between both + * origin and target entities + * + * That is, the two halves of the monitor as well as shared data + * are allocated in one single continuous memory block. The + * origin and target parts can separately each be inserted in + * either a (red-black) tree, a (circular double linked) list, or + * in a process signal queue. + * + * Each process and port contains: + * - a monitor list for local target monitors that is accessed + * via the ERTS_P_LT_MONITORS() macro, and + * - a monitor tree for other monitors that is accessed via the + * ERTS_P_MONITORS() macro + * + * These fields of processes/ports are protected by the main lock + * of the process/port. These are only intended to be accessed by + * the process/port itself. When setting up or tearing down a + * monitor one should *only* operate on the monitor tree/list of + * the currently executing process/port and send signals to the + * other involved process/port so it can modify its own monitor + * tree/list by itself (see erl_proc_sig_queue.h). One should + * absolutely *not* acquire the lock of the other involved + * process/port and operate on its monitor tree/list directly. + * + * Each dist entry contains a monitor/link dist structure that + * contains: + * - a monitor tree for origin named monitors that is accessed via + * the field 'orig_name_monitors', and + * - a monitor list for other monitors that is accessed via the + * 'monitors' field. + * Monitors in these fields contain information about all monitors + * over this specific connection. + * + * The fields of the dist structure are protected by a mutex in + * the same dist structure. Operations on these fields are + * normally performed by the locally involved process only, + * except when a connection is taken down. However in the case + * of distributed named monitors that originates from another + * node this is not possible. That is this operation is also + * performed from another context that the locally involved + * process. + * + * Access to monitor trees are performed using the + * erts_monitor_tree_* functions below. Access to monitor lists + * are performed using the erts_monitor_list_* functions below. + * + * + * The different monitor types: + * + * --- ERTS_MON_TYPE_PROC ---------------------------------------- + * + * A local process (origin) monitors another local process + * (target). + * + * Origin: + * Other Item: Target process identifier + * Target: + * Other Item: Origin process identifier + * Shared: + * Key: Reference + * Name: Name (atom) if by name + * + * Valid keys are only ordinary internal references. + * + * Origin part of the monitor is stored in the monitor tree of + * origin process and target part of the monitor is stored in + * monitor list for local targets on the target process. + * + * --- ERTS_MON_TYPE_PORT ---------------------------------------- + * + * A local process (origin) monitors a local port (target), or a + * local port (origin) monitors a local process (target). + * + * Origin: + * Other Item: Target process/port identifier + * Target: + * Other Item: Origin process/port identifier + * Shared: + * Key: Reference + * Name: Name (atom) if by name + * + * Valid keys are only ordinary internal references. + * + * Origin part of the monitor is stored in the monitor tree of + * origin process/port and target part of the monitor is stored + * in monitor list for local targets on the target process/port. + * + * + * --- ERTS_MON_TYPE_TIME_OFFSET --------------------------------- + * + * A local process (origin) monitors time offset (target) + * + * Origin: + * Other Item: clock_service + * Target: + * Other Item: Origin process identifier + * Shared: + * Key: Reference + * + * Valid keys are only ordinary internal references. + * + * Origin part of the monitor is stored in the monitor tree of + * origin process and target part of the monitor is stored in + * monitor list referred by the variable 'time_offset_monitors' + * (see erl_time_sup.c). + * + * + * --- ERTS_MON_TYPE_DIST_PROC ----------------------------------- + * + * A local process (origin) monitors a remote process (target). + * Origin node on local process and target node on dist entry. + * + * Origin: + * Other Item: Remote process identifier/Node name + * if by name + * Target: + * Other Item: Local process identifier + * Shared: + * Key: Reference + * Name: Name (atom) if by name + * Dist: Pointer to dist structure + * + * Valid keys are only ordinary internal references. + * + * Origin part of the monitor is stored in the monitor tree of + * origin process and target part of the monitor is stored in + * monitor list referred by 'monitors' field of the dist + * structure. + * + * + * A remote process (origin) monitors a local process (target). + * Origin node on dist entry and target node on local process. + * + * Origin: + * Other Item: Local process identifier + * Target: + * Other Item: Remote process identifier + * Shared: + * Key: Reference + * Name: Name (atom) if by name + * + * Valid keys are only external references. + * + * If monitor by name, the origin part of the monitor is stored + * in the monitor tree referred by 'orig_name_monitors' field in + * dist structure; otherwise in the monitor list referred by + * 'monitors' field in dist structure. The target part of the + * monitor is stored in the monitor tree of the local target + * process. + * + * + * --- ERTS_MON_TYPE_RESOURCE ------------------------------------ + * + * A NIF resource (origin) monitors a process (target). + * + * Origin: + * Other Item: Target process identifier + * Target: + * Other Ptr: Pointer to resource + * Shared: + * Key: Reference + * + * Valid keys are only ordinary internal references. + * + * Origin part of the monitor is stored in the monitor tree of + * origin resource (see erl_nif.c) and target part of the + * monitor is stored in monitor list for local targets on the + * target process. + * + * --- ERTS_MON_TYPE_NODE ---------------------------------------- + * + * A local process (origin) monitors a distribution connection + * (target) via erlang:monitor_node(). + * + * Origin: + * Other Item: Node name (atom) + * Key: Node name + * Target: + * Other Item: Origin process identifier + * Key: Origin process identifier + * Shared: + * Refc: Number of invocations + * + * Valid keys are only node-name atoms and internal process + * identifiers. + * + * Origin part of the monitor is stored in the monitor tree of + * origin process and target part of the monitor is stored in + * monitor list referred by 'monitors' field of the dist + * structure. + * + * --- ERTS_MON_TYPE_NODES --------------------------------------- + * + * A local process (origin) monitors all connections (target), + * via net_kernel:monitor_nodes(). + * + * Origin: + * Other Item: Bit mask (small) + * Key: Bit mask + * Target: + * Other Item: Origin process identifier + * Key: Origin process identifier + * Shared: + * Refc: Number of invocations + * + * Valid keys are only small integers and internal process + * identifiers. + * + * Origin part of the monitor is stored in the monitor tree of + * origin process and target part of the monitor is stored in + * monitor list referred by the variable 'nodes_monitors' (see + * dist.c). + * + * --- ERTS_MON_TYPE_SUSPEND ------------------------------------- + * + * Suspend monitor. + * + * Other Item: Suspendee process identifier + * Key: Suspendee process identifier + * + * Valid keys are only ordinary internal references. + * + * This type of monitor is a bit strange and the whole process + * suspend functionality should be improved... + * + * + * + * === Links ===================================================== + * + * The link data structure contains: + * - an 'a' part that should be inserted in a data structure of + * one entity and contains the identifier of the other involved + * entity (b) + * - a 'b' part that should be inserted in a data structure of + * the other involved entity and contains the identifier of the + * other involved entity (a) + * - shared part that contains information shared between both + * involved entities + * + * That is, the two halves of the link as well as shared data + * are allocated in one single continuous memory block. The 'a' + * and the 'b' parts can separately each be inserted in either + * a (red-black) tree, a (circular double linked) list, or in a + * process signal queue. + * + * Each process and port contains: + * - a link tree for links that is accessed via the + * ERTS_P_LINKS() macro + * + * This field of processes/ports is protected by the main lock of + * the process/port. It is only intended to be accessed by the + * process/port itself. When setting up or tearing down a link + * one should *only* operate on the link tree of the currently + * executing process/port and send signals to the other involved + * process/port so it can modify its own monitor tree by itself + * (see erl_proc_sig_queue.h). One should absolutely *not* + * acquire the lock of the other involved process/port and + * operate on its link tree directly. + * + * Each dist entry contains a monitor/link dist structure that + * contains: + * - a link list for links via the 'links' field. + * Links in this field contain information about all links over + * this specific connection. + * + * The fields of the dist structure are protected by a mutex in + * the same dist structure. Operation on the 'links' fields are + * normally performed by the locally involved process only, + * except when a connection is taken down. + * + * Access to link trees are performed using the erts_link_tree_* + * functions below. Access to link lists are performed using the + * erts_link_list_* functions below. + * + * There can only be one link between the same pair of + * processes/ports. Since a link can be simultaneously initiated + * from both ends we always save the link data structure with the + * lowest address if multiple links should appear between the + * same pair of processes/ports. + * + * + * The different link types: + * + * --- ERTS_LNK_TYPE_PROC ----------------------------------------- + * + * A link between a local process A and a local process B. + * + * A: + * Other Item: B process identifier + * Key: B process identifier + * B: + * Other Item: A process identifier + * Key: A process identifier + * + * Valid keys are only internal process identifiers. + * + * 'A' part of the link stored in the link tree of process A and + * 'B' part of the link is stored in link tree of process B. + * + * --- ERTS_LNK_TYPE_PORT ----------------------------------------- + * + * A link between a local process/port A and a local process/port + * B. + * + * A: + * Other Item: B process/port identifier + * Key: B process/port identifier + * B: + * Other Item: A process/port identifier + * Key: A process/port identifier + * + * Valid keys are internal process identifiers and internal port + * identifiers. + * + * 'A' part of the link stored in the link tree of process/port + * A and 'B' part of the link is stored in link tree of + * process/port B. + * + * --- ERTS_LNK_TYPE_DIST_PROC ------------------------------------ + * + * A link between a local process and a remote process. Either of + * the processes can be used as A or B. + * + * A: + * Other Item: B process identifier + * Key: B process identifier + * B: + * Other Item: A process identifier + * Key: A process identifier + * Shared: + * Dist: Pointer to dist structure + * + * Valid keys are internal and external process identifiers. + * + * The part of the link with a remote pid as "other item" is + * stored in the link tree of the local process. The part of + * the link with a local pid as "other item" is stored in the + * links list of the dist structure. + * + * =============================================================== + * + * Author: Rickard Green + * + */ + +#ifndef ERL_MONITOR_LINK_H__ +#define ERL_MONITOR_LINK_H__ + +#define ERTS_PROC_SIG_QUEUE_TYPE_ONLY +#include "erl_proc_sig_queue.h" +#undef ERTS_PROC_SIG_QUEUE_TYPE_ONLY + +#if defined(DEBUG) || 0 +# define ERTS_ML_DEBUG +#else +# undef ERTS_ML_DEBUG +#endif + +#ifdef ERTS_ML_DEBUG +# define ERTS_ML_ASSERT ERTS_ASSERT +#else +# define ERTS_ML_ASSERT(E) ((void) 1) +#endif + +#define ERTS_MON_TYPE_MAX ((Uint16) 7) + +#define ERTS_MON_TYPE_PROC ((Uint16) 0) +#define ERTS_MON_TYPE_PORT ((Uint16) 1) +#define ERTS_MON_TYPE_TIME_OFFSET ((Uint16) 2) +#define ERTS_MON_TYPE_DIST_PROC ((Uint16) 3) +#define ERTS_MON_TYPE_RESOURCE ((Uint16) 4) +#define ERTS_MON_TYPE_NODE ((Uint16) 5) +#define ERTS_MON_TYPE_NODES ((Uint16) 6) +#define ERTS_MON_TYPE_SUSPEND ERTS_MON_TYPE_MAX + +#define ERTS_MON_LNK_TYPE_MAX (ERTS_MON_TYPE_MAX + ((Uint16) 3)) +#define ERTS_LNK_TYPE_MAX ERTS_MON_LNK_TYPE_MAX + +#define ERTS_LNK_TYPE_PROC (ERTS_MON_TYPE_MAX + ((Uint16) 1)) +#define ERTS_LNK_TYPE_PORT (ERTS_MON_TYPE_MAX + ((Uint16) 2)) +#define ERTS_LNK_TYPE_DIST_PROC ERTS_LNK_TYPE_MAX + +#define ERTS_ML_FLG_TARGET (((Uint16) 1) << 0) +#define ERTS_ML_FLG_IN_TABLE (((Uint16) 1) << 1) +#define ERTS_ML_FLG_IN_SUBTABLE (((Uint16) 1) << 2) +#define ERTS_ML_FLG_NAME (((Uint16) 1) << 3) +#define ERTS_ML_FLG_EXTENDED (((Uint16) 1) << 4) + +#define ERTS_ML_FLG_DBG_VISITED (((Uint16) 1) << 15) + +/* Flags that should be the same on both monitor/link halves */ +#define ERTS_ML_FLGS_SAME \ + (ERTS_ML_FLG_EXTENDED|ERTS_ML_FLG_NAME) + +typedef struct ErtsMonLnkNode__ ErtsMonLnkNode; + +typedef struct { + UWord parent; /* Parent ptr and flags... */ + ErtsMonLnkNode *right; + ErtsMonLnkNode *left; +} ErtsMonLnkTreeNode; + +typedef struct { + ErtsMonLnkNode *next; + ErtsMonLnkNode *prev; +} ErtsMonLnkListNode; + +struct ErtsMonLnkNode__ { + union { + ErtsSignalCommon signal; + ErtsMonLnkTreeNode tree; + ErtsMonLnkListNode list; + } node; + union { + Eterm item; + void *ptr; + } other; + Uint16 offset; /* offset from monitor/link data to this structure (node) */ + Uint16 key_offset; /* offset from this structure (node) to key */ + Uint16 flags; + Uint16 type; +}; + +typedef struct { + Eterm nodename; + Uint32 connection_id; + erts_atomic_t refc; + erts_mtx_t mtx; + int alive; + ErtsMonLnkNode *links; /* Link double linked circular list */ + ErtsMonLnkNode *monitors; /* Monitor double linked circular list */ + ErtsMonLnkNode *orig_name_monitors; /* Origin named monitors + read-black tree */ +} ErtsMonLnkDist; + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Misc * +\* */ + +/** + * + * @brief Initialize monitor/link implementation + * + */ +void erts_monitor_link_init(void); + +/** + * + * @brief Create monitor/link dist structure to attach to dist entry + * + * Create dist structure containing monitor and link containers. This + * structure is to be attached to a connected dist entry. + * + * @param[in] nodename Node name as an atom + * + * @returns Pointer to dist structure + * + */ +ErtsMonLnkDist *erts_mon_link_dist_create(Eterm nodename); + +/** + * + * @brief Increase reference count of monitor/link dist structure + * + * @param[in] mld Pointer to dist structure + * + */ +ERTS_GLB_INLINE void erts_mon_link_dist_inc_refc(ErtsMonLnkDist *mld); + +/** + * + * @brief Decrease reference count of monitor/link dist structure + * + * @param[in] mld Pointer to dist structure + * + */ +ERTS_GLB_INLINE void erts_mon_link_dist_dec_refc(ErtsMonLnkDist *mld); + +/* internal functions... */ +ERTS_GLB_INLINE void erts_ml_dl_list_insert__(ErtsMonLnkNode **list, + ErtsMonLnkNode *ml); +ERTS_GLB_INLINE void erts_ml_dl_list_delete__(ErtsMonLnkNode **list, + ErtsMonLnkNode *ml); +ERTS_GLB_INLINE ErtsMonLnkNode *erts_ml_dl_list_first__(ErtsMonLnkNode *list); +ERTS_GLB_INLINE ErtsMonLnkNode *erts_ml_dl_list_last__(ErtsMonLnkNode *list); +void erts_mon_link_dist_destroy__(ErtsMonLnkDist *mld); +ERTS_GLB_INLINE void *erts_ml_node_to_main_struct__(ErtsMonLnkNode *mln); + +/* implementations for globally inlined misc functions... */ +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void +erts_mon_link_dist_inc_refc(ErtsMonLnkDist *mld) +{ + ERTS_ML_ASSERT(erts_atomic_read_nob(&mld->refc) > 0); + erts_atomic_inc_nob(&mld->refc); +} + +ERTS_GLB_INLINE void +erts_mon_link_dist_dec_refc(ErtsMonLnkDist *mld) +{ + ERTS_ML_ASSERT(erts_atomic_read_nob(&mld->refc) > 0); + if (erts_atomic_dec_read_nob(&mld->refc) == 0) + erts_mon_link_dist_destroy__(mld); +} + +ERTS_GLB_INLINE void * +erts_ml_node_to_main_struct__(ErtsMonLnkNode *mln) +{ + return (void *) (((char *) mln) - ((size_t) mln->offset)); +} + +ERTS_GLB_INLINE void +erts_ml_dl_list_insert__(ErtsMonLnkNode **list, ErtsMonLnkNode *ml) +{ + ErtsMonLnkNode *first = *list; + ERTS_ML_ASSERT(!(ml->flags & ERTS_ML_FLG_IN_TABLE)); + if (!first) { + ml->node.list.next = ml->node.list.prev = ml; + *list = ml; + } + else { + ERTS_ML_ASSERT(first->node.list.prev->node.list.next == first); + ERTS_ML_ASSERT(first->node.list.next->node.list.prev == first); + ml->node.list.next = first; + ml->node.list.prev = first->node.list.prev; + first->node.list.prev = ml; + ml->node.list.prev->node.list.next = ml; + } + ml->flags |= ERTS_ML_FLG_IN_TABLE; +} + +ERTS_GLB_INLINE void +erts_ml_dl_list_delete__(ErtsMonLnkNode **list, ErtsMonLnkNode *ml) +{ + ERTS_ML_ASSERT(ml->flags & ERTS_ML_FLG_IN_TABLE); + if (ml->node.list.next == ml) { + ERTS_ML_ASSERT(ml->node.list.prev == ml); + ERTS_ML_ASSERT(*list == ml); + + *list = NULL; + } + else { + ERTS_ML_ASSERT(ml->node.list.prev->node.list.next == ml); + ERTS_ML_ASSERT(ml->node.list.prev != ml); + ERTS_ML_ASSERT(ml->node.list.next->node.list.prev == ml); + ERTS_ML_ASSERT(ml->node.list.next != ml); + + if (*list == ml) + *list = ml->node.list.next; + ml->node.list.prev->node.list.next = ml->node.list.next; + ml->node.list.next->node.list.prev = ml->node.list.prev; + } + ml->flags &= ~ERTS_ML_FLG_IN_TABLE; +} + +ERTS_GLB_INLINE ErtsMonLnkNode * +erts_ml_dl_list_first__(ErtsMonLnkNode *list) +{ + return list; +} + +ERTS_GLB_INLINE ErtsMonLnkNode * +erts_ml_dl_list_last__(ErtsMonLnkNode *list) +{ + if (!list) + return NULL; + return list->node.list.prev; +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Monitor Operations * +\* */ + + +typedef struct ErtsMonLnkNode__ ErtsMonitor; + +typedef struct { + ErtsMonitor origin; + ErtsMonitor target; + Eterm ref; + erts_atomic32_t refc; +} ErtsMonitorData; + +typedef struct { + ErtsMonitorData md; + ErtsORefThing oref_thing; +} ErtsMonitorDataHeap; + +typedef struct ErtsMonitorDataExtended__ ErtsMonitorDataExtended; + +struct ErtsMonitorDataExtended__ { + ErtsMonitorData md; + union { + Eterm name; + Uint refc; + } u; + union { + struct erl_off_heap_header *ohhp; + ErtsMonitor *node_monitors; + } uptr; + ErtsMonLnkDist *dist; + Eterm heap[1]; /* heap start... */ +}; + +typedef struct { + ErtsMonitor mon; + int pending; + int active; +} ErtsMonitorSuspend; + +/* + * --- Monitor tree operations --- + */ + +/** + * + * @brief Lookup a monitor in a monitor tree + * + * + * @param[in] root Pointer to root of monitor tree + * + * @param[in] key Key of monitor to lookup + * + * @returns Pointer to a monitor with the + * key 'key', or NULL if no such + * monitor was found + * + */ +ErtsMonitor *erts_monitor_tree_lookup(ErtsMonitor *root, Eterm key); + +/** + * + * @brief Lookup or insert a monitor in a monitor tree + * + * When the funcion is called it is assumed that: + * - 'mon' monitor is not part of any tree or list + * If the above is not true, bad things will happen. + * + * @param[in,out] root Pointer to pointer to root of monitor tree + * + * @param[in] mon Monitor to insert if no monitor + * with the same key already exists + * + * @returns Pointer to a monitor with the + * key 'key'. If no monitor with the key + * 'key' was found and 'mon' was inserted + * 'mon' is returned. + * + */ +ErtsMonitor *erts_monotor_tree_lookup_insert(ErtsMonitor **root, + ErtsMonitor *mon); + +/** + * + * @brief Lookup or create a node or a nodes monitor in a monitor tree. + * + * Looks up an origin monitor with the key 'target' in the monitor tree. + * If it is not found, creates a monitor and returns a pointer to the + * origin monitor. + * + * When the funcion is called it is assumed that: + * - no target monitors with the key 'target' exists in the tree. + * If the above is not true, bad things will happen. + * + * @param[in,out] root Pointer to pointer to root of monitor tree + * + * @param[out] created Pointer to integer. The integer is set to + * a non-zero value if no monitor with key + * 'target' was found, and a new monitor + * was created. If a monitor was found, it + * is set to zero. + * + * @param[in] type ERTS_MON_TYPE_NODE | ERTS_MON_TYPE_NODES + * + * @param[in] origin The key of the origin + * + * @param[in] target The key of the target + * + */ +ErtsMonitor *erts_monitor_tree_lookup_create(ErtsMonitor **root, int *created, + Uint16 type, Eterm origin, + Eterm target); + +/** + * + * @brief Insert a monitor in a monitor tree + * + * When the funcion is called it is assumed that: + * - no monitors with the same key that 'mon' exist in the tree + * - 'mon' is not part of any list of tree + * If the above are not true, bad things will happen. + * + * @param[in,out] root Pointer to pointer to root of monitor tree + * + * @param[in] mon Monitor to insert. + * + */ +void erts_monitor_tree_insert(ErtsMonitor **root, ErtsMonitor *mon); + +/** + * + * @brief Replace a monitor in a monitor tree + * + * When the funcion is called it is assumed that: + * - 'old' monitor and 'new' monitor have exactly the same key + * - 'old' monitor is part of the tree + * - 'new' monitor is not part of any tree or list + * If the above are not true, bad things will happen. + * + * @param[in,out] root Pointer to pointer to root of monitor tree + * + * @param[in] old Monitor to remove from the tree + * + * @param[in] new Monitor to insert into the tree + * + */ +void erts_monitor_tree_replace(ErtsMonitor **root, ErtsMonitor *old, + ErtsMonitor *new); + +/** + * + * @brief Delete a monitor from a monitor tree + * + * When the funcion is called it is assumed that: + * - 'mon' monitor is part of the tree + * If the above is not true, bad things will happen. + * + * @param[in,out] root Pointer to pointer to root of monitor tree + * + * @param[in] mon Monitor to remove from the tree + * + */ +void erts_monitor_tree_delete(ErtsMonitor **root, ErtsMonitor *mon); + +/** + * + * @brief Call a function for each monitor in a monitor tree + * + * The funcion 'func' will be called with a pointer to a monitor + * as first argument and 'arg' as second argument for each monitor + * in the tree referred to by 'root'. + * + * @param[in] root Pointer to root of monitor tree + * + * @param[in] func Pointer to function to call + * + * @param[in] arg Argument to pass as second argument in + * calls to 'func' + * + */ +void erts_monitor_tree_foreach(ErtsMonitor *root, + void (*func)(ErtsMonitor *, void *), + void *arg); + +/** + * + * @brief Call a function for each monitor in a monitor tree. Yield + * if lots of monitors exist. + * + * The funcion 'func' will be called with a pointer to a monitor + * as first argument and 'arg' as second argument for each monitor + * in the tree referred to by 'root'. + * + * It is assumed that: + * - *yspp equals NULL on first call + * - this function is repetedly called with *yspp set + * as set when previous call returned until a non-zero + * value is returned. + * - no modifications are made on the tree between first call + * and the call that returns a non-zero value + * If the above are not true, bad things will happen. + * + * @param[in] root Pointer to root of monitor tree + * + * @param[in] func Pointer to function to call + * + * @param[in] arg Argument to pass as second argument in + * calls to 'func' + * + * @param[in,out] vyspp Pointer to a pointer to an internal state + * used by this function. At initial call + * *yspp should be NULL. When done *yspp + * will be NULL. + * + * @param[in] limit Maximum amount of monitors to process + * before yielding. + * + * @returns A non-zero value when all monitors has been + * processed, and zero when more work is needed. + * + */ +int erts_monitor_tree_foreach_yielding(ErtsMonitor *root, + void (*func)(ErtsMonitor *, void *), + void *arg, + void **vyspp, + Sint limit); + +/** + * + * @brief Delete all monitors from a monitor tree and call a function for + * each monitor + * + * The funcion 'func' will be called with a pointer to a monitor + * as first argument and 'arg' as second argument for each monitor + * in the tree referred to by 'root'. + * + * @param[in,out] root Pointer to pointer to root of monitor tree + * + * @param[in] func Pointer to function to call + * + * @param[in] arg Argument to pass as second argument in + * calls to 'func' + * + */ +void erts_monitor_tree_foreach_delete(ErtsMonitor **root, + void (*func)(ErtsMonitor *, void *), + void *arg); + +/** + * + * @brief Delete all monitors from a monitor tree and call a function for + * each monitor + * + * The funcion 'func' will be called with a pointer to a monitor + * as first argument and 'arg' as second argument for each monitor + * in the tree referred to by 'root'. + * + * It is assumed that: + * - *yspp equals NULL on first call + * - this function is repetededly called with *yspp set + * as set when previous call returned until a non-zero + * value is returned. + * - no modifications are made on the tree between first call + * and the call that returns a non-zero value + * If the above are not true, bad things will happen. + * + * @param[in,out] root Pointer to pointer to root of monitor tree + * + * @param[in] func Pointer to function to call + * + * @param[in] arg Argument to pass as second argument in + * calls to 'func' + * + * @param[in,out] vyspp Pointer to a pointer to an internal state + * used by this function. At initial call + * *yspp should be NULL. When done *yspp + * will be NULL. + * + * @param[in] limit Maximum amount of monitors to process + * before yielding. + * + * @returns A non-zero value when all monitors has been + * processed, and zero when more work is needed. + * + */ +int erts_monitor_tree_foreach_delete_yielding(ErtsMonitor **root, + void (*func)(ErtsMonitor *, void *), + void *arg, + void **vyspp, + Sint limit); + +/* + * --- Monitor list operations -- + */ + +/** + * + * @brief Insert a monitor in a monitor list + * + * When the funcion is called it is assumed that: + * - 'mon' monitor is not part of any list or tree + * If the above is not true, bad things will happen. + * + * @param[in,out] list Pointer to pointer to monitor list + * + * @param[in] mon Monitor to insert + * + */ +ERTS_GLB_INLINE void erts_monitor_list_insert(ErtsMonitor **list, ErtsMonitor *mon); + +/** + * + * @brief Delete a monitor from a monitor list + * + * When the funcion is called it is assumed that: + * - 'mon' monitor is part of the list + * If the above is not true, bad things will happen. + * + * @param[in,out] list Pointer to pointer to monitor list + * + * @param[in] mon Monitor to remove from the list + * + */ +ERTS_GLB_INLINE void erts_monitor_list_delete(ErtsMonitor **list, ErtsMonitor *mon); + +/** + * + * @brief Get a pointer to first monitor in a monitor list + * + * The monitor will still remain in the list after the return + * + * @param[in] list Pointer to monitor list + * + * @returns Pointer to first monitor in list if + * list is not empty. If list is empty + * NULL is returned. + * + */ +ERTS_GLB_INLINE ErtsMonitor *erts_monitor_list_first(ErtsMonitor *list); + +/** + * + * @brief Get a pointer to last monitor in a monitor list + * + * The monitor will still remain in the list after the return + * + * @param[in] list Pointer to monitor list + * + * @returns Pointer to last monitor in list if + * list is not empty. If list is empty + * NULL is returned. + * + */ +ERTS_GLB_INLINE ErtsMonitor *erts_monitor_list_last(ErtsMonitor *list); + +/** + * + * @brief Call a function for each monitor in a monitor list + * + * The funcion 'func' will be called with a pointer to a monitor + * as first argument and 'arg' as second argument for each monitor + * in the tree referred to by 'list'. + * + * @param[in] list Pointer to root of monitor list + * + * @param[in] func Pointer to function to call + * + * @param[in] arg Argument to pass as second argument in + * calls to 'func' + * + */ +void erts_monitor_list_foreach(ErtsMonitor *list, + void (*func)(ErtsMonitor *, void *), + void *arg); + +/** + * + * @brief Call a function for each monitor in a monitor list. Yield + * if lots of monitors exist. + * + * The funcion 'func' will be called with a pointer to a monitor + * as first argument and 'arg' as second argument for each monitor + * in the tree referred to by 'root'. + * + * It is assumed that: + * - *yspp equals NULL on first call + * - this function is repetedly called with *yspp set + * as set when previous call returned until a non-zero + * value is returned. + * - no modifications are made on the tree between first call + * and the call that returns a non-zero value + * If the above are not true, bad things will happen. + * + * @param[in] list Pointer to monitor list + * + * @param[in] func Pointer to function to call + * + * @param[in] arg Argument to pass as second argument in + * calls to 'func' + * + * @param[in,out] vyspp Pointer to a pointer to an internal state + * used by this function. At initial call + * *yspp should be NULL. When done *yspp + * will be NULL. + * + * @param[in] limit Maximum amount of monitors to process + * before yielding. + * + * @returns A non-zero value when all monitors has been + * processed, and zero when more work is needed. + * + */ +int erts_monitor_list_foreach_yielding(ErtsMonitor *list, + void (*func)(ErtsMonitor *, void *), + void *arg, + void **vyspp, + Sint limit); + +/** + * + * @brief Delete all monitors from a monitor list and call a function for + * each monitor + * + * The funcion 'func' will be called with a pointer to a monitor + * as first argument and 'arg' as second argument for each monitor + * in the tree referred to by 'root'. + * + * @param[in,out] list Pointer to pointer to monitor list + * + * @param[in] func Pointer to function to call + * + * @param[in] arg Argument to pass as second argument in + * calls to 'func' + * + */ +void erts_monitor_list_foreach_delete(ErtsMonitor **list, + void (*func)(ErtsMonitor *, void *), + void *arg); + +/** + * + * @brief Delete all monitors from a monitor list and call a function for + * each monitor + * + * The funcion 'func' will be called with a pointer to a monitor + * as first argument and 'arg' as second argument for each monitor + * in the tree referred to by 'root'. + * + * It is assumed that: + * - *yspp equals NULL on first call + * - this function is repetededly called with *yspp set + * as set when previous call returned until a non-zero + * value is returned. + * - no modifications are made on the tree between first + * and the call that returns a non-zero value + * If the above are not true, bad things will happen. + * + * @param[in,out] list Pointer to pointer to monitor list + * + * @param[in] func Pointer to function to call + * + * @param[in] arg Argument to pass as second argument in + * calls to 'func' + * + * @param[in,out] vyspp Pointer to a pointer to an internal state + * used by this function. At initial call + * *yspp should be NULL. When done *yspp + * will be NULL. + * + * @param[in] limit Maximum amount of monitors to process + * before yielding. + * + * @returns A non-zero value when all monitors has been + * processed, and zero when more work is needed. + * + */ +int erts_monitor_list_foreach_delete_yielding(ErtsMonitor **list, + void (*func)(ErtsMonitor *, void *), + void *arg, + void **vyspp, + Sint limit); + +/* + * --- Misc monitor operations --- + */ + +/** + * + * @brief Create a monitor + * + * Can create all types of monitors exept for suspend monitors + * + * When the funcion is called it is assumed that: + * - 'ref' is an internal ordinary reference if type is ERTS_MON_TYPE_PROC, + * ERTS_MON_TYPE_PORT, ERTS_MON_TYPE_TIME_OFFSET, or ERTS_MON_TYPE_RESOURCE + * - 'ref' is NIL if type is ERTS_MON_TYPE_NODE or ERTS_MON_TYPE_NODES + * - 'ref' is and ordinary internal reference or an external reference if + * type is ERTS_MON_TYPE_DIST_PROC + * - 'name' is an atom or NIL if type is ERTS_MON_TYPE_PROC, + * ERTS_MON_TYPE_PORT, or ERTS_MON_TYPE_DIST_PROC + * - 'name is NIL if type is ERTS_MON_TYPE_TIME_OFFSET, ERTS_MON_TYPE_RESOURCE, + * ERTS_MON_TYPE_NODE, or ERTS_MON_TYPE_NODES + * If the above is not true, bad things will happen. + * + * @param[in] type ERTS_MON_TYPE_PROC, ERTS_MON_TYPE_PORT, + * ERTS_MON_TYPE_TIME_OFFSET, ERTS_MON_TYPE_DIST_PROC, + * ERTS_MON_TYPE_RESOURCE, ERTS_MON_TYPE_NODE, + * or ERTS_MON_TYPE_NODES + * + * @param[in] ref A reference or NIL depending on type + * + * @param[in] origin The key of the origin + * + * @param[in] target The key of the target + * + */ +ErtsMonitorData *erts_monitor_create(Uint16 type, Eterm ref, Eterm origin, + Eterm target, Eterm name); + +/** + * + * @brief Get pointer to monitor data structure + * + * @param[in] mon Pointer to monitor + * + * @returns Pointer to monitor data structure + * + */ +ERTS_GLB_INLINE ErtsMonitorData *erts_monitor_to_data(ErtsMonitor *mon); + +/** + * + * @brief Check if monitor is a target monitor + * + * @param[in] mon Pointer to monitor to check + * + * @returns A non-zero value if target monitor; + * otherwise zero + * + */ +ERTS_GLB_INLINE int erts_monitor_is_target(ErtsMonitor *mon); + +/** + * + * @brief Check if monitor is an origin monitor + * + * @param[in] mon Pointer to monitor to check + * + * @returns A non-zero value if origin monitor; + * otherwise zero + * + */ +ERTS_GLB_INLINE int erts_monitor_is_origin(ErtsMonitor *mon); + +/** + * + * @brief Check if monitor is in tree or list + * + * @param[in] mon Pointer to monitor to check + * + * @returns A non-zero value if in tree or list; + * otherwise zero + * + */ +ERTS_GLB_INLINE int erts_monitor_is_in_table(ErtsMonitor *mon); + +/** + * + * @brief Release monitor + * + * When both the origin and the target part of the monitor have + * been released the monitor structure will be deallocated. + * + * When the funcion is called it is assumed that: + * - 'mon' monitor is not part of any list or tree + * - 'mon' is not referred to by any other structures + * If the above are not true, bad things will happen. + * + * @param[in] mon Pointer to monitor + * + */ +ERTS_GLB_INLINE void erts_monitor_release(ErtsMonitor *mon); + +/** + * + * @brief Release both target and origin monitor structures simultaneously + * + * Release both the origin and target parts of the monitor + * simultaneously and deallocate the structure. + * + * When the funcion is called it is assumed that: + * - Neither the origin part nor the target part of the monitor + * are not part of any list or tree + * - Neither the origin part nor the target part of the monitor + * are referred to by any other structures + * If the above are not true, bad things will happen. + * + * @param[in] mdp Pointer to monitor data structure + * + */ +ERTS_GLB_INLINE void erts_monitor_release_both(ErtsMonitorData *mdp); + +/** + * + * @brief Insert monitor in dist monitor tree or list + * + * When the funcion is called it is assumed that: + * - 'mon' monitor is not part of any list or tree + * If the above is not true, bad things will happen. + * + * @param[in] mon Pointer to monitor + * + * @param[in] dist Pointer to dist structure + * + * @returns A non-zero value if inserted; + * otherwise, zero. The monitor + * is not inserted if the dist + * structure has been set in a + * dead state. + * + */ +ERTS_GLB_INLINE int erts_monitor_dist_insert(ErtsMonitor *mon, ErtsMonLnkDist *dist); + +/** + * + * @brief Delete monitor from dist monitor tree or list + * + * When the funcion is called it is assumed that: + * - 'mon' monitor earler has been inserted into 'dist' + * If the above is not true, bad things will happen. + * + * @param[in] mon Pointer to monitor + * + * @param[in] dist Pointer to dist structure + * + * @returns A non-zero value if deleted; + * otherwise, zero. The monitor + * is not deleted if the dist + * structure has been set in a + * dead state or if it has already + * been deleted. + * + */ +ERTS_GLB_INLINE int erts_monitor_dist_delete(ErtsMonitor *mon); + +/** + * + * @brief Set dead dist structure on monitor + * + * @param[in] mon Pointer to monitor + * + * @param[in] nodename Name of remote node + * + */ +void +erts_monitor_set_dead_dist(ErtsMonitor *mon, Eterm nodename); + +/** + * + * @brief Get charged size of monitor + * + * If the other side of the monitor has been released, the + * whole size of the monitor data structure is returned; otherwise, + * half of the size is returned. + * + * When the funcion is called it is assumed that: + * - 'mon' has not been released + * If the above is not true, bad things will happen. + * + * @param[in] mon Pointer to monitor + * + * @returns Charged size in bytes + * + */ +Uint erts_monitor_size(ErtsMonitor *mon); + + +/* internal function... */ +void erts_monitor_destroy__(ErtsMonitorData *mdp); + +/* implementations for globally inlined monitor functions... */ +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE int +erts_monitor_is_target(ErtsMonitor *mon) +{ + return !!(mon->flags & ERTS_ML_FLG_TARGET); +} + +ERTS_GLB_INLINE int +erts_monitor_is_origin(ErtsMonitor *mon) +{ + return !(mon->flags & ERTS_ML_FLG_TARGET); +} + +ERTS_GLB_INLINE int +erts_monitor_is_in_table(ErtsMonitor *mon) +{ + return !!(mon->flags & ERTS_ML_FLG_IN_TABLE); +} + +ERTS_GLB_INLINE void +erts_monitor_list_insert(ErtsMonitor **list, ErtsMonitor *mon) +{ + erts_ml_dl_list_insert__((ErtsMonLnkNode **) list, (ErtsMonLnkNode *) mon); +} + +ERTS_GLB_INLINE void +erts_monitor_list_delete(ErtsMonitor **list, ErtsMonitor *mon) +{ + erts_ml_dl_list_delete__((ErtsMonLnkNode **) list, (ErtsMonLnkNode *) mon); +} + +ERTS_GLB_INLINE ErtsMonitor * +erts_monitor_list_first(ErtsMonitor *list) +{ + return (ErtsMonitor *) erts_ml_dl_list_first__((ErtsMonLnkNode *) list); +} + +ERTS_GLB_INLINE ErtsMonitor * +erts_monitor_list_last(ErtsMonitor *list) +{ + return (ErtsMonitor *) erts_ml_dl_list_last__((ErtsMonLnkNode *) list); +} + +#ifdef ERTS_ML_DEBUG +extern size_t erts_monitor_origin_offset; +extern size_t erts_monitor_origin_key_offset; +extern size_t erts_monitor_target_offset; +extern size_t erts_monitor_target_key_offset; +extern size_t erts_monitor_node_key_offset; +#endif + +ERTS_GLB_INLINE ErtsMonitorData * +erts_monitor_to_data(ErtsMonitor *mon) +{ + ErtsMonitorData *mdp = erts_ml_node_to_main_struct__((ErtsMonLnkNode *) mon); + +#ifdef ERTS_ML_DEBUG + ERTS_ML_ASSERT(!(mdp->origin.flags & ERTS_ML_FLG_TARGET)); + ERTS_ML_ASSERT(erts_monitor_origin_offset == (size_t) mdp->origin.offset); + ERTS_ML_ASSERT(!!(mdp->target.flags & ERTS_ML_FLG_TARGET)); + ERTS_ML_ASSERT(erts_monitor_target_offset == (size_t) mdp->target.offset); + if (mon->type == ERTS_MON_TYPE_NODE || mon->type == ERTS_MON_TYPE_NODES) { + ERTS_ML_ASSERT(erts_monitor_node_key_offset == (size_t) mdp->origin.key_offset); + ERTS_ML_ASSERT(erts_monitor_node_key_offset == (size_t) mdp->target.key_offset); + } + else { + ERTS_ML_ASSERT(erts_monitor_origin_key_offset == (size_t) mdp->origin.key_offset); + ERTS_ML_ASSERT(erts_monitor_target_key_offset == (size_t) mdp->target.key_offset); + } +#endif + + return mdp; +} + +ERTS_GLB_INLINE void +erts_monitor_release(ErtsMonitor *mon) +{ + ErtsMonitorData *mdp = erts_monitor_to_data(mon); + ERTS_ML_ASSERT(!(mon->flags & ERTS_ML_FLG_IN_TABLE)); + ERTS_ML_ASSERT(erts_atomic32_read_nob(&mdp->refc) > 0); + + if (erts_atomic32_dec_read_nob(&mdp->refc) == 0) + erts_monitor_destroy__(mdp); +} + +ERTS_GLB_INLINE void +erts_monitor_release_both(ErtsMonitorData *mdp) +{ + ERTS_ML_ASSERT((mdp->origin.flags & ERTS_ML_FLGS_SAME) + == (mdp->target.flags & ERTS_ML_FLGS_SAME)); + ERTS_ML_ASSERT(!(mdp->origin.flags & ERTS_ML_FLG_IN_TABLE)); + ERTS_ML_ASSERT(!(mdp->target.flags & ERTS_ML_FLG_IN_TABLE)); + ERTS_ML_ASSERT(erts_atomic32_read_nob(&mdp->refc) >= 2); + + if (erts_atomic32_add_read_nob(&mdp->refc, (erts_aint32_t) -2) == 0) + erts_monitor_destroy__(mdp); +} + +ERTS_GLB_INLINE int +erts_monitor_dist_insert(ErtsMonitor *mon, ErtsMonLnkDist *dist) +{ + ErtsMonitorDataExtended *mdep; + int insert; + + ERTS_ML_ASSERT(mon->flags & ERTS_ML_FLG_EXTENDED); + ERTS_ML_ASSERT(mon->type == ERTS_MON_TYPE_DIST_PROC + || mon->type == ERTS_MON_TYPE_NODE); + + mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon); + + ERTS_ML_ASSERT(!mdep->dist); + ERTS_ML_ASSERT(dist); + mdep->dist = dist; + + erts_mon_link_dist_inc_refc(dist); + + erts_mtx_lock(&dist->mtx); + + insert = dist->alive; + if (insert) { + if ((mon->flags & (ERTS_ML_FLG_NAME + | ERTS_ML_FLG_TARGET)) == ERTS_ML_FLG_NAME) + erts_monitor_tree_insert(&dist->orig_name_monitors, mon); + else + erts_monitor_list_insert(&dist->monitors, mon); + } + + erts_mtx_unlock(&dist->mtx); + + return insert; +} + +ERTS_GLB_INLINE int +erts_monitor_dist_delete(ErtsMonitor *mon) +{ + ErtsMonitorDataExtended *mdep; + ErtsMonLnkDist *dist; + Uint16 flags; + int delete; + + ERTS_ML_ASSERT(mon->flags & ERTS_ML_FLG_EXTENDED); + ERTS_ML_ASSERT(mon->type == ERTS_MON_TYPE_DIST_PROC + || mon->type == ERTS_MON_TYPE_NODE); + + mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon); + dist = mdep->dist; + ERTS_ML_ASSERT(dist); + + erts_mtx_lock(&dist->mtx); + + flags = mon->flags; + delete = !!dist->alive & !!(flags & ERTS_ML_FLG_IN_TABLE); + if (delete) { + if ((flags & (ERTS_ML_FLG_NAME + | ERTS_ML_FLG_TARGET)) == ERTS_ML_FLG_NAME) + erts_monitor_tree_delete(&dist->orig_name_monitors, mon); + else + erts_monitor_list_delete(&dist->monitors, mon); + } + + erts_mtx_unlock(&dist->mtx); + + return delete; +} + + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +/* suspend monitors... */ +ErtsMonitorSuspend *erts_monitor_suspend_create(Eterm pid); +ErtsMonitorSuspend *erts_monitor_suspend_tree_lookup_create(ErtsMonitor **root, + int *created, + Eterm pid); +void erts_monitor_suspend_destroy(ErtsMonitorSuspend *msp); + +ERTS_GLB_INLINE ErtsMonitorSuspend *erts_monitor_suspend(ErtsMonitor *mon); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE ErtsMonitorSuspend *erts_monitor_suspend(ErtsMonitor *mon) +{ + ERTS_ML_ASSERT(!mon || mon->type == ERTS_MON_TYPE_SUSPEND); + return (ErtsMonitorSuspend *) mon; +} + +#endif + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Link Operations * +\* */ + +typedef struct ErtsMonLnkNode__ ErtsLink; + +typedef struct { + ErtsLink a; + ErtsLink b; + erts_atomic32_t refc; +} ErtsLinkData; + +typedef struct { + ErtsLinkData ld; + struct erl_off_heap_header *ohhp; + ErtsMonLnkDist *dist; + Eterm heap[1]; /* heap start... */ +} ErtsLinkDataExtended; + +/* + * --- Link tree operations --- + */ + +/** + * + * @brief Lookup a link in a link tree + * + * + * @param[in] root Pointer to root of link tree + * + * @param[in] key Key of link to lookup + * + * @returns Pointer to a link with the + * key 'key', or NULL if no such + * link was found + * + */ +ErtsLink *erts_link_tree_lookup(ErtsLink *root, Eterm item); + +/** + * + * @brief Lookup or insert a link in a link tree + * + * When the funcion is called it is assumed that: + * - 'lnk' link is not part of any tree or list + * If the above is not true, bad things will happen. + * + * @param[in,out] root Pointer to pointer to root of link tree + * + * @param[in] lnk Link to insert if no link + * with the same key already exists + * + * @returns Pointer to a link with the + * key 'key'. If no link with the key + * 'key' was found and 'lnk' was inserted + * 'lnk' is returned. + * + */ +ErtsLink *erts_link_tree_lookup_insert(ErtsLink **root, ErtsLink *lnk); + +/** + * + * @brief Lookup or create a link in a link tree. + * + * Looks up a link with the key 'other' in the link tree. If it is not + * found, creates and insert a link with the key 'other'. + * + * @param[in,out] root Pointer to pointer to root of link tree + * + * @param[out] created Pointer to integer. The integer is set to + * a non-zero value if no link with key + * 'other' was found, and a new link + * was created. If a link was found, it + * is set to zero. + * + * @param[in] type Type of link + * + * @param[in] this Id of this entity + * + * @param[in] other Id of other entity + * + */ +ErtsLink *erts_link_tree_lookup_create(ErtsLink **root, int *created, + Uint16 type, Eterm this, Eterm other); + +/** + * + * @brief Insert a link in a link tree + * + * When the funcion is called it is assumed that: + * - no links with the same key that 'lnk' exist in the tree + * - 'lnk' is not part of any list of tree + * If the above are not true, bad things will happen. + * + * @param[in,out] root Pointer to pointer to root of link tree + * + * @param[in] lnk Link to insert. + * + */ +void erts_link_tree_insert(ErtsLink **root, ErtsLink *lnk); + +/** + * + * @brief Replace a link in a link tree + * + * When the funcion is called it is assumed that: + * - 'old' link and 'new' link have exactly the same key + * - 'old' link is part of the tree + * - 'new' link is not part of any tree or list + * If the above are not true, bad things will happen. + * + * @param[in,out] root Pointer to pointer to root of link tree + * + * @param[in] old Link to remove from the tree + * + * @param[in] new Link to insert into the tree + * + */ +void erts_link_tree_replace(ErtsLink **root, ErtsLink *old, ErtsLink *new); + +/** + * + * @brief Replace a link in a link tree if key already exist based on adress + * + * Inserts the link 'lnk' in the tree if no link with the same key + * already exists in tree. If a link with the same key exists in + * the tree and 'lnk' has a lower address than the link in the + * tree, the existing link in the tree is replaced by 'lnk'. + * + * When the funcion is called it is assumed that: + * - 'lnk' link is not part of any tree or list + * If the above are not true, bad things will happen. + * + * @param[in,out] root Pointer to pointer to root of link tree + * + * @param[in] lnk Link to insert into the tree + * + * @returns A pointer to the link that is not part + * of the tree after this operation. + * + */ +ERTS_GLB_INLINE ErtsLink *erts_link_tree_insert_addr_replace(ErtsLink **root, + ErtsLink *lnk); + +/** + * + * @brief Delete a link from a link tree + * + * When the funcion is called it is assumed that: + * - 'lnk' link is part of the tree + * If the above is not true, bad things will happen. + * + * @param[in,out] root Pointer to pointer to root of link tree + * + * @param[in] lnk Link to remove from the tree + * + */ +void erts_link_tree_delete(ErtsLink **root, ErtsLink *lnk); + +/** + * + * @brief Delete a link from a link tree based on key + * + * If link 'lnk' is in the tree, 'lnk' is deleted from the tree. + * If link 'lnk' is not in the tree, another link with the same + * key as 'lnk' is deleted from the tree if such a link exist. + * + * When the funcion is called it is assumed that: + * - if 'lnk' link is part of a tree or list, it is part of this tree + * If the above is not true, bad things will happen. + * + * @param[in,out] root Pointer to pointer to root of link tree + * + * @param[in] lnk Link to remove from the tree + * + * @returns A pointer to the link that was deleted + * from the tree, or NULL in case no link + * was deleted from the tree + * + */ +ERTS_GLB_INLINE ErtsLink *erts_link_tree_key_delete(ErtsLink **root, ErtsLink *lnk); + +/** + * + * @brief Call a function for each link in a link tree + * + * The funcion 'func' will be called with a pointer to a link + * as first argument and 'arg' as second argument for each link + * in the tree referred to by 'root'. + * + * @param[in] root Pointer to root of link tree + * + * @param[in] func Pointer to function to call + * + * @param[in] arg Argument to pass as second argument in + * calls to 'func' + * + */ +void erts_link_tree_foreach(ErtsLink *root, + void (*func)(ErtsLink *, void *), + void *arg); + +/** + * + * @brief Call a function for each link in a link tree. Yield if lots + * of links exist. + * + * The funcion 'func' will be called with a pointer to a link + * as first argument and 'arg' as second argument for each link + * in the tree referred to by 'root'. + * + * It is assumed that: + * - *yspp equals NULL on first call + * - this function is repetedly called with *yspp set + * as set when previous call returned until a non-zero + * value is returned. + * - no modifications are made on the tree between first call + * and the call that returns a non-zero value + * If the above are not true, bad things will happen. + * + * @param[in] root Pointer to root of link tree + * + * @param[in] func Pointer to function to call + * + * @param[in] arg Argument to pass as second argument in + * calls to 'func' + * + * @param[in,out] vyspp Pointer to a pointer to an internal state + * used by this function. At initial call + * *yspp should be NULL. When done *yspp + * will be NULL. + * + * @param[in] limit Maximum amount of links to process + * before yielding. + * + * @returns A non-zero value when all links has been + * processed, and zero when more work is needed. + * + */ +int erts_link_tree_foreach_yielding(ErtsLink *root, + void (*func)(ErtsLink *, void *), + void *arg, + void **vyspp, + Sint limit); + +/** + * + * @brief Delete all links from a link tree and call a function for + * each link + * + * The funcion 'func' will be called with a pointer to a link + * as first argument and 'arg' as second argument for each link + * in the tree referred to by 'root'. + * + * @param[in,out] root Pointer to pointer to root of link tree + * + * @param[in] func Pointer to function to call + * + * @param[in] arg Argument to pass as second argument in + * calls to 'func' + * + */ +void erts_link_tree_foreach_delete(ErtsLink **root, + void (*func)(ErtsLink *, void *), + void *arg); + +/** + * + * @brief Delete all links from a link tree and call a function for + * each link + * + * The funcion 'func' will be called with a pointer to a link + * as first argument and 'arg' as second argument for each link + * in the tree referred to by 'root'. + * + * It is assumed that: + * - *yspp equals NULL on first call + * - this function is repetededly called with *yspp set + * as set when previous call returned until a non-zero + * value is returned. + * - no modifications are made on the tree between first call + * and the call that returns a non-zero value + * If the above are not true, bad things will happen. + * + * @param[in,out] root Pointer to pointer to root of link tree + * + * @param[in] func Pointer to function to call + * + * @param[in] arg Argument to pass as second argument in + * calls to 'func' + * + * @param[in,out] vyspp Pointer to a pointer to an internal state + * used by this function. At initial call + * *yspp should be NULL. When done *yspp + * will be NULL. + * + * @param[in] limit Maximum amount of links to process + * before yielding. + * + * @returns A non-zero value when all links has been + * processed, and zero when more work is needed. + * + */ +int erts_link_tree_foreach_delete_yielding(ErtsLink **root, + void (*func)(ErtsLink *, void *), + void *arg, + void **vyspp, + Sint limit); + +/* + * --- Link list operations --- + */ + +/** + * + * @brief Insert a link in a link list + * + * When the funcion is called it is assumed that: + * - 'lnk' link is not part of any list or tree + * If the above is not true, bad things will happen. + * + * @param[in,out] list Pointer to pointer to link list + * + * @param[in] lnk Link to insert + * + */ +ERTS_GLB_INLINE void erts_link_list_insert(ErtsLink **list, ErtsLink *lnk); + +/** + * + * @brief Delete a link from a link list + * + * When the funcion is called it is assumed that: + * - 'lnk' link is part of the list + * If the above is not true, bad things will happen. + * + * @param[in,out] list Pointer to pointer to link list + * + * @param[in] lnk Link to remove from the list + * + */ +ERTS_GLB_INLINE void erts_link_list_delete(ErtsLink **list, ErtsLink *lnk); + +/** + * + * @brief Get a pointer to first link in a link list + * + * The link will still remain in the list after the return + * + * @param[in] list Pointer to link list + * + * @returns Pointer to first link in list if + * list is not empty. If list is empty + * NULL is returned. + * + */ +ERTS_GLB_INLINE ErtsLink *erts_link_list_first(ErtsLink *list); + +/** + * + * @brief Get a pointer to last link in a link list + * + * The link will still remain in the list after the return + * + * @param[in] list Pointer to link list + * + * @returns Pointer to last link in list if + * list is not empty. If list is empty + * NULL is returned. + * + */ +ERTS_GLB_INLINE ErtsLink *erts_link_list_last(ErtsLink *list); + +/** + * + * @brief Call a function for each link in a link list + * + * The funcion 'func' will be called with a pointer to a link + * as first argument and 'arg' as second argument for each link + * in the tree referred to by 'list'. + * + * @param[in] list Pointer to root of link list + * + * @param[in] func Pointer to function to call + * + * @param[in] arg Argument to pass as second argument in + * calls to 'func' + * + */ +void erts_link_list_foreach(ErtsLink *list, + void (*func)(ErtsLink *, void *), + void *arg); + +/** + * + * @brief Call a function for each link in a link list. Yield + * if lots of links exist. + * + * The funcion 'func' will be called with a pointer to a link + * as first argument and 'arg' as second argument for each link + * in the tree referred to by 'root'. + * + * It is assumed that: + * - *yspp equals NULL on first call + * - this function is repetedly called with *yspp set + * as set when previous call returned until a non-zero + * value is returned. + * - no modifications are made on the tree between first call + * and the call that returns a non-zero value + * If the above are not true, bad things will happen. + * + * @param[in] list Pointer to link list + * + * @param[in] func Pointer to function to call + * + * @param[in] arg Argument to pass as second argument in + * calls to 'func' + * + * @param[in,out] vyspp Pointer to a pointer to an internal state + * used by this function. At initial call + * *yspp should be NULL. When done *yspp + * will be NULL. + * + * @param[in] limit Maximum amount of links to process + * before yielding. + * + * @returns A non-zero value when all links has been + * processed, and zero when more work is needed. + * + */ +int erts_link_list_foreach_yielding(ErtsLink *list, + void (*func)(ErtsLink *, void *), + void *arg, + void **vyspp, + Sint limit); + +/** + * + * @brief Delete all links from a link list and call a function for + * each link + * + * The funcion 'func' will be called with a pointer to a link + * as first argument and 'arg' as second argument for each link + * in the tree referred to by 'root'. + * + * @param[in,out] list Pointer to pointer to link list + * + * @param[in] func Pointer to function to call + * + * @param[in] arg Argument to pass as second argument in + * calls to 'func' + * + */ +void erts_link_list_foreach_delete(ErtsLink **list, + void (*func)(ErtsLink *, void *), + void *arg); + +/** + * + * @brief Delete all links from a link list and call a function for + * each link + * + * The funcion 'func' will be called with a pointer to a link + * as first argument and 'arg' as second argument for each link + * in the tree referred to by 'root'. + * + * It is assumed that: + * - *yspp equals NULL on first call + * - this function is repetededly called with *yspp set + * as set when previous call returned until a non-zero + * value is returned. + * - no modifications are made on the tree between first + * and the call that returns a non-zero value + * If the above are not true, bad things will happen. + * + * @param[in,out] list Pointer to pointer to link list + * + * @param[in] func Pointer to function to call + * + * @param[in] arg Argument to pass as second argument in + * calls to 'func' + * + * @param[in,out] vyspp Pointer to a pointer to an internal state + * used by this function. At initial call + * *yspp should be NULL. When done *yspp + * will be NULL. + * + * @param[in] limit Maximum amount of links to process + * before yielding. + * + * @returns A non-zero value when all links has been + * processed, and zero when more work is needed. + * + */ +int erts_link_list_foreach_delete_yielding(ErtsLink **list, + void (*func)(ErtsLink *, void *), + void *arg, + void **vyspp, + Sint limit); + +/* + * --- Misc link operations --- + */ + +/** + * + * @brief Create a link + * + * Can create all types of links + * + * When the funcion is called it is assumed that: + * - 'ref' is an internal ordinary reference if type is ERTS_MON_TYPE_PROC, + * ERTS_MON_TYPE_PORT, ERTS_MON_TYPE_TIME_OFFSET, or ERTS_MON_TYPE_RESOURCE + * - 'ref' is NIL if type is ERTS_MON_TYPE_NODE or ERTS_MON_TYPE_NODES + * - 'ref' is and ordinary internal reference or an external reference if + * type is ERTS_MON_TYPE_DIST_PROC + * - 'name' is an atom or NIL if type is ERTS_MON_TYPE_PROC, + * ERTS_MON_TYPE_PORT, or ERTS_MON_TYPE_DIST_PROC + * - 'name is NIL if type is ERTS_MON_TYPE_TIME_OFFSET, ERTS_MON_TYPE_RESOURCE, + * ERTS_MON_TYPE_NODE, or ERTS_MON_TYPE_NODES + * If the above is not true, bad things will happen. + * + * @param[in] type ERTS_MON_TYPE_PROC, ERTS_MON_TYPE_PORT, + * ERTS_MON_TYPE_TIME_OFFSET, ERTS_MON_TYPE_DIST_PROC, + * ERTS_MON_TYPE_RESOURCE, ERTS_MON_TYPE_NODE, + * or ERTS_MON_TYPE_NODES + * + * @param[in] a The key of entity a. Link structure a will + * have field other.item set to 'b'. + * + * @param[in] b The key of entity b. Link structure b will + * have field other.item set to 'a'. + * + */ +ErtsLinkData *erts_link_create(Uint16 type, Eterm a, Eterm b); + +/** + * + * @brief Get pointer to link data structure + * + * @param[in] lnk Pointer to link + * + * @returns Pointer to link data structure + * + */ +ERTS_GLB_INLINE ErtsLinkData *erts_link_to_data(ErtsLink *lnk); + +/** + * + * @brief Get pointer to the other link structure part of the link + * + * @param[in] lnk Pointer to link + * + * @param[out] ldpp Pointer to pointer to link data structure, + * if a non-NULL value is passed in the call + * + * @returns Pointer to other link + * + */ +ERTS_GLB_INLINE ErtsLink *erts_link_to_other(ErtsLink *lnk, ErtsLinkData **ldpp); + +/** + * + * @brief Check if link is in tree or list + * + * @param[in] lnk Pointer to lnk to check + * + * @returns A non-zero value if in tree or list; + * otherwise zero + * + */ +ERTS_GLB_INLINE int erts_link_is_in_table(ErtsLink *lnk); + +/** + * + * @brief Release link + * + * When both link halves part of the link have been released the link + * structure will be deallocated. + * + * When the funcion is called it is assumed that: + * - 'lnk' link is not part of any list or tree + * - 'lnk' is not referred to by any other structures + * If the above are not true, bad things will happen. + * + * @param[in] lnk Pointer to link + * + */ +ERTS_GLB_INLINE void erts_link_release(ErtsLink *lnk); + +/** + * + * @brief Release both link halves of a link simultaneously + * + * Release both halves of a link simultaneously and deallocate + * the structure. + * + * When the funcion is called it is assumed that: + * - Neither of the parts of the link are part of any list or tree + * - Neither of the parts of the link or the link data structure + * are referred to by any other structures + * If the above are not true, bad things will happen. + * + * @param[in] mdp Pointer to link data structure + * + */ +ERTS_GLB_INLINE void erts_link_release_both(ErtsLinkData *ldp); + +/** + * + * @brief Insert link in dist link list + * + * When the funcion is called it is assumed that: + * - 'lnk' link is not part of any list or tree + * If the above is not true, bad things will happen. + * + * @param[in] lnk Pointer to link + * + * @param[in] dist Pointer to dist structure + * + * @returns A non-zero value if inserted; + * otherwise, zero. The link + * is not inserted if the dist + * structure has been set in a + * dead state. + * + */ +ERTS_GLB_INLINE int erts_link_dist_insert(ErtsLink *lnk, ErtsMonLnkDist *dist); + +/** + * + * @brief Delete link from dist link list + * + * When the funcion is called it is assumed that: + * - 'lnk' link earler has been inserted into 'dist' + * If the above is not true, bad things will happen. + * + * @param[in] lnk Pointer to link + * + * @param[in] dist Pointer to dist structure + * + * @returns A non-zero value if deleted; + * otherwise, zero. The link + * is not deleted if the dist + * structure has been set in a + * dead state or if it has already + * been deleted. + * + */ +ERTS_GLB_INLINE int erts_link_dist_delete(ErtsLink *lnk); + +/** + * + * @brief Set dead dist structure on link + * + * @param[in] lnk Pointer to link + * + * @param[in] nodename Name of remote node + * + */ +void +erts_link_set_dead_dist(ErtsLink *lnk, Eterm nodename); + +/** + * + * @brief Get charged size of link + * + * If the other side of the link has been released, the + * whole size of the link data structure is returned; otherwise, + * half of the size is returned. + * + * When the funcion is called it is assumed that: + * - 'lnk' has not been released + * If the above is not true, bad things will happen. + * + * @param[in] lnk Pointer to link + * + * @returns Charged size in bytes + * + */ +Uint erts_link_size(ErtsLink *lnk); + +/* internal function... */ +void erts_link_destroy__(ErtsLinkData *ldp); + +/* implementations for globally inlined link functions... */ +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +#ifdef ERTS_ML_DEBUG +extern size_t erts_link_a_offset; +extern size_t erts_link_b_offset; +extern size_t erts_link_key_offset; +#endif + +ERTS_GLB_INLINE ErtsLinkData * +erts_link_to_data(ErtsLink *lnk) +{ + ErtsLinkData *ldp = erts_ml_node_to_main_struct__((ErtsMonLnkNode *) lnk); + +#ifdef ERTS_ML_DEBUG + ERTS_ML_ASSERT(erts_link_a_offset == (size_t) ldp->a.offset); + ERTS_ML_ASSERT(erts_link_key_offset == (size_t) ldp->a.key_offset); + ERTS_ML_ASSERT(erts_link_b_offset == (size_t) ldp->b.offset); + ERTS_ML_ASSERT(erts_link_key_offset == (size_t) ldp->b.key_offset); +#endif + + return ldp; +} + +ERTS_GLB_INLINE ErtsLink * +erts_link_to_other(ErtsLink *lnk, ErtsLinkData **ldpp) +{ + ErtsLinkData *ldp = erts_link_to_data(lnk); + if (ldpp) + *ldpp = ldp; + return lnk == &ldp->a ? &ldp->b : &ldp->a; +} + +ERTS_GLB_INLINE int +erts_link_is_in_table(ErtsLink *lnk) +{ + return !!(lnk->flags & ERTS_ML_FLG_IN_TABLE); +} + +ERTS_GLB_INLINE void +erts_link_list_insert(ErtsLink **list, ErtsLink *lnk) +{ + erts_ml_dl_list_insert__((ErtsMonLnkNode **) list, (ErtsMonLnkNode *) lnk); +} + +ERTS_GLB_INLINE void +erts_link_list_delete(ErtsLink **list, ErtsLink *lnk) +{ + erts_ml_dl_list_delete__((ErtsMonLnkNode **) list, (ErtsMonLnkNode *) lnk); +} + +ERTS_GLB_INLINE ErtsLink * +erts_link_list_first(ErtsLink *list) +{ + return (ErtsLink *) erts_ml_dl_list_first__((ErtsMonLnkNode *) list); +} + +ERTS_GLB_INLINE ErtsLink * +erts_link_list_last(ErtsLink *list) +{ + return (ErtsLink *) erts_ml_dl_list_last__((ErtsMonLnkNode *) list); +} + +ERTS_GLB_INLINE void +erts_link_release(ErtsLink *lnk) +{ + ErtsLinkData *ldp = erts_link_to_data(lnk); + ERTS_ML_ASSERT(!(lnk->flags & ERTS_ML_FLG_IN_TABLE)); + ERTS_ML_ASSERT(erts_atomic32_read_nob(&ldp->refc) > 0); + if (erts_atomic32_dec_read_nob(&ldp->refc) == 0) + erts_link_destroy__(ldp); +} + +ERTS_GLB_INLINE void +erts_link_release_both(ErtsLinkData *ldp) +{ + ERTS_ML_ASSERT(!(ldp->a.flags & ERTS_ML_FLG_IN_TABLE)); + ERTS_ML_ASSERT(!(ldp->b.flags & ERTS_ML_FLG_IN_TABLE)); + ERTS_ML_ASSERT(erts_atomic32_read_nob(&ldp->refc) >= 2); + if (erts_atomic32_add_read_nob(&ldp->refc, (erts_aint32_t) -2) == 0) + erts_link_destroy__(ldp); +} + +ERTS_GLB_INLINE ErtsLink * +erts_link_tree_insert_addr_replace(ErtsLink **root, ErtsLink *lnk) +{ + ErtsLink *lnk2 = erts_link_tree_lookup_insert(root, lnk); + if (!lnk2) + return NULL; + if (lnk2 < lnk) + return lnk; + erts_link_tree_replace(root, lnk2, lnk); + return lnk2; +} + +ERTS_GLB_INLINE ErtsLink * +erts_link_tree_key_delete(ErtsLink **root, ErtsLink *lnk) +{ + ErtsLink *dlnk; + if (erts_link_is_in_table(lnk)) + dlnk = lnk; + else + dlnk = erts_link_tree_lookup(*root, lnk->other.item); + if (dlnk) + erts_link_tree_delete(root, dlnk); + return dlnk; +} + +ERTS_GLB_INLINE int +erts_link_dist_insert(ErtsLink *lnk, ErtsMonLnkDist *dist) +{ + ErtsLinkDataExtended *ldep; + int insert; + + ERTS_ML_ASSERT(lnk->flags & ERTS_ML_FLG_EXTENDED); + ERTS_ML_ASSERT(lnk->type == ERTS_LNK_TYPE_DIST_PROC); + + ldep = (ErtsLinkDataExtended *) erts_link_to_data(lnk); + + ERTS_ML_ASSERT(!ldep->dist); + ERTS_ML_ASSERT(dist); + ldep->dist = dist; + + erts_mon_link_dist_inc_refc(dist); + + erts_mtx_lock(&dist->mtx); + + insert = dist->alive; + if (insert) + erts_link_list_insert(&dist->links, lnk); + + erts_mtx_unlock(&dist->mtx); + + return insert; +} + +ERTS_GLB_INLINE int +erts_link_dist_delete(ErtsLink *lnk) +{ + ErtsLinkDataExtended *ldep; + ErtsMonLnkDist *dist; + int delete; + + ERTS_ML_ASSERT(lnk->flags & ERTS_ML_FLG_EXTENDED); + ERTS_ML_ASSERT(lnk->type == ERTS_LNK_TYPE_DIST_PROC); + + ldep = (ErtsLinkDataExtended *) erts_link_to_data(lnk); + dist = ldep->dist; + if (!dist) + return -1; + + erts_mtx_lock(&dist->mtx); + + delete = !!dist->alive & !!(lnk->flags & ERTS_ML_FLG_IN_TABLE); + if (delete) + erts_link_list_delete(&dist->links, lnk); + + erts_mtx_unlock(&dist->mtx); + + return delete; +} + + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* ERL_MONITOR_LINK_H__ */ diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c deleted file mode 100644 index 1c840d89f6..0000000000 --- a/erts/emulator/beam/erl_monitors.c +++ /dev/null @@ -1,1073 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2004-2017. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -/************************************************************************** - * Monitors and links data structure manipulation. - * Monitors and links are organized as AVL trees with the reference as - * key in the monitor case and the pid of the linked process as key in the - * link case. Lookups the order of the references is somewhat special. Local - * references are strictly smaller than remote references and are sorted - * by inlined comparison functionality. Remote references are handled by the - * usual cmp function. - * Each Monitor is tagged with different tags depending on which end of the - * monitor it is. - * A monitor is removed either explicitly by reference or all monitors are - * removed when the process exits. No need to access the monitor by pid. - **************************************************************************/ - -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "sys.h" -#include "erl_vm.h" -#include "global.h" -#include "erl_process.h" -#include "error.h" -#include "erl_db.h" -#include "bif.h" -#include "big.h" -#include "erl_monitors.h" -#include "erl_bif_unique.h" - -#define STACK_NEED 50 -#define MAX_MONITORS 0xFFFFFFFFUL - -#define DIR_LEFT 0 -#define DIR_RIGHT 1 -#define DIR_END 2 - -static erts_atomic_t tot_link_lh_size; - -/* Implements the sort order in monitor trees, which is different from - the ordinary term order. - No short local ref's should ever exist (the ref is created by the bif's - in runtime), therefore: - All local ref's are less than external ref's - Local ref's are inline-compared, - External ref's are compared by cmp */ - -#if 0 -#define CMP_MON_REF(Ref1,Ref2) \ -cmp((Ref1),(Ref2)) /* XXX, the inline comparision yet to be done */ -#else -#define CMP_MON_REF(Ref1,Ref2) cmp_mon_ref((Ref1),(Ref2)) -#endif - -static ERTS_INLINE int cmp_mon_ref(Eterm ref1, Eterm ref2) -{ - Eterm *b1, *b2; - - - b1 = boxed_val(ref1); - b2 = boxed_val(ref2); - if (is_ref_thing_header(*b1)) { - if (is_ref_thing_header(*b2)) { - Uint32 *num1, *num2; - if (is_ordinary_ref_thing(b1)) { - ErtsORefThing *rtp = (ErtsORefThing *) b1; - num1 = rtp->num; - } - else { - ErtsMRefThing *mrtp = (ErtsMRefThing *) b1; - num1 = mrtp->mb->refn; - } - if (is_ordinary_ref_thing(b2)) { - ErtsORefThing *rtp = (ErtsORefThing *) b2; - num2 = rtp->num; - } - else { - ErtsMRefThing *mrtp = (ErtsMRefThing *) b2; - num2 = mrtp->mb->refn; - } - return erts_internal_ref_number_cmp(num1, num2); - } - return -1; - } - if (is_ref_thing_header(*b2)) { - return 1; - } - return CMP(ref1,ref2); -} - -#define CP_LINK_VAL(To, Hp, From) \ -do { \ - if (is_immed(From)) \ - (To) = (From); \ - else { \ - Uint i__; \ - Uint len__; \ - ASSERT((Hp)); \ - ASSERT(is_internal_ordinary_ref((From)) \ - || is_external((From))); \ - (To) = make_boxed((Hp)); \ - len__ = thing_arityval(*boxed_val((From))) + 1; \ - for(i__ = 0; i__ < len__; i__++) \ - (*((Hp)++)) = boxed_val((From))[i__]; \ - if (is_external((To))) { \ - external_thing_ptr((To))->next = NULL; \ - erts_refc_inc(&(external_thing_ptr((To))->node->refc), 2);\ - } \ - } \ -} while (0) - -static ErtsMonitor *create_monitor(Uint type, Eterm ref, UWord entity, Eterm name) -{ - Uint mon_size = ERTS_MONITOR_SIZE; - ErtsMonitor *n; - Eterm *hp; - - mon_size += NC_HEAP_SIZE(ref); - if (type != MON_NIF_TARGET && is_not_immed(entity)) { - mon_size += NC_HEAP_SIZE(entity); - } - - if (mon_size <= ERTS_MONITOR_SH_SIZE) { - n = (ErtsMonitor *) erts_alloc(ERTS_ALC_T_MONITOR_SH, - mon_size*sizeof(Uint)); - } else { - n = (ErtsMonitor *) erts_alloc(ERTS_ALC_T_MONITOR_LH, - mon_size*sizeof(Uint)); - erts_atomic_add_nob(&tot_link_lh_size, mon_size*sizeof(Uint)); - } - hp = n->heap; - - - n->left = n->right = NULL; /* Always the same initial value*/ - n->type = (Uint16) type; - n->balance = 0; /* Always the same initial value */ - n->name = name; /* atom() or [] */ - CP_LINK_VAL(n->ref, hp, ref); /*XXX Unnecessary check, never immediate*/ - if (type == MON_NIF_TARGET) - n->u.resource = (ErtsResource*)entity; - else - CP_LINK_VAL(n->u.pid, hp, (Eterm)entity); - - return n; -} - -static ErtsLink *create_link(Uint type, Eterm pid) -{ - Uint lnk_size = ERTS_LINK_SIZE; - ErtsLink *n; - Eterm *hp; - - if (is_not_immed(pid)) { - lnk_size += NC_HEAP_SIZE(pid); - } - - if (lnk_size <= ERTS_LINK_SH_SIZE) { - n = (ErtsLink *) erts_alloc(ERTS_ALC_T_NLINK_SH, - lnk_size*sizeof(Uint)); - } else { - n = (ErtsLink *) erts_alloc(ERTS_ALC_T_NLINK_LH, - lnk_size*sizeof(Uint)); - erts_atomic_add_nob(&tot_link_lh_size, lnk_size*sizeof(Uint)); - } - hp = n->heap; - - - n->left = n->right = NULL; /* Always the same initial value*/ - n->type = (Uint16) type; - n->balance = 0; /* Always the same initial value */ - if (n->type == LINK_NODE) { - ERTS_LINK_REFC(n) = 0; - } else { - ERTS_LINK_ROOT(n) = NULL; - } - CP_LINK_VAL(n->pid, hp, pid); - - return n; -} - -#undef CP_LINK_VAL - -static ErtsSuspendMonitor *create_suspend_monitor(Eterm pid) -{ - ErtsSuspendMonitor *smon = erts_alloc(ERTS_ALC_T_SUSPEND_MON, - sizeof(ErtsSuspendMonitor)); - smon->left = smon->right = NULL; /* Always the same initial value */ - smon->balance = 0; /* Always the same initial value */ - smon->pending = 0; - smon->active = 0; - smon->pid = pid; - return smon; -} - -void -erts_init_monitors(void) -{ - erts_atomic_init_nob(&tot_link_lh_size, 0); -} - -Uint -erts_tot_link_lh_size(void) -{ - return (Uint) erts_atomic_read_nob(&tot_link_lh_size); -} - -void erts_destroy_monitor(ErtsMonitor *mon) -{ - Uint mon_size = ERTS_MONITOR_SIZE; - ErlNode *node; - - ASSERT(is_not_immed(mon->ref)); - mon_size += NC_HEAP_SIZE(mon->ref); - if (is_external(mon->ref)) { - node = external_thing_ptr(mon->ref)->node; - erts_deref_node_entry(node); - } - if (mon->type != MON_NIF_TARGET && is_not_immed(mon->u.pid)) { - mon_size += NC_HEAP_SIZE(mon->u.pid); - if (is_external(mon->u.pid)) { - node = external_thing_ptr(mon->u.pid)->node; - erts_deref_node_entry(node); - } - } - if (mon_size <= ERTS_MONITOR_SH_SIZE) { - erts_free(ERTS_ALC_T_MONITOR_SH, (void *) mon); - } else { - erts_free(ERTS_ALC_T_MONITOR_LH, (void *) mon); - erts_atomic_add_nob(&tot_link_lh_size, -1*mon_size*sizeof(Uint)); - } -} - -void erts_destroy_link(ErtsLink *lnk) -{ - Uint lnk_size = ERTS_LINK_SIZE; - ErlNode *node; - - ASSERT(lnk->type == LINK_NODE || ERTS_LINK_ROOT(lnk) == NULL); - - if (is_not_immed(lnk->pid)) { - lnk_size += NC_HEAP_SIZE(lnk->pid); - if (is_external(lnk->pid)) { - node = external_thing_ptr(lnk->pid)->node; - erts_deref_node_entry(node); - } - } - if (lnk_size <= ERTS_LINK_SH_SIZE) { - erts_free(ERTS_ALC_T_NLINK_SH, (void *) lnk); - } else { - erts_free(ERTS_ALC_T_NLINK_LH, (void *) lnk); - erts_atomic_add_nob(&tot_link_lh_size, -1*lnk_size*sizeof(Uint)); - } -} - -void erts_destroy_suspend_monitor(ErtsSuspendMonitor *smon) -{ - erts_free(ERTS_ALC_T_SUSPEND_MON, smon); -} - -static void insertion_rotation(int dstack[], int dpos, - void *tstack[], int tpos, - int state) { - - ErtsMonitorOrLink **this; - ErtsMonitorOrLink *p1, *p2, *p; - int dir; - - while (state && ( dir = dstack[--dpos] ) != DIR_END) { - this = tstack[--tpos]; - p = *this; - if (dir == DIR_LEFT) { - switch (p->balance) { - case 1: - p->balance = 0; - state = 0; - break; - case 0: - p->balance = -1; - break; - case -1: /* The icky case */ - p1 = p->left; - if (p1->balance == -1) { /* Single LL rotation */ - p->left = p1->right; - p1->right = p; - p->balance = 0; - (*this) = p1; - } else { /* Double RR rotation */ - p2 = p1->right; - p1->right = p2->left; - p2->left = p1; - p->left = p2->right; - p2->right = p; - p->balance = (p2->balance == -1) ? +1 : 0; - p1->balance = (p2->balance == 1) ? -1 : 0; - (*this) = p2; - } - (*this)->balance = 0; - state = 0; - break; - } - } else { /* dir == DIR_RIGHT */ - switch (p->balance) { - case -1: - p->balance = 0; - state = 0; - break; - case 0: - p->balance = 1; - break; - case 1: - p1 = p->right; - if (p1->balance == 1) { /* Single RR rotation */ - p->right = p1->left; - p1->left = p; - p->balance = 0; - (*this) = p1; - } else { /* Double RL rotation */ - p2 = p1->left; - p1->left = p2->right; - p2->right = p1; - p->right = p2->left; - p2->left = p; - p->balance = (p2->balance == 1) ? -1 : 0; - p1->balance = (p2->balance == -1) ? 1 : 0; - (*this) = p2; - } - (*this)->balance = 0; - state = 0; - break; - } - } - } -} - -void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, UWord entity, - Eterm name) -{ - void *tstack[STACK_NEED]; - int tpos = 0; - int dstack[STACK_NEED+1]; - int dpos = 1; - int state = 0; - ErtsMonitor **this = root; - Sint c; - - ASSERT(is_internal_ordinary_ref(ref) || is_external_ref(ref)); - - dstack[0] = DIR_END; - for (;;) { - if (!*this) { /* Found our place */ - state = 1; - *this = create_monitor(type,ref,entity,name); - break; - } else if ((c = CMP_MON_REF(ref,(*this)->ref)) < 0) { - /* go left */ - dstack[dpos++] = DIR_LEFT; - tstack[tpos++] = this; - this = &((*this)->left); - } else if (c > 0) { /* go right */ - dstack[dpos++] = DIR_RIGHT; - tstack[tpos++] = this; - this = &((*this)->right); - } else { /* Equal key is an error for monitors */ - erts_exit(ERTS_ERROR_EXIT,"Insertion of already present monitor!"); - break; - } - } - insertion_rotation(dstack, dpos, tstack, tpos, state); -} - - -/* Returns 0 if OK, < 0 if already present */ -int erts_add_link(ErtsLink **root, Uint type, Eterm pid) -{ - void *tstack[STACK_NEED]; - int tpos = 0; - int dstack[STACK_NEED+1]; - int dpos = 1; - int state = 0; - ErtsLink **this = root; - Sint c; - - dstack[0] = DIR_END; - for (;;) { - if (!*this) { /* Found our place */ - state = 1; - *this = create_link(type,pid); - break; - } else if ((c = CMP(pid,(*this)->pid)) < 0) { - /* go left */ - dstack[dpos++] = DIR_LEFT; - tstack[tpos++] = this; - this = &((*this)->left); - } else if (c > 0) { /* go right */ - dstack[dpos++] = DIR_RIGHT; - tstack[tpos++] = this; - this = &((*this)->right); - } else { /* Equal key is an error for monitors */ - return -1; - } - } - insertion_rotation(dstack, dpos, tstack, tpos, state); - return 0; -} - -ErtsSuspendMonitor * -erts_add_or_lookup_suspend_monitor(ErtsSuspendMonitor **root, Eterm pid) -{ - void *tstack[STACK_NEED]; - int tpos = 0; - int dstack[STACK_NEED+1]; - int dpos = 1; - int state = 0; - ErtsSuspendMonitor **this = root; - ErtsSuspendMonitor *res; - Sint c; - - dstack[0] = DIR_END; - for (;;) { - if (!*this) { /* Found our place */ - state = 1; - res = *this = create_suspend_monitor(pid); - break; - } else if ((c = CMP(pid,(*this)->pid)) < 0) { - /* go left */ - dstack[dpos++] = DIR_LEFT; - tstack[tpos++] = this; - this = &((*this)->left); - } else if (c > 0) { /* go right */ - dstack[dpos++] = DIR_RIGHT; - tstack[tpos++] = this; - this = &((*this)->right); - } else { /* Already here... */ - ASSERT((*this)->pid == pid); - return *this; - } - } - insertion_rotation(dstack, dpos, tstack, tpos, state); - return res; -} - - -/* Returns the new or old link structure */ -ErtsLink *erts_add_or_lookup_link(ErtsLink **root, Uint type, Eterm pid) -{ - void *tstack[STACK_NEED]; - int tpos = 0; - int dstack[STACK_NEED+1]; - int dpos = 1; - int state = 0; - ErtsLink **this = root; - Sint c; - ErtsLink *ret = NULL; - - dstack[0] = DIR_END; - for (;;) { - if (!*this) { /* Found our place */ - state = 1; - *this = create_link(type,pid); - ret = *this; - break; - } else if ((c = CMP(pid,(*this)->pid)) < 0) { - /* go left */ - dstack[dpos++] = DIR_LEFT; - tstack[tpos++] = this; - this = &((*this)->left); - } else if (c > 0) { /* go right */ - dstack[dpos++] = DIR_RIGHT; - tstack[tpos++] = this; - this = &((*this)->right); - } else { /* Equal key is an error for monitors */ - return *this; - } - } - insertion_rotation(dstack, dpos, tstack, tpos, state); - return ret; -} - - -/* - * Deletion helpers - */ -static int balance_left(ErtsMonitorOrLink **this) -{ - ErtsMonitorOrLink *p, *p1, *p2; - int b1, b2, h = 1; - - p = *this; - switch (p->balance) { - case -1: - p->balance = 0; - break; - case 0: - p->balance = 1; - h = 0; - break; - case 1: - p1 = p->right; - b1 = p1->balance; - if (b1 >= 0) { /* Single RR rotation */ - p->right = p1->left; - p1->left = p; - if (b1 == 0) { - p->balance = 1; - p1->balance = -1; - h = 0; - } else { - p->balance = p1->balance = 0; - } - (*this) = p1; - } else { /* Double RL rotation */ - p2 = p1->left; - b2 = p2->balance; - p1->left = p2->right; - p2->right = p1; - p->right = p2->left; - p2->left = p; - p->balance = (b2 == 1) ? -1 : 0; - p1->balance = (b2 == -1) ? 1 : 0; - p2->balance = 0; - (*this) = p2; - } - break; - } - return h; -} - -static int balance_right(ErtsMonitorOrLink **this) -{ - ErtsMonitorOrLink *p, *p1, *p2; - int b1, b2, h = 1; - - p = *this; - switch (p->balance) { - case 1: - p->balance = 0; - break; - case 0: - p->balance = -1; - h = 0; - break; - case -1: - p1 = p->left; - b1 = p1->balance; - if (b1 <= 0) { /* Single LL rotation */ - p->left = p1->right; - p1->right = p; - if (b1 == 0) { - p->balance = -1; - p1->balance = 1; - h = 0; - } else { - p->balance = p1->balance = 0; - } - (*this) = p1; - } else { /* Double LR rotation */ - p2 = p1->right; - b2 = p2->balance; - p1->right = p2->left; - p2->left = p1; - p->left = p2->right; - p2->right = p; - p->balance = (b2 == -1) ? 1 : 0; - p1->balance = (b2 == 1) ? -1 : 0; - p2->balance = 0; - (*this) = p2; - } - } - return h; -} - -static int delsub(ErtsMonitorOrLink **this) -{ - ErtsMonitorOrLink **tstack[STACK_NEED]; - int tpos = 0; - ErtsMonitorOrLink *q = (*this); - ErtsMonitorOrLink **r = &(q->left); - int h; - - /* - * Walk down the tree to the right and search - * for a void right child, pick that child out - * and return it to be put in the deleted - * object's place. - */ - - while ((*r)->right != NULL) { - tstack[tpos++] = r; - r = &((*r)->right); - } - *this = *r; - *r = (*r)->left; - (*this)->left = q->left; - (*this)->right = q->right; - (*this)->balance = q->balance; - tstack[0] = &((*this)->left); - h = 1; - while (tpos && h) { - r = tstack[--tpos]; - h = balance_right(r); - } - return h; -} - -ErtsMonitor *erts_remove_monitor(ErtsMonitor **root, Eterm ref) -{ - ErtsMonitor **tstack[STACK_NEED]; - int tpos = 0; - int dstack[STACK_NEED+1]; - int dpos = 1; - int state = 0; - ErtsMonitor **this = root; - Sint c; - int dir; - ErtsMonitor *q = NULL; - - dstack[0] = DIR_END; - for (;;) { - if (!*this) { /* Failure */ - return NULL; - } else if ((c = CMP_MON_REF(ref,(*this)->ref)) < 0) { - dstack[dpos++] = DIR_LEFT; - tstack[tpos++] = this; - this = &((*this)->left); - } else if (c > 0) { /* go right */ - dstack[dpos++] = DIR_RIGHT; - tstack[tpos++] = this; - this = &((*this)->right); - } else { /* Equal key, found the one to delete */ - q = (*this); - if (q->right == NULL) { - (*this) = q->left; - state = 1; - } else if (q->left == NULL) { - (*this) = q->right; - state = 1; - } else { - dstack[dpos++] = DIR_LEFT; - tstack[tpos++] = this; - state = delsub((ErtsMonitorOrLink **) this); - } - break; - } - } - while (state && ( dir = dstack[--dpos] ) != DIR_END) { - this = tstack[--tpos]; - if (dir == DIR_LEFT) { - state = balance_left((ErtsMonitorOrLink **) this); - } else { - state = balance_right((ErtsMonitorOrLink **) this); - } - } - return q; -} - -ErtsLink *erts_remove_link(ErtsLink **root, Eterm pid) -{ - ErtsLink **tstack[STACK_NEED]; - int tpos = 0; - int dstack[STACK_NEED+1]; - int dpos = 1; - int state = 0; - ErtsLink **this = root; - Sint c; - int dir; - ErtsLink *q = NULL; - - dstack[0] = DIR_END; - for (;;) { - if (!*this) { /* Failure */ - return NULL; - } else if ((c = CMP(pid,(*this)->pid)) < 0) { - dstack[dpos++] = DIR_LEFT; - tstack[tpos++] = this; - this = &((*this)->left); - } else if (c > 0) { /* go right */ - dstack[dpos++] = DIR_RIGHT; - tstack[tpos++] = this; - this = &((*this)->right); - } else { /* Equal key, found the one to delete */ - q = (*this); - if (q->right == NULL) { - (*this) = q->left; - state = 1; - } else if (q->left == NULL) { - (*this) = q->right; - state = 1; - } else { - dstack[dpos++] = DIR_LEFT; - tstack[tpos++] = this; - state = delsub((ErtsMonitorOrLink **) this); - } - break; - } - } - while (state && ( dir = dstack[--dpos] ) != DIR_END) { - this = tstack[--tpos]; - if (dir == DIR_LEFT) { - state = balance_left((ErtsMonitorOrLink **) this); - } else { - state = balance_right((ErtsMonitorOrLink **) this); - } - } - return q; -} - -void -erts_delete_suspend_monitor(ErtsSuspendMonitor **root, Eterm pid) -{ - ErtsSuspendMonitor **tstack[STACK_NEED]; - int tpos = 0; - int dstack[STACK_NEED+1]; - int dpos = 1; - int state = 0; - ErtsSuspendMonitor **this = root; - Sint c; - int dir; - ErtsSuspendMonitor *q = NULL; - - dstack[0] = DIR_END; - for (;;) { - if (!*this) { /* Nothing found */ - return; - } else if ((c = CMP(pid,(*this)->pid)) < 0) { - dstack[dpos++] = DIR_LEFT; - tstack[tpos++] = this; - this = &((*this)->left); - } else if (c > 0) { /* go right */ - dstack[dpos++] = DIR_RIGHT; - tstack[tpos++] = this; - this = &((*this)->right); - } else { /* Equal key, found the one to delete */ - q = (*this); - ASSERT(q->pid == pid); - if (q->right == NULL) { - (*this) = q->left; - state = 1; - } else if (q->left == NULL) { - (*this) = q->right; - state = 1; - } else { - dstack[dpos++] = DIR_LEFT; - tstack[tpos++] = this; - state = delsub((ErtsMonitorOrLink **) this); - } - erts_destroy_suspend_monitor(q); - break; - } - } - while (state && ( dir = dstack[--dpos] ) != DIR_END) { - this = tstack[--tpos]; - if (dir == DIR_LEFT) { - state = balance_left((ErtsMonitorOrLink **) this); - } else { - state = balance_right((ErtsMonitorOrLink **) this); - } - } -} - -ErtsMonitor *erts_lookup_monitor(ErtsMonitor *root, Eterm ref) -{ - Sint c; - - for (;;) { - if (root == NULL || (c = CMP_MON_REF(ref,root->ref)) == 0) { - return root; - } else if (c < 0) { - root = root->left; - } else { /* c > 0 */ - root = root->right; - } - } -} - -ErtsLink *erts_lookup_link(ErtsLink *root, Eterm pid) -{ - Sint c; - - for (;;) { - if (root == NULL || (c = CMP(pid,root->pid)) == 0) { - return root; - } else if (c < 0) { - root = root->left; - } else { /* c > 0 */ - root = root->right; - } - } -} - -ErtsSuspendMonitor * -erts_lookup_suspend_monitor(ErtsSuspendMonitor *root, Eterm pid) -{ - Sint c; - - for (;;) { - if (root == NULL || (c = CMP(pid,root->pid)) == 0) { - return root; - } else if (c < 0) { - root = root->left; - } else { /* c > 0 */ - root = root->right; - } - } -} - -void erts_sweep_monitors(ErtsMonitor *root, - void (*doit)(ErtsMonitor *, void *), - void *context) -{ - ErtsMonitor *tstack[STACK_NEED]; - int tpos = 0; - int dstack[STACK_NEED+1]; - int dpos = 1; - int dir; - - dstack[0] = DIR_END; - - for (;;) { - if (root == NULL) { - if ((dir = dstack[dpos-1]) == DIR_END) { - return; - } - if (dir == DIR_LEFT) { - /* Still has DIR_RIGHT to do */ - dstack[dpos-1] = DIR_RIGHT; - root = (tstack[tpos-1])->right; - } else { - /* stacktop is an object to be deleted */ - (*doit)(tstack[--tpos],context); /* expeted to do the - deletion */ - --dpos; - root = NULL; - } - } else { - dstack[dpos++] = DIR_LEFT; - tstack[tpos++] = root; - root = root->left; - } - } -} - -void erts_sweep_links(ErtsLink *root, - void (*doit)(ErtsLink *, void *), - void *context) -{ - ErtsLink *tstack[STACK_NEED]; - int tpos = 0; - int dstack[STACK_NEED+1]; - int dpos = 1; - int dir; - - dstack[0] = DIR_END; - - for (;;) { - if (root == NULL) { - if ((dir = dstack[dpos-1]) == DIR_END) { - return; - } - if (dir == DIR_LEFT) { - /* Still has DIR_RIGHT to do */ - dstack[dpos-1] = DIR_RIGHT; - root = (tstack[tpos-1])->right; - } else { - /* stacktop is an object to be deleted */ - (*doit)(tstack[--tpos],context); /* expeted to do the - deletion */ - --dpos; - root = NULL; - } - } else { - dstack[dpos++] = DIR_LEFT; - tstack[tpos++] = root; - root = root->left; - } - } -} - -void erts_sweep_suspend_monitors(ErtsSuspendMonitor *root, - void (*doit)(ErtsSuspendMonitor *, void *), - void *context) -{ - ErtsSuspendMonitor *tstack[STACK_NEED]; - int tpos = 0; - int dstack[STACK_NEED+1]; - int dpos = 1; - int dir; - - dstack[0] = DIR_END; - - for (;;) { - if (root == NULL) { - if ((dir = dstack[dpos-1]) == DIR_END) { - return; - } - if (dir == DIR_LEFT) { - /* Still has DIR_RIGHT to do */ - dstack[dpos-1] = DIR_RIGHT; - root = (tstack[tpos-1])->right; - } else { - /* stacktop is an object to be deleted */ - (*doit)(tstack[--tpos],context); /* expeted to do the - deletion */ - --dpos; - root = NULL; - } - } else { - dstack[dpos++] = DIR_LEFT; - tstack[tpos++] = root; - root = root->left; - } - } -} - - -/* Debug BIF, always present, but undocumented... */ - -static void erts_dump_monitors(ErtsMonitor *root, int indent) -{ - if (root == NULL) - return; - erts_dump_monitors(root->right,indent+2); - erts_printf("%*s[%b16d:%b16u:%T:%T", indent, "", root->balance, - root->type, root->ref, root->name); - if (root->type == MON_NIF_TARGET) - erts_printf(":%p]\n", root->u.resource); - else - erts_printf(":%T]\n", root->u.pid); - erts_dump_monitors(root->left,indent+2); -} - -static void erts_dump_links_aux(ErtsLink *root, int indent, - erts_dsprintf_buf_t *dsbufp) -{ - if (root == NULL) - return; - erts_dump_links_aux(root->right, indent+2, dsbufp); - dsbufp->str_len = 0; - erts_dsprintf(dsbufp, "%*s[%b16d:%b16u:%T:%p]", indent, "", - root->balance, root->type, root->pid, ERTS_LINK_ROOT(root)); - if (ERTS_LINK_ROOT(root) != NULL) { - ErtsLink *sub = ERTS_LINK_ROOT(root); - int len = dsbufp->str_len; - erts_dump_links_aux(sub->right, indent+len+5, dsbufp); - erts_dsprintf(dsbufp, "-> %*s[%b16d:%b16u:%T:%p]", indent, "", - sub->balance, sub->type, sub->pid, ERTS_LINK_ROOT(sub)); - erts_printf("%s\n", dsbufp->str); - erts_dump_links_aux(sub->left, indent+len+5, dsbufp); - } else { - erts_printf("%s\n", dsbufp->str); - } - erts_dump_links_aux(root->left, indent+2, dsbufp); -} - -static void erts_dump_links(ErtsLink *root, int indent) -{ - erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0); - erts_dump_links_aux(root, indent, dsbufp); - erts_destroy_tmp_dsbuf(dsbufp); -} - -Eterm erts_debug_dump_monitors_1(BIF_ALIST_1) -{ - Process *p = BIF_P; - Eterm pid = BIF_ARG_1; - Process *rp; - DistEntry *dep; - rp = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, pid, ERTS_PROC_LOCK_LINK); - if (!rp) { - ERTS_ASSERT_IS_NOT_EXITING(p); - if (is_atom(pid) && is_node_name_atom(pid) && - (dep = erts_find_dist_entry(pid)) != NULL) { - erts_printf("Dumping dist monitors-------------------\n"); - erts_de_links_lock(dep); - erts_dump_monitors(dep->monitors,0); - erts_de_links_unlock(dep); - erts_printf("Monitors dumped-------------------------\n"); - BIF_RET(am_true); - } else { - BIF_ERROR(p,BADARG); - } - } else { - erts_printf("Dumping pid monitors--------------------\n"); - erts_dump_monitors(ERTS_P_MONITORS(rp),0); - erts_printf("Monitors dumped-------------------------\n"); - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - BIF_RET(am_true); - } -} - -Eterm erts_debug_dump_links_1(BIF_ALIST_1) -{ - Process *p = BIF_P; - Eterm pid = BIF_ARG_1; - Process *rp; - DistEntry *dep; - if (is_internal_port(pid)) { - Port *rport = erts_id2port_sflgs(pid, - p, - ERTS_PROC_LOCK_MAIN, - ERTS_PORT_SFLGS_INVALID_LOOKUP); - if (rport) { - erts_printf("Dumping port links----------------------\n"); - erts_dump_links(ERTS_P_LINKS(rport), 0); - erts_printf("Links dumped----------------------------\n"); - erts_port_release(rport); - BIF_RET(am_true); - } else { - BIF_ERROR(p,BADARG); - } - } else { - rp = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, pid, ERTS_PROC_LOCK_LINK); - if (!rp) { - ERTS_ASSERT_IS_NOT_EXITING(p); - if (is_atom(pid) && is_node_name_atom(pid) && - (dep = erts_find_dist_entry(pid)) != NULL) { - erts_printf("Dumping dist links----------------------\n"); - erts_de_links_lock(dep); - erts_dump_links(dep->nlinks,0); - erts_de_links_unlock(dep); - erts_printf("Links dumped----------------------------\n"); - BIF_RET(am_true); - } else { - BIF_ERROR(p,BADARG); - } - - } else { - erts_printf("Dumping pid links-----------------------\n"); - erts_dump_links(ERTS_P_LINKS(rp), 0); - erts_printf("Links dumped----------------------------\n"); - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - BIF_RET(am_true); - } - } -} - -void erts_one_link_size(ErtsLink *lnk, void *vpu) -{ - Uint *pu = vpu; - *pu += ERTS_LINK_SIZE*sizeof(Uint); - if(is_not_immed(lnk->pid)) - *pu += NC_HEAP_SIZE(lnk->pid)*sizeof(Uint); - if (lnk->type != LINK_NODE && ERTS_LINK_ROOT(lnk) != NULL) { - erts_doforall_links(ERTS_LINK_ROOT(lnk),&erts_one_link_size,vpu); - } -} -void erts_one_mon_size(ErtsMonitor *mon, void *vpu) -{ - Uint *pu = vpu; - *pu += ERTS_MONITOR_SIZE*sizeof(Uint); - if(mon->type != MON_NIF_TARGET && is_not_immed(mon->u.pid)) - *pu += NC_HEAP_SIZE(mon->u.pid)*sizeof(Uint); - if(is_not_immed(mon->ref)) - *pu += NC_HEAP_SIZE(mon->ref)*sizeof(Uint); -} diff --git a/erts/emulator/beam/erl_monitors.h b/erts/emulator/beam/erl_monitors.h deleted file mode 100644 index 1cacecd7e9..0000000000 --- a/erts/emulator/beam/erl_monitors.h +++ /dev/null @@ -1,187 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2004-2017. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -/********************************************************************** - * Header for monitors and links data structures. - * Monitors are kept in an AVL tree and the data structures for - * the four different types of monitors are like this: - ********************************************************************** - * Local monitor by pid/port: - * (Ref is always same in all involved data structures) - ********************************************************************** - * Process/Port X Process Y - * +-------------+ +-------------+ - * Type: | MON_ORIGIN | | MON_TARGET | - * +-------------+ +-------------+ - * Pid: | Pid(Y) | | Pid/Port(X) | - * +-------------+ +-------------+ - * Name: | [] | | [] | - * +-------------+ +-------------+ - ********************************************************************** - * Local monitor by name: (Ref is always same in all involved data structures) - ********************************************************************** - * Process X Process Y (name foo) - * +-------------+ +-------------+ - * Type: | MON_ORIGIN | | MON_TARGET | - * +-------------+ +-------------+ - * Pid: | Pid(Y) | | Pid(X) | - * +-------------+ +-------------+ - * Name: | Atom(foo) | | Atom(foo) | - * +-------------+ +-------------+ - ********************************************************************** - * Remote monitor by pid: (Ref is always same in all involved data structures) - ********************************************************************** - * Node A | Node B - * ---------------------------------+---------------------------------- - * Process X (@A) Distentry @A Distentry @B Process Y (@B) - * for node B for node A - * +-------------+ +-------------+ +-------------+ +-------------+ - * Type: | MON_ORIGIN | | MON_TARGET | | MON_ORIGIN | | MON_TARGET | - * +-------------+ +-------------+ +-------------+ +-------------+ - * Pid: | Pid(Y) | | Pid(X) | | Pid(Y) | | Pid(X) | - * +-------------+ +-------------+ +-------------+ +-------------+ - * Name: | [] | | [] | | [] | | [] | - * +-------------+ +-------------+ +-------------+ +-------------+ - ********************************************************************** - * Remote monitor by name: (Ref is always same in all involved data structures) - ********************************************************************** - * Node A | Node B - * ---------------------------------+---------------------------------- - * Process X (@A) Distentry @A Distentry @B Process Y (@B) - * for node B for node A (name foo) - * +-------------+ +-------------+ +-------------+ +-------------+ - * Type: | MON_ORIGIN | | MON_TARGET | | MON_ORIGIN | | MON_TARGET | - * +-------------+ +-------------+ +-------------+ +-------------+ - * Pid: | Atom(node B)| | Pid(X) | | Pid(Y) | | Pid(X) | - * +-------------+ +-------------+ +-------------+ +-------------+ - * Name: | Atom(foo) | | Atom(foo) | | Atom(foo) | | Atom(foo) | - * +-------------+ +-------------+ +-------------+ +-------------+ - * The reason for the node atom in X->pid is that we don't know the actual - * pid of the monitored process on the other node when setting the monitor - * (which is done asyncronously). - **********************************************************************/ -#ifndef _ERL_MONITORS_H -#define _ERL_MONITORS_H - -/* Type tags for monitors */ -#define MON_ORIGIN 1 -#define MON_TARGET 2 -#define MON_NIF_TARGET 3 -#define MON_TIME_OFFSET 4 - -/* Type tags for links */ -#define LINK_PID 1 /* ...Or port */ -#define LINK_NODE 3 /* "Node monitor" */ - -/* Size of a monitor without heap, in words (fixalloc) */ -#define ERTS_MONITOR_SIZE ((sizeof(ErtsMonitor) - sizeof(Uint))/sizeof(Uint)) -#define ERTS_MONITOR_SH_SIZE (ERTS_MONITOR_SIZE + ERTS_REF_THING_SIZE) -#define ERTS_LINK_SIZE ((sizeof(ErtsLink) - sizeof(Uint))/sizeof(Uint)) -#define ERTS_LINK_SH_SIZE ERTS_LINK_SIZE /* Size of fix-alloced links */ - -/* ErtsMonitor and ErtsLink *need* to begin in a similar way as - ErtsMonitorOrLink */ -typedef struct erts_monitor_or_link { - struct erts_monitor_or_link *left, *right; - Sint16 balance; -} ErtsMonitorOrLink; - -typedef struct erts_monitor { - struct erts_monitor *left, *right; - Sint16 balance; - Uint16 type; /* MON_ORIGIN | MON_TARGET | MON_NIF_TARGET | MON_TIME_OFFSET */ - Eterm ref; - union { - Eterm pid; /* In case of distributed named monitor, this is the - * nodename atom in MON_ORIGIN process, otherwise a pid or, - * in case of a MON_TARGET, a port - */ - struct ErtsResource_* resource; /* MON_NIF_TARGET */ - }u; - Eterm name; /* When monitoring a named process: atom() else [] */ - Uint heap[1]; /* Larger in reality */ -} ErtsMonitor; - -typedef struct erts_link { - struct erts_link *left, *right; - Sint16 balance; - Uint16 type; /* LINK_PID | LINK_NODE */ - Eterm pid; /* When node monitor, - the node atom is here instead */ - union { - struct erts_link *root; /* Used only in dist entries */ - Uint refc; - } shared; - Uint heap[1]; /* Larger in reality */ -} ErtsLink; - -typedef struct erts_suspend_monitor { - struct erts_suspend_monitor *left, *right; - Sint16 balance; - - int pending; - int active; - Eterm pid; -} ErtsSuspendMonitor; - -#define ERTS_LINK_ROOT(Linkp) ((Linkp)->shared.root) -#define ERTS_LINK_REFC(Linkp) ((Linkp)->shared.refc) - -Uint erts_tot_link_lh_size(void); - - -/* Prototypes */ -void erts_destroy_monitor(ErtsMonitor *mon); -void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, UWord entity, - Eterm name); -ErtsMonitor *erts_remove_monitor(ErtsMonitor **root, Eterm ref); -ErtsMonitor *erts_lookup_monitor(ErtsMonitor *root, Eterm ref); -void erts_sweep_monitors(ErtsMonitor *root, - void (*doit)(ErtsMonitor *, void *), - void *context); - -void erts_destroy_link(ErtsLink *lnk); -/* Returns 0 if OK, < 0 if already present */ -int erts_add_link(ErtsLink **root, Uint type, Eterm pid); -ErtsLink *erts_add_or_lookup_link(ErtsLink **root, Uint type, Eterm pid); -ErtsLink *erts_remove_link(ErtsLink **root, Eterm pid); -ErtsLink *erts_lookup_link(ErtsLink *root, Eterm pid); -void erts_sweep_links(ErtsLink *root, - void (*doit)(ErtsLink *, void *), - void *context); - -void erts_destroy_suspend_monitor(ErtsSuspendMonitor *sproc); -void erts_sweep_suspend_monitors(ErtsSuspendMonitor *root, - void (*doit)(ErtsSuspendMonitor *, void *), - void *context); -ErtsSuspendMonitor *erts_add_or_lookup_suspend_monitor(ErtsSuspendMonitor **root, - Eterm pid); -ErtsSuspendMonitor *erts_lookup_suspend_monitor(ErtsSuspendMonitor *root, - Eterm pid); -void erts_delete_suspend_monitor(ErtsSuspendMonitor **root, Eterm pid); -void erts_init_monitors(void); -void erts_one_link_size(ErtsLink *lnk, void *vpu); -void erts_one_mon_size(ErtsMonitor *mon, void *vpu); - -#define erts_doforall_monitors erts_sweep_monitors -#define erts_doforall_links erts_sweep_links -#define erts_doforall_suspend_monitors erts_sweep_suspend_monitors - -#endif /* _ERL_MONITORS_H */ diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index c60cc7fecf..e208792868 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2017. All Rights Reserved. + * Copyright Ericsson AB 2009-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -57,6 +57,7 @@ #include "erl_bif_unique.h" #include "erl_utils.h" #include "erl_io_queue.h" +#include "erl_proc_sig_queue.h" #undef ERTS_WANT_NFUNC_SCHED_INTERNALS__ #define ERTS_WANT_NFUNC_SCHED_INTERNALS__ #include "erl_nfunc_sched.h" @@ -387,12 +388,18 @@ erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * erts_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC | ERTS_PSFLG_DIRTY_IO_PROC)); + ASSERT(esdp->current_nif == NULL); + esdp->current_nif = &env; + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); result = (*dirty_nif)(&env, codemfa->arity, argv); /* Call dirty NIF */ erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + ASSERT(esdp->current_nif == &env); + esdp->current_nif = NULL; + ASSERT(env.proc->static_flags & ERTS_STC_FLG_SHADOW_PROC); ASSERT(env.proc->next == c_p); @@ -658,7 +665,7 @@ int erts_flush_trace_messages(Process *c_p, ErtsProcLocks c_p_locks) rp_locks = 0; if (rp->common.id == c_p->common.id) rp_locks = c_p_locks; - erts_queue_messages(rp, rp_locks, first, last, len, c_p->common.id); + erts_queue_proc_messages(c_p, rp, rp_locks, first, last, len); if (rp->common.id == c_p->common.id) rp_locks &= ~c_p_locks; if (rp_locks) @@ -700,6 +707,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, Process* rp; Process* c_p; ErtsMessage *mp; + Eterm from; Eterm receiver = to_pid->pid; int scheduler; @@ -778,7 +786,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, msg = copy_struct_litopt(msg, sz, &hp, ohp, &litarea); } - ERL_MESSAGE_TERM(mp) = msg; + from = c_p ? c_p->common.id : am_undefined; if (!env || !env->tracee) { @@ -797,7 +805,6 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ErlTraceMessageQueue *msgq; Process *t_p = env->tracee; - erts_proc_lock(t_p, ERTS_PROC_LOCK_TRACE); msgq = t_p->trace_msg_q; @@ -817,6 +824,9 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, rp_locks & ERTS_PROC_LOCK_MSGQ || erts_proc_trylock(rp, ERTS_PROC_LOCK_MSGQ) == EBUSY) { + ERL_MESSAGE_TERM(mp) = msg; + ERL_MESSAGE_FROM(mp) = from; + if (!msgq) { msgq = erts_alloc(ERTS_ALC_T_TRACE_MSG_QUEUE, sizeof(ErlTraceMessageQueue)); @@ -846,8 +856,10 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, } } - erts_queue_message(rp, rp_locks, mp, msg, - c_p ? c_p->common.id : am_undefined); + if (c_p) + erts_queue_proc_message(c_p, rp, rp_locks, mp, msg); + else + erts_queue_message(rp, rp_locks, mp, msg, from); done: if (c_p == rp) @@ -1963,6 +1975,12 @@ ErlNifTid enif_thread_self(void) { return erl_drv_thread_self(); } int enif_equal_tids(ErlNifTid tid1, ErlNifTid tid2) { return erl_drv_equal_tids(tid1,tid2); } void enif_thread_exit(void *resp) { erl_drv_thread_exit(resp); } int enif_thread_join(ErlNifTid tid, void **respp) { return erl_drv_thread_join(tid,respp); } + +char* enif_mutex_name(ErlNifMutex *mtx) {return erl_drv_mutex_name(mtx); } +char* enif_cond_name(ErlNifCond *cnd) { return erl_drv_cond_name(cnd); } +char* enif_rwlock_name(ErlNifRWLock* rwlck) { return erl_drv_rwlock_name(rwlck); } +char* enif_thread_name(ErlNifTid tid) { return erl_drv_thread_name(tid); } + int enif_getenv(const char *key, char *value, size_t *value_size) { return erl_drv_getenv(key, value, value_size); } ErlNifTime enif_monotonic_time(ErlNifTimeUnit time_unit) @@ -1985,16 +2003,21 @@ enif_convert_time_unit(ErlNifTime val, (int) to); } -int enif_fprintf(void* filep, const char* format, ...) +int enif_fprintf(FILE* filep, const char* format, ...) { int ret; va_list arglist; va_start(arglist, format); - ret = erts_vfprintf((FILE*)filep, format, arglist); + ret = erts_vfprintf(filep, format, arglist); va_end(arglist); return ret; } +int enif_vfprintf(FILE* filep, const char *format, va_list ap) +{ + return erts_vfprintf(filep, format, ap); +} + int enif_snprintf(char *buffer, size_t size, const char* format, ...) { int ret; @@ -2005,6 +2028,12 @@ int enif_snprintf(char *buffer, size_t size, const char* format, ...) return ret; } +int enif_vsnprintf(char* buffer, size_t size, const char *format, va_list ap) +{ + return erts_vsnprintf(buffer, size, format, ap); +} + + /*********************************************************** ** Memory managed (GC'ed) "resource" objects ** ***********************************************************/ @@ -2206,71 +2235,61 @@ static void rollback_opened_resource_types(void) } } -struct destroy_monitor_ctx -{ - ErtsResource* resource; - int exiting_procs; - int scheduler; -}; +#ifdef ARCH_64 +# define ERTS_RESOURCE_DYING_FLAG (((Uint) 1) << 63) +#else +# define ERTS_RESOURCE_DYING_FLAG (((Uint) 1) << 31) +#endif +#define ERTS_RESOURCE_REFC_MASK (~ERTS_RESOURCE_DYING_FLAG) -static void destroy_one_monitor(ErtsMonitor* mon, void* context) +static ERTS_INLINE void +rmon_set_dying(ErtsResourceMonitors *rms) { - struct destroy_monitor_ctx* ctx = (struct destroy_monitor_ctx*) context; - Process* rp; - ErtsMonitor *rmon = NULL; - int is_exiting; + rms->refc |= ERTS_RESOURCE_DYING_FLAG; +} - ASSERT(mon->type == MON_ORIGIN); - ASSERT(is_internal_pid(mon->u.pid)); - ASSERT(is_internal_ref(mon->ref)); +static ERTS_INLINE int +rmon_is_dying(ErtsResourceMonitors *rms) +{ + return !!(rms->refc & ERTS_RESOURCE_DYING_FLAG); +} - if (ctx->scheduler > 0) { /* Normal scheduler */ - rp = erts_proc_lookup(mon->u.pid); - } - else { - rp = erts_proc_lookup_inc_refc(mon->u.pid); - } +static ERTS_INLINE void +rmon_refc_inc(ErtsResourceMonitors *rms) +{ + rms->refc++; +} - if (!rp) { - is_exiting = 1; - } - if (rp) { - erts_proc_lock(rp, ERTS_PROC_LOCK_LINK); - if (ERTS_PROC_IS_EXITING(rp)) { - is_exiting = 1; - } else { - rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref); - ASSERT(rmon); - is_exiting = 0; - } - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - if (ctx->scheduler <= 0) - erts_proc_dec_refc(rp); - } - if (is_exiting) { - ctx->resource->monitors->pending_failed_fire++; - } +static ERTS_INLINE Uint +rmon_refc_dec_read(ErtsResourceMonitors *rms) +{ + Uint res; + ASSERT((rms->refc & ERTS_RESOURCE_REFC_MASK) != 0); + res = --rms->refc; + return res & ERTS_RESOURCE_REFC_MASK; +} - /* ToDo: Delay destruction after monitor_locks */ - if (rmon) { - ASSERT(rmon->type == MON_NIF_TARGET); - ASSERT(rmon->u.resource == ctx->resource); - erts_destroy_monitor(rmon); - } - erts_destroy_monitor(mon); +static ERTS_INLINE void +rmon_refc_dec(ErtsResourceMonitors *rms) +{ + ASSERT((rms->refc & ERTS_RESOURCE_REFC_MASK) != 0); + --rms->refc; } -static void destroy_all_monitors(ErtsMonitor* monitors, ErtsResource* resource) +static ERTS_INLINE Uint +rmon_refc_read(ErtsResourceMonitors *rms) { - struct destroy_monitor_ctx ctx; + return rms->refc & ERTS_RESOURCE_REFC_MASK; +} - execution_state(NULL, NULL, &ctx.scheduler); +static void dtor_demonitor(ErtsMonitor* mon, void* context) +{ + ASSERT(erts_monitor_is_origin(mon)); + ASSERT(is_internal_pid(mon->other.item)); - ctx.resource = resource; - erts_sweep_monitors(monitors, &destroy_one_monitor, &ctx); + erts_proc_sig_send_demonitor(mon); } - # define NIF_RESOURCE_DTOR &nif_resource_dtor static int nif_resource_dtor(Binary* bin) @@ -2281,34 +2300,36 @@ static int nif_resource_dtor(Binary* bin) if (resource->monitors) { ErtsResourceMonitors* rm = resource->monitors; + int kill; + ErtsMonitor *root; + Uint refc; ASSERT(type->down); erts_mtx_lock(&rm->lock); ASSERT(erts_refc_read(&bin->intern.refc, 0) == 0); - if (rm->root) { - ASSERT(!rm->is_dying); - destroy_all_monitors(rm->root, resource); + kill = !rmon_is_dying(rm); + if (kill) { + rmon_set_dying(rm); + root = rm->root; rm->root = NULL; } - if (rm->pending_failed_fire) { - /* - * Resource death struggle prolonged to serve exiting process(es). - * Destructor will be called again when last exiting process - * tries to fire its MON_NIF_TARGET monitor (and fails). - * - * This resource is doomed. It has no "real" references and - * should get not get called upon to do anything except the - * final destructor call. - * - * We keep refc at 0 and use a separate counter for exiting - * processes to avoid resource getting revived by "dec_term". - */ - ASSERT(!rm->is_dying); - rm->is_dying = 1; - erts_mtx_unlock(&rm->lock); - return 0; - } + refc = rmon_refc_read(rm); erts_mtx_unlock(&rm->lock); + + if (kill) + erts_monitor_tree_foreach_delete(&root, + dtor_demonitor, + NULL); + + /* + * If resource->monitors->refc != 0 there are + * outstanding references to the resource from + * monitors that has not been removed yet. + * nif_resource_dtor() will be called again this + * reference count reach zero. + */ + if (refc != 0) + return 0; /* we'll be back... */ erts_mtx_destroy(&rm->lock); } @@ -2338,54 +2359,82 @@ void erts_resource_stop(ErtsResource* resource, ErlNifEvent e, post_nif_noproc(&msg_env); } -void erts_fire_nif_monitor(ErtsResource* resource, Eterm pid, Eterm ref) +void erts_nif_demonitored(ErtsResource* resource) { - ErtsMonitor* rmon; + ErtsResourceMonitors* rmp = resource->monitors; ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); + int free_me; + + ASSERT(rmp); + ASSERT(resource->type->down); + + erts_mtx_lock(&rmp->lock); + free_me = ((rmon_refc_dec_read(rmp) == 0) & !!rmon_is_dying(rmp)); + erts_mtx_unlock(&rmp->lock); + + if (free_me) + erts_bin_free(&bin->binary); +} + +void erts_fire_nif_monitor(ErtsMonitor *tmon) +{ + ErtsResource* resource; + ErtsMonitorData *mdp; + ErtsMonitor *omon; + ErtsBinary* bin; struct enif_msg_environment_t msg_env; ErlNifPid nif_pid; ErlNifMonitor nif_monitor; - ErtsResourceMonitors* rmp = resource->monitors; + ErtsResourceMonitors* rmp; + Uint mrefc, brefc; + int active, is_dying; + + ASSERT(tmon->type == ERTS_MON_TYPE_RESOURCE); + ASSERT(erts_monitor_is_target(tmon)); + + resource = tmon->other.ptr; + bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); + rmp = resource->monitors; + + mdp = erts_monitor_to_data(tmon); + omon = &mdp->origin; ASSERT(rmp); ASSERT(resource->type->down); erts_mtx_lock(&rmp->lock); - rmon = erts_remove_monitor(&rmp->root, ref); - if (!rmon) { - int free_me = (--rmp->pending_failed_fire == 0) && rmp->is_dying; - ASSERT(rmp->pending_failed_fire >= 0); - erts_mtx_unlock(&rmp->lock); - - if (free_me) { - ASSERT(erts_refc_read(&bin->binary.intern.refc, 0) == 0); - erts_bin_free(&bin->binary); - } - return; + + mrefc = rmon_refc_dec_read(rmp); + is_dying = rmon_is_dying(rmp); + active = !is_dying && erts_monitor_is_in_table(omon); + + if (active) { + erts_monitor_tree_delete(&rmp->root, omon); + brefc = (Uint) erts_refc_inc_unless(&bin->binary.intern.refc, 0, 0); } - ASSERT(!rmp->is_dying); - if (erts_refc_inc_unless(&bin->binary.intern.refc, 0, 0) == 0) { - /* - * Racing resource destruction. - * To avoid a more complex refc-dance with destructing thread - * we avoid calling 'down' and just silently remove the monitor. - * This can happen even for non smp as destructor calls may be scheduled. - */ - erts_mtx_unlock(&rmp->lock); + + erts_mtx_unlock(&rmp->lock); + + if (!active) { + ASSERT(!is_dying || erts_refc_read(&bin->binary.intern.refc, 0) == 0); + if (is_dying && mrefc == 0) + erts_bin_free(&bin->binary); + erts_monitor_release(tmon); } else { - erts_mtx_unlock(&rmp->lock); - - ASSERT(rmon->u.pid == pid); - erts_ref_to_driver_monitor(ref, &nif_monitor); - nif_pid.pid = pid; - pre_nif_noproc(&msg_env, resource->type->owner, NULL); - resource->type->down(&msg_env.env, resource->data, &nif_pid, &nif_monitor); - post_nif_noproc(&msg_env); + if (brefc > 0) { + ASSERT(is_internal_pid(omon->other.item)); + erts_ref_to_driver_monitor(mdp->ref, &nif_monitor); + nif_pid.pid = omon->other.item; + pre_nif_noproc(&msg_env, resource->type->owner, NULL); + resource->type->down(&msg_env.env, resource->data, &nif_pid, &nif_monitor); + post_nif_noproc(&msg_env); + + erts_bin_release(&bin->binary); + } - erts_bin_release(&bin->binary); + erts_monitor_release_both(mdp); } - erts_destroy_monitor(rmon); } void* enif_alloc_resource(ErlNifResourceType* type, size_t data_sz) @@ -2422,8 +2471,7 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t data_sz) erts_mtx_init(&resource->monitors->lock, "resource_monitors", NIL, ERTS_LOCK_FLAGS_CATEGORY_GENERIC); resource->monitors->root = NULL; - resource->monitors->pending_failed_fire = 0; - resource->monitors->is_dying = 0; + resource->monitors->refc = 0; resource->monitors->user_data_sz = data_sz; } else { @@ -2438,7 +2486,7 @@ void enif_release_resource(void* obj) ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == NIF_RESOURCE_DTOR); - ASSERT(!(resource->monitors && resource->monitors->is_dying)); + ASSERT(erts_refc_read(&bin->binary.intern.refc, 0) != 0); #ifdef DEBUG erts_refc_dec(&resource->nif_refc, 0); #endif @@ -2451,7 +2499,7 @@ void enif_keep_resource(void* obj) ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == NIF_RESOURCE_DTOR); - ASSERT(!(resource->monitors && resource->monitors->is_dying)); + ASSERT(erts_refc_read(&bin->binary.intern.refc, 0) != 0); #ifdef DEBUG erts_refc_inc(&resource->nif_refc, 1); #endif @@ -2461,7 +2509,7 @@ void enif_keep_resource(void* obj) Eterm erts_bld_resource_ref(Eterm** hpp, ErlOffHeap* oh, ErtsResource* resource) { ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); - ASSERT(!(resource->monitors && resource->monitors->is_dying)); + ASSERT(erts_refc_read(&bin->binary.intern.refc, 0) != 0); return erts_mk_magic_ref(hpp, oh, &bin->binary); } @@ -2470,7 +2518,7 @@ ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj) ErtsResource* resource = DATA_TO_RESOURCE(obj); ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); Eterm* hp = alloc_heap(env, ERTS_MAGIC_REF_THING_SIZE); - ASSERT(!(resource->monitors && resource->monitors->is_dying)); + ASSERT(erts_refc_read(&bin->binary.intern.refc, 0) != 0); return erts_mk_magic_ref(&hp, &MSO(env->proc), &bin->binary); } @@ -2895,6 +2943,44 @@ ERL_NIF_TERM enif_make_new_map(ErlNifEnv* env) return make_flatmap(mp); } +int enif_make_map_from_arrays(ErlNifEnv *env, + ERL_NIF_TERM keys[], + ERL_NIF_TERM values[], + size_t cnt, + ERL_NIF_TERM *map_out) +{ + ErtsHeapFactory factory; + int succeeded; + +#ifdef ERTS_NIF_ASSERT_IN_ENV + size_t index = 0; + + while (index < cnt) { + ASSERT_IN_ENV(env, keys[index], index, "key"); + ASSERT_IN_ENV(env, values[index], index, "value"); + index++; + } +#endif + + flush_env(env); + + erts_factory_proc_prealloc_init(&factory, env->proc, + cnt * 2 + MAP_HEADER_FLATMAP_SZ + 1); + + (*map_out) = erts_map_from_ks_and_vs(&factory, keys, values, cnt); + succeeded = (*map_out) != THE_NON_VALUE; + + if (!succeeded) { + erts_factory_undo(&factory); + } + + erts_factory_close(&factory); + + cache_env(env); + + return succeeded; +} + int enif_make_map_put(ErlNifEnv* env, Eterm map_in, Eterm key, @@ -3150,123 +3236,88 @@ int enif_map_iterator_get_pair(ErlNifEnv *env, int enif_monitor_process(ErlNifEnv* env, void* obj, const ErlNifPid* target_pid, ErlNifMonitor* monitor) { - int scheduler; ErtsResource* rsrc = DATA_TO_RESOURCE(obj); - Process *rp; Eterm tmp[ERTS_REF_THING_SIZE]; Eterm ref; - int retval; + ErtsResourceMonitors *rm; + ErtsMonitorData *mdp; ASSERT(ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rsrc)->magic_binary.destructor == NIF_RESOURCE_DTOR); - ASSERT(!(rsrc->monitors && rsrc->monitors->is_dying)); + ASSERT(erts_refc_read(&ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rsrc)->binary.intern.refc, 0) != 0); ASSERT(!rsrc->monitors == !rsrc->type->down); - - if (!rsrc->monitors) { + rm = rsrc->monitors; + if (!rm) { ASSERT(!rsrc->type->down); return -1; } ASSERT(rsrc->type->down); - execution_state(env, NULL, &scheduler); + ref = erts_make_ref_in_buffer(tmp); - if (scheduler > 0) /* Normal scheduler */ - rp = erts_proc_lookup_raw(target_pid->pid); - else - rp = erts_proc_lookup_raw_inc_refc(target_pid->pid); + mdp = erts_monitor_create(ERTS_MON_TYPE_RESOURCE, ref, + (Eterm) rsrc, target_pid->pid, NIL); + erts_mtx_lock(&rm->lock); + ASSERT(!rmon_is_dying(rm)); + erts_monitor_tree_insert(&rm->root, &mdp->origin); + rmon_refc_inc(rm); + erts_mtx_unlock(&rm->lock); - if (!rp) - return 1; + if (!erts_proc_sig_send_monitor(&mdp->target, target_pid->pid)) { + /* Failed to send monitor signal; cleanup... */ +#ifdef DEBUG + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rsrc); +#endif - ref = erts_make_ref_in_buffer(tmp); + erts_mtx_lock(&rm->lock); + ASSERT(!rmon_is_dying(rm)); + erts_monitor_tree_delete(&rm->root, &mdp->origin); + rmon_refc_dec(rm); + ASSERT(erts_refc_read(&bin->binary.intern.refc, 1) != 0); + erts_mtx_unlock(&rm->lock); + erts_monitor_release_both(mdp); - erts_mtx_lock(&rsrc->monitors->lock); - erts_proc_lock(rp, ERTS_PROC_LOCK_LINK); - if (ERTS_PSFLG_FREE & erts_atomic32_read_nob(&rp->state)) { - retval = 1; - } - else { - erts_add_monitor(&rsrc->monitors->root, MON_ORIGIN, ref, rp->common.id, NIL); - erts_add_monitor(&ERTS_P_MONITORS(rp), MON_NIF_TARGET, ref, (UWord)rsrc, NIL); - retval = 0; + return 1; } - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - erts_mtx_unlock(&rsrc->monitors->lock); - if (scheduler <= 0) - erts_proc_dec_refc(rp); if (monitor) erts_ref_to_driver_monitor(ref,monitor); - return retval; + return 0; } int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monitor) { - int scheduler; ErtsResource* rsrc = DATA_TO_RESOURCE(obj); #ifdef DEBUG ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rsrc); #endif - Process *rp; + ErtsResourceMonitors *rm; ErtsMonitor *mon; - ErtsMonitor *rmon = NULL; Eterm ref_heap[ERTS_REF_THING_SIZE]; Eterm ref; - int is_exiting; ASSERT(bin->magic_binary.destructor == NIF_RESOURCE_DTOR); - ASSERT(!(rsrc->monitors && rsrc->monitors->is_dying)); - - execution_state(env, NULL, &scheduler); + ASSERT(erts_refc_read(&bin->binary.intern.refc, 0) != 0); ref = erts_driver_monitor_to_ref(ref_heap, monitor); - erts_mtx_lock(&rsrc->monitors->lock); - mon = erts_remove_monitor(&rsrc->monitors->root, ref); + rm = rsrc->monitors; + erts_mtx_lock(&rm->lock); + ASSERT(!rmon_is_dying(rm)); + mon = erts_monitor_tree_lookup(rm->root, ref); + if (mon) + erts_monitor_tree_delete(&rm->root, mon); + erts_mtx_unlock(&rm->lock); - if (mon == NULL) { - erts_mtx_unlock(&rsrc->monitors->lock); + if (!mon) return 1; - } - ASSERT(mon->type == MON_ORIGIN); - ASSERT(is_internal_pid(mon->u.pid)); + ASSERT(erts_monitor_is_origin(mon)); + ASSERT(is_internal_pid(mon->other.item)); - if (scheduler > 0) /* Normal scheduler */ - rp = erts_proc_lookup(mon->u.pid); - else - rp = erts_proc_lookup_inc_refc(mon->u.pid); - - if (!rp) { - is_exiting = 1; - } - else { - erts_proc_lock(rp, ERTS_PROC_LOCK_LINK); - if (ERTS_PROC_IS_EXITING(rp)) { - is_exiting = 1; - } else { - rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); - ASSERT(rmon); - is_exiting = 0; - } - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - - if (scheduler <= 0) - erts_proc_dec_refc(rp); - } - if (is_exiting) { - rsrc->monitors->pending_failed_fire++; - } - erts_mtx_unlock(&rsrc->monitors->lock); - - if (rmon) { - ASSERT(rmon->type == MON_NIF_TARGET); - ASSERT(rmon->u.resource == rsrc); - erts_destroy_monitor(rmon); - } - erts_destroy_monitor(mon); + erts_proc_sig_send_demonitor(mon); return 0; } @@ -3726,16 +3777,26 @@ static Eterm mkatom(const char *str) return am_atom_put(str, sys_strlen(str)); } -static struct tainted_module_t +struct tainted_module_t { struct tainted_module_t* next; Eterm module_atom; -}*first_tainted_module = NULL; +}; + +erts_atomic_t first_taint; /* struct tainted_module_t* */ -static void add_taint(Eterm mod_atom) +void erts_add_taint(Eterm mod_atom) { - struct tainted_module_t* t; - for (t=first_tainted_module ; t!=NULL; t=t->next) { +#ifdef ERTS_ENABLE_LOCK_CHECK + extern erts_rwmtx_t erts_driver_list_lock; /* Mutex for driver list */ +#endif + struct tainted_module_t *first, *t; + + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_driver_list_lock) + || erts_thr_progress_is_blocking()); + + first = (struct tainted_module_t*) erts_atomic_read_nob(&first_taint); + for (t=first ; t; t=t->next) { if (t->module_atom == mod_atom) { return; } @@ -3743,22 +3804,24 @@ static void add_taint(Eterm mod_atom) t = erts_alloc_fnf(ERTS_ALC_T_TAINT, sizeof(*t)); if (t != NULL) { t->module_atom = mod_atom; - t->next = first_tainted_module; - first_tainted_module = t; + t->next = first; + erts_atomic_set_nob(&first_taint, (erts_aint_t)t); } } Eterm erts_nif_taints(Process* p) { - struct tainted_module_t* t; + struct tainted_module_t *first, *t; unsigned cnt = 0; Eterm list = NIL; Eterm* hp; - for (t=first_tainted_module ; t!=NULL; t=t->next) { + + first = (struct tainted_module_t*) erts_atomic_read_nob(&first_taint); + for (t=first ; t!=NULL; t=t->next) { cnt++; } hp = HAlloc(p,cnt*2); - for (t=first_tainted_module ; t!=NULL; t=t->next) { + for (t=first ; t!=NULL; t=t->next) { list = CONS(hp, t->module_atom, list); hp += 2; } @@ -3767,9 +3830,11 @@ Eterm erts_nif_taints(Process* p) void erts_print_nif_taints(fmtfn_t to, void* to_arg) { - struct tainted_module_t* t; + struct tainted_module_t *t; const char* delim = ""; - for (t=first_tainted_module ; t!=NULL; t=t->next) { + + t = (struct tainted_module_t*) erts_atomic_read_nob(&first_taint); + for ( ; t; t = t->next) { const Atom* atom = atom_tab(atom_val(t->module_atom)); erts_cbprintf(to,to_arg,"%s%.*s", delim, atom->len, atom->name); delim = ","; @@ -3856,6 +3921,11 @@ static struct erl_module_nif* create_lib(const ErlNifEntry* src) } else { dst->sizeof_ErlNifResourceTypeInit = 0; } + if (AT_LEAST_VERSION(src, 2, 14)) { + dst->min_erts = src->min_erts; + } else { + dst->min_erts = "erts-?"; + } return lib; }; @@ -3878,6 +3948,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ErtsSysDdllError errdesc = ERTS_SYS_DDLL_ERROR_INIT; Eterm ret = am_ok; int veto; + int taint = 1; struct erl_module_nif* lib = NULL; struct erl_module_instance* this_mi; struct erl_module_instance* prev_mi; @@ -3924,10 +3995,15 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ASSERT(module_p != NULL); mod_atomp = atom_tab(atom_val(mod_atom)); - init_func = erts_static_nif_get_nif_init((char*)mod_atomp->name, mod_atomp->len); - if (init_func != NULL) - handle = init_func; - + { + ErtsStaticNifEntry* sne; + sne = erts_static_nif_get_nif_init((char*)mod_atomp->name, mod_atomp->len); + if (sne != NULL) { + init_func = sne->nif_init; + handle = init_func; + taint = sne->taint; + } + } this_mi = &module_p->curr; prev_mi = &module_p->old; if (in_area(caller, module_p->old.code_hdr, module_p->old.code_length)) { @@ -3964,18 +4040,24 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) " function: '%s'", errdesc.str); } - else if ((add_taint(mod_atom), + else if ((taint ? erts_add_taint(mod_atom) : 0, (entry = erts_sys_ddll_call_nif_init(init_func)) == NULL)) { ret = load_nif_error(BIF_P, bad_lib, "Library init-call unsuccessful"); } + else if (entry->major > ERL_NIF_MAJOR_VERSION + || (entry->major == ERL_NIF_MAJOR_VERSION + && entry->minor > ERL_NIF_MINOR_VERSION)) { + char* fmt = "That '%T' NIF library needs %s or newer. Either try to" + " recompile the NIF lib or use a newer erts runtime."; + ret = load_nif_error(BIF_P, bad_lib, fmt, mod_atom, entry->min_erts); + } else if (entry->major < ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD - || (ERL_NIF_MAJOR_VERSION < entry->major - || (ERL_NIF_MAJOR_VERSION == entry->major - && ERL_NIF_MINOR_VERSION < entry->minor)) || (entry->major==2 && entry->minor == 5)) { /* experimental maps */ - ret = load_nif_error(BIF_P, bad_lib, "Library version (%d.%d) not compatible (with %d.%d).", - entry->major, entry->minor, ERL_NIF_MAJOR_VERSION, ERL_NIF_MINOR_VERSION); + char* fmt = "That old NIF library (%d.%d) is not compatible with this " + "erts runtime (%d.%d). Try recompile the NIF lib."; + ret = load_nif_error(BIF_P, bad_lib, fmt, entry->major, entry->minor, + ERL_NIF_MAJOR_VERSION, ERL_NIF_MINOR_VERSION); } else if (AT_LEAST_VERSION(entry, 2, 1) && sys_strcmp(entry->vm_variant, ERL_NIF_VM_VARIANT) != 0) { @@ -4185,6 +4267,10 @@ int erts_nif_get_funcs(struct erl_module_nif* mod, return mod->entry.num_of_funcs; } +Module *erts_nif_get_module(struct erl_module_nif *nif_mod) { + return nif_mod->mod; +} + Eterm erts_nif_call_function(Process *p, Process *tracee, struct erl_module_nif* mod, ErlNifFunc *fun, int argc, Eterm *argv) diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index a99b4db705..1906da732b 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -52,10 +52,12 @@ ** 2.11: 19.0 enif_snprintf ** 2.12: 20.0 add enif_select, enif_open_resource_type_x ** 2.13: 20.1 add enif_ioq -** 2.14: 21.0 add enif_ioq_peek_head +** 2.14: 21.0 add enif_ioq_peek_head, enif_(mutex|cond|rwlock|thread)_name +** enif_vfprintf, enif_vsnprintf, enif_make_map_from_arrays */ #define ERL_NIF_MAJOR_VERSION 2 #define ERL_NIF_MINOR_VERSION 14 +#define ERL_NIF_MIN_ERTS_VERSION "erts-10.0 (OTP-21)" /* * The emulator will refuse to load a nif-lib with a major version @@ -70,6 +72,8 @@ #define ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD 2 #include <stdlib.h> +#include <stdio.h> +#include <stdarg.h> #ifdef __cplusplus extern "C" { @@ -128,6 +132,9 @@ typedef struct enif_entry_t /* Added in 2.12 */ size_t sizeof_ErlNifResourceTypeInit; + + /* Added in 2.14 */ + const char* min_erts; }ErlNifEntry; @@ -350,7 +357,8 @@ ERL_NIF_INIT_DECL(NAME) \ LOAD, RELOAD, UPGRADE, UNLOAD, \ ERL_NIF_VM_VARIANT, \ 1, \ - sizeof(ErlNifResourceTypeInit) \ + sizeof(ErlNifResourceTypeInit), \ + ERL_NIF_MIN_ERTS_VERSION \ }; \ ERL_NIF_INIT_BODY; \ return &entry; \ diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 3750fd9b68..61f8fcf6ed 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -92,7 +92,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_thread_join,(ErlNifTid, void **respp)); ERL_NIF_API_FUNC_DECL(void*,enif_realloc,(void* ptr, size_t size)); ERL_NIF_API_FUNC_DECL(void,enif_system_info,(ErlNifSysInfo *sip, size_t si_size)); -ERL_NIF_API_FUNC_DECL(int,enif_fprintf,(void/* FILE* */ *filep, const char *format, ...)); +ERL_NIF_API_FUNC_DECL(int,enif_fprintf,(FILE* filep, const char *format, ...)); ERL_NIF_API_FUNC_DECL(int,enif_inspect_iolist_as_binary,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifBinary* bin)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_sub_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, size_t pos, size_t size)); ERL_NIF_API_FUNC_DECL(int,enif_get_string,(ErlNifEnv*, ERL_NIF_TERM list, char* buf, unsigned len, ErlNifCharEncoding)); @@ -200,6 +200,16 @@ ERL_NIF_API_FUNC_DECL(void,enif_free_iovec,(ErlNifIOVec *iov)); ERL_NIF_API_FUNC_DECL(int,enif_ioq_peek_head,(ErlNifEnv *env, ErlNifIOQueue *q, size_t *size, ERL_NIF_TERM *head)); +ERL_NIF_API_FUNC_DECL(char*,enif_mutex_name,(ErlNifMutex*)); +ERL_NIF_API_FUNC_DECL(char*,enif_cond_name,(ErlNifCond*)); +ERL_NIF_API_FUNC_DECL(char*,enif_rwlock_name,(ErlNifRWLock*)); +ERL_NIF_API_FUNC_DECL(char*,enif_thread_name,(ErlNifTid)); + +ERL_NIF_API_FUNC_DECL(int,enif_vfprintf,(FILE*, const char *fmt, va_list)); +ERL_NIF_API_FUNC_DECL(int,enif_vsnprintf,(char*, size_t, const char *fmt, va_list)); + +ERL_NIF_API_FUNC_DECL(int,enif_make_map_from_arrays,(ErlNifEnv *env, ERL_NIF_TERM keys[], ERL_NIF_TERM values[], size_t cnt, ERL_NIF_TERM *map_out)); + /* ** ADD NEW ENTRIES HERE (before this comment) !!! */ @@ -375,6 +385,13 @@ ERL_NIF_API_FUNC_DECL(int,enif_ioq_peek_head,(ErlNifEnv *env, ErlNifIOQueue *q, # define enif_inspect_iovec ERL_NIF_API_FUNC_MACRO(enif_inspect_iovec) # define enif_free_iovec ERL_NIF_API_FUNC_MACRO(enif_free_iovec) # define enif_ioq_peek_head ERL_NIF_API_FUNC_MACRO(enif_ioq_peek_head) +# define enif_mutex_name ERL_NIF_API_FUNC_MACRO(enif_mutex_name) +# define enif_cond_name ERL_NIF_API_FUNC_MACRO(enif_cond_name) +# define enif_rwlock_name ERL_NIF_API_FUNC_MACRO(enif_rwlock_name) +# define enif_thread_name ERL_NIF_API_FUNC_MACRO(enif_thread_name) +# define enif_vfprintf ERL_NIF_API_FUNC_MACRO(enif_vfprintf) +# define enif_vsnprintf ERL_NIF_API_FUNC_MACRO(enif_vsnprintf) +# define enif_make_map_from_arrays ERL_NIF_API_FUNC_MACRO(enif_make_map_from_arrays) /* ** ADD NEW ENTRIES HERE (before this comment) diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h index 6ec428e282..99e938266b 100644 --- a/erts/emulator/beam/erl_node_container_utils.h +++ b/erts/emulator/beam/erl_node_container_utils.h @@ -318,5 +318,3 @@ extern ErtsPTab erts_port; #define is_not_ref(x) (!is_ref(x)) #endif - - diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index e8901a652f..1f147011a8 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2017. All Rights Reserved. + * Copyright Ericsson AB 2001-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,6 +31,7 @@ #include "dtrace-wrapper.h" #include "erl_binary.h" #include "erl_bif_unique.h" +#include "erl_proc_sig_queue.h" Hash erts_dist_table; Hash erts_node_table; @@ -174,11 +175,7 @@ dist_table_alloc(void *dep_tmpl) dep->flags = 0; dep->version = 0; - erts_mtx_init(&dep->lnk_mtx, "dist_entry_links", sysname, - ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); - dep->node_links = NULL; - dep->nlinks = NULL; - dep->monitors = NULL; + dep->mld = NULL; erts_mtx_init(&dep->qlock, "dist_entry_out_queue", sysname, ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); @@ -225,9 +222,7 @@ dist_table_free(void *vdep) ASSERT(de_refc_read(dep, -1) == -1); ASSERT(dep->state == ERTS_DE_STATE_IDLE); ASSERT(is_nil(dep->cid)); - ASSERT(dep->nlinks == NULL); - ASSERT(dep->node_links == NULL); - ASSERT(dep->monitors == NULL); + ASSERT(dep->mld == NULL); /* Link out */ @@ -250,7 +245,6 @@ dist_table_free(void *vdep) ASSERT(!dep->cache); erts_rwmtx_destroy(&dep->rwmtx); - erts_mtx_destroy(&dep->lnk_mtx); erts_mtx_destroy(&dep->qlock); #ifdef DEBUG @@ -606,7 +600,9 @@ erts_set_dist_entry_not_connected(DistEntry *dep) void erts_set_dist_entry_pending(DistEntry *dep) { + ErtsMonLnkDist *mld = erts_mon_link_dist_create(dep->sysname); ERTS_LC_ASSERT(erts_lc_is_de_rwlocked(dep)); + erts_rwmtx_rwlock(&erts_dist_table_rwmtx); ASSERT(dep != erts_this_dist_entry); @@ -631,6 +627,10 @@ erts_set_dist_entry_pending(DistEntry *dep) dep->flags = (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY | DFLAG_NO_MAGIC); dep->connection_id = (dep->connection_id + 1) & ERTS_DIST_CON_ID_MASK; + ASSERT(!dep->mld); + mld->connection_id = dep->connection_id; + dep->mld = mld; + dep->prev = NULL; dep->next = erts_pending_dist_entries; if(erts_pending_dist_entries) { @@ -647,6 +647,8 @@ erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags) { erts_aint32_t set_qflgs; + ASSERT(dep->mld); + ERTS_LC_ASSERT(erts_lc_is_de_rwlocked(dep)); erts_rwmtx_rwlock(&erts_dist_table_rwmtx); @@ -1049,14 +1051,16 @@ static void erts_lcnt_enable_dist_lock_count(void *dep_raw, void *enable) { if(enable) { erts_lcnt_install_new_lock_info(&dep->rwmtx.lcnt, "dist_entry", dep->sysname, ERTS_LOCK_TYPE_RWMUTEX | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); - erts_lcnt_install_new_lock_info(&dep->lnk_mtx.lcnt, "dist_entry_links", dep->sysname, - ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); erts_lcnt_install_new_lock_info(&dep->qlock.lcnt, "dist_entry_out_queue", dep->sysname, ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); + if (dep->mld) + erts_lcnt_install_new_lock_info(&dep->mld->mtx.lcnt, "dist_entry_links", dep->sysname, + ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); } else { erts_lcnt_uninstall(&dep->rwmtx.lcnt); - erts_lcnt_uninstall(&dep->lnk_mtx.lcnt); erts_lcnt_uninstall(&dep->qlock.lcnt); + if (dep->mld) + erts_lcnt_uninstall(&dep->mld->mtx.lcnt); } } @@ -1099,6 +1103,7 @@ static Eterm AM_system; static Eterm AM_timer; static Eterm AM_delayed_delete_timer; static Eterm AM_thread_progress_delete_timer; +static Eterm AM_signal; static void setup_reference_table(void); static Eterm reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp); @@ -1120,6 +1125,7 @@ typedef struct node_referrer_ { int bin_ref; int timer_ref; int system_ref; + int signal_ref; Eterm id; Uint id_heap[ID_HEAP_SIZE]; ErlOffHeap off_heap; @@ -1137,6 +1143,7 @@ typedef struct dist_referrer_ { int node_ref; int ctrl_ref; int system_ref; + int signal_ref; Eterm id; Uint creation; Uint id_heap[ID_HEAP_SIZE]; @@ -1190,6 +1197,7 @@ erts_get_node_and_dist_references(struct process *proc) INIT_AM(system); INIT_AM(delayed_delete_timer); INIT_AM(thread_progress_delete_timer); + INIT_AM(signal); references_atoms_need_init = 0; } @@ -1226,6 +1234,7 @@ erts_get_node_and_dist_references(struct process *proc) #define MONITOR_REF 7 #define TIMER_REF 8 #define SYSTEM_REF 9 +#define SIGNAL_REF 10 #define INC_TAB_SZ 10 @@ -1260,6 +1269,7 @@ insert_dist_referrer(ReferredDist *referred_dist, drp->node_ref = 0; drp->ctrl_ref = 0; drp->system_ref = 0; + drp->signal_ref = 0; } switch (type) { @@ -1268,6 +1278,7 @@ insert_dist_referrer(ReferredDist *referred_dist, case HEAP_REF: drp->heap_ref++; break; case ETS_REF: drp->ets_ref++; break; case SYSTEM_REF: drp->system_ref++; break; + case SIGNAL_REF: drp->signal_ref++; break; default: ASSERT(0); } } @@ -1321,6 +1332,7 @@ insert_node_referrer(ReferredNode *referred_node, int type, Eterm id) nrp->bin_ref = 0; nrp->timer_ref = 0; nrp->system_ref = 0; + nrp->signal_ref = 0; } switch (type) { @@ -1331,6 +1343,7 @@ insert_node_referrer(ReferredNode *referred_node, int type, Eterm id) case MONITOR_REF: nrp->monitor_ref++; break; case TIMER_REF: nrp->timer_ref++; break; case SYSTEM_REF: nrp->system_ref++; break; + case SIGNAL_REF: nrp->signal_ref++; break; default: ASSERT(0); } } @@ -1438,49 +1451,145 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) } } -static void doit_insert_monitor(ErtsMonitor *monitor, void *p) +static void insert_monitor_data(ErtsMonitor *mon, int type, Eterm id) { - Eterm *idp = p; - if(monitor->type != MON_NIF_TARGET && is_external(monitor->u.pid)) - insert_node(external_thing_ptr(monitor->u.pid)->node, MONITOR_REF, *idp); - if(is_external(monitor->ref)) - insert_node(external_thing_ptr(monitor->ref)->node, MONITOR_REF, *idp); + ErtsMonitorData *mdp = erts_monitor_to_data(mon); + if ((mdp->origin.flags & (ERTS_ML_FLG_DBG_VISITED + | ERTS_ML_FLG_EXTENDED)) == ERTS_ML_FLG_EXTENDED) { + if (mon->type != ERTS_MON_TYPE_NODE) { + ErtsMonitorDataExtended *mdep = (ErtsMonitorDataExtended *) mdp; + ASSERT(mon->flags & ERTS_ML_FLG_EXTENDED); + if (mdep->uptr.ohhp) { + ErlOffHeap oh; + ERTS_INIT_OFF_HEAP(&oh); + oh.first = mdep->uptr.ohhp; + insert_offheap(&oh, type, id); + } + } + } + mdp->origin.flags |= ERTS_ML_FLG_DBG_VISITED; } -static void doit_insert_link(ErtsLink *lnk, void *p) +static void insert_monitor(ErtsMonitor *mon, void *idp) { - Eterm *idp = p; - if(is_external(lnk->pid)) - insert_node(external_thing_ptr(lnk->pid)->node, LINK_REF, - *idp); + Eterm id = *((Eterm *) idp); + insert_monitor_data(mon, MONITOR_REF, id); } +static void clear_visited_monitor(ErtsMonitor *mon, void *p) +{ + ErtsMonitorData *mdp = erts_monitor_to_data(mon); + mdp->origin.flags &= ~ERTS_ML_FLG_DBG_VISITED; +} static void -insert_monitors(ErtsMonitor *monitors, Eterm id) +insert_p_monitors(ErtsPTabElementCommon *p) { - erts_doforall_monitors(monitors,&doit_insert_monitor,&id); + Eterm id = p->id; + erts_monitor_tree_foreach(p->u.alive.monitors, + insert_monitor, + (void *) &id); + erts_monitor_list_foreach(p->u.alive.lt_monitors, + insert_monitor, + (void *) &id); +} + +static void +insert_dist_monitors(DistEntry *dep) +{ + if (dep->mld) { + erts_monitor_list_foreach(dep->mld->monitors, + insert_monitor, + (void *) &dep->sysname); + erts_monitor_tree_foreach(dep->mld->orig_name_monitors, + insert_monitor, + (void *) &dep->sysname); + } } static void -insert_links(ErtsLink *lnk, Eterm id) +clear_visited_p_monitors(ErtsPTabElementCommon *p) { - erts_doforall_links(lnk,&doit_insert_link,&id); + erts_monitor_tree_foreach(p->u.alive.monitors, + clear_visited_monitor, + NULL); + erts_monitor_list_foreach(p->u.alive.lt_monitors, + clear_visited_monitor, + NULL); } -static void doit_insert_link2(ErtsLink *lnk, void *p) +static void +clear_visited_dist_monitors(DistEntry *dep) +{ + if (dep->mld) { + erts_monitor_list_foreach(dep->mld->monitors, + clear_visited_monitor, + NULL); + erts_monitor_tree_foreach(dep->mld->orig_name_monitors, + clear_visited_monitor, + NULL); + } +} + +static void insert_link_data(ErtsLink *lnk, int type, Eterm id) +{ + ErtsLinkData *ldp = erts_link_to_data(lnk); + if ((ldp->a.flags & (ERTS_ML_FLG_DBG_VISITED + | ERTS_ML_FLG_EXTENDED)) == ERTS_ML_FLG_EXTENDED) { + ErtsLinkDataExtended *ldep = (ErtsLinkDataExtended *) ldp; + if (ldep->ohhp) { + ErlOffHeap oh; + ERTS_INIT_OFF_HEAP(&oh); + oh.first = ldep->ohhp; + insert_offheap(&oh, type, id); + } + } + ldp->a.flags |= ERTS_ML_FLG_DBG_VISITED; +} + +static void insert_link(ErtsLink *lnk, void *idp) { - Eterm *idp = p; - if(is_external(lnk->pid)) - insert_node(external_thing_ptr(lnk->pid)->node, LINK_REF, - *idp); - insert_links(ERTS_LINK_ROOT(lnk), *idp); + Eterm id = *((Eterm *) idp); + insert_link_data(lnk, LINK_REF, id); +} + +static void clear_visited_link(ErtsLink *lnk, void *p) +{ + ErtsLinkData *ldp = erts_link_to_data(lnk); + ldp->a.flags &= ~ERTS_ML_FLG_DBG_VISITED; } static void -insert_links2(ErtsLink *lnk, Eterm id) +insert_p_links(ErtsPTabElementCommon *p) { - erts_doforall_links(lnk,&doit_insert_link2,&id); + Eterm id = p->id; + erts_link_tree_foreach(p->u.alive.links, insert_link, (void *) &id); +} + +static void +insert_dist_links(DistEntry *dep) +{ + if (dep->mld) + erts_link_list_foreach(dep->mld->links, + insert_link, + (void *) &dep->sysname); +} + +static void +clear_visited_p_links(ErtsPTabElementCommon *p) +{ + erts_link_tree_foreach(p->u.alive.links, + clear_visited_link, + NULL); +} + +static void +clear_visited_dist_links(DistEntry *dep) +{ + if (dep->mld) + erts_link_list_foreach(dep->mld->links, + clear_visited_link, + NULL); } static void @@ -1576,6 +1685,60 @@ insert_delayed_delete_dist_entry(void *state, } static void +insert_message(ErtsMessage *msg, int type, Process *proc) +{ + ErlHeapFragment *heap_frag = NULL; + + ASSERT(ERTS_SIG_IS_MSG(msg)); + if (msg->data.attached) { + if (msg->data.attached == ERTS_MSG_COMBINED_HFRAG) + heap_frag = &msg->hfrag; + else if (ERTS_SIG_IS_INTERNAL_MSG(msg)) + heap_frag = msg->data.heap_frag; + else { + if (msg->data.dist_ext->dep) + insert_dist_entry(msg->data.dist_ext->dep, + type, proc->common.id, 0); + if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) + heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); + } + } + while (heap_frag) { + insert_offheap(&(heap_frag->off_heap), + type, + proc->common.id); + heap_frag = heap_frag->next; + } +} + +static void +insert_sig_msg(ErtsMessage *msg, void *arg) +{ + insert_message(msg, SIGNAL_REF, (Process *) arg); +} + +static void +insert_sig_offheap(ErlOffHeap *ohp, void *arg) +{ + Process *proc = arg; + insert_offheap(ohp, SIGNAL_REF, proc->common.id); +} + +static void +insert_sig_monitor(ErtsMonitor *mon, void *arg) +{ + Process *proc = arg; + insert_monitor_data(mon, SIGNAL_REF, proc->common.id); +} + +static void +insert_sig_link(ErtsLink *lnk, void *arg) +{ + Process *proc = arg; + insert_link_data(lnk, SIGNAL_REF, proc->common.id); +} + +static void setup_reference_table(void) { ErlHeapFragment *hfp; @@ -1630,10 +1793,7 @@ setup_reference_table(void) Process *proc = erts_pix2proc(i); if (proc) { int mli; - ErtsMessage *msg_list[] = { - proc->msg.first, - proc->msg_inq.first, - proc->msg_frag}; + ErtsMessage *msg_list[] = {proc->msg_frag}; /* Insert Heap */ insert_offheap(&(proc->off_heap), @@ -1648,34 +1808,24 @@ setup_reference_table(void) /* Insert msg buffers */ for (mli = 0; mli < sizeof(msg_list)/sizeof(msg_list[0]); mli++) { ErtsMessage *msg; - for (msg = msg_list[mli]; msg; msg = msg->next) { - ErlHeapFragment *heap_frag = NULL; - if (msg->data.attached) { - if (msg->data.attached == ERTS_MSG_COMBINED_HFRAG) - heap_frag = &msg->hfrag; - else if (is_value(ERL_MESSAGE_TERM(msg))) - heap_frag = msg->data.heap_frag; - else { - if (msg->data.dist_ext->dep) - insert_dist_entry(msg->data.dist_ext->dep, - HEAP_REF, proc->common.id, 0); - if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) - heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); - } - } - while (heap_frag) { - insert_offheap(&(heap_frag->off_heap), - HEAP_REF, - proc->common.id); - heap_frag = heap_frag->next; - } - } + for (msg = msg_list[mli]; msg; msg = msg->next) + insert_message(msg, HEAP_REF, proc); } + + /* Insert signal queue */ + erts_proc_sig_debug_foreach_sig(proc, + insert_sig_msg, + insert_sig_offheap, + insert_sig_monitor, + insert_sig_link, + (void *) proc); + /* Insert links */ - if (ERTS_P_LINKS(proc)) - insert_links(ERTS_P_LINKS(proc), proc->common.id); - if (ERTS_P_MONITORS(proc)) - insert_monitors(ERTS_P_MONITORS(proc), proc->common.id); + insert_p_links(&proc->common); + + /* Insert monitors */ + insert_p_monitors(&proc->common); + { DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(proc); if (dep) @@ -1705,11 +1855,9 @@ setup_reference_table(void) continue; /* Insert links */ - if (ERTS_P_LINKS(prt)) - insert_links(ERTS_P_LINKS(prt), prt->common.id); + insert_p_links(&prt->common); /* Insert monitors */ - if (ERTS_P_MONITORS(prt)) - insert_monitors(ERTS_P_MONITORS(prt), prt->common.id); + insert_p_monitors(&prt->common); /* Insert port data */ ohp = erts_port_data_offheap(prt); if (ohp) @@ -1758,41 +1906,25 @@ setup_reference_table(void) /* Insert all dist links */ for(dep = erts_visible_dist_entries; dep; dep = dep->next) { - if(dep->nlinks) - insert_links2(dep->nlinks, dep->sysname); - if(dep->node_links) - insert_links(dep->node_links, dep->sysname); - if(dep->monitors) - insert_monitors(dep->monitors, dep->sysname); + insert_dist_links(dep); + insert_dist_monitors(dep); } for(dep = erts_hidden_dist_entries; dep; dep = dep->next) { - if(dep->nlinks) - insert_links2(dep->nlinks, dep->sysname); - if(dep->node_links) - insert_links(dep->node_links, dep->sysname); - if(dep->monitors) - insert_monitors(dep->monitors, dep->sysname); + insert_dist_links(dep); + insert_dist_monitors(dep); } for(dep = erts_pending_dist_entries; dep; dep = dep->next) { - if(dep->nlinks) - insert_links2(dep->nlinks, dep->sysname); - if(dep->node_links) - insert_links(dep->node_links, dep->sysname); - if(dep->monitors) - insert_monitors(dep->monitors, dep->sysname); + insert_dist_links(dep); + insert_dist_monitors(dep); } /* Not connected dist entries should not have any links, but inspect them anyway */ for(dep = erts_not_connected_dist_entries; dep; dep = dep->next) { - if(dep->nlinks) - insert_links2(dep->nlinks, dep->sysname); - if(dep->node_links) - insert_links(dep->node_links, dep->sysname); - if(dep->monitors) - insert_monitors(dep->monitors, dep->sysname); + insert_dist_links(dep); + insert_dist_monitors(dep); } /* Insert all ets tables */ @@ -1877,6 +2009,10 @@ reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp) tup = MK_2TUP(AM_system, MK_UINT(nrp->system_ref)); nrl = MK_CONS(tup, nrl); } + if(nrp->signal_ref) { + tup = MK_2TUP(AM_signal, MK_UINT(nrp->signal_ref)); + nrl = MK_CONS(tup, nrl); + } nrid = nrp->id; if (!IS_CONST(nrp->id)) { @@ -1900,24 +2036,25 @@ reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp) } else if(is_internal_port(nrid)) { ASSERT(!nrp->heap_ref && !nrp->ets_ref && !nrp->bin_ref - && !nrp->timer_ref && !nrp->system_ref); + && !nrp->timer_ref && !nrp->system_ref && !nrp->signal_ref); tup = MK_2TUP(AM_port, nrid); } else if(nrp->ets_ref) { ASSERT(!nrp->heap_ref && !nrp->link_ref && !nrp->monitor_ref && !nrp->bin_ref - && !nrp->timer_ref && !nrp->system_ref); + && !nrp->timer_ref && !nrp->system_ref && !nrp->signal_ref); tup = MK_2TUP(AM_ets, nrid); } else if(nrp->bin_ref) { ASSERT(is_small(nrid) || is_big(nrid)); ASSERT(!nrp->heap_ref && !nrp->ets_ref && !nrp->link_ref && !nrp->monitor_ref && !nrp->timer_ref - && !nrp->system_ref); + && !nrp->system_ref && !nrp->signal_ref); tup = MK_2TUP(AM_match_spec, nrid); } else { - ASSERT(!nrp->heap_ref && !nrp->ets_ref && !nrp->bin_ref); + ASSERT(!nrp->heap_ref && !nrp->ets_ref && !nrp->bin_ref + && !nrp->signal_ref); ASSERT(is_atom(nrid)); tup = MK_2TUP(AM_dist, nrid); } @@ -1961,30 +2098,36 @@ reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp) tup = MK_2TUP(AM_system, MK_UINT(drp->system_ref)); drl = MK_CONS(tup, drl); } + if(drp->signal_ref) { + tup = MK_2TUP(AM_signal, MK_UINT(drp->signal_ref)); + drl = MK_CONS(tup, drl); + } if (is_internal_pid(drp->id)) { ASSERT(!drp->node_ref); tup = MK_2TUP(AM_process, drp->id); } else if(is_internal_port(drp->id)) { - ASSERT(drp->ctrl_ref && !drp->node_ref); + ASSERT(drp->ctrl_ref && !drp->node_ref && !drp->signal_ref); tup = MK_2TUP(AM_port, drp->id); } else if (is_tuple(drp->id)) { Eterm *t; ASSERT(drp->system_ref && !drp->node_ref - && !drp->ctrl_ref && !drp->heap_ref && !drp->ets_ref); + && !drp->ctrl_ref && !drp->heap_ref && !drp->ets_ref + && !drp->signal_ref); t = tuple_val(drp->id); ASSERT(2 == arityval(t[0])); tup = MK_2TUP(t[1], t[2]); } else if (drp->ets_ref) { ASSERT(!drp->heap_ref && !drp->node_ref && - !drp->ctrl_ref && !drp->system_ref); + !drp->ctrl_ref && !drp->system_ref + && !drp->signal_ref); tup = MK_2TUP(AM_ets, drp->id); } - else { - ASSERT(!drp->ctrl_ref && drp->node_ref); + else { + ASSERT(!drp->ctrl_ref && drp->node_ref && !drp->signal_ref); ASSERT(is_atom(drp->id)); tup = MK_2TUP(drp->id, MK_UINT(drp->creation)); tup = MK_2TUP(AM_node, tup); @@ -2019,10 +2162,21 @@ reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp) } +static void noop_sig_msg(ErtsMessage *msg, void *arg) +{ + +} + +static void noop_sig_offheap(ErlOffHeap *oh, void *arg) +{ + +} + static void delete_reference_table(void) { - Uint i; + DistEntry *dep; + int i, max; for(i = 0; i < no_referred_nodes; i++) { NodeReferrer *nrp; NodeReferrer *tnrp; @@ -2054,6 +2208,60 @@ delete_reference_table(void) inserted_bins = inserted_bins->next; erts_free(ERTS_ALC_T_NC_TMP, (void *)ib); } + + /* Cleanup... */ + + max = erts_ptab_max(&erts_proc); + for (i = 0; i < max; i++) { + Process *proc = erts_pix2proc(i); + if (proc) { + clear_visited_p_links(&proc->common); + clear_visited_p_monitors(&proc->common); + erts_proc_sig_debug_foreach_sig(proc, + noop_sig_msg, + noop_sig_offheap, + clear_visited_monitor, + clear_visited_link, + (void *) proc); + } + } + + max = erts_ptab_max(&erts_port); + for (i = 0; i < max; i++) { + erts_aint32_t state; + Port *prt; + + prt = erts_pix2port(i); + if (!prt) + continue; + + state = erts_atomic32_read_nob(&prt->state); + if (state & ERTS_PORT_SFLGS_DEAD) + continue; + + clear_visited_p_links(&prt->common); + clear_visited_p_monitors(&prt->common); + } + + for(dep = erts_visible_dist_entries; dep; dep = dep->next) { + clear_visited_dist_links(dep); + clear_visited_dist_monitors(dep); + } + + for(dep = erts_hidden_dist_entries; dep; dep = dep->next) { + clear_visited_dist_links(dep); + clear_visited_dist_monitors(dep); + } + + for(dep = erts_pending_dist_entries; dep; dep = dep->next) { + clear_visited_dist_links(dep); + clear_visited_dist_monitors(dep); + } + + for(dep = erts_not_connected_dist_entries; dep; dep = dep->next) { + clear_visited_dist_links(dep); + clear_visited_dist_monitors(dep); + } } void diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index 58279017c8..9a792b10b1 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2017. All Rights Reserved. + * Copyright Ericsson AB 2001-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,7 +18,17 @@ * %CopyrightEnd% */ -#ifndef ERL_NODE_TABLES_H__ +#ifndef ERL_NODE_TABLES_BASIC__ +#define ERL_NODE_TABLES_BASIC__ + +typedef struct dist_entry_ DistEntry; +typedef struct ErtsDistOutputBuf_ ErtsDistOutputBuf; +void erts_ref_dist_entry(DistEntry *dep); +void erts_deref_dist_entry(DistEntry *dep); + +#endif + +#if !defined(ERL_NODE_TABLES_BASIC_ONLY) && !defined(ERL_NODE_TABLES_H__) #define ERL_NODE_TABLES_H__ /* @@ -42,14 +52,14 @@ #include "sys.h" #include "hash.h" #include "erl_alloc.h" -#include "erl_process.h" -#include "erl_monitors.h" #define ERTS_PORT_TASK_ONLY_BASIC_TYPES__ #include "erl_port_task.h" #undef ERTS_PORT_TASK_ONLY_BASIC_TYPES__ +#include "erl_process.h" #define ERTS_BINARY_TYPES_ONLY__ #include "erl_binary.h" #undef ERTS_BINARY_TYPES_ONLY__ +#include "erl_monitor_link.h" #define ERTS_NODE_TAB_DELAY_GC_DEFAULT (60) #define ERTS_NODE_TAB_DELAY_GC_MAX (100*1000*1000) @@ -82,7 +92,6 @@ enum dist_entry_state { #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713) #endif -typedef struct ErtsDistOutputBuf_ ErtsDistOutputBuf; struct ErtsDistOutputBuf_ { #ifdef DEBUG Uint dbg_pattern; @@ -113,7 +122,7 @@ struct ErtsProcList_; * unlock mutexes with higher numbers before mutexes with higher numbers. */ -typedef struct dist_entry_ { +struct dist_entry_ { HashBucket hash_bucket; /* Hash bucket */ struct dist_entry_ *next; /* Next entry in dist_table (not sorted) */ struct dist_entry_ *prev; /* Previous entry in dist_table (not sorted) */ @@ -130,18 +139,7 @@ typedef struct dist_entry_ { atom cache etc. */ unsigned long version; /* Protocol version */ - - erts_mtx_t lnk_mtx; /* Protects node_links, nlinks, and - monitors. */ - ErtsLink *node_links; /* In a dist entry, node links are kept - in a separate tree, while they are - colocted with the ordinary link tree - for processes. It's not due to confusion, - it's because the link tree for the dist - entry is in two levels, see erl_monitors.h - */ - ErtsLink *nlinks; /* Link tree with subtrees */ - ErtsMonitor *monitors; /* Monitor tree */ + ErtsMonLnkDist *mld; /* Monitors and links */ erts_mtx_t qlock; /* Protects qflgs and out_queue */ erts_atomic32_t qflgs; @@ -163,7 +161,7 @@ typedef struct dist_entry_ { ErtsThrPrgrLaterOp later_op; struct transcode_context* transcode_ctx; -} DistEntry; +}; typedef struct erl_node_ { HashBucket hash_bucket; /* Hash bucket */ @@ -219,16 +217,12 @@ int erts_dist_entry_destructor(Binary *bin); DistEntry *erts_dhandle_to_dist_entry(Eterm dhandle); Eterm erts_build_dhandle(Eterm **hpp, ErlOffHeap*, DistEntry*); Eterm erts_make_dhandle(Process *c_p, DistEntry *dep); -void erts_ref_dist_entry(DistEntry *dep); -void erts_deref_dist_entry(DistEntry *dep); ERTS_GLB_INLINE void erts_deref_node_entry(ErlNode *np); ERTS_GLB_INLINE void erts_de_rlock(DistEntry *dep); ERTS_GLB_INLINE void erts_de_runlock(DistEntry *dep); ERTS_GLB_INLINE void erts_de_rwlock(DistEntry *dep); ERTS_GLB_INLINE void erts_de_rwunlock(DistEntry *dep); -ERTS_GLB_INLINE void erts_de_links_lock(DistEntry *dep); -ERTS_GLB_INLINE void erts_de_links_unlock(DistEntry *dep); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -264,18 +258,6 @@ erts_de_rwunlock(DistEntry *dep) erts_rwmtx_rwunlock(&dep->rwmtx); } -ERTS_GLB_INLINE void -erts_de_links_lock(DistEntry *dep) -{ - erts_mtx_lock(&dep->lnk_mtx); -} - -ERTS_GLB_INLINE void -erts_de_links_unlock(DistEntry *dep) -{ - erts_mtx_unlock(&dep->lnk_mtx); -} - #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ void erts_debug_test_node_tab_delayed_delete(Sint64 millisecs); diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index 0d148ee048..9b52b648e5 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012-2017. All Rights Reserved. + * Copyright Ericsson AB 2012-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -373,7 +373,7 @@ Eterm erts_request_io_bytes(Process *c_p); void print_port_info(Port *, fmtfn_t, void *); void erts_port_free(Port *); -void erts_fire_port_monitor(Port *prt, Eterm ref); +void erts_fire_port_monitor(Port *prt, ErtsMonitor *tmon); int erts_port_handle_xports(Port *); #if defined(ERTS_ENABLE_LOCK_CHECK) @@ -485,6 +485,8 @@ ERTS_GLB_INLINE Uint32 erts_portid2status(Eterm); ERTS_GLB_INLINE int erts_is_port_alive(Eterm); ERTS_GLB_INLINE int erts_is_valid_tracer_port(Eterm); ERTS_GLB_INLINE int erts_port_driver_callback_epilogue(Port *, erts_aint32_t *); +ERTS_GLB_INLINE Port *erts_get_current_port(void); +ERTS_GLB_INLINE Eterm erts_get_current_port_id(void); #define erts_drvport2port(Prt) erts_drvport2port_state((Prt), NULL) @@ -812,6 +814,20 @@ erts_port_driver_callback_epilogue(Port *prt, erts_aint32_t *statep) return reds; } +ERTS_GLB_INLINE +Port *erts_get_current_port(void) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + return esdp ? esdp->current_port : NULL; +} + +ERTS_GLB_INLINE +Eterm erts_get_current_port_id(void) +{ + Port *port = erts_get_current_port(); + return port ? port->common.id : THE_NON_VALUE; +} + #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ void erts_port_resume_procs(Port *); @@ -887,20 +903,20 @@ struct ErtsProc2PortSigData_ { Eterm item; } info; struct { - Eterm port; - Eterm to; + Eterm port_id; + ErtsLink *lnk; } link; struct { - Eterm from; + Eterm port_id; + ErtsLink *lnk; } unlink; struct { - Eterm origin; /* who receives monitor event, pid */ - Eterm name; /* either name for named monitor, or port id */ + Eterm port_id; + ErtsMonitor *mon; } monitor; struct { - Eterm origin; /* who is at the other end of the monitor, pid */ - Eterm name; /* port id */ - Uint32 ref[ERTS_MAX_REF_NUMBERS]; /* box contents of a ref */ + Eterm port_id; + ErtsMonitor *mon; } demonitor; } u; } ; @@ -946,7 +962,6 @@ typedef int (*ErtsProc2PortSigCallback)(Port *, typedef enum { ERTS_PORT_OP_BADARG, - ERTS_PORT_OP_CALLER_EXIT, ERTS_PORT_OP_BUSY, ERTS_PORT_OP_BUSY_SCHEDULED, ERTS_PORT_OP_SCHEDULED, @@ -982,32 +997,13 @@ ErtsPortOpResult erts_port_command(Process *, int, Port *, Eterm, Eterm *); ErtsPortOpResult erts_port_output(Process *, int, Port *, Eterm, Eterm, Eterm *); ErtsPortOpResult erts_port_exit(Process *, int, Port *, Eterm, Eterm, Eterm *); ErtsPortOpResult erts_port_connect(Process *, int, Port *, Eterm, Eterm, Eterm *); -ErtsPortOpResult erts_port_link(Process *, Port *, Eterm, Eterm *); -ErtsPortOpResult erts_port_unlink(Process *, Port *, Eterm, Eterm *); +ErtsPortOpResult erts_port_link(Process *, Port *, ErtsLink *, Eterm *); +ErtsPortOpResult erts_port_unlink(Process *, Port *, ErtsLink *, Eterm *); ErtsPortOpResult erts_port_control(Process *, Port *, unsigned int, Eterm, Eterm *); ErtsPortOpResult erts_port_call(Process *, Port *, unsigned int, Eterm, Eterm *); ErtsPortOpResult erts_port_info(Process *, Port *, Eterm, Eterm *); - -/* Creates monitor between Origin and Target. Ref must be initialized to - * a reference (ref may be rewritten to be used to serve additionally as a - * signal id). Name is atom if user monitors port by name or NIL */ -ErtsPortOpResult erts_port_monitor(Process *origin, Port *target, Eterm name, - Eterm *ref); - -typedef enum { - /* Normal demonitor rules apply with locking and reductions bump */ - ERTS_PORT_DEMONITOR_NORMAL = 1, - /* Relaxed demonitor rules when process is about to die, which means that - * pid lookup won't work, locks won't work, no reductions bump. */ - ERTS_PORT_DEMONITOR_ORIGIN_ON_DEATHBED = 2, -} ErtsDemonitorMode; - -/* Removes monitor between origin and target, identified by ref. - * origin_is_dying can be 0 (false, normal locking rules and reductions bump - * apply) or 1 (true, in case when we avoid origin locking) */ -ErtsPortOpResult erts_port_demonitor(Process *origin, ErtsDemonitorMode mode, - Port *target, Eterm ref, - Eterm *trap_ref); +ErtsPortOpResult erts_port_monitor(Process *, Port *, ErtsMonitor *); +ErtsPortOpResult erts_port_demonitor(Process *, Port *, ErtsMonitor *); /* defined in erl_bif_port.c */ Port *erts_sig_lookup_port(Process *c_p, Eterm id_or_name); diff --git a/erts/emulator/beam/erl_posix_str.c b/erts/emulator/beam/erl_posix_str.c index deb7e3e173..7b3e640d3f 100644 --- a/erts/emulator/beam/erl_posix_str.c +++ b/erts/emulator/beam/erl_posix_str.c @@ -156,6 +156,9 @@ erl_errno_id(error) #ifdef EFAULT case EFAULT: return "efault"; #endif +#ifdef EFTYPE + case EFTYPE: return "eftype"; +#endif #ifdef EFBIG case EFBIG: return "efbig"; #endif @@ -351,6 +354,9 @@ erl_errno_id(error) #if defined(EOPNOTSUPP) && (!defined(ENOTSUP) || (EOPNOTSUPP != ENOTSUP)) case EOPNOTSUPP: return "eopnotsupp"; #endif +#ifdef EOVERFLOW + case EOVERFLOW: return "eoverflow"; +#endif #ifdef EPERM case EPERM: return "eperm"; #endif diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index e6f8460164..910f241a3a 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -532,14 +532,13 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) { Atom* module = atom_tab(atom_val(ep->info.mfa.module)); Atom* name = atom_tab(atom_val(ep->info.mfa.function)); - PRINT_STRING(res, fn, arg, "#Fun<"); + PRINT_STRING(res, fn, arg, "fun "); PRINT_BUF(res, fn, arg, module->name, module->len); - PRINT_CHAR(res, fn, arg, '.'); + PRINT_CHAR(res, fn, arg, ':'); PRINT_BUF(res, fn, arg, name->name, name->len); - PRINT_CHAR(res, fn, arg, '.'); + PRINT_CHAR(res, fn, arg, '/'); PRINT_SWORD(res, fn, arg, 'd', 0, 1, (ErlPfSWord) ep->info.mfa.arity); - PRINT_CHAR(res, fn, arg, '>'); } break; case FUN_DEF: diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c new file mode 100644 index 0000000000..a2e6f1d39d --- /dev/null +++ b/erts/emulator/beam/erl_proc_sig_queue.c @@ -0,0 +1,4405 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Process signal queue implementation. + * + * Author: Rickard Green + */ + + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "global.h" +#include "dist.h" +#include "erl_process.h" +#include "erl_port_task.h" +#include "erl_trace.h" +#include "beam_bp.h" +#include "big.h" +#include "erl_gc.h" +#include "bif.h" +#include "erl_proc_sig_queue.h" +#include "dtrace-wrapper.h" + +#define ERTS_SIG_REDS_CNT_FACTOR 4 +#define ERTS_PROC_SIG_TRACE_COUNT_LIMIT 200 + +/* + * Note that not all signal are handled using this functionality! + */ + +#define ERTS_SIG_Q_OP_MAX 11 + +#define ERTS_SIG_Q_OP_EXIT 0 +#define ERTS_SIG_Q_OP_EXIT_LINKED 1 +#define ERTS_SIG_Q_OP_MONITOR_DOWN 2 +#define ERTS_SIG_Q_OP_MONITOR 3 +#define ERTS_SIG_Q_OP_DEMONITOR 4 +#define ERTS_SIG_Q_OP_LINK 5 +#define ERTS_SIG_Q_OP_UNLINK 6 +#define ERTS_SIG_Q_OP_GROUP_LEADER 7 +#define ERTS_SIG_Q_OP_TRACE_CHANGE_STATE 8 +#define ERTS_SIG_Q_OP_PERSISTENT_MON_MSG 9 +#define ERTS_SIG_Q_OP_IS_ALIVE 10 +#define ERTS_SIG_Q_OP_PROCESS_INFO ERTS_SIG_Q_OP_MAX + +#define ERTS_SIG_Q_TYPE_MAX (ERTS_MON_LNK_TYPE_MAX + 5) + +#define ERTS_SIG_Q_TYPE_UNDEFINED \ + (ERTS_MON_LNK_TYPE_MAX + 1) +#define ERTS_SIG_Q_TYPE_DIST_LINK \ + (ERTS_MON_LNK_TYPE_MAX + 2) +#define ERTS_SIG_Q_TYPE_GEN_EXIT \ + (ERTS_MON_LNK_TYPE_MAX + 3) +#define ERTS_SIG_Q_TYPE_DIST_PROC_DEMONITOR \ + (ERTS_MON_LNK_TYPE_MAX + 4) +#define ERTS_SIG_Q_TYPE_ADJUST_TRACE_INFO \ + ERTS_SIG_Q_TYPE_MAX + +Process *ERTS_WRITE_UNLIKELY(erts_dirty_process_signal_handler); +Process *ERTS_WRITE_UNLIKELY(erts_dirty_process_signal_handler_high); +Process *ERTS_WRITE_UNLIKELY(erts_dirty_process_signal_handler_max); + +void +erts_proc_sig_queue_init(void) +{ + ERTS_CT_ASSERT(ERTS_SIG_Q_OP_MASK > ERTS_SIG_Q_OP_MAX); + ERTS_CT_ASSERT(ERTS_SIG_Q_OP_MSGQ_LEN_OFFS_MARK > ERTS_SIG_Q_OP_MAX); + ERTS_CT_ASSERT(ERTS_SIG_Q_TYPE_MASK >= ERTS_SIG_Q_TYPE_MAX); +} + +typedef struct { + int active; + int procs; + struct { + int active; +#if defined(USE_VM_PROBES) + int vm_probes; + char receiver_name[DTRACE_TERM_BUF_SIZE]; +#endif + int receive_trace; + int bp_ix; + ErtsMessage **next; + ErtsTracingEvent *event; + } messages; +} ErtsSigRecvTracing; + +typedef struct { + Eterm message; + Eterm from; + Eterm reason; + union { + Eterm ref; + int normal_kills; + } u; +} ErtsExitSignalData; + +typedef struct { + Eterm message; + Eterm key; +} ErtsPersistMonMsg; + +typedef struct { + ErtsSignalCommon common; + Eterm local; /* internal pid (immediate) */ + Eterm remote; /* external pid (heap for it follow) */ + Eterm heap[EXTERNAL_THING_HEAD_SIZE + 1]; +} ErtsSigDistLinkOp; + +typedef struct { + ErtsSignalCommon common; + Uint flags_on; + Uint flags_off; + Eterm tracer; +} ErtsSigTraceInfo; + +#define ERTS_SIG_GL_FLG_ACTIVE (((erts_aint_t) 1) << 0) +#define ERTS_SIG_GL_FLG_RECEIVER (((erts_aint_t) 1) << 1) +#define ERTS_SIG_GL_FLG_SENDER (((erts_aint_t) 1) << 2) + +typedef struct { + ErtsSignalCommon common; + erts_atomic_t flags; + Eterm group_leader; + Eterm reply_to; + Eterm ref; + ErlOffHeap oh; + Eterm heap[1]; +} ErtsSigGroupLeader; + +typedef struct { + Eterm message; + Eterm requester; +} ErtsIsAliveRequest; + +typedef struct { + ErtsSignalCommon common; + Sint refc; + Sint delayed_len; + Sint len_offset; +} ErtsProcSigMsgQLenOffsetMarker; + +typedef struct { + ErtsSignalCommon common; + ErtsProcSigMsgQLenOffsetMarker marker; + Sint msgq_len_offset; + Eterm requester; + Eterm ref; + ErtsORefThing oref_thing; + Uint reserve_size; + Uint len; + int flags; + int item_ix[1]; /* of len size in reality... */ +} ErtsProcessInfoSig; + +#define ERTS_PROC_SIG_PI_MSGQ_LEN_IGNORE ((Sint) -1) +#define ERTS_PROC_SIG_PI_MSGQ_LEN_SYNC ((Sint) -2) + +static int handle_msg_tracing(Process *c_p, + ErtsSigRecvTracing *tracing, + ErtsMessage ***next_nm_sig); +static int handle_trace_change_state(Process *c_p, + ErtsSigRecvTracing *tracing, + Uint16 type, + ErtsMessage *sig, + ErtsMessage ***next_nm_sig); +static void getting_unlinked(Process *c_p, Eterm unlinker); +static void getting_linked(Process *c_p, Eterm linker); +static void group_leader_reply(Process *c_p, Eterm to, + Eterm ref, int success); +static int stretch_limit(Process *c_p, ErtsSigRecvTracing *tp, + int abs_lim, int *limp); + +#ifdef ERTS_PROC_SIG_HARD_DEBUG +#define ERTS_PROC_SIG_HDBG_PRIV_CHKQ(P, T, NMN) \ + do { \ + ErtsMessage **nm_next__ = *(NMN); \ + ErtsMessage **nm_last__ = (P)->sig_qs.nmsigs.last; \ + if (!nm_next__ || !*nm_next__) { \ + nm_next__ = NULL; \ + nm_last__ = NULL; \ + } \ + proc_sig_hdbg_check_queue((P), \ + 1, \ + &(P)->sig_qs.cont, \ + (P)->sig_qs.cont_last, \ + nm_next__, \ + nm_last__, \ + (T), \ + NULL, \ + ERTS_PSFLG_FREE); \ + } while (0); +static Sint +proc_sig_hdbg_check_queue(Process *c_p, + int privq, + ErtsMessage **sig_next, + ErtsMessage **sig_last, + ErtsMessage **sig_nm_next, + ErtsMessage **sig_nm_last, + ErtsSigRecvTracing *tracing, + int *found_saved_last_p, + erts_aint32_t sig_psflg); +#else +#define ERTS_PROC_SIG_HDBG_PRIV_CHKQ(P, T, NMN) +#endif + +typedef struct { + ErtsSignalCommon common; + Eterm ref; + Eterm heap[1]; +} ErtsSigDistProcDemonitor; + +static void +destroy_dist_proc_demonitor(ErtsSigDistProcDemonitor *dmon) +{ + Eterm ref = dmon->ref; + if (is_external(ref)) { + ExternalThing *etp = external_thing_ptr(ref); + erts_deref_node_entry(etp->node); + } + erts_free(ERTS_ALC_T_DIST_DEMONITOR, dmon); +} + +static ERTS_INLINE ErtsSigDistLinkOp * +make_sig_dist_link_op(int op, Eterm local, Eterm remote) +{ + Eterm *hp; + ErlOffHeap oh = {0}; + ErtsSigDistLinkOp *sdlnk = erts_alloc(ERTS_ALC_T_SIG_DATA, + sizeof(ErtsSigDistLinkOp)); + ASSERT(is_internal_pid(local)); + ASSERT(is_external_pid(remote)); + + hp = &sdlnk->heap[0]; + + sdlnk->common.tag = ERTS_PROC_SIG_MAKE_TAG(op, + ERTS_SIG_Q_TYPE_DIST_LINK, + 0); + sdlnk->local = local; + sdlnk->remote = STORE_NC(&hp, &oh, remote); + + ASSERT(&sdlnk->heap[0] < hp); + ASSERT(hp <= &sdlnk->heap[0] + sizeof(sdlnk->heap)/sizeof(sdlnk->heap[0])); + ASSERT(boxed_val(sdlnk->remote) == &sdlnk->heap[0]); + + return sdlnk; +} + +static ERTS_INLINE void +destroy_sig_dist_link_op(ErtsSigDistLinkOp *sdlnk) +{ + ASSERT(is_external_pid(sdlnk->remote)); + ASSERT(boxed_val(sdlnk->remote) == &sdlnk->heap[0]); + erts_deref_node_entry(((ExternalThing *) &sdlnk->heap[0])->node); + erts_free(ERTS_ALC_T_SIG_DATA, sdlnk); +} + +static ERTS_INLINE ErtsExitSignalData * +get_exit_signal_data(ErtsMessage *xsig) +{ + ASSERT(ERTS_SIG_IS_NON_MSG(xsig)); + ASSERT((ERTS_PROC_SIG_OP(((ErtsSignal *) xsig)->common.tag) + == ERTS_SIG_Q_OP_EXIT) + || (ERTS_PROC_SIG_OP(((ErtsSignal *) xsig)->common.tag) + == ERTS_SIG_Q_OP_EXIT_LINKED) + || (ERTS_PROC_SIG_OP(((ErtsSignal *) xsig)->common.tag) + == ERTS_SIG_Q_OP_MONITOR_DOWN)); + ASSERT(xsig->hfrag.alloc_size > xsig->hfrag.used_size); + ASSERT((xsig->hfrag.alloc_size - xsig->hfrag.used_size)*sizeof(UWord) + >= sizeof(ErtsExitSignalData)); + return (ErtsExitSignalData *) (char *) (&xsig->hfrag.mem[0] + + xsig->hfrag.used_size); +} + +static ERTS_INLINE void +destroy_trace_info(ErtsSigTraceInfo *ti) +{ + if (is_value(ti->tracer)) + erts_tracer_update(&ti->tracer, NIL); + erts_free(ERTS_ALC_T_SIG_DATA, ti); +} + +static void +destroy_sig_group_leader(ErtsSigGroupLeader *sgl) +{ + erts_cleanup_offheap(&sgl->oh); + erts_free(ERTS_ALC_T_SIG_DATA, sgl); +} + +static ERTS_INLINE void +sig_enqueue_trace(Process *c_p, ErtsMessage **sigp, int op, + Process *rp, ErtsMessage ***last_next) +{ + switch (op) { + case ERTS_SIG_Q_OP_LINK: + if (c_p + && ((!!IS_TRACED(c_p)) + & (ERTS_TRACE_FLAGS(c_p) & (F_TRACE_SOL + | F_TRACE_SOL1)))) { + ErtsSigTraceInfo *ti; + Eterm tag; + /* + * Set on link enabled. + * + * Prepend a trace-change-state signal before the + * link signal... + */ + tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_TRACE_CHANGE_STATE, + ERTS_SIG_Q_TYPE_ADJUST_TRACE_INFO, + 0); + ti = erts_alloc(ERTS_ALC_T_SIG_DATA, sizeof(ErtsSigTraceInfo)); + ti->common.next = *sigp; + ti->common.specific.next = &ti->common.next; + ti->common.tag = tag; + ti->flags_on = ERTS_TRACE_FLAGS(c_p) & TRACEE_FLAGS; + if (!(ti->flags_on & F_TRACE_SOL1)) + ti->flags_off = 0; + else { + ti->flags_off = F_TRACE_SOL1|F_TRACE_SOL; + erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + ERTS_TRACE_FLAGS(c_p) &= ~(F_TRACE_SOL1|F_TRACE_SOL); + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + } + erts_tracer_update(&ti->tracer, ERTS_TRACER(c_p)); + *sigp = (ErtsMessage *) ti; + if (!*last_next || *last_next == sigp) + *last_next = &ti->common.next; + } + break; + +#ifdef USE_VM_PROBES + case ERTS_SIG_Q_OP_EXIT: + case ERTS_SIG_Q_OP_EXIT_LINKED: + + if (DTRACE_ENABLED(process_exit_signal)) { + ErtsMessage* sig = *sigp; + Uint16 type = ERTS_PROC_SIG_TYPE(((ErtsSignal *) sig)->common.tag); + Eterm reason, from; + + if (type == ERTS_SIG_Q_TYPE_GEN_EXIT) { + ErtsExitSignalData *xsigd = get_exit_signal_data(sig); + reason = xsigd->reason; + from = xsigd->from; + } + else { + ErtsLink *lnk = (ErtsLink *) sig, *olnk; + + ASSERT(type == ERTS_LNK_TYPE_PROC + || type == ERTS_LNK_TYPE_PORT + || type == ERTS_LNK_TYPE_DIST_PROC); + + olnk = erts_link_to_other(lnk, NULL); + reason = lnk->other.item; + from = olnk->other.item; + } + + if (is_pid(from)) { + + DTRACE_CHARBUF(sender_str, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(receiver_str, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(reason_buf, DTRACE_TERM_BUF_SIZE); + + if (reason == am_kill) { + reason = am_killed; + } + + dtrace_pid_str(from, sender_str); + dtrace_proc_str(rp, receiver_str); + erts_snprintf(reason_buf, sizeof(DTRACE_CHARBUF_NAME(reason_buf)) - 1, "%T", reason); + DTRACE3(process_exit_signal, sender_str, receiver_str, reason_buf); + } + } + break; + +#endif + + default: + break; + } +} + +static void +sig_enqueue_trace_cleanup(ErtsMessage *first, ErtsSignal *sig, ErtsMessage *last) +{ + ErtsMessage *tmp; + + /* The usual case; no tracing signals... */ + if (sig == (ErtsSignal *) first && sig == (ErtsSignal *) last) { + sig->common.next = NULL; + return; + } + + /* Got trace signals to clean up... */ + + tmp = first; + + while (tmp) { + ErtsMessage *tmp_free = tmp; + tmp = tmp->next; + if (sig != (ErtsSignal *) tmp_free) { + switch (ERTS_PROC_SIG_OP(((ErtsSignal *) tmp_free)->common.tag)) { + case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE: + destroy_trace_info((ErtsSigTraceInfo *) tmp_free); + break; + default: + ERTS_INTERNAL_ERROR("Unexpected signal op"); + break; + } + } + } +} + +#ifdef DEBUG +static int dbg_count_nmsigs(ErtsMessage *first) +{ + ErtsMessage *sig; + int cnt = 0; + + for (sig = first; sig; sig = sig->next) { + if (ERTS_SIG_IS_NON_MSG(sig)) + ++cnt; + } + return cnt; +} +#endif + +static ERTS_INLINE erts_aint32_t +enqueue_signals(Process *rp, ErtsMessage *first, + ErtsMessage **last, ErtsMessage **last_next, + Uint num_msgs, + erts_aint32_t in_state) +{ + erts_aint32_t state = in_state; + ErtsMessage **this = rp->sig_inq.last; + + ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE(rp); + + ASSERT(!*this); + *this = first; + rp->sig_inq.last = last; + + if (!rp->sig_inq.nmsigs.next) { + ASSERT(!rp->sig_inq.nmsigs.last); + if (ERTS_SIG_IS_NON_MSG(first)) { + rp->sig_inq.nmsigs.next = this; + } + else if (last_next) { + ASSERT(first->next && ERTS_SIG_IS_NON_MSG(first->next)); + rp->sig_inq.nmsigs.next = &first->next; + } + else + goto no_nmsig; + + state = erts_atomic32_read_bor_nob(&rp->state, + ERTS_PSFLG_SIG_IN_Q); + no_nmsig: + ASSERT(!(state & ERTS_PSFLG_SIG_IN_Q)); + } + else { + ErtsSignal *sig; + ASSERT(rp->sig_inq.nmsigs.last); + + sig = (ErtsSignal *) *rp->sig_inq.nmsigs.last; + + ASSERT(sig && !sig->common.specific.next); + ASSERT(state & ERTS_PSFLG_SIG_IN_Q); + if (ERTS_SIG_IS_NON_MSG(first)) { + sig->common.specific.next = this; + } + else if (last_next) { + ASSERT(first->next && ERTS_SIG_IS_NON_MSG(first->next)); + sig->common.specific.next = &first->next; + } + } + + if (last_next) { + ASSERT(dbg_count_nmsigs(first) >= 2); + rp->sig_inq.nmsigs.last = last_next; + } + else if (ERTS_SIG_IS_NON_MSG(first)) { + ASSERT(dbg_count_nmsigs(first) == 1); + rp->sig_inq.nmsigs.last = this; + } + else + ASSERT(dbg_count_nmsigs(first) == 0); + + rp->sig_inq.len += num_msgs; + + ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE(rp); + + return state; +} + +erts_aint32_t erts_enqueue_signals(Process *rp, ErtsMessage *first, + ErtsMessage **last, ErtsMessage **last_next, + Uint num_msgs, + erts_aint32_t in_state) +{ + return enqueue_signals(rp, first, last, last_next, num_msgs, in_state); +} + +static ERTS_INLINE void +ensure_dirty_proc_handled(Eterm pid, + erts_aint32_t state, + erts_aint32_t prio) +{ + if (state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + Eterm *hp; + ErtsMessage *mp; + Process *sig_handler; + + if (prio < 0) + prio = (int) ERTS_PSFLGS_GET_USR_PRIO(state); + + switch (prio) { + case PRIORITY_MAX: + sig_handler = erts_dirty_process_signal_handler_max; + break; + case PRIORITY_HIGH: + sig_handler = erts_dirty_process_signal_handler_high; + break; + default: + sig_handler = erts_dirty_process_signal_handler; + break; + } + + /* Make sure signals are handled... */ + mp = erts_alloc_message(0, &hp); + erts_queue_message(sig_handler, 0, mp, pid, am_system); + } +} + +static void +check_push_msgq_len_offs_marker(Process *rp, ErtsSignal *sig); + +static int +proc_queue_signal(Process *c_p, Eterm pid, ErtsSignal *sig, int op) +{ + int res; + Process *rp; + ErtsMessage *first, *last, **last_next, **sigp; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + int is_normal_sched = !!esdp && esdp->type == ERTS_SCHED_NORMAL; + erts_aint32_t state; + ErtsSignal *pend_sig; + + if (is_normal_sched) { + pend_sig = esdp->pending_signal.sig; + if (op == ERTS_SIG_Q_OP_MONITOR + && ((ErtsMonitor*)sig)->type == ERTS_MON_TYPE_PROC) { + + if (!pend_sig) { + esdp->pending_signal.sig = sig; + esdp->pending_signal.to = pid; +#ifdef DEBUG + esdp->pending_signal.dbg_from = esdp->current_process; +#endif + return 1; + } + ASSERT(esdp->pending_signal.dbg_from == esdp->current_process); + if (pend_sig != sig) { + /* Switch them and send previously pending signal instead */ + Eterm pend_to = esdp->pending_signal.to; + esdp->pending_signal.sig = sig; + esdp->pending_signal.to = pid; + sig = pend_sig; + pid = pend_to; + } + else { + /* Caller wants to flush pending signal */ + ASSERT(pid == esdp->pending_signal.to); + esdp->pending_signal.sig = NULL; + esdp->pending_signal.to = THE_NON_VALUE; +#ifdef DEBUG + esdp->pending_signal.dbg_from = NULL; +#endif + pend_sig = NULL; + } + rp = erts_proc_lookup_raw(pid); + if (!rp) { + erts_proc_sig_send_monitor_down((ErtsMonitor*)sig, am_noproc); + return 1; + } + } + else if (pend_sig && pid == esdp->pending_signal.to) { + /* Flush pending signal to maintain signal order */ + esdp->pending_signal.sig = NULL; + esdp->pending_signal.to = THE_NON_VALUE; + + rp = erts_proc_lookup_raw(pid); + if (!rp) { + erts_proc_sig_send_monitor_down((ErtsMonitor*)pend_sig, am_noproc); + return 0; + } + + /* Prepend pending signal */ + pend_sig->common.next = (ErtsMessage*) sig; + pend_sig->common.specific.next = &pend_sig->common.next; + first = (ErtsMessage*) pend_sig; + last = (ErtsMessage*) sig; + sigp = last_next = &pend_sig->common.next; + goto first_last_done; + } + else { + pend_sig = NULL; + rp = erts_proc_lookup_raw(pid); + if (!rp) + return 0; + } + } + else { + rp = erts_proc_lookup_raw_inc_refc(pid); + if (!rp) + return 0; + pend_sig = NULL; + } + + first = last = (ErtsMessage *) sig; + last_next = NULL; + sigp = &first; + +first_last_done: + sig->common.specific.next = NULL; + + /* may add signals before and/or after sig */ + sig_enqueue_trace(c_p, sigp, op, rp, &last_next); + + last->next = NULL; + + erts_proc_lock(rp, ERTS_PROC_LOCK_MSGQ); + + state = erts_atomic32_read_nob(&rp->state); + + if (ERTS_PSFLG_FREE & state) + res = 0; + else { + state = enqueue_signals(rp, first, &last->next, last_next, 0, state); + if (ERTS_UNLIKELY(op == ERTS_SIG_Q_OP_PROCESS_INFO)) + check_push_msgq_len_offs_marker(rp, sig); + res = !0; + } + + erts_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ); + + if (res == 0) { + if (pend_sig) { + if (sig == pend_sig) { + /* We did a switch, callers signal is now pending (still ok) */ + ASSERT(esdp->pending_signal.sig); + res = 1; + } + else { + ASSERT(first == (ErtsMessage*)pend_sig); + first = first->next; + } + erts_proc_sig_send_monitor_down((ErtsMonitor*)pend_sig, am_noproc); + } + sig_enqueue_trace_cleanup(first, sig, last); + } + + if (!(state & (ERTS_PSFLG_EXITING + | ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_SIG_IN_Q))) { + /* Schedule process... */ + state = erts_proc_sys_schedule(rp, state, 0); + } + + ensure_dirty_proc_handled(rp->common.id, state, -1); + + if (!is_normal_sched) + erts_proc_dec_refc(rp); + + return res; +} + +void erts_proc_sig_send_pending(ErtsSchedulerData* esdp) +{ + ErtsSignal* sig = esdp->pending_signal.sig; + int op; + + ASSERT(esdp && esdp->type == ERTS_SCHED_NORMAL); + ASSERT(sig); + ASSERT(is_internal_pid(esdp->pending_signal.to)); + + op = ERTS_SIG_Q_OP_MONITOR; + ASSERT(op == ERTS_PROC_SIG_OP(sig->common.tag)); + + if (!proc_queue_signal(NULL, esdp->pending_signal.to, sig, op)) { + ErtsMonitor* mon = (ErtsMonitor*)sig; + erts_proc_sig_send_monitor_down(mon, am_noproc); + } +} + +static int +maybe_elevate_sig_handling_prio(Process *c_p, Eterm other) +{ + /* + * returns: + * > 0 -> elevated prio; process alive or exiting + * < 0 -> no elevation needed; process alive or exiting + * 0 -> process terminated (free) + */ + int res; + Process *rp; + erts_aint32_t state, my_prio, other_prio; + + rp = erts_proc_lookup_raw(other); + if (!rp) + res = 0; + else { + res = -1; + state = erts_atomic32_read_nob(&c_p->state); + my_prio = ERTS_PSFLGS_GET_USR_PRIO(state); + + state = erts_atomic32_read_nob(&rp->state); + other_prio = ERTS_PSFLGS_GET_USR_PRIO(state); + + if (other_prio > my_prio) { + /* Others prio is lower than mine; elevate it... */ + res = !!erts_sig_prio(other, my_prio); + if (res) { + /* ensure handled if dirty executing... */ + state = erts_atomic32_read_nob(&rp->state); + ensure_dirty_proc_handled(other, state, my_prio); + } + } + } + return res; +} + +void +erts_proc_sig_fetch__(Process *proc) +{ + ASSERT(proc->sig_inq.first); + + if (!proc->sig_inq.nmsigs.next) { + ASSERT(!(ERTS_PSFLG_SIG_IN_Q + & erts_atomic32_read_nob(&proc->state))); + ASSERT(!proc->sig_inq.nmsigs.last); + + if (proc->sig_qs.cont || ERTS_MSG_RECV_TRACED(proc)) { + *proc->sig_qs.cont_last = proc->sig_inq.first; + proc->sig_qs.cont_last = proc->sig_inq.last; + } + else { + *proc->sig_qs.last = proc->sig_inq.first; + proc->sig_qs.last = proc->sig_inq.last; + } + } + else { + erts_aint32_t s; + ASSERT(proc->sig_inq.nmsigs.last); + if (!proc->sig_qs.nmsigs.last) { + ASSERT(!proc->sig_qs.nmsigs.next); + if (proc->sig_inq.nmsigs.next == &proc->sig_inq.first) + proc->sig_qs.nmsigs.next = proc->sig_qs.cont_last; + else + proc->sig_qs.nmsigs.next = proc->sig_inq.nmsigs.next; + + s = erts_atomic32_read_bset_nob(&proc->state, + (ERTS_PSFLG_SIG_Q + | ERTS_PSFLG_SIG_IN_Q), + ERTS_PSFLG_SIG_Q); + + ASSERT((s & (ERTS_PSFLG_SIG_Q|ERTS_PSFLG_SIG_IN_Q)) + == ERTS_PSFLG_SIG_IN_Q); (void)s; + } + else { + ErtsSignal *sig; + ASSERT(proc->sig_qs.nmsigs.next); + sig = ((ErtsSignal *) *proc->sig_qs.nmsigs.last); + ASSERT(ERTS_SIG_IS_NON_MSG(sig)); + ASSERT(!sig->common.specific.next); + if (proc->sig_inq.nmsigs.next == &proc->sig_inq.first) + sig->common.specific.next = proc->sig_qs.cont_last; + else + sig->common.specific.next = proc->sig_inq.nmsigs.next; + + s = erts_atomic32_read_band_nob(&proc->state, + ~ERTS_PSFLG_SIG_IN_Q); + + ASSERT((s & (ERTS_PSFLG_SIG_Q|ERTS_PSFLG_SIG_IN_Q)) + == (ERTS_PSFLG_SIG_Q|ERTS_PSFLG_SIG_IN_Q)); (void)s; + } + if (proc->sig_inq.nmsigs.last == &proc->sig_inq.first) + proc->sig_qs.nmsigs.last = proc->sig_qs.cont_last; + else + proc->sig_qs.nmsigs.last = proc->sig_inq.nmsigs.last; + proc->sig_inq.nmsigs.next = NULL; + proc->sig_inq.nmsigs.last = NULL; + + *proc->sig_qs.cont_last = proc->sig_inq.first; + proc->sig_qs.cont_last = proc->sig_inq.last; + } + + proc->sig_qs.len += proc->sig_inq.len; + + proc->sig_inq.first = NULL; + proc->sig_inq.last = &proc->sig_inq.first; + proc->sig_inq.len = 0; + +} + +Sint +erts_proc_sig_fetch_msgq_len_offs__(Process *proc) +{ + ErtsProcSigMsgQLenOffsetMarker *marker + = (ErtsProcSigMsgQLenOffsetMarker *) proc->sig_inq.first; + + ASSERT(marker->common.tag == ERTS_PROC_SIG_MSGQ_LEN_OFFS_MARK); + + if (marker->common.next) { + Sint len; + + proc->flags |= F_DELAYED_PSIGQS_LEN; + + /* + * Prevent update of sig_qs.len in fetch. These + * updates are done via process-info signal(s) + * instead... + */ + len = proc->sig_inq.len; + marker->delayed_len += len; + marker->len_offset -= len; + proc->sig_inq.len = 0; + + /* + * Temorarily remove marker during fetch... + */ + + proc->sig_inq.first = marker->common.next; + if (proc->sig_inq.last == &marker->common.next) + proc->sig_inq.last = &proc->sig_inq.first; + if (proc->sig_inq.nmsigs.next == &marker->common.next) + proc->sig_inq.nmsigs.next = &proc->sig_inq.first; + if (proc->sig_inq.nmsigs.last == &marker->common.next) + proc->sig_inq.nmsigs.last = &proc->sig_inq.first; + + erts_proc_sig_fetch__(proc); + + marker->common.next = NULL; + proc->sig_inq.first = (ErtsMessage *) marker; + proc->sig_inq.last = &marker->common.next; + + } + + return marker->delayed_len; +} + +static ERTS_INLINE Sint +proc_sig_privqs_len(Process *c_p, int have_qlock) +{ + Sint res = c_p->sig_qs.len; + + ERTS_LC_ASSERT(!have_qlock + ? (ERTS_PROC_LOCK_MAIN + == erts_proc_lc_my_proc_locks(c_p)) + : ((ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_MAIN) + == ((ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_MAIN) + & erts_proc_lc_my_proc_locks(c_p)))); + + if (c_p->flags & F_DELAYED_PSIGQS_LEN) { + ErtsProcSigMsgQLenOffsetMarker *marker; + + if (!have_qlock) + erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ); + + marker = (ErtsProcSigMsgQLenOffsetMarker *) c_p->sig_inq.first; + ASSERT(marker); + ASSERT(marker->common.tag == ERTS_PROC_SIG_MSGQ_LEN_OFFS_MARK); + + res += marker->delayed_len; + + if (!have_qlock) + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); + } + +#ifdef ERTS_PROC_SIG_HARD_DEBUG_SIGQ_MSG_LEN + { + Sint len = 0; + ERTS_FOREACH_SIG_PRIVQS( + c_p, mp, + { + if (ERTS_SIG_IS_MSG(mp)) + len++; + }); + ERTS_ASSERT(res == len); + } +#endif + + return res; +} + +Sint +erts_proc_sig_privqs_len(Process *c_p) +{ + return proc_sig_privqs_len(c_p, 0); +} + +static void do_seq_trace_output(Eterm to, Eterm token, Eterm msg); + +static void +send_gen_exit_signal(Process *c_p, Eterm from_tag, + Eterm from, Eterm to, + Sint16 op, Eterm reason, Eterm ref, + Eterm token, int normal_kills) +{ + ErtsExitSignalData *xsigd; + Eterm *hp, *start_hp, s_reason, s_ref, s_message, s_token, s_from; + ErtsMessage *mp; + ErlHeapFragment *hfrag; + ErlOffHeap *ohp; + Uint hsz, from_sz, reason_sz, ref_sz, token_sz; + int seq_trace; +#ifdef USE_VM_PROBES + Eterm s_utag, utag; + Uint utag_sz; +#endif + + ASSERT(is_immed(from_tag)); + + hsz = sizeof(ErtsExitSignalData)/sizeof(Uint); + + seq_trace = c_p && have_seqtrace(token); + if (seq_trace) + seq_trace_update_send(c_p); + +#ifdef USE_VM_PROBES + utag_sz = 0; + utag = NIL; + if (c_p && token != NIL && (DT_UTAG_FLAGS(c_p) & DT_UTAG_SPREADING)) { + utag_sz = size_object(DT_UTAG(c_p)); + utag = DT_UTAG(c_p); + } + else if (token == am_have_dt_utag) { + token = NIL; + } + hsz += utag_sz; +#endif + + token_sz = is_immed(token) ? 0 : size_object(token); + hsz += token_sz; + + from_sz = is_immed(from) ? 0 : size_object(from); + hsz += from_sz; + + reason_sz = is_immed(reason) ? 0 : size_object(reason); + hsz += reason_sz; + + switch (op) { + case ERTS_SIG_Q_OP_EXIT: + case ERTS_SIG_Q_OP_EXIT_LINKED: { + /* {'EXIT', From, Reason} */ + hsz += 4; /* 3-tuple */ + ref_sz = 0; + break; + } + case ERTS_SIG_Q_OP_MONITOR_DOWN: { + /* {'DOWN', Ref, process, From, Reason} */ + hsz += 6; /* 5-tuple */ + ref_sz = NC_HEAP_SIZE(ref); + hsz += ref_sz; + break; + } + default: + ERTS_INTERNAL_ERROR("Invalid exit signal op"); + break; + } + + /* + * Allocate message combined with heap fragment... + */ + mp = erts_alloc_message(hsz, &hp); + hfrag = &mp->hfrag; + mp->next = NULL; + ohp = &hfrag->off_heap; + start_hp = hp; + + s_token = (is_immed(token) + ? token + : copy_struct(token, token_sz, &hp, ohp)); + + s_reason = (is_immed(reason) + ? reason + : copy_struct(reason, reason_sz, &hp, ohp)); + + s_from = (is_immed(from) + ? from + : copy_struct(from, from_sz, &hp, ohp)); + + if (!ref_sz) + s_ref = NIL; + else + s_ref = STORE_NC(&hp, ohp, ref); + + switch (op) { + case ERTS_SIG_Q_OP_EXIT: + case ERTS_SIG_Q_OP_EXIT_LINKED: + /* {'EXIT', From, Reason} */ + s_message = TUPLE3(hp, am_EXIT, s_from, s_reason); + hp += 4; + break; + case ERTS_SIG_Q_OP_MONITOR_DOWN: + /* {'DOWN', Ref, process, From, Reason} */ + s_message = TUPLE5(hp, am_DOWN, s_ref, am_process, s_from, s_reason); + hp += 6; + break; + } + +#ifdef USE_VM_PROBES + s_utag = (is_immed(utag) + ? utag + : copy_struct(utag, utag_sz, &hp, ohp)); + ERL_MESSAGE_DT_UTAG(mp) = s_utag; +#endif + + ERL_MESSAGE_TERM(mp) = ERTS_PROC_SIG_MAKE_TAG(op, + ERTS_SIG_Q_TYPE_GEN_EXIT, + 0); + ERL_MESSAGE_TOKEN(mp) = s_token; + ERL_MESSAGE_FROM(mp) = from_tag; /* immediate... */ + + hfrag->used_size = hp - start_hp; + + xsigd = (ErtsExitSignalData *) (char *) hp; + + xsigd->message = s_message; + xsigd->from = s_from; + xsigd->reason = s_reason; + if (is_nil(s_ref)) + xsigd->u.normal_kills = normal_kills; + else { + ASSERT(is_ref(s_ref)); + xsigd->u.ref = s_ref; + } + + if (seq_trace) + do_seq_trace_output(to, s_token, s_message); + + if (!proc_queue_signal(c_p, to, (ErtsSignal *) mp, op)) { + mp->next = NULL; + erts_cleanup_messages(mp); + } +} + +static void +do_seq_trace_output(Eterm to, Eterm token, Eterm msg) +{ + /* + * We could do this when enqueuing the signal and avoid some + * locking. However, the enqueuing code would then always + * have the penalty of this seq-tracing code which we do not + * want... + */ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + int is_normal_sched = !!esdp && esdp->type == ERTS_SCHED_NORMAL; + Process *rp; + + if (is_normal_sched) + rp = erts_proc_lookup_raw(to); + else + rp = erts_proc_lookup_raw_inc_refc(to); + + if (rp) { + erts_proc_lock(rp, ERTS_PROC_LOCK_MSGQ); + + if (!ERTS_PROC_IS_EXITING(rp)) + seq_trace_output(token, msg, SEQ_TRACE_SEND, to, rp); + + erts_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ); + + if (!is_normal_sched) + erts_proc_dec_refc(rp); + } +} + +void +erts_proc_sig_send_persistent_monitor_msg(Uint16 type, Eterm key, + Eterm from, Eterm to, + Eterm msg, Uint msg_sz) +{ + ErtsPersistMonMsg *prst_mon; + ErtsMessage *mp; + ErlHeapFragment *hfrag; + Eterm *hp, *start_hp, message; + ErlOffHeap *ohp; + Uint hsz = sizeof(ErtsPersistMonMsg) + msg_sz; + + /* + * Allocate message combined with heap fragment... + */ + mp = erts_alloc_message(hsz, &hp); + hfrag = &mp->hfrag; + mp->next = NULL; + ohp = &hfrag->off_heap; + start_hp = hp; + + ASSERT(msg_sz == size_object(msg)); + message = copy_struct(msg, msg_sz, &hp, ohp); + hfrag->used_size = hp - start_hp; + + prst_mon = (ErtsPersistMonMsg *) (char *) hp; + prst_mon->message = message; + + switch (type) { + case ERTS_MON_TYPE_NODES: + ASSERT(is_small(key)); + prst_mon->key = key; + break; + + case ERTS_MON_TYPE_TIME_OFFSET: + ASSERT(is_internal_ref(key)); + ASSERT(is_tuple_arity(message, 5)); + + prst_mon->key = tuple_val(message)[2]; + + ASSERT(eq(prst_mon->key, key)); + break; + + default: + ERTS_INTERNAL_ERROR("Invalid persistent monitor type"); + prst_mon->key = key; + break; + } + + ASSERT(is_immed(from)); + + ERL_MESSAGE_TERM(mp) = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_PERSISTENT_MON_MSG, + type, 0); + ERL_MESSAGE_FROM(mp) = from; + ERL_MESSAGE_TOKEN(mp) = NIL; +#ifdef USE_VM_PROBES + ERL_MESSAGE_DT_UTAG(mp) = NIL; +#endif + + if (!proc_queue_signal(NULL, to, (ErtsSignal *) mp, + ERTS_SIG_Q_OP_PERSISTENT_MON_MSG)) { + mp->next = NULL; + erts_cleanup_messages(mp); + } +} + +static ERTS_INLINE Eterm +get_persist_mon_msg(ErtsMessage *sig, Eterm *msg) +{ + ErtsPersistMonMsg *prst_mon; + prst_mon = ((ErtsPersistMonMsg *) + (char *) (&sig->hfrag.mem[0] + + sig->hfrag.used_size)); + *msg = prst_mon->message; + return prst_mon->key; +} + +void +erts_proc_sig_send_exit(Process *c_p, Eterm from, Eterm to, + Eterm reason, Eterm token, + int normal_kills) +{ + Eterm from_tag; + ASSERT(!c_p || c_p->common.id == from); + if (is_immed(from)) { + ASSERT(is_internal_pid(from) || is_internal_port(from)); + from_tag = from; + } + else { + DistEntry *dep; + ASSERT(is_external_pid(from)); + dep = external_pid_dist_entry(from); + from_tag = dep->sysname; + } + send_gen_exit_signal(c_p, from_tag, from, to, ERTS_SIG_Q_OP_EXIT, + reason, NIL, token, normal_kills); +} + +void +erts_proc_sig_send_link_exit(Process *c_p, Eterm from, ErtsLink *lnk, + Eterm reason, Eterm token) +{ + Eterm to; + ASSERT(!c_p || c_p->common.id == from); + ASSERT(lnk); + to = lnk->other.item; + if (is_not_immed(reason) || is_not_nil(token)) { + ASSERT(is_internal_pid(from) || is_internal_port(from)); + send_gen_exit_signal(c_p, from, from, to, ERTS_SIG_Q_OP_EXIT_LINKED, + reason, NIL, token, 0); + } + else { + /* Pass signal using old link structure... */ + ErtsSignal *sig = (ErtsSignal *) lnk; + lnk->other.item = reason; /* pass reason via this other.item */ + sig->common.tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_EXIT_LINKED, + lnk->type, 0); + if (proc_queue_signal(c_p, to, sig, ERTS_SIG_Q_OP_EXIT_LINKED)) + return; /* receiver will destroy lnk structure */ + } + if (lnk) + erts_link_release(lnk); +} + +int +erts_proc_sig_send_link(Process *c_p, Eterm to, ErtsLink *lnk) +{ + ErtsSignal *sig; + Uint16 type = lnk->type; + + ASSERT(!c_p || c_p->common.id == lnk->other.item); + ASSERT(lnk); + ASSERT(is_internal_pid(to)); + + sig = (ErtsSignal *) lnk; + sig->common.tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_LINK, + type, 0); + + return proc_queue_signal(c_p, to, sig, ERTS_SIG_Q_OP_LINK); +} + +void +erts_proc_sig_send_unlink(Process *c_p, ErtsLink *lnk) +{ + ErtsSignal *sig; + Eterm to; + + ASSERT(lnk); + + sig = (ErtsSignal *) lnk; + to = lnk->other.item; + + ASSERT(is_internal_pid(to)); + + sig->common.tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_UNLINK, + lnk->type, 0); + + if (!proc_queue_signal(c_p, to, sig, ERTS_SIG_Q_OP_UNLINK)) + erts_link_release(lnk); +} + +void +erts_proc_sig_send_dist_link_exit(DistEntry *dep, + Eterm from, Eterm to, + Eterm reason, Eterm token) +{ + send_gen_exit_signal(NULL, dep->sysname, from, to, ERTS_SIG_Q_OP_EXIT_LINKED, + reason, NIL, token, 0); +} + +void +erts_proc_sig_send_dist_unlink(DistEntry *dep, Eterm from, Eterm to) +{ + ErtsSignal *sig; + + ASSERT(is_internal_pid(to)); + ASSERT(is_external_pid(from)); + ASSERT(dep == external_pid_dist_entry(from)); + + sig = (ErtsSignal *) make_sig_dist_link_op(ERTS_SIG_Q_OP_UNLINK, + to, from); + + if (!proc_queue_signal(NULL, to, sig, ERTS_SIG_Q_OP_UNLINK)) + destroy_sig_dist_link_op((ErtsSigDistLinkOp *) sig); +} + +void +erts_proc_sig_send_dist_monitor_down(DistEntry *dep, Eterm ref, + Eterm from, Eterm to, + Eterm reason) +{ + Eterm monitored, heap[3]; + if (is_atom(from)) + monitored = TUPLE2(&heap[0], from, dep->sysname); + else + monitored = from; + send_gen_exit_signal(NULL, dep->sysname, monitored, + to, ERTS_SIG_Q_OP_MONITOR_DOWN, + reason, ref, NIL, 0); +} + +void +erts_proc_sig_send_monitor_down(ErtsMonitor *mon, Eterm reason) +{ + Eterm to; + + ASSERT(erts_monitor_is_target(mon)); + ASSERT(!erts_monitor_is_in_table(mon)); + + to = mon->other.item; + ASSERT(is_internal_pid(to)); + + if (is_immed(reason)) { + /* Pass signal using old monitor structure... */ + ErtsSignal *sig; + + mon->other.item = reason; /* Pass immed reason via other.item... */ + sig = (ErtsSignal *) mon; + sig->common.tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_MONITOR_DOWN, + mon->type, 0); + if (proc_queue_signal(NULL, to, sig, ERTS_SIG_Q_OP_MONITOR_DOWN)) + return; /* receiver will destroy mon structure */ + } + else { + ErtsMonitorData *mdp = erts_monitor_to_data(mon); + Eterm from_tag, monitored, heap[3]; + + if (!(mon->flags & ERTS_ML_FLG_NAME)) { + from_tag = monitored = mdp->origin.other.item; + if (is_external_pid(from_tag)) { + DistEntry *dep = external_pid_dist_entry(from_tag); + from_tag = dep->sysname; + } + } + else { + ErtsMonitorDataExtended *mdep; + Eterm name, node; + mdep = (ErtsMonitorDataExtended *) mdp; + name = mdep->u.name; + ASSERT(is_atom(name)); + if (mdep->dist) { + node = mdep->dist->nodename; + from_tag = node; + } + else { + node = erts_this_dist_entry->sysname; + from_tag = mdp->origin.other.item; + } + ASSERT(is_internal_port(from_tag) + || is_internal_pid(from_tag) + || is_atom(from_tag)); + monitored = TUPLE2(&heap[0], name, node); + } + send_gen_exit_signal(NULL, from_tag, monitored, + to, ERTS_SIG_Q_OP_MONITOR_DOWN, + reason, mdp->ref, NIL, 0); + } + erts_monitor_release(mon); +} + +void +erts_proc_sig_send_dist_demonitor(Eterm to, Eterm ref) +{ + ErtsSigDistProcDemonitor *dmon; + ErtsSignal *sig; + Eterm *hp; + ErlOffHeap oh; + size_t size; + + ERTS_INIT_OFF_HEAP(&oh); + + ASSERT(is_internal_pid(to)); + + size = sizeof(ErtsSigDistProcDemonitor) - sizeof(Eterm); + ASSERT(is_ref(ref)); + size += NC_HEAP_SIZE(ref)*sizeof(Eterm); + + dmon = erts_alloc(ERTS_ALC_T_DIST_DEMONITOR, size); + + hp = &dmon->heap[0]; + dmon->ref = STORE_NC(&hp, &oh, ref); + sig = (ErtsSignal *) dmon; + + sig->common.tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_DEMONITOR, + ERTS_SIG_Q_TYPE_DIST_PROC_DEMONITOR, + 0); + + if (!proc_queue_signal(NULL, to, sig, ERTS_SIG_Q_OP_DEMONITOR)) + destroy_dist_proc_demonitor(dmon); +} + +void +erts_proc_sig_send_demonitor(ErtsMonitor *mon) +{ + ErtsSignal *sig = (ErtsSignal *) mon; + Uint16 type = mon->type; + Eterm to = mon->other.item; + + ASSERT(is_internal_pid(to)); + ASSERT(erts_monitor_is_origin(mon)); + ASSERT(!erts_monitor_is_in_table(mon)); + + sig->common.tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_DEMONITOR, + type, 0); + + if (!proc_queue_signal(NULL, to, sig, ERTS_SIG_Q_OP_DEMONITOR)) + erts_monitor_release(mon); +} + +int +erts_proc_sig_send_monitor(ErtsMonitor *mon, Eterm to) +{ + ErtsSignal *sig = (ErtsSignal *) mon; + Uint16 type = mon->type; + + ASSERT(is_internal_pid(to) || to == am_undefined); + ASSERT(erts_monitor_is_target(mon)); + + sig->common.tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_MONITOR, + type, 0); + + return proc_queue_signal(NULL, to, sig, ERTS_SIG_Q_OP_MONITOR); +} + +void +erts_proc_sig_send_trace_change(Eterm to, Uint on, Uint off, Eterm tracer) +{ + ErtsSigTraceInfo *ti; + Eterm tag; + + ti = erts_alloc(ERTS_ALC_T_SIG_DATA, sizeof(ErtsSigTraceInfo)); + tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_TRACE_CHANGE_STATE, + ERTS_SIG_Q_TYPE_ADJUST_TRACE_INFO, + 0); + + ti->common.tag = tag; + ti->flags_off = off; + ti->flags_on = on; + ti->tracer = NIL; + if (is_not_nil(tracer)) + erts_tracer_update(&ti->tracer, tracer); + + if (!proc_queue_signal(NULL, to, (ErtsSignal *) ti, + ERTS_SIG_Q_OP_TRACE_CHANGE_STATE)) + destroy_trace_info(ti); +} + +void +erts_proc_sig_send_group_leader(Process *c_p, Eterm to, Eterm gl, Eterm ref) +{ + int res; + ErtsSigGroupLeader *sgl; + Eterm *hp; + Uint gl_sz, ref_sz, size; + erts_aint_t init_flags = ERTS_SIG_GL_FLG_ACTIVE|ERTS_SIG_GL_FLG_RECEIVER; + if (c_p) + init_flags |= ERTS_SIG_GL_FLG_SENDER; + + ASSERT(c_p ? is_internal_ref(ref) : ref == NIL); + + gl_sz = is_immed(gl) ? 0 : size_object(gl); + ref_sz = is_immed(ref) ? 0 : size_object(ref); + + size = sizeof(ErtsSigGroupLeader); + + size += (gl_sz + ref_sz - 1) * sizeof(Eterm); + + sgl = erts_alloc(ERTS_ALC_T_SIG_DATA, size); + + erts_atomic_init_nob(&sgl->flags, init_flags); + + ERTS_INIT_OFF_HEAP(&sgl->oh); + + hp = &sgl->heap[0]; + + sgl->group_leader = is_immed(gl) ? gl : copy_struct(gl, gl_sz, &hp, &sgl->oh); + sgl->reply_to = c_p ? c_p->common.id : NIL; + sgl->ref = is_immed(ref) ? ref : copy_struct(ref, ref_sz, &hp, &sgl->oh); + + sgl->common.tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_GROUP_LEADER, + ERTS_SIG_Q_TYPE_UNDEFINED, + 0); + + res = proc_queue_signal(c_p, to, (ErtsSignal *) sgl, + ERTS_SIG_Q_OP_GROUP_LEADER); + + if (!res) + destroy_sig_group_leader(sgl); + else if (c_p) { + erts_aint_t flags, rm_flags = ERTS_SIG_GL_FLG_SENDER; + int prio_res = maybe_elevate_sig_handling_prio(c_p, to); + if (!prio_res) + rm_flags |= ERTS_SIG_GL_FLG_ACTIVE; + flags = erts_atomic_read_band_nob(&sgl->flags, ~rm_flags); + if (!prio_res && (flags & ERTS_SIG_GL_FLG_ACTIVE)) + res = 0; /* We deactivated signal... */ + if ((flags & ~rm_flags) == 0) + destroy_sig_group_leader(sgl); + } + + if (!res && c_p) + group_leader_reply(c_p, c_p->common.id, ref, 0); +} + +void +erts_proc_sig_send_is_alive_request(Process *c_p, Eterm to, Eterm ref) +{ + ErlHeapFragment *hfrag; + Uint hsz; + Eterm *hp, *start_hp, ref_cpy, msg; + ErlOffHeap *ohp; + ErtsMessage *mp; + ErtsIsAliveRequest *alive_req; + + ASSERT(is_internal_ordinary_ref(ref)); + + hsz = ERTS_REF_THING_SIZE + 3 + sizeof(ErtsIsAliveRequest)/sizeof(Eterm); + + mp = erts_alloc_message(hsz, &hp); + hfrag = &mp->hfrag; + mp->next = NULL; + ohp = &hfrag->off_heap; + start_hp = hp; + + ref_cpy = STORE_NC(&hp, ohp, ref); + msg = TUPLE2(hp, ref_cpy, am_false); /* default res 'false' */ + hp += 3; + + hfrag->used_size = hp - start_hp; + + alive_req = (ErtsIsAliveRequest *) (char *) hp; + alive_req->message = msg; + alive_req->requester = c_p->common.id; + + ERL_MESSAGE_TERM(mp) = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_IS_ALIVE, + ERTS_SIG_Q_TYPE_UNDEFINED, + 0); + ERL_MESSAGE_TOKEN(mp) = NIL; + ERL_MESSAGE_FROM(mp) = am_system; +#ifdef USE_VM_PROBES + ERL_MESSAGE_DT_UTAG(mp) = NIL; +#endif + + if (proc_queue_signal(c_p, to, (ErtsSignal *) mp, ERTS_SIG_Q_OP_IS_ALIVE)) + (void) maybe_elevate_sig_handling_prio(c_p, to); + else { + /* It wasn't alive; reply to ourselves... */ + mp->next = NULL; + mp->data.attached = ERTS_MSG_COMBINED_HFRAG; + erts_queue_message(c_p, ERTS_PROC_LOCK_MAIN, mp, msg, am_system); + } +} + +int +erts_proc_sig_send_process_info_request(Process *c_p, + Eterm to, + int *item_ix, + int len, + int need_msgq_len, + int flags, + Uint reserve_size, + Eterm ref) +{ + Uint size = sizeof(ErtsProcessInfoSig) + (len - 1) * sizeof(int); + ErtsProcessInfoSig *pis = erts_alloc(ERTS_ALC_T_SIG_DATA, size); + int res; + + ASSERT(c_p); + ASSERT(item_ix); + ASSERT(len > 0); + ASSERT(is_internal_ordinary_ref(ref)); + + pis->common.tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_PROCESS_INFO, + 0, 0); + + if (!need_msgq_len) + pis->msgq_len_offset = ERTS_PROC_SIG_PI_MSGQ_LEN_IGNORE; + else { + pis->msgq_len_offset = ERTS_PROC_SIG_PI_MSGQ_LEN_SYNC; + pis->marker.common.next = NULL; + pis->marker.common.specific.next = NULL; + pis->marker.common.tag = ERTS_PROC_SIG_MSGQ_LEN_OFFS_MARK; + pis->marker.refc = 0; + pis->marker.delayed_len = 0; + pis->marker.len_offset = 0; + } + pis->requester = c_p->common.id; + sys_memcpy((void *) &pis->oref_thing, + (void *) internal_ref_val(ref), + sizeof(ErtsORefThing)); + pis->ref = make_internal_ref((char *) &pis->oref_thing); + pis->reserve_size = reserve_size; + pis->len = len; + pis->flags = flags; + sys_memcpy((void *) &pis->item_ix[0], + (void *) item_ix, + sizeof(int)*len); + res = proc_queue_signal(c_p, to, (ErtsSignal *) pis, + ERTS_SIG_Q_OP_PROCESS_INFO); + if (res) + (void) maybe_elevate_sig_handling_prio(c_p, to); + else + erts_free(ERTS_ALC_T_SIG_DATA, pis); + return res; +} + +static void +is_alive_response(Process *c_p, ErtsMessage *mp, int is_alive) +{ + /* + * Sender prepared the message for us. Just patch + * the result if necessary. The default prepared + * result is 'false'. + */ + Process *rp; + ErtsIsAliveRequest *alive_req; + + alive_req = (ErtsIsAliveRequest *) (char *) (&mp->hfrag.mem[0] + + mp->hfrag.used_size); + + + ASSERT(ERTS_SIG_IS_NON_MSG(mp)); + ASSERT(ERTS_PROC_SIG_OP(((ErtsSignal *) mp)->common.tag) + == ERTS_SIG_Q_OP_IS_ALIVE); + ASSERT(mp->hfrag.alloc_size > mp->hfrag.used_size); + ASSERT((mp->hfrag.alloc_size - mp->hfrag.used_size)*sizeof(UWord) + >= sizeof(ErtsIsAliveRequest)); + ASSERT(is_internal_pid(alive_req->requester)); + ASSERT(alive_req->requester != c_p->common.id); + ASSERT(is_tuple_arity(alive_req->message, 2)); + ASSERT(is_internal_ordinary_ref(tuple_val(alive_req->message)[1])); + ASSERT(tuple_val(alive_req->message)[2] == am_false); + + ERL_MESSAGE_TERM(mp) = alive_req->message; + mp->data.attached = ERTS_MSG_COMBINED_HFRAG; + mp->next = NULL; + + rp = erts_proc_lookup(alive_req->requester); + if (!rp) + erts_cleanup_messages(mp); + else { + if (is_alive) { /* patch result... */ + Eterm *tp = tuple_val(alive_req->message); + tp[2] = am_true; + } + erts_queue_message(rp, 0, mp, alive_req->message, am_system); + } +} + +static ERTS_INLINE void +adjust_tracing_state(Process *c_p, ErtsSigRecvTracing *tracing, int setup) +{ + if (!IS_TRACED(c_p) || (ERTS_TRACE_FLAGS(c_p) & F_SENSITIVE)) { + tracing->messages.active = 0; + tracing->messages.receive_trace = 0; + tracing->messages.event = NULL; + tracing->messages.next = NULL; + tracing->procs = 0; + tracing->active = 0; + } + else { + Uint flgs = ERTS_TRACE_FLAGS(c_p); + int procs_trace = !!(flgs & F_TRACE_PROCS); + int recv_trace = !!(flgs & F_TRACE_RECEIVE); + /* procs tracing enabled? */ + + tracing->procs = procs_trace; + + /* message receive tracing enabled? */ + tracing->messages.receive_trace = recv_trace; + if (!recv_trace) + tracing->messages.event = NULL; + else { + if (tracing->messages.bp_ix < 0) + tracing->messages.bp_ix = erts_active_bp_ix(); + tracing->messages.event = &erts_receive_tracing[tracing->messages.bp_ix]; + } + if (setup) { + if (recv_trace) + tracing->messages.next = &c_p->sig_qs.cont; + else + tracing->messages.next = NULL; + } + tracing->messages.active = recv_trace; + tracing->active = recv_trace | procs_trace; + } + +#if defined(USE_VM_PROBES) + /* vm probe message_queued enabled? */ + + tracing->messages.vm_probes = DTRACE_ENABLED(message_queued); + if (tracing->messages.vm_probes) { + dtrace_proc_str(c_p, tracing->messages.receiver_name); + tracing->messages.active = !0; + tracing->active = !0; + if (setup && !tracing->messages.next) + tracing->messages.next = &c_p->sig_qs.cont; + } + +#endif +} + +static ERTS_INLINE void +setup_tracing_state(Process *c_p, ErtsSigRecvTracing *tracing) +{ + tracing->messages.bp_ix = -1; + adjust_tracing_state(c_p, tracing, !0); +} + +static ERTS_INLINE void +remove_iq_sig(Process *c_p, ErtsMessage *sig, ErtsMessage **next_sig) +{ + /* + * Remove signal from inner queue. + */ + ASSERT(c_p->sig_qs.cont_last != &sig->next); + ASSERT(c_p->sig_qs.nmsigs.next != &sig->next); + ASSERT(c_p->sig_qs.nmsigs.last != &sig->next); + + if (c_p->sig_qs.save == &sig->next) + c_p->sig_qs.save = next_sig; + if (c_p->sig_qs.last == &sig->next) + c_p->sig_qs.last = next_sig; + if (c_p->sig_qs.saved_last == &sig->next) + c_p->sig_qs.saved_last = next_sig; + + *next_sig = sig->next; +} + +static ERTS_INLINE void +remove_mq_sig(Process *c_p, ErtsMessage *sig, + ErtsMessage **next_sig, ErtsMessage ***next_nm_sig) +{ + /* + * Remove signal from middle queue. + */ + ASSERT(c_p->sig_qs.save != &sig->next); + ASSERT(c_p->sig_qs.last != &sig->next); + + if (c_p->sig_qs.cont_last == &sig->next) + c_p->sig_qs.cont_last = next_sig; + if (c_p->sig_qs.saved_last == &sig->next) + c_p->sig_qs.saved_last = next_sig; + if (*next_nm_sig == &sig->next) + *next_nm_sig = next_sig; + if (c_p->sig_qs.nmsigs.last == &sig->next) + c_p->sig_qs.nmsigs.last = next_sig; + + *next_sig = sig->next; +} + +static ERTS_INLINE void +remove_nm_sig(Process *c_p, ErtsMessage *sig, ErtsMessage ***next_nm_sig) +{ + ErtsMessage **next_sig = *next_nm_sig; + ASSERT(ERTS_SIG_IS_NON_MSG(sig)); + ASSERT(*next_sig == sig); + *next_nm_sig = ((ErtsSignal *) sig)->common.specific.next; + remove_mq_sig(c_p, sig, next_sig, next_nm_sig); +} + +static ERTS_INLINE void +convert_to_msg(Process *c_p, ErtsMessage *sig, ErtsMessage *msg, + ErtsMessage ***next_nm_sig) +{ + ErtsMessage **next_sig = *next_nm_sig; + ASSERT(ERTS_SIG_IS_NON_MSG(sig)); + *next_nm_sig = ((ErtsSignal *) sig)->common.specific.next; + c_p->sig_qs.len++; + *next_sig = msg; + remove_mq_sig(c_p, sig, &msg->next, next_nm_sig); +} + +static ERTS_INLINE void +convert_to_msgs(Process *c_p, ErtsMessage *sig, Uint no_msgs, + ErtsMessage *first_msg, ErtsMessage *last_msg, + ErtsMessage ***next_nm_sig) +{ + ErtsMessage **next_sig = *next_nm_sig; + ASSERT(ERTS_SIG_IS_NON_MSG(sig)); + *next_nm_sig = ((ErtsSignal *) sig)->common.specific.next; + c_p->sig_qs.len += no_msgs; + *next_sig = first_msg; + remove_mq_sig(c_p, sig, &last_msg->next, next_nm_sig); +} + +static ERTS_INLINE void +insert_messages(Process *c_p, ErtsMessage **next, ErtsMessage *first, + ErtsMessage *last, Uint no_msgs, ErtsMessage ***next_nm_sig) +{ + last->next = *next; + if (c_p->sig_qs.cont_last == next) + c_p->sig_qs.cont_last = &last->next; + if (*next_nm_sig == next) + *next_nm_sig = &last->next; + if (c_p->sig_qs.nmsigs.last == next) + c_p->sig_qs.nmsigs.last = &last->next; + c_p->sig_qs.len += no_msgs; + *next = first; +} + +static ERTS_INLINE void +remove_mq_m_sig(Process *c_p, ErtsMessage *sig, ErtsMessage **next_sig, ErtsMessage ***next_nm_sig) +{ + /* Removing message... */ + ASSERT(!ERTS_SIG_IS_NON_MSG(sig)); + c_p->sig_qs.len--; + remove_mq_sig(c_p, sig, next_sig, next_nm_sig); +} + +static ERTS_INLINE void +remove_iq_m_sig(Process *c_p, ErtsMessage *sig, ErtsMessage **next_sig) +{ + /* Removing message... */ + ASSERT(!ERTS_SIG_IS_NON_MSG(sig)); + c_p->sig_qs.len--; + remove_iq_sig(c_p, sig, next_sig); +} + +static ERTS_INLINE void +convert_prepared_sig_to_msg(Process *c_p, ErtsMessage *sig, Eterm msg, + ErtsMessage ***next_nm_sig) +{ + /* + * Everything is already there except for the reference to + * the message and the combined hfrag marker that needs to be + * restored... + */ + *next_nm_sig = ((ErtsSignal *) sig)->common.specific.next; + sig->data.attached = ERTS_MSG_COMBINED_HFRAG; + ERL_MESSAGE_TERM(sig) = msg; + c_p->sig_qs.len++; +} + +static ERTS_INLINE int +handle_exit_signal(Process *c_p, ErtsSigRecvTracing *tracing, + ErtsMessage *sig, ErtsMessage ***next_nm_sig, + int *exited) +{ + ErtsMessage *conv_msg = NULL; + ErtsExitSignalData *xsigd = NULL; + ErtsLinkData *ldp = NULL; /* Avoid erroneous warning... */ + ErtsLink *dlnk = NULL; /* Avoid erroneous warning... */ + Eterm tag = ((ErtsSignal *) sig)->common.tag; + Uint16 type = ERTS_PROC_SIG_TYPE(tag); + int op = ERTS_PROC_SIG_OP(tag); + int destroy = 0; + int ignore = 0; + int save = 0; + int exit = 0; + int cnt = 1; + Eterm reason; + Eterm from; + + if (type == ERTS_SIG_Q_TYPE_GEN_EXIT) { + xsigd = get_exit_signal_data(sig); + from = xsigd->from; + reason = xsigd->reason; + if (op != ERTS_SIG_Q_OP_EXIT_LINKED) + ignore = 0; + else { + ErtsLink *llnk = erts_link_tree_lookup(ERTS_P_LINKS(c_p), from); + if (!llnk) { + /* Link no longer active; ignore... */ + ignore = !0; + destroy = !0; + } + else { + ignore = 0; + erts_link_tree_delete(&ERTS_P_LINKS(c_p), llnk); + if (llnk->type != ERTS_LNK_TYPE_DIST_PROC) + erts_link_release(llnk); + else { + dlnk = erts_link_to_other(llnk, &ldp); + if (erts_link_dist_delete(dlnk)) + erts_link_release_both(ldp); + else + erts_link_release(llnk); + } + } + } + + if (!ignore) { + + if ((op != ERTS_SIG_Q_OP_EXIT || reason != am_kill) + && (c_p->flags & F_TRAP_EXIT)) { + convert_prepared_sig_to_msg(c_p, sig, + xsigd->message, next_nm_sig); + conv_msg = sig; + } + else if (reason == am_normal && !xsigd->u.normal_kills) { + /* Ignore it... */ + destroy = !0; + ignore = !0; + } + else { + /* Terminate... */ + save = !0; + exit = !0; + if (op == ERTS_SIG_Q_OP_EXIT && reason == am_kill) + reason = am_killed; + } + } + } + else { /* Link exit */ + ErtsLink *slnk = (ErtsLink *) sig; + ErtsLink *llnk = erts_link_to_other(slnk, &ldp); + + ASSERT(type == ERTS_LNK_TYPE_PROC + || type == ERTS_LNK_TYPE_PORT + || type == ERTS_LNK_TYPE_DIST_PROC); + + from = llnk->other.item; + reason = slnk->other.item; /* reason in other.item ... */ + ASSERT(is_pid(from) || is_internal_port(from)); + ASSERT(is_immed(reason)); + ASSERT(op == ERTS_SIG_Q_OP_EXIT_LINKED); + dlnk = erts_link_tree_key_delete(&ERTS_P_LINKS(c_p), llnk); + if (!dlnk) { + ignore = !0; /* Link no longer active; ignore... */ + ldp = NULL; + } + else { + Eterm pid; + ErtsMessage *mp; + ErtsProcLocks locks; + Uint hsz; + Eterm *hp; + ErlOffHeap *ohp; + ignore = 0; + if (dlnk == llnk) + dlnk = NULL; + else + ldp = NULL; + + ASSERT(is_immed(reason)); + + if (!(c_p->flags & F_TRAP_EXIT)) { + if (reason == am_normal) + ignore = !0; /* Ignore it... */ + else + exit = !0; /* Terminate... */ + } + else { + + /* + * Create and EXIT message and replace + * the original signal with the message... + */ + + locks = ERTS_PROC_LOCK_MAIN; + + hsz = 4 + NC_HEAP_SIZE(from); + + mp = erts_alloc_message_heap(c_p, &locks, hsz, &hp, &ohp); + + if (locks != ERTS_PROC_LOCK_MAIN) + erts_proc_unlock(c_p, locks & ~ERTS_PROC_LOCK_MAIN); + + pid = STORE_NC(&hp, ohp, from); + + ERL_MESSAGE_TERM(mp) = TUPLE3(hp, am_EXIT, pid, reason); + ERL_MESSAGE_TOKEN(mp) = NIL; +#ifdef USE_VM_PROBES + ERL_MESSAGE_DT_UTAG(mp) = NIL; +#endif + if (is_immed(pid)) + ERL_MESSAGE_FROM(mp) = pid; + else { + DistEntry *dep; + ASSERT(is_external_pid(pid)); + dep = external_pid_dist_entry(pid); + ERL_MESSAGE_FROM(mp) = dep->sysname; + } + + /* Replace original signal with the exit message... */ + convert_to_msg(c_p, sig, mp, next_nm_sig); + + cnt += 4; + + conv_msg = mp; + } + } + destroy = !0; + } + + if (ignore|exit) { + remove_nm_sig(c_p, sig, next_nm_sig); + if (exit) { + if (save) { + sig->data.attached = ERTS_MSG_COMBINED_HFRAG; + ERL_MESSAGE_TERM(sig) = xsigd->message; + erts_save_message_in_proc(c_p, sig); + } + /* Exit process... */ + erts_set_self_exiting(c_p, reason); + + cnt++; + } + } + + if (!exit) { + if (conv_msg) + erts_proc_notify_new_message(c_p, ERTS_PROC_LOCK_MAIN); + if (op == ERTS_SIG_Q_OP_EXIT_LINKED && tracing->procs) + getting_unlinked(c_p, from); + } + + if (destroy) { + cnt++; + if (type == ERTS_SIG_Q_TYPE_GEN_EXIT) { + sig->next = NULL; + erts_cleanup_messages(sig); + } + else { + if (ldp) + erts_link_release_both(ldp); + else { + if (dlnk) + erts_link_release(dlnk); + erts_link_release((ErtsLink *) sig); + } + } + } + + *exited = exit; + + return cnt; +} + +static ERTS_INLINE int +convert_prepared_down_message(Process *c_p, ErtsMessage *sig, + Eterm msg, ErtsMessage ***next_nm_sig) +{ + convert_prepared_sig_to_msg(c_p, sig, msg, next_nm_sig); + erts_proc_notify_new_message(c_p, ERTS_PROC_LOCK_MAIN); + return 1; +} + +static int +convert_to_down_message(Process *c_p, + ErtsMessage *sig, + ErtsMonitorData *mdp, + Uint16 mon_type, + ErtsMessage ***next_nm_sig) +{ + /* + * Create a 'DOWN' message and replace the signal + * with it... + */ + int cnt = 0; + Eterm node = am_undefined; + ErtsMessage *mp; + ErtsProcLocks locks; + Uint hsz; + Eterm *hp, ref, from, type, reason; + ErlOffHeap *ohp; + + ASSERT(mdp); + ASSERT((mdp->origin.flags & ERTS_ML_FLGS_SAME) + == (mdp->target.flags & ERTS_ML_FLGS_SAME)); + + hsz = 6; /* 5-tuple */ + + if (mdp->origin.flags & ERTS_ML_FLG_NAME) + hsz += 3; /* reg name 2-tuple */ + else { + ASSERT(is_pid(mdp->origin.other.item) + || is_internal_port(mdp->origin.other.item)); + hsz += NC_HEAP_SIZE(mdp->origin.other.item); + } + + ASSERT(is_ref(mdp->ref)); + hsz += NC_HEAP_SIZE(mdp->ref); + + locks = ERTS_PROC_LOCK_MAIN; + + /* reason is mdp->target.other.item */ + reason = mdp->target.other.item; + ASSERT(is_immed(reason)); + + mp = erts_alloc_message_heap(c_p, &locks, hsz, &hp, &ohp); + + if (locks != ERTS_PROC_LOCK_MAIN) + erts_proc_unlock(c_p, locks & ~ERTS_PROC_LOCK_MAIN); + + cnt += 4; + + ref = STORE_NC(&hp, ohp, mdp->ref); + + if (!(mdp->origin.flags & ERTS_ML_FLG_NAME)) { + from = STORE_NC(&hp, ohp, mdp->origin.other.item); + } + else { + ErtsMonitorDataExtended *mdep; + ASSERT(mdp->origin.flags & ERTS_ML_FLG_EXTENDED); + mdep = (ErtsMonitorDataExtended *) mdp; + ASSERT(is_atom(mdep->u.name)); + if (mdep->dist) + node = mdep->dist->nodename; + else + node = erts_this_dist_entry->sysname; + from = TUPLE2(hp, mdep->u.name, node); + hp += 3; + } + + ASSERT(mdp->origin.type == mon_type); + switch (mon_type) { + case ERTS_MON_TYPE_PORT: + type = am_port; + if (mdp->origin.other.item == am_undefined) { + /* failed by name... */ + ERL_MESSAGE_FROM(mp) = am_system; + } + else { + ASSERT(is_internal_port(mdp->origin.other.item)); + ERL_MESSAGE_FROM(mp) = mdp->origin.other.item; + } + break; + case ERTS_MON_TYPE_PROC: + type = am_process; + if (mdp->origin.other.item == am_undefined) { + /* failed by name... */ + ERL_MESSAGE_FROM(mp) = am_system; + } + else { + ASSERT(is_internal_pid(mdp->origin.other.item)); + ERL_MESSAGE_FROM(mp) = mdp->origin.other.item; + } + break; + case ERTS_MON_TYPE_DIST_PROC: + type = am_process; + if (node == am_undefined) { + ErtsMonitorDataExtended *mdep; + ASSERT(mdp->origin.flags & ERTS_ML_FLG_EXTENDED); + mdep = (ErtsMonitorDataExtended *) mdp; + ASSERT(mdep->dist); + node = mdep->dist->nodename; + } + ASSERT(is_atom(node) && node != am_undefined); + ERL_MESSAGE_FROM(mp) = node; + break; + default: + ERTS_INTERNAL_ERROR("Unexpected monitor type"); + type = am_undefined; + ERL_MESSAGE_FROM(mp) = am_undefined; + break; + } + + ERL_MESSAGE_TERM(mp) = TUPLE5(hp, am_DOWN, ref, + type, from, reason); + hp += 6; + + ERL_MESSAGE_TOKEN(mp) = NIL; +#ifdef USE_VM_PROBES + ERL_MESSAGE_DT_UTAG(mp) = NIL; +#endif + /* Replace original signal with the exit message... */ + convert_to_msg(c_p, sig, mp, next_nm_sig); + + cnt += 4; + + erts_proc_notify_new_message(c_p, ERTS_PROC_LOCK_MAIN); + + return cnt; +} + +static ERTS_INLINE int +convert_to_nodedown_messages(Process *c_p, + ErtsMessage *sig, + ErtsMonitorData *mdp, + ErtsMessage ***next_nm_sig) +{ + int cnt = 1; + Uint n; + ErtsMonitorDataExtended *mdep = (ErtsMonitorDataExtended *) mdp; + + ASSERT((mdp->origin.flags & ERTS_ML_FLGS_SAME) + == (mdp->target.flags & ERTS_ML_FLGS_SAME)); + ASSERT(mdp->origin.flags & ERTS_ML_FLG_EXTENDED); + + n = mdep->u.refc; + + if (n == 0) + remove_nm_sig(c_p, sig, next_nm_sig); + else { + Uint i; + ErtsMessage *nd_first = NULL; + ErtsMessage *nd_last = NULL; + ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN; + Eterm node = mdep->dist->nodename; + + ASSERT(is_atom(node)); + ASSERT(n > 0); + + for (i = 0; i < n; i++) { + ErtsMessage *mp; + ErlOffHeap *ohp; + Eterm *hp; + + mp = erts_alloc_message_heap(c_p, &locks, 3, &hp, &ohp); + + ERL_MESSAGE_TERM(mp) = TUPLE2(hp, am_nodedown, node); + ERL_MESSAGE_FROM(mp) = am_system; + ERL_MESSAGE_TOKEN(mp) = NIL; +#ifdef USE_VM_PROBES + ERL_MESSAGE_DT_UTAG(mp) = NIL; +#endif + mp->next = nd_first; + nd_first = mp; + if (!nd_last) + nd_last = mp; + cnt++; + } + + if (locks != ERTS_PROC_LOCK_MAIN) + erts_proc_unlock(c_p, locks & ~ERTS_PROC_LOCK_MAIN); + + /* Replace signal with 'nodedown' messages */ + convert_to_msgs(c_p, sig, n, nd_first, nd_last, next_nm_sig); + + erts_proc_notify_new_message(c_p, ERTS_PROC_LOCK_MAIN); + } + return cnt; +} + +static int +handle_nodedown(Process *c_p, + ErtsMessage *sig, + ErtsMonitorData *mdp, + ErtsMessage ***next_nm_sig) +{ + ErtsMonitorDataExtended *mdep = (ErtsMonitorDataExtended *) mdp; + ErtsMonitor *omon = &mdp->origin; + int not_in_subtab = !(omon->flags & ERTS_ML_FLG_IN_SUBTABLE); + int cnt = 1; + + ASSERT(erts_monitor_is_in_table(omon)); + + if (not_in_subtab & !mdep->uptr.node_monitors) + erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p), omon); + else if (not_in_subtab) { + ErtsMonitor *sub_mon; + ErtsMonitorDataExtended *sub_mdep; + sub_mon = erts_monitor_list_last(mdep->uptr.node_monitors); + ASSERT(sub_mon); + erts_monitor_list_delete(&mdep->uptr.node_monitors, sub_mon); + sub_mon->flags &= ~ERTS_ML_FLG_IN_SUBTABLE; + sub_mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(sub_mon); + ASSERT(!sub_mdep->uptr.node_monitors); + sub_mdep->uptr.node_monitors = mdep->uptr.node_monitors; + mdep->uptr.node_monitors = NULL; + erts_monitor_tree_replace(&ERTS_P_MONITORS(c_p), omon, sub_mon); + cnt += 2; + } + else { + ErtsMonitorDataExtended *top_mdep; + ErtsMonitor *top_mon; + ASSERT(is_atom(omon->other.item)); + ASSERT(!mdep->uptr.node_monitors); + top_mon = erts_monitor_tree_lookup(ERTS_P_MONITORS(c_p), + omon->other.item); + ASSERT(top_mon); + top_mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(top_mon); + ASSERT(top_mdep->uptr.node_monitors); + erts_monitor_list_delete(&top_mdep->uptr.node_monitors, omon); + omon->flags &= ~ERTS_ML_FLG_IN_SUBTABLE; + cnt += 3; + } + + return cnt + convert_to_nodedown_messages(c_p, sig, mdp, next_nm_sig); +} + +static void +handle_persistent_mon_msg(Process *c_p, Uint16 type, + ErtsMonitor *mon, ErtsMessage *sig, + Eterm msg, ErtsMessage ***next_nm_sig) +{ + convert_prepared_sig_to_msg(c_p, sig, msg, next_nm_sig); + + switch (type) { + + case ERTS_MON_TYPE_TIME_OFFSET: + ASSERT(mon->type == ERTS_MON_TYPE_TIME_OFFSET); + break; + + case ERTS_MON_TYPE_NODES: { + ErtsMonitorDataExtended *mdep; + Uint n; + ASSERT(mon->type == ERTS_MON_TYPE_NODES); + mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon); + ERTS_ML_ASSERT(mdep->u.refc > 0); + n = mdep->u.refc; + n--; + if (n > 0) { + ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN; + ErtsMessage *first = NULL, *prev, *last; + Uint hsz = size_object(msg); + Uint i; + + for (i = 0; i < n; i++) { + Eterm *hp; + ErlOffHeap *ohp; + + last = erts_alloc_message_heap(c_p, &locks, hsz, &hp, &ohp); + + if (!first) + first = last; + else + prev->next = last; + prev = last; + + ERL_MESSAGE_TERM(last) = copy_struct(msg, hsz, &hp, ohp); + +#ifdef USE_VM_PROBES + ASSERT(is_immed(ERL_MESSAGE_DT_UTAG(sig))); + ERL_MESSAGE_DT_UTAG(last) = ERL_MESSAGE_DT_UTAG(sig); +#endif + ASSERT(is_immed(ERL_MESSAGE_TOKEN(sig))); + ERL_MESSAGE_TOKEN(last) = ERL_MESSAGE_TOKEN(sig); + ASSERT(is_immed(ERL_MESSAGE_FROM(sig))); + ERL_MESSAGE_FROM(last) = ERL_MESSAGE_FROM(sig); + + } + if (locks != ERTS_PROC_LOCK_MAIN) + erts_proc_unlock(c_p, locks & ~ERTS_PROC_LOCK_MAIN); + insert_messages(c_p, &sig->next, first, last, n, next_nm_sig); + } + break; + } + + default: + ERTS_INTERNAL_ERROR("Invalid type"); + break; + } + + erts_proc_notify_new_message(c_p, ERTS_PROC_LOCK_MAIN); +} + +static void +group_leader_reply(Process *c_p, Eterm to, Eterm ref, int success) +{ + Process *rp = erts_proc_lookup(to); + + if (rp) { + ErtsProcLocks locks; + Uint sz; + Eterm *hp, msg, ref_cpy, result; + ErlOffHeap *ohp; + ErtsMessage *mp; + + ASSERT(is_internal_ref(ref)); + + locks = c_p == rp ? ERTS_PROC_LOCK_MAIN : 0; + sz = size_object(ref); + + mp = erts_alloc_message_heap(rp, &locks, sz+3, + &hp, &ohp); + + ref_cpy = copy_struct(ref, sz, &hp, ohp); + result = success ? am_true : am_badarg; + msg = TUPLE2(hp, ref_cpy, result); + + erts_queue_message(rp, locks, mp, msg, am_system); + + if (c_p == rp) + locks &= ~ERTS_PROC_LOCK_MAIN; + + if (locks) + erts_proc_unlock(rp, locks); + } +} + +static void +handle_group_leader(Process *c_p, ErtsSigGroupLeader *sgl) +{ + erts_aint_t flags; + + flags = erts_atomic_read_band_nob(&sgl->flags, ~ERTS_SIG_GL_FLG_ACTIVE); + if (flags & ERTS_SIG_GL_FLG_ACTIVE) { + int res = erts_set_group_leader(c_p, sgl->group_leader); + if (is_internal_pid(sgl->reply_to)) + group_leader_reply(c_p, sgl->reply_to, sgl->ref, res); + } + + flags = erts_atomic_read_band_nob(&sgl->flags, ~ERTS_SIG_GL_FLG_RECEIVER); + if ((flags & ~ERTS_SIG_GL_FLG_RECEIVER) == 0) + destroy_sig_group_leader(sgl); +} + +static void +check_push_msgq_len_offs_marker(Process *rp, ErtsSignal *sig) +{ + ErtsProcessInfoSig *pisig = (ErtsProcessInfoSig *) sig; + + ASSERT(ERTS_PROC_SIG_OP(sig->common.tag) == ERTS_SIG_Q_OP_PROCESS_INFO); + + if (pisig->msgq_len_offset == ERTS_PROC_SIG_PI_MSGQ_LEN_SYNC) { + ErtsProcSigMsgQLenOffsetMarker *mrkr; + Sint len, msgq_len_offset; + ErtsMessage *first = rp->sig_inq.first; + ASSERT(first); + if (((ErtsSignal *) first)->common.tag == ERTS_PROC_SIG_MSGQ_LEN_OFFS_MARK) + mrkr = (ErtsProcSigMsgQLenOffsetMarker *) first; + else { + mrkr = &pisig->marker; + + ASSERT(mrkr->common.tag == ERTS_PROC_SIG_MSGQ_LEN_OFFS_MARK); + + mrkr->common.next = first; + ASSERT(rp->sig_inq.last != &rp->sig_inq.first); + if (rp->sig_inq.nmsigs.next == &rp->sig_inq.first) + rp->sig_inq.nmsigs.next = &mrkr->common.next; + if (rp->sig_inq.nmsigs.last == &rp->sig_inq.first) + rp->sig_inq.nmsigs.last = &mrkr->common.next; + rp->sig_inq.first = (ErtsMessage *) mrkr; + } + + len = rp->sig_inq.len; + msgq_len_offset = len - mrkr->len_offset; + + mrkr->len_offset = len; + mrkr->refc++; + + pisig->msgq_len_offset = msgq_len_offset; + +#ifdef DEBUG + /* save pointer to used marker... */ + pisig->marker.common.specific.attachment = (void *) mrkr; +#endif + + } +} + +static void +destroy_process_info_request(Process *c_p, ErtsProcessInfoSig *pisig) +{ + int dealloc_pisig = !0; + + if (pisig->msgq_len_offset != ERTS_PROC_SIG_PI_MSGQ_LEN_IGNORE) { + Sint refc; + int dealloc_marker = 0; + ErtsProcSigMsgQLenOffsetMarker *marker; +#ifdef ERTS_PROC_SIG_HARD_DEBUG_SIGQ_MSG_LEN + Sint delayed_len; +#endif + + ASSERT(pisig->msgq_len_offset >= 0); + + erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ); + marker = (ErtsProcSigMsgQLenOffsetMarker *) c_p->sig_inq.first; + ASSERT(marker); + ASSERT(marker->refc > 0); + ASSERT(pisig->marker.common.specific.attachment == (void *) marker); + + marker->delayed_len -= pisig->msgq_len_offset; +#ifdef ERTS_PROC_SIG_HARD_DEBUG_SIGQ_MSG_LEN + delayed_len = marker->delayed_len; +#endif + + refc = --marker->refc; + if (refc) { + if (marker == &pisig->marker) { + /* Another signal using our marker... */ + dealloc_pisig = 0; + } + } + else { + /* Marker unused; remove it... */ + ASSERT(marker->delayed_len + marker->len_offset == 0); +#ifdef ERTS_PROC_SIG_HARD_DEBUG_SIGQ_MSG_LEN + delayed_len += marker->len_offset; +#endif + if (marker != &pisig->marker) + dealloc_marker = !0; /* used another signals marker... */ + c_p->sig_inq.first = marker->common.next; + if (c_p->sig_inq.last == &marker->common.next) + c_p->sig_inq.last = &c_p->sig_inq.first; + if (c_p->sig_inq.nmsigs.next == &marker->common.next) + c_p->sig_inq.nmsigs.next = &c_p->sig_inq.first; + if (c_p->sig_inq.nmsigs.last == &marker->common.next) + c_p->sig_inq.nmsigs.last = &c_p->sig_inq.first; + } + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); + + if (!refc) { + c_p->flags &= ~F_DELAYED_PSIGQS_LEN; + /* Adjust msg len of inner+middle queue */ + ASSERT(marker->len_offset <= 0); + c_p->sig_qs.len -= marker->len_offset; + + ASSERT(c_p->sig_qs.len >= 0); + } + +#ifdef ERTS_PROC_SIG_HARD_DEBUG_SIGQ_MSG_LEN + { + Sint len = 0; + ERTS_FOREACH_SIG_PRIVQS( + c_p, mp, + { + if (ERTS_SIG_IS_MSG(mp)) + len++; + }); + ERTS_ASSERT(c_p->sig_qs.len + delayed_len == len); + } +#endif + + + if (dealloc_marker) { + ErtsProcessInfoSig *pisig2 + = (ErtsProcessInfoSig *) (((char *) marker) + - offsetof(ErtsProcessInfoSig, + marker)); + erts_free(ERTS_ALC_T_SIG_DATA, pisig2); + } + } + + if (dealloc_pisig) + erts_free(ERTS_ALC_T_SIG_DATA, pisig); +} + +static int +handle_process_info(Process *c_p, ErtsSigRecvTracing *tracing, + ErtsMessage *sig, ErtsMessage ***next_nm_sig, + int is_alive) +{ + ErtsProcessInfoSig *pisig = (ErtsProcessInfoSig *) sig; + Uint reds = 0; + Process *rp; + + if (pisig->msgq_len_offset != ERTS_PROC_SIG_PI_MSGQ_LEN_IGNORE) { + /* + * Request requires message queue data to be updated + * before inspection... + */ + + ASSERT(pisig->msgq_len_offset >= 0); + /* + * Update sig_qs.len to reflect the length + * of the message queue... + */ + c_p->sig_qs.len += pisig->msgq_len_offset; + + if (is_alive) { + /* + * Move messages part of message queue into inner + * signal queue... + */ + ASSERT(tracing); + + if (*next_nm_sig != &c_p->sig_qs.cont) { + if (*next_nm_sig == tracing->messages.next) + tracing->messages.next = &c_p->sig_qs.cont; + *c_p->sig_qs.last = c_p->sig_qs.cont; + c_p->sig_qs.last = *next_nm_sig; + + c_p->sig_qs.cont = **next_nm_sig; + if (c_p->sig_qs.nmsigs.last == *next_nm_sig) + c_p->sig_qs.nmsigs.last = &c_p->sig_qs.cont; + *next_nm_sig = &c_p->sig_qs.cont; + *c_p->sig_qs.last = NULL; + } + + if (!pisig->common.specific.next) { + /* + * No more signals in middle queue... + * + * Process-info 'status' needs sig-q + * process flag to be updated in order + * to show accurate result... + */ + erts_atomic32_read_band_nob(&c_p->state, + ~ERTS_PSFLG_SIG_Q); + } + +#ifdef ERTS_PROC_SIG_HARD_DEBUG_SIGQ_MSG_LEN + { + Sint len; + ErtsMessage *mp; + for (mp = c_p->sig_qs.first, len = 0; mp; mp = mp->next) { + ERTS_ASSERT(ERTS_SIG_IS_MSG(mp)); + len++; + } + ERTS_ASSERT(c_p->sig_qs.len == len); + } +#endif + } + } + if (is_alive) + remove_nm_sig(c_p, sig, next_nm_sig); + + rp = erts_proc_lookup(pisig->requester); + ASSERT(c_p != rp); + if (rp) { + Eterm msg, res, ref, *hp; + ErtsProcLocks locks = 0; + ErtsHeapFactory hfact; + ErtsMessage *mp; + Uint reserve_size = 3 + sizeof(pisig->oref_thing)/sizeof(Eterm); + + if (!is_alive) { + ErlOffHeap *ohp; + mp = erts_alloc_message_heap(rp, &locks, reserve_size, &hp, &ohp); + res = am_undefined; + } + else { + ErlHeapFragment *hfrag; + + reserve_size += pisig->reserve_size; + + mp = erts_alloc_message(0, NULL); + hfrag = new_message_buffer(reserve_size); + mp->data.heap_frag = hfrag; + erts_factory_selfcontained_message_init(&hfact, mp, &hfrag->mem[0]); + + res = erts_process_info(c_p, &hfact, c_p, ERTS_PROC_LOCK_MAIN, + pisig->item_ix, pisig->len, + pisig->flags, reserve_size, &reds); + + hp = erts_produce_heap(&hfact, + 3 + sizeof(pisig->oref_thing)/sizeof(Eterm), + 0); + } + + sys_memcpy((void *) hp, (void *) &pisig->oref_thing, + sizeof(pisig->oref_thing)); + ref = make_internal_ref(hp); + hp += sizeof(pisig->oref_thing)/sizeof(Eterm); + + msg = TUPLE2(hp, ref, res); + + if (is_alive) + erts_factory_trim_and_close(&hfact, &msg, 1); + + erts_queue_proc_message(c_p, rp, locks, mp, msg); + + if (!is_alive && locks) + erts_proc_unlock(rp, locks); + } + + destroy_process_info_request(c_p, pisig); + + if (reds > INT_MAX/8) + reds = INT_MAX/8; + + return ((int) reds)*4 + 8; +} + +/* + * Called in order to handle incoming signals. + */ + +int +erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep, + int *redsp, int max_reds, int local_only) +{ + Eterm tag; + erts_aint32_t state; + int cnt, limit, abs_lim, msg_tracing; + ErtsMessage *sig, ***next_nm_sig; + ErtsSigRecvTracing tracing; + + ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p, 0); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); + + if (local_only) + state = -1; /* can never be a valid state... */ + else { + state = erts_atomic32_read_nob(&c_p->state); + if (ERTS_PSFLG_SIG_IN_Q & state) { + erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ); + erts_proc_sig_fetch(c_p); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); + } + } + + limit = *redsp; + *redsp = 0; + + if (!c_p->sig_qs.cont) { + if (state == -1) + *statep = erts_atomic32_read_nob(&c_p->state); + else + *statep = state; + return !0; + } + + next_nm_sig = &c_p->sig_qs.nmsigs.next; + + setup_tracing_state(c_p, &tracing); + msg_tracing = tracing.messages.active; + + limit *= ERTS_SIG_REDS_CNT_FACTOR; + abs_lim = ERTS_SIG_REDS_CNT_FACTOR*max_reds; + if (limit > abs_lim) + limit = abs_lim; + + cnt = 0; + + do { + + if (msg_tracing) { + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + if (handle_msg_tracing(c_p, &tracing, next_nm_sig) != 0) { + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + break; /* tracing limit or end... */ + } + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + } + + if (!*next_nm_sig) + break; + + sig = **next_nm_sig; + + ASSERT(sig); + ASSERT(ERTS_SIG_IS_NON_MSG(sig)); + + tag = ((ErtsSignal *) sig)->common.tag; + + switch (ERTS_PROC_SIG_OP(tag)) { + + case ERTS_SIG_Q_OP_EXIT: + case ERTS_SIG_Q_OP_EXIT_LINKED: { + int exited; + + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + + cnt += handle_exit_signal(c_p, &tracing, sig, + next_nm_sig, &exited); + + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + + if (exited) + goto stop; /* terminated by signal */ + /* ignored or converted to exit message... */ + break; + } + + case ERTS_SIG_Q_OP_MONITOR_DOWN: { + Uint16 type = ERTS_PROC_SIG_TYPE(tag); + ErtsExitSignalData *xsigd = NULL; + ErtsMonitorData *mdp = NULL; + ErtsMonitor *omon = NULL, *tmon = NULL; + + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + + switch (type) { + case ERTS_MON_TYPE_DIST_PROC: + case ERTS_MON_TYPE_PROC: + case ERTS_MON_TYPE_PORT: + tmon = (ErtsMonitor *) sig; + ASSERT(erts_monitor_is_target(tmon)); + ASSERT(!erts_monitor_is_in_table(tmon)); + mdp = erts_monitor_to_data(tmon); + if (erts_monitor_is_in_table(&mdp->origin)) { + omon = &mdp->origin; + erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p), + omon); + cnt += convert_to_down_message(c_p, sig, mdp, + type, next_nm_sig); + } + break; + case ERTS_SIG_Q_TYPE_GEN_EXIT: + xsigd = get_exit_signal_data(sig); + omon = erts_monitor_tree_lookup(ERTS_P_MONITORS(c_p), + xsigd->u.ref); + if (omon) { + ASSERT(erts_monitor_is_origin(omon)); + if (omon->type == ERTS_MON_TYPE_DIST_PROC) { + mdp = erts_monitor_to_data(omon); + if (erts_monitor_dist_delete(&mdp->target)) + tmon = &mdp->target; + } + erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p), + omon); + cnt += convert_prepared_down_message(c_p, sig, + xsigd->message, + next_nm_sig); + } + break; + case ERTS_MON_TYPE_NODE: + tmon = (ErtsMonitor *) sig; + ASSERT(erts_monitor_is_target(tmon)); + ASSERT(!erts_monitor_is_in_table(tmon)); + mdp = erts_monitor_to_data(tmon); + if (erts_monitor_is_in_table(&mdp->origin)) { + omon = &mdp->origin; + cnt += handle_nodedown(c_p, sig, mdp, next_nm_sig); + } + break; + default: + ERTS_INTERNAL_ERROR("invalid monitor type"); + break; + } + + if (omon) { + if (tmon) + erts_monitor_release_both(mdp); + else + erts_monitor_release(omon); + } + else { + remove_nm_sig(c_p, sig, next_nm_sig); + if (xsigd) { + sig->next = NULL; + erts_cleanup_messages(sig); + } + if (tmon) + erts_monitor_release(tmon); + } + + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + break; + } + + case ERTS_SIG_Q_OP_PERSISTENT_MON_MSG: { + Uint16 type = ERTS_PROC_SIG_TYPE(tag); + ErtsMonitor *mon; + Eterm msg; + Eterm key; + + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + + key = get_persist_mon_msg(sig, &msg); + + cnt++; + mon = erts_monitor_tree_lookup(ERTS_P_MONITORS(c_p), key); + if (mon) { + ASSERT(erts_monitor_is_origin(mon)); + handle_persistent_mon_msg(c_p, type, mon, sig, + msg, next_nm_sig); + } + else { + cnt++; + remove_nm_sig(c_p, sig, next_nm_sig); + sig->next = NULL; + erts_cleanup_messages(sig); + } + + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + break; + } + + case ERTS_SIG_Q_OP_MONITOR: { + ErtsMonitor *mon = (ErtsMonitor *) sig; + + ASSERT(erts_monitor_is_target(mon)); + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + + remove_nm_sig(c_p, sig, next_nm_sig); + + if (mon->type == ERTS_MON_TYPE_DIST_PROC) + erts_monitor_tree_insert(&ERTS_P_MONITORS(c_p), mon); + else + erts_monitor_list_insert(&ERTS_P_LT_MONITORS(c_p), mon); + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + break; + } + + case ERTS_SIG_Q_OP_DEMONITOR: { + Uint16 type = ERTS_PROC_SIG_TYPE(tag); + + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + + remove_nm_sig(c_p, sig, next_nm_sig); + + if (type == ERTS_SIG_Q_TYPE_DIST_PROC_DEMONITOR) { + ErtsMonitor *tmon; + ErtsSigDistProcDemonitor *dmon; + dmon = (ErtsSigDistProcDemonitor *) sig; + tmon = erts_monitor_tree_lookup(ERTS_P_MONITORS(c_p), dmon->ref); + destroy_dist_proc_demonitor(dmon); + cnt++; + if (tmon) { + ErtsMonitorData *mdp = erts_monitor_to_data(tmon); + ASSERT(erts_monitor_is_target(tmon)); + erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p), tmon); + if (!erts_monitor_dist_delete(&mdp->origin)) + erts_monitor_release(tmon); + else + erts_monitor_release_both(mdp); + cnt += 2; + } + } + else { + ErtsMonitor *omon = (ErtsMonitor *) sig; + ErtsMonitorData *mdp = erts_monitor_to_data(omon); + ASSERT(omon->type == type); + ASSERT(erts_monitor_is_origin(omon)); + ASSERT(!erts_monitor_is_in_table(omon)); + if (!erts_monitor_is_in_table(&mdp->target)) + erts_monitor_release(omon); + else { + ErtsMonitor *tmon = &mdp->target; + ASSERT(tmon->type == type); + if (type == ERTS_MON_TYPE_DIST_PROC) + erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p), tmon); + else { + erts_monitor_list_delete(&ERTS_P_LT_MONITORS(c_p), tmon); + if (type == ERTS_MON_TYPE_RESOURCE) { + erts_nif_demonitored((ErtsResource *) tmon->other.ptr); + cnt++; + } + } + erts_monitor_release_both(mdp); + cnt++; + } + cnt++; + } + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + break; + } + + case ERTS_SIG_Q_OP_LINK: { + ErtsLink *rlnk, *lnk = (ErtsLink *) sig; + + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + + remove_nm_sig(c_p, sig, next_nm_sig); + rlnk = erts_link_tree_insert_addr_replace(&ERTS_P_LINKS(c_p), + lnk); + if (!rlnk) { + if (tracing.procs) + getting_linked(c_p, lnk->other.item); + } + else { + if (rlnk->type != ERTS_LNK_TYPE_DIST_PROC) + erts_link_release(rlnk); + else { + ErtsLinkData *ldp; + ErtsLink *dlnk = erts_link_to_other(rlnk, &ldp); + if (erts_link_dist_delete(dlnk)) + erts_link_release_both(ldp); + else + erts_link_release(rlnk); + } + } + + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + break; + } + + case ERTS_SIG_Q_OP_UNLINK: { + Uint16 type = ERTS_PROC_SIG_TYPE(tag); + ErtsLinkData *ldp; + ErtsLink *llnk; + + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + + remove_nm_sig(c_p, sig, next_nm_sig); + if (type == ERTS_SIG_Q_TYPE_DIST_LINK) { + ErtsSigDistLinkOp *sdlnk = (ErtsSigDistLinkOp *) sig; + ASSERT(type == ERTS_SIG_Q_TYPE_DIST_LINK); + ASSERT(is_external_pid(sdlnk->remote)); + llnk = erts_link_tree_lookup(ERTS_P_LINKS(c_p), sdlnk->remote); + if (llnk) { + ErtsLink *dlnk = erts_link_to_other(llnk, &ldp); + erts_link_tree_delete(&ERTS_P_LINKS(c_p), llnk); + if (erts_link_dist_delete(dlnk)) + erts_link_release_both(ldp); + else + erts_link_release(llnk); + cnt += 8; + if (tracing.procs) + getting_unlinked(c_p, sdlnk->remote); + } + destroy_sig_dist_link_op(sdlnk); + cnt++; + } + else { + ErtsLinkData *ldp; + ErtsLink *dlnk, *slnk; + slnk = (ErtsLink *) sig; + llnk = erts_link_to_other(slnk, &ldp); + dlnk = erts_link_tree_key_delete(&ERTS_P_LINKS(c_p), llnk); + if (!dlnk) + erts_link_release(slnk); + else { + if (tracing.procs) + getting_unlinked(c_p, llnk->other.item); + if (dlnk == llnk) + erts_link_release_both(ldp); + else { + erts_link_release(slnk); + erts_link_release(dlnk); + } + } + cnt += 2; + } + + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + break; + } + + case ERTS_SIG_Q_OP_GROUP_LEADER: { + ErtsSigGroupLeader *sgl = (ErtsSigGroupLeader *) sig; + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + remove_nm_sig(c_p, sig, next_nm_sig); + handle_group_leader(c_p, sgl); + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + break; + } + + case ERTS_SIG_Q_OP_IS_ALIVE: + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + remove_nm_sig(c_p, sig, next_nm_sig); + is_alive_response(c_p, sig, !0); + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + break; + + case ERTS_SIG_Q_OP_PROCESS_INFO: + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + handle_process_info(c_p, &tracing, sig, next_nm_sig, !0); + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + break; + + case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE: { + Uint16 type = ERTS_PROC_SIG_TYPE(tag); + + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + msg_tracing = handle_trace_change_state(c_p, &tracing, + type, sig, + next_nm_sig); + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig); + + break; + } + + default: + ERTS_INTERNAL_ERROR("Unknown signal"); + break; + } + + cnt++; + + } while (cnt <= limit || stretch_limit(c_p, &tracing, abs_lim, &limit)); + +stop: { + int deferred_save, deferred_saved_last, res; + + deferred_saved_last = !!(c_p->flags & F_DEFERRED_SAVED_LAST); + deferred_save = 0; + + if (!deferred_saved_last) + deferred_save = 0; + else { + if (c_p->sig_qs.saved_last == &c_p->sig_qs.cont) { + c_p->sig_qs.saved_last = c_p->sig_qs.last; + c_p->flags &= ~F_DEFERRED_SAVED_LAST; + deferred_saved_last = deferred_save = 0; + } + else { + if (c_p->sig_qs.save == c_p->sig_qs.last) + deferred_save = !0; + else + deferred_save = 0; + } + } + + ASSERT(c_p->sig_qs.saved_last != &c_p->sig_qs.cont); + + if (ERTS_UNLIKELY(msg_tracing != 0)) { + /* + * All messages that has been traced should + * be moved to inner queue. Next signal in + * middle queue should either be next message + * to trace or next non-message signal. + */ + ASSERT(tracing.messages.next); + if (*next_nm_sig) { + if (*next_nm_sig == tracing.messages.next) + *next_nm_sig = &c_p->sig_qs.cont; + if (c_p->sig_qs.nmsigs.last == tracing.messages.next) + c_p->sig_qs.nmsigs.last = &c_p->sig_qs.cont; + *statep = erts_atomic32_read_nob(&c_p->state); + } + else { + ASSERT(!c_p->sig_qs.nmsigs.next); + c_p->sig_qs.nmsigs.last = NULL; + state = erts_atomic32_read_band_nob(&c_p->state, + ~ERTS_PSFLG_SIG_Q); + state &= ~ERTS_PSFLG_SIG_Q; + *statep = state; + } + + if (tracing.messages.next != &c_p->sig_qs.cont) { + *c_p->sig_qs.last = c_p->sig_qs.cont; + c_p->sig_qs.last = tracing.messages.next; + + c_p->sig_qs.cont = *tracing.messages.next; + if (!c_p->sig_qs.cont) + c_p->sig_qs.cont_last = &c_p->sig_qs.cont; + *c_p->sig_qs.last = NULL; + } + + res = !c_p->sig_qs.cont; + } + else if (*next_nm_sig) { + /* + * All messages prior to next non-message + * signal should be moved to inner queue. + * Next non-message signal to handle should + * be first in middle queue. + */ + ASSERT(**next_nm_sig); + if (*next_nm_sig != &c_p->sig_qs.cont) { + *c_p->sig_qs.last = c_p->sig_qs.cont; + c_p->sig_qs.last = *next_nm_sig; + + c_p->sig_qs.cont = **next_nm_sig; + if (c_p->sig_qs.nmsigs.last == *next_nm_sig) + c_p->sig_qs.nmsigs.last = &c_p->sig_qs.cont; + *next_nm_sig = &c_p->sig_qs.cont; + *c_p->sig_qs.last = NULL; + } + + ASSERT(c_p->sig_qs.cont); + + *statep = erts_atomic32_read_nob(&c_p->state); + + res = 0; + } + else { + /* + * All non-message signals handled. All + * messages should be moved to inner queue. + * Middle queue should be empty. + */ + ASSERT(!c_p->sig_qs.nmsigs.next); + c_p->sig_qs.nmsigs.last = NULL; + + if (c_p->sig_qs.cont_last != &c_p->sig_qs.cont) { + ASSERT(!*c_p->sig_qs.last); + *c_p->sig_qs.last = c_p->sig_qs.cont; + c_p->sig_qs.last = c_p->sig_qs.cont_last; + ASSERT(!*c_p->sig_qs.last); + + c_p->sig_qs.cont_last = &c_p->sig_qs.cont; + c_p->sig_qs.cont = NULL; + } + + ASSERT(!c_p->sig_qs.cont); + + state = erts_atomic32_read_band_nob(&c_p->state, + ~ERTS_PSFLG_SIG_Q); + state &= ~ERTS_PSFLG_SIG_Q; + *statep = state; + res = !0; + } + + if (deferred_saved_last + && (c_p->sig_qs.saved_last == &c_p->sig_qs.cont)) { + c_p->sig_qs.saved_last = c_p->sig_qs.last; + c_p->flags &= ~F_DEFERRED_SAVED_LAST; + if (deferred_save) + c_p->sig_qs.save = c_p->sig_qs.saved_last; + } + else if (!res) { + if (deferred_save) { + c_p->sig_qs.save = c_p->sig_qs.last; + ASSERT(!PEEK_MESSAGE(c_p)); + } + } + else { + c_p->flags &= ~F_DEFERRED_SAVED_LAST; + if (deferred_save) + c_p->sig_qs.save = c_p->sig_qs.saved_last; + } + + ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p, 0); + + *redsp = cnt/4 + 1; + + return res; + } +} + +static int +stretch_limit(Process *c_p, ErtsSigRecvTracing *tp, + int abs_lim, int *limp) +{ + int lim; + /* + * Stretch limit up to a maximum of 'abs_lim' if + * there currently are no messages available to + * inspect by 'receive' and it might be possible + * to get messages available by processing + * signals (or trace messages). + */ + + lim = *limp; + ASSERT(abs_lim >= lim); + if (abs_lim == lim) + return 0; + + if (!(c_p->flags & F_DEFERRED_SAVED_LAST)) { + ErtsSignal *sig; + + if (PEEK_MESSAGE(c_p)) + return 0; + sig = (ErtsSignal *) c_p->sig_qs.cont; + if (!sig) + return 0; /* No signals to process available... */ + if (ERTS_SIG_IS_MSG(sig) && tp->messages.next != &c_p->sig_qs.cont) + return 0; + } + + lim += ERTS_SIG_REDS_CNT_FACTOR*100; + if (lim > abs_lim) + lim = abs_lim; + *limp = lim; + return !0; +} + + +int +erts_proc_sig_handle_exit(Process *c_p, int *redsp) +{ + int cnt, limit; + ErtsMessage *sig, ***next_nm_sig; + + ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p, 0); + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN); + + ASSERT(!(ERTS_PSFLG_SIG_IN_Q & erts_atomic32_read_nob(&c_p->state))); + + limit = *redsp; + limit *= ERTS_SIG_REDS_CNT_FACTOR; + + *redsp = 1; + + next_nm_sig = &c_p->sig_qs.nmsigs.next; + + if (!*next_nm_sig) { + ASSERT(!c_p->sig_qs.nmsigs.last); + return !0; /* done... */ + } + + cnt = 0; + + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, NULL, next_nm_sig); + + do { + Eterm tag; + Uint16 type; + int op; + + sig = **next_nm_sig; + + ASSERT(sig); + ASSERT(ERTS_SIG_IS_NON_MSG(sig)); + + tag = ((ErtsSignal *) sig)->common.tag; + type = ERTS_PROC_SIG_TYPE(tag); + op = ERTS_PROC_SIG_OP(tag); + + remove_nm_sig(c_p, sig, next_nm_sig); + + ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, NULL, next_nm_sig); + + cnt++; + + switch (op) { + + case ERTS_SIG_Q_OP_EXIT: + case ERTS_SIG_Q_OP_EXIT_LINKED: + case ERTS_SIG_Q_OP_MONITOR_DOWN: + switch (type) { + case ERTS_SIG_Q_TYPE_GEN_EXIT: + sig->next = NULL; + erts_cleanup_messages(sig); + break; + case ERTS_LNK_TYPE_PORT: + case ERTS_LNK_TYPE_PROC: + case ERTS_LNK_TYPE_DIST_PROC: + erts_link_release((ErtsLink *) sig); + break; + case ERTS_MON_TYPE_PORT: + case ERTS_MON_TYPE_PROC: + case ERTS_MON_TYPE_DIST_PROC: + case ERTS_MON_TYPE_NODE: + erts_monitor_release((ErtsMonitor *) sig); + break; + default: + ERTS_INTERNAL_ERROR("Unexpected sig type"); + break; + } + break; + + case ERTS_SIG_Q_OP_PERSISTENT_MON_MSG: + sig->next = NULL; + erts_cleanup_messages(sig); + break; + + case ERTS_SIG_Q_OP_MONITOR: { + ErtsProcExitContext pectxt = {c_p, am_noproc}; + erts_proc_exit_handle_monitor((ErtsMonitor *) sig, + (void *) &pectxt); + cnt += 4; + break; + } + + case ERTS_SIG_Q_OP_DEMONITOR: + if (type == ERTS_SIG_Q_TYPE_DIST_PROC_DEMONITOR) + destroy_dist_proc_demonitor((ErtsSigDistProcDemonitor *) sig); + else + erts_monitor_release((ErtsMonitor *) sig); + break; + + case ERTS_SIG_Q_OP_LINK: { + ErtsProcExitContext pectxt = {c_p, am_noproc}; + erts_proc_exit_handle_link((ErtsLink *) sig, (void *) &pectxt); + break; + } + + case ERTS_SIG_Q_OP_UNLINK: + if (type == ERTS_SIG_Q_TYPE_DIST_LINK) + destroy_sig_dist_link_op((ErtsSigDistLinkOp *) sig); + else + erts_link_release((ErtsLink *) sig); + break; + + case ERTS_SIG_Q_OP_GROUP_LEADER: { + ErtsSigGroupLeader *sgl = (ErtsSigGroupLeader *) sig; + handle_group_leader(c_p, sgl); + break; + } + + case ERTS_SIG_Q_OP_IS_ALIVE: + is_alive_response(c_p, sig, 0); + break; + + case ERTS_SIG_Q_OP_PROCESS_INFO: + handle_process_info(c_p, NULL, sig, next_nm_sig, 0); + break; + + case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE: + destroy_trace_info((ErtsSigTraceInfo *) sig); + break; + + default: + ERTS_INTERNAL_ERROR("Unknown signal"); + break; + } + + } while (cnt >= limit && *next_nm_sig); + + *redsp += cnt / ERTS_SIG_REDS_CNT_FACTOR; + + if (*next_nm_sig) + return 0; + + ASSERT(!c_p->sig_qs.nmsigs.next); + c_p->sig_qs.nmsigs.last = NULL; + (void) erts_atomic32_read_band_nob(&c_p->state, + ~ERTS_PSFLG_SIG_Q); + return !0; +} + +#ifdef USE_VM_PROBES +# define ERTS_CLEAR_SEQ_TOKEN(MP) \ + ERL_MESSAGE_TOKEN((MP)) = ((ERL_MESSAGE_DT_UTAG((MP)) != NIL) \ + ? am_have_dt_utag : NIL) +#else +# define ERTS_CLEAR_SEQ_TOKEN(MP) \ + ERL_MESSAGE_TOKEN((MP)) = NIL +#endif + +static ERTS_INLINE void +clear_seq_trace_token(ErtsMessage *sig) +{ + if (ERTS_SIG_IS_MSG((ErtsSignal *) sig)) + ERTS_CLEAR_SEQ_TOKEN(sig); + else { + Uint tag; + Uint16 op, type; + + tag = ((ErtsSignal *) sig)->common.tag; + type = ERTS_PROC_SIG_TYPE(tag); + op = ERTS_PROC_SIG_OP(tag); + + switch (op) { + + case ERTS_SIG_Q_OP_EXIT: + case ERTS_SIG_Q_OP_EXIT_LINKED: + case ERTS_SIG_Q_OP_MONITOR_DOWN: + switch (type) { + case ERTS_SIG_Q_TYPE_GEN_EXIT: + ERTS_CLEAR_SEQ_TOKEN(sig); + break; + case ERTS_LNK_TYPE_PORT: + case ERTS_LNK_TYPE_PROC: + case ERTS_LNK_TYPE_DIST_PROC: + case ERTS_MON_TYPE_PORT: + case ERTS_MON_TYPE_PROC: + case ERTS_MON_TYPE_DIST_PROC: + case ERTS_MON_TYPE_NODE: + break; + default: + ERTS_INTERNAL_ERROR("Unexpected sig type"); + break; + } + break; + + case ERTS_SIG_Q_OP_PERSISTENT_MON_MSG: + ERTS_CLEAR_SEQ_TOKEN(sig); + break; + + case ERTS_SIG_Q_OP_MONITOR: + case ERTS_SIG_Q_OP_DEMONITOR: + case ERTS_SIG_Q_OP_LINK: + case ERTS_SIG_Q_OP_UNLINK: + case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE: + break; + + default: + ERTS_INTERNAL_ERROR("Unknown signal"); + break; + } + } +} + +void +erts_proc_sig_clear_seq_trace_tokens(Process *c_p) +{ + ASSERT(erts_thr_progress_is_blocking()); + erts_proc_sig_fetch(c_p); + ERTS_FOREACH_SIG_PRIVQS(c_p, sig, clear_seq_trace_token(sig)); +} + +Uint +erts_proc_sig_signal_size(ErtsSignal *sig) +{ + Eterm tag; + Uint16 type; + int op; + Uint size = 0; + + ASSERT(sig); + ASSERT(ERTS_SIG_IS_NON_MSG(sig)); + + tag = sig->common.tag; + type = ERTS_PROC_SIG_TYPE(tag); + op = ERTS_PROC_SIG_OP(tag); + + switch (op) { + case ERTS_SIG_Q_OP_EXIT: + case ERTS_SIG_Q_OP_EXIT_LINKED: + case ERTS_SIG_Q_OP_MONITOR_DOWN: + switch (type) { + case ERTS_SIG_Q_TYPE_GEN_EXIT: + size = ((ErtsMessage *) sig)->hfrag.alloc_size; + size *= sizeof(Eterm); + size += sizeof(ErtsMessage) - sizeof(Eterm); + break; + case ERTS_LNK_TYPE_PORT: + case ERTS_LNK_TYPE_PROC: + case ERTS_LNK_TYPE_DIST_PROC: + size = erts_link_size((ErtsLink *) sig); + break; + case ERTS_MON_TYPE_PORT: + case ERTS_MON_TYPE_PROC: + case ERTS_MON_TYPE_DIST_PROC: + case ERTS_MON_TYPE_NODE: + size = erts_monitor_size((ErtsMonitor *) sig); + default: + ERTS_INTERNAL_ERROR("Unexpected sig type"); + break; + } + break; + + case ERTS_SIG_Q_OP_PERSISTENT_MON_MSG: + case ERTS_SIG_Q_OP_IS_ALIVE: + size = ((ErtsMessage *) sig)->hfrag.alloc_size; + size *= sizeof(Eterm); + size += sizeof(ErtsMessage) - sizeof(Eterm); + break; + + case ERTS_SIG_Q_OP_DEMONITOR: + if (type == ERTS_SIG_Q_TYPE_DIST_PROC_DEMONITOR) { + size = NC_HEAP_SIZE(((ErtsSigDistProcDemonitor *) sig)->ref); + size--; + size *= sizeof(Eterm); + size += sizeof(ErtsSigDistProcDemonitor); + break; + } + /* Fall through... */ + + case ERTS_SIG_Q_OP_MONITOR: + size = erts_monitor_size((ErtsMonitor *) sig); + break; + + case ERTS_SIG_Q_OP_UNLINK: + if (type == ERTS_SIG_Q_TYPE_DIST_LINK) { + size = NC_HEAP_SIZE(((ErtsSigDistLinkOp *) sig)->remote); + size--; + size *= sizeof(Eterm); + size += sizeof(ErtsSigDistLinkOp); + break; + } + /* Fall through... */ + + case ERTS_SIG_Q_OP_LINK: + size = erts_link_size((ErtsLink *) sig); + break; + + case ERTS_SIG_Q_OP_GROUP_LEADER: { + ErtsSigGroupLeader *sgl = (ErtsSigGroupLeader *) sig; + size = size_object(sgl->group_leader); + size += size_object(sgl->ref); + size *= sizeof(Eterm); + size += sizeof(ErtsSigGroupLeader) - sizeof(Eterm); + break; + } + + case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE: + size = sizeof(ErtsSigTraceInfo); + break; + + case ERTS_SIG_Q_OP_PROCESS_INFO: { + ErtsProcessInfoSig *pisig = (ErtsProcessInfoSig *) sig; + size = sizeof(ErtsProcessInfoSig); + size += (pisig->len - 1) * sizeof(int); + break; + } + + default: + ERTS_INTERNAL_ERROR("Unknown signal"); + break; + } + + return size; +} + +int +erts_proc_sig_receive_helper(Process *c_p, + int fcalls, + int neg_o_reds, + ErtsMessage **msgpp, + int *get_outp) +{ + ErtsMessage *msgp; + int reds, consumed_reds, left_reds, max_reds; + + /* + * Called from the loop-rec instruction when receive + * has reached end of inner (private) queue. This function + * tries to move more messages into the inner queue + * for the receive to handle. This by, processing the + * middle (private) queue and/or moving signals from + * the outer (public) queue into the middle queue. + * + * If this function succeeds in making more messages + * available in the inner queue, *msgpp points to next + * message. If *msgpp is set to NULL when: + * -- process became exiting. *get_outp is set to a + * value greater than zero. + * -- process needs to yield. *get_outp is set to a + * value less than zero. + * -- no more messages exist in any of the queues. + * *get_outp is set to zero and the message queue + * lock remains locked. This so the process can + * make its way to the appropriate wait instruction + * without racing with new incoming messages. + */ + + ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN); + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + ASSERT(!*msgpp); + + left_reds = fcalls - neg_o_reds; + consumed_reds = 0; + + while (!0) { + erts_aint32_t state; + + if (!c_p->sig_qs.cont) { + + consumed_reds += 4; + left_reds -= 4; + erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ); + erts_proc_sig_fetch(c_p); + /* + * Messages may have been moved directly to + * inner queue... + */ + msgp = PEEK_MESSAGE(c_p); + if (msgp) { + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); + *get_outp = 0; + *msgpp = msgp; + return consumed_reds; + } + + if (!c_p->sig_qs.cont) { + /* + * No messages! Return with message queue + * locked and let the process continue + * to wait instruction... + */ + *get_outp = 0; + *msgpp = NULL; + return consumed_reds; + } + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); + + if (left_reds <= 0) { + *get_outp = -1; /* yield */ + *msgpp = NULL; + + ASSERT(consumed_reds >= (fcalls - neg_o_reds)); + return consumed_reds; + } + + /* handle newly arrived signals... */ + } + + reds = ERTS_SIG_HANDLE_REDS_MAX_PREFERED; +#ifdef DEBUG + /* test that it works also with very few reds */ + max_reds = left_reds; + if (reds > left_reds) + reds = left_reds; +#else + /* At least work preferred amount of reds... */ + max_reds = left_reds; + if (max_reds < reds) + max_reds = reds; +#endif + (void) erts_proc_sig_handle_incoming(c_p, &state, &reds, + max_reds, !0); + consumed_reds += reds; + left_reds -= reds; + /* we may have exited by an incoming signal... */ + if (state & ERTS_PSFLG_EXITING) { + /* + * Process need to schedule out in order + * to terminate. Prepare this a bit... + */ + ASSERT(c_p->flags & F_DELAY_GC); + + c_p->flags &= ~F_DELAY_GC; + c_p->arity = 0; + c_p->current = NULL; + *get_outp = 1; + *msgpp = NULL; + return consumed_reds; + } + + msgp = PEEK_MESSAGE(c_p); + if (msgp) { + *get_outp = 0; + *msgpp = msgp; + return consumed_reds; + } + + if (left_reds <= 0) { + *get_outp = -1; /* yield */ + *msgpp = NULL; + + ASSERT(consumed_reds >= (fcalls - neg_o_reds)); + return consumed_reds; + } + + ASSERT(!c_p->sig_qs.cont); + /* Go fetch again... */ + } +} + +static int +handle_trace_change_state(Process *c_p, + ErtsSigRecvTracing *tracing, + Uint16 type, + ErtsMessage *sig, + ErtsMessage ***next_nm_sig) +{ + ErtsSigTraceInfo *trace_info = (ErtsSigTraceInfo *) sig; + ErtsMessage **next = *next_nm_sig; + int msgs_active, old_msgs_active = !!tracing->messages.active; + + ASSERT(sig == *next); + + erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + + ERTS_TRACE_FLAGS(c_p) |= trace_info->flags_on; + ERTS_TRACE_FLAGS(c_p) &= ~trace_info->flags_off; + if (is_value(trace_info->tracer)) + erts_tracer_replace(&c_p->common, trace_info->tracer); + + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + + remove_nm_sig(c_p, sig, next_nm_sig); + destroy_trace_info(trace_info); + /* + * Adjust tracing state according to modifications made by + * the trace info signal... + */ + adjust_tracing_state(c_p, tracing, 0); + msgs_active = !!tracing->messages.active; + + if (old_msgs_active ^ msgs_active) { + if (msgs_active) { + ASSERT(!tracing->messages.next); + tracing->messages.next = next; + } + else { + ASSERT(tracing->messages.next); + tracing->messages.next = NULL; + } + } + + ASSERT(!msgs_active || tracing->messages.next); + + return msgs_active; +} + +static void +getting_unlinked(Process *c_p, Eterm unlinker) +{ + trace_proc(c_p, ERTS_PROC_LOCK_MAIN, c_p, + am_getting_unlinked, unlinker); +} + +static void +getting_linked(Process *c_p, Eterm linker) +{ + trace_proc(c_p, ERTS_PROC_LOCK_MAIN, c_p, + am_getting_linked, linker); +} + +static ERTS_INLINE void +handle_message_enqueued_tracing(Process *c_p, + ErtsSigRecvTracing *tracing, + ErtsMessage *msg) +{ + ASSERT(ERTS_SIG_IS_INTERNAL_MSG(msg)); + +#if defined(USE_VM_PROBES) + if (tracing->messages.vm_probes && DTRACE_ENABLED(message_queued)) { + Sint tok_label = 0; + Sint tok_lastcnt = 0; + Sint tok_serial = 0; + Sint len = erts_proc_sig_privqs_len(c_p); + Eterm seq_trace_token = ERL_MESSAGE_TOKEN(msg); + + if (seq_trace_token != NIL && is_tuple(seq_trace_token)) { + tok_label = SEQ_TRACE_T_DTRACE_LABEL(seq_trace_token); + tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token)); + tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token)); + } + /* Message intentionally not passed... */ + DTRACE6(message_queued, + tracing->messages.receiver_name, + size_object(ERL_MESSAGE_TERM(msg)), + len, /* This is NOT message queue len, but its something... */ + tok_label, tok_lastcnt, tok_serial); + } +#endif + + if (tracing->messages.receive_trace && tracing->messages.event->on) { + ASSERT(IS_TRACED(c_p)); + trace_receive(c_p, + ERL_MESSAGE_FROM(msg), + ERL_MESSAGE_TERM(msg), + tracing->messages.event); + } +} + +static int +handle_msg_tracing(Process *c_p, ErtsSigRecvTracing *tracing, + ErtsMessage ***next_nm_sig) +{ + ErtsMessage **next_sig, *sig; + int cnt = 0, limit = ERTS_PROC_SIG_TRACE_COUNT_LIMIT; + + ASSERT(tracing->messages.next); + next_sig = tracing->messages.next; + sig = *next_sig; + + /* + * Receive tracing active. Handle all messages + * until next non-message signal... + */ + + while (sig && ERTS_SIG_IS_MSG(sig)) { + if (cnt > limit) { + tracing->messages.next = next_sig; + return -1; /* Yield... */ + } + if (ERTS_SIG_IS_EXTERNAL_MSG(sig)) { + cnt++; + if (!erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN, + sig, 0)) { + /* Bad dist message; remove it... */ + remove_mq_m_sig(c_p, sig, next_sig, next_nm_sig); + sig = *next_sig; + continue; + } + } + handle_message_enqueued_tracing(c_p, tracing, sig); + cnt++; + + next_sig = &(*next_sig)->next; + sig = *next_sig; + } + + tracing->messages.next = next_sig; + + if (!sig) { + ASSERT(!*next_nm_sig); + return 1; /* end... */ + } + + ASSERT(*next_nm_sig); + ASSERT(**next_nm_sig == sig); + + /* Next signal a non-message signal... */ + ASSERT(ERTS_SIG_IS_NON_MSG(sig)); + + /* + * Return and handle the non-message signal... + */ + + return 0; +} + +Uint +erts_proc_sig_prep_msgq_for_inspection(Process *c_p, + Process *rp, + ErtsProcLocks rp_locks, + int info_on_self, + ErtsMessageInfo *mip) +{ + Uint tot_heap_size; + ErtsMessage *mp, **mpp; + Sint i; + int self_on_heap; + + /* + * Prepare the message queue (inner signal queue) + * for inspection by process_info(). + * + * - Decode all messages on external format + * - Remove all corrupt dist messages from queue + * - Save pointer to, and heap size need of each + * message in the mip array. + * - Return total heap size need for all messages + * that needs to be copied. + * + */ + + ASSERT(!info_on_self || c_p == rp); + + self_on_heap = info_on_self && !(c_p->flags & F_OFF_HEAP_MSGQ); + + tot_heap_size = 0; + i = 0; + mpp = &rp->sig_qs.first; + mp = rp->sig_qs.first; + while (mp) { + Eterm msg = ERL_MESSAGE_TERM(mp); + + mip[i].size = 0; + + if (ERTS_SIG_IS_EXTERNAL_MSG(mp)) { + /* decode it... */ + if (mp->data.attached) + erts_decode_dist_message(rp, rp_locks, mp, !0); + + msg = ERL_MESSAGE_TERM(mp); + + if (is_non_value(msg)) { + ErtsMessage *bad_mp = mp; + /* + * Bad distribution message; remove + * it from the queue... + */ + ASSERT(!mp->data.attached); + + ASSERT(*mpp == bad_mp); + + remove_iq_m_sig(rp, mp, mpp); + + mp = *mpp; + + bad_mp->next = NULL; + erts_cleanup_messages(bad_mp); + continue; + } + } + + ASSERT(is_value(msg)); + + if (is_not_immed(msg) && (!self_on_heap || mp->data.attached)) { + Uint sz = size_object(msg); + mip[i].size = sz; + tot_heap_size += sz; + } + + mip[i].msgp = mp; + i++; + mpp = &mp->next; + mp = mp->next; + } + + ASSERT(c_p->sig_qs.len == i); + + return tot_heap_size; +} + +static ERTS_INLINE void +move_msg_to_heap(Process *c_p, ErtsMessage *mp) +{ + /* + * We leave not yet decoded distribution messages + * as they are in the queue since it is not + * possible to determine a maximum size until + * actual decoding... + * + * We also leave combined messages as they are... + */ + if (ERTS_SIG_IS_INTERNAL_MSG(mp) + && mp->data.attached + && mp->data.attached != ERTS_MSG_COMBINED_HFRAG) { + ErlHeapFragment *bp; + + bp = erts_message_to_heap_frag(mp); + + if (bp->next) + erts_move_multi_frags(&c_p->htop, &c_p->off_heap, bp, + mp->m, ERL_MESSAGE_REF_ARRAY_SZ, 0); + else + erts_copy_one_frag(&c_p->htop, &c_p->off_heap, bp, + mp->m, ERL_MESSAGE_REF_ARRAY_SZ); + + mp->data.heap_frag = NULL; + free_message_buffer(bp); + } +} + +void +erts_proc_sig_move_msgs_to_heap(Process *c_p) +{ + ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p, 0); + + ERTS_FOREACH_SIG_PRIVQS(c_p, sig, move_msg_to_heap(c_p, sig)); + + ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p, 0); +} + + +BIF_RETTYPE +erts_internal_dirty_process_handle_signals_1(BIF_ALIST_1) +{ + erts_aint32_t state, dirty, noproc; + int busy; + Process *rp; + + if (BIF_P != erts_dirty_process_signal_handler + && BIF_P != erts_dirty_process_signal_handler_high + && BIF_P != erts_dirty_process_signal_handler_max) + BIF_ERROR(BIF_P, EXC_NOTSUP); + + if (is_not_internal_pid(BIF_ARG_1)) + BIF_RET(am_false); + + rp = erts_proc_lookup_raw(BIF_ARG_1); + if (!rp) + BIF_RET(am_noproc); + + state = erts_atomic32_read_nob(&rp->state); + dirty = (state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)); + if (!dirty) + BIF_RET(am_normal); + + busy = erts_proc_trylock(rp, ERTS_PROC_LOCK_MAIN) == EBUSY; + + state = erts_atomic32_read_mb(&rp->state); + noproc = (state & ERTS_PSFLG_FREE); + dirty = (state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)); + + if (busy) { + if (noproc) + BIF_RET(am_noproc); + if (dirty) + BIF_RET(am_more); /* try again... */ + BIF_RET(am_normal); /* will handle signals itself... */ + } + else { + erts_aint32_t state; + int done; + Eterm res = am_false; + int reds = 0; + + if (noproc) + res = am_noproc; + else if (!dirty) + res = am_normal; /* will handle signals itself... */ + else { + reds = ERTS_BIF_REDS_LEFT(BIF_P); + done = erts_proc_sig_handle_incoming(rp, &state, &reds, + reds, 0); + if (done || (state & ERTS_PSFLG_EXITING)) + res = am_ok; + else + res = am_more; + } + + erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); + + if (reds) + BUMP_REDS(BIF_P, reds); + + BIF_RET(res); + } +} + +static void +debug_foreach_sig_heap_frags(ErlHeapFragment *hfrag, + void (*oh_func)(ErlOffHeap *, void *), + void *arg) +{ + ErlHeapFragment *hf = hfrag; + while (hf) { + oh_func(&(hf->off_heap), arg); + hf = hf->next; + } +} + +static void +debug_foreach_sig_fake_oh(Eterm term, + void (*oh_func)(ErlOffHeap *, void *), + void *arg) +{ + if (is_external(term)) { + ErlOffHeap oh; + oh.overhead = 0; + oh.first = ((struct erl_off_heap_header *) + (char *) external_thing_ptr(term)); + ASSERT(!oh.first->next); + oh_func(&oh, arg); + } + +} + +void +erts_proc_sig_debug_foreach_sig(Process *c_p, + void (*msg_func)(ErtsMessage *, void *), + void (*oh_func)(ErlOffHeap *, void *), + void (*mon_func)(ErtsMonitor *, void *), + void (*lnk_func)(ErtsLink *, void *), + void *arg) +{ + ErtsMessage *queue[] = {c_p->sig_qs.first, c_p->sig_qs.cont, c_p->sig_inq.first}; + int qix; + + for (qix = 0; qix < sizeof(queue)/sizeof(queue[0]); qix++) { + ErtsMessage *sig; + for (sig = queue[qix]; sig; sig = sig->next) { + + if (ERTS_SIG_IS_MSG(sig)) + msg_func(sig, arg); + else { + Eterm tag; + Uint16 type; + int op; + + ASSERT(sig); + ASSERT(ERTS_SIG_IS_NON_MSG(sig)); + + tag = ((ErtsSignal *) sig)->common.tag; + type = ERTS_PROC_SIG_TYPE(tag); + op = ERTS_PROC_SIG_OP(tag); + + switch (op) { + + case ERTS_SIG_Q_OP_EXIT: + case ERTS_SIG_Q_OP_EXIT_LINKED: + case ERTS_SIG_Q_OP_MONITOR_DOWN: + switch (type) { + case ERTS_SIG_Q_TYPE_GEN_EXIT: + debug_foreach_sig_heap_frags(&sig->hfrag, oh_func, arg); + break; + case ERTS_LNK_TYPE_PORT: + case ERTS_LNK_TYPE_PROC: + case ERTS_LNK_TYPE_DIST_PROC: + lnk_func((ErtsLink *) sig, arg); + break; + case ERTS_MON_TYPE_PORT: + case ERTS_MON_TYPE_PROC: + case ERTS_MON_TYPE_DIST_PROC: + case ERTS_MON_TYPE_NODE: + mon_func((ErtsMonitor *) sig, arg); + break; + default: + ERTS_INTERNAL_ERROR("Unexpected sig type"); + break; + } + break; + + case ERTS_SIG_Q_OP_PERSISTENT_MON_MSG: + debug_foreach_sig_heap_frags(&sig->hfrag, oh_func, arg); + break; + + case ERTS_SIG_Q_OP_DEMONITOR: + if (type == ERTS_SIG_Q_TYPE_DIST_PROC_DEMONITOR) { + debug_foreach_sig_fake_oh(((ErtsSigDistProcDemonitor *) sig)->ref, + oh_func, arg); + break; + } + /* Fall through... */ + + case ERTS_SIG_Q_OP_MONITOR: + mon_func((ErtsMonitor *) sig, arg); + break; + + case ERTS_SIG_Q_OP_UNLINK: + if (type == ERTS_SIG_Q_TYPE_DIST_LINK) { + debug_foreach_sig_fake_oh(((ErtsSigDistLinkOp *) sig)->remote, + oh_func, arg); + break; + } + /* Fall through... */ + + case ERTS_SIG_Q_OP_LINK: + lnk_func((ErtsLink *) sig, arg); + break; + + case ERTS_SIG_Q_OP_GROUP_LEADER: { + ErtsSigGroupLeader *sgl = (ErtsSigGroupLeader *) sig; + oh_func(&sgl->oh, arg); + break; + } + + case ERTS_SIG_Q_OP_IS_ALIVE: + case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE: + case ERTS_SIG_Q_OP_PROCESS_INFO: + break; + + default: + ERTS_INTERNAL_ERROR("Unknown signal"); + break; + } + + } + } + } +} + +#ifdef ERTS_PROC_SIG_HARD_DEBUG + +static void +chk_eterm(Process *c_p, int privq, ErtsMessage *mp, Eterm term) +{ + ErlHeapFragment *bp; + Eterm *ptr = NULL; + + switch (primary_tag(term)) { + case TAG_PRIMARY_IMMED1: + return; + case TAG_PRIMARY_LIST: + ptr = list_val(term); + ERTS_ASSERT(!is_header(CAR(ptr))); + ERTS_ASSERT(!is_header(CDR(ptr))); + break; + case TAG_PRIMARY_BOXED: + ptr = boxed_val(term); + ERTS_ASSERT(is_header(*ptr)); + break; + case TAG_PRIMARY_HEADER: + default: + ERTS_INTERNAL_ERROR("Not valid term"); + break; + } + + if (erts_is_literal(term, ptr)) + return; + + for (bp = erts_message_to_heap_frag(mp); bp; bp = bp->next) { + if (bp->mem <= ptr && ptr < bp->mem + bp->used_size) + return; + } + + ASSERT(erts_dbg_within_proc(ptr, c_p, NULL)); +} + +static Sint +proc_sig_hdbg_check_queue(Process *proc, + int privq, + ErtsMessage **sig_next, + ErtsMessage **sig_last, + ErtsMessage **sig_nm_next, + ErtsMessage **sig_nm_last, + ErtsSigRecvTracing *tracing, + int *found_saved_last_p, + erts_aint32_t sig_psflg) +{ + ErtsMessage **next, *sig, **nm_next, **nm_last; + int last_nm_sig_found, nm_sigs = 0, found_next_trace = 0, + found_save = 0, last_sig_found = 0, found_saved_last = 0; + Sint msg_len = 0; + ErtsMessage **next_trace = tracing ? tracing->messages.next : NULL; + ErtsMessage **save = proc->sig_qs.save; + ErtsMessage **saved_last = proc->sig_qs.saved_last; + + if (!privq) { + ErtsSignal *sig = (ErtsSignal *) *sig_next; + if (sig->common.tag == ERTS_PROC_SIG_MSGQ_LEN_OFFS_MARK) { + + } + } + + nm_next = sig_nm_next; + nm_last = sig_nm_last; + next = sig_next; + sig = *sig_next; + + last_nm_sig_found = !nm_last; + if (last_nm_sig_found) + ERTS_ASSERT(!nm_next); + else + ERTS_ASSERT(nm_next); + + while (1) { + ErtsSignal *nm_sig; + + if (next == sig_last) { + ASSERT(!*next); + last_sig_found = 1; + } + + if (next == save) + found_save = 1; + + if (next == saved_last) + found_saved_last = 1; + + if (next == next_trace) { + found_next_trace = 1; + ERTS_ASSERT(nm_sigs == 0); + } + + while (sig && ERTS_SIG_IS_MSG(sig)) { + int i; + if (ERTS_SIG_IS_EXTERNAL_MSG(sig)) + i = 1; + else + i = 0; + for (; i < ERL_MESSAGE_REF_ARRAY_SZ; i++) + chk_eterm(proc, privq, sig, sig->m[i]); + + msg_len++; + next = &sig->next; + sig = sig->next; + + if (next == sig_last) { + ASSERT(!*next); + last_sig_found = 1; + } + + if (next == save) + found_save = 1; + + if (next == saved_last) + found_saved_last = 1; + + if (next == next_trace) { + found_next_trace = 1; + ERTS_ASSERT(nm_sigs == 0); + } + } + + if (!sig) + break; + + nm_sig = (ErtsSignal *) sig; + + if (nm_sig->common.tag == ERTS_PROC_SIG_MSGQ_LEN_OFFS_MARK) { + ERTS_ASSERT(!privq); + ERTS_ASSERT(sig == *sig_next); + } + else { + nm_sigs++; + + ERTS_ASSERT(!last_nm_sig_found); + ERTS_ASSERT(ERTS_SIG_IS_NON_MSG(sig)); + + ERTS_ASSERT(nm_next == next); + + if (nm_last == next) { + ASSERT(!nm_sig->common.specific.next); + last_nm_sig_found = 1; + } + + nm_next = nm_sig->common.specific.next; + + } + + next = &nm_sig->common.next; + sig = nm_sig->common.next; + + } + + if (!privq) { + /* outer queue */ + ERTS_ASSERT(!found_save); + ERTS_ASSERT(!found_saved_last); + } + else if (privq > 0) { + /* middle queue */ + ERTS_ASSERT(!next_trace || found_next_trace); + ERTS_ASSERT(!found_save); + if (!found_saved_last_p) { + ERTS_ASSERT(!found_saved_last + || (proc->flags & F_DEFERRED_SAVED_LAST)); + } + else { + if (*found_saved_last_p) { + ERTS_ASSERT(!found_saved_last); + ERTS_ASSERT(!(proc->flags & F_DEFERRED_SAVED_LAST)); + } + else if (saved_last) { + ERTS_ASSERT(found_saved_last); + ERTS_ASSERT(proc->flags & F_DEFERRED_SAVED_LAST); + } + *found_saved_last_p |= found_saved_last; + } + } + else { + /* inner queue */ + ERTS_ASSERT(!found_next_trace); + ERTS_ASSERT(nm_sigs == 0); + ERTS_ASSERT(found_save); + ERTS_ASSERT(!saved_last + || (found_saved_last + || (proc->flags & F_DEFERRED_SAVED_LAST))); + if (found_saved_last_p) + *found_saved_last_p |= found_saved_last; + } + + ERTS_ASSERT(last_nm_sig_found); + ERTS_ASSERT(last_sig_found); + + if (sig_psflg != ERTS_PSFLG_FREE) { + erts_aint32_t state = erts_atomic32_read_nob(&proc->state); + ERTS_ASSERT(nm_sigs ? !!(state & sig_psflg) : !(state & sig_psflg)); + } + + return msg_len; +} + +void +erts_proc_sig_hdbg_check_priv_queue(Process *p, int qlock, char *what, char *file, int line) +{ + int found_saved_last = 0; + Sint len, len1, len2; + ERTS_LC_ASSERT(erts_thr_progress_is_blocking() + || ERTS_PROC_IS_EXITING(p) + || (ERTS_PROC_LOCK_MAIN + & erts_proc_lc_my_proc_locks(p))); + len1 = proc_sig_hdbg_check_queue(p, + -1, + &p->sig_qs.first, + p->sig_qs.last, + NULL, + NULL, + NULL, + &found_saved_last, + ERTS_PSFLG_FREE); + len2 = proc_sig_hdbg_check_queue(p, + 1, + &p->sig_qs.cont, + p->sig_qs.cont_last, + p->sig_qs.nmsigs.next, + p->sig_qs.nmsigs.last, + NULL, + &found_saved_last, + ERTS_PSFLG_SIG_Q); + if (p->sig_qs.saved_last) + ERTS_ASSERT(found_saved_last); + len = proc_sig_privqs_len(p, qlock); + ERTS_ASSERT(len == len1 + len2); +} + +void +erts_proc_sig_hdbg_check_in_queue(Process *p, char *what, char *file, int line) +{ + Sint len; + ERTS_LC_ASSERT(erts_thr_progress_is_blocking() + || ERTS_PROC_IS_EXITING(p) + || (ERTS_PROC_LOCK_MSGQ + & erts_proc_lc_my_proc_locks(p))); + len = proc_sig_hdbg_check_queue(p, + 0, + &p->sig_inq.first, + p->sig_inq.last, + p->sig_inq.nmsigs.next, + p->sig_inq.nmsigs.last, + NULL, + NULL, + ERTS_PSFLG_SIG_IN_Q); + ASSERT(p->sig_inq.len == len); (void)len; +} + +#endif /* ERTS_PROC_SIG_HARD_DEBUG */ diff --git a/erts/emulator/beam/erl_proc_sig_queue.h b/erts/emulator/beam/erl_proc_sig_queue.h new file mode 100644 index 0000000000..8b7cd35f61 --- /dev/null +++ b/erts/emulator/beam/erl_proc_sig_queue.h @@ -0,0 +1,884 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Process signal queue implementation. + * + * Currently the following signals are handled: + * - Messages + * - Exit + * - Monitor + * - Demonitor + * - Monitor down + * - Persistent monitor message + * - Link + * - Unlink + * - Group leader + * - Is process alive + * - Process info request + * - Trace change + * + * The signal queue consists of three parts: + * - Outer queue (sig_inq field in process struct) + * - Middle queue (sig_qs field in process struct) + * - Inner queue (sig_qs field in process struct) + * + * Incoming signals are placed in the outer queue + * by other processes, ports, or by the runtime system + * itself. This queue is protected by the msgq process + * lock and may be accessed by any other entity. While + * a signal is located in the outer queue, it is still + * in transit between sender and receiver. + * + * The middle and the inner queues are private to the + * receiving process and can only be accessed while + * holding the main process lock. The signal changes + * from being in transit to being received while in + * the middle queue. Non-message signals are handled + * immediately upon reception while message signals + * are moved into the inner queue. + * + * In the outer and middle queues both message signals + * and non-message signals are mixed. Signals in these + * queues are referenced using two single linked lists. + * One single linked list that go through all signals + * in the queue and another single linked list that + * goes through only non-message signals. The list + * through the non-message signals is used for fast + * access to these signals in the middle queue, since + * these should be handled immediately upon reception. + * + * The inner queue consists only of one single linked + * list through the message signals. A receive + * expression can only operate on messages once they + * have entered the inner queue. + * + * Author: Rickard Green + */ + +#ifndef ERTS_PROC_SIG_QUEUE_H_TYPE__ +#define ERTS_PROC_SIG_QUEUE_H_TYPE__ + +#if 0 +# define ERTS_PROC_SIG_HARD_DEBUG +#endif +#if 0 +# define ERTS_PROC_SIG_HARD_DEBUG_SIGQ_MSG_LEN +#endif + +struct erl_mesg; + +typedef struct { + struct erl_mesg *next; + union { + struct erl_mesg **next; + void *attachment; + } specific; + Eterm tag; +} ErtsSignalCommon; + +#define ERTS_SIG_HANDLE_REDS_MAX_PREFERED (CONTEXT_REDS/40) + +#ifdef ERTS_PROC_SIG_HARD_DEBUG +# define ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE(P) \ + ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE__((P), "") +# define ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(P, QL) \ + ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__((P), (QL), "") +# define ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE__(P, What) \ + erts_proc_sig_hdbg_check_in_queue((P), (What), __FILE__, __LINE__) +# define ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__(P, QL, What) \ + erts_proc_sig_hdbg_check_priv_queue((P), (QL), (What), __FILE__, __LINE__) +struct process; +void erts_proc_sig_hdbg_check_priv_queue(struct process *c_p, int qlock, + char *what, char *file, int line); +void erts_proc_sig_hdbg_check_in_queue(struct process *c_p, char *what, + char *file, int line); +#else +# define ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE(P) +# define ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(P, QL) +# define ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE__(P, What) +#define ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__(P, QL, What) +#endif + +#endif + +#if !defined(ERTS_PROC_SIG_QUEUE_H__) && !defined(ERTS_PROC_SIG_QUEUE_TYPE_ONLY) +#define ERTS_PROC_SIG_QUEUE_H__ + +#define ERTS_SIG_Q_OP_BITS 8 +#define ERTS_SIG_Q_OP_SHIFT 0 +#define ERTS_SIG_Q_OP_MASK ((1 << ERTS_SIG_Q_OP_BITS) - 1) + +#define ERTS_SIG_Q_TYPE_BITS 8 +#define ERTS_SIG_Q_TYPE_SHIFT ERTS_SIG_Q_OP_BITS +#define ERTS_SIG_Q_TYPE_MASK ((1 << ERTS_SIG_Q_TYPE_BITS) - 1) + +#define ERTS_SIG_Q_NON_X_BITS__ (_HEADER_ARITY_OFFS \ + + ERTS_SIG_Q_OP_BITS \ + + ERTS_SIG_Q_TYPE_BITS) + +#define ERTS_SIG_Q_XTRA_BITS (32 - ERTS_SIG_Q_NON_X_BITS__) +#define ERTS_SIG_Q_XTRA_SHIFT (ERTS_SIG_Q_OP_BITS \ + + ERTS_SIG_Q_TYPE_BITS) +#define ERTS_SIG_Q_XTRA_MASK ((1 << ERTS_SIG_Q_XTRA_BITS) - 1) + + +#define ERTS_PROC_SIG_OP(Tag) \ + ((int) (_unchecked_thing_arityval((Tag)) \ + >> ERTS_SIG_Q_OP_SHIFT) & ERTS_SIG_Q_OP_MASK) + +#define ERTS_PROC_SIG_TYPE(Tag) \ + ((Uint16) (_unchecked_thing_arityval((Tag)) \ + >> ERTS_SIG_Q_TYPE_SHIFT) & ERTS_SIG_Q_TYPE_MASK) + +#define ERTS_PROC_SIG_XTRA(Tag) \ + ((Uint32) (_unchecked_thing_arityval((Tag)) \ + >> ERTS_SIG_Q_XTRA_SHIFT) & ERTS_SIG_Q_XTRA_MASK) + +#define ERTS_PROC_SIG_MAKE_TAG(Op, Type, Xtra) \ + (ASSERT(0 <= (Xtra) && (Xtra) <= ERTS_SIG_Q_XTRA_MASK), \ + _make_header((((Type) & ERTS_SIG_Q_TYPE_MASK) \ + << ERTS_SIG_Q_TYPE_SHIFT) \ + | (((Op) & ERTS_SIG_Q_OP_MASK) \ + << ERTS_SIG_Q_OP_SHIFT) \ + | (((Xtra) & ERTS_SIG_Q_XTRA_MASK) \ + << ERTS_SIG_Q_XTRA_SHIFT), \ + _TAG_HEADER_EXTERNAL_PID)) + + +/* + * ERTS_SIG_Q_OP_MSGQ_LEN_OFFS_MARK is not an actual + * operation. We keep it at the top of the OP range, + * larger than ERTS_SIG_Q_OP_MAX. + */ +#define ERTS_SIG_Q_OP_MSGQ_LEN_OFFS_MARK ERTS_SIG_Q_OP_MASK + +#define ERTS_PROC_SIG_MSGQ_LEN_OFFS_MARK \ + ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_MSGQ_LEN_OFFS_MARK,0,0) + +struct dist_entry_; + +/* + * Send operations of currently supported process signals follow... + */ + +/** + * + * @brief Send an exit signal to a process. + * + * + * @param[in] c_p Pointer to process struct of + * currently executing process. + * + * @param[in] from Identifier of sender. + * + * @param[in] to Identifier of local process + * to send signal to. + * + * @param[in] reason Exit reason. + * + * @param[in] token Seq trace token. + * + * @param[in] normal_kills If non-zero, also normal exit + * reason will kill the receiver + * if it is not trapping exit. + * + */ +void +erts_proc_sig_send_exit(Process *c_p, Eterm from, Eterm to, + Eterm reason, Eterm token, int normal_kills); + +/** + * + * @brief Send an exit signal due to broken link to a process. + * + * + * @param[in] c_p Pointer to process struct of + * currently executing process. + * + * @param[in] from Identifier of sender. + * + * @param[in] lnk Pointer to link structure + * from the sending side. It + * should contain information + * about receiver. + * + * @param[in] reason Exit reason. + * + * @param[in] token Seq trace token. + * + */ +void +erts_proc_sig_send_link_exit(Process *c_p, Eterm from, ErtsLink *lnk, + Eterm reason, Eterm token); + +/** + * + * @brief Send an link signal to a process. + * + * + * @param[in] c_p Pointer to process struct of + * currently executing process. + * + * @param[in] to Identifier of receiver. + * + * @param[in] lnk Pointer to link structure to + * insert on receiver side. + * + * @return A non-zero value if + * signal was successfully + * sent. If a zero, value + * the signal was not sent + * due to the receiver not + * existing. The sender + * needs to deallocate the + * link structure. + * + */ +int +erts_proc_sig_send_link(Process *c_p, Eterm to, ErtsLink *lnk); + +/** + * + * @brief Send an unlink signal to a process. + * + * + * @param[in] c_p Pointer to process struct of + * currently executing process. + * + * @param[in] lnk Pointer to link structure from + * the sending side. It should + * contain information about + * receiver. + */ +void +erts_proc_sig_send_unlink(Process *c_p, ErtsLink *lnk); + +/** + * + * @brief Send an exit signal due to broken link to a process. + * + * This function is used instead of erts_proc_sig_send_link_exit() + * when the signal arrives via the distribution and + * no link structure is available. + * + * @param[in] dep Distribution entry of channel + * that the signal arrived on. + * + * @param[in] from Identifier of sender. + * + * @param[in] to Identifier of receiver. + * + * @param[in] reason Exit reason. + * + * @param[in] token Seq trace token. + * + */ +void +erts_proc_sig_send_dist_link_exit(struct dist_entry_ *dep, + Eterm from, Eterm to, + Eterm reason, Eterm token); + +/** + * + * @brief Send an unlink signal to a process. + * + * This function is used instead of erts_proc_sig_send_unlink() + * when the signal arrives via the distribution and + * no link structure is available. + * + * @param[in] dep Distribution entry of channel + * that the signal arrived on. + * + * @param[in] from Identifier of sender. + * + * @param[in] to Identifier of receiver. + * + */ +void +erts_proc_sig_send_dist_unlink(struct dist_entry_ *dep, + Eterm from, Eterm to); + +/** + * + * @brief Send a monitor down signal to a process. + * + * @param[in] mon Pointer to target monitor + * structure from the sending + * side. It should contain + * information about receiver. + * + * @param[in] reason Exit reason. + * + */ +void +erts_proc_sig_send_monitor_down(ErtsMonitor *mon, Eterm reason); + +/** + * + * @brief Send a demonitor signal to a process. + * + * @param[in] mon Pointer to origin monitor + * structure from the sending + * side. It should contain + * information about receiver. + * + * @param[in] reason Exit reason. + * + */ +void +erts_proc_sig_send_demonitor(ErtsMonitor *mon); + +/** + * + * @brief Send a monitor signal to a process. + * + * @param[in] mon Pointer to target monitor + * structure to insert on + * receiver side. + * + * @param[in] to Identifier of receiver. + * + * @return A non-zero value if + * signal was successfully + * sent. If a zero, value + * the signal was not sent + * due to the receiver not + * existing. The sender + * needs to deallocate the + * monitor structure. + * + */ +int +erts_proc_sig_send_monitor(ErtsMonitor *mon, Eterm to); + +/** + * + * @brief Send a monitor down signal to a process. + * + * This function is used instead of erts_proc_sig_send_monitor_down() + * when the signal arrives via the distribution and + * no link structure is available. + * + * @param[in] dep Pointer to distribution entry + * of channel that the signal + * arrived on. + * + * @param[in] ref Reference identifying the monitor. + * + * @param[in] from Identifier of sender. + * + * @param[in] to Identifier of receiver. + * + * @param[in] reason Exit reason. + * + */ +void +erts_proc_sig_send_dist_monitor_down(DistEntry *dep, Eterm ref, + Eterm from, Eterm to, + Eterm reason); + +/** + * + * @brief Send a demonitor signal to a process. + * + * This function is used instead of erts_proc_sig_send_demonitor() + * when the signal arrives via the distribution and + * no monitor structure is available. + * + * @param[in] to Identifier of receiver. + * + * @param[in] ref Reference identifying the monitor. + * + */ +void +erts_proc_sig_send_dist_demonitor(Eterm to, Eterm ref); + +/** + * + * @brief Send a persistent monitor triggered signal to a process. + * + * Used by monitors that are not auto disabled such as for + * example 'time_offset' monitors. + * + * @param[in] type Monitor type. + * + * @param[in] key Monitor key. + * + * @param[in] from Identifier of sender. + * + * @param[in] to Identifier of receiver. + * + * @param[in] msg Message template. + * + * @param[in] msg_sz Heap size of message template. + * + */ +void +erts_proc_sig_send_persistent_monitor_msg(Uint16 type, Eterm key, + Eterm from, Eterm to, + Eterm msg, Uint msg_sz); + +/** + * + * @brief Send a trace change signal to a process. + * + * @param[in] to Identifier of receiver. + * + * @param[in] on Trace flags to enable. + * + * @param[in] off Trace flags to disable. + * + * @param[in] tracer Tracer to set. If the non-value, + * tracer will not be changed. + * + */ +void +erts_proc_sig_send_trace_change(Eterm to, Uint on, Uint off, + Eterm tracer); + +/** + * + * @brief Send a group leader signal to a process. + * + * Set group-leader of receiving process. If sent locally, + * a response message '{Ref, Result}' is sent to the original + * sender when performed where Ref is the reference passed + * as 'ref' argument, and Result is either 'true' or 'badarg'. + * + * @param[in] c_p Pointer to process struct of + * currently executing process. + * NULL if signal arrived via + * distribution. + * + * @param[in] to Identifier of receiver. + * + * @param[in] gl Identifier of new group leader. + * + * @param[in] ref Reference to use in response + * message to locally sending + * process (i.e., c_p when c_p + * is non-null). + * + */ +void +erts_proc_sig_send_group_leader(Process *c_p, Eterm to, Eterm gl, + Eterm ref); + +/** + * + * @brief Send an 'is process alive' signal to a process. + * + * A response message '{Ref, Result}' is sent to the + * sender when performed where Ref is the reference passed + * as 'ref' argument, and Result is either 'true' or 'false'. + * + * @param[in] c_p Pointer to process struct of + * currently executing process. + * NULL if signal arrived via + * distribution. + * + * @param[in] to Identifier of receiver. + * + * @param[in] ref Reference to use in response + * message to the sending + * process (i.e., c_p). + * + */ +void +erts_proc_sig_send_is_alive_request(Process *c_p, Eterm to, + Eterm ref); + +/** + * + * @brief Send a 'process info request' signal to a process. + * + * A response message '{Ref, Result}' is sent to the + * sender when performed where Ref is the reference passed + * as 'ref' argument, and Result corresponds to return result + * from erlang:process_info/[1,2]. + * + * @param[in] c_p Pointer to process struct of + * currently executing process. + * NULL if signal arrived via + * distribution. + * + * @param[in] to Identifier of receiver. + * + * @param[in] item_ix Info index array to pass to + * erts_process_info() + * + * @param[in] len Lenght of info index array + * + * @param[in] need_msgq_len Non-zero if message queue + * length is needed; otherwise, + * zero. If non-zero, sig_qs.len + * will be set to correspond + * to the message queue length + * before call to + * erts_process_info() + * + * @param[in] flags Flags to pass to + * erts_process_info() + * + * @param[in] reserve_size Heap size that is known to + * be needed. May not be correct + * though. + * + * @param[in] ref Reference to use in response + * message to the sending + * process (i.e., c_p). + * + */ +int +erts_proc_sig_send_process_info_request(Process *c_p, + Eterm to, + int *item_ix, + int len, + int need_msgq_len, + int flags, + Uint reserve_size, + Eterm ref); + + +/* + * End of send operations of currently supported process signals. + */ + + +/** + * + * @brief Handle incoming signals. + * + * Called by an ordinary scheduler in order to handle incoming + * signals for a process. The work is done on the middle part + * of the signal queue. The maximum amount of signals handled + * is limited by the amount of reductions given when calling. + * Note that a reduction does not necessarily map to a signal. + * + * @param[in] c_p Pointer to process struct of + * currently executing process. + * + * @param[out] statep Pointer to process state after + * signal handling. May not be NULL. + * + * @param[in,out] redsp Pointer to an integer containing + * reductions. On input, the amount + * of preferred reductions to be + * used by the call. On output, the + * amount of reductions consumed. + * + * @param[in] max_reds Absolute maximum of reductions + * to use. If the process cannot + * make progress after the preferred + * amount of reductions has been + * consumed, signal handling may + * proceed up to a maximum of + * 'max_reds' in order to make + * the process able to proceed + * with other tasks after handling + * has finished. + * + * @param[in] local_only If is zero, new signals may be + * fetched from the outer queue and + * put in the middle queue before + * signal handling is performed. If + * non-zero, no new signals will be + * fetched before handling begins. + * + * @return Returns a non-zero value, when + * no more signals to handle in the + * middle queue remain. A zero + * return value means that there + * remains signals in the middle + * queue. + */ +int +erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep, + int *redsp, int max_reds, + int local_only); + +/** + * + * @brief Handle remaining signals for an exiting process + * + * Called as part of termination of a process. It will handle + * remaining signals. + * + * @param[in] c_p Pointer to process struct of + * currently executing process. + * + * @param[in,out] redsp Pointer to an integer containing + * reductions. On input, the amount + * of maximum reductions to be + * used by the call. On output, the + * amount of reductions consumed. + * + * @return Returns a non-zero value, when + * no more signals to handle in the + * middle queue remain. A zero + * return value means that there + * remains signals in the middle + * queue. + */ +int +erts_proc_sig_handle_exit(Process *c_p, int *redsp); + +/** + * + * @brief Helper for loop_rec instruction. + * + * This function should only be called from the loop_rec + * instruction (or equivalents). It is called when loop_rec + * reach the end of the inner queue (which is the only + * part of the signal queue that receive is allowed to + * operate on). When called, this function tries to make + * more messages available in the inner queue. This by + * fetching signals from the outer queue to the middle + * queue and/or processing signals in the middle queue. + * + * @param[in] c_p Pointer to process struct of + * currently executing process. + * + * @param[in] fcalls Content of FCALLS in + * process_main() + * + * @param[in] neg_o_reds Content of neg_o_reds in + * process_main() + * + * @param[out] msgpp Pointer to pointer to next + * available message to process. + * If *msgpp == NULL, no more + * messages are available. + * + * @param[out] get_outp Pointer to an integer + * indicating how to respond + * if no more messages are + * available (msgpp). If integer + * is set to zero, loop_rec + * should jump to an appropriate + * wait instruction. If zero, + * the message queue lock remain + * locked since the test for + * more messages was done. + * If the integer is set to a + * value larger that zero, the + * process exited. If the integer + * is set to a value less than + * zero, the process is required + * to yield. + * + * + * @return The amount of reductions + * consumed. + * + */ +int +erts_proc_sig_receive_helper(Process *c_p, int fcalls, + int neg_o_reds, ErtsMessage **msgpp, + int *get_outp); + +/** + * + * @brief Fetch signals from the outer queue + * + * Fetches signals from outer queue and places them in the + * middle queue ready for signal handling. If the middle + * queue is empty, only message signals were present in the + * outer queue, and no receive tracing has been enabled on + * the process, the middle queue is bypassed and messages + * are delivered directly to the inner queue instead. + * + * @param[in] c_p Pointer to process struct of + * currently executing process. + * @returns Amount of message signals in + * inner plus middle signal + * queues after fetch completed + * (NOT the message queue + * length). + */ +ERTS_GLB_INLINE Sint erts_proc_sig_fetch(Process *p); + +/** + * + * @brief Get amount of messages in private queues + * + * @param[in] c_p Pointer to process struct of + * currently executing process. + * + * @returns Amount of message signals in + * inner plus middle signal + * queues after fetch completed + * (NOT the message queue + * length). + */ +Sint +erts_proc_sig_privqs_len(Process *c_p); + + +/* SVERK: Doc me up! */ +erts_aint32_t +erts_enqueue_signals(Process *rp, ErtsMessage *first, + ErtsMessage **last, ErtsMessage **last_next, + Uint msg_cnt, + erts_aint32_t in_state); + +/* SVERK: Doc me up! */ +void +erts_proc_sig_send_pending(ErtsSchedulerData* esdp); + + +typedef struct { + Uint size; + ErtsMessage *msgp; +} ErtsMessageInfo; + +/** + * + * @brief Prepare signal queue for inspection by process_info() + * + * + * @param[in] c_p Pointer to process struct of + * currently executing process. + * + * @param[in] rp Pointer to process struct of + * process to inspect. + * + * @param[in] rp_locks Process locks held on 'rp'. + * + * @param[in] info_on_self Integer set to non-zero value + * if caller is inspecting itself; + * otherwise, zero. + * + * @param[in] mip Pointer to array of + * ErtsMessageInfo structures. + */ +Uint erts_proc_sig_prep_msgq_for_inspection(Process *c_p, + Process *rp, + ErtsProcLocks rp_locks, + int info_on_self, + ErtsMessageInfo *mip); + +/** + * + * @brief Move message data of messages in private queues to heap + * + * Move message data of messages in private queues to the heap. + * This is part of GC of processes that uses on-heap message + * data. + * + * @param[in] c_p Pointer to process struct of + * currently executing process. + * + */ +void erts_proc_sig_move_msgs_to_heap(Process *c_p); + +/** + * + * @brief Size of signal in bytes. + * + * @param[in] sig Signal to inspect. + * + */ +Uint erts_proc_sig_signal_size(ErtsSignal *sig); + + +/** + * + * @brief Clear seq trace tokens on all signals + * + * Assumes thread progress has been blocked! + * + * @param[in] c_p Pointer to process + * + */ +void +erts_proc_sig_clear_seq_trace_tokens(Process *c_p); + +/** + * @brief Initialize this functionality + */ +void erts_proc_sig_queue_init(void); + +void +erts_proc_sig_debug_foreach_sig(Process *c_p, + void (*msg_func)(ErtsMessage *, void *), + void (*oh_func)(ErlOffHeap *, void *), + void (*mon_func)(ErtsMonitor *, void *), + void (*lnk_func)(ErtsLink *, void *), + void *arg); + +extern Process *erts_dirty_process_signal_handler; +extern Process *erts_dirty_process_signal_handler_high; +extern Process *erts_dirty_process_signal_handler_max; + +void erts_proc_sig_fetch__(Process *proc); +Sint erts_proc_sig_fetch_msgq_len_offs__(Process *proc); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE Sint +erts_proc_sig_fetch(Process *proc) +{ + Sint res = 0; + ErtsSignal *sig; + + ERTS_LC_ASSERT(erts_thr_progress_is_blocking() + || ERTS_PROC_IS_EXITING(proc) + || ((erts_proc_lc_my_proc_locks(proc) + & (ERTS_PROC_LOCK_MAIN + | ERTS_PROC_LOCK_MSGQ)) + == (ERTS_PROC_LOCK_MAIN + | ERTS_PROC_LOCK_MSGQ))); + + ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE(proc); + ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(proc, !0); + + sig = (ErtsSignal *) proc->sig_inq.first; + if (sig) { + if (ERTS_LIKELY(sig->common.tag != ERTS_PROC_SIG_MSGQ_LEN_OFFS_MARK)) + erts_proc_sig_fetch__(proc); + else + res = erts_proc_sig_fetch_msgq_len_offs__(proc); + } + + res += proc->sig_qs.len; + + ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(proc, !0); + +#ifdef ERTS_PROC_SIG_HARD_DEBUG_SIGQ_MSG_LEN + { + Sint len = 0; + ERTS_FOREACH_SIG_PRIVQS( + proc, mp, + { + if (ERTS_SIG_IS_MSG(mp)) + len++; + }); + ERTS_ASSERT(res == len); + } +#endif + + return res; +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* ERTS_PROC_SIG_QUEUE_H__ */ diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 0d02d10ac9..ad7ac27ac3 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2017. All Rights Reserved. + * Copyright Ericsson AB 1996-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,7 +36,6 @@ #include "erl_db.h" #include "dist.h" #include "beam_catches.h" -#include "erl_instrument.h" #include "erl_threads.h" #include "erl_binary.h" #include "beam_bp.h" @@ -53,6 +52,7 @@ #include "erl_nfunc_sched.h" #include "erl_check_io.h" #include "erl_poll.h" +#include "erl_proc_sig_queue.h" #define ERTS_CHECK_TIME_REDS CONTEXT_REDS #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0) @@ -171,11 +171,19 @@ int erts_dio_sched_thread_suggested_stack_size = -1; ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE]; #endif -static struct ErtsSchedBusyWait_ { +typedef struct { int aux_work; int tse; int sys_schedule; -} sched_busy_wait; +} ErtsBusyWaitParams; + +static ErtsBusyWaitParams sched_busy_wait_params[ERTS_SCHED_TYPE_LAST + 1]; + +static ERTS_INLINE ErtsBusyWaitParams * +sched_get_busy_wait_params(ErtsSchedulerData *esdp) +{ + return &sched_busy_wait_params[esdp->type]; +} int erts_disable_proc_not_running_opt; @@ -431,7 +439,8 @@ typedef enum { ERTS_PSTT_CLA, /* Copy Literal Area */ ERTS_PSTT_COHMQ, /* Change off heap message queue */ ERTS_PSTT_FTMQ, /* Flush trace msg queue */ - ERTS_PSTT_ETS_FREE_FIXATION + ERTS_PSTT_ETS_FREE_FIXATION, + ERTS_PSTT_PRIO_SIG /* Elevate prio on signal management */ } ErtsProcSysTaskType; #define ERTS_MAX_PROC_SYS_TASK_ARGS 2 @@ -579,7 +588,6 @@ dbg_chk_aux_work_val(erts_aint32_t value) valid |= ERTS_SSI_AUX_WORK_CNCLD_TMRS; valid |= ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR; valid |= ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP; - valid |= ERTS_SSI_AUX_WORK_PENDING_EXITERS; #if HAVE_ERTS_MSEG valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK; #endif @@ -601,7 +609,6 @@ dbg_chk_aux_work_val(erts_aint32_t value) #define ERTS_DBG_CHK_SSI_AUX_WORK(SSI) #endif -static void do_handle_pending_exiters(ErtsProcList *); static void wake_scheduler(ErtsRunQueue *rq); #if defined(ERTS_ENABLE_LOCK_CHECK) @@ -666,8 +673,6 @@ erts_pre_init_process(void) = "MISC_THR_PRGR"; erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MISC_IX] = "MISC"; - erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_PENDING_EXITERS_IX] - = "PENDING_EXITERS"; erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_SET_TMO_IX] = "SET_TMO"; erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX] @@ -2502,6 +2507,8 @@ handle_yield(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) yield |= erts_handle_yielded_ets_all_request(awdp->esdp, &awdp->yield.ets_all); + yield |= erts_handle_yielded_alcu_blockscan(awdp->esdp, + &awdp->yield.alcu_blockscan); /* * Other yielding operations... @@ -2531,27 +2538,6 @@ handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiti static ERTS_INLINE erts_aint32_t -handle_pending_exiters(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) -{ - ErtsProcList *pnd_xtrs; - ErtsRunQueue *rq; - - rq = awdp->esdp->run_queue; - unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_PENDING_EXITERS); - - erts_runq_lock(rq); - pnd_xtrs = rq->procs.pending_exiters; - rq->procs.pending_exiters = NULL; - erts_runq_unlock(rq); - - if (erts_proclist_fetch(&pnd_xtrs, NULL)) - do_handle_pending_exiters(pnd_xtrs); - - return aux_work & ~ERTS_SSI_AUX_WORK_PENDING_EXITERS; -} - - -static ERTS_INLINE erts_aint32_t handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) { unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_SET_TMO); @@ -2632,9 +2618,6 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC, handle_misc_aux_work); - HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_PENDING_EXITERS, - handle_pending_exiters); - HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_SET_TMO, handle_setup_aux_work_timer); @@ -3288,7 +3271,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_runq_unlock(rq); - spincount = sched_busy_wait.tse; + spincount = sched_get_busy_wait_params(esdp)->tse; if (ERTS_SCHEDULER_IS_DIRTY(esdp)) dirty_sched_wall_time_change(esdp, working = 0); @@ -3390,7 +3373,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) } flgs = sched_prep_cont_spin_wait(ssi); - spincount = sched_busy_wait.aux_work; + spincount = sched_get_busy_wait_params(esdp)->aux_work; if (!(flgs & ERTS_SSI_FLG_WAITING)) { ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); @@ -5276,24 +5259,35 @@ typedef enum { #define ERTS_WAKEUP_OTHER_FIXED_INC_LEGACY (CONTEXT_REDS/10) -static struct { +typedef struct { ErtsSchedWakeupOtherThreshold threshold; ErtsSchedWakeupOtherType type; int limit; int dec_shift; int dec_mask; void (*check)(ErtsRunQueue *rq, Uint32 flags); -} wakeup_other; +} ErtsWakeupOtherParams; + +static ErtsWakeupOtherParams sched_wakeup_other_params[ERTS_SCHED_TYPE_LAST + 1]; + +static ERTS_INLINE ErtsWakeupOtherParams * +runq_get_wakeup_other_params(ErtsRunQueue *rq) +{ + ErtsSchedulerData *esdp = rq->scheduler; + return &sched_wakeup_other_params[esdp->type]; +} static void wakeup_other_check(ErtsRunQueue *rq, Uint32 flags) { + ErtsWakeupOtherParams *wo_params = runq_get_wakeup_other_params(rq); int wo_reds = rq->wakeup_other_reds; + if (wo_reds) { int left_len = erts_atomic32_read_dirty(&rq->len) - 1; if (left_len < 1) { - int wo_reduce = wo_reds << wakeup_other.dec_shift; - wo_reduce &= wakeup_other.dec_mask; + int wo_reduce = wo_reds << wo_params->dec_shift; + wo_reduce &= wo_params->dec_mask; rq->wakeup_other -= wo_reduce; if (rq->wakeup_other < 0) rq->wakeup_other = 0; @@ -5301,7 +5295,7 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags) else { rq->wakeup_other += (left_len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC); - if (rq->wakeup_other > wakeup_other.limit) { + if (rq->wakeup_other > wo_params->limit) { if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { if (rq->waiting) { wake_dirty_scheduler(rq); @@ -5323,42 +5317,44 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags) } static void -wakeup_other_set_limit(void) +wakeup_other_set_limit(ErtsWakeupOtherParams *params) { - switch (wakeup_other.threshold) { + switch (params->threshold) { case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH: - wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH; - wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_HIGH; + params->limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH; + params->dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_HIGH; break; case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH: - wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH; - wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_HIGH; + params->limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH; + params->dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_HIGH; break; case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM: - wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM; - wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_MEDIUM; + params->limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM; + params->dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_MEDIUM; break; case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW: - wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_LOW; - wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_LOW; + params->limit = ERTS_WAKEUP_OTHER_LIMIT_LOW; + params->dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_LOW; break; case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW: - wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW; - wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_LOW; + params->limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW; + params->dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_LOW; break; } - if (wakeup_other.dec_shift < 0) - wakeup_other.dec_mask = (1 << (sizeof(wakeup_other.dec_mask)*8 - + wakeup_other.dec_shift)) - 1; + + if (params->dec_shift < 0) + params->dec_mask = (1 << (sizeof(params->dec_mask)*8 + + params->dec_shift)) - 1; else { - wakeup_other.dec_mask = 0; - wakeup_other.dec_mask = ~wakeup_other.dec_mask; + params->dec_mask = 0; + params->dec_mask = ~params->dec_mask; } } static void wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags) { + ErtsWakeupOtherParams *wo_params = runq_get_wakeup_other_params(rq); int wo_reds = rq->wakeup_other_reds; if (wo_reds) { erts_aint32_t len = erts_atomic32_read_dirty(&rq->len); @@ -5367,7 +5363,7 @@ wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags) if (rq->wakeup_other < 0) rq->wakeup_other = 0; } - else if (rq->wakeup_other < wakeup_other.limit) + else if (rq->wakeup_other < wo_params->limit) rq->wakeup_other += len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC_LEGACY; else { if (flags & ERTS_RUNQ_FLG_PROTECTED) @@ -5383,23 +5379,23 @@ wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags) } static void -wakeup_other_set_limit_legacy(void) +wakeup_other_set_limit_legacy(ErtsWakeupOtherParams *params) { - switch (wakeup_other.threshold) { + switch (params->threshold) { case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH: - wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH_LEGACY; + params->limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH_LEGACY; break; case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH: - wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH_LEGACY; + params->limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH_LEGACY; break; case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM: - wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM_LEGACY; + params->limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM_LEGACY; break; case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW: - wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_LOW_LEGACY; + params->limit = ERTS_WAKEUP_OTHER_LIMIT_LOW_LEGACY; break; case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW: - wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW_LEGACY; + params->limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW_LEGACY; break; } } @@ -5407,15 +5403,21 @@ wakeup_other_set_limit_legacy(void) static void set_wakeup_other_data(void) { - switch (wakeup_other.type) { - case ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT: - wakeup_other.check = wakeup_other_check; - wakeup_other_set_limit(); - break; - case ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY: - wakeup_other.check = wakeup_other_check_legacy; - wakeup_other_set_limit_legacy(); - break; + ErtsSchedType type; + + for (type = ERTS_SCHED_TYPE_FIRST; type <= ERTS_SCHED_TYPE_LAST; type++) { + ErtsWakeupOtherParams *params = &sched_wakeup_other_params[type]; + + switch (params->type) { + case ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT: + params->check = wakeup_other_check; + wakeup_other_set_limit(params); + break; + case ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY: + params->check = wakeup_other_check_legacy; + wakeup_other_set_limit_legacy(params); + break; + } } } @@ -5470,56 +5472,64 @@ runq_supervisor(void *unused) void erts_early_init_scheduling(int no_schedulers) { + ErtsSchedType type; + aux_work_timeout_early_init(no_schedulers); - wakeup_other.threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM; - wakeup_other.type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT; - sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM; - sched_busy_wait.tse = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM - * ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT); - sched_busy_wait.aux_work = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM - * ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_MEDIUM); + + for (type = ERTS_SCHED_TYPE_FIRST; type <= ERTS_SCHED_TYPE_LAST; type++) { + erts_sched_set_wakeup_other_threshold(type, "medium"); + erts_sched_set_wakeup_other_type(type, "default"); + + erts_sched_set_busy_wait_threshold(type, "medium"); + } + + erts_sched_set_busy_wait_threshold(ERTS_SCHED_DIRTY_CPU, "short"); + erts_sched_set_busy_wait_threshold(ERTS_SCHED_DIRTY_IO, "short"); } int -erts_sched_set_wakeup_other_thresold(char *str) -{ - ErtsSchedWakeupOtherThreshold threshold; - if (sys_strcmp(str, "very_high") == 0) - threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH; - else if (sys_strcmp(str, "high") == 0) - threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH; - else if (sys_strcmp(str, "medium") == 0) - threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM; - else if (sys_strcmp(str, "low") == 0) - threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW; - else if (sys_strcmp(str, "very_low") == 0) - threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW; - else - return EINVAL; - wakeup_other.threshold = threshold; - set_wakeup_other_data(); +erts_sched_set_wakeup_other_threshold(ErtsSchedType sched_type, char *str) +{ + ErtsWakeupOtherParams *params = &sched_wakeup_other_params[sched_type]; + + if (sys_strcmp(str, "very_high") == 0) { + params->threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH; + } else if (sys_strcmp(str, "high") == 0) { + params->threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH; + } else if (sys_strcmp(str, "medium") == 0) { + params->threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM; + } else if (sys_strcmp(str, "low") == 0) { + params->threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW; + } else if (sys_strcmp(str, "very_low") == 0) { + params->threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW; + } else { + return EINVAL; + } + return 0; } int -erts_sched_set_wakeup_other_type(char *str) +erts_sched_set_wakeup_other_type(ErtsSchedType sched_type, char *str) { - ErtsSchedWakeupOtherType type; - if (sys_strcmp(str, "default") == 0) - type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT; - else if (sys_strcmp(str, "legacy") == 0) - type = ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY; - else - return EINVAL; - wakeup_other.type = type; + ErtsWakeupOtherParams *params = &sched_wakeup_other_params[sched_type]; + + if (sys_strcmp(str, "default") == 0) { + params->type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT; + } else if (sys_strcmp(str, "legacy") == 0) { + params->type = ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY; + } else { + return EINVAL; + } + return 0; } int -erts_sched_set_busy_wait_threshold(char *str) +erts_sched_set_busy_wait_threshold(ErtsSchedType sched_type, char *str) { - int sys_sched; - int aux_work_fact; + ErtsBusyWaitParams *params = &sched_busy_wait_params[sched_type]; + int aux_work_fact, sys_sched; if (sys_strcmp(str, "very_long") == 0) { sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_LONG; @@ -5549,9 +5559,9 @@ erts_sched_set_busy_wait_threshold(char *str) return EINVAL; } - sched_busy_wait.sys_schedule = sys_sched; - sched_busy_wait.tse = sys_sched*ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT; - sched_busy_wait.aux_work = sys_sched*aux_work_fact; + params->sys_schedule = sys_sched; + params->tse = sys_sched * ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT; + params->aux_work = sys_sched * aux_work_fact; return 0; } @@ -5689,6 +5699,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, esdp->ssi = ssi; esdp->current_process = NULL; esdp->current_port = NULL; + esdp->current_nif = NULL; esdp->virtual_reds = 0; esdp->cpu_id = -1; @@ -5704,6 +5715,12 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, esdp->io.out = (Uint64) 0; esdp->io.in = (Uint64) 0; + esdp->pending_signal.sig = NULL; + esdp->pending_signal.to = THE_NON_VALUE; +#ifdef DEBUG + esdp->pending_signal.dbg_from = NULL; +#endif + if (daww_ptr) { init_aux_work_data(&esdp->aux_work_data, esdp, *daww_ptr); *daww_ptr += daww_sz; @@ -5793,7 +5810,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_th rq->wakeup_other = 0; rq->wakeup_other_reds = 0; - rq->procs.pending_exiters = NULL; rq->procs.context_switches = 0; rq->procs.reductions = 0; @@ -6085,7 +6101,7 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio) proxy = prev_proxy; ASSERT(erts_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY); erts_atomic32_set_nob(&proxy->state, state); - (void) erts_set_runq_proc(proc, rq, &bound); + (void) erts_set_runq_proc(proxy, rq, &bound); } else { proxy = erts_alloc(ERTS_ALC_T_PROC, sizeof(Process)); @@ -6098,7 +6114,7 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio) } #endif erts_atomic32_init_nob(&proxy->state, state); - erts_init_runq_proc(proc, rq, bound); + erts_init_runq_proc(proxy, rq, bound); } proxy->common.id = proc->common.id; @@ -6328,11 +6344,36 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, int enqueue; /* < 0 -> use proxy */ ErtsRunQueue* runq; + ASSERT(!(state & (ERTS_PSFLG_DIRTY_IO_PROC + |ERTS_PSFLG_DIRTY_CPU_PROC)) + || (BeamIsOpCode(*p->i, op_call_nif) + || BeamIsOpCode(*p->i, op_apply_bif))); + + a = state; + + /* Clear activ-sys if needed... */ + while (1) { + n = e = a; + if (a & ERTS_PSFLG_ACTIVE_SYS) { + if (a & (ERTS_PSFLG_SIG_Q + | ERTS_PSFLG_SIG_IN_Q + | ERTS_PSFLG_SYS_TASKS)) + break; + /* Clear active-sys */ + n &= ~ERTS_PSFLG_ACTIVE_SYS; + } + a = erts_atomic32_cmpxchg_nob(&p->state, n, e); + if (a == e) { + a = n; + break; + } + } + if (!is_normal_sched) running_flgs = ERTS_PSFLG_DIRTY_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS; else { running_flgs = ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS; - if (state & ERTS_PSFLG_DIRTY_ACTIVE_SYS + if ((a & ERTS_PSFLG_DIRTY_ACTIVE_SYS) && (p->flags & (F_DELAY_GC|F_DISABLE_GC))) { /* * Delay dirty GC; will be enabled automatically @@ -6345,14 +6386,12 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, */ ASSERT(!(p->flags & (F_DIRTY_CLA | F_DIRTY_GC_HIBERNATE))); - state = erts_atomic32_read_band_nob(&p->state, - ~ERTS_PSFLG_DIRTY_ACTIVE_SYS); - state &= ~ERTS_PSFLG_DIRTY_ACTIVE_SYS; + a = erts_atomic32_read_band_nob(&p->state, + ~ERTS_PSFLG_DIRTY_ACTIVE_SYS); + a &= ~ERTS_PSFLG_DIRTY_ACTIVE_SYS; } } - a = state; - while (1) { n = e = a; @@ -6360,6 +6399,11 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, enqueue = ERTS_ENQUEUE_NOT; + ASSERT(((a & (ERTS_PSFLG_EXITING|ERTS_PSFLG_FREE)) + != ERTS_PSFLG_EXITING) + || ((a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) + == ERTS_PSFLG_ACTIVE)); + n &= ~running_flgs; if ((a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS)) || (a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) { @@ -6584,7 +6628,7 @@ change_proc_schedule_state(Process *p, } - *statep = a; + *statep = n; return enqueue; } @@ -6609,6 +6653,95 @@ erts_schedule_process(Process *p, erts_aint32_t state, ErtsProcLocks locks) schedule_process(p, state, locks); } +static ERTS_INLINE erts_aint32_t +active_sys_enqueue(Process *p, erts_aint32_t state, + erts_aint32_t enable_flags, int status_locked) +{ + /* + * This function may or may not be called with status locke held. + * It always returns without the status lock held! + */ + unsigned int prof_runnable_procs = erts_system_profile_flags.runnable_procs; + erts_aint32_t n, a = state, enq_prio = -1; + int slocked = status_locked; + int enqueue; /* < 0 -> use proxy */ + + /* Status lock prevents out of order "runnable proc" trace msgs */ + ERTS_LC_ASSERT(slocked || !(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p))); + ERTS_LC_ASSERT(!slocked || (ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p))); + + if (!prof_runnable_procs) { + if (slocked) { + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + slocked = 0; + } + } + else { + if (!slocked) { + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); + slocked = !0; + } + } + + ASSERT(!(state & ERTS_PSFLG_PROXY)); + + while (1) { + erts_aint32_t e; + n = e = a; + + if (a & ERTS_PSFLG_FREE) + goto cleanup; /* We don't want to schedule free processes... */ + + enqueue = ERTS_ENQUEUE_NOT; + n |= enable_flags; + n |= ERTS_PSFLG_ACTIVE_SYS; + if (!(a & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS))) + enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a); + a = erts_atomic32_cmpxchg_mb(&p->state, n, e); + if (a == e) + break; + if (a == n && enqueue == ERTS_ENQUEUE_NOT) + goto cleanup; + } + + if (prof_runnable_procs) { + + if (!(a & (ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) + && (!(a & ERTS_PSFLG_ACTIVE) || (a & ERTS_PSFLG_SUSPENDED))) { + /* We activated a prevously inactive process */ + profile_runnable_proc(p, am_active); + } + + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + slocked = 0; + } + + add2runq(enqueue, enq_prio, p, n, NULL); + +cleanup: + + if (slocked) + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + + ERTS_LC_ASSERT(!(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p))); + + return n; +} + +erts_aint32_t +erts_proc_sys_schedule(Process *p, erts_aint32_t state, erts_aint32_t enable_flag) +{ + /* We are not allowed to call this function with status lock held... */ + return active_sys_enqueue(p, state, enable_flag, 0); +} + static int schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, erts_aint32_t *fail_state_p) @@ -6616,19 +6749,16 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, int res; int locked; ErtsProcSysTaskQs *stqs, *free_stqs; - erts_aint32_t fail_state, state, a, n, enq_prio; - int enqueue; /* < 0 -> use proxy */ - unsigned int prof_runnable_procs; + erts_aint32_t fail_state, state; fail_state = *fail_state_p; res = 1; /* prepare for success */ st->next = st->prev = st; /* Prep for empty prio queue */ state = erts_atomic32_read_nob(&p->state); - prof_runnable_procs = erts_system_profile_flags.runnable_procs; locked = 0; free_stqs = NULL; - if (state & ERTS_PSFLG_ACTIVE_SYS) + if (state & ERTS_PSFLG_SYS_TASKS) stqs = NULL; else { alloc_qs: @@ -6648,6 +6778,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, state = erts_atomic32_read_nob(&p->state); if (state & fail_state) { + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); *fail_state_p = (state & fail_state); free_stqs = stqs; res = 0; @@ -6695,69 +6826,14 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, state = n; } - - a = state; - enq_prio = -1; - - /* Status lock prevents out of order "runnable proc" trace msgs */ - ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - - if (!prof_runnable_procs) { - erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - locked = 0; - } - - ASSERT(!(state & ERTS_PSFLG_PROXY)); - - while (1) { - erts_aint32_t e; - n = e = a; - - if (a & ERTS_PSFLG_FREE) - goto cleanup; /* We don't want to schedule free processes... */ - - enqueue = ERTS_ENQUEUE_NOT; - n |= ERTS_PSFLG_ACTIVE_SYS; - if (!(a & (ERTS_PSFLG_RUNNING - | ERTS_PSFLG_RUNNING_SYS - | ERTS_PSFLG_DIRTY_RUNNING - | ERTS_PSFLG_DIRTY_RUNNING_SYS))) - enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a); - a = erts_atomic32_cmpxchg_mb(&p->state, n, e); - if (a == e) - break; - if (a == n && enqueue == ERTS_ENQUEUE_NOT) - goto cleanup; - } - - if (prof_runnable_procs) { - - if (!(a & (ERTS_PSFLG_ACTIVE_SYS - | ERTS_PSFLG_RUNNING - | ERTS_PSFLG_RUNNING_SYS - | ERTS_PSFLG_DIRTY_RUNNING - | ERTS_PSFLG_DIRTY_RUNNING_SYS)) - && (!(a & ERTS_PSFLG_ACTIVE) || (a & ERTS_PSFLG_SUSPENDED))) { - /* We activated a prevously inactive process */ - profile_runnable_proc(p, am_active); - } - - erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - locked = 0; - } - - add2runq(enqueue, enq_prio, p, n, NULL); + /* active_sys_enqueue() always return with status lock unlocked */ + (void) active_sys_enqueue(p, state, ERTS_PSFLG_SYS_TASKS, locked); cleanup: - if (locked) - erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - if (free_stqs) proc_sys_task_queues_free(free_stqs); - ERTS_LC_ASSERT(!(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p))); - return res; } @@ -8265,6 +8341,7 @@ sched_thread_func(void *vesdp) ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL); #endif + erts_alcu_sched_spec_data_init(esdp); erts_ets_sched_spec_data_init(esdp); process_main(esdp->x_reg_array, esdp->f_reg_array); @@ -8536,11 +8613,13 @@ static ERTS_INLINE void cancel_suspend_of_suspendee(Process *p, ErtsProcLocks p_locks) { if (is_not_nil(p->suspendee)) { + ErtsMonitor *mon; + Eterm suspendee = p->suspendee; Process *rp; if (!(p_locks & ERTS_PROC_LOCK_STATUS)) erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); rp = erts_pid2proc(p, p_locks|ERTS_PROC_LOCK_STATUS, - p->suspendee, ERTS_PROC_LOCK_STATUS); + suspendee, ERTS_PROC_LOCK_STATUS); if (rp) { erts_resume(rp, ERTS_PROC_LOCK_STATUS); erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); @@ -8548,6 +8627,14 @@ cancel_suspend_of_suspendee(Process *p, ErtsProcLocks p_locks) if (!(p_locks & ERTS_PROC_LOCK_STATUS)) erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); p->suspendee = NIL; + + mon = erts_monitor_tree_lookup(p->suspend_monitors, + suspendee); + if (mon) { + erts_monitor_tree_delete(&p->suspend_monitors, + mon); + erts_monitor_suspend_destroy(erts_monitor_suspend(mon)); + } } } @@ -8706,8 +8793,7 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, { erts_aint32_t state; state = erts_atomic32_read_nob(&rp->state); - ASSERT((state & ERTS_PSFLG_PENDING_EXIT) - || !(state & ERTS_PSFLG_RUNNING)); + ASSERT(!(state & ERTS_PSFLG_RUNNING)); } #endif @@ -8762,7 +8848,7 @@ erts_pid2proc_nropt(Process *c_p, ErtsProcLocks c_p_locks, static ERTS_INLINE int do_bif_suspend_process(Process *c_p, - ErtsSuspendMonitor *smon, + ErtsMonitorSuspend *smon, Process *suspendee) { ASSERT(suspendee); @@ -8795,21 +8881,25 @@ handle_pend_bif_sync_suspend(Process *suspendee, suspender = erts_pid2proc(suspendee, suspendee_locks, suspender_pid, - ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); + ERTS_PROC_LOCK_STATUS); if (suspender) { + ErtsMonitorSuspend *smon; + ErtsMonitor *mon; + mon = erts_monitor_tree_lookup(suspender->suspend_monitors, + suspendee->common.id); + smon = erts_monitor_suspend(mon); + ASSERT(is_nil(suspender->suspendee)); - if (!suspendee_alive) - erts_delete_suspend_monitor(&suspender->suspend_monitors, - suspendee->common.id); + if (!suspendee_alive) { + if (mon) { + erts_monitor_tree_delete(&suspender->suspend_monitors, + mon); + erts_monitor_suspend_destroy(smon); + } + } else { #ifdef DEBUG - int res; -#endif - ErtsSuspendMonitor *smon; - smon = erts_lookup_suspend_monitor(suspender->suspend_monitors, - suspendee->common.id); -#ifdef DEBUG - res = + int res = #endif do_bif_suspend_process(suspendee, smon, suspendee); ASSERT(!smon || res != 0); @@ -8818,9 +8908,8 @@ handle_pend_bif_sync_suspend(Process *suspendee, /* suspender is suspended waiting for suspendee to suspend; resume suspender */ ASSERT(suspender != suspendee); - resume_process(suspender, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); - erts_proc_unlock(suspender, - ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); + resume_process(suspender, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(suspender, ERTS_PROC_LOCK_STATUS); } } @@ -8838,26 +8927,29 @@ handle_pend_bif_async_suspend(Process *suspendee, suspender = erts_pid2proc(suspendee, suspendee_locks, suspender_pid, - ERTS_PROC_LOCK_LINK); + ERTS_PROC_LOCK_STATUS); if (suspender) { + ErtsMonitorSuspend *smon; + ErtsMonitor *mon; + mon = erts_monitor_tree_lookup(suspender->suspend_monitors, + suspendee->common.id); + smon = erts_monitor_suspend(mon); ASSERT(is_nil(suspender->suspendee)); - if (!suspendee_alive) - erts_delete_suspend_monitor(&suspender->suspend_monitors, - suspendee->common.id); + if (!suspendee_alive) { + if (mon) { + erts_monitor_tree_delete(&suspender->suspend_monitors, + mon); + erts_monitor_suspend_destroy(smon); + } + } else { #ifdef DEBUG - int res; -#endif - ErtsSuspendMonitor *smon; - smon = erts_lookup_suspend_monitor(suspender->suspend_monitors, - suspendee->common.id); -#ifdef DEBUG - res = + int res = #endif - do_bif_suspend_process(suspendee, smon, suspendee); + do_bif_suspend_process(suspendee, smon, suspendee); ASSERT(!smon || res != 0); } - erts_proc_unlock(suspender, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(suspender, ERTS_PROC_LOCK_STATUS); } } @@ -8871,8 +8963,9 @@ suspend_process_2(BIF_ALIST_2) { Eterm res; Process* suspendee = NULL; - ErtsSuspendMonitor *smon; + ErtsMonitorSuspend *smon; ErtsProcLocks xlocks = (ErtsProcLocks) 0; + int created; /* Options and default values: */ int asynchronous = 0; @@ -8905,9 +8998,7 @@ suspend_process_2(BIF_ALIST_2) goto badarg; } - xlocks = ERTS_PROC_LOCK_LINK | (asynchronous - ? (ErtsProcLocks) 0 - : ERTS_PROC_LOCK_STATUS); + xlocks = ERTS_PROC_LOCK_STATUS; erts_proc_lock(BIF_P, xlocks); @@ -8918,15 +9009,14 @@ suspend_process_2(BIF_ALIST_2) if (!suspendee) goto no_suspendee; - smon = erts_add_or_lookup_suspend_monitor(&BIF_P->suspend_monitors, - BIF_ARG_1); - - /* ... but a little trickier with SMP support ... */ + smon = erts_monitor_suspend_tree_lookup_create(&BIF_P->suspend_monitors, + &created, + BIF_ARG_1); if (asynchronous) { /* --- Asynchronous suspend begin ---------------------------------- */ - ERTS_LC_ASSERT(ERTS_PROC_LOCK_LINK + ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(BIF_P)); ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS == erts_proc_lc_my_proc_locks(suspendee)); @@ -8968,9 +9058,9 @@ suspend_process_2(BIF_ALIST_2) else /* if (!asynchronous) */ { /* --- Synchronous suspend begin ----------------------------------- */ - ERTS_LC_ASSERT(((ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS) + ERTS_LC_ASSERT(((ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_STATUS) & erts_proc_lc_my_proc_locks(BIF_P)) - == (ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS)); + == (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_STATUS)); ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS == erts_proc_lc_my_proc_locks(suspendee)); @@ -9055,9 +9145,15 @@ suspend_process_2(BIF_ALIST_2) ERTS_BIF_PREP_ERROR(res, BIF_P, SYSTEM_LIMIT); goto do_return; - no_suspendee: - BIF_P->suspendee = NIL; - erts_delete_suspend_monitor(&BIF_P->suspend_monitors, BIF_ARG_1); + no_suspendee: { + ErtsMonitor *mon; + BIF_P->suspendee = NIL; + mon = erts_monitor_tree_lookup(BIF_P->suspend_monitors, BIF_ARG_1); + if (mon) { + erts_monitor_tree_delete(&BIF_P->suspend_monitors, mon); + erts_monitor_suspend_destroy(erts_monitor_suspend(mon)); + } + } badarg: ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); @@ -9084,15 +9180,17 @@ suspend_process_2(BIF_ALIST_2) BIF_RETTYPE resume_process_1(BIF_ALIST_1) { - ErtsSuspendMonitor *smon; + ErtsMonitor *mon; + ErtsMonitorSuspend *smon; Process *suspendee; int is_active; if (BIF_P->common.id == BIF_ARG_1) BIF_ERROR(BIF_P, BADARG); - erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); - smon = erts_lookup_suspend_monitor(BIF_P->suspend_monitors, BIF_ARG_1); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_STATUS); + mon = erts_monitor_tree_lookup(BIF_P->suspend_monitors, BIF_ARG_1); + smon = erts_monitor_suspend(mon); if (!smon) { /* No previous suspend or dead suspendee */ @@ -9109,20 +9207,17 @@ resume_process_1(BIF_ALIST_1) } else if (smon->active) { smon->active--; - ASSERT(smon->pending >= 0); + ASSERT(smon->pending == 0); is_active = 1; } else { /* No previous suspend or dead suspendee */ - goto error; + goto no_suspendee; } if (smon->active || smon->pending || !is_active) { /* Leave the suspendee as it is; just verify that it is still alive */ - suspendee = erts_pid2proc(BIF_P, - ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, - BIF_ARG_1, - 0); + suspendee = erts_proc_lookup(BIF_ARG_1); if (!suspendee) goto no_suspendee; @@ -9130,11 +9225,18 @@ resume_process_1(BIF_ALIST_1) else { /* Resume */ suspendee = erts_pid2proc(BIF_P, - ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, + ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS, BIF_ARG_1, ERTS_PROC_LOCK_STATUS); - if (!suspendee) + if (!suspendee) { + mon = erts_monitor_tree_lookup(BIF_P->suspend_monitors, BIF_ARG_1); + smon = erts_monitor_suspend(mon); + if (!mon) + goto error; goto no_suspendee; + } + + ASSERT(mon == erts_monitor_tree_lookup(BIF_P->suspend_monitors, BIF_ARG_1)); ASSERT(ERTS_PSFLG_SUSPENDED & erts_atomic32_read_nob(&suspendee->state)); @@ -9144,19 +9246,24 @@ resume_process_1(BIF_ALIST_1) erts_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); } - if (!smon->active && !smon->pending) - erts_delete_suspend_monitor(&BIF_P->suspend_monitors, BIF_ARG_1); + if (!smon->active && !smon->pending) { + ASSERT(mon); + erts_monitor_tree_delete(&BIF_P->suspend_monitors, mon); + erts_monitor_suspend_destroy(smon); + } - erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS); BIF_RET(am_true); no_suspendee: /* cleanup */ - erts_delete_suspend_monitor(&BIF_P->suspend_monitors, BIF_ARG_1); + ASSERT(mon); + erts_monitor_tree_delete(&BIF_P->suspend_monitors, mon); + erts_monitor_suspend_destroy(smon); error: - erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS); BIF_ERROR(BIF_P, BADARG); } @@ -9375,9 +9482,8 @@ erts_resume_processes(ErtsProcList *list) } Eterm -erts_get_process_priority(Process *p) +erts_get_process_priority(erts_aint32_t state) { - erts_aint32_t state = erts_atomic32_read_nob(&p->state); switch (ERTS_PSFLGS_GET_USR_PRIO(state)) { case PRIORITY_MAX: return am_max; case PRIORITY_HIGH: return am_high; @@ -9427,7 +9533,10 @@ erts_set_process_priority(Process *p, Eterm value) } max_qbit = 0; - if (a & ERTS_PSFLG_ACTIVE_SYS) + ASSERT((a & ERTS_PSFLG_SYS_TASKS) + ? !!p->sys_task_qs + : !p->sys_task_qs); + if (a & ERTS_PSFLG_SYS_TASKS) max_qbit |= p->sys_task_qs->qmask; if (a & ERTS_PSFLG_DELAYED_SYS) { ErtsProcSysTaskQs *qs; @@ -9450,8 +9559,8 @@ erts_set_process_priority(Process *p, Eterm value) aprio = PRIORITY_LOW; break; default: - ERTS_INTERNAL_ERROR("Invalid qmask"); - aprio = -1; + aprio = nprio; + break; } if (aprio > nprio) /* low value -> high prio */ @@ -9571,8 +9680,13 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) } else { is_normal_sched = !esdp; if (is_normal_sched) { - esdp = p->scheduler_data; + esdp = p->scheduler_data; ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); + + if (esdp->pending_signal.sig) { + ASSERT(esdp->pending_signal.dbg_from == p); + erts_proc_sig_send_pending(esdp); + } } else { ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)); @@ -9627,10 +9741,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) /* have to re-read state after taking lock */ state = erts_atomic32_read_nob(&p->state); - if (is_normal_sched && (state & ERTS_PSFLG_PENDING_EXIT)) - erts_handle_pending_exit(p, (ERTS_PROC_LOCK_MAIN - | ERTS_PROC_LOCK_TRACE - | ERTS_PROC_LOCK_STATUS)); if (p->pending_suspenders) handle_pending_suspend(p, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_TRACE @@ -9833,7 +9943,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) if (flags & ERTS_RUNQ_FLG_MISC_OP) exec_misc_ops(rq); - wakeup_other.check(rq, flags); + runq_get_wakeup_other_params(rq)->check(rq, flags); /* * Find a new port to run. @@ -9945,12 +10055,10 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) & ((state & (ERTS_PSFLG_SUSPENDED | ERTS_PSFLG_EXITING | ERTS_PSFLG_FREE - | ERTS_PSFLG_PENDING_EXIT | ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) != ERTS_PSFLG_SUSPENDED) - & (!(state & (ERTS_PSFLG_EXITING - | ERTS_PSFLG_PENDING_EXIT)) + & (!(state & ERTS_PSFLG_EXITING) | (!!is_normal_sched)) ); @@ -10042,7 +10150,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) goto sunlock_sched_out_proc; } if (state & (ERTS_PSFLG_ACTIVE_SYS - | ERTS_PSFLG_PENDING_EXIT | ERTS_PSFLG_EXITING)) { /* * IMPORTANT! We need to take care of @@ -10065,13 +10172,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) && (state & ERTS_PSFLG_DIRTY_IO_PROC))); } - if (state & ERTS_PSFLG_PENDING_EXIT) { - erts_handle_pending_exit(p, - ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); - state = erts_atomic32_read_nob(&p->state); - } - - erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); /* Clear tracer if it has been removed */ @@ -10093,24 +10193,47 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) } if (is_normal_sched) { - if (state & ERTS_PSFLG_RUNNING_SYS) { - /* - * GC is normally never delayed when a process - * is scheduled out, but might be when executing - * hand written beam assembly in - * prim_eval:'receive'. If GC is delayed we are - * not allowed to execute system tasks. - */ - if (!(p->flags & F_DELAY_GC)) { - int cost = execute_sys_tasks(p, &state, reds); - calls += cost; - reds -= cost; - if (reds <= 0) - goto sched_out_proc; - if (state & ERTS_PSFLGS_DIRTY_WORK) - goto sched_out_proc; + if (state & (ERTS_PSFLG_SIG_Q|ERTS_PSFLG_SIG_IN_Q)) { + int local_only = !!(p->flags & F_LOCAL_SIGS_ONLY); + if (!local_only || (state & ERTS_PSFLG_SIG_Q)) { + int sig_reds; + /* + * If we have dirty work scheduled we allow + * usage of all reductions since we need to + * handle all signals before doing dirty + * work... + */ + if (state & ERTS_PSFLGS_DIRTY_WORK) + sig_reds = reds; + else + sig_reds = ERTS_SIG_HANDLE_REDS_MAX_PREFERED; + (void) erts_proc_sig_handle_incoming(p, + &state, + &sig_reds, + sig_reds, + local_only); + reds -= sig_reds; + } } + if ((state & (ERTS_PSFLG_SYS_TASKS + | ERTS_PSFLG_EXITING)) == ERTS_PSFLG_SYS_TASKS) { + /* + * GC is normally never delayed when a process + * is scheduled out, but might be when executing + * hand written beam assembly in + * prim_eval:'receive'. If GC is delayed we are + * not allowed to execute system tasks. + */ + if (!(p->flags & F_DELAY_GC)) { + int cost = execute_sys_tasks(p, &state, reds); + calls += cost; + reds -= cost; + } + } + + if (reds <= 0 || (state & ERTS_PSFLGS_DIRTY_WORK)) + goto sched_out_proc; ASSERT(state & psflg_running_sys); ASSERT(!(state & psflg_running)); @@ -10120,7 +10243,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) if (((state & (ERTS_PSFLG_SUSPENDED | ERTS_PSFLG_ACTIVE)) != ERTS_PSFLG_ACTIVE) - && !(state & ERTS_PSFLG_EXITING)) { + & !(state & ERTS_PSFLG_EXITING)) { goto sched_out_proc; } @@ -10162,15 +10285,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); - /* Never run a suspended process */ -#ifdef DEBUG - { - erts_aint32_t dstate = erts_atomic32_read_nob(&p->state); - ASSERT(!(ERTS_PSFLG_SUSPENDED & dstate) - || (ERTS_PSFLG_DIRTY_RUNNING_SYS & dstate)); - } -#endif - ASSERT(erts_proc_read_refc(p) > 0); if (!(state & ERTS_PSFLG_EXITING) && ERTS_PTMR_IS_TIMED_OUT(p)) { @@ -10183,6 +10297,11 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) ERTS_PTMR_CLEAR(p); } + /* if exiting, we *shall* exit... */ + ASSERT(!(state & ERTS_PSFLG_EXITING) + || p->i == (BeamInstr *) beam_exit + || p->i == (BeamInstr *) beam_continue_exit); + #ifdef DEBUG if (is_normal_sched) { if (state & ERTS_PSFLGS_DIRTY_WORK) @@ -10198,6 +10317,18 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) ERTS_INTERNAL_ERROR("Executing normal code on dirty UNKNOWN scheduler"); } } + { + erts_aint32_t dstate = erts_atomic32_read_nob(&p->state); + + /* Never run a suspended process */ + ASSERT(!(ERTS_PSFLG_SUSPENDED & dstate) + || (ERTS_PSFLG_DIRTY_RUNNING_SYS & dstate)); + + /* Do not execute on the wrong type of scheduler... */ + ASSERT(is_normal_sched + ? !(dstate & ERTS_PSFLGS_DIRTY_WORK) + : !!(dstate & ERTS_PSFLGS_DIRTY_WORK)); + } #endif return p; @@ -10255,7 +10386,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, ASSERT(hp_start + hsz == hp); #endif - erts_queue_message(rp, rp_locks, mp, msg, c_p->common.id); + erts_queue_proc_message(c_p, rp, rp_locks, mp, msg); if (c_p == rp) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -10399,7 +10530,7 @@ fetch_sys_task(Process *c_p, erts_aint32_t state, int *qmaskp, int *priop) n |= (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET); if (!qmask) - n &= ~ERTS_PSFLG_ACTIVE_SYS; + n &= ~ERTS_PSFLG_SYS_TASKS; if (a == n) break; @@ -10419,6 +10550,8 @@ done: return st; } + +static void exit_permanent_prio_elevation(Process *c_p, erts_aint32_t state); static void save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio); static void save_dirty_task(Process *c_p, ErtsProcSysTask *st); @@ -10439,12 +10572,8 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) int st_prio; Eterm st_res; - if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) { - if (state & ERTS_PSFLG_PENDING_EXIT) - erts_handle_pending_exit(c_p, ERTS_PROC_LOCK_MAIN); - ASSERT(ERTS_PROC_IS_EXITING(c_p)); + if (state & ERTS_PSFLG_EXITING) break; - } st = fetch_sys_task(c_p, state, &qmask, &st_prio); if (!st) @@ -10540,6 +10669,43 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) reds -= erts_db_execute_free_fixation(c_p, (DbFixation*)st->arg[0]); st_res = am_true; break; + case ERTS_PSTT_PRIO_SIG: { + erts_aint32_t fail_state, state; + int local_only, sig_res, sig_reds = reds; + st_res = am_false; + + if (st->arg[0] == am_true) + local_only = !0; + else + local_only = 0; + + sig_reds = reds; + sig_res = erts_proc_sig_handle_incoming(c_p, &state, &sig_reds, + reds, local_only); + reds -= sig_reds; + + if (state & ERTS_PSFLG_EXITING) { + exit_permanent_prio_elevation(c_p, state); + break; + } + + if (sig_res) + break; + + st->arg[0] = am_true; + + fail_state = ERTS_PSFLG_EXITING; + + if (schedule_process_sys_task(c_p, st_prio, st, &fail_state)) { + /* Successfully rescheduled task... */ + st = NULL; + } + else { + state = erts_atomic32_read_nob(&c_p->state); + exit_permanent_prio_elevation(c_p, state); + } + break; + } default: ERTS_INTERNAL_ERROR("Invalid process sys task type"); st_res = am_false; @@ -10588,6 +10754,10 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) } switch (st->type) { + case ERTS_PSTT_PRIO_SIG: + state = erts_atomic32_read_nob(&c_p->state); + exit_permanent_prio_elevation(c_p, state); + /* fall through... */ case ERTS_PSTT_GC_MAJOR: case ERTS_PSTT_GC_MINOR: case ERTS_PSTT_CPC: @@ -10616,6 +10786,35 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) return reds; } +static void +exit_permanent_prio_elevation(Process *c_p, erts_aint32_t state) +{ + erts_aint32_t a; + /* + * we are about to terminate; permanently elevate + * prio in order to ensure high prio signal + * handling... + */ + a = state; + while (1) { + erts_aint32_t aprio, uprio, n, e; + ASSERT(a & ERTS_PSFLG_EXITING); + aprio = ERTS_PSFLGS_GET_ACT_PRIO(a); + uprio = ERTS_PSFLGS_GET_USR_PRIO(a); + if (aprio >= uprio) + break; /* user prio >= actual prio */ + /* + * actual prio is higher than user prio; raise + * user prio to actual prio... + */ + n = e = a; + n &= ~ERTS_PSFLGS_USR_PRIO_MASK; + n |= aprio << ERTS_PSFLGS_USR_PRIO_OFFSET; + a = erts_atomic32_cmpxchg_mb(&c_p->state, n, e); + if (a == e) + break; + } +} void erts_execute_dirty_system_task(Process *c_p) @@ -10645,8 +10844,7 @@ erts_execute_dirty_system_task(Process *c_p) if (c_p->flags & F_DIRTY_GC_HIBERNATE) { erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); - ERTS_MSGQ_MV_INQ2PRIVQ(c_p); - if (c_p->msg.len) + if (erts_proc_sig_fetch(c_p)) c_p->flags &= ~F_DIRTY_GC_HIBERNATE; /* operation aborted... */ else { erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); @@ -10718,7 +10916,7 @@ dispatch_system_task(Process *c_p, erts_aint_t fail_state, switch (st->type) { case ERTS_PSTT_CPC: - rp = erts_dirty_process_code_checker; + rp = erts_dirty_process_signal_handler; ASSERT(fail_state & (ERTS_PSFLG_DIRTY_RUNNING | ERTS_PSFLG_DIRTY_RUNNING_SYS)); if (c_p == rp) { @@ -10750,7 +10948,7 @@ dispatch_system_task(Process *c_p, erts_aint_t fail_state, msg = copy_struct(operation, osz, &hp, ohp); msg = TUPLE4(hp, st->requester, target, prio, msg); - erts_queue_message(rp, rp_locks, mp, msg, st->requester); + erts_queue_message(rp, rp_locks, mp, msg, am_system); if (rp_locks) erts_proc_unlock(rp, rp_locks); @@ -10935,13 +11133,15 @@ erts_internal_request_system_task_4(BIF_ALIST_4) BIF_ARG_2, BIF_ARG_3, BIF_ARG_4); } -static void -erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type, void* arg) +static int +schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type, + int prio, Eterm arg0, Eterm arg1) { - Process *rp = erts_proc_lookup(pid); + int res = 0; + Process *rp = erts_proc_lookup_raw(pid); if (rp) { ErtsProcSysTask *st; - erts_aint32_t state, fail_state; + erts_aint32_t st_prio, fail_state; st = erts_alloc(ERTS_ALC_T_PROC_SYS_TSK, ERTS_PROC_SYS_TASK_SIZE(0)); @@ -10950,31 +11150,46 @@ erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type, void* arg) st->reply_tag = NIL; st->req_id = NIL; st->req_id_sz = 0; - st->arg[0] = (Eterm)arg; + st->arg[0] = arg0; + st->arg[1] = arg1; ERTS_INIT_OFF_HEAP(&st->off_heap); - state = erts_atomic32_read_nob(&rp->state); - fail_state = ERTS_PSFLG_EXITING; - - if (!schedule_process_sys_task(rp, ERTS_PSFLGS_GET_USR_PRIO(state), - st, &fail_state)) + if (prio >= 0) { + st_prio = (erts_aint32_t) prio; + fail_state = ERTS_PSFLG_FREE; + } + else { + erts_aint32_t state = erts_atomic32_read_nob(&rp->state); + st_prio = ERTS_PSFLGS_GET_USR_PRIO(state); + fail_state = ERTS_PSFLG_EXITING; + } + res = schedule_process_sys_task(rp, st_prio, st, &fail_state); + if (!res) erts_free(ERTS_ALC_T_PROC_SYS_TSK, st); } + return res; } - void erts_schedule_complete_off_heap_message_queue_change(Eterm pid) { - erts_schedule_generic_sys_task(pid, ERTS_PSTT_COHMQ, NULL); + schedule_generic_sys_task(pid, ERTS_PSTT_COHMQ, + -1, NIL, NIL); } void erts_schedule_ets_free_fixation(Eterm pid, DbFixation* fix) { - erts_schedule_generic_sys_task(pid, ERTS_PSTT_ETS_FREE_FIXATION, fix); + schedule_generic_sys_task(pid, ERTS_PSTT_ETS_FREE_FIXATION, + -1, (Eterm) fix, NIL); } +int +erts_sig_prio(Eterm pid, int prio) +{ + return schedule_generic_sys_task(pid, ERTS_PSTT_PRIO_SIG, + prio, am_false, NIL); +} static void flush_dirty_trace_messages(void *vpid) @@ -11014,7 +11229,7 @@ erts_schedule_flush_trace_messages(Process *proc, int force_on_proc) dhndl = erts_thr_progress_unmanaged_delay(); - erts_schedule_generic_sys_task(pid, ERTS_PSTT_FTMQ, NULL); + schedule_generic_sys_task(pid, ERTS_PSTT_FTMQ, -1, NIL, NIL); erts_thr_progress_unmanaged_continue(dhndl); @@ -11210,9 +11425,11 @@ erts_set_gc_state(Process *c_p, int enable) #endif erts_atomic32_read_bset_nob(&c_p->state, - (ERTS_PSFLG_DELAYED_SYS - | ERTS_PSFLG_ACTIVE_SYS), - ERTS_PSFLG_ACTIVE_SYS); + (ERTS_PSFLG_DELAYED_SYS + | ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_SYS_TASKS), + (ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_SYS_TASKS)); #ifdef DEBUG ASSERT(state & ERTS_PSFLG_DELAYED_SYS); @@ -11675,7 +11892,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->common.u.alive.reg = NULL; ERTS_P_LINKS(p) = NULL; ERTS_P_MONITORS(p) = NULL; - p->nodes_monitors = NULL; + ERTS_P_LT_MONITORS(p) = NULL; p->suspend_monitors = NULL; ASSERT(is_pid(parent->group_leader)); @@ -11692,14 +11909,23 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). erts_get_default_proc_tracing(&ERTS_TRACE_FLAGS(p), &ERTS_TRACER(p)); - p->msg.first = NULL; - p->msg.last = &p->msg.first; - p->msg.save = &p->msg.first; - p->msg.saved_last = &p->msg.first; - p->msg.len = 0; - p->msg_inq.first = NULL; - p->msg_inq.last = &p->msg_inq.first; - p->msg_inq.len = 0; + p->sig_qs.first = NULL; + p->sig_qs.last = &p->sig_qs.first; + p->sig_qs.cont = NULL; + p->sig_qs.cont_last = &p->sig_qs.cont; + p->sig_qs.save = &p->sig_qs.first; + p->sig_qs.saved_last = NULL; + p->sig_qs.len = 0; + p->sig_qs.nmsigs.next = NULL; + p->sig_qs.nmsigs.last = NULL; + p->sig_inq.first = NULL; + p->sig_inq.last = &p->sig_inq.first; + p->sig_inq.len = 0; + p->sig_inq.nmsigs.next = NULL; + p->sig_inq.nmsigs.last = NULL; +#ifdef ERTS_PROC_SIG_HARD_DEBUG + p->sig_inq.may_contain_heap_terms = 0; +#endif p->bif_timers = NULL; p->mbuf = NULL; p->msg_frag = NULL; @@ -11726,8 +11952,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->scheduler_data = NULL; p->suspendee = NIL; p->pending_suspenders = NULL; - p->pending_exit.reason = THE_NON_VALUE; - p->pending_exit.bp = NULL; #if !defined(NO_FPE_SIGNALS) || defined(HIPE) p->fp_exception = 0; @@ -11781,30 +12005,33 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). */ if (so->flags & SPO_LINK) { -#ifdef DEBUG - int ret; -#endif -#ifdef DEBUG - ret = erts_add_link(&ERTS_P_LINKS(parent), LINK_PID, p->common.id); - ASSERT(ret == 0); - ret = erts_add_link(&ERTS_P_LINKS(p), LINK_PID, parent->common.id); - ASSERT(ret == 0); -#else - erts_add_link(&ERTS_P_LINKS(parent), LINK_PID, p->common.id); - erts_add_link(&ERTS_P_LINKS(p), LINK_PID, parent->common.id); -#endif - + ErtsLink *lnk; + ErtsLinkData *ldp = erts_link_create(ERTS_LNK_TYPE_PROC, + parent->common.id, + p->common.id); + lnk = erts_link_tree_lookup_insert(&ERTS_P_LINKS(parent), &ldp->a); + if (lnk) { + /* + * This should more or less never happen, but could + * potentially happen if pid:s wrap... + */ + erts_link_release(lnk); + } + erts_link_tree_insert(&ERTS_P_LINKS(p), &ldp->b); } /* * Test whether this process should be initially monitored by its parent. */ if (so->flags & SPO_MONITOR) { - Eterm mref; - - mref = erts_make_ref(parent); - erts_add_monitor(&ERTS_P_MONITORS(parent), MON_ORIGIN, mref, p->common.id, NIL); - erts_add_monitor(&ERTS_P_MONITORS(p), MON_TARGET, mref, parent->common.id, NIL); + Eterm mref = erts_make_ref(parent); + ErtsMonitorData *mdp = erts_monitor_create(ERTS_MON_TYPE_PROC, + mref, + parent->common.id, + p->common.id, + NIL); + erts_monitor_tree_insert(&ERTS_P_MONITORS(parent), &mdp->origin); + erts_monitor_list_insert(&ERTS_P_LT_MONITORS(p), &mdp->target); so->mref = mref; } @@ -11889,13 +12116,26 @@ void erts_init_empty_process(Process *p) p->mbuf_sz = 0; erts_atomic_init_nob(&p->psd, (erts_aint_t) NULL); ERTS_P_MONITORS(p) = NULL; + ERTS_P_LT_MONITORS(p) = NULL; ERTS_P_LINKS(p) = NULL; /* List of links */ - p->nodes_monitors = NULL; p->suspend_monitors = NULL; - p->msg.first = NULL; - p->msg.last = &p->msg.first; - p->msg.save = &p->msg.first; - p->msg.len = 0; + p->sig_qs.first = NULL; + p->sig_qs.last = &p->sig_qs.first; + p->sig_qs.cont = NULL; + p->sig_qs.cont_last = &p->sig_qs.cont; + p->sig_qs.save = &p->sig_qs.first; + p->sig_qs.saved_last = NULL; + p->sig_qs.len = 0; + p->sig_qs.nmsigs.next = NULL; + p->sig_qs.nmsigs.last = NULL; + p->sig_inq.first = NULL; + p->sig_inq.last = &p->sig_inq.first; + p->sig_inq.len = 0; + p->sig_inq.nmsigs.next = NULL; + p->sig_inq.nmsigs.last = NULL; +#ifdef ERTS_PROC_SIG_HARD_DEBUG + p->sig_inq.may_contain_heap_terms = 0; +#endif p->bif_timers = NULL; p->dictionary = NULL; p->seq_trace_clock = 0; @@ -11942,13 +12182,8 @@ void erts_init_empty_process(Process *p) erts_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL); p->scheduler_data = NULL; - p->msg_inq.first = NULL; - p->msg_inq.last = &p->msg_inq.first; - p->msg_inq.len = 0; p->suspendee = NIL; p->pending_suspenders = NULL; - p->pending_exit.reason = THE_NON_VALUE; - p->pending_exit.bp = NULL; erts_proc_lock_init(p); erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL); erts_init_runq_proc(p, ERTS_RUNQ_IX(0), 0); @@ -11984,11 +12219,11 @@ erts_debug_verify_clean_empty_process(Process* p) ASSERT(p->old_heap == NULL); ASSERT(ERTS_P_MONITORS(p) == NULL); + ASSERT(ERTS_P_LT_MONITORS(p) == NULL); ASSERT(ERTS_P_LINKS(p) == NULL); - ASSERT(p->nodes_monitors == NULL); ASSERT(p->suspend_monitors == NULL); - ASSERT(p->msg.first == NULL); - ASSERT(p->msg.len == 0); + ASSERT(p->sig_qs.first == NULL); + ASSERT(p->sig_qs.len == 0); ASSERT(p->bif_timers == NULL); ASSERT(p->dictionary == NULL); ASSERT(p->catches == 0); @@ -11998,12 +12233,10 @@ erts_debug_verify_clean_empty_process(Process* p) ASSERT(p->parent == NIL); - ASSERT(p->msg_inq.first == NULL); - ASSERT(p->msg_inq.len == 0); + ASSERT(p->sig_inq.first == NULL); + ASSERT(p->sig_inq.len == 0); ASSERT(p->suspendee == NIL); ASSERT(p->pending_suspenders == NULL); - ASSERT(p->pending_exit.reason == THE_NON_VALUE); - ASSERT(p->pending_exit.bp == NULL); /* Thing that erts_cleanup_empty_process() cleans up */ @@ -12104,817 +12337,328 @@ delete_process(Process* p) erts_erase_dicts(p); /* free all pending messages */ - erts_cleanup_messages(p->msg.first); - p->msg.first = NULL; + erts_cleanup_messages(p->sig_qs.first); + p->sig_qs.first = NULL; + erts_cleanup_messages(p->sig_qs.cont); + p->sig_qs.cont = NULL; - ASSERT(!p->nodes_monitors); ASSERT(!p->suspend_monitors); p->fvalue = NIL; } static ERTS_INLINE void -set_proc_exiting(Process *p, - erts_aint32_t in_state, - Eterm reason, - ErlHeapFragment *bp) +set_self_exiting(Process *c_p, Eterm reason, int *enqueue, + erts_aint32_t *prio, erts_aint32_t *state) { - erts_aint32_t state = in_state, enq_prio = -1; - int enqueue; - ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(p) == ERTS_PROC_LOCKS_ALL); - - enqueue = change_proc_schedule_state(p, - (ERTS_PSFLG_SUSPENDED - | ERTS_PSFLG_PENDING_EXIT - | ERTS_PSFLGS_DIRTY_WORK), - ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE, - &state, - &enq_prio, - ERTS_PROC_LOCKS_ALL); - - p->fvalue = reason; - if (bp) - erts_link_mbuf_to_proc(p, bp); - /* - * We used to set freason to EXC_EXIT here, but there is no need to - * save the stack trace since this process irreversibly is going to - * exit. - */ - p->freason = EXTAG_EXIT; - KILL_CATCHES(p); - p->i = (BeamInstr *) beam_exit; - - - add2runq(enqueue, enq_prio, p, state, NULL); -} - -static ERTS_INLINE erts_aint32_t -set_proc_self_exiting(Process *c_p) -{ -#ifdef DEBUG - int enqueue; -#endif - erts_aint32_t state, enq_prio = -1; + erts_aint32_t st, enq_prio = -1; + int enq; ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCKS_ALL); - state = erts_atomic32_read_nob(&c_p->state); - ASSERT(state & (ERTS_PSFLG_RUNNING - |ERTS_PSFLG_RUNNING_SYS - | ERTS_PSFLG_DIRTY_RUNNING - | ERTS_PSFLG_DIRTY_RUNNING_SYS)); + c_p->fvalue = reason; -#ifdef DEBUG - enqueue = -#endif - change_proc_schedule_state(c_p, - ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT, - ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE, - &state, - &enq_prio, - ERTS_PROC_LOCKS_ALL); - - ASSERT(!enqueue); - return state; -} + st = erts_atomic32_read_nob(&c_p->state); + ASSERT(enqueue || (st & (ERTS_PSFLG_RUNNING + |ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS))); + enq = change_proc_schedule_state(c_p, + (ERTS_PSFLG_SUSPENDED + | ERTS_PSFLGS_DIRTY_WORK), + ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE, + &st, + &enq_prio, + ERTS_PROC_LOCKS_ALL); -void -erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks) -{ - ErtsProcLocks xlocks; - ASSERT(is_value(c_p->pending_exit.reason)); - ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == locks); - ERTS_LC_ASSERT(locks & ERTS_PROC_LOCK_MAIN); - ERTS_LC_ASSERT(!((ERTS_PSFLG_EXITING|ERTS_PSFLG_FREE) - & erts_atomic32_read_nob(&c_p->state))); - - /* Ensure that all locks on c_p are locked before proceeding... */ - if (locks == ERTS_PROC_LOCKS_ALL) - xlocks = 0; - else { - xlocks = ~locks & ERTS_PROC_LOCKS_ALL; - if (erts_proc_trylock(c_p, xlocks) == EBUSY) { - erts_proc_unlock(c_p, locks & ~ERTS_PROC_LOCK_MAIN); - erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); - } - } - - set_proc_exiting(c_p, - erts_atomic32_read_acqb(&c_p->state), - c_p->pending_exit.reason, - c_p->pending_exit.bp); - c_p->pending_exit.reason = THE_NON_VALUE; - c_p->pending_exit.bp = NULL; - - if (xlocks) - erts_proc_unlock(c_p, xlocks); -} - -static void save_pending_exiter(Process *p, ErtsProcList *plp); - -static void -do_handle_pending_exiters(ErtsProcList *pnd_xtrs) -{ - /* 'list' is expected to have been fetched (i.e. not a ring anymore) */ - ErtsProcList *plp = pnd_xtrs; - - while (plp) { - ErtsProcList *next_plp = plp->next; - Process *p = erts_proc_lookup(plp->pid); - if (p) { - erts_aint32_t state; - /* - * If the process is running on a normal scheduler, the - * pending exit will soon be detected and handled by the - * scheduler running the process (at schedule in/out). - */ - if (erts_proc_trylock(p, ERTS_PROC_LOCKS_ALL) != EBUSY) { - if (erts_proclist_same(plp, p)) { - state = erts_atomic32_read_acqb(&p->state); - if (!(state & (ERTS_PSFLG_RUNNING - | ERTS_PSFLG_RUNNING_SYS - | ERTS_PSFLG_EXITING))) { - ASSERT(state & ERTS_PSFLG_PENDING_EXIT); - erts_handle_pending_exit(p, ERTS_PROC_LOCKS_ALL); - } - } - erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL); - } - else { - erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); - if (erts_proclist_same(plp, p)) { - state = erts_atomic32_read_acqb(&p->state); - if (!(state & (ERTS_PSFLG_RUNNING - | ERTS_PSFLG_RUNNING_SYS - | ERTS_PSFLG_EXITING))) { - /* - * Save process and try to acquire all - * locks at a later time... - */ - save_pending_exiter(p, plp); - plp = NULL; - } - } - erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - } - } - if (plp) - proclist_destroy(plp); - plp = next_plp; - } + ASSERT(enqueue || !enq); + if (enqueue) + *enqueue = enq; + if (prio) + *prio = enq_prio; + if (state) + *state = st; } -static void -save_pending_exiter(Process *p, ErtsProcList *plp) +void +erts_set_self_exiting(Process *c_p, Eterm reason) { - ErtsSchedulerSleepInfo *ssi; - ErtsRunQueue *rq; - - ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - - rq = erts_get_runq_proc(p, NULL); - ASSERT(rq && !ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); - - if (!plp) - plp = proclist_create(p); - - erts_runq_lock(rq); - - erts_proclist_store_last(&rq->procs.pending_exiters, plp); - - non_empty_runq(rq); - - ssi = rq->scheduler->ssi; - - erts_runq_unlock(rq); + int enqueue; + erts_aint32_t enq_prio, state; + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN); - set_aux_work_flags_wakeup_nob(ssi, ERTS_SSI_AUX_WORK_PENDING_EXITERS); -} + erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + set_self_exiting(c_p, reason, &enqueue, &enq_prio, &state); + c_p->freason = EXTAG_EXIT; + KILL_CATCHES(c_p); + c_p->i = (BeamInstr *) beam_exit; -/* - * This function delivers an EXIT message to a process - * which is trapping EXITs. - */ + /* Always active when exiting... */ + ASSERT(state & ERTS_PSFLG_ACTIVE); -static ERTS_INLINE void -send_exit_message(Process *to, ErtsProcLocks *to_locksp, - Eterm exit_term, Uint term_size, Eterm token) -{ - ErtsMessage *mp; - ErlOffHeap *ohp; - Eterm* hp; - Eterm mess; -#ifdef SHCOPY_SEND - erts_shcopy_t info; -#endif + /* + * If we are terminating a process that currently + * is executing on a dirty scheduler. It *should* + * be scheduled on a normal scheduler... + */ + ASSERT(!(state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) + || enqueue == ERTS_ENQUEUE_NORMAL_QUEUE + || enqueue == -ERTS_ENQUEUE_NORMAL_QUEUE); - if (!have_seqtrace(token)) { -#ifdef SHCOPY_SEND - INITIALIZE_SHCOPY(info); - term_size = copy_shared_calculate(exit_term, &info); - mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp); - mess = copy_shared_perform(exit_term, term_size, &info, &hp, ohp); - DESTROY_SHCOPY(info); -#else - mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp); - mess = copy_struct(exit_term, term_size, &hp, ohp); -#endif - erts_queue_message(to, *to_locksp, mp, mess, am_system); - } else { - Eterm temp_token; - Uint sz_token; - - ASSERT(is_tuple(token)); - sz_token = size_object(token); -#ifdef SHCOPY_SEND - INITIALIZE_SHCOPY(info); - term_size = copy_shared_calculate(exit_term, &info); - mp = erts_alloc_message_heap(to, to_locksp, term_size+sz_token, &hp, &ohp); - mess = copy_shared_perform(exit_term, term_size, &info, &hp, ohp); - DESTROY_SHCOPY(info); -#else - mp = erts_alloc_message_heap(to, to_locksp, term_size+sz_token, &hp, &ohp); - mess = copy_struct(exit_term, term_size, &hp, ohp); -#endif - /* the trace token must in this case be updated by the caller */ - seq_trace_output(token, mess, SEQ_TRACE_SEND, to->common.id, to); - temp_token = copy_struct(token, sz_token, &hp, ohp); - ERL_MESSAGE_TOKEN(mp) = temp_token; - erts_queue_message(to, *to_locksp, mp, mess, am_system); - } + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + if (enqueue) + add2runq(enqueue, enq_prio, c_p, state, NULL); } -/* - * - * *** Exit signal behavior *** - * - * Exit signals are asynchronous (truly asynchronous in the - * SMP emulator). When the signal is received the receiver receives an - * 'EXIT' message if it is trapping exits; otherwise, it will either - * ignore the signal if the exit reason is normal, or go into an - * exiting state (ERTS_PSFLG_EXITING). When a process has gone into the - * exiting state it will not execute any more Erlang code, but it might - * take a while before it actually exits. The exit signal is being - * received when the 'EXIT' message is put in the message queue, the - * signal is dropped, or when it changes state into exiting. The time it - * is in the exiting state before actually exiting is undefined (it - * might take a really long time under certain conditions). The - * receiver of the exit signal does not break links or trigger monitors - * until it actually exits. - * - * Exit signals and other signals, e.g. messages, have to be received - * by a receiver in the same order as sent by a sender. - * - * - * - * Exit signal implementation in the SMP emulator: - * - * If the receiver is trapping exits, the signal is transformed - * into an 'EXIT' message and sent as a normal message, if the - * reason is normal the signal is dropped; otherwise, the process - * is determined to be exited. The interesting case is when the - * process is to be exited and this is what is described below. - * - * If it is possible, the receiver is set in the exiting state straight - * away and we are done; otherwise, the sender places the exit reason - * in the pending_exit field of the process struct and if necessary - * adds the receiver to the run queue. It is typically not possible - * to set a scheduled process or a process which we cannot get all locks - * on without releasing locks on it in an exiting state straight away. - * - * The receiver will poll the pending_exit field when it reach certain - * places during it's execution. When it discovers the pending exit - * it will change state into the exiting state. If the receiver wasn't - * scheduled when the pending exit was set, the first scheduler that - * schedules a new process will set the receiving process in the exiting - * state just before it schedules next process. - * - * When the exit signal is placed in the pending_exit field, the signal - * is considered as being in transit on the Erlang level. The signal is - * actually in some kind of semi transit state, since we have already - * determined how it should be received. It will exit the process no - * matter what if it is received (the process may exit by itself before - * reception of the exit signal). The signal is received when it is - * discovered in the pending_exit field by the receiver. - * - * The receiver have to poll the pending_exit field at least before: - * - moving messages from the message in queue to the private message - * queue. This in order to preserve signal order. - * - unlink. Otherwise the process might get exited on a link that - * have been removed. - * - changing the trap_exit flag to true. This in order to simplify the - * implementation; otherwise, we would have to transform the signal - * into an 'EXIT' message when setting the trap_exit flag to true. We - * would also have to maintain a queue of exit signals in transit. - * - being scheduled in or out. - */ - -static ERTS_INLINE int -send_exit_signal(Process *c_p, /* current process if and only - if reason is stored on it */ - Eterm from, /* Id of sender of signal */ - Process *rp, /* receiving process */ - ErtsProcLocks *rp_locks,/* current locks on receiver */ - Eterm reason, /* exit reason */ - Eterm exit_tuple, /* Prebuild exit tuple - or THE_NON_VALUE */ - Uint exit_tuple_sz, /* Size of prebuilt exit tuple - (if exit_tuple != THE_NON_VALUE) */ - Eterm token, /* token */ - Process *token_update, /* token updater */ - Uint32 flags /* flags */ - ) -{ - erts_aint32_t state = erts_atomic32_read_nob(&rp->state); - Eterm rsn = reason == am_kill ? am_killed : reason; - - ERTS_LC_ASSERT(*rp_locks == erts_proc_lc_my_proc_locks(rp)); - ERTS_LC_ASSERT((*rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) - == ERTS_PROC_LOCKS_XSIG_SEND); - - ASSERT(reason != THE_NON_VALUE); - -#ifdef USE_VM_PROBES - if(DTRACE_ENABLED(process_exit_signal) && is_pid(from)) { - DTRACE_CHARBUF(sender_str, DTRACE_TERM_BUF_SIZE); - DTRACE_CHARBUF(receiver_str, DTRACE_TERM_BUF_SIZE); - DTRACE_CHARBUF(reason_buf, DTRACE_TERM_BUF_SIZE); - - dtrace_pid_str(from, sender_str); - dtrace_proc_str(rp, receiver_str); - erts_snprintf(reason_buf, sizeof(DTRACE_CHARBUF_NAME(reason_buf)) - 1, "%T", reason); - DTRACE3(process_exit_signal, sender_str, receiver_str, reason_buf); +void +erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt) +{ + Process *c_p = ((ErtsProcExitContext *) vctxt)->c_p; + Eterm reason = ((ErtsProcExitContext *) vctxt)->reason; + ErtsMonitorData *mdp = NULL; + + if (erts_monitor_is_target(mon)) { + /* We are being watched... */ + switch (mon->type) { + case ERTS_MON_TYPE_PROC: + erts_proc_sig_send_monitor_down(mon, reason); + mon = NULL; + break; + case ERTS_MON_TYPE_PORT: { + Port *prt; + ASSERT(is_internal_port(mon->other.item)); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + prt = erts_id2port(mon->other.item); + if (prt) { + erts_fire_port_monitor(prt, mon); + erts_port_release(prt); + mon = NULL; + } + erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + break; + } + case ERTS_MON_TYPE_RESOURCE: + erts_fire_nif_monitor(mon); + mon = NULL; + break; + case ERTS_MON_TYPE_DIST_PROC: { + ErtsMonLnkDist *dist; + DistEntry *dep; + ErtsDSigData dsd; + int code; + Eterm watcher; + Eterm watched; + + mdp = erts_monitor_to_data(mon); + + if (mon->flags & ERTS_ML_FLG_NAME) + watched = ((ErtsMonitorDataExtended *) mdp)->u.name; + else + watched = c_p->common.id; + ASSERT(is_internal_pid(watched) || is_atom(watched)); + + watcher = mon->other.item; + ASSERT(is_external_pid(watcher)); + dep = external_pid_dist_entry(watcher); + ASSERT(dep); + dist = ((ErtsMonitorDataExtended *) mdp)->dist; + ASSERT(dist); + code = erts_dsig_prepare(&dsd, dep, NULL, 0, + ERTS_DSP_NO_LOCK, 0, 0); + switch (code) { + case ERTS_DSIG_PREP_CONNECTED: + case ERTS_DSIG_PREP_PENDING: + if (dist->connection_id == dsd.connection_id) { + code = erts_dsig_send_m_exit(&dsd, + watcher, + watched, + mdp->ref, + reason); + ASSERT(code == ERTS_DSIG_SEND_OK); + } + default: + break; + } + if (!erts_monitor_dist_delete(&mdp->origin)) + mdp = NULL; + break; + } + default: + ERTS_INTERNAL_ERROR("Invalid target monitor type"); + break; + } } -#endif - - if ((state & ERTS_PSFLG_TRAP_EXIT) - && (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) { - /* have to release the status and trace lock in order to send the exit message */ - erts_proc_unlock(rp, *rp_locks & (ERTS_PROC_LOCKS_XSIG_SEND|ERTS_PROC_LOCK_TRACE)); - *rp_locks &= ~(ERTS_PROC_LOCKS_XSIG_SEND|ERTS_PROC_LOCK_TRACE); - if (have_seqtrace(token) && token_update) - seq_trace_update_send(token_update); - if (is_value(exit_tuple)) - send_exit_message(rp, rp_locks, exit_tuple, exit_tuple_sz, token); - else - erts_deliver_exit_message(from, rp, rp_locks, rsn, token); - return 1; /* Receiver will get a message */ - } - else if (reason != am_normal || (flags & ERTS_XSIG_FLG_NO_IGN_NORMAL)) { - if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))) { - ASSERT(!rp->pending_exit.bp); - - if (rp == c_p && (*rp_locks & ERTS_PROC_LOCK_MAIN)) { - /* Ensure that all locks on c_p are locked before - proceeding... */ - if (*rp_locks != ERTS_PROC_LOCKS_ALL) { - ErtsProcLocks need_locks = (~(*rp_locks) - & ERTS_PROC_LOCKS_ALL); - if (erts_proc_trylock(c_p, need_locks) == EBUSY) { - erts_proc_unlock(c_p, - *rp_locks & ~ERTS_PROC_LOCK_MAIN); - erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); - } - *rp_locks = ERTS_PROC_LOCKS_ALL; - } - set_proc_exiting(c_p, state, rsn, NULL); - } - else if (!(state & (ERTS_PSFLG_RUNNING - | ERTS_PSFLG_RUNNING_SYS - | ERTS_PSFLG_DIRTY_RUNNING_SYS))) { - /* Process not running ... */ - ErtsProcLocks need_locks = ~(*rp_locks) & ERTS_PROC_LOCKS_ALL; - ErlHeapFragment *bp = NULL; - Eterm rsn_cpy; - if (need_locks - && erts_proc_trylock(rp, need_locks) == EBUSY) { - /* ... but we havn't got all locks on it ... */ - save_pending_exiter(rp, NULL); - /* - * The pending exit will be discovered when next - * process is scheduled in - */ - goto set_pending_exit; - } - /* ...and we have all locks on it... */ - *rp_locks = ERTS_PROC_LOCKS_ALL; - - state = erts_atomic32_read_nob(&rp->state); - - if (is_immed(rsn)) - rsn_cpy = rsn; - else { - Eterm *hp; - ErlOffHeap *ohp; - Uint rsn_sz = size_object(rsn); - if (state & ERTS_PSFLG_DIRTY_RUNNING) { - bp = new_message_buffer(rsn_sz); - ohp = &bp->off_heap; - hp = &bp->mem[0]; - } - else - { - hp = HAlloc(rp, rsn_sz); - ohp = &rp->off_heap; - } - rsn_cpy = copy_struct(rsn, rsn_sz, &hp, ohp); - } - - set_proc_exiting(rp, state, rsn_cpy, bp); - } - else { /* Process running... */ - - /* - * The pending exit will be discovered when the process - * is scheduled out if not discovered earlier. - */ - - set_pending_exit: - if (is_immed(rsn)) { - rp->pending_exit.reason = rsn; - } - else { - Eterm *hp; - Uint sz = size_object(rsn); - ErlHeapFragment *bp = new_message_buffer(sz); - - hp = &bp->mem[0]; - rp->pending_exit.reason = copy_struct(rsn, - sz, - &hp, - &bp->off_heap); - rp->pending_exit.bp = bp; - } - - /* - * If no dirty work has been scheduled, pending exit will - * be discovered when the process is scheduled. If dirty work - * has been scheduled, we may need to add it to a normal run - * queue... - */ - { - erts_aint32_t a = erts_atomic32_read_nob(&rp->state); - while (1) { - erts_aint32_t n, e; - int dwork; - n = e = a; - n |= ERTS_PSFLG_PENDING_EXIT; - dwork = !!(n & ERTS_PSFLGS_DIRTY_WORK); - n &= ~ERTS_PSFLGS_DIRTY_WORK; - a = erts_atomic32_cmpxchg_mb(&rp->state, n, e); - if (a == e) { - if (dwork) - erts_schedule_process(rp, n, *rp_locks); - break; - } - } + else { /* Origin monitor */ + /* We are watching someone else... */ + switch (mon->type) { + case ERTS_MON_TYPE_PROC: + erts_proc_sig_send_demonitor(mon); + mon = NULL; + break; + case ERTS_MON_TYPE_TIME_OFFSET: + erts_demonitor_time_offset(mon); + mon = NULL; + break; + case ERTS_MON_TYPE_NODE: + mdp = erts_monitor_to_data(mon); + if (!erts_monitor_dist_delete(&mdp->target)) + mdp = NULL; + break; + case ERTS_MON_TYPE_NODES: + erts_monitor_nodes_delete(mon); + mon = NULL; + break; + case ERTS_MON_TYPE_PORT: { + Port *prt; + ASSERT(is_internal_port(mon->other.item)); + prt = erts_port_lookup_raw(mon->other.item); + if (prt) { + if (erts_port_demonitor(c_p, prt, mon) != ERTS_PORT_OP_DROPPED) + mon = NULL; + } + break; + } + case ERTS_MON_TYPE_DIST_PROC: { + ErtsMonLnkDist *dist; + DistEntry *dep; + ErtsDSigData dsd; + int code; + Eterm watched; + + mdp = erts_monitor_to_data(mon); + dist = ((ErtsMonitorDataExtended *) mdp)->dist; + ASSERT(dist); + if (mon->flags & ERTS_ML_FLG_NAME) { + watched = ((ErtsMonitorDataExtended *) mdp)->u.name; + ASSERT(is_atom(watched)); + dep = erts_sysname_to_connected_dist_entry(dist->nodename); + } + else { + watched = mon->other.item; + ASSERT(is_external_pid(watched)); + dep = external_pid_dist_entry(watched); + } + code = erts_dsig_prepare(&dsd, dep, NULL, 0, + ERTS_DSP_NO_LOCK, 0, 0); + switch (code) { + case ERTS_DSIG_PREP_CONNECTED: + case ERTS_DSIG_PREP_PENDING: + if (dist->connection_id == dsd.connection_id) { + code = erts_dsig_send_demonitor(&dsd, + c_p->common.id, + watched, + mdp->ref, + 1); + ASSERT(code == ERTS_DSIG_SEND_OK); } - } - } - /* else: - * - * The receiver already has a pending exit (or is exiting) - * so we drop this signal. - * - * NOTE: dropping this exit signal is based on the assumption - * that the receiver *will* exit; either on the pending - * exit or by itself before seeing the pending exit. - */ - return -1; /* Receiver will exit */ + default: + break; + } + if (!erts_monitor_dist_delete(&mdp->target)) + mdp = NULL; + break; + } + default: + ERTS_INTERNAL_ERROR("Invalid origin monitor type"); + break; + } } - return 0; /* Receiver unaffected */ -} - - -int -erts_send_exit_signal(Process *c_p, - Eterm from, - Process *rp, - ErtsProcLocks *rp_locks, - Eterm reason, - Eterm token, - Process *token_update, - Uint32 flags) -{ - return send_exit_signal(c_p, - from, - rp, - rp_locks, - reason, - THE_NON_VALUE, - 0, - token, - token_update, - flags); + if (mdp) + erts_monitor_release_both(mdp); + else if (mon) + erts_monitor_release(mon); } -typedef struct { - Eterm reason; - Process *p; -} ExitMonitorContext; - -static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) -{ - ExitMonitorContext *pcontext = vpcontext; - DistEntry *dep; - ErtsMonitor *rmon; - - switch (mon->type) { - case MON_ORIGIN: - /* We are monitoring someone else, we need to demonitor that one.. */ - if (is_atom(mon->u.pid)) { /* remote by name */ - ASSERT(is_node_name_atom(mon->u.pid)); - dep = erts_sysname_to_connected_dist_entry(mon->u.pid); - if (dep) { - erts_de_links_lock(dep); - rmon = erts_remove_monitor(&(dep->monitors), mon->ref); - erts_de_links_unlock(dep); - if (rmon) { - ErtsDSigData dsd; - int code = erts_dsig_prepare(&dsd, dep, NULL, 0, - ERTS_DSP_NO_LOCK, 0, 0); - if (code == ERTS_DSIG_PREP_CONNECTED || - code == ERTS_DSIG_PREP_PENDING) { - - code = erts_dsig_send_demonitor(&dsd, - rmon->u.pid, - mon->name, - mon->ref, - 1); - ASSERT(code == ERTS_DSIG_SEND_OK); - } - erts_destroy_monitor(rmon); - } - } - } else { - ASSERT(is_pid(mon->u.pid) || is_port(mon->u.pid)); - /* if is local by pid or name */ - if (is_internal_pid(mon->u.pid)) { - Process *rp = erts_pid2proc(NULL, 0, mon->u.pid, ERTS_PROC_LOCK_LINK); - if (!rp) { - goto done; - } - rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref); - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - if (rmon == NULL) { - goto done; - } - erts_destroy_monitor(rmon); - } else if (is_internal_port(mon->u.pid)) { - /* Is a local port */ - Port *prt = erts_port_lookup_raw(mon->u.pid); - if (!prt) { - goto done; - } - erts_port_demonitor(pcontext->p, - ERTS_PORT_DEMONITOR_ORIGIN_ON_DEATHBED, - prt, mon->ref, NULL); - } else { /* remote by pid */ - ASSERT(is_external_pid(mon->u.pid)); - dep = external_pid_dist_entry(mon->u.pid); - ASSERT(dep != NULL); - if (dep) { - erts_de_links_lock(dep); - rmon = erts_remove_monitor(&(dep->monitors), mon->ref); - erts_de_links_unlock(dep); - if (rmon) { - ErtsDSigData dsd; - int code = erts_dsig_prepare(&dsd, dep, NULL, 0, - ERTS_DSP_NO_LOCK, 0, 0); - if (code == ERTS_DSIG_PREP_CONNECTED || - code == ERTS_DSIG_PREP_PENDING) { - - code = erts_dsig_send_demonitor(&dsd, - rmon->u.pid, - mon->u.pid, - mon->ref, - 1); - ASSERT(code == ERTS_DSIG_SEND_OK); - } - erts_destroy_monitor(rmon); - } - } - } - } - break; - case MON_TARGET: - ASSERT(is_pid(mon->u.pid) || is_internal_port(mon->u.pid)); - if (is_internal_port(mon->u.pid)) { - Port *prt = erts_id2port(mon->u.pid); - if (prt == NULL) { - goto done; - } - erts_fire_port_monitor(prt, mon->ref); - erts_port_release(prt); - } else if (is_internal_pid(mon->u.pid)) {/* local by name or pid */ - Eterm watched; - Process *rp; - DeclareTmpHeapNoproc(lhp,3); - ErtsProcLocks rp_locks = (ERTS_PROC_LOCK_LINK - | ERTS_PROC_LOCKS_MSG_SEND); - rp = erts_pid2proc(NULL, 0, mon->u.pid, rp_locks); - if (rp == NULL) { - goto done; - } - UseTmpHeapNoproc(3); - rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref); - if (rmon) { - erts_destroy_monitor(rmon); - watched = (is_atom(mon->name) - ? TUPLE2(lhp, mon->name, - erts_this_dist_entry->sysname) - : pcontext->p->common.id); - erts_queue_monitor_message(rp, &rp_locks, mon->ref, am_process, - watched, pcontext->reason); - } - UnUseTmpHeapNoproc(3); - /* else: demonitor while we exited, i.e. do nothing... */ - erts_proc_unlock(rp, rp_locks); - } else { /* external by pid or name */ - ASSERT(is_external_pid(mon->u.pid)); - dep = external_pid_dist_entry(mon->u.pid); - ASSERT(dep != NULL); - if (dep) { - erts_de_links_lock(dep); - rmon = erts_remove_monitor(&(dep->monitors), mon->ref); - erts_de_links_unlock(dep); - if (rmon) { - ErtsDSigData dsd; - int code = erts_dsig_prepare(&dsd, dep, NULL, 0, - ERTS_DSP_NO_LOCK, 0, 0); - if (code == ERTS_DSIG_PREP_CONNECTED) { - code = erts_dsig_send_m_exit(&dsd, - mon->u.pid, - (rmon->name != NIL - ? rmon->name - : rmon->u.pid), - mon->ref, - pcontext->reason); - ASSERT(code == ERTS_DSIG_SEND_OK); - } - erts_destroy_monitor(rmon); - } - } - } - break; - case MON_NIF_TARGET: - erts_fire_nif_monitor(mon->u.resource, - pcontext->p->common.id, - mon->ref); +void +erts_proc_exit_handle_link(ErtsLink *lnk, void *vctxt) +{ + Process *c_p = ((ErtsProcExitContext *) vctxt)->c_p; + Eterm reason = ((ErtsProcExitContext *) vctxt)->reason; + ErtsLinkData *ldp = NULL; + + switch (lnk->type) { + case ERTS_LNK_TYPE_PROC: + ASSERT(is_internal_pid(lnk->other.item)); + erts_proc_sig_send_link_exit(c_p, c_p->common.id, lnk, + reason, SEQ_TRACE_TOKEN(c_p)); + lnk = NULL; + break; + case ERTS_LNK_TYPE_PORT: { + Port *prt; + ASSERT(is_internal_port(lnk->other.item)); + prt = erts_port_lookup(lnk->other.item, + ERTS_PORT_SFLGS_INVALID_LOOKUP); + if (prt) + erts_port_exit(NULL, + (ERTS_PORT_SIG_FLG_FORCE_SCHED + | ERTS_PORT_SIG_FLG_BROKEN_LINK), + prt, + c_p->common.id, + reason, + NULL); + break; + } + case ERTS_LNK_TYPE_DIST_PROC: { + DistEntry *dep; + ErtsMonLnkDist *dist; + ErtsLink *dlnk; + ErtsDSigData dsd; + int code; + + dlnk = erts_link_to_other(lnk, &ldp); + dist = ((ErtsLinkDataExtended *) ldp)->dist; + + ASSERT(is_external_pid(lnk->other.item)); + dep = external_pid_dist_entry(lnk->other.item); + + ASSERT(dep != erts_this_dist_entry); + + if (!erts_link_dist_delete(dlnk)) + ldp = NULL; + + code = erts_dsig_prepare(&dsd, dep, c_p, 0, ERTS_DSP_NO_LOCK, 0, 0); + switch (code) { + case ERTS_DSIG_PREP_CONNECTED: + case ERTS_DSIG_PREP_PENDING: + if (dist->connection_id == dsd.connection_id) { + code = erts_dsig_send_exit_tt(&dsd, + c_p->common.id, + lnk->other.item, + reason, + SEQ_TRACE_TOKEN(c_p)); + ASSERT(code == ERTS_DSIG_SEND_OK); + } + } break; - case MON_TIME_OFFSET: - erts_demonitor_time_offset(mon->ref); - break; - default: - ERTS_INTERNAL_ERROR("Invalid monitor type"); } - done: - /* As the monitors are previously removed from the process, - distribution operations will not cause monitors to disappear, - we can safely delete it. */ - - erts_destroy_monitor(mon); -} - -typedef struct { - Process *p; - Eterm reason; - Eterm exit_tuple; - Uint exit_tuple_sz; -} ExitLinkContext; - -static void doit_exit_link(ErtsLink *lnk, void *vpcontext) -{ - ExitLinkContext *pcontext = vpcontext; - /* Unpack context, it's readonly */ - Process *p = pcontext->p; - Eterm reason = pcontext->reason; - Eterm exit_tuple = pcontext->exit_tuple; - Uint exit_tuple_sz = pcontext->exit_tuple_sz; - Eterm item = lnk->pid; - ErtsLink *rlnk; - DistEntry *dep; - Process *rp; - - - switch(lnk->type) { - case LINK_PID: - if(is_internal_port(item)) { - Port *prt = erts_port_lookup(item, ERTS_PORT_SFLGS_INVALID_LOOKUP); - if (prt) - erts_port_exit(NULL, - (ERTS_PORT_SIG_FLG_FORCE_SCHED - | ERTS_PORT_SIG_FLG_BROKEN_LINK), - prt, - p->common.id, - reason, - NULL); - } - else if(is_external_port(item)) { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, - "Erroneous link between %T and external port %T " - "found\n", - p->common.id, - item); - erts_send_error_to_logger_nogl(dsbufp); - ASSERT(0); /* It isn't possible to setup such a link... */ - } - else if (is_internal_pid(item)) { - ErtsProcLocks rp_locks = (ERTS_PROC_LOCK_LINK - | ERTS_PROC_LOCKS_XSIG_SEND); - rp = erts_pid2proc(NULL, 0, item, rp_locks); - if (rp) { - rlnk = erts_remove_link(&ERTS_P_LINKS(rp), p->common.id); - /* If rlnk == NULL, we got unlinked while exiting, - i.e., do nothing... */ - if (rlnk) { - int xres; - erts_destroy_link(rlnk); - xres = send_exit_signal(NULL, - p->common.id, - rp, - &rp_locks, - reason, - exit_tuple, - exit_tuple_sz, - SEQ_TRACE_TOKEN(p), - p, - ERTS_XSIG_FLG_IGN_KILL); - if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) { - /* We didn't exit the process and it is traced */ - if (IS_TRACED_FL(rp, F_TRACE_PROCS)) { - if (rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) { - erts_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND); - rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND; - } - trace_proc(NULL, 0, rp, am_getting_unlinked, p->common.id); - } - } - } - ASSERT(rp != p); - erts_proc_unlock(rp, rp_locks); - } - } - else if (is_external_pid(item)) { - dep = external_pid_dist_entry(item); - if(dep != erts_this_dist_entry) { - ErtsDSigData dsd; - int code; - ErtsDistLinkData dld; - erts_remove_dist_link(&dld, p->common.id, item, dep); - if (dld.d_lnk) { - erts_proc_lock(p, ERTS_PROC_LOCK_MAIN); - code = erts_dsig_prepare(&dsd, dep, p, 0, ERTS_DSP_NO_LOCK, 0, 0); - if (code == ERTS_DSIG_PREP_CONNECTED || - code == ERTS_DSIG_PREP_PENDING) { - - code = erts_dsig_send_exit_tt(&dsd, p->common.id, item, - reason, SEQ_TRACE_TOKEN(p)); - ASSERT(code == ERTS_DSIG_SEND_OK); - } - erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - } - erts_destroy_dist_link(&dld); - } - } - break; - case LINK_NODE: - ASSERT(is_node_name_atom(item)); - dep = erts_sysname_to_connected_dist_entry(item); - if(dep) { - /* dist entries have node links in a separate structure to - avoid confusion */ - erts_de_links_lock(dep); - rlnk = erts_remove_link(&(dep->node_links), p->common.id); - erts_de_links_unlock(dep); - if (rlnk) - erts_destroy_link(rlnk); - } - break; - default: - erts_exit(ERTS_ERROR_EXIT, "bad type in link list\n"); - break; + ERTS_INTERNAL_ERROR("Unexpected link type"); + break; } - erts_destroy_link(lnk); + + if (ldp) + erts_link_release_both(ldp); + else if (lnk) + erts_link_release(lnk); } static void -resume_suspend_monitor(ErtsSuspendMonitor *smon, void *vc_p) +resume_suspend_monitor(ErtsMonitor *mon, void *vc_p) { + ErtsMonitorSuspend *smon = erts_monitor_suspend(mon); Process *suspendee = erts_pid2proc((Process *) vc_p, ERTS_PROC_LOCK_MAIN, - smon->pid, ERTS_PROC_LOCK_STATUS); + smon->mon.other.item, ERTS_PROC_LOCK_STATUS); if (suspendee) { ASSERT(suspendee != vc_p); if (smon->active) resume_process(suspendee, ERTS_PROC_LOCK_STATUS); erts_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); } - erts_destroy_suspend_monitor(smon); + erts_monitor_suspend_destroy(smon); } /* this function fishishes a process and propagates exit messages - called @@ -12923,8 +12667,6 @@ void erts_do_exit_process(Process* p, Eterm reason) { p->arity = 0; /* No live registers */ - p->fvalue = reason; - #ifdef USE_VM_PROBES if (DTRACE_ENABLED(process_exit)) { @@ -12948,19 +12690,10 @@ erts_do_exit_process(Process* p, Eterm reason) process from exiting until the lock has been released. */ erts_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); - if (ERTS_PSFLG_PENDING_EXIT & set_proc_self_exiting(p)) { - /* Process exited before pending exit was received... */ - p->pending_exit.reason = THE_NON_VALUE; - if (p->pending_exit.bp) { - free_message_buffer(p->pending_exit.bp); - p->pending_exit.bp = NULL; - } - } + set_self_exiting(p, reason, NULL, NULL, NULL); cancel_suspend_of_suspendee(p, ERTS_PROC_LOCKS_ALL); - ERTS_MSGQ_MV_INQ2PRIVQ(p); - if (IS_TRACED(p)) { if (IS_TRACED_FL(p, F_TRACE_CALLS)) erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_EXITING); @@ -12996,13 +12729,15 @@ erts_do_exit_process(Process* p, Eterm reason) void erts_continue_exit_process(Process *p) { - ErtsLink* lnk; - ErtsMonitor *mon; + ErtsLink *links; + ErtsMonitor *monitors; + ErtsMonitor *lt_monitors; ErtsProcLocks curr_locks = ERTS_PROC_LOCK_MAIN; Eterm reason = p->fvalue; DistEntry *dep = NULL; erts_aint32_t state; int delay_del_proc = 0; + ErtsProcExitContext pectxt; #ifdef DEBUG int yield_allowed = 1; @@ -13074,16 +12809,13 @@ erts_continue_exit_process(Process *p) erts_set_gc_state(p, 1); state = erts_atomic32_read_acqb(&p->state); - if (state & ERTS_PSFLG_ACTIVE_SYS - || p->dirty_sys_tasks - ) { + if ((state & ERTS_PSFLG_SYS_TASKS) || p->dirty_sys_tasks) { if (cleanup_sys_tasks(p, state, CONTEXT_REDS) >= CONTEXT_REDS/2) goto yield; } #ifdef DEBUG erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); - ASSERT(p->sys_task_qs == NULL); ASSERT(ERTS_PROC_GET_DELAYED_GC_TASK_QS(p) == NULL); ASSERT(p->dirty_sys_tasks == NULL); erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); @@ -13094,18 +12826,10 @@ erts_continue_exit_process(Process *p) p->flags &= ~F_USING_DDLL; } - if (p->nodes_monitors) { - erts_delete_nodes_monitors(p, ERTS_PROC_LOCK_MAIN); - p->nodes_monitors = NULL; - } - - - if (p->suspend_monitors) { - erts_sweep_suspend_monitors(p->suspend_monitors, - resume_suspend_monitor, - p); - p->suspend_monitors = NULL; - } + if (p->suspend_monitors) + erts_monitor_tree_foreach_delete(&p->suspend_monitors, + resume_suspend_monitor, + p); /* * The registered name *should* be the last "erlang resource" to @@ -13134,8 +12858,9 @@ erts_continue_exit_process(Process *p) * Note! The monitor and link fields will be overwritten * by erts_ptab_delete_element() below. */ - mon = ERTS_P_MONITORS(p); - lnk = ERTS_P_LINKS(p); + links = ERTS_P_LINKS(p); + monitors = ERTS_P_MONITORS(p); + lt_monitors = ERTS_P_LT_MONITORS(p); { /* Do *not* use erts_get_runq_proc() */ @@ -13172,7 +12897,9 @@ erts_continue_exit_process(Process *p) n = e = a; ASSERT(a & ERTS_PSFLG_EXITING); n |= ERTS_PSFLG_FREE; - n &= ~ERTS_PSFLG_ACTIVE; + n &= ~(ERTS_PSFLG_ACTIVE + | ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_ACTIVE_SYS); if ((n & ERTS_PSFLG_IN_RUNQ) && !refc_inced) { erts_proc_inc_refc(p); refc_inced = 1; @@ -13200,43 +12927,77 @@ erts_continue_exit_process(Process *p) ? ERTS_PROC_SET_DIST_ENTRY(p, NULL) : NULL); - erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL); + + /* + * It might show up signal prio elevation tasks until we + * have entered free state. Cleanup such tasks now. + */ + state = erts_atomic32_read_acqb(&p->state); + if (!(state & ERTS_PSFLG_SYS_TASKS)) + erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL); + else { + erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); + + do { + (void) cleanup_sys_tasks(p, state, CONTEXT_REDS); + state = erts_atomic32_read_acqb(&p->state); + } while (state & ERTS_PSFLG_SYS_TASKS); + + erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN); + } + +#ifdef DEBUG + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); + ASSERT(p->sys_task_qs == NULL); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); +#endif if (dep) { erts_do_net_exits(dep, (reason == am_kill) ? am_killed : reason); erts_deref_dist_entry(dep); } - /* - * Pre-build the EXIT tuple if there are any links. - */ - if (lnk) { - DeclareTmpHeap(tmp_heap,4,p); - Eterm exit_tuple; - Uint exit_tuple_sz; - Eterm* hp; + pectxt.c_p = p; + pectxt.reason = reason; - UseTmpHeap(4,p); - hp = &tmp_heap[0]; + erts_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_MSGQ); - exit_tuple = TUPLE3(hp, am_EXIT, p->common.id, reason); + erts_proc_sig_fetch(p); - exit_tuple_sz = size_object(exit_tuple); + erts_proc_unlock(p, ERTS_PROC_LOCK_MSGQ); - { - ExitLinkContext context = {p, reason, exit_tuple, exit_tuple_sz}; - erts_sweep_links(lnk, &doit_exit_link, &context); - } - UnUseTmpHeap(4,p); + if (links) { + erts_link_tree_foreach_delete(&links, + erts_proc_exit_handle_link, + (void *) &pectxt); + ASSERT(!links); } - { - ExitMonitorContext context = {reason, p}; - erts_sweep_monitors(mon,&doit_exit_monitor,&context); /* Allocates TmpHeap, but we - have none here */ + if (monitors) { + erts_monitor_tree_foreach_delete(&monitors, + erts_proc_exit_handle_monitor, + (void *) &pectxt); + ASSERT(!monitors); + } + + if (lt_monitors) { + erts_monitor_list_foreach_delete(<_monitors, + erts_proc_exit_handle_monitor, + (void *) &pectxt); + ASSERT(!lt_monitors); + } + + /* + * erts_proc_sig_handle_exit() implements yielding. + * However, this function cannot handle it yet... loop + * until done... + */ + while (!0) { + int reds = CONTEXT_REDS; + if (erts_proc_sig_handle_exit(p, &reds)) + break; } - erts_proc_lock(p, ERTS_PROC_LOCK_MAIN); ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); erts_flush_trace_messages(p, ERTS_PROC_LOCK_MAIN); @@ -13272,6 +13033,54 @@ erts_continue_exit_process(Process *p) BUMP_ALL_REDS(p); } +Process * +erts_try_lock_sig_free_proc(Eterm pid, ErtsProcLocks locks, + erts_aint32_t *statep) +{ + Process *rp = erts_proc_lookup_raw(pid); + erts_aint32_t state; + + if (!rp) { + if (statep) + *statep = ERTS_PSFLG_EXITING|ERTS_PSFLG_FREE; + return NULL; + } + + ERTS_LC_ASSERT(!erts_proc_lc_my_proc_locks(rp)); + + state = erts_atomic32_read_nob(&rp->state); + if (statep) + *statep = state; + + if (state & ERTS_PSFLG_FREE) + return NULL; + + if (state & (ERTS_PSFLG_SIG_IN_Q|ERTS_PSFLG_SIG_Q)) + return ERTS_PROC_LOCK_BUSY; + + if (!locks) + return rp; + + if (erts_proc_trylock(rp, locks) == EBUSY) + return ERTS_PROC_LOCK_BUSY; + + state = erts_atomic32_read_nob(&rp->state); + if (statep) + *statep = state; + + if (state & ERTS_PSFLG_FREE) { + erts_proc_unlock(rp, locks); + return NULL; + } + + if (state & (ERTS_PSFLG_SIG_IN_Q|ERTS_PSFLG_SIG_Q)) { + erts_proc_unlock(rp, locks); + return ERTS_PROC_LOCK_BUSY; + } + + return rp; +} + /* * Stack dump functions follow. */ diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index ebf990c837..b66272194c 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2017. All Rights Reserved. + * Copyright Ericsson AB 1996-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -51,7 +51,7 @@ typedef struct process Process; #include "erl_process_dict.h" #include "erl_node_container_utils.h" #include "erl_node_tables.h" -#include "erl_monitors.h" +#include "erl_monitor_link.h" #include "erl_hl_timer.h" #include "erl_time.h" #include "erl_atom_table.h" @@ -75,8 +75,6 @@ typedef struct process Process; #include "erl_thr_progress.h" #undef ERL_THR_PROGRESS_TSD_TYPE_ONLY -struct ErtsNodesMonitor_; - #define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 0 #define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT 0 @@ -311,7 +309,6 @@ typedef enum { ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX, ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX, ERTS_SSI_AUX_WORK_MISC_IX, - ERTS_SSI_AUX_WORK_PENDING_EXITERS_IX, ERTS_SSI_AUX_WORK_SET_TMO_IX, ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX, ERTS_SSI_AUX_WORK_YIELD_IX, @@ -345,8 +342,6 @@ typedef enum { (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX) #define ERTS_SSI_AUX_WORK_MISC \ (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX) -#define ERTS_SSI_AUX_WORK_PENDING_EXITERS \ - (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_PENDING_EXITERS_IX) #define ERTS_SSI_AUX_WORK_SET_TMO \ (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX) #define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \ @@ -405,9 +400,12 @@ typedef struct { } ErtsRunPrioQueue; typedef enum { - ERTS_SCHED_NORMAL, - ERTS_SCHED_DIRTY_CPU, - ERTS_SCHED_DIRTY_IO + ERTS_SCHED_NORMAL = 0, + ERTS_SCHED_DIRTY_CPU = 1, + ERTS_SCHED_DIRTY_IO = 2, + + ERTS_SCHED_TYPE_FIRST = ERTS_SCHED_NORMAL, + ERTS_SCHED_TYPE_LAST = ERTS_SCHED_DIRTY_IO } ErtsSchedType; typedef struct ErtsSchedulerData_ ErtsSchedulerData; @@ -491,7 +489,6 @@ struct ErtsRunQueue_ { int wakeup_other_reds; struct { - ErtsProcList *pending_exiters; Uint context_switches; Uint reductions; @@ -596,6 +593,7 @@ typedef struct { ErtsDelayedAuxWorkWakeupJob *job; } delayed_wakeup; struct { + ErtsAlcuBlockscanYieldData alcu_blockscan; ErtsEtsAllYieldData ets_all; /* Other yielding operations... */ } yield; @@ -640,6 +638,7 @@ struct ErtsSchedulerData_ { ErtsSchedType type; Uint no; /* Scheduler number for normal schedulers */ Uint dirty_no; /* Scheduler number for dirty schedulers */ + struct enif_environment_t *current_nif; Process *dirty_shadow_process; Port *current_port; ErtsRunQueue *run_queue; @@ -661,6 +660,13 @@ struct ErtsSchedulerData_ { Uint64 out; Uint64 in; } io; + struct { + ErtsSignal* sig; + Eterm to; +#ifdef DEBUG + Process* dbg_from; +#endif + } pending_signal; Uint64 reductions; ErtsSchedWallTime sched_wall_time; @@ -986,14 +992,10 @@ struct process { Process *next; /* Pointer to next process in run queue */ - struct ErtsNodesMonitor_ *nodes_monitors; - - ErtsSuspendMonitor *suspend_monitors; /* Processes suspended by - this process via - erlang:suspend_process/1 */ - - ErlMessageQueue msg; /* Message queue */ + ErtsMonitor *suspend_monitors; /* Processes suspended by this process via + erlang:suspend_process/1 */ + ErtsSignalPrivQueues sig_qs; /* Signal queues */ ErtsBifTimers *bif_timers; /* Bif timers aiming at this process */ ProcDict *dictionary; /* Process dictionary, may be NULL */ @@ -1052,9 +1054,8 @@ struct process { erts_atomic32_t state; /* Process state flags (see ERTS_PSFLG_*) */ erts_atomic32_t dirty_state; /* Process dirty state flags (see ERTS_PDSFLG_*) */ - ErlMessageInQueue msg_inq; + ErtsSignalInQueue sig_inq; ErlTraceMessageQueue *trace_msg_q; - ErtsPendExit pending_exit; erts_proc_lock_t lock; ErtsSchedulerData *scheduler_data; Eterm suspendee; @@ -1088,6 +1089,7 @@ struct process { #endif }; +extern Eterm erts_init_process_id; /* pid of init process */ extern const Process erts_invalid_process; #ifdef CHECK_FOR_HOLES @@ -1164,20 +1166,20 @@ void erts_check_for_holes(Process* p); #define ERTS_PSFLG_IN_PRQ_LOW ERTS_PSFLG_BIT(3) #define ERTS_PSFLG_FREE ERTS_PSFLG_BIT(4) #define ERTS_PSFLG_EXITING ERTS_PSFLG_BIT(5) -#define ERTS_PSFLG_PENDING_EXIT ERTS_PSFLG_BIT(6) +#define ERTS_PSFLG_UNUSED ERTS_PSFLG_BIT(6) #define ERTS_PSFLG_ACTIVE ERTS_PSFLG_BIT(7) #define ERTS_PSFLG_IN_RUNQ ERTS_PSFLG_BIT(8) #define ERTS_PSFLG_RUNNING ERTS_PSFLG_BIT(9) #define ERTS_PSFLG_SUSPENDED ERTS_PSFLG_BIT(10) #define ERTS_PSFLG_GC ERTS_PSFLG_BIT(11) -/* #define ERTS_PSFLG_ ERTS_PSFLG_BIT(12) */ -#define ERTS_PSFLG_TRAP_EXIT ERTS_PSFLG_BIT(13) +#define ERTS_PSFLG_SYS_TASKS ERTS_PSFLG_BIT(12) +#define ERTS_PSFLG_SIG_IN_Q ERTS_PSFLG_BIT(13) #define ERTS_PSFLG_ACTIVE_SYS ERTS_PSFLG_BIT(14) #define ERTS_PSFLG_RUNNING_SYS ERTS_PSFLG_BIT(15) #define ERTS_PSFLG_PROXY ERTS_PSFLG_BIT(16) #define ERTS_PSFLG_DELAYED_SYS ERTS_PSFLG_BIT(17) #define ERTS_PSFLG_OFF_HEAP_MSGQ ERTS_PSFLG_BIT(18) -/* #define ERTS_PSFLG_ ERTS_PSFLG_BIT(19) */ +#define ERTS_PSFLG_SIG_Q ERTS_PSFLG_BIT(19) #define ERTS_PSFLG_DIRTY_CPU_PROC ERTS_PSFLG_BIT(20) #define ERTS_PSFLG_DIRTY_IO_PROC ERTS_PSFLG_BIT(21) #define ERTS_PSFLG_DIRTY_ACTIVE_SYS ERTS_PSFLG_BIT(22) @@ -1196,7 +1198,6 @@ void erts_check_for_holes(Process* p); | ERTS_PSFLG_IN_PRQ_LOW) #define ERTS_PSFLGS_VOLATILE_HEAP (ERTS_PSFLG_EXITING \ - | ERTS_PSFLG_PENDING_EXIT \ | ERTS_PSFLG_DIRTY_RUNNING \ | ERTS_PSFLG_DIRTY_RUNNING_SYS) @@ -1261,7 +1262,24 @@ void erts_check_for_holes(Process* p); #define SEQ_TRACE_T_SENDER(token) (*(tuple_val(token) + 4)) #define SEQ_TRACE_T_LASTCNT(token) (*(tuple_val(token) + 5)) +#ifdef USE_VM_PROBES +/* The dtrace probe for seq_trace only supports 'int' labels, so we represent + * all values that won't fit into a 32-bit signed integer as ERTS_SINT32_MIN + * (bigints, tuples, etc). */ + +#define SEQ_TRACE_T_DTRACE_LABEL(token) \ + DTRACE_SEQ_TRACE_LABEL__(SEQ_TRACE_T_LABEL(token)) + +#define DTRACE_SEQ_TRACE_LABEL__(label_term) \ + (is_small((label_term)) ? \ + ((signed_val((label_term)) <= ERTS_SINT32_MAX && \ + signed_val((label_term)) >= ERTS_SINT32_MIN) ? \ + signed_val((label_term)) : ERTS_SINT32_MIN) \ + : ERTS_SINT32_MIN) +#endif + /* + * Possible flags for the flags field in ErlSpawnOpts below. */ @@ -1379,6 +1397,10 @@ extern int erts_system_profile_ts_type; #define F_DIRTY_MAJOR_GC (1 << 23) /* Dirty major GC scheduled */ #define F_DIRTY_MINOR_GC (1 << 24) /* Dirty minor GC scheduled */ #define F_HIBERNATED (1 << 25) /* Hibernated */ +#define F_LOCAL_SIGS_ONLY (1 << 26) +#define F_TRAP_EXIT (1 << 27) /* Trapping exit */ +#define F_DEFERRED_SAVED_LAST (1 << 28) +#define F_DELAYED_PSIGQS_LEN (1 << 29) /* * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent @@ -1444,7 +1466,7 @@ extern int erts_system_profile_ts_type; | F_TRACE_ARITY_ONLY | F_TRACE_RETURN_TO \ | F_TRACE_SILENT | F_TRACE_SCHED_PROCS | F_TRACE_PORTS \ | F_TRACE_SCHED_PORTS | F_TRACE_SCHED_NO \ - | F_TRACE_SCHED_EXIT) + | F_TRACE_SCHED_EXIT ) #define ERTS_TRACEE_MODIFIER_FLAGS \ @@ -1457,6 +1479,14 @@ extern int erts_system_profile_ts_type; #define SEQ_TRACE_FLAG(N) (1 << (ERTS_TRACE_TS_TYPE_BITS + (N))) +#define ERTS_SIG_ENABLE_TRACE_FLAGS \ + ( F_TRACE_RECEIVE | F_TRACE_PROCS) + +/* + * F_TRACE_RECEIVE is always enabled/disable via signaling. + * F_TRACE_PROCS enable/disable F_TRACE_PROCS_SIG via signaling. + */ + /* Sequential trace flags */ /* SEQ_TRACE_TIMESTAMP_MASK is a bit-field */ @@ -1483,10 +1513,6 @@ extern int erts_system_profile_ts_type; #define DT_UTAG_FLAGS(P) ((P)->dt_utag_flags) #endif -/* Option flags to erts_send_exit_signal() */ -#define ERTS_XSIG_FLG_IGN_KILL (((Uint32) 1) << 0) -#define ERTS_XSIG_FLG_NO_IGN_NORMAL (((Uint32) 1) << 1) - #define CANCEL_TIMER(P) \ do { \ if ((P)->flags & (F_INSLPQUEUE|F_TIMO)) { \ @@ -1716,9 +1742,9 @@ ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *list, #endif -int erts_sched_set_wakeup_other_thresold(char *str); -int erts_sched_set_wakeup_other_type(char *str); -int erts_sched_set_busy_wait_threshold(char *str); +int erts_sched_set_wakeup_other_threshold(ErtsSchedType sched_type, char *str); +int erts_sched_set_wakeup_other_type(ErtsSchedType sched_type, char *str); +int erts_sched_set_busy_wait_threshold(ErtsSchedType sched_type, char *str); int erts_sched_set_wake_cleanup_threshold(char *); void erts_schedule_thr_prgr_later_op(void (*)(void *), @@ -1733,6 +1759,7 @@ struct db_fixation; void erts_schedule_ets_free_fixation(Eterm pid, struct db_fixation*); void erts_schedule_flush_trace_messages(Process *proc, int force_on_proc); int erts_flush_trace_messages(Process *c_p, ErtsProcLocks locks); +int erts_sig_prio(Eterm pid, int prio); #if defined(ERTS_ENABLE_LOCK_CHECK) int erts_dbg_check_halloc_lock(Process *p); @@ -1781,8 +1808,10 @@ ErtsRunQueue *erts_schedid2runq(Uint); Process *erts_schedule(ErtsSchedulerData *, Process*, int); void erts_schedule_misc_op(void (*)(void *), void *); Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*); +void erts_set_self_exiting(Process *, Eterm); void erts_do_exit_process(Process*, Eterm); void erts_continue_exit_process(Process *); +void erts_proc_exit_link(Process *, ErtsLink *, Uint16, Eterm, Eterm); /* Begin System profile */ Uint erts_runnable_process_count(void); /* End System profile */ @@ -1798,8 +1827,19 @@ void erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp void erts_print_run_queue_info(fmtfn_t, void *to_arg, ErtsRunQueue*); void erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg); void erts_dump_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg); +Eterm erts_process_info(Process *c_p, ErtsHeapFactory *hfact, + Process *rp, ErtsProcLocks rp_locks, + int *item_ix, int item_ix_len, + int flags, Uint reserve_size, Uint *reds); -Eterm erts_get_process_priority(Process *p); +typedef struct { + Process *c_p; + Eterm reason; +} ErtsProcExitContext; +void erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt); +void erts_proc_exit_handle_link(ErtsLink *lnk, void *vctxt); + +Eterm erts_get_process_priority(erts_aint32_t state); Eterm erts_set_process_priority(Process *p, Eterm prio); Uint erts_get_total_context_switches(void); @@ -1817,18 +1857,6 @@ void erts_suspend(Process*, ErtsProcLocks, Port*); void erts_resume(Process*, ErtsProcLocks); int erts_resume_processes(ErtsProcList *); -int erts_send_exit_signal(Process *, - Eterm, - Process *, - ErtsProcLocks *, - Eterm, - Eterm, - Process *, - Uint32); -void erts_handle_pending_exit(Process *, ErtsProcLocks); -#define ERTS_PROC_PENDING_EXIT(P) \ - (ERTS_PSFLG_PENDING_EXIT & erts_atomic32_read_acqb(&(P)->state)) - void erts_deep_process_dump(fmtfn_t, void *); Eterm erts_get_reader_groups_map(Process *c_p); @@ -1841,7 +1869,7 @@ Uint erts_debug_nbalance(void); int erts_debug_wait_completed(Process *c_p, int flags); -Uint erts_process_memory(Process *c_p, int incl_msg_inq); +Uint erts_process_memory(Process *c_p, int include_sigs_in_transit); #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC # define ERTS_VERIFY_UNUSED_TEMP_ALLOC(P) \ @@ -1860,6 +1888,8 @@ do { \ ErtsSchedulerData *erts_get_scheduler_data(void); void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks); +erts_aint32_t erts_proc_sys_schedule(Process *p, erts_aint32_t state, + erts_aint32_t enable_flag); ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks); ERTS_GLB_INLINE void erts_schedule_dirty_sys_execution(Process *c_p); @@ -1891,8 +1921,7 @@ erts_schedule_dirty_sys_execution(Process *c_p) /* Don't set dirty-active-sys if we are about to exit... */ while (!(a & (ERTS_PSFLG_DIRTY_ACTIVE_SYS - | ERTS_PSFLG_EXITING - | ERTS_PSFLG_PENDING_EXIT))) { + | ERTS_PSFLG_EXITING))) { e = a; n = a | ERTS_PSFLG_DIRTY_ACTIVE_SYS; a = erts_atomic32_cmpxchg_mb(&c_p->state, n, e); @@ -2313,6 +2342,8 @@ erts_try_change_runq_proc(Process *p, ErtsRunQueue *rq) { erts_aint_t old_rqint, new_rqint; + ASSERT(rq); + new_rqint = (erts_aint_t) rq; old_rqint = (erts_aint_t) erts_atomic_read_nob(&p->run_queue); while (1) { @@ -2577,6 +2608,9 @@ ERTS_TIME2REDS_IMPL__(ErtsMonotonicTime start, ErtsMonotonicTime end) } #endif +Process *erts_try_lock_sig_free_proc(Eterm pid, + ErtsProcLocks locks, + erts_aint32_t *statep); Process *erts_pid2proc_not_running(Process *, ErtsProcLocks, diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index aee88841ae..38be3938cd 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -239,39 +239,70 @@ erts_erase_dicts(Process *p) /* * Called from process_info/1,2. */ -Eterm erts_dictionary_copy(Process *p, ProcDict *pd) +Eterm erts_dictionary_copy(ErtsHeapFactory *hfact, ProcDict *pd, Uint reserve_size) { - Eterm* hp; - Eterm* heap_start; - Eterm res = NIL; - Eterm tmp, tmp2; + Eterm res; unsigned int i, num; + Uint *sz; + Uint szi, rsz = reserve_size; - if (pd == NULL) { - return res; - } + if (pd == NULL) + return NIL; PD_CHECK(pd); num = HASH_RANGE(pd); - heap_start = hp = (Eterm *) erts_alloc(ERTS_ALC_T_TMP, - sizeof(Eterm) * pd->numElements * 2); - for (i = 0; i < num; ++i) { - tmp = ARRAY_GET(pd, i); + sz = (Uint *) erts_alloc(ERTS_ALC_T_TMP, sizeof(Uint) * pd->numElements); + + for (i = 0, szi = 0; i < num; ++i) { + Eterm tmp = ARRAY_GET(pd, i); if (is_boxed(tmp)) { + Uint size; ASSERT(is_tuple(tmp)); - res = CONS(hp, tmp, res); - hp += 2; - } else if (is_list(tmp)) { + size = size_object(tmp) + 2; + sz[szi++] = size; + rsz += size; + } + else if (is_list(tmp)) { while (tmp != NIL) { - tmp2 = TCAR(tmp); - res = CONS(hp, tmp2, res); - hp += 2; + Uint size = size_object(TCAR(tmp)) + 2; + sz[szi++] = size; + rsz += size; + tmp = TCDR(tmp); + } + } + } + + res = NIL; + + for (i = 0, szi = 0; i < num; ++i) { + Eterm tmp = ARRAY_GET(pd, i); + if (is_boxed(tmp)) { + Uint size; + Eterm el, *hp; + ASSERT(is_tuple(tmp)); + size = sz[szi++]; + rsz -= size; + hp = erts_produce_heap(hfact, size, rsz); + el = copy_struct(tmp, size-2, &hp, hfact->off_heap); + res = CONS(hp, el, res); + } + else if (is_list(tmp)) { + while (tmp != NIL) { + Uint size = sz[szi++]; + Eterm el, *hp; + rsz -= size; + hp = erts_produce_heap(hfact, size, rsz); + el = copy_struct(TCAR(tmp), size-2, &hp, hfact->off_heap); + res = CONS(hp, el, res); tmp = TCDR(tmp); } } } - res = copy_object(res, p); - erts_free(ERTS_ALC_T_TMP, (void *) heap_start); + + ASSERT(rsz == reserve_size); + + erts_free(ERTS_ALC_T_TMP, sz); + return res; } diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h index ab58f3c239..b89b387f5a 100644 --- a/erts/emulator/beam/erl_process_dict.h +++ b/erts/emulator/beam/erl_process_dict.h @@ -40,7 +40,7 @@ void erts_erase_dicts(struct process *p); void erts_dictionary_dump(fmtfn_t to, void *to_arg, ProcDict *pd); void erts_deep_dictionary_dump(fmtfn_t to, void *to_arg, ProcDict* pd, void (*cb)(fmtfn_t, void *, Eterm obj)); -Eterm erts_dictionary_copy(struct process *p, ProcDict *pd); +Eterm erts_dictionary_copy(ErtsHeapFactory *hfact, ProcDict *pd, Uint reserve_size); Eterm erts_pd_hash_get(struct process *p, Eterm id); Uint32 erts_pd_make_hx(Eterm key); diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 05e7bcdea2..243db4c734 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2017. All Rights Reserved. + * Copyright Ericsson AB 2003-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,6 +35,7 @@ #include "erl_map.h" #define ERTS_WANT_EXTERNAL_TAGS #include "external.h" +#include "erl_proc_sig_queue.h" #define PTR_FMT "%bpX" #define ETERM_FMT "%beX" @@ -96,72 +97,133 @@ erts_deep_process_dump(fmtfn_t to, void *to_arg) dump_binaries(to, to_arg, all_binaries); } -Uint erts_process_memory(Process *p, int incl_msg_inq) { - ErtsMessage *mp; - Uint size = 0; - struct saved_calls *scb; - size += sizeof(Process); - - if (incl_msg_inq) - ERTS_MSGQ_MV_INQ2PRIVQ(p); +static void +monitor_size(ErtsMonitor *mon, void *vsize) +{ + *((Uint *) vsize) += erts_monitor_size(mon); +} - erts_doforall_links(ERTS_P_LINKS(p), &erts_one_link_size, &size); - erts_doforall_monitors(ERTS_P_MONITORS(p), &erts_one_mon_size, &size); - size += (p->heap_sz + p->mbuf_sz) * sizeof(Eterm); - if (p->abandoned_heap) - size += (p->hend - p->heap) * sizeof(Eterm); - if (p->old_hend && p->old_heap) - size += (p->old_hend - p->old_heap) * sizeof(Eterm); +static void +link_size(ErtsMonitor *lnk, void *vsize) +{ + *((Uint *) vsize) += erts_link_size(lnk); +} +Uint erts_process_memory(Process *p, int include_sigs_in_transit) +{ + Uint size = 0; + struct saved_calls *scb; + + size += sizeof(Process); + + erts_link_tree_foreach(ERTS_P_LINKS(p), + link_size, (void *) &size); + erts_monitor_tree_foreach(ERTS_P_MONITORS(p), + monitor_size, (void *) &size); + erts_monitor_list_foreach(ERTS_P_LT_MONITORS(p), + monitor_size, (void *) &size); + size += (p->heap_sz + p->mbuf_sz) * sizeof(Eterm); + if (p->abandoned_heap) + size += (p->hend - p->heap) * sizeof(Eterm); + if (p->old_hend && p->old_heap) + size += (p->old_hend - p->old_heap) * sizeof(Eterm); + + if (!include_sigs_in_transit) { + /* + * Size of message queue! + * + * Note that this assumes that any part of message + * queue located in middle queue have been moved + * into the inner queue prior to this call. + * process_info() management ensures this is done- + */ + ErtsMessage *mp; + for (mp = p->sig_qs.first; mp; mp = mp->next) { + ASSERT(ERTS_SIG_IS_MSG((ErtsSignal *) mp)); + size += sizeof(ErtsMessage); + if (mp->data.attached) + size += erts_msg_attached_data_size(mp) * sizeof(Eterm); + } + } + else { + /* + * Size of message queue plus size of all signals + * in transit to the process! + */ + erts_proc_lock(p, ERTS_PROC_LOCK_MSGQ); + erts_proc_sig_fetch(p); + erts_proc_unlock(p, ERTS_PROC_LOCK_MSGQ); + + ERTS_FOREACH_SIG_PRIVQS( + p, mp, + { + size += sizeof(ErtsMessage); + if (ERTS_SIG_IS_NON_MSG((ErtsSignal *) mp)) + size += erts_proc_sig_signal_size((ErtsSignal *) mp); + else if (mp->data.attached) + size += erts_msg_attached_data_size(mp) * sizeof(Eterm); + }); + } - size += p->msg.len * sizeof(ErtsMessage); + if (p->arg_reg != p->def_arg_reg) { + size += p->arity * sizeof(p->arg_reg[0]); + } - for (mp = p->msg.first; mp; mp = mp->next) - if (mp->data.attached) - size += erts_msg_attached_data_size(mp)*sizeof(Eterm); + if (erts_atomic_read_nob(&p->psd) != (erts_aint_t) NULL) + size += sizeof(ErtsPSD); - if (p->arg_reg != p->def_arg_reg) { - size += p->arity * sizeof(p->arg_reg[0]); - } + scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p); + if (scb) { + size += (sizeof(struct saved_calls) + + (scb->len-1) * sizeof(scb->ct[0])); + } - if (erts_atomic_read_nob(&p->psd) != (erts_aint_t) NULL) - size += sizeof(ErtsPSD); + size += erts_dicts_mem_size(p); + return size; +} - scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p); - if (scb) { - size += (sizeof(struct saved_calls) - + (scb->len-1) * sizeof(scb->ct[0])); - } +static ERTS_INLINE void +dump_msg(fmtfn_t to, void *to_arg, ErtsMessage *mp) +{ + if (ERTS_SIG_IS_MSG((ErtsSignal *) mp)) { + Eterm mesg = ERL_MESSAGE_TERM(mp); + if (is_value(mesg)) + dump_element(to, to_arg, mesg); + else + dump_dist_ext(to, to_arg, mp->data.dist_ext); + mesg = ERL_MESSAGE_TOKEN(mp); + erts_print(to, to_arg, ":"); + dump_element(to, to_arg, mesg); + erts_print(to, to_arg, "\n"); + } +} - size += erts_dicts_mem_size(p); - return size; +static ERTS_INLINE void +heap_dump_msg(fmtfn_t to, void *to_arg, ErtsMessage *mp) +{ + if (ERTS_SIG_IS_MSG((ErtsSignal *) mp)) { + Eterm mesg = ERL_MESSAGE_TERM(mp); + if (is_value(mesg)) + heap_dump(to, to_arg, mesg); + mesg = ERL_MESSAGE_TOKEN(mp); + heap_dump(to, to_arg, mesg); + } } static void dump_process_info(fmtfn_t to, void *to_arg, Process *p) { Eterm* sp; - ErtsMessage* mp; int yreg = -1; if (ERTS_TRACE_FLAGS(p) & F_SENSITIVE) return; - ERTS_MSGQ_MV_INQ2PRIVQ(p); + erts_proc_sig_fetch(p); - if (p->msg.first) { + if (p->sig_qs.first || p->sig_qs.cont) { erts_print(to, to_arg, "=proc_messages:%T\n", p->common.id); - for (mp = p->msg.first; mp != NULL; mp = mp->next) { - Eterm mesg = ERL_MESSAGE_TERM(mp); - if (is_value(mesg)) - dump_element(to, to_arg, mesg); - else - dump_dist_ext(to, to_arg, mp->data.dist_ext); - mesg = ERL_MESSAGE_TOKEN(mp); - erts_print(to, to_arg, ":"); - dump_element(to, to_arg, mesg); - erts_print(to, to_arg, "\n"); - } + ERTS_FOREACH_SIG_PRIVQS(p, mp, dump_msg(to, to_arg, mp)); } if (p->dictionary) { @@ -183,13 +245,10 @@ dump_process_info(fmtfn_t to, void *to_arg, Process *p) heap_dump(to, to_arg, term); } } - for (mp = p->msg.first; mp != NULL; mp = mp->next) { - Eterm mesg = ERL_MESSAGE_TERM(mp); - if (is_value(mesg)) - heap_dump(to, to_arg, mesg); - mesg = ERL_MESSAGE_TOKEN(mp); - heap_dump(to, to_arg, mesg); - } + + if (p->sig_qs.first || p->sig_qs.cont) + ERTS_FOREACH_SIG_PRIVQS(p, mp, heap_dump_msg(to, to_arg, mp)); + if (p->dictionary) { erts_deep_dictionary_dump(to, to_arg, p->dictionary, heap_dump); } @@ -904,6 +963,9 @@ dump_module_literals(fmtfn_t to, void *to_arg, ErtsLiteralArea* lit_area) } erts_putc(to, to_arg, '\n'); } + } else if (is_export_header(w)) { + dump_externally(to, to_arg, term); + erts_putc(to, to_arg, '\n'); } size = 1 + header_arity(w); switch (w & _HEADER_SUBTAG_MASK) { @@ -997,8 +1059,8 @@ erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg) erts_print(to, to_arg, "FREE"); break; case ERTS_PSFLG_EXITING: erts_print(to, to_arg, "EXITING"); break; - case ERTS_PSFLG_PENDING_EXIT: - erts_print(to, to_arg, "PENDING_EXIT"); break; + case ERTS_PSFLG_UNUSED: + erts_print(to, to_arg, "UNUSED"); break; case ERTS_PSFLG_ACTIVE: erts_print(to, to_arg, "ACTIVE"); break; case ERTS_PSFLG_IN_RUNQ: @@ -1009,8 +1071,10 @@ erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg) erts_print(to, to_arg, "SUSPENDED"); break; case ERTS_PSFLG_GC: erts_print(to, to_arg, "GC"); break; - case ERTS_PSFLG_TRAP_EXIT: - erts_print(to, to_arg, "TRAP_EXIT"); break; + case ERTS_PSFLG_SYS_TASKS: + erts_print(to, to_arg, "SYS_TASKS"); break; + case ERTS_PSFLG_SIG_IN_Q: + erts_print(to, to_arg, "SIG_IN_Q"); break; case ERTS_PSFLG_ACTIVE_SYS: erts_print(to, to_arg, "ACTIVE_SYS"); break; case ERTS_PSFLG_RUNNING_SYS: @@ -1021,6 +1085,8 @@ erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg) erts_print(to, to_arg, "DELAYED_SYS"); break; case ERTS_PSFLG_OFF_HEAP_MSGQ: erts_print(to, to_arg, "OFF_HEAP_MSGQ"); break; + case ERTS_PSFLG_SIG_Q: + erts_print(to, to_arg, "SIG_Q"); break; case ERTS_PSFLG_DIRTY_CPU_PROC: erts_print(to, to_arg, "DIRTY_CPU_PROC"); break; case ERTS_PSFLG_DIRTY_IO_PROC: diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index 431867f27e..44c7892040 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2007-2017. All Rights Reserved. + * Copyright Ericsson AB 2007-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -101,7 +101,6 @@ static void cleanup_tse(void); #ifdef ERTS_ENABLE_LOCK_CHECK static struct { Sint16 proc_lock_main; - Sint16 proc_lock_link; Sint16 proc_lock_msgq; Sint16 proc_lock_btm; Sint16 proc_lock_status; @@ -140,7 +139,6 @@ erts_init_proc_lock(int cpus) #endif #ifdef ERTS_ENABLE_LOCK_CHECK lc_id.proc_lock_main = erts_lc_get_lock_order_id("proc_main"); - lc_id.proc_lock_link = erts_lc_get_lock_order_id("proc_link"); lc_id.proc_lock_msgq = erts_lc_get_lock_order_id("proc_msgq"); lc_id.proc_lock_btm = erts_lc_get_lock_order_id("proc_btm"); lc_id.proc_lock_status = erts_lc_get_lock_order_id("proc_status"); @@ -1055,12 +1053,6 @@ erts_proc_lock_init(Process *p) #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.main.lc); #endif - erts_mtx_init(&p->lock.link, "proc_link", p->common.id, - ERTS_LOCK_FLAGS_CATEGORY_PROCESS); - ethr_mutex_lock(&p->lock.link.mtx); -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_trylock(1, &p->lock.link.lc); -#endif erts_mtx_init(&p->lock.msgq, "proc_msgq", p->common.id, ERTS_LOCK_FLAGS_CATEGORY_PROCESS); ethr_mutex_lock(&p->lock.msgq.mtx); @@ -1102,7 +1094,6 @@ erts_proc_lock_fin(Process *p) { #if ERTS_PROC_LOCK_RAW_MUTEX_IMPL erts_mtx_destroy(&p->lock.main); - erts_mtx_destroy(&p->lock.link); erts_mtx_destroy(&p->lock.msgq); erts_mtx_destroy(&p->lock.btm); erts_mtx_destroy(&p->lock.status); @@ -1144,8 +1135,6 @@ void erts_lcnt_enable_proc_lock_count(Process *proc, int enable) { erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN, "proc_main", proc->common.id, ERTS_LOCK_TYPE_PROCLOCK); - erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK, - "proc_link", proc->common.id, ERTS_LOCK_TYPE_PROCLOCK); erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ, "proc_msgq", proc->common.id, ERTS_LOCK_TYPE_PROCLOCK); erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM, @@ -1200,10 +1189,6 @@ erts_proc_lc_lock(Process *p, ErtsProcLocks locks, char *file, unsigned int line lck.id = lc_id.proc_lock_main; erts_lc_lock_x(&lck,file,line); } - if (locks & ERTS_PROC_LOCK_LINK) { - lck.id = lc_id.proc_lock_link; - erts_lc_lock_x(&lck,file,line); - } if (locks & ERTS_PROC_LOCK_MSGQ) { lck.id = lc_id.proc_lock_msgq; erts_lc_lock_x(&lck,file,line); @@ -1233,10 +1218,6 @@ erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked, lck.id = lc_id.proc_lock_main; erts_lc_trylock_x(locked, &lck, file, line); } - if (locks & ERTS_PROC_LOCK_LINK) { - lck.id = lc_id.proc_lock_link; - erts_lc_trylock_x(locked, &lck, file, line); - } if (locks & ERTS_PROC_LOCK_MSGQ) { lck.id = lc_id.proc_lock_msgq; erts_lc_trylock_x(locked, &lck, file, line); @@ -1277,10 +1258,6 @@ erts_proc_lc_unlock(Process *p, ErtsProcLocks locks) lck.id = lc_id.proc_lock_msgq; erts_lc_unlock(&lck); } - if (locks & ERTS_PROC_LOCK_LINK) { - lck.id = lc_id.proc_lock_link; - erts_lc_unlock(&lck); - } if (locks & ERTS_PROC_LOCK_MAIN) { lck.id = lc_id.proc_lock_main; erts_lc_unlock(&lck); @@ -1312,10 +1289,6 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks) lck.id = lc_id.proc_lock_msgq; erts_lc_might_unlock(&lck); } - if (locks & ERTS_PROC_LOCK_LINK) { - lck.id = lc_id.proc_lock_link; - erts_lc_might_unlock(&lck); - } if (locks & ERTS_PROC_LOCK_MAIN) { lck.id = lc_id.proc_lock_main; erts_lc_might_unlock(&lck); @@ -1323,8 +1296,6 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks) #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL if (locks & ERTS_PROC_LOCK_MAIN) erts_lc_might_unlock(&p->lock.main.lc); - if (locks & ERTS_PROC_LOCK_LINK) - erts_lc_might_unlock(&p->lock.link.lc); if (locks & ERTS_PROC_LOCK_MSGQ) erts_lc_might_unlock(&p->lock.msgq.lc); if (locks & ERTS_PROC_LOCK_BTM) @@ -1348,10 +1319,6 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, char *file, lck.id = lc_id.proc_lock_main; erts_lc_require_lock(&lck, file, line); } - if (locks & ERTS_PROC_LOCK_LINK) { - lck.id = lc_id.proc_lock_link; - erts_lc_require_lock(&lck, file, line); - } if (locks & ERTS_PROC_LOCK_MSGQ) { lck.id = lc_id.proc_lock_msgq; erts_lc_require_lock(&lck, file, line); @@ -1371,8 +1338,6 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, char *file, #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL if (locks & ERTS_PROC_LOCK_MAIN) erts_lc_require_lock(&p->lock.main.lc, file, line); - if (locks & ERTS_PROC_LOCK_LINK) - erts_lc_require_lock(&p->lock.link.lc, file, line); if (locks & ERTS_PROC_LOCK_MSGQ) erts_lc_require_lock(&p->lock.msgq.lc, file, line); if (locks & ERTS_PROC_LOCK_BTM) @@ -1407,10 +1372,6 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks) lck.id = lc_id.proc_lock_msgq; erts_lc_unrequire_lock(&lck); } - if (locks & ERTS_PROC_LOCK_LINK) { - lck.id = lc_id.proc_lock_link; - erts_lc_unrequire_lock(&lck); - } if (locks & ERTS_PROC_LOCK_MAIN) { lck.id = lc_id.proc_lock_main; erts_lc_unrequire_lock(&lck); @@ -1418,8 +1379,6 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks) #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL if (locks & ERTS_PROC_LOCK_MAIN) erts_lc_unrequire_lock(&p->lock.main.lc); - if (locks & ERTS_PROC_LOCK_LINK) - erts_lc_unrequire_lock(&p->lock.link.lc); if (locks & ERTS_PROC_LOCK_MSGQ) erts_lc_unrequire_lock(&p->lock.msgq.lc); if (locks & ERTS_PROC_LOCK_BTM) @@ -1443,8 +1402,6 @@ erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks) if (locks & ERTS_PROC_LOCK_MAIN) lck.id = lc_id.proc_lock_main; - else if (locks & ERTS_PROC_LOCK_LINK) - lck.id = lc_id.proc_lock_link; else if (locks & ERTS_PROC_LOCK_MSGQ) lck.id = lc_id.proc_lock_msgq; else if (locks & ERTS_PROC_LOCK_BTM) @@ -1487,10 +1444,6 @@ void erts_proc_lc_chk_only_proc(Process *p, ErtsProcLocks locks) have_locks[have_locks_len].id = lc_id.proc_lock_main; have_locks[have_locks_len++].extra = p->common.id; } - if (locks & ERTS_PROC_LOCK_LINK) { - have_locks[have_locks_len].id = lc_id.proc_lock_link; - have_locks[have_locks_len++].extra = p->common.id; - } if (locks & ERTS_PROC_LOCK_MSGQ) { have_locks[have_locks_len].id = lc_id.proc_lock_msgq; have_locks[have_locks_len++].extra = p->common.id; @@ -1511,8 +1464,6 @@ void erts_proc_lc_chk_only_proc(Process *p, ErtsProcLocks locks) erts_lc_lock_t have_locks[6]; if (locks & ERTS_PROC_LOCK_MAIN) have_locks[have_locks_len++] = p->lock.main.lc; - if (locks & ERTS_PROC_LOCK_LINK) - have_locks[have_locks_len++] = p->lock.link.lc; if (locks & ERTS_PROC_LOCK_MSGQ) have_locks[have_locks_len++] = p->lock.msgq.lc; if (locks & ERTS_PROC_LOCK_BTM) @@ -1540,10 +1491,6 @@ erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks) have_locks[have_locks_len].id = lc_id.proc_lock_main; have_locks[have_locks_len++].extra = p->common.id; } - if (locks & ERTS_PROC_LOCK_LINK) { - have_locks[have_locks_len].id = lc_id.proc_lock_link; - have_locks[have_locks_len++].extra = p->common.id; - } if (locks & ERTS_PROC_LOCK_MSGQ) { have_locks[have_locks_len].id = lc_id.proc_lock_msgq; have_locks[have_locks_len++].extra = p->common.id; @@ -1564,8 +1511,6 @@ erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks) erts_lc_lock_t have_locks[6]; if (locks & ERTS_PROC_LOCK_MAIN) have_locks[have_locks_len++] = p->lock.main.lc; - if (locks & ERTS_PROC_LOCK_LINK) - have_locks[have_locks_len++] = p->lock.link.lc; if (locks & ERTS_PROC_LOCK_MSGQ) have_locks[have_locks_len++] = p->lock.msgq.lc; if (locks & ERTS_PROC_LOCK_BTM) @@ -1603,14 +1548,6 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) have_not_locks[have_not_locks_len].id = lc_id.proc_lock_main; have_not_locks[have_not_locks_len++].extra = p->common.id; } - if (locks & ERTS_PROC_LOCK_LINK) { - have_locks[have_locks_len].id = lc_id.proc_lock_link; - have_locks[have_locks_len++].extra = p->common.id; - } - else { - have_not_locks[have_not_locks_len].id = lc_id.proc_lock_link; - have_not_locks[have_not_locks_len++].extra = p->common.id; - } if (locks & ERTS_PROC_LOCK_MSGQ) { have_locks[have_locks_len].id = lc_id.proc_lock_msgq; have_locks[have_locks_len++].extra = p->common.id; @@ -1651,10 +1588,6 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) have_locks[have_locks_len++] = p->lock.main.lc; else have_not_locks[have_not_locks_len++] = p->lock.main.lc; - if (locks & ERTS_PROC_LOCK_LINK) - have_locks[have_locks_len++] = p->lock.link.lc; - else - have_not_locks[have_not_locks_len++] = p->lock.link.lc; if (locks & ERTS_PROC_LOCK_MSGQ) have_locks[have_locks_len++] = p->lock.msgq.lc; else @@ -1680,13 +1613,10 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) ErtsProcLocks erts_proc_lc_my_proc_locks(Process *p) { - int resv[6]; + int resv[5]; ErtsProcLocks res = 0; #if ERTS_PROC_LOCK_OWN_IMPL - erts_lc_lock_t locks[6] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main, - p->common.id, - ERTS_LOCK_TYPE_PROCLOCK), - ERTS_LC_LOCK_INIT(lc_id.proc_lock_link, + erts_lc_lock_t locks[5] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main, p->common.id, ERTS_LOCK_TYPE_PROCLOCK), ERTS_LC_LOCK_INIT(lc_id.proc_lock_msgq, @@ -1702,26 +1632,23 @@ erts_proc_lc_my_proc_locks(Process *p) p->common.id, ERTS_LOCK_TYPE_PROCLOCK)}; #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL - erts_lc_lock_t locks[6] = {p->lock.main.lc, - p->lock.link.lc, + erts_lc_lock_t locks[5] = {p->lock.main.lc, p->lock.msgq.lc, p->lock.btm.lc, p->lock.status.lc, p->lock.trace.lc}; #endif - erts_lc_have_locks(resv, locks, 6); + erts_lc_have_locks(resv, locks, 5); if (resv[0]) res |= ERTS_PROC_LOCK_MAIN; if (resv[1]) - res |= ERTS_PROC_LOCK_LINK; - if (resv[2]) res |= ERTS_PROC_LOCK_MSGQ; - if (resv[3]) + if (resv[2]) res |= ERTS_PROC_LOCK_BTM; - if (resv[4]) + if (resv[3]) res |= ERTS_PROC_LOCK_STATUS; - if (resv[5]) + if (resv[4]) res |= ERTS_PROC_LOCK_TRACE; return res; @@ -1730,15 +1657,14 @@ erts_proc_lc_my_proc_locks(Process *p) void erts_proc_lc_chk_no_proc_locks(char *file, int line) { - int resv[6]; - int ids[6] = {lc_id.proc_lock_main, - lc_id.proc_lock_link, + int resv[5]; + int ids[5] = {lc_id.proc_lock_main, lc_id.proc_lock_msgq, lc_id.proc_lock_btm, lc_id.proc_lock_status, lc_id.proc_lock_trace}; - erts_lc_have_lock_ids(resv, ids, 6); - if (!ERTS_IS_CRASH_DUMPING && (resv[0] || resv[1] || resv[2] || resv[3] || resv[4] || resv[5])) { + erts_lc_have_lock_ids(resv, ids, 5); + if (!ERTS_IS_CRASH_DUMPING && (resv[0] || resv[1] || resv[2] || resv[3] || resv[4])) { erts_lc_fail("%s:%d: Thread has process locks locked when expected " "not to have any process locks locked", file, line); diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index 9d5691d3c4..43f396c547 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2007-2017. All Rights Reserved. + * Copyright Ericsson AB 2007-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -66,7 +66,7 @@ #endif -#define ERTS_PROC_LOCK_MAX_BIT 5 +#define ERTS_PROC_LOCK_MAX_BIT 4 typedef erts_aint32_t ErtsProcLocks; @@ -82,19 +82,17 @@ typedef struct erts_proc_lock_t_ { /* Each erts_mtx_t has its own lock counter ^ */ #define ERTS_LCNT_PROCLOCK_IDX_MAIN 0 - #define ERTS_LCNT_PROCLOCK_IDX_LINK 1 - #define ERTS_LCNT_PROCLOCK_IDX_MSGQ 2 - #define ERTS_LCNT_PROCLOCK_IDX_BTM 3 - #define ERTS_LCNT_PROCLOCK_IDX_STATUS 4 - #define ERTS_LCNT_PROCLOCK_IDX_TRACE 5 + #define ERTS_LCNT_PROCLOCK_IDX_MSGQ 1 + #define ERTS_LCNT_PROCLOCK_IDX_BTM 2 + #define ERTS_LCNT_PROCLOCK_IDX_STATUS 3 + #define ERTS_LCNT_PROCLOCK_IDX_TRACE 4 - #define ERTS_LCNT_PROCLOCK_COUNT 6 + #define ERTS_LCNT_PROCLOCK_COUNT 5 erts_lcnt_ref_t lcnt_carrier; #endif #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL erts_mtx_t main; - erts_mtx_t link; erts_mtx_t msgq; erts_mtx_t btm; erts_mtx_t status; @@ -118,27 +116,18 @@ typedef struct erts_proc_lock_t_ { #define ERTS_PROC_LOCK_MAIN (((ErtsProcLocks) 1) << 0) /* - * Link lock: - * Protects the following fields in the process structure: - * * nlinks - * * monitors - * * suspend_monitors - */ -#define ERTS_PROC_LOCK_LINK (((ErtsProcLocks) 1) << 1) - -/* * Message queue lock: * Protects the following fields in the process structure: * * msg_inq */ -#define ERTS_PROC_LOCK_MSGQ (((ErtsProcLocks) 1) << 2) +#define ERTS_PROC_LOCK_MSGQ (((ErtsProcLocks) 1) << 1) /* * Bif timer lock: * Protects the following fields in the process structure: * * bif_timers */ -#define ERTS_PROC_LOCK_BTM (((ErtsProcLocks) 1) << 3) +#define ERTS_PROC_LOCK_BTM (((ErtsProcLocks) 1) << 2) /* * Status lock: @@ -148,7 +137,7 @@ typedef struct erts_proc_lock_t_ { * * sys_tasks * * ... */ -#define ERTS_PROC_LOCK_STATUS (((ErtsProcLocks) 1) << 4) +#define ERTS_PROC_LOCK_STATUS (((ErtsProcLocks) 1) << 3) /* * Trace message lock: @@ -277,9 +266,6 @@ void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks) { if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN); } - if (locks & ERTS_PROC_LOCK_LINK) { - erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK); - } if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ); } @@ -307,9 +293,6 @@ void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN, file, line); } - if (locks & ERTS_PROC_LOCK_LINK) { - erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK, file, line); - } if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ, file, line); } @@ -336,9 +319,6 @@ void erts_lcnt_proc_lock_unacquire(erts_proc_lock_t *lock, ErtsProcLocks locks) if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN); } - if (locks & ERTS_PROC_LOCK_LINK) { - erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK); - } if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ); } @@ -365,9 +345,6 @@ void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks) { if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN); } - if (locks & ERTS_PROC_LOCK_LINK) { - erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK); - } if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ); } @@ -394,9 +371,6 @@ void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN, res); } - if (locks & ERTS_PROC_LOCK_LINK) { - erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK, res); - } if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ, res); } @@ -640,9 +614,6 @@ erts_proc_raw_trylock__(Process *p, ErtsProcLocks locks) if (locks & ERTS_PROC_LOCK_MAIN) if (erts_mtx_trylock(&p->lock.main) == EBUSY) goto busy_main; - if (locks & ERTS_PROC_LOCK_LINK) - if (erts_mtx_trylock(&p->lock.link) == EBUSY) - goto busy_link; if (locks & ERTS_PROC_LOCK_MSGQ) if (erts_mtx_trylock(&p->lock.msgq) == EBUSY) goto busy_msgq; @@ -668,9 +639,6 @@ busy_btm: if (locks & ERTS_PROC_LOCK_MSGQ) erts_mtx_unlock(&p->lock.msgq); busy_msgq: - if (locks & ERTS_PROC_LOCK_LINK) - erts_mtx_unlock(&p->lock.link); -busy_link: if (locks & ERTS_PROC_LOCK_MAIN) erts_mtx_unlock(&p->lock.main); busy_main: @@ -741,8 +709,6 @@ erts_proc_lock__(Process *p, #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL if (locks & ERTS_PROC_LOCK_MAIN) erts_mtx_lock(&p->lock.main); - if (locks & ERTS_PROC_LOCK_LINK) - erts_mtx_lock(&p->lock.link); if (locks & ERTS_PROC_LOCK_MSGQ) erts_mtx_lock(&p->lock.msgq); if (locks & ERTS_PROC_LOCK_BTM) @@ -844,8 +810,6 @@ erts_proc_unlock__(Process *p, erts_mtx_unlock(&p->lock.btm); if (locks & ERTS_PROC_LOCK_MSGQ) erts_mtx_unlock(&p->lock.msgq); - if (locks & ERTS_PROC_LOCK_LINK) - erts_mtx_unlock(&p->lock.link); if (locks & ERTS_PROC_LOCK_MAIN) erts_mtx_unlock(&p->lock.main); #endif diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h index 4858cc8ab8..94f0247492 100644 --- a/erts/emulator/beam/erl_ptab.h +++ b/erts/emulator/beam/erl_ptab.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012-2016. All Rights Reserved. + * Copyright Ericsson AB 2012-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,7 +35,7 @@ #include "erl_thr_progress.h" #undef ERL_THR_PROGRESS_TSD_TYPE_ONLY #include "erl_alloc.h" -#include "erl_monitors.h" +#include "erl_monitor_link.h" #define ERTS_TRACER(P) ((P)->common.tracer) #define ERTS_TRACER_MODULE(T) (CAR(list_val(T))) @@ -44,6 +44,7 @@ #define ERTS_P_LINKS(P) ((P)->common.u.alive.links) #define ERTS_P_MONITORS(P) ((P)->common.u.alive.monitors) +#define ERTS_P_LT_MONITORS(P) ((P)->common.u.alive.lt_monitors) #define IS_TRACED(p) \ (ERTS_TRACER(p) != NIL) @@ -68,6 +69,7 @@ typedef struct { struct reg_proc *reg; ErtsLink *links; ErtsMonitor *monitors; + ErtsMonitor *lt_monitors; } alive; /* --- While being released --- */ diff --git a/erts/emulator/beam/erl_rbtree.h b/erts/emulator/beam/erl_rbtree.h index e59d6900b0..e50abf5cec 100644 --- a/erts/emulator/beam/erl_rbtree.h +++ b/erts/emulator/beam/erl_rbtree.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2015-2017. All Rights Reserved. + * Copyright Ericsson AB 2015-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -50,8 +50,14 @@ * - ERTS_RBT_GET_LEFT(T) - Get left child node. * - ERTS_RBT_SET_LEFT(T, L) - Set left child node. * - ERTS_RBT_GET_KEY(T) - Get key of node. - * - ERTS_RBT_IS_LT(KX, KY) - Is key KX less than key KY? - * - ERTS_RBT_IS_EQ(KX, KY) - Is key KX equal to key KY? + * Either: + * - ERTS_RBT_CMP_KEYS(KX, KY) - Compare keys... + * or: + * - ERTS_RBT_IS_LT(KX, KY) - Is key KX less than key KY? + * - ERTS_RBT_IS_EQ(KX, KY) - Is key KX equal to key KY? + * + * If ERTS_RBT_CMP_KEYS is defined ERTS_RBT_IS_LT and + * ERTS_RBT_IS_EQ will be redefined using ERTS_RBT_CMP_KEYS * * Optional defines: * @@ -337,6 +343,15 @@ * Should only be used for debuging. */ +#ifdef ERTS_RBT_CMP_KEYS + +# undef ERTS_RBT_IS_LT +# define ERTS_RBT_IS_LT(KX, KY) (ERTS_RBT_CMP_KEYS((KX), (KY)) < 0) + +# undef ERTS_RBT_IS_EQ +# define ERTS_RBT_IS_EQ(KX, KY) (ERTS_RBT_CMP_KEYS((KX), (KY)) == 0) + +#endif /* * Check that we have all mandatory defines @@ -396,6 +411,16 @@ # error Missing definition of ERTS_RBT_IS_EQ #endif +#undef ERTS_RBT_IS_GT__ +#ifdef ERTS_RBT_CMP_KEYS +# define ERTS_RBT_IS_GT__(KX, KY) \ + (ERTS_RBT_CMP_KEYS((KX), (KY)) > 0) +#else +# define ERTS_RBT_IS_GT__(KX, KY) \ + (!ERTS_RBT_IS_LT((KX), (KY)) && !ERTS_RBT_IS_EQ((KX), (KY))) + +#endif + #if defined(ERTS_RBT_HARD_DEBUG) || defined(DEBUG) # ifndef ERTS_RBT_DEBUG # define ERTS_RBT_DEBUG 1 @@ -1007,19 +1032,30 @@ ERTS_RBT_FUNC__(insert_aux__)(ERTS_RBT_T **root, ERTS_RBT_T *n, int lookup) ERTS_RBT_T *p, *x = *root; while (1) { - ERTS_RBT_KEY_T kx; + ERTS_RBT_KEY_T kx = ERTS_RBT_GET_KEY(x); ERTS_RBT_T *c; + int kres; +#ifdef ERTS_RBT_CMP_KEYS + int kcmp = ERTS_RBT_CMP_KEYS(kn, kx); + kres = kcmp == 0; +#else + kres = ERTS_RBT_IS_EQ(kn, kx); +#endif - kx = ERTS_RBT_GET_KEY(x); - - if (lookup && ERTS_RBT_IS_EQ(kn, kx)) { + if (lookup && kres) { ERTS_RBT_HDBG_CHECK_TREE__(*root, NULL); return x; } - if (ERTS_RBT_IS_LT(kn, kx)) { +#ifdef ERTS_RBT_CMP_KEYS + kres = kcmp < 0; +#else + kres = ERTS_RBT_IS_LT(kn, kx); +#endif + + if (kres) { c = ERTS_RBT_GET_LEFT(x); if (!c) { ERTS_RBT_SET_PARENT(n, x); @@ -1075,6 +1111,101 @@ ERTS_RBT_FUNC__(insert)(ERTS_RBT_T **root, ERTS_RBT_T *n) #endif /* ERTS_RBT_WANT_INSERT */ +#ifdef ERTS_RBT_WANT_LOOKUP_CREATE +static ERTS_INLINE ERTS_RBT_T * +ERTS_RBT_FUNC__(lookup_create)(ERTS_RBT_T **root, + ERTS_RBT_KEY_T kn, + ERTS_RBT_T *(*create)(ERTS_RBT_KEY_T, void *), + void *arg, + int *created) +{ + ERTS_RBT_T *n; + ERTS_RBT_HDBG_CHECK_TREE__(*root, NULL); + + if (!*root) { + n = (*create)(kn, arg); + ERTS_RBT_INIT_EMPTY_TNODE(n); + ERTS_RBT_ASSERT(ERTS_RBT_IS_EQ(ERTS_RBT_GET_KEY(n), kn)); + ERTS_RBT_SET_BLACK(n); + *root = n; + *created = !0; +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT + ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(NULL, n); +#endif + } + else { + ERTS_RBT_T *p, *x = *root; + + while (1) { + ERTS_RBT_KEY_T kx = ERTS_RBT_GET_KEY(x); + ERTS_RBT_T *c; + int kres; +#ifdef ERTS_RBT_CMP_KEYS + int kcmp = ERTS_RBT_CMP_KEYS(kn, kx); + kres = kcmp == 0; +#else + kres = ERTS_RBT_IS_EQ(kn, kx); +#endif + + if (kres) { + + ERTS_RBT_HDBG_CHECK_TREE__(*root, NULL); + + *created = 0; + return x; + } + +#ifdef ERTS_RBT_CMP_KEYS + kres = kcmp < 0; +#else + kres = ERTS_RBT_IS_LT(kn, kx); +#endif + + if (kres) { + c = ERTS_RBT_GET_LEFT(x); + if (!c) { + n = (*create)(kn, arg); + ERTS_RBT_INIT_EMPTY_TNODE(n); + ERTS_RBT_ASSERT(ERTS_RBT_IS_EQ(ERTS_RBT_GET_KEY(n), kn)); + *created = !0; + ERTS_RBT_SET_PARENT(n, x); + ERTS_RBT_SET_LEFT(x, n); + p = x; + break; + } + } + else { + c = ERTS_RBT_GET_RIGHT(x); + if (!c) { + n = (*create)(kn, arg); + ERTS_RBT_INIT_EMPTY_TNODE(n); + ERTS_RBT_ASSERT(ERTS_RBT_IS_EQ(ERTS_RBT_GET_KEY(n), kn)); + *created = !0; + ERTS_RBT_SET_PARENT(n, x); + ERTS_RBT_SET_RIGHT(x, n); + p = x; + break; + } + } + + x = c; + } + + ERTS_RBT_ASSERT(ERTS_RBT_IS_EQ(ERTS_RBT_GET_KEY(n), kn)); + ERTS_RBT_ASSERT(p); + + ERTS_RBT_SET_RED(n); + if (ERTS_RBT_IS_RED(p)) + ERTS_RBT_FUNC__(insert_fixup__)(root, n); + } + + ERTS_RBT_HDBG_CHECK_TREE__(*root, n); + + return n; +} + +#endif /* ERTS_RBT_WANT_LOOKUP_CREATE */ + #ifdef ERTS_RBT_WANT_LOOKUP static ERTS_RBT_API_INLINE__ ERTS_RBT_T * @@ -1088,11 +1219,24 @@ ERTS_RBT_FUNC__(lookup)(ERTS_RBT_T *root, ERTS_RBT_KEY_T key) while (1) { ERTS_RBT_KEY_T kx = ERTS_RBT_GET_KEY(x); ERTS_RBT_T *c; + int kres; +#ifdef ERTS_RBT_CMP_KEYS + int kcmp = ERTS_RBT_CMP_KEYS(key, kx); + kres = kcmp == 0; +#else + kres = ERTS_RBT_IS_EQ(key, kx); +#endif - if (ERTS_RBT_IS_EQ(key, kx)) + if (kres) return x; - if (ERTS_RBT_IS_LT(key, kx)) { +#ifdef ERTS_RBT_CMP_KEYS + kres = kcmp < 0; +#else + kres = ERTS_RBT_IS_LT(key, kx); +#endif + + if (kres) { c = ERTS_RBT_GET_LEFT(x); if (!c) return NULL; @@ -1426,14 +1570,14 @@ ERTS_RBT_FUNC__(foreach_large)(ERTS_RBT_T *root, #ifdef ERTS_RBT_WANT_FOREACH_YIELDING -static ERTS_RBT_API_INLINE__ void +static ERTS_RBT_API_INLINE__ int ERTS_RBT_FUNC__(foreach_yielding)(ERTS_RBT_T *root, void (*op)(ERTS_RBT_T *, void *), void *arg, ERTS_RBT_YIELD_STATE_T__ *ystate, Sint ylimit) { - (void) ERTS_RBT_FUNC__(foreach_unordered__)(*root, 0, op, arg, + return ERTS_RBT_FUNC__(foreach_unordered__)(&root, 0, op, arg, 1, ystate, ylimit); } @@ -1630,8 +1774,7 @@ ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root, ERTS_RBT_T *n) kx = ERTS_RBT_GET_KEY(x); kc = ERTS_RBT_GET_KEY(c); - ERTS_RBT_ASSERT(ERTS_RBT_IS_LT(kc, kx) - || ERTS_RBT_IS_EQ(kc, kx)); + ERTS_RBT_ASSERT(!ERTS_RBT_IS_GT__(kc, kx)); x = c; } @@ -1649,8 +1792,8 @@ ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root, ERTS_RBT_T *n) kx = ERTS_RBT_GET_KEY(x); kc = ERTS_RBT_GET_KEY(c); - ERTS_RBT_ASSERT(ERTS_RBT_IS_LT(kx, kc) - || ERTS_RBT_IS_EQ(kx, kc)); + ERTS_RBT_ASSERT(!ERTS_RBT_IS_GT__(kx, kc)); + x = c; } @@ -1672,8 +1815,8 @@ ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root, ERTS_RBT_T *n) kx = ERTS_RBT_GET_KEY(x); kc = ERTS_RBT_GET_KEY(c); - ERTS_RBT_ASSERT(ERTS_RBT_IS_LT(kx, kc) - || ERTS_RBT_IS_EQ(kx, kc)); + ERTS_RBT_ASSERT(!ERTS_RBT_IS_GT__(kx, kc)); + /* Go down tree of x's sibling... */ x = c; break; @@ -1707,6 +1850,7 @@ ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root, ERTS_RBT_T *n) #undef ERTS_RBT_NEED_FOREACH_ORDERED__ #undef ERTS_RBT_NEED_HDBG_CHECK_TREE__ #undef ERTS_RBT_HDBG_CHECK_TREE__ +#undef ERTS_RBT_IS_GT__ #ifdef ERTS_RBT_UNDEF # undef ERTS_RBT_PREFIX @@ -1727,6 +1871,7 @@ ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root, ERTS_RBT_T *n) # undef ERTS_RBT_GET_LEFT # undef ERTS_RBT_SET_LEFT # undef ERTS_RBT_GET_KEY +# undef ERTS_RBT_CMP_KEYS # undef ERTS_RBT_IS_LT # undef ERTS_RBT_IS_EQ # undef ERTS_RBT_UNDEF diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index 3dd1c2555c..bddf403b0a 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2017. All Rights Reserved. + * Copyright Ericsson AB 2000-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -345,6 +345,9 @@ _ET_DECLARE_CHECKED(Uint,thing_subtag,Eterm) * * To help find code which makes unwarranted assumptions about zero, * we now use a non-zero bit-pattern in debug mode. + * + * In order to be able to differentiata against values, the non-value + * needs to be tagged as a header of some sort. */ #if ET_DEBUG # ifdef HIPE @@ -355,7 +358,7 @@ _ET_DECLARE_CHECKED(Uint,thing_subtag,Eterm) # define THE_NON_VALUE _make_header(0,_TAG_HEADER_FLOAT) # endif #else -#define THE_NON_VALUE (0) +#define THE_NON_VALUE (TAG_PRIMARY_HEADER) #endif #define is_non_value(x) ((x) == THE_NON_VALUE) #define is_value(x) ((x) != THE_NON_VALUE) @@ -1181,6 +1184,54 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm) #define is_map(x) (is_boxed((x)) && is_map_header(*boxed_val(x))) #define is_not_map(x) (!is_map(x)) +#define MAP_HEADER(hp, sz, keys) \ + ((hp)[0] = MAP_HEADER_FLATMAP, \ + (hp)[1] = sz, \ + (hp)[2] = keys) + +#define MAP_SZ(sz) (MAP_HEADER_FLATMAP_SZ + 2*sz + 1) + +#define MAP0_SZ MAP_SZ(0) +#define MAP1_SZ MAP_SZ(1) +#define MAP2_SZ MAP_SZ(2) +#define MAP3_SZ MAP_SZ(3) +#define MAP4_SZ MAP_SZ(4) +#define MAP5_SZ MAP_SZ(5) +#define MAP0(hp) \ + (MAP_HEADER(hp, 0, TUPLE0(hp+MAP_HEADER_FLATMAP_SZ)), \ + make_flatmap(hp)) +#define MAP1(hp, k1, v1) \ + (MAP_HEADER(hp, 1, TUPLE1(hp+1+MAP_HEADER_FLATMAP_SZ, k1)), \ + (hp)[MAP_HEADER_FLATMAP_SZ+0] = v1, \ + make_flatmap(hp)) +#define MAP2(hp, k1, v1, k2, v2) \ + (MAP_HEADER(hp, 2, TUPLE2(hp+2+MAP_HEADER_FLATMAP_SZ, k1, k2)), \ + (hp)[MAP_HEADER_FLATMAP_SZ+0] = v1, \ + (hp)[MAP_HEADER_FLATMAP_SZ+1] = v2, \ + make_flatmap(hp)) +#define MAP3(hp, k1, v1, k2, v2, k3, v3) \ + (MAP_HEADER(hp, 3, TUPLE3(hp+3+MAP_HEADER_FLATMAP_SZ, k1, k2, k3)), \ + (hp)[MAP_HEADER_FLATMAP_SZ+0] = v1, \ + (hp)[MAP_HEADER_FLATMAP_SZ+1] = v2, \ + (hp)[MAP_HEADER_FLATMAP_SZ+2] = v3, \ + make_flatmap(hp)) +#define MAP4(hp, k1, v1, k2, v2, k3, v3, k4, v4) \ + (MAP_HEADER(hp, 4, TUPLE4(hp+4+MAP_HEADER_FLATMAP_SZ, k1, k2, k3, k4)), \ + (hp)[MAP_HEADER_FLATMAP_SZ+0] = v1, \ + (hp)[MAP_HEADER_FLATMAP_SZ+1] = v2, \ + (hp)[MAP_HEADER_FLATMAP_SZ+2] = v3, \ + (hp)[MAP_HEADER_FLATMAP_SZ+3] = v4, \ + make_flatmap(hp)) +#define MAP5(hp, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5) \ + (MAP_HEADER(hp, 5, TUPLE5(hp+5+MAP_HEADER_FLATMAP_SZ, k1, k2, k3, k4, k5)), \ + (hp)[MAP_HEADER_FLATMAP_SZ+0] = v1, \ + (hp)[MAP_HEADER_FLATMAP_SZ+1] = v2, \ + (hp)[MAP_HEADER_FLATMAP_SZ+2] = v3, \ + (hp)[MAP_HEADER_FLATMAP_SZ+3] = v4, \ + (hp)[MAP_HEADER_FLATMAP_SZ+4] = v5, \ + make_flatmap(hp)) + + /* number tests */ #define is_integer(x) (is_small(x) || is_big(x)) diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 65211e4e6f..968f21fd51 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2017. All Rights Reserved. + * Copyright Ericsson AB 2006-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,8 @@ #ifndef ERL_TIME_H__ #define ERL_TIME_H__ +#include "erl_monitor_link.h" + #if 0 # define ERTS_TW_DEBUG #endif @@ -79,8 +81,8 @@ typedef ErtsMonotonicTime * ErtsNextTimeoutRef; extern SysTimeval erts_first_emu_time; -void erts_monitor_time_offset(Eterm id, Eterm ref); -int erts_demonitor_time_offset(Eterm ref); +void erts_monitor_time_offset(ErtsMonitor *mon); +void erts_demonitor_time_offset(ErtsMonitor *mon); int erts_init_time_sup(int, ErtsTimeWarpMode); void erts_late_init_time_sup(void); diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 8cbdf9fa0f..4f91d9ad07 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2017. All Rights Reserved. + * Copyright Ericsson AB 1999-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,6 +35,7 @@ #include "erl_time.h" #include "erl_driver.h" #include "erl_nif.h" +#include "erl_proc_sig_queue.h" static erts_mtx_t erts_get_time_mtx; @@ -1870,36 +1871,33 @@ void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec) { #include "big.h" void -erts_monitor_time_offset(Eterm id, Eterm ref) +erts_monitor_time_offset(ErtsMonitor *mon) { erts_mtx_lock(&erts_get_time_mtx); - erts_add_monitor(&time_offset_monitors, MON_TIME_OFFSET, ref, id, NIL); + erts_monitor_list_insert(&time_offset_monitors, mon); no_time_offset_monitors++; erts_mtx_unlock(&erts_get_time_mtx); } -int -erts_demonitor_time_offset(Eterm ref) +void +erts_demonitor_time_offset(ErtsMonitor *mon) { - int res; - ErtsMonitor *mon; - ASSERT(is_internal_ref(ref)); + ErtsMonitorData *mdp = erts_monitor_to_data(mon); + ASSERT(erts_monitor_is_origin(mon)); + ASSERT(mon->type == ERTS_MON_TYPE_TIME_OFFSET); + erts_mtx_lock(&erts_get_time_mtx); - if (is_internal_ordinary_ref(ref)) - mon = erts_remove_monitor(&time_offset_monitors, ref); - else - mon = NULL; - if (!mon) - res = 0; - else { - ASSERT(no_time_offset_monitors > 0); - no_time_offset_monitors--; - res = 1; - } + + ASSERT(erts_monitor_is_in_table(&mdp->target)); + + erts_monitor_list_delete(&time_offset_monitors, &mdp->target); + + ASSERT(no_time_offset_monitors > 0); + no_time_offset_monitors--; + erts_mtx_unlock(&erts_get_time_mtx); - if (res) - erts_destroy_monitor(mon); - return res; + + erts_monitor_release_both(mdp); } typedef struct { @@ -1917,17 +1915,19 @@ static void save_time_offset_monitor(ErtsMonitor *mon, void *vcntxt) { ErtsTimeOffsetMonitorContext *cntxt; + ErtsMonitorData *mdp = erts_monitor_to_data(mon); Eterm *from_hp, *to_hp; Uint mix; int hix; cntxt = (ErtsTimeOffsetMonitorContext *) vcntxt; mix = (cntxt->ix)++; - cntxt->to_mon_info[mix].pid = mon->u.pid; + ASSERT(is_internal_pid(mon->other.item)); + cntxt->to_mon_info[mix].pid = mon->other.item; to_hp = &cntxt->to_mon_info[mix].heap[0]; - ASSERT(is_internal_ordinary_ref(mon->ref)); - from_hp = internal_ref_val(mon->ref); + ASSERT(is_internal_ordinary_ref(mdp->ref)); + from_hp = internal_ref_val(mdp->ref); ASSERT(thing_arityval(*from_hp) + 1 == ERTS_REF_THING_SIZE); for (hix = 0; hix < ERTS_REF_THING_SIZE; hix++) @@ -1972,9 +1972,9 @@ send_time_offset_changed_notifications(void *new_offsetp) cntxt.ix = 0; cntxt.to_mon_info = to_mon_info; - erts_doforall_monitors(time_offset_monitors, - save_time_offset_monitor, - &cntxt); + erts_monitor_list_foreach(time_offset_monitors, + save_time_offset_monitor, + &cntxt); ASSERT(cntxt.ix == no_monitors); } @@ -2008,26 +2008,14 @@ send_time_offset_changed_notifications(void *new_offsetp) ASSERT(*patch_refp == THE_NON_VALUE); for (mix = 0; mix < no_monitors; mix++) { - Process *rp = erts_proc_lookup(to_mon_info[mix].pid); - if (rp) { - Eterm ref = to_mon_info[mix].ref; - ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK; - erts_proc_lock(rp, ERTS_PROC_LOCK_LINK); - if (erts_lookup_monitor(ERTS_P_MONITORS(rp), ref)) { - ErtsMessage *mp; - ErlOffHeap *ohp; - Eterm message; - - mp = erts_alloc_message_heap(rp, &rp_locks, - hsz, &hp, &ohp); - *patch_refp = ref; - ASSERT(hsz == size_object(message_template)); - message = copy_struct(message_template, hsz, &hp, ohp); - erts_queue_message(rp, rp_locks, mp, message, am_clock_service); - } - erts_proc_unlock(rp, rp_locks); - } - } + *patch_refp = to_mon_info[mix].ref; + erts_proc_sig_send_persistent_monitor_msg(ERTS_MON_TYPE_TIME_OFFSET, + *patch_refp, + am_clock_service, + to_mon_info[mix].pid, + message_template, + hsz); + } erts_free(ERTS_ALC_T_TMP, tmp); } @@ -2216,6 +2204,8 @@ time_unit_conversion(Process *c_p, Eterm term, ErtsMonotonicTime val, ErtsMonoto ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); break; #endif + case am_perf_counter: + goto trap_to_erlang_code; default: { Eterm value, native_res; #ifndef ARCH_64 diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 018894f685..065a560b52 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -2044,7 +2044,7 @@ enqueue_sys_msg(enum ErtsSysMsgType type, void erts_queue_error_logger_message(Eterm from, Eterm msg, ErlHeapFragment *bp) { - enqueue_sys_msg(SYS_MSG_TYPE_ERRLGR, from, am_error_logger, msg, bp); + enqueue_sys_msg(SYS_MSG_TYPE_ERRLGR, from, am_logger, msg, bp); } void @@ -2110,13 +2110,13 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver) erts_thr_progress_unblock(); break; case SYS_MSG_TYPE_ERRLGR: { - char *no_elgger = "(no error logger present)"; + char *no_elgger = "(no logger present)"; Eterm *tp; Eterm tag; if (is_not_tuple(smqp->msg)) { unexpected_elmsg: erts_fprintf(stderr, - "%s unexpected error logger message: %T\n", + "%s unexpected logger message: %T\n", no_elgger, smqp->msg); } @@ -2284,7 +2284,7 @@ sys_msg_dispatcher_func(void *unused) } break; case SYS_MSG_TYPE_ERRLGR: - receiver = am_error_logger; + receiver = am_logger; break; default: receiver = NIL; @@ -2313,7 +2313,7 @@ sys_msg_dispatcher_func(void *unused) erts_proc_unlock(proc, proc_locks); } } - else if (receiver == am_error_logger) { + else if (receiver == am_logger) { proc = erts_whereis_process(NULL,0,receiver,proc_locks,0); if (!proc) goto failure; @@ -2379,7 +2379,7 @@ erts_foreach_sys_msg_in_q(void (*func)(Eterm, to = erts_get_system_profile(); break; case SYS_MSG_TYPE_ERRLGR: - to = am_error_logger; + to = am_logger; break; default: to = NIL; @@ -2593,7 +2593,7 @@ erts_term_to_tracer(Eterm prefix, Eterm t) state = tp[3]; } } else { - if (arityval(tp[0]) == 2 && is_atom(tp[2])) { + if (arityval(tp[0]) == 2 && is_atom(tp[1])) { module = tp[1]; state = tp[2]; } diff --git a/erts/emulator/beam/erlang_dtrace.d b/erts/emulator/beam/erlang_dtrace.d index d0b10e0306..c47a37eb62 100644 --- a/erts/emulator/beam/erlang_dtrace.d +++ b/erts/emulator/beam/erlang_dtrace.d @@ -55,7 +55,8 @@ provider erlang { * @param sender the PID (string form) of the sender * @param receiver the PID (string form) of the receiver * @param size the size of the message being delivered (words) - * @param token_label for the sender's sequential trace token + * @param token_label for the sender's sequential trace token. This will be + * INT_MIN if the label does not fit into a 32-bit integer. * @param token_previous count for the sender's sequential trace token * @param token_current count for the sender's sequential trace token */ @@ -73,7 +74,8 @@ provider erlang { * @param node_name the Erlang node name (string form) of the receiver * @param receiver the PID/name (string form) of the receiver * @param size the size of the message being delivered (words) - * @param token_label for the sender's sequential trace token + * @param token_label for the sender's sequential trace token. This will be + * INT_MIN if the label does not fit into a 32-bit integer. * @param token_previous count for the sender's sequential trace token * @param token_current count for the sender's sequential trace token */ @@ -98,7 +100,8 @@ provider erlang { * @param receiver the PID (string form) of the receiver * @param size the size of the message being delivered (words) * @param queue_len length of the queue of the receiving process - * @param token_label for the sender's sequential trace token + * @param token_label for the sender's sequential trace token. This will be + * INT_MIN if the label does not fit into a 32-bit integer. * @param token_previous count for the sender's sequential trace token * @param token_current count for the sender's sequential trace token */ @@ -122,7 +125,8 @@ provider erlang { * @param receiver the PID (string form) of the receiver * @param size the size of the message being delivered (words) * @param queue_len length of the queue of the receiving process - * @param token_label for the sender's sequential trace token + * @param token_label for the sender's sequential trace token. This will be + * INT_MIN if the label does not fit into a 32-bit integer. * @param token_previous count for the sender's sequential trace token * @param token_current count for the sender's sequential trace token */ @@ -273,7 +277,8 @@ provider erlang { * @param node_name the Erlang node name (string form) of the receiver * @param receiver the PID (string form) of the process receiving EXIT signal * @param reason the reason for the exit (may be truncated) - * @param token_label for the sender's sequential trace token + * @param token_label for the sender's sequential trace token. This will be + * INT_MIN if the label does not fit into a 32-bit integer. * @param token_previous count for the sender's sequential trace token * @param token_current count for the sender's sequential trace token */ diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index f9f8abcc27..bbd9b4bad2 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2016. All Rights Reserved. + * Copyright Ericsson AB 1996-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -83,7 +83,10 @@ #ifndef ERL_EXTERNAL_H__ #define ERL_EXTERNAL_H__ +#define ERL_NODE_TABLES_BASIC_ONLY #include "erl_node_tables.h" +#undef ERL_NODE_TABLES_BASIC_ONLY +#include "erl_alloc.h" #define ERTS_ATOM_CACHE_SIZE 2048 diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 0f23027752..2cf268162d 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2017. All Rights Reserved. + * Copyright Ericsson AB 1996-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -87,9 +87,7 @@ typedef struct { erts_mtx_t lock; ErtsMonitor* root; - int pending_failed_fire; - int is_dying; - + Uint refc; size_t user_data_sz; } ErtsResourceMonitors; @@ -116,13 +114,16 @@ extern void erts_pre_nif(struct enif_environment_t*, Process*, struct erl_module_nif*, Process* tracee); extern void erts_post_nif(struct enif_environment_t* env); extern void erts_resource_stop(ErtsResource*, ErlNifEvent, int is_direct_call); -void erts_fire_nif_monitor(ErtsResource*, Eterm pid, Eterm ref); +void erts_fire_nif_monitor(ErtsMonitor *tmon); +void erts_nif_demonitored(ErtsResource* resource); +extern void erts_add_taint(Eterm mod_atom); extern Eterm erts_nif_taints(Process* p); extern void erts_print_nif_taints(fmtfn_t to, void* to_arg); void erts_unload_nif(struct erl_module_nif* nif); extern void erl_nif_init(void); extern int erts_nif_get_funcs(struct erl_module_nif*, struct enif_func_t **funcs); +extern Module *erts_nif_get_module(struct erl_module_nif*); extern Eterm erts_nif_call_function(Process *p, Process *tracee, struct erl_module_nif*, struct enif_func_t *, @@ -200,6 +201,7 @@ typedef struct { struct erts_driver_t_ { erts_driver_t *next; erts_driver_t *prev; + Eterm name_atom; char *name; struct { int major; @@ -860,6 +862,8 @@ void erts_emasculate_writable_binary(ProcBin* pb); Eterm erts_new_heap_binary(Process *p, byte *buf, int len, byte** datap); Eterm erts_new_mso_binary(Process*, byte*, Uint); Eterm new_binary(Process*, byte*, Uint); +Eterm erts_heap_factory_new_binary(ErtsHeapFactory *hfact, byte *buf, + Uint len, Uint reserve_size); Eterm erts_realloc_binary(Eterm bin, size_t size); Eterm erts_build_proc_bin(ErlOffHeap*, Eterm*, Binary*); @@ -886,6 +890,7 @@ void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, Eterm (*bif)(Process*, Eterm*, BeamInstr*)); void erts_init_bif(void); Eterm erl_send(Process *p, Eterm to, Eterm msg); +int erts_set_group_leader(Process *proc, Eterm new_gl); /* erl_bif_op.c */ @@ -908,7 +913,6 @@ extern erts_atomic_t erts_copy_literal_area__; #define ERTS_COPY_LITERAL_AREA() \ ((ErtsLiteralArea *) erts_atomic_read_nob(&erts_copy_literal_area__)) extern Process *erts_literal_area_collector; -extern Process *erts_dirty_process_code_checker; extern Process *erts_code_purger; @@ -1072,7 +1076,7 @@ void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, Eterm* refs, unsigned nrefs, int literals); /* Utilities */ -extern void erts_delete_nodes_monitors(Process *, ErtsProcLocks); +void erts_monitor_nodes_delete(ErtsMonitor *); extern Eterm erts_monitor_nodes(Process *, Eterm, Eterm); extern Eterm erts_processes_monitoring_nodes(Process *); extern int erts_do_net_exits(DistEntry*, Eterm); @@ -1151,7 +1155,7 @@ typedef struct { #define ERTS_SPAWN_DRIVER 1 #define ERTS_SPAWN_EXECUTABLE 2 #define ERTS_SPAWN_ANY (ERTS_SPAWN_DRIVER | ERTS_SPAWN_EXECUTABLE) -int erts_add_driver_entry(ErlDrvEntry *drv, DE_Handle *handle, int driver_list_locked); +int erts_add_driver_entry(ErlDrvEntry *drv, DE_Handle *handle, int driver_list_locked, int taint); void erts_destroy_driver(erts_driver_t *drv); int erts_save_suspend_process_on_port(Port*, Process*); Port *erts_open_driver(erts_driver_t*, Eterm, char*, SysDriverOpts*, int *, int *); @@ -1176,8 +1180,17 @@ void erts_lcnt_update_port_locks(int enable); #endif /* driver_tab.c */ +typedef struct { + ErlDrvEntry* de; + int taint; +} ErtsStaticDriver; typedef void *(*ErtsStaticNifInitFPtr)(void); -ErtsStaticNifInitFPtr erts_static_nif_get_nif_init(const char *name, int len); +typedef struct ErtsStaticNifEntry_ { + const char *nif_name; + ErtsStaticNifInitFPtr nif_init; + int taint; +} ErtsStaticNifEntry; +ErtsStaticNifEntry* erts_static_nif_get_nif_init(const char *name, int len); int erts_is_static_nif(void *handle); void erts_init_static_drivers(void); diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c index 8548e30e8b..6a31489473 100644 --- a/erts/emulator/beam/hash.c +++ b/erts/emulator/beam/hash.c @@ -152,7 +152,7 @@ Hash* hash_init(int type, Hash* h, char* name, int size, HashFunctions fun) h->bucket = (HashBucket**) fun.meta_alloc(h->meta_alloc_type, sz); - sys_memzero(h->bucket, sz); + memzero(h->bucket, sz); h->is_allocated = 0; h->name = name; h->fun = fun; @@ -224,7 +224,7 @@ static void rehash(Hash* h, int grow) sz = h->size*sizeof(HashBucket*); new_bucket = (HashBucket **) h->fun.meta_alloc(h->meta_alloc_type, sz); - sys_memzero(new_bucket, sz); + memzero(new_bucket, sz); for (i = 0; i < old_size; i++) { HashBucket* b = h->bucket[i]; diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index e4a5f2b6b6..2446b3c074 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2017. All Rights Reserved. + * Copyright Ericsson AB 1996-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -53,6 +53,7 @@ #include "erl_hl_timer.h" #include "erl_time.h" #include "erl_io_queue.h" +#include "erl_proc_sig_queue.h" extern ErlDrvEntry fd_driver_entry; extern ErlDrvEntry vanilla_driver_entry; @@ -60,7 +61,7 @@ extern ErlDrvEntry spawn_driver_entry; #ifndef __WIN32__ extern ErlDrvEntry forker_driver_entry; #endif -extern ErlDrvEntry *driver_tab[]; /* table of static drivers, only used during initialization */ +extern ErtsStaticDriver driver_tab[]; /* table of static drivers, only used during initialization */ erts_driver_t *driver_list; /* List of all drivers, static and dynamic. */ erts_rwmtx_t erts_driver_list_lock; /* Mutex for driver list */ @@ -366,6 +367,7 @@ static Port *create_port(char *name, prt->drv_ptr = driver; ERTS_P_LINKS(prt) = NULL; ERTS_P_MONITORS(prt) = NULL; + ERTS_P_LT_MONITORS(prt) = NULL; prt->linebuf = NULL; prt->suspended = NULL; erts_init_port_data(prt); @@ -748,8 +750,8 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */ Port *creator_port; Port* port; erts_driver_t *driver; - Process *rp; erts_mtx_t *driver_lock = NULL; + ErtsLinkData *ldp; ERTS_CHK_NO_PROC_LOCKS; @@ -761,17 +763,13 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */ if (creator_port == ERTS_INVALID_ERL_DRV_PORT) return ERTS_INVALID_ERL_DRV_PORT; - rp = erts_proc_lookup(pid); - if (!rp) - return ERTS_INVALID_ERL_DRV_PORT; - ERTS_LC_ASSERT(erts_lc_is_port_locked(creator_port)); driver = creator_port->drv_ptr; erts_rwmtx_rlock(&erts_driver_list_lock); if (!erts_ddll_driver_ok(driver->handle)) { erts_rwmtx_runlock(&erts_driver_list_lock); - return ERTS_INVALID_ERL_DRV_PORT; + return ERTS_INVALID_ERL_DRV_PORT; } if (driver->handle != NULL) { @@ -799,9 +797,15 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */ } ERTS_LC_ASSERT(erts_lc_is_port_locked(port)); - erts_proc_lock(rp, ERTS_PROC_LOCK_LINK); - if (ERTS_PROC_IS_EXITING(rp)) { - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + ldp = erts_link_create(ERTS_LNK_TYPE_PORT, + port->common.id, pid); + ASSERT(ldp->a.other.item == pid); + ASSERT(ldp->b.other.item == port->common.id); + erts_link_tree_insert(&ERTS_P_LINKS(port), &ldp->a); + + if (!erts_proc_sig_send_link(NULL, pid, &ldp->b)) { + erts_link_tree_delete(&ERTS_P_LINKS(port), &ldp->a); + erts_link_release_both(ldp); if (driver->handle) { erts_rwmtx_rlock(&erts_driver_list_lock); erts_ddll_decrement_port_count(driver->handle); @@ -812,10 +816,6 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */ return ERTS_INVALID_ERL_DRV_PORT; } - erts_add_link(&ERTS_P_LINKS(port), LINK_PID, pid); - erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, port->common.id); - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - if (!driver_lock) { ErtsXPortsList *xplp = xports_list_alloc(); xplp->port = port; @@ -956,6 +956,9 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp) reds_left_in = ERTS_BIF_REDS_LEFT(c_p); erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + + ASSERT((c_p->scheduler_data)->current_port == NULL); + (c_p->scheduler_data)->current_port = prt; } ASSERT(0 <= reds_left_in && reds_left_in <= CONTEXT_REDS); @@ -1017,6 +1020,9 @@ finalize_imm_drv_call(ErtsTryImmDrvCallState *sp) erts_port_release(prt); if (c_p) { + ASSERT((c_p->scheduler_data)->current_port == prt); + (c_p->scheduler_data)->current_port = NULL; + erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); if (reds != (CONTEXT_REDS - sp->reds_left_in)) { @@ -1167,21 +1173,10 @@ erts_schedule_proc2port_signal(Process *c_p, * otherwise, next receive will *not* work * as expected! */ - erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - - if (ERTS_PROC_PENDING_EXIT(c_p)) { - /* need to exit caller instead */ - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - KILL_CATCHES(c_p); - c_p->freason = EXC_EXIT; - return ERTS_PORT_OP_CALLER_EXIT; - } - - ERTS_MSGQ_MV_INQ2PRIVQ(c_p); - c_p->msg.save = c_p->msg.last; - erts_proc_unlock(c_p, (ERTS_PROC_LOCKS_MSG_RECEIVE - | ERTS_PROC_LOCK_MAIN)); + ERTS_RECV_MARK_SAVE(c_p); + ERTS_RECV_MARK_SET(c_p); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); } @@ -1239,29 +1234,12 @@ erts_schedule_port2port_signal(Eterm port_num, ErtsProc2PortSigData *sigdp, static ERTS_INLINE void send_badsig(Port *prt) { - ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; - Process* rp; Eterm connected = ERTS_PORT_GET_CONNECTED(prt); ERTS_CHK_NO_PROC_LOCKS; ERTS_LC_ASSERT(erts_get_scheduler_id()); - ASSERT(is_internal_pid(connected)); - - rp = erts_proc_lookup_raw(connected); - if (rp) { - erts_proc_lock(rp, rp_locks); - if (!ERTS_PROC_IS_EXITING(rp)) - (void) erts_send_exit_signal(NULL, - prt->common.id, - rp, - &rp_locks, - am_badsig, - NIL, - NULL, - 0); - if (rp_locks) - erts_proc_unlock(rp, rp_locks); - } /* exit sent */ + erts_proc_sig_send_exit(NULL, prt->common.id, connected, + am_badsig, NIL, 0); } /* send_badsig */ static void @@ -2194,11 +2172,11 @@ call_deliver_port_exit(int bang_op, } if (broken_link) { - ErtsLink *lnk = erts_remove_link(&ERTS_P_LINKS(prt), from); - if (lnk) - erts_destroy_link(lnk); - else + ErtsLink *lnk = erts_link_tree_lookup(ERTS_P_LINKS(prt), from); + if (!lnk) return ERTS_PORT_OP_DROPPED; + erts_link_tree_delete(&ERTS_P_LINKS(prt), lnk); + erts_link_release(lnk); } if (IS_TRACED_FL(prt, F_TRACE_RECEIVE)) @@ -2367,27 +2345,45 @@ set_port_connected(int bang_op, #endif } else { /* Port BIF operation */ - Process *rp = erts_proc_lookup_raw(connect); - if (!rp) - return ERTS_PORT_OP_DROPPED; - erts_proc_lock(rp, ERTS_PROC_LOCK_LINK); - if (ERTS_PROC_IS_EXITING(rp)) { - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - return ERTS_PORT_OP_DROPPED; - } + int created; + ErtsLink *lnk; + + if (is_not_internal_pid(connect)) + return ERTS_PORT_OP_DROPPED; + + lnk = erts_link_tree_lookup_create(&ERTS_P_LINKS(prt), &created, + ERTS_LNK_TYPE_PORT, prt->common.id, + connect); + if (created) { + ErtsLinkData *ldp; + ErtsLink *olnk = erts_link_to_other(lnk, &ldp); + ASSERT(olnk->other.item == prt->common.id); + if (!erts_proc_sig_send_link(NULL, connect, olnk)) { + erts_link_tree_delete(&ERTS_P_LINKS(prt), lnk); + erts_link_release_both(ldp); + return ERTS_PORT_OP_DROPPED; + } + if (IS_TRACED_FL(prt, F_TRACE_PORTS)) + trace_port(prt, am_getting_linked, connect); + } - erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, prt->common.id); - erts_add_link(&ERTS_P_LINKS(prt), LINK_PID, connect); +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(port_connect)) { + Eterm old_connected = ERTS_PORT_GET_CONNECTED(prt); + DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(newprocess_str, DTRACE_TERM_BUF_SIZE); - if (IS_TRACED_FL(rp, F_TRACE_PROCS)) - trace_proc(NULL, 0, rp, am_getting_linked, prt->common.id); + dtrace_pid_str(old_connected, process_str); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), + "%T", prt->common.id); + dtrace_pid_str(connect, newprocess_str); + DTRACE4(port_connect, process_str, port_str, prt->name, newprocess_str); + } +#endif ERTS_PORT_SET_CONNECTED(prt, connect); - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - - if (IS_TRACED_FL(prt, F_TRACE_PORTS)) - trace_port(prt, am_getting_linked, connect); if (IS_TRACED_FL(prt, F_TRACE_RECEIVE)) trace_port_receive(prt, from, am_connect, connect); if (IS_TRACED_FL(prt, F_TRACE_SEND)) { @@ -2395,18 +2391,6 @@ set_port_connected(int bang_op, trace_port_send(prt, from, TUPLE2(hp, prt->common.id, am_connected), 1); } -#ifdef USE_VM_PROBES - if (DTRACE_ENABLED(port_connect)) { - DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE); - DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); - DTRACE_CHARBUF(newprocess_str, DTRACE_TERM_BUF_SIZE); - - dtrace_pid_str(connect, process_str); - erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id); - dtrace_proc_str(rp, newprocess_str); - DTRACE4(port_connect, process_str, port_str, prt->name, newprocess_str); - } -#endif } return ERTS_PORT_OP_DONE; @@ -2494,13 +2478,24 @@ erts_port_connect(Process *c_p, } static void -port_unlink(Port *prt, Eterm from) +port_unlink(Port *prt, ErtsLink *lnk) { - ErtsLink *lnk = erts_remove_link(&ERTS_P_LINKS(prt), from); - if (lnk) { + ErtsLinkData *ldp; + ErtsLink *dlnk, *llnk; + + llnk = erts_link_to_other(lnk, &ldp); + dlnk = erts_link_tree_key_delete(&ERTS_P_LINKS(prt), llnk); + if (!dlnk) + erts_link_release(lnk); + else { if (IS_TRACED_FL(prt, F_TRACE_PORTS)) - trace_port(prt, am_getting_unlinked, from); - erts_destroy_link(lnk); + trace_port(prt, am_getting_unlinked, llnk->other.item); + if (dlnk == llnk) + erts_link_release_both(ldp); + else { + erts_link_release(lnk); + erts_link_release(dlnk); + } } } @@ -2508,14 +2503,14 @@ static int port_sig_unlink(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp) { if (op == ERTS_PROC2PORT_SIG_EXEC) - port_unlink(prt, sigdp->u.unlink.from); + port_unlink(prt, sigdp->u.unlink.lnk); if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) port_sched_op_reply(sigdp->caller, sigdp->ref, am_true, prt); return ERTS_PORT_REDS_UNLINK; } ErtsPortOpResult -erts_port_unlink(Process *c_p, Port *prt, Eterm from, Eterm *refp) +erts_port_unlink(Process *c_p, Port *prt, ErtsLink *lnk, Eterm *refp) { ErtsProc2PortSigData *sigdp; ErtsTryImmDrvCallState try_call_state @@ -2528,7 +2523,7 @@ erts_port_unlink(Process *c_p, Port *prt, Eterm from, Eterm *refp) switch (try_imm_drv_call(&try_call_state)) { case ERTS_TRY_IMM_DRV_CALL_OK: - port_unlink(prt, from); + port_unlink(prt, lnk); finalize_imm_drv_call(&try_call_state); BUMP_REDS(c_p, ERTS_PORT_REDS_UNLINK); return ERTS_PORT_OP_DONE; @@ -2541,11 +2536,12 @@ erts_port_unlink(Process *c_p, Port *prt, Eterm from, Eterm *refp) sigdp = erts_port_task_alloc_p2p_sig_data(); sigdp->flags = ERTS_P2P_SIG_TYPE_UNLINK; - sigdp->u.unlink.from = from; - + sigdp->u.unlink.port_id = prt->common.id; + sigdp->u.unlink.lnk = lnk; + return erts_schedule_proc2port_signal(c_p, prt, - c_p ? c_p->common.id : from, + c_p->common.id, refp, sigdp, 0, @@ -2554,45 +2550,24 @@ erts_port_unlink(Process *c_p, Port *prt, Eterm from, Eterm *refp) } static void -port_link_failure(Eterm port_id, Eterm linker) +port_link_failure(Eterm port_id, ErtsLink *lnk) { - Process *rp; - ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCKS_XSIG_SEND; - ASSERT(is_internal_pid(linker)); - rp = erts_pid2proc(NULL, 0, linker, rp_locks); - if (rp) { - ErtsLink *rlnk = erts_remove_link(&ERTS_P_LINKS(rp), port_id); - if (rlnk) { - int xres = erts_send_exit_signal(NULL, - port_id, - rp, - &rp_locks, - am_noproc, - NIL, - NULL, - 0); - if (xres >= 0) { - /* We didn't exit the process and it is traced */ - if (IS_TRACED_FL(rp, F_TRACE_PROCS)) - trace_proc(NULL, 0, rp, am_getting_unlinked, port_id); - } - if (rp_locks) - erts_proc_unlock(rp, rp_locks); - } - } + erts_proc_sig_send_link_exit(NULL, port_id, lnk, am_noproc, NIL); } static void -port_link(Port *prt, erts_aint32_t state, Eterm to) +port_link(Port *prt, erts_aint32_t state, ErtsLink *lnk) { - if (IS_TRACED_FL(prt, F_TRACE_PORTS)) - trace_port(prt, am_getting_linked, to); - if (!(state & ERTS_PORT_SFLGS_INVALID_LOOKUP)) { - erts_add_link(&ERTS_P_LINKS(prt), LINK_PID, to); - } else { - port_link_failure(prt->common.id, to); - if (IS_TRACED_FL(prt, F_TRACE_PORTS)) - trace_port(prt, am_unlink, to); + if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP) + port_link_failure(prt->common.id, lnk); + else { + ErtsLink *rlnk; + rlnk = erts_link_tree_insert_addr_replace(&ERTS_P_LINKS(prt), + lnk); + if (rlnk) + erts_link_release(rlnk); + else if (IS_TRACED_FL(prt, F_TRACE_PORTS)) + trace_port(prt, am_getting_linked, lnk->other.item); } } @@ -2600,17 +2575,16 @@ static int port_sig_link(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp) { if (op == ERTS_PROC2PORT_SIG_EXEC) - port_link(prt, state, sigdp->u.link.to); - else { - port_link_failure(sigdp->u.link.port, sigdp->u.link.to); - } + port_link(prt, state, sigdp->u.link.lnk); + else + port_link_failure(sigdp->u.link.port_id, sigdp->u.link.lnk); if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) port_sched_op_reply(sigdp->caller, sigdp->ref, am_true, prt); return ERTS_PORT_REDS_LINK; } ErtsPortOpResult -erts_port_link(Process *c_p, Port *prt, Eterm to, Eterm *refp) +erts_port_link(Process *c_p, Port *prt, ErtsLink *lnk, Eterm *refp) { ErtsProc2PortSigData *sigdp; ErtsTryImmDrvCallState try_call_state @@ -2623,7 +2597,7 @@ erts_port_link(Process *c_p, Port *prt, Eterm to, Eterm *refp) switch (try_imm_drv_call(&try_call_state)) { case ERTS_TRY_IMM_DRV_CALL_OK: - port_link(prt, try_call_state.state, to); + port_link(prt, try_call_state.state, lnk); finalize_imm_drv_call(&try_call_state); BUMP_REDS(c_p, ERTS_PORT_REDS_LINK); return ERTS_PORT_OP_DONE; @@ -2636,12 +2610,12 @@ erts_port_link(Process *c_p, Port *prt, Eterm to, Eterm *refp) sigdp = erts_port_task_alloc_p2p_sig_data(); sigdp->flags = ERTS_P2P_SIG_TYPE_LINK; - sigdp->u.link.port = prt->common.id; - sigdp->u.link.to = to; + sigdp->u.link.port_id = prt->common.id; + sigdp->u.link.lnk = lnk; return erts_schedule_proc2port_signal(c_p, prt, - c_p ? c_p->common.id : to, + c_p->common.id, refp, sigdp, 0, @@ -2650,51 +2624,23 @@ erts_port_link(Process *c_p, Port *prt, Eterm to, Eterm *refp) } static void -port_monitor_failure(Eterm port_id, Eterm origin, Eterm ref_DOWN) +port_monitor_failure(Eterm port_id, ErtsMonitor *mon) { - Process *origin_p; - ErtsProcLocks p_locks = ERTS_PROC_LOCK_LINK; - ASSERT(is_internal_pid(origin)); - - origin_p = erts_pid2proc(NULL, 0, origin, p_locks); - if (! origin_p) { return; } - - /* Send the DOWN message immediately. Ref is made on the fly because - * caller has never seen it yet. */ - erts_queue_monitor_message(origin_p, &p_locks, ref_DOWN, - am_port, port_id, am_noproc); - erts_proc_unlock(origin_p, p_locks); + erts_proc_sig_send_monitor_down(mon, am_noproc); } /* Origin wants to monitor port Prt. State contains possible error, which has * happened just before. Name is either NIL or an atom, if user monitors * a port by name. Ref is premade reference that will be returned to user */ static void -port_monitor(Port *prt, erts_aint32_t state, Eterm origin, - Eterm name, Eterm ref) +port_monitor(Port *prt, erts_aint32_t state, ErtsMonitor *mon) { - Eterm name_or_nil = is_atom(name) ? name : NIL; - - ASSERT(is_pid(origin)); - ASSERT(is_atom(name) || is_port(name) || name == NIL); - ASSERT(is_internal_ordinary_ref(ref)); - - if (!(state & ERTS_PORT_SFLGS_INVALID_LOOKUP)) { - ErtsProcLocks p_locks = ERTS_PROC_LOCK_LINK; - - Process *origin_p = erts_pid2proc(NULL, 0, origin, p_locks); - if (! origin_p) { - goto failure; - } - erts_add_monitor(&ERTS_P_MONITORS(origin_p), MON_ORIGIN, ref, - prt->common.id, name_or_nil); - erts_add_monitor(&ERTS_P_MONITORS(prt), MON_TARGET, ref, - origin, name_or_nil); - - erts_proc_unlock(origin_p, p_locks); - } else { -failure: - port_monitor_failure(prt->common.id, origin, ref); + ASSERT(prt); + if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP) + port_monitor_failure(prt->common.id, mon); + else { + ASSERT(erts_monitor_is_target(mon)); + erts_monitor_list_insert(&ERTS_P_LT_MONITORS(prt), mon); } } @@ -2702,24 +2648,11 @@ static int port_sig_monitor(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp) { - Eterm hp[ERTS_REF_THING_SIZE]; - Eterm ref = make_internal_ref(&hp); - write_ref_thing(hp, sigdp->ref[0], sigdp->ref[1], sigdp->ref[2]); - - if (op == ERTS_PROC2PORT_SIG_EXEC) { - /* erts_add_monitor call inside port_monitor will copy ref from hp */ - port_monitor(prt, state, - sigdp->u.monitor.origin, - sigdp->u.monitor.name, - ref); - } else { - port_monitor_failure(sigdp->u.monitor.name, - sigdp->u.monitor.origin, - ref); - } - if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) { - port_sched_op_reply(sigdp->caller, sigdp->ref, am_true, prt); - } + if (op == ERTS_PROC2PORT_SIG_EXEC) + port_monitor(prt, state, sigdp->u.monitor.mon); + else + port_monitor_failure(sigdp->u.monitor.port_id, + sigdp->u.monitor.mon); return ERTS_PORT_REDS_MONITOR; } @@ -2727,95 +2660,63 @@ port_sig_monitor(Port *prt, erts_aint32_t state, int op, * a reference (ref may be rewritten to be used to serve additionally as a * signal id). Name is atom if user monitors port by name or NIL */ ErtsPortOpResult -erts_port_monitor(Process *origin, Port *port, Eterm name, Eterm *refp) +erts_port_monitor(Process *c_p, Port *port, ErtsMonitor *mon) { ErtsProc2PortSigData *sigdp; ErtsTryImmDrvCallState try_call_state = ERTS_INIT_TRY_IMM_DRV_CALL_STATE( - origin, port, ERTS_PORT_SFLGS_INVALID_LOOKUP, + c_p, + port, + ERTS_PORT_SFLGS_INVALID_LOOKUP, 0, - 0, /* trap_ref is always set so !trap_ref always is false */ + !0, am_monitor); - ASSERT(origin); + ASSERT(c_p); ASSERT(port); - ASSERT(is_atom(name) || is_port(name)); - ASSERT(refp); + ASSERT(mon); switch (try_imm_drv_call(&try_call_state)) { case ERTS_TRY_IMM_DRV_CALL_OK: - port_monitor(port, try_call_state.state, origin->common.id, name, *refp); + port_monitor(port, try_call_state.state, mon); finalize_imm_drv_call(&try_call_state); - BUMP_REDS(origin, ERTS_PORT_REDS_MONITOR); + BUMP_REDS(c_p, ERTS_PORT_REDS_MONITOR); return ERTS_PORT_OP_DONE; case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT: - return ERTS_PORT_OP_BADARG; + return ERTS_PORT_OP_DROPPED; default: break; /* Schedule call instead... */ } sigdp = erts_port_task_alloc_p2p_sig_data(); sigdp->flags = ERTS_P2P_SIG_TYPE_MONITOR; - sigdp->u.monitor.origin = origin->common.id; - sigdp->u.monitor.name = name; /* either named monitor, or port id */ + sigdp->u.monitor.port_id = port->common.id; + sigdp->u.monitor.mon = mon; /* Ref contents will be initialized here */ - return erts_schedule_proc2port_signal(origin, port, origin->common.id, - refp, sigdp, 0, NULL, + return erts_schedule_proc2port_signal(c_p, + port, + c_p->common.id, + NULL, + sigdp, + 0, + NULL, port_sig_monitor); } -static void -port_demonitor_failure(Eterm port_id, Eterm origin, Eterm ref) -{ - Process *origin_p; - ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK; - ErtsMonitor *mon1; - ASSERT(is_internal_pid(origin)); - - origin_p = erts_pid2proc(NULL, 0, origin, rp_locks); - if (! origin_p) { return; } - - /* do not send any DOWN messages, drop monitors on process */ - mon1 = erts_remove_monitor(&ERTS_P_MONITORS(origin_p), ref); - if (mon1 != NULL) { - erts_destroy_monitor(mon1); - } - - erts_proc_unlock(origin_p, rp_locks); -} - /* Origin wants to demonitor port Prt. State contains possible error, which has * happened just before. Ref is reference to monitor */ static void -port_demonitor(Port *port, erts_aint32_t state, Eterm origin, Eterm ref) +port_demonitor(Port *port, erts_aint32_t state, ErtsMonitor *mon) { - ASSERT(port); - ASSERT(is_pid(origin)); - ASSERT(is_internal_ref(ref)); - - if (!(state & ERTS_PORT_SFLGS_INVALID_LOOKUP)) { - ErtsProcLocks p_locks = ERTS_PROC_LOCK_LINK; - Process *origin_p = erts_pid2proc(NULL, 0, origin, p_locks); - if (origin_p) { - ErtsMonitor *mon1 = erts_remove_monitor(&ERTS_P_MONITORS(origin_p), - ref); - if (mon1 != NULL) { - erts_destroy_monitor(mon1); - } - } - if (1) { - ErtsMonitor *mon2 = erts_remove_monitor(&ERTS_P_MONITORS(port), - ref); - if (mon2 != NULL) { - erts_destroy_monitor(mon2); - } - } - if (origin_p) { /* when origin is dying, it won't be found */ - erts_proc_unlock(origin_p, p_locks); - } - } else { - port_demonitor_failure(port->common.id, origin, ref); + ErtsMonitorData *mdp = erts_monitor_to_data(mon); + ASSERT(port && mon); + ASSERT(erts_monitor_is_origin(mon)); + if (!erts_monitor_is_in_table(&mdp->target)) + erts_monitor_release(mon); + else { + erts_monitor_list_delete(&ERTS_P_LT_MONITORS(port), &mdp->target); + erts_monitor_release_both(mdp); } } @@ -2823,73 +2724,47 @@ static int port_sig_demonitor(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp) { - Eterm hp[ERTS_REF_THING_SIZE]; - Eterm ref = make_internal_ref(&hp); - write_ref_thing(hp, sigdp->u.demonitor.ref[0], - sigdp->u.demonitor.ref[1], - sigdp->u.demonitor.ref[2]); - if (op == ERTS_PROC2PORT_SIG_EXEC) { - port_demonitor(prt, state, sigdp->u.demonitor.origin, ref); - } else { - port_demonitor_failure(sigdp->u.demonitor.name, - sigdp->u.demonitor.origin, - ref); - } - if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) { - port_sched_op_reply(sigdp->caller, sigdp->ref, am_true, prt); - } + if (op == ERTS_PROC2PORT_SIG_EXEC) + port_demonitor(prt, state, sigdp->u.demonitor.mon); + else + erts_monitor_release(sigdp->u.demonitor.mon); return ERTS_PORT_REDS_DEMONITOR; } -/* Removes monitor between origin and target, identified by ref. - * Mode defines normal or relaxed demonitor rules (process is at death) */ -ErtsPortOpResult erts_port_demonitor(Process *origin, ErtsDemonitorMode mode, - Port *target, Eterm ref, - Eterm *trap_ref) +ErtsPortOpResult +erts_port_demonitor(Process *c_p, Port *prt, ErtsMonitor *mon) { - Process *c_p = mode == ERTS_PORT_DEMONITOR_NORMAL ? origin : NULL; ErtsProc2PortSigData *sigdp; ErtsTryImmDrvCallState try_call_state = ERTS_INIT_TRY_IMM_DRV_CALL_STATE( c_p, - target, ERTS_PORT_SFLGS_INVALID_LOOKUP, + prt, ERTS_PORT_SFLGS_INVALID_LOOKUP, 0, - !trap_ref, + !0, am_demonitor); - ASSERT(origin); - ASSERT(target); - ASSERT(is_internal_ref(ref)); + ASSERT(c_p && prt && mon); switch (try_imm_drv_call(&try_call_state)) { case ERTS_TRY_IMM_DRV_CALL_OK: - port_demonitor(target, try_call_state.state, origin->common.id, ref); + port_demonitor(prt, try_call_state.state, mon); finalize_imm_drv_call(&try_call_state); - if (mode == ERTS_PORT_DEMONITOR_NORMAL) { - BUMP_REDS(origin, ERTS_PORT_REDS_DEMONITOR); - } + BUMP_REDS(c_p, ERTS_PORT_REDS_DEMONITOR); return ERTS_PORT_OP_DONE; case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT: - return ERTS_PORT_OP_BADARG; + return ERTS_PORT_OP_DROPPED; default: break; /* Schedule call instead... */ } sigdp = erts_port_task_alloc_p2p_sig_data(); sigdp->flags = ERTS_P2P_SIG_TYPE_DEMONITOR; - sigdp->u.demonitor.origin = origin->common.id; - sigdp->u.demonitor.name = target->common.id; - { - Uint32 *nums = internal_ref_numbers(ref); - /* Start from 1 skip ref arity */ - sys_memcpy(sigdp->u.demonitor.ref, - nums, - sizeof(sigdp->u.demonitor.ref)); - } + sigdp->u.demonitor.port_id = prt->common.id; + sigdp->u.demonitor.mon = mon; /* Ref contents will be initialized here */ - return erts_schedule_proc2port_signal(c_p, target, origin->common.id, - trap_ref, sigdp, 0, NULL, + return erts_schedule_proc2port_signal(c_p, prt, c_p->common.id, + NULL, sigdp, 0, NULL, port_sig_demonitor); } @@ -2898,11 +2773,13 @@ init_ack_send_reply(Port *port, Eterm resp) { if (!is_internal_port(resp)) { - Process *rp = erts_proc_lookup_raw(port->async_open_port->to); - erts_proc_lock(rp, ERTS_PROC_LOCK_LINK); - erts_remove_link(&ERTS_P_LINKS(port), port->async_open_port->to); - erts_remove_link(&ERTS_P_LINKS(rp), port->common.id); - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + Eterm proc = port->async_open_port->to; + ErtsLink *lnk = erts_link_tree_lookup(ERTS_P_LINKS(port), + proc); + if (lnk) { + erts_link_tree_delete(&ERTS_P_LINKS(port), lnk); + erts_proc_sig_send_unlink(NULL, lnk); + } } port_sched_op_reply(port->async_open_port->to, port->async_open_port->ref, @@ -2964,7 +2841,7 @@ void erts_init_io(int port_tab_size, int port_tab_size_ignore_files, int legacy_port_tab) { - ErlDrvEntry** dp; + ErtsStaticDriver* dp; UWord common_element_size; erts_rwmtx_opt_t drv_list_rwmtx_opts = ERTS_RWMTX_OPT_DEFAULT_INITER; drv_list_rwmtx_opts.type = ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; @@ -3023,8 +2900,8 @@ void erts_init_io(int port_tab_size, init_driver(&forker_driver, &forker_driver_entry, NULL); #endif erts_init_static_drivers(); - for (dp = driver_tab; *dp != NULL; dp++) - erts_add_driver_entry(*dp, NULL, 1); + for (dp = driver_tab; dp->de != NULL; dp++) + erts_add_driver_entry(dp->de, NULL, 1, dp->taint); erts_tsd_set(driver_list_lock_status_key, NULL); erts_rwmtx_rwunlock(&erts_driver_list_lock); @@ -3035,11 +2912,8 @@ static void lcnt_enable_driver_lock_count(erts_driver_t *dp, int enable) { if (dp->lock) { if (enable) { - Eterm name_as_atom = erts_atom_put((byte*)dp->name, sys_strlen(dp->name), - ERTS_ATOM_ENC_LATIN1, 1); - - erts_lcnt_install_new_lock_info(&dp->lock->lcnt, "driver_lock", name_as_atom, - ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO); + erts_lcnt_install_new_lock_info(&dp->lock->lcnt, "driver_lock", + dp->name_atom, ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO); } else { erts_lcnt_uninstall(&dp->lock->lcnt); } @@ -3680,6 +3554,7 @@ terminate_port(Port *prt) ASSERT(!ERTS_P_LINKS(prt)); ASSERT(!ERTS_P_MONITORS(prt)); + ASSERT(!ERTS_P_LT_MONITORS(prt)); /* state may be altered by kill_port() below */ state = erts_atomic32_read_band_nob(&prt->state, @@ -3767,146 +3642,25 @@ erts_terminate_port(Port *pp) terminate_port(pp); } -static void port_fire_one_monitor(ErtsMonitor *mon, void *ctx0); -static void sweep_one_monitor(ErtsMonitor *mon, void *vpsc) -{ - switch (mon->type) { - case MON_ORIGIN: { - ErtsMonitor *rmon; - Process *rp; - - ASSERT(is_internal_pid(mon->u.pid)); - rp = erts_pid2proc(NULL, 0, mon->u.pid, ERTS_PROC_LOCK_LINK); - if (!rp) { - goto done; - } - rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref); - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - if (rmon == NULL) { - goto done; - } - erts_destroy_monitor(rmon); - } break; - case MON_TARGET: { - port_fire_one_monitor(mon, vpsc); /* forward call */ - } break; - } - done: - erts_destroy_monitor(mon); -} - - - typedef struct { - Port *port; + Eterm port_id; Eterm reason; -} SweepContext; +} ErtsPortExitContext; -static void sweep_one_link(ErtsLink *lnk, void *vpsc) +static void link_port_exit(ErtsLink *lnk, void *vpectxt) { - SweepContext *psc = vpsc; - DistEntry *dep; - Process *rp; - Eterm port_id = psc->port->common.id; - - ASSERT(lnk->type == LINK_PID); - - if (IS_TRACED_FL(psc->port, F_TRACE_PORTS)) - trace_port(psc->port, am_unlink, lnk->pid); - - if (is_external_pid(lnk->pid)) { - dep = external_pid_dist_entry(lnk->pid); - if(dep != erts_this_dist_entry) { - ErtsDistLinkData dld; - ErtsDSigData dsd; - int code; - code = erts_dsig_prepare(&dsd, dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0); - switch (code) { - case ERTS_DSIG_PREP_NOT_ALIVE: - case ERTS_DSIG_PREP_NOT_CONNECTED: - break; - case ERTS_DSIG_PREP_PENDING: - case ERTS_DSIG_PREP_CONNECTED: - erts_remove_dist_link(&dld, port_id, lnk->pid, dep); - erts_destroy_dist_link(&dld); - code = erts_dsig_send_exit(&dsd, port_id, lnk->pid, - psc->reason); - ASSERT(code == ERTS_DSIG_SEND_OK); - break; - default: - ASSERT(! "Invalid dsig prepare result"); - break; - } - } - } else { - ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCKS_XSIG_SEND; - ASSERT(is_internal_pid(lnk->pid)); - rp = erts_pid2proc(NULL, 0, lnk->pid, rp_locks); - if (rp) { - ErtsLink *rlnk = erts_remove_link(&ERTS_P_LINKS(rp), port_id); - - if (rlnk) { - int xres = erts_send_exit_signal(NULL, - port_id, - rp, - &rp_locks, - psc->reason, - NIL, - NULL, - 0); - if (xres >= 0) { - if (rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) { - erts_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND); - rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND; - } - /* We didn't exit the process and it is traced */ - if (IS_TRACED_FL(rp, F_TRACE_PROCS)) - trace_proc(NULL, 0, rp, am_getting_unlinked, port_id); - } - erts_destroy_link(rlnk); - } - - erts_proc_unlock(rp, rp_locks); - } - } - erts_destroy_link(lnk); + ErtsPortExitContext *pectxt = vpectxt; + erts_proc_sig_send_link_exit(NULL, pectxt->port_id, + lnk, pectxt->reason, NIL); } -static void -port_fire_one_monitor(ErtsMonitor *mon, void *ctx0) +static void monitor_port_exit(ErtsMonitor *mon, void *vpectxt) { - Process *origin; - ErtsProcLocks origin_locks; - - if (mon->type != MON_TARGET || ! is_pid(mon->u.pid)) { - return; - } - /* - * Proceed here if someone monitors us, we (port) are the target and - * origin is some process - */ - origin_locks = ERTS_PROC_LOCKS_MSG_SEND | ERTS_PROC_LOCK_LINK; - - origin = erts_pid2proc(NULL, 0, mon->u.pid, origin_locks); - if (origin) { - DeclareTmpHeapNoproc(lhp,3); - SweepContext *ctx = (SweepContext *)ctx0; - ErtsMonitor *rmon; - Eterm watched = (is_atom(mon->name) - ? TUPLE2(lhp, mon->name, erts_this_dist_entry->sysname) - : ctx->port->common.id); - - erts_queue_monitor_message(origin, &origin_locks, mon->ref, am_port, - watched, ctx->reason); - UnUseTmpHeapNoproc(3); - - rmon = erts_remove_monitor(&ERTS_P_MONITORS(origin), mon->ref); - erts_proc_unlock(origin, origin_locks); - - if (rmon) { - erts_destroy_monitor(rmon); - } - } + ErtsPortExitContext *pectxt = vpectxt; + if (erts_monitor_is_target(mon)) + erts_proc_sig_send_monitor_down(mon, pectxt->reason); + else + erts_proc_sig_send_demonitor(mon); } /* 'from' is sending 'this_port' an exit signal, (this_port must be internal). @@ -3923,9 +3677,12 @@ int erts_deliver_port_exit(Port *prt, Eterm from, Eterm reason, int send_closed, int drop_normal) { - ErtsLink *lnk; + ErtsLink *links; + ErtsMonitor *monitors; + ErtsMonitor *lt_monitors; Eterm modified_reason; erts_aint32_t state, set_state_flags; + ErtsPortExitContext pectxt; ERTS_CHK_NO_PROC_LOCKS; ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); @@ -3973,23 +3730,36 @@ erts_deliver_port_exit(Port *prt, Eterm from, Eterm reason, int send_closed, set_busy_port(ERTS_Port2ErlDrvPort(prt), 0); + links = ERTS_P_LINKS(prt); + ERTS_P_LINKS(prt) = NULL; + monitors = ERTS_P_MONITORS(prt); + ERTS_P_MONITORS(prt) = NULL; + lt_monitors = ERTS_P_LT_MONITORS(prt); + ERTS_P_LT_MONITORS(prt) = NULL; + if (prt->common.u.alive.reg != NULL) (void) erts_unregister_name(NULL, 0, prt, prt->common.u.alive.reg->name); - { - SweepContext sc = {prt, modified_reason}; - lnk = ERTS_P_LINKS(prt); - ERTS_P_LINKS(prt) = NULL; - erts_sweep_links(lnk, &sweep_one_link, &sc); + pectxt.port_id = prt->common.id; + pectxt.reason = modified_reason; + + if (links) + erts_monitor_tree_foreach_delete(&links, + link_port_exit, + (void *) &pectxt); + + if (monitors || lt_monitors) { + DRV_MONITOR_LOCK_PDL(prt); + if (monitors) + erts_monitor_tree_foreach_delete(&monitors, + monitor_port_exit, + (void *) &pectxt); + if (lt_monitors) + erts_monitor_list_foreach_delete(<_monitors, + monitor_port_exit, + (void *) &pectxt); + DRV_MONITOR_UNLOCK_PDL(prt); } - DRV_MONITOR_LOCK_PDL(prt); - { - SweepContext ctx = {prt, modified_reason}; - ErtsMonitor *moni = ERTS_P_MONITORS(prt); - ERTS_P_MONITORS(prt) = NULL; - erts_sweep_monitors(moni, &sweep_one_monitor, &ctx); - } - DRV_MONITOR_UNLOCK_PDL(prt); if ((state & ERTS_PORT_SFLG_DISTRIBUTION) && prt->dist_entry) { erts_do_net_exits(prt->dist_entry, modified_reason); @@ -5069,14 +4839,18 @@ typedef struct { static void prt_one_monitor(ErtsMonitor *mon, void *vprtd) { + ErtsMonitorData *mdp = erts_monitor_to_data(mon); prt_one_lnk_data *prtd = (prt_one_lnk_data *) vprtd; - erts_print(prtd->to, prtd->arg, "(%T,%T)", mon->u.pid, mon->ref); + if (mon->type == ERTS_MON_TYPE_RESOURCE && erts_monitor_is_target(mon)) + erts_print(prtd->to, prtd->arg, "(%p,%T)", mon->other.ptr, mdp->ref); + else + erts_print(prtd->to, prtd->arg, "(%T,%T)", mon->other.item, mdp->ref); } static void prt_one_lnk(ErtsLink *lnk, void *vprtd) { prt_one_lnk_data *prtd = (prt_one_lnk_data *) vprtd; - erts_print(prtd->to, prtd->arg, "%T", lnk->pid); + erts_print(prtd->to, prtd->arg, "%T", lnk->other.item); } static void dump_port_state(fmtfn_t to, void *arg, erts_aint32_t state) @@ -5188,15 +4962,18 @@ print_port_info(Port *p, fmtfn_t to, void *arg) prtd.to = to; prtd.arg = arg; erts_print(to, arg, "Links: "); - erts_doforall_links(ERTS_P_LINKS(p), &prt_one_lnk, &prtd); + erts_link_tree_foreach(ERTS_P_LINKS(p), prt_one_lnk, (void *) &prtd); erts_print(to, arg, "\n"); } - if (ERTS_P_MONITORS(p)) { + if (ERTS_P_MONITORS(p) || ERTS_P_LT_MONITORS(p)) { prt_one_lnk_data prtd; prtd.to = to; prtd.arg = arg; erts_print(to, arg, "Monitors: "); - erts_doforall_monitors(ERTS_P_MONITORS(p), &prt_one_monitor, &prtd); + erts_monitor_tree_foreach(ERTS_P_MONITORS(p), prt_one_monitor, + (void *) &prtd); + erts_monitor_list_foreach(ERTS_P_LT_MONITORS(p), prt_one_monitor, + (void *) &prtd); erts_print(to, arg, "\n"); } if (p->suspended) { @@ -7033,24 +6810,23 @@ static int do_driver_monitor_process(Port *prt, ErlDrvMonitor *monitor) { Eterm buf[ERTS_REF_THING_SIZE]; - Process *rp; Eterm ref; + ErtsMonitorData *mdp; - if (prt->drv_ptr->process_exit == NULL) { + if (!prt->drv_ptr->process_exit) return -1; - } - rp = erts_pid2proc_opt(NULL, 0, - (Eterm) process, ERTS_PROC_LOCK_LINK, - ERTS_P2P_FLG_ALLOW_OTHER_X); - if (!rp) { - return 1; - } ref = erts_make_ref_in_buffer(buf); - erts_add_monitor(&ERTS_P_MONITORS(prt), MON_ORIGIN, ref, rp->common.id, NIL); - erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, ref, prt->common.id, NIL); + mdp = erts_monitor_create(ERTS_MON_TYPE_PORT, ref, + prt->common.id, process, NIL); + + if (!erts_proc_sig_send_monitor(&mdp->target, process)) { + erts_monitor_release_both(mdp); + return 1; + } + + erts_monitor_tree_insert(&ERTS_P_MONITORS(prt), &mdp->origin); - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); erts_ref_to_driver_monitor(ref,monitor); return 0; } @@ -7083,37 +6859,17 @@ int driver_monitor_process(ErlDrvPort drvport, static int do_driver_demonitor_process(Port *prt, const ErlDrvMonitor *monitor) { Eterm heap[ERTS_REF_THING_SIZE]; - Process *rp; Eterm ref; ErtsMonitor *mon; - Eterm to; ref = erts_driver_monitor_to_ref(heap, monitor); - mon = erts_lookup_monitor(ERTS_P_MONITORS(prt), ref); - if (mon == NULL) { - return 1; - } - ASSERT(mon->type == MON_ORIGIN); - to = mon->u.pid; - ASSERT(is_internal_pid(to)); - rp = erts_pid2proc_opt(NULL, - 0, - to, - ERTS_PROC_LOCK_LINK, - ERTS_P2P_FLG_ALLOW_OTHER_X); - mon = erts_remove_monitor(&ERTS_P_MONITORS(prt), ref); - if (mon) { - erts_destroy_monitor(mon); - } - if (rp) { - ErtsMonitor *rmon; - rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - if (rmon != NULL) { - erts_destroy_monitor(rmon); - } - } + mon = erts_monitor_tree_lookup(ERTS_P_MONITORS(prt), ref); + if (!mon || !erts_monitor_is_origin(mon)) + return 1; + + erts_monitor_tree_delete(&ERTS_P_MONITORS(prt), mon); + erts_proc_sig_send_demonitor(mon); return 0; } @@ -7142,19 +6898,16 @@ static ErlDrvTermData do_driver_get_monitored_process(Port *prt,const ErlDrvMoni { Eterm ref; ErtsMonitor *mon; - Eterm to; Eterm heap[ERTS_REF_THING_SIZE]; ref = erts_driver_monitor_to_ref(heap, monitor); - mon = erts_lookup_monitor(ERTS_P_MONITORS(prt), ref); - if (mon == NULL) { + mon = erts_monitor_tree_lookup(ERTS_P_MONITORS(prt), ref); + if (!mon || !erts_monitor_is_origin(mon)) return driver_term_nil; - } - ASSERT(mon->type == MON_ORIGIN); - to = mon->u.pid; - ASSERT(is_internal_pid(to)); - return (ErlDrvTermData) to; + + ASSERT(is_internal_pid(mon->other.item)); + return (ErlDrvTermData) mon->other.item; } @@ -7186,24 +6939,27 @@ int driver_compare_monitors(const ErlDrvMonitor *monitor1, ERTS_REF_THING_SIZE*sizeof(Eterm)); } -void erts_fire_port_monitor(Port *prt, Eterm ref) +void erts_fire_port_monitor(Port *prt, ErtsMonitor *tmon) { - ErtsMonitor *rmon; + ErtsMonitorData *mdp; void (*callback)(ErlDrvData drv_data, ErlDrvMonitor *monitor); ErlDrvMonitor drv_monitor; int fpe_was_unmasked; ERTS_MSACC_PUSH_STATE_M(); ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); - ASSERT(prt->drv_ptr != NULL); + ASSERT(prt->drv_ptr != NULL); + ASSERT(erts_monitor_is_target(tmon)); + mdp = erts_monitor_to_data(tmon); DRV_MONITOR_LOCK_PDL(prt); - if (erts_lookup_monitor(ERTS_P_MONITORS(prt), ref) == NULL) { + if (!erts_monitor_is_in_table(&mdp->origin)) { DRV_MONITOR_UNLOCK_PDL(prt); + erts_monitor_release(tmon); return; } callback = prt->drv_ptr->process_exit; ASSERT(callback != NULL); - erts_ref_to_driver_monitor(ref,&drv_monitor); + erts_ref_to_driver_monitor(mdp->ref,&drv_monitor); ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT); DRV_MONITOR_UNLOCK_PDL(prt); #ifdef USE_VM_PROBES @@ -7227,11 +6983,9 @@ void erts_fire_port_monitor(Port *prt, Eterm ref) DRV_MONITOR_LOCK_PDL(prt); ERTS_MSACC_POP_STATE_M(); /* remove monitor *after* callback */ - rmon = erts_remove_monitor(&ERTS_P_MONITORS(prt), ref); + erts_monitor_tree_delete(&ERTS_P_MONITORS(prt), &mdp->origin); DRV_MONITOR_UNLOCK_PDL(prt); - if (rmon) { - erts_destroy_monitor(rmon); - } + erts_monitor_release_both(mdp); } @@ -7276,8 +7030,7 @@ driver_failure_term(ErlDrvPort ix, Eterm term, int eof) int driver_exit(ErlDrvPort ix, int err) { Port* prt = erts_drvport2port(ix); - Process* rp; - ErtsLink *lnk, *rlnk = NULL; + ErtsLink *lnk; Eterm connected; ERTS_CHK_NO_PROC_LOCKS; @@ -7286,22 +7039,10 @@ int driver_exit(ErlDrvPort ix, int err) return -1; connected = ERTS_PORT_GET_CONNECTED(prt); - rp = erts_pid2proc(NULL, 0, connected, ERTS_PROC_LOCK_LINK); - if (rp) { - rlnk = erts_remove_link(&ERTS_P_LINKS(rp),prt->common.id); - } - - lnk = erts_remove_link(&ERTS_P_LINKS(prt), connected); - - if (rp) - erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - - if (rlnk != NULL) { - erts_destroy_link(rlnk); - } - - if (lnk != NULL) { - erts_destroy_link(lnk); + lnk = erts_link_tree_lookup(ERTS_P_LINKS(prt), connected); + if (lnk) { + erts_link_tree_delete(&ERTS_P_LINKS(prt), lnk); + erts_proc_sig_send_unlink(NULL, lnk); } if (err == 0) @@ -7613,12 +7354,17 @@ no_stop_select_callback(ErlDrvEvent event, void* private) } #define IS_DRIVER_VERSION_GE(DE,MAJOR,MINOR) \ - ((DE)->major_version >= (MAJOR) && (DE)->minor_version >= (MINOR)) + ((DE)->major_version > (MAJOR) || \ + ((DE)->major_version == (MAJOR) && (DE)->minor_version >= (MINOR))) static int init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle) { + drv->name_atom = erts_atom_put((byte*)de->driver_name, + sys_strlen(de->driver_name), + ERTS_ATOM_ENC_LATIN1, 1); drv->name = de->driver_name; + ASSERT(de->extended_marker == ERL_DRV_EXTENDED_MARKER); ASSERT(de->major_version >= 2); drv->version.major = de->major_version; @@ -7628,13 +7374,10 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle) if (drv->flags & ERL_DRV_FLAG_USE_PORT_LOCKING) { drv->lock = NULL; } else { - Eterm driver_id = erts_atom_put((byte *) drv->name, - sys_strlen(drv->name), - ERTS_ATOM_ENC_LATIN1, 1); - drv->lock = erts_alloc(ERTS_ALC_T_DRIVER_LOCK, sizeof(erts_mtx_t)); - erts_mtx_init(drv->lock, "driver_lock", driver_id, ERTS_LOCK_FLAGS_CATEGORY_IO); + erts_mtx_init(drv->lock, "driver_lock", drv->name_atom, + ERTS_LOCK_FLAGS_CATEGORY_IO); } drv->entry = de; @@ -7696,13 +7439,14 @@ void add_driver_entry(ErlDrvEntry *drv){ * Ignore result of erts_add_driver_entry, the init is not * allowed to fail when drivers are added by drivers. */ - erts_add_driver_entry(drv, NULL, rec_lock != NULL); + erts_add_driver_entry(drv, NULL, rec_lock != NULL, 0); } -int erts_add_driver_entry(ErlDrvEntry *de, DE_Handle *handle, int driver_list_locked) +int erts_add_driver_entry(ErlDrvEntry *de, DE_Handle *handle, + int driver_list_locked, int taint) { erts_driver_t *dp = erts_alloc(ERTS_ALC_T_DRIVER, sizeof(erts_driver_t)); - int res; + int err = 0; if (!driver_list_locked) { erts_rwmtx_rwlock(&erts_driver_list_lock); @@ -7719,9 +7463,15 @@ int erts_add_driver_entry(ErlDrvEntry *de, DE_Handle *handle, int driver_list_lo erts_tsd_set(driver_list_lock_status_key, (void *) 1); } - res = init_driver(dp, de, handle); + if (!err) { + err = init_driver(dp, de, handle); + + if (taint) { + erts_add_taint(dp->name_atom); + } + } - if (res != 0) { + if (err) { /* * Remove it all again... */ @@ -7736,7 +7486,7 @@ int erts_add_driver_entry(ErlDrvEntry *de, DE_Handle *handle, int driver_list_lo erts_tsd_set(driver_list_lock_status_key, NULL); erts_rwmtx_rwunlock(&erts_driver_list_lock); } - return res; + return err; } /* Not allowed for dynamic drivers */ diff --git a/erts/emulator/beam/msg_instrs.tab b/erts/emulator/beam/msg_instrs.tab index d6d4d2fb49..26bea0efc6 100644 --- a/erts/emulator/beam/msg_instrs.tab +++ b/erts/emulator/beam/msg_instrs.tab @@ -2,7 +2,7 @@ // // %CopyrightBegin% // -// Copyright Ericsson AB 2017. All Rights Reserved. +// Copyright Ericsson AB 2017-2018. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -45,23 +45,16 @@ i_recv_mark() { /* - * Save the current position in message buffer. + * Save the current end of message queue */ - c_p->msg.saved_last = c_p->msg.last; + ERTS_RECV_MARK_SAVE(c_p); } i_recv_set() { /* - * If c_p->msg.saved_last is non-zero, it points to the first - * message that could possibly be matched out. - * - * If c_p->msg.saved_last is zero, it means that it was invalidated - * because another receive was executed before this i_recv_set() - * instruction was reached. + * If previously saved recv mark, set peek position to it */ - if (c_p->msg.saved_last) { - c_p->msg.save = c_p->msg.saved_last; - } + ERTS_RECV_MARK_SET(c_p); SET_I($NEXT_INSTRUCTION); goto loop_rec_top__; //| -no_next @@ -79,7 +72,6 @@ i_loop_rec(Dest) { /* Entry point from recv_set */ loop_rec_top__: - ; /* * We need to disable GC while matching messages @@ -89,34 +81,59 @@ i_loop_rec(Dest) { ASSERT(!(c_p->flags & F_DELAY_GC)); c_p->flags |= F_DELAY_GC; - /* Entry point from loop_rec_end */ + /* Entry point from loop_rec_end (and locally) */ loop_rec__: + if (FCALLS <= 0 && FCALLS <= neg_o_reds) { + $SET_CP_I_ABS(I); + c_p->flags &= ~F_DELAY_GC; + SWAPOUT; + c_p->arity = 0; + c_p->current = NULL; + goto do_schedule; + } + + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + PROCESS_MAIN_CHK_LOCKS(c_p); msgp = PEEK_MESSAGE(c_p); - if (!msgp) { - erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - /* Make sure messages wont pass exit signals... */ - if (ERTS_PROC_PENDING_EXIT(c_p)) { - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - SWAPOUT; - c_p->flags &= ~F_DELAY_GC; - c_p->arity = 0; - goto do_schedule; /* Will be rescheduled for exit */ - } - ERTS_MSGQ_MV_INQ2PRIVQ(c_p); - msgp = PEEK_MESSAGE(c_p); - if (msgp) { - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - } else { + if (ERTS_UNLIKELY(msgp == NULL)) { + int get_out; + SWAPOUT; + FCALLS -= erts_proc_sig_receive_helper(c_p, FCALLS, neg_o_reds, + &msgp, &get_out); + SWAPIN; + if (ERTS_UNLIKELY(msgp == NULL)) { + if (get_out) { + if (get_out < 0) { + ASSERT(FCALLS <= 0 && FCALLS <= neg_o_reds); + goto loop_rec__; /* yield */ + } + else { + ASSERT(ERTS_PROC_IS_EXITING(c_p)); + goto do_schedule; /* exit */ + } + } + + /* + * If there are no more messages in queue + * (and we are not yielding or exiting) + * erts_proc_sig_receive_helper() + * returns with message queue lock locked... + */ c_p->flags &= ~F_DELAY_GC; $SET_I_REL($Dest); Goto(*I); /* Jump to a wait or wait_timeout instruction */ } } - if (is_non_value(ERL_MESSAGE_TERM(msgp))) { + + ASSERT(msgp == PEEK_MESSAGE(c_p)); + ASSERT(msgp && ERTS_SIG_IS_MSG(msgp)); + + if (ERTS_UNLIKELY(ERTS_SIG_IS_EXTERNAL_MSG(msgp))) { + FCALLS -= 10; /* FIXME: bump appropriate amount... */ SWAPOUT; /* erts_decode_dist_message() may write to heap... */ if (!erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) { /* @@ -127,13 +144,16 @@ i_loop_rec(Dest) { ASSERT(HTOP == c_p->htop && E == c_p->stop); /* TODO: Add DTrace probe for this bad message situation? */ UNLINK_MESSAGE(c_p, msgp); - c_p->msg.saved_last = 0; /* Better safe than sorry. */ msgp->next = NULL; erts_cleanup_messages(msgp); goto loop_rec__; } SWAPIN; } + + ASSERT(msgp == PEEK_MESSAGE(c_p)); + ASSERT(ERTS_SIG_IS_INTERNAL_MSG(msgp)); + r(0) = ERL_MESSAGE_TERM(msgp); } @@ -205,17 +225,19 @@ remove_message() { Sint tok_label = 0; Sint tok_lastcnt = 0; Sint tok_serial = 0; + Sint len = erts_proc_sig_privqs_len(c_p); dtrace_proc_str(c_p, receiver_name); token2 = SEQ_TRACE_TOKEN(c_p); if (have_seqtrace(token2)) { - tok_label = signed_val(SEQ_TRACE_T_LABEL(token2)); + tok_label = SEQ_TRACE_T_DTRACE_LABEL(token2); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token2)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token2)); } DTRACE6(message_receive, receiver_name, size_object(ERL_MESSAGE_TERM(msgp)), - c_p->msg.len - 1, tok_label, tok_lastcnt, tok_serial); + len, /* This is NOT message queue len, but its something... */ + tok_label, tok_lastcnt, tok_serial); } #endif UNLINK_MESSAGE(c_p, msgp); @@ -252,17 +274,8 @@ loop_rec_end(Dest) { $SET_I_REL($Dest); SAVE_MESSAGE(c_p); - if (FCALLS > 0 || FCALLS > neg_o_reds) { - FCALLS--; - goto loop_rec__; - } - - c_p->flags &= ~F_DELAY_GC; - $SET_CP_I_ABS(I); - SWAPOUT; - c_p->arity = 0; - c_p->current = NULL; - goto do_schedule; + FCALLS--; + goto loop_rec__; } timeout_locked() { diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 77e375f2c0..8b2d9098a8 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -710,7 +710,8 @@ is_boolean Fail=f ac => jump Fail is_boolean f? xy %hot -is_function2 Fail=f acq Arity => jump Fail +is_function2 Fail=f Literal=q Arity | literal_is_export(Literal) => +is_function2 Fail=f c Arity => jump Fail is_function2 Fail=f Fun a => jump Fail is_function2 f? S s @@ -1413,7 +1414,7 @@ has_map_fields Fail Src Size Rest=* => \ ## Transform get_map_elements(s) #{ K1 := V1, K2 := V2 } -get_map_elements Fail Src=xy Size=u==2 Rest=* => \ +get_map_elements Fail Src Size=u==2 Rest=* => \ gen_get_map_element(Fail, Src, Size, Rest) get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \ gen_get_map_elements(Fail, Src, Size, Rest) @@ -1423,8 +1424,12 @@ i_get_map_elements f? s I i_get_map_element Fail Src=xy Key=y Dst => \ move Key x | i_get_map_element Fail Src x Dst +i_get_map_element_hash Fail Src=c Key Hash Dst => \ + move Src x | i_get_map_element_hash Fail x Key Hash Dst i_get_map_element_hash f? xy c I xy +i_get_map_element Fail Src=c Key Dst => \ + move Src x | i_get_map_element Fail x Key Dst i_get_map_element f? xy x xy # diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 13ae80e4a5..be6ab57eeb 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -366,29 +366,11 @@ typedef UWord BeamInstr; # define HAVE_INT64 1 typedef unsigned long Uint64; typedef long Sint64; -# ifdef ULONG_MAX -# define ERTS_UINT64_MAX ULONG_MAX -# endif -# ifdef LONG_MAX -# define ERTS_SINT64_MAX LONG_MAX -# endif -# ifdef LONG_MIN -# define ERTS_SINT64_MIN LONG_MIN -# endif # define ErtsStrToSint64 strtol # elif SIZEOF_LONG_LONG == 8 # define HAVE_INT64 1 typedef unsigned long long Uint64; typedef long long Sint64; -# ifdef ULLONG_MAX -# define ERTS_UINT64_MAX ULLONG_MAX -# endif -# ifdef LLONG_MAX -# define ERTS_SINT64_MAX LLONG_MAX -# endif -# ifdef LLONG_MIN -# define ERTS_SINT64_MIN LLONG_MIN -# endif # define ErtsStrToSint64 strtoll # else # error "No 64-bit integer type found" @@ -402,7 +384,7 @@ typedef long long Sint64; # define ERTS_SINT64_MAX ((Sint64) ((((Uint64) 1) << 63)-1)) #endif #ifndef ERTS_SINT64_MIN -# define ERTS_SINT64_MIN (-1*(((Sint64) 1) << 63)) +# define ERTS_SINT64_MIN ((Sint64) ((((Uint64) 1) << 63))) #endif #if SIZEOF_LONG == 4 @@ -415,6 +397,16 @@ typedef int Sint32; #error Found no appropriate type to use for 'Uint32' and 'Sint32' #endif +#ifndef ERTS_UINT32_MAX +# define ERTS_UINT32_MAX (~((Uint32) 0)) +#endif +#ifndef ERTS_SINT32_MAX +# define ERTS_SINT32_MAX ((Sint32) ((((Uint32) 1) << 31)-1)) +#endif +#ifndef ERTS_SINT32_MIN +# define ERTS_SINT32_MIN ((Sint32) ((((Uint32) 1) << 31))) +#endif + #if SIZEOF_INT == 2 typedef unsigned int Uint16; typedef int Sint16; @@ -425,6 +417,16 @@ typedef short Sint16; #error Found no appropriate type to use for 'Uint16' and 'Sint16' #endif +#ifndef ERTS_UINT16_MAX +# define ERTS_UINT16_MAX (~((Uint16) 0)) +#endif +#ifndef ERTS_SINT16_MAX +# define ERTS_SINT16_MAX ((Sint16) ((((Uint16) 1) << 15)-1)) +#endif +#ifndef ERTS_SINT16_MIN +# define ERTS_SINT16_MIN ((Sint16) ((((Uint16) 1) << 15))) +#endif + #if CHAR_BIT == 8 typedef unsigned char byte; #else @@ -994,7 +996,8 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val) return val; } -#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ +#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ + /* Thin wrappers around memcpy and friends, which should always be used in * place of plain memcpy, memset, etc. @@ -1003,29 +1006,72 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val) * may seemingly work when the length (if any) is zero; a compiler can take * this as a hint that the passed operand may *never* be NULL and then optimize * based on that information. - * - * (The weird casts in the assertions silence an "always evaluates to true" - * warning when an operand is the address of an lvalue) */ -#define sys_memcpy(s1,s2,n) \ - (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), memcpy(s1,s2,n)) -#define sys_memmove(s1,s2,n) \ - (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), memmove(s1,s2,n)) -#define sys_memcmp(s1,s2,n) \ - (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), memcmp(s1,s2,n)) -#define sys_memset(s,c,n) \ - (ASSERT((void*)(s) != NULL), memset(s,c,n)) -#define sys_memzero(s, n) \ - (ASSERT((void*)(s) != NULL), memset(s,'\0',n)) -#define sys_strcmp(s1,s2) \ - (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), strcmp(s1,s2)) -#define sys_strncmp(s1,s2,n) \ - (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), strncmp(s1,s2,n)) -#define sys_strcpy(s1,s2) \ - (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), strcpy(s1,s2)) -#define sys_strncpy(s1,s2,n) \ - (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), strncpy(s1,s2,n)) -#define sys_strlen(s) \ - (ASSERT((void*)(s) != NULL), strlen(s)) + */ +ERTS_GLB_INLINE void *sys_memcpy(void *dest, const void *src, size_t n); +ERTS_GLB_INLINE void *sys_memmove(void *dest, const void *src, size_t n); +ERTS_GLB_INLINE int sys_memcmp(const void *s1, const void *s2, size_t n); +ERTS_GLB_INLINE void *sys_memset(void *s, int c, size_t n); +ERTS_GLB_INLINE void *sys_memzero(void *s, size_t n); +ERTS_GLB_INLINE int sys_strcmp(const char *s1, const char *s2); +ERTS_GLB_INLINE int sys_strncmp(const char *s1, const char *s2, size_t n); +ERTS_GLB_INLINE char *sys_strcpy(char *dest, const char *src); +ERTS_GLB_INLINE char *sys_strncpy(char *dest, const char *src, size_t n); +ERTS_GLB_INLINE size_t sys_strlen(const char *s); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void *sys_memcpy(void *dest, const void *src, size_t n) +{ + ASSERT(dest != NULL && src != NULL); + return memcpy(dest,src,n); +} +ERTS_GLB_INLINE void *sys_memmove(void *dest, const void *src, size_t n) +{ + ASSERT(dest != NULL && src != NULL); + return memmove(dest,src,n); +} +ERTS_GLB_INLINE int sys_memcmp(const void *s1, const void *s2, size_t n) +{ + ASSERT(s1 != NULL && s2 != NULL); + return memcmp(s1,s2,n); +} +ERTS_GLB_INLINE void *sys_memset(void *s, int c, size_t n) +{ + ASSERT(s != NULL); + return memset(s,c,n); +} +ERTS_GLB_INLINE void *sys_memzero(void *s, size_t n) +{ + ASSERT(s != NULL); + return memset(s,'\0',n); +} +ERTS_GLB_INLINE int sys_strcmp(const char *s1, const char *s2) +{ + ASSERT(s1 != NULL && s2 != NULL); + return strcmp(s1,s2); +} +ERTS_GLB_INLINE int sys_strncmp(const char *s1, const char *s2, size_t n) +{ + ASSERT(s1 != NULL && s2 != NULL); + return strncmp(s1,s2,n); +} +ERTS_GLB_INLINE char *sys_strcpy(char *dest, const char *src) +{ + ASSERT(dest != NULL && src != NULL); + return strcpy(dest,src); + +} +ERTS_GLB_INLINE char *sys_strncpy(char *dest, const char *src, size_t n) +{ + ASSERT(dest != NULL && src != NULL); + return strncpy(dest,src,n); +} +ERTS_GLB_INLINE size_t sys_strlen(const char *s) +{ + ASSERT(s != NULL); + return strlen(s); +} +#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ /* define function symbols (needed in sys_drv_api) */ #define sys_fp_alloc sys_alloc diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 4bf60619ba..d74052d8b2 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -59,6 +59,7 @@ #endif #define ERTS_WANT_NFUNC_SCHED_INTERNALS__ #include "erl_nfunc_sched.h" +#include "erl_proc_sig_queue.h" #undef M_TRIM_THRESHOLD #undef M_TOP_PAD @@ -139,7 +140,7 @@ Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz) { Eterm* p = ptr; - int i; + Uint i; for (i = 0; i < sz; i++) { *p++ = ERTS_HOLE_MARKER; @@ -1923,155 +1924,165 @@ make_internal_hash(Eterm term, Uint32 salt) #undef HCONST #undef MIX +/* error_logger ! + {log, Level, format, [args], #{ gl, pid, time, error_logger => #{tag, emulator => true} }} +*/ static Eterm -do_allocate_logger_message(Eterm gleader, Eterm **hp, ErlOffHeap **ohp, - ErlHeapFragment **bp, Process **p, Uint sz) +do_allocate_logger_message(Eterm gleader, ErtsMonotonicTime *ts, Eterm *pid, + Eterm **hp, ErlOffHeap **ohp, + ErlHeapFragment **bp, Uint sz) { Uint gl_sz; gl_sz = IS_CONST(gleader) ? 0 : size_object(gleader); - sz = sz + gl_sz; + sz = sz + gl_sz + 6 /*outer 5-tuple*/ + + MAP2_SZ /* error_logger map */; + + *pid = erts_get_current_pid(); + + if (is_nil(gleader) && is_non_value(*pid)) { + sz += MAP2_SZ /* metadata map no gl, no pid */; + } else if (is_nil(gleader) || is_non_value(*pid)) + sz += MAP3_SZ /* metadata map no gl or no pid*/; + else + sz += MAP4_SZ /* metadata map w gl w pid*/; + + *ts = ERTS_MONOTONIC_TO_USEC(erts_get_monotonic_time(NULL)) + ERTS_MONOTONIC_OFFSET_USEC; + erts_bld_sint64(NULL, &sz, *ts); *bp = new_message_buffer(sz); *ohp = &(*bp)->off_heap; *hp = (*bp)->mem; - return (is_nil(gleader) - ? am_noproc - : (IS_CONST(gleader) - ? gleader - : copy_struct(gleader,gl_sz,hp,*ohp))); + return copy_struct(gleader,gl_sz,hp,*ohp); } -static void do_send_logger_message(Eterm *hp, ErlOffHeap *ohp, ErlHeapFragment *bp, - Process *p, Eterm message) +static void do_send_logger_message(Eterm gl, Eterm tag, Eterm format, Eterm args, + ErtsMonotonicTime ts, Eterm pid, + Eterm *hp, ErlHeapFragment *bp) { -#ifdef HARDDEBUG - erts_fprintf(stderr, "%T\n", message); -#endif - { - Eterm from = erts_get_current_pid(); - if (is_not_internal_pid(from)) - from = NIL; - erts_queue_error_logger_message(from, message, bp); + Eterm message, md, el_tag = tag; + Eterm time = erts_bld_sint64(&hp, NULL, ts); + + /* This mapping is needed for the backwards compatible error_logger */ + switch (tag) { + case am_info: el_tag = am_info_msg; break; + case am_warning: el_tag = am_warning_msg; break; + default: + ASSERT(am_error); + break; } + + md = MAP2(hp, am_emulator, am_true, + am_atom_put("tag", 3), el_tag); + hp += MAP2_SZ; + + if (is_nil(gl) && is_non_value(pid)) { + /* no gl and no pid, probably from a port */ + md = MAP2(hp, + am_error_logger, md, + am_time, time); + hp += MAP2_SZ; + pid = NIL; + } else if (is_nil(gl)) { + /* no gl */ + md = MAP3(hp, + am_error_logger, md, + am_pid, pid, + am_time, time); + hp += MAP3_SZ; + } else if (is_non_value(pid)) { + /* no gl */ + md = MAP3(hp, + am_error_logger, md, + am_atom_put("gl", 2), gl, + am_time, time); + hp += MAP3_SZ; + pid = NIL; + } else { + md = MAP4(hp, + am_error_logger, md, + am_atom_put("gl", 2), gl, + am_pid, pid, + am_time, time); + hp += MAP4_SZ; + } + + message = TUPLE5(hp, am_log, tag, format, args, md); + erts_queue_error_logger_message(pid, message, bp); } -/* error_logger ! - {notify,{info_msg,gleader,{emulator,format,[args]}}} | - {notify,{error,gleader,{emulator,format,[args]}}} | - {notify,{warning_msg,gleader,{emulator,format,[args}]}} */ -static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len) +static int do_send_to_logger(Eterm tag, Eterm gl, char *buf, size_t len) { Uint sz; - Eterm gl; - Eterm list,args,format,tuple1,tuple2,tuple3; + Eterm list, args, format, pid; + ErtsMonotonicTime ts; Eterm *hp = NULL; ErlOffHeap *ohp = NULL; ErlHeapFragment *bp = NULL; - Process *p = NULL; - - ASSERT(is_atom(tag)); - - if (len <= 0) { - return -1; - } sz = len * 2 /* message list */ + 2 /* cons surrounding message list */ - + 3 /*outer 2-tuple*/ + 4 /* middle 3-tuple */ + 4 /*inner 3-tuple */ + 8 /* "~s~n" */; /* gleader size is accounted and allocated next */ - gl = do_allocate_logger_message(gleader, &hp, &ohp, &bp, &p, sz); - - if(is_nil(gl)) { - /* buf *always* points to a null terminated string */ - erts_fprintf(stderr, "(no error logger present) %T: \"%s\"\n", - tag, buf); - return 0; - } + gl = do_allocate_logger_message(gl, &ts, &pid, &hp, &ohp, &bp, sz); list = buf_to_intlist(&hp, buf, len, NIL); args = CONS(hp,list,NIL); hp += 2; format = buf_to_intlist(&hp, "~s~n", 4, NIL); - tuple1 = TUPLE3(hp, am_emulator, format, args); - hp += 4; - tuple2 = TUPLE3(hp, tag, gl, tuple1); - hp += 4; - tuple3 = TUPLE2(hp, am_notify, tuple2); - do_send_logger_message(hp, ohp, bp, p, tuple3); + do_send_logger_message(gl, tag, format, args, ts, pid, hp, bp); return 0; } -static int do_send_term_to_logger(Eterm tag, Eterm gleader, - char *buf, int len, Eterm args) +static int do_send_term_to_logger(Eterm tag, Eterm gl, + char *buf, size_t len, Eterm args) { Uint sz; - Eterm gl; Uint args_sz; - Eterm format,tuple1,tuple2,tuple3; + Eterm format, pid; + ErtsMonotonicTime ts; Eterm *hp = NULL; ErlOffHeap *ohp = NULL; ErlHeapFragment *bp = NULL; - Process *p = NULL; - ASSERT(is_atom(tag)); + ASSERT(len > 0); args_sz = size_object(args); - sz = len * 2 /* format */ + args_sz - + 3 /*outer 2-tuple*/ + 4 /* middle 3-tuple */ + 4 /*inner 3-tuple */; + sz = len * 2 /* format */ + args_sz; /* gleader size is accounted and allocated next */ - gl = do_allocate_logger_message(gleader, &hp, &ohp, &bp, &p, sz); - - if(is_nil(gl)) { - /* buf *always* points to a null terminated string */ - erts_fprintf(stderr, "(no error logger present) %T: \"%s\" %T\n", - tag, buf, args); - return 0; - } + gl = do_allocate_logger_message(gl, &ts, &pid, &hp, &ohp, &bp, sz); format = buf_to_intlist(&hp, buf, len, NIL); args = copy_struct(args, args_sz, &hp, ohp); - tuple1 = TUPLE3(hp, am_emulator, format, args); - hp += 4; - tuple2 = TUPLE3(hp, tag, gl, tuple1); - hp += 4; - tuple3 = TUPLE2(hp, am_notify, tuple2); - do_send_logger_message(hp, ohp, bp, p, tuple3); + do_send_logger_message(gl, tag, format, args, ts, pid, hp, bp); return 0; } static ERTS_INLINE int -send_info_to_logger(Eterm gleader, char *buf, int len) +send_info_to_logger(Eterm gleader, char *buf, size_t len) { - return do_send_to_logger(am_info_msg, gleader, buf, len); + return do_send_to_logger(am_info, gleader, buf, len); } static ERTS_INLINE int -send_warning_to_logger(Eterm gleader, char *buf, int len) +send_warning_to_logger(Eterm gleader, char *buf, size_t len) { - Eterm tag; - switch (erts_error_logger_warnings) { - case am_info: tag = am_info_msg; break; - case am_warning: tag = am_warning_msg; break; - default: tag = am_error; break; - } - return do_send_to_logger(tag, gleader, buf, len); + return do_send_to_logger(erts_error_logger_warnings, gleader, buf, len); } static ERTS_INLINE int -send_error_to_logger(Eterm gleader, char *buf, int len) +send_error_to_logger(Eterm gleader, char *buf, size_t len) { return do_send_to_logger(am_error, gleader, buf, len); } static ERTS_INLINE int -send_error_term_to_logger(Eterm gleader, char *buf, int len, Eterm args) +send_error_term_to_logger(Eterm gleader, char *buf, size_t len, Eterm args) { return do_send_term_to_logger(am_error, gleader, buf, len, args); } diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index 4294fb4f46..1a68f65b52 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -9930,6 +9930,12 @@ static int tcp_recv_closed(tcp_descriptor* desc) set_busy_port(desc->inet.port, 0); inet_reply_error_am(INETP(desc), am_closed); DEBUGF(("tcp_recv_closed(%ld): busy reply 'closed'\r\n", port)); + } else { + /* No blocking send op to reply to right now. + * If next op is a send, make sure it returns {error,closed} + * rather than {error,enotconn}. + */ + desc->tcp_add_flags |= TCP_ADDF_DELAYED_CLOSE_SEND; } if (!desc->inet.active) { /* We must cancel any timer here ! */ diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c index 3323e8640b..74e793577c 100644 --- a/erts/emulator/hipe/hipe_mkliterals.c +++ b/erts/emulator/hipe/hipe_mkliterals.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2016. All Rights Reserved. + * Copyright Ericsson AB 2001-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -519,8 +519,8 @@ static const struct rts_param rts_params[] = { 1, offsetof(struct process, hipe.bif_callee) #endif }, - { 49, "P_MSG_FIRST", 1, offsetof(struct process, msg.first) }, - { 50, "P_MSG_SAVE", 1, offsetof(struct process, msg.save) }, + { 49, "P_MSG_FIRST", 1, offsetof(struct process, sig_qs.first) }, + { 50, "P_MSG_SAVE", 1, offsetof(struct process, sig_qs.save) }, { 51, "P_CALLEE_EXP", 1, offsetof(struct process, hipe.u.callee_exp) }, { 52, "THE_NON_VALUE", 1, (int)THE_NON_VALUE }, @@ -531,8 +531,8 @@ static const struct rts_param rts_params[] = { #endif }, - { 54, "P_MSG_LAST", 1, offsetof(struct process, msg.last) }, - { 55, "P_MSG_SAVED_LAST", 1, offsetof(struct process, msg.saved_last) }, + { 54, "P_MSG_LAST", 1, offsetof(struct process, sig_qs.last) }, + { 55, "P_MSG_SAVED_LAST", 1, offsetof(struct process, sig_qs.saved_last) }, }; #define NR_PARAMS ARRAY_SIZE(rts_params) diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c index 8b497c9970..bc9a700204 100644 --- a/erts/emulator/hipe/hipe_mode_switch.c +++ b/erts/emulator/hipe/hipe_mode_switch.c @@ -496,8 +496,10 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) erts_proc_lock(p, ERTS_PROC_LOCKS_MSG_RECEIVE); p->i = hipe_beam_pc_resume; p->arity = 0; - erts_atomic32_read_band_relb(&p->state, - ~ERTS_PSFLG_ACTIVE); + if (erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_EXITING) + ASSERT(erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_ACTIVE); + else + erts_atomic32_read_band_relb(&p->state, ~ERTS_PSFLG_ACTIVE); erts_proc_unlock(p, ERTS_PROC_LOCKS_MSG_RECEIVE); do_schedule: { @@ -662,7 +664,8 @@ void hipe_inc_nstack(Process *p) Eterm *new_nstack = erts_alloc(ERTS_ALC_T_HIPE_STK, new_size*sizeof(Eterm)); unsigned used_size = p->hipe.nstend - p->hipe.nsp; - sys_memcpy(new_nstack+new_size-used_size, p->hipe.nsp, used_size*sizeof(Eterm)); + if (used_size) + sys_memcpy(new_nstack+new_size-used_size, p->hipe.nsp, used_size*sizeof(Eterm)); if (p->hipe.nstgraylim) p->hipe.nstgraylim = new_nstack + new_size - (p->hipe.nstend - p->hipe.nstgraylim); if (p->hipe.nstblacklim) diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c index 498b43ac6b..cf8c4139be 100644 --- a/erts/emulator/hipe/hipe_native_bif.c +++ b/erts/emulator/hipe/hipe_native_bif.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2017. All Rights Reserved. + * Copyright Ericsson AB 2001-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,6 +35,7 @@ #include "hipe_native_bif.h" #include "hipe_arch.h" #include "hipe_stack.h" +#include "erl_proc_sig_queue.h" /* * These are wrappers for BIFs that may trigger a native @@ -254,7 +255,7 @@ void hipe_handle_exception(Process *c_p) /* Synthesized to avoid having to generate code for it. */ c_p->def_arg_reg[0] = exception_tag[GET_EXC_CLASS(c_p->freason)]; - c_p->msg.saved_last = 0; /* No longer safe to use this position */ + ERTS_RECV_MARK_CLEAR(c_p); /* No longer safe to use this position */ hipe_find_handler(c_p); } @@ -544,38 +545,57 @@ Eterm hipe_check_get_msg(Process *c_p) msgp = PEEK_MESSAGE(c_p); if (!msgp) { - erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - /* Make sure messages wont pass exit signals... */ - if (ERTS_PROC_PENDING_EXIT(c_p)) { - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - return THE_NON_VALUE; /* Will be rescheduled for exit */ - } - ERTS_MSGQ_MV_INQ2PRIVQ(c_p); - msgp = PEEK_MESSAGE(c_p); - if (msgp) - erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - else { - /* XXX: BEAM doesn't need this */ - c_p->hipe_smp.have_receive_locks = 1; - c_p->flags &= ~F_DELAY_GC; - return THE_NON_VALUE; - } + int get_out; + (void) erts_proc_sig_receive_helper(c_p, CONTEXT_REDS, 0, + &msgp, &get_out); + /* FIXME: Need to bump reductions... */ + if (!msgp) { + if (get_out) { + if (get_out < 0) { + /* + * FIXME: We should get out yielding + * here... + */ + goto next_message; + } + /* Go exit... */ + return THE_NON_VALUE; + } + + /* + * If there are no more messages in queue + * (and we are not yielding or exiting) + * erts_proc_sig_receive_helper() + * returns with message queue lock locked... + */ + + /* XXX: BEAM doesn't need this */ + c_p->hipe_smp.have_receive_locks = 1; + c_p->flags &= ~F_DELAY_GC; + return THE_NON_VALUE; + } } - if (is_non_value(ERL_MESSAGE_TERM(msgp)) - && !erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) { - /* - * A corrupt distribution message that we weren't able to decode; - * remove it... - */ - ASSERT(!msgp->data.attached); - UNLINK_MESSAGE(c_p, msgp); - msgp->next = NULL; - erts_cleanup_messages(msgp); - goto next_message; + ASSERT(msgp == PEEK_MESSAGE(c_p)); + ASSERT(msgp && ERTS_SIG_IS_MSG(msgp)); + + if (ERTS_SIG_IS_EXTERNAL_MSG(msgp)) { + /* FIXME: bump appropriate amount... */ + if (!erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) { + /* + * A corrupt distribution message that we weren't able to decode; + * remove it... + */ + /* TODO: Add DTrace probe for this bad message situation? */ + UNLINK_MESSAGE(c_p, msgp); + msgp->next = NULL; + erts_cleanup_messages(msgp); + goto next_message; + } } - ASSERT(is_value(ERL_MESSAGE_TERM(msgp))); + ASSERT(msgp == PEEK_MESSAGE(c_p)); + ASSERT(ERTS_SIG_IS_INTERNAL_MSG(msgp)); return ERL_MESSAGE_TERM(msgp); } diff --git a/erts/emulator/nifs/common/prim_file_nif.c b/erts/emulator/nifs/common/prim_file_nif.c index 6874f41d75..bbd9becb47 100644 --- a/erts/emulator/nifs/common/prim_file_nif.c +++ b/erts/emulator/nifs/common/prim_file_nif.c @@ -891,10 +891,10 @@ static ERL_NIF_TERM set_owner_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM a posix_errno_t posix_errno; efile_path_t path; - Uint32 uid, gid; + Sint32 uid, gid; - if(argc != 3 || !enif_get_uint(env, argv[1], &uid) - || !enif_get_uint(env, argv[2], &gid)) { + if(argc != 3 || !enif_get_int(env, argv[1], &uid) + || !enif_get_int(env, argv[2], &gid)) { return enif_make_badarg(env); } diff --git a/erts/emulator/nifs/common/prim_file_nif.h b/erts/emulator/nifs/common/prim_file_nif.h index cc9bc8f5c3..4194cdc7d9 100644 --- a/erts/emulator/nifs/common/prim_file_nif.h +++ b/erts/emulator/nifs/common/prim_file_nif.h @@ -177,7 +177,7 @@ posix_errno_t efile_set_permissions(const efile_path_t *path, Uint32 permissions /** @brief On Unix, this will set the owner/group to the given values. It will * do nothing on other platforms. */ -posix_errno_t efile_set_owner(const efile_path_t *path, Uint32 owner, Uint32 group); +posix_errno_t efile_set_owner(const efile_path_t *path, Sint32 owner, Sint32 group); /** @brief Resolves the final path of the given link. */ posix_errno_t efile_read_link(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result); diff --git a/erts/emulator/nifs/unix/unix_prim_file.c b/erts/emulator/nifs/unix/unix_prim_file.c index 4a6c476882..1637f9cb71 100644 --- a/erts/emulator/nifs/unix/unix_prim_file.c +++ b/erts/emulator/nifs/unix/unix_prim_file.c @@ -687,7 +687,7 @@ posix_errno_t efile_set_permissions(const efile_path_t *path, Uint32 permissions return 0; } -posix_errno_t efile_set_owner(const efile_path_t *path, Uint32 owner, Uint32 group) { +posix_errno_t efile_set_owner(const efile_path_t *path, Sint32 owner, Sint32 group) { if(chown((const char*)path->data, owner, group) < 0) { return errno; } diff --git a/erts/emulator/nifs/win32/win_prim_file.c b/erts/emulator/nifs/win32/win_prim_file.c index 9b79182f2c..8058350b25 100644 --- a/erts/emulator/nifs/win32/win_prim_file.c +++ b/erts/emulator/nifs/win32/win_prim_file.c @@ -801,7 +801,7 @@ posix_errno_t efile_set_permissions(const efile_path_t *path, Uint32 permissions return windows_to_posix_errno(GetLastError()); } -posix_errno_t efile_set_owner(const efile_path_t *path, Uint32 owner, Uint32 group) { +posix_errno_t efile_set_owner(const efile_path_t *path, Sint32 owner, Sint32 group) { (void)path; (void)owner; (void)group; diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index f93d4c1557..3e77dce1cd 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2017. All Rights Reserved. + * Copyright Ericsson AB 2006-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -885,7 +885,7 @@ enif_select(ErlNifEnv* env, ErtsDrvSelectDataState *free_select = NULL; ErtsNifSelectDataState *free_nif = NULL; - ASSERT(!(resource->monitors && resource->monitors->is_dying)); + ASSERT(!resource->monitors); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS if (!grow_drv_ev_state(fd)) { @@ -1736,7 +1736,7 @@ erts_check_io(ErtsPollThread *psi) } } if (resource) { - erts_resource_stop(resource, (ErlNifEvent)fd, 1); + erts_resource_stop(resource, (ErlNifEvent)fd, 0); enif_release_resource(resource->data); } if (free_select) diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index 7aa53e8f36..ced8a4a2a7 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -782,10 +782,14 @@ update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events) struct kevent evts[2]; struct timespec ts = {0, 0}; -#ifdef EV_DISPATCH - /* If we have EV_DISPATCH we use it. The kevent descriptions for both - read and write are added on OP_ADD and removed on OP_DEL. And then - after than only EV_ENABLE|EV_DISPATCH are used. +#if defined(EV_DISPATCH) && !defined(__OpenBSD__) + /* If we have EV_DISPATCH we use it, unless we are on OpenBSD as the + behavior of EV_EOF seems to be edge triggered there and we need it + to be level triggered. + + The kevent descriptions for both read and write are added on OP_ADD + and removed on OP_DEL. And then after than only EV_ENABLE|EV_DISPATCH + are used. It could be possible to not modify the pollset when disabling and/or deleting events, but that may cause the poll threads to be awoken diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c index 96bdbacb9e..41a6fcb7e1 100644 --- a/erts/emulator/sys/common/erl_sys_common_misc.c +++ b/erts/emulator/sys/common/erl_sys_common_misc.c @@ -142,7 +142,16 @@ sys_double_to_chars(double fp, char *buffer, size_t buffer_size) return sys_double_to_chars_ext(fp, buffer, buffer_size, SYS_DEFAULT_FLOAT_DECIMALS); } -/* Convert float to string using fixed point notation. + +#if SIZEOF_LONG == 8 +# define round_int64 lround +#elif SIZEOF_LONG_LONG == 8 +# define round_int64 llround +#else +# error "No 64-bit integer type?" +#endif + +/* Convert float to string * decimals must be >= 0 * if compact != 0, the trailing 0's will be truncated */ @@ -154,80 +163,35 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals, #define FRAC_SIZE 52 #define EXP_SIZE 11 #define EXP_MASK (((Uint64)1 << EXP_SIZE) - 1) - #define MAX_DECIMALS (sizeof(cs_sys_double_pow10) \ - / sizeof(cs_sys_double_pow10[0])) + #define MAX_DECIMALS (sizeof(pow10v) / sizeof(pow10v[0])) #define FRAC_MASK (((Uint64)1 << FRAC_SIZE) - 1) #define FRAC_MASK2 (((Uint64)1 << (FRAC_SIZE + 1)) - 1) #define MAX_FLOAT ((Uint64)1 << (FRAC_SIZE+1)) - static const double cs_sys_double_pow10[] = { - SYS_DOUBLE_RND_CONST / 1e0, - SYS_DOUBLE_RND_CONST / 1e1, - SYS_DOUBLE_RND_CONST / 1e2, - SYS_DOUBLE_RND_CONST / 1e3, - SYS_DOUBLE_RND_CONST / 1e4, - SYS_DOUBLE_RND_CONST / 1e5, - SYS_DOUBLE_RND_CONST / 1e6, - SYS_DOUBLE_RND_CONST / 1e7, - SYS_DOUBLE_RND_CONST / 1e8, - SYS_DOUBLE_RND_CONST / 1e9, - SYS_DOUBLE_RND_CONST / 1e10, - SYS_DOUBLE_RND_CONST / 1e11, - SYS_DOUBLE_RND_CONST / 1e12, - SYS_DOUBLE_RND_CONST / 1e13, - SYS_DOUBLE_RND_CONST / 1e14, - SYS_DOUBLE_RND_CONST / 1e15, - SYS_DOUBLE_RND_CONST / 1e16, - SYS_DOUBLE_RND_CONST / 1e17, - SYS_DOUBLE_RND_CONST / 1e18 + static const double pow10v[] = { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, + 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18 }; - Uint64 mantissa, int_part, frac_part; - int exp; - int fbits; - int max; + double af; + Uint64 int_part, frac_part; int neg; - double fr; - union { Uint64 L; double F; } x; char *p = buffer; if (decimals < 0) return -1; - if (f >= 0) { - neg = 0; - fr = decimals < MAX_DECIMALS ? (f + cs_sys_double_pow10[decimals]) : f; - x.F = fr; - } else { + if (f < 0) { neg = 1; - fr = decimals < MAX_DECIMALS ? (f - cs_sys_double_pow10[decimals]) : f; - x.F = -fr; + af = -f; } - - exp = (x.L >> FRAC_SIZE) & EXP_MASK; - mantissa = x.L & FRAC_MASK; - - if (exp == EXP_MASK) { - if (mantissa == 0) { - if (neg) - *p++ = '-'; - *p++ = 'i'; - *p++ = 'n'; - *p++ = 'f'; - } else { - *p++ = 'n'; - *p++ = 'a'; - *p++ = 'n'; - } - *p = '\0'; - return p - buffer; + else { + neg = 0; + af = f; } - exp -= EXP_MASK >> 1; - mantissa |= ((Uint64)1 << FRAC_SIZE); - /* Don't bother with optimizing too large numbers or too large precision */ - if (x.F > MAX_FLOAT || decimals >= MAX_DECIMALS) { + if (af > MAX_FLOAT || decimals >= MAX_DECIMALS) { int len = erts_snprintf(buffer, buffer_size, "%.*f", decimals, f); char* p = buffer + len; if (len >= buffer_size) @@ -237,77 +201,64 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals, p = find_first_trailing_zero(p); *p = '\0'; return p - buffer; - } else if (exp >= FRAC_SIZE) { - int_part = mantissa << (exp - FRAC_SIZE); - frac_part = 0; - fbits = FRAC_SIZE; /* not important as frac_part==0 */ - } else if (exp >= 0) { - fbits = FRAC_SIZE - exp; - int_part = mantissa >> fbits; - frac_part = mantissa & (((Uint64)1 << fbits) -1); - } else /* if (exp < 0) */ { - int_part = 0; - frac_part = mantissa; - fbits = FRAC_SIZE - exp; - } - - if (!int_part) { - if (neg) - *p++ = '-'; - *p++ = '0'; - } else { - int ret, i, n; - while (int_part != 0) { - *p++ = (char)((int_part % 10) + '0'); - int_part /= 10; - } - if (neg) - *p++ = '-'; - /* Reverse string */ - ret = p - buffer; - for (i = 0, n = ret/2; i < n; i++) { - int j = ret - i - 1; - char c = buffer[i]; - buffer[i] = buffer[j]; - buffer[j] = c; - } } - if (decimals > 0) { - int i; - *p++ = '.'; + if (decimals) { + double int_f = floor(af); + double frac_f = round((af - int_f) * pow10v[decimals]); - max = buffer_size - (p - buffer) - 1 /* leave room for trailing '\0' */; + int_part = (Uint64)int_f; + frac_part = (Uint64)frac_f; - if (decimals > max) - return -1; /* the number is not large enough to fit in the buffer */ - - max = decimals; + if (frac_f >= pow10v[decimals]) { + /* rounding overflow carry into int_part */ + int_part++; + frac_part = 0; + } - for (i = 0; i < max; i++) { - if (frac_part > (ERTS_UINT64_MAX/5)) { - frac_part >>= 3; - fbits -= 3; + do { + Uint64 n; + if (!frac_part) { + do { + *p++ = '0'; + } while (--decimals); + break; } + n = frac_part / 10; + *p++ = (char)((frac_part - n*10) + '0'); + frac_part = n; + } while (--decimals); - /* Multiply by 10 (5*2) to extract decimal digit as integer part */ - frac_part *= 5; - fbits--; + *p++ = '.'; + } + else + int_part = (Uint64)round_int64(af); - if (fbits >= 64) { - *p++ = '0'; - } - else { - *p++ = (char)((frac_part >> fbits) + '0'); - frac_part &= ((Uint64)1 << fbits) - 1; - } + if (!int_part) { + *p++ = '0'; + } else { + do { + Uint64 n = int_part / 10; + *p++ = (char)((int_part - n*10) + '0'); + int_part = n; + } while (int_part); + } + if (neg) + *p++ = '-'; + + {/* Reverse string */ + int i = 0; + int j = p - buffer - 1; + for ( ; i < j; i++, j--) { + char tmp = buffer[i]; + buffer[i] = buffer[j]; + buffer[j] = tmp; } - - /* Delete trailing zeroes */ - if (compact) - p = find_first_trailing_zero(p); } + /* Delete trailing zeroes */ + if (compact) + p = find_first_trailing_zero(p); *p = '\0'; return p - buffer; } diff --git a/erts/emulator/sys/unix/erl_child_setup.c b/erts/emulator/sys/unix/erl_child_setup.c index 57973b10d7..10601529a4 100644 --- a/erts/emulator/sys/unix/erl_child_setup.c +++ b/erts/emulator/sys/unix/erl_child_setup.c @@ -56,6 +56,8 @@ #include <stdio.h> #include <stdarg.h> #include <sys/wait.h> +#include <sys/types.h> +#include <sys/socket.h> #define WANT_NONBLOCKING diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c index b7ac89d89a..117855acf0 100644 --- a/erts/emulator/sys/unix/sys_drivers.c +++ b/erts/emulator/sys/unix/sys_drivers.c @@ -50,6 +50,9 @@ #include <sys/ioctl.h> #endif +#include <sys/types.h> +#include <sys/socket.h> + #define WANT_NONBLOCKING /* must define this to pull in defs from sys.h */ #include "sys.h" diff --git a/erts/emulator/sys/unix/sys_uds.c b/erts/emulator/sys/unix/sys_uds.c index dd0a3b03ff..278c6b6ba1 100644 --- a/erts/emulator/sys/unix/sys_uds.c +++ b/erts/emulator/sys/unix/sys_uds.c @@ -18,6 +18,42 @@ * %CopyrightEnd% */ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#if defined(__sun__) && !defined(_XOPEN_SOURCE) +#define _XOPEN_SOURCE 500 +#endif + +#include <limits.h> + +#include <sys/types.h> +#include <sys/socket.h> + +#ifdef HAVE_SYS_SOCKETIO_H +# include <sys/socketio.h> +#endif +#ifdef HAVE_SYS_SOCKIO_H +# include <sys/sockio.h> +#endif + +#ifdef HAVE_NET_ERRNO_H +#include <net/errno.h> +#endif + +#ifdef HAVE_DIRENT_H +# include <dirent.h> +#endif + +#ifdef HAVE_UNISTD_H +# include <unistd.h> +#endif + +#include <stdlib.h> +#include <string.h> +#include <errno.h> + #include "sys_uds.h" int diff --git a/erts/emulator/sys/unix/sys_uds.h b/erts/emulator/sys/unix/sys_uds.h index a598102d5c..26c91d6a00 100644 --- a/erts/emulator/sys/unix/sys_uds.h +++ b/erts/emulator/sys/unix/sys_uds.h @@ -21,18 +21,6 @@ #ifndef _ERL_UNIX_UDS_H #define _ERL_UNIX_UDS_H -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#if defined(__sun__) && !defined(_XOPEN_SOURCE) -#define _XOPEN_SOURCE 500 -#endif - -#include <limits.h> - -#include <sys/types.h> -#include <sys/socket.h> #include <sys/uio.h> #if defined IOV_MAX @@ -43,8 +31,6 @@ #define MAXIOV 16 #endif -#include "sys.h" - int sys_uds_readv(int fd, struct iovec *iov, size_t iov_len, int *fds, int fd_count, int flags); int sys_uds_read(int fd, char *buff, size_t len, diff --git a/erts/emulator/test/beam_literals_SUITE.erl b/erts/emulator/test/beam_literals_SUITE.erl index 09761263e2..b447ca0210 100644 --- a/erts/emulator/test/beam_literals_SUITE.erl +++ b/erts/emulator/test/beam_literals_SUITE.erl @@ -248,35 +248,58 @@ literal_type_tests(Config) when is_list(Config) -> ok. make_test([{is_function=T,L}|Ts]) -> - [test(T, L),test(T, 0, L)|make_test(Ts)]; + [guard_test(T, L),guard_test(T, 0, L),body_test(T, L),body_test(T, 0, L)|make_test(Ts)]; make_test([{T,L}|Ts]) -> - [test(T, L)|make_test(Ts)]; + [guard_test(T, L),body_test(T, L)|make_test(Ts)]; make_test([]) -> []. -test(T, L) -> - S = lists:flatten(io_lib:format("begin io:format(\"~~p~n\", [{~p,~p}]), if ~w(~w) -> true; true -> false end end. ", [T, L, T, L])), - {ok,Toks,_Line} = erl_scan:string(S), - {ok,E} = erl_parse:parse_exprs(Toks), - {value,Val,_Bs} = erl_eval:exprs(E, []), +guard_test(_, L) when is_function(L) -> + %% Skip guard tests with exports - they are not literals + {atom,erl_anno:new(0),true}; +guard_test(T, L) -> + S = io_lib:format("begin io:format(\"~~p~n\", [{~p,~p}]), if ~w(~w) -> true; true -> false end end. ", [T, L, T, L]), + {Val,Expr} = eval_string(S), + Anno = erl_anno:new(0), + {match,Anno,{atom,Anno,Val},Expr}. + +guard_test(_, _, L) when is_function(L) -> + %% Skip guard tests with exports - they are not literals + {atom,erl_anno:new(0),true}; +guard_test(T, A, L) -> + S = io_lib:format("begin io:format(\"~~p~n\", [{~p,~p,~p}]), if ~w(~w, ~w) -> true; true -> false end end. ", [T,L,A,T,L,A]), + {Val,Expr} = eval_string(S), + Anno = erl_anno:new(0), + {match,Anno,{atom,Anno,Val},Expr}. + +body_test(T, L) -> + S = io_lib:format("begin io:format(\"~~p~n\", [{~p,~p}]), ~w(~w) end. ", [T,L,T,L]), + {Val,Expr} = eval_string(S), Anno = erl_anno:new(0), - {match,Anno,{atom,Anno,Val},hd(E)}. + {match,Anno,{atom,Anno,Val},Expr}. -test(T, A, L) -> - S = lists:flatten(io_lib:format("begin io:format(\"~~p~n\", [{~p,~p,~p}]), if ~w(~w, ~w) -> true; true -> false end end. ", - [T,L,A,T,L,A])), - {ok,Toks,_Line} = erl_scan:string(S), +body_test(T, A, L) -> + S = io_lib:format("begin io:format(\"~~p~n\", [{~p,~p,~p}]), ~w(~w,~w) end. ", [T,L,A,T,L,A]), + {Val,Expr} = eval_string(S), + Anno = erl_anno:new(0), + {match,Anno,{atom,Anno,Val},Expr}. + +eval_string(S) -> + {ok,Toks,_Line} = erl_scan:string(lists:flatten(S)), {ok,E} = erl_parse:parse_exprs(Toks), {value,Val,_Bs} = erl_eval:exprs(E, []), - Anno = erl_anno:new(0), - {match,Anno,{atom,Anno,Val},hd(E)}. - + {Val,hd(E)}. + literals() -> [42, 3.14, -3, 32982724987789283473473838474, [], - xxxx]. + "abc", + <<"abc">>, + {}, + xxxx, + fun erlang:erase/0]. type_tests() -> [is_boolean, diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl index e1b42e5d85..32bfcd5520 100644 --- a/erts/emulator/test/bif_SUITE.erl +++ b/erts/emulator/test/bif_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2016. All Rights Reserved. +%% Copyright Ericsson AB 2005-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -33,7 +33,10 @@ atom_to_binary/1,min_max/1, erlang_halt/1, erl_crash_dump_bytes/1, is_builtin/1, error_stacktrace/1, - error_stacktrace_during_call_trace/1]). + error_stacktrace_during_call_trace/1, + group_leader_prio/1, group_leader_prio_dirty/1, + is_process_alive/1, + process_info_blast/1]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -46,7 +49,9 @@ all() -> display, display_string, list_to_utf8_atom, atom_to_binary, binary_to_atom, binary_to_existing_atom, erl_crash_dump_bytes, min_max, erlang_halt, is_builtin, - error_stacktrace, error_stacktrace_during_call_trace]. + error_stacktrace, error_stacktrace_during_call_trace, + group_leader_prio, group_leader_prio_dirty, + is_process_alive, process_info_blast]. %% Uses erlang:display to test that erts_printf does not do deep recursion display(Config) when is_list(Config) -> @@ -825,7 +830,6 @@ error_stacktrace_during_call_trace(Config) when is_list(Config) -> end end, ok. - error_stacktrace_test() -> Types = [apply_const_last, apply_const, apply_last, @@ -963,9 +967,210 @@ do_error_1(call) -> erlang:error(id(oops)). +group_leader_prio(Config) when is_list(Config) -> + group_leader_prio_test(false). + +group_leader_prio_dirty(Config) when is_list(Config) -> + group_leader_prio_test(true). + +group_leader_prio_test(Dirty) -> + %% + %% Unfortunately back in the days node local group_leader/2 was not + %% implemented as sending an asynchronous signal to the process to change + %% group leader for. Instead it has always been synchronously changed, and + %% nothing in the documentation have hinted otherwise... Therefore I do not + %% dare the change this. + %% + %% In order to prevent priority inversion, the priority of the receiver of + %% the group leader signal is elevated while handling incoming signals if + %% the sender has a higher priority than the receiver. This test tests that + %% the priority elevation actually works... + %% + Tester = self(), + Init = erlang:whereis(init), + GL = erlang:group_leader(), + process_flag(priority, max), + {TestProcFun, NTestProcs} + = case Dirty of + false -> + %% These processes will handle all incoming signals + %% by them selves... + {fun () -> + Tester ! {alive, self()}, + receive after infinity -> ok end + end, + 100}; + true -> + %% These processes wont handle incoming signals by + %% them selves since they are stuck on dirty schedulers + %% when we try to change group leader. A dirty process + %% signal handler process (system process) will be notified + %% of the need to handle incoming signals for these processes, + %% and will instead handle the signal for these processes... + {fun () -> + %% The following sends the message '{alive, self()}' + %% to Tester once on a dirty io scheduler, then wait + %% there until the process terminates... + erts_debug:dirty_io(alive_waitexiting, Tester) + end, + erlang:system_info(dirty_io_schedulers)} + end, + TPs = lists:map(fun (_) -> + spawn_opt(TestProcFun, + [link, {priority, normal}]) + end, lists:seq(1, NTestProcs)), + lists:foreach(fun (TP) -> receive {alive, TP} -> ok end end, TPs), + TLs = lists:map(fun (_) -> + spawn_opt(fun () -> tok_loop() end, + [link, {priority, high}]) + end, + lists:seq(1, 2*erlang:system_info(schedulers))), + %% Wait to ensure distribution of high prio processes over schedulers... + receive after 1000 -> ok end, + %% + %% Test that we can get group-leader signals through to normal prio + %% processes from a max prio process even though all schedulers are filled + %% with executing high prio processes. + %% + lists:foreach(fun (_) -> + lists:foreach(fun (TP) -> + erlang:yield(), + %% whitebox -- Enqueue some signals on it + %% preventing us from hogging its main lock + %% and set group-leader directly.... + erlang:demonitor(erlang:monitor(process, TP)), + true = erlang:group_leader(Init, TP), + {group_leader, Init} = process_info(TP, group_leader), + erlang:demonitor(erlang:monitor(process, TP)), + true = erlang:group_leader(GL, TP), + {group_leader, GL} = process_info(TP, group_leader) + end, + TPs) + end, + lists:seq(1,100)), + %% + %% Also test when it is exiting... + %% + lists:foreach(fun (TP) -> + erlang:yield(), + M = erlang:monitor(process, TP), + unlink(TP), + exit(TP, bang), + badarg = try + true = erlang:group_leader(Init, TP) + catch + error : What -> What + end, + receive + {'DOWN', M, process, TP, Reason} -> + bang = Reason + end + end, + TPs), + lists:foreach(fun (TL) -> + M = erlang:monitor(process, TL), + unlink(TL), + exit(TL, bang), + receive + {'DOWN', M, process, TL, Reason} -> + bang = Reason + end + end, + TLs), + ok. +is_process_alive(Config) when is_list(Config) -> + process_flag(priority, max), + Ps = lists:map(fun (_) -> + spawn_opt(fun () -> tok_loop() end, + [{priority, high}, link]) + end, + lists:seq(1, 2*erlang:system_info(schedulers))), + receive after 1000 -> ok end, %% Wait for load to spread + lists:foreach(fun (P) -> + %% Ensure that signal order is preserved + %% and that we are not starved due to + %% priority inversion + true = erlang:is_process_alive(P), + unlink(P), + true = erlang:is_process_alive(P), + exit(P, kill), + false = erlang:is_process_alive(P) + end, + Ps), + ok. -%% Helpers +process_info_blast(Config) when is_list(Config) -> + Tester = self(), + NoAttackers = 1000, + NoAL = lists:seq(1, NoAttackers), + Consume = make_ref(), + Victim = spawn_link(fun () -> + receive + Consume -> + ok + end, + consume_msgs() + end), + AFun = fun () -> + Victim ! hej, + Res = process_info(Victim, message_queue_len), + Tester ! {self(), Res} + end, + Attackers0 = lists:map(fun (_) -> + spawn_link(AFun) + end, + NoAL), + lists:foreach(fun (A) -> + receive + {A, Res} -> + case Res of + {message_queue_len, Len} when Len > 0, Len =< NoAttackers -> + Len; + Error -> + exit({unexpected, Error}) + end + end + end, + Attackers0), + Attackers1 = lists:map(fun (_) -> + spawn_link(AFun) + end, + NoAL), + Victim ! Consume, + lists:foreach(fun (A) -> + receive + {A, Res} -> + case Res of + {message_queue_len, Len} when Len >= 0, Len =< 2*NoAttackers+1 -> + ok; + undefined -> + ok; + Error -> + exit({unexpected, Error}) + end + end + end, + Attackers1), + KillFun = fun (P) -> + unlink(P), + exit(P, kill), + false = erlang:is_process_alive(P) + end, + lists:foreach(fun (A) -> KillFun(A) end, Attackers0), + lists:foreach(fun (A) -> KillFun(A) end, Attackers1), + KillFun(Victim), + ok. + +consume_msgs() -> + receive + _ -> + consume_msgs() + after 0 -> + ok + end. + +%% helpers id(I) -> I. @@ -1005,3 +1210,11 @@ hostname([$@ | Hostname]) -> list_to_atom(Hostname); hostname([_C | Cs]) -> hostname(Cs). + +tok_loop() -> + tok_loop(hej). + +tok_loop(hej) -> + tok_loop(hopp); +tok_loop(hopp) -> + tok_loop(hej). diff --git a/erts/emulator/test/dgawd_handler.erl b/erts/emulator/test/dgawd_handler.erl index 52cdd26427..29b9d6ac7b 100644 --- a/erts/emulator/test/dgawd_handler.erl +++ b/erts/emulator/test/dgawd_handler.erl @@ -42,10 +42,10 @@ %%==================================================================== install() -> - gen_event:add_handler(error_logger, ?MODULE, []). + error_logger:add_report_handler(?MODULE, []). restore() -> - gen_event:delete_handler(error_logger, ?MODULE, []). + error_logger:delete_report_handler(?MODULE). got_dgawd_report() -> gen_event:call(error_logger, ?MODULE, got_dgawd_report, 10*60*1000). diff --git a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c index 2a8b999307..a94a2c0b02 100644 --- a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c +++ b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2017. All Rights Reserved. + * Copyright Ericsson AB 2009-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -112,15 +112,12 @@ static ERL_NIF_TERM send_from_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_ { ERL_NIF_TERM result; ErlNifPid pid; - ErlNifEnv* menv; int res; if (!enif_get_local_pid(env, argv[0], &pid)) return enif_make_badarg(env); result = enif_make_tuple2(env, enif_make_atom(env, "ok"), enif_make_pid(env, &pid)); - menv = enif_alloc_env(); - res = enif_send(env, &pid, menv, result); - enif_free_env(menv); + res = enif_send(env, &pid, NULL, result); if (!res) return enif_make_badarg(env); else @@ -131,15 +128,12 @@ static ERL_NIF_TERM send_wait_from_dirty_nif(ErlNifEnv* env, int argc, const ERL { ERL_NIF_TERM result; ErlNifPid pid; - ErlNifEnv* menv; int res; if (!enif_get_local_pid(env, argv[0], &pid)) return enif_make_badarg(env); result = enif_make_tuple2(env, enif_make_atom(env, "ok"), enif_make_pid(env, &pid)); - menv = enif_alloc_env(); - res = enif_send(env, &pid, menv, result); - enif_free_env(menv); + res = enif_send(env, &pid, NULL, result); #ifdef __WIN32__ Sleep(2000); @@ -211,10 +205,8 @@ dirty_sleeper(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) /* If we get a pid argument, it indicates a process involved in the test wants a message from us. Prior to the sleep we send a 'ready' message, and then after the sleep, send a 'done' message. */ - if (argc == 1 && enif_get_local_pid(env, argv[0], &pid)) { - msg_env = enif_alloc_env(); - enif_send(env, &pid, msg_env, enif_make_atom(msg_env, "ready")); - } + if (argc == 1 && enif_get_local_pid(env, argv[0], &pid)) + enif_send(env, &pid, NULL, enif_make_atom(env, "ready")); #ifdef __WIN32__ Sleep(2000); @@ -222,11 +214,8 @@ dirty_sleeper(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) sleep(2); #endif - if (argc == 1) { - assert(msg_env != NULL); - enif_send(env, &pid, msg_env, enif_make_atom(msg_env, "done")); - enif_free_env(msg_env); - } + if (argc == 1) + enif_send(env, &pid, NULL, enif_make_atom(env, "done")); return enif_make_atom(env, "ok"); } @@ -247,8 +236,8 @@ static ERL_NIF_TERM dirty_call_while_terminated_nif(ErlNifEnv* env, int argc, co self_term = enif_make_pid(env, &self); - result = enif_make_tuple2(env, enif_make_atom(env, "dirty_alive"), self_term); menv = enif_alloc_env(); + result = enif_make_tuple2(menv, enif_make_atom(menv, "dirty_alive"), self_term); res = enif_send(env, &to, menv, result); enif_free_env(menv); if (!res) @@ -259,9 +248,7 @@ static ERL_NIF_TERM dirty_call_while_terminated_nif(ErlNifEnv* env, int argc, co ; result = enif_make_tuple2(env, enif_make_atom(env, "dirty_dead"), self_term); - menv = enif_alloc_env(); - res = enif_send(env, &to, menv, result); - enif_free_env(menv); + res = enif_send(env, &to, NULL, result); #ifdef __WIN32__ Sleep(1000); diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl index 294c42780d..40c7cc11e1 100644 --- a/erts/emulator/test/driver_SUITE.erl +++ b/erts/emulator/test/driver_SUITE.erl @@ -2589,8 +2589,18 @@ stop_driver(Port, Name) -> ok = erl_ddll:stop(). load_driver(Dir, Driver) -> + Before = erlang:system_info(taints), case erl_ddll:load_driver(Dir, Driver) of - ok -> ok; + ok -> + After = erlang:system_info(taints), + case lists:member(Driver, Before) of + true -> + After = Before; + false -> + true = lists:member(Driver, After), + Before = lists:delete(Driver, After) + end, + ok; {error, Error} = Res -> io:format("~s\n", [erl_ddll:format_error(Error)]), Res diff --git a/erts/emulator/test/dump_SUITE.erl b/erts/emulator/test/dump_SUITE.erl index 38fa198ea6..8d18d46d92 100644 --- a/erts/emulator/test/dump_SUITE.erl +++ b/erts/emulator/test/dump_SUITE.erl @@ -96,12 +96,12 @@ get_dump_when_done(Dump) -> {ok, #file_info{ size = Sz }} -> get_dump_when_done(Dump, Sz); {error, enoent} -> - timer:sleep(100), + timer:sleep(1000), get_dump_when_done(Dump) end. get_dump_when_done(Dump, Sz) -> - timer:sleep(100), + timer:sleep(1000), case file:read_file_info(Dump) of {ok, #file_info{ size = Sz }} -> file:read_file(Dump); diff --git a/erts/emulator/test/erl_link_SUITE.erl b/erts/emulator/test/erl_link_SUITE.erl index a66ca7a57d..ed444f2599 100644 --- a/erts/emulator/test/erl_link_SUITE.erl +++ b/erts/emulator/test/erl_link_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2017. All Rights Reserved. +%% Copyright Ericsson AB 2001-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -53,26 +53,17 @@ -export([test_proc/0]). --define(LINK_UNDEF, 0). --define(LINK_PID, 1). --define(LINK_NODE, 3). - - -% These are to be kept in sync with erl_monitors.h --define(MON_ORIGIN, 1). --define(MON_TARGET, 2). - - --record(erl_link, {type = ?LINK_UNDEF, +-record(erl_link, {type, % process | port | dist_process pid = [], - targets = []}). + id}). % This is to be kept in sync with erl_bif_info.c (make_monitor_list) --record(erl_monitor, {type, % MON_ORIGIN or MON_TARGET - ref, +-record(erl_monitor, {type, % process | port | time_offset | dist_process | resource | node | nodes | suspend + dir, % origin | target + ref, % Reference | [] pid, % Process or nodename - name = []}). % registered name or [] + extra = []}). % registered name, integer or, [] suite() -> @@ -106,7 +97,7 @@ end_per_suite(_Config) -> links(Config) when is_list(Config) -> common_link_test(node(), node()), true = link(self()), - [] = find_erl_link(self(), ?LINK_PID, self()), + [] = find_erl_link(self(), process, self()), true = unlink(self()), ok. @@ -191,6 +182,7 @@ monitor_nodes(Config) when is_list(Config) -> monitor_node(A, false), monitor_node(B, true), monitor_node(C, true), + receive after 1000 -> ok end, monitor_node(C, false), monitor_node(C, true), monitor_node(B, true), @@ -304,7 +296,8 @@ run_common_process_monitors(TP1, TP2) -> wait_until(fun () -> is_proc_dead(TP2) end), ok = tp_call(TP1, fun () -> receive - {'DOWN',R2,process,TP2O,bye} -> + {'DOWN',R2,process,TP2O,Reason1} -> + bye = Reason1, ok end end), @@ -313,7 +306,8 @@ run_common_process_monitors(TP1, TP2) -> R3 = tp_call(TP1, fun () -> erlang:monitor(process, TP2) end), ok = tp_call(TP1, fun () -> receive - {'DOWN',R3,process,TP2O,noproc} -> + {'DOWN',R3,process,TP2O,Reason2} -> + noproc = Reason2, ok end end), @@ -698,22 +692,10 @@ test_proc() -> end, test_proc(). -expand_link_list([#erl_link{type = ?LINK_NODE, targets = N} = Rec | T]) -> - lists:duplicate(N,Rec#erl_link{targets = []}) ++ expand_link_list(T); -expand_link_list([#erl_link{targets = [#erl_link{pid = Pid}]} = Rec | T]) -> - [Rec#erl_link{targets = [Pid]} | expand_link_list(T)]; -expand_link_list([#erl_link{targets = [#erl_link{pid = Pid}|TT]} = Rec | T]) -> - [ Rec#erl_link{targets = [Pid]} | expand_link_list( - [Rec#erl_link{targets = TT} | T])]; -expand_link_list([#erl_link{targets = []} = Rec | T]) -> - [Rec | expand_link_list(T)]; -expand_link_list([]) -> - []. - get_local_link_list(Obj) -> case catch erts_debug:get_internal_state({link_list, Obj}) of LL when is_list(LL) -> - expand_link_list(LL); + LL; _ -> [] end. @@ -722,7 +704,7 @@ get_remote_link_list(Node, Obj) -> case catch rpc:call(Node, erts_debug, get_internal_state, [{link_list, Obj}]) of LL when is_list(LL) -> - expand_link_list(LL); + LL; _ -> [] end. @@ -776,82 +758,106 @@ get_monitor_list(undefined) -> find_erl_monitor(Pid, Ref) when is_reference(Ref) -> + MonitorList = get_monitor_list(Pid), + io:format("~p MonitorList: ~p~n", [Pid, MonitorList]), lists:foldl(fun (#erl_monitor{ref = R} = EL, Acc) when R == Ref -> [EL|Acc]; (_, Acc) -> Acc end, [], - get_monitor_list(Pid)). - -% find_erl_link(Obj, Ref) when is_reference(Ref) -> -% lists:foldl(fun (#erl_link{ref = R} = EL, Acc) when R == Ref -> -% [EL|Acc]; -% (_, Acc) -> -% Acc -% end, -% [], -% get_link_list(Obj)). - -find_erl_link(Obj, Type, [Item, Data]) when is_pid(Item); - is_port(Item); - is_atom(Item) -> - lists:foldl(fun (#erl_link{type = T, pid = I, targets = D} = EL, + MonitorList); +find_erl_monitor(Pid, Item) -> + MonitorList = get_monitor_list(Pid), + io:format("~p MonitorList: ~p~n", [Pid, MonitorList]), + lists:foldl(fun (#erl_monitor{pid = I} = EL, Acc) when I == Item -> + [EL|Acc]; + (_, Acc) -> + Acc + end, + [], + MonitorList). + + +find_erl_link(Obj, Type, Item) when is_pid(Item); is_port(Item) -> + LinkList = get_link_list(Obj), + io:format("~p LinkList: ~p~n", [Obj, LinkList]), + lists:foldl(fun (#erl_link{type = T, pid = I} = EL, Acc) when T == Type, I == Item -> - case Data of - D -> - [EL|Acc]; - [] -> - [EL|Acc]; - _ -> - Acc - end; + [EL|Acc]; + (_, Acc) -> + Acc + end, + [], + LinkList); +find_erl_link(Obj, Type, Id) when is_integer(Id) -> + %% Find by Id + LinkList = get_link_list(Obj), + io:format("~p LinkList: ~p~n", [Obj, LinkList]), + lists:foldl(fun (#erl_link{type = T, id = I} = EL, + Acc) when T == Type, I == Id -> + [EL|Acc]; (_, Acc) -> Acc end, [], - get_link_list(Obj)); -find_erl_link(Obj, Type, Item) when is_pid(Item); is_port(Item); is_atom(Item) -> - find_erl_link(Obj, Type, [Item, []]). + LinkList). +get_link_type(A, B) when is_port(A); + is_port(B) -> + port; +get_link_type(A, B) when is_pid(A), + is_pid(B) -> + case node(A) == node(B) of + true -> + process; + false -> + dist_process + end. +check_link(A, B) when node(A) == node(B) -> + LinkType = get_link_type(A, B), + [#erl_link{type = LinkType, + pid = B, + id = Id}] = find_erl_link(A, LinkType, B), + [#erl_link{type = LinkType, + pid = A, + id = Id}] = find_erl_link(B, LinkType, A), + [] = find_erl_link({node(A), node(B)}, + LinkType, + A), + [] = find_erl_link({node(B), node(A)}, + LinkType, + B), + ok; check_link(A, B) -> - [#erl_link{type = ?LINK_PID, + [#erl_link{type = dist_process, pid = B, - targets = []}] = find_erl_link(A, ?LINK_PID, B), - [#erl_link{type = ?LINK_PID, + id = IdA}] = find_erl_link(A, dist_process, B), + [#erl_link{type = dist_process, pid = A, - targets = []}] = find_erl_link(B, ?LINK_PID, A), - case node(A) == node(B) of - false -> - [#erl_link{type = ?LINK_PID, - pid = A, - targets = [B]}] = find_erl_link({node(A), - node(B)}, - ?LINK_PID, - [A, [B]]), - [#erl_link{type = ?LINK_PID, - pid = B, - targets = [A]}] = find_erl_link({node(B), - node(A)}, - ?LINK_PID, - [B, [A]]); - true -> - [] = find_erl_link({node(A), node(B)}, - ?LINK_PID, - [A, [B]]), - [] = find_erl_link({node(B), node(A)}, - ?LINK_PID, - [B, [A]]) - end, + id = IdA}] = find_erl_link({node(A), + node(B)}, + dist_process, + IdA), + [#erl_link{type = dist_process, + pid = A, + id = IdB}] = find_erl_link(B, dist_process, A), + [#erl_link{type = dist_process, + pid = B, + id = IdB}] = find_erl_link({node(B), + node(A)}, + dist_process, + IdB), ok. check_unlink(A, B) -> - [] = find_erl_link(A, ?LINK_PID, B), - [] = find_erl_link(B, ?LINK_PID, A), - [] = find_erl_link({node(A), node(B)}, ?LINK_PID, [A, [B]]), - [] = find_erl_link({node(B), node(A)}, ?LINK_PID, [B, [A]]), + LinkType = get_link_type(A, B), + [] = find_erl_link(A, LinkType, B), + [] = find_erl_link(B, LinkType, A), + [] = find_erl_link({node(A), node(B)}, dist_process, A), + [] = find_erl_link({node(B), node(A)}, dist_process, B), ok. check_process_monitor(From, {Name, Node}, Ref) when is_pid(From), @@ -864,22 +870,26 @@ check_process_monitor(From, {Name, Node}, Ref) when is_pid(From), is_atom(Node), is_reference(Ref) -> MonitoredPid = rpc:call(Node, erlang, whereis, [Name]), - [#erl_monitor{type = ?MON_ORIGIN, + [#erl_monitor{type = dist_process, + dir = origin, ref = Ref, pid = Node, - name = Name}] = find_erl_monitor(From, Ref), - [#erl_monitor{type = ?MON_TARGET, + extra = Name}] = find_erl_monitor(From, Ref), + [#erl_monitor{type = dist_process, + dir = target, ref = Ref, pid = From, - name = Name}] = find_erl_monitor({node(From), Node}, Ref), - [#erl_monitor{type = ?MON_ORIGIN, + extra = Name}] = find_erl_monitor({node(From), Node}, Ref), + [#erl_monitor{type = dist_process, + dir = origin, ref = Ref, pid = MonitoredPid, - name = Name}] = find_erl_monitor({Node, node(From)}, Ref), - [#erl_monitor{type = ?MON_TARGET, + extra = Name}] = find_erl_monitor({Node, node(From)}, Ref), + [#erl_monitor{type = dist_process, + dir = target, ref = Ref, pid = From, - name = Name}] = find_erl_monitor(MonitoredPid, Ref), + extra = Name}] = find_erl_monitor(MonitoredPid, Ref), ok; check_process_monitor(From, Name, Ref) when is_pid(From), is_atom(Name), @@ -887,27 +897,36 @@ check_process_monitor(From, Name, Ref) when is_pid(From), is_reference(Ref) -> MonitoredPid = rpc:call(node(From), erlang, whereis, [Name]), - [#erl_monitor{type = ?MON_ORIGIN, + [#erl_monitor{type = process, + dir = origin, ref = Ref, pid = MonitoredPid, - name = Name}] = find_erl_monitor(From, Ref), + extra = Name}] = find_erl_monitor(From, Ref), - [#erl_monitor{type = ?MON_TARGET, + [#erl_monitor{type = process, + dir = target, ref = Ref, pid = From, - name = Name}] = find_erl_monitor(MonitoredPid,Ref), + extra = Name}] = find_erl_monitor(MonitoredPid,Ref), ok; check_process_monitor(From, To, Ref) when is_pid(From), is_pid(To), is_reference(Ref) -> - OriMon = [#erl_monitor{type = ?MON_ORIGIN, + MonType = case node(From) == node(To) of + true -> process; + false -> dist_process + end, + + OriMon = [#erl_monitor{type = MonType, + dir = origin, ref = Ref, pid = To}], OriMon = find_erl_monitor(From, Ref), - TargMon = [#erl_monitor{type = ?MON_TARGET, + TargMon = [#erl_monitor{type = MonType, + dir = target, ref = Ref, pid = From}], TargMon = find_erl_monitor(To, Ref), @@ -915,7 +934,11 @@ check_process_monitor(From, To, Ref) when is_pid(From), case node(From) == node(To) of false -> - TargMon = find_erl_monitor({node(From), node(To)}, Ref), + DistTargMon = [#erl_monitor{type = dist_process, + dir = target, + ref = Ref, + pid = From}], + DistTargMon = find_erl_monitor({node(From), node(To)}, Ref), OriMon = find_erl_monitor({node(To), node(From)}, Ref); true -> [] = find_erl_monitor({node(From), node(From)}, Ref) @@ -986,19 +1009,36 @@ check_process_demonitor(From, To, Ref) when is_pid(From), ok. no_of_monitor_node(From, Node) when is_pid(From), is_atom(Node) -> - length(find_erl_link(From, ?LINK_NODE, Node)). + case find_erl_monitor(From, Node) of + [] -> 0; + [#erl_monitor{type = node, + dir = origin, + pid = Node, + extra = N}] -> N + end. +check_monitor_node(From, Node, 0) when is_pid(From), + is_atom(Node) -> + [] = find_erl_monitor(From, Node), + [] = find_erl_monitor({node(From), Node}, From); check_monitor_node(From, Node, No) when is_pid(From), is_atom(Node), is_integer(No), - No >= 0 -> - LL = lists:duplicate(No, #erl_link{type = ?LINK_NODE, pid = Node}), - DLL = lists:duplicate(No, #erl_link{type = ?LINK_NODE, pid = From}), - LL = find_erl_link(From, ?LINK_NODE, Node), - DLL = find_erl_link({node(From), Node}, ?LINK_NODE, From), - ok. - - + No > 0 -> + [#erl_monitor{type = node, + dir = origin, + pid = Node, + extra = No}] = find_erl_monitor(From, Node), + [#erl_monitor{type = node, + dir = target, + pid = From}] = find_erl_monitor({node(From), Node}, From). + +connection_id(Node) -> + try + erts_debug:get_internal_state({connection_id, Node}) + catch + _:_ -> -1 + end. hostname() -> from($@, atom_to_list(node())). diff --git a/erts/emulator/test/exception_SUITE.erl b/erts/emulator/test/exception_SUITE.erl index da0292f385..60d14ce841 100644 --- a/erts/emulator/test/exception_SUITE.erl +++ b/erts/emulator/test/exception_SUITE.erl @@ -23,7 +23,8 @@ -export([all/0, suite/0, badmatch/1, pending_errors/1, nil_arith/1, top_of_stacktrace/1, stacktrace/1, nested_stacktrace/1, raise/1, gunilla/1, per/1, - exception_with_heap_frag/1, line_numbers/1]). + exception_with_heap_frag/1, backtrace_depth/1, + line_numbers/1]). -export([bad_guy/2]). -export([crash/1]). @@ -42,7 +43,7 @@ suite() -> all() -> [badmatch, pending_errors, nil_arith, top_of_stacktrace, stacktrace, nested_stacktrace, raise, gunilla, per, - exception_with_heap_frag, line_numbers]. + exception_with_heap_frag, backtrace_depth, line_numbers]. -define(try_match(E), catch ?MODULE:bar(), @@ -572,6 +573,57 @@ do_exception_with_heap_frag(Bin, [Sz|Sizes]) -> do_exception_with_heap_frag(Bin, Sizes); do_exception_with_heap_frag(_, []) -> ok. +backtrace_depth(Config) when is_list(Config) -> + _ = [do_backtrace_depth(D) || D <- lists:seq(0, 8)], + ok. + +do_backtrace_depth(D) -> + Old = erlang:system_flag(backtrace_depth, D), + try + Expected = max(1, D), + do_backtrace_depth_1(Expected) + after + _ = erlang:system_flag(backtrace_depth, Old) + end. + +do_backtrace_depth_1(D) -> + Exit = fun() -> + error(reason) + end, + HandCrafted = fun() -> + {'EXIT',{_,Stk0}} = (catch error(get_stacktrace)), + %% Fool the compiler to force a hand-crafted + %% stacktrace. + Stk = [hd(Stk0)|tl(Stk0)], + erlang:raise(error, reason, Stk) + end, + PassedOn = fun() -> + try error(get_stacktrace) + catch error:_:Stk -> + %% Just pass on the given stacktrace. + erlang:raise(error, reason, Stk) + end + end, + do_backtrace_depth_2(D, Exit), + do_backtrace_depth_2(D, HandCrafted), + do_backtrace_depth_2(D, PassedOn), + ok. + +do_backtrace_depth_2(D, Exc) -> + try + Exc() + catch + error:reason:Stk -> + if + length(Stk) =/= D -> + io:format("Expected depth: ~p\n", [D]), + io:format("~p\n", [Stk]), + error(bad_depth); + true -> + ok + end + end. + line_numbers(Config) when is_list(Config) -> {'EXIT',{{case_clause,bad_tag}, [{?MODULE,line1,2, diff --git a/erts/emulator/test/lcnt_SUITE.erl b/erts/emulator/test/lcnt_SUITE.erl index 4e52c2813c..dfffd662e2 100644 --- a/erts/emulator/test/lcnt_SUITE.erl +++ b/erts/emulator/test/lcnt_SUITE.erl @@ -87,8 +87,9 @@ wait_for_empty_lock_list() -> wait_for_empty_lock_list(10). wait_for_empty_lock_list(Tries) when Tries > 0 -> try_flush_cleanup_ops(), - case erts_debug:lcnt_collect() of - [{duration, _}, {locks, []}] -> + [{duration, _}, {locks, Locks}] = erts_debug:lcnt_collect(), + case remove_untoggleable_locks(Locks) of + [] -> ok; _ -> timer:sleep(50), @@ -124,7 +125,7 @@ toggle_lock_counting(Config) when is_list(Config) -> get_lock_info_for(Categories) when is_list(Categories) -> ok = erts_debug:lcnt_control(mask, Categories), [{duration, _}, {locks, Locks}] = erts_debug:lcnt_collect(), - Locks; + remove_untoggleable_locks(Locks); get_lock_info_for(Category) when is_atom(Category) -> get_lock_info_for([Category]). @@ -178,3 +179,13 @@ registered_db_tables(Config) when is_list(Config) -> (_Lock) -> false end, DbLocks), ok. + +%% Not all locks can be toggled on or off due to technical limitations, so we +%% need to filter them out when checking whether we successfully disabled lock +%% counting. +remove_untoggleable_locks([]) -> + []; +remove_untoggleable_locks([{resource_monitors, _, _, _} | T]) -> + remove_untoggleable_locks(T); +remove_untoggleable_locks([H | T]) -> + [H | remove_untoggleable_locks(T)]. diff --git a/erts/emulator/test/map_SUITE.erl b/erts/emulator/test/map_SUITE.erl index c9e971af8a..43807b4388 100644 --- a/erts/emulator/test/map_SUITE.erl +++ b/erts/emulator/test/map_SUITE.erl @@ -36,6 +36,7 @@ t_map_equal/1, t_map_compare/1, t_map_size/1, + t_map_get/1, t_is_map/1, %% Specific Map BIFs @@ -124,7 +125,7 @@ all() -> [t_build_and_match_literals, t_build_and_match_literals_large, %% erlang t_erlang_hash, t_map_encode_decode, t_gc_rare_map_overflow, - t_map_size, t_is_map, + t_map_size, t_map_get, t_is_map, %% non specific BIF related t_bif_build_and_check, @@ -680,6 +681,48 @@ t_map_size(Config) when is_list(Config) -> end), ok. +t_map_get(Config) when is_list(Config) -> + %% small map + 1 = map_get(a, id(#{a=>1})), + 2 = map_get(b, id(#{a=>1, b=>2})), + "hi" = map_get("hello", id(#{a=>1, "hello"=>"hi"})), + "tuple hi" = map_get({1,1.0}, id(#{a=>a, {1,1.0}=>"tuple hi"})), + + M0 = id(#{ k1=>"v1", <<"k2">> => <<"v3">> }), + "v4" = map_get(<<"k2">>, M0#{<<"k2">> => "v4"}), + + %% large map + M1 = maps:from_list([{I,I}||I<-lists:seq(1,100)] ++ + [{a,1},{b,2},{"hello","hi"},{{1,1.0},"tuple hi"}, + {k1,"v1"},{<<"k2">>,"v3"}]), + 1 = map_get(a, M1), + 2 = map_get(b, M1), + "hi" = map_get("hello", M1), + "tuple hi" = map_get({1,1.0}, M1), + "v3" = map_get(<<"k2">>, M1), + + %% error cases + do_badmap(fun(T) -> + {'EXIT',{{badmap,T},[{erlang,map_get,_,_}|_]}} = + (catch map_get(a, T)) + end), + + {'EXIT',{{badkey,{1,1}},[{erlang,map_get,_,_}|_]}} = + (catch map_get({1,1}, id(#{{1,1.0}=>"tuple"}))), + {'EXIT',{{badkey,a},[{erlang,map_get,_,_}|_]}} = (catch map_get(a, id(#{}))), + {'EXIT',{{badkey,a},[{erlang,map_get,_,_}|_]}} = + (catch map_get(a, id(#{b=>1, c=>2}))), + + %% in guards + M2 = id(#{a=>1}), + true = if map_get(a, M2) =:= 1 -> true; true -> false end, + false = if map_get(x, M2) =:= 1 -> true; true -> false end, + do_badmap(fun + (T) when map_get(T, x) =:= 1 -> ok; + (T) -> false = is_map(T) + end), + ok. + build_and_check_size([K|Ks],N,M0) -> N = map_size(M0), M1 = M0#{ K => K }, diff --git a/erts/emulator/test/match_spec_SUITE.erl b/erts/emulator/test/match_spec_SUITE.erl index 08a7b4560c..c1bc01f01e 100644 --- a/erts/emulator/test/match_spec_SUITE.erl +++ b/erts/emulator/test/match_spec_SUITE.erl @@ -885,6 +885,19 @@ maps(Config) when is_list(Config) -> erlang:match_spec_test(#{<<"b">> =>"camembert","c"=>"cabécou", "wat"=>"hi", b=><<"other">>}, [{#{<<"b">> => '$1',"wat" => '$2'},[],[#{a=>'$1',b=>'$2'}]}], table), + + {ok,1,[],[]} = erlang:match_spec_test(#{a => 1}, [{'$1',[],[{map_size,'$1'}]}],table), + {ok,'EXIT',[],[]} = erlang:match_spec_test(not_a_map, [{'$1',[],[{map_size,'$1'}]}], table), + {ok,false,[],[]} = erlang:match_spec_test(not_a_map, [{'$1',[{map_size,'$1'}],['$_']}], table), + {ok,true,[],[]} = erlang:match_spec_test(#{a => 1}, [{'$1',[{'=:=',{map_size,'$1'},1}],[true]}], table), + + {ok,1,[],[]} = erlang:match_spec_test(#{a => 1}, [{'$1',[],[{map_get,a,'$1'}]}], table), + {ok,'EXIT',[],[]} = erlang:match_spec_test(#{a => 1}, [{'$1',[],[{map_get,b,'$1'}]}], table), + {ok,'EXIT',[],[]} = erlang:match_spec_test(not_a_map, [{'$1',[],[{map_get,b,'$1'}]}], table), + {ok,false,[],[]} = erlang:match_spec_test(#{a => 1}, [{'$1',[{map_get,b,'$1'}],['$_']}], table), + {ok,false,[],[]} = erlang:match_spec_test(not_a_map, [{'$1',[{map_get,b,'$1'}],['$_']}], table), + {ok,true,[],[]} = erlang:match_spec_test(#{a => true}, [{'$1',[{map_get,a,'$1'}],[true]}], table), + %% large maps Ls0 = [{I,<<I:32>>}||I <- lists:seq(1,415)], diff --git a/erts/emulator/test/module_info_SUITE.erl b/erts/emulator/test/module_info_SUITE.erl index ba9b564fdc..46a3bba732 100644 --- a/erts/emulator/test/module_info_SUITE.erl +++ b/erts/emulator/test/module_info_SUITE.erl @@ -23,7 +23,7 @@ -include_lib("common_test/include/ct.hrl"). -export([all/0, suite/0, - exports/1,functions/1,deleted/1,native/1,info/1]). + exports/1,functions/1,deleted/1,native/1,info/1,nifs/1]). %%-compile(native). @@ -38,7 +38,7 @@ all() -> modules(). modules() -> - [exports, functions, deleted, native, info]. + [exports, functions, deleted, native, info, nifs]. %% Should return all functions exported from this module. (local) all_exported() -> @@ -62,12 +62,24 @@ exports(Config) when is_list(Config) -> All = lists:sort(?MODULE:module_info(exports)), ok. -%% Test that the list of exported functions from this module is correct. +%% Test that the list of local and exported functions from this module is +%% correct. functions(Config) when is_list(Config) -> All = all_functions(), All = lists:sort(?MODULE:module_info(functions)), ok. +nifs(Config) when is_list(Config) -> + [] = ?MODULE:module_info(nifs), + + %% erl_tracer is guaranteed to be present and contain these NIFs + TraceNIFs = erl_tracer:module_info(nifs), + true = lists:member({enabled, 3}, TraceNIFs), + true = lists:member({trace, 5}, TraceNIFs), + 2 = length(TraceNIFs), + + ok. + %% Test that deleted modules cause badarg deleted(Config) when is_list(Config) -> Data = proplists:get_value(data_dir, Config), diff --git a/erts/emulator/test/monitor_SUITE.erl b/erts/emulator/test/monitor_SUITE.erl index 9d772480d9..c7250a9d26 100644 --- a/erts/emulator/test/monitor_SUITE.erl +++ b/erts/emulator/test/monitor_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2017. All Rights Reserved. +%% Copyright Ericsson AB 1999-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -86,7 +86,7 @@ case_2(Config) when is_list(Config) -> R = erlang:monitor(process, B), B ! R, receive - {'EXIT', _} -> ok; + true -> ok; Other -> ct:fail({rec, Other}) end, @@ -98,7 +98,7 @@ case_2a(Config) when is_list(Config) -> {B,R} = spawn_monitor(?MODULE, y2, [self()]), B ! R, receive - {'EXIT', _} -> ok; + true -> ok; Other -> ct:fail({rec, Other}) end, @@ -182,7 +182,7 @@ demon_e_1(Config) when is_list(Config) -> end ), receive {P2, ref, R2} -> - demon_error(R2, badarg), + true = erlang:demonitor(R2), P2 ! {self(), stop}; Other2 -> ct:fail({rec, Other2}) @@ -729,8 +729,8 @@ named_down(Config) when is_list(Config) -> end), ?assertEqual(true, register(Name, NamedProc)), unlink(NamedProc), - exit(NamedProc, bang), Mon = erlang:monitor(process, Name), + exit(NamedProc, bang), receive {'DOWN',Mon, _, _, bang} -> ok after 3000 -> ?assert(false) end, ?assertEqual(true, register(Name, self())), diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index b6e15ababb..df521311e3 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -653,7 +653,7 @@ select_steal(Config) when is_list(Config) -> check_stop_ret(select_nif(RFd, ?ERL_NIF_SELECT_STOP, RFd, null, Ref)), ?assertMatch([{fd_resource_stop, RPtr, _}], flush()), - {1, {RPtr, 1}} = last_fd_stop_call(), + {1, {RPtr, _DirectCall}} = last_fd_stop_call(), ?assert(is_closed_nif(WFd)), @@ -1170,6 +1170,12 @@ maps(Config) when is_list(Config) -> {1, M2} = make_map_remove_nif(M2, "key3"), {0, undefined} = make_map_remove_nif(self(), key), + M1 = maps_from_list_nif(maps:to_list(M1)), + M2 = maps_from_list_nif(maps:to_list(M2)), + M3 = maps_from_list_nif(maps:to_list(M3)), + + has_duplicate_keys = maps_from_list_nif([{1,1},{1,1}]), + verify_tmpmem(TmpMem), ok. diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c index fa9ae1015c..a0aef60cf1 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c +++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c @@ -2140,24 +2140,45 @@ static ERL_NIF_TERM make_map_remove_nif(ErlNifEnv* env, int argc, const ERL_NIF_ /* maps */ static ERL_NIF_TERM maps_from_list_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - ERL_NIF_TERM cell = argv[0]; - ERL_NIF_TERM map = enif_make_new_map(env); - ERL_NIF_TERM tuple; - const ERL_NIF_TERM *pair; - int arity = -1; + ERL_NIF_TERM *keys, *values; + ERL_NIF_TERM result, cell; + unsigned count; - if (argc != 1 && !enif_is_list(env, cell)) return enif_make_badarg(env); + if (argc != 1 || !enif_get_list_length(env, argv[0], &count)) { + return enif_make_badarg(env); + } - /* assume sorted keys */ + keys = enif_alloc(sizeof(ERL_NIF_TERM) * count * 2); + values = keys + count; - while (!enif_is_empty_list(env,cell)) { - if (!enif_get_list_cell(env, cell, &tuple, &cell)) return enif_make_badarg(env); - if (enif_get_tuple(env,tuple,&arity,&pair)) { - enif_make_map_put(env, map, pair[0], pair[1], &map); - } + cell = argv[0]; + count = 0; + + while (!enif_is_empty_list(env, cell)) { + const ERL_NIF_TERM *pair; + ERL_NIF_TERM tuple; + int arity; + + if (!enif_get_list_cell(env, cell, &tuple, &cell) + || !enif_get_tuple(env, tuple, &arity, &pair) + || arity != 2) { + enif_free(keys); + return enif_make_badarg(env); + } + + keys[count] = pair[0]; + values[count] = pair[1]; + + count++; + } + + if (!enif_make_map_from_arrays(env, keys, values, count, &result)) { + result = enif_make_atom(env, "has_duplicate_keys"); } - return map; + enif_free(keys); + + return result; } static ERL_NIF_TERM sorted_list_from_maps_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { @@ -2838,7 +2859,7 @@ unsigned rand_bits(struct frenzy_rand_bits* rnd, unsigned int nbits) struct frenzy_monitor { ErlNifMutex* lock; - enum { + volatile enum { MON_FREE, MON_FREE_DOWN, MON_FREE_DEMONITOR, MON_TRYING, MON_ACTIVE, MON_PENDING } state; @@ -3200,13 +3221,24 @@ static void frenzy_resource_down(ErlNifEnv* env, void* obj, ErlNifPid* pid, DBG_TRACE3("DOWN pid=%T, r=%p rix=%u\n", pid->pid, r, r->rix); for (mix = 0; mix < FRENZY_MONITORS_MAX; mix++) { - if (r->monv[mix].pid.pid == pid->pid && r->monv[mix].state >= MON_TRYING) { + int state1 = r->monv[mix].state; + /* First do dirty access of pid and state without the lock */ + if (r->monv[mix].pid.pid == pid->pid && state1 >= MON_TRYING) { + int state2; enif_mutex_lock(r->monv[mix].lock); - if (enif_compare_monitors(mon, &r->monv[mix].mon) == 0) { - assert(r->monv[mix].state >= MON_ACTIVE); - r->monv[mix].state = MON_FREE_DOWN; - enif_mutex_unlock(r->monv[mix].lock); - return; + state2 = r->monv[mix].state; + if (state2 >= MON_ACTIVE) { + if (enif_compare_monitors(mon, &r->monv[mix].mon) == 0) { + r->monv[mix].state = MON_FREE_DOWN; + enif_mutex_unlock(r->monv[mix].lock); + return; + } + } + else { + assert(state2 != MON_TRYING); + assert(state1 == MON_TRYING || /* racing monitor failed */ + state2 == MON_FREE_DEMONITOR || /* racing demonitor */ + state2 == MON_FREE_DOWN); /* racing down */ } enif_mutex_unlock(r->monv[mix].lock); } diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl index 592542405f..290bb61fc8 100644 --- a/erts/emulator/test/num_bif_SUITE.erl +++ b/erts/emulator/test/num_bif_SUITE.erl @@ -213,6 +213,20 @@ fts_rand_float_decimals(N) -> [begin F0 = rand_float_reasonable(), L0 = float_to_list(F0, [{decimals, D}]), + case conform_with_io_lib_format_os(F0,D) of + false -> ok; + true -> + IOL = lists:flatten(io_lib:format("~.*f", [D, F0])), + true = case L0 =:= IOL of + true -> true; + false -> + io:format("F0 = ~w ~w\n", [F0, <<F0/float>>]), + io:format("decimals = ~w\n", [D]), + io:format("float_to_list = ~s\n", [L0]), + io:format("io_lib:format = ~s\n", [IOL]), + false + end + end, L1 = case D of 0 -> L0 ++ ".0"; _ -> L0 @@ -234,6 +248,26 @@ fts_rand_float_decimals(N) -> fts_rand_float_decimals(N-1). +conform_with_io_lib_format_os(F, D) -> + case os:type() of + {win32,_} -> + %% io_lib:format("~.*f") buggy on windows? OTP-15010 + false; + _ -> + conform_with_io_lib_format(F, D) + end. + +conform_with_io_lib_format(_, 0) -> + %% io_lib:format("~.*f") does not support zero decimals + false; +conform_with_io_lib_format(_, D) when D > 10 -> + %% Seems float_to_list gets it slightly wrong sometimes for many decimals + false; +conform_with_io_lib_format(F, D) -> + %% io_lib:format prints '0' for input bits beyond mantissa precision + %% float_to_list treats those unknown input bits as if they were zeros. + math:log2(abs(F) * math:pow(10,D)) < 54. + max_diff_decimals(F, D) -> IntBits = floor(math:log2(abs(F))) + 1, FracBits = (52 - IntBits), diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl index e46665a881..5b39d05df8 100644 --- a/erts/emulator/test/port_SUITE.erl +++ b/erts/emulator/test/port_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2017. All Rights Reserved. +%% Copyright Ericsson AB 1997-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -109,7 +109,6 @@ mon_port_pid_demonitor/1, mon_port_remote_on_remote/1, mon_port_driver_die/1, - mon_port_driver_die_demonitor/1, mul_basic/1, mul_slow_writes/1, name1/1, @@ -180,8 +179,7 @@ all() -> mon_port_bad_named, mon_port_pid_demonitor, mon_port_name_demonitor, - mon_port_driver_die, - mon_port_driver_die_demonitor + mon_port_driver_die ]. groups() -> @@ -2783,7 +2781,7 @@ mon_port_driver_die(Config) -> end, ok. - +-ifdef(DISABLED_TESTCASE). %% 1. Spawn a port which will sleep 3 seconds %% 2. Monitor port %% 3. Port driver and dies horribly (via C driver_failure call). This should @@ -2819,6 +2817,7 @@ mon_port_driver_die_demonitor(Config) -> after 5000 -> ?assert(false) end, ok. +-endif. %% @doc Makes a controllable port for testing. Underlying mechanism of this %% port is not important, only important is our ability to close/kill it or diff --git a/erts/emulator/test/port_trace_SUITE.erl b/erts/emulator/test/port_trace_SUITE.erl index a1986397a8..eba8f194e0 100644 --- a/erts/emulator/test/port_trace_SUITE.erl +++ b/erts/emulator/test/port_trace_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2017. All Rights Reserved. +%% Copyright Ericsson AB 1999-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -202,8 +202,7 @@ ports(_Config) -> erlang:port_close(Prt), [{trace,Prt,closed,normal}, - {trace,Prt,unregister,port_trace_SUITE}, - {trace,Prt,unlink,S}] = flush(), + {trace,Prt,unregister,port_trace_SUITE}] = flush(), ok. @@ -475,8 +474,7 @@ failure_test(Failure, Reason) -> process_flag(trap_exit, false) end, [{trace, Prt, 'receive', {S, {command, Failure}}}, - {trace, Prt, closed, Reason}, - {trace, Prt, unlink, S}] = flush(), + {trace, Prt, closed, Reason}] = flush(), ok. @@ -599,13 +597,11 @@ close(Prt, Flags) -> if Recv, Ports -> [{trace, Prt, 'receive', {S, close}}, - {trace, Prt, closed, normal}, - {trace, Prt, unlink, S}] = flush(); + {trace, Prt, closed, normal}] = flush(); Recv -> [{trace, Prt, 'receive', {S, close}}] = flush(); Ports -> - [{trace, Prt, closed, normal}, - {trace, Prt, unlink, S}] = flush(); + [{trace, Prt, closed, normal}] = flush(); true -> [] = flush() end. diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl index a8bcfac84d..585c5a1871 100644 --- a/erts/emulator/test/process_SUITE.erl +++ b/erts/emulator/test/process_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2017. All Rights Reserved. +%% Copyright Ericsson AB 1997-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -42,8 +42,9 @@ process_info_lock_reschedule2/1, process_info_lock_reschedule3/1, process_info_garbage_collection/1, + process_info_smoke_all/1, + process_info_status_handled_signal/1, bump_reductions/1, low_prio/1, binary_owner/1, yield/1, yield2/1, - process_status_exiting/1, otp_4725/1, bad_register/1, garbage_collect/1, otp_6237/1, process_info_messages/1, process_flag_badarg/1, process_flag_heap_size/1, spawn_opt_heap_size/1, spawn_opt_max_heap_size/1, @@ -80,7 +81,8 @@ all() -> process_info_lock_reschedule2, process_info_lock_reschedule3, process_info_garbage_collection, - process_status_exiting, + process_info_smoke_all, + process_info_status_handled_signal, bump_reductions, low_prio, yield, yield2, otp_4725, bad_register, garbage_collect, process_info_messages, process_flag_badarg, process_flag_heap_size, @@ -512,14 +514,20 @@ pio_current_location(N, Pid, Pi, Looper) -> case Where of {erlang,process_info,2,[]} -> pio_current_location(N-1, Pid, Pi+1, Looper); + {erts_internal,await_result,1, Loc} when is_list(Loc) -> + pio_current_location(N-1, Pid, Pi+1, Looper); {?MODULE,process_info_looper,1,Loc} when is_list(Loc) -> - pio_current_location(N-1, Pid, Pi, Looper+1) + pio_current_location(N-1, Pid, Pi, Looper+1); + _ -> + exit({unexpected_location, Where}) end. pio_current_stacktrace() -> L = [begin - {current_stacktrace,Stk} = process_info(P, current_stacktrace), - {P,Stk} + case process_info(P, current_stacktrace) of + {current_stacktrace, Stk} -> {P,Stk}; + undefined -> {P, []} + end end || P <- processes()], [erlang:garbage_collect(P) || {P,_} <- L], erlang:garbage_collect(), @@ -841,28 +849,6 @@ process_info_lock_reschedule3(Config) when is_list(Config) -> ct:fail(BadStatus) end. -process_status_exiting(Config) when is_list(Config) -> - %% Make sure that erts_debug:get_internal_state({process_status,P}) - %% returns exiting if it is in status P_EXITING. - erts_debug:set_internal_state(available_internal_state,true), - Prio = process_flag(priority, max), - P = spawn_opt(fun () -> receive after infinity -> ok end end, - [{priority, normal}]), - erlang:yield(), - %% The tok_loop processes are here to make it hard for the exiting - %% process to be scheduled in for exit... - TokLoops = lists:map(fun (_) -> - spawn_opt(fun tok_loop/0, - [link,{priority, high}]) - end, lists:seq(1, erlang:system_info(schedulers_online))), - exit(P, boom), - wait_until(fun() -> - exiting =:= erts_debug:get_internal_state({process_status,P}) - end), - lists:foreach(fun (Tok) -> unlink(Tok), exit(Tok,bang) end, TokLoops), - process_flag(priority, Prio), - ok. - otp_4725(Config) when is_list(Config) -> Tester = self(), Ref1 = make_ref(), @@ -997,10 +983,110 @@ process_info_garbage_collection(_Config) -> gv(Key,List) -> proplists:get_value(Key,List). +process_info_smoke_all_tester() -> + register(process_info_smoke_all_tester, self()), + put(ets_ref, ets:new(blupp, [])), + put(binary, [list_to_binary(lists:duplicate(1000, 1)), + list_to_binary(lists:duplicate(1000, 2))]), + process_info_smoke_all_tester_loop(). + +process_info_smoke_all_tester_loop() -> + receive + {other_process, Pid} -> + case get(procs) of + undefined -> put(procs, [Pid]); + Procs -> put(procs, [Pid|Procs]) + end, + erlang:monitor(process, Pid), + link(Pid), + process_info_smoke_all_tester_loop() + end. + +process_info_smoke_all(Config) when is_list(Config) -> + AllPIOptions = [registered_name, + current_function, + initial_call, + messages, + message_queue_len, + links, + monitors, + monitored_by, + dictionary, + trap_exit, + error_handler, + heap_size, + stack_size, + memory, + garbage_collection, + group_leader, + reductions, + priority, + trace, + binary, + sequential_trace_token, + catchlevel, + backtrace, + last_calls, + total_heap_size, + suspending, + min_heap_size, + min_bin_vheap_size, + max_heap_size, + current_location, + current_stacktrace, + message_queue_data, + garbage_collection_info, + magic_ref, + fullsweep_after], + + {ok, Node} = start_node(Config, ""), + RP = spawn_link(Node, fun process_info_smoke_all_tester/0), + LP = spawn_link(fun process_info_smoke_all_tester/0), + RP ! {other_process, LP}, + LP ! {other_process, RP}, + LP ! {other_process, self()}, + LP ! ets:new(blapp, []), + LP ! ets:new(blipp, []), + LP ! list_to_binary(lists:duplicate(1000, 3)), + receive after 1000 -> ok end, + _MLP = erlang:monitor(process, LP), + true = is_process_alive(LP), + PI = process_info(LP, AllPIOptions), + io:format("~p~n", [PI]), + garbage_collect(), + unlink(RP), + unlink(LP), + exit(RP, kill), + exit(LP, kill), + false = is_process_alive(LP), + stop_node(Node), + ok. + +process_info_status_handled_signal(Config) when is_list(Config) -> + P = spawn_link(fun () -> + receive after infinity -> ok end + end), + wait_until(fun () -> + process_info(P, status) == {status, waiting} + end), + %% + %% The 'messages' option will force a process-info-request + %% signal to be scheduled on the process. Ensure that status + %% 'waiting' is reported even though it is actually running + %% when handling the request. We want it to report the status + %% it would have had if it had not been handling the + %% process-info-request... + %% + [{status, waiting}, {messages, []}] = process_info(P, [status, messages]), + unlink(P), + exit(P, kill), + false = erlang:is_process_alive(P), + ok. + %% Tests erlang:bump_reductions/1. bump_reductions(Config) when is_list(Config) -> erlang:garbage_collect(), - receive after 1 -> ok end, % Clear reductions. + erlang:yield(), % Clear reductions. {reductions,R1} = process_info(self(), reductions), true = erlang:bump_reductions(100), {reductions,R2} = process_info(self(), reductions), @@ -2131,36 +2217,44 @@ handle_event(Event, Pid) -> processes_term_proc_list(Config) when is_list(Config) -> Tester = self(), - as_expected = processes_term_proc_list_test(false), - {ok, Node} = start_node(Config, "+Mis true"), - RT = spawn_link(Node, fun () -> - receive after 1000 -> ok end, - processes_term_proc_list_test(false), - Tester ! {it_worked, self()} - end), - receive {it_worked, RT} -> ok end, - stop_node(Node), + + Run = fun(Args) -> + {ok, Node} = start_node(Config, Args), + RT = spawn_link(Node, fun () -> + receive after 1000 -> ok end, + as_expected = processes_term_proc_list_test(false), + Tester ! {it_worked, self()} + end), + receive {it_worked, RT} -> ok end, + stop_node(Node) + end, + + %% We have to run this test case with +S1 since instrument:allocations() + %% will report a free()'d block as present until it's actually deallocated + %% by its employer. + Run("+MSe true +MSatags false +S1"), + Run("+MSe true +MSatags true +S1"), + ok. - + -define(CHK_TERM_PROC_LIST(MC, XB), chk_term_proc_list(?LINE, MC, XB)). chk_term_proc_list(Line, MustChk, ExpectBlks) -> - case {MustChk, instrument:memory_status(types)} of - {false, false} -> + Allocs = instrument:allocations(#{ allocator_types => [sl_alloc] }), + case {MustChk, Allocs} of + {false, {error, not_enabled}} -> not_enabled; - {_, MS} -> - {value, - {ptab_list_deleted_el, - DL}} = lists:keysearch(ptab_list_deleted_el, 1, MS), - case lists:keysearch(blocks, 1, DL) of - {value, {blocks, ExpectBlks, _, _}} -> - ok; - {value, {blocks, Blks, _, _}} -> - exit({line, Line, - mismatch, expected, ExpectBlks, actual, Blks}); - Unexpected -> - exit(Unexpected) + {_, {ok, {_Shift, _Unscanned, ByOrigin}}} -> + ByType = maps:get(system, ByOrigin, #{}), + Hist = maps:get(ptab_list_deleted_el, ByType, {}), + case lists:sum(tuple_to_list(Hist)) of + ExpectBlks -> + ok; + Blks -> + exit({line, Line, mismatch, + expected, ExpectBlks, + actual, Blks}) end end, ok. diff --git a/erts/emulator/test/signal_SUITE.erl b/erts/emulator/test/signal_SUITE.erl index 61a8617165..fab2f45f28 100644 --- a/erts/emulator/test/signal_SUITE.erl +++ b/erts/emulator/test/signal_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2017. All Rights Reserved. +%% Copyright Ericsson AB 2006-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -34,23 +34,9 @@ -export([init_per_testcase/2, end_per_testcase/2]). % Test cases --export([xm_sig_order/1, - pending_exit_unlink_process/1, - pending_exit_unlink_dist_process/1, - pending_exit_unlink_port/1, - pending_exit_trap_exit/1, - pending_exit_receive/1, - pending_exit_exit/1, - pending_exit_gc/1, - pending_exit_is_process_alive/1, - pending_exit_process_display/1, - pending_exit_process_info_1/1, - pending_exit_process_info_2/1, - pending_exit_group_leader/1, - exit_before_pending_exit/1]). +-export([xm_sig_order/1]). init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> - available_internal_state(true), [{testcase, Func}|Config]. end_per_testcase(_Func, _Config) -> @@ -60,24 +46,14 @@ init_per_suite(Config) -> Config. end_per_suite(_Config) -> - available_internal_state(true), - catch erts_debug:set_internal_state(not_running_optimization, true), - available_internal_state(false). + ok. suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap, {minutes, 2}}]. all() -> - [xm_sig_order, pending_exit_unlink_process, - pending_exit_unlink_dist_process, - pending_exit_unlink_port, pending_exit_trap_exit, - pending_exit_receive, pending_exit_trap_exit, - pending_exit_gc, pending_exit_is_process_alive, - pending_exit_process_display, - pending_exit_process_info_1, - pending_exit_process_info_2, pending_exit_group_leader, - exit_before_pending_exit]. + [xm_sig_order]. %% Test that exit signals and messages are received in correct order @@ -113,362 +89,9 @@ xm_sig_order_proc() -> end, xm_sig_order_proc(). -pending_exit_unlink_process(Config) when is_list(Config) -> - pending_exit_test(self(), unlink). - -pending_exit_unlink_dist_process(Config) when is_list(Config) -> - {ok, Node} = start_node(Config), - From = spawn(Node, fun () -> receive after infinity -> ok end end), - Res = pending_exit_test(From, unlink), - stop_node(Node), - Res. - -pending_exit_unlink_port(Config) when is_list(Config) -> - pending_exit_test(hd(erlang:ports()), unlink). - -pending_exit_trap_exit(Config) when is_list(Config) -> - pending_exit_test(self(), trap_exit). - -pending_exit_receive(Config) when is_list(Config) -> - pending_exit_test(self(), 'receive'). - -pending_exit_exit(Config) when is_list(Config) -> - pending_exit_test(self(), exit). - -pending_exit_gc(Config) when is_list(Config) -> - pending_exit_test(self(), gc). - -pending_exit_test(From, Type) -> - OTE = process_flag(trap_exit, true), - Ref = make_ref(), - Master = self(), - ExitBySignal = case Type of - gc -> - lists:duplicate(10000, - exit_by_signal); - _ -> - exit_by_signal - end, - Pid = spawn_link( - fun () -> - receive go -> ok end, - false = have_pending_exit(), - exit = fake_exit(From, - self(), - ExitBySignal), - true = have_pending_exit(), - Master ! {self(), Ref, Type}, - case Type of - gc -> - force_gc(), - erlang:yield(); - unlink -> - unlink(From); - trap_exit -> - process_flag(trap_exit, true); - 'receive' -> - receive _ -> ok - after 0 -> ok - end; - exit -> - ok - end, - exit(exit_by_myself) - end), - Mon = erlang:monitor(process, Pid), - Pid ! go, - Reason = receive - {'DOWN', Mon, process, Pid, R} -> - receive - {Pid, Ref, Type} -> - ok - after 0 -> - ct:fail(premature_exit) - end, - case Type of - exit -> - exit_by_myself = R; - _ -> - ExitBySignal = R - end - end, - receive - {'EXIT', Pid, R2} -> - Reason = R2 - end, - process_flag(trap_exit, OTE), - ok, - {comment, "Test only valid with current SMP emulator."}. - - - -exit_before_pending_exit(Config) when is_list(Config) -> - %% This is a testcase testcase very specific to the smp - %% implementation as it is of the time of writing. - %% - %% The testcase tries to check that a process can - %% exit by itself even though it has a pending exit. - OTE = process_flag(trap_exit, true), - Master = self(), - Tester = spawn_link( - fun () -> - Opts = case {erlang:system_info(run_queues), - erlang:system_info(schedulers_online)} of - {RQ, SO} when RQ =:= 1; SO =:= 1 -> []; - _ -> - process_flag(scheduler, 1), - [{scheduler, 2}] - end, - P = self(), - Exiter = spawn_opt(fun () -> - receive - {exit_me, P, R} -> - exit(P, R) - end - end, Opts), - erlang:yield(), - Exiter ! {exit_me, self(), exited_by_exiter}, - %% We want to get a pending exit - %% before we exit ourselves. We - %% don't want to be scheduled out - %% since we will then see the - %% pending exit. - %% - %% Do something that takes - %% relatively long time but - %% consumes few reductions... - repeat(fun() -> erlang:system_info(procs) end,10), - %% ... then exit. - Master ! {self(), - pending_exit, - have_pending_exit()}, - exit(exited_by_myself) - end), - PendingExit = receive {Tester, pending_exit, PE} -> PE end, - receive - {'EXIT', Tester, exited_by_myself} -> - process_flag(trap_exit, OTE), - ok; - Msg -> - ct:fail({unexpected_message, Msg}) - end, - NoScheds = integer_to_list(erlang:system_info(schedulers_online)), - {comment, - "Was " - ++ case PendingExit of - true -> ""; - false ->"*not*" - end ++ " able to trigger a pending exit. " - ++ "Running on " ++ NoScheds ++ " scheduler(s). " - ++ "This test is only interesting with at least two schedulers."}. - --define(PE_INFO_REPEAT, 100). - -pending_exit_is_process_alive(Config) when is_list(Config) -> - S = exit_op_test_init(), - TestFun = fun (P) -> false = is_process_alive(P) end, - repeated_exit_op_test(TestFun, ?PE_INFO_REPEAT), - verify_pending_exit_success(S), - comment(). - -pending_exit_process_info_1(Config) when is_list(Config) -> - S = exit_op_test_init(), - TestFun = fun (P) -> - undefined = process_info(P) - end, - repeated_exit_op_test(TestFun, ?PE_INFO_REPEAT), - verify_pending_exit_success(S), - comment(). - -pending_exit_process_info_2(Config) when is_list(Config) -> - S0 = exit_op_test_init(), - repeated_exit_op_test(fun (P) -> - undefined = process_info(P, messages) - end, ?PE_INFO_REPEAT), - S1 = verify_pending_exit_success(S0), - repeated_exit_op_test(fun (P) -> - undefined = process_info(P, status) - end, ?PE_INFO_REPEAT), - S2 = verify_pending_exit_success(S1), - repeated_exit_op_test(fun (P) -> - undefined = process_info(P, links) - end, ?PE_INFO_REPEAT), - S3 = verify_pending_exit_success(S2), - repeated_exit_op_test(fun (P) -> - undefined = process_info(P, [messages]) - end, ?PE_INFO_REPEAT), - S4 = verify_pending_exit_success(S3), - repeated_exit_op_test(fun (P) -> - undefined = process_info(P, [status]) - end, ?PE_INFO_REPEAT), - S5 = verify_pending_exit_success(S4), - repeated_exit_op_test(fun (P) -> - undefined = process_info(P, [links]) - end, ?PE_INFO_REPEAT), - S6 = verify_pending_exit_success(S5), - repeated_exit_op_test(fun (P) -> - undefined = process_info(P, [status, - links]) - end, ?PE_INFO_REPEAT), - S7 = verify_pending_exit_success(S6), - repeated_exit_op_test(fun (P) -> - undefined = process_info(P, [messages, - status]) - end, ?PE_INFO_REPEAT), - S8 = verify_pending_exit_success(S7), - repeated_exit_op_test(fun (P) -> - undefined = process_info(P, [messages, - links]) - end, ?PE_INFO_REPEAT), - S9 = verify_pending_exit_success(S8), - repeated_exit_op_test( - fun (P) -> - undefined = process_info(P, [message_queue_len, - status]) - end, ?PE_INFO_REPEAT), - S10 = verify_pending_exit_success(S9), - repeated_exit_op_test(fun (P) -> - undefined = process_info(P, [messages, - links, - status]) - end, ?PE_INFO_REPEAT), - verify_pending_exit_success(S10), - comment(). - -pending_exit_process_display(Config) when is_list(Config) -> - S = exit_op_test_init(), - TestFun = fun (P) -> - badarg = try - erlang:process_display(P, backtrace) - catch - error:badarg -> badarg - end - end, - repeated_exit_op_test(TestFun, ?PE_INFO_REPEAT), - verify_pending_exit_success(S), - comment(). - -pending_exit_group_leader(Config) when is_list(Config) -> - S = exit_op_test_init(), - TestFun = fun (P) -> - badarg = try - group_leader(self(), P) - catch - error:badarg -> badarg - end - end, - repeated_exit_op_test(TestFun, ?PE_INFO_REPEAT), - verify_pending_exit_success(S), - comment(). - %% %% -- Internal utils -------------------------------------------------------- %% -exit_op_test_init() -> - put(no_pending_exit_success, 0), - put(no_pending_exit_tries, 0), - {case {erlang:system_info(run_queues), - erlang:system_info(schedulers_online)} of - {RQ, SO} when RQ =:= 1; SO =:= 1 -> false; - _ -> true - end, 0, 0}. - -verify_pending_exit_success({false, _, _} = S) -> - S; -verify_pending_exit_success({true, S, T}) -> - NewS = get(no_pending_exit_success), - NewT = get(no_pending_exit_tries), - case NewT =:= T of - true -> ok; - _ -> case NewS > S of - true -> ok; - _ -> exit(no_pending_exits) - end - end, - {true, NewS, NewT}. - -comment() -> - {comment, - "Pending exit trigger ratio " - ++ integer_to_list(get(no_pending_exit_success)) - ++ "/" - ++ integer_to_list(get(no_pending_exit_tries)) - ++ "." - ++ case get(not_running_opt_test) of - true -> " No 'not running optimization' to disable."; - _ -> "" - end}. - -repeated_exit_op_test(TestFun, N) -> - WorkFun0 = fun () -> - lists:sort(lists:reverse(lists:seq(1, 1000))) - end, - repeat(fun () -> exit_op_test(TestFun, WorkFun0) end, N), - try erts_debug:set_internal_state(not_running_optimization, false) of - Bool when Bool == true; Bool == false -> - WorkFun1 = fun () -> - erts_debug:set_internal_state(sleep, 0), - lists:sort(lists:reverse(lists:seq(1, 1000))) - end, - repeat(fun () -> - exit_op_test(TestFun, WorkFun1) - end, N) - catch - error:notsup -> put(not_running_opt_test, true) - after - catch erts_debug:set_internal_state(not_running_optimization, true) - end. - -exit_op_test(TestFun, WorkFun) -> - Opts = case {erlang:system_info(run_queues), - erlang:system_info(schedulers_online)} of - {RQ, SO} when RQ =:= 1; SO =:= 1 -> []; - _ -> - process_flag(scheduler, 1), - [{scheduler, 2}] - end, - Master = self(), - Going = make_ref(), - P = spawn_opt(fun () -> - loop(10, WorkFun), - Master ! Going, - loop(infinity, WorkFun) - end, Opts), - receive Going -> ok end, - loop(10, WorkFun), - erlang:yield(), - exit(P, bang), - PE0 = have_pending_exit(P), - TestFun(P), - PE = case PE0 of - true -> true; - _ -> false - end, - case {PE, get(no_pending_exit_success), get(no_pending_exit_tries)} of - {true, undefined, undefined} -> - put(no_pending_exit_success, 1), - put(no_pending_exit_tries, 1); - {false, undefined, undefined} -> - put(no_pending_exit_success, 0), - put(no_pending_exit_tries, 1); - {true, S, T} -> - put(no_pending_exit_success, S+1), - put(no_pending_exit_tries, T+1); - {false, _S, T} -> - put(no_pending_exit_tries, T+1) - end, - ok. - -loop(infinity, WorkFun) -> - do_loop(infinity, WorkFun); -loop(0, _WorkFun) -> - ok; -loop(N, WorkFun) when is_integer(N) -> - do_loop(N-1, WorkFun). - -do_loop(N, WorkFun) -> - WorkFun(), - loop(N, WorkFun). repeat(_Fun, N) when is_integer(N), N =< 0 -> ok; @@ -486,30 +109,3 @@ start_node(Config) -> stop_node(Node) -> test_server:stop_node(Node). - -have_pending_exit() -> - have_pending_exit(self()). - -have_pending_exit(Pid) -> - erts_debug:get_internal_state({have_pending_exit, Pid}). - -force_gc() -> - erts_debug:set_internal_state(force_gc, self()). - -fake_exit(From, To, Reason) -> - erts_debug:set_internal_state(send_fake_exit_signal, {From, To, Reason}). - -available_internal_state(Bool) when Bool == true; Bool == false -> - case {Bool, - (catch erts_debug:get_internal_state(available_internal_state))} of - {true, true} -> - true; - {false, true} -> - erts_debug:set_internal_state(available_internal_state, false), - true; - {true, _} -> - erts_debug:set_internal_state(available_internal_state, true), - false; - {false, _} -> - false - end. diff --git a/erts/emulator/test/smoke_test_SUITE.erl b/erts/emulator/test/smoke_test_SUITE.erl index adc6f56c06..b3d34103f1 100644 --- a/erts/emulator/test/smoke_test_SUITE.erl +++ b/erts/emulator/test/smoke_test_SUITE.erl @@ -70,6 +70,20 @@ boot_combo(Config) when is_list(Config) -> chk_boot(Config, "+Ktrue", NOOP), chk_boot(Config, "+A42", A42), chk_boot(Config, "+Ktrue +A42", A42), + + WBTArgs = ["very_short", "short", "medium", "long", "very_long"], + WTArgs = ["very_low", "low", "medium", "high", "very_high"], + [chk_boot(Config, + " +sbwt " ++ WBT ++ + " +sbwtdcpu " ++ WBT ++ + " +sbwtdio " ++ WBT ++ + " +swt " ++ WT ++ + " +swtdcpu " ++ WT ++ + " +swtdio " ++ WT, NOOP) || WBT <- WBTArgs, WT <- WTArgs], + + WSArgs = ["legacy", "default"], + [chk_boot(Config, " +sws " ++ WS, NOOP) || WS <- WSArgs], + %% A lot more combos could be implemented... ok after diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl index 029a6de897..3f2897242e 100644 --- a/erts/emulator/test/statistics_SUITE.erl +++ b/erts/emulator/test/statistics_SUITE.erl @@ -310,6 +310,13 @@ scheduler_wall_time_all(Config) when is_list(Config) -> scheduler_wall_time_test(scheduler_wall_time_all). scheduler_wall_time_test(Type) -> + case string:find(erlang:system_info(system_version), + "dirty-schedulers-TEST") == nomatch of + true -> run_scheduler_wall_time_test(Type); + false -> {skip, "Cannot be run with dirty-schedulers-TEST build"} + end. + +run_scheduler_wall_time_test(Type) -> %% Should return undefined if system_flag is not turned on yet undefined = statistics(Type), %% Turn on statistics diff --git a/erts/emulator/test/trace_SUITE.erl b/erts/emulator/test/trace_SUITE.erl index a81aa64057..def25dba7d 100644 --- a/erts/emulator/test/trace_SUITE.erl +++ b/erts/emulator/test/trace_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2017. All Rights Reserved. +%% Copyright Ericsson AB 1997-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -235,7 +235,7 @@ link_receive_call_correlation(Config) when is_list(Config) -> 1 = erlang:trace(Receiver, true, ['receive', procs, call, timestamp, scheduler_id]), 1 = erlang:trace_pattern({?MODULE, receive_msg, '_'}, [], [local]), - Num = 100000, + Num = 100, (fun F(0) -> []; F(N) -> @@ -255,7 +255,7 @@ link_receive_call_correlation(Config) when is_list(Config) -> Msgs = (fun F() -> receive M -> [M | F()] after 1 -> [] end end)(), - case check_consistent(Receiver, Num, Num, Num, Msgs) of + case check_consistent(Receiver, Num, Num, Num, Msgs, false, undefined) of ok -> ok; {error, Reason} -> @@ -265,20 +265,63 @@ link_receive_call_correlation(Config) when is_list(Config) -> -define(schedid, , _). -check_consistent(_Pid, Recv, Call, _LU, [Msg | _]) when Recv > Call -> +check_consistent(_Pid, Recv, Call, _LU, [Msg | _], _Received, _LinkedN) when Recv > Call -> {error, Msg}; -check_consistent(Pid, Recv, Call, LU, [Msg | Msgs]) -> +check_consistent(Pid, Recv, Call, LU, [Msg | Msgs], false, undefined) -> case Msg of {trace, Pid, 'receive', Recv ?schedid} -> - check_consistent(Pid,Recv - 1, Call, LU, Msgs); + check_consistent(Pid,Recv - 1, Call, LU, Msgs, true, undefined); {trace_ts, Pid, 'receive', Recv ?schedid, _} -> - check_consistent(Pid,Recv - 1, Call, LU, Msgs); + check_consistent(Pid,Recv - 1, Call, LU, Msgs, true, undefined); {trace, Pid, call, {?MODULE, receive_msg, [Call]} ?schedid} -> - check_consistent(Pid,Recv, Call - 1, LU, Msgs); + check_consistent(Pid,Recv, Call - 1, LU, Msgs, false, undefined); {trace_ts, Pid, call, {?MODULE, receive_msg, [Call]} ?schedid, _} -> - check_consistent(Pid,Recv, Call - 1, LU, Msgs); + check_consistent(Pid,Recv, Call - 1, LU, Msgs, false, undefined); + + {trace, Pid, _, _Self ?schedid} -> + check_consistent(Pid, Recv, Call, LU, Msgs, false, undefined); + {trace_ts, Pid, _, _Self ?schedid, _} -> + check_consistent(Pid, Recv, Call, LU, Msgs, false, undefined); + + Msg -> + {error, Msg} + end; +check_consistent(Pid, Recv, Call, LU, [Msg | Msgs], true, undefined) -> + + case Msg of + {trace, Pid, call, {?MODULE, receive_msg, [Call]} ?schedid} -> + check_consistent(Pid,Recv, Call - 1, LU, Msgs, true, undefined); + {trace_ts, Pid, call, {?MODULE, receive_msg, [Call]} ?schedid, _} -> + check_consistent(Pid,Recv, Call - 1, LU, Msgs, true, undefined); + + {trace, Pid, getting_linked, _Self ?schedid} -> + check_consistent(Pid, Recv, Call, LU - 1, Msgs, true, Recv rem 2); + {trace_ts, Pid, getting_linked, _Self ?schedid, _} -> + check_consistent(Pid, Recv, Call, LU - 1, Msgs, true, Recv rem 2); + + {trace, Pid, getting_unlinked, _Self ?schedid} -> + check_consistent(Pid, Recv, Call, LU - 1, Msgs, true, (Recv+1) rem 2); + {trace_ts, Pid, getting_unlinked, _Self ?schedid, _} -> + check_consistent(Pid, Recv, Call, LU - 1, Msgs, true, (Recv+1) rem 2); + + Msg -> + {error, Msg} + end; +check_consistent(Pid, Recv, Call, LU, [Msg | Msgs], true, LinkedN) -> + UnlinkedN = (LinkedN + 1) rem 2, + + case Msg of + {trace, Pid, 'receive', Recv ?schedid} when Recv == LU -> + check_consistent(Pid,Recv - 1, Call, LU, Msgs, true, LinkedN); + {trace_ts, Pid, 'receive', Recv ?schedid, _} when Recv == LU -> + check_consistent(Pid,Recv - 1, Call, LU, Msgs, true, LinkedN); + + {trace, Pid, call, {?MODULE, receive_msg, [Call]} ?schedid} -> + check_consistent(Pid,Recv, Call - 1, LU, Msgs, true, LinkedN); + {trace_ts, Pid, call, {?MODULE, receive_msg, [Call]} ?schedid, _} -> + check_consistent(Pid,Recv, Call - 1, LU, Msgs, true, LinkedN); %% We check that for each receive we have gotten a %% getting_linked or getting_unlinked message. Also @@ -286,38 +329,38 @@ check_consistent(Pid, Recv, Call, LU, [Msg | Msgs]) -> %% message we expect to receive is an even number %% and odd number for getting_unlinked. {trace, Pid, getting_linked, _Self ?schedid} - when Recv rem 2 == 0, Recv == LU -> - check_consistent(Pid, Recv, Call, LU - 1, Msgs); + when Recv rem 2 == LinkedN -> + check_consistent(Pid, Recv, Call, LU - 1, Msgs, true, LinkedN); {trace_ts, Pid, getting_linked, _Self ?schedid, _} - when Recv rem 2 == 0, Recv == LU -> - check_consistent(Pid, Recv, Call, LU - 1, Msgs); + when Recv rem 2 == LinkedN -> + check_consistent(Pid, Recv, Call, LU - 1, Msgs, true, LinkedN); {trace, Pid, getting_unlinked, _Self ?schedid} - when Recv rem 2 == 1, Recv == LU -> - check_consistent(Pid, Recv, Call, LU - 1, Msgs); + when Recv rem 2 == UnlinkedN -> + check_consistent(Pid, Recv, Call, LU - 1, Msgs, true, LinkedN); {trace_ts, Pid, getting_unlinked, _Self ?schedid, _} - when Recv rem 2 == 1, Recv == LU -> - check_consistent(Pid, Recv, Call, LU - 1, Msgs); + when Recv rem 2 == UnlinkedN -> + check_consistent(Pid, Recv, Call, LU - 1, Msgs, true, LinkedN); {trace,Pid,'receive',Ignore ?schedid} when Ignore == stop; Ignore == timeout -> - check_consistent(Pid, Recv, Call, LU, Msgs); + check_consistent(Pid, Recv, Call, LU, Msgs, true, LinkedN); {trace_ts,Pid,'receive',Ignore ?schedid,_} when Ignore == stop; Ignore == timeout -> - check_consistent(Pid, Recv, Call, LU, Msgs); + check_consistent(Pid, Recv, Call, LU, Msgs, true, LinkedN); {trace, Pid, exit, normal ?schedid} -> - check_consistent(Pid, Recv, Call, LU, Msgs); + check_consistent(Pid, Recv, Call, LU, Msgs, true, LinkedN); {trace_ts, Pid, exit, normal ?schedid, _} -> - check_consistent(Pid, Recv, Call, LU, Msgs); + check_consistent(Pid, Recv, Call, LU, Msgs, true, LinkedN); {'EXIT', Pid, normal} -> - check_consistent(Pid, Recv, Call, LU, Msgs); + check_consistent(Pid, Recv, Call, LU, Msgs, true, LinkedN); Msg -> {error, Msg} end; -check_consistent(_, 0, 0, 0, []) -> +check_consistent(_, 0, 0, 1, [], true, _) -> ok; -check_consistent(_, Recv, Call, LU, []) -> +check_consistent(_, Recv, Call, LU, [], _, _) -> {error,{Recv, Call, LU}}. receive_msg(M) -> diff --git a/erts/emulator/test/tracer_SUITE.erl b/erts/emulator/test/tracer_SUITE.erl index ab7d047bc3..e1362ef07a 100644 --- a/erts/emulator/test/tracer_SUITE.erl +++ b/erts/emulator/test/tracer_SUITE.erl @@ -30,7 +30,8 @@ -export([load/1, unload/1, reload/1, invalid_tracers/1]). -export([send/1, recv/1, call/1, call_return/1, spawn/1, exit/1, link/1, unlink/1, getting_linked/1, getting_unlinked/1, - register/1, unregister/1, in/1, out/1, gc_start/1, gc_end/1]). + register/1, unregister/1, in/1, out/1, gc_start/1, gc_end/1, + seq_trace/1]). suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap, {minutes, 1}}]. @@ -41,7 +42,8 @@ all() -> groups() -> [{ basic, [], [send, recv, call, call_return, spawn, exit, link, unlink, getting_linked, getting_unlinked, - register, unregister, in, out, gc_start, gc_end]}]. + register, unregister, in, out, gc_start, gc_end, + seq_trace]}]. init_per_suite(Config) -> erlang:trace_pattern({'_','_','_'}, false, [local]), @@ -583,6 +585,24 @@ gc_end(_Config) -> test(gc_major_end, garbage_collection, Tc, Expect, false). +seq_trace(_Config) -> + + seq_trace:set_system_tracer({tracer_test, + {#{ seq_trace => trace }, self(), []}}), + erlang:spawn(fun() -> + seq_trace:set_token(label,17), + seq_trace:set_token(print,true), + seq_trace:print(17,"**** Trace Started ****") + end), + receive + {seq_trace, _, 17, {print, _, _, _, _}, _} -> + ok; + M -> + ct:fail("~p~n",[M]) + after 100 -> + ct:fail(timeout) + end. + test(Event, Tc, Expect) -> test(Event, Tc, Expect, false). test(Event, Tc, Expect, Removes) -> diff --git a/erts/emulator/test/z_SUITE.erl b/erts/emulator/test/z_SUITE.erl index ac3df8bfbf..103f9f1550 100644 --- a/erts/emulator/test/z_SUITE.erl +++ b/erts/emulator/test/z_SUITE.erl @@ -37,6 +37,7 @@ -export([schedulers_alive/1, node_container_refc_check/1, long_timers/1, pollset_size/1, check_io_debug/1, get_check_io_info/0, + lc_graph/1, leaked_processes/1]). suite() -> @@ -46,6 +47,7 @@ suite() -> all() -> [schedulers_alive, node_container_refc_check, long_timers, pollset_size, check_io_debug, + lc_graph, %% Make sure that the leaked_processes/1 is always %% run last. leaked_processes]. @@ -289,6 +291,12 @@ has_gethost([P|T]) -> has_gethost([]) -> false. +lc_graph(Config) when is_list(Config) -> + %% Create "lc_graph" file in current working dir + %% if lock checker is enabled + erts_debug:lc_graph(), + ok. + leaked_processes(Config) when is_list(Config) -> %% Replace the defualt timetrap with a timetrap with %% known pid. diff --git a/erts/emulator/utils/make_driver_tab b/erts/emulator/utils/make_driver_tab index ffb5f58ebf..b7bca1dc3a 100755 --- a/erts/emulator/utils/make_driver_tab +++ b/erts/emulator/utils/make_driver_tab @@ -30,6 +30,7 @@ use File::Basename; my $file = ""; my $nif = ""; my @emu_drivers = (); +my @emu_nifs = (); my @static_drivers = (); my @static_nifs = (); my $mode = 1; @@ -61,7 +62,7 @@ while (@ARGV) { } elsif ($mode == 2) { $d = basename $d; $d =~ s/_nif(\..*|)$//; # strip nif.* or just nif - push(@static_nifs, $d); + push(@emu_nifs, $d); next; } $d = basename $d; @@ -94,37 +95,33 @@ foreach (@static_drivers) { } # The array itself -print "\nErlDrvEntry *driver_tab[] =\n{\n"; +print "\nErtsStaticDriver driver_tab[] =\n{\n"; foreach (@emu_drivers) { - print " &${_}driver_entry,\n"; + print " {&${_}driver_entry, 0},\n"; } foreach (@static_drivers) { - print " NULL, /* ${_} */\n"; + print " {NULL, 1}, /* ${_} */\n"; } -print " NULL\n};\n"; +print " {NULL}\n};\n"; print "void erts_init_static_drivers() {\n"; my $index = 0; foreach (@static_drivers) { - print " driver_tab[".(scalar @emu_drivers+$index)."] = ${_}_driver_init();\n"; + print " driver_tab[".(scalar @emu_drivers+$index)."].de = ${_}_driver_init();\n"; $index++; } print "}\n"; -print <<EOF; - -typedef struct ErtsStaticNifEntry_ { - const char *nif_name; - ErtsStaticNifInitFPtr nif_init; -} ErtsStaticNifEntry; - -EOF - # prototypes +foreach (@emu_nifs) { + my $d = ${_}; + $d =~ s/\.debug//; # strip .debug + print "void *".$d."_nif_init(void);\n"; +} foreach (@static_nifs) { my $d = ${_}; $d =~ s/\.debug//; # strip .debug @@ -134,20 +131,25 @@ foreach (@static_nifs) { # The array itself print "static ErtsStaticNifEntry static_nif_tab[] =\n{\n"; +foreach (@emu_nifs) { + my $d = ${_}; + $d =~ s/\.debug//; # strip .debug + print " {\"${_}\", &".$d."_nif_init, 0},\n"; +} foreach (@static_nifs) { my $d = ${_}; $d =~ s/\.debug//; # strip .debug - print "{\"${_}\",&".$d."_nif_init},\n"; + print " {\"${_}\", &".$d."_nif_init, 1},\n"; } print " {NULL,NULL}\n};\n"; print <<EOF; -ErtsStaticNifInitFPtr erts_static_nif_get_nif_init(const char *name, int len) { +ErtsStaticNifEntry* erts_static_nif_get_nif_init(const char *name, int len) { ErtsStaticNifEntry* p; for (p = static_nif_tab; p->nif_name != NULL; p++) if (strncmp(p->nif_name, name, len) == 0 && p->nif_name[len] == 0) - return p->nif_init; + return p; return NULL; } |