From 5aff60d96efac96a41b514ed167f13eb787a415f Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 26 Sep 2016 17:05:50 +0200 Subject: Support for dirty BIFs --- erts/emulator/beam/atom.names | 9 + erts/emulator/beam/beam_bif_load.c | 6 + erts/emulator/beam/beam_debug.c | 347 +++++++++++++++++ erts/emulator/beam/beam_emu.c | 117 +++--- erts/emulator/beam/bif.c | 317 +++++++++++++++- erts/emulator/beam/bif.h | 87 ++++- erts/emulator/beam/bif.tab | 3 + erts/emulator/beam/erl_bif_info.c | 3 + erts/emulator/beam/erl_dirty_bif.tab | 82 ++++ erts/emulator/beam/erl_gc.c | 24 +- erts/emulator/beam/erl_msacc.c | 4 +- erts/emulator/beam/erl_msacc.h | 2 +- erts/emulator/beam/erl_nfunc_sched.c | 144 +++++++ erts/emulator/beam/erl_nfunc_sched.h | 289 ++++++++++++++ erts/emulator/beam/erl_nif.c | 707 +++++++++++++---------------------- erts/emulator/beam/erl_process.c | 18 +- erts/emulator/beam/erl_process.h | 8 +- erts/emulator/beam/error.h | 88 +++-- erts/emulator/beam/global.h | 14 +- erts/emulator/beam/utils.c | 2 + 20 files changed, 1663 insertions(+), 608 deletions(-) create mode 100644 erts/emulator/beam/erl_dirty_bif.tab create mode 100644 erts/emulator/beam/erl_nfunc_sched.c create mode 100644 erts/emulator/beam/erl_nfunc_sched.h (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index b1aeed7889..7df350116a 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -76,6 +76,7 @@ atom ac atom accessor atom active atom active_tasks +atom alive atom all atom all_but_first atom all_names @@ -199,10 +200,15 @@ atom dexit atom depth atom dgroup_leader atom dictionary +atom dirty_bif_exception +atom dirty_bif_result +atom dirty_bif_trap atom dirty_cpu atom dirty_cpu_schedulers_online atom dirty_execution atom dirty_io +atom dirty_nif_exception +atom dirty_nif_finalizer atom disable_trace atom disabled atom discard @@ -243,6 +249,7 @@ atom ERROR='ERROR' atom error_handler atom error_logger atom erts_code_purger +atom erts_debug atom erts_internal atom ets atom ETS_TRANSFER='ETS-TRANSFER' @@ -573,6 +580,7 @@ atom scientific atom scope atom second atom seconds +atom send atom send_to_non_existing_process atom sensitive atom sequential_tracer @@ -665,6 +673,7 @@ atom value atom values atom version atom visible +atom wait atom waiting atom wall_clock atom warning diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 09331eb8ad..36bbe81ed5 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -36,6 +36,7 @@ #include "erl_nif.h" #include "erl_bits.h" #include "erl_thr_progress.h" +#include "erl_nfunc_sched.h" #ifdef HIPE # include "hipe_bif0.h" # define IF_HIPE(X) (X) @@ -1102,6 +1103,11 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) *redsp += 1; + if (erts_check_nif_export_in_area(rp, mod_start, mod_size)) + return am_true; + + *redsp += 1; + if (erts_check_nif_export_in_area(rp, mod_start, mod_size)) return am_true; diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index e72d7f8de4..3ffbb0364c 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -39,6 +39,7 @@ #include "beam_bp.h" #include "erl_binary.h" #include "erl_thr_progress.h" +#include "erl_nfunc_sched.h" #ifdef ARCH_64 # define HEXF "%016bpX" @@ -764,3 +765,349 @@ static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif) erts_print(to, to_arg, "%T/%u", name, arity); } } + +/* + * Dirty BIF testing. + * + * The erts_debug:dirty_cpu/2, erts_debug:dirty_io/1, and + * erts_debug:dirty/3 BIFs are used by the dirty_bif_SUITE + * test suite. + */ + +#ifdef ERTS_DIRTY_SCHEDULERS +static int ms_wait(Process *c_p, Eterm etimeout, int busy); +static int dirty_send_message(Process *c_p, Eterm to, Eterm tag); +#endif +static BIF_RETTYPE dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I); + +/* + * erts_debug:dirty_cpu/2 is statically determined to execute on + * a dirty CPU scheduler (see erts_dirty_bif.tab). + */ +BIF_RETTYPE +erts_debug_dirty_cpu_2(BIF_ALIST_2) +{ + return dirty_test(BIF_P, am_dirty_cpu, BIF_ARG_1, BIF_ARG_2, BIF_I); +} + +/* + * erts_debug:dirty_io/2 is statically determined to execute on + * a dirty I/O scheduler (see erts_dirty_bif.tab). + */ +BIF_RETTYPE +erts_debug_dirty_io_2(BIF_ALIST_2) +{ + return dirty_test(BIF_P, am_dirty_io, BIF_ARG_1, BIF_ARG_2, BIF_I); +} + +/* + * erts_debug:dirty/3 executes on a normal scheduler. + */ +BIF_RETTYPE +erts_debug_dirty_3(BIF_ALIST_3) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + Eterm argv[2]; + switch (BIF_ARG_1) { + case am_normal: + return dirty_test(BIF_P, am_normal, BIF_ARG_2, BIF_ARG_3, BIF_I); + case am_dirty_cpu: + argv[0] = BIF_ARG_2; + argv[1] = BIF_ARG_3; + return erts_schedule_bif(BIF_P, + argv, + BIF_I, + erts_debug_dirty_cpu_2, + ERTS_SCHED_DIRTY_CPU, + am_erts_debug, + am_dirty_cpu, + 2); + case am_dirty_io: + argv[0] = BIF_ARG_2; + argv[1] = BIF_ARG_3; + return erts_schedule_bif(BIF_P, + argv, + BIF_I, + erts_debug_dirty_io_2, + ERTS_SCHED_DIRTY_IO, + am_erts_debug, + am_dirty_io, + 2); + default: + BIF_ERROR(BIF_P, EXC_BADARG); + } +#else + BIF_ERROR(BIF_P, EXC_UNDEF); +#endif +} + + +static BIF_RETTYPE +dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I) +{ + BIF_RETTYPE ret; +#ifdef ERTS_DIRTY_SCHEDULERS + if (am_scheduler == arg1) { + ErtsSchedulerData *esdp; + if (arg2 != am_type) + goto badarg; + esdp = erts_proc_sched_data(c_p); + if (!esdp) + ERTS_BIF_PREP_RET(ret, am_error); + else if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) + ERTS_BIF_PREP_RET(ret, am_normal); + else if (ERTS_SCHEDULER_IS_DIRTY_CPU(esdp)) + ERTS_BIF_PREP_RET(ret, am_dirty_cpu); + else if (ERTS_SCHEDULER_IS_DIRTY_IO(esdp)) + ERTS_BIF_PREP_RET(ret, am_dirty_io); + else + ERTS_BIF_PREP_RET(ret, am_error); + } + else if (am_error == arg1) { + switch (arg2) { + case am_notsup: + ERTS_BIF_PREP_ERROR(ret, c_p, EXC_NOTSUP); + break; + case am_undef: + ERTS_BIF_PREP_ERROR(ret, c_p, EXC_UNDEF); + break; + case am_badarith: + ERTS_BIF_PREP_ERROR(ret, c_p, EXC_BADARITH); + break; + case am_noproc: + ERTS_BIF_PREP_ERROR(ret, c_p, EXC_NOPROC); + break; + case am_system_limit: + ERTS_BIF_PREP_ERROR(ret, c_p, SYSTEM_LIMIT); + break; + case am_badarg: + default: + goto badarg; + } + } + else if (am_copy == arg1) { + int i; + Eterm res; + + for (res = NIL, i = 0; i < 1000; i++) { + Eterm *hp, sz; + Eterm cpy; + /* We do not want this to be optimized, + but rather the oposite... */ + sz = size_object(arg2); + hp = HAlloc(c_p, sz); + cpy = copy_struct(arg2, sz, &hp, &c_p->off_heap); + hp = HAlloc(c_p, 2); + res = CONS(hp, cpy, res); + } + + ERTS_BIF_PREP_RET(ret, res); + } + else if (am_send == arg1) { + dirty_send_message(c_p, arg2, am_ok); + ERTS_BIF_PREP_RET(ret, am_ok); + } + else if (ERTS_IS_ATOM_STR("wait", arg1)) { + if (!ms_wait(c_p, arg2, type == am_dirty_cpu)) + goto badarg; + ERTS_BIF_PREP_RET(ret, am_ok); + } + else if (ERTS_IS_ATOM_STR("reschedule", arg1)) { + /* + * Reschedule operation after decrement of two until we reach + * zero. Switch between dirty scheduler types when 'n' is + * evenly divided by 4. If the initial value wasn't evenly + * dividable by 2, throw badarg exception. + */ + Eterm next_type; + Sint n; + if (!term_to_Sint(arg2, &n) || n < 0) + goto badarg; + if (n == 0) + ERTS_BIF_PREP_RET(ret, am_ok); + else { + Eterm argv[3]; + Eterm eint = erts_make_integer((Uint) (n - 2), c_p); + if (n % 4 != 0) + next_type = type; + else { + switch (type) { + case am_dirty_cpu: next_type = am_dirty_io; break; + case am_dirty_io: next_type = am_normal; break; + case am_normal: next_type = am_dirty_cpu; break; + default: goto badarg; + } + } + switch (next_type) { + case am_dirty_io: + argv[0] = arg1; + argv[1] = eint; + ret = erts_schedule_bif(c_p, + argv, + I, + erts_debug_dirty_io_2, + ERTS_SCHED_DIRTY_IO, + am_erts_debug, + am_dirty_io, + 2); + break; + case am_dirty_cpu: + argv[0] = arg1; + argv[1] = eint; + ret = erts_schedule_bif(c_p, + argv, + I, + erts_debug_dirty_cpu_2, + ERTS_SCHED_DIRTY_CPU, + am_erts_debug, + am_dirty_cpu, + 2); + break; + case am_normal: + argv[0] = am_normal; + argv[1] = arg1; + argv[2] = eint; + ret = erts_schedule_bif(c_p, + argv, + I, + erts_debug_dirty_3, + ERTS_SCHED_NORMAL, + am_erts_debug, + am_dirty, + 3); + break; + default: + goto badarg; + } + } + } + else if (ERTS_IS_ATOM_STR("ready_wait6_done", arg1)) { + ERTS_DECL_AM(ready); + ERTS_DECL_AM(done); + dirty_send_message(c_p, arg2, AM_ready); + ms_wait(c_p, make_small(6000), 0); + dirty_send_message(c_p, arg2, AM_done); + ERTS_BIF_PREP_RET(ret, am_ok); + } + else if (ERTS_IS_ATOM_STR("alive_waitexiting", arg1)) { + Process *real_c_p = erts_proc_shadow2real(c_p); + Eterm *hp, *hp2; + Uint sz; + int i; + if (ERTS_PROC_IS_EXITING(real_c_p)) + goto badarg; + dirty_send_message(c_p, arg2, am_alive); + /* Wait until dead */ + + while (!ERTS_PROC_IS_EXITING(real_c_p)) + erts_thr_yield(); + + ms_wait(c_p, make_small(1000), 0); + + /* Should still be able to allocate memory */ + hp = HAlloc(c_p, 3); /* Likely on heap */ + sz = 10000; + hp2 = HAlloc(c_p, sz); /* Likely in heap fragment */ + *hp2 = make_pos_bignum_header(sz); + for (i = 1; i < sz; i++) + hp2[i] = (Eterm) 4711; + ERTS_BIF_PREP_RET(ret, TUPLE2(hp, am_ok, make_big(hp2))); + } + else { + badarg: + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + } +#else + ERTS_BIF_PREP_ERROR(ret, c_p, EXC_UNDEF); +#endif + return ret; +} + +#ifdef ERTS_DIRTY_SCHEDULERS + +static int +dirty_send_message(Process *c_p, Eterm to, Eterm tag) +{ + ErtsProcLocks c_p_locks, rp_locks; + Process *rp, *real_c_p; + Eterm msg, *hp; + ErlOffHeap *ohp; + ErtsMessage *mp; + + ASSERT(is_immed(tag)); + + real_c_p = erts_proc_shadow2real(c_p); + if (real_c_p != c_p) + c_p_locks = 0; + else + c_p_locks = ERTS_PROC_LOCK_MAIN; + + ASSERT(real_c_p->common.id == c_p->common.id); + + rp = erts_pid2proc_opt(real_c_p, c_p_locks, + to, 0, + ERTS_P2P_FLG_INC_REFC); + + if (!rp) + return 0; + + rp_locks = 0; + mp = erts_alloc_message_heap(rp, &rp_locks, 3, &hp, &ohp); + + msg = TUPLE2(hp, tag, c_p->common.id); + erts_queue_message(rp, rp_locks, mp, msg, c_p->common.id); + + if (rp == real_c_p) + rp_locks &= ~c_p_locks; + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + + erts_proc_dec_refc(rp); + + return 1; +} + +static int +ms_wait(Process *c_p, Eterm etimeout, int busy) +{ + ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); + ErtsMonotonicTime time, timeout_time; + Sint64 ms; + + if (!term_to_Sint64(etimeout, &ms)) + return 0; + + time = erts_get_monotonic_time(esdp); + + if (ms < 0) + timeout_time = time; + else + timeout_time = time + ERTS_MSEC_TO_MONOTONIC(ms); + + while (time < timeout_time) { + if (busy) + erts_thr_yield(); + else { + ErtsMonotonicTime timeout = timeout_time - time; + +#ifdef __WIN32__ + Sleep((DWORD) ERTS_MONOTONIC_TO_MSEC(timeout)); +#else + { + ErtsMonotonicTime to = ERTS_MONOTONIC_TO_USEC(timeout); + struct timeval tv; + + tv.tv_sec = (long) to / (1000*1000); + tv.tv_usec = (long) to % (1000*1000); + + select(0, NULL, NULL, NULL, &tv); + } +#endif + } + + time = erts_get_monotonic_time(esdp); + } + return 1; +} + +#endif /* ERTS_DIRTY_SCHEDULERS */ diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index bb77dbd955..befd2989f9 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -38,6 +38,7 @@ #include "beam_bp.h" #include "beam_catches.h" #include "erl_thr_progress.h" +#include "erl_nfunc_sched.h" #ifdef HIPE #include "hipe_mode_switch.h" #include "hipe_bif1.h" @@ -214,6 +215,7 @@ BeamInstr beam_continue_exit[1]; BeamInstr* em_call_error_handler; BeamInstr* em_apply_bif; BeamInstr* em_call_nif; +BeamInstr* em_call_bif_e; /* NOTE These should be the only variables containing trace instructions. @@ -2581,7 +2583,7 @@ do { \ OpCase(bif1_fbsd): { - Eterm (*bf)(Process*, Eterm*); + ErtsBifFunc bf; Eterm tmp_reg[1]; Eterm result; @@ -2592,7 +2594,7 @@ do { \ PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, tmp_reg); + result = (*bf)(c_p, tmp_reg, I); ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -2613,19 +2615,19 @@ do { \ OpCase(bif1_body_bsd): { - Eterm (*bf)(Process*, Eterm*); + ErtsBifFunc bf; Eterm tmp_reg[1]; Eterm result; GetArg1(1, tmp_reg[0]); - bf = (BifFunction) Arg(0); + bf = (ErtsBifFunc) Arg(0); ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, tmp_reg); + result = (*bf)(c_p, tmp_reg, I); ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -2775,17 +2777,17 @@ do { \ OpCase(i_bif2_fbssd): { Eterm tmp_reg[2]; - Eterm (*bf)(Process*, Eterm*); + ErtsBifFunc bf; Eterm result; GetArg2(2, tmp_reg[0], tmp_reg[1]); - bf = (BifFunction) Arg(1); + bf = (ErtsBifFunc) Arg(1); ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, tmp_reg); + result = (*bf)(c_p, tmp_reg, I); ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -2806,15 +2808,15 @@ do { \ OpCase(i_bif2_body_bssd): { Eterm tmp_reg[2]; - Eterm (*bf)(Process*, Eterm*); + ErtsBifFunc bf; Eterm result; GetArg2(1, tmp_reg[0], tmp_reg[1]); - bf = (BifFunction) Arg(0); + bf = (ErtsBifFunc) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, tmp_reg); + result = (*bf)(c_p, tmp_reg, I); ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -2837,7 +2839,7 @@ do { \ */ OpCase(call_bif_e): { - Eterm (*bf)(Process*, Eterm*, BeamInstr*); + ErtsBifFunc bf; Eterm result; BeamInstr *next; ErlHeapFragment *live_hf_end; @@ -3644,7 +3646,7 @@ do { \ ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); { - Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf; + ErtsBifFunc bf = vbf; ASSERT(!ERTS_PROC_IS_EXITING(c_p)); live_hf_end = c_p->mbuf; ERTS_CHK_MBUF_SZ(c_p); @@ -5137,6 +5139,7 @@ do { \ em_call_error_handler = OpCode(call_error_handler); em_apply_bif = OpCode(apply_bif); em_call_nif = OpCode(call_nif); + em_call_bif_e = OpCode(call_bif_e); beam_apply[0] = (BeamInstr) OpCode(i_apply); beam_apply[1] = (BeamInstr) OpCode(normal_exit); @@ -5342,8 +5345,6 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) I = c_p->i; - ASSERT(em_call_nif == (BeamInstr *) *I); - /* * Set fcalls even though we ignore it, so we don't * confuse code accessing it... @@ -5379,10 +5380,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) } { -#ifdef DEBUG - Eterm result; -#endif - Eterm arity; + int exiting; { /* @@ -5398,7 +5396,6 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) * * This layout is determined by the NifExport struct */ - BifFunction vbf; ErtsCodeMFA *codemfa; ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); @@ -5406,44 +5403,29 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) codemfa = erts_code_to_codemfa(I); DTRACE_NIF_ENTRY(c_p, codemfa); - /* current and vbf set to please handle_error */ c_p->current = codemfa; SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); - arity = codemfa->arity; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - { - typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); - NifF* fp = vbf = (NifF*) I[1]; - struct enif_environment_t env; - ASSERT(!c_p->scheduler_data); - - erts_pre_dirty_nif(esdp, &env, c_p, - (struct erl_module_nif*)I[2]); - -#ifdef DEBUG - result = -#else - (void) -#endif - (*fp)(&env, arity, reg); - - erts_post_dirty_nif(&env); + if (em_apply_bif == (BeamInstr *) *I) { + exiting = erts_call_dirty_bif(esdp, c_p, I, reg); + } + else { + ASSERT(em_call_nif == (BeamInstr *) *I); + exiting = erts_call_dirty_nif(esdp, c_p, I, reg); + } - ASSERT(!is_value(result)); - ASSERT(c_p->freason == TRAP); - ASSERT(!(c_p->flags & F_HIBERNATE_SCHED)); + ASSERT(!(c_p->flags & F_HIBERNATE_SCHED)); - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); - if (env.exiting) - goto do_dirty_schedule; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - } + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); + if (exiting) + goto do_dirty_schedule; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); DTRACE_NIF_RETURN(c_p, codemfa); ERTS_HOLE_CHECK(c_p); @@ -5529,10 +5511,15 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) Eterm* hp; Eterm Value = c_p->fvalue; Eterm Args = am_true; - c_p->i = pc; /* In case we call erts_exit(). */ ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */ + if (c_p->freason & EXF_RESTORE_NIF) { + erts_nif_export_restore_error(c_p, &pc, reg, &bf); + } + + c_p->i = pc; /* In case we call erts_exit(). */ + /* * Check if we have an arglist for the top level call. If so, this * is encoded in Value, so we have to dig out the real Value as well @@ -5829,6 +5816,18 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) { * error terms.) */ +static void +save_stacktrace_current(struct StackTrace *s, Process *c_p, ErtsCodeMFA *mfa) +{ + NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + if (!nep || &nep->exp.info.mfa != mfa) + s->current = mfa; + else { + s->mfa = *mfa; + s->current = &s->mfa; + } +} + static void save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, Eterm args) { @@ -5858,12 +5857,10 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, && bf != throw_1 && bf != wrap_error_1 && bf != wrap_error_2 && bf != wrap_exit_1 && bf != wrap_throw_1) { int i; - int a = 0; + int a; for (i = 0; i < BIF_SIZE; i++) { if (bf == bif_table[i].f || bf == bif_table[i].traced) { - Export *ep = bif_export[i]; - s->current = &ep->info.mfa; - a = bif_table[i].arity; + save_stacktrace_current(s, c_p, &bif_export[i]->info.mfa); break; } } @@ -5875,9 +5872,11 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, * OR it is a NIF called by call_nif where current is also set. */ ASSERT(c_p->current); - s->current = c_p->current; - a = s->current->arity; + save_stacktrace_current(s, c_p, c_p->current); } + + a = s->current->arity; + /* Save first stack entry */ ASSERT(pc); if (depth > 0) { @@ -5892,7 +5891,9 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, s->pc = NULL; args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */ } else { - s->current = c_p->current; + + save_stacktrace_current(s, c_p, c_p->current); + /* * For a function_clause error, the arguments are in the beam * registers, c_p->cp is valid, and c_p->current is set. diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index d886c2985e..7bcb7c196d 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -4958,7 +4958,7 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, Export bif_return_trap_export; void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, - Eterm (*bif)(BIF_ALIST_0)) + Eterm (*bif)(BIF_ALIST)) { int i; sys_memset((void *) ep, 0, sizeof(Export)); @@ -5019,6 +5019,321 @@ void erts_init_bif(void) erts_smp_atomic32_init_nob(&msacc, ERTS_MSACC_IS_ENABLED()); } +/* + * Scheduling of BIFs via NifExport... + */ +#define ERTS_WANT_NFUNC_SCHED_INTERNALS__ +#include "erl_nfunc_sched.h" + +#define ERTS_SCHED_BIF_TRAP_MARKER ((void *) (UWord) 1) + +static void +schedule(Process *c_p, Process *dirty_shadow_proc, + ErtsCodeMFA *mfa, BifFunction *nif, BeamInstr *pc, + ErtsBifFunc dfunc, void *ifunc, + Eterm module, Eterm function, + int argc, Eterm *argv) +{ + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); + (void) erts_nif_export_schedule(c_p, dirty_shadow_proc, + mfa, nif, pc, + (BeamInstr) em_apply_bif, + dfunc, ifunc, + module, function, + argc, argv); +} + +#ifdef ERTS_DIRTY_SCHEDULERS + +static BIF_RETTYPE dirty_bif_result(BIF_ALIST_1) +{ + NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(BIF_P); + erts_nif_export_restore(BIF_P, nep); + BIF_RET(BIF_ARG_1); +} + +static BIF_RETTYPE dirty_bif_trap(BIF_ALIST) +{ + NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(BIF_P); + + /* + * Arity and argument registers already set + * correct by call to dirty_bif_trap()... + */ + + ASSERT(BIF_P->arity == nep->exp.info.mfa.arity); + + erts_nif_export_restore(BIF_P, nep); + + BIF_P->i = (BeamInstr *) nep->func; + BIF_P->freason = TRAP; + return THE_NON_VALUE; +} + +static BIF_RETTYPE dirty_bif_exception(BIF_ALIST_2) +{ + Eterm freason; + + ASSERT(is_small(BIF_ARG_1)); + + freason = signed_val(BIF_ARG_1); + + /* Restore orig info for error and clear nif export in handle_error() */ + freason |= EXF_RESTORE_NIF; + + BIF_P->fvalue = BIF_ARG_2; + + BIF_ERROR(BIF_P, freason); +} + +#endif /* ERTS_DIRTY_SCHEDULERS */ + +extern BeamInstr* em_call_bif_e; +static BIF_RETTYPE call_bif(Process *c_p, Eterm *reg, BeamInstr *I); + +BIF_RETTYPE +erts_schedule_bif(Process *proc, + Eterm *argv, + BeamInstr *i, + ErtsBifFunc bif, + ErtsSchedType sched_type, + Eterm mod, + Eterm func, + int argc) +{ + Process *c_p, *dirty_shadow_proc; + ErtsCodeMFA *mfa; + +#ifdef ERTS_DIRTY_SCHEDULERS + if (proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { + dirty_shadow_proc = proc; + c_p = proc->next; + ASSERT(c_p->common.id == dirty_shadow_proc->common.id); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + } + else +#endif + { + dirty_shadow_proc = NULL; + c_p = proc; + } + + if (!ERTS_PROC_IS_EXITING(c_p)) { + Export *exp; + BifFunction obif, dbif, ibif; + BeamInstr *pc; + + /* + * obif - original bif + * dbif - direct bif + * ibif - indirect bif + */ + +#ifdef ERTS_DIRTY_SCHEDULERS + erts_aint32_t set, mask; + mask = (ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_IO_PROC); + switch (sched_type) { + case ERTS_SCHED_DIRTY_CPU: + set = ERTS_PSFLG_DIRTY_CPU_PROC; + dbif = bif; + ibif = NULL; + break; + case ERTS_SCHED_DIRTY_IO: + set = ERTS_PSFLG_DIRTY_IO_PROC; + dbif = bif; + ibif = NULL; + break; + case ERTS_SCHED_NORMAL: + default: + set = 0; + dbif = call_bif; + ibif = bif; + break; + } + + (void) erts_smp_atomic32_read_bset_nob(&c_p->state, mask, set); +#else + dbif = call_bif; + ibif = bif; +#endif + + if (i == NULL) { + ERTS_INTERNAL_ERROR("Missing instruction pointer"); + obif = NULL; + } +#ifdef HIPE + else if (proc->flags & F_HIPE_MODE) { + /* Pointer to bif export in i */ + exp = (Export *) i; + pc = c_p->cp; + obif = (BifFunction) exp->beam[1]; + mfa = &exp->info.mfa; + } +#endif + else if (em_call_bif_e == (BeamInstr *) *i) { + /* Pointer to bif export in i+1 */ + exp = (Export *) i[1]; + pc = i; + obif = (BifFunction) exp->beam[1]; + mfa = &exp->info.mfa; + } + else if (em_apply_bif == (BeamInstr *) *i) { + /* Pointer to bif in i+1, and mfa in i-3 */ + obif = (BifFunction) i[1]; + pc = c_p->cp; + mfa = erts_code_to_codemfa(i); + } + else { + ERTS_INTERNAL_ERROR("erts_schedule_bif() called " + "from unexpected instruction"); + obif = NULL; + } + ASSERT(bif); + + if (argc < 0) { /* reschedule original call */ + mod = mfa->module; + func = mfa->function; + argc = (int) mfa->arity; + } + + schedule(c_p, dirty_shadow_proc, mfa, obif, + pc, dbif, ibif, mod, func, argc, argv); + } + + if (dirty_shadow_proc) + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + + return THE_NON_VALUE; +} + +static BIF_RETTYPE +call_bif(Process *c_p, Eterm *reg, BeamInstr *I) +{ + NifExport *nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I); + ErtsBifFunc bif = (ErtsBifFunc) nep->func; + BIF_RETTYPE ret; + + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); + + nep->func = ERTS_SCHED_BIF_TRAP_MARKER; + + ASSERT(bif); + + ret = (*bif)(c_p, reg, I); + + if (is_value(ret)) + erts_nif_export_restore(c_p, nep); + else if (c_p->freason != TRAP) + c_p->freason |= EXF_RESTORE_NIF; /* restore in handle_error() */ + else if (nep->func == ERTS_SCHED_BIF_TRAP_MARKER) { + /* BIF did an ordinary trap... */ + erts_nif_export_restore(c_p, nep); + } + /* else: + * BIF rescheduled itself using erts_schedule_bif(). + */ + + return ret; +} + +#ifdef ERTS_DIRTY_SCHEDULERS + +int +erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *reg) +{ + BIF_RETTYPE result; + int exiting; + Process *dirty_shadow_proc; + ErtsBifFunc bf; + NifExport *nep; +#ifdef DEBUG + Eterm *c_p_htop; + erts_aint32_t state; + + ASSERT(!c_p->scheduler_data); + state = erts_smp_atomic32_read_nob(&c_p->state); + ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING) + && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))); + ASSERT(esdp); + +#endif + + nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I); + ASSERT(nep == ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p)); + + nep->func = ERTS_SCHED_BIF_TRAP_MARKER; + + bf = (ErtsBifFunc) I[1]; + + erts_smp_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_IO_PROC)); + + dirty_shadow_proc = erts_make_dirty_shadow_proc(esdp, c_p); + + dirty_shadow_proc->freason = c_p->freason; + dirty_shadow_proc->fvalue = c_p->fvalue; + dirty_shadow_proc->ftrace = c_p->ftrace; + dirty_shadow_proc->cp = c_p->cp; + dirty_shadow_proc->i = c_p->i; + +#ifdef DEBUG + c_p_htop = c_p->htop; +#endif + + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + + result = (*bf)(dirty_shadow_proc, reg, I); + + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + + ASSERT(c_p_htop == c_p->htop); + ASSERT(dirty_shadow_proc->static_flags & ERTS_STC_FLG_SHADOW_PROC); + ASSERT(dirty_shadow_proc->next == c_p); + + exiting = ERTS_PROC_IS_EXITING(c_p); + + if (!exiting) { + if (is_value(result)) + schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL, dirty_bif_result, + NULL, am_erts_internal, am_dirty_bif_result, 1, &result); + else if (dirty_shadow_proc->freason != TRAP) { + Eterm argv[2]; + ASSERT(dirty_shadow_proc->freason <= MAX_SMALL); + argv[0] = make_small(dirty_shadow_proc->freason); + argv[1] = dirty_shadow_proc->fvalue; + schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL, + dirty_bif_exception, NULL, am_erts_internal, + am_dirty_bif_exception, 2, argv); + } + else if (nep->func == ERTS_SCHED_BIF_TRAP_MARKER) { + /* Dirty BIF did an ordinary trap... */ + ASSERT(!(erts_smp_atomic32_read_nob(&c_p->state) + & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC))); + schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL, + dirty_bif_trap, (void *) dirty_shadow_proc->i, + am_erts_internal, am_dirty_bif_trap, + dirty_shadow_proc->arity, reg); + } + /* else: + * BIF rescheduled itself using erts_schedule_bif(). + */ + c_p->freason = dirty_shadow_proc->freason; + c_p->fvalue = dirty_shadow_proc->fvalue; + c_p->ftrace = dirty_shadow_proc->ftrace; + c_p->cp = dirty_shadow_proc->cp; + c_p->i = dirty_shadow_proc->i; + c_p->arity = dirty_shadow_proc->arity; + } + + erts_flush_dirty_shadow_proc(dirty_shadow_proc); + + return exiting; +} + +#endif /* ERTS_DIRTY_SCHEDULERS */ + + #ifdef HARDDEBUG /* You'll need this line in bif.tab to be able to use this debug bif diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index 0c85e19ef0..ad55b19d0a 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -29,17 +29,35 @@ extern Export *erts_convert_time_unit_trap; #define BIF_P A__p -#define BIF_ALIST_0 Process* A__p, Eterm* BIF__ARGS -#define BIF_ALIST_1 Process* A__p, Eterm* BIF__ARGS -#define BIF_ALIST_2 Process* A__p, Eterm* BIF__ARGS -#define BIF_ALIST_3 Process* A__p, Eterm* BIF__ARGS -#define BIF_ALIST_4 Process* A__p, Eterm* BIF__ARGS +#define BIF_ALIST Process* A__p, Eterm* BIF__ARGS, BeamInstr *A__I +#define BIF_CALL_ARGS A__p, BIF__ARGS, A__I + +#define BIF_ALIST_0 BIF_ALIST +#define BIF_ALIST_1 BIF_ALIST +#define BIF_ALIST_2 BIF_ALIST +#define BIF_ALIST_3 BIF_ALIST +#define BIF_ALIST_4 BIF_ALIST #define BIF_ARG_1 (BIF__ARGS[0]) #define BIF_ARG_2 (BIF__ARGS[1]) #define BIF_ARG_3 (BIF__ARGS[2]) #define BIF_ARG_4 (BIF__ARGS[3]) +#define BIF_I A__I + +/* NBIF_* is for bif calls from native code... */ + +#define NBIF_ALIST Process* A__p, Eterm* BIF__ARGS +#define NBIF_CALL_ARGS A__p, BIF__ARGS + +#define NBIF_ALIST_0 NBIF_ALIST +#define NBIF_ALIST_1 NBIF_ALIST +#define NBIF_ALIST_2 NBIF_ALIST +#define NBIF_ALIST_3 NBIF_ALIST +#define NBIF_ALIST_4 NBIF_ALIST + +typedef BIF_RETTYPE (*ErtsBifFunc)(BIF_ALIST); + #define ERTS_IS_PROC_OUT_OF_REDS(p) \ ((p)->fcalls > 0 \ ? 0 \ @@ -480,6 +498,49 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, Eterm args[], int nargs); +typedef enum { + ERTS_SCHED_NORMAL, + ERTS_SCHED_DIRTY_CPU, + ERTS_SCHED_DIRTY_IO +} ErtsSchedType; + +#ifdef ERTS_DIRTY_SCHEDULERS +int erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, + BeamInstr *I, Eterm *reg); +#endif + +BIF_RETTYPE +erts_schedule_bif(Process *proc, + Eterm *argv, + BeamInstr *i, + ErtsBifFunc dbf, + ErtsSchedType sched_type, + Eterm mod, + Eterm func, + int argc); + +ERTS_GLB_INLINE BIF_RETTYPE +erts_reschedule_bif(Process *proc, + Eterm *argv, + BeamInstr *i, + ErtsBifFunc dbf, + ErtsSchedType sched_type); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE BIF_RETTYPE +erts_reschedule_bif(Process *proc, + Eterm *argv, + BeamInstr *i, + ErtsBifFunc dbf, + ErtsSchedType sched_type) +{ + return erts_schedule_bif(proc, argv, i, dbf, sched_type, + THE_NON_VALUE, THE_NON_VALUE, -1); +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + #ifdef ERL_WANT_HIPE_BIF_WRAPPER__ #ifndef HIPE @@ -510,16 +571,16 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, #define HIPE_WRAPPER_BIF_DISABLE_GC(BIF_NAME, ARITY) \ -BIF_RETTYPE hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (Process* c_p, \ - Eterm* args); \ -BIF_RETTYPE hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (Process* c_p, \ - Eterm* args) \ +BIF_RETTYPE \ +nbif_impl_hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (NBIF_ALIST); \ +BIF_RETTYPE \ +nbif_impl_hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (NBIF_ALIST) \ { \ BIF_RETTYPE res; \ - hipe_reserve_beam_trap_frame(c_p, args, ARITY); \ - res = BIF_NAME ## _ ## ARITY (c_p, args); \ - if (is_value(res) || c_p->freason != TRAP) { \ - hipe_unreserve_beam_trap_frame(c_p); \ + hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, ARITY); \ + res = nbif_impl_ ## BIF_NAME ## _ ## ARITY (NBIF_CALL_ARGS); \ + if (is_value(res) || BIF_P->freason != TRAP) { \ + hipe_unreserve_beam_trap_frame(BIF_P); \ } \ return res; \ } diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 32600f4338..f461da913e 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -420,6 +420,9 @@ bif erts_debug:set_internal_state/2 bif erts_debug:display/1 bif erts_debug:dist_ext_to_term/2 bif erts_debug:instructions/0 +bif erts_debug:dirty_cpu/2 +bif erts_debug:dirty_io/2 +bif erts_debug:dirty/3 # # Monitor testing bif's... diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 009d2b72d3..4659cb5a7b 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -93,6 +93,9 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE #ifdef USE_THREADS #if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP) " [ds:%beu:%beu:%beu]" +#endif +#if defined(ERTS_DIRTY_SCHEDULERS_TEST) + " [dirty-schedulers-TEST]" #endif " [async-threads:%d]" #endif diff --git a/erts/emulator/beam/erl_dirty_bif.tab b/erts/emulator/beam/erl_dirty_bif.tab new file mode 100644 index 0000000000..69421dcfcc --- /dev/null +++ b/erts/emulator/beam/erl_dirty_bif.tab @@ -0,0 +1,82 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2016. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# %CopyrightEnd% +# + +# +# Static declaration of BIFs that should execute on dirty schedulers. +# +# ::= +# ::= ":" "/" +# ::= dirty-cpu | dirty-io | dirty-cpu-test | dirty-io-test +# +# When dirty scheduler support is available, a BIF declared with the +# 'dirty-cpu' type will unconditionally execute on a dirty CPU scheduler, +# and a BIF declared with the type 'dirty-io' will unconditionally execute +# on a dirty IO scheduler. When dirty scheduler support is not available +# all BIFs will of course execute on normal schedulers. +# +# When the emulator has been configured with the debug option +# '--enable-dirty-schedulers-test', BIFs with the types 'dirty-cpu-test', +# and 'dirty-io-test' will unconditionally execute on dirty schedulers. +# When this debug option has not been enabled, these BIFs will be executed +# on normal schedulers. +# +# BIFs marked as 'ubif' in ./bif.tab will be ignored, i.e., will always +# execute on normal schedulers. +# + +# --- Dirty BIFs --- + +dirty-cpu erts_debug:dirty_cpu/2 +dirty-io erts_debug:dirty_io/2 + +# --- TEST of Dirty BIF functionality --- +# Functions below will execute on dirty schedulers when emulator has +# been configured for testing dirty schedulers. This is used for test +# and debug purposes only. We really do *not* want to execute these +# on dirty schedulers on a real system. + +dirty-cpu-test erlang:'++'/2 +dirty-cpu-test erlang:append/2 +dirty-cpu-test erlang:'--'/2 +dirty-cpu-test erlang:subtract/2 +dirty-cpu-test erlang:iolist_size/1 +dirty-cpu-test erlang:make_tuple/2 +dirty-cpu-test erlang:make_tuple/3 +dirty-cpu-test erlang:append_element/2 +dirty-cpu-test erlang:insert_element/3 +dirty-cpu-test erlang:delete_element/2 +dirty-cpu-test erlang:atom_to_list/1 +dirty-cpu-test erlang:list_to_atom/1 +dirty-cpu-test erlang:list_to_existing_atom/1 +dirty-cpu-test erlang:integer_to_list/1 +dirty-cpu-test erlang:string_to_integer/1 +dirty-cpu-test erlang:list_to_integer/1 +dirty-cpu-test erlang:list_to_integer/2 +dirty-cpu-test erlang:float_to_list/1 +dirty-cpu-test erlang:float_to_list/2 +dirty-cpu-test erlang:float_to_binary/1 +dirty-cpu-test erlang:float_to_binary/2 +dirty-cpu-test erlang:string_to_float/1 +dirty-cpu-test erlang:list_to_float/1 +dirty-cpu-test erlang:binary_to_float/1 +dirty-cpu-test erlang:tuple_to_list/1 +dirty-cpu-test erlang:list_to_tuple/1 +dirty-cpu-test erlang:display/1 +dirty-cpu-test erlang:display_string/1 diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 643b46c861..6093b0cf39 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -42,6 +42,7 @@ #include "dtrace-wrapper.h" #include "erl_bif_unique.h" #include "dist.h" +#include "erl_nfunc_sched.h" #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20 @@ -2423,17 +2424,10 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) } /* - * If a NIF has saved arguments, they need to be added + * If a NIF or BIF has saved arguments, they need to be added */ - if (ERTS_PROC_GET_NIF_TRAP_EXPORT(p)) { - Eterm* argv; - int argc; - if (erts_setup_nif_gc(p, &argv, &argc)) { - roots[n].v = argv; - roots[n].sz = argc; - n++; - } - } + if (erts_setup_nif_export_rootset(p, &roots[n].v, &roots[n].sz)) + n++; ASSERT(n <= rootset->size); @@ -2985,6 +2979,8 @@ static void ERTS_INLINE offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size, Eterm* objv, int nobj) { + Eterm *v; + Uint sz; if (p->dictionary) { offset_heap(ERTS_PD_START(p->dictionary), ERTS_PD_SIZE(p->dictionary), @@ -3005,12 +3001,8 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size, offset_heap_ptr(objv, nobj, offs, area, area_size); } offset_off_heap(p, offs, area, area_size); - if (ERTS_PROC_GET_NIF_TRAP_EXPORT(p)) { - Eterm* argv; - int argc; - if (erts_setup_nif_gc(p, &argv, &argc)) - offset_heap_ptr(argv, argc, offs, area, area_size); - } + if (erts_setup_nif_export_rootset(p, &v, &sz)) + offset_heap_ptr(v, sz, offs, area, area_size); } static void diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c index 7ddf49937f..66bb55e6c8 100644 --- a/erts/emulator/beam/erl_msacc.c +++ b/erts/emulator/beam/erl_msacc.c @@ -137,8 +137,8 @@ void erts_msacc_init_thread(char *type, int id, int managed) { void erts_msacc_set_bif_state(ErtsMsAcc *__erts_msacc_cache, Eterm mod, void *fn) { #ifdef ERTS_MSACC_EXTENDED_BIFS -#define BIF_LIST(Mod,Func,Arity,FuncAddr,Num) \ - if (fn == &FuncAddr) { \ +#define BIF_LIST(Mod,Func,Arity,BifFuncAddr,FuncAddr,Num) \ + if (fn == &BifFuncAddr) { \ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATIC_STATE_COUNT + Num); \ } else #include "erl_bif_list.h" diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h index 9ef86a1293..d64ef8c8b9 100644 --- a/erts/emulator/beam/erl_msacc.h +++ b/erts/emulator/beam/erl_msacc.h @@ -122,7 +122,7 @@ static char *erts_msacc_states[] = { "sleep", "timers" #ifdef ERTS_MSACC_EXTENDED_BIFS -#define BIF_LIST(Mod,Func,Arity,FuncAddr,Num) \ +#define BIF_LIST(Mod,Func,Arity,BifFuncAddr,FuncAddr,Num) \ ,"bif_" #Mod "_" #Func "_" #Arity #include "erl_bif_list.h" #undef BIF_LIST diff --git a/erts/emulator/beam/erl_nfunc_sched.c b/erts/emulator/beam/erl_nfunc_sched.c new file mode 100644 index 0000000000..0333545255 --- /dev/null +++ b/erts/emulator/beam/erl_nfunc_sched.c @@ -0,0 +1,144 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#define ERTS_WANT_NFUNC_SCHED_INTERNALS__ + +#include "global.h" +#include "erl_process.h" +#include "bif.h" +#include "erl_nfunc_sched.h" + +NifExport * +erts_new_proc_nif_export(Process *c_p, int argc) +{ + size_t size; + int i; + NifExport *nep, *old_nep; + + size = sizeof(NifExport) + (argc-1)*sizeof(Eterm); + nep = erts_alloc(ERTS_ALC_T_NIF_TRAP_EXPORT, size); + + for (i = 0; i < ERTS_NUM_CODE_IX; i++) + nep->exp.addressv[i] = &nep->exp.beam[0]; + + nep->argc = -1; /* unused marker */ + nep->argv_size = argc; + old_nep = ERTS_PROC_SET_NIF_TRAP_EXPORT(c_p, nep); + if (old_nep) + erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, old_nep); + return nep; +} + +void +erts_destroy_nif_export(Process *p) +{ + NifExport *nep = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL); + if (nep) { + if (nep->m) + erts_nif_export_cleanup_nif_mod(nep); + erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, nep); + } +} + +NifExport * +erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, + ErtsCodeMFA *mfa, void *nif, BeamInstr *pc, + BeamInstr instr, + void *dfunc, void *ifunc, + Eterm mod, Eterm func, + int argc, const Eterm *argv) +{ + Process *used_proc; + ErtsSchedulerData *esdp; + Eterm* reg; + NifExport* nep; + int i; + + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); + + if (dirty_shadow_proc) { + esdp = erts_get_scheduler_data(); + ASSERT(esdp && ERTS_SCHEDULER_IS_DIRTY(esdp)); + + used_proc = dirty_shadow_proc; + } + else { + esdp = erts_proc_sched_data(c_p); + ASSERT(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)); + + used_proc = c_p; + ERTS_VBUMP_ALL_REDS(c_p); + } + + reg = esdp->x_reg_array; + + if (mfa) + nep = erts_get_proc_nif_export(c_p, (int) mfa->arity); + else { + /* If no mfa, this is not the first schedule... */ + nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + ASSERT(nep && nep->argc >= 0); + } + + if (nep->argc < 0) { + /* + * First schedule; save things that might + * need to be restored... + */ + for (i = 0; i < (int) mfa->arity; i++) + nep->argv[i] = reg[i]; + nep->pc = pc; + nep->cp = c_p->cp; + ASSERT(nif); + nep->nif = nif; + nep->mfa = mfa; + nep->current = c_p->current; + ASSERT(argc >= 0); + nep->argc = (int) mfa->arity; + nep->m = NULL; + + ASSERT(!erts_check_nif_export_in_area(c_p, + (char *) nep, + (sizeof(NifExport) + + (sizeof(Eterm) + *(nep->argc-1))))); + } + /* Copy new arguments into register array if necessary... */ + if (reg != argv) { + for (i = 0; i < argc; i++) + reg[i] = argv[i]; + } + ASSERT(is_atom(mod) && is_atom(func)); + nep->exp.info.mfa.module = mod; + nep->exp.info.mfa.function = func; + nep->exp.info.mfa.arity = (Uint) argc; + nep->exp.beam[0] = (BeamInstr) instr; /* call_nif || apply_bif */ + nep->exp.beam[1] = (BeamInstr) dfunc; + nep->func = ifunc; + used_proc->arity = argc; + used_proc->freason = TRAP; + used_proc->i = (BeamInstr*) nep->exp.addressv[0]; + return nep; +} diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h new file mode 100644 index 0000000000..72de7f0eb5 --- /dev/null +++ b/erts/emulator/beam/erl_nfunc_sched.h @@ -0,0 +1,289 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef ERL_NFUNC_SCHED_H__ +#define ERL_NFUNC_SCHED_H__ + +#include "erl_process.h" +#include "bif.h" + +/* + * NIF exports need a few more items than the Export struct provides, + * including the erl_module_nif* and a NIF function pointer, so the + * NifExport below adds those. The Export member must be first in the + * struct. A number of values are stored for error handling purposes + * only. + * + * 'argc' is >= 0 when NifExport is in use, and < 0 when not. + */ + +typedef struct { + Export exp; + struct erl_module_nif* m; /* NIF module, or NULL if BIF */ + void *func; /* Indirect NIF or BIF to execute (may be unused) */ + ErtsCodeMFA *current;/* Current as set when originally called */ + /* --- The following is only used on error --- */ + BeamInstr *pc; /* Program counter */ + BeamInstr *cp; /* Continuation pointer */ + void *nif; /* Original NIF/BIF call */ + ErtsCodeMFA *mfa; /* MFA of original call */ + int argc; /* Number of arguments in original call */ + int argv_size; /* Allocated size of argv */ + Eterm argv[1]; /* Saved arguments from the original call */ +} NifExport; + +NifExport *erts_new_proc_nif_export(Process *c_p, int argc); +void erts_destroy_nif_export(Process *p); +NifExport *erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, + ErtsCodeMFA *mfa, void *nif, BeamInstr *pc, + BeamInstr instr, + void *dfunc, void *ifunc, + Eterm mod, Eterm func, + int argc, const Eterm *argv); +void erts_nif_export_cleanup_nif_mod(NifExport *ep); /* erl_nif.c */ +ERTS_GLB_INLINE NifExport *erts_get_proc_nif_export(Process *c_p, int extra); +ERTS_GLB_INLINE int erts_setup_nif_export_rootset(Process* proc, Eterm** objv, + Uint* nobj); +ERTS_GLB_INLINE int erts_check_nif_export_in_area(Process *p, + char *start, Uint size); +ERTS_GLB_INLINE void erts_nif_export_restore(Process *c_p, NifExport *ep); +ERTS_GLB_INLINE void erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, + Eterm *reg, void **nif); +ERTS_GLB_INLINE Process *erts_proc_shadow2real(Process *c_p); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE NifExport * +erts_get_proc_nif_export(Process *c_p, int argc) +{ + NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + if (!nep || (nep->argc < 0 && nep->argv_size < argc)) + return erts_new_proc_nif_export(c_p, argc); + return nep; +} + +/* + * If a process has saved arguments, they need to be part of the GC + * rootset. The function below is called from setup_rootset() in + * erl_gc.c. Any exception term saved in the NifExport is also made + * part of the GC rootset here; it always resides in rootset[0]. + */ +ERTS_GLB_INLINE int +erts_setup_nif_export_rootset(Process* proc, Eterm** objv, Uint* nobj) +{ + NifExport* ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + + if (!ep || ep->argc <= 0) + return 0; + + *objv = ep->argv; + *nobj = ep->argc; + return 1; +} + +/* + * Check if nif export points into code area... + */ +ERTS_GLB_INLINE int +erts_check_nif_export_in_area(Process *p, char *start, Uint size) +{ + NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(p); + if (!nep || nep->argc < 0) + return 0; + if (ErtsInArea(nep->pc, start, size)) + return 1; + if (ErtsInArea(nep->cp, start, size)) + return 1; + if (ErtsInArea(nep->mfa, start, size)) + return 1; + if (ErtsInArea(nep->current, start, size)) + return 1; + return 0; +} + +ERTS_GLB_INLINE void +erts_nif_export_restore(Process *c_p, NifExport *ep) +{ + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); + ERTS_SMP_LC_ASSERT(!(c_p->static_flags + & ERTS_STC_FLG_SHADOW_PROC)); + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); + + c_p->current = ep->current; + ep->argc = -1; /* Unused nif-export marker... */ +} + +ERTS_GLB_INLINE void +erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, Eterm *reg, + void **nif) +{ + NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + int ix; + + ASSERT(nep); + *pc = nep->pc; + c_p->cp = nep->cp; + *nif = nep->nif; + for (ix = 0; ix < nep->argc; ix++) + reg[ix] = nep->argv[ix]; + erts_nif_export_restore(c_p, nep); +} + +ERTS_GLB_INLINE Process * +erts_proc_shadow2real(Process *c_p) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + if (c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC) { + Process *real_c_p = c_p->next; + ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); + ASSERT(real_c_p->common.id == c_p->common.id); + return real_c_p; + } + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); +#endif + return c_p; +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* ERL_NFUNC_SCHED_H__ */ + +#if defined(ERTS_WANT_NFUNC_SCHED_INTERNALS__) && !defined(ERTS_NFUNC_SCHED_INTERNALS__) +#define ERTS_NFUNC_SCHED_INTERNALS__ + +#define ERTS_I_BEAM_OP_TO_NIF_EXPORT(I) \ + (ASSERT(BeamOp(op_apply_bif) == (BeamInstr *) (*(I)) \ + || BeamOp(op_call_nif) == (BeamInstr *) (*(I))), \ + ((NifExport *) (((char *) (I)) - offsetof(NifExport, exp.beam[0])))) + +#ifdef ERTS_DIRTY_SCHEDULERS + +#include "erl_message.h" +#include + +ERTS_GLB_INLINE void erts_flush_dirty_shadow_proc(Process *sproc); +ERTS_GLB_INLINE void erts_cache_dirty_shadow_proc(Process *sproc); +ERTS_GLB_INLINE Process *erts_make_dirty_shadow_proc(ErtsSchedulerData *esdp, + Process *c_p); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void +erts_flush_dirty_shadow_proc(Process *sproc) +{ + Process *c_p = sproc->next; + + ASSERT(sproc->common.id == c_p->common.id); + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); + + ASSERT(c_p->stop == sproc->stop); + ASSERT(c_p->hend == sproc->hend); + ASSERT(c_p->heap == sproc->heap); + ASSERT(c_p->abandoned_heap == sproc->abandoned_heap); + ASSERT(c_p->heap_sz == sproc->heap_sz); + ASSERT(c_p->high_water == sproc->high_water); + ASSERT(c_p->old_heap == sproc->old_heap); + ASSERT(c_p->old_htop == sproc->old_htop); + ASSERT(c_p->old_hend == sproc->old_hend); + + ASSERT(c_p->htop <= sproc->htop && sproc->htop <= c_p->stop); + + c_p->htop = sproc->htop; + + if (!c_p->mbuf) + c_p->mbuf = sproc->mbuf; + else if (sproc->mbuf) { + ErlHeapFragment *bp; + for (bp = sproc->mbuf; bp->next; bp = bp->next) + ASSERT(!bp->off_heap.first); + bp->next = c_p->mbuf; + c_p->mbuf = sproc->mbuf; + } + + c_p->mbuf_sz += sproc->mbuf_sz; + + if (!c_p->off_heap.first) + c_p->off_heap.first = sproc->off_heap.first; + else if (sproc->off_heap.first) { + struct erl_off_heap_header *ohhp; + for (ohhp = sproc->off_heap.first; ohhp->next; ohhp = ohhp->next) + ; + ohhp->next = c_p->off_heap.first; + c_p->off_heap.first = sproc->off_heap.first; + } + + c_p->off_heap.overhead += sproc->off_heap.overhead; +} + +ERTS_GLB_INLINE void +erts_cache_dirty_shadow_proc(Process *sproc) +{ + Process *c_p = sproc->next; + ASSERT(c_p); + ASSERT(sproc->common.id == c_p->common.id); + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); + + sproc->htop = c_p->htop; + sproc->stop = c_p->stop; + sproc->hend = c_p->hend; + sproc->heap = c_p->heap; + sproc->abandoned_heap = c_p->abandoned_heap; + sproc->heap_sz = c_p->heap_sz; + sproc->high_water = c_p->high_water; + sproc->old_hend = c_p->old_hend; + sproc->old_htop = c_p->old_htop; + sproc->old_heap = c_p->old_heap; + sproc->mbuf = NULL; + sproc->mbuf_sz = 0; + ERTS_INIT_OFF_HEAP(&sproc->off_heap); +} + +ERTS_GLB_INLINE Process * +erts_make_dirty_shadow_proc(ErtsSchedulerData *esdp, Process *c_p) +{ + Process *sproc; + + ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)); + + sproc = esdp->dirty_shadow_process; + ASSERT(sproc); + ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC); + ASSERT(erts_smp_atomic32_read_nob(&sproc->state) + == (ERTS_PSFLG_ACTIVE + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_PROXY)); + + sproc->next = c_p; + sproc->common.id = c_p->common.id; + + erts_cache_dirty_shadow_proc(sproc); + + return sproc; +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* ERTS_DIRTY_SCHEDULERS */ + +#endif /* defined(ERTS_WANT_NFUNC_SCHED_INTERNALS__) && !defined(ERTS_NFUNC_SCHED_INTERNALS__) */ + diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 5499512dd1..ee4b22f7ca 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -55,6 +55,9 @@ #include "dtrace-wrapper.h" #include "erl_process.h" #include "erl_bif_unique.h" +#undef ERTS_WANT_NFUNC_SCHED_INTERNALS__ +#define ERTS_WANT_NFUNC_SCHED_INTERNALS__ +#include "erl_nfunc_sched.h" #if defined(USE_DYNAMIC_TRACE) && (defined(USE_DTRACE) || defined(USE_SYSTEMTAP)) #define HAVE_USE_DTRACE 1 #endif @@ -79,8 +82,11 @@ struct erl_module_nif { ErlNifFunc _funcs_copy_[1]; /* only used for old libs */ }; +typedef ERL_NIF_TERM (*NativeFunPtr)(ErlNifEnv*, int, const ERL_NIF_TERM[]); + #ifdef DEBUG # define READONLY_CHECK +# define ERTS_DBG_NIF_NOT_SCHED_MARKER ((void *) (UWord) 1) #endif #ifdef READONLY_CHECK # define ADD_READONLY_CHECK(ENV,PTR,SIZE) add_readonly_check(ENV,PTR,SIZE) @@ -219,38 +225,6 @@ static void cache_env(ErlNifEnv* env); static void full_flush_env(ErlNifEnv *env); static void flush_env(ErlNifEnv* env); -#ifdef ERTS_DIRTY_SCHEDULERS -void erts_pre_dirty_nif(ErtsSchedulerData *esdp, - ErlNifEnv* env, Process* p, - struct erl_module_nif* mod_nif) -{ - Process *sproc; -#ifdef DEBUG - erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); - - ASSERT(!p->scheduler_data); - ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING) - && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))); - ASSERT(esdp); -#endif - - erts_pre_nif(env, p, mod_nif, NULL); - - sproc = esdp->dirty_shadow_process; - ASSERT(sproc); - ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC); - ASSERT(erts_smp_atomic32_read_nob(&sproc->state) - == (ERTS_PSFLG_ACTIVE - | ERTS_PSFLG_DIRTY_RUNNING - | ERTS_PSFLG_PROXY)); - - sproc->next = p; - sproc->common.id = p->common.id; - env->proc = sproc; - full_cache_env(env); -} -#endif - /* Temporary object header, auto-deallocated when NIF returns * or when independent environment is cleared. */ @@ -278,115 +252,158 @@ void erts_post_nif(ErlNifEnv* env) env->exiting = ERTS_PROC_IS_EXITING(env->proc); } -#ifdef ERTS_DIRTY_SCHEDULERS -void erts_post_dirty_nif(ErlNifEnv* env) + +/* + * Initialize a NifExport struct. Create it if needed and store it in the + * proc. The direct_fp function is what will be invoked by op_call_nif, and + * the indirect_fp function, if not NULL, is what the direct_fp function + * will call. If the allocated NifExport isn't enough to hold all of argv, + * allocate a larger one. Save 'current' and registers if first time this + * call is scheduled. + */ + +static ERTS_INLINE ERL_NIF_TERM +schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp, + Eterm mod, Eterm func_name, int argc, const ERL_NIF_TERM argv[]) { - Process *c_p; - ASSERT(env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC); - ASSERT(env->proc->next); - erts_unblock_fpe(env->fpe_was_unmasked); - full_flush_env(env); - free_tmp_objs(env); - c_p = env->proc->next; - env->exiting = ERTS_PROC_IS_EXITING(c_p); - ERTS_VBUMP_ALL_REDS(c_p); + Export *exp; + NifExport *ep; + Process *c_p, *dirty_shadow_proc; + + execution_state(env, &c_p, NULL); + if (c_p == env->proc) + dirty_shadow_proc = NULL; + else + dirty_shadow_proc = env->proc; + + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); + + exp = ErtsContainerStruct(c_p->current, Export, info.mfa); + + ep = erts_nif_export_schedule(c_p, dirty_shadow_proc, + c_p->current, + (BifFunction) exp->beam[1], + c_p->cp, + (BeamInstr) em_call_nif, + direct_fp, indirect_fp, + mod, func_name, + argc, (const Eterm *) argv); + if (!ep->m) { + /* First time this call is scheduled... */ + erts_refc_inc(&env->mod_nif->rt_dtor_cnt, 1); + ep->m = env->mod_nif; + } + return (ERL_NIF_TERM) THE_NON_VALUE; } -#endif -static void full_flush_env(ErlNifEnv* env) -{ #ifdef ERTS_DIRTY_SCHEDULERS - if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { - /* Dirty nif call using shadow process struct */ - Process *c_p = env->proc->next; - - ASSERT(is_scheduler() < 0); - ASSERT(env->proc->common.id == c_p->common.id); - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) - & ERTS_PROC_LOCK_MAIN); - - if (!env->heap_frag) { - ASSERT(env->hp_end == HEAP_LIMIT(c_p)); - ASSERT(env->hp >= HEAP_TOP(c_p)); - ASSERT(env->hp <= HEAP_LIMIT(c_p)); - HEAP_TOP(c_p) = env->hp; + +static ERL_NIF_TERM dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +int +erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *reg) +{ + int exiting; + ERL_NIF_TERM *argv = (ERL_NIF_TERM *) reg; + NifExport *nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I); + ErtsCodeMFA *codemfa = erts_code_to_codemfa(I); + NativeFunPtr dirty_nif = (NativeFunPtr) I[1]; + ErlNifEnv env; + ERL_NIF_TERM result; +#ifdef DEBUG + erts_aint32_t state = erts_smp_atomic32_read_nob(&c_p->state); + + ASSERT(nep == ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p)); + + ASSERT(!c_p->scheduler_data); + ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING) + && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))); + ASSERT(esdp); + + nep->func = ERTS_DBG_NIF_NOT_SCHED_MARKER; +#endif + + erts_pre_nif(&env, c_p, nep->m, NULL); + + env.proc = erts_make_dirty_shadow_proc(esdp, c_p); + + env.proc->freason = EXC_NULL; + env.proc->fvalue = NIL; + env.proc->ftrace = NIL; + env.proc->i = c_p->i; + + ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p))); + + erts_smp_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_IO_PROC)); + + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + + result = (*dirty_nif)(&env, codemfa->arity, argv); /* Call dirty NIF */ + + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + + ASSERT(env.proc->static_flags & ERTS_STC_FLG_SHADOW_PROC); + ASSERT(env.proc->next == c_p); + + exiting = ERTS_PROC_IS_EXITING(c_p); + + if (!exiting) { + if (env.exception_thrown) { + schedule_exception: + schedule(&env, dirty_nif_exception, NULL, + am_erts_internal, am_dirty_nif_exception, + 1, &env.proc->fvalue); + } + else if (is_value(result)) { + schedule(&env, dirty_nif_finalizer, NULL, + am_erts_internal, am_dirty_nif_finalizer, + 1, &result); + } + else if (env.proc->freason != TRAP) { /* user returned garbage... */ + ERTS_DECL_AM(badreturn); + (void) enif_raise_exception(&env, AM_badreturn); + goto schedule_exception; } else { - Uint usz; - ASSERT(env->hp_end != HEAP_LIMIT(c_p)); - ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); - - HEAP_TOP(c_p) = HEAP_TOP(env->proc); - usz = env->hp - env->heap_frag->mem; - env->proc->mbuf_sz += usz - env->heap_frag->used_size; - env->heap_frag->used_size = usz; - - ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); - - if (c_p->mbuf) { - ErlHeapFragment *bp; - for (bp = env->proc->mbuf; bp->next; bp = bp->next) - ; - bp->next = c_p->mbuf; - } + /* Rescheduled by dirty NIF call... */ + ASSERT(nep->func != ERTS_DBG_NIF_NOT_SCHED_MARKER); + } + c_p->i = env.proc->i; + c_p->arity = env.proc->arity; + } - c_p->mbuf = env->proc->mbuf; - c_p->mbuf_sz += env->proc->mbuf_sz; +#ifdef DEBUG + if (nep->func == ERTS_DBG_NIF_NOT_SCHED_MARKER) + nep->func = NULL; +#endif - } + erts_unblock_fpe(env.fpe_was_unmasked); + full_flush_env(&env); + free_tmp_objs(&env); - if (!c_p->off_heap.first) - c_p->off_heap.first = env->proc->off_heap.first; - else if (env->proc->off_heap.first) { - struct erl_off_heap_header *ohhp; - for (ohhp = env->proc->off_heap.first; ohhp->next; ohhp = ohhp->next) - ; - ohhp->next = c_p->off_heap.first; - c_p->off_heap.first = env->proc->off_heap.first; - } - c_p->off_heap.overhead += env->proc->off_heap.overhead; + return exiting; +} - return; - } #endif +static void full_flush_env(ErlNifEnv* env) +{ flush_env(env); +#ifdef ERTS_DIRTY_SCHEDULERS + if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) + /* Dirty nif call using shadow process struct */ + erts_flush_dirty_shadow_proc(env->proc); +#endif } static void full_cache_env(ErlNifEnv* env) { #ifdef ERTS_DIRTY_SCHEDULERS - if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { - /* Dirty nif call using shadow process struct */ - Process *sproc = env->proc; - Process *c_p = sproc->next; - ASSERT(c_p); - ASSERT(is_scheduler() < 0); - ASSERT(env->proc->common.id == c_p->common.id); - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) - & ERTS_PROC_LOCK_MAIN); - - sproc->htop = c_p->htop; - sproc->stop = c_p->stop; - sproc->hend = c_p->hend; - sproc->heap = c_p->heap; - sproc->abandoned_heap = c_p->abandoned_heap; - sproc->heap_sz = c_p->heap_sz; - sproc->high_water = c_p->high_water; - sproc->old_hend = c_p->old_hend; - sproc->old_htop = c_p->old_htop; - sproc->old_heap = c_p->old_heap; - sproc->mbuf = NULL; - sproc->mbuf_sz = 0; - ERTS_INIT_OFF_HEAP(&sproc->off_heap); - - env->hp_end = HEAP_LIMIT(c_p); - env->hp = HEAP_TOP(c_p); - env->heap_frag = NULL; - return; - } + if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) + erts_cache_dirty_shadow_proc(env->proc); #endif - cache_env(env); } @@ -1303,22 +1320,15 @@ Eterm enif_make_badarg(ErlNifEnv* env) Eterm enif_raise_exception(ErlNifEnv* env, ERL_NIF_TERM reason) { - Process *c_p; - - execution_state(env, &c_p, NULL); - env->exception_thrown = 1; - c_p->fvalue = reason; - BIF_ERROR(c_p, EXC_ERROR); + env->proc->fvalue = reason; + BIF_ERROR(env->proc, EXC_ERROR); } int enif_has_pending_exception(ErlNifEnv* env, ERL_NIF_TERM* reason) { - if (env->exception_thrown && reason != NULL) { - Process *c_p; - execution_state(env, &c_p, NULL); - *reason = c_p->fvalue; - } + if (env->exception_thrown && reason != NULL) + *reason = env->proc->fvalue; return env->exception_thrown; } @@ -2269,188 +2279,28 @@ int enif_consume_timeslice(ErlNifEnv* env, int percent) return ERTS_BIF_REDS_LEFT(proc) == 0; } -/* - * NIF exports need a few more items than the Export struct provides, - * including the erl_module_nif* and a NIF function pointer, so the - * NifExport below adds those. The Export member must be first in the - * struct. The saved_current, exception_thrown, saved_argc, rootset_extra, and - * rootset members are used to track the MFA, any pending exception, and - * arguments of the top NIF in case a chain of one or more - * enif_schedule_nif() calls results in an exception, since in that case - * the original MFA and registers have to be restored before returning to - * Erlang to ensure stacktrace information associated with the exception is - * correct. - */ -typedef ERL_NIF_TERM (*NativeFunPtr)(ErlNifEnv*, int, const ERL_NIF_TERM[]); - -typedef struct { - Export exp; - struct erl_module_nif* m; - NativeFunPtr fp; - ErtsCodeMFA *saved_current; - int exception_thrown; - int saved_argc; - int rootset_extra; - Eterm rootset[1]; -} NifExport; - -/* - * If a process has saved arguments, they need to be part of the GC - * rootset. The function below is called from setup_rootset() in - * erl_gc.c. This function is declared in erl_process.h. Any exception term - * saved in the NifExport is also made part of the GC rootset here; it - * always resides in rootset[0]. - */ -int -erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj) -{ - NifExport* ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - int gc = ep && (ep->saved_argc > 0 || ep->rootset[0] != NIL); - - if (gc) { - *objv = ep->rootset; - *nobj = 1 + ep->saved_argc; - } - return gc; -} - -int -erts_check_nif_export_in_area(Process *p, char *start, Uint size) -{ - NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(p); - if (!nep || !nep->saved_current) - return 0; - if (ErtsInArea(nep->saved_current, start, size)) - return 1; - return 0; -} - -/* - * Allocate a NifExport and set it in proc specific data - */ -static NifExport* -allocate_nif_sched_data(Process* proc, int argc) -{ - NifExport* ep; - size_t total; - int i; - - total = sizeof(NifExport) + argc*sizeof(Eterm); - ep = erts_alloc(ERTS_ALC_T_NIF_TRAP_EXPORT, total); - sys_memset((void*) ep, 0, total); - ep->rootset_extra = argc; - ep->rootset[0] = NIL; - for (i=0; iexp.addressv[i] = &ep->exp.beam[0]; - } - ep->exp.beam[0] = (BeamInstr) em_call_nif; - (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ep); - return ep; -} - static ERTS_INLINE void -destroy_nif_export(NifExport *nif_export) +nif_export_cleanup_nif_mod(NifExport *ep) { - erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, (void *) nif_export); + if (erts_refc_dectest(&ep->m->rt_dtor_cnt, 0) == 0 && ep->m->mod == NULL) + close_lib(ep->m); + ep->m = NULL; } void -erts_destroy_nif_export(void *nif_export) +erts_nif_export_cleanup_nif_mod(NifExport *ep) { - destroy_nif_export((NifExport *) nif_export); + nif_export_cleanup_nif_mod(ep); } -/* - * Initialize a NifExport struct. Create it if needed and store it in the - * proc. The direct_fp function is what will be invoked by op_call_nif, and - * the indirect_fp function, if not NULL, is what the direct_fp function - * will call. If the allocated NifExport isn't enough to hold all of argv, - * allocate a larger one. Save MFA and registers only if the need_save - * parameter is true. - */ -static ERL_NIF_TERM -init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp, - int need_save, int argc, const ERL_NIF_TERM argv[]) +static ERTS_INLINE void +nif_export_restore(Process *c_p, NifExport *ep) { - Process* proc; - Eterm* reg; - NifExport* ep; - int i, scheduler; - int orig_argc; - - execution_state(env, &proc, &scheduler); - - ASSERT(scheduler); - - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc) - & ERTS_PROC_LOCK_MAIN); - - reg = erts_proc_sched_data(proc)->x_reg_array; - - ASSERT(!need_save || proc->current); - orig_argc = need_save ? (int) proc->current->arity : 0; - - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - if (!ep) - ep = allocate_nif_sched_data(proc, orig_argc); - else if (need_save && ep->rootset_extra < orig_argc) { - NifExport* new_ep = allocate_nif_sched_data(proc, orig_argc); - destroy_nif_export(ep); - ep = new_ep; - } - if (env->exception_thrown) { - ep->exception_thrown = 1; - ep->rootset[0] = proc->fvalue; - } else { - ep->exception_thrown = 0; - ep->rootset[0] = NIL; - } - if (scheduler > 0) - ERTS_VBUMP_ALL_REDS(proc); - if (need_save) { - ep->saved_current = proc->current; - ep->saved_argc = orig_argc; - for (i = 0; i < orig_argc; i++) - ep->rootset[i+1] = reg[i]; - } - for (i = 0; i < argc; i++) - reg[i] = (Eterm) argv[i]; - proc->i = (BeamInstr*) ep->exp.addressv[0]; - ep->exp.info.mfa.module = proc->current->module; - ep->exp.info.mfa.function = proc->current->function; - ep->exp.info.mfa.arity = argc; - ep->exp.beam[1] = (BeamInstr) direct_fp; - ep->m = env->mod_nif; - ep->fp = indirect_fp; - proc->freason = TRAP; - proc->arity = argc; - return THE_NON_VALUE; + erts_nif_export_restore(c_p, ep); + ASSERT(ep->m); + nif_export_cleanup_nif_mod(ep); } -/* - * Restore saved MFA and registers. Registers are restored only when the - * exception flag is true. - */ -static void -restore_nif_mfa(Process* proc, NifExport* ep, int exception) -{ - int i; - - ERTS_SMP_LC_ASSERT(!(proc->static_flags - & ERTS_STC_FLG_SHADOW_PROC)); - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc) - & ERTS_PROC_LOCK_MAIN); - - ASSERT(ep->saved_current != &ep->exp.info.mfa); - proc->current = ep->saved_current; - ep->saved_current = NULL; - if (exception) { - Eterm* reg = erts_proc_sched_data(proc)->x_reg_array; - for (i = 0; i < ep->saved_argc; i++) - reg[i] = ep->rootset[i+1]; - } - ep->saved_argc = 0; -} #ifdef ERTS_DIRTY_SCHEDULERS @@ -2459,7 +2309,7 @@ restore_nif_mfa(Process* proc, NifExport* ep, int exception) * switch the process off a dirty scheduler thread and back onto a regular * scheduler thread, and then return the result from the dirty NIF. It also * restores the original NIF MFA when necessary based on the value of - * ep->fp set by execute_dirty_nif via init_nif_sched_data -- non-NULL + * ep->func set by execute_dirty_nif via init_nif_sched_data -- non-NULL * means restore, NULL means do not restore. */ static ERL_NIF_TERM @@ -2474,9 +2324,7 @@ dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep); - ASSERT(!ep->exception_thrown); - if (ep->fp) - restore_nif_mfa(proc, ep, 0); + nif_export_restore(proc, ep); return argv[0]; } @@ -2486,148 +2334,100 @@ dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) static ERL_NIF_TERM dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + ERL_NIF_TERM ret; Process* proc; NifExport* ep; + Eterm exception; execution_state(env, &proc, NULL); + ASSERT(argc == 1); ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep); - ASSERT(ep->exception_thrown); - if (ep->fp) - restore_nif_mfa(proc, ep, 1); - return enif_raise_exception(env, ep->rootset[0]); + exception = argv[0]; /* argv overwritten by restore below... */ + nif_export_cleanup_nif_mod(ep); + ret = enif_raise_exception(env, exception); + + /* Restore orig info for error and clear nif export in handle_error() */ + proc->freason |= EXF_RESTORE_NIF; + return ret; } /* - * Dirty NIF execution wrapper function. Invoke an application's dirty NIF, - * then check the result and schedule the appropriate finalizer function - * where needed. Also restore the original NIF MFA when appropriate. + * Dirty NIF scheduling wrapper function. Schedule a dirty NIF to execute. + * The dirty scheduler thread type (CPU or I/O) is indicated in flags + * parameter. */ -static ERL_NIF_TERM -execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +static ERTS_INLINE ERL_NIF_TERM +schedule_dirty_nif(ErlNifEnv* env, int flags, NativeFunPtr fp, + Eterm func_name, int argc, const ERL_NIF_TERM argv[]) { Process* proc; - NativeFunPtr fp; - NifExport* ep; - ERL_NIF_TERM result; - - execution_state(env, &proc, NULL); - ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); - fp = ep->fp; - - ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); - - /* - * Set ep->fp to NULL before the native call so we know later whether it scheduled another NIF for execution - */ - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - ASSERT(ep && fp); - - ep->fp = NULL; - erts_smp_atomic32_read_band_mb(&proc->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC - | ERTS_PSFLG_DIRTY_IO_PROC)); + ASSERT(is_atom(func_name)); + ASSERT(fp); - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); + ASSERT(flags==ERL_NIF_DIRTY_JOB_IO_BOUND || flags==ERL_NIF_DIRTY_JOB_CPU_BOUND); - result = (*fp)(env, argc, argv); + execution_state(env, &proc, NULL); - erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); + (void) erts_smp_atomic32_read_bset_nob(&proc->state, + (ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_IO_PROC), + (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND + ? ERTS_PSFLG_DIRTY_CPU_PROC + : ERTS_PSFLG_DIRTY_IO_PROC)); - if (erts_refc_dectest(&env->mod_nif->rt_dtor_cnt, 0) == 0 && env->mod_nif->mod == NULL) - close_lib(env->mod_nif); - /* - * If no more NIFs were scheduled by the native call via - * enif_schedule_nif(), then ep->fp will still be NULL as set above, in - * which case we need to restore the original NIF calling - * context. Reuse fp essentially as a boolean for this, passing it to - * init_nif_sched_data below. Both dirty_nif_exception and - * dirty_nif_finalizer then check ep->fp to decide whether or not to - * restore the original calling context. - */ - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - ASSERT(ep); - if (ep->fp) - fp = NULL; - if (is_non_value(result) || env->exception_thrown) { - if (proc->freason != TRAP) { - return init_nif_sched_data(env, dirty_nif_exception, fp, 0, argc, argv); - } else { - if (ep->fp == NULL) - restore_nif_mfa(proc, ep, 1); - return THE_NON_VALUE; - } - } - else - return init_nif_sched_data(env, dirty_nif_finalizer, fp, 0, 1, &result); + return schedule(env, fp, NULL, proc->current->module, func_name, argc, argv); } -/* - * Dirty NIF scheduling wrapper function. Schedule a dirty NIF to execute - * via the execute_dirty_nif() wrapper function. The dirty scheduler thread - * type (CPU or I/O) is indicated in flags parameter. - */ static ERTS_INLINE ERL_NIF_TERM -schedule_dirty_nif(ErlNifEnv* env, int flags, int argc, const ERL_NIF_TERM argv[]) +static_schedule_dirty_nif(ErlNifEnv* env, erts_aint32_t dirty_psflg, + int argc, const ERL_NIF_TERM argv[]) { - ERL_NIF_TERM result; - erts_aint32_t act, dirty_flag; - Process* proc; + Process *proc; + NifExport *ep; + Eterm mod, func; NativeFunPtr fp; - NifExport* ep; - int need_save, scheduler; - execution_state(env, &proc, &scheduler); - if (scheduler <= 0) { - ASSERT(scheduler < 0); - erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); - } + execution_state(env, &proc, NULL); + + /* + * Called in order to schedule statically determined + * dirty NIF calls... + * + * Note that 'current' does not point into a NifExport + * structure; only a structure with similar + * parts (located in code). + */ ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); - fp = ep->fp; + mod = proc->current->module; + func = proc->current->function; + fp = (NativeFunPtr) ep->func; + ASSERT(is_atom(mod) && is_atom(func)); ASSERT(fp); - ASSERT(flags==ERL_NIF_DIRTY_JOB_IO_BOUND || flags==ERL_NIF_DIRTY_JOB_CPU_BOUND); - - if (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND) - dirty_flag = ERTS_PSFLG_DIRTY_CPU_PROC; - else - dirty_flag = ERTS_PSFLG_DIRTY_IO_PROC; - - act = erts_smp_atomic32_read_bor_nob(&proc->state, dirty_flag); - if (!(act & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC))) - erts_refc_inc(&env->mod_nif->rt_dtor_cnt, 1); - else if ((act & (ERTS_PSFLG_DIRTY_CPU_PROC - | ERTS_PSFLG_DIRTY_IO_PROC)) & ~dirty_flag) { - /* clear other flag... */ - if (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND) - dirty_flag = ERTS_PSFLG_DIRTY_IO_PROC; - else - dirty_flag = ERTS_PSFLG_DIRTY_CPU_PROC; - erts_smp_atomic32_read_band_nob(&proc->state, ~dirty_flag); - } + (void) erts_smp_atomic32_read_bset_nob(&proc->state, + (ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_IO_PROC), + dirty_psflg); - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - need_save = (ep == NULL || !ep->saved_current); - result = init_nif_sched_data(env, execute_dirty_nif, fp, need_save, argc, argv); - if (scheduler <= 0) - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); - return result; + return schedule(env, fp, NULL, mod, func, argc, argv); } static ERL_NIF_TERM -schedule_dirty_io_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +static_schedule_dirty_io_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - return schedule_dirty_nif(env, ERL_NIF_DIRTY_JOB_IO_BOUND, argc, argv); + return static_schedule_dirty_nif(env, ERTS_PSFLG_DIRTY_IO_PROC, argc, argv); } static ERL_NIF_TERM -schedule_dirty_cpu_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +static_schedule_dirty_cpu_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - return schedule_dirty_nif(env, ERL_NIF_DIRTY_JOB_CPU_BOUND, argc, argv); + return static_schedule_dirty_nif(env, ERTS_PSFLG_DIRTY_CPU_PROC, argc, argv); } #endif /* ERTS_DIRTY_SCHEDULERS */ @@ -2646,23 +2446,42 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ERL_NIF_TERM result; execution_state(env, &proc, NULL); - ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); - fp = ep->fp; - ASSERT(!env->exception_thrown); - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); + fp = ep->func; ASSERT(ep); - ep->fp = NULL; + ASSERT(!env->exception_thrown); + + fp = (NativeFunPtr) ep->func; + +#ifdef DEBUG + ep->func = ERTS_DBG_NIF_NOT_SCHED_MARKER; +#endif + result = (*fp)(env, argc, argv); - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - ASSERT(ep); - /* - * If no NIFs were scheduled by the native call via - * enif_schedule_nif(), then ep->fp will still be NULL as set above, in - * which case we need to restore the original NIF MFA. - */ - if (ep->fp == NULL) - restore_nif_mfa(proc, ep, env->exception_thrown); + + ASSERT(ep == ERTS_PROC_GET_NIF_TRAP_EXPORT(proc)); + + if (is_value(result) || proc->freason != TRAP) { + /* Done (not rescheduled)... */ + ASSERT(ep->func == ERTS_DBG_NIF_NOT_SCHED_MARKER); + if (!env->exception_thrown) + nif_export_restore(proc, ep); + else { + nif_export_cleanup_nif_mod(ep); + /* + * Restore orig info for error and clear nif + * export in handle_error() + */ + proc->freason |= EXF_RESTORE_NIF; + } + } + +#ifdef DEBUG + if (ep->func == ERTS_DBG_NIF_NOT_SCHED_MARKER) + ep->func = NULL; +#endif + return result; } @@ -2672,9 +2491,8 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, int argc, const ERL_NIF_TERM argv[]) { Process* proc; - NifExport* ep; ERL_NIF_TERM fun_name_atom, result; - int need_save, scheduler; + int scheduler; if (argc > MAX_ARG) return enif_make_badarg(env); @@ -2689,35 +2507,16 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); } - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - need_save = (ep == NULL || !ep->saved_current); - - if (flags) { + if (flags == 0) + result = schedule(env, execute_nif, fp, proc->current->module, + fun_name_atom, argc, argv); #ifdef ERTS_DIRTY_SCHEDULERS - NativeFunPtr sched_fun; - int chkflgs = (flags & (ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND)); - if (chkflgs == ERL_NIF_DIRTY_JOB_IO_BOUND) - sched_fun = schedule_dirty_io_nif; - else if (chkflgs == ERL_NIF_DIRTY_JOB_CPU_BOUND) - sched_fun = schedule_dirty_cpu_nif; - else { - result = enif_make_badarg(env); - goto done; - } - result = init_nif_sched_data(env, sched_fun, fp, need_save, argc, argv); -#else - result = enif_make_badarg(env); + else if (!(flags & ~(ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND))) + result = schedule_dirty_nif(env, flags, fp, fun_name_atom, argc, argv); #endif - goto done; - } else - result = init_nif_sched_data(env, execute_nif, fp, need_save, argc, argv); - - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - ASSERT(ep); - ep->exp.info.mfa.function = (BeamInstr) fun_name_atom; + result = enif_make_badarg(env); -done: if (scheduler < 0) erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); @@ -3416,8 +3215,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) if (f->flags) { code_ptr[3] = (BeamInstr) f->fptr; code_ptr[1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ? - (BeamInstr) schedule_dirty_io_nif : - (BeamInstr) schedule_dirty_cpu_nif; + (BeamInstr) static_schedule_dirty_io_nif : + (BeamInstr) static_schedule_dirty_cpu_nif; } else #endif diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 6928b282ee..955b98b35b 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -48,6 +48,7 @@ #include "erl_bif_unique.h" #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" +#include "erl_nfunc_sched.h" #define ERTS_CHECK_TIME_REDS CONTEXT_REDS #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0) @@ -193,12 +194,6 @@ static ErtsAuxWorkData *aux_thread_aux_work_data; #define ERTS_SCHDLR_SSPND_CHNG_ONLN (((erts_aint32_t) 1) << 2) #define ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN (((erts_aint32_t) 1) << 3) -typedef enum { - ERTS_SCHED_NORMAL, - ERTS_SCHED_DIRTY_CPU, - ERTS_SCHED_DIRTY_IO -} ErtsSchedType; - typedef struct { int ongoing; ErtsProcList *blckrs; @@ -10102,9 +10097,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) goto sunlock_sched_out_proc; } - ASSERT((state & ERTS_PSFLG_DIRTY_ACTIVE_SYS) - || *p->i == (BeamInstr) em_call_nif); - ASSERT(rq == ERTS_DIRTY_CPU_RUNQ ? (state & (ERTS_PSFLG_DIRTY_CPU_PROC | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) @@ -11976,7 +11968,6 @@ delete_process(Process* p) ErtsPSD *psd; struct saved_calls *scb; process_breakpoint_time_t *pbt; - void *nif_export; VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id)); VERBOSE(DEBUG_SHCOPY, ("[pid=%T] delete process: %p %p %p %p\n", p->common.id, @@ -11993,9 +11984,7 @@ delete_process(Process* p) if (pbt) erts_free(ERTS_ALC_T_BPD, (void *) pbt); - nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL); - if (nif_export) - erts_destroy_nif_export(nif_export); + erts_destroy_nif_export(p); /* Cleanup psd */ @@ -13564,6 +13553,9 @@ erts_dbg_check_halloc_lock(Process *p) ErtsSchedulerData *esdp; if (ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)) return 1; + if ((p->static_flags & ERTS_STC_FLG_SHADOW_PROC) + && ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())) + return 1; if (p->common.id == ERTS_INVALID_PID) return 1; esdp = erts_proc_sched_data(p); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 7193b7d1db..58a50f8cf4 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -838,8 +838,8 @@ typedef struct { #define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN -#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ((ErtsProcLocks) 0) -#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ((ErtsProcLocks) 0) +#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN typedef struct { ErtsProcLocks get_locks; @@ -1582,10 +1582,6 @@ Uint64 erts_get_proc_interval(void); Uint64 erts_ensure_later_proc_interval(Uint64); Uint64 erts_step_proc_interval(void); -int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */ -void erts_destroy_nif_export(void *); /* see erl_nif.c */ -int erts_check_nif_export_in_area(Process *p, char *start, Uint size); - ErtsProcList *erts_proclist_create(Process *); ErtsProcList *erts_proclist_copy(ErtsProcList *); void erts_proclist_destroy(ErtsProcList *); diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h index e431c3051b..99a8ff6bad 100644 --- a/erts/emulator/beam/error.h +++ b/erts/emulator/beam/error.h @@ -38,15 +38,12 @@ * Some of these have convenient aliases, like BADARG and BADARITH. */ -/* - * Bits 0-1 index the 'exception class tag' table. - */ -#define EXC_CLASSBITS 3 -#define GET_EXC_CLASS(x) ((x) & EXC_CLASSBITS) - /* * Exception class tags (indices into the 'exception_tag' array) */ +#define EXTAG_OFFSET 0 +#define EXTAG_BITS 2 + #define EXTAG_ERROR 0 #define EXTAG_EXIT 1 #define EXTAG_THROWN 2 @@ -54,20 +51,31 @@ #define NUMBER_EXC_TAGS 3 /* The number of exception class tags */ /* - * Exit code flags (bits 2-7) + * Index to the 'exception class tag' table. + */ +#define EXC_CLASSBITS ((1<> 8) +#define EXC_OFFSET (EXF_OFFSET+EXF_BITS) +#define EXC_BITS 5 + +#define EXC_INDEXBITS (((1<<(EXC_BITS+EXC_OFFSET))-1) \ + & ~((1<<(EXC_OFFSET))-1)) + +#define GET_EXC_INDEX(x) (((x) & EXC_INDEXBITS) >> EXC_OFFSET) /* * Exit codes used for raising a fresh exception. The primary exceptions @@ -107,46 +120,46 @@ /* Error with given arglist term * (exit reason in p->fvalue) */ -#define EXC_NORMAL ((1 << 8) | EXC_EXIT) +#define EXC_NORMAL ((1 << EXC_OFFSET) | EXC_EXIT) /* Normal exit (reason 'normal') */ -#define EXC_INTERNAL_ERROR ((2 << 8) | EXC_ERROR | EXF_PANIC) +#define EXC_INTERNAL_ERROR ((2 << EXC_OFFSET) | EXC_ERROR | EXF_PANIC) /* Things that shouldn't happen */ -#define EXC_BADARG ((3 << 8) | EXC_ERROR) +#define EXC_BADARG ((3 << EXC_OFFSET) | EXC_ERROR) /* Bad argument to a BIF */ -#define EXC_BADARITH ((4 << 8) | EXC_ERROR) +#define EXC_BADARITH ((4 << EXC_OFFSET) | EXC_ERROR) /* Bad arithmetic */ -#define EXC_BADMATCH ((5 << 8) | EXC_ERROR) +#define EXC_BADMATCH ((5 << EXC_OFFSET) | EXC_ERROR) /* Bad match in function body */ -#define EXC_FUNCTION_CLAUSE ((6 << 8) | EXC_ERROR) +#define EXC_FUNCTION_CLAUSE ((6 << EXC_OFFSET) | EXC_ERROR) /* No matching function head */ -#define EXC_CASE_CLAUSE ((7 << 8) | EXC_ERROR) +#define EXC_CASE_CLAUSE ((7 << EXC_OFFSET) | EXC_ERROR) /* No matching case clause */ -#define EXC_IF_CLAUSE ((8 << 8) | EXC_ERROR) +#define EXC_IF_CLAUSE ((8 << EXC_OFFSET) | EXC_ERROR) /* No matching if clause */ -#define EXC_UNDEF ((9 << 8) | EXC_ERROR) +#define EXC_UNDEF ((9 << EXC_OFFSET) | EXC_ERROR) /* No farity that matches */ -#define EXC_BADFUN ((10 << 8) | EXC_ERROR) +#define EXC_BADFUN ((10 << EXC_OFFSET) | EXC_ERROR) /* Not an existing fun */ -#define EXC_BADARITY ((11 << 8) | EXC_ERROR) +#define EXC_BADARITY ((11 << EXC_OFFSET) | EXC_ERROR) /* Attempt to call fun with * wrong number of arguments. */ -#define EXC_TIMEOUT_VALUE ((12 << 8) | EXC_ERROR) +#define EXC_TIMEOUT_VALUE ((12 << EXC_OFFSET) | EXC_ERROR) /* Bad time out value */ -#define EXC_NOPROC ((13 << 8) | EXC_ERROR) +#define EXC_NOPROC ((13 << EXC_OFFSET) | EXC_ERROR) /* No process or port */ -#define EXC_NOTALIVE ((14 << 8) | EXC_ERROR) +#define EXC_NOTALIVE ((14 << EXC_OFFSET) | EXC_ERROR) /* Not distributed */ -#define EXC_SYSTEM_LIMIT ((15 << 8) | EXC_ERROR) +#define EXC_SYSTEM_LIMIT ((15 << EXC_OFFSET) | EXC_ERROR) /* Ran out of something */ -#define EXC_TRY_CLAUSE ((16 << 8) | EXC_ERROR) +#define EXC_TRY_CLAUSE ((16 << EXC_OFFSET) | EXC_ERROR) /* No matching try clause */ -#define EXC_NOTSUP ((17 << 8) | EXC_ERROR) +#define EXC_NOTSUP ((17 << EXC_OFFSET) | EXC_ERROR) /* Not supported */ -#define EXC_BADMAP ((18 << 8) | EXC_ERROR) +#define EXC_BADMAP ((18 << EXC_OFFSET) | EXC_ERROR) /* Bad map */ -#define EXC_BADKEY ((19 << 8) | EXC_ERROR) +#define EXC_BADKEY ((19 << EXC_OFFSET) | EXC_ERROR) /* Bad key in map */ #define NUMBER_EXIT_CODES 20 /* The number of exit code indices */ @@ -154,7 +167,7 @@ /* * Internal pseudo-error codes. */ -#define TRAP (1 << 8) /* BIF Trap to erlang code */ +#define TRAP (1 << EXC_OFFSET) /* BIF Trap to erlang code */ /* * Aliases for some common exit codes. @@ -201,6 +214,7 @@ struct StackTrace { BeamInstr* pc; ErtsCodeMFA* current; int depth; /* number of saved pointers in trace[] */ + ErtsCodeMFA mfa; /* in case we need to make a copy of mfa */ BeamInstr *trace[1]; /* varying size - must be last in struct */ }; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 2b2f3c5cdc..d5ca3b04eb 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -61,12 +61,6 @@ struct enif_environment_t /* ErlNifEnv */ extern void erts_pre_nif(struct enif_environment_t*, Process*, struct erl_module_nif*, Process* tracee); extern void erts_post_nif(struct enif_environment_t* env); -#ifdef ERTS_DIRTY_SCHEDULERS -extern void erts_pre_dirty_nif(ErtsSchedulerData *, - struct enif_environment_t*, Process*, - struct erl_module_nif*); -extern void erts_post_dirty_nif(struct enif_environment_t* env); -#endif extern Eterm erts_nif_taints(Process* p); extern void erts_print_nif_taints(fmtfn_t to, void* to_arg); void erts_unload_nif(struct erl_module_nif* nif); @@ -78,6 +72,12 @@ extern Eterm erts_nif_call_function(Process *p, Process *tracee, struct enif_func_t *, int argc, Eterm *argv); +#ifdef ERTS_DIRTY_SCHEDULERS +int erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, + BeamInstr *I, Eterm *reg); +#endif /* ERTS_DIRTY_SCHEDULERS */ + + /* Driver handle (wrapper for old plain handle) */ #define ERL_DE_OK 0 #define ERL_DE_UNLOAD 1 @@ -993,7 +993,7 @@ void erts_queue_monitor_message(Process *, Eterm, Eterm); void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, - Eterm (*bif)(Process*,Eterm*)); + Eterm (*bif)(Process*, Eterm*, BeamInstr*)); void erts_init_bif(void); Eterm erl_send(Process *p, Eterm to, Eterm msg); diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index ec502d5a78..3fa48da1ec 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -56,6 +56,8 @@ #ifdef HIPE # include "hipe_mode_switch.h" #endif +#define ERTS_WANT_NFUNC_SCHED_INTERNALS__ +#include "erl_nfunc_sched.h" #undef M_TRIM_THRESHOLD #undef M_TOP_PAD -- cgit v1.2.3