From d82ed483ae6bae52745d2725017d77cb2c7810c7 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 22 Nov 2016 15:53:51 +0100 Subject: Fix saving of original arguments when rescheduling via NifExport --- erts/emulator/beam/erl_nif.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 3a547982da..55bd9f7455 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2360,6 +2360,7 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec Eterm* reg; NifExport* ep; int i, scheduler; + int orig_argc; execution_state(env, &proc, &scheduler); @@ -2370,11 +2371,14 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec reg = erts_proc_sched_data(proc)->x_reg_array; + ASSERT(!need_save || proc->current); + orig_argc = need_save ? (int) proc->current[2] : 0; + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); if (!ep) - ep = allocate_nif_sched_data(proc, argc); - else if (need_save && ep->rootset_extra < argc) { - NifExport* new_ep = allocate_nif_sched_data(proc, argc); + ep = allocate_nif_sched_data(proc, orig_argc); + else if (need_save && ep->rootset_extra < orig_argc) { + NifExport* new_ep = allocate_nif_sched_data(proc, orig_argc); destroy_nif_export(ep); ep = new_ep; } @@ -2387,16 +2391,14 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec } if (scheduler > 0) ERTS_VBUMP_ALL_REDS(proc); - for (i = 0; i < argc; i++) { - if (need_save) - ep->rootset[i+1] = reg[i]; - reg[i] = (Eterm) argv[i]; - } if (need_save) { - ASSERT(proc->current); ep->saved_current = proc->current; - ep->saved_argc = argc; + ep->saved_argc = orig_argc; + for (i = 0; i < orig_argc; i++) + ep->rootset[i+1] = reg[i]; } + for (i = 0; i < argc; i++) + reg[i] = (Eterm) argv[i]; proc->i = (BeamInstr*) ep->exp.addressv[0]; ep->exp.code[0] = (BeamInstr) proc->current[0]; ep->exp.code[1] = (BeamInstr) proc->current[1]; -- cgit v1.2.3 From 7a6973a2fa238468bb47d175908e99d230fbc0b1 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 22 Nov 2016 16:00:25 +0100 Subject: Fix GC when NifExport is in use --- erts/emulator/beam/erl_gc.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 6f641a1ea7..af799d09da 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -2325,6 +2325,11 @@ move_msgq_to_heap(Process *p) static Uint setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) { + /* + * NOTE! + * Remember to update offset_rootset() when changing + * this function. + */ Roots* roots; Uint n; @@ -2969,6 +2974,12 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size, offset_heap_ptr(objv, nobj, offs, area, area_size); } offset_off_heap(p, offs, area, area_size); + if (ERTS_PROC_GET_NIF_TRAP_EXPORT(p)) { + Eterm* argv; + int argc; + if (erts_setup_nif_gc(p, &argv, &argc)) + offset_heap_ptr(argv, argc, offs, area, area_size); + } } static void -- cgit v1.2.3 From ad94a4bdb24c93d3cafc7351c4c98f346a5f53cc Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 22 Nov 2016 16:14:57 +0100 Subject: Fix check_process_code() when NifExport is in use --- erts/emulator/beam/beam_bif_load.c | 11 +++++++++++ erts/emulator/beam/erl_nif.c | 11 +++++++++++ erts/emulator/beam/erl_process.h | 1 + 3 files changed, 23 insertions(+) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 5969197168..74a333ecbe 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1086,6 +1086,11 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls || ErtsInArea(rp->cp, mod_start, mod_size)) { return am_true; } + + *redsp += 1; + + if (erts_check_nif_export_in_area(rp, mod_start, mod_size)) + return am_true; *redsp += (STACK_START(rp) - rp->stop) / 32; @@ -1161,6 +1166,12 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls || ErtsInArea(rp->cp, mod_start, mod_size)) { return am_true; } + + *redsp += 1; + + if (erts_check_nif_export_in_area(rp, mod_start, mod_size)) + return am_true; + /* * Check all continuation pointers stored on the stack. diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 55bd9f7455..e2f208badf 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2309,6 +2309,17 @@ erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj) return gc; } +int +erts_check_nif_export_in_area(Process *p, char *start, Uint size) +{ + NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(p); + if (!nep || !nep->saved_current) + return 0; + if (ErtsInArea(nep->saved_current, start, size)) + return 1; + return 0; +} + /* * Allocate a NifExport and set it in proc specific data */ diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 3347a7a60e..435b53fb5f 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1586,6 +1586,7 @@ Uint64 erts_step_proc_interval(void); int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */ void erts_destroy_nif_export(void *); /* see erl_nif.c */ +int erts_check_nif_export_in_area(Process *p, char *start, Uint size); ErtsProcList *erts_proclist_create(Process *); ErtsProcList *erts_proclist_copy(ErtsProcList *); -- cgit v1.2.3 From 80490fe3e8f636f4de3af1d71cdb41721e88ac7c Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 22 Nov 2016 16:19:55 +0100 Subject: Silence debug warning when no beam jump table is used with dirty schedulers --- erts/emulator/beam/beam_emu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index ef4cdf9d5a..c2000929d5 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -5325,7 +5325,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) I = c_p->i; - ASSERT(BeamOp(op_call_nif) == (BeamInstr *) *I); + ASSERT(em_call_nif == (BeamInstr *) *I); /* * Set fcalls even though we ignore it, so we don't -- cgit v1.2.3 From 696167d5e64365c28d626e117c136cb81e4c4028 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 21 Oct 2016 14:16:24 +0200 Subject: Fix alloc-util hard-debug --- erts/emulator/beam/erl_alloc_util.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 2995f2f822..7abe3846be 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -4153,9 +4153,9 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) ASSERT(IS_LAST_BLK(blk)); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - (*allctr->link_free_block)(allctr, blk, 0); + (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, blk, 0); + (*allctr->unlink_free_block)(allctr, blk); #endif } #endif @@ -6440,11 +6440,6 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) ASSERT(SBC2BLK(allctr, sbc) == iblk); ASSERT(CARRIER_SZ(sbc) - SBC_HEADER_SIZE >= SBC_BLK_SZ(iblk)); -#if HAVE_ERTS_MSEG - if (IS_MSEG_CARRIER(sbc)) { - ASSERT(CARRIER_SZ(sbc) % ERTS_SACRR_UNIT_SZ == 0); - } -#endif crr = sbc; cl = &allctr->sbc_list; } -- cgit v1.2.3 From a532df810da29fcb28d142d244b6b3c812fa33ca Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 17 Oct 2016 12:00:32 +0200 Subject: Fix dirty scheduler process priority --- erts/emulator/beam/erl_process.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index bc59147c6c..59ee84d115 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -6271,7 +6271,11 @@ check_dirty_enqueue_in_prio_queue(Process *c_p, return -1*queue; } - *newp |= ERTS_PSFLG_IN_RUNQ; + /* + * Enqueue using process struct. + */ + *newp &= ~ERTS_PSFLGS_PRQ_PRIO_MASK; + *newp |= ERTS_PSFLG_IN_RUNQ | (aprio << ERTS_PSFLGS_PRQ_PRIO_OFFSET); return queue; } -- cgit v1.2.3 From dd2e99bdd726d4322c8e07c81731ad66ae05176e Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 21 Oct 2016 14:05:11 +0200 Subject: Fix send of exit signal to process executing dirty --- erts/emulator/beam/erl_process.c | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 59ee84d115..86a997de65 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -12405,6 +12405,8 @@ send_exit_signal(Process *c_p, /* current process if and only else if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))) { /* Process not running ... */ ErtsProcLocks need_locks = ~(*rp_locks) & ERTS_PROC_LOCKS_ALL; + ErlHeapFragment *bp = NULL; + Eterm rsn_cpy; if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { /* ... but we havn't got all locks on it ... */ @@ -12417,12 +12419,32 @@ send_exit_signal(Process *c_p, /* current process if and only } /* ...and we have all locks on it... */ *rp_locks = ERTS_PROC_LOCKS_ALL; - set_proc_exiting(rp, - state, - (is_immed(rsn) - ? rsn - : copy_object(rsn, rp)), - NULL); + + state = erts_smp_atomic32_read_nob(&rp->state); + + if (is_immed(rsn)) + rsn_cpy = rsn; + else { + Eterm *hp; + ErlOffHeap *ohp; + Uint rsn_sz = size_object(rsn); +#ifdef ERTS_DIRTY_SCHEDULERS + if (state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + bp = new_message_buffer(rsn_sz); + ohp = &bp->off_heap; + hp = &bp->mem[0]; + } + else +#endif + { + hp = HAlloc(rp, rsn_sz); + ohp = &rp->off_heap; + } + rsn_cpy = copy_struct(rsn, rsn_sz, &hp, ohp); + } + + set_proc_exiting(rp, state, rsn_cpy, bp); } else { /* Process running... */ -- cgit v1.2.3 From e9ac0c64a78c858d474757d435f3c7dce2a224b1 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 1 Nov 2016 16:08:21 +0100 Subject: Fix call time tracing with dirty schedulers --- erts/emulator/beam/beam_bp.c | 2 +- erts/emulator/beam/beam_bp.h | 2 +- erts/emulator/beam/erl_process.c | 4 ++++ erts/emulator/beam/erl_process.h | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 920c8b1ed0..bbb2e4f34f 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -1517,7 +1517,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, ASSERT((bp->flags & ERTS_BPF_TIME_TRACE) == 0); bdt = Alloc(sizeof(BpDataTime)); erts_refc_init(&bdt->refc, 1); - bdt->n = erts_no_schedulers; + bdt->n = erts_no_total_schedulers; bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n)); for (i = 0; i < bdt->n; i++) { bp_hash_init(&(bdt->hash[i]), 32); diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 541af77211..7206ef471a 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -80,7 +80,7 @@ typedef struct generic_bp { #define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2) #ifdef ERTS_SMP -#define bp_sched2ix_proc(p) (erts_proc_sched_data(p)->no - 1) +#define bp_sched2ix_proc(p) (erts_proc_sched_data(p)->thr_id - 1) #else #define bp_sched2ix_proc(p) (0) #endif diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 86a997de65..adf7779fe5 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -154,6 +154,7 @@ int ERTS_WRITE_UNLIKELY(erts_eager_check_io) = 1; int ERTS_WRITE_UNLIKELY(erts_sched_compact_load); int ERTS_WRITE_UNLIKELY(erts_sched_balance_util) = 0; Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers); +Uint ERTS_WRITE_UNLIKELY(erts_no_total_schedulers); Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers) = 0; Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers) = 0; @@ -5908,9 +5909,12 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online n = (int) no_schedulers; erts_no_schedulers = n; + erts_no_total_schedulers = n; #ifdef ERTS_DIRTY_SCHEDULERS erts_no_dirty_cpu_schedulers = no_dirty_cpu_schedulers; + erts_no_total_schedulers += no_dirty_cpu_schedulers; erts_no_dirty_io_schedulers = no_dirty_io_schedulers; + erts_no_total_schedulers += no_dirty_io_schedulers; #endif /* Create and initialize scheduler sleep info */ diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 435b53fb5f..9214d05456 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -111,6 +111,7 @@ extern int erts_eager_check_io; extern int erts_sched_compact_load; extern int erts_sched_balance_util; extern Uint erts_no_schedulers; +extern Uint erts_no_total_schedulers; #ifdef ERTS_DIRTY_SCHEDULERS extern Uint erts_no_dirty_cpu_schedulers; extern Uint erts_no_dirty_io_schedulers; -- cgit v1.2.3 From ea15a660908c48a2b8cefe0ed90fc5c4a89fe572 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 22 Nov 2016 15:31:44 +0100 Subject: Fix scheduling of system tasks on processes executing dirty --- erts/emulator/beam/erl_process.c | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index adf7779fe5..92b3d9d1a8 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -6691,15 +6691,8 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, erts_aint32_t fail_state, state, a, n, enq_prio; int enqueue; /* < 0 -> use proxy */ unsigned int prof_runnable_procs; - int strict_fail_state; fail_state = *fail_state_p; - /* - * If fail state something other than just exiting process, - * ensure that the task wont be scheduled when the - * receiver is in the failure state. - */ - strict_fail_state = fail_state != ERTS_PSFLG_EXITING; res = 1; /* prepare for success */ st->next = st->prev = st; /* Prep for empty prio queue */ @@ -6781,7 +6774,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, /* Status lock prevents out of order "runnable proc" trace msgs */ ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - if (!prof_runnable_procs && !strict_fail_state) { + if (!prof_runnable_procs) { erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); locked = 0; } @@ -6792,11 +6785,6 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, erts_aint32_t e; n = e = a; - if (strict_fail_state && (a & fail_state)) { - *fail_state_p = (a & fail_state); - goto cleanup; - } - if (a & ERTS_PSFLG_FREE) goto cleanup; /* We don't want to schedule free processes... */ -- cgit v1.2.3