Merge branch 'rickard/dirty-scheduling-fixes' into maint

OTP-14051

* rickard/dirty-scheduling-fixes:
  Update etp-commands for dirty schedulers
  Fix scheduling of system tasks on processes executing dirty
  Fix call time tracing with dirty schedulers
  Fix send of exit signal to process executing dirty
  Fix dirty scheduler process priority
  Fix alloc-util hard-debug
  Silence debug warning when no beam jump table is used with dirty schedulers

Conflicts:
	erts/etc/unix/etp-commands.in

7 files changed, 131 insertions, 66 deletions

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 920c8b1ed0..bbb2e4f34f 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -1517,7 +1517,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags,
 	ASSERT((bp->flags & ERTS_BPF_TIME_TRACE) == 0);
 	bdt = Alloc(sizeof(BpDataTime));
 	erts_refc_init(&bdt->refc, 1);
-	bdt->n = erts_no_schedulers;
+	bdt->n = erts_no_total_schedulers;
 	bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n));
 	for (i = 0; i < bdt->n; i++) {
 	    bp_hash_init(&(bdt->hash[i]), 32);
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index 541af77211..7206ef471a 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -80,7 +80,7 @@ typedef struct generic_bp {
 #define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2)
 
 #ifdef ERTS_SMP
-#define bp_sched2ix_proc(p) (erts_proc_sched_data(p)->no - 1)
+#define bp_sched2ix_proc(p) (erts_proc_sched_data(p)->thr_id - 1)
 #else
 #define bp_sched2ix_proc(p) (0)
 #endif
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 3be5c0d24c..59a9ea1417 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -5325,7 +5325,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 
 	I = c_p->i;
 
-	ASSERT(BeamOp(op_call_nif) == (BeamInstr *) *I);
+	ASSERT(em_call_nif == (BeamInstr *) *I);
 
 	/*
 	 * Set fcalls even though we ignore it, so we don't
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index f4ca689040..230ca6ccbb 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -4153,9 +4153,9 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp)
 	    ASSERT(IS_LAST_BLK(blk));
 
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
-	    (*allctr->link_free_block)(allctr, blk, 0);
+	    (*allctr->link_free_block)(allctr, blk);
 	    HARD_CHECK_BLK_CARRIER(allctr, blk);
-	    (*allctr->unlink_free_block)(allctr, blk, 0);
+	    (*allctr->unlink_free_block)(allctr, blk);
 #endif
 	}
 #endif
@@ -6440,11 +6440,6 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
 
 	ASSERT(SBC2BLK(allctr, sbc) == iblk);
 	ASSERT(CARRIER_SZ(sbc) - SBC_HEADER_SIZE >= SBC_BLK_SZ(iblk));
-#if HAVE_ERTS_MSEG
-	if (IS_MSEG_CARRIER(sbc)) {
-	    ASSERT(CARRIER_SZ(sbc) % ERTS_SACRR_UNIT_SZ == 0);
-	}
-#endif
 	crr = sbc;
 	cl = &allctr->sbc_list;
     }
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b989d3be7a..b345c35a7e 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -154,6 +154,7 @@ int ERTS_WRITE_UNLIKELY(erts_eager_check_io) = 1;
 int ERTS_WRITE_UNLIKELY(erts_sched_compact_load);
 int ERTS_WRITE_UNLIKELY(erts_sched_balance_util) = 0;
 Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers);
+Uint ERTS_WRITE_UNLIKELY(erts_no_total_schedulers);
 Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers) = 0;
 Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers) = 0;
 
@@ -5908,9 +5909,12 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 
     n = (int) no_schedulers;
     erts_no_schedulers = n;
+    erts_no_total_schedulers = n;
 #ifdef ERTS_DIRTY_SCHEDULERS
     erts_no_dirty_cpu_schedulers = no_dirty_cpu_schedulers;
+    erts_no_total_schedulers += no_dirty_cpu_schedulers;
     erts_no_dirty_io_schedulers = no_dirty_io_schedulers;
+    erts_no_total_schedulers += no_dirty_io_schedulers;
 #endif
 
     /* Create and initialize scheduler sleep info */
@@ -6271,7 +6275,11 @@ check_dirty_enqueue_in_prio_queue(Process *c_p,
 	return -1*queue;
     }
 
-    *newp |= ERTS_PSFLG_IN_RUNQ;
+    /*
+     * Enqueue using process struct.
+     */
+    *newp &= ~ERTS_PSFLGS_PRQ_PRIO_MASK;
+    *newp |= ERTS_PSFLG_IN_RUNQ | (aprio << ERTS_PSFLGS_PRQ_PRIO_OFFSET);
     return queue;
 }
 
@@ -6683,15 +6691,8 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
     erts_aint32_t fail_state, state, a, n, enq_prio;
     int enqueue; /* < 0 -> use proxy */
     unsigned int prof_runnable_procs;
-    int strict_fail_state;
 
     fail_state = *fail_state_p;
-    /*
-     * If fail state something other than just exiting process,
-     * ensure that the task wont be scheduled when the 
-     * receiver is in the failure state.
-     */
-    strict_fail_state = fail_state != ERTS_PSFLG_EXITING;
 
     res = 1; /* prepare for success */
     st->next = st->prev = st; /* Prep for empty prio queue */
@@ -6773,7 +6774,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
     /* Status lock prevents out of order "runnable proc" trace msgs */
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
 
-    if (!prof_runnable_procs && !strict_fail_state) {
+    if (!prof_runnable_procs) {
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 	locked = 0;
     }
@@ -6784,11 +6785,6 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
 	erts_aint32_t e;
 	n = e = a;
 
-	if (strict_fail_state && (a & fail_state)) {
-	    *fail_state_p = (a & fail_state);
-	    goto cleanup;
-	}
-
 	if (a & ERTS_PSFLG_FREE)
 	    goto cleanup; /* We don't want to schedule free processes... */
 
@@ -12401,6 +12397,8 @@ send_exit_signal(Process *c_p,		/* current process if and only
 	    else if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))) {
 		/* Process not running ... */
 		ErtsProcLocks need_locks = ~(*rp_locks) & ERTS_PROC_LOCKS_ALL;
+		ErlHeapFragment *bp = NULL;
+		Eterm rsn_cpy;
 		if (need_locks
 		    && erts_smp_proc_trylock(rp, need_locks) == EBUSY) {
 		    /* ... but we havn't got all locks on it ... */
@@ -12413,12 +12411,32 @@ send_exit_signal(Process *c_p,		/* current process if and only
 		}
                 /* ...and we have all locks on it... */
                 *rp_locks = ERTS_PROC_LOCKS_ALL;
-                set_proc_exiting(rp,
-                                 state,
-                                 (is_immed(rsn)
-                                  ? rsn
-                                  : copy_object(rsn, rp)),
-                                 NULL);
+
+		state = erts_smp_atomic32_read_nob(&rp->state);
+
+		if (is_immed(rsn))
+		    rsn_cpy = rsn;
+		else {
+		    Eterm *hp;
+		    ErlOffHeap *ohp;
+		    Uint rsn_sz = size_object(rsn);
+#ifdef ERTS_DIRTY_SCHEDULERS
+		    if (state & (ERTS_PSFLG_DIRTY_RUNNING
+				 | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
+			bp = new_message_buffer(rsn_sz);
+			ohp = &bp->off_heap;
+			hp = &bp->mem[0];
+		    }
+		    else
+#endif
+		    {
+			hp = HAlloc(rp, rsn_sz);
+			ohp = &rp->off_heap;
+		    }
+		    rsn_cpy = copy_struct(rsn, rsn_sz, &hp, ohp);
+		}
+
+		set_proc_exiting(rp, state, rsn_cpy, bp);
 	    }
 	    else { /* Process running... */
 
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index d7eff3605a..9f7084c127 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -111,6 +111,7 @@ extern int erts_eager_check_io;
 extern int erts_sched_compact_load;
 extern int erts_sched_balance_util;
 extern Uint erts_no_schedulers;
+extern Uint erts_no_total_schedulers;
 #ifdef ERTS_DIRTY_SCHEDULERS
 extern Uint erts_no_dirty_cpu_schedulers;
 extern Uint erts_no_dirty_io_schedulers;
diff --git a/erts/etc/unix/etp-commands.in b/erts/etc/unix/etp-commands.in
index 1a4f641301..fa68bd26ee 100644
--- a/erts/etc/unix/etp-commands.in
+++ b/erts/etc/unix/etp-commands.in
@@ -2443,46 +2443,36 @@ define etp-schedulers
   if (!erts_initialized)
     printf "No schedulers, since system isn't initialized!\n"
   else
+    set $sched_type = 0
     set $sched_ix = 0
     while $sched_ix < erts_no_schedulers
-      printf "--- Scheduler %d ---\n", $sched_ix+1
-      printf " IX: %d\n", $sched_ix
-      if (erts_aligned_scheduler_data[$sched_ix].esd.cpu_id < 0)
-        printf " CPU Binding: unbound\n"
-      else
-        printf " CPU Binding: %d\n", erts_aligned_scheduler_data[$sched_ix].esd.cpu_id
-      end
-      printf " Aux work Flags:"
-      set $aux_work_flags = *((Uint32 *) &erts_aligned_scheduler_data[$sched_ix].esd.ssi->aux_work)
-      etp-aux-work-flags $aux_work_flags
-      printf " Sleep Info Flags:"
-      set $ssi_flags = *((Uint32 *) &erts_aligned_scheduler_data[$sched_ix].esd.ssi->flags)
-      etp-ssi-flags $ssi_flags
-      printf " Pointer: (ErtsSchedulerData *) %p\n", &erts_aligned_scheduler_data[$sched_ix].esd
-      printf " - Run Queue -\n"
-      if (etp_smp_compiled)
-        set $runq = erts_aligned_scheduler_data[$sched_ix].esd.run_queue
-      else
-        set $runq = &erts_aligned_run_queues[0].runq
-      end
-      printf "  Length: total=%d", *((Uint32 *) &($runq->len))
-      printf ", max=%d", *((Uint32 *) &($runq->procs.prio_info[0].len))
-      printf ", high=%d", *((Uint32 *) &($runq->procs.prio_info[1].len))
-      printf ", normal=%d", *((Uint32 *) &($runq->procs.prio_info[2].len))
-      printf ", low=%d", *((Uint32 *) &($runq->procs.prio_info[3].len))
-      printf ", port=%d\n", *((Uint32 *) &($runq->ports.info.len))
-      if ($runq->misc.start)
-        printf "  Misc Jobs: yes\n"
-      else
-        printf "  Misc Jobs: no\n"
-      end
-      set $rq_flags = *((Uint32 *) &($runq->flags))
-      etp-rq-flags-int $rq_flags
-      printf "  Pointer: (ErtsRunQueue *) %p\n", $runq
-
+      etp-scheduler-info-internal
+      etp-run-queue-info-internal
       set $sched_ix++
     end
-    printf "-------------------\n",
+    printf "---------------------\n"
+    if (erts_no_dirty_cpu_schedulers)
+       printf "\n\n"
+       set $sched_type = 1
+       set $sched_ix = 0
+       while $sched_ix < erts_no_dirty_cpu_schedulers
+         etp-scheduler-info-internal
+         set $sched_ix++
+       end
+       etp-run-queue-info-internal
+       printf "---------------------\n"
+    end
+    if (erts_no_dirty_io_schedulers)
+       printf "\n\n"
+       set $sched_type = 2
+       set $sched_ix = 0
+       while $sched_ix < erts_no_dirty_io_schedulers
+         etp-scheduler-info-internal
+         set $sched_ix++
+       end
+       etp-run-queue-info-internal
+       printf "---------------------\n"
+    end
   end
 end
 
@@ -2494,6 +2484,67 @@ document etp-schedulers
 %---------------------------------------------------------------------------
 end
 
+define etp-scheduler-info-internal
+  if ($sched_type == 0)
+    printf "--- Scheduler %d ---\n", $sched_ix+1
+    set $sched_data=&erts_aligned_scheduler_data[$sched_ix].esd
+  else
+    if ($sched_type == 1)
+      printf "--- Dirty CPU Scheduler %d ---\n", $sched_ix+1
+      set $sched_data=&erts_aligned_dirty_cpu_scheduler_data[$sched_ix].esd
+    else
+      printf "--- Dirty I/O Scheduler %d ---\n", $sched_ix+1
+      set $sched_data=&erts_aligned_dirty_io_scheduler_data[$sched_ix].esd
+    end
+  end
+  printf " IX: %d\n", $sched_ix
+  if ($sched_data->cpu_id < 0)
+    printf " CPU Binding: unbound\n"
+  else
+    printf " CPU Binding: %d\n", $sched_data->cpu_id
+  end
+  printf " Aux work Flags:"
+  set $aux_work_flags = *((Uint32 *) &$sched_data->ssi->aux_work)
+  etp-aux-work-flags $aux_work_flags
+  printf " Sleep Info Flags:"
+  set $ssi_flags = *((Uint32 *) &$sched_data->ssi->flags)
+  etp-ssi-flags $ssi_flags
+  printf " Pointer: (ErtsSchedulerData *) %p\n", $sched_data
+end
+
+define etp-run-queue-info-internal
+  if ($sched_type == 0)
+    printf " - Run Queue -\n"
+    if (etp_smp_compiled)
+      set $runq = erts_aligned_scheduler_data[$sched_ix].esd.run_queue
+    else
+      set $runq = &erts_aligned_run_queues[0].runq
+    end
+  else
+    if ($sched_type == 1)
+      printf "\n--- Dirty CPU Run Queue ---\n"
+      set $runq = &erts_aligned_run_queues[-1].runq
+    else
+      printf "\n--- Dirty I/O Run Queue ---\n"
+      set $runq = &erts_aligned_run_queues[-2].runq
+    end
+  end
+  printf "  Length: total=%d", *((Uint32 *) &($runq->len))
+  printf ", max=%d", *((Uint32 *) &($runq->procs.prio_info[0].len))
+  printf ", high=%d", *((Uint32 *) &($runq->procs.prio_info[1].len))
+  printf ", normal=%d", *((Uint32 *) &($runq->procs.prio_info[2].len))
+  printf ", low=%d", *((Uint32 *) &($runq->procs.prio_info[3].len))
+  printf ", port=%d\n", *((Uint32 *) &($runq->ports.info.len))
+  if ($runq->misc.start)
+    printf "  Misc Jobs: yes\n"
+  else
+    printf "  Misc Jobs: no\n"
+  end
+  set $rq_flags = *((Uint32 *) &($runq->flags))
+  etp-rq-flags-int $rq_flags
+  printf "  Pointer: (ErtsRunQueue *) %p\n", $runq
+
+
 define etp-disasm-1
   set $code_ptr = ((BeamInstr*)$arg0)
   set $addr = *$code_ptr