Merge branch 'maint'

* maint:
  Update etp-commands for dirty schedulers
  Fix scheduling of system tasks on processes executing dirty
  Fix call time tracing with dirty schedulers
  Fix send of exit signal to process executing dirty
  Fix dirty scheduler process priority
  Fix alloc-util hard-debug
  Silence debug warning when no beam jump table is used with dirty schedulers
  Fix check_process_code() when NifExport is in use
  Fix GC when NifExport is in use
  Fix saving of original arguments when rescheduling via NifExport

Conflicts:
	erts/emulator/beam/beam_bif_load.c
	erts/emulator/beam/erl_nif.c

10 files changed, 171 insertions, 76 deletions

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index e7301b1b4b..5c3565f498 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1093,6 +1093,11 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls)
 	|| ErtsInArea(rp->cp, mod_start, mod_size)) {
 	return am_true;
     }
+ 
+    *redsp += 1;
+
+    if (erts_check_nif_export_in_area(rp, mod_start, mod_size))
+	return am_true;
 
     *redsp += (STACK_START(rp) - rp->stop) / 32;
 
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 62464f3864..73158205b3 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -1526,7 +1526,7 @@ set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags,
 	ASSERT((bp->flags & ERTS_BPF_TIME_TRACE) == 0);
 	bdt = Alloc(sizeof(BpDataTime));
 	erts_refc_init(&bdt->refc, 1);
-	bdt->n = erts_no_schedulers;
+	bdt->n = erts_no_total_schedulers;
 	bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n));
 	for (i = 0; i < bdt->n; i++) {
 	    bp_hash_init(&(bdt->hash[i]), 32);
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index d32a6bcba5..224b46407d 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -80,7 +80,7 @@ typedef struct generic_bp {
 #define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2)
 
 #ifdef ERTS_SMP
-#define bp_sched2ix_proc(p) (erts_proc_sched_data(p)->no - 1)
+#define bp_sched2ix_proc(p) (erts_proc_sched_data(p)->thr_id - 1)
 #else
 #define bp_sched2ix_proc(p) (0)
 #endif
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index b24995ca3d..613bd5c14e 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -5333,7 +5333,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 
 	I = c_p->i;
 
-	ASSERT(BeamOp(op_call_nif) == (BeamInstr *) *I);
+	ASSERT(em_call_nif == (BeamInstr *) *I);
 
 	/*
 	 * Set fcalls even though we ignore it, so we don't
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index b4ee9cba33..61cce224dd 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -4197,9 +4197,9 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp)
 	    ASSERT(IS_LAST_BLK(blk));
 
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
-	    (*allctr->link_free_block)(allctr, blk, 0);
+	    (*allctr->link_free_block)(allctr, blk);
 	    HARD_CHECK_BLK_CARRIER(allctr, blk);
-	    (*allctr->unlink_free_block)(allctr, blk, 0);
+	    (*allctr->unlink_free_block)(allctr, blk);
 #endif
 	}
 #endif
@@ -6484,11 +6484,6 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
 
 	ASSERT(SBC2BLK(allctr, sbc) == iblk);
 	ASSERT(CARRIER_SZ(sbc) - SBC_HEADER_SIZE >= SBC_BLK_SZ(iblk));
-#if HAVE_ERTS_MSEG
-	if (IS_MSEG_CARRIER(sbc)) {
-	    ASSERT(CARRIER_SZ(sbc) % ERTS_SACRR_UNIT_SZ == 0);
-	}
-#endif
 	crr = sbc;
 	cl = &allctr->sbc_list;
     }
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 6c752b9741..28b2b02914 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2317,6 +2317,11 @@ move_msgq_to_heap(Process *p)
 static Uint
 setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
 {
+    /*
+     * NOTE!
+     *   Remember to update offset_rootset() when changing
+     *   this function.
+     */
     Roots* roots;
     Uint n;
 
@@ -2961,6 +2966,12 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size,
 	offset_heap_ptr(objv, nobj, offs, area, area_size);
     }
     offset_off_heap(p, offs, area, area_size);
+    if (ERTS_PROC_GET_NIF_TRAP_EXPORT(p)) {
+	Eterm* argv;
+	int argc;
+	if (erts_setup_nif_gc(p, &argv, &argc))
+	    offset_heap_ptr(argv, argc, offs, area, area_size);
+    }
 }
 
 static void
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 19adff610b..01169852b2 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2312,6 +2312,17 @@ erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj)
     return gc;
 }
 
+int
+erts_check_nif_export_in_area(Process *p, char *start, Uint size)
+{
+    NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(p);
+    if (!nep || !nep->saved_current)
+	return 0;
+    if (ErtsInArea(nep->saved_current, start, size))
+	return 1;
+    return 0;
+}
+
 /*
  * Allocate a NifExport and set it in proc specific data
  */
@@ -2363,6 +2374,7 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec
     Eterm* reg;
     NifExport* ep;
     int i, scheduler;
+    int orig_argc;
 
     execution_state(env, &proc, &scheduler);
 
@@ -2373,11 +2385,14 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec
 
     reg = erts_proc_sched_data(proc)->x_reg_array;
 
+    ASSERT(!need_save || proc->current);
+    orig_argc = need_save ? (int) proc->current->arity : 0;
+
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     if (!ep)
-	ep = allocate_nif_sched_data(proc, argc);
-    else if (need_save && ep->rootset_extra < argc) {
-	NifExport* new_ep = allocate_nif_sched_data(proc, argc);
+	ep = allocate_nif_sched_data(proc, orig_argc);
+    else if (need_save && ep->rootset_extra < orig_argc) {
+	NifExport* new_ep = allocate_nif_sched_data(proc, orig_argc);
 	destroy_nif_export(ep);
 	ep = new_ep;
     }
@@ -2390,16 +2405,14 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec
     }
     if (scheduler > 0)
 	ERTS_VBUMP_ALL_REDS(proc);
-    for (i = 0; i < argc; i++) {
-	if (need_save)
-	    ep->rootset[i+1] = reg[i];
-	reg[i] = (Eterm) argv[i];
-    }
     if (need_save) {
-	ASSERT(proc->current);
 	ep->saved_current = proc->current;
-	ep->saved_argc = argc;
+	ep->saved_argc = orig_argc;
+	for (i = 0; i < orig_argc; i++)
+	    ep->rootset[i+1] = reg[i];
     }
+    for (i = 0; i < argc; i++)
+	reg[i] = (Eterm) argv[i];
     proc->i = (BeamInstr*) ep->exp.addressv[0];
     ep->exp.info.mfa.module = proc->current->module;
     ep->exp.info.mfa.function = proc->current->function;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 22c6b3310a..4fb0f9e975 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -154,6 +154,7 @@ int ERTS_WRITE_UNLIKELY(erts_eager_check_io) = 1;
 int ERTS_WRITE_UNLIKELY(erts_sched_compact_load);
 int ERTS_WRITE_UNLIKELY(erts_sched_balance_util) = 0;
 Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers);
+Uint ERTS_WRITE_UNLIKELY(erts_no_total_schedulers);
 Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers) = 0;
 Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers) = 0;
 
@@ -5907,9 +5908,12 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 
     n = (int) no_schedulers;
     erts_no_schedulers = n;
+    erts_no_total_schedulers = n;
 #ifdef ERTS_DIRTY_SCHEDULERS
     erts_no_dirty_cpu_schedulers = no_dirty_cpu_schedulers;
+    erts_no_total_schedulers += no_dirty_cpu_schedulers;
     erts_no_dirty_io_schedulers = no_dirty_io_schedulers;
+    erts_no_total_schedulers += no_dirty_io_schedulers;
 #endif
 
     /* Create and initialize scheduler sleep info */
@@ -6270,7 +6274,11 @@ check_dirty_enqueue_in_prio_queue(Process *c_p,
 	return -1*queue;
     }
 
-    *newp |= ERTS_PSFLG_IN_RUNQ;
+    /*
+     * Enqueue using process struct.
+     */
+    *newp &= ~ERTS_PSFLGS_PRQ_PRIO_MASK;
+    *newp |= ERTS_PSFLG_IN_RUNQ | (aprio << ERTS_PSFLGS_PRQ_PRIO_OFFSET);
     return queue;
 }
 
@@ -6682,15 +6690,8 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
     erts_aint32_t fail_state, state, a, n, enq_prio;
     int enqueue; /* < 0 -> use proxy */
     unsigned int prof_runnable_procs;
-    int strict_fail_state;
 
     fail_state = *fail_state_p;
-    /*
-     * If fail state something other than just exiting process,
-     * ensure that the task wont be scheduled when the 
-     * receiver is in the failure state.
-     */
-    strict_fail_state = fail_state != ERTS_PSFLG_EXITING;
 
     res = 1; /* prepare for success */
     st->next = st->prev = st; /* Prep for empty prio queue */
@@ -6772,7 +6773,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
     /* Status lock prevents out of order "runnable proc" trace msgs */
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
 
-    if (!prof_runnable_procs && !strict_fail_state) {
+    if (!prof_runnable_procs) {
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 	locked = 0;
     }
@@ -6783,11 +6784,6 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
 	erts_aint32_t e;
 	n = e = a;
 
-	if (strict_fail_state && (a & fail_state)) {
-	    *fail_state_p = (a & fail_state);
-	    goto cleanup;
-	}
-
 	if (a & ERTS_PSFLG_FREE)
 	    goto cleanup; /* We don't want to schedule free processes... */
 
@@ -12404,6 +12400,8 @@ send_exit_signal(Process *c_p,		/* current process if and only
 	    else if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))) {
 		/* Process not running ... */
 		ErtsProcLocks need_locks = ~(*rp_locks) & ERTS_PROC_LOCKS_ALL;
+		ErlHeapFragment *bp = NULL;
+		Eterm rsn_cpy;
 		if (need_locks
 		    && erts_smp_proc_trylock(rp, need_locks) == EBUSY) {
 		    /* ... but we havn't got all locks on it ... */
@@ -12416,12 +12414,32 @@ send_exit_signal(Process *c_p,		/* current process if and only
 		}
                 /* ...and we have all locks on it... */
                 *rp_locks = ERTS_PROC_LOCKS_ALL;
-                set_proc_exiting(rp,
-                                 state,
-                                 (is_immed(rsn)
-                                  ? rsn
-                                  : copy_object(rsn, rp)),
-                                 NULL);
+
+		state = erts_smp_atomic32_read_nob(&rp->state);
+
+		if (is_immed(rsn))
+		    rsn_cpy = rsn;
+		else {
+		    Eterm *hp;
+		    ErlOffHeap *ohp;
+		    Uint rsn_sz = size_object(rsn);
+#ifdef ERTS_DIRTY_SCHEDULERS
+		    if (state & (ERTS_PSFLG_DIRTY_RUNNING
+				 | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
+			bp = new_message_buffer(rsn_sz);
+			ohp = &bp->off_heap;
+			hp = &bp->mem[0];
+		    }
+		    else
+#endif
+		    {
+			hp = HAlloc(rp, rsn_sz);
+			ohp = &rp->off_heap;
+		    }
+		    rsn_cpy = copy_struct(rsn, rsn_sz, &hp, ohp);
+		}
+
+		set_proc_exiting(rp, state, rsn_cpy, bp);
 	    }
 	    else { /* Process running... */
 
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 4d0093d0d8..1c4b9f149d 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -107,6 +107,7 @@ extern int erts_eager_check_io;
 extern int erts_sched_compact_load;
 extern int erts_sched_balance_util;
 extern Uint erts_no_schedulers;
+extern Uint erts_no_total_schedulers;
 #ifdef ERTS_DIRTY_SCHEDULERS
 extern Uint erts_no_dirty_cpu_schedulers;
 extern Uint erts_no_dirty_io_schedulers;
@@ -1583,6 +1584,7 @@ Uint64 erts_step_proc_interval(void);
 
 int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */
 void erts_destroy_nif_export(void *); /* see erl_nif.c */
+int erts_check_nif_export_in_area(Process *p, char *start, Uint size);
 
 ErtsProcList *erts_proclist_create(Process *);
 ErtsProcList *erts_proclist_copy(ErtsProcList *);
diff --git a/erts/etc/unix/etp-commands.in b/erts/etc/unix/etp-commands.in
index 0e7193daf9..0990fcf8a7 100644
--- a/erts/etc/unix/etp-commands.in
+++ b/erts/etc/unix/etp-commands.in
@@ -2456,46 +2456,36 @@ define etp-schedulers
   if (!erts_initialized)
     printf "No schedulers, since system isn't initialized!\n"
   else
+    set $sched_type = 0
     set $sched_ix = 0
     while $sched_ix < erts_no_schedulers
-      printf "--- Scheduler %d ---\n", $sched_ix+1
-      printf " IX: %d\n", $sched_ix
-      if (erts_aligned_scheduler_data[$sched_ix].esd.cpu_id < 0)
-        printf " CPU Binding: unbound\n"
-      else
-        printf " CPU Binding: %d\n", erts_aligned_scheduler_data[$sched_ix].esd.cpu_id
-      end
-      printf " Aux work Flags:"
-      set $aux_work_flags = *((Uint32 *) &erts_aligned_scheduler_data[$sched_ix].esd.ssi->aux_work)
-      etp-aux-work-flags $aux_work_flags
-      printf " Sleep Info Flags:"
-      set $ssi_flags = *((Uint32 *) &erts_aligned_scheduler_data[$sched_ix].esd.ssi->flags)
-      etp-ssi-flags $ssi_flags
-      printf " Pointer: (ErtsSchedulerData *) %p\n", &erts_aligned_scheduler_data[$sched_ix].esd
-      printf " - Run Queue -\n"
-      if (etp_smp_compiled)
-        set $runq = erts_aligned_scheduler_data[$sched_ix].esd.run_queue
-      else
-        set $runq = &erts_aligned_run_queues[0].runq
-      end
-      printf "  Length: total=%d", *((Uint32 *) &($runq->len))
-      printf ", max=%d", *((Uint32 *) &($runq->procs.prio_info[0].len))
-      printf ", high=%d", *((Uint32 *) &($runq->procs.prio_info[1].len))
-      printf ", normal=%d", *((Uint32 *) &($runq->procs.prio_info[2].len))
-      printf ", low=%d", *((Uint32 *) &($runq->procs.prio_info[3].len))
-      printf ", port=%d\n", *((Uint32 *) &($runq->ports.info.len))
-      if ($runq->misc.start)
-        printf "  Misc Jobs: yes\n"
-      else
-        printf "  Misc Jobs: no\n"
-      end
-      set $rq_flags = *((Uint32 *) &($runq->flags))
-      etp-rq-flags-int $rq_flags
-      printf "  Pointer: (ErtsRunQueue *) %p\n", $runq
-
+      etp-scheduler-info-internal
+      etp-run-queue-info-internal
       set $sched_ix++
     end
-    printf "-------------------\n",
+    printf "---------------------\n"
+    if (erts_no_dirty_cpu_schedulers)
+       printf "\n\n"
+       set $sched_type = 1
+       set $sched_ix = 0
+       while $sched_ix < erts_no_dirty_cpu_schedulers
+         etp-scheduler-info-internal
+         set $sched_ix++
+       end
+       etp-run-queue-info-internal
+       printf "---------------------\n"
+    end
+    if (erts_no_dirty_io_schedulers)
+       printf "\n\n"
+       set $sched_type = 2
+       set $sched_ix = 0
+       while $sched_ix < erts_no_dirty_io_schedulers
+         etp-scheduler-info-internal
+         set $sched_ix++
+       end
+       etp-run-queue-info-internal
+       printf "---------------------\n"
+    end
   end
 end
 
@@ -2507,6 +2497,67 @@ document etp-schedulers
 %---------------------------------------------------------------------------
 end
 
+define etp-scheduler-info-internal
+  if ($sched_type == 0)
+    printf "--- Scheduler %d ---\n", $sched_ix+1
+    set $sched_data=&erts_aligned_scheduler_data[$sched_ix].esd
+  else
+    if ($sched_type == 1)
+      printf "--- Dirty CPU Scheduler %d ---\n", $sched_ix+1
+      set $sched_data=&erts_aligned_dirty_cpu_scheduler_data[$sched_ix].esd
+    else
+      printf "--- Dirty I/O Scheduler %d ---\n", $sched_ix+1
+      set $sched_data=&erts_aligned_dirty_io_scheduler_data[$sched_ix].esd
+    end
+  end
+  printf " IX: %d\n", $sched_ix
+  if ($sched_data->cpu_id < 0)
+    printf " CPU Binding: unbound\n"
+  else
+    printf " CPU Binding: %d\n", $sched_data->cpu_id
+  end
+  printf " Aux work Flags:"
+  set $aux_work_flags = *((Uint32 *) &$sched_data->ssi->aux_work)
+  etp-aux-work-flags $aux_work_flags
+  printf " Sleep Info Flags:"
+  set $ssi_flags = *((Uint32 *) &$sched_data->ssi->flags)
+  etp-ssi-flags $ssi_flags
+  printf " Pointer: (ErtsSchedulerData *) %p\n", $sched_data
+end
+
+define etp-run-queue-info-internal
+  if ($sched_type == 0)
+    printf " - Run Queue -\n"
+    if (etp_smp_compiled)
+      set $runq = erts_aligned_scheduler_data[$sched_ix].esd.run_queue
+    else
+      set $runq = &erts_aligned_run_queues[0].runq
+    end
+  else
+    if ($sched_type == 1)
+      printf "\n--- Dirty CPU Run Queue ---\n"
+      set $runq = &erts_aligned_run_queues[-1].runq
+    else
+      printf "\n--- Dirty I/O Run Queue ---\n"
+      set $runq = &erts_aligned_run_queues[-2].runq
+    end
+  end
+  printf "  Length: total=%d", *((Uint32 *) &($runq->len))
+  printf ", max=%d", *((Uint32 *) &($runq->procs.prio_info[0].len))
+  printf ", high=%d", *((Uint32 *) &($runq->procs.prio_info[1].len))
+  printf ", normal=%d", *((Uint32 *) &($runq->procs.prio_info[2].len))
+  printf ", low=%d", *((Uint32 *) &($runq->procs.prio_info[3].len))
+  printf ", port=%d\n", *((Uint32 *) &($runq->ports.info.len))
+  if ($runq->misc.start)
+    printf "  Misc Jobs: yes\n"
+  else
+    printf "  Misc Jobs: no\n"
+  end
+  set $rq_flags = *((Uint32 *) &($runq->flags))
+  etp-rq-flags-int $rq_flags
+  printf "  Pointer: (ErtsRunQueue *) %p\n", $runq
+
+
 define etp-disasm-1
   set $code_ptr = ((BeamInstr*)$arg0)
   set $addr = *$code_ptr