Merge branch 'rickard/proc-sched/OTP-9892'

* rickard/proc-sched/OTP-9892:
  Teach etp-commands to understand new emulator internal data structures
  Optimize process state changes
  Optimize process table access
  Implement possibility to use ordinary mutexes as process locks

Conflicts:
	erts/emulator/beam/erl_alloc.types

1 files changed, 173 insertions, 38 deletions

diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index c1be206b90..12fea779da 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -144,8 +144,8 @@ typedef struct ErtsXPortsList_ ErtsXPortsList;
 struct port {
     ErtsPortTaskSched sched;
     ErtsPortTaskHandle timeout_task;
-#ifdef ERTS_SMP
     erts_smp_atomic_t refc;
+#ifdef ERTS_SMP
     erts_smp_mtx_t *lock;
     ErtsXPortsList *xports;
     erts_smp_atomic_t run_queue;
@@ -198,6 +198,8 @@ erts_port_runq(Port *prt)
 #ifdef ERTS_SMP
     ErtsRunQueue *rq1, *rq2;
     rq1 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
+    if (!rq1)
+	return NULL;
     while (1) {
 	erts_smp_runq_lock(rq1);
 	rq2 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
@@ -205,6 +207,8 @@ erts_port_runq(Port *prt)
 	    return rq1;
 	erts_smp_runq_unlock(rq1);
 	rq1 = rq2;
+	if (!rq1)
+	    return NULL;
     }
 #else
     return ERTS_RUNQ_IX(0);
@@ -1237,28 +1241,29 @@ erts_smp_port_state_unlock(Port *prt)
 ERTS_GLB_INLINE int
 erts_smp_port_trylock(Port *prt)
 {
-#ifdef ERTS_SMP
     int res;
 
     ASSERT(erts_smp_atomic_read_nob(&prt->refc) > 0);
     erts_smp_atomic_inc_nob(&prt->refc);
+
+#ifdef ERTS_SMP
     res = erts_smp_mtx_trylock(prt->lock);
     if (res == EBUSY) {
 	erts_smp_atomic_dec_nob(&prt->refc);
     }
+#else
+    res = 0;
+#endif
 
     return res;
-#else /* !ERTS_SMP */
-    return 0;
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_port_lock(Port *prt)
 {
-#ifdef ERTS_SMP
     ASSERT(erts_smp_atomic_read_nob(&prt->refc) > 0);
     erts_smp_atomic_inc_nob(&prt->refc);
+#ifdef ERTS_SMP
     erts_smp_mtx_lock(prt->lock);
 #endif
 }
@@ -1266,14 +1271,14 @@ erts_smp_port_lock(Port *prt)
 ERTS_GLB_INLINE void
 erts_smp_port_unlock(Port *prt)
 {
-#ifdef ERTS_SMP
     erts_aint_t refc;
+#ifdef ERTS_SMP
     erts_smp_mtx_unlock(prt->lock);
+#endif
     refc = erts_smp_atomic_dec_read_nob(&prt->refc);
     ASSERT(refc >= 0);
     if (refc == 0)
 	erts_port_cleanup(prt);
-#endif
 }
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
@@ -1336,12 +1341,12 @@ erts_id2port_sflgs(Eterm id, Process *c_p, ErtsProcLocks c_p_locks, Uint32 sflgs
 	erts_smp_port_state_unlock(prt);
 	prt = NULL;
     }
-#ifdef ERTS_SMP
     else {
 	erts_smp_atomic_inc_nob(&prt->refc);
 	erts_smp_port_state_unlock(prt);
 
-	if (no_proc_locks)
+#ifdef ERTS_SMP
+ 	if (no_proc_locks)
 	    erts_smp_mtx_lock(prt->lock);
 	else if (erts_smp_mtx_trylock(prt->lock) == EBUSY) {
 	    /* Unlock process locks, and acquire locks in lock order... */
@@ -1357,21 +1362,17 @@ erts_id2port_sflgs(Eterm id, Process *c_p, ErtsProcLocks c_p_locks, Uint32 sflgs
 	    erts_smp_port_unlock(prt); /* Also decrements refc... */
 	    prt = NULL;
 	}
-    }
 #endif
 
+    }
+
     return prt;
 }
 
 ERTS_GLB_INLINE void
 erts_port_release(Port *prt)
 {
-#ifdef ERTS_SMP
     erts_smp_port_unlock(prt);
-#else
-    if (prt->status & ERTS_PORT_SFLGS_DEAD)
-	erts_port_cleanup(prt);
-#endif
 }
 
 ERTS_GLB_INLINE Port*
@@ -1488,6 +1489,122 @@ void erl_drv_thr_init(void);
 
 /* utils.c */
 
+typedef struct {
+#ifdef DEBUG
+    int smp_api;
+#endif
+    union {
+	Uint64 not_atomic;
+#ifdef ARCH_64
+	erts_atomic_t atomic;
+#else
+	erts_dw_atomic_t atomic;
+#endif
+    } counter;
+} erts_interval_t;
+
+void erts_interval_init(erts_interval_t *);
+void erts_smp_interval_init(erts_interval_t *);
+Uint64 erts_step_interval_nob(erts_interval_t *);
+Uint64 erts_step_interval_relb(erts_interval_t *);
+Uint64 erts_smp_step_interval_nob(erts_interval_t *);
+Uint64 erts_smp_step_interval_relb(erts_interval_t *);
+Uint64 erts_ensure_later_interval_nob(erts_interval_t *, Uint64);
+Uint64 erts_ensure_later_interval_acqb(erts_interval_t *, Uint64);
+Uint64 erts_smp_ensure_later_interval_nob(erts_interval_t *, Uint64);
+Uint64 erts_smp_ensure_later_interval_acqb(erts_interval_t *, Uint64);
+#ifdef ARCH_32
+ERTS_GLB_INLINE Uint64 erts_interval_dw_aint_to_val__(erts_dw_aint_t *);
+#endif
+ERTS_GLB_INLINE Uint64 erts_current_interval_nob__(erts_interval_t *);
+ERTS_GLB_INLINE Uint64 erts_current_interval_acqb__(erts_interval_t *);
+ERTS_GLB_INLINE Uint64 erts_current_interval_nob(erts_interval_t *);
+ERTS_GLB_INLINE Uint64 erts_current_interval_acqb(erts_interval_t *);
+ERTS_GLB_INLINE Uint64 erts_smp_current_interval_nob(erts_interval_t *);
+ERTS_GLB_INLINE Uint64 erts_smp_current_interval_acqb(erts_interval_t *);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+#ifdef ARCH_32
+
+ERTS_GLB_INLINE Uint64
+erts_interval_dw_aint_to_val__(erts_dw_aint_t *dw)
+{
+#ifdef ETHR_SU_DW_NAINT_T__
+    return (Uint64) dw->dw_sint;
+#else
+    Uint64 res;
+    res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]);
+    res <<= 32;
+    res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]);
+    return res;
+#endif
+}
+
+#endif
+
+ERTS_GLB_INLINE Uint64
+erts_current_interval_nob__(erts_interval_t *icp)
+{
+#ifdef ARCH_64
+    return (Uint64) erts_atomic_read_nob(&icp->counter.atomic);
+#else
+    erts_dw_aint_t dw;
+    erts_dw_atomic_read_nob(&icp->counter.atomic, &dw);
+    return erts_interval_dw_aint_to_val__(&dw);
+#endif
+}
+
+ERTS_GLB_INLINE Uint64
+erts_current_interval_acqb__(erts_interval_t *icp)
+{
+#ifdef ARCH_64
+    return (Uint64) erts_atomic_read_acqb(&icp->counter.atomic);
+#else
+    erts_dw_aint_t dw;
+    erts_dw_atomic_read_acqb(&icp->counter.atomic, &dw);
+    return erts_interval_dw_aint_to_val__(&dw);
+#endif
+}
+
+ERTS_GLB_INLINE Uint64
+erts_current_interval_nob(erts_interval_t *icp)
+{
+    ASSERT(!icp->smp_api);
+    return erts_current_interval_nob__(icp);
+}
+
+ERTS_GLB_INLINE Uint64
+erts_current_interval_acqb(erts_interval_t *icp)
+{
+    ASSERT(!icp->smp_api);
+    return erts_current_interval_acqb__(icp);
+}
+
+ERTS_GLB_INLINE Uint64
+erts_smp_current_interval_nob(erts_interval_t *icp)
+{
+    ASSERT(icp->smp_api);
+#ifdef ERTS_SMP
+    return erts_current_interval_nob__(icp);
+#else
+    return icp->counter.not_atomic;
+#endif
+}
+
+ERTS_GLB_INLINE Uint64
+erts_smp_current_interval_acqb(erts_interval_t *icp)
+{
+    ASSERT(icp->smp_api);
+#ifdef ERTS_SMP
+    return erts_current_interval_acqb__(icp);
+#else
+    return icp->counter.not_atomic;
+#endif
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
 /*
  * To be used to silence unused result warnings, but do not abuse it.
  */
@@ -1495,7 +1612,8 @@ void erts_silence_warn_unused_result(long unused);
 
 void erts_cleanup_offheap(ErlOffHeap *offheap);
 
-Uint erts_fit_in_bits(Uint);
+int erts_fit_in_bits_int64(Sint64);
+int erts_fit_in_bits_int32(Sint32);
 int list_length(Eterm);
 Export* erts_find_function(Eterm, Eterm, unsigned int, ErtsCodeIndex);
 int erts_is_builtin(Eterm, Eterm, int);
@@ -1836,6 +1954,13 @@ extern erts_driver_t spawn_driver;
 extern erts_driver_t fd_driver;
 
 /* Should maybe be placed in erl_message.h, but then we get an include mess. */
+ERTS_GLB_INLINE Eterm *
+erts_alloc_message_heap_state(Uint size,
+			      ErlHeapFragment **bpp,
+			      ErlOffHeap **ohpp,
+			      Process *receiver,
+			      ErtsProcLocks *receiver_locks,
+			      erts_aint32_t *statep);
 
 ERTS_GLB_INLINE Eterm *
 erts_alloc_message_heap(Uint size,
@@ -1858,16 +1983,22 @@ erts_alloc_message_heap(Uint size,
  */
 
 ERTS_GLB_INLINE Eterm *
-erts_alloc_message_heap(Uint size,
-			ErlHeapFragment **bpp,
-			ErlOffHeap **ohpp,
-			Process *receiver,
-			ErtsProcLocks *receiver_locks)
+erts_alloc_message_heap_state(Uint size,
+			      ErlHeapFragment **bpp,
+			      ErlOffHeap **ohpp,
+			      Process *receiver,
+			      ErtsProcLocks *receiver_locks,
+			      erts_aint32_t *statep)
 {
     Eterm *hp;
+    erts_aint32_t state;
 #ifdef ERTS_SMP
     int locked_main = 0;
-    ErtsProcLocks ulocks = *receiver_locks & ERTS_PROC_LOCKS_MSG_SEND;
+    state = erts_smp_atomic32_read_acqb(&receiver->state);
+    if (statep)
+	*statep = state;
+    if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
+	goto allocate_in_mbuf;
 #endif
 
     if (size > (Uint) INT_MAX)
@@ -1883,20 +2014,19 @@ erts_alloc_message_heap(Uint size,
 #ifdef ERTS_SMP
     try_allocate_on_heap:
 #endif
-	if (ERTS_PROC_IS_EXITING(receiver)
+	state = erts_smp_atomic32_read_nob(&receiver->state);
+	if (statep)
+	    *statep = state;
+	if ((state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
 	    || HEAP_LIMIT(receiver) - HEAP_TOP(receiver) <= size) {
 #ifdef ERTS_SMP
-	    if (locked_main)
-		ulocks |= ERTS_PROC_LOCK_MAIN;
+	    if (locked_main) {
+		*receiver_locks &= ~ERTS_PROC_LOCK_MAIN;
+		erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MAIN);
+	    }
 #endif
 	    goto allocate_in_mbuf;
 	}
-#ifdef ERTS_SMP
-	if (ulocks) {
-	    erts_smp_proc_unlock(receiver, ulocks);
-	    *receiver_locks &= ~ulocks;
-	}
-#endif
 	hp = HEAP_TOP(receiver);
 	HEAP_TOP(receiver) = hp + size;
 	*bpp = NULL;
@@ -1912,12 +2042,6 @@ erts_alloc_message_heap(Uint size,
     else {
 	ErlHeapFragment *bp;
     allocate_in_mbuf:
-#ifdef ERTS_SMP
-	if (ulocks) {
-	    *receiver_locks &= ~ulocks;
-	    erts_smp_proc_unlock(receiver, ulocks);
-	}
-#endif
 	bp = new_message_buffer(size);
 	hp = bp->mem;
 	*bpp = bp;
@@ -1927,6 +2051,17 @@ erts_alloc_message_heap(Uint size,
     return hp;
 }
 
+ERTS_GLB_INLINE Eterm *
+erts_alloc_message_heap(Uint size,
+			ErlHeapFragment **bpp,
+			ErlOffHeap **ohpp,
+			Process *receiver,
+			ErtsProcLocks *receiver_locks)
+{
+    return erts_alloc_message_heap_state(size, bpp, ohpp, receiver,
+					 receiver_locks, NULL);
+}
+
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 #if !HEAP_ON_C_STACK