Merge branch 'sverker/mbcs-pool-stats/ERL-88/OTP-14795'

* sverker/mbcs-pool-stats/ERL-88/OTP-14795:
  Fix erlang:memory
  Remove duplicate stats for 'fetch'
  Do not include new stats for 'allocator_sizes'
  Add more cpool stats
  erts: Add statistics for cpoll fetch attempts

4 files changed, 141 insertions, 26 deletions

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 230ca6ccbb..d8346c4239 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -3279,8 +3279,16 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 	ASSERT(crr->cpool.orig_allctr == allctr);
 	dl = dl->next;
 	exp = erts_smp_atomic_read_rb(&crr->allctr);
-	if ((exp & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL
-	    && erts_atomic_read_nob(&crr->cpool.max_size) >= size) {
+        if (erts_atomic_read_nob(&crr->cpool.max_size) < size) {
+            INC_CC(allctr->cpool.stat.skip_size);
+        }
+        else if ((exp & ERTS_CRR_ALCTR_FLG_MASK) != ERTS_CRR_ALCTR_FLG_IN_POOL) {
+            if (exp & ERTS_CRR_ALCTR_FLG_BUSY)
+                INC_CC(allctr->cpool.stat.skip_busy);
+            else
+                INC_CC(allctr->cpool.stat.skip_not_pooled);
+        }
+        else {
 	    /* Try to fetch it... */
 	    act = erts_smp_atomic_cmpxchg_mb(&crr->allctr,
 					     (erts_aint_t) allctr,
@@ -3306,6 +3314,7 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 	if (--i <= 0) {
 	    /* Move sentinel to continue next search from here */
 	    relink_edl_before(dl, &allctr->cpool.pooled_list);
+            INC_CC(allctr->cpool.stat.fail_pooled);
 	    return NULL;
 	}
     }
@@ -3326,22 +3335,27 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 	dl = dl->next;
 	exp = erts_smp_atomic_read_rb(&crr->allctr);
 	if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) {
-	    if (!(exp & ERTS_CRR_ALCTR_FLG_BUSY)
-		&& erts_atomic_read_nob(&crr->cpool.max_size) >= size) {
-		/* Try to fetch it... */
-		act = erts_smp_atomic_cmpxchg_mb(&crr->allctr,
-						 (erts_aint_t) allctr,
-						 exp);
-		if (act == exp) {
-		    cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr);
-		    unlink_abandoned_carrier(crr);
-
-		    /* Move sentinel to continue next search from here */
-		    relink_edl_before(dl, &allctr->cpool.traitor_list);
-		    return crr;
-		}
-		exp = act;
-	    }
+            if (erts_atomic_read_nob(&crr->cpool.max_size) < size) {
+                INC_CC(allctr->cpool.stat.skip_size);
+            }
+            else if (exp & ERTS_CRR_ALCTR_FLG_BUSY) {
+                INC_CC(allctr->cpool.stat.skip_busy);
+            }
+            else {
+                /* Try to fetch it... */
+                act = erts_smp_atomic_cmpxchg_mb(&crr->allctr,
+                                                 (erts_aint_t) allctr,
+                                                 exp);
+                if (act == exp) {
+                    cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr);
+                    unlink_abandoned_carrier(crr);
+
+                    /* Move sentinel to continue next search from here */
+                    relink_edl_before(dl, &allctr->cpool.traitor_list);
+                    return crr;
+                }
+                exp = act;
+            }
 	    if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) {
 		if (!cpool_entrance)
 		    cpool_entrance = &crr->cpool;
@@ -3351,13 +3365,18 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 		link_abandoned_carrier(&allctr->cpool.pooled_list, crr);
 	    }
 	}
+        else {
+            INC_CC(allctr->cpool.stat.skip_traitor);
+        }
 	if (--i <= i_stop) {
 	    /* Move sentinel to continue next search from here */
 	    relink_edl_before(dl, &allctr->cpool.traitor_list);
 	    if (i > 0)
 		break;
-	    else
+	    else {
+                INC_CC(allctr->cpool.stat.fail_traitor);
 		return NULL;
+            }
 	}
     }
 
@@ -3406,8 +3425,17 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 	}
 	crr = (Carrier_t *)(((char *)cpdp) - offsetof(Carrier_t, cpool));
 	exp = erts_smp_atomic_read_rb(&crr->allctr);
-	if (((exp & (ERTS_CRR_ALCTR_FLG_MASK)) == ERTS_CRR_ALCTR_FLG_IN_POOL)
-	    && (erts_atomic_read_nob(&cpdp->max_size) >= size)) {
+
+        if (erts_atomic_read_nob(&cpdp->max_size) < size) {
+            INC_CC(allctr->cpool.stat.skip_size);
+        }
+        else if ((exp & (ERTS_CRR_ALCTR_FLG_MASK)) != ERTS_CRR_ALCTR_FLG_IN_POOL) {
+            if (exp & ERTS_CRR_ALCTR_FLG_BUSY)
+                INC_CC(allctr->cpool.stat.skip_busy);
+            else
+                INC_CC(allctr->cpool.stat.skip_race);
+        }
+	else {
 	    erts_aint_t act;
 	    /* Try to fetch it... */
 	    act = erts_smp_atomic_cmpxchg_mb(&crr->allctr,
@@ -3421,8 +3449,10 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 		return crr;
 	    }
 	}
-	if (--i <= 0)
+	if (--i <= 0) {
+            INC_CC(allctr->cpool.stat.fail_shared);
 	    return NULL;
+        }
     }
 
 check_dc_list:
@@ -3445,10 +3475,15 @@ check_dc_list:
 	    return crr;
 	}
 	crr = crr->prev;
-	if (--i <= 0)
+	if (--i <= 0) {
+            INC_CC(allctr->cpool.stat.fail_pend_dealloc);
 	    return NULL;
+        }
     }
 
+    if (i != ERTS_ALC_CPOOL_MAX_FETCH_INSPECT)
+        INC_CC(allctr->cpool.stat.fail);
+
     return NULL;
 }
 
@@ -3801,6 +3836,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 	crr = cpool_fetch(allctr, blk_sz);
 	if (crr) {
 	    STAT_MBC_CPOOL_FETCH(allctr, crr);
+            INC_CC(allctr->cpool.stat.fetch);
 	    link_carrier(&allctr->mbc_list, crr);
 	    (*allctr->add_mbc)(allctr, crr);
 	    blk = (*allctr->get_free_block)(allctr, blk_sz, NULL, 0);
@@ -4252,6 +4288,17 @@ static struct {
     Eterm mbcs;
 #ifdef ERTS_SMP
     Eterm mbcs_pool;
+    Eterm fetch;
+    Eterm fail_pooled;
+    Eterm fail_traitor;
+    Eterm fail_shared;
+    Eterm fail_pend_dealloc;
+    Eterm fail;
+    Eterm skip_size;
+    Eterm skip_busy;
+    Eterm skip_not_pooled;
+    Eterm skip_traitor;
+    Eterm skip_race;
 #endif
     Eterm sbcs;
 
@@ -4342,6 +4389,17 @@ init_atoms(Allctr_t *allctr)
 	AM_INIT(mbcs);
 #ifdef ERTS_SMP
 	AM_INIT(mbcs_pool);
+	AM_INIT(fetch);
+        AM_INIT(fail_pooled);
+        AM_INIT(fail_traitor);
+        AM_INIT(fail_shared);
+        AM_INIT(fail_pend_dealloc);
+        AM_INIT(fail);
+        AM_INIT(skip_size);
+        AM_INIT(skip_busy);
+        AM_INIT(skip_not_pooled);
+        AM_INIT(skip_traitor);
+        AM_INIT(skip_race);
 #endif
 	AM_INIT(sbcs);
 
@@ -4627,9 +4685,56 @@ info_cpool(Allctr_t *allctr,
 
     if (hpp || szp) {
 	res = NIL;
+
+      if (!sz_only) {
+        add_3tup(hpp, szp, &res, am.fail_pooled,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail_pooled)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail_pooled)));
+
+        add_3tup(hpp, szp, &res, am.fail_traitor,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail_traitor)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail_traitor)));
+
+        add_3tup(hpp, szp, &res, am.fail_shared,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail_shared)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail_shared)));
+
+        add_3tup(hpp, szp, &res, am.fail_pend_dealloc,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail_pend_dealloc)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail_pend_dealloc)));
+
+        add_3tup(hpp, szp, &res, am.fail,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail)));
+
+        add_3tup(hpp, szp, &res, am.fetch,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fetch)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fetch)));
+
+        add_3tup(hpp, szp, &res, am.skip_size,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_size)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_size)));
+
+        add_3tup(hpp, szp, &res, am.skip_busy,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_busy)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_busy)));
+
+        add_3tup(hpp, szp, &res, am.skip_not_pooled,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_not_pooled)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_not_pooled)));
+
+        add_3tup(hpp, szp, &res, am.skip_traitor,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_traitor)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_traitor)));
+
+        add_3tup(hpp, szp, &res, am.skip_race,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_race)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_race)));
+
 	add_2tup(hpp, szp, &res,
 		 am.carriers_size,
 		 bld_unstable_uint(hpp, szp, csz));
+      }
 	if (!sz_only)
 	    add_2tup(hpp, szp, &res,
 		     am.carriers,
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 81180382af..e69b7744ce 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -574,6 +574,17 @@ struct Allctr_t_ {
 	    erts_atomic_t	no_blocks;
 	    erts_atomic_t	carriers_size;
 	    erts_atomic_t	no_carriers;
+            CallCounter_t       fail_pooled;
+            CallCounter_t       fail_traitor;
+            CallCounter_t       fail_shared;
+            CallCounter_t       fail_pend_dealloc;
+            CallCounter_t       fail;
+            CallCounter_t       fetch;
+	    CallCounter_t       skip_size;
+	    CallCounter_t       skip_busy;
+	    CallCounter_t       skip_not_pooled;
+	    CallCounter_t       skip_traitor;
+	    CallCounter_t       skip_race;
 	} stat;
     } cpool;
 #endif
diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam
index 72898e6d8a..ae3f11cd9d 100644
--- a/erts/preloaded/ebin/erlang.beam
+++ b/erts/preloaded/ebin/erlang.beam
diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl
index 8771089b65..3abb4538dc 100644
--- a/erts/preloaded/src/erlang.erl
+++ b/erts/preloaded/src/erlang.erl
@@ -3713,15 +3713,14 @@ memory_is_supported() ->
 
 get_blocks_size([{blocks_size, Sz, _, _} | Rest], Acc) ->
     get_blocks_size(Rest, Acc+Sz);
-get_blocks_size([{_, _, _, _} | Rest], Acc) ->
-    get_blocks_size(Rest, Acc);
 get_blocks_size([{blocks_size, Sz} | Rest], Acc) ->
     get_blocks_size(Rest, Acc+Sz);
-get_blocks_size([{_, _} | Rest], Acc) ->
+get_blocks_size([_ | Rest], Acc) ->
     get_blocks_size(Rest, Acc);
 get_blocks_size([], Acc) ->
     Acc.
 
+
 blocks_size([{Carriers, SizeList} | Rest], Acc) when Carriers == mbcs;
 						     Carriers == mbcs_pool;
 						     Carriers == sbcs ->