erts: Refactor carrier dealloc migration

by adding a dedicated 'homecoming_dd_block' to use in dd-queue.

+ Preparation for dd-queue-migration of non-empty carriers.

+ Get rid of ugly hack where blk->carrier pointer is overwritten
  by dd-queue and then have to be restored.

2 files changed, 69 insertions, 66 deletions

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 230ca6ccbb..f0597e778a 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -2083,27 +2083,14 @@ handle_delayed_dealloc(Allctr_t *allctr,
 	res = 1;
 
 	blk = UMEM2BLK(ptr);
-	if (IS_FREE_LAST_MBC_BLK(blk)) {
+	if (blk->bhdr == HOMECOMING_MBC_BLK_HDR) {
 	    /*
 	     * A multiblock carrier that previously has been migrated away
 	     * from us and now is back to be deallocated. For more info
 	     * see schedule_dealloc_carrier().
-	     *
-	     * Note that we cannot use FBLK_TO_MBC(blk) since it
-	     * data has been overwritten by the queue.
 	     */
-	    Carrier_t *crr = FIRST_BLK_TO_MBC(allctr, blk);
-
-	     /* Restore word overwritten by the dd-queue as it will be read
-	      * if this carrier is pulled from dc_list by cpool_fetch()
-	      */
-	    ERTS_ALC_CPOOL_ASSERT(FBLK_TO_MBC(blk) != crr);
-	    ERTS_CT_ASSERT(sizeof(ErtsAllctrDDBlock_t) == sizeof(void*));
-#ifdef MBC_ABLK_OFFSET_BITS
-	    blk->u.carrier = crr;
-#else
-	    blk->carrier = crr;
-#endif
+	    Carrier_t *crr = ErtsContainerStruct(blk, Carrier_t,
+                                                 cpool.homecoming_dd.blk);
 
 	    ERTS_ALC_CPOOL_ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr));
 	    ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr);
@@ -2116,6 +2103,7 @@ handle_delayed_dealloc(Allctr_t *allctr,
 	    schedule_dealloc_carrier(allctr, crr);
 	}
 	else {
+            ASSERT(IS_SBC_BLK(blk) || !IS_FREE_BLK(blk));
 
 	    INC_CC(allctr->calls.this_free);
 
@@ -3543,6 +3531,8 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr)
 			      == (erts_smp_atomic_read_nob(&crr->allctr)
 				  & ~ERTS_CRR_ALCTR_FLG_MASK));
 
+        blk = &crr->cpool.homecoming_dd.blk;
+        ERTS_ALC_CPOOL_ASSERT(blk->bhdr == HOMECOMING_MBC_BLK_HDR);
 	if (ddq_enqueue(&orig_allctr->dd.q, BLK2UMEM(blk), cinit))
 	    erts_alloc_notify_delayed_dealloc(orig_allctr->ix);
 	return;
@@ -3581,6 +3571,7 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr)
 static ERTS_INLINE void
 cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr)
 {
+    crr->cpool.homecoming_dd.blk.bhdr = HOMECOMING_MBC_BLK_HDR;
     erts_atomic_init_nob(&crr->cpool.next, ERTS_AINT_NULL);
     erts_atomic_init_nob(&crr->cpool.prev, ERTS_AINT_NULL);
     crr->cpool.orig_allctr = allctr;
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 81180382af..1291a6c10e 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -305,45 +305,7 @@ void  erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int supe
 
 typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t;
 
-#ifdef ERTS_SMP
-
-typedef struct ErtsDoubleLink_t_ {
-    struct ErtsDoubleLink_t_ *next;
-    struct ErtsDoubleLink_t_ *prev;
-}ErtsDoubleLink_t;
-
-typedef struct {
-    erts_atomic_t next;
-    erts_atomic_t prev;
-    Allctr_t *orig_allctr;      /* read-only while carrier is alive */
-    ErtsThrPrgrVal thr_prgr;
-    erts_atomic_t max_size;
-    UWord abandon_limit;
-    UWord blocks;
-    UWord blocks_size;
-    ErtsDoubleLink_t abandoned; /* node in pooled_list or traitor_list */
-} ErtsAlcCPoolData_t;
-
-#endif
-
 typedef struct Carrier_t_ Carrier_t;
-struct Carrier_t_ {
-    UWord chdr;
-    Carrier_t *next;
-    Carrier_t *prev;
-    erts_smp_atomic_t allctr;
-#ifdef ERTS_SMP
-    ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */
-#endif
-};
-
-#define ERTS_ALC_CARRIER_TO_ALLCTR(C) \
-  ((Allctr_t *) (erts_smp_atomic_read_nob(&(C)->allctr) & ~FLG_MASK))
-
-typedef struct {
-    Carrier_t *first;
-    Carrier_t *last;
-} CarrierList_t;
 
 typedef struct {
     UWord bhdr;
@@ -357,6 +319,22 @@ typedef struct {
 #endif
 } Block_t;
 
+typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t;
+
+union ErtsAllctrDDBlock_t_ {
+    erts_atomic_t atmc_next;
+    ErtsAllctrDDBlock_t *ptr_next;
+};
+
+typedef struct {
+    Block_t blk;
+#if !MBC_ABLK_OFFSET_BITS
+    ErtsAllctrDDBlock_t umem_;
+#endif
+} ErtsFakeDDBlock_t;
+
+
+
 #define THIS_FREE_BLK_HDR_FLG 	(((UWord) 1) << 0)
 #define PREV_FREE_BLK_HDR_FLG 	(((UWord) 1) << 1)
 #define LAST_BLK_HDR_FLG 	(((UWord) 1) << 2)
@@ -365,14 +343,13 @@ typedef struct {
     (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)
 
 /*
- * FREE_LAST_MBC_BLK_HDR_FLGS is a special flag combo used for
- * distinguishing empty mbc's from allocated blocks in
- * handle_delayed_dealloc().
+ * HOMECOMING_MBC_BLK_HDR is a special block header combo used for
+ * distinguishing MBC's from allocated blocks in handle_delayed_dealloc().
  */
-#define FREE_LAST_MBC_BLK_HDR_FLGS (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)
+#define HOMECOMING_MBC_BLK_HDR (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)
 
 #define IS_FREE_LAST_MBC_BLK(B) \
-    (((B)->bhdr & FLG_MASK) == FREE_LAST_MBC_BLK_HDR_FLGS)
+    (((B)->bhdr & FLG_MASK) == (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG))
 
 #define IS_SBC_BLK(B) (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG)
 #define IS_MBC_BLK(B) (!IS_SBC_BLK((B)))
@@ -396,6 +373,48 @@ typedef struct {
 
 typedef UWord FreeBlkFtr_t; /* Footer of a free block */
 
+
+#ifdef ERTS_SMP
+
+typedef struct ErtsDoubleLink_t_ {
+    struct ErtsDoubleLink_t_ *next;
+    struct ErtsDoubleLink_t_ *prev;
+}ErtsDoubleLink_t;
+
+typedef struct {
+    ErtsFakeDDBlock_t homecoming_dd;
+    erts_atomic_t next;
+    erts_atomic_t prev;
+    Allctr_t *orig_allctr;      /* read-only while carrier is alive */
+    ErtsThrPrgrVal thr_prgr;
+    erts_atomic_t max_size;
+    UWord abandon_limit;
+    UWord blocks;
+    UWord blocks_size;
+    ErtsDoubleLink_t abandoned; /* node in pooled_list or traitor_list */
+} ErtsAlcCPoolData_t;
+
+#endif
+
+struct Carrier_t_ {
+    UWord chdr;
+    Carrier_t *next;
+    Carrier_t *prev;
+    erts_smp_atomic_t allctr;
+#ifdef ERTS_SMP
+    ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */
+#endif
+};
+
+#define ERTS_ALC_CARRIER_TO_ALLCTR(C) \
+  ((Allctr_t *) (erts_smp_atomic_read_nob(&(C)->allctr) & ~FLG_MASK))
+
+typedef struct {
+    Carrier_t *first;
+    Carrier_t *last;
+} CarrierList_t;
+
+
 typedef Uint64 CallCounter_t;
 
 typedef struct {
@@ -433,13 +452,6 @@ typedef struct {
 
 #ifdef ERTS_SMP
 
-typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t;
-
-union ErtsAllctrDDBlock_t_ {
-    erts_atomic_t atmc_next;
-    ErtsAllctrDDBlock_t *ptr_next;
-};
-
 typedef struct {
     ErtsAllctrDDBlock_t marker;
     erts_atomic_t last;