erts: Add carrier pointer to header of free block

6 files changed, 154 insertions, 82 deletions

diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c
index ad7a5822df..02171e050f 100644
--- a/erts/emulator/beam/erl_afit_alloc.c
+++ b/erts/emulator/beam/erl_afit_alloc.c
@@ -42,7 +42,7 @@ struct AFFreeBlock_t_ {
     AFFreeBlock_t *prev;
     AFFreeBlock_t *next;
 };
-#define AF_BLK_SZ(B) MBC_BLK_SZ(&(B)->block_head)
+#define AF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->block_head)
 
 #define MIN_MBC_SZ		(16*1024)
 #define MIN_MBC_FIRST_FREE_SZ	(4*1024)
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 3f8d3f23c3..c8e2d28c49 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -153,7 +153,7 @@ MBC after deallocating first block:
 #define UNUSED1_BLK_FTR_FLG	(((UWord) 1) << 1)
 #define UNUSED2_BLK_FTR_FLG	(((UWord) 1) << 2)
 
-#define ABLK_HDR_SZ (sizeof(Block_t))
+#define ABLK_HDR_SZ (offsetof(Block_t,u))
 #define FBLK_FTR_SZ (sizeof(FreeBlkFtr_t))
 
 #define UMEMSZ2BLKSZ(AP, SZ)						\
@@ -177,17 +177,15 @@ MBC after deallocating first block:
 */
 #define SBC_BLK_HDR_FLG (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)
 
-#define SET_MBC_BLK_SZ(B, SZ) \
+#define SET_MBC_ABLK_SZ(B, SZ) \
   (ASSERT(((SZ) & FLG_MASK) == 0), \
-   (B)->bhdr = (((B)->bhdr) & ~MBC_BLK_SZ_MASK) | (SZ))
+   (B)->bhdr = (((B)->bhdr) & ~MBC_ABLK_SZ_MASK) | (SZ))
+#define SET_MBC_FBLK_SZ(B, SZ) \
+  (ASSERT(((SZ) & FLG_MASK) == 0), \
+   (B)->bhdr = (((B)->bhdr) & ~MBC_FBLK_SZ_MASK) | (SZ))
 #define SET_SBC_BLK_SZ(B, SZ) \
   (ASSERT(((SZ) & FLG_MASK) == 0), \
    (B)->bhdr = (((B)->bhdr) & ~SBC_BLK_SZ_MASK) | (SZ))
-#define SET_BLK_FREE(B) \
-  (ASSERT(!IS_PREV_BLK_FREE(B)), \
-   (B)->bhdr |= THIS_FREE_BLK_HDR_FLG)
-#define SET_BLK_ALLOCED(B) \
-  ((B)->bhdr &= ~THIS_FREE_BLK_HDR_FLG)
 #define SET_PREV_BLK_FREE(AP,B) \
   (ASSERT(!IS_MBC_FIRST_BLK(AP,B)), \
    ASSERT(!IS_FREE_BLK(B)), \
@@ -208,32 +206,79 @@ MBC after deallocating first block:
 
 #  define BLK_CARRIER_OFFSET(B, C) (((char*)(B) - (char*)(C)) >> MSEG_UNIT_SHIFT)
 
-#  define SET_MBC_BLK_HDR(B, Sz, F, C) \
-    (ASSERT(((Sz) & FLG_MASK) == 0), \
-     ASSERT((UWord)(F) < SBC_BLK_HDR_FLG), \
+#  define SET_MBC_ABLK_HDR(B, Sz, F, C) \
+    (ASSERT(((Sz) & ~MBC_ABLK_SZ_MASK) == 0), \
+     ASSERT(!((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG))), \
      (B)->bhdr = ((Sz) | (F) | (BLK_CARRIER_OFFSET(B,C) << CARRIER_OFFSET_SHIFT)))
 
-#  define BLK_TO_MBC(B) \
-    (ASSERT(IS_MBC_BLK(B)), \
+#  define SET_MBC_FBLK_HDR(B, Sz, F, C) \
+    (ASSERT(((Sz) & ~MBC_FBLK_SZ_MASK) == 0), \
+     ASSERT(((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \
+     (B)->bhdr = ((Sz) | (F)), \
+     (B)->u.carrier = (C))
+
+#  define ABLK_TO_MBC(B) \
+    (ASSERT(IS_MBC_BLK(B) && IS_ALLOCED_BLK(B)), \
      (Carrier_t*)((MSEG_UNIT_FLOOR((UWord)(B)) - \
 		  (((B)->bhdr >> CARRIER_OFFSET_SHIFT) << MSEG_UNIT_SHIFT))))
 
-#  define IS_MBC_FIRST_BLK(AP,B) \
+#  define FBLK_TO_MBC(B) \
+    (ASSERT(IS_MBC_BLK(B) && IS_FREE_BLK(B)), \
+     (B)->u.carrier)
+      
+#  define BLK_TO_MBC(B) (IS_FREE_BLK(B) ? FBLK_TO_MBC(B) : ABLK_TO_MBC(B))
+
+#  define IS_MBC_FIRST_ABLK(AP,B) \
   ((((UWord)(B) & ~MSEG_UNIT_MASK) == (AP)->mbc_header_size) \
    && ((B)->bhdr & CARRIER_OFFSET_MASK) == 0)
 
+#  define IS_MBC_FIRST_FBLK(AP,B) \
+  ((char*)(B) == (char*)((B)->u.carrier) + (AP)->mbc_header_size)
+
+#  define IS_MBC_FIRST_BLK(AP,B) \
+  (IS_FREE_BLK(B) ? IS_MBC_FIRST_FBLK(AP,B) : IS_MBC_FIRST_ABLK(AP,B))
+
+#  define SET_BLK_FREE(B) \
+  (ASSERT(!IS_PREV_BLK_FREE(B)), \
+   (B)->u.carrier = ABLK_TO_MBC(B), \
+   (B)->bhdr |= THIS_FREE_BLK_HDR_FLG, \
+   (B)->bhdr &= (MBC_ABLK_SZ_MASK|FLG_MASK))
+
+#  define SET_BLK_ALLOCED(B) \
+  (ASSERT(((B)->bhdr & (CARRIER_OFFSET_MASK|THIS_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \
+   (B)->bhdr &= ~THIS_FREE_BLK_HDR_FLG, \
+   (B)->bhdr |= (BLK_CARRIER_OFFSET(B,(B)->u.carrier) << CARRIER_OFFSET_SHIFT))
+
 #else /* !HAVE_SUPER_ALIGNED_MB_CARRIERS */
 
-#  define SET_MBC_BLK_HDR(B, Sz, F, C) \
+#  define SET_MBC_ABLK_HDR(B, Sz, F, C) \
     (ASSERT(((Sz) & FLG_MASK) == 0), \
+     ASSERT(!((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG))), \
      ASSERT((UWord)(F) < SBC_BLK_HDR_FLG), \
      (B)->bhdr = ((Sz) | (F)), \
      (B)->carrier = (C))
 
+#  define SET_MBC_FBLK_HDR(B, Sz, F, C) \
+    (ASSERT(((Sz) & FLG_MASK) == 0), \
+     ASSERT(((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \
+     (B)->bhdr = ((Sz) | (F)), \
+     (B)->carrier = (C))
+
 #  define BLK_TO_MBC(B) ((B)->carrier)
+#  define ABLK_TO_MBC(B) BLK_TO_MBC(B)
+#  define FBLK_TO_MBC(B) BLK_TO_MBC(B)
 
 #  define IS_MBC_FIRST_BLK(AP,B) \
   ((char*)(B) == (char*)((B)->carrier) + (AP)->mbc_header_size)
+#  define IS_MBC_FIRST_ABLK(AP,B) IS_MBC_FIRST_BLK(AP,B)
+#  define IS_MBC_FIRST_FBLK(AP,B) IS_MBC_FIRST_BLK(AP,B)
+
+#  define SET_BLK_FREE(B) \
+  (ASSERT(!IS_PREV_BLK_FREE(B)), \
+   (B)->bhdr |= THIS_FREE_BLK_HDR_FLG)
+
+#  define SET_BLK_ALLOCED(B) \
+  ((B)->bhdr &= ~THIS_FREE_BLK_HDR_FLG)
 
 #endif /* !HAVE_SUPER_ALIGNED_MB_CARRIERS */
 
@@ -270,12 +315,18 @@ MBC after deallocating first block:
 #define IS_MBC_BLK(B) \
   (!IS_SBC_BLK((B)))
 
+#define MBC_BLK_SZ(B) (IS_FREE_BLK(B) ? MBC_FBLK_SZ(B) : MBC_ABLK_SZ(B))
+
 #define NXT_BLK(B) \
-  ((Block_t *) (((char *) (B)) + MBC_BLK_SZ((B))))
+  (ASSERT(IS_MBC_BLK(B)), \
+   (Block_t *) (((char *) (B)) + MBC_BLK_SZ((B))))
 #define PREV_BLK(B) \
   ((Block_t *) (((char *) (B)) - PREV_BLK_SZ((B))))
 
-#define BLK_SZ(B) ((B)->bhdr & (IS_SBC_BLK(B) ? SBC_BLK_SZ_MASK : MBC_BLK_SZ_MASK))
+#define BLK_AFTER(B,Sz) \
+  ((Block_t *) (((char *) (B)) + (Sz)))
+
+#define BLK_SZ(B) ((B)->bhdr & (((B)->bhdr & THIS_FREE_BLK_HDR_FLG) ? MBC_FBLK_SZ_MASK : MBC_ABLK_SZ_MASK))
 
 /* Carriers ... */
 
@@ -846,10 +897,17 @@ static ERTS_INLINE Allctr_t*
 get_used_allctr(void *extra, void *p, UWord *sizep)
 {
     Block_t* blk = UMEM2BLK(p);
-    Carrier_t* crr = IS_SBC_BLK(blk) ? BLK_TO_SBC(blk) : BLK_TO_MBC(blk);
+    Carrier_t* crr;
 
-    if (sizep) {
-	*sizep = BLK_UMEM_SZ(blk);
+    if (IS_SBC_BLK(blk)) {
+	crr = BLK_TO_SBC(blk);
+	if (sizep)
+	    *sizep = SBC_BLK_SZ(blk) - ABLK_HDR_SZ;  
+    }
+    else {
+	crr = ABLK_TO_MBC(blk);
+	if (sizep)
+	    *sizep = MBC_ABLK_SZ(blk) - ABLK_HDR_SZ;
     }
     return crr->allctr;
 }
@@ -1283,6 +1341,7 @@ mbc_alloc_finalize(Allctr_t *allctr,
 		   Block_t *blk,
 		   Uint org_blk_sz,
 		   UWord flags,
+		   Carrier_t *crr,
 		   Uint want_blk_sz,
 		   int valid_blk_info,
 		   Uint32 alcu_flgs)
@@ -1291,7 +1350,6 @@ mbc_alloc_finalize(Allctr_t *allctr,
     Uint nxt_blk_sz;
     Block_t *nxt_blk;
     UWord prev_free_flg = flags & PREV_FREE_BLK_HDR_FLG;
-    Carrier_t* crr = BLK_TO_MBC(blk);
 
     ASSERT(org_blk_sz >= want_blk_sz);
     ASSERT(blk);
@@ -1304,17 +1362,17 @@ mbc_alloc_finalize(Allctr_t *allctr,
 	/* Shrink block... */
 	blk_sz = want_blk_sz;
 	nxt_blk_sz = org_blk_sz - blk_sz;
-	SET_MBC_BLK_HDR(blk, blk_sz, prev_free_flg, crr);
+	SET_MBC_ABLK_HDR(blk, blk_sz, prev_free_flg, crr);
 
-	nxt_blk = NXT_BLK(blk);
-	SET_MBC_BLK_HDR(nxt_blk, nxt_blk_sz,
-			SBH_THIS_FREE|(flags & LAST_BLK_HDR_FLG),
-			crr);
+	nxt_blk = BLK_AFTER(blk, blk_sz);
+	SET_MBC_FBLK_HDR(nxt_blk, nxt_blk_sz,
+			 SBH_THIS_FREE|(flags & LAST_BLK_HDR_FLG),
+			 crr);
 
 	if (!(flags & LAST_BLK_HDR_FLG)) {
 	    SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz);
 	    if (!valid_blk_info) {
-		Block_t *nxt_nxt_blk = NXT_BLK(nxt_blk);
+		Block_t *nxt_nxt_blk = BLK_AFTER(nxt_blk, nxt_blk_sz);
 		SET_PREV_BLK_FREE(allctr, nxt_nxt_blk);
 	    }
 	}
@@ -1332,8 +1390,8 @@ mbc_alloc_finalize(Allctr_t *allctr,
 	ASSERT(nxt_blk_sz == MBC_BLK_SZ(nxt_blk));
 	ASSERT(nxt_blk_sz % sizeof(Unit_t) == 0);
 	ASSERT(nxt_blk_sz >= allctr->min_block_size);
-	ASSERT(BLK_TO_MBC(blk) == crr);
-	ASSERT(BLK_TO_MBC(nxt_blk) == crr);
+	ASSERT(ABLK_TO_MBC(blk) == crr);
+	ASSERT(FBLK_TO_MBC(nxt_blk) == crr);
     }
     else {
 	blk_sz = org_blk_sz;
@@ -1341,21 +1399,21 @@ mbc_alloc_finalize(Allctr_t *allctr,
 	    if (valid_blk_info)
 		SET_BLK_ALLOCED(blk);
 	    else
-		SET_MBC_BLK_HDR(blk, blk_sz, SBH_LAST_BLK|prev_free_flg, crr);
+		SET_MBC_ABLK_HDR(blk, blk_sz, SBH_LAST_BLK|prev_free_flg, crr);
 	}
 	else {
 	    if (valid_blk_info)
 		SET_BLK_ALLOCED(blk);
 	    else
-		SET_MBC_BLK_HDR(blk, blk_sz, prev_free_flg, crr);
-	    nxt_blk = NXT_BLK(blk);
+		SET_MBC_ABLK_HDR(blk, blk_sz, prev_free_flg, crr);
+	    nxt_blk = BLK_AFTER(blk, blk_sz);
 	    SET_PREV_BLK_ALLOCED(nxt_blk);
 	}
 
 	ASSERT((flags & LAST_BLK_HDR_FLG)
 	       ? IS_LAST_BLK(blk)
 	       : IS_NOT_LAST_BLK(blk));
-	ASSERT(BLK_TO_MBC(blk) == crr);
+	ASSERT(ABLK_TO_MBC(blk) == crr);
     }
 
     STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs);
@@ -1387,6 +1445,7 @@ mbc_alloc(Allctr_t *allctr, Uint size)
 			   blk,
 			   MBC_BLK_SZ(blk),
 			   GET_BLK_HDR_FLGS(blk),
+			   FBLK_TO_MBC(blk),
 			   blk_sz,
 			   1,
 			   alcu_flgs);
@@ -1418,7 +1477,7 @@ mbc_free(Allctr_t *allctr, void *p)
 
     STAT_MBC_BLK_FREE(allctr, blk_sz, alcu_flgs);
 
-    is_first_blk = IS_MBC_FIRST_BLK(allctr, blk);
+    is_first_blk = IS_MBC_FIRST_ABLK(allctr, blk);
     is_last_blk = IS_LAST_BLK(blk);
 
     if (IS_PREV_BLK_FREE(blk)) {
@@ -1428,8 +1487,8 @@ mbc_free(Allctr_t *allctr, void *p)
 	(*allctr->unlink_free_block)(allctr, blk, alcu_flgs);
 
 	blk_sz += MBC_BLK_SZ(blk);
-	is_first_blk = IS_MBC_FIRST_BLK(allctr, blk);
-	SET_MBC_BLK_SZ(blk, blk_sz);
+	is_first_blk = IS_MBC_FIRST_FBLK(allctr, blk);
+	SET_MBC_FBLK_SZ(blk, blk_sz);
     }
     else {
 	SET_BLK_FREE(blk);
@@ -1438,12 +1497,12 @@ mbc_free(Allctr_t *allctr, void *p)
     if (is_last_blk)
 	SET_LAST_BLK(blk);
     else {
-	nxt_blk = NXT_BLK(blk);
+	nxt_blk = BLK_AFTER(blk, blk_sz);
 	if (IS_FREE_BLK(nxt_blk)) {
 	    /* Coalesce with next block... */
 	    (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs);
-	    blk_sz += MBC_BLK_SZ(nxt_blk);
-	    SET_MBC_BLK_SZ(blk, blk_sz);
+	    blk_sz += MBC_FBLK_SZ(nxt_blk);
+	    SET_MBC_FBLK_SZ(blk, blk_sz);
 
 	    is_last_blk = IS_LAST_BLK(nxt_blk);
 	    if (is_last_blk) 
@@ -1461,9 +1520,9 @@ mbc_free(Allctr_t *allctr, void *p)
 
     }
 
-    ASSERT(!is_last_blk  == !IS_LAST_BLK(blk));
-    ASSERT(!is_first_blk == !IS_MBC_FIRST_BLK(allctr, blk));
     ASSERT(IS_FREE_BLK(blk));
+    ASSERT(!is_last_blk  == !IS_LAST_BLK(blk));
+    ASSERT(!is_first_blk == !IS_MBC_FIRST_FBLK(allctr, blk));
     ASSERT(is_first_blk || IS_PREV_BLK_ALLOCED(blk));
     ASSERT(is_last_blk  || IS_PREV_BLK_FREE(NXT_BLK(blk)));
     ASSERT(blk_sz == MBC_BLK_SZ(blk));
@@ -1559,12 +1618,12 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
 		cand_blk = blk;
 	    }
 	    else {
-		ASSERT(!IS_MBC_FIRST_BLK(allctr, blk));
+		ASSERT(!IS_MBC_FIRST_ABLK(allctr, blk));
 		cand_blk = PREV_BLK(blk);
 		cand_blk_sz += PREV_BLK_SZ(blk);
 	    }
 	    if (!is_last_blk) {
-		nxt_blk = NXT_BLK(blk);
+		nxt_blk = BLK_AFTER(blk, old_blk_sz);
 		if (IS_FREE_BLK(nxt_blk))
 		    cand_blk_sz += MBC_BLK_SZ(nxt_blk);
 	    }
@@ -1582,18 +1641,18 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
 
 	nxt_blk_sz = old_blk_sz - blk_sz;
 
-	if ((is_last_blk || IS_ALLOCED_BLK(NXT_BLK(blk)))
+	if ((is_last_blk || IS_ALLOCED_BLK(BLK_AFTER(blk,old_blk_sz)))
 	    && (nxt_blk_sz < allctr->min_block_size))
 	    return p;
 
 	HARD_CHECK_BLK_CARRIER(allctr, blk);
 
-	nxt_nxt_blk = NXT_BLK(blk);
+	nxt_nxt_blk = BLK_AFTER(blk, old_blk_sz);
 
-	SET_MBC_BLK_SZ(blk, blk_sz);
+	SET_MBC_ABLK_SZ(blk, blk_sz);
 	SET_NOT_LAST_BLK(blk);
 
-	nxt_blk = NXT_BLK(blk);
+	nxt_blk = BLK_AFTER(blk, blk_sz);
 
 	STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs);
 	STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs);
@@ -1614,8 +1673,8 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
 	    SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz);
 	}
 
-	crr = BLK_TO_MBC(blk);
-	SET_MBC_BLK_HDR(nxt_blk, nxt_blk_sz,
+	crr = ABLK_TO_MBC(blk);
+	SET_MBC_FBLK_HDR(nxt_blk, nxt_blk_sz,
 			SBH_THIS_FREE | (is_last_blk ? SBH_LAST_BLK : 0),
 			crr);
 
@@ -1638,7 +1697,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
 	ASSERT(is_last_blk ? IS_LAST_BLK(nxt_blk) : IS_NOT_LAST_BLK(nxt_blk));
 	ASSERT(is_last_blk || nxt_blk == PREV_BLK(NXT_BLK(nxt_blk)));
 	ASSERT(is_last_blk || IS_PREV_BLK_FREE(NXT_BLK(nxt_blk)));
-	ASSERT(BLK_TO_MBC(nxt_blk) == crr);
+	ASSERT(FBLK_TO_MBC(nxt_blk) == crr);
 		    
 	HARD_CHECK_BLK_CARRIER(allctr, blk);
 
@@ -1648,7 +1707,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
     /* Need larger block... */
 
     if (!is_last_blk) {
-	nxt_blk = NXT_BLK(blk);
+	nxt_blk = BLK_AFTER(blk, old_blk_sz);
 	nxt_blk_sz = MBC_BLK_SZ(nxt_blk);
 	if (IS_FREE_BLK(nxt_blk) && get_blk_sz <= old_blk_sz + nxt_blk_sz) {
 	    /* Grow into next block... */
@@ -1662,7 +1721,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
 	    if (nxt_blk_sz < allctr->min_block_size) {
 		blk_sz += nxt_blk_sz;
 
-		SET_MBC_BLK_SZ(blk, blk_sz);
+		SET_MBC_ABLK_SZ(blk, blk_sz);
 
 		if (is_last_blk) {
 		    SET_LAST_BLK(blk);
@@ -1671,7 +1730,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
 #endif
 		}
 		else {
-		    nxt_blk = NXT_BLK(blk);
+		    nxt_blk = BLK_AFTER(blk, blk_sz);
 		    SET_PREV_BLK_ALLOCED(nxt_blk);
 #ifdef DEBUG
 		    is_last_blk = IS_LAST_BLK(nxt_blk);
@@ -1680,11 +1739,11 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
 		}
 	    }
 	    else {
-		Carrier_t* crr = BLK_TO_MBC(blk);
-		SET_MBC_BLK_SZ(blk, blk_sz);
+		Carrier_t* crr = ABLK_TO_MBC(blk);
+		SET_MBC_ABLK_SZ(blk, blk_sz);
 
-		nxt_blk = NXT_BLK(blk);
-		SET_MBC_BLK_HDR(nxt_blk, nxt_blk_sz, SBH_THIS_FREE, crr);
+		nxt_blk = BLK_AFTER(blk, blk_sz);
+		SET_MBC_FBLK_HDR(nxt_blk, nxt_blk_sz, SBH_THIS_FREE, crr);
 
 		if (is_last_blk)
 		    SET_LAST_BLK(nxt_blk);
@@ -1694,7 +1753,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
 		(*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs);
 
 		ASSERT(IS_FREE_BLK(nxt_blk));
-		ASSERT(BLK_TO_MBC(nxt_blk) == crr);
+		ASSERT(FBLK_TO_MBC(nxt_blk) == crr);
 	    }
 
 	    STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs);
@@ -1739,12 +1798,12 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
 	cand_blk_sz = 0;
     }
     else {
-	ASSERT(!IS_MBC_FIRST_BLK(allctr, blk));
+	ASSERT(!IS_MBC_FIRST_ABLK(allctr, blk));
 	cand_blk = PREV_BLK(blk);
 	cand_blk_sz = old_blk_sz + PREV_BLK_SZ(blk);
 
 	if (!is_last_blk) {
-	    nxt_blk = NXT_BLK(blk);
+	    nxt_blk = BLK_AFTER(blk, old_blk_sz);
 	    if (IS_FREE_BLK(nxt_blk))
 		cand_blk_sz += MBC_BLK_SZ(nxt_blk);
 	}
@@ -1784,6 +1843,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
 			       new_blk,
 			       MBC_BLK_SZ(new_blk),
 			       GET_BLK_HDR_FLGS(new_blk),
+			       FBLK_TO_MBC(new_blk),
 			       blk_sz,
 			       1,
 			       alcu_flgs);
@@ -1793,6 +1853,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
 	    return new_p;
 	}
 	else {
+	    Carrier_t* crr;
 	    Uint new_blk_sz;
 	    UWord new_blk_flgs;
 	    Uint prev_blk_sz;
@@ -1813,7 +1874,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
 	    if (is_last_blk) 
 		new_blk_flgs |= LAST_BLK_HDR_FLG;
 	    else {
-		nxt_blk = NXT_BLK(blk);
+		nxt_blk = BLK_AFTER(blk, old_blk_sz);
 		if (IS_FREE_BLK(nxt_blk)) {
 		    new_blk_flgs |= GET_LAST_BLK_HDR_FLG(nxt_blk);
 		    new_blk_sz += MBC_BLK_SZ(nxt_blk);
@@ -1829,6 +1890,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
 
 	    new_p = BLK2UMEM(new_blk);
 	    blk_cpy_sz = MIN(blk_sz, old_blk_sz);
+	    crr = FBLK_TO_MBC(new_blk);
 
 	    if (prev_blk_sz >= blk_cpy_sz)
 		sys_memcpy(new_p, p, blk_cpy_sz - ABLK_HDR_SZ);
@@ -1839,6 +1901,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
 			       new_blk,
 			       new_blk_sz,
 			       new_blk_flgs,
+			       crr,
 			       blk_sz,
 			       0,
 			       alcu_flgs);
@@ -1864,18 +1927,20 @@ static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C,
 {
     ASSERT(IS_LAST_BLK((B)));
     ASSERT((CSZ) == CARRIER_SZ((C)));
-    ASSERT((BSZ) == BLK_SZ((B)));
     ASSERT((BSZ) % sizeof(Unit_t) == 0);
     if ((SBC)) {
+	ASSERT((BSZ) == SBC_BLK_SZ((B)));
 	ASSERT((char*)B == (char*)C + A->sbc_header_size);
 	ASSERT(IS_SBC_BLK((B)));
 	ASSERT(IS_SB_CARRIER((C)));
     }
     else {
-	ASSERT(IS_MBC_FIRST_BLK(A, (B)));
+	ASSERT(IS_FREE_BLK(B));
+	ASSERT((BSZ) == MBC_FBLK_SZ((B)));
+	ASSERT(IS_MBC_FIRST_FBLK(A, (B)));
 	ASSERT(IS_MBC_BLK((B)));
 	ASSERT(IS_MB_CARRIER((C)));
-	ASSERT(BLK_TO_MBC(B) == (C));
+	ASSERT(FBLK_TO_MBC(B) == (C));
     }
     if ((MSEGED)) {
 	ASSERT(IS_MSEG_CARRIER((C)));
@@ -1913,7 +1978,7 @@ create_sbmbc(Allctr_t *allctr, Uint umem_sz)
 
     blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size);
 
-    SET_MBC_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr);
+    SET_MBC_FBLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr);
 
     link_carrier(&allctr->sbmbc_list, crr);
 
@@ -1933,7 +1998,7 @@ destroy_sbmbc(Allctr_t *allctr, Block_t *blk)
     Carrier_t *crr;
 
     ASSERT(IS_MBC_BLK(blk));
-    ASSERT(IS_MBC_FIRST_BLK(allctr, blk));
+    ASSERT(IS_MBC_FIRST_FBLK(allctr, blk));
 
     crr = FIRST_BLK_TO_MBC(allctr, blk);
     crr_sz = CARRIER_SZ(crr);
@@ -2114,7 +2179,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 
 	blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size);
 
-	SET_MBC_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr);
+	SET_MBC_FBLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr);
 
 	if (flags & CFLG_MAIN_CARRIER) {
 	    ASSERT(!allctr->main_carrier);
@@ -2300,7 +2365,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
 
     }
     else {
-	ASSERT(IS_MBC_FIRST_BLK(allctr, blk));
+	ASSERT(IS_MBC_FIRST_FBLK(allctr, blk));
 	crr = FIRST_BLK_TO_MBC(allctr, blk);
 	crr_sz = CARRIER_SZ(crr);
 
@@ -4272,7 +4337,7 @@ erts_alcu_stop(Allctr_t *allctr)
 void
 erts_alcu_init(AlcUInit_t *init)
 {
-
+    ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */
 #if HAVE_ERTS_MSEG
     ASSERT(erts_mseg_unit_size() == MSEG_UNIT_SZ);
 # if HAVE_SUPER_ALIGNED_MB_CARRIERS
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 7d6f61fdff..06a5a5a346 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -218,6 +218,7 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);
 
 #define FLG_MASK		INV_UNIT_MASK
 #define SBC_BLK_SZ_MASK         UNIT_MASK
+#define MBC_FBLK_SZ_MASK        UNIT_MASK
 #define CARRIER_SZ_MASK         UNIT_MASK
 
 
@@ -225,12 +226,13 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);
 #  define CARRIER_OFFSET_BITS   13
 #  define CARRIER_OFFSET_SHIFT  (sizeof(UWord)*8 - CARRIER_OFFSET_BITS)
 #  define CARRIER_OFFSET_MASK   (~((UWord)0) << CARRIER_OFFSET_SHIFT)
-#  define MBC_BLK_SZ_MASK	(~CARRIER_OFFSET_MASK & ~FLG_MASK)
+#  define MBC_ABLK_SZ_MASK	(~CARRIER_OFFSET_MASK & ~FLG_MASK)
 #else
-#  define MBC_BLK_SZ_MASK	(~FLG_MASK)
+#  define MBC_ABLK_SZ_MASK	(~FLG_MASK)
 #endif
 
-#define MBC_BLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_BLK_SZ_MASK)
+#define MBC_ABLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_ABLK_SZ_MASK)
+#define MBC_FBLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_FBLK_SZ_MASK)
 #define SBC_BLK_SZ(B) (ASSERT_EXPR(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK)
 
 #define CARRIER_SZ(C) \
@@ -257,9 +259,14 @@ typedef struct {
     UWord bhdr;
 #if !HAVE_SUPER_ALIGNED_MB_CARRIERS
     Carrier_t *carrier;
-#endif 
+#endif
+    union {
+	Carrier_t *carrier;     /* if free */
+	char       udata__[1];  /* if allocated */
+    }u;
 } Block_t;
-typedef UWord FreeBlkFtr_t;
+
+typedef UWord FreeBlkFtr_t; /* Footer of a free block */
 
 typedef struct {
     UWord giga_no;
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
index 4768dcc939..ded58e89f4 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.c
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -91,7 +91,7 @@ struct AOFF_RBTree_t_ {
     AOFF_RBTree_t *right;
     Uint max_sz;  /* of all blocks in this sub-tree */
 };
-#define AOFF_BLK_SZ(B) MBC_BLK_SZ(&(B)->hdr)
+#define AOFF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->hdr)
 
 #ifdef HARD_DEBUG
 static AOFF_RBTree_t * check_tree(AOFF_RBTree_t* root, Uint);
diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c
index 619206bbcd..4d3ee98a4c 100644
--- a/erts/emulator/beam/erl_bestfit_alloc.c
+++ b/erts/emulator/beam/erl_bestfit_alloc.c
@@ -73,7 +73,7 @@
 #define SET_RED(N)		(((RBTree_t *) (N))->flags |= RED_FLG)
 #define SET_BLACK(N)		(((RBTree_t *) (N))->flags &= ~RED_FLG)
 
-#define BF_BLK_SZ(B)            MBC_BLK_SZ(&(B)->hdr)
+#define BF_BLK_SZ(B)            MBC_FBLK_SZ(&(B)->hdr)
 
 #undef ASSERT
 #define ASSERT ASSERT_EXPR
diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c
index 9db89652f7..264561a965 100644
--- a/erts/emulator/beam/erl_goodfit_alloc.c
+++ b/erts/emulator/beam/erl_goodfit_alloc.c
@@ -363,7 +363,7 @@ search_bucket(Allctr_t *allctr, int ix, Uint size)
 	 blk && i < max_blk_search;
 	 blk = blk->next, i++) {
 
-	blk_sz = MBC_BLK_SZ(&blk->block_head);
+	blk_sz = MBC_FBLK_SZ(&blk->block_head);
 	blk_on_lambc = (((char *) blk) < gfallctr->last_aux_mbc_end
 			&& gfallctr->last_aux_mbc_start <= ((char *) blk));
 
@@ -402,7 +402,7 @@ get_free_block(Allctr_t *allctr, Uint size,
     if (min_bi == unsafe_bi) {
 	blk = search_bucket(allctr, min_bi, size);
 	if (blk) {
-	    if (cand_blk && cand_size <= MBC_BLK_SZ(blk))
+	    if (cand_blk && cand_size <= MBC_FBLK_SZ(blk))
 		return NULL; /* cand_blk was better */
 	    unlink_free_block(allctr, blk, flags);
 	    return blk;
@@ -422,7 +422,7 @@ get_free_block(Allctr_t *allctr, Uint size,
     /* We are guaranteed to find a block that fits in this bucket */
     blk = search_bucket(allctr, min_bi, size);
     ASSERT(blk);
-    if (cand_blk && cand_size <= MBC_BLK_SZ(blk))
+    if (cand_blk && cand_size <= MBC_FBLK_SZ(blk))
 	return NULL; /* cand_blk was better */
     unlink_free_block(allctr, blk, flags);
     return blk;
@@ -435,7 +435,7 @@ link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
 {
     GFAllctr_t *gfallctr = (GFAllctr_t *) allctr;
     GFFreeBlock_t *blk = (GFFreeBlock_t *) block;
-    Uint sz = MBC_BLK_SZ(&blk->block_head);
+    Uint sz = MBC_FBLK_SZ(&blk->block_head);
     int i = BKT_IX(gfallctr, sz);
 
     ASSERT(sz >= MIN_BLK_SZ);
@@ -456,7 +456,7 @@ unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
 {
     GFAllctr_t *gfallctr = (GFAllctr_t *) allctr;
     GFFreeBlock_t *blk = (GFFreeBlock_t *) block;
-    Uint sz = MBC_BLK_SZ(&blk->block_head);
+    Uint sz = MBC_FBLK_SZ(&blk->block_head);
     int i = BKT_IX(gfallctr, sz);
 
     if (!blk->prev) {