Merge branch 'rickard/aligned-sys_alloc-carriers/OTP-11318' into rickard/aligned-sys_alloc-carriers_maint/OTP-11318

Conflicts:
	erts/emulator/beam/erl_alloc.c
	erts/emulator/beam/erl_alloc_util.c
	erts/emulator/beam/erl_alloc_util.h

10 files changed, 186 insertions, 80 deletions

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 5eacff8829..6dec383cee 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -267,7 +267,6 @@ set_default_sl_alloc_opts(struct au_init *ip)
     ip->atype			= GOODFIT;
 #endif
     ip->init.util.name_prefix	= "sl_";
-    ip->init.util.mmmbc		= 5;
     ip->init.util.alloc_no	= ERTS_ALC_A_SHORT_LIVED;
 #ifndef SMALL_MEMORY
     ip->init.util.mmbcs 	= 128*1024; /* Main carrier size */
@@ -295,7 +294,6 @@ set_default_std_alloc_opts(struct au_init *ip)
     ip->atype			= BESTFIT;
 #endif
     ip->init.util.name_prefix	= "std_";
-    ip->init.util.mmmbc		= 5;
     ip->init.util.alloc_no	= ERTS_ALC_A_STANDARD;
 #ifndef SMALL_MEMORY
     ip->init.util.mmbcs 	= 128*1024; /* Main carrier size */
@@ -319,7 +317,6 @@ set_default_ll_alloc_opts(struct au_init *ip)
 #endif
     ip->init.util.ramv		= 0;
     ip->init.util.mmsbc		= 0;
-    ip->init.util.mmmbc		= 0;
     ip->init.util.sbct		= ~((UWord) 0);
     ip->init.util.name_prefix	= "ll_";
     ip->init.util.alloc_no	= ERTS_ALC_A_LONG_LIVED;
@@ -370,7 +367,6 @@ set_default_eheap_alloc_opts(struct au_init *ip)
     ip->thr_spec		= 1;
     ip->atype			= GOODFIT;
 #endif
-    ip->init.util.mmmbc		= 100;
     ip->init.util.name_prefix	= "eheap_";
     ip->init.util.alloc_no	= ERTS_ALC_A_EHEAP;
 #ifndef SMALL_MEMORY
@@ -397,7 +393,6 @@ set_default_binary_alloc_opts(struct au_init *ip)
     ip->thr_spec		= 1;
     ip->atype			= BESTFIT;
 #endif
-    ip->init.util.mmmbc		= 50;
     ip->init.util.name_prefix	= "binary_";
     ip->init.util.alloc_no	= ERTS_ALC_A_BINARY;
 #ifndef SMALL_MEMORY
@@ -419,7 +414,6 @@ set_default_ets_alloc_opts(struct au_init *ip)
     ip->thr_spec		= 1;
     ip->atype			= BESTFIT;
 #endif
-    ip->init.util.mmmbc		= 100;
     ip->init.util.name_prefix	= "ets_";
     ip->init.util.alloc_no	= ERTS_ALC_A_ETS;
 #ifndef SMALL_MEMORY
@@ -495,13 +489,6 @@ adjust_tpref(struct au_init *ip, int no_sched)
 	/* ... shrink smallest multi-block carrier size */
 	if (ip->default_.smbcs)
 	    ip->init.util.smbcs /= ERTS_MIN(4, no_sched);
-	/* ... and more than three allocators shrink
-	   max mseg multi-block carriers */
-	if (ip->default_.mmmbc && no_sched > 2) {
-	    ip->init.util.mmmbc /= ERTS_MIN(4, no_sched - 1);
-	    if (ip->init.util.mmmbc < 3)
-		ip->init.util.mmmbc = 3;
-	}
     }
 }
 
@@ -1560,7 +1547,6 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 				aui[a]->thr_spec = 0;
 				check_disable_carrier_migration(aui[a]);
 				aui[a]->init.util.ramv = 0;
-				aui[a]->init.util.mmmbc = 10;
 				aui[a]->init.util.lmbcs = 5*1024*1024;
 			    }
 			}
@@ -2716,8 +2702,8 @@ erts_allocator_options(void *proc)
 #endif
     Uint sz, *szp, *hp, **hpp;
     Eterm res, features, settings;
-    Eterm atoms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+5];
-    Uint terms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+5];
+    Eterm atoms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+6];
+    Uint terms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+6];
     int a, length;
     SysAllocStat sas;
     Uint *endp = NULL;
@@ -2830,6 +2816,9 @@ erts_allocator_options(void *proc)
     if (use_mseg)
 	terms[length++] = am_atom_put("mseg_alloc", 10);
 #endif
+#if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
+    terms[length++] = am_atom_put("sys_aligned_alloc", 17);
+#endif
 
     features = length ? erts_bld_list(hpp, szp, length, terms) : NIL;
 
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index b5975c6c32..f83f6b39cf 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -54,6 +54,16 @@ void erts_sys_alloc_init(void);
 void *erts_sys_alloc(ErtsAlcType_t, void *, Uint);
 void *erts_sys_realloc(ErtsAlcType_t, void *, void *, Uint);
 void erts_sys_free(ErtsAlcType_t, void *, void *);
+#if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
+/*
+ * Note 'alignment' must remain the same in calls to
+ * 'erts_sys_aligned_realloc()' and 'erts_sys_aligned_free()'
+ * as in the initial call to 'erts_sys_aligned_alloc()'.
+ */
+void *erts_sys_aligned_alloc(UWord alignment, UWord size);
+void *erts_sys_aligned_realloc(UWord alignment, void *ptr, UWord size, UWord old_size);
+void erts_sys_aligned_free(UWord alignment, void *ptr);
+#endif
 
 Eterm erts_memory(int *, void *, void *, Eterm);
 Eterm erts_allocated_areas(int *, void *, void *);
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index e6d9f83aed..825b68bb85 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -81,7 +81,6 @@
 static int atoms_initialized = 0;
 static int initialized = 0;
 
-
 #define INV_SYS_ALLOC_CARRIER_MASK	((UWord) (sys_alloc_carrier_size - 1))
 #define SYS_ALLOC_CARRIER_MASK		(~INV_SYS_ALLOC_CARRIER_MASK)
 #define SYS_ALLOC_CARRIER_FLOOR(X)	((X) & SYS_ALLOC_CARRIER_MASK)
@@ -194,9 +193,9 @@ MBC after deallocating first block:
 
 #if MBC_ABLK_OFFSET_BITS
 
-#  define MBC_SZ_MAX_LIMIT ((((UWord)1 << MBC_ABLK_OFFSET_BITS) - 1) << MSEG_ALIGN_BITS)
+#  define MBC_SZ_MAX_LIMIT ((((UWord)1 << MBC_ABLK_OFFSET_BITS) - 1) << ERTS_SUPER_ALIGN_BITS)
 
-#  define BLK_CARRIER_OFFSET(B, C) (((char*)(B) - (char*)(C)) >> MSEG_UNIT_SHIFT)
+#  define BLK_CARRIER_OFFSET(B, C) (((char*)(B) - (char*)(C)) >> ERTS_SACRR_UNIT_SHIFT)
 
 #  define SET_MBC_ABLK_HDR(B, Sz, F, C) \
     (ASSERT(((Sz) & ~MBC_ABLK_SZ_MASK) == 0), \
@@ -210,7 +209,7 @@ MBC after deallocating first block:
      (B)->u.carrier = (C))
       
 #  define IS_MBC_FIRST_ABLK(AP,B) \
-  ((((UWord)(B) & ~MSEG_UNIT_MASK) == MBC_HEADER_SIZE(AP)) \
+  ((((UWord)(B) & ~ERTS_SACRR_UNIT_MASK) == MBC_HEADER_SIZE(AP)) \
    && ((B)->bhdr & MBC_ABLK_OFFSET_MASK) == 0)
 
 #  define IS_MBC_FIRST_FBLK(AP,B) \
@@ -787,11 +786,15 @@ alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags)
 #endif
 
 static ERTS_INLINE void *
-alcu_sys_alloc(Allctr_t *allctr, Uint size)
+alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign)
 {
     void *res;
-
-    res = erts_sys_alloc(0, NULL, size);
+#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
+    if (superalign)
+	res = erts_sys_aligned_alloc(ERTS_SACRR_UNIT_SZ, size);
+    else
+#endif
+	res = erts_sys_alloc(0, NULL, size);
     INC_CC(allctr->calls.sys_alloc);
     if (erts_mtrace_enabled)
 	erts_mtrace_crr_alloc(res, allctr->alloc_no, ERTS_ALC_A_SYSTEM, size);
@@ -799,11 +802,16 @@ alcu_sys_alloc(Allctr_t *allctr, Uint size)
 }
 
 static ERTS_INLINE void *
-alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size)
+alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign)
 {
     void *res;
 
-    res = erts_sys_realloc(0, NULL, ptr, size);
+#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
+    if (superalign)
+	res = erts_sys_aligned_realloc(ERTS_SACRR_UNIT_SZ, ptr, size, old_size);
+    else
+#endif
+	res = erts_sys_realloc(0, NULL, ptr, size);
     INC_CC(allctr->calls.sys_realloc);
     if (erts_mtrace_enabled)
 	erts_mtrace_crr_realloc(res,
@@ -815,9 +823,14 @@ alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size)
 }
 
 static ERTS_INLINE void
-alcu_sys_free(Allctr_t *allctr, void *ptr)
+alcu_sys_free(Allctr_t *allctr, void *ptr, int superalign)
 {
-    erts_sys_free(0, NULL, ptr);
+#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
+    if (superalign)
+	erts_sys_aligned_free(ERTS_SACRR_UNIT_SZ, ptr);
+    else
+#endif
+	erts_sys_free(0, NULL, ptr);
     INC_CC(allctr->calls.sys_free);
     if (erts_mtrace_enabled)
 	erts_mtrace_crr_free(allctr->alloc_no, ERTS_ALC_A_SYSTEM, ptr);
@@ -1334,7 +1347,7 @@ erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
 	return fix_nocpool_alloc_shrink(allctr, flgs);
 }
 
-static void dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, Uint mseg_flags);
+static void dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, int superaligned);
 
 #ifdef ERTS_SMP
 
@@ -1942,7 +1955,7 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp)
 
     if (!blk) {
 	blk = create_carrier(allctr, get_blk_sz, CFLG_MBC);
-#if !HALFWORD_HEAP && !HAVE_SUPER_ALIGNED_MB_CARRIERS
+#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY
 	if (!blk) {
 	    /* Emergency! We couldn't create the carrier as we wanted.
 	       Try to place it in a sys_alloced sbc. */
@@ -2978,7 +2991,7 @@ check_pending_dealloc_carrier(Allctr_t *allctr,
 
 	    dcrr = crr;
 	    crr = crr->next;
-	    dealloc_carrier(allctr, dcrr, ERTS_MSEG_FLG_2POW);
+	    dealloc_carrier(allctr, dcrr, 1);
 	    i++;
 	} while (crr && i < ERTS_ALC_MAX_DEALLOC_CARRIER);
 
@@ -3013,7 +3026,7 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr)
     erts_aint_t max_size;
 
     if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) {
-	dealloc_carrier(allctr, crr, ERTS_MSEG_FLG_2POW);
+	dealloc_carrier(allctr, crr, 1);
 	return;
     }
 
@@ -3053,7 +3066,7 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr)
 
     if (crr->cpool.thr_prgr == ERTS_THR_PRGR_INVALID
 	|| erts_thr_progress_has_reached(crr->cpool.thr_prgr)) {
-	dealloc_carrier(allctr, crr, ERTS_MSEG_FLG_2POW);
+	dealloc_carrier(allctr, crr, 1);
 	return;
     }
 
@@ -3188,10 +3201,10 @@ cpool_read_stat(Allctr_t *allctr, UWord *nocp, UWord *cszp, UWord *nobp, UWord *
 
 #ifdef DEBUG
 
-#if HAVE_ERTS_MSEG
-#define ASSERT_MSEG_UNIT_SIZE_MULTIPLE(CSZ) ASSERT((CSZ) % MSEG_UNIT_SZ == 0)
+#if ERTS_SA_MB_CARRIERS
+#define ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE(CSZ) ASSERT((CSZ) % ERTS_SACRR_UNIT_SZ == 0)
 #else
-#define ASSERT_MSEG_UNIT_SIZE_MULTIPLE(CSZ)
+#define ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE(CSZ)
 #endif
 
 static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C,
@@ -3216,7 +3229,7 @@ static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C,
     }
     if ((MSEGED)) {
 	ASSERT(IS_MSEG_CARRIER((C)));
-	ASSERT_MSEG_UNIT_SIZE_MULTIPLE((CSZ));
+	ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE((CSZ));
     }
     else {
 	ASSERT(IS_SYS_ALLOC_CARRIER((C)));
@@ -3244,7 +3257,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 
 #if HALFWORD_HEAP
     flags |= CFLG_FORCE_MSEG;
-#elif HAVE_SUPER_ALIGNED_MB_CARRIERS
+#elif ERTS_SUPER_ALIGNED_MSEG_ONLY
     if (flags & CFLG_MBC) {
 	flags |= CFLG_FORCE_MSEG;
     }
@@ -3287,7 +3300,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 	if (allctr->sbcs.curr.norm.mseg.no >= allctr->max_mseg_sbcs)
 	    goto try_sys_alloc;
     }
-#if !HAVE_SUPER_ALIGNED_MB_CARRIERS
+#if !ERTS_SUPER_ALIGNED_MSEG_ONLY
     else {
 	if (allctr->mbcs.curr.norm.mseg.no >= allctr->max_mseg_mbcs)
 	    goto try_sys_alloc;
@@ -3350,12 +3363,12 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 	      ? UNIT_CEILING(bcrr_sz)
 	      : SYS_ALLOC_CARRIER_CEILING(bcrr_sz));
 
-    crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz);
+    crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz, flags & CFLG_MBC);
 	
     if (!crr) {
 	if (crr_sz > UNIT_CEILING(bcrr_sz)) {
 	    crr_sz = UNIT_CEILING(bcrr_sz);
-	    crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz);
+	    crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz, flags & CFLG_MBC);
 	}
 	if (!crr) {
 #if HAVE_ERTS_MSEG
@@ -3453,7 +3466,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
 	if (!(flags & CFLG_FORCE_SYS_ALLOC)) {
 
 	    new_crr_sz = new_blk_sz + SBC_HEADER_SIZE;
-	    new_crr_sz = MSEG_UNIT_CEILING(new_crr_sz);
+	    new_crr_sz = ERTS_SACRR_UNIT_CEILING(new_crr_sz);
 	    new_crr = (Carrier_t *) alcu_mseg_realloc(allctr,
 						      old_crr,
 						      old_crr_sz,
@@ -3503,7 +3516,9 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
 
 	    new_crr = (Carrier_t *) alcu_sys_realloc(allctr,
 						     (void *) old_crr,
-						     new_crr_sz);
+						     new_crr_sz,
+						     old_crr_sz,
+						     0);
 	    if (new_crr) {
 	    sys_realloc_success:
 		SET_CARRIER_SZ(new_crr, new_crr_sz);
@@ -3522,7 +3537,9 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
 		new_crr_sz = UNIT_CEILING(new_crr_sz);
 		new_crr = (Carrier_t *) alcu_sys_realloc(allctr,
 							 (void *) old_crr,
-							 new_crr_sz);
+							 new_crr_sz,
+							 old_crr_sz,
+							 0);
 		if (new_crr)
 		    goto sys_realloc_success;
 	    }
@@ -3541,7 +3558,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
 		       (void *) BLK2UMEM(old_blk),
 		       MIN(new_blk_sz, old_blk_sz) - ABLK_HDR_SZ);
 	    unlink_carrier(&allctr->sbc_list, old_crr);
-	    alcu_sys_free(allctr, old_crr);
+	    alcu_sys_free(allctr, old_crr, 0);
 	}
 	else {
 	    /* Old carrier unchanged; restore... */
@@ -3554,14 +3571,17 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
 }
 
 static void
-dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, Uint mseg_flags)
+dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, int superaligned)
 {
 #if HAVE_ERTS_MSEG
     if (IS_MSEG_CARRIER(crr))
-	alcu_mseg_dealloc(allctr, crr, CARRIER_SZ(crr), mseg_flags);
+	alcu_mseg_dealloc(allctr, crr, CARRIER_SZ(crr),
+			  (superaligned
+			   ? ERTS_MSEG_FLG_2POW
+			   : ERTS_MSEG_FLG_NONE));
     else
 #endif
-	alcu_sys_free(allctr, crr);
+	alcu_sys_free(allctr, crr, superaligned);
 }
 
 static void
@@ -3581,7 +3601,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp)
 
 #if HAVE_ERTS_MSEG
 	if (IS_MSEG_CARRIER(crr)) {
-	    ASSERT(crr_sz % MSEG_UNIT_SZ == 0);
+	    ASSERT(crr_sz % ERTS_SACRR_UNIT_SZ == 0);
 	    STAT_MSEG_SBC_FREE(allctr, crr_sz, blk_sz);
 	}
 	else
@@ -3590,7 +3610,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp)
 
 	unlink_carrier(&allctr->sbc_list, crr);
 
-	dealloc_carrier(allctr, crr, ERTS_MSEG_FLG_NONE);
+	dealloc_carrier(allctr, crr, 0);
     }
     else {
 	ASSERT(IS_MBC_FIRST_FBLK(allctr, blk));
@@ -3624,7 +3644,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp)
 	    unlink_carrier(&allctr->mbc_list, crr);
 #if HAVE_ERTS_MSEG
 	    if (IS_MSEG_CARRIER(crr)) {
-		ASSERT(crr_sz % MSEG_UNIT_SZ == 0);
+		ASSERT(crr_sz % ERTS_SACRR_UNIT_SZ == 0);
 		STAT_MSEG_MBC_FREE(allctr, crr_sz);
 	    }
 	    else
@@ -3635,10 +3655,9 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp)
 #ifdef ERTS_SMP
 	schedule_dealloc_carrier(allctr, crr);
 #else
-	dealloc_carrier(allctr, crr, ERTS_MSEG_FLG_2POW);
+	dealloc_carrier(allctr, crr, 1);
 #endif
     }
-
 }
 
 
@@ -5086,7 +5105,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
 		crr_sz = SYS_ALLOC_CARRIER_CEILING(used_sz);
 #if HAVE_ERTS_MSEG
 	    else
-		crr_sz = MSEG_UNIT_CEILING(used_sz);
+		crr_sz = ERTS_SACRR_UNIT_CEILING(used_sz);
 #endif
 	    diff_sz_val = crr_sz - used_sz;
 	    if (diff_sz_val < (~((Uint) 0) / 100))
@@ -5454,7 +5473,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
     allctr->mbc_move_threshold		= init->rmbcmt;
 #if HAVE_ERTS_MSEG
     allctr->max_mseg_sbcs		= init->mmsbc;
-# if HAVE_SUPER_ALIGNED_MB_CARRIERS
+# if ERTS_SUPER_ALIGNED_MSEG_ONLY
     allctr->max_mseg_mbcs		= ~(Uint)0;
 # else
     allctr->max_mseg_mbcs		= init->mmmbc;
@@ -5570,7 +5589,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 			     CFLG_MBC
 			     | CFLG_FORCE_SIZE
 			     | CFLG_NO_CPOOL
-#if !HALFWORD_HEAP && !HAVE_SUPER_ALIGNED_MB_CARRIERS
+#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY
 			     | CFLG_FORCE_SYS_ALLOC
 #endif
 			     | CFLG_MAIN_CARRIER);
@@ -5656,9 +5675,9 @@ erts_alcu_init(AlcUInit_t *init)
 #endif
     ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */
 #if HAVE_ERTS_MSEG
-    ASSERT(erts_mseg_unit_size() == MSEG_UNIT_SZ);
+    ASSERT(erts_mseg_unit_size() == ERTS_SACRR_UNIT_SZ);
     max_mseg_carriers = init->mmc;
-    sys_alloc_carrier_size = MSEG_UNIT_CEILING(init->ycs);
+    sys_alloc_carrier_size = ERTS_SACRR_UNIT_CEILING(init->ycs);
 #else /* #if HAVE_ERTS_MSEG */
     sys_alloc_carrier_size = ((init->ycs + 4095) / 4096) * 4096;
 #endif
@@ -5819,7 +5838,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
 	ASSERT(CARRIER_SZ(sbc) - SBC_HEADER_SIZE >= SBC_BLK_SZ(iblk));
 #if HAVE_ERTS_MSEG
 	if (IS_MSEG_CARRIER(sbc)) {
-	    ASSERT(CARRIER_SZ(sbc) % MSEG_UNIT_SZ == 0);
+	    ASSERT(CARRIER_SZ(sbc) % ERTS_SACRR_UNIT_SZ == 0);
 	}
 #endif
 	crr = sbc;
@@ -5904,7 +5923,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
 
 #if HAVE_ERTS_MSEG
 	if (IS_MSEG_CARRIER(crr)) {
-	    ASSERT(CARRIER_SZ(crr) % MSEG_UNIT_SZ == 0);
+	    ASSERT(CARRIER_SZ(crr) % ERTS_SACRR_UNIT_SZ == 0);
 	}
 #endif
 	cl = &allctr->mbc_list;
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 02cbe5c5d0..70ecf28172 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -75,7 +75,7 @@ typedef struct {
 
 #define ERTS_DEFAULT_ALCU_INIT {                                           \
     1024*1024,		/* (bytes)  ycs:    sys_alloc carrier size       */\
-    1024      		/* (amount) mmc:    max mseg carriers            */\
+    ~((UWord) 0)	/* (amount) mmc:    max mseg carriers            */ \
 }
 
 #define ERTS_DEFAULT_ALLCTR_INIT {                                         \
@@ -95,7 +95,7 @@ typedef struct {
     50,			/* (%)      rmbcmt: rel mbc move threshold       */\
     1024*1024,		/* (bytes)  mmbcs:  main multiblock carrier size */\
     256,		/* (amount) mmsbc:  max mseg sbcs                */\
-    10,			/* (amount) mmmbc:  max mseg mbcs                */\
+    ~((UWord) 0),	/* (amount) mmmbc:  max mseg mbcs                */ \
     10*1024*1024,	/* (bytes)  lmbcs:  largest mbc size             */\
     1024*1024,		/* (bytes)  smbcs:  smallest mbc size            */\
     10,			/* (amount) mbcgs:  mbc growth stages            */\
@@ -128,7 +128,7 @@ typedef struct {
     80,			/* (%)      rsbcmt: rel sbc move threshold       */\
     128*1024,		/* (bytes)  mmbcs:  main multiblock carrier size */\
     256,		/* (amount) mmsbc:  max mseg sbcs                */\
-    10,			/* (amount) mmmbc:  max mseg mbcs                */\
+    ~((UWord) 0),	/* (amount) mmmbc:  max mseg mbcs                */ \
     1024*1024,		/* (bytes)  lmbcs:  largest mbc size             */\
     128*1024,		/* (bytes)  smbcs:  smallest mbc size            */\
     10,			/* (amount) mbcgs:  mbc growth stages            */\
@@ -217,25 +217,34 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);
 #define MBC_FBLK_SZ_MASK        UNIT_MASK
 #define CARRIER_SZ_MASK         UNIT_MASK
 
-#if HAVE_ERTS_MSEG
-
-#  define MSEG_UNIT_SHIFT         MSEG_ALIGN_BITS
-#  define MSEG_UNIT_SZ            (1 << MSEG_UNIT_SHIFT)
-#  define MSEG_UNIT_MASK		((~(UWord)0) << MSEG_UNIT_SHIFT)
-
-#  define MSEG_UNIT_FLOOR(X)	((X) & MSEG_UNIT_MASK)
-#  define MSEG_UNIT_CEILING(X)	MSEG_UNIT_FLOOR((X) + ~MSEG_UNIT_MASK)
-
+#if ERTS_HAVE_MSEG_SUPER_ALIGNED \
+    || (!HAVE_ERTS_MSEG && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC)
+#  ifndef MSEG_ALIGN_BITS
+#    define ERTS_SUPER_ALIGN_BITS MSEG_ALIGN_BITS
+#  else
+#    define ERTS_SUPER_ALIGN_BITS 18
+#  endif
 #  ifdef ARCH_64 
 #    define MBC_ABLK_OFFSET_BITS   24
-#  elif HAVE_SUPER_ALIGNED_MB_CARRIERS
+#  else
 #    define MBC_ABLK_OFFSET_BITS   9
      /* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */
 #  endif
-#endif
-#ifndef MBC_ABLK_OFFSET_BITS
+#  define ERTS_SACRR_UNIT_SHIFT		ERTS_SUPER_ALIGN_BITS
+#  define ERTS_SACRR_UNIT_SZ		(1 << ERTS_SACRR_UNIT_SHIFT)
+#  define ERTS_SACRR_UNIT_MASK		((~(UWord)0) << ERTS_SACRR_UNIT_SHIFT)
+#  define ERTS_SACRR_UNIT_FLOOR(X)	((X) & ERTS_SACRR_UNIT_MASK)
+#  define ERTS_SACRR_UNIT_CEILING(X)	ERTS_SACRR_UNIT_FLOOR((X) + ~ERTS_SACRR_UNIT_MASK)
+#  define ERTS_SA_MB_CARRIERS 1
+#else
+#  define ERTS_SA_MB_CARRIERS 0
 #  define MBC_ABLK_OFFSET_BITS   0 /* no carrier offset in block header */
 #endif
+#if ERTS_HAVE_MSEG_SUPER_ALIGNED && !ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
+#  define ERTS_SUPER_ALIGNED_MSEG_ONLY 1
+#else
+#  define ERTS_SUPER_ALIGNED_MSEG_ONLY 0
+#endif
 
 #if MBC_ABLK_OFFSET_BITS
 #  define MBC_ABLK_OFFSET_SHIFT  (sizeof(UWord)*8 - MBC_ABLK_OFFSET_BITS)
@@ -327,8 +336,8 @@ typedef struct {
 			  (B)->u.carrier)
 #  define ABLK_TO_MBC(B) \
     (ASSERT(IS_MBC_BLK(B) && !IS_FREE_BLK(B)), \
-     (Carrier_t*)((MSEG_UNIT_FLOOR((UWord)(B)) - \
-		  (((B)->bhdr >> MBC_ABLK_OFFSET_SHIFT) << MSEG_UNIT_SHIFT))))
+     (Carrier_t*)((ERTS_SACRR_UNIT_FLOOR((UWord)(B)) - \
+		  (((B)->bhdr >> MBC_ABLK_OFFSET_SHIFT) << ERTS_SACRR_UNIT_SHIFT))))
 #  define BLK_TO_MBC(B) (IS_FREE_BLK(B) ? FBLK_TO_MBC(B) : ABLK_TO_MBC(B))
 #else
 #  define FBLK_TO_MBC(B) ((B)->carrier)
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 05bff430e3..096394b878 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -1012,6 +1012,9 @@ void erl_bin_write(unsigned char *, int, int);
 #define ERTS_SMALL_ABS(Small) labs(Small) 
 #endif
 
+#ifndef ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
+#  define ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC 0
+#endif
 
 #ifdef __WIN32__
 void call_break_handler(void);
diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h
index a4f250ceab..a1b000f51c 100644
--- a/erts/emulator/sys/common/erl_mseg.h
+++ b/erts/emulator/sys/common/erl_mseg.h
@@ -32,13 +32,13 @@
 
 #if HAVE_MMAP
 #  define HAVE_ERTS_MSEG 1
-#  define HAVE_SUPER_ALIGNED_MB_CARRIERS 1
+#  define ERTS_HAVE_MSEG_SUPER_ALIGNED 1
 #else
 #  define HAVE_ERTS_MSEG 0
-#  define HAVE_SUPER_ALIGNED_MB_CARRIERS 0
+#  define ERTS_HAVE_MSEG_SUPER_ALIGNED 0
 #endif
 
-#if HAVE_SUPER_ALIGNED_MB_CARRIERS
+#if ERTS_HAVE_MSEG_SUPER_ALIGNED
 #  define MSEG_ALIGN_BITS (18)
    /* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */
 #else
diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h
index c8fcec8547..2c47aa06c2 100644
--- a/erts/emulator/sys/unix/erl_unix_sys.h
+++ b/erts/emulator/sys/unix/erl_unix_sys.h
@@ -107,6 +107,10 @@
 #endif
 #include <netdb.h>
 
+#ifdef HAVE_POSIX_MEMALIGN
+#  define ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC 1
+#endif
+
 /*
  * Make sure that MAXPATHLEN is defined.
  */
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index a7ea4b2490..fdc3167c62 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -2524,6 +2524,52 @@ void erts_sys_alloc_init(void)
 {
 }
 
+#if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
+void *erts_sys_aligned_alloc(UWord alignment, UWord size)
+{
+#ifdef HAVE_POSIX_MEMALIGN
+    void *ptr = NULL;
+    int error;
+    ASSERT(alignment && (alignment & ~alignment) == 0); /* power of 2 */
+    error = posix_memalign(&ptr, (size_t) alignment, (size_t) size);
+#if HAVE_ERTS_MSEG
+    if (error || !ptr) {
+	erts_mseg_clear_cache();
+	error = posix_memalign(&ptr, (size_t) alignment, (size_t) size);
+    }
+#endif
+    if (error) {
+	errno = error;
+	return NULL;
+    }
+    if (!ptr)
+	errno = ENOMEM;
+    ASSERT(!ptr || (((UWord) ptr) & (alignment - 1)) == 0);
+    return ptr;
+#else
+#  error "Missing erts_sys_aligned_alloc() implementation"
+#endif
+}
+
+void erts_sys_aligned_free(UWord alignment, void *ptr)
+{
+    ASSERT(alignment && (alignment & ~alignment) == 0); /* power of 2 */
+    free(ptr);
+}
+
+void *erts_sys_aligned_realloc(UWord alignment, void *ptr, UWord size, UWord old_size)
+{
+    void *new_ptr = erts_sys_aligned_alloc(alignment, size);
+    if (new_ptr) {
+	UWord copy_size = old_size < size ? old_size : size;
+	sys_memcpy(new_ptr, ptr, (size_t) copy_size);
+	erts_sys_aligned_free(alignment, ptr);
+    }
+    return new_ptr;
+}
+
+#endif
+
 void *erts_sys_alloc(ErtsAlcType_t t, void *x, Uint sz)
 {
     void *res = malloc((size_t) sz);
diff --git a/erts/emulator/sys/win32/erl_win_sys.h b/erts/emulator/sys/win32/erl_win_sys.h
index 5ce1a61303..c44c5d9492 100644
--- a/erts/emulator/sys/win32/erl_win_sys.h
+++ b/erts/emulator/sys/win32/erl_win_sys.h
@@ -95,6 +95,8 @@
 #  define ERTS_I64_LITERAL(X) X##i64
 #endif
 
+#define ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC 1
+
 /*
  * Practial Windows specific macros.
  */
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index 922967c698..6a1d6b08f4 100755
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -32,7 +32,7 @@
 #include "erl_threads.h"
 #include "../../drivers/win32/win_con.h"
 #include "erl_cpu_topology.h"
-
+#include <malloc.h>
 
 void erts_sys_init_float(void);
 
@@ -2912,6 +2912,30 @@ void erts_sys_free(ErtsAlcType_t t, void *x, void *p)
     free(p);
 }
 
+void *erts_sys_aligned_alloc(UWord alignment, UWord size)
+{
+    void *ptr;
+    ASSERT(alignment && (alignment & ~alignment) == 0); /* power of 2 */
+    ptr = _aligned_malloc((size_t) size, (size_t) alignment);
+    ASSERT(!ptr || (((UWord) ptr) & (alignment - 1)) == 0);
+    return ptr;
+}
+
+void erts_sys_aligned_free(UWord alignment, void *ptr)
+{
+    ASSERT(alignment && (alignment & ~alignment) == 0); /* power of 2 */
+    _aligned_free(ptr);
+}
+
+void *erts_sys_aligned_realloc(UWord alignment, void *ptr, UWord size, UWord old_size)
+{
+    void *new_ptr;
+    ASSERT(alignment && (alignment & ~alignment) == 0); /* power of 2 */
+    new_ptr = _aligned_realloc(ptr, (size_t) size, (size_t) alignment);
+    ASSERT(!new_ptr || (((UWord) new_ptr) & (alignment - 1)) == 0);
+    return new_ptr;
+}
+
 static Preload* preloaded = NULL;
 static unsigned* res_name = NULL;
 static int num_preloaded = 0;