aboutsummaryrefslogtreecommitdiffstats
path: root/erts
diff options
context:
space:
mode:
authorSverker Eriksson <[email protected]>2012-11-06 22:42:47 +0100
committerSverker Eriksson <[email protected]>2012-11-15 14:20:14 +0100
commitde03018e74bea97795895f1611abf2a50a484449 (patch)
tree43d33e133e538a92e8d1f8495f71194022248233 /erts
parent495cadb1cbc315fa5250f99894b20508bb8a4c50 (diff)
downloadotp-de03018e74bea97795895f1611abf2a50a484449.tar.gz
otp-de03018e74bea97795895f1611abf2a50a484449.tar.bz2
otp-de03018e74bea97795895f1611abf2a50a484449.zip
erts: Add carrier offset to internal allocation headers
Diffstat (limited to 'erts')
-rw-r--r--erts/emulator/beam/erl_afit_alloc.c10
-rw-r--r--erts/emulator/beam/erl_afit_alloc.h5
-rw-r--r--erts/emulator/beam/erl_alloc_util.c419
-rw-r--r--erts/emulator/beam/erl_alloc_util.h33
-rw-r--r--erts/emulator/beam/erl_ao_firstfit_alloc.c15
-rw-r--r--erts/emulator/beam/erl_bestfit_alloc.c44
-rw-r--r--erts/emulator/beam/erl_goodfit_alloc.c12
-rw-r--r--erts/emulator/sys/common/erl_mseg.h2
8 files changed, 277 insertions, 263 deletions
diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c
index 570cc59be2..ad7a5822df 100644
--- a/erts/emulator/beam/erl_afit_alloc.c
+++ b/erts/emulator/beam/erl_afit_alloc.c
@@ -37,6 +37,12 @@
#define GET_ERL_AF_ALLOC_IMPL
#include "erl_afit_alloc.h"
+struct AFFreeBlock_t_ {
+ Block_t block_head;
+ AFFreeBlock_t *prev;
+ AFFreeBlock_t *next;
+};
+#define AF_BLK_SZ(B) MBC_BLK_SZ(&(B)->block_head)
#define MIN_MBC_SZ (16*1024)
#define MIN_MBC_FIRST_FREE_SZ (4*1024)
@@ -118,7 +124,7 @@ get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size,
ASSERT(!cand_blk || cand_size >= size);
- if (afallctr->free_list && BLK_SZ(afallctr->free_list) >= size) {
+ if (afallctr->free_list && AF_BLK_SZ(afallctr->free_list) >= size) {
AFFreeBlock_t *res = afallctr->free_list;
afallctr->free_list = res->next;
if (res->next)
@@ -135,7 +141,7 @@ link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
AFFreeBlock_t *blk = (AFFreeBlock_t *) block;
AFAllctr_t *afallctr = (AFAllctr_t *) allctr;
- if (afallctr->free_list && BLK_SZ(afallctr->free_list) > BLK_SZ(blk)) {
+ if (afallctr->free_list && AF_BLK_SZ(afallctr->free_list) > AF_BLK_SZ(blk)) {
blk->next = afallctr->free_list->next;
blk->prev = afallctr->free_list;
afallctr->free_list->next = blk;
diff --git a/erts/emulator/beam/erl_afit_alloc.h b/erts/emulator/beam/erl_afit_alloc.h
index ea408a7194..87caccac20 100644
--- a/erts/emulator/beam/erl_afit_alloc.h
+++ b/erts/emulator/beam/erl_afit_alloc.h
@@ -49,11 +49,6 @@ Allctr_t *erts_afalc_start(AFAllctr_t *, AFAllctrInit_t *, AllctrInit_t *);
#include "erl_alloc_util.h"
typedef struct AFFreeBlock_t_ AFFreeBlock_t;
-struct AFFreeBlock_t_ {
- Block_t block_head;
- AFFreeBlock_t *prev;
- AFFreeBlock_t *next;
-};
struct AFAllctr_t_ {
Allctr_t allctr; /* Has to be first! */
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 1cceacce5c..6b674debbd 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -78,10 +78,12 @@ int erts_have_sbmbc_alloc;
#if HAVE_ERTS_MSEG
-#define INV_MSEG_UNIT_MASK ((UWord) (mseg_unit_size - 1))
-#define MSEG_UNIT_MASK (~INV_MSEG_UNIT_MASK)
+#define MSEG_UNIT_SHIFT ((UWord)12)
+#define MSEG_UNIT_SZ (1 << MSEG_UNIT_SHIFT)
+#define MSEG_UNIT_MASK ((~(UWord)0) << MSEG_UNIT_SHIFT)
+
#define MSEG_UNIT_FLOOR(X) ((X) & MSEG_UNIT_MASK)
-#define MSEG_UNIT_CEILING(X) MSEG_UNIT_FLOOR((X) + INV_MSEG_UNIT_MASK)
+#define MSEG_UNIT_CEILING(X) MSEG_UNIT_FLOOR((X) + ~MSEG_UNIT_MASK)
#endif
@@ -104,7 +106,6 @@ int erts_have_sbmbc_alloc;
static Uint sys_alloc_carrier_size;
#if HAVE_ERTS_MSEG
static Uint max_mseg_carriers;
-static Uint mseg_unit_size;
#endif
#define ONE_GIGA (1000000000)
@@ -137,7 +138,7 @@ static Uint mseg_unit_size;
#define BLK2UMEM(P) ((void *) (((char *) (P)) + ABLK_HDR_SZ))
#define PREV_BLK_SZ(B) \
- ((UWord) (*(((UWord *) (B)) - 1) & SZ_MASK))
+ ((UWord) (((UWord *)(B))[-1] & MBC_BLK_SZ_MASK))
#define SET_BLK_SZ_FTR(B, SZ) \
(*((UWord *) (((char *) (B)) + (SZ) - sizeof(UWord))) = (SZ))
@@ -146,55 +147,80 @@ static Uint mseg_unit_size;
#define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1)
#define LAST_BLK_HDR_FLG (((UWord) 1) << 2)
-#define SET_BLK_SZ(B, SZ) \
+#define SET_MBC_BLK_SZ(B, SZ) \
+ (ASSERT(((SZ) & FLG_MASK) == 0), \
+ (B)->bhdr = (((B)->bhdr) & ~MBC_BLK_SZ_MASK) | (SZ))
+#define SET_SBC_BLK_SZ(B, SZ) \
(ASSERT(((SZ) & FLG_MASK) == 0), \
- (*((Block_t *) (B)) = ((*((Block_t *) (B)) & FLG_MASK) | (SZ))))
+ (B)->bhdr = (((B)->bhdr) & ~SBC_BLK_SZ_MASK) | (SZ))
#define SET_BLK_FREE(B) \
- (*((Block_t *) (B)) |= THIS_FREE_BLK_HDR_FLG)
+ ((B)->bhdr |= THIS_FREE_BLK_HDR_FLG)
#define SET_BLK_ALLOCED(B) \
- (*((Block_t *) (B)) &= ~THIS_FREE_BLK_HDR_FLG)
+ ((B)->bhdr &= ~THIS_FREE_BLK_HDR_FLG)
#define SET_PREV_BLK_FREE(B) \
- (*((Block_t *) (B)) |= PREV_FREE_BLK_HDR_FLG)
+ ((B)->bhdr |= PREV_FREE_BLK_HDR_FLG)
#define SET_PREV_BLK_ALLOCED(B) \
- (*((Block_t *) (B)) &= ~PREV_FREE_BLK_HDR_FLG)
+ ((B)->bhdr &= ~PREV_FREE_BLK_HDR_FLG)
#define SET_LAST_BLK(B) \
- (*((Block_t *) (B)) |= LAST_BLK_HDR_FLG)
+ ((B)->bhdr |= LAST_BLK_HDR_FLG)
#define SET_NOT_LAST_BLK(B) \
- (*((Block_t *) (B)) &= ~LAST_BLK_HDR_FLG)
+ ((B)->bhdr &= ~LAST_BLK_HDR_FLG)
#define SBH_THIS_FREE THIS_FREE_BLK_HDR_FLG
-#define SBH_THIS_ALLOCED ((UWord) 0)
#define SBH_PREV_FREE PREV_FREE_BLK_HDR_FLG
-#define SBH_PREV_ALLOCED ((UWord) 0)
#define SBH_LAST_BLK LAST_BLK_HDR_FLG
-#define SBH_NOT_LAST_BLK ((UWord) 0)
-#define SET_BLK_HDR(B, Sz, F) \
- (ASSERT(((Sz) & FLG_MASK) == 0), *((Block_t *) (B)) = ((Sz) | (F)))
+#if HAVE_SUPER_ALIGNED_MB_CARRIERS
+
+# define BLK_CARRIER_OFFSET(B, C) (((char*)(B) - (char*)(C)) >> MSEG_UNIT_SHIFT)
+
+# define SET_MBC_BLK_HDR(B, Sz, F, C) \
+ (ASSERT(((Sz) & FLG_MASK) == 0), \
+ (B)->bhdr = ((Sz) | (F) | (BLK_CARRIER_OFFSET(B,C) << CARRIER_OFFSET_SHIFT)))
+
+# define GET_MB_CARRIER(B) \
+ (ASSERT(IS_MBC_BLK(B)), \
+ (Carrier_t*)((MSEG_UNIT_FLOOR((UWord)(B)) - \
+ (((B)->bhdr >> CARRIER_OFFSET_SHIFT) << MSEG_UNIT_SHIFT))))
+
+#else /* !HAVE_SUPER_ALIGNED_MB_CARRIERS */
+
+# define SET_MBC_BLK_HDR(B, Sz, F, C) \
+ (ASSERT(((Sz) & FLG_MASK) == 0), \
+ (B)->bhdr = ((Sz) | (F)), \
+ (B)->carrier = (C))
+
+# define GET_MB_CARRIER(B) ((B)->carrier)
+
+#endif /* !HAVE_SUPER_ALIGNED_MB_CARRIERS */
+
+#define SET_SBC_BLK_HDR(B, Sz, F) \
+ (ASSERT(((Sz) & FLG_MASK) == 0), (B)->bhdr = ((Sz) | (F)))
+
#define BLK_UMEM_SZ(B) \
(BLK_SZ(B) - (ABLK_HDR_SZ))
#define IS_PREV_BLK_FREE(B) \
- (*((Block_t *) (B)) & PREV_FREE_BLK_HDR_FLG)
+ ((B)->bhdr & PREV_FREE_BLK_HDR_FLG)
#define IS_PREV_BLK_ALLOCED(B) \
(!IS_PREV_BLK_FREE((B)))
#define IS_FREE_BLK(B) \
- (*((Block_t *) (B)) & THIS_FREE_BLK_HDR_FLG)
+ ((B)->bhdr & THIS_FREE_BLK_HDR_FLG)
#define IS_ALLOCED_BLK(B) \
(!IS_FREE_BLK((B)))
#define IS_LAST_BLK(B) \
- (*((Block_t *) (B)) & LAST_BLK_HDR_FLG)
+ ((B)->bhdr & LAST_BLK_HDR_FLG)
#define IS_NOT_LAST_BLK(B) \
(!IS_LAST_BLK((B)))
#define GET_LAST_BLK_HDR_FLG(B) \
- (*((Block_t*) (B)) & LAST_BLK_HDR_FLG)
+ ((B)->bhdr & LAST_BLK_HDR_FLG)
#define GET_THIS_FREE_BLK_HDR_FLG(B) \
- (*((Block_t*) (B)) & THIS_FREE_BLK_HDR_FLG)
+ ((B)->bhdr & THIS_FREE_BLK_HDR_FLG)
#define GET_PREV_FREE_BLK_HDR_FLG(B) \
- (*((Block_t*) (B)) & PREV_FREE_BLK_HDR_FLG)
+ ((B)->bhdr & PREV_FREE_BLK_HDR_FLG)
#define GET_BLK_HDR_FLGS(B) \
- (*((Block_t*) (B)) & FLG_MASK)
+ ((B)->bhdr & FLG_MASK)
#define IS_FIRST_BLK(B) \
(IS_PREV_BLK_FREE((B)) && (PREV_BLK_SZ((B)) == 0))
@@ -212,7 +238,7 @@ static Uint mseg_unit_size;
(!IS_SBC_BLK((B)))
#define NXT_BLK(B) \
- ((Block_t *) (((char *) (B)) + BLK_SZ((B))))
+ ((Block_t *) (((char *) (B)) + MBC_BLK_SZ((B))))
#define PREV_BLK(B) \
((Block_t *) (((char *) (B)) - PREV_BLK_SZ((B))))
@@ -1209,10 +1235,8 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp, Uint32 *alcu_flgsp)
if ((*alcu_flgsp) & ERTS_ALCU_FLG_SBMBC)
blk = create_sbmbc(allctr, get_blk_sz);
else {
-#if HALFWORD_HEAP
- blk = create_carrier(allctr, get_blk_sz, CFLG_MBC|CFLG_FORCE_MSEG);
-#else
blk = create_carrier(allctr, get_blk_sz, CFLG_MBC);
+#if !HALFWORD_HEAP && !HAVE_SUPER_ALIGNED_MB_CARRIERS
if (!blk) {
/* Emergency! We couldn't create the carrier as we wanted.
Try to place it in a sys_alloced sbc. */
@@ -1250,6 +1274,7 @@ mbc_alloc_finalize(Allctr_t *allctr,
Uint nxt_blk_sz;
Block_t *nxt_blk;
UWord prev_free_flg = flags & PREV_FREE_BLK_HDR_FLG;
+ Carrier_t* crr = GET_MB_CARRIER(blk);
ASSERT(org_blk_sz >= want_blk_sz);
ASSERT(blk);
@@ -1262,16 +1287,12 @@ mbc_alloc_finalize(Allctr_t *allctr,
/* Shrink block... */
blk_sz = want_blk_sz;
nxt_blk_sz = org_blk_sz - blk_sz;
- SET_BLK_HDR(blk,
- blk_sz,
- SBH_THIS_ALLOCED|SBH_NOT_LAST_BLK|prev_free_flg);
+ SET_MBC_BLK_HDR(blk, blk_sz, prev_free_flg, crr);
nxt_blk = NXT_BLK(blk);
- SET_BLK_HDR(nxt_blk,
- nxt_blk_sz,
- (SBH_THIS_FREE
- | SBH_PREV_ALLOCED
- | (flags & LAST_BLK_HDR_FLG)));
+ SET_MBC_BLK_HDR(nxt_blk, nxt_blk_sz,
+ SBH_THIS_FREE|(flags & LAST_BLK_HDR_FLG),
+ crr);
if (!(flags & LAST_BLK_HDR_FLG)) {
SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz);
@@ -1291,9 +1312,11 @@ mbc_alloc_finalize(Allctr_t *allctr,
|| nxt_blk == PREV_BLK(NXT_BLK(nxt_blk)));
ASSERT((flags & LAST_BLK_HDR_FLG)
|| IS_PREV_BLK_FREE(NXT_BLK(nxt_blk)));
- ASSERT(nxt_blk_sz == BLK_SZ(nxt_blk));
+ ASSERT(nxt_blk_sz == MBC_BLK_SZ(nxt_blk));
ASSERT(nxt_blk_sz % sizeof(Unit_t) == 0);
ASSERT(nxt_blk_sz >= allctr->min_block_size);
+ ASSERT(GET_MB_CARRIER(blk) == crr);
+ ASSERT(GET_MB_CARRIER(nxt_blk) == crr);
}
else {
blk_sz = org_blk_sz;
@@ -1301,17 +1324,13 @@ mbc_alloc_finalize(Allctr_t *allctr,
if (valid_blk_info)
SET_BLK_ALLOCED(blk);
else
- SET_BLK_HDR(blk,
- blk_sz,
- SBH_THIS_ALLOCED|SBH_LAST_BLK|prev_free_flg);
+ SET_MBC_BLK_HDR(blk, blk_sz, SBH_LAST_BLK|prev_free_flg, crr);
}
else {
if (valid_blk_info)
SET_BLK_ALLOCED(blk);
else
- SET_BLK_HDR(blk,
- blk_sz,
- SBH_THIS_ALLOCED|SBH_NOT_LAST_BLK|prev_free_flg);
+ SET_MBC_BLK_HDR(blk, blk_sz, prev_free_flg, crr);
nxt_blk = NXT_BLK(blk);
SET_PREV_BLK_ALLOCED(nxt_blk);
}
@@ -1319,12 +1338,13 @@ mbc_alloc_finalize(Allctr_t *allctr,
ASSERT((flags & LAST_BLK_HDR_FLG)
? IS_LAST_BLK(blk)
: IS_NOT_LAST_BLK(blk));
+ ASSERT(GET_MB_CARRIER(blk) == crr);
}
STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs);
ASSERT(IS_ALLOCED_BLK(blk));
- ASSERT(blk_sz == BLK_SZ(blk));
+ ASSERT(blk_sz == MBC_BLK_SZ(blk));
ASSERT(blk_sz % sizeof(Unit_t) == 0);
ASSERT(blk_sz >= allctr->min_block_size);
ASSERT(blk_sz >= want_blk_sz);
@@ -1348,7 +1368,7 @@ mbc_alloc(Allctr_t *allctr, Uint size)
if (IS_MBC_BLK(blk))
mbc_alloc_finalize(allctr,
blk,
- BLK_SZ(blk),
+ MBC_BLK_SZ(blk),
GET_BLK_HDR_FLGS(blk),
blk_sz,
1,
@@ -1370,7 +1390,7 @@ mbc_free(Allctr_t *allctr, void *p)
ASSERT(p);
blk = UMEM2BLK(p);
- blk_sz = BLK_SZ(blk);
+ blk_sz = MBC_BLK_SZ(blk);
if (blk_sz < allctr->sbmbc_threshold)
alcu_flgs |= ERTS_ALCU_FLG_SBMBC;
@@ -1389,9 +1409,9 @@ mbc_free(Allctr_t *allctr, void *p)
blk = PREV_BLK(blk);
(*allctr->unlink_free_block)(allctr, blk, alcu_flgs);
- blk_sz += BLK_SZ(blk);
+ blk_sz += MBC_BLK_SZ(blk);
is_first_blk = IS_FIRST_BLK(blk);
- SET_BLK_SZ(blk, blk_sz);
+ SET_MBC_BLK_SZ(blk, blk_sz);
}
else {
SET_BLK_FREE(blk);
@@ -1404,8 +1424,8 @@ mbc_free(Allctr_t *allctr, void *p)
if (IS_FREE_BLK(nxt_blk)) {
/* Coalesce with next block... */
(*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs);
- blk_sz += BLK_SZ(nxt_blk);
- SET_BLK_SZ(blk, blk_sz);
+ blk_sz += MBC_BLK_SZ(nxt_blk);
+ SET_MBC_BLK_SZ(blk, blk_sz);
is_last_blk = IS_LAST_BLK(nxt_blk);
if (is_last_blk)
@@ -1428,7 +1448,7 @@ mbc_free(Allctr_t *allctr, void *p)
ASSERT(IS_FREE_BLK(blk));
ASSERT(is_first_blk || IS_PREV_BLK_ALLOCED(blk));
ASSERT(is_last_blk || IS_PREV_BLK_FREE(NXT_BLK(blk)));
- ASSERT(blk_sz == BLK_SZ(blk));
+ ASSERT(blk_sz == MBC_BLK_SZ(blk));
ASSERT(is_last_blk || blk == PREV_BLK(NXT_BLK(blk)));
ASSERT(blk_sz % sizeof(Unit_t) == 0);
ASSERT(IS_MBC_BLK(blk));
@@ -1472,7 +1492,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
ASSERT(size < allctr->sbc_threshold);
blk = (Block_t *) UMEM2BLK(p);
- old_blk_sz = BLK_SZ(blk);
+ old_blk_sz = MBC_BLK_SZ(blk);
ASSERT(old_blk_sz >= allctr->min_block_size);
@@ -1497,6 +1517,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
return p;
else if (blk_sz < old_blk_sz) {
/* Shrink block... */
+ Carrier_t* crr;
Block_t *nxt_nxt_blk;
Uint diff_sz_val = old_blk_sz - blk_sz;
Uint old_blk_sz_val = old_blk_sz;
@@ -1525,7 +1546,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
if (!is_last_blk) {
nxt_blk = NXT_BLK(blk);
if (IS_FREE_BLK(nxt_blk))
- cand_blk_sz += BLK_SZ(nxt_blk);
+ cand_blk_sz += MBC_BLK_SZ(nxt_blk);
}
new_blk = (*allctr->get_free_block)(allctr,
@@ -1547,46 +1568,42 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
HARD_CHECK_BLK_CARRIER(allctr, blk);
- SET_BLK_SZ(blk, blk_sz);
+ nxt_nxt_blk = NXT_BLK(blk);
+
+ SET_MBC_BLK_SZ(blk, blk_sz);
SET_NOT_LAST_BLK(blk);
nxt_blk = NXT_BLK(blk);
- SET_BLK_HDR(nxt_blk,
- nxt_blk_sz,
- SBH_THIS_FREE|SBH_PREV_ALLOCED|SBH_NOT_LAST_BLK);
STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs);
STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs);
- ASSERT(BLK_SZ(blk) >= allctr->min_block_size);
+ ASSERT(MBC_BLK_SZ(blk) >= allctr->min_block_size);
- if (is_last_blk)
- SET_LAST_BLK(nxt_blk);
- else {
- nxt_nxt_blk = NXT_BLK(nxt_blk);
+ if (!is_last_blk) {
if (IS_FREE_BLK(nxt_nxt_blk)) {
/* Coalesce with next free block... */
- nxt_blk_sz += BLK_SZ(nxt_nxt_blk);
+ nxt_blk_sz += MBC_BLK_SZ(nxt_nxt_blk);
(*allctr->unlink_free_block)(allctr, nxt_nxt_blk, alcu_flgs);
- SET_BLK_SZ(nxt_blk, nxt_blk_sz);
- is_last_blk = IS_LAST_BLK(nxt_nxt_blk);
- if (is_last_blk)
- SET_LAST_BLK(nxt_blk);
- else
- SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz);
+ is_last_blk = GET_LAST_BLK_HDR_FLG(nxt_nxt_blk);
}
else {
- SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz);
SET_PREV_BLK_FREE(nxt_nxt_blk);
}
+ SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz);
}
+ crr = GET_MB_CARRIER(blk);
+ SET_MBC_BLK_HDR(nxt_blk, nxt_blk_sz,
+ SBH_THIS_FREE | (is_last_blk ? SBH_LAST_BLK : 0),
+ crr);
+
(*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs);
ASSERT(IS_ALLOCED_BLK(blk));
- ASSERT(blk_sz == BLK_SZ(blk));
+ ASSERT(blk_sz == MBC_BLK_SZ(blk));
ASSERT(blk_sz % sizeof(Unit_t) == 0);
ASSERT(blk_sz >= allctr->min_block_size);
ASSERT(blk_sz >= size + ABLK_HDR_SZ);
@@ -1594,14 +1611,15 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
ASSERT(IS_FREE_BLK(nxt_blk));
ASSERT(IS_PREV_BLK_ALLOCED(nxt_blk));
- ASSERT(nxt_blk_sz == BLK_SZ(nxt_blk));
+ ASSERT(nxt_blk_sz == MBC_BLK_SZ(nxt_blk));
ASSERT(nxt_blk_sz % sizeof(Unit_t) == 0);
ASSERT(nxt_blk_sz >= allctr->min_block_size);
ASSERT(IS_MBC_BLK(nxt_blk));
ASSERT(is_last_blk ? IS_LAST_BLK(nxt_blk) : IS_NOT_LAST_BLK(nxt_blk));
ASSERT(is_last_blk || nxt_blk == PREV_BLK(NXT_BLK(nxt_blk)));
ASSERT(is_last_blk || IS_PREV_BLK_FREE(NXT_BLK(nxt_blk)));
-
+ ASSERT(GET_MB_CARRIER(nxt_blk) == crr);
+
HARD_CHECK_BLK_CARRIER(allctr, blk);
return p;
@@ -1611,7 +1629,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
if (!is_last_blk) {
nxt_blk = NXT_BLK(blk);
- nxt_blk_sz = BLK_SZ(nxt_blk);
+ nxt_blk_sz = MBC_BLK_SZ(nxt_blk);
if (IS_FREE_BLK(nxt_blk) && get_blk_sz <= old_blk_sz + nxt_blk_sz) {
/* Grow into next block... */
@@ -1624,7 +1642,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
if (nxt_blk_sz < allctr->min_block_size) {
blk_sz += nxt_blk_sz;
- SET_BLK_SZ(blk, blk_sz);
+ SET_MBC_BLK_SZ(blk, blk_sz);
if (is_last_blk) {
SET_LAST_BLK(blk);
@@ -1637,17 +1655,16 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
SET_PREV_BLK_ALLOCED(nxt_blk);
#ifdef DEBUG
is_last_blk = IS_LAST_BLK(nxt_blk);
- nxt_blk_sz = BLK_SZ(nxt_blk);
+ nxt_blk_sz = MBC_BLK_SZ(nxt_blk);
#endif
}
}
else {
- SET_BLK_SZ(blk, blk_sz);
+ Carrier_t* crr = GET_MB_CARRIER(blk);
+ SET_MBC_BLK_SZ(blk, blk_sz);
nxt_blk = NXT_BLK(blk);
- SET_BLK_HDR(nxt_blk,
- nxt_blk_sz,
- SBH_THIS_FREE|SBH_PREV_ALLOCED|SBH_NOT_LAST_BLK);
+ SET_MBC_BLK_HDR(nxt_blk, nxt_blk_sz, SBH_THIS_FREE, crr);
if (is_last_blk)
SET_LAST_BLK(nxt_blk);
@@ -1657,6 +1674,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
(*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs);
ASSERT(IS_FREE_BLK(nxt_blk));
+ ASSERT(GET_MB_CARRIER(nxt_blk) == crr);
}
STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs);
@@ -1664,14 +1682,14 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
ASSERT(IS_ALLOCED_BLK(blk));
- ASSERT(blk_sz == BLK_SZ(blk));
+ ASSERT(blk_sz == MBC_BLK_SZ(blk));
ASSERT(blk_sz % sizeof(Unit_t) == 0);
ASSERT(blk_sz >= allctr->min_block_size);
ASSERT(blk_sz >= size + ABLK_HDR_SZ);
ASSERT(IS_MBC_BLK(blk));
ASSERT(!nxt_blk || IS_PREV_BLK_ALLOCED(nxt_blk));
- ASSERT(!nxt_blk || nxt_blk_sz == BLK_SZ(nxt_blk));
+ ASSERT(!nxt_blk || nxt_blk_sz == MBC_BLK_SZ(nxt_blk));
ASSERT(!nxt_blk || nxt_blk_sz % sizeof(Unit_t) == 0);
ASSERT(!nxt_blk || nxt_blk_sz >= allctr->min_block_size);
ASSERT(!nxt_blk || IS_MBC_BLK(nxt_blk));
@@ -1707,7 +1725,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
if (!is_last_blk) {
nxt_blk = NXT_BLK(blk);
if (IS_FREE_BLK(nxt_blk))
- cand_blk_sz += BLK_SZ(nxt_blk);
+ cand_blk_sz += MBC_BLK_SZ(nxt_blk);
}
}
@@ -1743,7 +1761,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
if (new_blk) {
mbc_alloc_finalize(allctr,
new_blk,
- BLK_SZ(new_blk),
+ MBC_BLK_SZ(new_blk),
GET_BLK_HDR_FLGS(new_blk),
blk_sz,
1,
@@ -1777,7 +1795,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
nxt_blk = NXT_BLK(blk);
if (IS_FREE_BLK(nxt_blk)) {
new_blk_flgs |= GET_LAST_BLK_HDR_FLG(nxt_blk);
- new_blk_sz += BLK_SZ(nxt_blk);
+ new_blk_sz += MBC_BLK_SZ(nxt_blk);
(*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs);
}
}
@@ -1815,35 +1833,37 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
#ifdef DEBUG
#if HAVE_ERTS_MSEG
-#define ASSERT_MSEG_UNIT_SIZE_MULTIPLE(CSZ) ASSERT((CSZ) % mseg_unit_size == 0)
+#define ASSERT_MSEG_UNIT_SIZE_MULTIPLE(CSZ) ASSERT((CSZ) % MSEG_UNIT_SZ == 0)
#else
#define ASSERT_MSEG_UNIT_SIZE_MULTIPLE(CSZ)
#endif
-#define CHECK_1BLK_CARRIER(A, SBC, MSEGED, C, CSZ, B, BSZ) \
-do { \
- ASSERT(IS_FIRST_BLK((B))); \
- ASSERT(IS_LAST_BLK((B))); \
- ASSERT((CSZ) == CARRIER_SZ((C))); \
- ASSERT((BSZ) == BLK_SZ((B))); \
- ASSERT((BSZ) % sizeof(Unit_t) == 0); \
- if ((SBC)) { \
- ASSERT(IS_SBC_BLK((B))); \
- ASSERT(IS_SB_CARRIER((C))); \
- } \
- else { \
- ASSERT(IS_MBC_BLK((B))); \
- ASSERT(IS_MB_CARRIER((C))); \
- } \
- if ((MSEGED)) { \
- ASSERT(IS_MSEG_CARRIER((C))); \
- ASSERT_MSEG_UNIT_SIZE_MULTIPLE((CSZ)); \
- } \
- else { \
- ASSERT(IS_SYS_ALLOC_CARRIER((C))); \
- ASSERT((CSZ) % sizeof(Unit_t) == 0); \
- } \
-} while (0)
+static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C,
+ UWord CSZ, Block_t* B, UWord BSZ)
+{
+ ASSERT(IS_FIRST_BLK((B)));
+ ASSERT(IS_LAST_BLK((B)));
+ ASSERT((CSZ) == CARRIER_SZ((C)));
+ ASSERT((BSZ) == BLK_SZ((B)));
+ ASSERT((BSZ) % sizeof(Unit_t) == 0);
+ if ((SBC)) {
+ ASSERT(IS_SBC_BLK((B)));
+ ASSERT(IS_SB_CARRIER((C)));
+ }
+ else {
+ ASSERT(IS_MBC_BLK((B)));
+ ASSERT(IS_MB_CARRIER((C)));
+ ASSERT(GET_MB_CARRIER(B) == (C));
+ }
+ if ((MSEGED)) {
+ ASSERT(IS_MSEG_CARRIER((C)));
+ ASSERT_MSEG_UNIT_SIZE_MULTIPLE((CSZ));
+ }
+ else {
+ ASSERT(IS_SYS_ALLOC_CARRIER((C)));
+ ASSERT((CSZ) % sizeof(Unit_t) == 0);
+ }
+}
#else
#define CHECK_1BLK_CARRIER(A, SBC, MSEGED, C, CSZ, B, BSZ)
@@ -1869,33 +1889,15 @@ create_sbmbc(Allctr_t *allctr, Uint umem_sz)
blk = MBC2FBLK(allctr, crr);
-#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
- if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
- crr_sz -= sizeof(UWord);
-#endif
-
blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size);
SET_MBC_BLK_FTR(((UWord *) blk)[-1]);
- SET_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_PREV_FREE|SBH_LAST_BLK);
-
-#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
- *((Carrier_t **) NXT_BLK(blk)) = crr;
-#endif
+ SET_MBC_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_PREV_FREE|SBH_LAST_BLK, crr);
link_carrier(&allctr->sbmbc_list, crr);
-#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
- if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
- crr_sz += sizeof(UWord);
-#endif
-
STAT_SBMBC_ALLOC(allctr, crr_sz);
CHECK_1BLK_CARRIER(allctr, 0, 0, crr, crr_sz, blk, blk_sz);
-#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
- if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
- crr_sz -= sizeof(UWord);
-#endif
if (allctr->creating_mbc)
(*allctr->creating_mbc)(allctr, crr, ERTS_ALCU_FLG_SBMBC);
@@ -1957,9 +1959,19 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
int is_mseg = 0;
#endif
+#if HALFWORD_HEAP
+ flags |= CFLG_FORCE_MSEG;
+#elif HAVE_SUPER_ALIGNED_MB_CARRIERS
+ if (flags & CFLG_MBC) {
+ flags |= CFLG_FORCE_MSEG;
+ }
+#endif
+
ASSERT((flags & CFLG_SBC && !(flags & CFLG_MBC))
|| (flags & CFLG_MBC && !(flags & CFLG_SBC)));
+ ASSERT(!(flags & CFLG_FORCE_MSEG && flags & CFLG_FORCE_SYS_ALLOC));
+
blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz);
#if HAVE_ERTS_MSEG
@@ -1974,10 +1986,12 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
if (allctr->sbcs.curr.norm.mseg.no >= allctr->max_mseg_sbcs)
goto try_sys_alloc;
}
+#if !HAVE_SUPER_ALIGNED_MB_CARRIERS
else {
if (allctr->mbcs.curr.norm.mseg.no >= allctr->max_mseg_mbcs)
goto try_sys_alloc;
}
+#endif
try_mseg:
@@ -1988,13 +2002,9 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
crr_sz = (*allctr->get_next_mbc_size)(allctr);
if (crr_sz < allctr->mbc_header_size + blk_sz)
crr_sz = allctr->mbc_header_size + blk_sz;
-#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
- if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
- crr_sz += sizeof(UWord);
-#endif
}
crr_sz = MSEG_UNIT_CEILING(crr_sz);
- ASSERT(crr_sz % mseg_unit_size == 0);
+ ASSERT(crr_sz % MSEG_UNIT_SZ == 0);
crr = (Carrier_t *) alcu_mseg_alloc(allctr, &crr_sz);
if (!crr) {
@@ -2019,6 +2029,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
}
try_sys_alloc:
+
#endif /* #if HAVE_ERTS_MSEG */
if (flags & CFLG_SBC) {
@@ -2029,11 +2040,6 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
if (!(flags & CFLG_MAIN_CARRIER)
&& bcrr_sz < allctr->smallest_mbc_size)
bcrr_sz = allctr->smallest_mbc_size;
-#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
- if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
- bcrr_sz += sizeof(UWord);
-#endif
-
}
crr_sz = (flags & CFLG_FORCE_SIZE
@@ -2067,7 +2073,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
blk = SBC2BLK(allctr, crr);
SET_SBC_BLK_FTR(((UWord *) blk)[-1]);
- SET_BLK_HDR(blk, blk_sz, SBH_THIS_ALLOCED|SBH_PREV_FREE|SBH_LAST_BLK);
+ SET_SBC_BLK_HDR(blk, blk_sz, SBH_PREV_FREE|SBH_LAST_BLK);
link_carrier(&allctr->sbc_list, crr);
@@ -2084,19 +2090,10 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
blk = MBC2FBLK(allctr, crr);
-#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
- if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
- crr_sz -= sizeof(UWord);
-#endif
-
blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size);
SET_MBC_BLK_FTR(((UWord *) blk)[-1]);
- SET_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_PREV_FREE|SBH_LAST_BLK);
-
-#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
- *((Carrier_t **) NXT_BLK(blk)) = crr;
-#endif
+ SET_MBC_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_PREV_FREE|SBH_LAST_BLK, crr);
if (flags & CFLG_MAIN_CARRIER) {
ASSERT(!allctr->main_carrier);
@@ -2105,15 +2102,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
link_carrier(&allctr->mbc_list, crr);
-#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
- if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
- crr_sz += sizeof(UWord);
-#endif
CHECK_1BLK_CARRIER(allctr, 0, is_mseg, crr, crr_sz, blk, blk_sz);
-#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
- if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
- crr_sz -= sizeof(UWord);
-#endif
if (allctr->creating_mbc)
(*allctr->creating_mbc)(allctr, crr, 0);
@@ -2142,7 +2131,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
HARD_CHECK_BLK_CARRIER(allctr, old_blk);
- old_blk_sz = BLK_SZ(old_blk);
+ old_blk_sz = SBC_BLK_SZ(old_blk);
old_crr = BLK2SBC(allctr, old_blk);
old_crr_sz = CARRIER_SZ(old_crr);
ASSERT(IS_SB_CARRIER(old_crr));
@@ -2166,7 +2155,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
if (new_crr) {
SET_CARRIER_SZ(new_crr, new_crr_sz);
new_blk = SBC2BLK(allctr, new_crr);
- SET_BLK_SZ(new_blk, new_blk_sz);
+ SET_SBC_BLK_SZ(new_blk, new_blk_sz);
STAT_MSEG_SBC_ALLOC(allctr, new_crr_sz, new_blk_sz);
relink_carrier(&allctr->sbc_list, new_crr);
CHECK_1BLK_CARRIER(allctr, 1, 1, new_crr, new_crr_sz,
@@ -2174,6 +2163,11 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
DEBUG_SAVE_ALIGNMENT(new_crr);
return new_blk;
}
+#if HALFWORD_HEAP
+ /* Old carrier unchanged; restore stat */
+ STAT_MSEG_SBC_ALLOC(allctr, old_crr_sz, old_blk_sz);
+ return NULL;
+#endif
create_flags |= CFLG_FORCE_SYS_ALLOC; /* since mseg_realloc()
failed */
}
@@ -2208,7 +2202,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
sys_realloc_success:
SET_CARRIER_SZ(new_crr, new_crr_sz);
new_blk = SBC2BLK(allctr, new_crr);
- SET_BLK_SZ(new_blk, new_blk_sz);
+ SET_SBC_BLK_SZ(new_blk, new_blk_sz);
STAT_SYS_ALLOC_SBC_FREE(allctr, old_crr_sz, old_blk_sz);
STAT_SYS_ALLOC_SBC_ALLOC(allctr, new_crr_sz, new_blk_sz);
relink_carrier(&allctr->sbc_list, new_crr);
@@ -2265,7 +2259,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
ASSERT(IS_FIRST_BLK(blk));
if (IS_SBC_BLK(blk)) {
- Uint blk_sz = BLK_SZ(blk);
+ Uint blk_sz = SBC_BLK_SZ(blk);
crr = BLK2SBC(allctr, blk);
crr_sz = CARRIER_SZ(crr);
@@ -2276,7 +2270,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
#if HAVE_ERTS_MSEG
if (IS_MSEG_CARRIER(crr)) {
is_mseg++;
- ASSERT(crr_sz % mseg_unit_size == 0);
+ ASSERT(crr_sz % MSEG_UNIT_SZ == 0);
STAT_MSEG_SBC_FREE(allctr, crr_sz, blk_sz);
}
else
@@ -2305,7 +2299,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
#if HAVE_ERTS_MSEG
if (IS_MSEG_CARRIER(crr)) {
is_mseg++;
- ASSERT(crr_sz % mseg_unit_size == 0);
+ ASSERT(crr_sz % MSEG_UNIT_SZ == 0);
STAT_MSEG_MBC_FREE(allctr, crr_sz);
}
else
@@ -3430,12 +3424,7 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size)
if (allctr->dd.use)
ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1);
#endif
-#if HALFWORD_HEAP
- blk = create_carrier(allctr, size,
- CFLG_SBC | CFLG_FORCE_MSEG);
-#else
blk = create_carrier(allctr, size, CFLG_SBC);
-#endif
res = blk ? BLK2UMEM(blk) : NULL;
}
else
@@ -3775,7 +3764,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
if (res) {
sys_memcpy((void*) res,
(void*) p,
- MIN(BLK_SZ(blk) - ABLK_HDR_SZ, size));
+ MIN(SBC_BLK_SZ(blk) - ABLK_HDR_SZ, size));
destroy_carrier(allctr, blk);
}
}
@@ -3798,16 +3787,12 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
else if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
return NULL;
else {
-#if HALFWORD_HEAP
- new_blk = create_carrier(allctr, size, CFLG_SBC | CFLG_FORCE_MSEG);
-#else
new_blk = create_carrier(allctr, size, CFLG_SBC);
-#endif
if (new_blk) {
res = BLK2UMEM(new_blk);
sys_memcpy((void *) res,
(void *) p,
- MIN(BLK_SZ(blk) - ABLK_HDR_SZ, size));
+ MIN(MBC_BLK_SZ(blk) - ABLK_HDR_SZ, size));
mbc_free(allctr, p);
}
else
@@ -4120,7 +4105,11 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
allctr->mbc_move_threshold = init->rmbcmt;
#if HAVE_ERTS_MSEG
allctr->max_mseg_sbcs = init->mmsbc;
+# if HAVE_SUPER_ALIGNED_MB_CARRIERS
+ allctr->max_mseg_mbcs = ~(Uint)0;
+# else
allctr->max_mseg_mbcs = init->mmmbc;
+# endif
#endif
allctr->largest_mbc_size = MAX(init->lmbcs, init->smbcs);
@@ -4241,21 +4230,14 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
if (allctr->main_carrier_size) {
Block_t *blk;
-#if HALFWORD_HEAP
- blk = create_carrier(allctr,
- allctr->main_carrier_size,
- CFLG_MBC
- | CFLG_FORCE_SIZE
- | CFLG_FORCE_MSEG
- | CFLG_MAIN_CARRIER);
-#else
blk = create_carrier(allctr,
allctr->main_carrier_size,
CFLG_MBC
| CFLG_FORCE_SIZE
+#if !HALFWORD_HEAP && !HAVE_SUPER_ALIGNED_MB_CARRIERS
| CFLG_FORCE_SYS_ALLOC
- | CFLG_MAIN_CARRIER);
#endif
+ | CFLG_MAIN_CARRIER);
if (!blk)
goto error;
@@ -4321,15 +4303,12 @@ erts_alcu_init(AlcUInit_t *init)
{
#if HAVE_ERTS_MSEG
- mseg_unit_size = erts_mseg_unit_size();
-
- if (mseg_unit_size % sizeof(Unit_t)) /* A little paranoid... */
- erl_exit(-1,
- "Mseg unit size (%d) not evenly divideble by "
- "internal unit size of alloc_util (%d)\n",
- mseg_unit_size,
- sizeof(Unit_t));
-
+ ASSERT(erts_mseg_unit_size() == MSEG_UNIT_SZ);
+# if HAVE_SUPER_ALIGNED_MB_CARRIERS
+ /*SVERK Add assert about CARRIER_OFFSET_BITS and max MBC size
+ *SVERK Add assert about CARRIER_OFFSET_SHIFT and max MBC block size
+ */
+# endif
max_mseg_carriers = init->mmc;
sys_alloc_carrier_size = MSEG_UNIT_CEILING(init->ycs);
#else /* #if HAVE_ERTS_MSEG */
@@ -4432,6 +4411,13 @@ erts_alcu_verify_unused_ts(Allctr_t *allctr)
#endif
}
+#ifdef DEBUG
+int is_sbc_blk(Block_t* blk)
+{
+ return IS_SBC_BLK(blk);
+}
+#endif
+
#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
static void
@@ -4447,28 +4433,34 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
ASSERT(IS_ALLOCED_BLK(iblk));
ASSERT(IS_FIRST_BLK(iblk));
ASSERT(IS_LAST_BLK(iblk));
- ASSERT(CARRIER_SZ(sbc) - allctr->sbc_header_size >= BLK_SZ(iblk));
+ ASSERT(CARRIER_SZ(sbc) - allctr->sbc_header_size >= SBC_BLK_SZ(iblk));
#if HAVE_ERTS_MSEG
if (IS_MSEG_CARRIER(sbc)) {
- ASSERT(CARRIER_SZ(sbc) % mseg_unit_size == 0);
+ ASSERT(CARRIER_SZ(sbc) % MSEG_UNIT_SZ == 0);
}
#endif
crr = sbc;
cl = &allctr->sbc_list;
}
else {
- Carrier_t *mbc = NULL;
Block_t *prev_blk = NULL;
Block_t *blk;
char *carrier_end;
Uint is_free_blk;
Uint tot_blk_sz;
Uint blk_sz;
+ int has_wrapped_around = 0;
blk = iblk;
tot_blk_sz = 0;
+ crr = GET_MB_CARRIER(blk);
+ ASSERT(IS_MB_CARRIER(crr));
+ /* Step around the carrier one whole lap starting at 'iblk'
+ */
while (1) {
+ ASSERT(IS_MBC_BLK(blk));
+ ASSERT(GET_MB_CARRIER(blk) == crr);
if (prev_blk) {
ASSERT(NXT_BLK(prev_blk) == blk);
@@ -4481,18 +4473,16 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
}
}
- if (mbc) {
+ if (has_wrapped_around) {
+ ASSERT(((Block_t *) crr) < blk);
if (blk == iblk)
break;
- ASSERT(((Block_t *) mbc) < blk && blk < iblk);
+ ASSERT(blk < iblk);
}
else
ASSERT(blk >= iblk);
-
- ASSERT(IS_MBC_BLK(blk));
-
- blk_sz = BLK_SZ(blk);
+ blk_sz = MBC_BLK_SZ(blk);
ASSERT(blk_sz % sizeof(Unit_t) == 0);
ASSERT(blk_sz >= allctr->min_block_size);
@@ -4511,9 +4501,9 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
if (IS_LAST_BLK(blk)) {
carrier_end = ((char *) NXT_BLK(blk));
- mbc = *((Carrier_t **) NXT_BLK(blk));
+ has_wrapped_around = 1;
prev_blk = NULL;
- blk = MBC2FBLK(allctr, mbc);
+ blk = MBC2FBLK(allctr, crr);
ASSERT(IS_FIRST_BLK(blk));
}
else {
@@ -4521,23 +4511,21 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
blk = NXT_BLK(blk);
}
}
-
- ASSERT(IS_MB_CARRIER(mbc));
- ASSERT((((char *) mbc)
+
+ ASSERT((((char *) crr)
+ allctr->mbc_header_size
+ tot_blk_sz) == carrier_end);
- ASSERT(((char *) mbc) + CARRIER_SZ(mbc) - sizeof(Unit_t) <= carrier_end
- && carrier_end <= ((char *) mbc) + CARRIER_SZ(mbc));
+ ASSERT(((char *) crr) + CARRIER_SZ(crr) - sizeof(Unit_t) <= carrier_end
+ && carrier_end <= ((char *) crr) + CARRIER_SZ(crr));
if (allctr->check_mbc)
- (*allctr->check_mbc)(allctr, mbc);
+ (*allctr->check_mbc)(allctr, crr);
#if HAVE_ERTS_MSEG
- if (IS_MSEG_CARRIER(mbc)) {
- ASSERT(CARRIER_SZ(mbc) % mseg_unit_size == 0);
+ if (IS_MSEG_CARRIER(crr)) {
+ ASSERT(CARRIER_SZ(crr) % MSEG_UNIT_SZ == 0);
}
#endif
- crr = mbc;
cl = &allctr->mbc_list;
}
@@ -4559,4 +4547,5 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
#endif
}
-#endif
+#endif /* ERTS_ALLOC_UTIL_HARD_DEBUG */
+
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index cedf4ccf85..f440752c4a 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -216,16 +216,27 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);
#define UNIT_FLOOR(X) ((X) & UNIT_MASK)
#define UNIT_CEILING(X) UNIT_FLOOR((X) + INV_UNIT_MASK)
+#define FLG_MASK INV_UNIT_MASK
+#define SBC_BLK_SZ_MASK UNIT_MASK
+#define CARRIER_SZ_MASK UNIT_MASK
+
+
+#if HAVE_SUPER_ALIGNED_MB_CARRIERS
+# define CARRIER_OFFSET_BITS 13
+# define CARRIER_OFFSET_SHIFT (sizeof(UWord)*8 - CARRIER_OFFSET_BITS)
+# define CARRIER_OFFSET_MASK (~((UWord)0) << CARRIER_OFFSET_SHIFT)
+# define MBC_BLK_SZ_MASK (~CARRIER_OFFSET_MASK & ~FLG_MASK)
+#else
+# define MBC_BLK_SZ_MASK (~FLG_MASK)
+#endif
-#define SZ_MASK (~((UWord) 0) << 3)
-#define FLG_MASK (~(SZ_MASK))
-
+#define MBC_BLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_BLK_SZ_MASK)
+#define SBC_BLK_SZ(B) (ASSERT_EXPR(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK)
-#define BLK_SZ(B) \
- (*((Block_t *) (B)) & SZ_MASK)
+#define BLK_SZ(B) ((B)->bhdr & (IS_SBC_BLK(B) ? SBC_BLK_SZ_MASK : MBC_BLK_SZ_MASK))
#define CARRIER_SZ(C) \
- ((C)->chdr & SZ_MASK)
+ ((C)->chdr & CARRIER_SZ_MASK)
extern int erts_have_sbmbc_alloc;
@@ -243,7 +254,12 @@ typedef struct {
Carrier_t *last;
} CarrierList_t;
-typedef UWord Block_t;
+typedef struct {
+ UWord bhdr;
+#if !HAVE_SUPER_ALIGNED_MB_CARRIERS
+ Carrier_t *carrier;
+#endif
+} Block_t;
typedef UWord FreeBlkFtr_t;
typedef struct {
@@ -469,6 +485,9 @@ void erts_alcu_verify_unused_ts(Allctr_t *allctr);
unsigned long erts_alcu_test(unsigned long, unsigned long, unsigned long);
+#ifdef DEBUG
+int is_sbc_blk(Block_t*);
+#endif
#endif /* #if defined(GET_ERL_ALLOC_UTIL_IMPL)
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
index 5bdb752d3a..4768dcc939 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.c
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -91,6 +91,7 @@ struct AOFF_RBTree_t_ {
AOFF_RBTree_t *right;
Uint max_sz; /* of all blocks in this sub-tree */
};
+#define AOFF_BLK_SZ(B) MBC_BLK_SZ(&(B)->hdr)
#ifdef HARD_DEBUG
static AOFF_RBTree_t * check_tree(AOFF_RBTree_t* root, Uint);
@@ -102,7 +103,7 @@ static AOFF_RBTree_t * check_tree(AOFF_RBTree_t* root, Uint);
*/
static ERTS_INLINE Uint node_max_size(AOFF_RBTree_t *x)
{
- Uint sz = BLK_SZ(x);
+ Uint sz = AOFF_BLK_SZ(x);
if (x->left && x->left->max_sz > sz) {
sz = x->left->max_sz;
}
@@ -587,7 +588,7 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
AOFF_RBTree_t *blk = (AOFF_RBTree_t *) block;
AOFF_RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC)
? &alc->sbmbc_root : &alc->mbc_root);
- Uint blk_sz = BLK_SZ(blk);
+ Uint blk_sz = AOFF_BLK_SZ(blk);
#ifdef HARD_DEBUG
check_tree(*root, 0);
@@ -659,7 +660,7 @@ aoff_get_free_block(Allctr_t *allctr, Uint size,
if (x->left && x->left->max_sz >= size) {
x = x->left;
}
- else if (BLK_SZ(x) >= size) {
+ else if (AOFF_BLK_SZ(x) >= size) {
blk = x;
break;
}
@@ -910,12 +911,12 @@ check_tree(AOFF_RBTree_t* root, Uint size)
ASSERT(x->right > x);
ASSERT(x->right->max_sz <= x->max_sz);
}
- ASSERT(x->max_sz >= BLK_SZ(x));
- ASSERT(x->max_sz == BLK_SZ(x)
+ ASSERT(x->max_sz >= AOFF_BLK_SZ(x));
+ ASSERT(x->max_sz == AOFF_BLK_SZ(x)
|| x->max_sz == (x->left ? x->left->max_sz : 0)
|| x->max_sz == (x->right ? x->right->max_sz : 0));
- if (size && BLK_SZ(x) >= size) {
+ if (size && AOFF_BLK_SZ(x) >= size) {
if (!res || x < res) {
res = x;
}
@@ -956,7 +957,7 @@ print_tree_aux(AOFF_RBTree_t *x, int indent)
}
fprintf(stderr, "%s: sz=%lu addr=0x%lx max_size=%lu\r\n",
IS_BLACK(x) ? "BLACK" : "RED",
- BLK_SZ(x), (Uint)x, x->max_sz);
+ AOFF_BLK_SZ(x), (Uint)x, x->max_sz);
print_tree_aux(x->left, indent + INDENT_STEP);
}
}
diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c
index c50fdeb4e8..619206bbcd 100644
--- a/erts/emulator/beam/erl_bestfit_alloc.c
+++ b/erts/emulator/beam/erl_bestfit_alloc.c
@@ -73,6 +73,8 @@
#define SET_RED(N) (((RBTree_t *) (N))->flags |= RED_FLG)
#define SET_BLACK(N) (((RBTree_t *) (N))->flags &= ~RED_FLG)
+#define BF_BLK_SZ(B) MBC_BLK_SZ(&(B)->hdr)
+
#undef ASSERT
#define ASSERT ASSERT_EXPR
@@ -592,7 +594,7 @@ aobf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
? &bfallctr->sbmbc_root
: &bfallctr->mbc_root);
RBTree_t *blk = (RBTree_t *) block;
- Uint blk_sz = BLK_SZ(blk);
+ Uint blk_sz = BF_BLK_SZ(blk);
@@ -610,7 +612,7 @@ aobf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
while (1) {
Uint size;
- size = BLK_SZ(x);
+ size = BF_BLK_SZ(x);
if (blk_sz < size || (blk_sz == size && blk < x)) {
if (!x->left) {
@@ -668,7 +670,7 @@ aobf_get_free_block(Allctr_t *allctr, Uint size,
ASSERT(!cand_blk || cand_size >= size);
while (x) {
- blk_sz = BLK_SZ(x);
+ blk_sz = BF_BLK_SZ(x);
if (blk_sz < size) {
x = x->right;
}
@@ -686,7 +688,7 @@ aobf_get_free_block(Allctr_t *allctr, Uint size,
#endif
if (cand_blk) {
- blk_sz = BLK_SZ(blk);
+ blk_sz = BF_BLK_SZ(blk);
if (cand_size < blk_sz)
return NULL; /* cand_blk was better */
if (cand_size == blk_sz && ((void *) cand_blk) < ((void *) blk))
@@ -711,7 +713,7 @@ bf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
? &bfallctr->sbmbc_root
: &bfallctr->mbc_root);
RBTree_t *blk = (RBTree_t *) block;
- Uint blk_sz = BLK_SZ(blk);
+ Uint blk_sz = BF_BLK_SZ(blk);
SET_TREE_NODE(blk);
@@ -730,7 +732,7 @@ bf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
while (1) {
Uint size;
- size = BLK_SZ(x);
+ size = BF_BLK_SZ(x);
if (blk_sz == size) {
@@ -796,7 +798,7 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
else if (LIST_NEXT(x)) {
/* Replace tree node by next element in list... */
- ASSERT(BLK_SZ(LIST_NEXT(x)) == BLK_SZ(x));
+ ASSERT(BF_BLK_SZ(LIST_NEXT(x)) == BF_BLK_SZ(x));
ASSERT(IS_TREE_NODE(x));
ASSERT(IS_LIST_ELEM(LIST_NEXT(x)));
@@ -834,7 +836,7 @@ bf_get_free_block(Allctr_t *allctr, Uint size,
ASSERT(!cand_blk || cand_size >= size);
while (x) {
- blk_sz = BLK_SZ(x);
+ blk_sz = BF_BLK_SZ(x);
if (blk_sz < size) {
x = x->right;
}
@@ -855,11 +857,11 @@ bf_get_free_block(Allctr_t *allctr, Uint size,
#ifdef HARD_DEBUG
{
RBTree_t *ct_blk = check_tree(root, 0, size);
- ASSERT(BLK_SZ(ct_blk) == BLK_SZ(blk));
+ ASSERT(BF_BLK_SZ(ct_blk) == BF_BLK_SZ(blk));
}
#endif
- if (cand_blk && cand_size <= BLK_SZ(blk))
+ if (cand_blk && cand_size <= BF_BLK_SZ(blk))
return NULL; /* cand_blk was better */
/* Use next block if it exist in order to avoid replacing
@@ -1093,36 +1095,36 @@ check_tree(RBTree_t *root, int ao, Uint size)
if (x->left) {
ASSERT(x->left->parent == x);
if (ao) {
- ASSERT(BLK_SZ(x->left) < BLK_SZ(x)
- || (BLK_SZ(x->left) == BLK_SZ(x) && x->left < x));
+ ASSERT(BF_BLK_SZ(x->left) < BF_BLK_SZ(x)
+ || (BF_BLK_SZ(x->left) == BF_BLK_SZ(x) && x->left < x));
}
else {
ASSERT(IS_TREE_NODE(x->left));
- ASSERT(BLK_SZ(x->left) < BLK_SZ(x));
+ ASSERT(BF_BLK_SZ(x->left) < BF_BLK_SZ(x));
}
}
if (x->right) {
ASSERT(x->right->parent == x);
if (ao) {
- ASSERT(BLK_SZ(x->right) > BLK_SZ(x)
- || (BLK_SZ(x->right) == BLK_SZ(x) && x->right > x));
+ ASSERT(BF_BLK_SZ(x->right) > BF_BLK_SZ(x)
+ || (BF_BLK_SZ(x->right) == BF_BLK_SZ(x) && x->right > x));
}
else {
ASSERT(IS_TREE_NODE(x->right));
- ASSERT(BLK_SZ(x->right) > BLK_SZ(x));
+ ASSERT(BF_BLK_SZ(x->right) > BF_BLK_SZ(x));
}
}
- if (size && BLK_SZ(x) >= size) {
+ if (size && BF_BLK_SZ(x) >= size) {
if (ao) {
if (!res
- || BLK_SZ(x) < BLK_SZ(res)
- || (BLK_SZ(x) == BLK_SZ(res) && x < res))
+ || BF_BLK_SZ(x) < BF_BLK_SZ(res)
+ || (BF_BLK_SZ(x) == BF_BLK_SZ(res) && x < res))
res = x;
}
else {
- if (!res || BLK_SZ(x) < BLK_SZ(res))
+ if (!res || BF_BLK_SZ(x) < BF_BLK_SZ(res))
res = x;
}
}
@@ -1168,7 +1170,7 @@ print_tree_aux(RBTree_t *x, int indent)
}
fprintf(stderr, "%s: sz=%lu addr=0x%lx\r\n",
IS_BLACK(x) ? "BLACK" : "RED",
- BLK_SZ(x),
+ BF_BLK_SZ(x),
(Uint) x);
print_tree_aux(x->left, indent + INDENT_STEP);
}
diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c
index e7d4ac2b67..9db89652f7 100644
--- a/erts/emulator/beam/erl_goodfit_alloc.c
+++ b/erts/emulator/beam/erl_goodfit_alloc.c
@@ -363,7 +363,7 @@ search_bucket(Allctr_t *allctr, int ix, Uint size)
blk && i < max_blk_search;
blk = blk->next, i++) {
- blk_sz = BLK_SZ(blk);
+ blk_sz = MBC_BLK_SZ(&blk->block_head);
blk_on_lambc = (((char *) blk) < gfallctr->last_aux_mbc_end
&& gfallctr->last_aux_mbc_start <= ((char *) blk));
@@ -402,7 +402,7 @@ get_free_block(Allctr_t *allctr, Uint size,
if (min_bi == unsafe_bi) {
blk = search_bucket(allctr, min_bi, size);
if (blk) {
- if (cand_blk && cand_size <= BLK_SZ(blk))
+ if (cand_blk && cand_size <= MBC_BLK_SZ(blk))
return NULL; /* cand_blk was better */
unlink_free_block(allctr, blk, flags);
return blk;
@@ -422,7 +422,7 @@ get_free_block(Allctr_t *allctr, Uint size,
/* We are guaranteed to find a block that fits in this bucket */
blk = search_bucket(allctr, min_bi, size);
ASSERT(blk);
- if (cand_blk && cand_size <= BLK_SZ(blk))
+ if (cand_blk && cand_size <= MBC_BLK_SZ(blk))
return NULL; /* cand_blk was better */
unlink_free_block(allctr, blk, flags);
return blk;
@@ -435,7 +435,7 @@ link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
{
GFAllctr_t *gfallctr = (GFAllctr_t *) allctr;
GFFreeBlock_t *blk = (GFFreeBlock_t *) block;
- Uint sz = BLK_SZ(blk);
+ Uint sz = MBC_BLK_SZ(&blk->block_head);
int i = BKT_IX(gfallctr, sz);
ASSERT(sz >= MIN_BLK_SZ);
@@ -456,7 +456,7 @@ unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
{
GFAllctr_t *gfallctr = (GFAllctr_t *) allctr;
GFFreeBlock_t *blk = (GFFreeBlock_t *) block;
- Uint sz = BLK_SZ(blk);
+ Uint sz = MBC_BLK_SZ(&blk->block_head);
int i = BKT_IX(gfallctr, sz);
if (!blk->prev) {
@@ -618,7 +618,7 @@ check_block(Allctr_t *allctr, Block_t * blk, int free_block)
GFFreeBlock_t *fblk;
if(free_block) {
- Uint blk_sz = BLK_SZ(blk);
+ Uint blk_sz = is_sbc_blk(blk) ? SBC_BLK_SZ(blk) : MBC_BLK_SZ(blk);
bi = BKT_IX(gfallctr, blk_sz);
ASSERT(gfallctr->bucket_mask.main & (((UWord) 1) << IX2SMIX(bi)));
diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h
index 644887462a..460e4a514a 100644
--- a/erts/emulator/sys/common/erl_mseg.h
+++ b/erts/emulator/sys/common/erl_mseg.h
@@ -32,8 +32,10 @@
#if HAVE_MMAP
# define HAVE_ERTS_MSEG 1
+# define HAVE_SUPER_ALIGNED_MB_CARRIERS 1
#else
# define HAVE_ERTS_MSEG 0
+# define HAVE_SUPER_ALIGNED_MB_CARRIERS 0
#endif
#if HAVE_ERTS_MSEG