aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator')
-rw-r--r--erts/emulator/beam/erl_afit_alloc.c2
-rw-r--r--erts/emulator/beam/erl_alloc_util.c203
-rw-r--r--erts/emulator/beam/erl_alloc_util.h17
-rw-r--r--erts/emulator/beam/erl_ao_firstfit_alloc.c2
-rw-r--r--erts/emulator/beam/erl_bestfit_alloc.c2
-rw-r--r--erts/emulator/beam/erl_goodfit_alloc.c10
-rw-r--r--erts/emulator/sys/common/erl_mseg.h6
7 files changed, 159 insertions, 83 deletions
diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c
index ad7a5822df..02171e050f 100644
--- a/erts/emulator/beam/erl_afit_alloc.c
+++ b/erts/emulator/beam/erl_afit_alloc.c
@@ -42,7 +42,7 @@ struct AFFreeBlock_t_ {
AFFreeBlock_t *prev;
AFFreeBlock_t *next;
};
-#define AF_BLK_SZ(B) MBC_BLK_SZ(&(B)->block_head)
+#define AF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->block_head)
#define MIN_MBC_SZ (16*1024)
#define MIN_MBC_FIRST_FREE_SZ (4*1024)
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 3f8d3f23c3..c8e2d28c49 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -153,7 +153,7 @@ MBC after deallocating first block:
#define UNUSED1_BLK_FTR_FLG (((UWord) 1) << 1)
#define UNUSED2_BLK_FTR_FLG (((UWord) 1) << 2)
-#define ABLK_HDR_SZ (sizeof(Block_t))
+#define ABLK_HDR_SZ (offsetof(Block_t,u))
#define FBLK_FTR_SZ (sizeof(FreeBlkFtr_t))
#define UMEMSZ2BLKSZ(AP, SZ) \
@@ -177,17 +177,15 @@ MBC after deallocating first block:
*/
#define SBC_BLK_HDR_FLG (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)
-#define SET_MBC_BLK_SZ(B, SZ) \
+#define SET_MBC_ABLK_SZ(B, SZ) \
(ASSERT(((SZ) & FLG_MASK) == 0), \
- (B)->bhdr = (((B)->bhdr) & ~MBC_BLK_SZ_MASK) | (SZ))
+ (B)->bhdr = (((B)->bhdr) & ~MBC_ABLK_SZ_MASK) | (SZ))
+#define SET_MBC_FBLK_SZ(B, SZ) \
+ (ASSERT(((SZ) & FLG_MASK) == 0), \
+ (B)->bhdr = (((B)->bhdr) & ~MBC_FBLK_SZ_MASK) | (SZ))
#define SET_SBC_BLK_SZ(B, SZ) \
(ASSERT(((SZ) & FLG_MASK) == 0), \
(B)->bhdr = (((B)->bhdr) & ~SBC_BLK_SZ_MASK) | (SZ))
-#define SET_BLK_FREE(B) \
- (ASSERT(!IS_PREV_BLK_FREE(B)), \
- (B)->bhdr |= THIS_FREE_BLK_HDR_FLG)
-#define SET_BLK_ALLOCED(B) \
- ((B)->bhdr &= ~THIS_FREE_BLK_HDR_FLG)
#define SET_PREV_BLK_FREE(AP,B) \
(ASSERT(!IS_MBC_FIRST_BLK(AP,B)), \
ASSERT(!IS_FREE_BLK(B)), \
@@ -208,32 +206,79 @@ MBC after deallocating first block:
# define BLK_CARRIER_OFFSET(B, C) (((char*)(B) - (char*)(C)) >> MSEG_UNIT_SHIFT)
-# define SET_MBC_BLK_HDR(B, Sz, F, C) \
- (ASSERT(((Sz) & FLG_MASK) == 0), \
- ASSERT((UWord)(F) < SBC_BLK_HDR_FLG), \
+# define SET_MBC_ABLK_HDR(B, Sz, F, C) \
+ (ASSERT(((Sz) & ~MBC_ABLK_SZ_MASK) == 0), \
+ ASSERT(!((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG))), \
(B)->bhdr = ((Sz) | (F) | (BLK_CARRIER_OFFSET(B,C) << CARRIER_OFFSET_SHIFT)))
-# define BLK_TO_MBC(B) \
- (ASSERT(IS_MBC_BLK(B)), \
+# define SET_MBC_FBLK_HDR(B, Sz, F, C) \
+ (ASSERT(((Sz) & ~MBC_FBLK_SZ_MASK) == 0), \
+ ASSERT(((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \
+ (B)->bhdr = ((Sz) | (F)), \
+ (B)->u.carrier = (C))
+
+# define ABLK_TO_MBC(B) \
+ (ASSERT(IS_MBC_BLK(B) && IS_ALLOCED_BLK(B)), \
(Carrier_t*)((MSEG_UNIT_FLOOR((UWord)(B)) - \
(((B)->bhdr >> CARRIER_OFFSET_SHIFT) << MSEG_UNIT_SHIFT))))
-# define IS_MBC_FIRST_BLK(AP,B) \
+# define FBLK_TO_MBC(B) \
+ (ASSERT(IS_MBC_BLK(B) && IS_FREE_BLK(B)), \
+ (B)->u.carrier)
+
+# define BLK_TO_MBC(B) (IS_FREE_BLK(B) ? FBLK_TO_MBC(B) : ABLK_TO_MBC(B))
+
+# define IS_MBC_FIRST_ABLK(AP,B) \
((((UWord)(B) & ~MSEG_UNIT_MASK) == (AP)->mbc_header_size) \
&& ((B)->bhdr & CARRIER_OFFSET_MASK) == 0)
+# define IS_MBC_FIRST_FBLK(AP,B) \
+ ((char*)(B) == (char*)((B)->u.carrier) + (AP)->mbc_header_size)
+
+# define IS_MBC_FIRST_BLK(AP,B) \
+ (IS_FREE_BLK(B) ? IS_MBC_FIRST_FBLK(AP,B) : IS_MBC_FIRST_ABLK(AP,B))
+
+# define SET_BLK_FREE(B) \
+ (ASSERT(!IS_PREV_BLK_FREE(B)), \
+ (B)->u.carrier = ABLK_TO_MBC(B), \
+ (B)->bhdr |= THIS_FREE_BLK_HDR_FLG, \
+ (B)->bhdr &= (MBC_ABLK_SZ_MASK|FLG_MASK))
+
+# define SET_BLK_ALLOCED(B) \
+ (ASSERT(((B)->bhdr & (CARRIER_OFFSET_MASK|THIS_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \
+ (B)->bhdr &= ~THIS_FREE_BLK_HDR_FLG, \
+ (B)->bhdr |= (BLK_CARRIER_OFFSET(B,(B)->u.carrier) << CARRIER_OFFSET_SHIFT))
+
#else /* !HAVE_SUPER_ALIGNED_MB_CARRIERS */
-# define SET_MBC_BLK_HDR(B, Sz, F, C) \
+# define SET_MBC_ABLK_HDR(B, Sz, F, C) \
(ASSERT(((Sz) & FLG_MASK) == 0), \
+ ASSERT(!((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG))), \
ASSERT((UWord)(F) < SBC_BLK_HDR_FLG), \
(B)->bhdr = ((Sz) | (F)), \
(B)->carrier = (C))
+# define SET_MBC_FBLK_HDR(B, Sz, F, C) \
+ (ASSERT(((Sz) & FLG_MASK) == 0), \
+ ASSERT(((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \
+ (B)->bhdr = ((Sz) | (F)), \
+ (B)->carrier = (C))
+
# define BLK_TO_MBC(B) ((B)->carrier)
+# define ABLK_TO_MBC(B) BLK_TO_MBC(B)
+# define FBLK_TO_MBC(B) BLK_TO_MBC(B)
# define IS_MBC_FIRST_BLK(AP,B) \
((char*)(B) == (char*)((B)->carrier) + (AP)->mbc_header_size)
+# define IS_MBC_FIRST_ABLK(AP,B) IS_MBC_FIRST_BLK(AP,B)
+# define IS_MBC_FIRST_FBLK(AP,B) IS_MBC_FIRST_BLK(AP,B)
+
+# define SET_BLK_FREE(B) \
+ (ASSERT(!IS_PREV_BLK_FREE(B)), \
+ (B)->bhdr |= THIS_FREE_BLK_HDR_FLG)
+
+# define SET_BLK_ALLOCED(B) \
+ ((B)->bhdr &= ~THIS_FREE_BLK_HDR_FLG)
#endif /* !HAVE_SUPER_ALIGNED_MB_CARRIERS */
@@ -270,12 +315,18 @@ MBC after deallocating first block:
#define IS_MBC_BLK(B) \
(!IS_SBC_BLK((B)))
+#define MBC_BLK_SZ(B) (IS_FREE_BLK(B) ? MBC_FBLK_SZ(B) : MBC_ABLK_SZ(B))
+
#define NXT_BLK(B) \
- ((Block_t *) (((char *) (B)) + MBC_BLK_SZ((B))))
+ (ASSERT(IS_MBC_BLK(B)), \
+ (Block_t *) (((char *) (B)) + MBC_BLK_SZ((B))))
#define PREV_BLK(B) \
((Block_t *) (((char *) (B)) - PREV_BLK_SZ((B))))
-#define BLK_SZ(B) ((B)->bhdr & (IS_SBC_BLK(B) ? SBC_BLK_SZ_MASK : MBC_BLK_SZ_MASK))
+#define BLK_AFTER(B,Sz) \
+ ((Block_t *) (((char *) (B)) + (Sz)))
+
+#define BLK_SZ(B) ((B)->bhdr & (((B)->bhdr & THIS_FREE_BLK_HDR_FLG) ? MBC_FBLK_SZ_MASK : MBC_ABLK_SZ_MASK))
/* Carriers ... */
@@ -846,10 +897,17 @@ static ERTS_INLINE Allctr_t*
get_used_allctr(void *extra, void *p, UWord *sizep)
{
Block_t* blk = UMEM2BLK(p);
- Carrier_t* crr = IS_SBC_BLK(blk) ? BLK_TO_SBC(blk) : BLK_TO_MBC(blk);
+ Carrier_t* crr;
- if (sizep) {
- *sizep = BLK_UMEM_SZ(blk);
+ if (IS_SBC_BLK(blk)) {
+ crr = BLK_TO_SBC(blk);
+ if (sizep)
+ *sizep = SBC_BLK_SZ(blk) - ABLK_HDR_SZ;
+ }
+ else {
+ crr = ABLK_TO_MBC(blk);
+ if (sizep)
+ *sizep = MBC_ABLK_SZ(blk) - ABLK_HDR_SZ;
}
return crr->allctr;
}
@@ -1283,6 +1341,7 @@ mbc_alloc_finalize(Allctr_t *allctr,
Block_t *blk,
Uint org_blk_sz,
UWord flags,
+ Carrier_t *crr,
Uint want_blk_sz,
int valid_blk_info,
Uint32 alcu_flgs)
@@ -1291,7 +1350,6 @@ mbc_alloc_finalize(Allctr_t *allctr,
Uint nxt_blk_sz;
Block_t *nxt_blk;
UWord prev_free_flg = flags & PREV_FREE_BLK_HDR_FLG;
- Carrier_t* crr = BLK_TO_MBC(blk);
ASSERT(org_blk_sz >= want_blk_sz);
ASSERT(blk);
@@ -1304,17 +1362,17 @@ mbc_alloc_finalize(Allctr_t *allctr,
/* Shrink block... */
blk_sz = want_blk_sz;
nxt_blk_sz = org_blk_sz - blk_sz;
- SET_MBC_BLK_HDR(blk, blk_sz, prev_free_flg, crr);
+ SET_MBC_ABLK_HDR(blk, blk_sz, prev_free_flg, crr);
- nxt_blk = NXT_BLK(blk);
- SET_MBC_BLK_HDR(nxt_blk, nxt_blk_sz,
- SBH_THIS_FREE|(flags & LAST_BLK_HDR_FLG),
- crr);
+ nxt_blk = BLK_AFTER(blk, blk_sz);
+ SET_MBC_FBLK_HDR(nxt_blk, nxt_blk_sz,
+ SBH_THIS_FREE|(flags & LAST_BLK_HDR_FLG),
+ crr);
if (!(flags & LAST_BLK_HDR_FLG)) {
SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz);
if (!valid_blk_info) {
- Block_t *nxt_nxt_blk = NXT_BLK(nxt_blk);
+ Block_t *nxt_nxt_blk = BLK_AFTER(nxt_blk, nxt_blk_sz);
SET_PREV_BLK_FREE(allctr, nxt_nxt_blk);
}
}
@@ -1332,8 +1390,8 @@ mbc_alloc_finalize(Allctr_t *allctr,
ASSERT(nxt_blk_sz == MBC_BLK_SZ(nxt_blk));
ASSERT(nxt_blk_sz % sizeof(Unit_t) == 0);
ASSERT(nxt_blk_sz >= allctr->min_block_size);
- ASSERT(BLK_TO_MBC(blk) == crr);
- ASSERT(BLK_TO_MBC(nxt_blk) == crr);
+ ASSERT(ABLK_TO_MBC(blk) == crr);
+ ASSERT(FBLK_TO_MBC(nxt_blk) == crr);
}
else {
blk_sz = org_blk_sz;
@@ -1341,21 +1399,21 @@ mbc_alloc_finalize(Allctr_t *allctr,
if (valid_blk_info)
SET_BLK_ALLOCED(blk);
else
- SET_MBC_BLK_HDR(blk, blk_sz, SBH_LAST_BLK|prev_free_flg, crr);
+ SET_MBC_ABLK_HDR(blk, blk_sz, SBH_LAST_BLK|prev_free_flg, crr);
}
else {
if (valid_blk_info)
SET_BLK_ALLOCED(blk);
else
- SET_MBC_BLK_HDR(blk, blk_sz, prev_free_flg, crr);
- nxt_blk = NXT_BLK(blk);
+ SET_MBC_ABLK_HDR(blk, blk_sz, prev_free_flg, crr);
+ nxt_blk = BLK_AFTER(blk, blk_sz);
SET_PREV_BLK_ALLOCED(nxt_blk);
}
ASSERT((flags & LAST_BLK_HDR_FLG)
? IS_LAST_BLK(blk)
: IS_NOT_LAST_BLK(blk));
- ASSERT(BLK_TO_MBC(blk) == crr);
+ ASSERT(ABLK_TO_MBC(blk) == crr);
}
STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs);
@@ -1387,6 +1445,7 @@ mbc_alloc(Allctr_t *allctr, Uint size)
blk,
MBC_BLK_SZ(blk),
GET_BLK_HDR_FLGS(blk),
+ FBLK_TO_MBC(blk),
blk_sz,
1,
alcu_flgs);
@@ -1418,7 +1477,7 @@ mbc_free(Allctr_t *allctr, void *p)
STAT_MBC_BLK_FREE(allctr, blk_sz, alcu_flgs);
- is_first_blk = IS_MBC_FIRST_BLK(allctr, blk);
+ is_first_blk = IS_MBC_FIRST_ABLK(allctr, blk);
is_last_blk = IS_LAST_BLK(blk);
if (IS_PREV_BLK_FREE(blk)) {
@@ -1428,8 +1487,8 @@ mbc_free(Allctr_t *allctr, void *p)
(*allctr->unlink_free_block)(allctr, blk, alcu_flgs);
blk_sz += MBC_BLK_SZ(blk);
- is_first_blk = IS_MBC_FIRST_BLK(allctr, blk);
- SET_MBC_BLK_SZ(blk, blk_sz);
+ is_first_blk = IS_MBC_FIRST_FBLK(allctr, blk);
+ SET_MBC_FBLK_SZ(blk, blk_sz);
}
else {
SET_BLK_FREE(blk);
@@ -1438,12 +1497,12 @@ mbc_free(Allctr_t *allctr, void *p)
if (is_last_blk)
SET_LAST_BLK(blk);
else {
- nxt_blk = NXT_BLK(blk);
+ nxt_blk = BLK_AFTER(blk, blk_sz);
if (IS_FREE_BLK(nxt_blk)) {
/* Coalesce with next block... */
(*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs);
- blk_sz += MBC_BLK_SZ(nxt_blk);
- SET_MBC_BLK_SZ(blk, blk_sz);
+ blk_sz += MBC_FBLK_SZ(nxt_blk);
+ SET_MBC_FBLK_SZ(blk, blk_sz);
is_last_blk = IS_LAST_BLK(nxt_blk);
if (is_last_blk)
@@ -1461,9 +1520,9 @@ mbc_free(Allctr_t *allctr, void *p)
}
- ASSERT(!is_last_blk == !IS_LAST_BLK(blk));
- ASSERT(!is_first_blk == !IS_MBC_FIRST_BLK(allctr, blk));
ASSERT(IS_FREE_BLK(blk));
+ ASSERT(!is_last_blk == !IS_LAST_BLK(blk));
+ ASSERT(!is_first_blk == !IS_MBC_FIRST_FBLK(allctr, blk));
ASSERT(is_first_blk || IS_PREV_BLK_ALLOCED(blk));
ASSERT(is_last_blk || IS_PREV_BLK_FREE(NXT_BLK(blk)));
ASSERT(blk_sz == MBC_BLK_SZ(blk));
@@ -1559,12 +1618,12 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
cand_blk = blk;
}
else {
- ASSERT(!IS_MBC_FIRST_BLK(allctr, blk));
+ ASSERT(!IS_MBC_FIRST_ABLK(allctr, blk));
cand_blk = PREV_BLK(blk);
cand_blk_sz += PREV_BLK_SZ(blk);
}
if (!is_last_blk) {
- nxt_blk = NXT_BLK(blk);
+ nxt_blk = BLK_AFTER(blk, old_blk_sz);
if (IS_FREE_BLK(nxt_blk))
cand_blk_sz += MBC_BLK_SZ(nxt_blk);
}
@@ -1582,18 +1641,18 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
nxt_blk_sz = old_blk_sz - blk_sz;
- if ((is_last_blk || IS_ALLOCED_BLK(NXT_BLK(blk)))
+ if ((is_last_blk || IS_ALLOCED_BLK(BLK_AFTER(blk,old_blk_sz)))
&& (nxt_blk_sz < allctr->min_block_size))
return p;
HARD_CHECK_BLK_CARRIER(allctr, blk);
- nxt_nxt_blk = NXT_BLK(blk);
+ nxt_nxt_blk = BLK_AFTER(blk, old_blk_sz);
- SET_MBC_BLK_SZ(blk, blk_sz);
+ SET_MBC_ABLK_SZ(blk, blk_sz);
SET_NOT_LAST_BLK(blk);
- nxt_blk = NXT_BLK(blk);
+ nxt_blk = BLK_AFTER(blk, blk_sz);
STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs);
STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs);
@@ -1614,8 +1673,8 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz);
}
- crr = BLK_TO_MBC(blk);
- SET_MBC_BLK_HDR(nxt_blk, nxt_blk_sz,
+ crr = ABLK_TO_MBC(blk);
+ SET_MBC_FBLK_HDR(nxt_blk, nxt_blk_sz,
SBH_THIS_FREE | (is_last_blk ? SBH_LAST_BLK : 0),
crr);
@@ -1638,7 +1697,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
ASSERT(is_last_blk ? IS_LAST_BLK(nxt_blk) : IS_NOT_LAST_BLK(nxt_blk));
ASSERT(is_last_blk || nxt_blk == PREV_BLK(NXT_BLK(nxt_blk)));
ASSERT(is_last_blk || IS_PREV_BLK_FREE(NXT_BLK(nxt_blk)));
- ASSERT(BLK_TO_MBC(nxt_blk) == crr);
+ ASSERT(FBLK_TO_MBC(nxt_blk) == crr);
HARD_CHECK_BLK_CARRIER(allctr, blk);
@@ -1648,7 +1707,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
/* Need larger block... */
if (!is_last_blk) {
- nxt_blk = NXT_BLK(blk);
+ nxt_blk = BLK_AFTER(blk, old_blk_sz);
nxt_blk_sz = MBC_BLK_SZ(nxt_blk);
if (IS_FREE_BLK(nxt_blk) && get_blk_sz <= old_blk_sz + nxt_blk_sz) {
/* Grow into next block... */
@@ -1662,7 +1721,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
if (nxt_blk_sz < allctr->min_block_size) {
blk_sz += nxt_blk_sz;
- SET_MBC_BLK_SZ(blk, blk_sz);
+ SET_MBC_ABLK_SZ(blk, blk_sz);
if (is_last_blk) {
SET_LAST_BLK(blk);
@@ -1671,7 +1730,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
#endif
}
else {
- nxt_blk = NXT_BLK(blk);
+ nxt_blk = BLK_AFTER(blk, blk_sz);
SET_PREV_BLK_ALLOCED(nxt_blk);
#ifdef DEBUG
is_last_blk = IS_LAST_BLK(nxt_blk);
@@ -1680,11 +1739,11 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
}
}
else {
- Carrier_t* crr = BLK_TO_MBC(blk);
- SET_MBC_BLK_SZ(blk, blk_sz);
+ Carrier_t* crr = ABLK_TO_MBC(blk);
+ SET_MBC_ABLK_SZ(blk, blk_sz);
- nxt_blk = NXT_BLK(blk);
- SET_MBC_BLK_HDR(nxt_blk, nxt_blk_sz, SBH_THIS_FREE, crr);
+ nxt_blk = BLK_AFTER(blk, blk_sz);
+ SET_MBC_FBLK_HDR(nxt_blk, nxt_blk_sz, SBH_THIS_FREE, crr);
if (is_last_blk)
SET_LAST_BLK(nxt_blk);
@@ -1694,7 +1753,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
(*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs);
ASSERT(IS_FREE_BLK(nxt_blk));
- ASSERT(BLK_TO_MBC(nxt_blk) == crr);
+ ASSERT(FBLK_TO_MBC(nxt_blk) == crr);
}
STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs);
@@ -1739,12 +1798,12 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
cand_blk_sz = 0;
}
else {
- ASSERT(!IS_MBC_FIRST_BLK(allctr, blk));
+ ASSERT(!IS_MBC_FIRST_ABLK(allctr, blk));
cand_blk = PREV_BLK(blk);
cand_blk_sz = old_blk_sz + PREV_BLK_SZ(blk);
if (!is_last_blk) {
- nxt_blk = NXT_BLK(blk);
+ nxt_blk = BLK_AFTER(blk, old_blk_sz);
if (IS_FREE_BLK(nxt_blk))
cand_blk_sz += MBC_BLK_SZ(nxt_blk);
}
@@ -1784,6 +1843,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
new_blk,
MBC_BLK_SZ(new_blk),
GET_BLK_HDR_FLGS(new_blk),
+ FBLK_TO_MBC(new_blk),
blk_sz,
1,
alcu_flgs);
@@ -1793,6 +1853,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
return new_p;
}
else {
+ Carrier_t* crr;
Uint new_blk_sz;
UWord new_blk_flgs;
Uint prev_blk_sz;
@@ -1813,7 +1874,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
if (is_last_blk)
new_blk_flgs |= LAST_BLK_HDR_FLG;
else {
- nxt_blk = NXT_BLK(blk);
+ nxt_blk = BLK_AFTER(blk, old_blk_sz);
if (IS_FREE_BLK(nxt_blk)) {
new_blk_flgs |= GET_LAST_BLK_HDR_FLG(nxt_blk);
new_blk_sz += MBC_BLK_SZ(nxt_blk);
@@ -1829,6 +1890,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
new_p = BLK2UMEM(new_blk);
blk_cpy_sz = MIN(blk_sz, old_blk_sz);
+ crr = FBLK_TO_MBC(new_blk);
if (prev_blk_sz >= blk_cpy_sz)
sys_memcpy(new_p, p, blk_cpy_sz - ABLK_HDR_SZ);
@@ -1839,6 +1901,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
new_blk,
new_blk_sz,
new_blk_flgs,
+ crr,
blk_sz,
0,
alcu_flgs);
@@ -1864,18 +1927,20 @@ static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C,
{
ASSERT(IS_LAST_BLK((B)));
ASSERT((CSZ) == CARRIER_SZ((C)));
- ASSERT((BSZ) == BLK_SZ((B)));
ASSERT((BSZ) % sizeof(Unit_t) == 0);
if ((SBC)) {
+ ASSERT((BSZ) == SBC_BLK_SZ((B)));
ASSERT((char*)B == (char*)C + A->sbc_header_size);
ASSERT(IS_SBC_BLK((B)));
ASSERT(IS_SB_CARRIER((C)));
}
else {
- ASSERT(IS_MBC_FIRST_BLK(A, (B)));
+ ASSERT(IS_FREE_BLK(B));
+ ASSERT((BSZ) == MBC_FBLK_SZ((B)));
+ ASSERT(IS_MBC_FIRST_FBLK(A, (B)));
ASSERT(IS_MBC_BLK((B)));
ASSERT(IS_MB_CARRIER((C)));
- ASSERT(BLK_TO_MBC(B) == (C));
+ ASSERT(FBLK_TO_MBC(B) == (C));
}
if ((MSEGED)) {
ASSERT(IS_MSEG_CARRIER((C)));
@@ -1913,7 +1978,7 @@ create_sbmbc(Allctr_t *allctr, Uint umem_sz)
blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size);
- SET_MBC_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr);
+ SET_MBC_FBLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr);
link_carrier(&allctr->sbmbc_list, crr);
@@ -1933,7 +1998,7 @@ destroy_sbmbc(Allctr_t *allctr, Block_t *blk)
Carrier_t *crr;
ASSERT(IS_MBC_BLK(blk));
- ASSERT(IS_MBC_FIRST_BLK(allctr, blk));
+ ASSERT(IS_MBC_FIRST_FBLK(allctr, blk));
crr = FIRST_BLK_TO_MBC(allctr, blk);
crr_sz = CARRIER_SZ(crr);
@@ -2114,7 +2179,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size);
- SET_MBC_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr);
+ SET_MBC_FBLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr);
if (flags & CFLG_MAIN_CARRIER) {
ASSERT(!allctr->main_carrier);
@@ -2300,7 +2365,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
}
else {
- ASSERT(IS_MBC_FIRST_BLK(allctr, blk));
+ ASSERT(IS_MBC_FIRST_FBLK(allctr, blk));
crr = FIRST_BLK_TO_MBC(allctr, blk);
crr_sz = CARRIER_SZ(crr);
@@ -4272,7 +4337,7 @@ erts_alcu_stop(Allctr_t *allctr)
void
erts_alcu_init(AlcUInit_t *init)
{
-
+ ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */
#if HAVE_ERTS_MSEG
ASSERT(erts_mseg_unit_size() == MSEG_UNIT_SZ);
# if HAVE_SUPER_ALIGNED_MB_CARRIERS
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 7d6f61fdff..06a5a5a346 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -218,6 +218,7 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);
#define FLG_MASK INV_UNIT_MASK
#define SBC_BLK_SZ_MASK UNIT_MASK
+#define MBC_FBLK_SZ_MASK UNIT_MASK
#define CARRIER_SZ_MASK UNIT_MASK
@@ -225,12 +226,13 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);
# define CARRIER_OFFSET_BITS 13
# define CARRIER_OFFSET_SHIFT (sizeof(UWord)*8 - CARRIER_OFFSET_BITS)
# define CARRIER_OFFSET_MASK (~((UWord)0) << CARRIER_OFFSET_SHIFT)
-# define MBC_BLK_SZ_MASK (~CARRIER_OFFSET_MASK & ~FLG_MASK)
+# define MBC_ABLK_SZ_MASK (~CARRIER_OFFSET_MASK & ~FLG_MASK)
#else
-# define MBC_BLK_SZ_MASK (~FLG_MASK)
+# define MBC_ABLK_SZ_MASK (~FLG_MASK)
#endif
-#define MBC_BLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_BLK_SZ_MASK)
+#define MBC_ABLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_ABLK_SZ_MASK)
+#define MBC_FBLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_FBLK_SZ_MASK)
#define SBC_BLK_SZ(B) (ASSERT_EXPR(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK)
#define CARRIER_SZ(C) \
@@ -257,9 +259,14 @@ typedef struct {
UWord bhdr;
#if !HAVE_SUPER_ALIGNED_MB_CARRIERS
Carrier_t *carrier;
-#endif
+#endif
+ union {
+ Carrier_t *carrier; /* if free */
+ char udata__[1]; /* if allocated */
+ }u;
} Block_t;
-typedef UWord FreeBlkFtr_t;
+
+typedef UWord FreeBlkFtr_t; /* Footer of a free block */
typedef struct {
UWord giga_no;
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
index 4768dcc939..ded58e89f4 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.c
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -91,7 +91,7 @@ struct AOFF_RBTree_t_ {
AOFF_RBTree_t *right;
Uint max_sz; /* of all blocks in this sub-tree */
};
-#define AOFF_BLK_SZ(B) MBC_BLK_SZ(&(B)->hdr)
+#define AOFF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->hdr)
#ifdef HARD_DEBUG
static AOFF_RBTree_t * check_tree(AOFF_RBTree_t* root, Uint);
diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c
index 619206bbcd..4d3ee98a4c 100644
--- a/erts/emulator/beam/erl_bestfit_alloc.c
+++ b/erts/emulator/beam/erl_bestfit_alloc.c
@@ -73,7 +73,7 @@
#define SET_RED(N) (((RBTree_t *) (N))->flags |= RED_FLG)
#define SET_BLACK(N) (((RBTree_t *) (N))->flags &= ~RED_FLG)
-#define BF_BLK_SZ(B) MBC_BLK_SZ(&(B)->hdr)
+#define BF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->hdr)
#undef ASSERT
#define ASSERT ASSERT_EXPR
diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c
index 9db89652f7..264561a965 100644
--- a/erts/emulator/beam/erl_goodfit_alloc.c
+++ b/erts/emulator/beam/erl_goodfit_alloc.c
@@ -363,7 +363,7 @@ search_bucket(Allctr_t *allctr, int ix, Uint size)
blk && i < max_blk_search;
blk = blk->next, i++) {
- blk_sz = MBC_BLK_SZ(&blk->block_head);
+ blk_sz = MBC_FBLK_SZ(&blk->block_head);
blk_on_lambc = (((char *) blk) < gfallctr->last_aux_mbc_end
&& gfallctr->last_aux_mbc_start <= ((char *) blk));
@@ -402,7 +402,7 @@ get_free_block(Allctr_t *allctr, Uint size,
if (min_bi == unsafe_bi) {
blk = search_bucket(allctr, min_bi, size);
if (blk) {
- if (cand_blk && cand_size <= MBC_BLK_SZ(blk))
+ if (cand_blk && cand_size <= MBC_FBLK_SZ(blk))
return NULL; /* cand_blk was better */
unlink_free_block(allctr, blk, flags);
return blk;
@@ -422,7 +422,7 @@ get_free_block(Allctr_t *allctr, Uint size,
/* We are guaranteed to find a block that fits in this bucket */
blk = search_bucket(allctr, min_bi, size);
ASSERT(blk);
- if (cand_blk && cand_size <= MBC_BLK_SZ(blk))
+ if (cand_blk && cand_size <= MBC_FBLK_SZ(blk))
return NULL; /* cand_blk was better */
unlink_free_block(allctr, blk, flags);
return blk;
@@ -435,7 +435,7 @@ link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
{
GFAllctr_t *gfallctr = (GFAllctr_t *) allctr;
GFFreeBlock_t *blk = (GFFreeBlock_t *) block;
- Uint sz = MBC_BLK_SZ(&blk->block_head);
+ Uint sz = MBC_FBLK_SZ(&blk->block_head);
int i = BKT_IX(gfallctr, sz);
ASSERT(sz >= MIN_BLK_SZ);
@@ -456,7 +456,7 @@ unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
{
GFAllctr_t *gfallctr = (GFAllctr_t *) allctr;
GFFreeBlock_t *blk = (GFFreeBlock_t *) block;
- Uint sz = MBC_BLK_SZ(&blk->block_head);
+ Uint sz = MBC_FBLK_SZ(&blk->block_head);
int i = BKT_IX(gfallctr, sz);
if (!blk->prev) {
diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h
index 91e335b225..f260c6506f 100644
--- a/erts/emulator/sys/common/erl_mseg.h
+++ b/erts/emulator/sys/common/erl_mseg.h
@@ -33,10 +33,14 @@
#if HAVE_MMAP
# define HAVE_ERTS_MSEG 1
# define HAVE_SUPER_ALIGNED_MB_CARRIERS 1
-# define MSEG_ALIGN_BITS (17) /*SVERK Configure me! */
#else
# define HAVE_ERTS_MSEG 0
# define HAVE_SUPER_ALIGNED_MB_CARRIERS 0
+#endif
+
+#if HAVE_SUPER_ALIGNED_MB_CARRIERS
+# define MSEG_ALIGN_BITS (17) /*SVERK Configure me! */
+#else
# define MSEG_ALIGN_BITS (12) /*SVERK Configure me! */
#endif
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-11 16:55:18 +0000