From 770ee09dfbe5c9024432ac8696be6b91d2ec9a9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 2 Nov 2012 17:38:46 +0100 Subject: erts: New mseg allocator cache * utilize the power of two --- erts/emulator/beam/erl_alloc_util.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 97ba306a79..1cceacce5c 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -510,7 +510,7 @@ alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p) { void *res; - res = erts_mseg_alloc_opt(allctr->alloc_no, size_p, &allctr->mseg_opt); + res = erts_mseg_alloc_opt(allctr->alloc_no, size_p, (Uint)0, &allctr->mseg_opt); INC_CC(allctr->calls.mseg_alloc); return res; } @@ -521,7 +521,7 @@ alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p) void *res; res = erts_mseg_realloc_opt(allctr->alloc_no, seg, old_size, new_size_p, - &allctr->mseg_opt); + (Uint)0, &allctr->mseg_opt); INC_CC(allctr->calls.mseg_realloc); return res; } -- cgit v1.2.3 From 495cadb1cbc315fa5250f99894b20508bb8a4c50 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 6 Nov 2012 18:39:21 +0100 Subject: erts: Fix compiler warnings for halfword vm --- erts/emulator/beam/beam_bp.c | 16 ++++++++-------- erts/emulator/beam/erl_trace.c | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 58e0090a76..3fd8c9cbfb 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -65,10 +65,10 @@ #define ERTS_BPF_ALL 0xFF -extern Eterm beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ -extern Eterm beam_return_trace[1]; /* OpCode(i_return_trace) */ -extern Eterm beam_exception_trace[1]; /* OpCode(i_exception_trace) */ -extern Eterm beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ +extern BeamInstr beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ +extern BeamInstr beam_return_trace[1]; /* OpCode(i_return_trace) */ +extern BeamInstr beam_exception_trace[1]; /* OpCode(i_exception_trace) */ +extern BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ erts_smp_atomic32_t erts_active_bp_index; erts_smp_atomic32_t erts_staging_bp_index; @@ -161,7 +161,7 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) for (current = 0; current < num_modules; current++) { BeamInstr** code_base = (BeamInstr **) module[current]->curr.code; BeamInstr* code; - Uint num_functions = (Uint) code_base[MI_NUM_FUNCTIONS]; + Uint num_functions = (Uint)(UWord) code_base[MI_NUM_FUNCTIONS]; Uint fi; if (specified > 0) { @@ -172,7 +172,7 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) } for (fi = 0; fi < num_functions; fi++) { - Eterm* pc; + BeamInstr* pc; int wi; code = code_base[MI_FUNCTIONS+fi]; @@ -555,7 +555,7 @@ erts_clear_module_break(Module *modp) { if (code_base == NULL) { return 0; } - n = (Uint) code_base[MI_NUM_FUNCTIONS]; + n = (Uint)(UWord) code_base[MI_NUM_FUNCTIONS]; for (i = 0; i < n; ++i) { BeamInstr* pc; @@ -919,7 +919,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, E -= 1; ASSERT(c_p->htop <= E && E <= c_p->hend); E[0] = make_cp(c_p->cp); - c_p->cp = (BeamInstr *) beam_return_to_trace; + c_p->cp = beam_return_to_trace; } if (flags & MATCH_SET_RX_TRACE) { E -= 3; diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index d04a91f18c..88d4d137b3 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -44,9 +44,9 @@ #undef DEBUG_PRINTOUTS #endif -extern Eterm beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ -extern Eterm beam_return_trace[1]; /* OpCode(i_return_trace) */ -extern Eterm beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ +extern BeamInstr beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ +extern BeamInstr beam_return_trace[1]; /* OpCode(i_return_trace) */ +extern BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ /* Pseudo export entries. Never filled in with data, only used to yield unique pointers of the correct type. */ -- cgit v1.2.3 From de03018e74bea97795895f1611abf2a50a484449 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 6 Nov 2012 22:42:47 +0100 Subject: erts: Add carrier offset to internal allocation headers --- erts/emulator/beam/erl_afit_alloc.c | 10 +- erts/emulator/beam/erl_afit_alloc.h | 5 - erts/emulator/beam/erl_alloc_util.c | 419 ++++++++++++++--------------- erts/emulator/beam/erl_alloc_util.h | 33 ++- erts/emulator/beam/erl_ao_firstfit_alloc.c | 15 +- erts/emulator/beam/erl_bestfit_alloc.c | 44 +-- erts/emulator/beam/erl_goodfit_alloc.c | 12 +- 7 files changed, 275 insertions(+), 263 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c index 570cc59be2..ad7a5822df 100644 --- a/erts/emulator/beam/erl_afit_alloc.c +++ b/erts/emulator/beam/erl_afit_alloc.c @@ -37,6 +37,12 @@ #define GET_ERL_AF_ALLOC_IMPL #include "erl_afit_alloc.h" +struct AFFreeBlock_t_ { + Block_t block_head; + AFFreeBlock_t *prev; + AFFreeBlock_t *next; +}; +#define AF_BLK_SZ(B) MBC_BLK_SZ(&(B)->block_head) #define MIN_MBC_SZ (16*1024) #define MIN_MBC_FIRST_FREE_SZ (4*1024) @@ -118,7 +124,7 @@ get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size, ASSERT(!cand_blk || cand_size >= size); - if (afallctr->free_list && BLK_SZ(afallctr->free_list) >= size) { + if (afallctr->free_list && AF_BLK_SZ(afallctr->free_list) >= size) { AFFreeBlock_t *res = afallctr->free_list; afallctr->free_list = res->next; if (res->next) @@ -135,7 +141,7 @@ link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) AFFreeBlock_t *blk = (AFFreeBlock_t *) block; AFAllctr_t *afallctr = (AFAllctr_t *) allctr; - if (afallctr->free_list && BLK_SZ(afallctr->free_list) > BLK_SZ(blk)) { + if (afallctr->free_list && AF_BLK_SZ(afallctr->free_list) > AF_BLK_SZ(blk)) { blk->next = afallctr->free_list->next; blk->prev = afallctr->free_list; afallctr->free_list->next = blk; diff --git a/erts/emulator/beam/erl_afit_alloc.h b/erts/emulator/beam/erl_afit_alloc.h index ea408a7194..87caccac20 100644 --- a/erts/emulator/beam/erl_afit_alloc.h +++ b/erts/emulator/beam/erl_afit_alloc.h @@ -49,11 +49,6 @@ Allctr_t *erts_afalc_start(AFAllctr_t *, AFAllctrInit_t *, AllctrInit_t *); #include "erl_alloc_util.h" typedef struct AFFreeBlock_t_ AFFreeBlock_t; -struct AFFreeBlock_t_ { - Block_t block_head; - AFFreeBlock_t *prev; - AFFreeBlock_t *next; -}; struct AFAllctr_t_ { Allctr_t allctr; /* Has to be first! */ diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 1cceacce5c..6b674debbd 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -78,10 +78,12 @@ int erts_have_sbmbc_alloc; #if HAVE_ERTS_MSEG -#define INV_MSEG_UNIT_MASK ((UWord) (mseg_unit_size - 1)) -#define MSEG_UNIT_MASK (~INV_MSEG_UNIT_MASK) +#define MSEG_UNIT_SHIFT ((UWord)12) +#define MSEG_UNIT_SZ (1 << MSEG_UNIT_SHIFT) +#define MSEG_UNIT_MASK ((~(UWord)0) << MSEG_UNIT_SHIFT) + #define MSEG_UNIT_FLOOR(X) ((X) & MSEG_UNIT_MASK) -#define MSEG_UNIT_CEILING(X) MSEG_UNIT_FLOOR((X) + INV_MSEG_UNIT_MASK) +#define MSEG_UNIT_CEILING(X) MSEG_UNIT_FLOOR((X) + ~MSEG_UNIT_MASK) #endif @@ -104,7 +106,6 @@ int erts_have_sbmbc_alloc; static Uint sys_alloc_carrier_size; #if HAVE_ERTS_MSEG static Uint max_mseg_carriers; -static Uint mseg_unit_size; #endif #define ONE_GIGA (1000000000) @@ -137,7 +138,7 @@ static Uint mseg_unit_size; #define BLK2UMEM(P) ((void *) (((char *) (P)) + ABLK_HDR_SZ)) #define PREV_BLK_SZ(B) \ - ((UWord) (*(((UWord *) (B)) - 1) & SZ_MASK)) + ((UWord) (((UWord *)(B))[-1] & MBC_BLK_SZ_MASK)) #define SET_BLK_SZ_FTR(B, SZ) \ (*((UWord *) (((char *) (B)) + (SZ) - sizeof(UWord))) = (SZ)) @@ -146,55 +147,80 @@ static Uint mseg_unit_size; #define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1) #define LAST_BLK_HDR_FLG (((UWord) 1) << 2) -#define SET_BLK_SZ(B, SZ) \ +#define SET_MBC_BLK_SZ(B, SZ) \ + (ASSERT(((SZ) & FLG_MASK) == 0), \ + (B)->bhdr = (((B)->bhdr) & ~MBC_BLK_SZ_MASK) | (SZ)) +#define SET_SBC_BLK_SZ(B, SZ) \ (ASSERT(((SZ) & FLG_MASK) == 0), \ - (*((Block_t *) (B)) = ((*((Block_t *) (B)) & FLG_MASK) | (SZ)))) + (B)->bhdr = (((B)->bhdr) & ~SBC_BLK_SZ_MASK) | (SZ)) #define SET_BLK_FREE(B) \ - (*((Block_t *) (B)) |= THIS_FREE_BLK_HDR_FLG) + ((B)->bhdr |= THIS_FREE_BLK_HDR_FLG) #define SET_BLK_ALLOCED(B) \ - (*((Block_t *) (B)) &= ~THIS_FREE_BLK_HDR_FLG) + ((B)->bhdr &= ~THIS_FREE_BLK_HDR_FLG) #define SET_PREV_BLK_FREE(B) \ - (*((Block_t *) (B)) |= PREV_FREE_BLK_HDR_FLG) + ((B)->bhdr |= PREV_FREE_BLK_HDR_FLG) #define SET_PREV_BLK_ALLOCED(B) \ - (*((Block_t *) (B)) &= ~PREV_FREE_BLK_HDR_FLG) + ((B)->bhdr &= ~PREV_FREE_BLK_HDR_FLG) #define SET_LAST_BLK(B) \ - (*((Block_t *) (B)) |= LAST_BLK_HDR_FLG) + ((B)->bhdr |= LAST_BLK_HDR_FLG) #define SET_NOT_LAST_BLK(B) \ - (*((Block_t *) (B)) &= ~LAST_BLK_HDR_FLG) + ((B)->bhdr &= ~LAST_BLK_HDR_FLG) #define SBH_THIS_FREE THIS_FREE_BLK_HDR_FLG -#define SBH_THIS_ALLOCED ((UWord) 0) #define SBH_PREV_FREE PREV_FREE_BLK_HDR_FLG -#define SBH_PREV_ALLOCED ((UWord) 0) #define SBH_LAST_BLK LAST_BLK_HDR_FLG -#define SBH_NOT_LAST_BLK ((UWord) 0) -#define SET_BLK_HDR(B, Sz, F) \ - (ASSERT(((Sz) & FLG_MASK) == 0), *((Block_t *) (B)) = ((Sz) | (F))) +#if HAVE_SUPER_ALIGNED_MB_CARRIERS + +# define BLK_CARRIER_OFFSET(B, C) (((char*)(B) - (char*)(C)) >> MSEG_UNIT_SHIFT) + +# define SET_MBC_BLK_HDR(B, Sz, F, C) \ + (ASSERT(((Sz) & FLG_MASK) == 0), \ + (B)->bhdr = ((Sz) | (F) | (BLK_CARRIER_OFFSET(B,C) << CARRIER_OFFSET_SHIFT))) + +# define GET_MB_CARRIER(B) \ + (ASSERT(IS_MBC_BLK(B)), \ + (Carrier_t*)((MSEG_UNIT_FLOOR((UWord)(B)) - \ + (((B)->bhdr >> CARRIER_OFFSET_SHIFT) << MSEG_UNIT_SHIFT)))) + +#else /* !HAVE_SUPER_ALIGNED_MB_CARRIERS */ + +# define SET_MBC_BLK_HDR(B, Sz, F, C) \ + (ASSERT(((Sz) & FLG_MASK) == 0), \ + (B)->bhdr = ((Sz) | (F)), \ + (B)->carrier = (C)) + +# define GET_MB_CARRIER(B) ((B)->carrier) + +#endif /* !HAVE_SUPER_ALIGNED_MB_CARRIERS */ + +#define SET_SBC_BLK_HDR(B, Sz, F) \ + (ASSERT(((Sz) & FLG_MASK) == 0), (B)->bhdr = ((Sz) | (F))) + #define BLK_UMEM_SZ(B) \ (BLK_SZ(B) - (ABLK_HDR_SZ)) #define IS_PREV_BLK_FREE(B) \ - (*((Block_t *) (B)) & PREV_FREE_BLK_HDR_FLG) + ((B)->bhdr & PREV_FREE_BLK_HDR_FLG) #define IS_PREV_BLK_ALLOCED(B) \ (!IS_PREV_BLK_FREE((B))) #define IS_FREE_BLK(B) \ - (*((Block_t *) (B)) & THIS_FREE_BLK_HDR_FLG) + ((B)->bhdr & THIS_FREE_BLK_HDR_FLG) #define IS_ALLOCED_BLK(B) \ (!IS_FREE_BLK((B))) #define IS_LAST_BLK(B) \ - (*((Block_t *) (B)) & LAST_BLK_HDR_FLG) + ((B)->bhdr & LAST_BLK_HDR_FLG) #define IS_NOT_LAST_BLK(B) \ (!IS_LAST_BLK((B))) #define GET_LAST_BLK_HDR_FLG(B) \ - (*((Block_t*) (B)) & LAST_BLK_HDR_FLG) + ((B)->bhdr & LAST_BLK_HDR_FLG) #define GET_THIS_FREE_BLK_HDR_FLG(B) \ - (*((Block_t*) (B)) & THIS_FREE_BLK_HDR_FLG) + ((B)->bhdr & THIS_FREE_BLK_HDR_FLG) #define GET_PREV_FREE_BLK_HDR_FLG(B) \ - (*((Block_t*) (B)) & PREV_FREE_BLK_HDR_FLG) + ((B)->bhdr & PREV_FREE_BLK_HDR_FLG) #define GET_BLK_HDR_FLGS(B) \ - (*((Block_t*) (B)) & FLG_MASK) + ((B)->bhdr & FLG_MASK) #define IS_FIRST_BLK(B) \ (IS_PREV_BLK_FREE((B)) && (PREV_BLK_SZ((B)) == 0)) @@ -212,7 +238,7 @@ static Uint mseg_unit_size; (!IS_SBC_BLK((B))) #define NXT_BLK(B) \ - ((Block_t *) (((char *) (B)) + BLK_SZ((B)))) + ((Block_t *) (((char *) (B)) + MBC_BLK_SZ((B)))) #define PREV_BLK(B) \ ((Block_t *) (((char *) (B)) - PREV_BLK_SZ((B)))) @@ -1209,10 +1235,8 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp, Uint32 *alcu_flgsp) if ((*alcu_flgsp) & ERTS_ALCU_FLG_SBMBC) blk = create_sbmbc(allctr, get_blk_sz); else { -#if HALFWORD_HEAP - blk = create_carrier(allctr, get_blk_sz, CFLG_MBC|CFLG_FORCE_MSEG); -#else blk = create_carrier(allctr, get_blk_sz, CFLG_MBC); +#if !HALFWORD_HEAP && !HAVE_SUPER_ALIGNED_MB_CARRIERS if (!blk) { /* Emergency! We couldn't create the carrier as we wanted. Try to place it in a sys_alloced sbc. */ @@ -1250,6 +1274,7 @@ mbc_alloc_finalize(Allctr_t *allctr, Uint nxt_blk_sz; Block_t *nxt_blk; UWord prev_free_flg = flags & PREV_FREE_BLK_HDR_FLG; + Carrier_t* crr = GET_MB_CARRIER(blk); ASSERT(org_blk_sz >= want_blk_sz); ASSERT(blk); @@ -1262,16 +1287,12 @@ mbc_alloc_finalize(Allctr_t *allctr, /* Shrink block... */ blk_sz = want_blk_sz; nxt_blk_sz = org_blk_sz - blk_sz; - SET_BLK_HDR(blk, - blk_sz, - SBH_THIS_ALLOCED|SBH_NOT_LAST_BLK|prev_free_flg); + SET_MBC_BLK_HDR(blk, blk_sz, prev_free_flg, crr); nxt_blk = NXT_BLK(blk); - SET_BLK_HDR(nxt_blk, - nxt_blk_sz, - (SBH_THIS_FREE - | SBH_PREV_ALLOCED - | (flags & LAST_BLK_HDR_FLG))); + SET_MBC_BLK_HDR(nxt_blk, nxt_blk_sz, + SBH_THIS_FREE|(flags & LAST_BLK_HDR_FLG), + crr); if (!(flags & LAST_BLK_HDR_FLG)) { SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); @@ -1291,9 +1312,11 @@ mbc_alloc_finalize(Allctr_t *allctr, || nxt_blk == PREV_BLK(NXT_BLK(nxt_blk))); ASSERT((flags & LAST_BLK_HDR_FLG) || IS_PREV_BLK_FREE(NXT_BLK(nxt_blk))); - ASSERT(nxt_blk_sz == BLK_SZ(nxt_blk)); + ASSERT(nxt_blk_sz == MBC_BLK_SZ(nxt_blk)); ASSERT(nxt_blk_sz % sizeof(Unit_t) == 0); ASSERT(nxt_blk_sz >= allctr->min_block_size); + ASSERT(GET_MB_CARRIER(blk) == crr); + ASSERT(GET_MB_CARRIER(nxt_blk) == crr); } else { blk_sz = org_blk_sz; @@ -1301,17 +1324,13 @@ mbc_alloc_finalize(Allctr_t *allctr, if (valid_blk_info) SET_BLK_ALLOCED(blk); else - SET_BLK_HDR(blk, - blk_sz, - SBH_THIS_ALLOCED|SBH_LAST_BLK|prev_free_flg); + SET_MBC_BLK_HDR(blk, blk_sz, SBH_LAST_BLK|prev_free_flg, crr); } else { if (valid_blk_info) SET_BLK_ALLOCED(blk); else - SET_BLK_HDR(blk, - blk_sz, - SBH_THIS_ALLOCED|SBH_NOT_LAST_BLK|prev_free_flg); + SET_MBC_BLK_HDR(blk, blk_sz, prev_free_flg, crr); nxt_blk = NXT_BLK(blk); SET_PREV_BLK_ALLOCED(nxt_blk); } @@ -1319,12 +1338,13 @@ mbc_alloc_finalize(Allctr_t *allctr, ASSERT((flags & LAST_BLK_HDR_FLG) ? IS_LAST_BLK(blk) : IS_NOT_LAST_BLK(blk)); + ASSERT(GET_MB_CARRIER(blk) == crr); } STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); ASSERT(IS_ALLOCED_BLK(blk)); - ASSERT(blk_sz == BLK_SZ(blk)); + ASSERT(blk_sz == MBC_BLK_SZ(blk)); ASSERT(blk_sz % sizeof(Unit_t) == 0); ASSERT(blk_sz >= allctr->min_block_size); ASSERT(blk_sz >= want_blk_sz); @@ -1348,7 +1368,7 @@ mbc_alloc(Allctr_t *allctr, Uint size) if (IS_MBC_BLK(blk)) mbc_alloc_finalize(allctr, blk, - BLK_SZ(blk), + MBC_BLK_SZ(blk), GET_BLK_HDR_FLGS(blk), blk_sz, 1, @@ -1370,7 +1390,7 @@ mbc_free(Allctr_t *allctr, void *p) ASSERT(p); blk = UMEM2BLK(p); - blk_sz = BLK_SZ(blk); + blk_sz = MBC_BLK_SZ(blk); if (blk_sz < allctr->sbmbc_threshold) alcu_flgs |= ERTS_ALCU_FLG_SBMBC; @@ -1389,9 +1409,9 @@ mbc_free(Allctr_t *allctr, void *p) blk = PREV_BLK(blk); (*allctr->unlink_free_block)(allctr, blk, alcu_flgs); - blk_sz += BLK_SZ(blk); + blk_sz += MBC_BLK_SZ(blk); is_first_blk = IS_FIRST_BLK(blk); - SET_BLK_SZ(blk, blk_sz); + SET_MBC_BLK_SZ(blk, blk_sz); } else { SET_BLK_FREE(blk); @@ -1404,8 +1424,8 @@ mbc_free(Allctr_t *allctr, void *p) if (IS_FREE_BLK(nxt_blk)) { /* Coalesce with next block... */ (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); - blk_sz += BLK_SZ(nxt_blk); - SET_BLK_SZ(blk, blk_sz); + blk_sz += MBC_BLK_SZ(nxt_blk); + SET_MBC_BLK_SZ(blk, blk_sz); is_last_blk = IS_LAST_BLK(nxt_blk); if (is_last_blk) @@ -1428,7 +1448,7 @@ mbc_free(Allctr_t *allctr, void *p) ASSERT(IS_FREE_BLK(blk)); ASSERT(is_first_blk || IS_PREV_BLK_ALLOCED(blk)); ASSERT(is_last_blk || IS_PREV_BLK_FREE(NXT_BLK(blk))); - ASSERT(blk_sz == BLK_SZ(blk)); + ASSERT(blk_sz == MBC_BLK_SZ(blk)); ASSERT(is_last_blk || blk == PREV_BLK(NXT_BLK(blk))); ASSERT(blk_sz % sizeof(Unit_t) == 0); ASSERT(IS_MBC_BLK(blk)); @@ -1472,7 +1492,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) ASSERT(size < allctr->sbc_threshold); blk = (Block_t *) UMEM2BLK(p); - old_blk_sz = BLK_SZ(blk); + old_blk_sz = MBC_BLK_SZ(blk); ASSERT(old_blk_sz >= allctr->min_block_size); @@ -1497,6 +1517,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) return p; else if (blk_sz < old_blk_sz) { /* Shrink block... */ + Carrier_t* crr; Block_t *nxt_nxt_blk; Uint diff_sz_val = old_blk_sz - blk_sz; Uint old_blk_sz_val = old_blk_sz; @@ -1525,7 +1546,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (!is_last_blk) { nxt_blk = NXT_BLK(blk); if (IS_FREE_BLK(nxt_blk)) - cand_blk_sz += BLK_SZ(nxt_blk); + cand_blk_sz += MBC_BLK_SZ(nxt_blk); } new_blk = (*allctr->get_free_block)(allctr, @@ -1547,46 +1568,42 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) HARD_CHECK_BLK_CARRIER(allctr, blk); - SET_BLK_SZ(blk, blk_sz); + nxt_nxt_blk = NXT_BLK(blk); + + SET_MBC_BLK_SZ(blk, blk_sz); SET_NOT_LAST_BLK(blk); nxt_blk = NXT_BLK(blk); - SET_BLK_HDR(nxt_blk, - nxt_blk_sz, - SBH_THIS_FREE|SBH_PREV_ALLOCED|SBH_NOT_LAST_BLK); STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); - ASSERT(BLK_SZ(blk) >= allctr->min_block_size); + ASSERT(MBC_BLK_SZ(blk) >= allctr->min_block_size); - if (is_last_blk) - SET_LAST_BLK(nxt_blk); - else { - nxt_nxt_blk = NXT_BLK(nxt_blk); + if (!is_last_blk) { if (IS_FREE_BLK(nxt_nxt_blk)) { /* Coalesce with next free block... */ - nxt_blk_sz += BLK_SZ(nxt_nxt_blk); + nxt_blk_sz += MBC_BLK_SZ(nxt_nxt_blk); (*allctr->unlink_free_block)(allctr, nxt_nxt_blk, alcu_flgs); - SET_BLK_SZ(nxt_blk, nxt_blk_sz); - is_last_blk = IS_LAST_BLK(nxt_nxt_blk); - if (is_last_blk) - SET_LAST_BLK(nxt_blk); - else - SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); + is_last_blk = GET_LAST_BLK_HDR_FLG(nxt_nxt_blk); } else { - SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); SET_PREV_BLK_FREE(nxt_nxt_blk); } + SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); } + crr = GET_MB_CARRIER(blk); + SET_MBC_BLK_HDR(nxt_blk, nxt_blk_sz, + SBH_THIS_FREE | (is_last_blk ? SBH_LAST_BLK : 0), + crr); + (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); ASSERT(IS_ALLOCED_BLK(blk)); - ASSERT(blk_sz == BLK_SZ(blk)); + ASSERT(blk_sz == MBC_BLK_SZ(blk)); ASSERT(blk_sz % sizeof(Unit_t) == 0); ASSERT(blk_sz >= allctr->min_block_size); ASSERT(blk_sz >= size + ABLK_HDR_SZ); @@ -1594,14 +1611,15 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) ASSERT(IS_FREE_BLK(nxt_blk)); ASSERT(IS_PREV_BLK_ALLOCED(nxt_blk)); - ASSERT(nxt_blk_sz == BLK_SZ(nxt_blk)); + ASSERT(nxt_blk_sz == MBC_BLK_SZ(nxt_blk)); ASSERT(nxt_blk_sz % sizeof(Unit_t) == 0); ASSERT(nxt_blk_sz >= allctr->min_block_size); ASSERT(IS_MBC_BLK(nxt_blk)); ASSERT(is_last_blk ? IS_LAST_BLK(nxt_blk) : IS_NOT_LAST_BLK(nxt_blk)); ASSERT(is_last_blk || nxt_blk == PREV_BLK(NXT_BLK(nxt_blk))); ASSERT(is_last_blk || IS_PREV_BLK_FREE(NXT_BLK(nxt_blk))); - + ASSERT(GET_MB_CARRIER(nxt_blk) == crr); + HARD_CHECK_BLK_CARRIER(allctr, blk); return p; @@ -1611,7 +1629,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (!is_last_blk) { nxt_blk = NXT_BLK(blk); - nxt_blk_sz = BLK_SZ(nxt_blk); + nxt_blk_sz = MBC_BLK_SZ(nxt_blk); if (IS_FREE_BLK(nxt_blk) && get_blk_sz <= old_blk_sz + nxt_blk_sz) { /* Grow into next block... */ @@ -1624,7 +1642,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (nxt_blk_sz < allctr->min_block_size) { blk_sz += nxt_blk_sz; - SET_BLK_SZ(blk, blk_sz); + SET_MBC_BLK_SZ(blk, blk_sz); if (is_last_blk) { SET_LAST_BLK(blk); @@ -1637,17 +1655,16 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) SET_PREV_BLK_ALLOCED(nxt_blk); #ifdef DEBUG is_last_blk = IS_LAST_BLK(nxt_blk); - nxt_blk_sz = BLK_SZ(nxt_blk); + nxt_blk_sz = MBC_BLK_SZ(nxt_blk); #endif } } else { - SET_BLK_SZ(blk, blk_sz); + Carrier_t* crr = GET_MB_CARRIER(blk); + SET_MBC_BLK_SZ(blk, blk_sz); nxt_blk = NXT_BLK(blk); - SET_BLK_HDR(nxt_blk, - nxt_blk_sz, - SBH_THIS_FREE|SBH_PREV_ALLOCED|SBH_NOT_LAST_BLK); + SET_MBC_BLK_HDR(nxt_blk, nxt_blk_sz, SBH_THIS_FREE, crr); if (is_last_blk) SET_LAST_BLK(nxt_blk); @@ -1657,6 +1674,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); ASSERT(IS_FREE_BLK(nxt_blk)); + ASSERT(GET_MB_CARRIER(nxt_blk) == crr); } STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); @@ -1664,14 +1682,14 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) ASSERT(IS_ALLOCED_BLK(blk)); - ASSERT(blk_sz == BLK_SZ(blk)); + ASSERT(blk_sz == MBC_BLK_SZ(blk)); ASSERT(blk_sz % sizeof(Unit_t) == 0); ASSERT(blk_sz >= allctr->min_block_size); ASSERT(blk_sz >= size + ABLK_HDR_SZ); ASSERT(IS_MBC_BLK(blk)); ASSERT(!nxt_blk || IS_PREV_BLK_ALLOCED(nxt_blk)); - ASSERT(!nxt_blk || nxt_blk_sz == BLK_SZ(nxt_blk)); + ASSERT(!nxt_blk || nxt_blk_sz == MBC_BLK_SZ(nxt_blk)); ASSERT(!nxt_blk || nxt_blk_sz % sizeof(Unit_t) == 0); ASSERT(!nxt_blk || nxt_blk_sz >= allctr->min_block_size); ASSERT(!nxt_blk || IS_MBC_BLK(nxt_blk)); @@ -1707,7 +1725,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (!is_last_blk) { nxt_blk = NXT_BLK(blk); if (IS_FREE_BLK(nxt_blk)) - cand_blk_sz += BLK_SZ(nxt_blk); + cand_blk_sz += MBC_BLK_SZ(nxt_blk); } } @@ -1743,7 +1761,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (new_blk) { mbc_alloc_finalize(allctr, new_blk, - BLK_SZ(new_blk), + MBC_BLK_SZ(new_blk), GET_BLK_HDR_FLGS(new_blk), blk_sz, 1, @@ -1777,7 +1795,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) nxt_blk = NXT_BLK(blk); if (IS_FREE_BLK(nxt_blk)) { new_blk_flgs |= GET_LAST_BLK_HDR_FLG(nxt_blk); - new_blk_sz += BLK_SZ(nxt_blk); + new_blk_sz += MBC_BLK_SZ(nxt_blk); (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); } } @@ -1815,35 +1833,37 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) #ifdef DEBUG #if HAVE_ERTS_MSEG -#define ASSERT_MSEG_UNIT_SIZE_MULTIPLE(CSZ) ASSERT((CSZ) % mseg_unit_size == 0) +#define ASSERT_MSEG_UNIT_SIZE_MULTIPLE(CSZ) ASSERT((CSZ) % MSEG_UNIT_SZ == 0) #else #define ASSERT_MSEG_UNIT_SIZE_MULTIPLE(CSZ) #endif -#define CHECK_1BLK_CARRIER(A, SBC, MSEGED, C, CSZ, B, BSZ) \ -do { \ - ASSERT(IS_FIRST_BLK((B))); \ - ASSERT(IS_LAST_BLK((B))); \ - ASSERT((CSZ) == CARRIER_SZ((C))); \ - ASSERT((BSZ) == BLK_SZ((B))); \ - ASSERT((BSZ) % sizeof(Unit_t) == 0); \ - if ((SBC)) { \ - ASSERT(IS_SBC_BLK((B))); \ - ASSERT(IS_SB_CARRIER((C))); \ - } \ - else { \ - ASSERT(IS_MBC_BLK((B))); \ - ASSERT(IS_MB_CARRIER((C))); \ - } \ - if ((MSEGED)) { \ - ASSERT(IS_MSEG_CARRIER((C))); \ - ASSERT_MSEG_UNIT_SIZE_MULTIPLE((CSZ)); \ - } \ - else { \ - ASSERT(IS_SYS_ALLOC_CARRIER((C))); \ - ASSERT((CSZ) % sizeof(Unit_t) == 0); \ - } \ -} while (0) +static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C, + UWord CSZ, Block_t* B, UWord BSZ) +{ + ASSERT(IS_FIRST_BLK((B))); + ASSERT(IS_LAST_BLK((B))); + ASSERT((CSZ) == CARRIER_SZ((C))); + ASSERT((BSZ) == BLK_SZ((B))); + ASSERT((BSZ) % sizeof(Unit_t) == 0); + if ((SBC)) { + ASSERT(IS_SBC_BLK((B))); + ASSERT(IS_SB_CARRIER((C))); + } + else { + ASSERT(IS_MBC_BLK((B))); + ASSERT(IS_MB_CARRIER((C))); + ASSERT(GET_MB_CARRIER(B) == (C)); + } + if ((MSEGED)) { + ASSERT(IS_MSEG_CARRIER((C))); + ASSERT_MSEG_UNIT_SIZE_MULTIPLE((CSZ)); + } + else { + ASSERT(IS_SYS_ALLOC_CARRIER((C))); + ASSERT((CSZ) % sizeof(Unit_t) == 0); + } +} #else #define CHECK_1BLK_CARRIER(A, SBC, MSEGED, C, CSZ, B, BSZ) @@ -1869,33 +1889,15 @@ create_sbmbc(Allctr_t *allctr, Uint umem_sz) blk = MBC2FBLK(allctr, crr); -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (allctr->mbc_header_size % sizeof(Unit_t) == 0) - crr_sz -= sizeof(UWord); -#endif - blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size); SET_MBC_BLK_FTR(((UWord *) blk)[-1]); - SET_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_PREV_FREE|SBH_LAST_BLK); - -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - *((Carrier_t **) NXT_BLK(blk)) = crr; -#endif + SET_MBC_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_PREV_FREE|SBH_LAST_BLK, crr); link_carrier(&allctr->sbmbc_list, crr); -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (allctr->mbc_header_size % sizeof(Unit_t) == 0) - crr_sz += sizeof(UWord); -#endif - STAT_SBMBC_ALLOC(allctr, crr_sz); CHECK_1BLK_CARRIER(allctr, 0, 0, crr, crr_sz, blk, blk_sz); -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (allctr->mbc_header_size % sizeof(Unit_t) == 0) - crr_sz -= sizeof(UWord); -#endif if (allctr->creating_mbc) (*allctr->creating_mbc)(allctr, crr, ERTS_ALCU_FLG_SBMBC); @@ -1957,9 +1959,19 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) int is_mseg = 0; #endif +#if HALFWORD_HEAP + flags |= CFLG_FORCE_MSEG; +#elif HAVE_SUPER_ALIGNED_MB_CARRIERS + if (flags & CFLG_MBC) { + flags |= CFLG_FORCE_MSEG; + } +#endif + ASSERT((flags & CFLG_SBC && !(flags & CFLG_MBC)) || (flags & CFLG_MBC && !(flags & CFLG_SBC))); + ASSERT(!(flags & CFLG_FORCE_MSEG && flags & CFLG_FORCE_SYS_ALLOC)); + blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz); #if HAVE_ERTS_MSEG @@ -1974,10 +1986,12 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) if (allctr->sbcs.curr.norm.mseg.no >= allctr->max_mseg_sbcs) goto try_sys_alloc; } +#if !HAVE_SUPER_ALIGNED_MB_CARRIERS else { if (allctr->mbcs.curr.norm.mseg.no >= allctr->max_mseg_mbcs) goto try_sys_alloc; } +#endif try_mseg: @@ -1988,13 +2002,9 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) crr_sz = (*allctr->get_next_mbc_size)(allctr); if (crr_sz < allctr->mbc_header_size + blk_sz) crr_sz = allctr->mbc_header_size + blk_sz; -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (allctr->mbc_header_size % sizeof(Unit_t) == 0) - crr_sz += sizeof(UWord); -#endif } crr_sz = MSEG_UNIT_CEILING(crr_sz); - ASSERT(crr_sz % mseg_unit_size == 0); + ASSERT(crr_sz % MSEG_UNIT_SZ == 0); crr = (Carrier_t *) alcu_mseg_alloc(allctr, &crr_sz); if (!crr) { @@ -2019,6 +2029,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) } try_sys_alloc: + #endif /* #if HAVE_ERTS_MSEG */ if (flags & CFLG_SBC) { @@ -2029,11 +2040,6 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) if (!(flags & CFLG_MAIN_CARRIER) && bcrr_sz < allctr->smallest_mbc_size) bcrr_sz = allctr->smallest_mbc_size; -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (allctr->mbc_header_size % sizeof(Unit_t) == 0) - bcrr_sz += sizeof(UWord); -#endif - } crr_sz = (flags & CFLG_FORCE_SIZE @@ -2067,7 +2073,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) blk = SBC2BLK(allctr, crr); SET_SBC_BLK_FTR(((UWord *) blk)[-1]); - SET_BLK_HDR(blk, blk_sz, SBH_THIS_ALLOCED|SBH_PREV_FREE|SBH_LAST_BLK); + SET_SBC_BLK_HDR(blk, blk_sz, SBH_PREV_FREE|SBH_LAST_BLK); link_carrier(&allctr->sbc_list, crr); @@ -2084,19 +2090,10 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) blk = MBC2FBLK(allctr, crr); -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (allctr->mbc_header_size % sizeof(Unit_t) == 0) - crr_sz -= sizeof(UWord); -#endif - blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size); SET_MBC_BLK_FTR(((UWord *) blk)[-1]); - SET_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_PREV_FREE|SBH_LAST_BLK); - -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - *((Carrier_t **) NXT_BLK(blk)) = crr; -#endif + SET_MBC_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_PREV_FREE|SBH_LAST_BLK, crr); if (flags & CFLG_MAIN_CARRIER) { ASSERT(!allctr->main_carrier); @@ -2105,15 +2102,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) link_carrier(&allctr->mbc_list, crr); -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (allctr->mbc_header_size % sizeof(Unit_t) == 0) - crr_sz += sizeof(UWord); -#endif CHECK_1BLK_CARRIER(allctr, 0, is_mseg, crr, crr_sz, blk, blk_sz); -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (allctr->mbc_header_size % sizeof(Unit_t) == 0) - crr_sz -= sizeof(UWord); -#endif if (allctr->creating_mbc) (*allctr->creating_mbc)(allctr, crr, 0); @@ -2142,7 +2131,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) HARD_CHECK_BLK_CARRIER(allctr, old_blk); - old_blk_sz = BLK_SZ(old_blk); + old_blk_sz = SBC_BLK_SZ(old_blk); old_crr = BLK2SBC(allctr, old_blk); old_crr_sz = CARRIER_SZ(old_crr); ASSERT(IS_SB_CARRIER(old_crr)); @@ -2166,7 +2155,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) if (new_crr) { SET_CARRIER_SZ(new_crr, new_crr_sz); new_blk = SBC2BLK(allctr, new_crr); - SET_BLK_SZ(new_blk, new_blk_sz); + SET_SBC_BLK_SZ(new_blk, new_blk_sz); STAT_MSEG_SBC_ALLOC(allctr, new_crr_sz, new_blk_sz); relink_carrier(&allctr->sbc_list, new_crr); CHECK_1BLK_CARRIER(allctr, 1, 1, new_crr, new_crr_sz, @@ -2174,6 +2163,11 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) DEBUG_SAVE_ALIGNMENT(new_crr); return new_blk; } +#if HALFWORD_HEAP + /* Old carrier unchanged; restore stat */ + STAT_MSEG_SBC_ALLOC(allctr, old_crr_sz, old_blk_sz); + return NULL; +#endif create_flags |= CFLG_FORCE_SYS_ALLOC; /* since mseg_realloc() failed */ } @@ -2208,7 +2202,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) sys_realloc_success: SET_CARRIER_SZ(new_crr, new_crr_sz); new_blk = SBC2BLK(allctr, new_crr); - SET_BLK_SZ(new_blk, new_blk_sz); + SET_SBC_BLK_SZ(new_blk, new_blk_sz); STAT_SYS_ALLOC_SBC_FREE(allctr, old_crr_sz, old_blk_sz); STAT_SYS_ALLOC_SBC_ALLOC(allctr, new_crr_sz, new_blk_sz); relink_carrier(&allctr->sbc_list, new_crr); @@ -2265,7 +2259,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) ASSERT(IS_FIRST_BLK(blk)); if (IS_SBC_BLK(blk)) { - Uint blk_sz = BLK_SZ(blk); + Uint blk_sz = SBC_BLK_SZ(blk); crr = BLK2SBC(allctr, blk); crr_sz = CARRIER_SZ(crr); @@ -2276,7 +2270,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(crr)) { is_mseg++; - ASSERT(crr_sz % mseg_unit_size == 0); + ASSERT(crr_sz % MSEG_UNIT_SZ == 0); STAT_MSEG_SBC_FREE(allctr, crr_sz, blk_sz); } else @@ -2305,7 +2299,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(crr)) { is_mseg++; - ASSERT(crr_sz % mseg_unit_size == 0); + ASSERT(crr_sz % MSEG_UNIT_SZ == 0); STAT_MSEG_MBC_FREE(allctr, crr_sz); } else @@ -3430,12 +3424,7 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) if (allctr->dd.use) ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); #endif -#if HALFWORD_HEAP - blk = create_carrier(allctr, size, - CFLG_SBC | CFLG_FORCE_MSEG); -#else blk = create_carrier(allctr, size, CFLG_SBC); -#endif res = blk ? BLK2UMEM(blk) : NULL; } else @@ -3775,7 +3764,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, if (res) { sys_memcpy((void*) res, (void*) p, - MIN(BLK_SZ(blk) - ABLK_HDR_SZ, size)); + MIN(SBC_BLK_SZ(blk) - ABLK_HDR_SZ, size)); destroy_carrier(allctr, blk); } } @@ -3798,16 +3787,12 @@ do_erts_alcu_realloc(ErtsAlcType_t type, else if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) return NULL; else { -#if HALFWORD_HEAP - new_blk = create_carrier(allctr, size, CFLG_SBC | CFLG_FORCE_MSEG); -#else new_blk = create_carrier(allctr, size, CFLG_SBC); -#endif if (new_blk) { res = BLK2UMEM(new_blk); sys_memcpy((void *) res, (void *) p, - MIN(BLK_SZ(blk) - ABLK_HDR_SZ, size)); + MIN(MBC_BLK_SZ(blk) - ABLK_HDR_SZ, size)); mbc_free(allctr, p); } else @@ -4120,7 +4105,11 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->mbc_move_threshold = init->rmbcmt; #if HAVE_ERTS_MSEG allctr->max_mseg_sbcs = init->mmsbc; +# if HAVE_SUPER_ALIGNED_MB_CARRIERS + allctr->max_mseg_mbcs = ~(Uint)0; +# else allctr->max_mseg_mbcs = init->mmmbc; +# endif #endif allctr->largest_mbc_size = MAX(init->lmbcs, init->smbcs); @@ -4241,21 +4230,14 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (allctr->main_carrier_size) { Block_t *blk; -#if HALFWORD_HEAP - blk = create_carrier(allctr, - allctr->main_carrier_size, - CFLG_MBC - | CFLG_FORCE_SIZE - | CFLG_FORCE_MSEG - | CFLG_MAIN_CARRIER); -#else blk = create_carrier(allctr, allctr->main_carrier_size, CFLG_MBC | CFLG_FORCE_SIZE +#if !HALFWORD_HEAP && !HAVE_SUPER_ALIGNED_MB_CARRIERS | CFLG_FORCE_SYS_ALLOC - | CFLG_MAIN_CARRIER); #endif + | CFLG_MAIN_CARRIER); if (!blk) goto error; @@ -4321,15 +4303,12 @@ erts_alcu_init(AlcUInit_t *init) { #if HAVE_ERTS_MSEG - mseg_unit_size = erts_mseg_unit_size(); - - if (mseg_unit_size % sizeof(Unit_t)) /* A little paranoid... */ - erl_exit(-1, - "Mseg unit size (%d) not evenly divideble by " - "internal unit size of alloc_util (%d)\n", - mseg_unit_size, - sizeof(Unit_t)); - + ASSERT(erts_mseg_unit_size() == MSEG_UNIT_SZ); +# if HAVE_SUPER_ALIGNED_MB_CARRIERS + /*SVERK Add assert about CARRIER_OFFSET_BITS and max MBC size + *SVERK Add assert about CARRIER_OFFSET_SHIFT and max MBC block size + */ +# endif max_mseg_carriers = init->mmc; sys_alloc_carrier_size = MSEG_UNIT_CEILING(init->ycs); #else /* #if HAVE_ERTS_MSEG */ @@ -4432,6 +4411,13 @@ erts_alcu_verify_unused_ts(Allctr_t *allctr) #endif } +#ifdef DEBUG +int is_sbc_blk(Block_t* blk) +{ + return IS_SBC_BLK(blk); +} +#endif + #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG static void @@ -4447,28 +4433,34 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) ASSERT(IS_ALLOCED_BLK(iblk)); ASSERT(IS_FIRST_BLK(iblk)); ASSERT(IS_LAST_BLK(iblk)); - ASSERT(CARRIER_SZ(sbc) - allctr->sbc_header_size >= BLK_SZ(iblk)); + ASSERT(CARRIER_SZ(sbc) - allctr->sbc_header_size >= SBC_BLK_SZ(iblk)); #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(sbc)) { - ASSERT(CARRIER_SZ(sbc) % mseg_unit_size == 0); + ASSERT(CARRIER_SZ(sbc) % MSEG_UNIT_SZ == 0); } #endif crr = sbc; cl = &allctr->sbc_list; } else { - Carrier_t *mbc = NULL; Block_t *prev_blk = NULL; Block_t *blk; char *carrier_end; Uint is_free_blk; Uint tot_blk_sz; Uint blk_sz; + int has_wrapped_around = 0; blk = iblk; tot_blk_sz = 0; + crr = GET_MB_CARRIER(blk); + ASSERT(IS_MB_CARRIER(crr)); + /* Step around the carrier one whole lap starting at 'iblk' + */ while (1) { + ASSERT(IS_MBC_BLK(blk)); + ASSERT(GET_MB_CARRIER(blk) == crr); if (prev_blk) { ASSERT(NXT_BLK(prev_blk) == blk); @@ -4481,18 +4473,16 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) } } - if (mbc) { + if (has_wrapped_around) { + ASSERT(((Block_t *) crr) < blk); if (blk == iblk) break; - ASSERT(((Block_t *) mbc) < blk && blk < iblk); + ASSERT(blk < iblk); } else ASSERT(blk >= iblk); - - ASSERT(IS_MBC_BLK(blk)); - - blk_sz = BLK_SZ(blk); + blk_sz = MBC_BLK_SZ(blk); ASSERT(blk_sz % sizeof(Unit_t) == 0); ASSERT(blk_sz >= allctr->min_block_size); @@ -4511,9 +4501,9 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) if (IS_LAST_BLK(blk)) { carrier_end = ((char *) NXT_BLK(blk)); - mbc = *((Carrier_t **) NXT_BLK(blk)); + has_wrapped_around = 1; prev_blk = NULL; - blk = MBC2FBLK(allctr, mbc); + blk = MBC2FBLK(allctr, crr); ASSERT(IS_FIRST_BLK(blk)); } else { @@ -4521,23 +4511,21 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) blk = NXT_BLK(blk); } } - - ASSERT(IS_MB_CARRIER(mbc)); - ASSERT((((char *) mbc) + + ASSERT((((char *) crr) + allctr->mbc_header_size + tot_blk_sz) == carrier_end); - ASSERT(((char *) mbc) + CARRIER_SZ(mbc) - sizeof(Unit_t) <= carrier_end - && carrier_end <= ((char *) mbc) + CARRIER_SZ(mbc)); + ASSERT(((char *) crr) + CARRIER_SZ(crr) - sizeof(Unit_t) <= carrier_end + && carrier_end <= ((char *) crr) + CARRIER_SZ(crr)); if (allctr->check_mbc) - (*allctr->check_mbc)(allctr, mbc); + (*allctr->check_mbc)(allctr, crr); #if HAVE_ERTS_MSEG - if (IS_MSEG_CARRIER(mbc)) { - ASSERT(CARRIER_SZ(mbc) % mseg_unit_size == 0); + if (IS_MSEG_CARRIER(crr)) { + ASSERT(CARRIER_SZ(crr) % MSEG_UNIT_SZ == 0); } #endif - crr = mbc; cl = &allctr->mbc_list; } @@ -4559,4 +4547,5 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) #endif } -#endif +#endif /* ERTS_ALLOC_UTIL_HARD_DEBUG */ + diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index cedf4ccf85..f440752c4a 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -216,16 +216,27 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); #define UNIT_FLOOR(X) ((X) & UNIT_MASK) #define UNIT_CEILING(X) UNIT_FLOOR((X) + INV_UNIT_MASK) +#define FLG_MASK INV_UNIT_MASK +#define SBC_BLK_SZ_MASK UNIT_MASK +#define CARRIER_SZ_MASK UNIT_MASK + + +#if HAVE_SUPER_ALIGNED_MB_CARRIERS +# define CARRIER_OFFSET_BITS 13 +# define CARRIER_OFFSET_SHIFT (sizeof(UWord)*8 - CARRIER_OFFSET_BITS) +# define CARRIER_OFFSET_MASK (~((UWord)0) << CARRIER_OFFSET_SHIFT) +# define MBC_BLK_SZ_MASK (~CARRIER_OFFSET_MASK & ~FLG_MASK) +#else +# define MBC_BLK_SZ_MASK (~FLG_MASK) +#endif -#define SZ_MASK (~((UWord) 0) << 3) -#define FLG_MASK (~(SZ_MASK)) - +#define MBC_BLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_BLK_SZ_MASK) +#define SBC_BLK_SZ(B) (ASSERT_EXPR(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK) -#define BLK_SZ(B) \ - (*((Block_t *) (B)) & SZ_MASK) +#define BLK_SZ(B) ((B)->bhdr & (IS_SBC_BLK(B) ? SBC_BLK_SZ_MASK : MBC_BLK_SZ_MASK)) #define CARRIER_SZ(C) \ - ((C)->chdr & SZ_MASK) + ((C)->chdr & CARRIER_SZ_MASK) extern int erts_have_sbmbc_alloc; @@ -243,7 +254,12 @@ typedef struct { Carrier_t *last; } CarrierList_t; -typedef UWord Block_t; +typedef struct { + UWord bhdr; +#if !HAVE_SUPER_ALIGNED_MB_CARRIERS + Carrier_t *carrier; +#endif +} Block_t; typedef UWord FreeBlkFtr_t; typedef struct { @@ -469,6 +485,9 @@ void erts_alcu_verify_unused_ts(Allctr_t *allctr); unsigned long erts_alcu_test(unsigned long, unsigned long, unsigned long); +#ifdef DEBUG +int is_sbc_blk(Block_t*); +#endif #endif /* #if defined(GET_ERL_ALLOC_UTIL_IMPL) diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 5bdb752d3a..4768dcc939 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -91,6 +91,7 @@ struct AOFF_RBTree_t_ { AOFF_RBTree_t *right; Uint max_sz; /* of all blocks in this sub-tree */ }; +#define AOFF_BLK_SZ(B) MBC_BLK_SZ(&(B)->hdr) #ifdef HARD_DEBUG static AOFF_RBTree_t * check_tree(AOFF_RBTree_t* root, Uint); @@ -102,7 +103,7 @@ static AOFF_RBTree_t * check_tree(AOFF_RBTree_t* root, Uint); */ static ERTS_INLINE Uint node_max_size(AOFF_RBTree_t *x) { - Uint sz = BLK_SZ(x); + Uint sz = AOFF_BLK_SZ(x); if (x->left && x->left->max_sz > sz) { sz = x->left->max_sz; } @@ -587,7 +588,7 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) AOFF_RBTree_t *blk = (AOFF_RBTree_t *) block; AOFF_RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) ? &alc->sbmbc_root : &alc->mbc_root); - Uint blk_sz = BLK_SZ(blk); + Uint blk_sz = AOFF_BLK_SZ(blk); #ifdef HARD_DEBUG check_tree(*root, 0); @@ -659,7 +660,7 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, if (x->left && x->left->max_sz >= size) { x = x->left; } - else if (BLK_SZ(x) >= size) { + else if (AOFF_BLK_SZ(x) >= size) { blk = x; break; } @@ -910,12 +911,12 @@ check_tree(AOFF_RBTree_t* root, Uint size) ASSERT(x->right > x); ASSERT(x->right->max_sz <= x->max_sz); } - ASSERT(x->max_sz >= BLK_SZ(x)); - ASSERT(x->max_sz == BLK_SZ(x) + ASSERT(x->max_sz >= AOFF_BLK_SZ(x)); + ASSERT(x->max_sz == AOFF_BLK_SZ(x) || x->max_sz == (x->left ? x->left->max_sz : 0) || x->max_sz == (x->right ? x->right->max_sz : 0)); - if (size && BLK_SZ(x) >= size) { + if (size && AOFF_BLK_SZ(x) >= size) { if (!res || x < res) { res = x; } @@ -956,7 +957,7 @@ print_tree_aux(AOFF_RBTree_t *x, int indent) } fprintf(stderr, "%s: sz=%lu addr=0x%lx max_size=%lu\r\n", IS_BLACK(x) ? "BLACK" : "RED", - BLK_SZ(x), (Uint)x, x->max_sz); + AOFF_BLK_SZ(x), (Uint)x, x->max_sz); print_tree_aux(x->left, indent + INDENT_STEP); } } diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c index c50fdeb4e8..619206bbcd 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.c +++ b/erts/emulator/beam/erl_bestfit_alloc.c @@ -73,6 +73,8 @@ #define SET_RED(N) (((RBTree_t *) (N))->flags |= RED_FLG) #define SET_BLACK(N) (((RBTree_t *) (N))->flags &= ~RED_FLG) +#define BF_BLK_SZ(B) MBC_BLK_SZ(&(B)->hdr) + #undef ASSERT #define ASSERT ASSERT_EXPR @@ -592,7 +594,7 @@ aobf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) ? &bfallctr->sbmbc_root : &bfallctr->mbc_root); RBTree_t *blk = (RBTree_t *) block; - Uint blk_sz = BLK_SZ(blk); + Uint blk_sz = BF_BLK_SZ(blk); @@ -610,7 +612,7 @@ aobf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) while (1) { Uint size; - size = BLK_SZ(x); + size = BF_BLK_SZ(x); if (blk_sz < size || (blk_sz == size && blk < x)) { if (!x->left) { @@ -668,7 +670,7 @@ aobf_get_free_block(Allctr_t *allctr, Uint size, ASSERT(!cand_blk || cand_size >= size); while (x) { - blk_sz = BLK_SZ(x); + blk_sz = BF_BLK_SZ(x); if (blk_sz < size) { x = x->right; } @@ -686,7 +688,7 @@ aobf_get_free_block(Allctr_t *allctr, Uint size, #endif if (cand_blk) { - blk_sz = BLK_SZ(blk); + blk_sz = BF_BLK_SZ(blk); if (cand_size < blk_sz) return NULL; /* cand_blk was better */ if (cand_size == blk_sz && ((void *) cand_blk) < ((void *) blk)) @@ -711,7 +713,7 @@ bf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) ? &bfallctr->sbmbc_root : &bfallctr->mbc_root); RBTree_t *blk = (RBTree_t *) block; - Uint blk_sz = BLK_SZ(blk); + Uint blk_sz = BF_BLK_SZ(blk); SET_TREE_NODE(blk); @@ -730,7 +732,7 @@ bf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) while (1) { Uint size; - size = BLK_SZ(x); + size = BF_BLK_SZ(x); if (blk_sz == size) { @@ -796,7 +798,7 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) else if (LIST_NEXT(x)) { /* Replace tree node by next element in list... */ - ASSERT(BLK_SZ(LIST_NEXT(x)) == BLK_SZ(x)); + ASSERT(BF_BLK_SZ(LIST_NEXT(x)) == BF_BLK_SZ(x)); ASSERT(IS_TREE_NODE(x)); ASSERT(IS_LIST_ELEM(LIST_NEXT(x))); @@ -834,7 +836,7 @@ bf_get_free_block(Allctr_t *allctr, Uint size, ASSERT(!cand_blk || cand_size >= size); while (x) { - blk_sz = BLK_SZ(x); + blk_sz = BF_BLK_SZ(x); if (blk_sz < size) { x = x->right; } @@ -855,11 +857,11 @@ bf_get_free_block(Allctr_t *allctr, Uint size, #ifdef HARD_DEBUG { RBTree_t *ct_blk = check_tree(root, 0, size); - ASSERT(BLK_SZ(ct_blk) == BLK_SZ(blk)); + ASSERT(BF_BLK_SZ(ct_blk) == BF_BLK_SZ(blk)); } #endif - if (cand_blk && cand_size <= BLK_SZ(blk)) + if (cand_blk && cand_size <= BF_BLK_SZ(blk)) return NULL; /* cand_blk was better */ /* Use next block if it exist in order to avoid replacing @@ -1093,36 +1095,36 @@ check_tree(RBTree_t *root, int ao, Uint size) if (x->left) { ASSERT(x->left->parent == x); if (ao) { - ASSERT(BLK_SZ(x->left) < BLK_SZ(x) - || (BLK_SZ(x->left) == BLK_SZ(x) && x->left < x)); + ASSERT(BF_BLK_SZ(x->left) < BF_BLK_SZ(x) + || (BF_BLK_SZ(x->left) == BF_BLK_SZ(x) && x->left < x)); } else { ASSERT(IS_TREE_NODE(x->left)); - ASSERT(BLK_SZ(x->left) < BLK_SZ(x)); + ASSERT(BF_BLK_SZ(x->left) < BF_BLK_SZ(x)); } } if (x->right) { ASSERT(x->right->parent == x); if (ao) { - ASSERT(BLK_SZ(x->right) > BLK_SZ(x) - || (BLK_SZ(x->right) == BLK_SZ(x) && x->right > x)); + ASSERT(BF_BLK_SZ(x->right) > BF_BLK_SZ(x) + || (BF_BLK_SZ(x->right) == BF_BLK_SZ(x) && x->right > x)); } else { ASSERT(IS_TREE_NODE(x->right)); - ASSERT(BLK_SZ(x->right) > BLK_SZ(x)); + ASSERT(BF_BLK_SZ(x->right) > BF_BLK_SZ(x)); } } - if (size && BLK_SZ(x) >= size) { + if (size && BF_BLK_SZ(x) >= size) { if (ao) { if (!res - || BLK_SZ(x) < BLK_SZ(res) - || (BLK_SZ(x) == BLK_SZ(res) && x < res)) + || BF_BLK_SZ(x) < BF_BLK_SZ(res) + || (BF_BLK_SZ(x) == BF_BLK_SZ(res) && x < res)) res = x; } else { - if (!res || BLK_SZ(x) < BLK_SZ(res)) + if (!res || BF_BLK_SZ(x) < BF_BLK_SZ(res)) res = x; } } @@ -1168,7 +1170,7 @@ print_tree_aux(RBTree_t *x, int indent) } fprintf(stderr, "%s: sz=%lu addr=0x%lx\r\n", IS_BLACK(x) ? "BLACK" : "RED", - BLK_SZ(x), + BF_BLK_SZ(x), (Uint) x); print_tree_aux(x->left, indent + INDENT_STEP); } diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index e7d4ac2b67..9db89652f7 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -363,7 +363,7 @@ search_bucket(Allctr_t *allctr, int ix, Uint size) blk && i < max_blk_search; blk = blk->next, i++) { - blk_sz = BLK_SZ(blk); + blk_sz = MBC_BLK_SZ(&blk->block_head); blk_on_lambc = (((char *) blk) < gfallctr->last_aux_mbc_end && gfallctr->last_aux_mbc_start <= ((char *) blk)); @@ -402,7 +402,7 @@ get_free_block(Allctr_t *allctr, Uint size, if (min_bi == unsafe_bi) { blk = search_bucket(allctr, min_bi, size); if (blk) { - if (cand_blk && cand_size <= BLK_SZ(blk)) + if (cand_blk && cand_size <= MBC_BLK_SZ(blk)) return NULL; /* cand_blk was better */ unlink_free_block(allctr, blk, flags); return blk; @@ -422,7 +422,7 @@ get_free_block(Allctr_t *allctr, Uint size, /* We are guaranteed to find a block that fits in this bucket */ blk = search_bucket(allctr, min_bi, size); ASSERT(blk); - if (cand_blk && cand_size <= BLK_SZ(blk)) + if (cand_blk && cand_size <= MBC_BLK_SZ(blk)) return NULL; /* cand_blk was better */ unlink_free_block(allctr, blk, flags); return blk; @@ -435,7 +435,7 @@ link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; GFFreeBlock_t *blk = (GFFreeBlock_t *) block; - Uint sz = BLK_SZ(blk); + Uint sz = MBC_BLK_SZ(&blk->block_head); int i = BKT_IX(gfallctr, sz); ASSERT(sz >= MIN_BLK_SZ); @@ -456,7 +456,7 @@ unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; GFFreeBlock_t *blk = (GFFreeBlock_t *) block; - Uint sz = BLK_SZ(blk); + Uint sz = MBC_BLK_SZ(&blk->block_head); int i = BKT_IX(gfallctr, sz); if (!blk->prev) { @@ -618,7 +618,7 @@ check_block(Allctr_t *allctr, Block_t * blk, int free_block) GFFreeBlock_t *fblk; if(free_block) { - Uint blk_sz = BLK_SZ(blk); + Uint blk_sz = is_sbc_blk(blk) ? SBC_BLK_SZ(blk) : MBC_BLK_SZ(blk); bi = BKT_IX(gfallctr, blk_sz); ASSERT(gfallctr->bucket_mask.main & (((UWord) 1) << IX2SMIX(bi))); -- cgit v1.2.3 From 4ca2d66fb51e1f3bb85d420c339fb73c5fb6bc62 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 8 Nov 2012 11:24:40 +0100 Subject: erts: Move carrier alignment define to erl_msg.h --- erts/emulator/beam/erl_alloc_util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 6b674debbd..32a42f9274 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -78,7 +78,7 @@ int erts_have_sbmbc_alloc; #if HAVE_ERTS_MSEG -#define MSEG_UNIT_SHIFT ((UWord)12) +#define MSEG_UNIT_SHIFT MSEG_ALIGN_BITS #define MSEG_UNIT_SZ (1 << MSEG_UNIT_SHIFT) #define MSEG_UNIT_MASK ((~(UWord)0) << MSEG_UNIT_SHIFT) -- cgit v1.2.3 From 1a32cca7ec4c97d0d006e5dd6704008586e006d7 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 8 Nov 2012 11:31:18 +0100 Subject: erts: Use new mseg flag argument for carrier sizing --- erts/emulator/beam/erl_alloc_util.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 32a42f9274..bba1e06af9 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -532,11 +532,11 @@ static void mbc_free(Allctr_t *allctr, void *p); #if HAVE_ERTS_MSEG static ERTS_INLINE void * -alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p) +alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { void *res; - res = erts_mseg_alloc_opt(allctr->alloc_no, size_p, (Uint)0, &allctr->mseg_opt); + res = erts_mseg_alloc_opt(allctr->alloc_no, size_p, flags, &allctr->mseg_opt); INC_CC(allctr->calls.mseg_alloc); return res; } @@ -547,7 +547,7 @@ alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p) void *res; res = erts_mseg_realloc_opt(allctr->alloc_no, seg, old_size, new_size_p, - (Uint)0, &allctr->mseg_opt); + ERTS_MSEG_FLG_NONE, &allctr->mseg_opt); INC_CC(allctr->calls.mseg_realloc); return res; } @@ -1954,6 +1954,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) Uint blk_sz, bcrr_sz, crr_sz; #if HAVE_ERTS_MSEG int have_tried_sys_alloc = 0, have_tried_mseg = 0; + Uint mseg_flags; #endif #ifdef DEBUG int is_mseg = 0; @@ -1997,16 +1998,18 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) if (flags & CFLG_SBC) { crr_sz = blk_sz + allctr->sbc_header_size; + mseg_flags = ERTS_MSEG_FLG_NONE; } else { crr_sz = (*allctr->get_next_mbc_size)(allctr); if (crr_sz < allctr->mbc_header_size + blk_sz) crr_sz = allctr->mbc_header_size + blk_sz; + mseg_flags = ERTS_MSEG_FLG_2POW; } crr_sz = MSEG_UNIT_CEILING(crr_sz); ASSERT(crr_sz % MSEG_UNIT_SZ == 0); - crr = (Carrier_t *) alcu_mseg_alloc(allctr, &crr_sz); + crr = (Carrier_t *) alcu_mseg_alloc(allctr, &crr_sz, mseg_flags); if (!crr) { have_tried_mseg = 1; if (!(have_tried_sys_alloc || flags & CFLG_FORCE_MSEG)) -- cgit v1.2.3 From 0b18830887c6925a45a74eef97ee6e4c81e4f94d Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 8 Nov 2012 21:30:32 +0100 Subject: erts: Change single carrier block header flags to allow realloc to determine block size (in MBC or SBC) without having to read the footer of the previous block that might be written to by concurrent thread. --- erts/emulator/beam/erl_alloc_util.c | 88 +++++++++++++++++++++---------------- erts/emulator/beam/erl_alloc_util.h | 2 - 2 files changed, 51 insertions(+), 39 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index bba1e06af9..470c045ee6 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -122,7 +122,7 @@ static Uint max_mseg_carriers; /* Blocks ... */ -#define SBC_BLK_FTR_FLG (((UWord) 1) << 0) +#define UNUSED0_BLK_FTR_FLG (((UWord) 1) << 0) #define UNUSED1_BLK_FTR_FLG (((UWord) 1) << 1) #define UNUSED2_BLK_FTR_FLG (((UWord) 1) << 2) @@ -147,6 +147,10 @@ static Uint max_mseg_carriers; #define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1) #define LAST_BLK_HDR_FLG (((UWord) 1) << 2) +/* Special flag combo for (allocated) SBC blocks +*/ +#define SBC_BLK_HDR_FLG (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) + #define SET_MBC_BLK_SZ(B, SZ) \ (ASSERT(((SZ) & FLG_MASK) == 0), \ (B)->bhdr = (((B)->bhdr) & ~MBC_BLK_SZ_MASK) | (SZ)) @@ -154,11 +158,14 @@ static Uint max_mseg_carriers; (ASSERT(((SZ) & FLG_MASK) == 0), \ (B)->bhdr = (((B)->bhdr) & ~SBC_BLK_SZ_MASK) | (SZ)) #define SET_BLK_FREE(B) \ - ((B)->bhdr |= THIS_FREE_BLK_HDR_FLG) + (ASSERT(!IS_PREV_BLK_FREE(B)), \ + (B)->bhdr |= THIS_FREE_BLK_HDR_FLG) #define SET_BLK_ALLOCED(B) \ ((B)->bhdr &= ~THIS_FREE_BLK_HDR_FLG) -#define SET_PREV_BLK_FREE(B) \ - ((B)->bhdr |= PREV_FREE_BLK_HDR_FLG) +#define SET_PREV_BLK_FREE(AP,B) \ + (ASSERT(!IS_MBC_FIRST_BLK(AP,B)), \ + ASSERT(!IS_FREE_BLK(B)), \ + (B)->bhdr |= PREV_FREE_BLK_HDR_FLG) #define SET_PREV_BLK_ALLOCED(B) \ ((B)->bhdr &= ~PREV_FREE_BLK_HDR_FLG) #define SET_LAST_BLK(B) \ @@ -170,12 +177,14 @@ static Uint max_mseg_carriers; #define SBH_PREV_FREE PREV_FREE_BLK_HDR_FLG #define SBH_LAST_BLK LAST_BLK_HDR_FLG + #if HAVE_SUPER_ALIGNED_MB_CARRIERS # define BLK_CARRIER_OFFSET(B, C) (((char*)(B) - (char*)(C)) >> MSEG_UNIT_SHIFT) # define SET_MBC_BLK_HDR(B, Sz, F, C) \ (ASSERT(((Sz) & FLG_MASK) == 0), \ + ASSERT((UWord)(F) < SBC_BLK_HDR_FLG), \ (B)->bhdr = ((Sz) | (F) | (BLK_CARRIER_OFFSET(B,C) << CARRIER_OFFSET_SHIFT))) # define GET_MB_CARRIER(B) \ @@ -183,19 +192,27 @@ static Uint max_mseg_carriers; (Carrier_t*)((MSEG_UNIT_FLOOR((UWord)(B)) - \ (((B)->bhdr >> CARRIER_OFFSET_SHIFT) << MSEG_UNIT_SHIFT)))) +# define IS_MBC_FIRST_BLK(AP,B) \ + ((((UWord)(B) & ~MSEG_UNIT_MASK) == (AP)->mbc_header_size) \ + && ((B)->bhdr & CARRIER_OFFSET_MASK) == 0) + #else /* !HAVE_SUPER_ALIGNED_MB_CARRIERS */ # define SET_MBC_BLK_HDR(B, Sz, F, C) \ (ASSERT(((Sz) & FLG_MASK) == 0), \ + ASSERT((UWord)(F) < SBC_BLK_HDR_FLG), \ (B)->bhdr = ((Sz) | (F)), \ (B)->carrier = (C)) # define GET_MB_CARRIER(B) ((B)->carrier) +# define IS_MBC_FIRST_BLK(AP,B) \ + ((char*)(B) == (char*)((B)->carrier) + (AP)->mbc_header_size) + #endif /* !HAVE_SUPER_ALIGNED_MB_CARRIERS */ -#define SET_SBC_BLK_HDR(B, Sz, F) \ - (ASSERT(((Sz) & FLG_MASK) == 0), (B)->bhdr = ((Sz) | (F))) +#define SET_SBC_BLK_HDR(B, Sz) \ + (ASSERT(((Sz) & FLG_MASK) == 0), (B)->bhdr = ((Sz) | (SBC_BLK_HDR_FLG))) #define BLK_UMEM_SZ(B) \ @@ -205,7 +222,7 @@ static Uint max_mseg_carriers; #define IS_PREV_BLK_ALLOCED(B) \ (!IS_PREV_BLK_FREE((B))) #define IS_FREE_BLK(B) \ - ((B)->bhdr & THIS_FREE_BLK_HDR_FLG) + (ASSERT(!IS_SBC_BLK(B)), (B)->bhdr & THIS_FREE_BLK_HDR_FLG) #define IS_ALLOCED_BLK(B) \ (!IS_FREE_BLK((B))) #define IS_LAST_BLK(B) \ @@ -222,18 +239,13 @@ static Uint max_mseg_carriers; #define GET_BLK_HDR_FLGS(B) \ ((B)->bhdr & FLG_MASK) -#define IS_FIRST_BLK(B) \ - (IS_PREV_BLK_FREE((B)) && (PREV_BLK_SZ((B)) == 0)) -#define IS_NOT_FIRST_BLK(B) \ - (!IS_FIRST_BLK((B))) - #define SET_SBC_BLK_FTR(FTR) \ - ((FTR) = (0 | SBC_BLK_FTR_FLG)) + ((FTR) = 0) #define SET_MBC_BLK_FTR(FTR) \ ((FTR) = 0) #define IS_SBC_BLK(B) \ - (IS_PREV_BLK_FREE((B)) && (((UWord *) (B))[-1] & SBC_BLK_FTR_FLG)) + (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG) #define IS_MBC_BLK(B) \ (!IS_SBC_BLK((B))) @@ -242,6 +254,8 @@ static Uint max_mseg_carriers; #define PREV_BLK(B) \ ((Block_t *) (((char *) (B)) - PREV_BLK_SZ((B)))) +#define BLK_SZ(B) ((B)->bhdr & (IS_SBC_BLK(B) ? SBC_BLK_SZ_MASK : MBC_BLK_SZ_MASK)) + /* Carriers ... */ #define MSEG_CARRIER_HDR_FLAG (((UWord) 1) << 0) @@ -1298,7 +1312,7 @@ mbc_alloc_finalize(Allctr_t *allctr, SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); if (!valid_blk_info) { Block_t *nxt_nxt_blk = NXT_BLK(nxt_blk); - SET_PREV_BLK_FREE(nxt_nxt_blk); + SET_PREV_BLK_FREE(allctr, nxt_nxt_blk); } } (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); @@ -1401,16 +1415,17 @@ mbc_free(Allctr_t *allctr, void *p) STAT_MBC_BLK_FREE(allctr, blk_sz, alcu_flgs); - is_first_blk = IS_FIRST_BLK(blk); + is_first_blk = IS_MBC_FIRST_BLK(allctr, blk); is_last_blk = IS_LAST_BLK(blk); - if (!is_first_blk && IS_PREV_BLK_FREE(blk)) { + if (IS_PREV_BLK_FREE(blk)) { + ASSERT(!is_first_blk); /* Coalesce with previous block... */ blk = PREV_BLK(blk); (*allctr->unlink_free_block)(allctr, blk, alcu_flgs); blk_sz += MBC_BLK_SZ(blk); - is_first_blk = IS_FIRST_BLK(blk); + is_first_blk = IS_MBC_FIRST_BLK(allctr, blk); SET_MBC_BLK_SZ(blk, blk_sz); } else { @@ -1436,15 +1451,15 @@ mbc_free(Allctr_t *allctr, void *p) } } else { - SET_PREV_BLK_FREE(nxt_blk); + SET_PREV_BLK_FREE(allctr, nxt_blk); SET_NOT_LAST_BLK(blk); SET_BLK_SZ_FTR(blk, blk_sz); } } - ASSERT(is_last_blk ? IS_LAST_BLK(blk) : IS_NOT_LAST_BLK(blk)); - ASSERT(is_first_blk ? IS_FIRST_BLK(blk) : IS_NOT_FIRST_BLK(blk)); + ASSERT(!is_last_blk == !IS_LAST_BLK(blk)); + ASSERT(!is_first_blk == !IS_MBC_FIRST_BLK(allctr, blk)); ASSERT(IS_FREE_BLK(blk)); ASSERT(is_first_blk || IS_PREV_BLK_ALLOCED(blk)); ASSERT(is_last_blk || IS_PREV_BLK_FREE(NXT_BLK(blk))); @@ -1537,9 +1552,11 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) return NULL; cand_blk_sz = old_blk_sz; - if (!IS_PREV_BLK_FREE(blk) || IS_FIRST_BLK(blk)) + if (!IS_PREV_BLK_FREE(blk)) { cand_blk = blk; + } else { + ASSERT(!IS_MBC_FIRST_BLK(allctr, blk)); cand_blk = PREV_BLK(blk); cand_blk_sz += PREV_BLK_SZ(blk); } @@ -1589,7 +1606,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) is_last_blk = GET_LAST_BLK_HDR_FLG(nxt_nxt_blk); } else { - SET_PREV_BLK_FREE(nxt_nxt_blk); + SET_PREV_BLK_FREE(allctr, nxt_nxt_blk); } SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); } @@ -1714,11 +1731,12 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) /* Need to grow in another block */ - if (!IS_PREV_BLK_FREE(blk) || IS_FIRST_BLK(blk)) { + if (!IS_PREV_BLK_FREE(blk)) { cand_blk = NULL; cand_blk_sz = 0; } else { + ASSERT(!IS_MBC_FIRST_BLK(allctr, blk)); cand_blk = PREV_BLK(blk); cand_blk_sz = old_blk_sz + PREV_BLK_SZ(blk); @@ -1841,16 +1859,17 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C, UWord CSZ, Block_t* B, UWord BSZ) { - ASSERT(IS_FIRST_BLK((B))); ASSERT(IS_LAST_BLK((B))); ASSERT((CSZ) == CARRIER_SZ((C))); ASSERT((BSZ) == BLK_SZ((B))); ASSERT((BSZ) % sizeof(Unit_t) == 0); if ((SBC)) { + ASSERT((char*)B == (char*)C + A->sbc_header_size); ASSERT(IS_SBC_BLK((B))); ASSERT(IS_SB_CARRIER((C))); } else { + ASSERT(IS_MBC_FIRST_BLK(A, (B))); ASSERT(IS_MBC_BLK((B))); ASSERT(IS_MB_CARRIER((C))); ASSERT(GET_MB_CARRIER(B) == (C)); @@ -1892,7 +1911,7 @@ create_sbmbc(Allctr_t *allctr, Uint umem_sz) blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size); SET_MBC_BLK_FTR(((UWord *) blk)[-1]); - SET_MBC_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_PREV_FREE|SBH_LAST_BLK, crr); + SET_MBC_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr); link_carrier(&allctr->sbmbc_list, crr); @@ -1911,9 +1930,8 @@ destroy_sbmbc(Allctr_t *allctr, Block_t *blk) Uint crr_sz; Carrier_t *crr; - ASSERT(IS_FIRST_BLK(blk)); - ASSERT(IS_MBC_BLK(blk)); + ASSERT(IS_MBC_FIRST_BLK(allctr, blk)); crr = FBLK2MBC(allctr, blk); crr_sz = CARRIER_SZ(crr); @@ -2076,7 +2094,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) blk = SBC2BLK(allctr, crr); SET_SBC_BLK_FTR(((UWord *) blk)[-1]); - SET_SBC_BLK_HDR(blk, blk_sz, SBH_PREV_FREE|SBH_LAST_BLK); + SET_SBC_BLK_HDR(blk, blk_sz); link_carrier(&allctr->sbc_list, crr); @@ -2096,7 +2114,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size); SET_MBC_BLK_FTR(((UWord *) blk)[-1]); - SET_MBC_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_PREV_FREE|SBH_LAST_BLK, crr); + SET_MBC_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr); if (flags & CFLG_MAIN_CARRIER) { ASSERT(!allctr->main_carrier); @@ -2259,8 +2277,6 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) Uint is_mseg = 0; #endif - ASSERT(IS_FIRST_BLK(blk)); - if (IS_SBC_BLK(blk)) { Uint blk_sz = SBC_BLK_SZ(blk); crr = BLK2SBC(allctr, blk); @@ -2284,6 +2300,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) } else { + ASSERT(IS_MBC_FIRST_BLK(allctr, blk)); crr = FBLK2MBC(allctr, blk); crr_sz = CARRIER_SZ(crr); @@ -4370,7 +4387,7 @@ erts_alcu_test(unsigned long op, unsigned long a1, unsigned long a2) case 0x017: return (unsigned long) ((Allctr_t *) a1)->min_block_size; case 0x018: return (unsigned long) NXT_BLK((Block_t *) a1); case 0x019: return (unsigned long) PREV_BLK((Block_t *) a1); - case 0x01a: return (unsigned long) IS_FIRST_BLK((Block_t *) a1); + case 0x01a: return (unsigned long) IS_MBC_FIRST_BLK((Allctr_t*)a1, (Block_t *) a2); case 0x01b: return (unsigned long) sizeof(Unit_t); default: ASSERT(0); return ~((unsigned long) 0); } @@ -4433,9 +4450,6 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) Carrier_t *sbc = BLK2SBC(allctr, iblk); ASSERT(SBC2BLK(allctr, sbc) == iblk); - ASSERT(IS_ALLOCED_BLK(iblk)); - ASSERT(IS_FIRST_BLK(iblk)); - ASSERT(IS_LAST_BLK(iblk)); ASSERT(CARRIER_SZ(sbc) - allctr->sbc_header_size >= SBC_BLK_SZ(iblk)); #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(sbc)) { @@ -4507,7 +4521,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) has_wrapped_around = 1; prev_blk = NULL; blk = MBC2FBLK(allctr, crr); - ASSERT(IS_FIRST_BLK(blk)); + ASSERT(IS_MBC_FIRST_BLK(allctr,blk)); } else { prev_blk = blk; diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index f440752c4a..1e6ba5736a 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -233,8 +233,6 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); #define MBC_BLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_BLK_SZ_MASK) #define SBC_BLK_SZ(B) (ASSERT_EXPR(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK) -#define BLK_SZ(B) ((B)->bhdr & (IS_SBC_BLK(B) ? SBC_BLK_SZ_MASK : MBC_BLK_SZ_MASK)) - #define CARRIER_SZ(C) \ ((C)->chdr & CARRIER_SZ_MASK) -- cgit v1.2.3 From e0ac12699f685cb6de71f72ed7420a70db047d7f Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 9 Nov 2012 16:33:34 +0100 Subject: erts: Refactor erl_alloc_util.c Make use of type FreeBlkFtr_t Rename GET_MB_CARRIER into BLK2MBC (like BLK2SBC) Remove some unused macros --- erts/emulator/beam/erl_alloc_util.c | 57 +++++++++++++++---------------------- 1 file changed, 23 insertions(+), 34 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 470c045ee6..b3c948c3e0 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -127,7 +127,7 @@ static Uint max_mseg_carriers; #define UNUSED2_BLK_FTR_FLG (((UWord) 1) << 2) #define ABLK_HDR_SZ (sizeof(Block_t)) -#define FBLK_FTR_SZ (sizeof(UWord)) +#define FBLK_FTR_SZ (sizeof(FreeBlkFtr_t)) #define UMEMSZ2BLKSZ(AP, SZ) \ (ABLK_HDR_SZ + (SZ) <= (AP)->min_block_size \ @@ -138,10 +138,10 @@ static Uint max_mseg_carriers; #define BLK2UMEM(P) ((void *) (((char *) (P)) + ABLK_HDR_SZ)) #define PREV_BLK_SZ(B) \ - ((UWord) (((UWord *)(B))[-1] & MBC_BLK_SZ_MASK)) + ((UWord) (((FreeBlkFtr_t *)(B))[-1] & MBC_BLK_SZ_MASK)) #define SET_BLK_SZ_FTR(B, SZ) \ - (*((UWord *) (((char *) (B)) + (SZ) - sizeof(UWord))) = (SZ)) + (((FreeBlkFtr_t *) (((char *) (B)) + (SZ)))[-1] = (SZ)) #define THIS_FREE_BLK_HDR_FLG (((UWord) 1) << 0) #define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1) @@ -187,7 +187,7 @@ static Uint max_mseg_carriers; ASSERT((UWord)(F) < SBC_BLK_HDR_FLG), \ (B)->bhdr = ((Sz) | (F) | (BLK_CARRIER_OFFSET(B,C) << CARRIER_OFFSET_SHIFT))) -# define GET_MB_CARRIER(B) \ +# define BLK2MBC(B) \ (ASSERT(IS_MBC_BLK(B)), \ (Carrier_t*)((MSEG_UNIT_FLOOR((UWord)(B)) - \ (((B)->bhdr >> CARRIER_OFFSET_SHIFT) << MSEG_UNIT_SHIFT)))) @@ -204,7 +204,7 @@ static Uint max_mseg_carriers; (B)->bhdr = ((Sz) | (F)), \ (B)->carrier = (C)) -# define GET_MB_CARRIER(B) ((B)->carrier) +# define BLK2MBC(B) ((B)->carrier) # define IS_MBC_FIRST_BLK(AP,B) \ ((char*)(B) == (char*)((B)->carrier) + (AP)->mbc_header_size) @@ -290,15 +290,6 @@ static Uint max_mseg_carriers; #define IS_MB_CARRIER(C) \ (!IS_SB_CARRIER((C))) -#define SET_MSEG_CARRIER(C) \ - ((C)->chdr |= MSEG_CARRIER_HDR_FLAG) -#define SET_SYS_ALLOC_CARRIER(C) \ - ((C)->chdr &= ~MSEG_CARRIER_HDR_FLAG) -#define SET_SB_CARRIER(C) \ - ((C)->chdr |= SBC_CARRIER_HDR_FLAG) -#define SET_MB_CARRIER(C) \ - ((C)->chdr &= ~SBC_CARRIER_HDR_FLAG) - #define SET_CARRIER_SZ(C, SZ) \ (ASSERT(((SZ) & FLG_MASK) == 0), \ ((C)->chdr = ((C)->chdr & FLG_MASK) | (SZ))) @@ -1288,7 +1279,7 @@ mbc_alloc_finalize(Allctr_t *allctr, Uint nxt_blk_sz; Block_t *nxt_blk; UWord prev_free_flg = flags & PREV_FREE_BLK_HDR_FLG; - Carrier_t* crr = GET_MB_CARRIER(blk); + Carrier_t* crr = BLK2MBC(blk); ASSERT(org_blk_sz >= want_blk_sz); ASSERT(blk); @@ -1329,8 +1320,8 @@ mbc_alloc_finalize(Allctr_t *allctr, ASSERT(nxt_blk_sz == MBC_BLK_SZ(nxt_blk)); ASSERT(nxt_blk_sz % sizeof(Unit_t) == 0); ASSERT(nxt_blk_sz >= allctr->min_block_size); - ASSERT(GET_MB_CARRIER(blk) == crr); - ASSERT(GET_MB_CARRIER(nxt_blk) == crr); + ASSERT(BLK2MBC(blk) == crr); + ASSERT(BLK2MBC(nxt_blk) == crr); } else { blk_sz = org_blk_sz; @@ -1352,7 +1343,7 @@ mbc_alloc_finalize(Allctr_t *allctr, ASSERT((flags & LAST_BLK_HDR_FLG) ? IS_LAST_BLK(blk) : IS_NOT_LAST_BLK(blk)); - ASSERT(GET_MB_CARRIER(blk) == crr); + ASSERT(BLK2MBC(blk) == crr); } STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); @@ -1611,7 +1602,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); } - crr = GET_MB_CARRIER(blk); + crr = BLK2MBC(blk); SET_MBC_BLK_HDR(nxt_blk, nxt_blk_sz, SBH_THIS_FREE | (is_last_blk ? SBH_LAST_BLK : 0), crr); @@ -1635,7 +1626,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) ASSERT(is_last_blk ? IS_LAST_BLK(nxt_blk) : IS_NOT_LAST_BLK(nxt_blk)); ASSERT(is_last_blk || nxt_blk == PREV_BLK(NXT_BLK(nxt_blk))); ASSERT(is_last_blk || IS_PREV_BLK_FREE(NXT_BLK(nxt_blk))); - ASSERT(GET_MB_CARRIER(nxt_blk) == crr); + ASSERT(BLK2MBC(nxt_blk) == crr); HARD_CHECK_BLK_CARRIER(allctr, blk); @@ -1677,7 +1668,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) } } else { - Carrier_t* crr = GET_MB_CARRIER(blk); + Carrier_t* crr = BLK2MBC(blk); SET_MBC_BLK_SZ(blk, blk_sz); nxt_blk = NXT_BLK(blk); @@ -1691,7 +1682,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); ASSERT(IS_FREE_BLK(nxt_blk)); - ASSERT(GET_MB_CARRIER(nxt_blk) == crr); + ASSERT(BLK2MBC(nxt_blk) == crr); } STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); @@ -1872,7 +1863,7 @@ static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C, ASSERT(IS_MBC_FIRST_BLK(A, (B))); ASSERT(IS_MBC_BLK((B))); ASSERT(IS_MB_CARRIER((C))); - ASSERT(GET_MB_CARRIER(B) == (C)); + ASSERT(BLK2MBC(B) == (C)); } if ((MSEGED)) { ASSERT(IS_MSEG_CARRIER((C))); @@ -1910,7 +1901,7 @@ create_sbmbc(Allctr_t *allctr, Uint umem_sz) blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size); - SET_MBC_BLK_FTR(((UWord *) blk)[-1]); + SET_MBC_BLK_FTR(((FreeBlkFtr_t *) blk)[-1]); SET_MBC_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr); link_carrier(&allctr->sbmbc_list, crr); @@ -2093,7 +2084,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) blk = SBC2BLK(allctr, crr); - SET_SBC_BLK_FTR(((UWord *) blk)[-1]); + SET_SBC_BLK_FTR(((FreeBlkFtr_t *) blk)[-1]); SET_SBC_BLK_HDR(blk, blk_sz); link_carrier(&allctr->sbc_list, crr); @@ -2113,7 +2104,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size); - SET_MBC_BLK_FTR(((UWord *) blk)[-1]); + SET_MBC_BLK_FTR(((FreeBlkFtr_t *) blk)[-1]); SET_MBC_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr); if (flags & CFLG_MAIN_CARRIER) { @@ -4139,7 +4130,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (allctr->min_block_size < ABLK_HDR_SZ) goto error; allctr->min_block_size = UNIT_CEILING(allctr->min_block_size - + sizeof(UWord)); + + sizeof(FreeBlkFtr_t)); #if ERTS_SMP if (init->tpref) { Uint sz = sizeof(Block_t); @@ -4470,14 +4461,14 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) blk = iblk; tot_blk_sz = 0; - crr = GET_MB_CARRIER(blk); + crr = BLK2MBC(blk); ASSERT(IS_MB_CARRIER(crr)); /* Step around the carrier one whole lap starting at 'iblk' */ while (1) { ASSERT(IS_MBC_BLK(blk)); - ASSERT(GET_MB_CARRIER(blk) == crr); + ASSERT(BLK2MBC(blk) == crr); if (prev_blk) { ASSERT(NXT_BLK(prev_blk) == blk); @@ -4507,11 +4498,9 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) tot_blk_sz += blk_sz; is_free_blk = (int) IS_FREE_BLK(blk); - if(is_free_blk) { - if (IS_NOT_LAST_BLK(blk)) - ASSERT(*((UWord *) (((char *) blk)+blk_sz-sizeof(UWord))) - == blk_sz); - } + ASSERT(!is_free_blk + || IS_LAST_BLK(blk) + || PREV_BLK_SZ(((char *) blk)+blk_sz) == blk_sz); if (allctr->check_block) (*allctr->check_block)(allctr, blk, (int) is_free_blk); -- cgit v1.2.3 From b07f01c20f5c01f8a6b5a60e760aad3381e1ff9a Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 9 Nov 2012 16:37:28 +0100 Subject: erts: Save one word per block for thread preferred allocators by making use of the new block header scheme to find the carrier header and thereby the allocator. --- erts/emulator/beam/erl_alloc_util.c | 178 ++++++++++++------------------------ erts/emulator/beam/erl_alloc_util.h | 1 + 2 files changed, 62 insertions(+), 117 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index b3c948c3e0..d1335598b0 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -258,6 +258,11 @@ static Uint max_mseg_carriers; /* Carriers ... */ +#define SIZEOF_SBC_HDR (UNIT_CEILING(sizeof(Carrier_t) \ + + FBLK_FTR_SZ \ + + ABLK_HDR_SZ) \ + - ABLK_HDR_SZ) + #define MSEG_CARRIER_HDR_FLAG (((UWord) 1) << 0) #define SBC_CARRIER_HDR_FLAG (((UWord) 1) << 1) @@ -266,11 +271,11 @@ static Uint max_mseg_carriers; #define SCH_MBC 0 #define SCH_SBC SBC_CARRIER_HDR_FLAG -#define SET_CARRIER_HDR(C, Sz, F) \ - (ASSERT(((Sz) & FLG_MASK) == 0), (C)->chdr = ((Sz) | (F))) +#define SET_CARRIER_HDR(C, Sz, F, AP) \ + (ASSERT(((Sz) & FLG_MASK) == 0), (C)->chdr = ((Sz) | (F)), (C)->allctr = (AP)) -#define BLK2SBC(AP, B) \ - ((Carrier_t *) (((char *) (B)) - (AP)->sbc_header_size)) +#define BLK2SBC(B) \ + ((Carrier_t *) (((char *) (B)) - SIZEOF_SBC_HDR)) #define FBLK2MBC(AP, B) \ ((Carrier_t *) (((char *) (B)) - (AP)->mbc_header_size)) @@ -796,13 +801,9 @@ erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) #define ERTS_ALCU_DD_FIX_TYPE_OFFS \ ((sizeof(ErtsAllctrDDBlock_t)-1)/sizeof(UWord) + 1) -#define ERTS_AU_PREF_ALLOC_IX_MASK \ - ((((UWord) 1) << ERTS_AU_PREF_ALLOC_BITS) - 1) -#define ERTS_AU_PREF_ALLOC_SIZE_MASK \ - ((((UWord) 1) << (sizeof(UWord)*8 - ERTS_AU_PREF_ALLOC_BITS)) - 1) -static ERTS_INLINE int -get_pref_allctr(void *extra, Allctr_t **allctr) +static ERTS_INLINE Allctr_t* +get_pref_allctr(void *extra) { ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; int pref_ix; @@ -812,34 +813,26 @@ get_pref_allctr(void *extra, Allctr_t **allctr) ASSERT(sizeof(UWord) == sizeof(Allctr_t *)); ASSERT(0 <= pref_ix && pref_ix < tspec->size); - *allctr = tspec->allctr[pref_ix]; - return pref_ix; + return tspec->allctr[pref_ix]; } -static ERTS_INLINE void * -get_used_allctr(void *extra, void *p, Allctr_t **allctr, UWord *sizep) +/* SMP note: + * get_used_allctr() must be safe WITHOUT locking the allocator while + * concurrent threads may be updating adjacent blocks. + * We rely on getting a consistent result (without atomic op) when reading + * the block header word even if a concurrent thread is updating + * the "PREV_FREE" flag bit. + */ +static ERTS_INLINE Allctr_t* +get_used_allctr(void *extra, void *p, UWord *sizep) { - ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; - void *ptr = (void *) (((char *) p) - sizeof(UWord)); - UWord ainfo = *((UWord *) ptr); - int aix = (int) (ainfo & ERTS_AU_PREF_ALLOC_IX_MASK); - *allctr = tspec->allctr[aix]; - if (sizep) - *sizep = ((ainfo >> ERTS_AU_PREF_ALLOC_BITS) - & ERTS_AU_PREF_ALLOC_SIZE_MASK); - return ptr; -} + Block_t* blk = UMEM2BLK(p); + Carrier_t* crr = IS_SBC_BLK(blk) ? BLK2SBC(blk) : BLK2MBC(blk); -static ERTS_INLINE void * -put_used_allctr(void *p, int ix, UWord size) -{ - UWord ainfo = (size >= ERTS_AU_PREF_ALLOC_SIZE_MASK - ? ERTS_AU_PREF_ALLOC_SIZE_MASK - : size); - ainfo <<= ERTS_AU_PREF_ALLOC_BITS; - ainfo |= (UWord) ix; - *((UWord *) p) = ainfo; - return (void *) (((char *) p) + sizeof(UWord)); + if (sizep) { + *sizep = BLK_UMEM_SZ(blk); + } + return crr->allctr; } static void @@ -1895,7 +1888,7 @@ create_sbmbc(Allctr_t *allctr, Uint umem_sz) crr = erts_alloc(ERTS_ALC_T_SBMBC, crr_sz); INC_CC(allctr->calls.sbmbc_alloc); - SET_CARRIER_HDR(crr, crr_sz, SCH_SYS_ALLOC|SCH_MBC); + SET_CARRIER_HDR(crr, crr_sz, SCH_SYS_ALLOC|SCH_MBC, allctr); blk = MBC2FBLK(allctr, crr); @@ -2030,12 +2023,12 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) is_mseg = 1; #endif if (flags & CFLG_SBC) { - SET_CARRIER_HDR(crr, crr_sz, SCH_MSEG|SCH_SBC); + SET_CARRIER_HDR(crr, crr_sz, SCH_MSEG|SCH_SBC, allctr); STAT_MSEG_SBC_ALLOC(allctr, crr_sz, blk_sz); goto sbc_final_touch; } else { - SET_CARRIER_HDR(crr, crr_sz, SCH_MSEG|SCH_MBC); + SET_CARRIER_HDR(crr, crr_sz, SCH_MSEG|SCH_MBC, allctr); STAT_MSEG_MBC_ALLOC(allctr, crr_sz); goto mbc_final_touch; } @@ -2075,7 +2068,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) } } if (flags & CFLG_SBC) { - SET_CARRIER_HDR(crr, crr_sz, SCH_SYS_ALLOC|SCH_SBC); + SET_CARRIER_HDR(crr, crr_sz, SCH_SYS_ALLOC|SCH_SBC, allctr); STAT_SYS_ALLOC_SBC_ALLOC(allctr, crr_sz, blk_sz); #if HAVE_ERTS_MSEG @@ -2093,7 +2086,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) } else { - SET_CARRIER_HDR(crr, crr_sz, SCH_SYS_ALLOC|SCH_MBC); + SET_CARRIER_HDR(crr, crr_sz, SCH_SYS_ALLOC|SCH_MBC, allctr); STAT_SYS_ALLOC_MBC_ALLOC(allctr, crr_sz); #if HAVE_ERTS_MSEG @@ -2144,7 +2137,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) HARD_CHECK_BLK_CARRIER(allctr, old_blk); old_blk_sz = SBC_BLK_SZ(old_blk); - old_crr = BLK2SBC(allctr, old_blk); + old_crr = BLK2SBC(old_blk); old_crr_sz = CARRIER_SZ(old_crr); ASSERT(IS_SB_CARRIER(old_crr)); ASSERT(IS_SBC_BLK(old_blk)); @@ -2270,7 +2263,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) if (IS_SBC_BLK(blk)) { Uint blk_sz = SBC_BLK_SZ(blk); - crr = BLK2SBC(allctr, blk); + crr = BLK2SBC(blk); crr_sz = CARRIER_SZ(crr); ASSERT(IS_LAST_BLK(blk)); @@ -3506,24 +3499,20 @@ erts_alcu_alloc_thr_spec(ErtsAlcType_t type, void *extra, Uint size) void * erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size) { - int pref_ix; Allctr_t *pref_allctr; void *res; - pref_ix = get_pref_allctr(extra, &pref_allctr); + pref_allctr = get_pref_allctr(extra); if (pref_allctr->thread_safe) erts_mtx_lock(&pref_allctr->mutex); ERTS_ALCU_DBG_CHK_THR_ACCESS(pref_allctr); - res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(UWord)); + res = do_erts_alcu_alloc(type, pref_allctr, size); if (pref_allctr->thread_safe) erts_mtx_unlock(&pref_allctr->mutex); - if (res) - res = put_used_allctr(res, pref_ix, size); - DEBUG_CHECK_ALIGNMENT(res); @@ -3644,21 +3633,20 @@ erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p) { if (p) { Allctr_t *pref_allctr, *used_allctr; - void *ptr; - get_pref_allctr(extra, &pref_allctr); - ptr = get_used_allctr(extra, p, &used_allctr, NULL); + pref_allctr = get_pref_allctr(extra); + used_allctr = get_used_allctr(extra, p, NULL); if (pref_allctr != used_allctr) enqueue_dealloc_other_instance(type, used_allctr, - ptr, + p, (used_allctr->dd.ix - pref_allctr->dd.ix)); else { if (used_allctr->thread_safe) erts_mtx_lock(&used_allctr->mutex); ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr); - do_erts_alcu_free(type, used_allctr, ptr); + do_erts_alcu_free(type, used_allctr, p); if (used_allctr->thread_safe) erts_mtx_unlock(&used_allctr->mutex); } @@ -3745,7 +3733,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, Uint crr_sz_val; #if HAVE_ERTS_MSEG - if (IS_SYS_ALLOC_CARRIER(BLK2SBC(allctr, blk))) + if (IS_SYS_ALLOC_CARRIER(BLK2SBC(blk))) #endif crr_sz = SYS_ALLOC_CARRIER_CEILING(used_sz); #if HAVE_ERTS_MSEG @@ -3962,16 +3950,15 @@ static ERTS_INLINE void * realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size, int force_move) { - int pref_ix; - void *ptr, *res; + void *res; Allctr_t *pref_allctr, *used_allctr; UWord old_user_size; if (!p) return erts_alcu_alloc_thr_pref(type, extra, size); - pref_ix = get_pref_allctr(extra, &pref_allctr); - ptr = get_used_allctr(extra, p, &used_allctr, &old_user_size); + pref_allctr = get_pref_allctr(extra); + used_allctr = get_used_allctr(extra, p, &old_user_size); ASSERT(used_allctr && pref_allctr); @@ -3981,56 +3968,33 @@ realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size, ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr); res = do_erts_alcu_realloc(type, used_allctr, - ptr, - size + sizeof(UWord), + p, + size, 0); if (used_allctr->thread_safe) erts_mtx_unlock(&used_allctr->mutex); - if (res) - res = put_used_allctr(res, pref_ix, size); } else { if (pref_allctr->thread_safe) erts_mtx_lock(&pref_allctr->mutex); - res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(UWord)); - if (pref_allctr->thread_safe && (!force_move - || used_allctr != pref_allctr)) + res = do_erts_alcu_alloc(type, pref_allctr, size); + if (pref_allctr->thread_safe && used_allctr != pref_allctr) { erts_mtx_unlock(&pref_allctr->mutex); + } if (res) { - Block_t *blk; - size_t cpy_size; - - res = put_used_allctr(res, pref_ix, size); - DEBUG_CHECK_ALIGNMENT(res); - blk = UMEM2BLK(ptr); - if (old_user_size != ERTS_AU_PREF_ALLOC_SIZE_MASK) - cpy_size = old_user_size; - else { - if (used_allctr->thread_safe && (!force_move - || used_allctr != pref_allctr)) - erts_mtx_lock(&used_allctr->mutex); - ERTS_SMP_LC_ASSERT(!used_allctr->thread_safe || - erts_lc_mtx_is_locked(&used_allctr->mutex)); - cpy_size = BLK_SZ(blk); - if (used_allctr->thread_safe && (!force_move - || used_allctr != pref_allctr)) - erts_mtx_unlock(&used_allctr->mutex); - cpy_size -= ABLK_HDR_SZ + sizeof(UWord); - } - if (cpy_size > size) - cpy_size = size; - sys_memcpy(res, p, cpy_size); + sys_memcpy(res, p, MIN(size,old_user_size)); - if (!force_move || used_allctr != pref_allctr) + if (used_allctr != pref_allctr) { enqueue_dealloc_other_instance(type, used_allctr, - ptr, + p, (used_allctr->dd.ix - pref_allctr->dd.ix)); + } else { - do_erts_alcu_free(type, used_allctr, ptr); + do_erts_alcu_free(type, used_allctr, p); ASSERT(pref_allctr == used_allctr); if (pref_allctr->thread_safe) erts_mtx_unlock(&pref_allctr->mutex); @@ -4208,35 +4172,16 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #ifdef ERTS_SMP allctr->dd.use = 0; if (init->tpref) { - allctr->mbc_header_size = (UNIT_CEILING(allctr->mbc_header_size - + FBLK_FTR_SZ - + ABLK_HDR_SZ - + sizeof(UWord)) - - ABLK_HDR_SZ - - sizeof(UWord)); - allctr->sbc_header_size = (UNIT_CEILING(sizeof(Carrier_t) - + FBLK_FTR_SZ - + ABLK_HDR_SZ - + sizeof(UWord)) - - ABLK_HDR_SZ - - sizeof(UWord)); - allctr->dd.use = 1; init_dd_queue(&allctr->dd.q); allctr->dd.ix = init->ix; } - else #endif - { - allctr->mbc_header_size = (UNIT_CEILING(allctr->mbc_header_size - + FBLK_FTR_SZ - + ABLK_HDR_SZ) - - ABLK_HDR_SZ); - allctr->sbc_header_size = (UNIT_CEILING(sizeof(Carrier_t) - + FBLK_FTR_SZ - + ABLK_HDR_SZ) - - ABLK_HDR_SZ); - } + allctr->mbc_header_size = (UNIT_CEILING(allctr->mbc_header_size + + FBLK_FTR_SZ + + ABLK_HDR_SZ) + - ABLK_HDR_SZ); + allctr->sbc_header_size = SIZEOF_SBC_HDR; if (allctr->main_carrier_size) { Block_t *blk; @@ -4362,8 +4307,7 @@ erts_alcu_test(unsigned long op, unsigned long a1, unsigned long a2) case 0x00b: return (unsigned long) CARRIER_SZ((Carrier_t *) a1); case 0x00c: return (unsigned long) SBC2BLK((Allctr_t *) a1, (Carrier_t *) a2); - case 0x00d: return (unsigned long) BLK2SBC((Allctr_t *) a1, - (Block_t *) a2); + case 0x00d: return (unsigned long) BLK2SBC((Block_t *) a2); case 0x00e: return (unsigned long) MBC2FBLK((Allctr_t *) a1, (Carrier_t *) a2); case 0x00f: return (unsigned long) FBLK2MBC((Allctr_t *) a1, @@ -4438,7 +4382,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) CarrierList_t *cl; if (IS_SBC_BLK(iblk)) { - Carrier_t *sbc = BLK2SBC(allctr, iblk); + Carrier_t *sbc = BLK2SBC(iblk); ASSERT(SBC2BLK(allctr, sbc) == iblk); ASSERT(CARRIER_SZ(sbc) - allctr->sbc_header_size >= SBC_BLK_SZ(iblk)); diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index 1e6ba5736a..7d6f61fdff 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -245,6 +245,7 @@ struct Carrier_t_ { UWord chdr; Carrier_t *next; Carrier_t *prev; + Allctr_t *allctr; }; typedef struct { -- cgit v1.2.3 From 9a763d644ca0e549f2733fd414aee9e77691b376 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 9 Nov 2012 19:06:01 +0100 Subject: erts: Refactor removing dummy footer in carrier header --- erts/emulator/beam/erl_alloc_util.c | 45 ++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 15 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index d1335598b0..6ce872895b 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -118,7 +118,34 @@ static Uint max_mseg_carriers; ? ((CC).giga_no--, (CC).no = ONE_GIGA - 1) \ : (CC).no--) -/* ... */ +/* Multi block carrier (MBC) memory layout in R16: + +Empty MBC: +[Carrier_t|pad|Block_t L0T|fhdr| free... ] + +MBC after allocating first block: +[Carrier_t|pad|Block_t 000| udata |pad|Block_t L0T|fhdr| free... ] + +MBC after allocating second block: +[Carrier_t|pad|Block_t 000| udata |pad|Block_t 000| udata |pad|Block_t L0T|fhdr| free... ] + +MBC after deallocating first block: +[Carrier_t|pad|Block_t 00T|fhdr| free |FreeBlkFtr_t|Block_t 0P0| udata |pad|Block_t L0T|fhdr| free... ] + + + udata = Allocated user data + pad = Padding to ensure correct alignment for user data + fhdr = Allocator specific header to keep track of free block + free = Unused free memory + T = This block is free (THIS_FREE_BLK_HDR_FLG) + P = Previous block is free (PREV_FREE_BLK_HDR_FLG) + L = Last block in carrier (LAST_BLK_HDR_FLG) +*/ + +/* Single block carrier (SBC): +[Carrier_t|pad|Block_t 111| udata... ] +*/ + /* Blocks ... */ @@ -137,8 +164,7 @@ static Uint max_mseg_carriers; #define UMEM2BLK(P) ((Block_t *) (((char *) (P)) - ABLK_HDR_SZ)) #define BLK2UMEM(P) ((void *) (((char *) (P)) + ABLK_HDR_SZ)) -#define PREV_BLK_SZ(B) \ - ((UWord) (((FreeBlkFtr_t *)(B))[-1] & MBC_BLK_SZ_MASK)) +#define PREV_BLK_SZ(B) ((UWord) (((FreeBlkFtr_t *)(B))[-1])) #define SET_BLK_SZ_FTR(B, SZ) \ (((FreeBlkFtr_t *) (((char *) (B)) + (SZ)))[-1] = (SZ)) @@ -239,11 +265,6 @@ static Uint max_mseg_carriers; #define GET_BLK_HDR_FLGS(B) \ ((B)->bhdr & FLG_MASK) -#define SET_SBC_BLK_FTR(FTR) \ - ((FTR) = 0) -#define SET_MBC_BLK_FTR(FTR) \ - ((FTR) = 0) - #define IS_SBC_BLK(B) \ (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG) #define IS_MBC_BLK(B) \ @@ -258,9 +279,7 @@ static Uint max_mseg_carriers; /* Carriers ... */ -#define SIZEOF_SBC_HDR (UNIT_CEILING(sizeof(Carrier_t) \ - + FBLK_FTR_SZ \ - + ABLK_HDR_SZ) \ +#define SIZEOF_SBC_HDR (UNIT_CEILING(sizeof(Carrier_t) + ABLK_HDR_SZ) \ - ABLK_HDR_SZ) #define MSEG_CARRIER_HDR_FLAG (((UWord) 1) << 0) @@ -1894,7 +1913,6 @@ create_sbmbc(Allctr_t *allctr, Uint umem_sz) blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size); - SET_MBC_BLK_FTR(((FreeBlkFtr_t *) blk)[-1]); SET_MBC_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr); link_carrier(&allctr->sbmbc_list, crr); @@ -2077,7 +2095,6 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) blk = SBC2BLK(allctr, crr); - SET_SBC_BLK_FTR(((FreeBlkFtr_t *) blk)[-1]); SET_SBC_BLK_HDR(blk, blk_sz); link_carrier(&allctr->sbc_list, crr); @@ -2097,7 +2114,6 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size); - SET_MBC_BLK_FTR(((FreeBlkFtr_t *) blk)[-1]); SET_MBC_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr); if (flags & CFLG_MAIN_CARRIER) { @@ -4178,7 +4194,6 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) } #endif allctr->mbc_header_size = (UNIT_CEILING(allctr->mbc_header_size - + FBLK_FTR_SZ + ABLK_HDR_SZ) - ABLK_HDR_SZ); allctr->sbc_header_size = SIZEOF_SBC_HDR; -- cgit v1.2.3 From 51277f5dcf29a085dc478e560d3b92a438c8ae2f Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 15 Nov 2012 12:31:59 +0100 Subject: erts: Refactor renaming a couple of macros in alloc_util To get a consistent naming scheme where FBLK = Free block ABLK = Allocated block --- erts/emulator/beam/erl_alloc_util.c | 64 ++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 32 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 6ce872895b..3f8d3f23c3 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -213,7 +213,7 @@ MBC after deallocating first block: ASSERT((UWord)(F) < SBC_BLK_HDR_FLG), \ (B)->bhdr = ((Sz) | (F) | (BLK_CARRIER_OFFSET(B,C) << CARRIER_OFFSET_SHIFT))) -# define BLK2MBC(B) \ +# define BLK_TO_MBC(B) \ (ASSERT(IS_MBC_BLK(B)), \ (Carrier_t*)((MSEG_UNIT_FLOOR((UWord)(B)) - \ (((B)->bhdr >> CARRIER_OFFSET_SHIFT) << MSEG_UNIT_SHIFT)))) @@ -230,7 +230,7 @@ MBC after deallocating first block: (B)->bhdr = ((Sz) | (F)), \ (B)->carrier = (C)) -# define BLK2MBC(B) ((B)->carrier) +# define BLK_TO_MBC(B) ((B)->carrier) # define IS_MBC_FIRST_BLK(AP,B) \ ((char*)(B) == (char*)((B)->carrier) + (AP)->mbc_header_size) @@ -293,12 +293,12 @@ MBC after deallocating first block: #define SET_CARRIER_HDR(C, Sz, F, AP) \ (ASSERT(((Sz) & FLG_MASK) == 0), (C)->chdr = ((Sz) | (F)), (C)->allctr = (AP)) -#define BLK2SBC(B) \ +#define BLK_TO_SBC(B) \ ((Carrier_t *) (((char *) (B)) - SIZEOF_SBC_HDR)) -#define FBLK2MBC(AP, B) \ +#define FIRST_BLK_TO_MBC(AP, B) \ ((Carrier_t *) (((char *) (B)) - (AP)->mbc_header_size)) -#define MBC2FBLK(AP, P) \ +#define MBC_TO_FIRST_BLK(AP, P) \ ((Block_t *) (((char *) (P)) + (AP)->mbc_header_size)) #define SBC2BLK(AP, P) \ ((Block_t *) (((char *) (P)) + (AP)->sbc_header_size)) @@ -846,7 +846,7 @@ static ERTS_INLINE Allctr_t* get_used_allctr(void *extra, void *p, UWord *sizep) { Block_t* blk = UMEM2BLK(p); - Carrier_t* crr = IS_SBC_BLK(blk) ? BLK2SBC(blk) : BLK2MBC(blk); + Carrier_t* crr = IS_SBC_BLK(blk) ? BLK_TO_SBC(blk) : BLK_TO_MBC(blk); if (sizep) { *sizep = BLK_UMEM_SZ(blk); @@ -1291,7 +1291,7 @@ mbc_alloc_finalize(Allctr_t *allctr, Uint nxt_blk_sz; Block_t *nxt_blk; UWord prev_free_flg = flags & PREV_FREE_BLK_HDR_FLG; - Carrier_t* crr = BLK2MBC(blk); + Carrier_t* crr = BLK_TO_MBC(blk); ASSERT(org_blk_sz >= want_blk_sz); ASSERT(blk); @@ -1332,8 +1332,8 @@ mbc_alloc_finalize(Allctr_t *allctr, ASSERT(nxt_blk_sz == MBC_BLK_SZ(nxt_blk)); ASSERT(nxt_blk_sz % sizeof(Unit_t) == 0); ASSERT(nxt_blk_sz >= allctr->min_block_size); - ASSERT(BLK2MBC(blk) == crr); - ASSERT(BLK2MBC(nxt_blk) == crr); + ASSERT(BLK_TO_MBC(blk) == crr); + ASSERT(BLK_TO_MBC(nxt_blk) == crr); } else { blk_sz = org_blk_sz; @@ -1355,7 +1355,7 @@ mbc_alloc_finalize(Allctr_t *allctr, ASSERT((flags & LAST_BLK_HDR_FLG) ? IS_LAST_BLK(blk) : IS_NOT_LAST_BLK(blk)); - ASSERT(BLK2MBC(blk) == crr); + ASSERT(BLK_TO_MBC(blk) == crr); } STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); @@ -1473,7 +1473,7 @@ mbc_free(Allctr_t *allctr, void *p) if (is_first_blk && is_last_blk - && allctr->main_carrier != FBLK2MBC(allctr, blk)) { + && allctr->main_carrier != FIRST_BLK_TO_MBC(allctr, blk)) { if (alcu_flgs & ERTS_ALCU_FLG_SBMBC) destroy_sbmbc(allctr, blk); else @@ -1614,7 +1614,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); } - crr = BLK2MBC(blk); + crr = BLK_TO_MBC(blk); SET_MBC_BLK_HDR(nxt_blk, nxt_blk_sz, SBH_THIS_FREE | (is_last_blk ? SBH_LAST_BLK : 0), crr); @@ -1638,7 +1638,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) ASSERT(is_last_blk ? IS_LAST_BLK(nxt_blk) : IS_NOT_LAST_BLK(nxt_blk)); ASSERT(is_last_blk || nxt_blk == PREV_BLK(NXT_BLK(nxt_blk))); ASSERT(is_last_blk || IS_PREV_BLK_FREE(NXT_BLK(nxt_blk))); - ASSERT(BLK2MBC(nxt_blk) == crr); + ASSERT(BLK_TO_MBC(nxt_blk) == crr); HARD_CHECK_BLK_CARRIER(allctr, blk); @@ -1680,7 +1680,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) } } else { - Carrier_t* crr = BLK2MBC(blk); + Carrier_t* crr = BLK_TO_MBC(blk); SET_MBC_BLK_SZ(blk, blk_sz); nxt_blk = NXT_BLK(blk); @@ -1694,7 +1694,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); ASSERT(IS_FREE_BLK(nxt_blk)); - ASSERT(BLK2MBC(nxt_blk) == crr); + ASSERT(BLK_TO_MBC(nxt_blk) == crr); } STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); @@ -1875,7 +1875,7 @@ static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C, ASSERT(IS_MBC_FIRST_BLK(A, (B))); ASSERT(IS_MBC_BLK((B))); ASSERT(IS_MB_CARRIER((C))); - ASSERT(BLK2MBC(B) == (C)); + ASSERT(BLK_TO_MBC(B) == (C)); } if ((MSEGED)) { ASSERT(IS_MSEG_CARRIER((C))); @@ -1909,7 +1909,7 @@ create_sbmbc(Allctr_t *allctr, Uint umem_sz) INC_CC(allctr->calls.sbmbc_alloc); SET_CARRIER_HDR(crr, crr_sz, SCH_SYS_ALLOC|SCH_MBC, allctr); - blk = MBC2FBLK(allctr, crr); + blk = MBC_TO_FIRST_BLK(allctr, crr); blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size); @@ -1935,7 +1935,7 @@ destroy_sbmbc(Allctr_t *allctr, Block_t *blk) ASSERT(IS_MBC_BLK(blk)); ASSERT(IS_MBC_FIRST_BLK(allctr, blk)); - crr = FBLK2MBC(allctr, blk); + crr = FIRST_BLK_TO_MBC(allctr, blk); crr_sz = CARRIER_SZ(crr); #ifdef DEBUG @@ -2110,7 +2110,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) mbc_final_touch: #endif - blk = MBC2FBLK(allctr, crr); + blk = MBC_TO_FIRST_BLK(allctr, crr); blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size); @@ -2153,7 +2153,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) HARD_CHECK_BLK_CARRIER(allctr, old_blk); old_blk_sz = SBC_BLK_SZ(old_blk); - old_crr = BLK2SBC(old_blk); + old_crr = BLK_TO_SBC(old_blk); old_crr_sz = CARRIER_SZ(old_crr); ASSERT(IS_SB_CARRIER(old_crr)); ASSERT(IS_SBC_BLK(old_blk)); @@ -2279,7 +2279,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) if (IS_SBC_BLK(blk)) { Uint blk_sz = SBC_BLK_SZ(blk); - crr = BLK2SBC(blk); + crr = BLK_TO_SBC(blk); crr_sz = CARRIER_SZ(crr); ASSERT(IS_LAST_BLK(blk)); @@ -2301,7 +2301,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) } else { ASSERT(IS_MBC_FIRST_BLK(allctr, blk)); - crr = FBLK2MBC(allctr, blk); + crr = FIRST_BLK_TO_MBC(allctr, blk); crr_sz = CARRIER_SZ(crr); #ifdef DEBUG @@ -3749,7 +3749,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, Uint crr_sz_val; #if HAVE_ERTS_MSEG - if (IS_SYS_ALLOC_CARRIER(BLK2SBC(blk))) + if (IS_SYS_ALLOC_CARRIER(BLK_TO_SBC(blk))) #endif crr_sz = SYS_ALLOC_CARRIER_CEILING(used_sz); #if HAVE_ERTS_MSEG @@ -4256,9 +4256,9 @@ erts_alcu_stop(Allctr_t *allctr) while (allctr->sbc_list.first) destroy_carrier(allctr, SBC2BLK(allctr, allctr->sbc_list.first)); while (allctr->mbc_list.first) - destroy_carrier(allctr, MBC2FBLK(allctr, allctr->mbc_list.first)); + destroy_carrier(allctr, MBC_TO_FIRST_BLK(allctr, allctr->mbc_list.first)); while (allctr->sbmbc_list.first) - destroy_sbmbc(allctr, MBC2FBLK(allctr, allctr->sbmbc_list.first)); + destroy_sbmbc(allctr, MBC_TO_FIRST_BLK(allctr, allctr->sbmbc_list.first)); #ifdef USE_THREADS if (allctr->thread_safe) @@ -4322,10 +4322,10 @@ erts_alcu_test(unsigned long op, unsigned long a1, unsigned long a2) case 0x00b: return (unsigned long) CARRIER_SZ((Carrier_t *) a1); case 0x00c: return (unsigned long) SBC2BLK((Allctr_t *) a1, (Carrier_t *) a2); - case 0x00d: return (unsigned long) BLK2SBC((Block_t *) a2); - case 0x00e: return (unsigned long) MBC2FBLK((Allctr_t *) a1, + case 0x00d: return (unsigned long) BLK_TO_SBC((Block_t *) a2); + case 0x00e: return (unsigned long) MBC_TO_FIRST_BLK((Allctr_t *) a1, (Carrier_t *) a2); - case 0x00f: return (unsigned long) FBLK2MBC((Allctr_t *) a1, + case 0x00f: return (unsigned long) FIRST_BLK_TO_MBC((Allctr_t *) a1, (Block_t *) a2); case 0x010: return (unsigned long) ((Allctr_t *) a1)->mbc_list.first; case 0x011: return (unsigned long) ((Allctr_t *) a1)->mbc_list.last; @@ -4397,7 +4397,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) CarrierList_t *cl; if (IS_SBC_BLK(iblk)) { - Carrier_t *sbc = BLK2SBC(iblk); + Carrier_t *sbc = BLK_TO_SBC(iblk); ASSERT(SBC2BLK(allctr, sbc) == iblk); ASSERT(CARRIER_SZ(sbc) - allctr->sbc_header_size >= SBC_BLK_SZ(iblk)); @@ -4420,14 +4420,14 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) blk = iblk; tot_blk_sz = 0; - crr = BLK2MBC(blk); + crr = BLK_TO_MBC(blk); ASSERT(IS_MB_CARRIER(crr)); /* Step around the carrier one whole lap starting at 'iblk' */ while (1) { ASSERT(IS_MBC_BLK(blk)); - ASSERT(BLK2MBC(blk) == crr); + ASSERT(BLK_TO_MBC(blk) == crr); if (prev_blk) { ASSERT(NXT_BLK(prev_blk) == blk); @@ -4468,7 +4468,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) carrier_end = ((char *) NXT_BLK(blk)); has_wrapped_around = 1; prev_blk = NULL; - blk = MBC2FBLK(allctr, crr); + blk = MBC_TO_FIRST_BLK(allctr, crr); ASSERT(IS_MBC_FIRST_BLK(allctr,blk)); } else { -- cgit v1.2.3 From 1ff4fffe893346160e5136a3e4a1999e8927b5ec Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 14 Nov 2012 19:25:09 +0100 Subject: erts: Add carrier pointer to header of free block --- erts/emulator/beam/erl_afit_alloc.c | 2 +- erts/emulator/beam/erl_alloc_util.c | 203 +++++++++++++++++++---------- erts/emulator/beam/erl_alloc_util.h | 17 ++- erts/emulator/beam/erl_ao_firstfit_alloc.c | 2 +- erts/emulator/beam/erl_bestfit_alloc.c | 2 +- erts/emulator/beam/erl_goodfit_alloc.c | 10 +- 6 files changed, 154 insertions(+), 82 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c index ad7a5822df..02171e050f 100644 --- a/erts/emulator/beam/erl_afit_alloc.c +++ b/erts/emulator/beam/erl_afit_alloc.c @@ -42,7 +42,7 @@ struct AFFreeBlock_t_ { AFFreeBlock_t *prev; AFFreeBlock_t *next; }; -#define AF_BLK_SZ(B) MBC_BLK_SZ(&(B)->block_head) +#define AF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->block_head) #define MIN_MBC_SZ (16*1024) #define MIN_MBC_FIRST_FREE_SZ (4*1024) diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 3f8d3f23c3..c8e2d28c49 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -153,7 +153,7 @@ MBC after deallocating first block: #define UNUSED1_BLK_FTR_FLG (((UWord) 1) << 1) #define UNUSED2_BLK_FTR_FLG (((UWord) 1) << 2) -#define ABLK_HDR_SZ (sizeof(Block_t)) +#define ABLK_HDR_SZ (offsetof(Block_t,u)) #define FBLK_FTR_SZ (sizeof(FreeBlkFtr_t)) #define UMEMSZ2BLKSZ(AP, SZ) \ @@ -177,17 +177,15 @@ MBC after deallocating first block: */ #define SBC_BLK_HDR_FLG (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) -#define SET_MBC_BLK_SZ(B, SZ) \ +#define SET_MBC_ABLK_SZ(B, SZ) \ (ASSERT(((SZ) & FLG_MASK) == 0), \ - (B)->bhdr = (((B)->bhdr) & ~MBC_BLK_SZ_MASK) | (SZ)) + (B)->bhdr = (((B)->bhdr) & ~MBC_ABLK_SZ_MASK) | (SZ)) +#define SET_MBC_FBLK_SZ(B, SZ) \ + (ASSERT(((SZ) & FLG_MASK) == 0), \ + (B)->bhdr = (((B)->bhdr) & ~MBC_FBLK_SZ_MASK) | (SZ)) #define SET_SBC_BLK_SZ(B, SZ) \ (ASSERT(((SZ) & FLG_MASK) == 0), \ (B)->bhdr = (((B)->bhdr) & ~SBC_BLK_SZ_MASK) | (SZ)) -#define SET_BLK_FREE(B) \ - (ASSERT(!IS_PREV_BLK_FREE(B)), \ - (B)->bhdr |= THIS_FREE_BLK_HDR_FLG) -#define SET_BLK_ALLOCED(B) \ - ((B)->bhdr &= ~THIS_FREE_BLK_HDR_FLG) #define SET_PREV_BLK_FREE(AP,B) \ (ASSERT(!IS_MBC_FIRST_BLK(AP,B)), \ ASSERT(!IS_FREE_BLK(B)), \ @@ -208,32 +206,79 @@ MBC after deallocating first block: # define BLK_CARRIER_OFFSET(B, C) (((char*)(B) - (char*)(C)) >> MSEG_UNIT_SHIFT) -# define SET_MBC_BLK_HDR(B, Sz, F, C) \ - (ASSERT(((Sz) & FLG_MASK) == 0), \ - ASSERT((UWord)(F) < SBC_BLK_HDR_FLG), \ +# define SET_MBC_ABLK_HDR(B, Sz, F, C) \ + (ASSERT(((Sz) & ~MBC_ABLK_SZ_MASK) == 0), \ + ASSERT(!((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG))), \ (B)->bhdr = ((Sz) | (F) | (BLK_CARRIER_OFFSET(B,C) << CARRIER_OFFSET_SHIFT))) -# define BLK_TO_MBC(B) \ - (ASSERT(IS_MBC_BLK(B)), \ +# define SET_MBC_FBLK_HDR(B, Sz, F, C) \ + (ASSERT(((Sz) & ~MBC_FBLK_SZ_MASK) == 0), \ + ASSERT(((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ + (B)->bhdr = ((Sz) | (F)), \ + (B)->u.carrier = (C)) + +# define ABLK_TO_MBC(B) \ + (ASSERT(IS_MBC_BLK(B) && IS_ALLOCED_BLK(B)), \ (Carrier_t*)((MSEG_UNIT_FLOOR((UWord)(B)) - \ (((B)->bhdr >> CARRIER_OFFSET_SHIFT) << MSEG_UNIT_SHIFT)))) -# define IS_MBC_FIRST_BLK(AP,B) \ +# define FBLK_TO_MBC(B) \ + (ASSERT(IS_MBC_BLK(B) && IS_FREE_BLK(B)), \ + (B)->u.carrier) + +# define BLK_TO_MBC(B) (IS_FREE_BLK(B) ? FBLK_TO_MBC(B) : ABLK_TO_MBC(B)) + +# define IS_MBC_FIRST_ABLK(AP,B) \ ((((UWord)(B) & ~MSEG_UNIT_MASK) == (AP)->mbc_header_size) \ && ((B)->bhdr & CARRIER_OFFSET_MASK) == 0) +# define IS_MBC_FIRST_FBLK(AP,B) \ + ((char*)(B) == (char*)((B)->u.carrier) + (AP)->mbc_header_size) + +# define IS_MBC_FIRST_BLK(AP,B) \ + (IS_FREE_BLK(B) ? IS_MBC_FIRST_FBLK(AP,B) : IS_MBC_FIRST_ABLK(AP,B)) + +# define SET_BLK_FREE(B) \ + (ASSERT(!IS_PREV_BLK_FREE(B)), \ + (B)->u.carrier = ABLK_TO_MBC(B), \ + (B)->bhdr |= THIS_FREE_BLK_HDR_FLG, \ + (B)->bhdr &= (MBC_ABLK_SZ_MASK|FLG_MASK)) + +# define SET_BLK_ALLOCED(B) \ + (ASSERT(((B)->bhdr & (CARRIER_OFFSET_MASK|THIS_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ + (B)->bhdr &= ~THIS_FREE_BLK_HDR_FLG, \ + (B)->bhdr |= (BLK_CARRIER_OFFSET(B,(B)->u.carrier) << CARRIER_OFFSET_SHIFT)) + #else /* !HAVE_SUPER_ALIGNED_MB_CARRIERS */ -# define SET_MBC_BLK_HDR(B, Sz, F, C) \ +# define SET_MBC_ABLK_HDR(B, Sz, F, C) \ (ASSERT(((Sz) & FLG_MASK) == 0), \ + ASSERT(!((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG))), \ ASSERT((UWord)(F) < SBC_BLK_HDR_FLG), \ (B)->bhdr = ((Sz) | (F)), \ (B)->carrier = (C)) +# define SET_MBC_FBLK_HDR(B, Sz, F, C) \ + (ASSERT(((Sz) & FLG_MASK) == 0), \ + ASSERT(((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ + (B)->bhdr = ((Sz) | (F)), \ + (B)->carrier = (C)) + # define BLK_TO_MBC(B) ((B)->carrier) +# define ABLK_TO_MBC(B) BLK_TO_MBC(B) +# define FBLK_TO_MBC(B) BLK_TO_MBC(B) # define IS_MBC_FIRST_BLK(AP,B) \ ((char*)(B) == (char*)((B)->carrier) + (AP)->mbc_header_size) +# define IS_MBC_FIRST_ABLK(AP,B) IS_MBC_FIRST_BLK(AP,B) +# define IS_MBC_FIRST_FBLK(AP,B) IS_MBC_FIRST_BLK(AP,B) + +# define SET_BLK_FREE(B) \ + (ASSERT(!IS_PREV_BLK_FREE(B)), \ + (B)->bhdr |= THIS_FREE_BLK_HDR_FLG) + +# define SET_BLK_ALLOCED(B) \ + ((B)->bhdr &= ~THIS_FREE_BLK_HDR_FLG) #endif /* !HAVE_SUPER_ALIGNED_MB_CARRIERS */ @@ -270,12 +315,18 @@ MBC after deallocating first block: #define IS_MBC_BLK(B) \ (!IS_SBC_BLK((B))) +#define MBC_BLK_SZ(B) (IS_FREE_BLK(B) ? MBC_FBLK_SZ(B) : MBC_ABLK_SZ(B)) + #define NXT_BLK(B) \ - ((Block_t *) (((char *) (B)) + MBC_BLK_SZ((B)))) + (ASSERT(IS_MBC_BLK(B)), \ + (Block_t *) (((char *) (B)) + MBC_BLK_SZ((B)))) #define PREV_BLK(B) \ ((Block_t *) (((char *) (B)) - PREV_BLK_SZ((B)))) -#define BLK_SZ(B) ((B)->bhdr & (IS_SBC_BLK(B) ? SBC_BLK_SZ_MASK : MBC_BLK_SZ_MASK)) +#define BLK_AFTER(B,Sz) \ + ((Block_t *) (((char *) (B)) + (Sz))) + +#define BLK_SZ(B) ((B)->bhdr & (((B)->bhdr & THIS_FREE_BLK_HDR_FLG) ? MBC_FBLK_SZ_MASK : MBC_ABLK_SZ_MASK)) /* Carriers ... */ @@ -846,10 +897,17 @@ static ERTS_INLINE Allctr_t* get_used_allctr(void *extra, void *p, UWord *sizep) { Block_t* blk = UMEM2BLK(p); - Carrier_t* crr = IS_SBC_BLK(blk) ? BLK_TO_SBC(blk) : BLK_TO_MBC(blk); + Carrier_t* crr; - if (sizep) { - *sizep = BLK_UMEM_SZ(blk); + if (IS_SBC_BLK(blk)) { + crr = BLK_TO_SBC(blk); + if (sizep) + *sizep = SBC_BLK_SZ(blk) - ABLK_HDR_SZ; + } + else { + crr = ABLK_TO_MBC(blk); + if (sizep) + *sizep = MBC_ABLK_SZ(blk) - ABLK_HDR_SZ; } return crr->allctr; } @@ -1283,6 +1341,7 @@ mbc_alloc_finalize(Allctr_t *allctr, Block_t *blk, Uint org_blk_sz, UWord flags, + Carrier_t *crr, Uint want_blk_sz, int valid_blk_info, Uint32 alcu_flgs) @@ -1291,7 +1350,6 @@ mbc_alloc_finalize(Allctr_t *allctr, Uint nxt_blk_sz; Block_t *nxt_blk; UWord prev_free_flg = flags & PREV_FREE_BLK_HDR_FLG; - Carrier_t* crr = BLK_TO_MBC(blk); ASSERT(org_blk_sz >= want_blk_sz); ASSERT(blk); @@ -1304,17 +1362,17 @@ mbc_alloc_finalize(Allctr_t *allctr, /* Shrink block... */ blk_sz = want_blk_sz; nxt_blk_sz = org_blk_sz - blk_sz; - SET_MBC_BLK_HDR(blk, blk_sz, prev_free_flg, crr); + SET_MBC_ABLK_HDR(blk, blk_sz, prev_free_flg, crr); - nxt_blk = NXT_BLK(blk); - SET_MBC_BLK_HDR(nxt_blk, nxt_blk_sz, - SBH_THIS_FREE|(flags & LAST_BLK_HDR_FLG), - crr); + nxt_blk = BLK_AFTER(blk, blk_sz); + SET_MBC_FBLK_HDR(nxt_blk, nxt_blk_sz, + SBH_THIS_FREE|(flags & LAST_BLK_HDR_FLG), + crr); if (!(flags & LAST_BLK_HDR_FLG)) { SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); if (!valid_blk_info) { - Block_t *nxt_nxt_blk = NXT_BLK(nxt_blk); + Block_t *nxt_nxt_blk = BLK_AFTER(nxt_blk, nxt_blk_sz); SET_PREV_BLK_FREE(allctr, nxt_nxt_blk); } } @@ -1332,8 +1390,8 @@ mbc_alloc_finalize(Allctr_t *allctr, ASSERT(nxt_blk_sz == MBC_BLK_SZ(nxt_blk)); ASSERT(nxt_blk_sz % sizeof(Unit_t) == 0); ASSERT(nxt_blk_sz >= allctr->min_block_size); - ASSERT(BLK_TO_MBC(blk) == crr); - ASSERT(BLK_TO_MBC(nxt_blk) == crr); + ASSERT(ABLK_TO_MBC(blk) == crr); + ASSERT(FBLK_TO_MBC(nxt_blk) == crr); } else { blk_sz = org_blk_sz; @@ -1341,21 +1399,21 @@ mbc_alloc_finalize(Allctr_t *allctr, if (valid_blk_info) SET_BLK_ALLOCED(blk); else - SET_MBC_BLK_HDR(blk, blk_sz, SBH_LAST_BLK|prev_free_flg, crr); + SET_MBC_ABLK_HDR(blk, blk_sz, SBH_LAST_BLK|prev_free_flg, crr); } else { if (valid_blk_info) SET_BLK_ALLOCED(blk); else - SET_MBC_BLK_HDR(blk, blk_sz, prev_free_flg, crr); - nxt_blk = NXT_BLK(blk); + SET_MBC_ABLK_HDR(blk, blk_sz, prev_free_flg, crr); + nxt_blk = BLK_AFTER(blk, blk_sz); SET_PREV_BLK_ALLOCED(nxt_blk); } ASSERT((flags & LAST_BLK_HDR_FLG) ? IS_LAST_BLK(blk) : IS_NOT_LAST_BLK(blk)); - ASSERT(BLK_TO_MBC(blk) == crr); + ASSERT(ABLK_TO_MBC(blk) == crr); } STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); @@ -1387,6 +1445,7 @@ mbc_alloc(Allctr_t *allctr, Uint size) blk, MBC_BLK_SZ(blk), GET_BLK_HDR_FLGS(blk), + FBLK_TO_MBC(blk), blk_sz, 1, alcu_flgs); @@ -1418,7 +1477,7 @@ mbc_free(Allctr_t *allctr, void *p) STAT_MBC_BLK_FREE(allctr, blk_sz, alcu_flgs); - is_first_blk = IS_MBC_FIRST_BLK(allctr, blk); + is_first_blk = IS_MBC_FIRST_ABLK(allctr, blk); is_last_blk = IS_LAST_BLK(blk); if (IS_PREV_BLK_FREE(blk)) { @@ -1428,8 +1487,8 @@ mbc_free(Allctr_t *allctr, void *p) (*allctr->unlink_free_block)(allctr, blk, alcu_flgs); blk_sz += MBC_BLK_SZ(blk); - is_first_blk = IS_MBC_FIRST_BLK(allctr, blk); - SET_MBC_BLK_SZ(blk, blk_sz); + is_first_blk = IS_MBC_FIRST_FBLK(allctr, blk); + SET_MBC_FBLK_SZ(blk, blk_sz); } else { SET_BLK_FREE(blk); @@ -1438,12 +1497,12 @@ mbc_free(Allctr_t *allctr, void *p) if (is_last_blk) SET_LAST_BLK(blk); else { - nxt_blk = NXT_BLK(blk); + nxt_blk = BLK_AFTER(blk, blk_sz); if (IS_FREE_BLK(nxt_blk)) { /* Coalesce with next block... */ (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); - blk_sz += MBC_BLK_SZ(nxt_blk); - SET_MBC_BLK_SZ(blk, blk_sz); + blk_sz += MBC_FBLK_SZ(nxt_blk); + SET_MBC_FBLK_SZ(blk, blk_sz); is_last_blk = IS_LAST_BLK(nxt_blk); if (is_last_blk) @@ -1461,9 +1520,9 @@ mbc_free(Allctr_t *allctr, void *p) } - ASSERT(!is_last_blk == !IS_LAST_BLK(blk)); - ASSERT(!is_first_blk == !IS_MBC_FIRST_BLK(allctr, blk)); ASSERT(IS_FREE_BLK(blk)); + ASSERT(!is_last_blk == !IS_LAST_BLK(blk)); + ASSERT(!is_first_blk == !IS_MBC_FIRST_FBLK(allctr, blk)); ASSERT(is_first_blk || IS_PREV_BLK_ALLOCED(blk)); ASSERT(is_last_blk || IS_PREV_BLK_FREE(NXT_BLK(blk))); ASSERT(blk_sz == MBC_BLK_SZ(blk)); @@ -1559,12 +1618,12 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) cand_blk = blk; } else { - ASSERT(!IS_MBC_FIRST_BLK(allctr, blk)); + ASSERT(!IS_MBC_FIRST_ABLK(allctr, blk)); cand_blk = PREV_BLK(blk); cand_blk_sz += PREV_BLK_SZ(blk); } if (!is_last_blk) { - nxt_blk = NXT_BLK(blk); + nxt_blk = BLK_AFTER(blk, old_blk_sz); if (IS_FREE_BLK(nxt_blk)) cand_blk_sz += MBC_BLK_SZ(nxt_blk); } @@ -1582,18 +1641,18 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) nxt_blk_sz = old_blk_sz - blk_sz; - if ((is_last_blk || IS_ALLOCED_BLK(NXT_BLK(blk))) + if ((is_last_blk || IS_ALLOCED_BLK(BLK_AFTER(blk,old_blk_sz))) && (nxt_blk_sz < allctr->min_block_size)) return p; HARD_CHECK_BLK_CARRIER(allctr, blk); - nxt_nxt_blk = NXT_BLK(blk); + nxt_nxt_blk = BLK_AFTER(blk, old_blk_sz); - SET_MBC_BLK_SZ(blk, blk_sz); + SET_MBC_ABLK_SZ(blk, blk_sz); SET_NOT_LAST_BLK(blk); - nxt_blk = NXT_BLK(blk); + nxt_blk = BLK_AFTER(blk, blk_sz); STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); @@ -1614,8 +1673,8 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); } - crr = BLK_TO_MBC(blk); - SET_MBC_BLK_HDR(nxt_blk, nxt_blk_sz, + crr = ABLK_TO_MBC(blk); + SET_MBC_FBLK_HDR(nxt_blk, nxt_blk_sz, SBH_THIS_FREE | (is_last_blk ? SBH_LAST_BLK : 0), crr); @@ -1638,7 +1697,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) ASSERT(is_last_blk ? IS_LAST_BLK(nxt_blk) : IS_NOT_LAST_BLK(nxt_blk)); ASSERT(is_last_blk || nxt_blk == PREV_BLK(NXT_BLK(nxt_blk))); ASSERT(is_last_blk || IS_PREV_BLK_FREE(NXT_BLK(nxt_blk))); - ASSERT(BLK_TO_MBC(nxt_blk) == crr); + ASSERT(FBLK_TO_MBC(nxt_blk) == crr); HARD_CHECK_BLK_CARRIER(allctr, blk); @@ -1648,7 +1707,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) /* Need larger block... */ if (!is_last_blk) { - nxt_blk = NXT_BLK(blk); + nxt_blk = BLK_AFTER(blk, old_blk_sz); nxt_blk_sz = MBC_BLK_SZ(nxt_blk); if (IS_FREE_BLK(nxt_blk) && get_blk_sz <= old_blk_sz + nxt_blk_sz) { /* Grow into next block... */ @@ -1662,7 +1721,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (nxt_blk_sz < allctr->min_block_size) { blk_sz += nxt_blk_sz; - SET_MBC_BLK_SZ(blk, blk_sz); + SET_MBC_ABLK_SZ(blk, blk_sz); if (is_last_blk) { SET_LAST_BLK(blk); @@ -1671,7 +1730,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) #endif } else { - nxt_blk = NXT_BLK(blk); + nxt_blk = BLK_AFTER(blk, blk_sz); SET_PREV_BLK_ALLOCED(nxt_blk); #ifdef DEBUG is_last_blk = IS_LAST_BLK(nxt_blk); @@ -1680,11 +1739,11 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) } } else { - Carrier_t* crr = BLK_TO_MBC(blk); - SET_MBC_BLK_SZ(blk, blk_sz); + Carrier_t* crr = ABLK_TO_MBC(blk); + SET_MBC_ABLK_SZ(blk, blk_sz); - nxt_blk = NXT_BLK(blk); - SET_MBC_BLK_HDR(nxt_blk, nxt_blk_sz, SBH_THIS_FREE, crr); + nxt_blk = BLK_AFTER(blk, blk_sz); + SET_MBC_FBLK_HDR(nxt_blk, nxt_blk_sz, SBH_THIS_FREE, crr); if (is_last_blk) SET_LAST_BLK(nxt_blk); @@ -1694,7 +1753,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); ASSERT(IS_FREE_BLK(nxt_blk)); - ASSERT(BLK_TO_MBC(nxt_blk) == crr); + ASSERT(FBLK_TO_MBC(nxt_blk) == crr); } STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); @@ -1739,12 +1798,12 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) cand_blk_sz = 0; } else { - ASSERT(!IS_MBC_FIRST_BLK(allctr, blk)); + ASSERT(!IS_MBC_FIRST_ABLK(allctr, blk)); cand_blk = PREV_BLK(blk); cand_blk_sz = old_blk_sz + PREV_BLK_SZ(blk); if (!is_last_blk) { - nxt_blk = NXT_BLK(blk); + nxt_blk = BLK_AFTER(blk, old_blk_sz); if (IS_FREE_BLK(nxt_blk)) cand_blk_sz += MBC_BLK_SZ(nxt_blk); } @@ -1784,6 +1843,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) new_blk, MBC_BLK_SZ(new_blk), GET_BLK_HDR_FLGS(new_blk), + FBLK_TO_MBC(new_blk), blk_sz, 1, alcu_flgs); @@ -1793,6 +1853,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) return new_p; } else { + Carrier_t* crr; Uint new_blk_sz; UWord new_blk_flgs; Uint prev_blk_sz; @@ -1813,7 +1874,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (is_last_blk) new_blk_flgs |= LAST_BLK_HDR_FLG; else { - nxt_blk = NXT_BLK(blk); + nxt_blk = BLK_AFTER(blk, old_blk_sz); if (IS_FREE_BLK(nxt_blk)) { new_blk_flgs |= GET_LAST_BLK_HDR_FLG(nxt_blk); new_blk_sz += MBC_BLK_SZ(nxt_blk); @@ -1829,6 +1890,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) new_p = BLK2UMEM(new_blk); blk_cpy_sz = MIN(blk_sz, old_blk_sz); + crr = FBLK_TO_MBC(new_blk); if (prev_blk_sz >= blk_cpy_sz) sys_memcpy(new_p, p, blk_cpy_sz - ABLK_HDR_SZ); @@ -1839,6 +1901,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) new_blk, new_blk_sz, new_blk_flgs, + crr, blk_sz, 0, alcu_flgs); @@ -1864,18 +1927,20 @@ static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C, { ASSERT(IS_LAST_BLK((B))); ASSERT((CSZ) == CARRIER_SZ((C))); - ASSERT((BSZ) == BLK_SZ((B))); ASSERT((BSZ) % sizeof(Unit_t) == 0); if ((SBC)) { + ASSERT((BSZ) == SBC_BLK_SZ((B))); ASSERT((char*)B == (char*)C + A->sbc_header_size); ASSERT(IS_SBC_BLK((B))); ASSERT(IS_SB_CARRIER((C))); } else { - ASSERT(IS_MBC_FIRST_BLK(A, (B))); + ASSERT(IS_FREE_BLK(B)); + ASSERT((BSZ) == MBC_FBLK_SZ((B))); + ASSERT(IS_MBC_FIRST_FBLK(A, (B))); ASSERT(IS_MBC_BLK((B))); ASSERT(IS_MB_CARRIER((C))); - ASSERT(BLK_TO_MBC(B) == (C)); + ASSERT(FBLK_TO_MBC(B) == (C)); } if ((MSEGED)) { ASSERT(IS_MSEG_CARRIER((C))); @@ -1913,7 +1978,7 @@ create_sbmbc(Allctr_t *allctr, Uint umem_sz) blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size); - SET_MBC_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr); + SET_MBC_FBLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr); link_carrier(&allctr->sbmbc_list, crr); @@ -1933,7 +1998,7 @@ destroy_sbmbc(Allctr_t *allctr, Block_t *blk) Carrier_t *crr; ASSERT(IS_MBC_BLK(blk)); - ASSERT(IS_MBC_FIRST_BLK(allctr, blk)); + ASSERT(IS_MBC_FIRST_FBLK(allctr, blk)); crr = FIRST_BLK_TO_MBC(allctr, blk); crr_sz = CARRIER_SZ(crr); @@ -2114,7 +2179,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size); - SET_MBC_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr); + SET_MBC_FBLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr); if (flags & CFLG_MAIN_CARRIER) { ASSERT(!allctr->main_carrier); @@ -2300,7 +2365,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) } else { - ASSERT(IS_MBC_FIRST_BLK(allctr, blk)); + ASSERT(IS_MBC_FIRST_FBLK(allctr, blk)); crr = FIRST_BLK_TO_MBC(allctr, blk); crr_sz = CARRIER_SZ(crr); @@ -4272,7 +4337,7 @@ erts_alcu_stop(Allctr_t *allctr) void erts_alcu_init(AlcUInit_t *init) { - + ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */ #if HAVE_ERTS_MSEG ASSERT(erts_mseg_unit_size() == MSEG_UNIT_SZ); # if HAVE_SUPER_ALIGNED_MB_CARRIERS diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index 7d6f61fdff..06a5a5a346 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -218,6 +218,7 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); #define FLG_MASK INV_UNIT_MASK #define SBC_BLK_SZ_MASK UNIT_MASK +#define MBC_FBLK_SZ_MASK UNIT_MASK #define CARRIER_SZ_MASK UNIT_MASK @@ -225,12 +226,13 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); # define CARRIER_OFFSET_BITS 13 # define CARRIER_OFFSET_SHIFT (sizeof(UWord)*8 - CARRIER_OFFSET_BITS) # define CARRIER_OFFSET_MASK (~((UWord)0) << CARRIER_OFFSET_SHIFT) -# define MBC_BLK_SZ_MASK (~CARRIER_OFFSET_MASK & ~FLG_MASK) +# define MBC_ABLK_SZ_MASK (~CARRIER_OFFSET_MASK & ~FLG_MASK) #else -# define MBC_BLK_SZ_MASK (~FLG_MASK) +# define MBC_ABLK_SZ_MASK (~FLG_MASK) #endif -#define MBC_BLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_BLK_SZ_MASK) +#define MBC_ABLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_ABLK_SZ_MASK) +#define MBC_FBLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_FBLK_SZ_MASK) #define SBC_BLK_SZ(B) (ASSERT_EXPR(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK) #define CARRIER_SZ(C) \ @@ -257,9 +259,14 @@ typedef struct { UWord bhdr; #if !HAVE_SUPER_ALIGNED_MB_CARRIERS Carrier_t *carrier; -#endif +#endif + union { + Carrier_t *carrier; /* if free */ + char udata__[1]; /* if allocated */ + }u; } Block_t; -typedef UWord FreeBlkFtr_t; + +typedef UWord FreeBlkFtr_t; /* Footer of a free block */ typedef struct { UWord giga_no; diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 4768dcc939..ded58e89f4 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -91,7 +91,7 @@ struct AOFF_RBTree_t_ { AOFF_RBTree_t *right; Uint max_sz; /* of all blocks in this sub-tree */ }; -#define AOFF_BLK_SZ(B) MBC_BLK_SZ(&(B)->hdr) +#define AOFF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->hdr) #ifdef HARD_DEBUG static AOFF_RBTree_t * check_tree(AOFF_RBTree_t* root, Uint); diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c index 619206bbcd..4d3ee98a4c 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.c +++ b/erts/emulator/beam/erl_bestfit_alloc.c @@ -73,7 +73,7 @@ #define SET_RED(N) (((RBTree_t *) (N))->flags |= RED_FLG) #define SET_BLACK(N) (((RBTree_t *) (N))->flags &= ~RED_FLG) -#define BF_BLK_SZ(B) MBC_BLK_SZ(&(B)->hdr) +#define BF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->hdr) #undef ASSERT #define ASSERT ASSERT_EXPR diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index 9db89652f7..264561a965 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -363,7 +363,7 @@ search_bucket(Allctr_t *allctr, int ix, Uint size) blk && i < max_blk_search; blk = blk->next, i++) { - blk_sz = MBC_BLK_SZ(&blk->block_head); + blk_sz = MBC_FBLK_SZ(&blk->block_head); blk_on_lambc = (((char *) blk) < gfallctr->last_aux_mbc_end && gfallctr->last_aux_mbc_start <= ((char *) blk)); @@ -402,7 +402,7 @@ get_free_block(Allctr_t *allctr, Uint size, if (min_bi == unsafe_bi) { blk = search_bucket(allctr, min_bi, size); if (blk) { - if (cand_blk && cand_size <= MBC_BLK_SZ(blk)) + if (cand_blk && cand_size <= MBC_FBLK_SZ(blk)) return NULL; /* cand_blk was better */ unlink_free_block(allctr, blk, flags); return blk; @@ -422,7 +422,7 @@ get_free_block(Allctr_t *allctr, Uint size, /* We are guaranteed to find a block that fits in this bucket */ blk = search_bucket(allctr, min_bi, size); ASSERT(blk); - if (cand_blk && cand_size <= MBC_BLK_SZ(blk)) + if (cand_blk && cand_size <= MBC_FBLK_SZ(blk)) return NULL; /* cand_blk was better */ unlink_free_block(allctr, blk, flags); return blk; @@ -435,7 +435,7 @@ link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; GFFreeBlock_t *blk = (GFFreeBlock_t *) block; - Uint sz = MBC_BLK_SZ(&blk->block_head); + Uint sz = MBC_FBLK_SZ(&blk->block_head); int i = BKT_IX(gfallctr, sz); ASSERT(sz >= MIN_BLK_SZ); @@ -456,7 +456,7 @@ unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; GFFreeBlock_t *blk = (GFFreeBlock_t *) block; - Uint sz = MBC_BLK_SZ(&blk->block_head); + Uint sz = MBC_FBLK_SZ(&blk->block_head); int i = BKT_IX(gfallctr, sz); if (!blk->prev) { -- cgit v1.2.3 From facbc93c04c757c1e5c10097d90187d177d0c376 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 15 Nov 2012 11:38:59 +0100 Subject: erts: Ensure MBC limits due to super alignment --- erts/emulator/beam/erl_alloc_util.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index c8e2d28c49..c0884c0013 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -204,6 +204,8 @@ MBC after deallocating first block: #if HAVE_SUPER_ALIGNED_MB_CARRIERS +# define MBC_SZ_MAX_LIMIT ((((UWord)1 << CARRIER_OFFSET_BITS) - 1) << MSEG_ALIGN_BITS) + # define BLK_CARRIER_OFFSET(B, C) (((char*)(B) - (char*)(C)) >> MSEG_UNIT_SHIFT) # define SET_MBC_ABLK_HDR(B, Sz, F, C) \ @@ -251,6 +253,8 @@ MBC after deallocating first block: #else /* !HAVE_SUPER_ALIGNED_MB_CARRIERS */ +# define MBC_SZ_MAX_LIMIT ((UWord)~0) + # define SET_MBC_ABLK_HDR(B, Sz, F, C) \ (ASSERT(((Sz) & FLG_MASK) == 0), \ ASSERT(!((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG))), \ @@ -2111,6 +2115,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) goto sbc_final_touch; } else { + ASSERT(crr_sz <= MBC_SZ_MAX_LIMIT); SET_CARRIER_HDR(crr, crr_sz, SCH_MSEG|SCH_MBC, allctr); STAT_MSEG_MBC_ALLOC(allctr, crr_sz); goto mbc_final_touch; @@ -4153,6 +4158,12 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->ramv = init->ramv; allctr->main_carrier_size = init->mmbcs; allctr->sbc_threshold = init->sbct; +#if HAVE_SUPER_ALIGNED_MB_CARRIERS + if (allctr->sbc_threshold > MBC_ABLK_SZ_MASK - ABLK_HDR_SZ) { + allctr->sbc_threshold = MBC_ABLK_SZ_MASK - ABLK_HDR_SZ + 1; + } +#endif + #if HAVE_ERTS_MSEG allctr->mseg_opt.abs_shrink_th = init->asbcst; allctr->mseg_opt.rel_shrink_th = init->rsbcst; @@ -4169,6 +4180,12 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #endif allctr->largest_mbc_size = MAX(init->lmbcs, init->smbcs); +#if HAVE_SUPER_ALIGNED_MB_CARRIERS + if (allctr->largest_mbc_size > MBC_SZ_MAX_LIMIT) { + allctr->largest_mbc_size = MBC_SZ_MAX_LIMIT; + } +#endif + allctr->smallest_mbc_size = init->smbcs; allctr->mbc_growth_stages = MAX(1, init->mbcgs); @@ -4340,11 +4357,6 @@ erts_alcu_init(AlcUInit_t *init) ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */ #if HAVE_ERTS_MSEG ASSERT(erts_mseg_unit_size() == MSEG_UNIT_SZ); -# if HAVE_SUPER_ALIGNED_MB_CARRIERS - /*SVERK Add assert about CARRIER_OFFSET_BITS and max MBC size - *SVERK Add assert about CARRIER_OFFSET_SHIFT and max MBC block size - */ -# endif max_mseg_carriers = init->mmc; sys_alloc_carrier_size = MSEG_UNIT_CEILING(init->ycs); #else /* #if HAVE_ERTS_MSEG */ -- cgit v1.2.3 From 8f1bd655d8bbfc80efb082daca495c19aff02f3d Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 15 Nov 2012 20:24:16 +0100 Subject: erts: Enable new header scheme without super alignment on 64-bit --- erts/emulator/beam/erl_alloc_util.c | 28 ++++++++++++++-------------- erts/emulator/beam/erl_alloc_util.h | 22 +++++++++++++++------- 2 files changed, 29 insertions(+), 21 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index c0884c0013..cef30d95a2 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -153,7 +153,11 @@ MBC after deallocating first block: #define UNUSED1_BLK_FTR_FLG (((UWord) 1) << 1) #define UNUSED2_BLK_FTR_FLG (((UWord) 1) << 2) -#define ABLK_HDR_SZ (offsetof(Block_t,u)) +#if MBC_ABLK_OFFSET_BITS +# define ABLK_HDR_SZ (offsetof(Block_t,u)) +#else +# define ABLK_HDR_SZ (sizeof(Block_t)) +#endif #define FBLK_FTR_SZ (sizeof(FreeBlkFtr_t)) #define UMEMSZ2BLKSZ(AP, SZ) \ @@ -202,16 +206,16 @@ MBC after deallocating first block: #define SBH_LAST_BLK LAST_BLK_HDR_FLG -#if HAVE_SUPER_ALIGNED_MB_CARRIERS +#if MBC_ABLK_OFFSET_BITS -# define MBC_SZ_MAX_LIMIT ((((UWord)1 << CARRIER_OFFSET_BITS) - 1) << MSEG_ALIGN_BITS) +# define MBC_SZ_MAX_LIMIT ((((UWord)1 << MBC_ABLK_OFFSET_BITS) - 1) << MSEG_ALIGN_BITS) # define BLK_CARRIER_OFFSET(B, C) (((char*)(B) - (char*)(C)) >> MSEG_UNIT_SHIFT) # define SET_MBC_ABLK_HDR(B, Sz, F, C) \ (ASSERT(((Sz) & ~MBC_ABLK_SZ_MASK) == 0), \ ASSERT(!((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG))), \ - (B)->bhdr = ((Sz) | (F) | (BLK_CARRIER_OFFSET(B,C) << CARRIER_OFFSET_SHIFT))) + (B)->bhdr = ((Sz) | (F) | (BLK_CARRIER_OFFSET(B,C) << MBC_ABLK_OFFSET_SHIFT))) # define SET_MBC_FBLK_HDR(B, Sz, F, C) \ (ASSERT(((Sz) & ~MBC_FBLK_SZ_MASK) == 0), \ @@ -222,7 +226,7 @@ MBC after deallocating first block: # define ABLK_TO_MBC(B) \ (ASSERT(IS_MBC_BLK(B) && IS_ALLOCED_BLK(B)), \ (Carrier_t*)((MSEG_UNIT_FLOOR((UWord)(B)) - \ - (((B)->bhdr >> CARRIER_OFFSET_SHIFT) << MSEG_UNIT_SHIFT)))) + (((B)->bhdr >> MBC_ABLK_OFFSET_SHIFT) << MSEG_UNIT_SHIFT)))) # define FBLK_TO_MBC(B) \ (ASSERT(IS_MBC_BLK(B) && IS_FREE_BLK(B)), \ @@ -232,7 +236,7 @@ MBC after deallocating first block: # define IS_MBC_FIRST_ABLK(AP,B) \ ((((UWord)(B) & ~MSEG_UNIT_MASK) == (AP)->mbc_header_size) \ - && ((B)->bhdr & CARRIER_OFFSET_MASK) == 0) + && ((B)->bhdr & MBC_ABLK_OFFSET_MASK) == 0) # define IS_MBC_FIRST_FBLK(AP,B) \ ((char*)(B) == (char*)((B)->u.carrier) + (AP)->mbc_header_size) @@ -247,11 +251,11 @@ MBC after deallocating first block: (B)->bhdr &= (MBC_ABLK_SZ_MASK|FLG_MASK)) # define SET_BLK_ALLOCED(B) \ - (ASSERT(((B)->bhdr & (CARRIER_OFFSET_MASK|THIS_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ + (ASSERT(((B)->bhdr & (MBC_ABLK_OFFSET_MASK|THIS_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ (B)->bhdr &= ~THIS_FREE_BLK_HDR_FLG, \ - (B)->bhdr |= (BLK_CARRIER_OFFSET(B,(B)->u.carrier) << CARRIER_OFFSET_SHIFT)) + (B)->bhdr |= (BLK_CARRIER_OFFSET(B,(B)->u.carrier) << MBC_ABLK_OFFSET_SHIFT)) -#else /* !HAVE_SUPER_ALIGNED_MB_CARRIERS */ +#else /* !MBC_ABLK_OFFSET_BITS */ # define MBC_SZ_MAX_LIMIT ((UWord)~0) @@ -284,7 +288,7 @@ MBC after deallocating first block: # define SET_BLK_ALLOCED(B) \ ((B)->bhdr &= ~THIS_FREE_BLK_HDR_FLG) -#endif /* !HAVE_SUPER_ALIGNED_MB_CARRIERS */ +#endif /* !MBC_ABLK_OFFSET_BITS */ #define SET_SBC_BLK_HDR(B, Sz) \ (ASSERT(((Sz) & FLG_MASK) == 0), (B)->bhdr = ((Sz) | (SBC_BLK_HDR_FLG))) @@ -4158,11 +4162,9 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->ramv = init->ramv; allctr->main_carrier_size = init->mmbcs; allctr->sbc_threshold = init->sbct; -#if HAVE_SUPER_ALIGNED_MB_CARRIERS if (allctr->sbc_threshold > MBC_ABLK_SZ_MASK - ABLK_HDR_SZ) { allctr->sbc_threshold = MBC_ABLK_SZ_MASK - ABLK_HDR_SZ + 1; } -#endif #if HAVE_ERTS_MSEG allctr->mseg_opt.abs_shrink_th = init->asbcst; @@ -4180,11 +4182,9 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #endif allctr->largest_mbc_size = MAX(init->lmbcs, init->smbcs); -#if HAVE_SUPER_ALIGNED_MB_CARRIERS if (allctr->largest_mbc_size > MBC_SZ_MAX_LIMIT) { allctr->largest_mbc_size = MBC_SZ_MAX_LIMIT; } -#endif allctr->smallest_mbc_size = init->smbcs; allctr->mbc_growth_stages = MAX(1, init->mbcgs); diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index 06a5a5a346..e3dde2d54c 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -222,11 +222,18 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); #define CARRIER_SZ_MASK UNIT_MASK -#if HAVE_SUPER_ALIGNED_MB_CARRIERS -# define CARRIER_OFFSET_BITS 13 -# define CARRIER_OFFSET_SHIFT (sizeof(UWord)*8 - CARRIER_OFFSET_BITS) -# define CARRIER_OFFSET_MASK (~((UWord)0) << CARRIER_OFFSET_SHIFT) -# define MBC_ABLK_SZ_MASK (~CARRIER_OFFSET_MASK & ~FLG_MASK) +#ifdef ARCH_64 +# define MBC_ABLK_OFFSET_BITS 24 +#elif HAVE_SUPER_ALIGNED_MB_CARRIERS +# define MBC_ABLK_OFFSET_BITS 13 +#else +# define MBC_ABLK_OFFSET_BITS 0 /* no carrier offset in block header */ +#endif + +#if MBC_ABLK_OFFSET_BITS +# define MBC_ABLK_OFFSET_SHIFT (sizeof(UWord)*8 - MBC_ABLK_OFFSET_BITS) +# define MBC_ABLK_OFFSET_MASK (~((UWord)0) << MBC_ABLK_OFFSET_SHIFT) +# define MBC_ABLK_SZ_MASK (~MBC_ABLK_OFFSET_MASK & ~FLG_MASK) #else # define MBC_ABLK_SZ_MASK (~FLG_MASK) #endif @@ -257,13 +264,14 @@ typedef struct { typedef struct { UWord bhdr; -#if !HAVE_SUPER_ALIGNED_MB_CARRIERS +#if !MBC_ABLK_OFFSET_BITS Carrier_t *carrier; -#endif +#else union { Carrier_t *carrier; /* if free */ char udata__[1]; /* if allocated */ }u; +#endif } Block_t; typedef UWord FreeBlkFtr_t; /* Footer of a free block */ -- cgit v1.2.3 From 048841ed243974562ead2f8d65435e2ffaf974c6 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 16 Nov 2012 15:00:51 +0100 Subject: erts: Fix compile warnings in alloc_util --- erts/emulator/beam/erl_alloc_util.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index cef30d95a2..1bdd5d79b5 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -2119,7 +2119,9 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) goto sbc_final_touch; } else { +#ifndef ARCH_64 ASSERT(crr_sz <= MBC_SZ_MAX_LIMIT); +#endif SET_CARRIER_HDR(crr, crr_sz, SCH_MSEG|SCH_MBC, allctr); STAT_MSEG_MBC_ALLOC(allctr, crr_sz); goto mbc_final_touch; @@ -4162,9 +4164,11 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->ramv = init->ramv; allctr->main_carrier_size = init->mmbcs; allctr->sbc_threshold = init->sbct; +#ifndef ARCH_64 if (allctr->sbc_threshold > MBC_ABLK_SZ_MASK - ABLK_HDR_SZ) { allctr->sbc_threshold = MBC_ABLK_SZ_MASK - ABLK_HDR_SZ + 1; } +#endif #if HAVE_ERTS_MSEG allctr->mseg_opt.abs_shrink_th = init->asbcst; @@ -4182,10 +4186,11 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #endif allctr->largest_mbc_size = MAX(init->lmbcs, init->smbcs); +#ifndef ARCH_64 if (allctr->largest_mbc_size > MBC_SZ_MAX_LIMIT) { allctr->largest_mbc_size = MBC_SZ_MAX_LIMIT; } - +#endif allctr->smallest_mbc_size = init->smbcs; allctr->mbc_growth_stages = MAX(1, init->mbcgs); -- cgit v1.2.3 From ea0b2bd94d0c1413bce84880196077c44b65f6f8 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 19 Nov 2012 14:39:57 +0100 Subject: erts: Cleanup minor things in alloc_util --- erts/emulator/beam/erl_alloc_util.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 1bdd5d79b5..64490a1c02 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -2099,8 +2099,6 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) crr_sz = allctr->mbc_header_size + blk_sz; mseg_flags = ERTS_MSEG_FLG_2POW; } - crr_sz = MSEG_UNIT_CEILING(crr_sz); - ASSERT(crr_sz % MSEG_UNIT_SZ == 0); crr = (Carrier_t *) alcu_mseg_alloc(allctr, &crr_sz, mseg_flags); if (!crr) { @@ -4200,7 +4198,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) + sizeof(FreeBlkFtr_t)); #if ERTS_SMP if (init->tpref) { - Uint sz = sizeof(Block_t); + Uint sz = ABLK_HDR_SZ; sz += ERTS_ALCU_DD_FIX_TYPE_OFFS*sizeof(UWord); if (init->fix) sz += sizeof(UWord); -- cgit v1.2.3 From 2ab1d972f6fd37c17b05fe8c69d8617874ffe733 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 21 Nov 2012 14:25:13 +0100 Subject: erts: Make carrier header sizes compile time constants No allocator strategy is using customized carrier headers anyway. --- erts/emulator/beam/erl_afit_alloc.c | 1 - erts/emulator/beam/erl_alloc_util.c | 58 ++++++++++++++---------------- erts/emulator/beam/erl_alloc_util.h | 2 -- erts/emulator/beam/erl_ao_firstfit_alloc.c | 1 - erts/emulator/beam/erl_bestfit_alloc.c | 1 - erts/emulator/beam/erl_goodfit_alloc.c | 1 - 6 files changed, 27 insertions(+), 37 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c index 02171e050f..306b32691c 100644 --- a/erts/emulator/beam/erl_afit_alloc.c +++ b/erts/emulator/beam/erl_afit_alloc.c @@ -86,7 +86,6 @@ erts_afalc_start(AFAllctr_t *afallctr, init->sbmbct = 0; /* Small mbc not supported by afit */ - allctr->mbc_header_size = sizeof(Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; allctr->min_block_size = sizeof(AFFreeBlock_t); diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 64490a1c02..e6aa93b9d8 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -235,11 +235,11 @@ MBC after deallocating first block: # define BLK_TO_MBC(B) (IS_FREE_BLK(B) ? FBLK_TO_MBC(B) : ABLK_TO_MBC(B)) # define IS_MBC_FIRST_ABLK(AP,B) \ - ((((UWord)(B) & ~MSEG_UNIT_MASK) == (AP)->mbc_header_size) \ + ((((UWord)(B) & ~MSEG_UNIT_MASK) == MBC_HEADER_SIZE(AP)) \ && ((B)->bhdr & MBC_ABLK_OFFSET_MASK) == 0) # define IS_MBC_FIRST_FBLK(AP,B) \ - ((char*)(B) == (char*)((B)->u.carrier) + (AP)->mbc_header_size) + ((char*)(B) == (char*)((B)->u.carrier) + MBC_HEADER_SIZE(AP)) # define IS_MBC_FIRST_BLK(AP,B) \ (IS_FREE_BLK(B) ? IS_MBC_FIRST_FBLK(AP,B) : IS_MBC_FIRST_ABLK(AP,B)) @@ -277,7 +277,7 @@ MBC after deallocating first block: # define FBLK_TO_MBC(B) BLK_TO_MBC(B) # define IS_MBC_FIRST_BLK(AP,B) \ - ((char*)(B) == (char*)((B)->carrier) + (AP)->mbc_header_size) + ((char*)(B) == (char*)((B)->carrier) + MBC_HEADER_SIZE(AP)) # define IS_MBC_FIRST_ABLK(AP,B) IS_MBC_FIRST_BLK(AP,B) # define IS_MBC_FIRST_FBLK(AP,B) IS_MBC_FIRST_BLK(AP,B) @@ -338,8 +338,10 @@ MBC after deallocating first block: /* Carriers ... */ -#define SIZEOF_SBC_HDR (UNIT_CEILING(sizeof(Carrier_t) + ABLK_HDR_SZ) \ - - ABLK_HDR_SZ) +#define SBC_HEADER_SIZE (UNIT_CEILING(sizeof(Carrier_t) + ABLK_HDR_SZ) \ + - ABLK_HDR_SZ) +#define MBC_HEADER_SIZE(AP) SBC_HEADER_SIZE + #define MSEG_CARRIER_HDR_FLAG (((UWord) 1) << 0) #define SBC_CARRIER_HDR_FLAG (((UWord) 1) << 1) @@ -353,16 +355,16 @@ MBC after deallocating first block: (ASSERT(((Sz) & FLG_MASK) == 0), (C)->chdr = ((Sz) | (F)), (C)->allctr = (AP)) #define BLK_TO_SBC(B) \ - ((Carrier_t *) (((char *) (B)) - SIZEOF_SBC_HDR)) + ((Carrier_t *) (((char *) (B)) - SBC_HEADER_SIZE)) #define FIRST_BLK_TO_MBC(AP, B) \ - ((Carrier_t *) (((char *) (B)) - (AP)->mbc_header_size)) + ((Carrier_t *) (((char *) (B)) - MBC_HEADER_SIZE(AP))) #define MBC_TO_FIRST_BLK(AP, P) \ - ((Block_t *) (((char *) (P)) + (AP)->mbc_header_size)) + ((Block_t *) (((char *) (P)) + MBC_HEADER_SIZE(AP))) #define SBC2BLK(AP, P) \ - ((Block_t *) (((char *) (P)) + (AP)->sbc_header_size)) + ((Block_t *) (((char *) (P)) + SBC_HEADER_SIZE)) #define SBC2UMEM(AP, P) \ - ((void *) (((char *) (P)) + ((AP)->sbc_header_size + ABLK_HDR_SZ))) + ((void *) (((char *) (P)) + (SBC_HEADER_SIZE + ABLK_HDR_SZ))) #define IS_MSEG_CARRIER(C) \ ((C)->chdr & MSEG_CARRIER_HDR_FLAG) @@ -1938,7 +1940,7 @@ static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C, ASSERT((BSZ) % sizeof(Unit_t) == 0); if ((SBC)) { ASSERT((BSZ) == SBC_BLK_SZ((B))); - ASSERT((char*)B == (char*)C + A->sbc_header_size); + ASSERT((char*)B == (char*)C + SBC_HEADER_SIZE); ASSERT(IS_SBC_BLK((B))); ASSERT(IS_SB_CARRIER((C))); } @@ -1984,7 +1986,7 @@ create_sbmbc(Allctr_t *allctr, Uint umem_sz) blk = MBC_TO_FIRST_BLK(allctr, crr); - blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size); + blk_sz = UNIT_FLOOR(crr_sz - MBC_HEADER_SIZE(allctr)); SET_MBC_FBLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr); @@ -2090,13 +2092,13 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) try_mseg: if (flags & CFLG_SBC) { - crr_sz = blk_sz + allctr->sbc_header_size; + crr_sz = blk_sz + SBC_HEADER_SIZE; mseg_flags = ERTS_MSEG_FLG_NONE; } else { crr_sz = (*allctr->get_next_mbc_size)(allctr); - if (crr_sz < allctr->mbc_header_size + blk_sz) - crr_sz = allctr->mbc_header_size + blk_sz; + if (crr_sz < MBC_HEADER_SIZE(allctr) + blk_sz) + crr_sz = MBC_HEADER_SIZE(allctr) + blk_sz; mseg_flags = ERTS_MSEG_FLG_2POW; } @@ -2130,10 +2132,10 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) #endif /* #if HAVE_ERTS_MSEG */ if (flags & CFLG_SBC) { - bcrr_sz = blk_sz + allctr->sbc_header_size; + bcrr_sz = blk_sz + SBC_HEADER_SIZE; } else { - bcrr_sz = allctr->mbc_header_size + blk_sz; + bcrr_sz = MBC_HEADER_SIZE(allctr) + blk_sz; if (!(flags & CFLG_MAIN_CARRIER) && bcrr_sz < allctr->smallest_mbc_size) bcrr_sz = allctr->smallest_mbc_size; @@ -2186,7 +2188,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) blk = MBC_TO_FIRST_BLK(allctr, crr); - blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size); + blk_sz = UNIT_FLOOR(crr_sz - MBC_HEADER_SIZE(allctr)); SET_MBC_FBLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr); @@ -2241,7 +2243,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) if (!(flags & CFLG_FORCE_SYS_ALLOC)) { - new_crr_sz = new_blk_sz + allctr->sbc_header_size; + new_crr_sz = new_blk_sz + SBC_HEADER_SIZE; new_crr_sz = MSEG_UNIT_CEILING(new_crr_sz); new_crr = (Carrier_t *) alcu_mseg_realloc(allctr, old_crr, @@ -2285,7 +2287,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) else { if (!(flags & CFLG_FORCE_MSEG)) { #endif /* #if HAVE_ERTS_MSEG */ - new_bcrr_sz = new_blk_sz + allctr->sbc_header_size; + new_bcrr_sz = new_blk_sz + SBC_HEADER_SIZE; new_crr_sz = (flags & CFLG_FORCE_SIZE ? UNIT_CEILING(new_bcrr_sz) : SYS_ALLOC_CARRIER_CEILING(new_bcrr_sz)); @@ -2307,7 +2309,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) return new_blk; } else if (new_crr_sz > UNIT_CEILING(new_bcrr_sz)) { - new_crr_sz = new_blk_sz + allctr->sbc_header_size; + new_crr_sz = new_blk_sz + SBC_HEADER_SIZE; new_crr_sz = UNIT_CEILING(new_crr_sz); new_crr = (Carrier_t *) alcu_sys_realloc(allctr, (void *) old_crr, @@ -3817,7 +3819,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, if (IS_MBC_BLK(blk)) res = mbc_realloc(allctr, p, size, alcu_flgs); else { - Uint used_sz = allctr->sbc_header_size + ABLK_HDR_SZ + size; + Uint used_sz = SBC_HEADER_SIZE + ABLK_HDR_SZ + size; Uint crr_sz; Uint diff_sz_val; Uint crr_sz_val; @@ -4223,7 +4225,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->sbmbc_size = init->sbmbcs; min_size = allctr->sbmbc_threshold; min_size += allctr->min_block_size; - min_size += allctr->mbc_header_size; + min_size += MBC_HEADER_SIZE(allctr); if (allctr->sbmbc_size < min_size) allctr->sbmbc_size = min_size; } @@ -4268,8 +4270,6 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (!allctr->get_next_mbc_size) allctr->get_next_mbc_size = get_next_mbc_size; - if (allctr->mbc_header_size < sizeof(Carrier_t)) - goto error; #ifdef ERTS_SMP allctr->dd.use = 0; if (init->tpref) { @@ -4278,10 +4278,6 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->dd.ix = init->ix; } #endif - allctr->mbc_header_size = (UNIT_CEILING(allctr->mbc_header_size - + ABLK_HDR_SZ) - - ABLK_HDR_SZ); - allctr->sbc_header_size = SIZEOF_SBC_HDR; if (allctr->main_carrier_size) { Block_t *blk; @@ -4480,7 +4476,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) Carrier_t *sbc = BLK_TO_SBC(iblk); ASSERT(SBC2BLK(allctr, sbc) == iblk); - ASSERT(CARRIER_SZ(sbc) - allctr->sbc_header_size >= SBC_BLK_SZ(iblk)); + ASSERT(CARRIER_SZ(sbc) - SBC_HEADER_SIZE >= SBC_BLK_SZ(iblk)); #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(sbc)) { ASSERT(CARRIER_SZ(sbc) % MSEG_UNIT_SZ == 0); @@ -4558,7 +4554,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) } ASSERT((((char *) crr) - + allctr->mbc_header_size + + MBC_HEADER_SIZE(allctr) + tot_blk_sz) == carrier_end); ASSERT(((char *) crr) + CARRIER_SZ(crr) - sizeof(Unit_t) <= carrier_end && carrier_end <= ((char *) crr) + CARRIER_SZ(crr)); diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index e3dde2d54c..18ddad5f5b 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -411,8 +411,6 @@ struct Allctr_t_ { #endif /* */ - Uint mbc_header_size; - Uint sbc_header_size; Uint min_mbc_size; Uint min_mbc_first_free_size; Uint min_block_size; diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index ded58e89f4..86b4696d8f 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -184,7 +184,6 @@ erts_aoffalc_start(AOFFAllctr_t *alc, sys_memcpy((void *) alc, (void *) &zero.allctr, sizeof(AOFFAllctr_t)); - allctr->mbc_header_size = sizeof(Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; allctr->min_block_size = sizeof(AOFF_RBTree_t); diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c index 4d3ee98a4c..743cbd93c9 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.c +++ b/erts/emulator/beam/erl_bestfit_alloc.c @@ -179,7 +179,6 @@ erts_bfalc_start(BFAllctr_t *bfallctr, bfallctr->address_order = bfinit->ao; - allctr->mbc_header_size = sizeof(Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; allctr->min_block_size = (bfinit->ao diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index 264561a965..f98d377fd6 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -205,7 +205,6 @@ erts_gfalc_start(GFAllctr_t *gfallctr, init->sbmbct = 0; /* Small mbc not yet supported by goodfit */ - allctr->mbc_header_size = sizeof(Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; allctr->min_block_size = sizeof(GFFreeBlock_t); -- cgit v1.2.3 From 38930473052af252b8f527200f4db9ba29f435ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Wed, 14 Nov 2012 18:15:08 +0100 Subject: erts: Remove unused mseg options amcbf and rmcbf --- erts/emulator/beam/erl_alloc.c | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 3eee53eba3..50385c9f8c 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -1315,25 +1315,12 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) handle_au_arg(&init->ll_alloc, &argv[i][3], argv, &i); break; case 'M': - if (has_prefix("amcbf", argv[i]+3)) { -#if HAVE_ERTS_MSEG - init->mseg.amcbf = -#endif - get_kb_value(argv[i]+8, argv, &i); - } - else if (has_prefix("rmcbf", argv[i]+3)) { -#if HAVE_ERTS_MSEG - init->mseg.rmcbf = -#endif - get_amount_value(argv[i]+8, argv, &i); - } - else if (has_prefix("mcs", argv[i]+3)) { + if (has_prefix("mcs", argv[i]+3)) { #if HAVE_ERTS_MSEG init->mseg.mcs = #endif get_amount_value(argv[i]+6, argv, &i); - } - else { + } else { bad_param(param, param+2); } break; -- cgit v1.2.3 From 2eaecbf3847120978cd61479d656d2381d533134 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 22 Nov 2012 09:33:27 +0100 Subject: erts: Fix new header scheme for win64 --- erts/emulator/beam/erl_alloc_util.h | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index 18ddad5f5b..9c15547574 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -222,11 +222,14 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); #define CARRIER_SZ_MASK UNIT_MASK -#ifdef ARCH_64 -# define MBC_ABLK_OFFSET_BITS 24 -#elif HAVE_SUPER_ALIGNED_MB_CARRIERS -# define MBC_ABLK_OFFSET_BITS 13 -#else +#if HAVE_ERTS_MSEG +# ifdef ARCH_64 +# define MBC_ABLK_OFFSET_BITS 24 +# elif HAVE_SUPER_ALIGNED_MB_CARRIERS +# define MBC_ABLK_OFFSET_BITS 13 +# endif +#endif +#ifndef MBC_ABLK_OFFSET_BITS # define MBC_ABLK_OFFSET_BITS 0 /* no carrier offset in block header */ #endif -- cgit v1.2.3 From 860f0fdb4b14619fdc57621ea820381287eb8711 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 23 Nov 2012 19:45:42 +0100 Subject: erts: Force sbmbc to be disabled in a crude way --- erts/emulator/beam/erl_alloc.c | 18 ++++++++++++++++-- erts/emulator/beam/erl_alloc_util.h | 2 ++ 2 files changed, 18 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 50385c9f8c..b9c3b43498 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -1173,6 +1173,11 @@ handle_au_arg(struct au_init *auip, break; case 'e': auip->enable = get_bool_value(sub_param+1, argv, ip); +#if !HAVE_ERTS_SBMBC + if (auip->init.util.alloc_no == ERTS_ALC_A_SBMBC) { + auip->enable = 0; + } +#endif break; case 'l': if (has_prefix("lmbcs", sub_param)) { @@ -1233,10 +1238,16 @@ handle_au_arg(struct au_init *auip, auip->init.util.sbct = get_kb_value(sub_param + 4, argv, ip); } else if (has_prefix("sbmbcs", sub_param)) { - auip->init.util.sbmbcs = get_byte_value(sub_param + 6, argv, ip); +#if HAVE_ERTS_SBMBC + auip->init.util.sbmbcs = +#endif + get_byte_value(sub_param + 6, argv, ip); } else if (has_prefix("sbmbct", sub_param)) { - auip->init.util.sbmbct = get_byte_value(sub_param + 6, argv, ip); +#if HAVE_ERTS_SBMBC + auip->init.util.sbmbct = +#endif + get_byte_value(sub_param + 6, argv, ip); } else if (has_prefix("smbcs", sub_param)) { auip->default_.smbcs = 0; @@ -1390,6 +1401,9 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) else if (strcmp("max", arg) == 0) { for (a = 0; a < aui_sz; a++) aui[a]->enable = 1; +#if !HAVE_ERTS_SBMBC + init->sbmbc_alloc.enable = 0; +#endif } else if (strcmp("config", arg) == 0) { init->erts_alloc_config = 1; diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index 9c15547574..423bb0517e 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -237,8 +237,10 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); # define MBC_ABLK_OFFSET_SHIFT (sizeof(UWord)*8 - MBC_ABLK_OFFSET_BITS) # define MBC_ABLK_OFFSET_MASK (~((UWord)0) << MBC_ABLK_OFFSET_SHIFT) # define MBC_ABLK_SZ_MASK (~MBC_ABLK_OFFSET_MASK & ~FLG_MASK) +# define HAVE_ERTS_SBMBC 0 #else # define MBC_ABLK_SZ_MASK (~FLG_MASK) +# define HAVE_ERTS_SBMBC 1 #endif #define MBC_ABLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_ABLK_SZ_MASK) -- cgit v1.2.3 From 971d054018d94dd89c2323e237f9c6c3afa91e3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Wed, 28 Nov 2012 17:58:23 +0100 Subject: erts: Make gc sizes fit into MB Carrier blocks * Account for block header size in gc-sizes * Also slow down growth to 20% instead of 25% when size threshold is reached --- erts/emulator/beam/erl_gc.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 6075a527c3..cf06cedf7f 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -168,15 +168,21 @@ erts_init_gc(void) * we really don't want that growth when the heaps are that big. */ - heap_sizes[0] = 34; - heap_sizes[1] = 55; - for (i = 2; i < 23; i++) { - heap_sizes[i] = heap_sizes[i-1] + heap_sizes[i-2]; + /* Growth stage 1 - Fibonacci + 1*/ + /* 12,38 will hit size 233, the old default */ + + heap_sizes[0] = 12; + heap_sizes[1] = 38; + + for(i = 2; i < 23; i++) { + /* one extra word for block header */ + heap_sizes[i] = heap_sizes[i-1] + heap_sizes[i-2] + 1; } + /* Growth stage 2 - 20% growth */ /* At 1.3 mega words heap, we start to slow down. */ for (i = 23; i < ALENGTH(heap_sizes); i++) { - heap_sizes[i] = 5*(heap_sizes[i-1]/4); + heap_sizes[i] = heap_sizes[i-1] + heap_sizes[i-1]/5; if (heap_sizes[i] < 0) { /* Size turned negative. Discard this last size. */ i--; -- cgit v1.2.3 From b27bb5275d58b0abeadc4664ed27febe625f8ff0 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 29 Nov 2012 19:25:07 +0100 Subject: erts: Fix bug when allocating size near sbc_threshold A block larger than sbc_threshold can be allocated in MBC if the subsequent "residue block" is smaller than min_block_size. Solved by lowering sbc_threshold to ("hard limit" - min_block_size). --- erts/emulator/beam/erl_alloc_util.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index e6aa93b9d8..f4e6f0bba0 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -1404,6 +1404,7 @@ mbc_alloc_finalize(Allctr_t *allctr, ASSERT(FBLK_TO_MBC(nxt_blk) == crr); } else { + ASSERT(org_blk_sz <= MBC_ABLK_SZ_MASK); blk_sz = org_blk_sz; if (flags & LAST_BLK_HDR_FLG) { if (valid_blk_info) @@ -4163,12 +4164,6 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->ramv = init->ramv; allctr->main_carrier_size = init->mmbcs; - allctr->sbc_threshold = init->sbct; -#ifndef ARCH_64 - if (allctr->sbc_threshold > MBC_ABLK_SZ_MASK - ABLK_HDR_SZ) { - allctr->sbc_threshold = MBC_ABLK_SZ_MASK - ABLK_HDR_SZ + 1; - } -#endif #if HAVE_ERTS_MSEG allctr->mseg_opt.abs_shrink_th = init->asbcst; @@ -4210,6 +4205,23 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) } #endif + allctr->sbc_threshold = init->sbct; +#ifndef ARCH_64 + if (allctr->sbc_threshold > 0) { + Uint max_mbc_block_sz = UNIT_CEILING(allctr->sbc_threshold - 1 + ABLK_HDR_SZ); + if (max_mbc_block_sz + UNIT_FLOOR(allctr->min_block_size - 1) > MBC_ABLK_SZ_MASK + || max_mbc_block_sz < allctr->sbc_threshold) { /* wrap around */ + /* + * By limiting sbc_threshold to (hard limit - min_block_size) + * we avoid having to split off free "residue blocks" + * smaller than min_block_size. + */ + max_mbc_block_sz = MBC_ABLK_SZ_MASK - UNIT_FLOOR(allctr->min_block_size - 1); + allctr->sbc_threshold = max_mbc_block_sz - ABLK_HDR_SZ + 1; + } + } +#endif + allctr->sbmbc_threshold = init->sbmbct; -- cgit v1.2.3 From 4c4149e3ec93a2c501f1ca4bdeb3a70940df5b27 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 3 Dec 2012 16:52:32 +0100 Subject: erts: Optimize erl_alloc_util.c by substitute MBC_BLK_SZ with either MBC_ABLK_SZ or MBC_FBLK_SZ in all cases when we already know what kind of block it is. --- erts/emulator/beam/erl_alloc_util.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index f4e6f0bba0..3d6761339b 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -1454,7 +1454,7 @@ mbc_alloc(Allctr_t *allctr, Uint size) if (IS_MBC_BLK(blk)) mbc_alloc_finalize(allctr, blk, - MBC_BLK_SZ(blk), + MBC_FBLK_SZ(blk), GET_BLK_HDR_FLGS(blk), FBLK_TO_MBC(blk), blk_sz, @@ -1477,7 +1477,7 @@ mbc_free(Allctr_t *allctr, void *p) ASSERT(p); blk = UMEM2BLK(p); - blk_sz = MBC_BLK_SZ(blk); + blk_sz = MBC_ABLK_SZ(blk); if (blk_sz < allctr->sbmbc_threshold) alcu_flgs |= ERTS_ALCU_FLG_SBMBC; @@ -1497,7 +1497,7 @@ mbc_free(Allctr_t *allctr, void *p) blk = PREV_BLK(blk); (*allctr->unlink_free_block)(allctr, blk, alcu_flgs); - blk_sz += MBC_BLK_SZ(blk); + blk_sz += MBC_FBLK_SZ(blk); is_first_blk = IS_MBC_FIRST_FBLK(allctr, blk); SET_MBC_FBLK_SZ(blk, blk_sz); } @@ -1580,7 +1580,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) ASSERT(size < allctr->sbc_threshold); blk = (Block_t *) UMEM2BLK(p); - old_blk_sz = MBC_BLK_SZ(blk); + old_blk_sz = MBC_ABLK_SZ(blk); ASSERT(old_blk_sz >= allctr->min_block_size); @@ -1636,7 +1636,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (!is_last_blk) { nxt_blk = BLK_AFTER(blk, old_blk_sz); if (IS_FREE_BLK(nxt_blk)) - cand_blk_sz += MBC_BLK_SZ(nxt_blk); + cand_blk_sz += MBC_FBLK_SZ(nxt_blk); } new_blk = (*allctr->get_free_block)(allctr, @@ -1673,7 +1673,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (!is_last_blk) { if (IS_FREE_BLK(nxt_nxt_blk)) { /* Coalesce with next free block... */ - nxt_blk_sz += MBC_BLK_SZ(nxt_nxt_blk); + nxt_blk_sz += MBC_FBLK_SZ(nxt_nxt_blk); (*allctr->unlink_free_block)(allctr, nxt_nxt_blk, alcu_flgs); is_last_blk = GET_LAST_BLK_HDR_FLG(nxt_nxt_blk); @@ -1816,7 +1816,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (!is_last_blk) { nxt_blk = BLK_AFTER(blk, old_blk_sz); if (IS_FREE_BLK(nxt_blk)) - cand_blk_sz += MBC_BLK_SZ(nxt_blk); + cand_blk_sz += MBC_FBLK_SZ(nxt_blk); } } @@ -1852,7 +1852,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (new_blk) { mbc_alloc_finalize(allctr, new_blk, - MBC_BLK_SZ(new_blk), + MBC_FBLK_SZ(new_blk), GET_BLK_HDR_FLGS(new_blk), FBLK_TO_MBC(new_blk), blk_sz, @@ -1888,7 +1888,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) nxt_blk = BLK_AFTER(blk, old_blk_sz); if (IS_FREE_BLK(nxt_blk)) { new_blk_flgs |= GET_LAST_BLK_HDR_FLG(nxt_blk); - new_blk_sz += MBC_BLK_SZ(nxt_blk); + new_blk_sz += MBC_FBLK_SZ(nxt_blk); (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); } } @@ -3884,7 +3884,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, res = BLK2UMEM(new_blk); sys_memcpy((void *) res, (void *) p, - MIN(MBC_BLK_SZ(blk) - ABLK_HDR_SZ, size)); + MIN(MBC_ABLK_SZ(blk) - ABLK_HDR_SZ, size)); mbc_free(allctr, p); } else -- cgit v1.2.3 From e976eb64736435b4c79bb53947a77d8bc04a0481 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Mon, 3 Dec 2012 18:31:02 +0100 Subject: erts: Reintroduce mseg options amcbf and rmcbf Used with new sbc cache --- erts/emulator/beam/erl_alloc.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index b9c3b43498..1fc36fbd26 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -1326,12 +1326,25 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) handle_au_arg(&init->ll_alloc, &argv[i][3], argv, &i); break; case 'M': - if (has_prefix("mcs", argv[i]+3)) { + if (has_prefix("amcbf", argv[i]+3)) { +#if HAVE_ERTS_MSEG + init->mseg.amcbf = +#endif + get_kb_value(argv[i]+8, argv, &i); + } + else if (has_prefix("rmcbf", argv[i]+3)) { +#if HAVE_ERTS_MSEG + init->mseg.rmcbf = +#endif + get_amount_value(argv[i]+8, argv, &i); + } + else if (has_prefix("mcs", argv[i]+3)) { #if HAVE_ERTS_MSEG init->mseg.mcs = #endif get_amount_value(argv[i]+6, argv, &i); - } else { + } + else { bad_param(param, param+2); } break; -- cgit v1.2.3 From 0fb68ef1f8c7f44c6f7edc1cc461d2b598b96fd9 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 4 Dec 2012 17:57:51 +0100 Subject: erts: Set super alignment (256kb) and limits for sbct (8Mb) and lmbcs (128Mb) --- erts/emulator/beam/erl_alloc_util.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index 423bb0517e..e0754e7f69 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -226,7 +226,8 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); # ifdef ARCH_64 # define MBC_ABLK_OFFSET_BITS 24 # elif HAVE_SUPER_ALIGNED_MB_CARRIERS -# define MBC_ABLK_OFFSET_BITS 13 +# define MBC_ABLK_OFFSET_BITS 9 + /* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */ # endif #endif #ifndef MBC_ABLK_OFFSET_BITS -- cgit v1.2.3 From 53cabc7eda09dbe38df565cbf87d2f1ab3cd68e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Fri, 7 Dec 2012 17:41:42 +0100 Subject: erts: Reduce max heap sizes Reduces max heap sizes to max addressable memory space. In reality this could be even less since we need at least twice as much adressable memory to do a garbage collection. The rest of the system also uses memory thus further constraining heap memory space and in the 64 bit case we really only have 48 bits mappable memory. --- erts/emulator/beam/erl_gc.c | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index cf06cedf7f..2917f6526a 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -129,7 +129,7 @@ static void disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int n #if defined(ARCH_64) && !HALFWORD_HEAP # define MAX_HEAP_SIZES 154 #else -# define MAX_HEAP_SIZES 55 +# define MAX_HEAP_SIZES 59 #endif static Sint heap_sizes[MAX_HEAP_SIZES]; /* Suitable heap sizes. */ @@ -144,6 +144,7 @@ void erts_init_gc(void) { int i = 0; + Sint max_heap_size = 0; ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word)); ASSERT(offsetof(ProcBin,thing_word) == offsetof(ErlFunThing,thing_word)); @@ -179,11 +180,19 @@ erts_init_gc(void) heap_sizes[i] = heap_sizes[i-1] + heap_sizes[i-2] + 1; } + + /* for 32 bit we want max_heap_size to be MAX(32bit) / 4 [words] (and halfword) + * for 64 bit we want max_heap_size to be MAX(52bit) / 8 [words] + */ + + max_heap_size = sizeof(Eterm) < 8 ? (Sint)((~(Uint)0)/(sizeof(Eterm))) : + (Sint)(((Uint64)1 << 53)/sizeof(Eterm)); + /* Growth stage 2 - 20% growth */ /* At 1.3 mega words heap, we start to slow down. */ for (i = 23; i < ALENGTH(heap_sizes); i++) { heap_sizes[i] = heap_sizes[i-1] + heap_sizes[i-1]/5; - if (heap_sizes[i] < 0) { + if ((heap_sizes[i] < 0) || heap_sizes[i] > max_heap_size) { /* Size turned negative. Discard this last size. */ i--; break; @@ -860,14 +869,12 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) } } - if (wanted < MIN_HEAP_SIZE(p)) { - wanted = MIN_HEAP_SIZE(p); - } else { - wanted = next_heap_size(p, wanted, 0); - } + wanted = wanted < MIN_HEAP_SIZE(p) ? MIN_HEAP_SIZE(p) + : next_heap_size(p, wanted, 0); if (wanted < HEAP_SIZE(p)) { shrink_new_heap(p, wanted, objv, nobj); } + ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); return 1; /* We are done. */ } @@ -1426,11 +1433,10 @@ adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int I think this is better as fullsweep is used mainly on small memory systems, but I could be wrong... */ wanted = 2 * need_after; - if (wanted < p->min_heap_size) { - sz = p->min_heap_size; - } else { - sz = next_heap_size(p, wanted, 0); - } + + sz = wanted < p->min_heap_size ? p->min_heap_size + : next_heap_size(p, wanted, 0); + if (sz < HEAP_SIZE(p)) { shrink_new_heap(p, sz, objv, nobj); } -- cgit v1.2.3 From 4795e4510804f5c1a0160443b9a7b56caa9d369e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Thu, 13 Dec 2012 20:19:33 +0100 Subject: erts: Make ll main mbc fit into 2pow size --- erts/emulator/beam/erl_alloc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 1fc36fbd26..152d8e8c65 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -310,9 +310,9 @@ set_default_ll_alloc_opts(struct au_init *ip) ip->init.util.name_prefix = "ll_"; ip->init.util.alloc_no = ERTS_ALC_A_LONG_LIVED; #ifndef SMALL_MEMORY - ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */ + ip->init.util.mmbcs = 2*1024*1024 - 40; /* Main carrier size */ #else - ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */ + ip->init.util.mmbcs = 1*1024*1024 - 40; /* Main carrier size */ #endif ip->init.util.ts = ERTS_ALC_MTA_LONG_LIVED; ip->init.util.asbcst = 0; -- cgit v1.2.3