diff options
Diffstat (limited to 'erts/emulator/beam/erl_alloc_util.c')
| -rw-r--r-- | erts/emulator/beam/erl_alloc_util.c | 2776 |
1 files changed, 2036 insertions, 740 deletions
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 53062979a6..b19e603a5f 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -71,21 +71,16 @@ #define ALLOC_ZERO_EQ_NULL 0 +#ifndef ERTS_MSEG_FLG_2POW +# define ERTS_MSEG_FLG_2POW 0 +#endif +#ifndef ERTS_MSEG_FLG_NONE +# define ERTS_MSEG_FLG_NONE 0 +#endif + static int atoms_initialized = 0; static int initialized = 0; -int erts_have_sbmbc_alloc; - -#if HAVE_ERTS_MSEG - -#define MSEG_UNIT_SHIFT MSEG_ALIGN_BITS -#define MSEG_UNIT_SZ (1 << MSEG_UNIT_SHIFT) -#define MSEG_UNIT_MASK ((~(UWord)0) << MSEG_UNIT_SHIFT) - -#define MSEG_UNIT_FLOOR(X) ((X) & MSEG_UNIT_MASK) -#define MSEG_UNIT_CEILING(X) MSEG_UNIT_FLOOR((X) + ~MSEG_UNIT_MASK) - -#endif #define INV_SYS_ALLOC_CARRIER_MASK ((UWord) (sys_alloc_carrier_size - 1)) #define SYS_ALLOC_CARRIER_MASK (~INV_SYS_ALLOC_CARRIER_MASK) @@ -110,13 +105,12 @@ static Uint max_mseg_carriers; #define ONE_GIGA (1000000000) -#define INC_CC(CC) ((CC).no == ONE_GIGA - 1 \ - ? ((CC).giga_no++, (CC).no = 0) \ - : (CC).no++) +#define ERTS_ALC_CC_GIGA_VAL(CC) ((CC) / ONE_GIGA) +#define ERTS_ALC_CC_VAL(CC) ((CC) % ONE_GIGA) + +#define INC_CC(CC) ((CC)++) -#define DEC_CC(CC) ((CC).no == 0 \ - ? ((CC).giga_no--, (CC).no = ONE_GIGA - 1) \ - : (CC).no--) +#define DEC_CC(CC) ((CC)--) /* Multi block carrier (MBC) memory layout in R16: @@ -173,14 +167,6 @@ MBC after deallocating first block: #define SET_BLK_SZ_FTR(B, SZ) \ (((FreeBlkFtr_t *) (((char *) (B)) + (SZ)))[-1] = (SZ)) -#define THIS_FREE_BLK_HDR_FLG (((UWord) 1) << 0) -#define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1) -#define LAST_BLK_HDR_FLG (((UWord) 1) << 2) - -/* Special flag combo for (allocated) SBC blocks -*/ -#define SBC_BLK_HDR_FLG (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) - #define SET_MBC_ABLK_SZ(B, SZ) \ (ASSERT(((SZ) & FLG_MASK) == 0), \ (B)->bhdr = (((B)->bhdr) & ~MBC_ABLK_SZ_MASK) | (SZ)) @@ -222,18 +208,7 @@ MBC after deallocating first block: ASSERT(((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ (B)->bhdr = ((Sz) | (F)), \ (B)->u.carrier = (C)) - -# define ABLK_TO_MBC(B) \ - (ASSERT(IS_MBC_BLK(B) && IS_ALLOCED_BLK(B)), \ - (Carrier_t*)((MSEG_UNIT_FLOOR((UWord)(B)) - \ - (((B)->bhdr >> MBC_ABLK_OFFSET_SHIFT) << MSEG_UNIT_SHIFT)))) - -# define FBLK_TO_MBC(B) \ - (ASSERT(IS_MBC_BLK(B) && IS_FREE_BLK(B)), \ - (B)->u.carrier) -# define BLK_TO_MBC(B) (IS_FREE_BLK(B) ? FBLK_TO_MBC(B) : ABLK_TO_MBC(B)) - # define IS_MBC_FIRST_ABLK(AP,B) \ ((((UWord)(B) & ~MSEG_UNIT_MASK) == MBC_HEADER_SIZE(AP)) \ && ((B)->bhdr & MBC_ABLK_OFFSET_MASK) == 0) @@ -272,10 +247,6 @@ MBC after deallocating first block: (B)->bhdr = ((Sz) | (F)), \ (B)->carrier = (C)) -# define BLK_TO_MBC(B) ((B)->carrier) -# define ABLK_TO_MBC(B) BLK_TO_MBC(B) -# define FBLK_TO_MBC(B) BLK_TO_MBC(B) - # define IS_MBC_FIRST_BLK(AP,B) \ ((char*)(B) == (char*)((B)->carrier) + MBC_HEADER_SIZE(AP)) # define IS_MBC_FIRST_ABLK(AP,B) IS_MBC_FIRST_BLK(AP,B) @@ -300,8 +271,6 @@ MBC after deallocating first block: ((B)->bhdr & PREV_FREE_BLK_HDR_FLG) #define IS_PREV_BLK_ALLOCED(B) \ (!IS_PREV_BLK_FREE((B))) -#define IS_FREE_BLK(B) \ - (ASSERT(!IS_SBC_BLK(B)), (B)->bhdr & THIS_FREE_BLK_HDR_FLG) #define IS_ALLOCED_BLK(B) \ (!IS_FREE_BLK((B))) #define IS_LAST_BLK(B) \ @@ -318,13 +287,6 @@ MBC after deallocating first block: #define GET_BLK_HDR_FLGS(B) \ ((B)->bhdr & FLG_MASK) -#define IS_SBC_BLK(B) \ - (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG) -#define IS_MBC_BLK(B) \ - (!IS_SBC_BLK((B))) - -#define MBC_BLK_SZ(B) (IS_FREE_BLK(B) ? MBC_FBLK_SZ(B) : MBC_ABLK_SZ(B)) - #define NXT_BLK(B) \ (ASSERT(IS_MBC_BLK(B)), \ (Block_t *) (((char *) (B)) + MBC_BLK_SZ((B)))) @@ -338,9 +300,96 @@ MBC after deallocating first block: /* Carriers ... */ -#define SBC_HEADER_SIZE (UNIT_CEILING(sizeof(Carrier_t) + ABLK_HDR_SZ) \ - - ABLK_HDR_SZ) -#define MBC_HEADER_SIZE(AP) SBC_HEADER_SIZE +/* #define ERTS_ALC_CPOOL_DEBUG */ + +#if defined(DEBUG) && !defined(ERTS_ALC_CPOOL_DEBUG) +# define ERTS_ALC_CPOOL_DEBUG +#endif + +#ifndef ERTS_SMP +# undef ERTS_ALC_CPOOL_DEBUG +#endif + +#ifdef ERTS_ALC_CPOOL_DEBUG +# define ERTS_ALC_CPOOL_ASSERT(A) \ + ((void) ((A) \ + ? 1 \ + : (erts_alcu_assert_failed(#A, \ + (char *) __FILE__, \ + __LINE__, \ + (char *) __func__), \ + 0))) +#else +# define ERTS_ALC_CPOOL_ASSERT(A) ((void) 1) +#endif + +#ifdef ERTS_SMP +#define ERTS_ALC_IS_CPOOL_ENABLED(A) ((A)->cpool.util_limit) +#else +#define ERTS_ALC_IS_CPOOL_ENABLED(A) (0) +#endif + +#ifdef ERTS_SMP + +#define ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON 1000 +#define ERTS_ALC_CPOOL_ALLOC_OP_INC 8 +#define ERTS_ALC_CPOOL_FREE_OP_DEC 10 + +#define ERTS_ALC_CPOOL_ALLOC_OP(A) \ +do { \ + if ((A)->cpool.disable_abandon < ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON) { \ + (A)->cpool.disable_abandon += ERTS_ALC_CPOOL_ALLOC_OP_INC; \ + if ((A)->cpool.disable_abandon > ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON) \ + (A)->cpool.disable_abandon = ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON; \ + } \ +} while (0) + + +#if ERTS_ALC_CPOOL_ALLOC_OP_INC >= ERTS_ALC_CPOOL_FREE_OP_DEC +# error "Implementation assume ERTS_ALC_CPOOL_ALLOC_OP_INC < ERTS_ALC_CPOOL_FREE_OP_DEC" +#endif + +#define ERTS_ALC_CPOOL_REALLOC_OP(A) \ +do { \ + if ((A)->cpool.disable_abandon) { \ + (A)->cpool.disable_abandon -= (ERTS_ALC_CPOOL_FREE_OP_DEC \ + - ERTS_ALC_CPOOL_ALLOC_OP_INC); \ + if ((A)->cpool.disable_abandon < 0) \ + (A)->cpool.disable_abandon = 0; \ + } \ +} while (0) + +#define ERTS_ALC_CPOOL_FREE_OP(A) \ +do { \ + if ((A)->cpool.disable_abandon) { \ + (A)->cpool.disable_abandon -= ERTS_ALC_CPOOL_FREE_OP_DEC; \ + if ((A)->cpool.disable_abandon < 0) \ + (A)->cpool.disable_abandon = 0; \ + } \ +} while (0) + +#else +#define ERTS_ALC_CPOOL_ALLOC_OP(A) +#define ERTS_ALC_CPOOL_REALLOC_OP(A) +#define ERTS_ALC_CPOOL_FREE_OP(A) +#endif + +#define ERTS_CRR_ALCTR_FLG_IN_POOL (((erts_aint_t) 1) << 0) +#define ERTS_CRR_ALCTR_FLG_BUSY (((erts_aint_t) 1) << 1) + +#ifdef ERTS_SMP +#define SBC_HEADER_SIZE \ + (UNIT_CEILING(sizeof(Carrier_t) \ + - sizeof(ErtsAlcCPoolData_t) \ + + ABLK_HDR_SZ) \ + - ABLK_HDR_SZ) +#else +#define SBC_HEADER_SIZE \ + (UNIT_CEILING(sizeof(Carrier_t) \ + + ABLK_HDR_SZ) \ + - ABLK_HDR_SZ) +#endif +#define MBC_HEADER_SIZE(AP) ((AP)->mbc_header_size) #define MSEG_CARRIER_HDR_FLAG (((UWord) 1) << 0) @@ -352,7 +401,8 @@ MBC after deallocating first block: #define SCH_SBC SBC_CARRIER_HDR_FLAG #define SET_CARRIER_HDR(C, Sz, F, AP) \ - (ASSERT(((Sz) & FLG_MASK) == 0), (C)->chdr = ((Sz) | (F)), (C)->allctr = (AP)) + (ASSERT(((Sz) & FLG_MASK) == 0), (C)->chdr = ((Sz) | (F)), \ + erts_smp_atomic_init_nob(&(C)->allctr, (erts_aint_t) (AP))) #define BLK_TO_SBC(B) \ ((Carrier_t *) (((char *) (B)) - SBC_HEADER_SIZE)) @@ -385,6 +435,7 @@ MBC after deallocating first block: #define CFLG_FORCE_SYS_ALLOC (1 << 3) #define CFLG_FORCE_SIZE (1 << 4) #define CFLG_MAIN_CARRIER (1 << 5) +#define CFLG_NO_CPOOL (1 << 6) #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG static void check_blk_carrier(Allctr_t *, Block_t *); @@ -412,11 +463,7 @@ static void check_blk_carrier(Allctr_t *, Block_t *); ASSERT(((AP)->mbcs.curr.norm.sys_alloc.no \ && (AP)->mbcs.curr.norm.sys_alloc.size) \ || (!(AP)->mbcs.curr.norm.sys_alloc.no \ - && !(AP)->mbcs.curr.norm.sys_alloc.size)); \ - ASSERT(((AP)->sbmbcs.curr.small_block.no \ - && (AP)->sbmbcs.curr.small_block.size) \ - || (!(AP)->sbmbcs.curr.small_block.no \ - && !(AP)->sbmbcs.curr.small_block.size)) + && !(AP)->mbcs.curr.norm.sys_alloc.size)); #else #define DEBUG_CHECK_CARRIER_NO_SZ(AP) @@ -487,17 +534,6 @@ do { \ + (AP)->mbcs.curr.norm.sys_alloc.size) -#define STAT_SBMBC_ALLOC(AP, CSZ) \ -do { \ - (AP)->sbmbcs.curr.small_block.no++; \ - (AP)->sbmbcs.curr.small_block.size += (CSZ); \ - if ((AP)->sbmbcs.max.no < (AP)->sbmbcs.curr.small_block.no) \ - (AP)->sbmbcs.max.no = (AP)->sbmbcs.curr.small_block.no; \ - if ((AP)->sbmbcs.max.size < (AP)->sbmbcs.curr.small_block.size) \ - (AP)->sbmbcs.max.size = (AP)->sbmbcs.curr.small_block.size; \ - DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ -} while (0) - #define STAT_MSEG_MBC_ALLOC(AP, CSZ) \ do { \ (AP)->mbcs.curr.norm.mseg.no++; \ @@ -514,13 +550,19 @@ do { \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) -#define STAT_SBMBC_FREE(AP, CSZ) \ +#define STAT_MBC_CPOOL_FETCH(AP, CRR) \ do { \ - ASSERT((AP)->sbmbcs.curr.small_block.no > 0); \ - (AP)->sbmbcs.curr.small_block.no--; \ - ASSERT((AP)->sbmbcs.curr.small_block.size >= (CSZ)); \ - (AP)->sbmbcs.curr.small_block.size -= (CSZ); \ - DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ + UWord csz__ = CARRIER_SZ((CRR)); \ + if (IS_MSEG_CARRIER((CRR))) \ + STAT_MSEG_MBC_ALLOC((AP), csz__); \ + else \ + STAT_SYS_ALLOC_MBC_ALLOC((AP), csz__); \ + (AP)->mbcs.blocks.curr.no += (CRR)->cpool.blocks; \ + if ((AP)->mbcs.blocks.max.no < (AP)->mbcs.blocks.curr.no) \ + (AP)->mbcs.blocks.max.no = (AP)->mbcs.blocks.curr.no; \ + (AP)->mbcs.blocks.curr.size += (CRR)->cpool.blocks_size; \ + if ((AP)->mbcs.blocks.max.size < (AP)->mbcs.blocks.curr.size) \ + (AP)->mbcs.blocks.max.size = (AP)->mbcs.blocks.curr.size; \ } while (0) #define STAT_MSEG_MBC_FREE(AP, CSZ) \ @@ -541,28 +583,88 @@ do { \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) -#define STAT_MBC_BLK_ALLOC(AP, BSZ, FLGS) \ +#define STAT_MBC_CPOOL_INSERT(AP, CRR) \ +do { \ + UWord csz__ = CARRIER_SZ((CRR)); \ + if (IS_MSEG_CARRIER((CRR))) \ + STAT_MSEG_MBC_FREE((AP), csz__); \ + else \ + STAT_SYS_ALLOC_MBC_FREE((AP), csz__); \ + ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.no \ + >= (CRR)->cpool.blocks); \ + (AP)->mbcs.blocks.curr.no -= (CRR)->cpool.blocks; \ + ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.size \ + >= (CRR)->cpool.blocks_size); \ + (AP)->mbcs.blocks.curr.size -= (CRR)->cpool.blocks_size; \ +} while (0) + +#ifdef ERTS_SMP +#define STAT_MBC_BLK_ALLOC_CRR(CRR, BSZ) \ +do { \ + (CRR)->cpool.blocks++; \ + (CRR)->cpool.blocks_size += (BSZ); \ +} while (0) +#else +#define STAT_MBC_BLK_ALLOC_CRR(CRR, BSZ) ((void) (CRR)) /* Get rid of warning */ +#endif + +#define STAT_MBC_BLK_ALLOC(AP, CRR, BSZ, FLGS) \ do { \ - CarriersStats_t *cstats__ = (((FLGS) & ERTS_ALCU_FLG_SBMBC) \ - ? &(AP)->sbmbcs \ - : &(AP)->mbcs); \ + CarriersStats_t *cstats__ = &(AP)->mbcs; \ cstats__->blocks.curr.no++; \ if (cstats__->blocks.max.no < cstats__->blocks.curr.no) \ cstats__->blocks.max.no = cstats__->blocks.curr.no; \ cstats__->blocks.curr.size += (BSZ); \ if (cstats__->blocks.max.size < cstats__->blocks.curr.size) \ cstats__->blocks.max.size = cstats__->blocks.curr.size; \ + STAT_MBC_BLK_ALLOC_CRR((CRR), (BSZ)); \ } while (0) -#define STAT_MBC_BLK_FREE(AP, BSZ, FLGS) \ +static ERTS_INLINE int +stat_cpool_mbc_blk_free(Allctr_t *allctr, + Carrier_t *crr, + Carrier_t **busy_pcrr_pp, + UWord blksz) +{ +#ifdef ERTS_SMP + + ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks > 0); + crr->cpool.blocks--; + ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks_size >= blksz); + crr->cpool.blocks_size -= blksz; + + if (!busy_pcrr_pp || !*busy_pcrr_pp) + return 0; + + ERTS_ALC_CPOOL_ASSERT(crr == *busy_pcrr_pp); + +#ifdef ERTS_ALC_CPOOL_DEBUG + ERTS_ALC_CPOOL_ASSERT( + erts_atomic_dec_read_nob(&allctr->cpool.stat.no_blocks) >= 0); + ERTS_ALC_CPOOL_ASSERT( + erts_atomic_add_read_nob(&allctr->cpool.stat.blocks_size, + -((erts_aint_t) blksz)) >= 0); +#else + erts_atomic_dec_nob(&allctr->cpool.stat.no_blocks); + erts_atomic_add_nob(&allctr->cpool.stat.blocks_size, + -((erts_aint_t) blksz)); +#endif + + return 1; +#else + return 0; +#endif +} + +#define STAT_MBC_BLK_FREE(AP, CRR, BPCRRPP, BSZ, FLGS) \ do { \ - CarriersStats_t *cstats__ = (((FLGS) & ERTS_ALCU_FLG_SBMBC) \ - ? &(AP)->sbmbcs \ - : &(AP)->mbcs); \ - ASSERT(cstats__->blocks.curr.no > 0); \ - cstats__->blocks.curr.no--; \ - ASSERT(cstats__->blocks.curr.size >= (BSZ)); \ - cstats__->blocks.curr.size -= (BSZ); \ + if (!stat_cpool_mbc_blk_free((AP), (CRR), (BPCRRPP), (BSZ))) { \ + CarriersStats_t *cstats__ = &(AP)->mbcs; \ + ASSERT(cstats__->blocks.curr.no > 0); \ + cstats__->blocks.curr.no--; \ + ASSERT(cstats__->blocks.curr.size >= (BSZ)); \ + cstats__->blocks.curr.size -= (BSZ); \ + } \ } while (0) /* Debug stuff... */ @@ -613,13 +715,42 @@ do { \ #define ERTS_ALCU_DBG_CHK_THR_ACCESS(A) #endif - static void make_name_atoms(Allctr_t *allctr); static Block_t *create_carrier(Allctr_t *, Uint, UWord); -static void destroy_carrier(Allctr_t *, Block_t *); -static void mbc_free(Allctr_t *allctr, void *p); +static void destroy_carrier(Allctr_t *, Block_t *, Carrier_t **); +static void mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp); +static void dealloc_block(Allctr_t *, void *, int); + +/* internal data... */ + +#if 0 + +static ERTS_INLINE void * +internal_alloc(UWord size) +{ + void *res = erts_sys_alloc(0, NULL, size); + if (!res) + erts_alloc_enomem(ERTS_ALC_T_UNDEF, size); + return res; +} +static ERTS_INLINE void * +internal_realloc(void *ptr, UWord size) +{ + void *res = erts_sys_realloc(0, NULL, ptr, size); + if (!res) + erts_alloc_enomem(ERTS_ALC_T_UNDEF, size); + return res; +} + +static ERTS_INLINE void +internal_free(void *ptr) +{ + erts_sys_free(0, NULL, ptr); +} + +#endif /* mseg ... */ @@ -781,10 +912,35 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr) } } -static Block_t *create_sbmbc(Allctr_t *allctr, Uint umem_sz); -static void destroy_sbmbc(Allctr_t *allctr, Block_t *blk); -static Block_t *create_carrier(Allctr_t *, Uint, UWord); -static void destroy_carrier(Allctr_t *, Block_t *); +#ifdef ERTS_SMP + +static ERTS_INLINE void +clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr) +{ + if (crr) { + erts_aint_t max_size; + erts_aint_t new_val; + + max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); + erts_atomic_set_nob(&crr->cpool.max_size, max_size); + + new_val = (((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); + +#ifdef ERTS_ALC_CPOOL_DEBUG + { + erts_aint_t old_val = new_val|ERTS_CRR_ALCTR_FLG_BUSY; + + ERTS_ALC_CPOOL_ASSERT(old_val + == erts_smp_atomic_xchg_relb(&crr->allctr, + new_val)); + } +#else + erts_smp_atomic_set_relb(&crr->allctr, new_val); +#endif + } +} + +#endif #if 0 #define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) \ @@ -806,13 +962,154 @@ chk_fix_list(Allctr_t *allctr, ErtsAlcFixList_t *fix, int ix, int before) #define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) #endif -erts_aint32_t -erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) +static void *mbc_alloc(Allctr_t *allctr, Uint size); + +#ifdef ERTS_SMP +typedef struct { + ErtsAllctrDDBlock_t ddblock__; /* must be first */ + ErtsAlcType_t fix_type; +} ErtsAllctrFixDDBlock_t; +#endif + +static ERTS_INLINE void +dealloc_fix_block(Allctr_t *allctr, + ErtsAlcType_t type, + void *ptr, + int dec_cc_on_redirect) +{ +#ifdef ERTS_SMP + /* May be redirected... */ + ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type; +#endif + dealloc_block(allctr, ptr, dec_cc_on_redirect); +} + +static ERTS_INLINE void +sched_fix_shrink(Allctr_t *allctr, int on) +{ + if (on && !allctr->fix_shrink_scheduled) { + allctr->fix_shrink_scheduled = 1; + erts_set_aux_work_timeout(allctr->ix, + (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), + 1); + } + else if (!on && allctr->fix_shrink_scheduled) { + allctr->fix_shrink_scheduled = 0; + erts_set_aux_work_timeout(allctr->ix, + (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), + 0); + } +} + +static ERTS_INLINE void +fix_cpool_check_shrink(Allctr_t *allctr, + ErtsAlcType_t type, + ErtsAlcFixList_t *fix, + Carrier_t **busy_pcrr_pp) +{ + if (fix->u.cpool.shrink_list > 0) { + if (fix->list_size == 0) + fix->u.cpool.shrink_list = 0; + else { + void *p; +#ifdef ERTS_SMP + if (busy_pcrr_pp) { + clear_busy_pool_carrier(allctr, *busy_pcrr_pp); + *busy_pcrr_pp = NULL; + } +#endif + fix->u.cpool.shrink_list--; + p = fix->list; + fix->list = *((void **) p); + fix->list_size--; + if (fix->u.cpool.min_list_size > fix->list_size) + fix->u.cpool.min_list_size = fix->list_size; + + fix->u.cpool.allocated--; + dealloc_fix_block(allctr, type, p, 0); + } + } +} + +static ERTS_INLINE void * +fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) +{ + void *res; + ErtsAlcFixList_t *fix; + + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type + && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + + fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + + res = fix->list; + if (res) { + fix->list = *((void **) res); + fix->list_size--; + if (fix->u.cpool.min_list_size > fix->list_size) + fix->u.cpool.min_list_size = fix->list_size; + fix->u.cpool.used++; + fix_cpool_check_shrink(allctr, type, fix, NULL); + return res; + } + if (size < 2*sizeof(UWord)) + size += sizeof(UWord); + if (size >= allctr->sbc_threshold) { + Block_t *blk; + blk = create_carrier(allctr, size, CFLG_SBC); + res = blk ? BLK2UMEM(blk) : NULL; + } + else + res = mbc_alloc(allctr, size); + if (res) { + fix->u.cpool.used++; + fix->u.cpool.allocated++; + } + return res; +} + +static ERTS_INLINE void +fix_cpool_free(Allctr_t *allctr, + ErtsAlcType_t type, + void *p, + Carrier_t **busy_pcrr_pp) +{ + ErtsAlcFixList_t *fix; + + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type + && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + + fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + + fix->u.cpool.used--; + + if ((!busy_pcrr_pp || !*busy_pcrr_pp) + && !fix->u.cpool.shrink_list + && fix->list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { + *((void **) p) = fix->list; + fix->list = p; + fix->list_size++; + sched_fix_shrink(allctr, 1); + } + else { + Block_t *blk = UMEM2BLK(p); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk, NULL); + else + mbc_free(allctr, p, busy_pcrr_pp); + fix->u.cpool.allocated--; + fix_cpool_check_shrink(allctr, type, fix, busy_pcrr_pp); + } +} + +static ERTS_INLINE erts_aint32_t +fix_cpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) { int all_empty = 1; erts_aint32_t res = 0; int ix, o; - ErtsAlcFixList_t *fix = allctr->fix; int flush = flgs == 0; #ifdef USE_THREADS @@ -821,56 +1118,204 @@ erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) #endif for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) { + ErtsAlcFixList_t *fix = &allctr->fix[ix]; + ErtsAlcType_t type; ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); - if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) { - fix[ix].limit = fix[ix].max_used; - if (fix[ix].limit < fix[ix].used) - fix[ix].limit = fix[ix].used; - fix[ix].max_used = fix[ix].used; - ASSERT(fix[ix].limit >= 0); - - } - if (flush) { - fix[ix].limit = 0; - fix[ix].max_used = fix[ix].used; - ASSERT(fix[ix].limit >= 0); + if (flush) + fix->u.cpool.shrink_list = fix->list_size; + else if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) { + fix->u.cpool.shrink_list = fix->u.cpool.min_list_size; + fix->u.cpool.min_list_size = fix->list_size; } + type = (ErtsAlcType_t) (ix + ERTS_ALC_N_MIN_A_FIXED_SIZE); for (o = 0; o < ERTS_ALC_FIX_MAX_SHRINK_OPS || flush; o++) { - Block_t *blk; void *ptr; - if (!flush && fix[ix].limit >= fix[ix].allocated) + if (fix->u.cpool.shrink_list == 0) break; - if (fix[ix].list_size == 0) + if (fix->list_size == 0) { + fix->u.cpool.shrink_list = 0; break; - ptr = fix[ix].list; - fix[ix].list = *((void **) ptr); - fix[ix].list_size--; + } + ptr = fix->list; + fix->list = *((void **) ptr); + fix->list_size--; + fix->u.cpool.shrink_list--; + fix->u.cpool.allocated--; + dealloc_fix_block(allctr, type, ptr, 0); + } + if (fix->u.cpool.min_list_size > fix->list_size) + fix->u.cpool.min_list_size = fix->list_size; + if (fix->list_size != 0) { + if (fix->u.cpool.shrink_list > 0) + res |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC; + all_empty = 0; + } + } + + if (all_empty) + sched_fix_shrink(allctr, 0); + +#ifdef USE_THREADS + if (allctr->thread_safe) + erts_mtx_unlock(&allctr->mutex); +#endif - blk = UMEM2BLK(ptr); + return res; +} +static ERTS_INLINE void * +fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) +{ + ErtsAlcFixList_t *fix; + void *res; + + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type + && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + + fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); + fix->u.nocpool.used++; + res = fix->list; + if (res) { + fix->list_size--; + fix->list = *((void **) res); + if (fix->list && fix->u.nocpool.allocated > fix->u.nocpool.limit) { + Block_t *blk; + void *p = fix->list; + fix->list = *((void **) p); + fix->list_size--; + blk = UMEM2BLK(p); if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); + destroy_carrier(allctr, blk, NULL); else - mbc_free(allctr, ptr); + mbc_free(allctr, p, NULL); + fix->u.nocpool.allocated--; + } + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + return res; + } + if (size < 2*sizeof(UWord)) + size += sizeof(UWord); + if (fix->u.nocpool.limit < fix->u.nocpool.used) + fix->u.nocpool.limit = fix->u.nocpool.used; + if (fix->u.nocpool.max_used < fix->u.nocpool.used) + fix->u.nocpool.max_used = fix->u.nocpool.used; + fix->u.nocpool.allocated++; - fix[ix].allocated--; + if (size >= allctr->sbc_threshold) { + Block_t *blk; + blk = create_carrier(allctr, size, CFLG_SBC); + res = blk ? BLK2UMEM(blk) : NULL; + } + else + res = mbc_alloc(allctr, size); + + if (!res) { + fix->u.nocpool.allocated--; + fix->u.nocpool.used--; + } + return res; +} + +static ERTS_INLINE void +fix_nocpool_free(Allctr_t *allctr, + ErtsAlcType_t type, + void *p) +{ + Block_t *blk; + ErtsAlcFixList_t *fix; + + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type + && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + + fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); + fix->u.nocpool.used--; + if (fix->u.nocpool.allocated < fix->u.nocpool.limit + && fix->list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { + *((void **) p) = fix->list; + fix->list = p; + fix->list_size++; + sched_fix_shrink(allctr, 1); + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + return; + } + fix->u.nocpool.allocated--; + if (fix->list && fix->u.nocpool.allocated > fix->u.nocpool.limit) { + blk = UMEM2BLK(p); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk, NULL); + else + mbc_free(allctr, p, NULL); + p = fix->list; + fix->list = *((void **) p); + fix->list_size--; + fix->u.nocpool.allocated--; + } + + blk = UMEM2BLK(p); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk, NULL); + else + mbc_free(allctr, p, NULL); + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); +} + +static ERTS_INLINE erts_aint32_t +fix_nocpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) +{ + int all_empty = 1; + erts_aint32_t res = 0; + int ix, o; + int flush = flgs == 0; + +#ifdef USE_THREADS + if (allctr->thread_safe) + erts_mtx_lock(&allctr->mutex); +#endif + + for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) { + ErtsAlcFixList_t *fix = &allctr->fix[ix]; + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); + if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) { + fix->u.nocpool.limit = fix->u.nocpool.max_used; + if (fix->u.nocpool.limit < fix->u.nocpool.used) + fix->u.nocpool.limit = fix->u.nocpool.used; + fix->u.nocpool.max_used = fix->u.nocpool.used; + ASSERT(fix->u.nocpool.limit >= 0); + + } + if (flush) { + fix->u.nocpool.limit = 0; + fix->u.nocpool.max_used = fix->u.nocpool.used; + ASSERT(fix->u.nocpool.limit >= 0); } - if (fix[ix].list_size != 0) { - if (fix[ix].limit < fix[ix].allocated) + for (o = 0; o < ERTS_ALC_FIX_MAX_SHRINK_OPS || flush; o++) { + void *ptr; + + if (!flush && fix->u.nocpool.limit >= fix->u.nocpool.allocated) + break; + if (fix->list_size == 0) + break; + ptr = fix->list; + fix->list = *((void **) ptr); + fix->list_size--; + dealloc_block(allctr, ptr, 0); + fix->u.nocpool.allocated--; + } + if (fix->list_size != 0) { + if (fix->u.nocpool.limit < fix->u.nocpool.allocated) res |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC; all_empty = 0; } ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); } - if (all_empty && allctr->fix_shrink_scheduled) { - allctr->fix_shrink_scheduled = 0; - erts_set_aux_work_timeout(allctr->ix, - (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM - | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), - 0); - } + if (all_empty) + sched_fix_shrink(allctr, 0); #ifdef USE_THREADS if (allctr->thread_safe) @@ -880,13 +1325,18 @@ erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) return res; } -#ifdef ERTS_SMP +erts_aint32_t +erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) +{ + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + return fix_cpool_alloc_shrink(allctr, flgs); + else + return fix_nocpool_alloc_shrink(allctr, flgs); +} -typedef struct { - ErtsAllctrDDBlock_t ddblock__; /* must be first */ - ErtsAlcType_t fix_type; -}ErtsAllctrFixDDBlock_t; +static void dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, Uint mseg_flags); +#ifdef ERTS_SMP static ERTS_INLINE Allctr_t* get_pref_allctr(void *extra) @@ -902,6 +1352,9 @@ get_pref_allctr(void *extra) return tspec->allctr[pref_ix]; } +#define ERTS_ALC_TS_PREF_LOCK_IF_USED (1) +#define ERTS_ALC_TS_PREF_LOCK_NO (0) + /* SMP note: * get_used_allctr() must be safe WITHOUT locking the allocator while * concurrent threads may be updating adjacent blocks. @@ -910,22 +1363,85 @@ get_pref_allctr(void *extra) * the "PREV_FREE" flag bit. */ static ERTS_INLINE Allctr_t* -get_used_allctr(void *extra, void *p, UWord *sizep) +get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep, + Carrier_t **busy_pcrr_pp) { Block_t* blk = UMEM2BLK(p); - Carrier_t* crr; + Carrier_t *crr; + erts_aint_t iallctr; + Allctr_t *used_allctr; + + *busy_pcrr_pp = NULL; if (IS_SBC_BLK(blk)) { crr = BLK_TO_SBC(blk); if (sizep) *sizep = SBC_BLK_SZ(blk) - ABLK_HDR_SZ; + iallctr = erts_smp_atomic_read_dirty(&crr->allctr); } else { crr = ABLK_TO_MBC(blk); + if (sizep) *sizep = MBC_ABLK_SZ(blk) - ABLK_HDR_SZ; + if (!ERTS_ALC_IS_CPOOL_ENABLED(pref_allctr)) + iallctr = erts_smp_atomic_read_dirty(&crr->allctr); + else { + int locked_pref_allctr = 0; + iallctr = erts_smp_atomic_read_ddrb(&crr->allctr); + + if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock + && pref_allctr->thread_safe) { + used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK); + if (pref_allctr == used_allctr) { + erts_mtx_lock(&pref_allctr->mutex); + locked_pref_allctr = 1; + } + } + + while ((iallctr & ((~FLG_MASK)|ERTS_CRR_ALCTR_FLG_IN_POOL)) + == (((erts_aint_t) pref_allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL)) { + erts_aint_t act; + + ERTS_ALC_CPOOL_ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY)); + act = erts_smp_atomic_cmpxchg_ddrb(&crr->allctr, + iallctr|ERTS_CRR_ALCTR_FLG_BUSY, + iallctr); + if (act == iallctr) { + *busy_pcrr_pp = crr; + break; + } + iallctr = act; + } + + used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK); + + if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock) { + if (locked_pref_allctr && used_allctr != pref_allctr) { + /* Was taken out of pool; now owned by someone else */ + erts_mtx_unlock(&pref_allctr->mutex); + } + } + + ERTS_ALC_CPOOL_ASSERT( + (((iallctr & ~FLG_MASK) == (erts_aint_t) pref_allctr) + ? (((iallctr & FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL) + || ((iallctr & FLG_MASK) == 0)) + : 1)); + + return used_allctr; + } + } + + used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK); + + if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock + && used_allctr == pref_allctr + && pref_allctr->thread_safe) { + erts_mtx_lock(&pref_allctr->mutex); } - return crr->allctr; + + return used_allctr; } static void @@ -1020,7 +1536,7 @@ check_insert_marker(ErtsAllctrDDQueue_t *ddq, erts_aint_t ilast) } static ERTS_INLINE int -ddq_enqueue(ErtsAlcType_t type, ErtsAllctrDDQueue_t *ddq, void *ptr, int cinit) +ddq_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr, int cinit) { int last_elem; int um_refc_ix = 0; @@ -1121,6 +1637,59 @@ store_earliest_thr_prgr(ErtsThrPrgrVal *prev_val, ErtsAllctrDDQueue_t *ddq) } } +static void +check_pending_dealloc_carrier(Allctr_t *allctr, + int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, + int *need_more_work); + +static void +handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr) +{ + ErtsAlcType_t type; + + type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type; + + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type + && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + fix_nocpool_free(allctr, type, ptr); + else { + Block_t *blk = UMEM2BLK(ptr); + Carrier_t *busy_pcrr_p; + Allctr_t *used_allctr; + + if (IS_SBC_BLK(blk)) { + busy_pcrr_p = NULL; + goto doit; + } + + used_allctr = get_used_allctr(allctr, ERTS_ALC_TS_PREF_LOCK_NO, ptr, + NULL, &busy_pcrr_p); + if (used_allctr == allctr) { + doit: + fix_cpool_free(allctr, type, ptr, &busy_pcrr_p); + clear_busy_pool_carrier(allctr, busy_pcrr_p); + } + else { + /* Carrier migrated; need to redirect block to new owner... */ + int cinit = used_allctr->dd.ix - allctr->dd.ix; + + ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); + + DEC_CC(allctr->calls.this_free); + + ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type; + if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit)) + erts_alloc_notify_delayed_dealloc(used_allctr->ix); + } + } +} + +static void +schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr); + static ERTS_INLINE int handle_delayed_dealloc(Allctr_t *allctr, int allctr_locked, @@ -1152,7 +1721,6 @@ handle_delayed_dealloc(Allctr_t *allctr, while (1) { Block_t *blk; void *ptr; - int ix; if (use_limit && ++ops > ops_limit) { if (ddq->head.first != ddq->head.unref_end) { @@ -1181,52 +1749,29 @@ handle_delayed_dealloc(Allctr_t *allctr, res = 1; - INC_CC(allctr->calls.this_free); - - if (fix) { - ErtsAlcType_t type; - - type = ((ErtsAllctrFixDDBlock_t*) ptr)->fix_type; - ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); - fix[ix].used--; - if (fix[ix].allocated < fix[ix].limit - && fix[ix].list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { - *((void **) ptr) = fix[ix].list; - fix[ix].list = ptr; - fix[ix].list_size++; - if (!allctr->fix_shrink_scheduled) { - allctr->fix_shrink_scheduled = 1; - erts_set_aux_work_timeout( - allctr->ix, - (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM - | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), - 1); - } - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); - continue; - } - fix[ix].allocated--; - if (fix[ix].list && fix[ix].allocated > fix[ix].limit) { - blk = UMEM2BLK(ptr); - if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); - else - mbc_free(allctr, ptr); - ptr = fix[ix].list; - fix[ix].list = *((void **) ptr); - fix[ix].list_size--; - fix[ix].allocated--; - } + blk = UMEM2BLK(ptr); + if (IS_FREE_LAST_MBC_BLK(blk)) { + /* + * A multiblock carrier that previously has been migrated away + * from us and now is back to be deallocated... + * + * Note that we cannot use FBLK_TO_MBC(blk) since it + * data has been overwritten by the queue. + */ + Carrier_t *crr = FIRST_BLK_TO_MBC(allctr, blk); + ERTS_ALC_CPOOL_ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr)); + ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); + schedule_dealloc_carrier(allctr, crr); } + else { - blk = UMEM2BLK(ptr); + INC_CC(allctr->calls.this_free); - if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); - else - mbc_free(allctr, ptr); - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + if (fix) + handle_delayed_fix_dealloc(allctr, ptr); + else + dealloc_block(allctr, ptr, 1); + } } if (need_thr_progress && !(need_thr_prgr | need_mr_wrk)) { @@ -1236,6 +1781,12 @@ handle_delayed_dealloc(Allctr_t *allctr, store_earliest_thr_prgr(thr_prgr_p, ddq); } + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + check_pending_dealloc_carrier(allctr, + need_thr_progress, + thr_prgr_p, + need_more_work); + if (allctr->thread_safe && !allctr_locked) erts_mtx_unlock(&allctr->mutex); return res; @@ -1250,13 +1801,59 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type, if (allctr->fix) ((ErtsAllctrFixDDBlock_t*) ptr)->fix_type = type; - if (ddq_enqueue(type, &allctr->dd.q, ptr, cinit)) + if (ddq_enqueue(&allctr->dd.q, ptr, cinit)) erts_alloc_notify_delayed_dealloc(allctr->ix); } #endif #ifdef ERTS_SMP +static void +set_new_allctr_abandon_limit(Allctr_t *allctr); +static void +abandon_carrier(Allctr_t *allctr, Carrier_t *crr); + + +static ERTS_INLINE void +check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) +{ + Carrier_t *crr; + + if (busy_pcrr_pp && *busy_pcrr_pp) + return; + + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + return; + + allctr->cpool.check_limit_count--; + if (--allctr->cpool.check_limit_count <= 0) + set_new_allctr_abandon_limit(allctr); + + if (!erts_thr_progress_is_managed_thread()) + return; + + if (allctr->cpool.disable_abandon) + return; + + if (allctr->mbcs.blocks.curr.size > allctr->cpool.abandon_limit) + return; + + + crr = FBLK_TO_MBC(fblk); + + if (allctr->main_carrier == crr) + return; + + if (crr->cpool.blocks_size > crr->cpool.abandon_limit) + return; + + if (crr->cpool.thr_prgr != ERTS_THR_PRGR_INVALID + && !erts_thr_progress_has_reached(crr->cpool.thr_prgr)) + return; + + abandon_carrier(allctr, crr); +} + void erts_alcu_check_delayed_dealloc(Allctr_t *allctr, int limit, @@ -1274,78 +1871,88 @@ erts_alcu_check_delayed_dealloc(Allctr_t *allctr, } #endif -#define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \ - handle_delayed_dealloc((Allctr), (Locked), 1, \ +#define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \ + handle_delayed_dealloc((Allctr), (Locked), 1, \ ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL, NULL) +static void +dealloc_block(Allctr_t *allctr, void *ptr, int dec_cc_on_redirect) +{ + Block_t *blk = UMEM2BLK(ptr); + + ERTS_SMP_LC_ASSERT(!allctr->thread_safe + || erts_lc_mtx_is_locked(&allctr->mutex)); + + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk, NULL); +#ifndef ERTS_SMP + else + mbc_free(allctr, ptr, NULL); +#else + else if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + mbc_free(allctr, ptr, NULL); + else { + Carrier_t *busy_pcrr_p; + Allctr_t *used_allctr; + used_allctr = get_used_allctr(allctr, ERTS_ALC_TS_PREF_LOCK_NO, ptr, + NULL, &busy_pcrr_p); + if (used_allctr == allctr) { + mbc_free(allctr, ptr, &busy_pcrr_p); + clear_busy_pool_carrier(allctr, busy_pcrr_p); + } + else { + /* Carrier migrated; need to redirect block to new owner... */ + int cinit = used_allctr->dd.ix - allctr->dd.ix; + + ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); + + if (dec_cc_on_redirect) + DEC_CC(allctr->calls.this_free); + if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit)) + erts_alloc_notify_delayed_dealloc(used_allctr->ix); + } + } +#endif +} + /* Multi block carrier alloc/realloc/free ... */ /* NOTE! mbc_alloc() may in case of memory shortage place the requested * block in a sbc. */ static ERTS_INLINE void * -mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp, Uint32 *alcu_flgsp) +mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp) { Block_t *blk; Uint get_blk_sz; - Uint sbmbct; ASSERT(size); ASSERT(size < allctr->sbc_threshold); *blk_szp = get_blk_sz = UMEMSZ2BLKSZ(allctr, size); - sbmbct = allctr->sbmbc_threshold; - if (sbmbct) { - if (get_blk_sz < sbmbct) { - *alcu_flgsp |= ERTS_ALCU_FLG_SBMBC; - if (get_blk_sz + allctr->min_block_size > sbmbct) { - /* Since we use block size to determine if blocks are - located in sbmbc or not... */ - get_blk_sz += allctr->min_block_size; - } - } - } - -#ifdef ERTS_SMP - if (allctr->dd.use) - ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); -#endif - - blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0, *alcu_flgsp); - -#ifdef ERTS_SMP - if (!blk && allctr->dd.use) { - if (ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1)) - blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0, - *alcu_flgsp); - } -#endif + blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0); if (!blk) { - if ((*alcu_flgsp) & ERTS_ALCU_FLG_SBMBC) - blk = create_sbmbc(allctr, get_blk_sz); - else { - blk = create_carrier(allctr, get_blk_sz, CFLG_MBC); + blk = create_carrier(allctr, get_blk_sz, CFLG_MBC); #if !HALFWORD_HEAP && !HAVE_SUPER_ALIGNED_MB_CARRIERS - if (!blk) { - /* Emergency! We couldn't create the carrier as we wanted. - Try to place it in a sys_alloced sbc. */ - blk = create_carrier(allctr, - size, - (CFLG_SBC - | CFLG_FORCE_SIZE - | CFLG_FORCE_SYS_ALLOC)); - } -#endif + if (!blk) { + /* Emergency! We couldn't create the carrier as we wanted. + Try to place it in a sys_alloced sbc. */ + blk = create_carrier(allctr, + size, + (CFLG_SBC + | CFLG_FORCE_SIZE + | CFLG_FORCE_SYS_ALLOC)); } +#endif } #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG if (IS_MBC_BLK(blk)) { - (*allctr->link_free_block)(allctr, blk, *alcu_flgsp); + (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, blk, *alcu_flgsp); + (*allctr->unlink_free_block)(allctr, blk); } #endif @@ -1359,8 +1966,7 @@ mbc_alloc_finalize(Allctr_t *allctr, UWord flags, Carrier_t *crr, Uint want_blk_sz, - int valid_blk_info, - Uint32 alcu_flgs) + int valid_blk_info) { Uint blk_sz; Uint nxt_blk_sz; @@ -1392,7 +1998,7 @@ mbc_alloc_finalize(Allctr_t *allctr, SET_PREV_BLK_FREE(allctr, nxt_nxt_blk); } } - (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->link_free_block)(allctr, nxt_blk); ASSERT(IS_NOT_LAST_BLK(blk)); ASSERT(IS_FREE_BLK(nxt_blk)); @@ -1433,7 +2039,8 @@ mbc_alloc_finalize(Allctr_t *allctr, ASSERT(ABLK_TO_MBC(blk) == crr); } - STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); + ERTS_ALC_CPOOL_ALLOC_OP(allctr); + STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs); ASSERT(IS_ALLOCED_BLK(blk)); ASSERT(blk_sz == MBC_BLK_SZ(blk)); @@ -1453,8 +2060,7 @@ mbc_alloc(Allctr_t *allctr, Uint size) { Block_t *blk; Uint blk_sz; - Uint32 alcu_flgs = 0; - blk = mbc_alloc_block(allctr, size, &blk_sz, &alcu_flgs); + blk = mbc_alloc_block(allctr, size, &blk_sz); if (!blk) return NULL; if (IS_MBC_BLK(blk)) @@ -1464,35 +2070,34 @@ mbc_alloc(Allctr_t *allctr, Uint size) GET_BLK_HDR_FLGS(blk), FBLK_TO_MBC(blk), blk_sz, - 1, - alcu_flgs); + 1); return BLK2UMEM(blk); } static void -mbc_free(Allctr_t *allctr, void *p) +mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp) { Uint is_first_blk; Uint is_last_blk; - Uint32 alcu_flgs = 0; Uint blk_sz; Block_t *blk; Block_t *nxt_blk; - + Carrier_t *crr; ASSERT(p); blk = UMEM2BLK(p); blk_sz = MBC_ABLK_SZ(blk); - if (blk_sz < allctr->sbmbc_threshold) - alcu_flgs |= ERTS_ALCU_FLG_SBMBC; ASSERT(IS_MBC_BLK(blk)); ASSERT(blk_sz >= allctr->min_block_size); HARD_CHECK_BLK_CARRIER(allctr, blk); - STAT_MBC_BLK_FREE(allctr, blk_sz, alcu_flgs); + crr = ABLK_TO_MBC(blk); + + ERTS_ALC_CPOOL_FREE_OP(allctr); + STAT_MBC_BLK_FREE(allctr, crr, busy_pcrr_pp, blk_sz, alcu_flgs); is_first_blk = IS_MBC_FIRST_ABLK(allctr, blk); is_last_blk = IS_LAST_BLK(blk); @@ -1501,7 +2106,7 @@ mbc_free(Allctr_t *allctr, void *p) ASSERT(!is_first_blk); /* Coalesce with previous block... */ blk = PREV_BLK(blk); - (*allctr->unlink_free_block)(allctr, blk, alcu_flgs); + (*allctr->unlink_free_block)(allctr, blk); blk_sz += MBC_FBLK_SZ(blk); is_first_blk = IS_MBC_FIRST_FBLK(allctr, blk); @@ -1517,7 +2122,7 @@ mbc_free(Allctr_t *allctr, void *p) nxt_blk = BLK_AFTER(blk, blk_sz); if (IS_FREE_BLK(nxt_blk)) { /* Coalesce with next block... */ - (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->unlink_free_block)(allctr, nxt_blk); blk_sz += MBC_FBLK_SZ(nxt_blk); SET_MBC_FBLK_SZ(blk, blk_sz); @@ -1550,19 +2155,20 @@ mbc_free(Allctr_t *allctr, void *p) if (is_first_blk && is_last_blk && allctr->main_carrier != FIRST_BLK_TO_MBC(allctr, blk)) { - if (alcu_flgs & ERTS_ALCU_FLG_SBMBC) - destroy_sbmbc(allctr, blk); - else - destroy_carrier(allctr, blk); + destroy_carrier(allctr, blk, busy_pcrr_pp); } else { - (*allctr->link_free_block)(allctr, blk, alcu_flgs); + (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); +#ifdef ERTS_SMP + check_abandon_carrier(allctr, blk, busy_pcrr_pp); +#endif } } static void * -mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) +mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, + Carrier_t **busy_pcrr_pp) { void *new_p; Uint old_blk_sz; @@ -1576,11 +2182,6 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) Uint is_last_blk; #endif /* #ifndef MBC_REALLOC_ALWAYS_MOVES */ -#ifdef ERTS_SMP - if (allctr->dd.use) - ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); -#endif - ASSERT(p); ASSERT(size); ASSERT(size < allctr->sbc_threshold); @@ -1594,13 +2195,13 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) return NULL; #else /* !MBC_REALLOC_ALWAYS_MOVES */ + +#ifdef ERTS_SMP + if (busy_pcrr_pp && *busy_pcrr_pp) + goto realloc_move; /* Don't want to use carrier in pool */ +#endif + get_blk_sz = blk_sz = UMEMSZ2BLKSZ(allctr, size); - if ((alcu_flgs & ERTS_ALCU_FLG_SBMBC) - && (blk_sz + allctr->min_block_size > allctr->sbmbc_threshold)) { - /* Since we use block size to determine if blocks are - located in sbmbc or not... */ - get_blk_sz = blk_sz + allctr->min_block_size; - } ASSERT(IS_ALLOCED_BLK(blk)); ASSERT(IS_MBC_BLK(blk)); @@ -1648,8 +2249,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) new_blk = (*allctr->get_free_block)(allctr, get_blk_sz, cand_blk, - cand_blk_sz, - alcu_flgs); + cand_blk_sz); if (new_blk || cand_blk != blk) goto move_into_new_blk; } @@ -1671,8 +2271,11 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) nxt_blk = BLK_AFTER(blk, blk_sz); - STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); - STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); + crr = ABLK_TO_MBC(blk); + + ERTS_ALC_CPOOL_REALLOC_OP(allctr); + STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs); + STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs); ASSERT(MBC_BLK_SZ(blk) >= allctr->min_block_size); @@ -1680,7 +2283,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (IS_FREE_BLK(nxt_nxt_blk)) { /* Coalesce with next free block... */ nxt_blk_sz += MBC_FBLK_SZ(nxt_nxt_blk); - (*allctr->unlink_free_block)(allctr, nxt_nxt_blk, alcu_flgs); + (*allctr->unlink_free_block)(allctr, nxt_nxt_blk); is_last_blk = GET_LAST_BLK_HDR_FLG(nxt_nxt_blk); } @@ -1690,12 +2293,11 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); } - crr = ABLK_TO_MBC(blk); SET_MBC_FBLK_HDR(nxt_blk, nxt_blk_sz, SBH_THIS_FREE | (is_last_blk ? SBH_LAST_BLK : 0), crr); - (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->link_free_block)(allctr, nxt_blk); ASSERT(IS_ALLOCED_BLK(blk)); @@ -1718,6 +2320,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) HARD_CHECK_BLK_CARRIER(allctr, blk); +#ifdef ERTS_SMP + check_abandon_carrier(allctr, nxt_blk, NULL); +#endif + return p; } @@ -1727,11 +2333,12 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) nxt_blk = BLK_AFTER(blk, old_blk_sz); nxt_blk_sz = MBC_BLK_SZ(nxt_blk); if (IS_FREE_BLK(nxt_blk) && get_blk_sz <= old_blk_sz + nxt_blk_sz) { + Carrier_t* crr = ABLK_TO_MBC(blk); /* Grow into next block... */ HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->unlink_free_block)(allctr, nxt_blk); nxt_blk_sz -= blk_sz - old_blk_sz; is_last_blk = IS_LAST_BLK(nxt_blk); @@ -1756,7 +2363,6 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) } } else { - Carrier_t* crr = ABLK_TO_MBC(blk); SET_MBC_ABLK_SZ(blk, blk_sz); nxt_blk = BLK_AFTER(blk, blk_sz); @@ -1767,15 +2373,15 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) else SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); - (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->link_free_block)(allctr, nxt_blk); ASSERT(IS_FREE_BLK(nxt_blk)); ASSERT(FBLK_TO_MBC(nxt_blk) == crr); } - STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); - STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); - + ERTS_ALC_CPOOL_REALLOC_OP(allctr); + STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs); + STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs); ASSERT(IS_ALLOCED_BLK(blk)); ASSERT(blk_sz == MBC_BLK_SZ(blk)); @@ -1828,13 +2434,16 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (cand_blk_sz < get_blk_sz) { /* We wont fit in cand_blk get a new one */ +#ifdef ERTS_SMP + realloc_move: +#endif #endif /* !MBC_REALLOC_ALWAYS_MOVES */ new_p = mbc_alloc(allctr, size); if (!new_p) return NULL; sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ)); - mbc_free(allctr, p); + mbc_free(allctr, p, busy_pcrr_pp); return new_p; @@ -1847,8 +2456,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) new_blk = (*allctr->get_free_block)(allctr, get_blk_sz, cand_blk, - cand_blk_sz, - alcu_flgs); + cand_blk_sz); move_into_new_blk: /* * new_blk, and cand_blk have to be correctly set @@ -1862,11 +2470,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) GET_BLK_HDR_FLGS(new_blk), FBLK_TO_MBC(new_blk), blk_sz, - 1, - alcu_flgs); + 1); new_p = BLK2UMEM(new_blk); sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ)); - mbc_free(allctr, p); + mbc_free(allctr, p, NULL); return new_p; } else { @@ -1886,7 +2493,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, new_blk, alcu_flgs); /* prev */ + (*allctr->unlink_free_block)(allctr, new_blk); /* prev */ if (is_last_blk) new_blk_flgs |= LAST_BLK_HDR_FLG; @@ -1895,7 +2502,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (IS_FREE_BLK(nxt_blk)) { new_blk_flgs |= GET_LAST_BLK_HDR_FLG(nxt_blk); new_blk_sz += MBC_FBLK_SZ(nxt_blk); - (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->unlink_free_block)(allctr, nxt_blk); } } @@ -1920,10 +2527,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) new_blk_flgs, crr, blk_sz, - 0, - alcu_flgs); + 0); - STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); + ERTS_ALC_CPOOL_FREE_OP(allctr); + STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs); return new_p; } @@ -1931,6 +2538,635 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) #endif /* !MBC_REALLOC_ALWAYS_MOVES */ } +#ifdef ERTS_SMP + +#define ERTS_ALC_MAX_DEALLOC_CARRIER 10 +#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 10 +#define ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT 100 +#define ERTS_ALC_CPOOL_MAX_NO_CARRIERS 5 +#define ERTS_ALC_CPOOL_INSERT_ALLOWED_OFFSET 100 +#define ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS 3 + +#define ERTS_ALC_CPOOL_PTR_MOD_MRK (((erts_aint_t) 1) << 0) +#define ERTS_ALC_CPOOL_PTR_DEL_MRK (((erts_aint_t) 1) << 1) + +#define ERTS_ALC_CPOOL_PTR_MRKS \ + (ERTS_ALC_CPOOL_PTR_MOD_MRK | ERTS_ALC_CPOOL_PTR_DEL_MRK) + +/* + * When setting multiple mod markers we always + * set mod markers in pointer order and always + * on next pointers before prev pointers. + */ + +typedef union { + ErtsAlcCPoolData_t sentinel; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAlcCPoolData_t))]; +} ErtsAlcCrrPool_t; + +#if ERTS_ALC_A_INVALID != 0 +# error "Carrier pool implementation assumes ERTS_ALC_A_INVALID == 0" +#endif +#if ERTS_ALC_A_MIN <= ERTS_ALC_A_INVALID +# error "Carrier pool implementation assumes ERTS_ALC_A_MIN > ERTS_ALC_A_INVALID" +#endif + +/* + * The pool is only allowed to be manipulated by managed + * threads except in the alloc_SUITE:cpool case. In this + * test case carrier_pool[ERTS_ALC_A_INVALID] will be + * used. + */ + +static ErtsAlcCrrPool_t carrier_pool[ERTS_ALC_A_MAX+1] erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +#define ERTS_ALC_CPOOL_MAX_BACKOFF (1 << 8) + +static int +backoff(int n) +{ + int i; + + for (i = 0; i < n; i++) + ERTS_SPIN_BODY; + + if (n >= ERTS_ALC_CPOOL_MAX_BACKOFF) + return ERTS_ALC_CPOOL_MAX_BACKOFF; + else + return n << 1; +} + +static int +cpool_dbg_is_in_pool(Allctr_t *allctr, Carrier_t *crr) +{ + ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + ErtsAlcCPoolData_t *cpdp = sentinel; + Carrier_t *tmp_crr; + + while (1) { + cpdp = (ErtsAlcCPoolData_t *) (erts_atomic_read_ddrb(&cpdp->next) & ~FLG_MASK); + if (cpdp == sentinel) + return 0; + tmp_crr = (Carrier_t *) (((char *) cpdp) - offsetof(Carrier_t, cpool)); + if (tmp_crr == crr) + return 1; + } +} + +static int +cpool_is_empty(Allctr_t *allctr) +{ + ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + return ((erts_atomic_read_rb(&sentinel->next) == (erts_aint_t) sentinel) + && (erts_atomic_read_rb(&sentinel->prev) == (erts_aint_t) sentinel)); +} + +static ERTS_INLINE ErtsAlcCPoolData_t * +cpool_aint2cpd(erts_aint_t aint) +{ + return (ErtsAlcCPoolData_t *) (aint & ~ERTS_ALC_CPOOL_PTR_MRKS); +} + +static ERTS_INLINE erts_aint_t +cpool_read(erts_atomic_t *aptr) +{ + return erts_atomic_read_acqb(aptr); +} + +static ERTS_INLINE void +cpool_init(erts_atomic_t *aptr, erts_aint_t val) +{ + erts_atomic_set_nob(aptr, val); +} + +static ERTS_INLINE void +cpool_set_mod_marked(erts_atomic_t *aptr, erts_aint_t new, erts_aint_t old) +{ +#ifdef ERTS_ALC_CPOOL_DEBUG + erts_aint_t act = erts_atomic_xchg_relb(aptr, new); + ERTS_ALC_CPOOL_ASSERT(act == (old | ERTS_ALC_CPOOL_PTR_MOD_MRK)); +#else + erts_atomic_set_relb(aptr, new); +#endif +} + + +static ERTS_INLINE erts_aint_t +cpool_try_mod_mark_exp(erts_atomic_t *aptr, erts_aint_t exp) +{ + ERTS_ALC_CPOOL_ASSERT((exp & ERTS_ALC_CPOOL_PTR_MOD_MRK) == 0); + return erts_atomic_cmpxchg_nob(aptr, exp | ERTS_ALC_CPOOL_PTR_MOD_MRK, exp); +} + +static ERTS_INLINE erts_aint_t +cpool_mod_mark_exp(erts_atomic_t *aptr, erts_aint_t exp) +{ + int b; + erts_aint_t act; + ERTS_ALC_CPOOL_ASSERT((exp & ERTS_ALC_CPOOL_PTR_MOD_MRK) == 0); + while (1) { + act = erts_atomic_cmpxchg_nob(aptr, + exp | ERTS_ALC_CPOOL_PTR_MOD_MRK, + exp); + if (act == exp) + return exp; + b = 1; + do { + if ((act & ~ERTS_ALC_CPOOL_PTR_MOD_MRK) != exp) + return act; + b = backoff(b); + act = erts_atomic_read_nob(aptr); + } while (act != exp); + } +} + +static ERTS_INLINE erts_aint_t +cpool_mod_mark(erts_atomic_t *aptr) +{ + int b; + erts_aint_t act, exp; + act = cpool_read(aptr); + while (1) { + b = 1; + while (act & ERTS_ALC_CPOOL_PTR_MOD_MRK) { + b = backoff(b); + act = erts_atomic_read_nob(aptr); + } + exp = act; + act = erts_atomic_cmpxchg_acqb(aptr, + exp | ERTS_ALC_CPOOL_PTR_MOD_MRK, + exp); + if (act == exp) + return exp; + } +} + +static void +cpool_insert(Allctr_t *allctr, Carrier_t *crr) +{ + ErtsAlcCPoolData_t *cpd1p, *cpd2p; + erts_aint_t val; + ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + + ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ + || erts_thr_progress_is_managed_thread()); + ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_read_nob(&crr->allctr) + == (erts_aint_t) allctr); + + erts_atomic_add_nob(&allctr->cpool.stat.blocks_size, + (erts_aint_t) crr->cpool.blocks_size); + erts_atomic_add_nob(&allctr->cpool.stat.no_blocks, + (erts_aint_t) crr->cpool.blocks); + erts_atomic_add_nob(&allctr->cpool.stat.carriers_size, + (erts_aint_t) CARRIER_SZ(crr)); + erts_atomic_inc_nob(&allctr->cpool.stat.no_carriers); + + erts_smp_atomic_set_nob(&crr->allctr, + ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); + + /* + * We search in 'next' direction and begin by passing + * one element before trying to insert. This in order to + * avoid contention with threads fetching elements. + */ + + val = cpool_read(&sentinel->next); + + /* Find a predecessor to be, and set mod marker on its next ptr */ + + while (1) { + cpd1p = cpool_aint2cpd(val); + if (cpd1p == sentinel) { + val = cpool_mod_mark(&cpd1p->next); + break; + } + val = cpool_read(&cpd1p->next); + if (!(val & ERTS_ALC_CPOOL_PTR_MRKS)) { + erts_aint_t tmp = cpool_try_mod_mark_exp(&cpd1p->next, val); + if (tmp == val) { + val = tmp; + break; + } + val = tmp; + } + } + + /* Set mod marker on prev ptr of the to be successor */ + + cpd2p = cpool_aint2cpd(val); + + cpool_init(&crr->cpool.next, (erts_aint_t) cpd2p); + cpool_init(&crr->cpool.prev, (erts_aint_t) cpd1p); + + val = (erts_aint_t) cpd1p; + + while (1) { + int b; + erts_aint_t tmp; + + tmp = cpool_mod_mark_exp(&cpd2p->prev, val); + if (tmp == val) + break; + b = 1; + do { + b = backoff(b); + tmp = cpool_read(&cpd2p->prev); + } while (tmp != val); + } + + /* Write pointers to this element in successor and predecessor */ + + cpool_set_mod_marked(&cpd1p->next, + (erts_aint_t) &crr->cpool, + (erts_aint_t) cpd2p); + cpool_set_mod_marked(&cpd2p->prev, + (erts_aint_t) &crr->cpool, + (erts_aint_t) cpd1p); +} + +static void +cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr) +{ + ErtsAlcCPoolData_t *cpd1p, *cpd2p; + erts_aint_t val; +#ifdef ERTS_ALC_CPOOL_DEBUG + ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; +#endif + + ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ + || erts_thr_progress_is_managed_thread()); + ERTS_ALC_CPOOL_ASSERT(sentinel != &crr->cpool); + + /* Set mod marker on next ptr of our predecessor */ + + val = (erts_aint_t) &crr->cpool; + while (1) { + erts_aint_t tmp; + cpd1p = cpool_aint2cpd(cpool_read(&crr->cpool.prev)); + tmp = cpool_mod_mark_exp(&cpd1p->next, val); + if (tmp == val) + break; + } + + /* Set mod marker on our next ptr */ + + val = cpool_mod_mark(&crr->cpool.next); + + /* Set mod marker on the prev ptr of our successor */ + + cpd2p = cpool_aint2cpd(val); + + val = (erts_aint_t) &crr->cpool; + + while (1) { + int b; + erts_aint_t tmp; + + tmp = cpool_mod_mark_exp(&cpd2p->prev, val); + if (tmp == val) + break; + b = 1; + do { + b = backoff(b); + tmp = cpool_read(&cpd2p->prev); + } while (tmp != val); + } + + /* Set mod marker on our prev ptr */ + + val = (erts_aint_t) cpd1p; + + while (1) { + int b; + erts_aint_t tmp; + + tmp = cpool_mod_mark_exp(&crr->cpool.prev, val); + if (tmp == val) + break; + b = 1; + do { + b = backoff(b); + tmp = cpool_read(&cpd2p->prev); + } while (tmp != val); + } + + /* Write pointers past this element in predecessor and successor */ + + cpool_set_mod_marked(&cpd1p->next, + (erts_aint_t) cpd2p, + (erts_aint_t) &crr->cpool); + cpool_set_mod_marked(&cpd2p->prev, + (erts_aint_t) cpd1p, + (erts_aint_t) &crr->cpool); + + /* Repleace mod markers with delete markers on this element */ + cpool_set_mod_marked(&crr->cpool.next, + ((erts_aint_t) cpd2p) | ERTS_ALC_CPOOL_PTR_DEL_MRK, + ((erts_aint_t) cpd2p) | ERTS_ALC_CPOOL_PTR_MOD_MRK); + cpool_set_mod_marked(&crr->cpool.prev, + ((erts_aint_t) cpd1p) | ERTS_ALC_CPOOL_PTR_DEL_MRK, + ((erts_aint_t) cpd1p) | ERTS_ALC_CPOOL_PTR_MOD_MRK); + + crr->cpool.thr_prgr = erts_thr_progress_later(NULL); + + erts_atomic_add_nob(&prev_allctr->cpool.stat.blocks_size, + -((erts_aint_t) crr->cpool.blocks_size)); + erts_atomic_add_nob(&prev_allctr->cpool.stat.no_blocks, + -((erts_aint_t) crr->cpool.blocks)); + erts_atomic_add_nob(&prev_allctr->cpool.stat.carriers_size, + -((erts_aint_t) CARRIER_SZ(crr))); + erts_atomic_dec_wb(&prev_allctr->cpool.stat.no_carriers); + +} + +static Carrier_t * +cpool_fetch(Allctr_t *allctr, UWord size) +{ + int i; + Carrier_t *crr; + ErtsAlcCPoolData_t *cpdp; + ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + + ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ + || erts_thr_progress_is_managed_thread()); + + i = 0; + + /* First; check our own pending dealloc carrier list... */ + crr = allctr->cpool.dc_list.last; + while (crr && i < ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) { + if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) { + unlink_carrier(&allctr->cpool.dc_list, crr); +#ifdef ERTS_ALC_CPOOL_DEBUG + ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr, + ((erts_aint_t) allctr)) + == (((erts_aint_t) allctr) & ~FLG_MASK)); +#else + erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); +#endif + return crr; + } + crr = crr->prev; + i++; + } + + /* ... then the pool ... */ + + /* + * We search in 'prev' direction and begin by passing + * one element before trying to fetch. This in order to + * avoid contention with threads inserting elements. + */ + + cpdp = cpool_aint2cpd(cpool_read(&sentinel->prev)); + if (cpdp == sentinel) + return NULL; + + while (i < ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) { + erts_aint_t exp; + cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); + if (cpdp == sentinel) { + cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); + if (cpdp == sentinel) + return NULL; + i = ERTS_ALC_CPOOL_MAX_FETCH_INSPECT; /* Last one to inspect */ + } + crr = (Carrier_t *) (((char *) cpdp) - offsetof(Carrier_t, cpool)); + exp = erts_smp_atomic_read_rb(&crr->allctr); + if (((exp & (ERTS_CRR_ALCTR_FLG_IN_POOL|ERTS_CRR_ALCTR_FLG_BUSY)) + == ERTS_CRR_ALCTR_FLG_IN_POOL) + && (erts_atomic_read_nob(&cpdp->max_size) >= size)) { + erts_aint_t act; + /* Try to fetch it... */ + act = erts_smp_atomic_cmpxchg_mb(&crr->allctr, + (erts_aint_t) allctr, + exp); + if (act == exp) { + cpool_delete(allctr, ((Allctr_t *) (act & ~FLG_MASK)), crr); + return crr; + } + } + i++; + } + return NULL; +} + +static void +check_pending_dealloc_carrier(Allctr_t *allctr, + int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, + int *need_more_work) +{ + Carrier_t *crr = allctr->cpool.dc_list.first; + + if (crr) { + ErtsThrPrgrVal current = erts_thr_progress_current(); + int i = 0; + + do { + Carrier_t *dcrr; + + if (!erts_thr_progress_has_reached_this(current, crr->cpool.thr_prgr)) + break; + + dcrr = crr; + crr = crr->next; + dealloc_carrier(allctr, dcrr, ERTS_MSEG_FLG_2POW); + i++; + } while (crr && i < ERTS_ALC_MAX_DEALLOC_CARRIER); + + allctr->cpool.dc_list.first = crr; + if (!crr) + allctr->cpool.dc_list.last = NULL; + else { + crr->prev = NULL; + + if (need_more_work) { + ERTS_ALC_CPOOL_ASSERT(need_thr_progress && thr_prgr_p); + if (erts_thr_progress_has_reached_this(current, crr->cpool.thr_prgr)) + *need_more_work = 1; + else { + *need_thr_progress = 1; + if (*thr_prgr_p == ERTS_THR_PRGR_INVALID + || erts_thr_progress_cmp(crr->cpool.thr_prgr, + *thr_prgr_p) < 0) { + *thr_prgr_p = crr->cpool.thr_prgr; + } + } + } + } + } +} + +static void +schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) +{ + Allctr_t *used_allctr; + int check_pending_dealloc; + erts_aint_t max_size; + + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + dealloc_carrier(allctr, crr, ERTS_MSEG_FLG_2POW); + return; + } + + used_allctr = crr->cpool.orig_allctr; + + if (allctr != used_allctr) { + Block_t *blk = MBC_TO_FIRST_BLK(allctr, crr); + int cinit = used_allctr->dd.ix - allctr->dd.ix; + + /* + * Receiver will recognize that this is a carrier to + * deallocate since the block is an mbc block that + * is free and last in carrier... + */ + ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(blk)); + + ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, blk)); + ERTS_ALC_CPOOL_ASSERT(crr == FBLK_TO_MBC(blk)); + ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(used_allctr, blk)); + + if (ddq_enqueue(&used_allctr->dd.q, BLK2UMEM(blk), cinit)) + erts_alloc_notify_delayed_dealloc(used_allctr->ix); + return; + } + + if (crr->cpool.thr_prgr == ERTS_THR_PRGR_INVALID + || erts_thr_progress_has_reached(crr->cpool.thr_prgr)) { + dealloc_carrier(allctr, crr, ERTS_MSEG_FLG_2POW); + return; + } + + max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); + erts_atomic_set_nob(&crr->cpool.max_size, max_size); + + crr->next = NULL; + crr->prev = allctr->cpool.dc_list.last; + if (allctr->cpool.dc_list.last) { + check_pending_dealloc = 1; + allctr->cpool.dc_list.last->next = crr; + } + else { + check_pending_dealloc = 0; + allctr->cpool.dc_list.first = crr; + } + allctr->cpool.dc_list.last = crr; + if (check_pending_dealloc) + check_pending_dealloc_carrier(allctr, NULL, NULL, NULL); + erts_alloc_ensure_handle_delayed_dealloc_call(allctr->ix); +} + +static ERTS_INLINE void +cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr) +{ + erts_atomic_init_nob(&crr->cpool.next, ERTS_AINT_NULL); + erts_atomic_init_nob(&crr->cpool.prev, ERTS_AINT_NULL); + crr->cpool.orig_allctr = allctr; + crr->cpool.thr_prgr = ERTS_THR_PRGR_INVALID; + erts_atomic_init_nob(&crr->cpool.max_size, 0); + crr->cpool.blocks = 0; + crr->cpool.blocks_size = 0; + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + crr->cpool.abandon_limit = 0; + else { + UWord csz = CARRIER_SZ(crr); + UWord limit = csz*allctr->cpool.util_limit; + if (limit > csz) + limit /= 100; + else + limit = (csz/100)*allctr->cpool.util_limit; + crr->cpool.abandon_limit = limit; + } +} + +static void +set_new_allctr_abandon_limit(Allctr_t *allctr) +{ + UWord limit; + UWord csz; + + allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT; + + csz = allctr->mbcs.curr.norm.mseg.size; + csz += allctr->mbcs.curr.norm.sys_alloc.size; + + limit = csz*allctr->cpool.util_limit; + if (limit > csz) + limit /= 100; + else + limit = (csz/100)*allctr->cpool.util_limit; + + allctr->cpool.abandon_limit = limit; +} + +static void +abandon_carrier(Allctr_t *allctr, Carrier_t *crr) +{ + erts_aint_t max_size; + + STAT_MBC_CPOOL_INSERT(allctr, crr); + + unlink_carrier(&allctr->mbc_list, crr); + + allctr->remove_mbc(allctr, crr); + + max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); + erts_atomic_set_nob(&crr->cpool.max_size, max_size); + + cpool_insert(allctr, crr); + + set_new_allctr_abandon_limit(allctr); +} + +static void +cpool_read_stat(Allctr_t *allctr, UWord *nocp, UWord *cszp, UWord *nobp, UWord *bszp) +{ + int i; + UWord noc = 0, csz = 0, nob = 0, bsz = 0; + + /* + * We try to get consistent values, but after + * ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS failed + * tries we give up and present what we got... + */ + for (i = 0; i <= ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS; i++) { + UWord tnoc, tcsz, tnob, tbsz; + + tnoc = (UWord) (nocp + ? erts_atomic_read_nob(&allctr->cpool.stat.no_carriers) + : 0); + tcsz = (UWord) (cszp + ? erts_atomic_read_nob(&allctr->cpool.stat.carriers_size) + : 0); + tnob = (UWord) (nobp + ? erts_atomic_read_nob(&allctr->cpool.stat.no_blocks) + : 0); + tbsz = (UWord) (bszp + ? erts_atomic_read_nob(&allctr->cpool.stat.blocks_size) + : 0); + if (tnoc == noc && tcsz == csz && tnob == nob && tbsz == bsz) + break; + noc = tnoc; + csz = tcsz; + nob = tnob; + bsz = tbsz; + ERTS_THR_READ_MEMORY_BARRIER; + } + + if (nocp) + *nocp = noc; + if (cszp) + *cszp = csz; + if (nobp) + *nobp = nob; + if (bszp) + *bszp = bsz; +} + + +#endif /* ERTS_SMP */ + #ifdef DEBUG #if HAVE_ERTS_MSEG @@ -1974,81 +3210,6 @@ static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C, #endif static Block_t * -create_sbmbc(Allctr_t *allctr, Uint umem_sz) -{ - Block_t *blk; - Uint blk_sz; - Uint crr_sz = allctr->sbmbc_size; - Carrier_t *crr; - -#if HALFWORD_HEAP - if (allctr->mseg_opt.low_mem) - crr = erts_alloc(ERTS_ALC_T_SBMBC_LOW, crr_sz); - else -#endif - crr = erts_alloc(ERTS_ALC_T_SBMBC, crr_sz); - - INC_CC(allctr->calls.sbmbc_alloc); - SET_CARRIER_HDR(crr, crr_sz, SCH_SYS_ALLOC|SCH_MBC, allctr); - - blk = MBC_TO_FIRST_BLK(allctr, crr); - - blk_sz = UNIT_FLOOR(crr_sz - MBC_HEADER_SIZE(allctr)); - - SET_MBC_FBLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr); - - link_carrier(&allctr->sbmbc_list, crr); - - STAT_SBMBC_ALLOC(allctr, crr_sz); - CHECK_1BLK_CARRIER(allctr, 0, 0, crr, crr_sz, blk, blk_sz); - if (allctr->creating_mbc) - (*allctr->creating_mbc)(allctr, crr, ERTS_ALCU_FLG_SBMBC); - - DEBUG_SAVE_ALIGNMENT(crr); - return blk; -} - -static void -destroy_sbmbc(Allctr_t *allctr, Block_t *blk) -{ - Uint crr_sz; - Carrier_t *crr; - - ASSERT(IS_MBC_BLK(blk)); - ASSERT(IS_MBC_FIRST_FBLK(allctr, blk)); - - crr = FIRST_BLK_TO_MBC(allctr, blk); - crr_sz = CARRIER_SZ(crr); - -#ifdef DEBUG - if (!allctr->stopped) { - ASSERT(IS_LAST_BLK(blk)); - -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - (*allctr->link_free_block)(allctr, blk, ERTS_ALCU_FLG_SBMBC); - HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, blk, ERTS_ALCU_FLG_SBMBC); -#endif - } -#endif - - STAT_SBMBC_FREE(allctr, crr_sz); - - unlink_carrier(&allctr->sbmbc_list, crr); - if (allctr->destroying_mbc) - (*allctr->destroying_mbc)(allctr, crr, ERTS_ALCU_FLG_SBMBC); - - INC_CC(allctr->calls.sbmbc_free); - -#if HALFWORD_HEAP - if (allctr->mseg_opt.low_mem) - erts_free(ERTS_ALC_T_SBMBC_LOW, crr); - else -#endif - erts_free(ERTS_ALC_T_SBMBC, crr); -} - -static Block_t * create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) { Block_t *blk; @@ -2077,6 +3238,24 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz); +#ifdef ERTS_SMP + allctr->cpool.disable_abandon = ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON; + + if ((flags & (CFLG_MBC|CFLG_NO_CPOOL)) == CFLG_MBC + && ERTS_ALC_IS_CPOOL_ENABLED(allctr) + && erts_thr_progress_is_managed_thread()) { + crr = cpool_fetch(allctr, blk_sz); + if (crr) { + STAT_MBC_CPOOL_FETCH(allctr, crr); + link_carrier(&allctr->mbc_list, crr); + (*allctr->add_mbc)(allctr, crr); + blk = (*allctr->get_free_block)(allctr, blk_sz, NULL, 0); + ASSERT(blk); + return blk; + } + } +#endif + #if HAVE_ERTS_MSEG if (flags & CFLG_FORCE_SYS_ALLOC) @@ -2204,11 +3383,15 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) allctr->main_carrier = crr; } +#ifdef ERTS_SMP + cpool_init_carrier_data(allctr, crr); +#endif + link_carrier(&allctr->mbc_list, crr); CHECK_1BLK_CARRIER(allctr, 0, is_mseg, crr, crr_sz, blk, blk_sz); if (allctr->creating_mbc) - (*allctr->creating_mbc)(allctr, crr, 0); + (*allctr->creating_mbc)(allctr, crr); } @@ -2352,14 +3535,21 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) } static void -destroy_carrier(Allctr_t *allctr, Block_t *blk) +dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, Uint mseg_flags) { - Uint crr_sz; - Carrier_t *crr; #if HAVE_ERTS_MSEG - Uint is_mseg = 0; - Uint mseg_flags = ERTS_MSEG_FLG_NONE; + if (IS_MSEG_CARRIER(crr)) + alcu_mseg_dealloc(allctr, crr, CARRIER_SZ(crr), mseg_flags); + else #endif + alcu_sys_free(allctr, crr); +} + +static void +destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) +{ + Uint crr_sz; + Carrier_t *crr; if (IS_SBC_BLK(blk)) { Uint blk_sz = SBC_BLK_SZ(blk); @@ -2372,7 +3562,6 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(crr)) { - is_mseg++; ASSERT(crr_sz % MSEG_UNIT_SZ == 0); STAT_MSEG_SBC_FREE(allctr, crr_sz, blk_sz); } @@ -2382,6 +3571,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) unlink_carrier(&allctr->sbc_list, crr); + dealloc_carrier(allctr, crr, ERTS_MSEG_FLG_NONE); } else { ASSERT(IS_MBC_FIRST_FBLK(allctr, blk)); @@ -2400,30 +3590,36 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) } #endif -#if HAVE_ERTS_MSEG - if (IS_MSEG_CARRIER(crr)) { - is_mseg++; - ASSERT(crr_sz % MSEG_UNIT_SZ == 0); - STAT_MSEG_MBC_FREE(allctr, crr_sz); - mseg_flags = ERTS_MSEG_FLG_2POW; + if (allctr->destroying_mbc) + (*allctr->destroying_mbc)(allctr, crr); + +#ifdef ERTS_SMP + if (busy_pcrr_pp && *busy_pcrr_pp) { + ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr); + *busy_pcrr_pp = NULL; + cpool_delete(allctr, allctr, crr); } else #endif - STAT_SYS_ALLOC_MBC_FREE(allctr, crr_sz); + { + unlink_carrier(&allctr->mbc_list, crr); +#if HAVE_ERTS_MSEG + if (IS_MSEG_CARRIER(crr)) { + ASSERT(crr_sz % MSEG_UNIT_SZ == 0); + STAT_MSEG_MBC_FREE(allctr, crr_sz); + } + else +#endif + STAT_SYS_ALLOC_MBC_FREE(allctr, crr_sz); + } - unlink_carrier(&allctr->mbc_list, crr); - if (allctr->destroying_mbc) - (*allctr->destroying_mbc)(allctr, crr, 0); +#ifdef ERTS_SMP + schedule_dealloc_carrier(allctr, crr); +#else + dealloc_carrier(allctr, crr, ERTS_MSEG_FLG_2POW); +#endif } - -#if HAVE_ERTS_MSEG - if (is_mseg) { - alcu_mseg_dealloc(allctr, crr, crr_sz, mseg_flags); - } - else -#endif - alcu_sys_free(allctr, crr); } @@ -2457,19 +3653,19 @@ static struct { Eterm lmbcs; Eterm smbcs; Eterm mbcgs; - Eterm sbmbcs; - Eterm sbmbct; + Eterm acul; #if HAVE_ERTS_MSEG Eterm mmc; #endif Eterm ycs; - /* Eterm sbmbcs; */ - Eterm fix_types; Eterm mbcs; +#ifdef ERTS_SMP + Eterm mbcs_pool; +#endif Eterm sbcs; Eterm sys_alloc_carriers_size; @@ -2494,8 +3690,6 @@ static struct { Eterm mseg_dealloc; Eterm mseg_realloc; #endif - Eterm sbmbc_alloc; - Eterm sbmbc_free; #ifdef DEBUG Eterm end_of_atoms; #endif @@ -2551,19 +3745,19 @@ init_atoms(Allctr_t *allctr) AM_INIT(lmbcs); AM_INIT(smbcs); AM_INIT(mbcgs); - AM_INIT(sbmbcs); - AM_INIT(sbmbct); + AM_INIT(acul); #if HAVE_ERTS_MSEG AM_INIT(mmc); #endif AM_INIT(ycs); - /*AM_INIT(sbmbcs);*/ - AM_INIT(fix_types); AM_INIT(mbcs); +#ifdef ERTS_SMP + AM_INIT(mbcs_pool); +#endif AM_INIT(sbcs); AM_INIT(sys_alloc_carriers_size); @@ -2588,8 +3782,6 @@ init_atoms(Allctr_t *allctr) AM_INIT(mseg_dealloc); AM_INIT(mseg_realloc); #endif - AM_INIT(sbmbc_free); - AM_INIT(sbmbc_alloc); #ifdef DEBUG for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) { @@ -2637,19 +3829,22 @@ ensure_atoms_initialized(Allctr_t *allctr) * that would fit a small when size check is done may need to be built * as a big when the actual build is performed. Caller is required to * HRelease after build. + * + * Note, bld_unstable_uint() should have been called bld_unstable_uword() + * but we do not want to rename it... */ static ERTS_INLINE Eterm -bld_unstable_uint(Uint **hpp, Uint *szp, Uint ui) +bld_unstable_uint(Uint **hpp, Uint *szp, UWord ui) { Eterm res = THE_NON_VALUE; if (szp) - *szp += BIG_UINT_HEAP_SIZE; + *szp += BIG_UWORD_HEAP_SIZE(~((UWord) 0)); if (hpp) { if (IS_USMALL(0, ui)) res = make_small(ui); else { - res = uint_to_big(ui, *hpp); - *hpp += BIG_UINT_HEAP_SIZE; + res = uword_to_big(ui, *hpp); + *hpp += BIG_UWORD_HEAP_SIZE(ui); } } return res; @@ -2675,8 +3870,24 @@ add_4tup(Uint **hpp, Uint *szp, Eterm *lp, bld_cons(hpp, szp, bld_tuple(hpp, szp, 4, el1, el2, el3, el4), *lp); } +static ERTS_INLINE void +add_fix_types(Allctr_t *allctr, int internal, Uint **hpp, Uint *szp, + Eterm *lp, Eterm fix) +{ + if (allctr->fix) { + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + add_2tup(hpp, szp, lp, am.fix_types, fix); + else if (internal) + add_3tup(hpp, szp, lp, + am.fix_types, + erts_bld_uword(hpp, szp, ~((UWord) 0)), + fix); + } +} + static Eterm sz_info_fix(Allctr_t *allctr, + int internal, int *print_to_p, void *print_to_arg, Uint **hpp, @@ -2684,36 +3895,67 @@ sz_info_fix(Allctr_t *allctr, { Eterm res; int ix; - ErtsAlcFixList_t *fix = allctr->fix; - ASSERT(fix); + ASSERT(allctr->fix); res = NIL; - for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) { - ErtsAlcType_t n = ix + ERTS_ALC_N_MIN_A_FIXED_SIZE; - Uint alloced = (fix[ix].type_size * fix[ix].allocated); - Uint used = fix[ix].type_size*fix[ix].used; - - if (print_to_p) { - int to = *print_to_p; - void *arg = print_to_arg; - erts_print(to, - arg, - "fix type: %s %bpu %bpu\n", - (char *) ERTS_ALC_N2TD(n), - alloced, - used); - } + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + + if (internal) { + for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) { + ErtsAlcFixList_t *fix = &allctr->fix[ix]; + UWord alloced = fix->type_size * fix->u.cpool.allocated; + UWord used = fix->type_size * fix->u.cpool.used; + + if (print_to_p) { + int to = *print_to_p; + void *arg = print_to_arg; + erts_print(to, + arg, + "fix type internal: %s %bpu %bpu\n", + (char *) ERTS_ALC_N2TD(ERTS_ALC_N_MIN_A_FIXED_SIZE + + ix), + alloced, + used); + } - if (hpp || szp) { - add_3tup(hpp, szp, &res, - fix_type_atoms[ix], - bld_unstable_uint(hpp, szp, alloced), - bld_unstable_uint(hpp, szp, used)); + if (hpp || szp) { + add_3tup(hpp, szp, &res, + fix_type_atoms[ix], + bld_unstable_uint(hpp, szp, alloced), + bld_unstable_uint(hpp, szp, used)); + } + } } } + else { + for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) { + ErtsAlcFixList_t *fix = &allctr->fix[ix]; + UWord alloced = fix->type_size * fix->u.nocpool.allocated; + UWord used = fix->type_size*fix->u.nocpool.used; + + if (print_to_p) { + int to = *print_to_p; + void *arg = print_to_arg; + erts_print(to, + arg, + "fix type: %s %bpu %bpu\n", + (char *) ERTS_ALC_N2TD(ERTS_ALC_N_MIN_A_FIXED_SIZE + + ix), + alloced, + used); + } + + if (hpp || szp) { + add_3tup(hpp, szp, &res, + fix_type_atoms[ix], + bld_unstable_uint(hpp, szp, alloced), + bld_unstable_uint(hpp, szp, used)); + } + } + } return res; } @@ -2727,9 +3969,7 @@ sz_info_carriers(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; - Uint curr_size = (cs == &allctr->sbmbcs - ? cs->curr.small_block.size - : cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size); + UWord curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; if (print_to_p) { int to = *print_to_p; @@ -2743,7 +3983,7 @@ sz_info_carriers(Allctr_t *allctr, cs->blocks.max_ever.size); erts_print(to, arg, - "%scarriers size: %beu %bpu %bpu\n", + "%scarriers size: %bpu %bpu %bpu\n", prefix, curr_size, cs->max.size, @@ -2767,6 +4007,62 @@ sz_info_carriers(Allctr_t *allctr, return res; } +#ifdef ERTS_SMP + +static Eterm +info_cpool(Allctr_t *allctr, + int sz_only, + char *prefix, + int *print_to_p, + void *print_to_arg, + Uint **hpp, + Uint *szp) +{ + Eterm res = THE_NON_VALUE; + UWord noc, csz, nob, bsz; + + noc = csz = nob = bsz = ~0; + if (print_to_p || hpp) { + if (sz_only) + cpool_read_stat(allctr, NULL, &csz, NULL, &bsz); + else + cpool_read_stat(allctr, &noc, &csz, &nob, &bsz); + } + + if (print_to_p) { + int to = *print_to_p; + void *arg = print_to_arg; + if (!sz_only) + erts_print(to, arg, "%sblocks: %bpu\n", prefix, nob); + erts_print(to, arg, "%sblocks size: %bpu\n", prefix, bsz); + if (!sz_only) + erts_print(to, arg, "%scarriers: %bpu\n", prefix, noc); + erts_print(to, arg, "%scarriers size: %bpu\n", prefix, csz); + } + + if (hpp || szp) { + res = NIL; + add_2tup(hpp, szp, &res, + am.carriers_size, + bld_unstable_uint(hpp, szp, csz)); + if (!sz_only) + add_2tup(hpp, szp, &res, + am.carriers, + bld_unstable_uint(hpp, szp, noc)); + add_2tup(hpp, szp, &res, + am.blocks_size, + bld_unstable_uint(hpp, szp, bsz)); + if (!sz_only) + add_2tup(hpp, szp, &res, + am.blocks, + bld_unstable_uint(hpp, szp, nob)); + } + + return res; +} + +#endif /* ERTS_SMP */ + static Eterm info_carriers(Allctr_t *allctr, CarriersStats_t *cs, @@ -2777,17 +4073,10 @@ info_carriers(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; - Uint curr_no, curr_size; - int small_block = cs == &allctr->sbmbcs; - - if (small_block) { - curr_no = cs->curr.small_block.no; - curr_size = cs->curr.small_block.size; - } - else { - curr_no = cs->curr.norm.mseg.no + cs->curr.norm.sys_alloc.no; - curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; - } + UWord curr_no, curr_size; + + curr_no = cs->curr.norm.mseg.no + cs->curr.norm.sys_alloc.no; + curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; if (print_to_p) { int to = *print_to_p; @@ -2808,75 +4097,67 @@ info_carriers(Allctr_t *allctr, cs->blocks.max_ever.size); erts_print(to, arg, - "%scarriers: %beu %bpu %bpu\n", + "%scarriers: %bpu %bpu %bpu\n", prefix, curr_no, cs->max.no, cs->max_ever.no); - if (!small_block) { #if HAVE_ERTS_MSEG - erts_print(to, - arg, - "%smseg carriers: %bpu\n", - prefix, - cs->curr.norm.mseg.no); -#endif - erts_print(to, - arg, - "%ssys_alloc carriers: %bpu\n", - prefix, - cs->curr.norm.sys_alloc.no); - } erts_print(to, arg, - "%scarriers size: %beu %bpu %bpu\n", + "%smseg carriers: %bpu\n", + prefix, + cs->curr.norm.mseg.no); +#endif + erts_print(to, + arg, + "%ssys_alloc carriers: %bpu\n", + prefix, + cs->curr.norm.sys_alloc.no); + erts_print(to, + arg, + "%scarriers size: %bpu %bpu %bpu\n", prefix, curr_size, cs->max.size, cs->max_ever.size); - if (!small_block) { #if HAVE_ERTS_MSEG - erts_print(to, - arg, - "%smseg carriers size: %bpu\n", - prefix, - cs->curr.norm.mseg.size); -#endif - erts_print(to, - arg, - "%ssys_alloc carriers size: %bpu\n", - prefix, - cs->curr.norm.sys_alloc.size); - } + erts_print(to, + arg, + "%smseg carriers size: %bpu\n", + prefix, + cs->curr.norm.mseg.size); +#endif + erts_print(to, + arg, + "%ssys_alloc carriers size: %bpu\n", + prefix, + cs->curr.norm.sys_alloc.size); } if (hpp || szp) { res = NIL; - if (!small_block) { - add_2tup(hpp, szp, &res, - am.sys_alloc_carriers_size, - bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.size)); + add_2tup(hpp, szp, &res, + am.sys_alloc_carriers_size, + bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.size)); #if HAVE_ERTS_MSEG - add_2tup(hpp, szp, &res, - am.mseg_alloc_carriers_size, - bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.size)); + add_2tup(hpp, szp, &res, + am.mseg_alloc_carriers_size, + bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.size)); #endif - } add_4tup(hpp, szp, &res, am.carriers_size, bld_unstable_uint(hpp, szp, curr_size), bld_unstable_uint(hpp, szp, cs->max.size), bld_unstable_uint(hpp, szp, cs->max_ever.size)); - if (!small_block) { - add_2tup(hpp, szp, &res, - am.sys_alloc_carriers, - bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.no)); + add_2tup(hpp, szp, &res, + am.sys_alloc_carriers, + bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.no)); #if HAVE_ERTS_MSEG - add_2tup(hpp, szp, &res, - am.mseg_alloc_carriers, - bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.no)); + add_2tup(hpp, szp, &res, + am.mseg_alloc_carriers, + bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.no)); #endif - } add_4tup(hpp, szp, &res, am.carriers, bld_unstable_uint(hpp, szp, curr_no), @@ -2935,16 +4216,10 @@ info_calls(Allctr_t *allctr, if (print_to_p) { #define PRINT_CC_4(TO, TOA, NAME, CC) \ - if ((CC).giga_no == 0) \ - erts_print(TO, TOA, "%s calls: %b32u\n", NAME, CC.no); \ - else \ - erts_print(TO, TOA, "%s calls: %b32u%09lu\n", NAME, CC.giga_no, CC.no) + erts_print(TO, TOA, "%s calls: %b64u\n", NAME, CC) #define PRINT_CC_5(TO, TOA, PRFX, NAME, CC) \ - if ((CC).giga_no == 0) \ - erts_print(TO, TOA, "%s%s calls: %b32u\n",PRFX,NAME,CC.no); \ - else \ - erts_print(TO, TOA, "%s%s calls: %b32u%09lu\n",PRFX,NAME,CC.giga_no,CC.no) + erts_print(TO, TOA, "%s%s calls: %b64u\n",PRFX,NAME,CC) char *prefix = allctr->name_prefix; int to = *print_to_p; @@ -2954,9 +4229,6 @@ info_calls(Allctr_t *allctr, PRINT_CC_5(to, arg, prefix, "free", allctr->calls.this_free); PRINT_CC_5(to, arg, prefix, "realloc", allctr->calls.this_realloc); - PRINT_CC_4(to, arg, "sbmbc_alloc", allctr->calls.sbmbc_alloc); - PRINT_CC_4(to, arg, "sbmbc_free", allctr->calls.sbmbc_free); - #if HAVE_ERTS_MSEG PRINT_CC_4(to, arg, "mseg_alloc", allctr->calls.mseg_alloc); PRINT_CC_4(to, arg, "mseg_dealloc", allctr->calls.mseg_dealloc); @@ -2983,50 +4255,42 @@ info_calls(Allctr_t *allctr, add_3tup(hpp, szp, &res, am.sys_realloc, - bld_unstable_uint(hpp, szp, allctr->calls.sys_realloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.sys_realloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.sys_realloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.sys_realloc))); add_3tup(hpp, szp, &res, am.sys_free, - bld_unstable_uint(hpp, szp, allctr->calls.sys_free.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.sys_free.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.sys_free)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.sys_free))); add_3tup(hpp, szp, &res, am.sys_alloc, - bld_unstable_uint(hpp, szp, allctr->calls.sys_alloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.sys_alloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.sys_alloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.sys_alloc))); #if HAVE_ERTS_MSEG add_3tup(hpp, szp, &res, am.mseg_realloc, - bld_unstable_uint(hpp, szp, allctr->calls.mseg_realloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.mseg_realloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.mseg_realloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.mseg_realloc))); add_3tup(hpp, szp, &res, am.mseg_dealloc, - bld_unstable_uint(hpp, szp, allctr->calls.mseg_dealloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.mseg_dealloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.mseg_dealloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.mseg_dealloc))); add_3tup(hpp, szp, &res, am.mseg_alloc, - bld_unstable_uint(hpp, szp, allctr->calls.mseg_alloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.mseg_alloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.mseg_alloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.mseg_alloc))); #endif add_3tup(hpp, szp, &res, - am.sbmbc_free, - bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_free.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_free.no)); - add_3tup(hpp, szp, &res, - am.sbmbc_alloc, - bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_alloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_alloc.no)); - add_3tup(hpp, szp, &res, allctr->name.realloc, - bld_unstable_uint(hpp, szp, allctr->calls.this_realloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.this_realloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.this_realloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.this_realloc))); add_3tup(hpp, szp, &res, allctr->name.free, - bld_unstable_uint(hpp, szp, allctr->calls.this_free.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.this_free.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.this_free)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.this_free))); add_3tup(hpp, szp, &res, allctr->name.alloc, - bld_unstable_uint(hpp, szp, allctr->calls.this_alloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.this_alloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.this_alloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.this_alloc))); } return res; @@ -3040,6 +4304,7 @@ info_options(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; + int acul; if (!allctr) { if (print_to_p) @@ -3051,6 +4316,12 @@ info_options(Allctr_t *allctr, return res; } +#ifdef ERTS_SMP + acul = allctr->cpool.util_limit; +#else + acul = 0; +#endif + if (print_to_p) { char topt[21]; /* Enough for any 64-bit integer */ if (allctr->t) @@ -3079,9 +4350,8 @@ info_options(Allctr_t *allctr, #endif "option lmbcs: %beu\n" "option smbcs: %beu\n" - "option mbcgs: %beu\n" - "option sbmbcs: %beu\n" - "option sbmbct: %beu\n", + "option mbcgs: %beu\n", + "option acul: %d\n", topt, allctr->ramv ? "true" : "false", #if HALFWORD_HEAP @@ -3102,8 +4372,7 @@ info_options(Allctr_t *allctr, allctr->largest_mbc_size, allctr->smallest_mbc_size, allctr->mbc_growth_stages, - allctr->sbmbc_size, - allctr->sbmbc_threshold); + acul); } res = (*allctr->info_options)(allctr, "option ", print_to_p, print_to_arg, @@ -3111,11 +4380,8 @@ info_options(Allctr_t *allctr, if (hpp || szp) { add_2tup(hpp, szp, &res, - am.sbmbct, - bld_uint(hpp, szp, allctr->sbmbc_threshold)); - add_2tup(hpp, szp, &res, - am.sbmbcs, - bld_uint(hpp, szp, allctr->sbmbc_size)); + am.acul, + bld_uint(hpp, szp, (UWord) acul)); add_2tup(hpp, szp, &res, am.mbcgs, bld_uint(hpp, szp, allctr->mbc_growth_stages)); @@ -3261,13 +4527,17 @@ erts_alcu_info_options(Allctr_t *allctr, Eterm erts_alcu_sz_info(Allctr_t *allctr, + int internal, int begin_max_period, int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) { - Eterm res, sbmbcs, mbcs, sbcs, fix = THE_NON_VALUE; + Eterm res, mbcs, sbcs, fix = THE_NON_VALUE; +#ifdef ERTS_SMP + Eterm mbcs_pool; +#endif res = THE_NON_VALUE; @@ -3296,30 +4566,35 @@ erts_alcu_sz_info(Allctr_t *allctr, = allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no; allctr->sbcs.blocks.max.no = allctr->sbcs.max.no; - update_max_ever_values(&allctr->sbmbcs); update_max_ever_values(&allctr->mbcs); update_max_ever_values(&allctr->sbcs); if (allctr->fix) - fix = sz_info_fix(allctr, print_to_p, print_to_arg, hpp, szp); - sbmbcs = sz_info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p, - print_to_arg, hpp, szp); + fix = sz_info_fix(allctr, internal, print_to_p, print_to_arg, hpp, szp); mbcs = sz_info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, print_to_arg, hpp, szp); +#ifdef ERTS_SMP + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + mbcs_pool = info_cpool(allctr, 1, "mbcs_pool ", print_to_p, + print_to_arg, hpp, szp); + else + mbcs_pool = THE_NON_VALUE; /* shut up annoying warning... */ +#endif sbcs = sz_info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p, print_to_arg, hpp, szp); if (hpp || szp) { res = NIL; add_2tup(hpp, szp, &res, am.sbcs, sbcs); +#ifdef ERTS_SMP + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + add_2tup(hpp, szp, &res, am.mbcs_pool, mbcs_pool); +#endif add_2tup(hpp, szp, &res, am.mbcs, mbcs); - add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs); - if (allctr->fix) - add_2tup(hpp, szp, &res, am.fix_types, fix); + add_fix_types(allctr, internal, hpp, szp, &res, fix); } if (begin_max_period) { - reset_max_values(&allctr->sbmbcs); reset_max_values(&allctr->mbcs); reset_max_values(&allctr->sbcs); } @@ -3338,13 +4613,17 @@ erts_alcu_sz_info(Allctr_t *allctr, Eterm erts_alcu_info(Allctr_t *allctr, + int internal, int begin_max_period, int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) { - Eterm res, sett, sbmbcs, mbcs, sbcs, calls, fix = THE_NON_VALUE; + Eterm res, sett, mbcs, sbcs, calls, fix = THE_NON_VALUE; +#ifdef ERTS_SMP + Eterm mbcs_pool; +#endif res = THE_NON_VALUE; @@ -3373,7 +4652,6 @@ erts_alcu_info(Allctr_t *allctr, = allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no; allctr->sbcs.blocks.max.no = allctr->sbcs.max.no; - update_max_ever_values(&allctr->sbmbcs); update_max_ever_values(&allctr->mbcs); update_max_ever_values(&allctr->sbcs); @@ -3387,11 +4665,16 @@ erts_alcu_info(Allctr_t *allctr, sett = info_options(allctr, print_to_p, print_to_arg, hpp, szp); if (allctr->fix) - fix = sz_info_fix(allctr, print_to_p, print_to_arg, hpp, szp); - sbmbcs = info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p, - print_to_arg, hpp, szp); + fix = sz_info_fix(allctr, internal, print_to_p, print_to_arg, hpp, szp); mbcs = info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, print_to_arg, hpp, szp); +#ifdef ERTS_SMP + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + mbcs_pool = info_cpool(allctr, 0, "mbcs_pool ", print_to_p, + print_to_arg, hpp, szp); + else + mbcs_pool = THE_NON_VALUE; /* shut up annoying warning... */ +#endif sbcs = info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p, print_to_arg, hpp, szp); calls = info_calls(allctr, print_to_p, print_to_arg, hpp, szp); @@ -3401,10 +4684,12 @@ erts_alcu_info(Allctr_t *allctr, add_2tup(hpp, szp, &res, am.calls, calls); add_2tup(hpp, szp, &res, am.sbcs, sbcs); +#ifdef ERTS_SMP + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + add_2tup(hpp, szp, &res, am.mbcs_pool, mbcs_pool); +#endif add_2tup(hpp, szp, &res, am.mbcs, mbcs); - add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs); - if (allctr->fix) - add_2tup(hpp, szp, &res, am.fix_types, fix); + add_fix_types(allctr, internal, hpp, szp, &res, fix); add_2tup(hpp, szp, &res, am.options, sett); add_3tup(hpp, szp, &res, am.versions, @@ -3413,7 +4698,6 @@ erts_alcu_info(Allctr_t *allctr, } if (begin_max_period) { - reset_max_values(&allctr->sbmbcs); reset_max_values(&allctr->mbcs); reset_max_values(&allctr->sbcs); } @@ -3441,22 +4725,37 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t * size->carriers = allctr->mbcs.curr.norm.mseg.size; size->carriers += allctr->mbcs.curr.norm.sys_alloc.size; - size->carriers += allctr->sbmbcs.curr.small_block.size; size->carriers += allctr->sbcs.curr.norm.mseg.size; size->carriers += allctr->sbcs.curr.norm.sys_alloc.size; size->blocks = allctr->mbcs.blocks.curr.size; - size->blocks += allctr->sbmbcs.blocks.curr.size; size->blocks += allctr->sbcs.blocks.curr.size; +#ifdef ERTS_SMP + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + UWord csz, bsz; + cpool_read_stat(allctr, NULL, &csz, NULL, &bsz); + size->blocks += bsz; + size->carriers += csz; + } +#endif + if (fi) { int ix; for (ix = 0; ix < fisz; ix++) { if (allctr->fix) { - fi[ix].allocated += (allctr->fix[ix].type_size - * allctr->fix[ix].allocated); - fi[ix].used += (allctr->fix[ix].type_size - * allctr->fix[ix].used); + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + fi[ix].allocated += (allctr->fix[ix].type_size + * allctr->fix[ix].u.cpool.allocated); + fi[ix].used += (allctr->fix[ix].type_size + * allctr->fix[ix].u.cpool.used); + } + else { + fi[ix].allocated += (allctr->fix[ix].type_size + * allctr->fix[ix].u.nocpool.allocated); + fi[ix].used += (allctr->fix[ix].type_size + * allctr->fix[ix].u.nocpool.used); + } } } } @@ -3474,7 +4773,6 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) { Allctr_t *allctr = (Allctr_t *) extra; void *res; - ErtsAlcFixList_t *fix; ASSERT(initialized); @@ -3492,57 +4790,21 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) INC_CC(allctr->calls.this_alloc); - fix = allctr->fix; - if (fix) { - int ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; - ASSERT((unsigned)ix < ERTS_ALC_NO_FIXED_SIZES); - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); - ASSERT(size <= fix[ix].type_size); - fix[ix].used++; - res = fix[ix].list; - if (res) { - fix[ix].list_size--; - fix[ix].list = *((void **) res); - if (fix[ix].list && fix[ix].allocated > fix[ix].limit) { - void *p = fix[ix].list; - Block_t *blk; - fix[ix].list = *((void **) p); - fix[ix].list_size--; - blk = UMEM2BLK(p); - if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); - else - mbc_free(allctr, p); - fix[ix].allocated--; - } - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); - return res; - } - size = fix[ix].type_size; - if (fix[ix].limit < fix[ix].used) - fix[ix].limit = fix[ix].used; - if (fix[ix].max_used < fix[ix].used) - fix[ix].max_used = fix[ix].used; - fix[ix].allocated++; + if (allctr->fix) { + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + return fix_cpool_alloc(allctr, type, size); + else + return fix_nocpool_alloc(allctr, type, size); } if (size >= allctr->sbc_threshold) { Block_t *blk; -#ifdef ERTS_SMP - if (allctr->dd.use) - ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); -#endif blk = create_carrier(allctr, size, CFLG_SBC); res = blk ? BLK2UMEM(blk) : NULL; } else res = mbc_alloc(allctr, size); - if (!res && fix) { - int ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; - fix[ix].allocated--; - fix[ix].used--; - } return res; } @@ -3611,9 +4873,22 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size) if (pref_allctr->thread_safe) erts_mtx_lock(&pref_allctr->mutex); +#ifdef ERTS_SMP + ASSERT(pref_allctr->dd.use); + ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1); +#endif + ERTS_ALCU_DBG_CHK_THR_ACCESS(pref_allctr); res = do_erts_alcu_alloc(type, pref_allctr, size); + +#ifdef ERTS_SMP + if (!res && ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1)) { + /* Cleaned up a bit more; try one more time... */ + res = do_erts_alcu_alloc(type, pref_allctr, size); + } +#endif + if (pref_allctr->thread_safe) erts_mtx_unlock(&pref_allctr->mutex); @@ -3630,9 +4905,9 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size) /* ------------------------------------------------------------------------- */ static ERTS_INLINE void -do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p) +do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p, + Carrier_t **busy_pcrr_pp) { - int ix; Allctr_t *allctr = (Allctr_t *) extra; ASSERT(initialized); @@ -3644,57 +4919,28 @@ do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p) ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); if (p) { - ErtsAlcFixList_t *fix = allctr->fix; - Block_t *blk; INC_CC(allctr->calls.this_free); - if (fix) { - ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); - fix[ix].used--; - if (fix[ix].allocated < fix[ix].limit - && fix[ix].list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { - *((void **) p) = fix[ix].list; - fix[ix].list = p; - fix[ix].list_size++; - if (!allctr->fix_shrink_scheduled) { - allctr->fix_shrink_scheduled = 1; - erts_set_aux_work_timeout( - allctr->ix, - (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM - | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), - 1); - } - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); - return; - } - fix[ix].allocated--; - if (fix[ix].list && fix[ix].allocated > fix[ix].limit) { - blk = UMEM2BLK(p); - if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); - else - mbc_free(allctr, p); - p = fix[ix].list; - fix[ix].list = *((void **) p); - fix[ix].list_size--; - fix[ix].allocated--; - } + if (allctr->fix) { + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + fix_cpool_free(allctr, type, p, busy_pcrr_pp); + else + fix_nocpool_free(allctr, type, p); + } + else { + Block_t *blk = UMEM2BLK(p); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk, NULL); + else + mbc_free(allctr, p, busy_pcrr_pp); } - - blk = UMEM2BLK(p); - if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); - else - mbc_free(allctr, p); - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); } } void erts_alcu_free(ErtsAlcType_t type, void *extra, void *p) { - do_erts_alcu_free(type, extra, p); + do_erts_alcu_free(type, extra, p, NULL); } #ifdef USE_THREADS @@ -3704,7 +4950,7 @@ erts_alcu_free_ts(ErtsAlcType_t type, void *extra, void *p) { Allctr_t *allctr = (Allctr_t *) extra; erts_mtx_lock(&allctr->mutex); - do_erts_alcu_free(type, extra, p); + do_erts_alcu_free(type, extra, p, NULL); erts_mtx_unlock(&allctr->mutex); } @@ -3726,7 +4972,7 @@ erts_alcu_free_thr_spec(ErtsAlcType_t type, void *extra, void *p) if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); - do_erts_alcu_free(type, allctr, p); + do_erts_alcu_free(type, allctr, p, NULL); if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); @@ -3736,10 +4982,12 @@ void erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p) { if (p) { + Carrier_t *busy_pcrr_p; Allctr_t *pref_allctr, *used_allctr; pref_allctr = get_pref_allctr(extra); - used_allctr = get_used_allctr(extra, p, NULL); + used_allctr = get_used_allctr(pref_allctr, ERTS_ALC_TS_PREF_LOCK_IF_USED, + p, NULL, &busy_pcrr_p); if (pref_allctr != used_allctr) enqueue_dealloc_other_instance(type, used_allctr, @@ -3747,12 +4995,11 @@ erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p) (used_allctr->dd.ix - pref_allctr->dd.ix)); else { - if (used_allctr->thread_safe) - erts_mtx_lock(&used_allctr->mutex); ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr); - do_erts_alcu_free(type, used_allctr, p); - if (used_allctr->thread_safe) - erts_mtx_unlock(&used_allctr->mutex); + do_erts_alcu_free(type, used_allctr, p, &busy_pcrr_p); + clear_busy_pool_carrier(used_allctr, busy_pcrr_p); + if (pref_allctr->thread_safe) + erts_mtx_unlock(&pref_allctr->mutex); } } } @@ -3768,7 +5015,8 @@ do_erts_alcu_realloc(ErtsAlcType_t type, void *extra, void *p, Uint size, - Uint32 alcu_flgs) + Uint32 alcu_flgs, + Carrier_t **busy_pcrr_pp) { Allctr_t *allctr = (Allctr_t *) extra; Block_t *blk; @@ -3793,7 +5041,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, #if ALLOC_ZERO_EQ_NULL if (!size) { ASSERT(p); - do_erts_alcu_free(type, extra, p); + do_erts_alcu_free(type, extra, p, busy_pcrr_pp); INC_CC(allctr->calls.this_realloc); DEC_CC(allctr->calls.this_free); return NULL; @@ -3804,32 +5052,9 @@ do_erts_alcu_realloc(ErtsAlcType_t type, blk = UMEM2BLK(p); - if (allctr->sbmbc_threshold > 0) { - Uint old_sz, new_sz, lim; - lim = allctr->sbmbc_threshold; - old_sz = BLK_SZ(blk); - new_sz = UMEMSZ2BLKSZ(allctr, size); - if ((old_sz < lim && lim <= new_sz) - || (new_sz < lim && lim <= old_sz)) { - /* *Need* to move it... */ - - INC_CC(allctr->calls.this_realloc); - res = do_erts_alcu_alloc(type, extra, size); - DEC_CC(allctr->calls.this_alloc); - - sys_memcpy(res, p, MIN(size, old_sz - ABLK_HDR_SZ)); - - do_erts_alcu_free(type, extra, p); - DEC_CC(allctr->calls.this_free); - return res; - } - if (old_sz < lim) - alcu_flgs |= ERTS_ALCU_FLG_SBMBC; - } - if (size < allctr->sbc_threshold) { if (IS_MBC_BLK(blk)) - res = mbc_realloc(allctr, p, size, alcu_flgs); + res = mbc_realloc(allctr, p, size, alcu_flgs, busy_pcrr_pp); else { Uint used_sz = SBC_HEADER_SIZE + ABLK_HDR_SZ + size; Uint crr_sz; @@ -3868,16 +5093,12 @@ do_erts_alcu_realloc(ErtsAlcType_t type, sys_memcpy((void*) res, (void*) p, MIN(SBC_BLK_SZ(blk) - ABLK_HDR_SZ, size)); - destroy_carrier(allctr, blk); + destroy_carrier(allctr, blk, NULL); } } } else { Block_t *new_blk; -#ifdef ERTS_SMP - if (allctr->dd.use) - ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); -#endif if(IS_SBC_BLK(blk)) { do_carrier_resize: #if HALFWORD_HEAP @@ -3896,7 +5117,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, sys_memcpy((void *) res, (void *) p, MIN(MBC_ABLK_SZ(blk) - ABLK_HDR_SZ, size)); - mbc_free(allctr, p); + mbc_free(allctr, p, busy_pcrr_pp); } else res = NULL; @@ -3910,7 +5131,7 @@ void * erts_alcu_realloc(ErtsAlcType_t type, void *extra, void *p, Uint size) { void *res; - res = do_erts_alcu_realloc(type, extra, p, size, 0); + res = do_erts_alcu_realloc(type, extra, p, size, 0, NULL); DEBUG_CHECK_ALIGNMENT(res); return res; } @@ -3931,7 +5152,7 @@ erts_alcu_realloc_mv(ErtsAlcType_t type, void *extra, void *p, Uint size) if (cpy_size > size) cpy_size = size; sys_memcpy(res, p, cpy_size); - do_erts_alcu_free(type, extra, p); + do_erts_alcu_free(type, extra, p, NULL); } DEBUG_CHECK_ALIGNMENT(res); return res; @@ -3946,7 +5167,7 @@ erts_alcu_realloc_ts(ErtsAlcType_t type, void *extra, void *ptr, Uint size) Allctr_t *allctr = (Allctr_t *) extra; void *res; erts_mtx_lock(&allctr->mutex); - res = do_erts_alcu_realloc(type, extra, ptr, size, 0); + res = do_erts_alcu_realloc(type, extra, ptr, size, 0, NULL); erts_mtx_unlock(&allctr->mutex); DEBUG_CHECK_ALIGNMENT(res); return res; @@ -3970,7 +5191,7 @@ erts_alcu_realloc_mv_ts(ErtsAlcType_t type, void *extra, void *p, Uint size) if (cpy_size > size) cpy_size = size; sys_memcpy(res, p, cpy_size); - do_erts_alcu_free(type, extra, p); + do_erts_alcu_free(type, extra, p, NULL); } erts_mtx_unlock(&allctr->mutex); DEBUG_CHECK_ALIGNMENT(res); @@ -3997,7 +5218,7 @@ erts_alcu_realloc_thr_spec(ErtsAlcType_t type, void *extra, if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); - res = do_erts_alcu_realloc(type, allctr, ptr, size, 0); + res = do_erts_alcu_realloc(type, allctr, ptr, size, 0, NULL); if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); @@ -4040,7 +5261,7 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra, if (cpy_size > size) cpy_size = size; sys_memcpy(res, ptr, cpy_size); - do_erts_alcu_free(type, allctr, ptr); + do_erts_alcu_free(type, allctr, ptr, NULL); if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); } @@ -4057,40 +5278,64 @@ realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size, void *res; Allctr_t *pref_allctr, *used_allctr; UWord old_user_size; + Carrier_t *busy_pcrr_p; +#ifdef ERTS_SMP + int retried; +#endif if (!p) return erts_alcu_alloc_thr_pref(type, extra, size); pref_allctr = get_pref_allctr(extra); - used_allctr = get_used_allctr(extra, p, &old_user_size); + + if (pref_allctr->thread_safe) + erts_mtx_lock(&pref_allctr->mutex); + +#ifdef ERTS_SMP + ASSERT(pref_allctr->dd.use); + ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1); + retried = 0; +restart: +#endif + + used_allctr = get_used_allctr(pref_allctr, ERTS_ALC_TS_PREF_LOCK_NO, + p, &old_user_size, &busy_pcrr_p); ASSERT(used_allctr && pref_allctr); if (!force_move && used_allctr == pref_allctr) { - if (used_allctr->thread_safe) - erts_mtx_lock(&used_allctr->mutex); ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr); res = do_erts_alcu_realloc(type, used_allctr, p, size, - 0); - if (used_allctr->thread_safe) - erts_mtx_unlock(&used_allctr->mutex); + 0, + &busy_pcrr_p); + clear_busy_pool_carrier(used_allctr, busy_pcrr_p); +#ifdef ERTS_SMP + if (!res && !retried && ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1)) { + /* Cleaned up a bit more; try one more time... */ + retried = 1; + goto restart; + } +#endif + if (pref_allctr->thread_safe) + erts_mtx_unlock(&pref_allctr->mutex); } else { - if (pref_allctr->thread_safe) - erts_mtx_lock(&pref_allctr->mutex); res = do_erts_alcu_alloc(type, pref_allctr, size); - if (pref_allctr->thread_safe && used_allctr != pref_allctr) { - erts_mtx_unlock(&pref_allctr->mutex); - } - if (res) { - DEBUG_CHECK_ALIGNMENT(res); + if (!res) + goto unlock_ts_return; + else { - sys_memcpy(res, p, MIN(size,old_user_size)); + DEBUG_CHECK_ALIGNMENT(res); if (used_allctr != pref_allctr) { + if (pref_allctr->thread_safe) + erts_mtx_unlock(&pref_allctr->mutex); + + sys_memcpy(res, p, MIN(size, old_user_size)); + enqueue_dealloc_other_instance(type, used_allctr, p, @@ -4098,8 +5343,14 @@ realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size, - pref_allctr->dd.ix)); } else { - do_erts_alcu_free(type, used_allctr, p); + + sys_memcpy(res, p, MIN(size, old_user_size)); + + do_erts_alcu_free(type, used_allctr, p, &busy_pcrr_p); ASSERT(pref_allctr == used_allctr); + clear_busy_pool_carrier(used_allctr, busy_pcrr_p); + + unlock_ts_return: if (pref_allctr->thread_safe) erts_mtx_unlock(&pref_allctr->mutex); } @@ -4213,6 +5464,17 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (sz > allctr->min_block_size) allctr->min_block_size = sz; } + + allctr->cpool.dc_list.first = NULL; + allctr->cpool.dc_list.last = NULL; + allctr->cpool.abandon_limit = 0; + allctr->cpool.disable_abandon = 0; + erts_atomic_init_nob(&allctr->cpool.stat.blocks_size, 0); + erts_atomic_init_nob(&allctr->cpool.stat.no_blocks, 0); + erts_atomic_init_nob(&allctr->cpool.stat.carriers_size, 0); + erts_atomic_init_nob(&allctr->cpool.stat.no_carriers, 0); + allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT; + allctr->cpool.util_limit = init->acul; #endif allctr->sbc_threshold = init->sbct; @@ -4232,27 +5494,6 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) } #endif - - - allctr->sbmbc_threshold = init->sbmbct; - - if (!erts_have_sbmbc_alloc - || ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no)) - allctr->sbmbc_threshold = 0; - - if (!allctr->sbmbc_threshold) - allctr->sbmbc_size = 0; - else { - Uint min_size; - allctr->sbmbc_size = init->sbmbcs; - min_size = allctr->sbmbc_threshold; - min_size += allctr->min_block_size; - min_size += MBC_HEADER_SIZE(allctr); - if (allctr->sbmbc_size < min_size) - allctr->sbmbc_size = min_size; - } - - #if HAVE_ERTS_MSEG if (allctr->mseg_opt.abs_shrink_th > ~((UWord) 0) / 100) allctr->mseg_opt.abs_shrink_th = ~((UWord) 0) / 100; @@ -4264,16 +5505,12 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #ifdef ERTS_ENABLE_LOCK_COUNT erts_mtx_init_x_opt(&allctr->mutex, - ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no) - ? "sbmbc_alloc" - : "alcu_allocator", + "alcu_allocator", make_small(allctr->alloc_no), ERTS_LCNT_LT_ALLOC); #else erts_mtx_init_x(&allctr->mutex, - ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no) - ? "sbmbc_alloc" - : "alcu_allocator", + "alcu_allocator", make_small(allctr->alloc_no)); #endif /*ERTS_ENABLE_LOCK_COUNT*/ @@ -4292,6 +5529,8 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (!allctr->get_next_mbc_size) allctr->get_next_mbc_size = get_next_mbc_size; + if (allctr->mbc_header_size < sizeof(Carrier_t)) + goto error; #ifdef ERTS_SMP allctr->dd.use = 0; if (init->tpref) { @@ -4300,6 +5539,9 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->dd.ix = init->ix; } #endif + allctr->mbc_header_size = (UNIT_CEILING(allctr->mbc_header_size + + ABLK_HDR_SZ) + - ABLK_HDR_SZ); if (allctr->main_carrier_size) { Block_t *blk; @@ -4308,6 +5550,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->main_carrier_size, CFLG_MBC | CFLG_FORCE_SIZE + | CFLG_NO_CPOOL #if !HALFWORD_HEAP && !HAVE_SUPER_ALIGNED_MB_CARRIERS | CFLG_FORCE_SYS_ALLOC #endif @@ -4315,7 +5558,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (!blk) goto error; - (*allctr->link_free_block)(allctr, blk, 0); + (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); @@ -4326,16 +5569,24 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->fix = init->fix; allctr->fix_shrink_scheduled = 0; for (i = 0; i < ERTS_ALC_NO_FIXED_SIZES; i++) { - allctr->fix[i].max_used = 0; - allctr->fix[i].limit = 0; allctr->fix[i].type_size = init->fix_type_size[i]; allctr->fix[i].list_size = 0; allctr->fix[i].list = NULL; - allctr->fix[i].allocated = 0; - allctr->fix[i].used = 0; #ifdef ERTS_SMP ASSERT(allctr->fix[i].type_size >= sizeof(ErtsAllctrFixDDBlock_t)); #endif + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + allctr->fix[i].u.cpool.min_list_size = 0; + allctr->fix[i].u.cpool.shrink_list = 0; + allctr->fix[i].u.cpool.allocated = 0; + allctr->fix[i].u.cpool.used = 0; + } + else { + allctr->fix[i].u.nocpool.max_used = 0; + allctr->fix[i].u.nocpool.limit = 0; + allctr->fix[i].u.nocpool.allocated = 0; + allctr->fix[i].u.nocpool.used = 0; + } } } @@ -4360,11 +5611,9 @@ erts_alcu_stop(Allctr_t *allctr) allctr->stopped = 1; while (allctr->sbc_list.first) - destroy_carrier(allctr, SBC2BLK(allctr, allctr->sbc_list.first)); + destroy_carrier(allctr, SBC2BLK(allctr, allctr->sbc_list.first), NULL); while (allctr->mbc_list.first) - destroy_carrier(allctr, MBC_TO_FIRST_BLK(allctr, allctr->mbc_list.first)); - while (allctr->sbmbc_list.first) - destroy_sbmbc(allctr, MBC_TO_FIRST_BLK(allctr, allctr->sbmbc_list.first)); + destroy_carrier(allctr, MBC_TO_FIRST_BLK(allctr, allctr->mbc_list.first), NULL); #ifdef USE_THREADS if (allctr->thread_safe) @@ -4378,6 +5627,14 @@ erts_alcu_stop(Allctr_t *allctr) void erts_alcu_init(AlcUInit_t *init) { +#ifdef ERTS_SMP + int i; + for (i = 0; i <= ERTS_ALC_A_MAX; i++) { + ErtsAlcCPoolData_t *sentinel = &carrier_pool[i].sentinel; + erts_atomic_init_nob(&sentinel->next, (erts_aint_t) sentinel); + erts_atomic_init_nob(&sentinel->prev, (erts_aint_t) sentinel); + } +#endif ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */ #if HAVE_ERTS_MSEG ASSERT(erts_mseg_unit_size() == MSEG_UNIT_SZ); @@ -4440,8 +5697,35 @@ erts_alcu_test(UWord op, UWord a1, UWord a2) case 0x019: return (UWord) PREV_BLK((Block_t *) a1); case 0x01a: return (UWord) IS_MBC_FIRST_BLK((Allctr_t*)a1, (Block_t *) a2); case 0x01b: return (UWord) sizeof(Unit_t); + case 0x01c: return (unsigned long) BLK_TO_MBC((Block_t*) a1); + case 0x01d: ((Allctr_t*) a1)->add_mbc((Allctr_t*)a1, (Carrier_t*)a2); break; + case 0x01e: ((Allctr_t*) a1)->remove_mbc((Allctr_t*)a1, (Carrier_t*)a2); break; +#ifdef ERTS_SMP + case 0x01f: return (UWord) sizeof(ErtsAlcCrrPool_t); + case 0x020: + SET_CARRIER_HDR((Carrier_t *) a2, 0, SCH_SYS_ALLOC|SCH_MBC, (Allctr_t *) a1); + cpool_init_carrier_data((Allctr_t *) a1, (Carrier_t *) a2); + return (UWord) a2; + case 0x021: + cpool_insert((Allctr_t *) a1, (Carrier_t *) a2); + return (UWord) a2; + case 0x022: + cpool_delete((Allctr_t *) a1, (Allctr_t *) a1, (Carrier_t *) a2); + return (UWord) a2; + case 0x023: return (UWord) cpool_is_empty((Allctr_t *) a1); + case 0x024: return (UWord) cpool_dbg_is_in_pool((Allctr_t *) a1, (Carrier_t *) a2); +#else + case 0x01f: return (UWord) 0; + case 0x020: return (UWord) 0; + case 0x021: return (UWord) 0; + case 0x022: return (UWord) 0; + case 0x023: return (UWord) 0; + case 0x024: return (UWord) 0; +#endif + default: ASSERT(0); return ~((UWord) 0); } + return 0; } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ @@ -4449,6 +5733,20 @@ erts_alcu_test(UWord op, UWord a1, UWord a2) \* */ void +erts_alcu_assert_failed(char* expr, char* file, int line, char *func) +{ + fflush(stdout); + fprintf(stderr, "%s:%d:%s(): Assertion failed: %s\n", + file, line, func, expr); + fflush(stderr); +#if defined(__WIN__) || defined(__WIN32__) + DebugBreak(); +#else + abort(); +#endif +} + +void erts_alcu_verify_unused(Allctr_t *allctr) { UWord no; @@ -4456,12 +5754,10 @@ erts_alcu_verify_unused(Allctr_t *allctr) no = allctr->sbcs.curr.norm.mseg.no; no += allctr->sbcs.curr.norm.sys_alloc.no; no += allctr->mbcs.blocks.curr.no; - no += allctr->sbmbcs.blocks.curr.no; if (no) { UWord sz = allctr->sbcs.blocks.curr.size; sz += allctr->mbcs.blocks.curr.size; - sz += allctr->sbmbcs.blocks.curr.size; erl_exit(ERTS_ABORT_EXIT, "%salloc() used when expected to be unused!\n" "Total amount of blocks allocated: %bpu\n" @@ -4595,7 +5891,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) cl = &allctr->mbc_list; } -#if 0 /* FIXIT sbmbc */ +#ifdef DEBUG if (cl->first == crr) { ASSERT(!crr->prev); } |