aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/erl_alloc_util.c
diff options
context:
space:
mode:
authorRickard Green <[email protected]>2013-06-04 11:48:13 +0200
committerRickard Green <[email protected]>2013-06-04 11:48:13 +0200
commite4f06fc54f6a8ae3bcdc5faa646a7bc720fe7479 (patch)
tree8d55ef4992ddccf03999da08adfb0d7957fbf5ad /erts/emulator/beam/erl_alloc_util.c
parent4a25ec13e624989af191a2297f904df87d8e3248 (diff)
parentb7081f310a8503855d10aeda35f4bc056be24bb3 (diff)
downloadotp-e4f06fc54f6a8ae3bcdc5faa646a7bc720fe7479.tar.gz
otp-e4f06fc54f6a8ae3bcdc5faa646a7bc720fe7479.tar.bz2
otp-e4f06fc54f6a8ae3bcdc5faa646a7bc720fe7479.zip
Merge branch 'maint'
* maint: erts: Document the +M<S>acul command line argument erts: Carrier pool information in allocator information erts: Use carrier pool for migration of carriers erts: Implement test case for carrier pool erts: Implement carrier pool erts: Fix type errors in info functions erts: Use Uint64 for call counts erts: Fix failing testcase alloc_SUITE:rbtree erts: Rename allocator aoffcbf to aoffcaobf erts: Remove unnecessary flag arguments in allocators erts: Remove SBMBC allocator erts: Add test for add_mbc and remove_mbc callbacks erts: Fix deallocation in removed carrier erts: Change naive list to rb-tree of carriers in AOFF allocator erts: Prepare aoff allocator for carrier migration erts: Make carrier header sizes customizable erts: Add "bestfit within carrier" for aoff allocator (aoffcbf) Conflicts: erts/preloaded/ebin/erlang.beam
Diffstat (limited to 'erts/emulator/beam/erl_alloc_util.c')
-rw-r--r--erts/emulator/beam/erl_alloc_util.c2776
1 files changed, 2036 insertions, 740 deletions
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 53062979a6..b19e603a5f 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -71,21 +71,16 @@
#define ALLOC_ZERO_EQ_NULL 0
+#ifndef ERTS_MSEG_FLG_2POW
+# define ERTS_MSEG_FLG_2POW 0
+#endif
+#ifndef ERTS_MSEG_FLG_NONE
+# define ERTS_MSEG_FLG_NONE 0
+#endif
+
static int atoms_initialized = 0;
static int initialized = 0;
-int erts_have_sbmbc_alloc;
-
-#if HAVE_ERTS_MSEG
-
-#define MSEG_UNIT_SHIFT MSEG_ALIGN_BITS
-#define MSEG_UNIT_SZ (1 << MSEG_UNIT_SHIFT)
-#define MSEG_UNIT_MASK ((~(UWord)0) << MSEG_UNIT_SHIFT)
-
-#define MSEG_UNIT_FLOOR(X) ((X) & MSEG_UNIT_MASK)
-#define MSEG_UNIT_CEILING(X) MSEG_UNIT_FLOOR((X) + ~MSEG_UNIT_MASK)
-
-#endif
#define INV_SYS_ALLOC_CARRIER_MASK ((UWord) (sys_alloc_carrier_size - 1))
#define SYS_ALLOC_CARRIER_MASK (~INV_SYS_ALLOC_CARRIER_MASK)
@@ -110,13 +105,12 @@ static Uint max_mseg_carriers;
#define ONE_GIGA (1000000000)
-#define INC_CC(CC) ((CC).no == ONE_GIGA - 1 \
- ? ((CC).giga_no++, (CC).no = 0) \
- : (CC).no++)
+#define ERTS_ALC_CC_GIGA_VAL(CC) ((CC) / ONE_GIGA)
+#define ERTS_ALC_CC_VAL(CC) ((CC) % ONE_GIGA)
+
+#define INC_CC(CC) ((CC)++)
-#define DEC_CC(CC) ((CC).no == 0 \
- ? ((CC).giga_no--, (CC).no = ONE_GIGA - 1) \
- : (CC).no--)
+#define DEC_CC(CC) ((CC)--)
/* Multi block carrier (MBC) memory layout in R16:
@@ -173,14 +167,6 @@ MBC after deallocating first block:
#define SET_BLK_SZ_FTR(B, SZ) \
(((FreeBlkFtr_t *) (((char *) (B)) + (SZ)))[-1] = (SZ))
-#define THIS_FREE_BLK_HDR_FLG (((UWord) 1) << 0)
-#define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1)
-#define LAST_BLK_HDR_FLG (((UWord) 1) << 2)
-
-/* Special flag combo for (allocated) SBC blocks
-*/
-#define SBC_BLK_HDR_FLG (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)
-
#define SET_MBC_ABLK_SZ(B, SZ) \
(ASSERT(((SZ) & FLG_MASK) == 0), \
(B)->bhdr = (((B)->bhdr) & ~MBC_ABLK_SZ_MASK) | (SZ))
@@ -222,18 +208,7 @@ MBC after deallocating first block:
ASSERT(((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \
(B)->bhdr = ((Sz) | (F)), \
(B)->u.carrier = (C))
-
-# define ABLK_TO_MBC(B) \
- (ASSERT(IS_MBC_BLK(B) && IS_ALLOCED_BLK(B)), \
- (Carrier_t*)((MSEG_UNIT_FLOOR((UWord)(B)) - \
- (((B)->bhdr >> MBC_ABLK_OFFSET_SHIFT) << MSEG_UNIT_SHIFT))))
-
-# define FBLK_TO_MBC(B) \
- (ASSERT(IS_MBC_BLK(B) && IS_FREE_BLK(B)), \
- (B)->u.carrier)
-# define BLK_TO_MBC(B) (IS_FREE_BLK(B) ? FBLK_TO_MBC(B) : ABLK_TO_MBC(B))
-
# define IS_MBC_FIRST_ABLK(AP,B) \
((((UWord)(B) & ~MSEG_UNIT_MASK) == MBC_HEADER_SIZE(AP)) \
&& ((B)->bhdr & MBC_ABLK_OFFSET_MASK) == 0)
@@ -272,10 +247,6 @@ MBC after deallocating first block:
(B)->bhdr = ((Sz) | (F)), \
(B)->carrier = (C))
-# define BLK_TO_MBC(B) ((B)->carrier)
-# define ABLK_TO_MBC(B) BLK_TO_MBC(B)
-# define FBLK_TO_MBC(B) BLK_TO_MBC(B)
-
# define IS_MBC_FIRST_BLK(AP,B) \
((char*)(B) == (char*)((B)->carrier) + MBC_HEADER_SIZE(AP))
# define IS_MBC_FIRST_ABLK(AP,B) IS_MBC_FIRST_BLK(AP,B)
@@ -300,8 +271,6 @@ MBC after deallocating first block:
((B)->bhdr & PREV_FREE_BLK_HDR_FLG)
#define IS_PREV_BLK_ALLOCED(B) \
(!IS_PREV_BLK_FREE((B)))
-#define IS_FREE_BLK(B) \
- (ASSERT(!IS_SBC_BLK(B)), (B)->bhdr & THIS_FREE_BLK_HDR_FLG)
#define IS_ALLOCED_BLK(B) \
(!IS_FREE_BLK((B)))
#define IS_LAST_BLK(B) \
@@ -318,13 +287,6 @@ MBC after deallocating first block:
#define GET_BLK_HDR_FLGS(B) \
((B)->bhdr & FLG_MASK)
-#define IS_SBC_BLK(B) \
- (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG)
-#define IS_MBC_BLK(B) \
- (!IS_SBC_BLK((B)))
-
-#define MBC_BLK_SZ(B) (IS_FREE_BLK(B) ? MBC_FBLK_SZ(B) : MBC_ABLK_SZ(B))
-
#define NXT_BLK(B) \
(ASSERT(IS_MBC_BLK(B)), \
(Block_t *) (((char *) (B)) + MBC_BLK_SZ((B))))
@@ -338,9 +300,96 @@ MBC after deallocating first block:
/* Carriers ... */
-#define SBC_HEADER_SIZE (UNIT_CEILING(sizeof(Carrier_t) + ABLK_HDR_SZ) \
- - ABLK_HDR_SZ)
-#define MBC_HEADER_SIZE(AP) SBC_HEADER_SIZE
+/* #define ERTS_ALC_CPOOL_DEBUG */
+
+#if defined(DEBUG) && !defined(ERTS_ALC_CPOOL_DEBUG)
+# define ERTS_ALC_CPOOL_DEBUG
+#endif
+
+#ifndef ERTS_SMP
+# undef ERTS_ALC_CPOOL_DEBUG
+#endif
+
+#ifdef ERTS_ALC_CPOOL_DEBUG
+# define ERTS_ALC_CPOOL_ASSERT(A) \
+ ((void) ((A) \
+ ? 1 \
+ : (erts_alcu_assert_failed(#A, \
+ (char *) __FILE__, \
+ __LINE__, \
+ (char *) __func__), \
+ 0)))
+#else
+# define ERTS_ALC_CPOOL_ASSERT(A) ((void) 1)
+#endif
+
+#ifdef ERTS_SMP
+#define ERTS_ALC_IS_CPOOL_ENABLED(A) ((A)->cpool.util_limit)
+#else
+#define ERTS_ALC_IS_CPOOL_ENABLED(A) (0)
+#endif
+
+#ifdef ERTS_SMP
+
+#define ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON 1000
+#define ERTS_ALC_CPOOL_ALLOC_OP_INC 8
+#define ERTS_ALC_CPOOL_FREE_OP_DEC 10
+
+#define ERTS_ALC_CPOOL_ALLOC_OP(A) \
+do { \
+ if ((A)->cpool.disable_abandon < ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON) { \
+ (A)->cpool.disable_abandon += ERTS_ALC_CPOOL_ALLOC_OP_INC; \
+ if ((A)->cpool.disable_abandon > ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON) \
+ (A)->cpool.disable_abandon = ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON; \
+ } \
+} while (0)
+
+
+#if ERTS_ALC_CPOOL_ALLOC_OP_INC >= ERTS_ALC_CPOOL_FREE_OP_DEC
+# error "Implementation assume ERTS_ALC_CPOOL_ALLOC_OP_INC < ERTS_ALC_CPOOL_FREE_OP_DEC"
+#endif
+
+#define ERTS_ALC_CPOOL_REALLOC_OP(A) \
+do { \
+ if ((A)->cpool.disable_abandon) { \
+ (A)->cpool.disable_abandon -= (ERTS_ALC_CPOOL_FREE_OP_DEC \
+ - ERTS_ALC_CPOOL_ALLOC_OP_INC); \
+ if ((A)->cpool.disable_abandon < 0) \
+ (A)->cpool.disable_abandon = 0; \
+ } \
+} while (0)
+
+#define ERTS_ALC_CPOOL_FREE_OP(A) \
+do { \
+ if ((A)->cpool.disable_abandon) { \
+ (A)->cpool.disable_abandon -= ERTS_ALC_CPOOL_FREE_OP_DEC; \
+ if ((A)->cpool.disable_abandon < 0) \
+ (A)->cpool.disable_abandon = 0; \
+ } \
+} while (0)
+
+#else
+#define ERTS_ALC_CPOOL_ALLOC_OP(A)
+#define ERTS_ALC_CPOOL_REALLOC_OP(A)
+#define ERTS_ALC_CPOOL_FREE_OP(A)
+#endif
+
+#define ERTS_CRR_ALCTR_FLG_IN_POOL (((erts_aint_t) 1) << 0)
+#define ERTS_CRR_ALCTR_FLG_BUSY (((erts_aint_t) 1) << 1)
+
+#ifdef ERTS_SMP
+#define SBC_HEADER_SIZE \
+ (UNIT_CEILING(sizeof(Carrier_t) \
+ - sizeof(ErtsAlcCPoolData_t) \
+ + ABLK_HDR_SZ) \
+ - ABLK_HDR_SZ)
+#else
+#define SBC_HEADER_SIZE \
+ (UNIT_CEILING(sizeof(Carrier_t) \
+ + ABLK_HDR_SZ) \
+ - ABLK_HDR_SZ)
+#endif
+#define MBC_HEADER_SIZE(AP) ((AP)->mbc_header_size)
#define MSEG_CARRIER_HDR_FLAG (((UWord) 1) << 0)
@@ -352,7 +401,8 @@ MBC after deallocating first block:
#define SCH_SBC SBC_CARRIER_HDR_FLAG
#define SET_CARRIER_HDR(C, Sz, F, AP) \
- (ASSERT(((Sz) & FLG_MASK) == 0), (C)->chdr = ((Sz) | (F)), (C)->allctr = (AP))
+ (ASSERT(((Sz) & FLG_MASK) == 0), (C)->chdr = ((Sz) | (F)), \
+ erts_smp_atomic_init_nob(&(C)->allctr, (erts_aint_t) (AP)))
#define BLK_TO_SBC(B) \
((Carrier_t *) (((char *) (B)) - SBC_HEADER_SIZE))
@@ -385,6 +435,7 @@ MBC after deallocating first block:
#define CFLG_FORCE_SYS_ALLOC (1 << 3)
#define CFLG_FORCE_SIZE (1 << 4)
#define CFLG_MAIN_CARRIER (1 << 5)
+#define CFLG_NO_CPOOL (1 << 6)
#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
static void check_blk_carrier(Allctr_t *, Block_t *);
@@ -412,11 +463,7 @@ static void check_blk_carrier(Allctr_t *, Block_t *);
ASSERT(((AP)->mbcs.curr.norm.sys_alloc.no \
&& (AP)->mbcs.curr.norm.sys_alloc.size) \
|| (!(AP)->mbcs.curr.norm.sys_alloc.no \
- && !(AP)->mbcs.curr.norm.sys_alloc.size)); \
- ASSERT(((AP)->sbmbcs.curr.small_block.no \
- && (AP)->sbmbcs.curr.small_block.size) \
- || (!(AP)->sbmbcs.curr.small_block.no \
- && !(AP)->sbmbcs.curr.small_block.size))
+ && !(AP)->mbcs.curr.norm.sys_alloc.size));
#else
#define DEBUG_CHECK_CARRIER_NO_SZ(AP)
@@ -487,17 +534,6 @@ do { \
+ (AP)->mbcs.curr.norm.sys_alloc.size)
-#define STAT_SBMBC_ALLOC(AP, CSZ) \
-do { \
- (AP)->sbmbcs.curr.small_block.no++; \
- (AP)->sbmbcs.curr.small_block.size += (CSZ); \
- if ((AP)->sbmbcs.max.no < (AP)->sbmbcs.curr.small_block.no) \
- (AP)->sbmbcs.max.no = (AP)->sbmbcs.curr.small_block.no; \
- if ((AP)->sbmbcs.max.size < (AP)->sbmbcs.curr.small_block.size) \
- (AP)->sbmbcs.max.size = (AP)->sbmbcs.curr.small_block.size; \
- DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
-} while (0)
-
#define STAT_MSEG_MBC_ALLOC(AP, CSZ) \
do { \
(AP)->mbcs.curr.norm.mseg.no++; \
@@ -514,13 +550,19 @@ do { \
DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
} while (0)
-#define STAT_SBMBC_FREE(AP, CSZ) \
+#define STAT_MBC_CPOOL_FETCH(AP, CRR) \
do { \
- ASSERT((AP)->sbmbcs.curr.small_block.no > 0); \
- (AP)->sbmbcs.curr.small_block.no--; \
- ASSERT((AP)->sbmbcs.curr.small_block.size >= (CSZ)); \
- (AP)->sbmbcs.curr.small_block.size -= (CSZ); \
- DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
+ UWord csz__ = CARRIER_SZ((CRR)); \
+ if (IS_MSEG_CARRIER((CRR))) \
+ STAT_MSEG_MBC_ALLOC((AP), csz__); \
+ else \
+ STAT_SYS_ALLOC_MBC_ALLOC((AP), csz__); \
+ (AP)->mbcs.blocks.curr.no += (CRR)->cpool.blocks; \
+ if ((AP)->mbcs.blocks.max.no < (AP)->mbcs.blocks.curr.no) \
+ (AP)->mbcs.blocks.max.no = (AP)->mbcs.blocks.curr.no; \
+ (AP)->mbcs.blocks.curr.size += (CRR)->cpool.blocks_size; \
+ if ((AP)->mbcs.blocks.max.size < (AP)->mbcs.blocks.curr.size) \
+ (AP)->mbcs.blocks.max.size = (AP)->mbcs.blocks.curr.size; \
} while (0)
#define STAT_MSEG_MBC_FREE(AP, CSZ) \
@@ -541,28 +583,88 @@ do { \
DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
} while (0)
-#define STAT_MBC_BLK_ALLOC(AP, BSZ, FLGS) \
+#define STAT_MBC_CPOOL_INSERT(AP, CRR) \
+do { \
+ UWord csz__ = CARRIER_SZ((CRR)); \
+ if (IS_MSEG_CARRIER((CRR))) \
+ STAT_MSEG_MBC_FREE((AP), csz__); \
+ else \
+ STAT_SYS_ALLOC_MBC_FREE((AP), csz__); \
+ ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.no \
+ >= (CRR)->cpool.blocks); \
+ (AP)->mbcs.blocks.curr.no -= (CRR)->cpool.blocks; \
+ ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.size \
+ >= (CRR)->cpool.blocks_size); \
+ (AP)->mbcs.blocks.curr.size -= (CRR)->cpool.blocks_size; \
+} while (0)
+
+#ifdef ERTS_SMP
+#define STAT_MBC_BLK_ALLOC_CRR(CRR, BSZ) \
+do { \
+ (CRR)->cpool.blocks++; \
+ (CRR)->cpool.blocks_size += (BSZ); \
+} while (0)
+#else
+#define STAT_MBC_BLK_ALLOC_CRR(CRR, BSZ) ((void) (CRR)) /* Get rid of warning */
+#endif
+
+#define STAT_MBC_BLK_ALLOC(AP, CRR, BSZ, FLGS) \
do { \
- CarriersStats_t *cstats__ = (((FLGS) & ERTS_ALCU_FLG_SBMBC) \
- ? &(AP)->sbmbcs \
- : &(AP)->mbcs); \
+ CarriersStats_t *cstats__ = &(AP)->mbcs; \
cstats__->blocks.curr.no++; \
if (cstats__->blocks.max.no < cstats__->blocks.curr.no) \
cstats__->blocks.max.no = cstats__->blocks.curr.no; \
cstats__->blocks.curr.size += (BSZ); \
if (cstats__->blocks.max.size < cstats__->blocks.curr.size) \
cstats__->blocks.max.size = cstats__->blocks.curr.size; \
+ STAT_MBC_BLK_ALLOC_CRR((CRR), (BSZ)); \
} while (0)
-#define STAT_MBC_BLK_FREE(AP, BSZ, FLGS) \
+static ERTS_INLINE int
+stat_cpool_mbc_blk_free(Allctr_t *allctr,
+ Carrier_t *crr,
+ Carrier_t **busy_pcrr_pp,
+ UWord blksz)
+{
+#ifdef ERTS_SMP
+
+ ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks > 0);
+ crr->cpool.blocks--;
+ ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks_size >= blksz);
+ crr->cpool.blocks_size -= blksz;
+
+ if (!busy_pcrr_pp || !*busy_pcrr_pp)
+ return 0;
+
+ ERTS_ALC_CPOOL_ASSERT(crr == *busy_pcrr_pp);
+
+#ifdef ERTS_ALC_CPOOL_DEBUG
+ ERTS_ALC_CPOOL_ASSERT(
+ erts_atomic_dec_read_nob(&allctr->cpool.stat.no_blocks) >= 0);
+ ERTS_ALC_CPOOL_ASSERT(
+ erts_atomic_add_read_nob(&allctr->cpool.stat.blocks_size,
+ -((erts_aint_t) blksz)) >= 0);
+#else
+ erts_atomic_dec_nob(&allctr->cpool.stat.no_blocks);
+ erts_atomic_add_nob(&allctr->cpool.stat.blocks_size,
+ -((erts_aint_t) blksz));
+#endif
+
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+#define STAT_MBC_BLK_FREE(AP, CRR, BPCRRPP, BSZ, FLGS) \
do { \
- CarriersStats_t *cstats__ = (((FLGS) & ERTS_ALCU_FLG_SBMBC) \
- ? &(AP)->sbmbcs \
- : &(AP)->mbcs); \
- ASSERT(cstats__->blocks.curr.no > 0); \
- cstats__->blocks.curr.no--; \
- ASSERT(cstats__->blocks.curr.size >= (BSZ)); \
- cstats__->blocks.curr.size -= (BSZ); \
+ if (!stat_cpool_mbc_blk_free((AP), (CRR), (BPCRRPP), (BSZ))) { \
+ CarriersStats_t *cstats__ = &(AP)->mbcs; \
+ ASSERT(cstats__->blocks.curr.no > 0); \
+ cstats__->blocks.curr.no--; \
+ ASSERT(cstats__->blocks.curr.size >= (BSZ)); \
+ cstats__->blocks.curr.size -= (BSZ); \
+ } \
} while (0)
/* Debug stuff... */
@@ -613,13 +715,42 @@ do { \
#define ERTS_ALCU_DBG_CHK_THR_ACCESS(A)
#endif
-
static void make_name_atoms(Allctr_t *allctr);
static Block_t *create_carrier(Allctr_t *, Uint, UWord);
-static void destroy_carrier(Allctr_t *, Block_t *);
-static void mbc_free(Allctr_t *allctr, void *p);
+static void destroy_carrier(Allctr_t *, Block_t *, Carrier_t **);
+static void mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp);
+static void dealloc_block(Allctr_t *, void *, int);
+
+/* internal data... */
+
+#if 0
+
+static ERTS_INLINE void *
+internal_alloc(UWord size)
+{
+ void *res = erts_sys_alloc(0, NULL, size);
+ if (!res)
+ erts_alloc_enomem(ERTS_ALC_T_UNDEF, size);
+ return res;
+}
+static ERTS_INLINE void *
+internal_realloc(void *ptr, UWord size)
+{
+ void *res = erts_sys_realloc(0, NULL, ptr, size);
+ if (!res)
+ erts_alloc_enomem(ERTS_ALC_T_UNDEF, size);
+ return res;
+}
+
+static ERTS_INLINE void
+internal_free(void *ptr)
+{
+ erts_sys_free(0, NULL, ptr);
+}
+
+#endif
/* mseg ... */
@@ -781,10 +912,35 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr)
}
}
-static Block_t *create_sbmbc(Allctr_t *allctr, Uint umem_sz);
-static void destroy_sbmbc(Allctr_t *allctr, Block_t *blk);
-static Block_t *create_carrier(Allctr_t *, Uint, UWord);
-static void destroy_carrier(Allctr_t *, Block_t *);
+#ifdef ERTS_SMP
+
+static ERTS_INLINE void
+clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr)
+{
+ if (crr) {
+ erts_aint_t max_size;
+ erts_aint_t new_val;
+
+ max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr);
+ erts_atomic_set_nob(&crr->cpool.max_size, max_size);
+
+ new_val = (((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL);
+
+#ifdef ERTS_ALC_CPOOL_DEBUG
+ {
+ erts_aint_t old_val = new_val|ERTS_CRR_ALCTR_FLG_BUSY;
+
+ ERTS_ALC_CPOOL_ASSERT(old_val
+ == erts_smp_atomic_xchg_relb(&crr->allctr,
+ new_val));
+ }
+#else
+ erts_smp_atomic_set_relb(&crr->allctr, new_val);
+#endif
+ }
+}
+
+#endif
#if 0
#define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) \
@@ -806,13 +962,154 @@ chk_fix_list(Allctr_t *allctr, ErtsAlcFixList_t *fix, int ix, int before)
#define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B)
#endif
-erts_aint32_t
-erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
+static void *mbc_alloc(Allctr_t *allctr, Uint size);
+
+#ifdef ERTS_SMP
+typedef struct {
+ ErtsAllctrDDBlock_t ddblock__; /* must be first */
+ ErtsAlcType_t fix_type;
+} ErtsAllctrFixDDBlock_t;
+#endif
+
+static ERTS_INLINE void
+dealloc_fix_block(Allctr_t *allctr,
+ ErtsAlcType_t type,
+ void *ptr,
+ int dec_cc_on_redirect)
+{
+#ifdef ERTS_SMP
+ /* May be redirected... */
+ ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type;
+#endif
+ dealloc_block(allctr, ptr, dec_cc_on_redirect);
+}
+
+static ERTS_INLINE void
+sched_fix_shrink(Allctr_t *allctr, int on)
+{
+ if (on && !allctr->fix_shrink_scheduled) {
+ allctr->fix_shrink_scheduled = 1;
+ erts_set_aux_work_timeout(allctr->ix,
+ (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
+ | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
+ 1);
+ }
+ else if (!on && allctr->fix_shrink_scheduled) {
+ allctr->fix_shrink_scheduled = 0;
+ erts_set_aux_work_timeout(allctr->ix,
+ (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
+ | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
+ 0);
+ }
+}
+
+static ERTS_INLINE void
+fix_cpool_check_shrink(Allctr_t *allctr,
+ ErtsAlcType_t type,
+ ErtsAlcFixList_t *fix,
+ Carrier_t **busy_pcrr_pp)
+{
+ if (fix->u.cpool.shrink_list > 0) {
+ if (fix->list_size == 0)
+ fix->u.cpool.shrink_list = 0;
+ else {
+ void *p;
+#ifdef ERTS_SMP
+ if (busy_pcrr_pp) {
+ clear_busy_pool_carrier(allctr, *busy_pcrr_pp);
+ *busy_pcrr_pp = NULL;
+ }
+#endif
+ fix->u.cpool.shrink_list--;
+ p = fix->list;
+ fix->list = *((void **) p);
+ fix->list_size--;
+ if (fix->u.cpool.min_list_size > fix->list_size)
+ fix->u.cpool.min_list_size = fix->list_size;
+
+ fix->u.cpool.allocated--;
+ dealloc_fix_block(allctr, type, p, 0);
+ }
+ }
+}
+
+static ERTS_INLINE void *
+fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size)
+{
+ void *res;
+ ErtsAlcFixList_t *fix;
+
+ ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type
+ && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE);
+
+ fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE];
+
+ res = fix->list;
+ if (res) {
+ fix->list = *((void **) res);
+ fix->list_size--;
+ if (fix->u.cpool.min_list_size > fix->list_size)
+ fix->u.cpool.min_list_size = fix->list_size;
+ fix->u.cpool.used++;
+ fix_cpool_check_shrink(allctr, type, fix, NULL);
+ return res;
+ }
+ if (size < 2*sizeof(UWord))
+ size += sizeof(UWord);
+ if (size >= allctr->sbc_threshold) {
+ Block_t *blk;
+ blk = create_carrier(allctr, size, CFLG_SBC);
+ res = blk ? BLK2UMEM(blk) : NULL;
+ }
+ else
+ res = mbc_alloc(allctr, size);
+ if (res) {
+ fix->u.cpool.used++;
+ fix->u.cpool.allocated++;
+ }
+ return res;
+}
+
+static ERTS_INLINE void
+fix_cpool_free(Allctr_t *allctr,
+ ErtsAlcType_t type,
+ void *p,
+ Carrier_t **busy_pcrr_pp)
+{
+ ErtsAlcFixList_t *fix;
+
+ ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type
+ && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE);
+
+ fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE];
+
+ fix->u.cpool.used--;
+
+ if ((!busy_pcrr_pp || !*busy_pcrr_pp)
+ && !fix->u.cpool.shrink_list
+ && fix->list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) {
+ *((void **) p) = fix->list;
+ fix->list = p;
+ fix->list_size++;
+ sched_fix_shrink(allctr, 1);
+ }
+ else {
+ Block_t *blk = UMEM2BLK(p);
+ if (IS_SBC_BLK(blk))
+ destroy_carrier(allctr, blk, NULL);
+ else
+ mbc_free(allctr, p, busy_pcrr_pp);
+ fix->u.cpool.allocated--;
+ fix_cpool_check_shrink(allctr, type, fix, busy_pcrr_pp);
+ }
+}
+
+static ERTS_INLINE erts_aint32_t
+fix_cpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
{
int all_empty = 1;
erts_aint32_t res = 0;
int ix, o;
- ErtsAlcFixList_t *fix = allctr->fix;
int flush = flgs == 0;
#ifdef USE_THREADS
@@ -821,56 +1118,204 @@ erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
#endif
for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) {
+ ErtsAlcFixList_t *fix = &allctr->fix[ix];
+ ErtsAlcType_t type;
ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
- if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) {
- fix[ix].limit = fix[ix].max_used;
- if (fix[ix].limit < fix[ix].used)
- fix[ix].limit = fix[ix].used;
- fix[ix].max_used = fix[ix].used;
- ASSERT(fix[ix].limit >= 0);
-
- }
- if (flush) {
- fix[ix].limit = 0;
- fix[ix].max_used = fix[ix].used;
- ASSERT(fix[ix].limit >= 0);
+ if (flush)
+ fix->u.cpool.shrink_list = fix->list_size;
+ else if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) {
+ fix->u.cpool.shrink_list = fix->u.cpool.min_list_size;
+ fix->u.cpool.min_list_size = fix->list_size;
}
+ type = (ErtsAlcType_t) (ix + ERTS_ALC_N_MIN_A_FIXED_SIZE);
for (o = 0; o < ERTS_ALC_FIX_MAX_SHRINK_OPS || flush; o++) {
- Block_t *blk;
void *ptr;
- if (!flush && fix[ix].limit >= fix[ix].allocated)
+ if (fix->u.cpool.shrink_list == 0)
break;
- if (fix[ix].list_size == 0)
+ if (fix->list_size == 0) {
+ fix->u.cpool.shrink_list = 0;
break;
- ptr = fix[ix].list;
- fix[ix].list = *((void **) ptr);
- fix[ix].list_size--;
+ }
+ ptr = fix->list;
+ fix->list = *((void **) ptr);
+ fix->list_size--;
+ fix->u.cpool.shrink_list--;
+ fix->u.cpool.allocated--;
+ dealloc_fix_block(allctr, type, ptr, 0);
+ }
+ if (fix->u.cpool.min_list_size > fix->list_size)
+ fix->u.cpool.min_list_size = fix->list_size;
+ if (fix->list_size != 0) {
+ if (fix->u.cpool.shrink_list > 0)
+ res |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC;
+ all_empty = 0;
+ }
+ }
+
+ if (all_empty)
+ sched_fix_shrink(allctr, 0);
+
+#ifdef USE_THREADS
+ if (allctr->thread_safe)
+ erts_mtx_unlock(&allctr->mutex);
+#endif
- blk = UMEM2BLK(ptr);
+ return res;
+}
+static ERTS_INLINE void *
+fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size)
+{
+ ErtsAlcFixList_t *fix;
+ void *res;
+
+ ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type
+ && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE);
+
+ fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE];
+
+ ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
+ fix->u.nocpool.used++;
+ res = fix->list;
+ if (res) {
+ fix->list_size--;
+ fix->list = *((void **) res);
+ if (fix->list && fix->u.nocpool.allocated > fix->u.nocpool.limit) {
+ Block_t *blk;
+ void *p = fix->list;
+ fix->list = *((void **) p);
+ fix->list_size--;
+ blk = UMEM2BLK(p);
if (IS_SBC_BLK(blk))
- destroy_carrier(allctr, blk);
+ destroy_carrier(allctr, blk, NULL);
else
- mbc_free(allctr, ptr);
+ mbc_free(allctr, p, NULL);
+ fix->u.nocpool.allocated--;
+ }
+ ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
+ return res;
+ }
+ if (size < 2*sizeof(UWord))
+ size += sizeof(UWord);
+ if (fix->u.nocpool.limit < fix->u.nocpool.used)
+ fix->u.nocpool.limit = fix->u.nocpool.used;
+ if (fix->u.nocpool.max_used < fix->u.nocpool.used)
+ fix->u.nocpool.max_used = fix->u.nocpool.used;
+ fix->u.nocpool.allocated++;
- fix[ix].allocated--;
+ if (size >= allctr->sbc_threshold) {
+ Block_t *blk;
+ blk = create_carrier(allctr, size, CFLG_SBC);
+ res = blk ? BLK2UMEM(blk) : NULL;
+ }
+ else
+ res = mbc_alloc(allctr, size);
+
+ if (!res) {
+ fix->u.nocpool.allocated--;
+ fix->u.nocpool.used--;
+ }
+ return res;
+}
+
+static ERTS_INLINE void
+fix_nocpool_free(Allctr_t *allctr,
+ ErtsAlcType_t type,
+ void *p)
+{
+ Block_t *blk;
+ ErtsAlcFixList_t *fix;
+
+ ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type
+ && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE);
+
+ fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE];
+
+ ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
+ fix->u.nocpool.used--;
+ if (fix->u.nocpool.allocated < fix->u.nocpool.limit
+ && fix->list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) {
+ *((void **) p) = fix->list;
+ fix->list = p;
+ fix->list_size++;
+ sched_fix_shrink(allctr, 1);
+ ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
+ return;
+ }
+ fix->u.nocpool.allocated--;
+ if (fix->list && fix->u.nocpool.allocated > fix->u.nocpool.limit) {
+ blk = UMEM2BLK(p);
+ if (IS_SBC_BLK(blk))
+ destroy_carrier(allctr, blk, NULL);
+ else
+ mbc_free(allctr, p, NULL);
+ p = fix->list;
+ fix->list = *((void **) p);
+ fix->list_size--;
+ fix->u.nocpool.allocated--;
+ }
+
+ blk = UMEM2BLK(p);
+ if (IS_SBC_BLK(blk))
+ destroy_carrier(allctr, blk, NULL);
+ else
+ mbc_free(allctr, p, NULL);
+ ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
+}
+
+static ERTS_INLINE erts_aint32_t
+fix_nocpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
+{
+ int all_empty = 1;
+ erts_aint32_t res = 0;
+ int ix, o;
+ int flush = flgs == 0;
+
+#ifdef USE_THREADS
+ if (allctr->thread_safe)
+ erts_mtx_lock(&allctr->mutex);
+#endif
+
+ for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) {
+ ErtsAlcFixList_t *fix = &allctr->fix[ix];
+ ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
+ if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) {
+ fix->u.nocpool.limit = fix->u.nocpool.max_used;
+ if (fix->u.nocpool.limit < fix->u.nocpool.used)
+ fix->u.nocpool.limit = fix->u.nocpool.used;
+ fix->u.nocpool.max_used = fix->u.nocpool.used;
+ ASSERT(fix->u.nocpool.limit >= 0);
+
+ }
+ if (flush) {
+ fix->u.nocpool.limit = 0;
+ fix->u.nocpool.max_used = fix->u.nocpool.used;
+ ASSERT(fix->u.nocpool.limit >= 0);
}
- if (fix[ix].list_size != 0) {
- if (fix[ix].limit < fix[ix].allocated)
+ for (o = 0; o < ERTS_ALC_FIX_MAX_SHRINK_OPS || flush; o++) {
+ void *ptr;
+
+ if (!flush && fix->u.nocpool.limit >= fix->u.nocpool.allocated)
+ break;
+ if (fix->list_size == 0)
+ break;
+ ptr = fix->list;
+ fix->list = *((void **) ptr);
+ fix->list_size--;
+ dealloc_block(allctr, ptr, 0);
+ fix->u.nocpool.allocated--;
+ }
+ if (fix->list_size != 0) {
+ if (fix->u.nocpool.limit < fix->u.nocpool.allocated)
res |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC;
all_empty = 0;
}
ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
}
- if (all_empty && allctr->fix_shrink_scheduled) {
- allctr->fix_shrink_scheduled = 0;
- erts_set_aux_work_timeout(allctr->ix,
- (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
- | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
- 0);
- }
+ if (all_empty)
+ sched_fix_shrink(allctr, 0);
#ifdef USE_THREADS
if (allctr->thread_safe)
@@ -880,13 +1325,18 @@ erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
return res;
}
-#ifdef ERTS_SMP
+erts_aint32_t
+erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
+{
+ if (ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ return fix_cpool_alloc_shrink(allctr, flgs);
+ else
+ return fix_nocpool_alloc_shrink(allctr, flgs);
+}
-typedef struct {
- ErtsAllctrDDBlock_t ddblock__; /* must be first */
- ErtsAlcType_t fix_type;
-}ErtsAllctrFixDDBlock_t;
+static void dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, Uint mseg_flags);
+#ifdef ERTS_SMP
static ERTS_INLINE Allctr_t*
get_pref_allctr(void *extra)
@@ -902,6 +1352,9 @@ get_pref_allctr(void *extra)
return tspec->allctr[pref_ix];
}
+#define ERTS_ALC_TS_PREF_LOCK_IF_USED (1)
+#define ERTS_ALC_TS_PREF_LOCK_NO (0)
+
/* SMP note:
* get_used_allctr() must be safe WITHOUT locking the allocator while
* concurrent threads may be updating adjacent blocks.
@@ -910,22 +1363,85 @@ get_pref_allctr(void *extra)
* the "PREV_FREE" flag bit.
*/
static ERTS_INLINE Allctr_t*
-get_used_allctr(void *extra, void *p, UWord *sizep)
+get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep,
+ Carrier_t **busy_pcrr_pp)
{
Block_t* blk = UMEM2BLK(p);
- Carrier_t* crr;
+ Carrier_t *crr;
+ erts_aint_t iallctr;
+ Allctr_t *used_allctr;
+
+ *busy_pcrr_pp = NULL;
if (IS_SBC_BLK(blk)) {
crr = BLK_TO_SBC(blk);
if (sizep)
*sizep = SBC_BLK_SZ(blk) - ABLK_HDR_SZ;
+ iallctr = erts_smp_atomic_read_dirty(&crr->allctr);
}
else {
crr = ABLK_TO_MBC(blk);
+
if (sizep)
*sizep = MBC_ABLK_SZ(blk) - ABLK_HDR_SZ;
+ if (!ERTS_ALC_IS_CPOOL_ENABLED(pref_allctr))
+ iallctr = erts_smp_atomic_read_dirty(&crr->allctr);
+ else {
+ int locked_pref_allctr = 0;
+ iallctr = erts_smp_atomic_read_ddrb(&crr->allctr);
+
+ if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock
+ && pref_allctr->thread_safe) {
+ used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK);
+ if (pref_allctr == used_allctr) {
+ erts_mtx_lock(&pref_allctr->mutex);
+ locked_pref_allctr = 1;
+ }
+ }
+
+ while ((iallctr & ((~FLG_MASK)|ERTS_CRR_ALCTR_FLG_IN_POOL))
+ == (((erts_aint_t) pref_allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL)) {
+ erts_aint_t act;
+
+ ERTS_ALC_CPOOL_ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY));
+ act = erts_smp_atomic_cmpxchg_ddrb(&crr->allctr,
+ iallctr|ERTS_CRR_ALCTR_FLG_BUSY,
+ iallctr);
+ if (act == iallctr) {
+ *busy_pcrr_pp = crr;
+ break;
+ }
+ iallctr = act;
+ }
+
+ used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK);
+
+ if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock) {
+ if (locked_pref_allctr && used_allctr != pref_allctr) {
+ /* Was taken out of pool; now owned by someone else */
+ erts_mtx_unlock(&pref_allctr->mutex);
+ }
+ }
+
+ ERTS_ALC_CPOOL_ASSERT(
+ (((iallctr & ~FLG_MASK) == (erts_aint_t) pref_allctr)
+ ? (((iallctr & FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL)
+ || ((iallctr & FLG_MASK) == 0))
+ : 1));
+
+ return used_allctr;
+ }
+ }
+
+ used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK);
+
+ if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock
+ && used_allctr == pref_allctr
+ && pref_allctr->thread_safe) {
+ erts_mtx_lock(&pref_allctr->mutex);
}
- return crr->allctr;
+
+ return used_allctr;
}
static void
@@ -1020,7 +1536,7 @@ check_insert_marker(ErtsAllctrDDQueue_t *ddq, erts_aint_t ilast)
}
static ERTS_INLINE int
-ddq_enqueue(ErtsAlcType_t type, ErtsAllctrDDQueue_t *ddq, void *ptr, int cinit)
+ddq_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr, int cinit)
{
int last_elem;
int um_refc_ix = 0;
@@ -1121,6 +1637,59 @@ store_earliest_thr_prgr(ErtsThrPrgrVal *prev_val, ErtsAllctrDDQueue_t *ddq)
}
}
+static void
+check_pending_dealloc_carrier(Allctr_t *allctr,
+ int *need_thr_progress,
+ ErtsThrPrgrVal *thr_prgr_p,
+ int *need_more_work);
+
+static void
+handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr)
+{
+ ErtsAlcType_t type;
+
+ type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type;
+
+ ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type
+ && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE);
+
+ if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ fix_nocpool_free(allctr, type, ptr);
+ else {
+ Block_t *blk = UMEM2BLK(ptr);
+ Carrier_t *busy_pcrr_p;
+ Allctr_t *used_allctr;
+
+ if (IS_SBC_BLK(blk)) {
+ busy_pcrr_p = NULL;
+ goto doit;
+ }
+
+ used_allctr = get_used_allctr(allctr, ERTS_ALC_TS_PREF_LOCK_NO, ptr,
+ NULL, &busy_pcrr_p);
+ if (used_allctr == allctr) {
+ doit:
+ fix_cpool_free(allctr, type, ptr, &busy_pcrr_p);
+ clear_busy_pool_carrier(allctr, busy_pcrr_p);
+ }
+ else {
+ /* Carrier migrated; need to redirect block to new owner... */
+ int cinit = used_allctr->dd.ix - allctr->dd.ix;
+
+ ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p);
+
+ DEC_CC(allctr->calls.this_free);
+
+ ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type;
+ if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit))
+ erts_alloc_notify_delayed_dealloc(used_allctr->ix);
+ }
+ }
+}
+
+static void
+schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr);
+
static ERTS_INLINE int
handle_delayed_dealloc(Allctr_t *allctr,
int allctr_locked,
@@ -1152,7 +1721,6 @@ handle_delayed_dealloc(Allctr_t *allctr,
while (1) {
Block_t *blk;
void *ptr;
- int ix;
if (use_limit && ++ops > ops_limit) {
if (ddq->head.first != ddq->head.unref_end) {
@@ -1181,52 +1749,29 @@ handle_delayed_dealloc(Allctr_t *allctr,
res = 1;
- INC_CC(allctr->calls.this_free);
-
- if (fix) {
- ErtsAlcType_t type;
-
- type = ((ErtsAllctrFixDDBlock_t*) ptr)->fix_type;
- ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE;
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
- fix[ix].used--;
- if (fix[ix].allocated < fix[ix].limit
- && fix[ix].list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) {
- *((void **) ptr) = fix[ix].list;
- fix[ix].list = ptr;
- fix[ix].list_size++;
- if (!allctr->fix_shrink_scheduled) {
- allctr->fix_shrink_scheduled = 1;
- erts_set_aux_work_timeout(
- allctr->ix,
- (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
- | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
- 1);
- }
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
- continue;
- }
- fix[ix].allocated--;
- if (fix[ix].list && fix[ix].allocated > fix[ix].limit) {
- blk = UMEM2BLK(ptr);
- if (IS_SBC_BLK(blk))
- destroy_carrier(allctr, blk);
- else
- mbc_free(allctr, ptr);
- ptr = fix[ix].list;
- fix[ix].list = *((void **) ptr);
- fix[ix].list_size--;
- fix[ix].allocated--;
- }
+ blk = UMEM2BLK(ptr);
+ if (IS_FREE_LAST_MBC_BLK(blk)) {
+ /*
+ * A multiblock carrier that previously has been migrated away
+ * from us and now is back to be deallocated...
+ *
+ * Note that we cannot use FBLK_TO_MBC(blk) since it
+ * data has been overwritten by the queue.
+ */
+ Carrier_t *crr = FIRST_BLK_TO_MBC(allctr, blk);
+ ERTS_ALC_CPOOL_ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr));
+ ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr);
+ schedule_dealloc_carrier(allctr, crr);
}
+ else {
- blk = UMEM2BLK(ptr);
+ INC_CC(allctr->calls.this_free);
- if (IS_SBC_BLK(blk))
- destroy_carrier(allctr, blk);
- else
- mbc_free(allctr, ptr);
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
+ if (fix)
+ handle_delayed_fix_dealloc(allctr, ptr);
+ else
+ dealloc_block(allctr, ptr, 1);
+ }
}
if (need_thr_progress && !(need_thr_prgr | need_mr_wrk)) {
@@ -1236,6 +1781,12 @@ handle_delayed_dealloc(Allctr_t *allctr,
store_earliest_thr_prgr(thr_prgr_p, ddq);
}
+ if (ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ check_pending_dealloc_carrier(allctr,
+ need_thr_progress,
+ thr_prgr_p,
+ need_more_work);
+
if (allctr->thread_safe && !allctr_locked)
erts_mtx_unlock(&allctr->mutex);
return res;
@@ -1250,13 +1801,59 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type,
if (allctr->fix)
((ErtsAllctrFixDDBlock_t*) ptr)->fix_type = type;
- if (ddq_enqueue(type, &allctr->dd.q, ptr, cinit))
+ if (ddq_enqueue(&allctr->dd.q, ptr, cinit))
erts_alloc_notify_delayed_dealloc(allctr->ix);
}
#endif
#ifdef ERTS_SMP
+static void
+set_new_allctr_abandon_limit(Allctr_t *allctr);
+static void
+abandon_carrier(Allctr_t *allctr, Carrier_t *crr);
+
+
+static ERTS_INLINE void
+check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp)
+{
+ Carrier_t *crr;
+
+ if (busy_pcrr_pp && *busy_pcrr_pp)
+ return;
+
+ if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ return;
+
+ allctr->cpool.check_limit_count--;
+ if (--allctr->cpool.check_limit_count <= 0)
+ set_new_allctr_abandon_limit(allctr);
+
+ if (!erts_thr_progress_is_managed_thread())
+ return;
+
+ if (allctr->cpool.disable_abandon)
+ return;
+
+ if (allctr->mbcs.blocks.curr.size > allctr->cpool.abandon_limit)
+ return;
+
+
+ crr = FBLK_TO_MBC(fblk);
+
+ if (allctr->main_carrier == crr)
+ return;
+
+ if (crr->cpool.blocks_size > crr->cpool.abandon_limit)
+ return;
+
+ if (crr->cpool.thr_prgr != ERTS_THR_PRGR_INVALID
+ && !erts_thr_progress_has_reached(crr->cpool.thr_prgr))
+ return;
+
+ abandon_carrier(allctr, crr);
+}
+
void
erts_alcu_check_delayed_dealloc(Allctr_t *allctr,
int limit,
@@ -1274,78 +1871,88 @@ erts_alcu_check_delayed_dealloc(Allctr_t *allctr,
}
#endif
-#define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \
- handle_delayed_dealloc((Allctr), (Locked), 1, \
+#define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \
+ handle_delayed_dealloc((Allctr), (Locked), 1, \
ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL, NULL)
+static void
+dealloc_block(Allctr_t *allctr, void *ptr, int dec_cc_on_redirect)
+{
+ Block_t *blk = UMEM2BLK(ptr);
+
+ ERTS_SMP_LC_ASSERT(!allctr->thread_safe
+ || erts_lc_mtx_is_locked(&allctr->mutex));
+
+ if (IS_SBC_BLK(blk))
+ destroy_carrier(allctr, blk, NULL);
+#ifndef ERTS_SMP
+ else
+ mbc_free(allctr, ptr, NULL);
+#else
+ else if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ mbc_free(allctr, ptr, NULL);
+ else {
+ Carrier_t *busy_pcrr_p;
+ Allctr_t *used_allctr;
+ used_allctr = get_used_allctr(allctr, ERTS_ALC_TS_PREF_LOCK_NO, ptr,
+ NULL, &busy_pcrr_p);
+ if (used_allctr == allctr) {
+ mbc_free(allctr, ptr, &busy_pcrr_p);
+ clear_busy_pool_carrier(allctr, busy_pcrr_p);
+ }
+ else {
+ /* Carrier migrated; need to redirect block to new owner... */
+ int cinit = used_allctr->dd.ix - allctr->dd.ix;
+
+ ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p);
+
+ if (dec_cc_on_redirect)
+ DEC_CC(allctr->calls.this_free);
+ if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit))
+ erts_alloc_notify_delayed_dealloc(used_allctr->ix);
+ }
+ }
+#endif
+}
+
/* Multi block carrier alloc/realloc/free ... */
/* NOTE! mbc_alloc() may in case of memory shortage place the requested
* block in a sbc.
*/
static ERTS_INLINE void *
-mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp, Uint32 *alcu_flgsp)
+mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp)
{
Block_t *blk;
Uint get_blk_sz;
- Uint sbmbct;
ASSERT(size);
ASSERT(size < allctr->sbc_threshold);
*blk_szp = get_blk_sz = UMEMSZ2BLKSZ(allctr, size);
- sbmbct = allctr->sbmbc_threshold;
- if (sbmbct) {
- if (get_blk_sz < sbmbct) {
- *alcu_flgsp |= ERTS_ALCU_FLG_SBMBC;
- if (get_blk_sz + allctr->min_block_size > sbmbct) {
- /* Since we use block size to determine if blocks are
- located in sbmbc or not... */
- get_blk_sz += allctr->min_block_size;
- }
- }
- }
-
-#ifdef ERTS_SMP
- if (allctr->dd.use)
- ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1);
-#endif
-
- blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0, *alcu_flgsp);
-
-#ifdef ERTS_SMP
- if (!blk && allctr->dd.use) {
- if (ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1))
- blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0,
- *alcu_flgsp);
- }
-#endif
+ blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0);
if (!blk) {
- if ((*alcu_flgsp) & ERTS_ALCU_FLG_SBMBC)
- blk = create_sbmbc(allctr, get_blk_sz);
- else {
- blk = create_carrier(allctr, get_blk_sz, CFLG_MBC);
+ blk = create_carrier(allctr, get_blk_sz, CFLG_MBC);
#if !HALFWORD_HEAP && !HAVE_SUPER_ALIGNED_MB_CARRIERS
- if (!blk) {
- /* Emergency! We couldn't create the carrier as we wanted.
- Try to place it in a sys_alloced sbc. */
- blk = create_carrier(allctr,
- size,
- (CFLG_SBC
- | CFLG_FORCE_SIZE
- | CFLG_FORCE_SYS_ALLOC));
- }
-#endif
+ if (!blk) {
+ /* Emergency! We couldn't create the carrier as we wanted.
+ Try to place it in a sys_alloced sbc. */
+ blk = create_carrier(allctr,
+ size,
+ (CFLG_SBC
+ | CFLG_FORCE_SIZE
+ | CFLG_FORCE_SYS_ALLOC));
}
+#endif
}
#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
if (IS_MBC_BLK(blk)) {
- (*allctr->link_free_block)(allctr, blk, *alcu_flgsp);
+ (*allctr->link_free_block)(allctr, blk);
HARD_CHECK_BLK_CARRIER(allctr, blk);
- (*allctr->unlink_free_block)(allctr, blk, *alcu_flgsp);
+ (*allctr->unlink_free_block)(allctr, blk);
}
#endif
@@ -1359,8 +1966,7 @@ mbc_alloc_finalize(Allctr_t *allctr,
UWord flags,
Carrier_t *crr,
Uint want_blk_sz,
- int valid_blk_info,
- Uint32 alcu_flgs)
+ int valid_blk_info)
{
Uint blk_sz;
Uint nxt_blk_sz;
@@ -1392,7 +1998,7 @@ mbc_alloc_finalize(Allctr_t *allctr,
SET_PREV_BLK_FREE(allctr, nxt_nxt_blk);
}
}
- (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs);
+ (*allctr->link_free_block)(allctr, nxt_blk);
ASSERT(IS_NOT_LAST_BLK(blk));
ASSERT(IS_FREE_BLK(nxt_blk));
@@ -1433,7 +2039,8 @@ mbc_alloc_finalize(Allctr_t *allctr,
ASSERT(ABLK_TO_MBC(blk) == crr);
}
- STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs);
+ ERTS_ALC_CPOOL_ALLOC_OP(allctr);
+ STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs);
ASSERT(IS_ALLOCED_BLK(blk));
ASSERT(blk_sz == MBC_BLK_SZ(blk));
@@ -1453,8 +2060,7 @@ mbc_alloc(Allctr_t *allctr, Uint size)
{
Block_t *blk;
Uint blk_sz;
- Uint32 alcu_flgs = 0;
- blk = mbc_alloc_block(allctr, size, &blk_sz, &alcu_flgs);
+ blk = mbc_alloc_block(allctr, size, &blk_sz);
if (!blk)
return NULL;
if (IS_MBC_BLK(blk))
@@ -1464,35 +2070,34 @@ mbc_alloc(Allctr_t *allctr, Uint size)
GET_BLK_HDR_FLGS(blk),
FBLK_TO_MBC(blk),
blk_sz,
- 1,
- alcu_flgs);
+ 1);
return BLK2UMEM(blk);
}
static void
-mbc_free(Allctr_t *allctr, void *p)
+mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp)
{
Uint is_first_blk;
Uint is_last_blk;
- Uint32 alcu_flgs = 0;
Uint blk_sz;
Block_t *blk;
Block_t *nxt_blk;
-
+ Carrier_t *crr;
ASSERT(p);
blk = UMEM2BLK(p);
blk_sz = MBC_ABLK_SZ(blk);
- if (blk_sz < allctr->sbmbc_threshold)
- alcu_flgs |= ERTS_ALCU_FLG_SBMBC;
ASSERT(IS_MBC_BLK(blk));
ASSERT(blk_sz >= allctr->min_block_size);
HARD_CHECK_BLK_CARRIER(allctr, blk);
- STAT_MBC_BLK_FREE(allctr, blk_sz, alcu_flgs);
+ crr = ABLK_TO_MBC(blk);
+
+ ERTS_ALC_CPOOL_FREE_OP(allctr);
+ STAT_MBC_BLK_FREE(allctr, crr, busy_pcrr_pp, blk_sz, alcu_flgs);
is_first_blk = IS_MBC_FIRST_ABLK(allctr, blk);
is_last_blk = IS_LAST_BLK(blk);
@@ -1501,7 +2106,7 @@ mbc_free(Allctr_t *allctr, void *p)
ASSERT(!is_first_blk);
/* Coalesce with previous block... */
blk = PREV_BLK(blk);
- (*allctr->unlink_free_block)(allctr, blk, alcu_flgs);
+ (*allctr->unlink_free_block)(allctr, blk);
blk_sz += MBC_FBLK_SZ(blk);
is_first_blk = IS_MBC_FIRST_FBLK(allctr, blk);
@@ -1517,7 +2122,7 @@ mbc_free(Allctr_t *allctr, void *p)
nxt_blk = BLK_AFTER(blk, blk_sz);
if (IS_FREE_BLK(nxt_blk)) {
/* Coalesce with next block... */
- (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs);
+ (*allctr->unlink_free_block)(allctr, nxt_blk);
blk_sz += MBC_FBLK_SZ(nxt_blk);
SET_MBC_FBLK_SZ(blk, blk_sz);
@@ -1550,19 +2155,20 @@ mbc_free(Allctr_t *allctr, void *p)
if (is_first_blk
&& is_last_blk
&& allctr->main_carrier != FIRST_BLK_TO_MBC(allctr, blk)) {
- if (alcu_flgs & ERTS_ALCU_FLG_SBMBC)
- destroy_sbmbc(allctr, blk);
- else
- destroy_carrier(allctr, blk);
+ destroy_carrier(allctr, blk, busy_pcrr_pp);
}
else {
- (*allctr->link_free_block)(allctr, blk, alcu_flgs);
+ (*allctr->link_free_block)(allctr, blk);
HARD_CHECK_BLK_CARRIER(allctr, blk);
+#ifdef ERTS_SMP
+ check_abandon_carrier(allctr, blk, busy_pcrr_pp);
+#endif
}
}
static void *
-mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
+mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs,
+ Carrier_t **busy_pcrr_pp)
{
void *new_p;
Uint old_blk_sz;
@@ -1576,11 +2182,6 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
Uint is_last_blk;
#endif /* #ifndef MBC_REALLOC_ALWAYS_MOVES */
-#ifdef ERTS_SMP
- if (allctr->dd.use)
- ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1);
-#endif
-
ASSERT(p);
ASSERT(size);
ASSERT(size < allctr->sbc_threshold);
@@ -1594,13 +2195,13 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
return NULL;
#else /* !MBC_REALLOC_ALWAYS_MOVES */
+
+#ifdef ERTS_SMP
+ if (busy_pcrr_pp && *busy_pcrr_pp)
+ goto realloc_move; /* Don't want to use carrier in pool */
+#endif
+
get_blk_sz = blk_sz = UMEMSZ2BLKSZ(allctr, size);
- if ((alcu_flgs & ERTS_ALCU_FLG_SBMBC)
- && (blk_sz + allctr->min_block_size > allctr->sbmbc_threshold)) {
- /* Since we use block size to determine if blocks are
- located in sbmbc or not... */
- get_blk_sz = blk_sz + allctr->min_block_size;
- }
ASSERT(IS_ALLOCED_BLK(blk));
ASSERT(IS_MBC_BLK(blk));
@@ -1648,8 +2249,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
new_blk = (*allctr->get_free_block)(allctr,
get_blk_sz,
cand_blk,
- cand_blk_sz,
- alcu_flgs);
+ cand_blk_sz);
if (new_blk || cand_blk != blk)
goto move_into_new_blk;
}
@@ -1671,8 +2271,11 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
nxt_blk = BLK_AFTER(blk, blk_sz);
- STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs);
- STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs);
+ crr = ABLK_TO_MBC(blk);
+
+ ERTS_ALC_CPOOL_REALLOC_OP(allctr);
+ STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs);
+ STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs);
ASSERT(MBC_BLK_SZ(blk) >= allctr->min_block_size);
@@ -1680,7 +2283,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
if (IS_FREE_BLK(nxt_nxt_blk)) {
/* Coalesce with next free block... */
nxt_blk_sz += MBC_FBLK_SZ(nxt_nxt_blk);
- (*allctr->unlink_free_block)(allctr, nxt_nxt_blk, alcu_flgs);
+ (*allctr->unlink_free_block)(allctr, nxt_nxt_blk);
is_last_blk = GET_LAST_BLK_HDR_FLG(nxt_nxt_blk);
}
@@ -1690,12 +2293,11 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz);
}
- crr = ABLK_TO_MBC(blk);
SET_MBC_FBLK_HDR(nxt_blk, nxt_blk_sz,
SBH_THIS_FREE | (is_last_blk ? SBH_LAST_BLK : 0),
crr);
- (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs);
+ (*allctr->link_free_block)(allctr, nxt_blk);
ASSERT(IS_ALLOCED_BLK(blk));
@@ -1718,6 +2320,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
HARD_CHECK_BLK_CARRIER(allctr, blk);
+#ifdef ERTS_SMP
+ check_abandon_carrier(allctr, nxt_blk, NULL);
+#endif
+
return p;
}
@@ -1727,11 +2333,12 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
nxt_blk = BLK_AFTER(blk, old_blk_sz);
nxt_blk_sz = MBC_BLK_SZ(nxt_blk);
if (IS_FREE_BLK(nxt_blk) && get_blk_sz <= old_blk_sz + nxt_blk_sz) {
+ Carrier_t* crr = ABLK_TO_MBC(blk);
/* Grow into next block... */
HARD_CHECK_BLK_CARRIER(allctr, blk);
- (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs);
+ (*allctr->unlink_free_block)(allctr, nxt_blk);
nxt_blk_sz -= blk_sz - old_blk_sz;
is_last_blk = IS_LAST_BLK(nxt_blk);
@@ -1756,7 +2363,6 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
}
}
else {
- Carrier_t* crr = ABLK_TO_MBC(blk);
SET_MBC_ABLK_SZ(blk, blk_sz);
nxt_blk = BLK_AFTER(blk, blk_sz);
@@ -1767,15 +2373,15 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
else
SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz);
- (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs);
+ (*allctr->link_free_block)(allctr, nxt_blk);
ASSERT(IS_FREE_BLK(nxt_blk));
ASSERT(FBLK_TO_MBC(nxt_blk) == crr);
}
- STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs);
- STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs);
-
+ ERTS_ALC_CPOOL_REALLOC_OP(allctr);
+ STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs);
+ STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs);
ASSERT(IS_ALLOCED_BLK(blk));
ASSERT(blk_sz == MBC_BLK_SZ(blk));
@@ -1828,13 +2434,16 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
if (cand_blk_sz < get_blk_sz) {
/* We wont fit in cand_blk get a new one */
+#ifdef ERTS_SMP
+ realloc_move:
+#endif
#endif /* !MBC_REALLOC_ALWAYS_MOVES */
new_p = mbc_alloc(allctr, size);
if (!new_p)
return NULL;
sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ));
- mbc_free(allctr, p);
+ mbc_free(allctr, p, busy_pcrr_pp);
return new_p;
@@ -1847,8 +2456,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
new_blk = (*allctr->get_free_block)(allctr,
get_blk_sz,
cand_blk,
- cand_blk_sz,
- alcu_flgs);
+ cand_blk_sz);
move_into_new_blk:
/*
* new_blk, and cand_blk have to be correctly set
@@ -1862,11 +2470,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
GET_BLK_HDR_FLGS(new_blk),
FBLK_TO_MBC(new_blk),
blk_sz,
- 1,
- alcu_flgs);
+ 1);
new_p = BLK2UMEM(new_blk);
sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ));
- mbc_free(allctr, p);
+ mbc_free(allctr, p, NULL);
return new_p;
}
else {
@@ -1886,7 +2493,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
HARD_CHECK_BLK_CARRIER(allctr, blk);
- (*allctr->unlink_free_block)(allctr, new_blk, alcu_flgs); /* prev */
+ (*allctr->unlink_free_block)(allctr, new_blk); /* prev */
if (is_last_blk)
new_blk_flgs |= LAST_BLK_HDR_FLG;
@@ -1895,7 +2502,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
if (IS_FREE_BLK(nxt_blk)) {
new_blk_flgs |= GET_LAST_BLK_HDR_FLG(nxt_blk);
new_blk_sz += MBC_FBLK_SZ(nxt_blk);
- (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs);
+ (*allctr->unlink_free_block)(allctr, nxt_blk);
}
}
@@ -1920,10 +2527,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
new_blk_flgs,
crr,
blk_sz,
- 0,
- alcu_flgs);
+ 0);
- STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs);
+ ERTS_ALC_CPOOL_FREE_OP(allctr);
+ STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs);
return new_p;
}
@@ -1931,6 +2538,635 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
#endif /* !MBC_REALLOC_ALWAYS_MOVES */
}
+#ifdef ERTS_SMP
+
+#define ERTS_ALC_MAX_DEALLOC_CARRIER 10
+#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 10
+#define ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT 100
+#define ERTS_ALC_CPOOL_MAX_NO_CARRIERS 5
+#define ERTS_ALC_CPOOL_INSERT_ALLOWED_OFFSET 100
+#define ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS 3
+
+#define ERTS_ALC_CPOOL_PTR_MOD_MRK (((erts_aint_t) 1) << 0)
+#define ERTS_ALC_CPOOL_PTR_DEL_MRK (((erts_aint_t) 1) << 1)
+
+#define ERTS_ALC_CPOOL_PTR_MRKS \
+ (ERTS_ALC_CPOOL_PTR_MOD_MRK | ERTS_ALC_CPOOL_PTR_DEL_MRK)
+
+/*
+ * When setting multiple mod markers we always
+ * set mod markers in pointer order and always
+ * on next pointers before prev pointers.
+ */
+
+typedef union {
+ ErtsAlcCPoolData_t sentinel;
+ char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAlcCPoolData_t))];
+} ErtsAlcCrrPool_t;
+
+#if ERTS_ALC_A_INVALID != 0
+# error "Carrier pool implementation assumes ERTS_ALC_A_INVALID == 0"
+#endif
+#if ERTS_ALC_A_MIN <= ERTS_ALC_A_INVALID
+# error "Carrier pool implementation assumes ERTS_ALC_A_MIN > ERTS_ALC_A_INVALID"
+#endif
+
+/*
+ * The pool is only allowed to be manipulated by managed
+ * threads except in the alloc_SUITE:cpool case. In this
+ * test case carrier_pool[ERTS_ALC_A_INVALID] will be
+ * used.
+ */
+
+static ErtsAlcCrrPool_t carrier_pool[ERTS_ALC_A_MAX+1] erts_align_attribute(ERTS_CACHE_LINE_SIZE);
+
+#define ERTS_ALC_CPOOL_MAX_BACKOFF (1 << 8)
+
+static int
+backoff(int n)
+{
+ int i;
+
+ for (i = 0; i < n; i++)
+ ERTS_SPIN_BODY;
+
+ if (n >= ERTS_ALC_CPOOL_MAX_BACKOFF)
+ return ERTS_ALC_CPOOL_MAX_BACKOFF;
+ else
+ return n << 1;
+}
+
+static int
+cpool_dbg_is_in_pool(Allctr_t *allctr, Carrier_t *crr)
+{
+ ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel;
+ ErtsAlcCPoolData_t *cpdp = sentinel;
+ Carrier_t *tmp_crr;
+
+ while (1) {
+ cpdp = (ErtsAlcCPoolData_t *) (erts_atomic_read_ddrb(&cpdp->next) & ~FLG_MASK);
+ if (cpdp == sentinel)
+ return 0;
+ tmp_crr = (Carrier_t *) (((char *) cpdp) - offsetof(Carrier_t, cpool));
+ if (tmp_crr == crr)
+ return 1;
+ }
+}
+
+static int
+cpool_is_empty(Allctr_t *allctr)
+{
+ ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel;
+ return ((erts_atomic_read_rb(&sentinel->next) == (erts_aint_t) sentinel)
+ && (erts_atomic_read_rb(&sentinel->prev) == (erts_aint_t) sentinel));
+}
+
+static ERTS_INLINE ErtsAlcCPoolData_t *
+cpool_aint2cpd(erts_aint_t aint)
+{
+ return (ErtsAlcCPoolData_t *) (aint & ~ERTS_ALC_CPOOL_PTR_MRKS);
+}
+
+static ERTS_INLINE erts_aint_t
+cpool_read(erts_atomic_t *aptr)
+{
+ return erts_atomic_read_acqb(aptr);
+}
+
+static ERTS_INLINE void
+cpool_init(erts_atomic_t *aptr, erts_aint_t val)
+{
+ erts_atomic_set_nob(aptr, val);
+}
+
+static ERTS_INLINE void
+cpool_set_mod_marked(erts_atomic_t *aptr, erts_aint_t new, erts_aint_t old)
+{
+#ifdef ERTS_ALC_CPOOL_DEBUG
+ erts_aint_t act = erts_atomic_xchg_relb(aptr, new);
+ ERTS_ALC_CPOOL_ASSERT(act == (old | ERTS_ALC_CPOOL_PTR_MOD_MRK));
+#else
+ erts_atomic_set_relb(aptr, new);
+#endif
+}
+
+
+static ERTS_INLINE erts_aint_t
+cpool_try_mod_mark_exp(erts_atomic_t *aptr, erts_aint_t exp)
+{
+ ERTS_ALC_CPOOL_ASSERT((exp & ERTS_ALC_CPOOL_PTR_MOD_MRK) == 0);
+ return erts_atomic_cmpxchg_nob(aptr, exp | ERTS_ALC_CPOOL_PTR_MOD_MRK, exp);
+}
+
+static ERTS_INLINE erts_aint_t
+cpool_mod_mark_exp(erts_atomic_t *aptr, erts_aint_t exp)
+{
+ int b;
+ erts_aint_t act;
+ ERTS_ALC_CPOOL_ASSERT((exp & ERTS_ALC_CPOOL_PTR_MOD_MRK) == 0);
+ while (1) {
+ act = erts_atomic_cmpxchg_nob(aptr,
+ exp | ERTS_ALC_CPOOL_PTR_MOD_MRK,
+ exp);
+ if (act == exp)
+ return exp;
+ b = 1;
+ do {
+ if ((act & ~ERTS_ALC_CPOOL_PTR_MOD_MRK) != exp)
+ return act;
+ b = backoff(b);
+ act = erts_atomic_read_nob(aptr);
+ } while (act != exp);
+ }
+}
+
+static ERTS_INLINE erts_aint_t
+cpool_mod_mark(erts_atomic_t *aptr)
+{
+ int b;
+ erts_aint_t act, exp;
+ act = cpool_read(aptr);
+ while (1) {
+ b = 1;
+ while (act & ERTS_ALC_CPOOL_PTR_MOD_MRK) {
+ b = backoff(b);
+ act = erts_atomic_read_nob(aptr);
+ }
+ exp = act;
+ act = erts_atomic_cmpxchg_acqb(aptr,
+ exp | ERTS_ALC_CPOOL_PTR_MOD_MRK,
+ exp);
+ if (act == exp)
+ return exp;
+ }
+}
+
+static void
+cpool_insert(Allctr_t *allctr, Carrier_t *crr)
+{
+ ErtsAlcCPoolData_t *cpd1p, *cpd2p;
+ erts_aint_t val;
+ ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel;
+
+ ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */
+ || erts_thr_progress_is_managed_thread());
+ ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_read_nob(&crr->allctr)
+ == (erts_aint_t) allctr);
+
+ erts_atomic_add_nob(&allctr->cpool.stat.blocks_size,
+ (erts_aint_t) crr->cpool.blocks_size);
+ erts_atomic_add_nob(&allctr->cpool.stat.no_blocks,
+ (erts_aint_t) crr->cpool.blocks);
+ erts_atomic_add_nob(&allctr->cpool.stat.carriers_size,
+ (erts_aint_t) CARRIER_SZ(crr));
+ erts_atomic_inc_nob(&allctr->cpool.stat.no_carriers);
+
+ erts_smp_atomic_set_nob(&crr->allctr,
+ ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL);
+
+ /*
+ * We search in 'next' direction and begin by passing
+ * one element before trying to insert. This in order to
+ * avoid contention with threads fetching elements.
+ */
+
+ val = cpool_read(&sentinel->next);
+
+ /* Find a predecessor to be, and set mod marker on its next ptr */
+
+ while (1) {
+ cpd1p = cpool_aint2cpd(val);
+ if (cpd1p == sentinel) {
+ val = cpool_mod_mark(&cpd1p->next);
+ break;
+ }
+ val = cpool_read(&cpd1p->next);
+ if (!(val & ERTS_ALC_CPOOL_PTR_MRKS)) {
+ erts_aint_t tmp = cpool_try_mod_mark_exp(&cpd1p->next, val);
+ if (tmp == val) {
+ val = tmp;
+ break;
+ }
+ val = tmp;
+ }
+ }
+
+ /* Set mod marker on prev ptr of the to be successor */
+
+ cpd2p = cpool_aint2cpd(val);
+
+ cpool_init(&crr->cpool.next, (erts_aint_t) cpd2p);
+ cpool_init(&crr->cpool.prev, (erts_aint_t) cpd1p);
+
+ val = (erts_aint_t) cpd1p;
+
+ while (1) {
+ int b;
+ erts_aint_t tmp;
+
+ tmp = cpool_mod_mark_exp(&cpd2p->prev, val);
+ if (tmp == val)
+ break;
+ b = 1;
+ do {
+ b = backoff(b);
+ tmp = cpool_read(&cpd2p->prev);
+ } while (tmp != val);
+ }
+
+ /* Write pointers to this element in successor and predecessor */
+
+ cpool_set_mod_marked(&cpd1p->next,
+ (erts_aint_t) &crr->cpool,
+ (erts_aint_t) cpd2p);
+ cpool_set_mod_marked(&cpd2p->prev,
+ (erts_aint_t) &crr->cpool,
+ (erts_aint_t) cpd1p);
+}
+
+static void
+cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr)
+{
+ ErtsAlcCPoolData_t *cpd1p, *cpd2p;
+ erts_aint_t val;
+#ifdef ERTS_ALC_CPOOL_DEBUG
+ ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel;
+#endif
+
+ ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */
+ || erts_thr_progress_is_managed_thread());
+ ERTS_ALC_CPOOL_ASSERT(sentinel != &crr->cpool);
+
+ /* Set mod marker on next ptr of our predecessor */
+
+ val = (erts_aint_t) &crr->cpool;
+ while (1) {
+ erts_aint_t tmp;
+ cpd1p = cpool_aint2cpd(cpool_read(&crr->cpool.prev));
+ tmp = cpool_mod_mark_exp(&cpd1p->next, val);
+ if (tmp == val)
+ break;
+ }
+
+ /* Set mod marker on our next ptr */
+
+ val = cpool_mod_mark(&crr->cpool.next);
+
+ /* Set mod marker on the prev ptr of our successor */
+
+ cpd2p = cpool_aint2cpd(val);
+
+ val = (erts_aint_t) &crr->cpool;
+
+ while (1) {
+ int b;
+ erts_aint_t tmp;
+
+ tmp = cpool_mod_mark_exp(&cpd2p->prev, val);
+ if (tmp == val)
+ break;
+ b = 1;
+ do {
+ b = backoff(b);
+ tmp = cpool_read(&cpd2p->prev);
+ } while (tmp != val);
+ }
+
+ /* Set mod marker on our prev ptr */
+
+ val = (erts_aint_t) cpd1p;
+
+ while (1) {
+ int b;
+ erts_aint_t tmp;
+
+ tmp = cpool_mod_mark_exp(&crr->cpool.prev, val);
+ if (tmp == val)
+ break;
+ b = 1;
+ do {
+ b = backoff(b);
+ tmp = cpool_read(&cpd2p->prev);
+ } while (tmp != val);
+ }
+
+ /* Write pointers past this element in predecessor and successor */
+
+ cpool_set_mod_marked(&cpd1p->next,
+ (erts_aint_t) cpd2p,
+ (erts_aint_t) &crr->cpool);
+ cpool_set_mod_marked(&cpd2p->prev,
+ (erts_aint_t) cpd1p,
+ (erts_aint_t) &crr->cpool);
+
+ /* Repleace mod markers with delete markers on this element */
+ cpool_set_mod_marked(&crr->cpool.next,
+ ((erts_aint_t) cpd2p) | ERTS_ALC_CPOOL_PTR_DEL_MRK,
+ ((erts_aint_t) cpd2p) | ERTS_ALC_CPOOL_PTR_MOD_MRK);
+ cpool_set_mod_marked(&crr->cpool.prev,
+ ((erts_aint_t) cpd1p) | ERTS_ALC_CPOOL_PTR_DEL_MRK,
+ ((erts_aint_t) cpd1p) | ERTS_ALC_CPOOL_PTR_MOD_MRK);
+
+ crr->cpool.thr_prgr = erts_thr_progress_later(NULL);
+
+ erts_atomic_add_nob(&prev_allctr->cpool.stat.blocks_size,
+ -((erts_aint_t) crr->cpool.blocks_size));
+ erts_atomic_add_nob(&prev_allctr->cpool.stat.no_blocks,
+ -((erts_aint_t) crr->cpool.blocks));
+ erts_atomic_add_nob(&prev_allctr->cpool.stat.carriers_size,
+ -((erts_aint_t) CARRIER_SZ(crr)));
+ erts_atomic_dec_wb(&prev_allctr->cpool.stat.no_carriers);
+
+}
+
+static Carrier_t *
+cpool_fetch(Allctr_t *allctr, UWord size)
+{
+ int i;
+ Carrier_t *crr;
+ ErtsAlcCPoolData_t *cpdp;
+ ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel;
+
+ ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */
+ || erts_thr_progress_is_managed_thread());
+
+ i = 0;
+
+ /* First; check our own pending dealloc carrier list... */
+ crr = allctr->cpool.dc_list.last;
+ while (crr && i < ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) {
+ if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) {
+ unlink_carrier(&allctr->cpool.dc_list, crr);
+#ifdef ERTS_ALC_CPOOL_DEBUG
+ ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr,
+ ((erts_aint_t) allctr))
+ == (((erts_aint_t) allctr) & ~FLG_MASK));
+#else
+ erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr));
+#endif
+ return crr;
+ }
+ crr = crr->prev;
+ i++;
+ }
+
+ /* ... then the pool ... */
+
+ /*
+ * We search in 'prev' direction and begin by passing
+ * one element before trying to fetch. This in order to
+ * avoid contention with threads inserting elements.
+ */
+
+ cpdp = cpool_aint2cpd(cpool_read(&sentinel->prev));
+ if (cpdp == sentinel)
+ return NULL;
+
+ while (i < ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) {
+ erts_aint_t exp;
+ cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev));
+ if (cpdp == sentinel) {
+ cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev));
+ if (cpdp == sentinel)
+ return NULL;
+ i = ERTS_ALC_CPOOL_MAX_FETCH_INSPECT; /* Last one to inspect */
+ }
+ crr = (Carrier_t *) (((char *) cpdp) - offsetof(Carrier_t, cpool));
+ exp = erts_smp_atomic_read_rb(&crr->allctr);
+ if (((exp & (ERTS_CRR_ALCTR_FLG_IN_POOL|ERTS_CRR_ALCTR_FLG_BUSY))
+ == ERTS_CRR_ALCTR_FLG_IN_POOL)
+ && (erts_atomic_read_nob(&cpdp->max_size) >= size)) {
+ erts_aint_t act;
+ /* Try to fetch it... */
+ act = erts_smp_atomic_cmpxchg_mb(&crr->allctr,
+ (erts_aint_t) allctr,
+ exp);
+ if (act == exp) {
+ cpool_delete(allctr, ((Allctr_t *) (act & ~FLG_MASK)), crr);
+ return crr;
+ }
+ }
+ i++;
+ }
+ return NULL;
+}
+
+static void
+check_pending_dealloc_carrier(Allctr_t *allctr,
+ int *need_thr_progress,
+ ErtsThrPrgrVal *thr_prgr_p,
+ int *need_more_work)
+{
+ Carrier_t *crr = allctr->cpool.dc_list.first;
+
+ if (crr) {
+ ErtsThrPrgrVal current = erts_thr_progress_current();
+ int i = 0;
+
+ do {
+ Carrier_t *dcrr;
+
+ if (!erts_thr_progress_has_reached_this(current, crr->cpool.thr_prgr))
+ break;
+
+ dcrr = crr;
+ crr = crr->next;
+ dealloc_carrier(allctr, dcrr, ERTS_MSEG_FLG_2POW);
+ i++;
+ } while (crr && i < ERTS_ALC_MAX_DEALLOC_CARRIER);
+
+ allctr->cpool.dc_list.first = crr;
+ if (!crr)
+ allctr->cpool.dc_list.last = NULL;
+ else {
+ crr->prev = NULL;
+
+ if (need_more_work) {
+ ERTS_ALC_CPOOL_ASSERT(need_thr_progress && thr_prgr_p);
+ if (erts_thr_progress_has_reached_this(current, crr->cpool.thr_prgr))
+ *need_more_work = 1;
+ else {
+ *need_thr_progress = 1;
+ if (*thr_prgr_p == ERTS_THR_PRGR_INVALID
+ || erts_thr_progress_cmp(crr->cpool.thr_prgr,
+ *thr_prgr_p) < 0) {
+ *thr_prgr_p = crr->cpool.thr_prgr;
+ }
+ }
+ }
+ }
+ }
+}
+
+static void
+schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr)
+{
+ Allctr_t *used_allctr;
+ int check_pending_dealloc;
+ erts_aint_t max_size;
+
+ if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) {
+ dealloc_carrier(allctr, crr, ERTS_MSEG_FLG_2POW);
+ return;
+ }
+
+ used_allctr = crr->cpool.orig_allctr;
+
+ if (allctr != used_allctr) {
+ Block_t *blk = MBC_TO_FIRST_BLK(allctr, crr);
+ int cinit = used_allctr->dd.ix - allctr->dd.ix;
+
+ /*
+ * Receiver will recognize that this is a carrier to
+ * deallocate since the block is an mbc block that
+ * is free and last in carrier...
+ */
+ ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(blk));
+
+ ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, blk));
+ ERTS_ALC_CPOOL_ASSERT(crr == FBLK_TO_MBC(blk));
+ ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(used_allctr, blk));
+
+ if (ddq_enqueue(&used_allctr->dd.q, BLK2UMEM(blk), cinit))
+ erts_alloc_notify_delayed_dealloc(used_allctr->ix);
+ return;
+ }
+
+ if (crr->cpool.thr_prgr == ERTS_THR_PRGR_INVALID
+ || erts_thr_progress_has_reached(crr->cpool.thr_prgr)) {
+ dealloc_carrier(allctr, crr, ERTS_MSEG_FLG_2POW);
+ return;
+ }
+
+ max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr);
+ erts_atomic_set_nob(&crr->cpool.max_size, max_size);
+
+ crr->next = NULL;
+ crr->prev = allctr->cpool.dc_list.last;
+ if (allctr->cpool.dc_list.last) {
+ check_pending_dealloc = 1;
+ allctr->cpool.dc_list.last->next = crr;
+ }
+ else {
+ check_pending_dealloc = 0;
+ allctr->cpool.dc_list.first = crr;
+ }
+ allctr->cpool.dc_list.last = crr;
+ if (check_pending_dealloc)
+ check_pending_dealloc_carrier(allctr, NULL, NULL, NULL);
+ erts_alloc_ensure_handle_delayed_dealloc_call(allctr->ix);
+}
+
+static ERTS_INLINE void
+cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr)
+{
+ erts_atomic_init_nob(&crr->cpool.next, ERTS_AINT_NULL);
+ erts_atomic_init_nob(&crr->cpool.prev, ERTS_AINT_NULL);
+ crr->cpool.orig_allctr = allctr;
+ crr->cpool.thr_prgr = ERTS_THR_PRGR_INVALID;
+ erts_atomic_init_nob(&crr->cpool.max_size, 0);
+ crr->cpool.blocks = 0;
+ crr->cpool.blocks_size = 0;
+ if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ crr->cpool.abandon_limit = 0;
+ else {
+ UWord csz = CARRIER_SZ(crr);
+ UWord limit = csz*allctr->cpool.util_limit;
+ if (limit > csz)
+ limit /= 100;
+ else
+ limit = (csz/100)*allctr->cpool.util_limit;
+ crr->cpool.abandon_limit = limit;
+ }
+}
+
+static void
+set_new_allctr_abandon_limit(Allctr_t *allctr)
+{
+ UWord limit;
+ UWord csz;
+
+ allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT;
+
+ csz = allctr->mbcs.curr.norm.mseg.size;
+ csz += allctr->mbcs.curr.norm.sys_alloc.size;
+
+ limit = csz*allctr->cpool.util_limit;
+ if (limit > csz)
+ limit /= 100;
+ else
+ limit = (csz/100)*allctr->cpool.util_limit;
+
+ allctr->cpool.abandon_limit = limit;
+}
+
+static void
+abandon_carrier(Allctr_t *allctr, Carrier_t *crr)
+{
+ erts_aint_t max_size;
+
+ STAT_MBC_CPOOL_INSERT(allctr, crr);
+
+ unlink_carrier(&allctr->mbc_list, crr);
+
+ allctr->remove_mbc(allctr, crr);
+
+ max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr);
+ erts_atomic_set_nob(&crr->cpool.max_size, max_size);
+
+ cpool_insert(allctr, crr);
+
+ set_new_allctr_abandon_limit(allctr);
+}
+
+static void
+cpool_read_stat(Allctr_t *allctr, UWord *nocp, UWord *cszp, UWord *nobp, UWord *bszp)
+{
+ int i;
+ UWord noc = 0, csz = 0, nob = 0, bsz = 0;
+
+ /*
+ * We try to get consistent values, but after
+ * ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS failed
+ * tries we give up and present what we got...
+ */
+ for (i = 0; i <= ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS; i++) {
+ UWord tnoc, tcsz, tnob, tbsz;
+
+ tnoc = (UWord) (nocp
+ ? erts_atomic_read_nob(&allctr->cpool.stat.no_carriers)
+ : 0);
+ tcsz = (UWord) (cszp
+ ? erts_atomic_read_nob(&allctr->cpool.stat.carriers_size)
+ : 0);
+ tnob = (UWord) (nobp
+ ? erts_atomic_read_nob(&allctr->cpool.stat.no_blocks)
+ : 0);
+ tbsz = (UWord) (bszp
+ ? erts_atomic_read_nob(&allctr->cpool.stat.blocks_size)
+ : 0);
+ if (tnoc == noc && tcsz == csz && tnob == nob && tbsz == bsz)
+ break;
+ noc = tnoc;
+ csz = tcsz;
+ nob = tnob;
+ bsz = tbsz;
+ ERTS_THR_READ_MEMORY_BARRIER;
+ }
+
+ if (nocp)
+ *nocp = noc;
+ if (cszp)
+ *cszp = csz;
+ if (nobp)
+ *nobp = nob;
+ if (bszp)
+ *bszp = bsz;
+}
+
+
+#endif /* ERTS_SMP */
+
#ifdef DEBUG
#if HAVE_ERTS_MSEG
@@ -1974,81 +3210,6 @@ static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C,
#endif
static Block_t *
-create_sbmbc(Allctr_t *allctr, Uint umem_sz)
-{
- Block_t *blk;
- Uint blk_sz;
- Uint crr_sz = allctr->sbmbc_size;
- Carrier_t *crr;
-
-#if HALFWORD_HEAP
- if (allctr->mseg_opt.low_mem)
- crr = erts_alloc(ERTS_ALC_T_SBMBC_LOW, crr_sz);
- else
-#endif
- crr = erts_alloc(ERTS_ALC_T_SBMBC, crr_sz);
-
- INC_CC(allctr->calls.sbmbc_alloc);
- SET_CARRIER_HDR(crr, crr_sz, SCH_SYS_ALLOC|SCH_MBC, allctr);
-
- blk = MBC_TO_FIRST_BLK(allctr, crr);
-
- blk_sz = UNIT_FLOOR(crr_sz - MBC_HEADER_SIZE(allctr));
-
- SET_MBC_FBLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr);
-
- link_carrier(&allctr->sbmbc_list, crr);
-
- STAT_SBMBC_ALLOC(allctr, crr_sz);
- CHECK_1BLK_CARRIER(allctr, 0, 0, crr, crr_sz, blk, blk_sz);
- if (allctr->creating_mbc)
- (*allctr->creating_mbc)(allctr, crr, ERTS_ALCU_FLG_SBMBC);
-
- DEBUG_SAVE_ALIGNMENT(crr);
- return blk;
-}
-
-static void
-destroy_sbmbc(Allctr_t *allctr, Block_t *blk)
-{
- Uint crr_sz;
- Carrier_t *crr;
-
- ASSERT(IS_MBC_BLK(blk));
- ASSERT(IS_MBC_FIRST_FBLK(allctr, blk));
-
- crr = FIRST_BLK_TO_MBC(allctr, blk);
- crr_sz = CARRIER_SZ(crr);
-
-#ifdef DEBUG
- if (!allctr->stopped) {
- ASSERT(IS_LAST_BLK(blk));
-
-#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
- (*allctr->link_free_block)(allctr, blk, ERTS_ALCU_FLG_SBMBC);
- HARD_CHECK_BLK_CARRIER(allctr, blk);
- (*allctr->unlink_free_block)(allctr, blk, ERTS_ALCU_FLG_SBMBC);
-#endif
- }
-#endif
-
- STAT_SBMBC_FREE(allctr, crr_sz);
-
- unlink_carrier(&allctr->sbmbc_list, crr);
- if (allctr->destroying_mbc)
- (*allctr->destroying_mbc)(allctr, crr, ERTS_ALCU_FLG_SBMBC);
-
- INC_CC(allctr->calls.sbmbc_free);
-
-#if HALFWORD_HEAP
- if (allctr->mseg_opt.low_mem)
- erts_free(ERTS_ALC_T_SBMBC_LOW, crr);
- else
-#endif
- erts_free(ERTS_ALC_T_SBMBC, crr);
-}
-
-static Block_t *
create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
{
Block_t *blk;
@@ -2077,6 +3238,24 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz);
+#ifdef ERTS_SMP
+ allctr->cpool.disable_abandon = ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON;
+
+ if ((flags & (CFLG_MBC|CFLG_NO_CPOOL)) == CFLG_MBC
+ && ERTS_ALC_IS_CPOOL_ENABLED(allctr)
+ && erts_thr_progress_is_managed_thread()) {
+ crr = cpool_fetch(allctr, blk_sz);
+ if (crr) {
+ STAT_MBC_CPOOL_FETCH(allctr, crr);
+ link_carrier(&allctr->mbc_list, crr);
+ (*allctr->add_mbc)(allctr, crr);
+ blk = (*allctr->get_free_block)(allctr, blk_sz, NULL, 0);
+ ASSERT(blk);
+ return blk;
+ }
+ }
+#endif
+
#if HAVE_ERTS_MSEG
if (flags & CFLG_FORCE_SYS_ALLOC)
@@ -2204,11 +3383,15 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
allctr->main_carrier = crr;
}
+#ifdef ERTS_SMP
+ cpool_init_carrier_data(allctr, crr);
+#endif
+
link_carrier(&allctr->mbc_list, crr);
CHECK_1BLK_CARRIER(allctr, 0, is_mseg, crr, crr_sz, blk, blk_sz);
if (allctr->creating_mbc)
- (*allctr->creating_mbc)(allctr, crr, 0);
+ (*allctr->creating_mbc)(allctr, crr);
}
@@ -2352,14 +3535,21 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
}
static void
-destroy_carrier(Allctr_t *allctr, Block_t *blk)
+dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, Uint mseg_flags)
{
- Uint crr_sz;
- Carrier_t *crr;
#if HAVE_ERTS_MSEG
- Uint is_mseg = 0;
- Uint mseg_flags = ERTS_MSEG_FLG_NONE;
+ if (IS_MSEG_CARRIER(crr))
+ alcu_mseg_dealloc(allctr, crr, CARRIER_SZ(crr), mseg_flags);
+ else
#endif
+ alcu_sys_free(allctr, crr);
+}
+
+static void
+destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp)
+{
+ Uint crr_sz;
+ Carrier_t *crr;
if (IS_SBC_BLK(blk)) {
Uint blk_sz = SBC_BLK_SZ(blk);
@@ -2372,7 +3562,6 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
#if HAVE_ERTS_MSEG
if (IS_MSEG_CARRIER(crr)) {
- is_mseg++;
ASSERT(crr_sz % MSEG_UNIT_SZ == 0);
STAT_MSEG_SBC_FREE(allctr, crr_sz, blk_sz);
}
@@ -2382,6 +3571,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
unlink_carrier(&allctr->sbc_list, crr);
+ dealloc_carrier(allctr, crr, ERTS_MSEG_FLG_NONE);
}
else {
ASSERT(IS_MBC_FIRST_FBLK(allctr, blk));
@@ -2400,30 +3590,36 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
}
#endif
-#if HAVE_ERTS_MSEG
- if (IS_MSEG_CARRIER(crr)) {
- is_mseg++;
- ASSERT(crr_sz % MSEG_UNIT_SZ == 0);
- STAT_MSEG_MBC_FREE(allctr, crr_sz);
- mseg_flags = ERTS_MSEG_FLG_2POW;
+ if (allctr->destroying_mbc)
+ (*allctr->destroying_mbc)(allctr, crr);
+
+#ifdef ERTS_SMP
+ if (busy_pcrr_pp && *busy_pcrr_pp) {
+ ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr);
+ *busy_pcrr_pp = NULL;
+ cpool_delete(allctr, allctr, crr);
}
else
#endif
- STAT_SYS_ALLOC_MBC_FREE(allctr, crr_sz);
+ {
+ unlink_carrier(&allctr->mbc_list, crr);
+#if HAVE_ERTS_MSEG
+ if (IS_MSEG_CARRIER(crr)) {
+ ASSERT(crr_sz % MSEG_UNIT_SZ == 0);
+ STAT_MSEG_MBC_FREE(allctr, crr_sz);
+ }
+ else
+#endif
+ STAT_SYS_ALLOC_MBC_FREE(allctr, crr_sz);
+ }
- unlink_carrier(&allctr->mbc_list, crr);
- if (allctr->destroying_mbc)
- (*allctr->destroying_mbc)(allctr, crr, 0);
+#ifdef ERTS_SMP
+ schedule_dealloc_carrier(allctr, crr);
+#else
+ dealloc_carrier(allctr, crr, ERTS_MSEG_FLG_2POW);
+#endif
}
-
-#if HAVE_ERTS_MSEG
- if (is_mseg) {
- alcu_mseg_dealloc(allctr, crr, crr_sz, mseg_flags);
- }
- else
-#endif
- alcu_sys_free(allctr, crr);
}
@@ -2457,19 +3653,19 @@ static struct {
Eterm lmbcs;
Eterm smbcs;
Eterm mbcgs;
- Eterm sbmbcs;
- Eterm sbmbct;
+ Eterm acul;
#if HAVE_ERTS_MSEG
Eterm mmc;
#endif
Eterm ycs;
- /* Eterm sbmbcs; */
-
Eterm fix_types;
Eterm mbcs;
+#ifdef ERTS_SMP
+ Eterm mbcs_pool;
+#endif
Eterm sbcs;
Eterm sys_alloc_carriers_size;
@@ -2494,8 +3690,6 @@ static struct {
Eterm mseg_dealloc;
Eterm mseg_realloc;
#endif
- Eterm sbmbc_alloc;
- Eterm sbmbc_free;
#ifdef DEBUG
Eterm end_of_atoms;
#endif
@@ -2551,19 +3745,19 @@ init_atoms(Allctr_t *allctr)
AM_INIT(lmbcs);
AM_INIT(smbcs);
AM_INIT(mbcgs);
- AM_INIT(sbmbcs);
- AM_INIT(sbmbct);
+ AM_INIT(acul);
#if HAVE_ERTS_MSEG
AM_INIT(mmc);
#endif
AM_INIT(ycs);
- /*AM_INIT(sbmbcs);*/
-
AM_INIT(fix_types);
AM_INIT(mbcs);
+#ifdef ERTS_SMP
+ AM_INIT(mbcs_pool);
+#endif
AM_INIT(sbcs);
AM_INIT(sys_alloc_carriers_size);
@@ -2588,8 +3782,6 @@ init_atoms(Allctr_t *allctr)
AM_INIT(mseg_dealloc);
AM_INIT(mseg_realloc);
#endif
- AM_INIT(sbmbc_free);
- AM_INIT(sbmbc_alloc);
#ifdef DEBUG
for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) {
@@ -2637,19 +3829,22 @@ ensure_atoms_initialized(Allctr_t *allctr)
* that would fit a small when size check is done may need to be built
* as a big when the actual build is performed. Caller is required to
* HRelease after build.
+ *
+ * Note, bld_unstable_uint() should have been called bld_unstable_uword()
+ * but we do not want to rename it...
*/
static ERTS_INLINE Eterm
-bld_unstable_uint(Uint **hpp, Uint *szp, Uint ui)
+bld_unstable_uint(Uint **hpp, Uint *szp, UWord ui)
{
Eterm res = THE_NON_VALUE;
if (szp)
- *szp += BIG_UINT_HEAP_SIZE;
+ *szp += BIG_UWORD_HEAP_SIZE(~((UWord) 0));
if (hpp) {
if (IS_USMALL(0, ui))
res = make_small(ui);
else {
- res = uint_to_big(ui, *hpp);
- *hpp += BIG_UINT_HEAP_SIZE;
+ res = uword_to_big(ui, *hpp);
+ *hpp += BIG_UWORD_HEAP_SIZE(ui);
}
}
return res;
@@ -2675,8 +3870,24 @@ add_4tup(Uint **hpp, Uint *szp, Eterm *lp,
bld_cons(hpp, szp, bld_tuple(hpp, szp, 4, el1, el2, el3, el4), *lp);
}
+static ERTS_INLINE void
+add_fix_types(Allctr_t *allctr, int internal, Uint **hpp, Uint *szp,
+ Eterm *lp, Eterm fix)
+{
+ if (allctr->fix) {
+ if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ add_2tup(hpp, szp, lp, am.fix_types, fix);
+ else if (internal)
+ add_3tup(hpp, szp, lp,
+ am.fix_types,
+ erts_bld_uword(hpp, szp, ~((UWord) 0)),
+ fix);
+ }
+}
+
static Eterm
sz_info_fix(Allctr_t *allctr,
+ int internal,
int *print_to_p,
void *print_to_arg,
Uint **hpp,
@@ -2684,36 +3895,67 @@ sz_info_fix(Allctr_t *allctr,
{
Eterm res;
int ix;
- ErtsAlcFixList_t *fix = allctr->fix;
- ASSERT(fix);
+ ASSERT(allctr->fix);
res = NIL;
- for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) {
- ErtsAlcType_t n = ix + ERTS_ALC_N_MIN_A_FIXED_SIZE;
- Uint alloced = (fix[ix].type_size * fix[ix].allocated);
- Uint used = fix[ix].type_size*fix[ix].used;
-
- if (print_to_p) {
- int to = *print_to_p;
- void *arg = print_to_arg;
- erts_print(to,
- arg,
- "fix type: %s %bpu %bpu\n",
- (char *) ERTS_ALC_N2TD(n),
- alloced,
- used);
- }
+ if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) {
+
+ if (internal) {
+ for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) {
+ ErtsAlcFixList_t *fix = &allctr->fix[ix];
+ UWord alloced = fix->type_size * fix->u.cpool.allocated;
+ UWord used = fix->type_size * fix->u.cpool.used;
+
+ if (print_to_p) {
+ int to = *print_to_p;
+ void *arg = print_to_arg;
+ erts_print(to,
+ arg,
+ "fix type internal: %s %bpu %bpu\n",
+ (char *) ERTS_ALC_N2TD(ERTS_ALC_N_MIN_A_FIXED_SIZE
+ + ix),
+ alloced,
+ used);
+ }
- if (hpp || szp) {
- add_3tup(hpp, szp, &res,
- fix_type_atoms[ix],
- bld_unstable_uint(hpp, szp, alloced),
- bld_unstable_uint(hpp, szp, used));
+ if (hpp || szp) {
+ add_3tup(hpp, szp, &res,
+ fix_type_atoms[ix],
+ bld_unstable_uint(hpp, szp, alloced),
+ bld_unstable_uint(hpp, szp, used));
+ }
+ }
}
}
+ else {
+ for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) {
+ ErtsAlcFixList_t *fix = &allctr->fix[ix];
+ UWord alloced = fix->type_size * fix->u.nocpool.allocated;
+ UWord used = fix->type_size*fix->u.nocpool.used;
+
+ if (print_to_p) {
+ int to = *print_to_p;
+ void *arg = print_to_arg;
+ erts_print(to,
+ arg,
+ "fix type: %s %bpu %bpu\n",
+ (char *) ERTS_ALC_N2TD(ERTS_ALC_N_MIN_A_FIXED_SIZE
+ + ix),
+ alloced,
+ used);
+ }
+
+ if (hpp || szp) {
+ add_3tup(hpp, szp, &res,
+ fix_type_atoms[ix],
+ bld_unstable_uint(hpp, szp, alloced),
+ bld_unstable_uint(hpp, szp, used));
+ }
+ }
+ }
return res;
}
@@ -2727,9 +3969,7 @@ sz_info_carriers(Allctr_t *allctr,
Uint *szp)
{
Eterm res = THE_NON_VALUE;
- Uint curr_size = (cs == &allctr->sbmbcs
- ? cs->curr.small_block.size
- : cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size);
+ UWord curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size;
if (print_to_p) {
int to = *print_to_p;
@@ -2743,7 +3983,7 @@ sz_info_carriers(Allctr_t *allctr,
cs->blocks.max_ever.size);
erts_print(to,
arg,
- "%scarriers size: %beu %bpu %bpu\n",
+ "%scarriers size: %bpu %bpu %bpu\n",
prefix,
curr_size,
cs->max.size,
@@ -2767,6 +4007,62 @@ sz_info_carriers(Allctr_t *allctr,
return res;
}
+#ifdef ERTS_SMP
+
+static Eterm
+info_cpool(Allctr_t *allctr,
+ int sz_only,
+ char *prefix,
+ int *print_to_p,
+ void *print_to_arg,
+ Uint **hpp,
+ Uint *szp)
+{
+ Eterm res = THE_NON_VALUE;
+ UWord noc, csz, nob, bsz;
+
+ noc = csz = nob = bsz = ~0;
+ if (print_to_p || hpp) {
+ if (sz_only)
+ cpool_read_stat(allctr, NULL, &csz, NULL, &bsz);
+ else
+ cpool_read_stat(allctr, &noc, &csz, &nob, &bsz);
+ }
+
+ if (print_to_p) {
+ int to = *print_to_p;
+ void *arg = print_to_arg;
+ if (!sz_only)
+ erts_print(to, arg, "%sblocks: %bpu\n", prefix, nob);
+ erts_print(to, arg, "%sblocks size: %bpu\n", prefix, bsz);
+ if (!sz_only)
+ erts_print(to, arg, "%scarriers: %bpu\n", prefix, noc);
+ erts_print(to, arg, "%scarriers size: %bpu\n", prefix, csz);
+ }
+
+ if (hpp || szp) {
+ res = NIL;
+ add_2tup(hpp, szp, &res,
+ am.carriers_size,
+ bld_unstable_uint(hpp, szp, csz));
+ if (!sz_only)
+ add_2tup(hpp, szp, &res,
+ am.carriers,
+ bld_unstable_uint(hpp, szp, noc));
+ add_2tup(hpp, szp, &res,
+ am.blocks_size,
+ bld_unstable_uint(hpp, szp, bsz));
+ if (!sz_only)
+ add_2tup(hpp, szp, &res,
+ am.blocks,
+ bld_unstable_uint(hpp, szp, nob));
+ }
+
+ return res;
+}
+
+#endif /* ERTS_SMP */
+
static Eterm
info_carriers(Allctr_t *allctr,
CarriersStats_t *cs,
@@ -2777,17 +4073,10 @@ info_carriers(Allctr_t *allctr,
Uint *szp)
{
Eterm res = THE_NON_VALUE;
- Uint curr_no, curr_size;
- int small_block = cs == &allctr->sbmbcs;
-
- if (small_block) {
- curr_no = cs->curr.small_block.no;
- curr_size = cs->curr.small_block.size;
- }
- else {
- curr_no = cs->curr.norm.mseg.no + cs->curr.norm.sys_alloc.no;
- curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size;
- }
+ UWord curr_no, curr_size;
+
+ curr_no = cs->curr.norm.mseg.no + cs->curr.norm.sys_alloc.no;
+ curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size;
if (print_to_p) {
int to = *print_to_p;
@@ -2808,75 +4097,67 @@ info_carriers(Allctr_t *allctr,
cs->blocks.max_ever.size);
erts_print(to,
arg,
- "%scarriers: %beu %bpu %bpu\n",
+ "%scarriers: %bpu %bpu %bpu\n",
prefix,
curr_no,
cs->max.no,
cs->max_ever.no);
- if (!small_block) {
#if HAVE_ERTS_MSEG
- erts_print(to,
- arg,
- "%smseg carriers: %bpu\n",
- prefix,
- cs->curr.norm.mseg.no);
-#endif
- erts_print(to,
- arg,
- "%ssys_alloc carriers: %bpu\n",
- prefix,
- cs->curr.norm.sys_alloc.no);
- }
erts_print(to,
arg,
- "%scarriers size: %beu %bpu %bpu\n",
+ "%smseg carriers: %bpu\n",
+ prefix,
+ cs->curr.norm.mseg.no);
+#endif
+ erts_print(to,
+ arg,
+ "%ssys_alloc carriers: %bpu\n",
+ prefix,
+ cs->curr.norm.sys_alloc.no);
+ erts_print(to,
+ arg,
+ "%scarriers size: %bpu %bpu %bpu\n",
prefix,
curr_size,
cs->max.size,
cs->max_ever.size);
- if (!small_block) {
#if HAVE_ERTS_MSEG
- erts_print(to,
- arg,
- "%smseg carriers size: %bpu\n",
- prefix,
- cs->curr.norm.mseg.size);
-#endif
- erts_print(to,
- arg,
- "%ssys_alloc carriers size: %bpu\n",
- prefix,
- cs->curr.norm.sys_alloc.size);
- }
+ erts_print(to,
+ arg,
+ "%smseg carriers size: %bpu\n",
+ prefix,
+ cs->curr.norm.mseg.size);
+#endif
+ erts_print(to,
+ arg,
+ "%ssys_alloc carriers size: %bpu\n",
+ prefix,
+ cs->curr.norm.sys_alloc.size);
}
if (hpp || szp) {
res = NIL;
- if (!small_block) {
- add_2tup(hpp, szp, &res,
- am.sys_alloc_carriers_size,
- bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.size));
+ add_2tup(hpp, szp, &res,
+ am.sys_alloc_carriers_size,
+ bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.size));
#if HAVE_ERTS_MSEG
- add_2tup(hpp, szp, &res,
- am.mseg_alloc_carriers_size,
- bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.size));
+ add_2tup(hpp, szp, &res,
+ am.mseg_alloc_carriers_size,
+ bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.size));
#endif
- }
add_4tup(hpp, szp, &res,
am.carriers_size,
bld_unstable_uint(hpp, szp, curr_size),
bld_unstable_uint(hpp, szp, cs->max.size),
bld_unstable_uint(hpp, szp, cs->max_ever.size));
- if (!small_block) {
- add_2tup(hpp, szp, &res,
- am.sys_alloc_carriers,
- bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.no));
+ add_2tup(hpp, szp, &res,
+ am.sys_alloc_carriers,
+ bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.no));
#if HAVE_ERTS_MSEG
- add_2tup(hpp, szp, &res,
- am.mseg_alloc_carriers,
- bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.no));
+ add_2tup(hpp, szp, &res,
+ am.mseg_alloc_carriers,
+ bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.no));
#endif
- }
add_4tup(hpp, szp, &res,
am.carriers,
bld_unstable_uint(hpp, szp, curr_no),
@@ -2935,16 +4216,10 @@ info_calls(Allctr_t *allctr,
if (print_to_p) {
#define PRINT_CC_4(TO, TOA, NAME, CC) \
- if ((CC).giga_no == 0) \
- erts_print(TO, TOA, "%s calls: %b32u\n", NAME, CC.no); \
- else \
- erts_print(TO, TOA, "%s calls: %b32u%09lu\n", NAME, CC.giga_no, CC.no)
+ erts_print(TO, TOA, "%s calls: %b64u\n", NAME, CC)
#define PRINT_CC_5(TO, TOA, PRFX, NAME, CC) \
- if ((CC).giga_no == 0) \
- erts_print(TO, TOA, "%s%s calls: %b32u\n",PRFX,NAME,CC.no); \
- else \
- erts_print(TO, TOA, "%s%s calls: %b32u%09lu\n",PRFX,NAME,CC.giga_no,CC.no)
+ erts_print(TO, TOA, "%s%s calls: %b64u\n",PRFX,NAME,CC)
char *prefix = allctr->name_prefix;
int to = *print_to_p;
@@ -2954,9 +4229,6 @@ info_calls(Allctr_t *allctr,
PRINT_CC_5(to, arg, prefix, "free", allctr->calls.this_free);
PRINT_CC_5(to, arg, prefix, "realloc", allctr->calls.this_realloc);
- PRINT_CC_4(to, arg, "sbmbc_alloc", allctr->calls.sbmbc_alloc);
- PRINT_CC_4(to, arg, "sbmbc_free", allctr->calls.sbmbc_free);
-
#if HAVE_ERTS_MSEG
PRINT_CC_4(to, arg, "mseg_alloc", allctr->calls.mseg_alloc);
PRINT_CC_4(to, arg, "mseg_dealloc", allctr->calls.mseg_dealloc);
@@ -2983,50 +4255,42 @@ info_calls(Allctr_t *allctr,
add_3tup(hpp, szp, &res,
am.sys_realloc,
- bld_unstable_uint(hpp, szp, allctr->calls.sys_realloc.giga_no),
- bld_unstable_uint(hpp, szp, allctr->calls.sys_realloc.no));
+ bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.sys_realloc)),
+ bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.sys_realloc)));
add_3tup(hpp, szp, &res,
am.sys_free,
- bld_unstable_uint(hpp, szp, allctr->calls.sys_free.giga_no),
- bld_unstable_uint(hpp, szp, allctr->calls.sys_free.no));
+ bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.sys_free)),
+ bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.sys_free)));
add_3tup(hpp, szp, &res,
am.sys_alloc,
- bld_unstable_uint(hpp, szp, allctr->calls.sys_alloc.giga_no),
- bld_unstable_uint(hpp, szp, allctr->calls.sys_alloc.no));
+ bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.sys_alloc)),
+ bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.sys_alloc)));
#if HAVE_ERTS_MSEG
add_3tup(hpp, szp, &res,
am.mseg_realloc,
- bld_unstable_uint(hpp, szp, allctr->calls.mseg_realloc.giga_no),
- bld_unstable_uint(hpp, szp, allctr->calls.mseg_realloc.no));
+ bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.mseg_realloc)),
+ bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.mseg_realloc)));
add_3tup(hpp, szp, &res,
am.mseg_dealloc,
- bld_unstable_uint(hpp, szp, allctr->calls.mseg_dealloc.giga_no),
- bld_unstable_uint(hpp, szp, allctr->calls.mseg_dealloc.no));
+ bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.mseg_dealloc)),
+ bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.mseg_dealloc)));
add_3tup(hpp, szp, &res,
am.mseg_alloc,
- bld_unstable_uint(hpp, szp, allctr->calls.mseg_alloc.giga_no),
- bld_unstable_uint(hpp, szp, allctr->calls.mseg_alloc.no));
+ bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.mseg_alloc)),
+ bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.mseg_alloc)));
#endif
add_3tup(hpp, szp, &res,
- am.sbmbc_free,
- bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_free.giga_no),
- bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_free.no));
- add_3tup(hpp, szp, &res,
- am.sbmbc_alloc,
- bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_alloc.giga_no),
- bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_alloc.no));
- add_3tup(hpp, szp, &res,
allctr->name.realloc,
- bld_unstable_uint(hpp, szp, allctr->calls.this_realloc.giga_no),
- bld_unstable_uint(hpp, szp, allctr->calls.this_realloc.no));
+ bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.this_realloc)),
+ bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.this_realloc)));
add_3tup(hpp, szp, &res,
allctr->name.free,
- bld_unstable_uint(hpp, szp, allctr->calls.this_free.giga_no),
- bld_unstable_uint(hpp, szp, allctr->calls.this_free.no));
+ bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.this_free)),
+ bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.this_free)));
add_3tup(hpp, szp, &res,
allctr->name.alloc,
- bld_unstable_uint(hpp, szp, allctr->calls.this_alloc.giga_no),
- bld_unstable_uint(hpp, szp, allctr->calls.this_alloc.no));
+ bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.this_alloc)),
+ bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.this_alloc)));
}
return res;
@@ -3040,6 +4304,7 @@ info_options(Allctr_t *allctr,
Uint *szp)
{
Eterm res = THE_NON_VALUE;
+ int acul;
if (!allctr) {
if (print_to_p)
@@ -3051,6 +4316,12 @@ info_options(Allctr_t *allctr,
return res;
}
+#ifdef ERTS_SMP
+ acul = allctr->cpool.util_limit;
+#else
+ acul = 0;
+#endif
+
if (print_to_p) {
char topt[21]; /* Enough for any 64-bit integer */
if (allctr->t)
@@ -3079,9 +4350,8 @@ info_options(Allctr_t *allctr,
#endif
"option lmbcs: %beu\n"
"option smbcs: %beu\n"
- "option mbcgs: %beu\n"
- "option sbmbcs: %beu\n"
- "option sbmbct: %beu\n",
+ "option mbcgs: %beu\n",
+ "option acul: %d\n",
topt,
allctr->ramv ? "true" : "false",
#if HALFWORD_HEAP
@@ -3102,8 +4372,7 @@ info_options(Allctr_t *allctr,
allctr->largest_mbc_size,
allctr->smallest_mbc_size,
allctr->mbc_growth_stages,
- allctr->sbmbc_size,
- allctr->sbmbc_threshold);
+ acul);
}
res = (*allctr->info_options)(allctr, "option ", print_to_p, print_to_arg,
@@ -3111,11 +4380,8 @@ info_options(Allctr_t *allctr,
if (hpp || szp) {
add_2tup(hpp, szp, &res,
- am.sbmbct,
- bld_uint(hpp, szp, allctr->sbmbc_threshold));
- add_2tup(hpp, szp, &res,
- am.sbmbcs,
- bld_uint(hpp, szp, allctr->sbmbc_size));
+ am.acul,
+ bld_uint(hpp, szp, (UWord) acul));
add_2tup(hpp, szp, &res,
am.mbcgs,
bld_uint(hpp, szp, allctr->mbc_growth_stages));
@@ -3261,13 +4527,17 @@ erts_alcu_info_options(Allctr_t *allctr,
Eterm
erts_alcu_sz_info(Allctr_t *allctr,
+ int internal,
int begin_max_period,
int *print_to_p,
void *print_to_arg,
Uint **hpp,
Uint *szp)
{
- Eterm res, sbmbcs, mbcs, sbcs, fix = THE_NON_VALUE;
+ Eterm res, mbcs, sbcs, fix = THE_NON_VALUE;
+#ifdef ERTS_SMP
+ Eterm mbcs_pool;
+#endif
res = THE_NON_VALUE;
@@ -3296,30 +4566,35 @@ erts_alcu_sz_info(Allctr_t *allctr,
= allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no;
allctr->sbcs.blocks.max.no = allctr->sbcs.max.no;
- update_max_ever_values(&allctr->sbmbcs);
update_max_ever_values(&allctr->mbcs);
update_max_ever_values(&allctr->sbcs);
if (allctr->fix)
- fix = sz_info_fix(allctr, print_to_p, print_to_arg, hpp, szp);
- sbmbcs = sz_info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p,
- print_to_arg, hpp, szp);
+ fix = sz_info_fix(allctr, internal, print_to_p, print_to_arg, hpp, szp);
mbcs = sz_info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p,
print_to_arg, hpp, szp);
+#ifdef ERTS_SMP
+ if (ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ mbcs_pool = info_cpool(allctr, 1, "mbcs_pool ", print_to_p,
+ print_to_arg, hpp, szp);
+ else
+ mbcs_pool = THE_NON_VALUE; /* shut up annoying warning... */
+#endif
sbcs = sz_info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p,
print_to_arg, hpp, szp);
if (hpp || szp) {
res = NIL;
add_2tup(hpp, szp, &res, am.sbcs, sbcs);
+#ifdef ERTS_SMP
+ if (ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ add_2tup(hpp, szp, &res, am.mbcs_pool, mbcs_pool);
+#endif
add_2tup(hpp, szp, &res, am.mbcs, mbcs);
- add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs);
- if (allctr->fix)
- add_2tup(hpp, szp, &res, am.fix_types, fix);
+ add_fix_types(allctr, internal, hpp, szp, &res, fix);
}
if (begin_max_period) {
- reset_max_values(&allctr->sbmbcs);
reset_max_values(&allctr->mbcs);
reset_max_values(&allctr->sbcs);
}
@@ -3338,13 +4613,17 @@ erts_alcu_sz_info(Allctr_t *allctr,
Eterm
erts_alcu_info(Allctr_t *allctr,
+ int internal,
int begin_max_period,
int *print_to_p,
void *print_to_arg,
Uint **hpp,
Uint *szp)
{
- Eterm res, sett, sbmbcs, mbcs, sbcs, calls, fix = THE_NON_VALUE;
+ Eterm res, sett, mbcs, sbcs, calls, fix = THE_NON_VALUE;
+#ifdef ERTS_SMP
+ Eterm mbcs_pool;
+#endif
res = THE_NON_VALUE;
@@ -3373,7 +4652,6 @@ erts_alcu_info(Allctr_t *allctr,
= allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no;
allctr->sbcs.blocks.max.no = allctr->sbcs.max.no;
- update_max_ever_values(&allctr->sbmbcs);
update_max_ever_values(&allctr->mbcs);
update_max_ever_values(&allctr->sbcs);
@@ -3387,11 +4665,16 @@ erts_alcu_info(Allctr_t *allctr,
sett = info_options(allctr, print_to_p, print_to_arg, hpp, szp);
if (allctr->fix)
- fix = sz_info_fix(allctr, print_to_p, print_to_arg, hpp, szp);
- sbmbcs = info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p,
- print_to_arg, hpp, szp);
+ fix = sz_info_fix(allctr, internal, print_to_p, print_to_arg, hpp, szp);
mbcs = info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p,
print_to_arg, hpp, szp);
+#ifdef ERTS_SMP
+ if (ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ mbcs_pool = info_cpool(allctr, 0, "mbcs_pool ", print_to_p,
+ print_to_arg, hpp, szp);
+ else
+ mbcs_pool = THE_NON_VALUE; /* shut up annoying warning... */
+#endif
sbcs = info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p,
print_to_arg, hpp, szp);
calls = info_calls(allctr, print_to_p, print_to_arg, hpp, szp);
@@ -3401,10 +4684,12 @@ erts_alcu_info(Allctr_t *allctr,
add_2tup(hpp, szp, &res, am.calls, calls);
add_2tup(hpp, szp, &res, am.sbcs, sbcs);
+#ifdef ERTS_SMP
+ if (ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ add_2tup(hpp, szp, &res, am.mbcs_pool, mbcs_pool);
+#endif
add_2tup(hpp, szp, &res, am.mbcs, mbcs);
- add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs);
- if (allctr->fix)
- add_2tup(hpp, szp, &res, am.fix_types, fix);
+ add_fix_types(allctr, internal, hpp, szp, &res, fix);
add_2tup(hpp, szp, &res, am.options, sett);
add_3tup(hpp, szp, &res,
am.versions,
@@ -3413,7 +4698,6 @@ erts_alcu_info(Allctr_t *allctr,
}
if (begin_max_period) {
- reset_max_values(&allctr->sbmbcs);
reset_max_values(&allctr->mbcs);
reset_max_values(&allctr->sbcs);
}
@@ -3441,22 +4725,37 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t *
size->carriers = allctr->mbcs.curr.norm.mseg.size;
size->carriers += allctr->mbcs.curr.norm.sys_alloc.size;
- size->carriers += allctr->sbmbcs.curr.small_block.size;
size->carriers += allctr->sbcs.curr.norm.mseg.size;
size->carriers += allctr->sbcs.curr.norm.sys_alloc.size;
size->blocks = allctr->mbcs.blocks.curr.size;
- size->blocks += allctr->sbmbcs.blocks.curr.size;
size->blocks += allctr->sbcs.blocks.curr.size;
+#ifdef ERTS_SMP
+ if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) {
+ UWord csz, bsz;
+ cpool_read_stat(allctr, NULL, &csz, NULL, &bsz);
+ size->blocks += bsz;
+ size->carriers += csz;
+ }
+#endif
+
if (fi) {
int ix;
for (ix = 0; ix < fisz; ix++) {
if (allctr->fix) {
- fi[ix].allocated += (allctr->fix[ix].type_size
- * allctr->fix[ix].allocated);
- fi[ix].used += (allctr->fix[ix].type_size
- * allctr->fix[ix].used);
+ if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) {
+ fi[ix].allocated += (allctr->fix[ix].type_size
+ * allctr->fix[ix].u.cpool.allocated);
+ fi[ix].used += (allctr->fix[ix].type_size
+ * allctr->fix[ix].u.cpool.used);
+ }
+ else {
+ fi[ix].allocated += (allctr->fix[ix].type_size
+ * allctr->fix[ix].u.nocpool.allocated);
+ fi[ix].used += (allctr->fix[ix].type_size
+ * allctr->fix[ix].u.nocpool.used);
+ }
}
}
}
@@ -3474,7 +4773,6 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size)
{
Allctr_t *allctr = (Allctr_t *) extra;
void *res;
- ErtsAlcFixList_t *fix;
ASSERT(initialized);
@@ -3492,57 +4790,21 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size)
INC_CC(allctr->calls.this_alloc);
- fix = allctr->fix;
- if (fix) {
- int ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE;
- ASSERT((unsigned)ix < ERTS_ALC_NO_FIXED_SIZES);
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
- ASSERT(size <= fix[ix].type_size);
- fix[ix].used++;
- res = fix[ix].list;
- if (res) {
- fix[ix].list_size--;
- fix[ix].list = *((void **) res);
- if (fix[ix].list && fix[ix].allocated > fix[ix].limit) {
- void *p = fix[ix].list;
- Block_t *blk;
- fix[ix].list = *((void **) p);
- fix[ix].list_size--;
- blk = UMEM2BLK(p);
- if (IS_SBC_BLK(blk))
- destroy_carrier(allctr, blk);
- else
- mbc_free(allctr, p);
- fix[ix].allocated--;
- }
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
- return res;
- }
- size = fix[ix].type_size;
- if (fix[ix].limit < fix[ix].used)
- fix[ix].limit = fix[ix].used;
- if (fix[ix].max_used < fix[ix].used)
- fix[ix].max_used = fix[ix].used;
- fix[ix].allocated++;
+ if (allctr->fix) {
+ if (ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ return fix_cpool_alloc(allctr, type, size);
+ else
+ return fix_nocpool_alloc(allctr, type, size);
}
if (size >= allctr->sbc_threshold) {
Block_t *blk;
-#ifdef ERTS_SMP
- if (allctr->dd.use)
- ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1);
-#endif
blk = create_carrier(allctr, size, CFLG_SBC);
res = blk ? BLK2UMEM(blk) : NULL;
}
else
res = mbc_alloc(allctr, size);
- if (!res && fix) {
- int ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE;
- fix[ix].allocated--;
- fix[ix].used--;
- }
return res;
}
@@ -3611,9 +4873,22 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size)
if (pref_allctr->thread_safe)
erts_mtx_lock(&pref_allctr->mutex);
+#ifdef ERTS_SMP
+ ASSERT(pref_allctr->dd.use);
+ ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1);
+#endif
+
ERTS_ALCU_DBG_CHK_THR_ACCESS(pref_allctr);
res = do_erts_alcu_alloc(type, pref_allctr, size);
+
+#ifdef ERTS_SMP
+ if (!res && ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1)) {
+ /* Cleaned up a bit more; try one more time... */
+ res = do_erts_alcu_alloc(type, pref_allctr, size);
+ }
+#endif
+
if (pref_allctr->thread_safe)
erts_mtx_unlock(&pref_allctr->mutex);
@@ -3630,9 +4905,9 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size)
/* ------------------------------------------------------------------------- */
static ERTS_INLINE void
-do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p)
+do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p,
+ Carrier_t **busy_pcrr_pp)
{
- int ix;
Allctr_t *allctr = (Allctr_t *) extra;
ASSERT(initialized);
@@ -3644,57 +4919,28 @@ do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p)
ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr);
if (p) {
- ErtsAlcFixList_t *fix = allctr->fix;
- Block_t *blk;
INC_CC(allctr->calls.this_free);
- if (fix) {
- ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE;
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
- fix[ix].used--;
- if (fix[ix].allocated < fix[ix].limit
- && fix[ix].list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) {
- *((void **) p) = fix[ix].list;
- fix[ix].list = p;
- fix[ix].list_size++;
- if (!allctr->fix_shrink_scheduled) {
- allctr->fix_shrink_scheduled = 1;
- erts_set_aux_work_timeout(
- allctr->ix,
- (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
- | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
- 1);
- }
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
- return;
- }
- fix[ix].allocated--;
- if (fix[ix].list && fix[ix].allocated > fix[ix].limit) {
- blk = UMEM2BLK(p);
- if (IS_SBC_BLK(blk))
- destroy_carrier(allctr, blk);
- else
- mbc_free(allctr, p);
- p = fix[ix].list;
- fix[ix].list = *((void **) p);
- fix[ix].list_size--;
- fix[ix].allocated--;
- }
+ if (allctr->fix) {
+ if (ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ fix_cpool_free(allctr, type, p, busy_pcrr_pp);
+ else
+ fix_nocpool_free(allctr, type, p);
+ }
+ else {
+ Block_t *blk = UMEM2BLK(p);
+ if (IS_SBC_BLK(blk))
+ destroy_carrier(allctr, blk, NULL);
+ else
+ mbc_free(allctr, p, busy_pcrr_pp);
}
-
- blk = UMEM2BLK(p);
- if (IS_SBC_BLK(blk))
- destroy_carrier(allctr, blk);
- else
- mbc_free(allctr, p);
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
}
}
void erts_alcu_free(ErtsAlcType_t type, void *extra, void *p)
{
- do_erts_alcu_free(type, extra, p);
+ do_erts_alcu_free(type, extra, p, NULL);
}
#ifdef USE_THREADS
@@ -3704,7 +4950,7 @@ erts_alcu_free_ts(ErtsAlcType_t type, void *extra, void *p)
{
Allctr_t *allctr = (Allctr_t *) extra;
erts_mtx_lock(&allctr->mutex);
- do_erts_alcu_free(type, extra, p);
+ do_erts_alcu_free(type, extra, p, NULL);
erts_mtx_unlock(&allctr->mutex);
}
@@ -3726,7 +4972,7 @@ erts_alcu_free_thr_spec(ErtsAlcType_t type, void *extra, void *p)
if (allctr->thread_safe)
erts_mtx_lock(&allctr->mutex);
- do_erts_alcu_free(type, allctr, p);
+ do_erts_alcu_free(type, allctr, p, NULL);
if (allctr->thread_safe)
erts_mtx_unlock(&allctr->mutex);
@@ -3736,10 +4982,12 @@ void
erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p)
{
if (p) {
+ Carrier_t *busy_pcrr_p;
Allctr_t *pref_allctr, *used_allctr;
pref_allctr = get_pref_allctr(extra);
- used_allctr = get_used_allctr(extra, p, NULL);
+ used_allctr = get_used_allctr(pref_allctr, ERTS_ALC_TS_PREF_LOCK_IF_USED,
+ p, NULL, &busy_pcrr_p);
if (pref_allctr != used_allctr)
enqueue_dealloc_other_instance(type,
used_allctr,
@@ -3747,12 +4995,11 @@ erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p)
(used_allctr->dd.ix
- pref_allctr->dd.ix));
else {
- if (used_allctr->thread_safe)
- erts_mtx_lock(&used_allctr->mutex);
ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr);
- do_erts_alcu_free(type, used_allctr, p);
- if (used_allctr->thread_safe)
- erts_mtx_unlock(&used_allctr->mutex);
+ do_erts_alcu_free(type, used_allctr, p, &busy_pcrr_p);
+ clear_busy_pool_carrier(used_allctr, busy_pcrr_p);
+ if (pref_allctr->thread_safe)
+ erts_mtx_unlock(&pref_allctr->mutex);
}
}
}
@@ -3768,7 +5015,8 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
void *extra,
void *p,
Uint size,
- Uint32 alcu_flgs)
+ Uint32 alcu_flgs,
+ Carrier_t **busy_pcrr_pp)
{
Allctr_t *allctr = (Allctr_t *) extra;
Block_t *blk;
@@ -3793,7 +5041,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
#if ALLOC_ZERO_EQ_NULL
if (!size) {
ASSERT(p);
- do_erts_alcu_free(type, extra, p);
+ do_erts_alcu_free(type, extra, p, busy_pcrr_pp);
INC_CC(allctr->calls.this_realloc);
DEC_CC(allctr->calls.this_free);
return NULL;
@@ -3804,32 +5052,9 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
blk = UMEM2BLK(p);
- if (allctr->sbmbc_threshold > 0) {
- Uint old_sz, new_sz, lim;
- lim = allctr->sbmbc_threshold;
- old_sz = BLK_SZ(blk);
- new_sz = UMEMSZ2BLKSZ(allctr, size);
- if ((old_sz < lim && lim <= new_sz)
- || (new_sz < lim && lim <= old_sz)) {
- /* *Need* to move it... */
-
- INC_CC(allctr->calls.this_realloc);
- res = do_erts_alcu_alloc(type, extra, size);
- DEC_CC(allctr->calls.this_alloc);
-
- sys_memcpy(res, p, MIN(size, old_sz - ABLK_HDR_SZ));
-
- do_erts_alcu_free(type, extra, p);
- DEC_CC(allctr->calls.this_free);
- return res;
- }
- if (old_sz < lim)
- alcu_flgs |= ERTS_ALCU_FLG_SBMBC;
- }
-
if (size < allctr->sbc_threshold) {
if (IS_MBC_BLK(blk))
- res = mbc_realloc(allctr, p, size, alcu_flgs);
+ res = mbc_realloc(allctr, p, size, alcu_flgs, busy_pcrr_pp);
else {
Uint used_sz = SBC_HEADER_SIZE + ABLK_HDR_SZ + size;
Uint crr_sz;
@@ -3868,16 +5093,12 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
sys_memcpy((void*) res,
(void*) p,
MIN(SBC_BLK_SZ(blk) - ABLK_HDR_SZ, size));
- destroy_carrier(allctr, blk);
+ destroy_carrier(allctr, blk, NULL);
}
}
}
else {
Block_t *new_blk;
-#ifdef ERTS_SMP
- if (allctr->dd.use)
- ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1);
-#endif
if(IS_SBC_BLK(blk)) {
do_carrier_resize:
#if HALFWORD_HEAP
@@ -3896,7 +5117,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
sys_memcpy((void *) res,
(void *) p,
MIN(MBC_ABLK_SZ(blk) - ABLK_HDR_SZ, size));
- mbc_free(allctr, p);
+ mbc_free(allctr, p, busy_pcrr_pp);
}
else
res = NULL;
@@ -3910,7 +5131,7 @@ void *
erts_alcu_realloc(ErtsAlcType_t type, void *extra, void *p, Uint size)
{
void *res;
- res = do_erts_alcu_realloc(type, extra, p, size, 0);
+ res = do_erts_alcu_realloc(type, extra, p, size, 0, NULL);
DEBUG_CHECK_ALIGNMENT(res);
return res;
}
@@ -3931,7 +5152,7 @@ erts_alcu_realloc_mv(ErtsAlcType_t type, void *extra, void *p, Uint size)
if (cpy_size > size)
cpy_size = size;
sys_memcpy(res, p, cpy_size);
- do_erts_alcu_free(type, extra, p);
+ do_erts_alcu_free(type, extra, p, NULL);
}
DEBUG_CHECK_ALIGNMENT(res);
return res;
@@ -3946,7 +5167,7 @@ erts_alcu_realloc_ts(ErtsAlcType_t type, void *extra, void *ptr, Uint size)
Allctr_t *allctr = (Allctr_t *) extra;
void *res;
erts_mtx_lock(&allctr->mutex);
- res = do_erts_alcu_realloc(type, extra, ptr, size, 0);
+ res = do_erts_alcu_realloc(type, extra, ptr, size, 0, NULL);
erts_mtx_unlock(&allctr->mutex);
DEBUG_CHECK_ALIGNMENT(res);
return res;
@@ -3970,7 +5191,7 @@ erts_alcu_realloc_mv_ts(ErtsAlcType_t type, void *extra, void *p, Uint size)
if (cpy_size > size)
cpy_size = size;
sys_memcpy(res, p, cpy_size);
- do_erts_alcu_free(type, extra, p);
+ do_erts_alcu_free(type, extra, p, NULL);
}
erts_mtx_unlock(&allctr->mutex);
DEBUG_CHECK_ALIGNMENT(res);
@@ -3997,7 +5218,7 @@ erts_alcu_realloc_thr_spec(ErtsAlcType_t type, void *extra,
if (allctr->thread_safe)
erts_mtx_lock(&allctr->mutex);
- res = do_erts_alcu_realloc(type, allctr, ptr, size, 0);
+ res = do_erts_alcu_realloc(type, allctr, ptr, size, 0, NULL);
if (allctr->thread_safe)
erts_mtx_unlock(&allctr->mutex);
@@ -4040,7 +5261,7 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra,
if (cpy_size > size)
cpy_size = size;
sys_memcpy(res, ptr, cpy_size);
- do_erts_alcu_free(type, allctr, ptr);
+ do_erts_alcu_free(type, allctr, ptr, NULL);
if (allctr->thread_safe)
erts_mtx_unlock(&allctr->mutex);
}
@@ -4057,40 +5278,64 @@ realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size,
void *res;
Allctr_t *pref_allctr, *used_allctr;
UWord old_user_size;
+ Carrier_t *busy_pcrr_p;
+#ifdef ERTS_SMP
+ int retried;
+#endif
if (!p)
return erts_alcu_alloc_thr_pref(type, extra, size);
pref_allctr = get_pref_allctr(extra);
- used_allctr = get_used_allctr(extra, p, &old_user_size);
+
+ if (pref_allctr->thread_safe)
+ erts_mtx_lock(&pref_allctr->mutex);
+
+#ifdef ERTS_SMP
+ ASSERT(pref_allctr->dd.use);
+ ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1);
+ retried = 0;
+restart:
+#endif
+
+ used_allctr = get_used_allctr(pref_allctr, ERTS_ALC_TS_PREF_LOCK_NO,
+ p, &old_user_size, &busy_pcrr_p);
ASSERT(used_allctr && pref_allctr);
if (!force_move && used_allctr == pref_allctr) {
- if (used_allctr->thread_safe)
- erts_mtx_lock(&used_allctr->mutex);
ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr);
res = do_erts_alcu_realloc(type,
used_allctr,
p,
size,
- 0);
- if (used_allctr->thread_safe)
- erts_mtx_unlock(&used_allctr->mutex);
+ 0,
+ &busy_pcrr_p);
+ clear_busy_pool_carrier(used_allctr, busy_pcrr_p);
+#ifdef ERTS_SMP
+ if (!res && !retried && ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1)) {
+ /* Cleaned up a bit more; try one more time... */
+ retried = 1;
+ goto restart;
+ }
+#endif
+ if (pref_allctr->thread_safe)
+ erts_mtx_unlock(&pref_allctr->mutex);
}
else {
- if (pref_allctr->thread_safe)
- erts_mtx_lock(&pref_allctr->mutex);
res = do_erts_alcu_alloc(type, pref_allctr, size);
- if (pref_allctr->thread_safe && used_allctr != pref_allctr) {
- erts_mtx_unlock(&pref_allctr->mutex);
- }
- if (res) {
- DEBUG_CHECK_ALIGNMENT(res);
+ if (!res)
+ goto unlock_ts_return;
+ else {
- sys_memcpy(res, p, MIN(size,old_user_size));
+ DEBUG_CHECK_ALIGNMENT(res);
if (used_allctr != pref_allctr) {
+ if (pref_allctr->thread_safe)
+ erts_mtx_unlock(&pref_allctr->mutex);
+
+ sys_memcpy(res, p, MIN(size, old_user_size));
+
enqueue_dealloc_other_instance(type,
used_allctr,
p,
@@ -4098,8 +5343,14 @@ realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size,
- pref_allctr->dd.ix));
}
else {
- do_erts_alcu_free(type, used_allctr, p);
+
+ sys_memcpy(res, p, MIN(size, old_user_size));
+
+ do_erts_alcu_free(type, used_allctr, p, &busy_pcrr_p);
ASSERT(pref_allctr == used_allctr);
+ clear_busy_pool_carrier(used_allctr, busy_pcrr_p);
+
+ unlock_ts_return:
if (pref_allctr->thread_safe)
erts_mtx_unlock(&pref_allctr->mutex);
}
@@ -4213,6 +5464,17 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
if (sz > allctr->min_block_size)
allctr->min_block_size = sz;
}
+
+ allctr->cpool.dc_list.first = NULL;
+ allctr->cpool.dc_list.last = NULL;
+ allctr->cpool.abandon_limit = 0;
+ allctr->cpool.disable_abandon = 0;
+ erts_atomic_init_nob(&allctr->cpool.stat.blocks_size, 0);
+ erts_atomic_init_nob(&allctr->cpool.stat.no_blocks, 0);
+ erts_atomic_init_nob(&allctr->cpool.stat.carriers_size, 0);
+ erts_atomic_init_nob(&allctr->cpool.stat.no_carriers, 0);
+ allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT;
+ allctr->cpool.util_limit = init->acul;
#endif
allctr->sbc_threshold = init->sbct;
@@ -4232,27 +5494,6 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
}
#endif
-
-
- allctr->sbmbc_threshold = init->sbmbct;
-
- if (!erts_have_sbmbc_alloc
- || ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no))
- allctr->sbmbc_threshold = 0;
-
- if (!allctr->sbmbc_threshold)
- allctr->sbmbc_size = 0;
- else {
- Uint min_size;
- allctr->sbmbc_size = init->sbmbcs;
- min_size = allctr->sbmbc_threshold;
- min_size += allctr->min_block_size;
- min_size += MBC_HEADER_SIZE(allctr);
- if (allctr->sbmbc_size < min_size)
- allctr->sbmbc_size = min_size;
- }
-
-
#if HAVE_ERTS_MSEG
if (allctr->mseg_opt.abs_shrink_th > ~((UWord) 0) / 100)
allctr->mseg_opt.abs_shrink_th = ~((UWord) 0) / 100;
@@ -4264,16 +5505,12 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
#ifdef ERTS_ENABLE_LOCK_COUNT
erts_mtx_init_x_opt(&allctr->mutex,
- ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no)
- ? "sbmbc_alloc"
- : "alcu_allocator",
+ "alcu_allocator",
make_small(allctr->alloc_no),
ERTS_LCNT_LT_ALLOC);
#else
erts_mtx_init_x(&allctr->mutex,
- ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no)
- ? "sbmbc_alloc"
- : "alcu_allocator",
+ "alcu_allocator",
make_small(allctr->alloc_no));
#endif /*ERTS_ENABLE_LOCK_COUNT*/
@@ -4292,6 +5529,8 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
if (!allctr->get_next_mbc_size)
allctr->get_next_mbc_size = get_next_mbc_size;
+ if (allctr->mbc_header_size < sizeof(Carrier_t))
+ goto error;
#ifdef ERTS_SMP
allctr->dd.use = 0;
if (init->tpref) {
@@ -4300,6 +5539,9 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
allctr->dd.ix = init->ix;
}
#endif
+ allctr->mbc_header_size = (UNIT_CEILING(allctr->mbc_header_size
+ + ABLK_HDR_SZ)
+ - ABLK_HDR_SZ);
if (allctr->main_carrier_size) {
Block_t *blk;
@@ -4308,6 +5550,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
allctr->main_carrier_size,
CFLG_MBC
| CFLG_FORCE_SIZE
+ | CFLG_NO_CPOOL
#if !HALFWORD_HEAP && !HAVE_SUPER_ALIGNED_MB_CARRIERS
| CFLG_FORCE_SYS_ALLOC
#endif
@@ -4315,7 +5558,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
if (!blk)
goto error;
- (*allctr->link_free_block)(allctr, blk, 0);
+ (*allctr->link_free_block)(allctr, blk);
HARD_CHECK_BLK_CARRIER(allctr, blk);
@@ -4326,16 +5569,24 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
allctr->fix = init->fix;
allctr->fix_shrink_scheduled = 0;
for (i = 0; i < ERTS_ALC_NO_FIXED_SIZES; i++) {
- allctr->fix[i].max_used = 0;
- allctr->fix[i].limit = 0;
allctr->fix[i].type_size = init->fix_type_size[i];
allctr->fix[i].list_size = 0;
allctr->fix[i].list = NULL;
- allctr->fix[i].allocated = 0;
- allctr->fix[i].used = 0;
#ifdef ERTS_SMP
ASSERT(allctr->fix[i].type_size >= sizeof(ErtsAllctrFixDDBlock_t));
#endif
+ if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) {
+ allctr->fix[i].u.cpool.min_list_size = 0;
+ allctr->fix[i].u.cpool.shrink_list = 0;
+ allctr->fix[i].u.cpool.allocated = 0;
+ allctr->fix[i].u.cpool.used = 0;
+ }
+ else {
+ allctr->fix[i].u.nocpool.max_used = 0;
+ allctr->fix[i].u.nocpool.limit = 0;
+ allctr->fix[i].u.nocpool.allocated = 0;
+ allctr->fix[i].u.nocpool.used = 0;
+ }
}
}
@@ -4360,11 +5611,9 @@ erts_alcu_stop(Allctr_t *allctr)
allctr->stopped = 1;
while (allctr->sbc_list.first)
- destroy_carrier(allctr, SBC2BLK(allctr, allctr->sbc_list.first));
+ destroy_carrier(allctr, SBC2BLK(allctr, allctr->sbc_list.first), NULL);
while (allctr->mbc_list.first)
- destroy_carrier(allctr, MBC_TO_FIRST_BLK(allctr, allctr->mbc_list.first));
- while (allctr->sbmbc_list.first)
- destroy_sbmbc(allctr, MBC_TO_FIRST_BLK(allctr, allctr->sbmbc_list.first));
+ destroy_carrier(allctr, MBC_TO_FIRST_BLK(allctr, allctr->mbc_list.first), NULL);
#ifdef USE_THREADS
if (allctr->thread_safe)
@@ -4378,6 +5627,14 @@ erts_alcu_stop(Allctr_t *allctr)
void
erts_alcu_init(AlcUInit_t *init)
{
+#ifdef ERTS_SMP
+ int i;
+ for (i = 0; i <= ERTS_ALC_A_MAX; i++) {
+ ErtsAlcCPoolData_t *sentinel = &carrier_pool[i].sentinel;
+ erts_atomic_init_nob(&sentinel->next, (erts_aint_t) sentinel);
+ erts_atomic_init_nob(&sentinel->prev, (erts_aint_t) sentinel);
+ }
+#endif
ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */
#if HAVE_ERTS_MSEG
ASSERT(erts_mseg_unit_size() == MSEG_UNIT_SZ);
@@ -4440,8 +5697,35 @@ erts_alcu_test(UWord op, UWord a1, UWord a2)
case 0x019: return (UWord) PREV_BLK((Block_t *) a1);
case 0x01a: return (UWord) IS_MBC_FIRST_BLK((Allctr_t*)a1, (Block_t *) a2);
case 0x01b: return (UWord) sizeof(Unit_t);
+ case 0x01c: return (unsigned long) BLK_TO_MBC((Block_t*) a1);
+ case 0x01d: ((Allctr_t*) a1)->add_mbc((Allctr_t*)a1, (Carrier_t*)a2); break;
+ case 0x01e: ((Allctr_t*) a1)->remove_mbc((Allctr_t*)a1, (Carrier_t*)a2); break;
+#ifdef ERTS_SMP
+ case 0x01f: return (UWord) sizeof(ErtsAlcCrrPool_t);
+ case 0x020:
+ SET_CARRIER_HDR((Carrier_t *) a2, 0, SCH_SYS_ALLOC|SCH_MBC, (Allctr_t *) a1);
+ cpool_init_carrier_data((Allctr_t *) a1, (Carrier_t *) a2);
+ return (UWord) a2;
+ case 0x021:
+ cpool_insert((Allctr_t *) a1, (Carrier_t *) a2);
+ return (UWord) a2;
+ case 0x022:
+ cpool_delete((Allctr_t *) a1, (Allctr_t *) a1, (Carrier_t *) a2);
+ return (UWord) a2;
+ case 0x023: return (UWord) cpool_is_empty((Allctr_t *) a1);
+ case 0x024: return (UWord) cpool_dbg_is_in_pool((Allctr_t *) a1, (Carrier_t *) a2);
+#else
+ case 0x01f: return (UWord) 0;
+ case 0x020: return (UWord) 0;
+ case 0x021: return (UWord) 0;
+ case 0x022: return (UWord) 0;
+ case 0x023: return (UWord) 0;
+ case 0x024: return (UWord) 0;
+#endif
+
default: ASSERT(0); return ~((UWord) 0);
}
+ return 0;
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
@@ -4449,6 +5733,20 @@ erts_alcu_test(UWord op, UWord a1, UWord a2)
\* */
void
+erts_alcu_assert_failed(char* expr, char* file, int line, char *func)
+{
+ fflush(stdout);
+ fprintf(stderr, "%s:%d:%s(): Assertion failed: %s\n",
+ file, line, func, expr);
+ fflush(stderr);
+#if defined(__WIN__) || defined(__WIN32__)
+ DebugBreak();
+#else
+ abort();
+#endif
+}
+
+void
erts_alcu_verify_unused(Allctr_t *allctr)
{
UWord no;
@@ -4456,12 +5754,10 @@ erts_alcu_verify_unused(Allctr_t *allctr)
no = allctr->sbcs.curr.norm.mseg.no;
no += allctr->sbcs.curr.norm.sys_alloc.no;
no += allctr->mbcs.blocks.curr.no;
- no += allctr->sbmbcs.blocks.curr.no;
if (no) {
UWord sz = allctr->sbcs.blocks.curr.size;
sz += allctr->mbcs.blocks.curr.size;
- sz += allctr->sbmbcs.blocks.curr.size;
erl_exit(ERTS_ABORT_EXIT,
"%salloc() used when expected to be unused!\n"
"Total amount of blocks allocated: %bpu\n"
@@ -4595,7 +5891,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
cl = &allctr->mbc_list;
}
-#if 0 /* FIXIT sbmbc */
+#ifdef DEBUG
if (cl->first == crr) {
ASSERT(!crr->prev);
}