diff options
Diffstat (limited to 'erts/emulator/beam/erl_alloc_util.c')
-rw-r--r-- | erts/emulator/beam/erl_alloc_util.c | 655 |
1 files changed, 499 insertions, 156 deletions
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index bf8a37c71b..d9eaa7b32c 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2002-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -81,16 +82,12 @@ static int atoms_initialized = 0; static int initialized = 0; - #define INV_SYS_ALLOC_CARRIER_MASK ((UWord) (sys_alloc_carrier_size - 1)) #define SYS_ALLOC_CARRIER_MASK (~INV_SYS_ALLOC_CARRIER_MASK) #define SYS_ALLOC_CARRIER_FLOOR(X) ((X) & SYS_ALLOC_CARRIER_MASK) #define SYS_ALLOC_CARRIER_CEILING(X) \ SYS_ALLOC_CARRIER_FLOOR((X) + INV_SYS_ALLOC_CARRIER_MASK) -#undef ASSERT -#define ASSERT ASSERT_EXPR - #if 0 /* Can be useful for debugging */ #define MBC_REALLOC_ALWAYS_MOVES @@ -102,6 +99,7 @@ static Uint sys_alloc_carrier_size; #if HAVE_ERTS_MSEG static Uint max_mseg_carriers; #endif +static int allow_sys_alloc_carriers; #define ONE_GIGA (1000000000) @@ -194,9 +192,9 @@ MBC after deallocating first block: #if MBC_ABLK_OFFSET_BITS -# define MBC_SZ_MAX_LIMIT ((((UWord)1 << MBC_ABLK_OFFSET_BITS) - 1) << MSEG_ALIGN_BITS) +# define MBC_SZ_MAX_LIMIT ((((UWord)1 << MBC_ABLK_OFFSET_BITS) - 1) << ERTS_SUPER_ALIGN_BITS) -# define BLK_CARRIER_OFFSET(B, C) (((char*)(B) - (char*)(C)) >> MSEG_UNIT_SHIFT) +# define BLK_CARRIER_OFFSET(B, C) (((char*)(B) - (char*)(C)) >> ERTS_SACRR_UNIT_SHIFT) # define SET_MBC_ABLK_HDR(B, Sz, F, C) \ (ASSERT(((Sz) & ~MBC_ABLK_SZ_MASK) == 0), \ @@ -208,9 +206,9 @@ MBC after deallocating first block: ASSERT(((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ (B)->bhdr = ((Sz) | (F)), \ (B)->u.carrier = (C)) - + # define IS_MBC_FIRST_ABLK(AP,B) \ - ((((UWord)(B) & ~MSEG_UNIT_MASK) == MBC_HEADER_SIZE(AP)) \ + ((((UWord)(B) & ~ERTS_SACRR_UNIT_MASK) == MBC_HEADER_SIZE(AP)) \ && ((B)->bhdr & MBC_ABLK_OFFSET_MASK) == 0) # define IS_MBC_FIRST_FBLK(AP,B) \ @@ -376,12 +374,13 @@ do { \ #define ERTS_CRR_ALCTR_FLG_IN_POOL (((erts_aint_t) 1) << 0) #define ERTS_CRR_ALCTR_FLG_BUSY (((erts_aint_t) 1) << 1) +#define ERTS_CRR_ALCTR_FLG_MASK (ERTS_CRR_ALCTR_FLG_IN_POOL | \ + ERTS_CRR_ALCTR_FLG_BUSY) #ifdef ERTS_SMP #define SBC_HEADER_SIZE \ - (UNIT_CEILING(sizeof(Carrier_t) \ - - sizeof(ErtsAlcCPoolData_t) \ - + ABLK_HDR_SZ) \ + (UNIT_CEILING(offsetof(Carrier_t, cpool) \ + + ABLK_HDR_SZ) \ - ABLK_HDR_SZ) #else #define SBC_HEADER_SIZE \ @@ -720,7 +719,7 @@ static void make_name_atoms(Allctr_t *allctr); static Block_t *create_carrier(Allctr_t *, Uint, UWord); static void destroy_carrier(Allctr_t *, Block_t *, Carrier_t **); static void mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp); -static void dealloc_block(Allctr_t *, void *, int); +static void dealloc_block(Allctr_t *, void *, ErtsAlcFixList_t *, int); /* internal data... */ @@ -760,8 +759,9 @@ static ERTS_INLINE void * alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { void *res; - - res = erts_mseg_alloc_opt(allctr->alloc_no, size_p, flags, &allctr->mseg_opt); + UWord size = (UWord) *size_p; + res = erts_mseg_alloc_opt(allctr->alloc_no, &size, flags, &allctr->mseg_opt); + *size_p = (Uint) size; INC_CC(allctr->calls.mseg_alloc); return res; } @@ -770,9 +770,10 @@ static ERTS_INLINE void * alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p) { void *res; - - res = erts_mseg_realloc_opt(allctr->alloc_no, seg, old_size, new_size_p, + UWord new_size = (UWord) *new_size_p; + res = erts_mseg_realloc_opt(allctr->alloc_no, seg, (UWord) old_size, &new_size, ERTS_MSEG_FLG_NONE, &allctr->mseg_opt); + *new_size_p = (Uint) new_size; INC_CC(allctr->calls.mseg_realloc); return res; } @@ -780,18 +781,22 @@ alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p) static ERTS_INLINE void alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) { - erts_mseg_dealloc_opt(allctr->alloc_no, seg, size, flags, &allctr->mseg_opt); + erts_mseg_dealloc_opt(allctr->alloc_no, seg, (UWord) size, flags, &allctr->mseg_opt); INC_CC(allctr->calls.mseg_dealloc); } #endif static ERTS_INLINE void * -alcu_sys_alloc(Allctr_t *allctr, Uint size) +alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign) { void *res; - - res = erts_sys_alloc(0, NULL, size); +#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC + if (superalign) + res = erts_sys_aligned_alloc(ERTS_SACRR_UNIT_SZ, size); + else +#endif + res = erts_sys_alloc(0, NULL, size); INC_CC(allctr->calls.sys_alloc); if (erts_mtrace_enabled) erts_mtrace_crr_alloc(res, allctr->alloc_no, ERTS_ALC_A_SYSTEM, size); @@ -799,11 +804,16 @@ alcu_sys_alloc(Allctr_t *allctr, Uint size) } static ERTS_INLINE void * -alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size) +alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign) { void *res; - res = erts_sys_realloc(0, NULL, ptr, size); +#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC + if (superalign) + res = erts_sys_aligned_realloc(ERTS_SACRR_UNIT_SZ, ptr, size, old_size); + else +#endif + res = erts_sys_realloc(0, NULL, ptr, size); INC_CC(allctr->calls.sys_realloc); if (erts_mtrace_enabled) erts_mtrace_crr_realloc(res, @@ -815,9 +825,14 @@ alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size) } static ERTS_INLINE void -alcu_sys_free(Allctr_t *allctr, void *ptr) +alcu_sys_free(Allctr_t *allctr, void *ptr, int superalign) { - erts_sys_free(0, NULL, ptr); +#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC + if (superalign) + erts_sys_aligned_free(ERTS_SACRR_UNIT_SZ, ptr); + else +#endif + erts_sys_free(0, NULL, ptr); INC_CC(allctr->calls.sys_free); if (erts_mtrace_enabled) erts_mtrace_crr_free(allctr->alloc_no, ERTS_ALC_A_SYSTEM, ptr); @@ -914,6 +929,88 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr) #ifdef ERTS_SMP +#ifdef DEBUG +static int is_in_list(ErtsDoubleLink_t* sentinel, ErtsDoubleLink_t* node) +{ + ErtsDoubleLink_t* p; + + ASSERT(node != sentinel); + for (p = sentinel->next; p != sentinel; p = p->next) { + if (p == node) + return 1; + } + return 0; +} +#endif /* DEBUG */ + +static ERTS_INLINE void +link_edl_after(ErtsDoubleLink_t* after_me, ErtsDoubleLink_t* node) +{ + ErtsDoubleLink_t* before_me = after_me->next; + ASSERT(node != after_me && node != before_me); + node->next = before_me; + node->prev = after_me; + before_me->prev = node; + after_me->next = node; +} + +static ERTS_INLINE void +link_edl_before(ErtsDoubleLink_t* before_me, ErtsDoubleLink_t* node) +{ + ErtsDoubleLink_t* after_me = before_me->prev; + ASSERT(node != before_me && node != after_me); + node->next = before_me; + node->prev = after_me; + before_me->prev = node; + after_me->next = node; +} + +static ERTS_INLINE void +unlink_edl(ErtsDoubleLink_t* node) +{ + node->next->prev = node->prev; + node->prev->next = node->next; +} + +static ERTS_INLINE void +relink_edl_before(ErtsDoubleLink_t* before_me, ErtsDoubleLink_t* node) +{ + if (node != before_me && node != before_me->prev) { + unlink_edl(node); + link_edl_before(before_me, node); + } +} + +static ERTS_INLINE int is_abandoned(Carrier_t *crr) +{ + return crr->cpool.abandoned.next != NULL; +} + +static ERTS_INLINE void +link_abandoned_carrier(ErtsDoubleLink_t* list, Carrier_t *crr) +{ + ASSERT(!is_abandoned(crr)); + + link_edl_after(list, &crr->cpool.abandoned); + + ASSERT(crr->cpool.abandoned.next != &crr->cpool.abandoned); + ASSERT(crr->cpool.abandoned.prev != &crr->cpool.abandoned); +} + +static ERTS_INLINE void +unlink_abandoned_carrier(Carrier_t *crr) +{ + ASSERT(is_in_list(&crr->cpool.orig_allctr->cpool.pooled_list, + &crr->cpool.abandoned) || + is_in_list(&crr->cpool.orig_allctr->cpool.traitor_list, + &crr->cpool.abandoned)); + + unlink_edl(&crr->cpool.abandoned); + + crr->cpool.abandoned.next = NULL; + crr->cpool.abandoned.prev = NULL; +} + static ERTS_INLINE void clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr) { @@ -940,7 +1037,7 @@ clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr) } } -#endif +#endif /* ERTS_SMP */ #if 0 #define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) \ @@ -971,17 +1068,21 @@ typedef struct { } ErtsAllctrFixDDBlock_t; #endif +#define ERTS_ALC_FIX_NO_UNUSE (((ErtsAlcType_t) 1) << ERTS_ALC_N_BITS) + static ERTS_INLINE void dealloc_fix_block(Allctr_t *allctr, ErtsAlcType_t type, void *ptr, + ErtsAlcFixList_t *fix, int dec_cc_on_redirect) { #ifdef ERTS_SMP /* May be redirected... */ - ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type; + ASSERT((type & ERTS_ALC_FIX_NO_UNUSE) == 0); + ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type | ERTS_ALC_FIX_NO_UNUSE; #endif - dealloc_block(allctr, ptr, dec_cc_on_redirect); + dealloc_block(allctr, ptr, fix, dec_cc_on_redirect); } static ERTS_INLINE void @@ -1027,8 +1128,7 @@ fix_cpool_check_shrink(Allctr_t *allctr, if (fix->u.cpool.min_list_size > fix->list_size) fix->u.cpool.min_list_size = fix->list_size; - fix->u.cpool.allocated--; - dealloc_fix_block(allctr, type, p, 0); + dealloc_fix_block(allctr, type, p, fix, 0); } } } @@ -1074,7 +1174,8 @@ static ERTS_INLINE void fix_cpool_free(Allctr_t *allctr, ErtsAlcType_t type, void *p, - Carrier_t **busy_pcrr_pp) + Carrier_t **busy_pcrr_pp, + int unuse) { ErtsAlcFixList_t *fix; @@ -1082,8 +1183,9 @@ fix_cpool_free(Allctr_t *allctr, && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; - - fix->u.cpool.used--; + + if (unuse) + fix->u.cpool.used--; if ((!busy_pcrr_pp || !*busy_pcrr_pp) && !fix->u.cpool.shrink_list @@ -1141,8 +1243,7 @@ fix_cpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) fix->list = *((void **) ptr); fix->list_size--; fix->u.cpool.shrink_list--; - fix->u.cpool.allocated--; - dealloc_fix_block(allctr, type, ptr, 0); + dealloc_fix_block(allctr, type, ptr, fix, 0); } if (fix->u.cpool.min_list_size > fix->list_size) fix->u.cpool.min_list_size = fix->list_size; @@ -1303,7 +1404,7 @@ fix_nocpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) ptr = fix->list; fix->list = *((void **) ptr); fix->list_size--; - dealloc_block(allctr, ptr, 0); + dealloc_block(allctr, ptr, NULL, 0); fix->u.nocpool.allocated--; } if (fix->list_size != 0) { @@ -1334,7 +1435,17 @@ erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) return fix_nocpool_alloc_shrink(allctr, flgs); } -static void dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, Uint mseg_flags); +static void dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, int superaligned); + +static ERTS_INLINE void +dealloc_mbc(Allctr_t *allctr, Carrier_t *crr) +{ + ASSERT(IS_MB_CARRIER(crr)); + if (allctr->destroying_mbc) + allctr->destroying_mbc(allctr, crr); + + dealloc_carrier(allctr, crr, 1); +} #ifdef ERTS_SMP @@ -1346,7 +1457,7 @@ get_pref_allctr(void *extra) pref_ix = ERTS_ALC_GET_THR_IX(); - ASSERT(sizeof(UWord) == sizeof(Allctr_t *)); + ERTS_CT_ASSERT(sizeof(UWord) == sizeof(Allctr_t *)); ASSERT(0 <= pref_ix && pref_ix < tspec->size); return tspec->allctr[pref_ix]; @@ -1392,14 +1503,14 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep, if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock && pref_allctr->thread_safe) { - used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK); + used_allctr = (Allctr_t *) (iallctr & ~ERTS_CRR_ALCTR_FLG_MASK); if (pref_allctr == used_allctr) { erts_mtx_lock(&pref_allctr->mutex); locked_pref_allctr = 1; } } - while ((iallctr & ((~FLG_MASK)|ERTS_CRR_ALCTR_FLG_IN_POOL)) + while ((iallctr & ((~ERTS_CRR_ALCTR_FLG_MASK)|ERTS_CRR_ALCTR_FLG_IN_POOL)) == (((erts_aint_t) pref_allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL)) { erts_aint_t act; @@ -1414,7 +1525,7 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep, iallctr = act; } - used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK); + used_allctr = (Allctr_t *) (iallctr & ~ERTS_CRR_ALCTR_FLG_MASK); if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock) { if (locked_pref_allctr && used_allctr != pref_allctr) { @@ -1424,16 +1535,16 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep, } ERTS_ALC_CPOOL_ASSERT( - (((iallctr & ~FLG_MASK) == (erts_aint_t) pref_allctr) - ? (((iallctr & FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL) - || ((iallctr & FLG_MASK) == 0)) + (((iallctr & ~ERTS_CRR_ALCTR_FLG_MASK) == (erts_aint_t) pref_allctr) + ? (((iallctr & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL) + || ((iallctr & ERTS_CRR_ALCTR_FLG_MASK) == 0)) : 1)); return used_allctr; } } - used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK); + used_allctr = (Allctr_t *) (iallctr & ~ERTS_CRR_ALCTR_FLG_MASK); if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock && used_allctr == pref_allctr @@ -1650,11 +1761,13 @@ handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr) type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type; - ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type - && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE + <= (type & ~ERTS_ALC_FIX_NO_UNUSE)); + ASSERT((type & ~ERTS_ALC_FIX_NO_UNUSE) + <= ERTS_ALC_N_MAX_A_FIXED_SIZE); if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) - fix_nocpool_free(allctr, type, ptr); + fix_nocpool_free(allctr, (type & ~ERTS_ALC_FIX_NO_UNUSE), ptr); else { Block_t *blk = UMEM2BLK(ptr); Carrier_t *busy_pcrr_p; @@ -1669,7 +1782,9 @@ handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr) NULL, &busy_pcrr_p); if (used_allctr == allctr) { doit: - fix_cpool_free(allctr, type, ptr, &busy_pcrr_p); + fix_cpool_free(allctr, (type & ~ERTS_ALC_FIX_NO_UNUSE), + ptr, &busy_pcrr_p, + !(type & ERTS_ALC_FIX_NO_UNUSE)); clear_busy_pool_carrier(allctr, busy_pcrr_p); } else { @@ -1760,11 +1875,23 @@ handle_delayed_dealloc(Allctr_t *allctr, * data has been overwritten by the queue. */ Carrier_t *crr = FIRST_BLK_TO_MBC(allctr, blk); + + /* Restore word overwritten by the dd-queue as it will be read + * if this carrier is pulled from dc_list by cpool_fetch() + */ + ERTS_ALC_CPOOL_ASSERT(FBLK_TO_MBC(blk) != crr); + ERTS_CT_ASSERT(sizeof(ErtsAllctrDDBlock_t) == sizeof(void*)); +#ifdef MBC_ABLK_OFFSET_BITS + blk->u.carrier = crr; +#else + blk->carrier = crr; +#endif + ERTS_ALC_CPOOL_ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr)); ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) != (erts_smp_atomic_read_nob(&crr->allctr) - & ~FLG_MASK)); + & ~ERTS_CRR_ALCTR_FLG_MASK)); erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); @@ -1777,7 +1904,7 @@ handle_delayed_dealloc(Allctr_t *allctr, if (fix) handle_delayed_fix_dealloc(allctr, ptr); else - dealloc_block(allctr, ptr, 1); + dealloc_block(allctr, ptr, NULL, 1); } } @@ -1883,15 +2010,24 @@ erts_alcu_check_delayed_dealloc(Allctr_t *allctr, ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL, NULL) static void -dealloc_block(Allctr_t *allctr, void *ptr, int dec_cc_on_redirect) +dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_redirect) { Block_t *blk = UMEM2BLK(ptr); ERTS_SMP_LC_ASSERT(!allctr->thread_safe || erts_lc_mtx_is_locked(&allctr->mutex)); - if (IS_SBC_BLK(blk)) + if (IS_SBC_BLK(blk)) { destroy_carrier(allctr, blk, NULL); +#ifdef ERTS_SMP + if (fix && ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + ErtsAlcType_t type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type; + if (!(type & ERTS_ALC_FIX_NO_UNUSE)) + fix->u.cpool.used--; + fix->u.cpool.allocated--; + } +#endif + } #ifndef ERTS_SMP else mbc_free(allctr, ptr, NULL); @@ -1904,6 +2040,12 @@ dealloc_block(Allctr_t *allctr, void *ptr, int dec_cc_on_redirect) used_allctr = get_used_allctr(allctr, ERTS_ALC_TS_PREF_LOCK_NO, ptr, NULL, &busy_pcrr_p); if (used_allctr == allctr) { + if (fix) { + ErtsAlcType_t type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type; + if (!(type & ERTS_ALC_FIX_NO_UNUSE)) + fix->u.cpool.used--; + fix->u.cpool.allocated--; + } mbc_free(allctr, ptr, &busy_pcrr_p); clear_busy_pool_carrier(allctr, busy_pcrr_p); } @@ -1942,7 +2084,7 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp) if (!blk) { blk = create_carrier(allctr, get_blk_sz, CFLG_MBC); -#if !HALFWORD_HEAP && !HAVE_SUPER_ALIGNED_MB_CARRIERS +#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY if (!blk) { /* Emergency! We couldn't create the carrier as we wanted. Try to place it in a sys_alloced sbc. */ @@ -2548,10 +2690,9 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, #ifdef ERTS_SMP #define ERTS_ALC_MAX_DEALLOC_CARRIER 10 -#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 10 +#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 20 +#define ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT 10 #define ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT 100 -#define ERTS_ALC_CPOOL_MAX_NO_CARRIERS 5 -#define ERTS_ALC_CPOOL_INSERT_ALLOWED_OFFSET 100 #define ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS 3 #define ERTS_ALC_CPOOL_PTR_MOD_MRK (((erts_aint_t) 1) << 0) @@ -2728,9 +2869,6 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) (erts_aint_t) CARRIER_SZ(crr)); erts_atomic_inc_nob(&allctr->cpool.stat.no_carriers); - erts_smp_atomic_set_nob(&crr->allctr, - ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); - /* * We search in 'next' direction and begin by passing * one element before trying to insert. This in order to @@ -2789,6 +2927,9 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) cpool_set_mod_marked(&cpd2p->prev, (erts_aint_t) &crr->cpool, (erts_aint_t) cpd1p); + + erts_smp_atomic_set_wb(&crr->allctr, + ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); } static void @@ -2889,59 +3030,163 @@ cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr) static Carrier_t * cpool_fetch(Allctr_t *allctr, UWord size) { - int i; + int i, i_stop, has_passed_sentinel; Carrier_t *crr; ErtsAlcCPoolData_t *cpdp; - ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + ErtsAlcCPoolData_t *cpool_entrance; + ErtsAlcCPoolData_t *sentinel; + ErtsDoubleLink_t* dl; + ErtsDoubleLink_t* first_old_traitor; ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ || erts_thr_progress_is_managed_thread()); - i = 0; + i = ERTS_ALC_CPOOL_MAX_FETCH_INSPECT; + first_old_traitor = allctr->cpool.traitor_list.next; + cpool_entrance = NULL; - /* First; check our own pending dealloc carrier list... */ - crr = allctr->cpool.dc_list.last; - while (crr && i < ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) { - if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) { - unlink_carrier(&allctr->cpool.dc_list, crr); -#ifdef ERTS_ALC_CPOOL_DEBUG - ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr, - ((erts_aint_t) allctr)) - == (((erts_aint_t) allctr) & ~FLG_MASK)); -#else - erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); -#endif - return crr; + /* + * Search my own pooled_list, + * i.e my abandoned carriers that were in the pool last time I checked. + */ + + dl = allctr->cpool.pooled_list.next; + while(dl != &allctr->cpool.pooled_list) { + erts_aint_t exp, act; + crr = (Carrier_t *) (((char *) dl) - offsetof(Carrier_t, cpool.abandoned)); + + ASSERT(!is_in_list(&allctr->cpool.traitor_list, dl)); + ASSERT(crr->cpool.orig_allctr == allctr); + dl = dl->next; + exp = erts_smp_atomic_read_rb(&crr->allctr); + if ((exp & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL + && erts_atomic_read_nob(&crr->cpool.max_size) >= size) { + /* Try to fetch it... */ + act = erts_smp_atomic_cmpxchg_mb(&crr->allctr, + (erts_aint_t) allctr, + exp); + if (act == exp) { + cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); + unlink_abandoned_carrier(crr); + + /* Move sentinel to continue next search from here */ + relink_edl_before(dl, &allctr->cpool.pooled_list); + return crr; + } + exp = act; + } + if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { + if (!cpool_entrance) + cpool_entrance = &crr->cpool; + } + else { /* Not in pool, move to traitor_list */ + unlink_abandoned_carrier(crr); + link_abandoned_carrier(&allctr->cpool.traitor_list, crr); + } + if (--i <= 0) { + /* Move sentinel to continue next search from here */ + relink_edl_before(dl, &allctr->cpool.pooled_list); + return NULL; } - crr = crr->prev; - i++; } - /* ... then the pool ... */ + /* Now search traitor_list. + * i.e carriers employed by other allocators last time I checked. + * They might have been abandoned since then. + */ + + i_stop = (i < ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT ? + 0 : i - ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT); + dl = first_old_traitor; + while(dl != &allctr->cpool.traitor_list) { + erts_aint_t exp, act; + crr = (Carrier_t *) (((char *) dl) - offsetof(Carrier_t, cpool.abandoned)); + ASSERT(dl != &allctr->cpool.pooled_list); + ASSERT(crr->cpool.orig_allctr == allctr); + dl = dl->next; + exp = erts_smp_atomic_read_rb(&crr->allctr); + if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { + if (!(exp & ERTS_CRR_ALCTR_FLG_BUSY) + && erts_atomic_read_nob(&crr->cpool.max_size) >= size) { + /* Try to fetch it... */ + act = erts_smp_atomic_cmpxchg_mb(&crr->allctr, + (erts_aint_t) allctr, + exp); + if (act == exp) { + cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); + unlink_abandoned_carrier(crr); + + /* Move sentinel to continue next search from here */ + relink_edl_before(dl, &allctr->cpool.traitor_list); + return crr; + } + exp = act; + } + if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { + if (!cpool_entrance) + cpool_entrance = &crr->cpool; + + /* Move to pooled_list */ + unlink_abandoned_carrier(crr); + link_abandoned_carrier(&allctr->cpool.pooled_list, crr); + } + } + if (--i <= i_stop) { + /* Move sentinel to continue next search from here */ + relink_edl_before(dl, &allctr->cpool.traitor_list); + if (i > 0) + break; + else + return NULL; + } + } /* - * We search in 'prev' direction and begin by passing - * one element before trying to fetch. This in order to - * avoid contention with threads inserting elements. + * Finally search the shared pool and try employ foreign carriers */ - cpdp = cpool_aint2cpd(cpool_read(&sentinel->prev)); - if (cpdp == sentinel) - return NULL; + sentinel = &carrier_pool[allctr->alloc_no].sentinel; + if (cpool_entrance) { + /* We saw a pooled carried above, use it as entrance into the pool + */ + cpdp = cpool_entrance; + } + else { + /* No pooled carried seen above. Start search at cpool sentinel, + * but begin by passing one element before trying to fetch. + * This in order to avoid contention with threads inserting elements. + */ + cpool_entrance = sentinel; + cpdp = cpool_aint2cpd(cpool_read(&cpool_entrance->prev)); + if (cpdp == sentinel) + goto check_dc_list; + } - while (i < ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) { + has_passed_sentinel = 0; + while (1) { erts_aint_t exp; cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); - if (cpdp == sentinel) { + if (cpdp == cpool_entrance) { + if (cpool_entrance == sentinel) { + cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); + if (cpdp == sentinel) + break; + } + i = 0; /* Last one to inspect */ + } + else if (cpdp == sentinel) { + if (has_passed_sentinel) { + /* We been here before. cpool_entrance must have been removed */ + break; + } cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); if (cpdp == sentinel) - return NULL; - i = ERTS_ALC_CPOOL_MAX_FETCH_INSPECT; /* Last one to inspect */ + break; + has_passed_sentinel = 1; } - crr = (Carrier_t *) (((char *) cpdp) - offsetof(Carrier_t, cpool)); + crr = (Carrier_t *)(((char *)cpdp) - offsetof(Carrier_t, cpool)); exp = erts_smp_atomic_read_rb(&crr->allctr); - if (((exp & (ERTS_CRR_ALCTR_FLG_IN_POOL|ERTS_CRR_ALCTR_FLG_BUSY)) - == ERTS_CRR_ALCTR_FLG_IN_POOL) + if (((exp & (ERTS_CRR_ALCTR_FLG_MASK)) == ERTS_CRR_ALCTR_FLG_IN_POOL) && (erts_atomic_read_nob(&cpdp->max_size) >= size)) { erts_aint_t act; /* Try to fetch it... */ @@ -2949,12 +3194,41 @@ cpool_fetch(Allctr_t *allctr, UWord size) (erts_aint_t) allctr, exp); if (act == exp) { - cpool_delete(allctr, ((Allctr_t *) (act & ~FLG_MASK)), crr); + cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); + if (crr->cpool.orig_allctr == allctr) { + unlink_abandoned_carrier(crr); + } return crr; } } - i++; + if (--i <= 0) + return NULL; + } + +check_dc_list: + /* Last; check our own pending dealloc carrier list... */ + crr = allctr->cpool.dc_list.last; + while (crr) { + if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) { + Block_t* blk; + unlink_carrier(&allctr->cpool.dc_list, crr); +#ifdef ERTS_ALC_CPOOL_DEBUG + ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr, + ((erts_aint_t) allctr)) + == (((erts_aint_t) allctr) & ~ERTS_CRR_ALCTR_FLG_MASK)); +#else + erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); +#endif + blk = MBC_TO_FIRST_BLK(allctr, crr); + ASSERT(FBLK_TO_MBC(blk) == crr); + allctr->link_free_block(allctr, blk); + return crr; + } + crr = crr->prev; + if (--i <= 0) + return NULL; } + return NULL; } @@ -2978,7 +3252,7 @@ check_pending_dealloc_carrier(Allctr_t *allctr, dcrr = crr; crr = crr->next; - dealloc_carrier(allctr, dcrr, ERTS_MSEG_FLG_2POW); + dealloc_mbc(allctr, dcrr); i++; } while (crr && i < ERTS_ALC_MAX_DEALLOC_CARRIER); @@ -3009,18 +3283,20 @@ static void schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) { Allctr_t *orig_allctr; + Block_t *blk; int check_pending_dealloc; erts_aint_t max_size; + ASSERT(IS_MB_CARRIER(crr)); + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { - dealloc_carrier(allctr, crr, ERTS_MSEG_FLG_2POW); + dealloc_mbc(allctr, crr); return; } orig_allctr = crr->cpool.orig_allctr; if (allctr != orig_allctr) { - Block_t *blk = MBC_TO_FIRST_BLK(allctr, crr); int cinit = orig_allctr->dd.ix - allctr->dd.ix; /* @@ -3037,6 +3313,7 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) * since the block is an mbc block that is free and last * in the carrier. */ + blk = MBC_TO_FIRST_BLK(allctr, crr); ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(blk)); ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, blk)); @@ -3044,20 +3321,25 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(allctr, blk)); ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) == (erts_smp_atomic_read_nob(&crr->allctr) - & ~FLG_MASK)); + & ~ERTS_CRR_ALCTR_FLG_MASK)); if (ddq_enqueue(&orig_allctr->dd.q, BLK2UMEM(blk), cinit)) erts_alloc_notify_delayed_dealloc(orig_allctr->ix); return; } + if (is_abandoned(crr)) + unlink_abandoned_carrier(crr); + if (crr->cpool.thr_prgr == ERTS_THR_PRGR_INVALID || erts_thr_progress_has_reached(crr->cpool.thr_prgr)) { - dealloc_carrier(allctr, crr, ERTS_MSEG_FLG_2POW); + dealloc_mbc(allctr, crr); return; } - max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); + blk = MBC_TO_FIRST_BLK(allctr, crr); + ASSERT(IS_FREE_LAST_MBC_BLK(blk)); + max_size = (erts_aint_t) MBC_FBLK_SZ(blk); erts_atomic_set_nob(&crr->cpool.max_size, max_size); crr->next = NULL; @@ -3097,6 +3379,8 @@ cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr) limit = (csz/100)*allctr->cpool.util_limit; crr->cpool.abandon_limit = limit; } + crr->cpool.abandoned.next = NULL; + crr->cpool.abandoned.prev = NULL; } static void @@ -3127,6 +3411,9 @@ abandon_carrier(Allctr_t *allctr, Carrier_t *crr) STAT_MBC_CPOOL_INSERT(allctr, crr); unlink_carrier(&allctr->mbc_list, crr); + if (crr->cpool.orig_allctr == allctr) { + link_abandoned_carrier(&allctr->cpool.pooled_list, crr); + } allctr->remove_mbc(allctr, crr); @@ -3188,10 +3475,10 @@ cpool_read_stat(Allctr_t *allctr, UWord *nocp, UWord *cszp, UWord *nobp, UWord * #ifdef DEBUG -#if HAVE_ERTS_MSEG -#define ASSERT_MSEG_UNIT_SIZE_MULTIPLE(CSZ) ASSERT((CSZ) % MSEG_UNIT_SZ == 0) +#if ERTS_SA_MB_CARRIERS +#define ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE(CSZ) ASSERT((CSZ) % ERTS_SACRR_UNIT_SZ == 0) #else -#define ASSERT_MSEG_UNIT_SIZE_MULTIPLE(CSZ) +#define ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE(CSZ) #endif static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C, @@ -3213,10 +3500,12 @@ static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C, ASSERT(IS_MBC_BLK((B))); ASSERT(IS_MB_CARRIER((C))); ASSERT(FBLK_TO_MBC(B) == (C)); + if ((MSEGED)) { + ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE((CSZ)); + } } if ((MSEGED)) { ASSERT(IS_MSEG_CARRIER((C))); - ASSERT_MSEG_UNIT_SIZE_MULTIPLE((CSZ)); } else { ASSERT(IS_SYS_ALLOC_CARRIER((C))); @@ -3242,19 +3531,30 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) int is_mseg = 0; #endif -#if HALFWORD_HEAP - flags |= CFLG_FORCE_MSEG; -#elif HAVE_SUPER_ALIGNED_MB_CARRIERS - if (flags & CFLG_MBC) { + if (HALFWORD_HEAP + || (ERTS_SUPER_ALIGNED_MSEG_ONLY && (flags & CFLG_MBC)) + || !allow_sys_alloc_carriers) { flags |= CFLG_FORCE_MSEG; - } + flags &= ~CFLG_FORCE_SYS_ALLOC; +#if !HAVE_ERTS_MSEG + return NULL; #endif + } ASSERT((flags & CFLG_SBC && !(flags & CFLG_MBC)) || (flags & CFLG_MBC && !(flags & CFLG_SBC))); ASSERT(!(flags & CFLG_FORCE_MSEG && flags & CFLG_FORCE_SYS_ALLOC)); + if (umem_sz > (ERTS_UINT_MAX - ERTS_UINT_MAX/100)) { + /* Do an overly conservative _overflow_ check here so we don't + * have to deal with it from here on. I guess we could be more accurate + * but I don't think the need to allocate over 99% of the address space + * will ever arise on any machine, neither 32 nor 64 bit. + */ + return NULL; + } + blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz); #ifdef ERTS_SMP @@ -3287,7 +3587,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) if (allctr->sbcs.curr.norm.mseg.no >= allctr->max_mseg_sbcs) goto try_sys_alloc; } -#if !HAVE_SUPER_ALIGNED_MB_CARRIERS +#if !ERTS_SUPER_ALIGNED_MSEG_ONLY else { if (allctr->mbcs.curr.norm.mseg.no >= allctr->max_mseg_mbcs) goto try_sys_alloc; @@ -3350,12 +3650,12 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) ? UNIT_CEILING(bcrr_sz) : SYS_ALLOC_CARRIER_CEILING(bcrr_sz)); - crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz); + crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz, flags & CFLG_MBC); if (!crr) { if (crr_sz > UNIT_CEILING(bcrr_sz)) { crr_sz = UNIT_CEILING(bcrr_sz); - crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz); + crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz, flags & CFLG_MBC); } if (!crr) { #if HAVE_ERTS_MSEG @@ -3453,7 +3753,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) if (!(flags & CFLG_FORCE_SYS_ALLOC)) { new_crr_sz = new_blk_sz + SBC_HEADER_SIZE; - new_crr_sz = MSEG_UNIT_CEILING(new_crr_sz); + new_crr_sz = ERTS_SACRR_UNIT_CEILING(new_crr_sz); new_crr = (Carrier_t *) alcu_mseg_realloc(allctr, old_crr, old_crr_sz, @@ -3503,7 +3803,9 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) new_crr = (Carrier_t *) alcu_sys_realloc(allctr, (void *) old_crr, - new_crr_sz); + new_crr_sz, + old_crr_sz, + 0); if (new_crr) { sys_realloc_success: SET_CARRIER_SZ(new_crr, new_crr_sz); @@ -3522,7 +3824,9 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) new_crr_sz = UNIT_CEILING(new_crr_sz); new_crr = (Carrier_t *) alcu_sys_realloc(allctr, (void *) old_crr, - new_crr_sz); + new_crr_sz, + old_crr_sz, + 0); if (new_crr) goto sys_realloc_success; } @@ -3541,27 +3845,29 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) (void *) BLK2UMEM(old_blk), MIN(new_blk_sz, old_blk_sz) - ABLK_HDR_SZ); unlink_carrier(&allctr->sbc_list, old_crr); - alcu_sys_free(allctr, old_crr); + alcu_sys_free(allctr, old_crr, 0); } else { /* Old carrier unchanged; restore... */ STAT_SYS_ALLOC_SBC_ALLOC(allctr, old_crr_sz, old_blk_sz); } - DEBUG_SAVE_ALIGNMENT(new_crr); return new_blk; } #endif } static void -dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, Uint mseg_flags) +dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, int superaligned) { #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(crr)) - alcu_mseg_dealloc(allctr, crr, CARRIER_SZ(crr), mseg_flags); + alcu_mseg_dealloc(allctr, crr, CARRIER_SZ(crr), + (superaligned + ? ERTS_MSEG_FLG_2POW + : ERTS_MSEG_FLG_NONE)); else #endif - alcu_sys_free(allctr, crr); + alcu_sys_free(allctr, crr, superaligned); } static void @@ -3581,7 +3887,6 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(crr)) { - ASSERT(crr_sz % MSEG_UNIT_SZ == 0); STAT_MSEG_SBC_FREE(allctr, crr_sz, blk_sz); } else @@ -3590,7 +3895,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) unlink_carrier(&allctr->sbc_list, crr); - dealloc_carrier(allctr, crr, ERTS_MSEG_FLG_NONE); + dealloc_carrier(allctr, crr, 0); } else { ASSERT(IS_MBC_FIRST_FBLK(allctr, blk)); @@ -3609,13 +3914,15 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) } #endif - if (allctr->destroying_mbc) - (*allctr->destroying_mbc)(allctr, crr); - #ifdef ERTS_SMP if (busy_pcrr_pp && *busy_pcrr_pp) { ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr); *busy_pcrr_pp = NULL; + ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_read_nob(&crr->allctr) + == (((erts_aint_t) allctr) + | ERTS_CRR_ALCTR_FLG_IN_POOL + | ERTS_CRR_ALCTR_FLG_BUSY)); + erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); cpool_delete(allctr, allctr, crr); } else @@ -3624,21 +3931,23 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) unlink_carrier(&allctr->mbc_list, crr); #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(crr)) { - ASSERT(crr_sz % MSEG_UNIT_SZ == 0); + ASSERT(crr_sz % ERTS_SACRR_UNIT_SZ == 0); STAT_MSEG_MBC_FREE(allctr, crr_sz); } else #endif STAT_SYS_ALLOC_MBC_FREE(allctr, crr_sz); + + if (allctr->remove_mbc) + allctr->remove_mbc(allctr, crr); } #ifdef ERTS_SMP schedule_dealloc_carrier(allctr, crr); #else - dealloc_carrier(allctr, crr, ERTS_MSEG_FLG_2POW); + dealloc_mbc(allctr, crr); #endif } - } @@ -3678,6 +3987,7 @@ static struct { Eterm mmc; #endif Eterm ycs; + Eterm sac; Eterm fix_types; @@ -3770,6 +4080,7 @@ init_atoms(Allctr_t *allctr) AM_INIT(mmc); #endif AM_INIT(ycs); + AM_INIT(sac); AM_INIT(fix_types); @@ -4207,7 +4518,7 @@ make_name_atoms(Allctr_t *allctr) size_t prefix_len = strlen(allctr->name_prefix); if (prefix_len > MAX_ATOM_CHARACTERS + sizeof(realloc) - 1) - erl_exit(1,"Too long allocator name: %salloc\n",allctr->name_prefix); + erts_exit(ERTS_ERROR_EXIT,"Too long allocator name: %salloc\n",allctr->name_prefix); memcpy((void *) buf, (void *) allctr->name_prefix, prefix_len); @@ -4369,7 +4680,7 @@ info_options(Allctr_t *allctr, #endif "option lmbcs: %beu\n" "option smbcs: %beu\n" - "option mbcgs: %beu\n", + "option mbcgs: %beu\n" "option acul: %d\n", topt, allctr->ramv ? "true" : "false", @@ -4490,17 +4801,22 @@ erts_alcu_au_info_options(int *print_to_p, void *print_to_arg, #if HAVE_ERTS_MSEG "option mmc: %beu\n" #endif - "option ycs: %beu\n", + "option ycs: %beu\n" + "option sac: %s\n", #if HAVE_ERTS_MSEG max_mseg_carriers, #endif - sys_alloc_carrier_size); + sys_alloc_carrier_size, + allow_sys_alloc_carriers ? "true" : "false"); } if (hpp || szp) { res = NIL; ensure_atoms_initialized(NULL); add_2tup(hpp, szp, &res, + am.sac, + allow_sys_alloc_carriers ? am_true : am_false); + add_2tup(hpp, szp, &res, am.ycs, bld_uint(hpp, szp, sys_alloc_carrier_size)); #if HAVE_ERTS_MSEG @@ -4943,7 +5259,7 @@ do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p, if (allctr->fix) { if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) - fix_cpool_free(allctr, type, p, busy_pcrr_pp); + fix_cpool_free(allctr, type, p, busy_pcrr_pp, 1); else fix_nocpool_free(allctr, type, p); } @@ -5086,7 +5402,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, crr_sz = SYS_ALLOC_CARRIER_CEILING(used_sz); #if HAVE_ERTS_MSEG else - crr_sz = MSEG_UNIT_CEILING(used_sz); + crr_sz = ERTS_SACRR_UNIT_CEILING(used_sz); #endif diff_sz_val = crr_sz - used_sz; if (diff_sz_val < (~((Uint) 0) / 100)) @@ -5403,6 +5719,11 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) { /* erts_alcu_start assumes that allctr has been zeroed */ + if (((UWord)allctr & ERTS_CRR_ALCTR_FLG_MASK) != 0) { + erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_alcu_start: Alignment error\n", + __FILE__, __LINE__); + } + if (!initialized) goto error; @@ -5454,7 +5775,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->mbc_move_threshold = init->rmbcmt; #if HAVE_ERTS_MSEG allctr->max_mseg_sbcs = init->mmsbc; -# if HAVE_SUPER_ALIGNED_MB_CARRIERS +# if ERTS_SUPER_ALIGNED_MSEG_ONLY allctr->max_mseg_mbcs = ~(Uint)0; # else allctr->max_mseg_mbcs = init->mmmbc; @@ -5484,6 +5805,10 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->min_block_size = sz; } + allctr->cpool.pooled_list.next = &allctr->cpool.pooled_list; + allctr->cpool.pooled_list.prev = &allctr->cpool.pooled_list; + allctr->cpool.traitor_list.next = &allctr->cpool.traitor_list; + allctr->cpool.traitor_list.prev = &allctr->cpool.traitor_list; allctr->cpool.dc_list.first = NULL; allctr->cpool.dc_list.last = NULL; allctr->cpool.abandon_limit = 0; @@ -5493,7 +5818,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) erts_atomic_init_nob(&allctr->cpool.stat.carriers_size, 0); erts_atomic_init_nob(&allctr->cpool.stat.no_carriers, 0); allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT; - allctr->cpool.util_limit = init->acul; + allctr->cpool.util_limit = init->ts ? 0 : init->acul; #endif allctr->sbc_threshold = init->sbct; @@ -5526,11 +5851,11 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) erts_mtx_init_x_opt(&allctr->mutex, "alcu_allocator", make_small(allctr->alloc_no), - ERTS_LCNT_LT_ALLOC); + ERTS_LCNT_LT_ALLOC,1); #else erts_mtx_init_x(&allctr->mutex, "alcu_allocator", - make_small(allctr->alloc_no)); + make_small(allctr->alloc_no),1); #endif /*ERTS_ENABLE_LOCK_COUNT*/ #ifdef DEBUG @@ -5570,12 +5895,19 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) CFLG_MBC | CFLG_FORCE_SIZE | CFLG_NO_CPOOL -#if !HALFWORD_HEAP && !HAVE_SUPER_ALIGNED_MB_CARRIERS +#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY | CFLG_FORCE_SYS_ALLOC #endif | CFLG_MAIN_CARRIER); - if (!blk) - goto error; + if (!blk) { +#ifdef USE_THREADS + if (allctr->thread_safe) + erts_mtx_destroy(&allctr->mutex); +#endif + erts_exit(ERTS_ABORT_EXIT, + "Failed to create main carrier for %salloc\n", + init->name_prefix); + } (*allctr->link_free_block)(allctr, blk); @@ -5654,14 +5986,15 @@ erts_alcu_init(AlcUInit_t *init) erts_atomic_init_nob(&sentinel->prev, (erts_aint_t) sentinel); } #endif - ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */ + ERTS_CT_ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */ #if HAVE_ERTS_MSEG - ASSERT(erts_mseg_unit_size() == MSEG_UNIT_SZ); + ASSERT(erts_mseg_unit_size() == ERTS_SACRR_UNIT_SZ); max_mseg_carriers = init->mmc; - sys_alloc_carrier_size = MSEG_UNIT_CEILING(init->ycs); + sys_alloc_carrier_size = ERTS_SACRR_UNIT_CEILING(init->ycs); #else /* #if HAVE_ERTS_MSEG */ sys_alloc_carrier_size = ((init->ycs + 4095) / 4096) * 4096; #endif + allow_sys_alloc_carriers = init->sac; #ifdef DEBUG carrier_alignment = sizeof(Unit_t); @@ -5716,7 +6049,7 @@ erts_alcu_test(UWord op, UWord a1, UWord a2) case 0x019: return (UWord) PREV_BLK((Block_t *) a1); case 0x01a: return (UWord) IS_MBC_FIRST_BLK((Allctr_t*)a1, (Block_t *) a2); case 0x01b: return (UWord) sizeof(Unit_t); - case 0x01c: return (unsigned long) BLK_TO_MBC((Block_t*) a1); + case 0x01c: return (UWord) BLK_TO_MBC((Block_t*) a1); case 0x01d: ((Allctr_t*) a1)->add_mbc((Allctr_t*)a1, (Carrier_t*)a2); break; case 0x01e: ((Allctr_t*) a1)->remove_mbc((Allctr_t*)a1, (Carrier_t*)a2); break; #ifdef ERTS_SMP @@ -5741,6 +6074,16 @@ erts_alcu_test(UWord op, UWord a1, UWord a2) case 0x023: return (UWord) 0; case 0x024: return (UWord) 0; #endif + case 0x025: /* UMEM2BLK_TEST*/ +#ifdef DEBUG +# ifdef HARD_DEBUG + return (UWord)UMEM2BLK(a1-3*sizeof(UWord)); +# else + return (UWord)UMEM2BLK(a1-2*sizeof(UWord)); +# endif +#else + return (UWord)UMEM2BLK(a1); +#endif default: ASSERT(0); return ~((UWord) 0); } @@ -5777,7 +6120,7 @@ erts_alcu_verify_unused(Allctr_t *allctr) if (no) { UWord sz = allctr->sbcs.blocks.curr.size; sz += allctr->mbcs.blocks.curr.size; - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%salloc() used when expected to be unused!\n" "Total amount of blocks allocated: %bpu\n" "Total amount of bytes allocated: %bpu\n", @@ -5819,7 +6162,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) ASSERT(CARRIER_SZ(sbc) - SBC_HEADER_SIZE >= SBC_BLK_SZ(iblk)); #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(sbc)) { - ASSERT(CARRIER_SZ(sbc) % MSEG_UNIT_SZ == 0); + ASSERT(CARRIER_SZ(sbc) % ERTS_SACRR_UNIT_SZ == 0); } #endif crr = sbc; @@ -5904,7 +6247,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(crr)) { - ASSERT(CARRIER_SZ(crr) % MSEG_UNIT_SZ == 0); + ASSERT(CARRIER_SZ(crr) % ERTS_SACRR_UNIT_SZ == 0); } #endif cl = &allctr->mbc_list; |