diff options
Diffstat (limited to 'erts/emulator/beam/erl_alloc_util.c')
-rw-r--r-- | erts/emulator/beam/erl_alloc_util.c | 648 |
1 files changed, 516 insertions, 132 deletions
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 236ee35d18..af86ad0548 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2013. All Rights Reserved. + * Copyright Ericsson AB 2002-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -52,6 +52,7 @@ #ifdef ERTS_ENABLE_LOCK_COUNT #include "erl_lock_count.h" #endif +#include "lttng-wrapper.h" #if defined(ERTS_ALLOC_UTIL_HARD_DEBUG) && defined(__GNUC__) #warning "* * * * * * * * * *" @@ -751,12 +752,77 @@ internal_free(void *ptr) #endif +#ifdef ARCH_32 + +/* + * Bit vector for the entire 32-bit virtual address space + * with one bit for each super aligned memory segment. + */ + +#define VSPACE_MAP_BITS (1 << (32 - ERTS_MMAP_SUPERALIGNED_BITS)) +#define VSPACE_MAP_SZ (VSPACE_MAP_BITS / ERTS_VSPACE_WORD_BITS) + +static ERTS_INLINE void set_bit(UWord* map, Uint ix) +{ + ASSERT(ix / ERTS_VSPACE_WORD_BITS < VSPACE_MAP_SZ); + map[ix / ERTS_VSPACE_WORD_BITS] + |= ((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS)); +} + +static ERTS_INLINE void clr_bit(UWord* map, Uint ix) +{ + ASSERT(ix / ERTS_VSPACE_WORD_BITS < VSPACE_MAP_SZ); + map[ix / ERTS_VSPACE_WORD_BITS] + &= ~((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS)); +} + +static ERTS_INLINE int is_bit_set(UWord* map, Uint ix) +{ + ASSERT(ix / ERTS_VSPACE_WORD_BITS < VSPACE_MAP_SZ); + return map[ix / ERTS_VSPACE_WORD_BITS] + & ((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS)); +} + +UWord erts_literal_vspace_map[VSPACE_MAP_SZ]; + +static void set_literal_range(void* start, Uint size) +{ + Uint ix = (UWord)start >> ERTS_MMAP_SUPERALIGNED_BITS; + Uint n = size >> ERTS_MMAP_SUPERALIGNED_BITS; + + ASSERT(!((UWord)start & ERTS_INV_SUPERALIGNED_MASK)); + ASSERT(!((UWord)size & ERTS_INV_SUPERALIGNED_MASK)); + ASSERT(n); + while (n--) { + ASSERT(!is_bit_set(erts_literal_vspace_map, ix)); + set_bit(erts_literal_vspace_map, ix); + ix++; + } +} + +static void clear_literal_range(void* start, Uint size) +{ + Uint ix = (UWord)start >> ERTS_MMAP_SUPERALIGNED_BITS; + Uint n = size >> ERTS_MMAP_SUPERALIGNED_BITS; + + ASSERT(!((UWord)start & ERTS_INV_SUPERALIGNED_MASK)); + ASSERT(!((UWord)size & ERTS_INV_SUPERALIGNED_MASK)); + ASSERT(n); + while (n--) { + ASSERT(is_bit_set(erts_literal_vspace_map, ix)); + clr_bit(erts_literal_vspace_map, ix); + ix++; + } +} + +#endif /* ARCH_32 */ + /* mseg ... */ #if HAVE_ERTS_MSEG -static ERTS_INLINE void * -alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) +static void* +erts_alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { void *res; UWord size = (UWord) *size_p; @@ -766,8 +832,9 @@ alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) return res; } -static ERTS_INLINE void * -alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p) +static void* +erts_alcu_mseg_realloc(Allctr_t *allctr, void *seg, + Uint old_size, Uint *new_size_p) { void *res; UWord new_size = (UWord) *new_size_p; @@ -778,19 +845,162 @@ alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p) return res; } -static ERTS_INLINE void -alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) +static void +erts_alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) { erts_mseg_dealloc_opt(allctr->alloc_no, seg, (UWord) size, flags, &allctr->mseg_opt); INC_CC(allctr->calls.mseg_dealloc); } -#endif -static ERTS_INLINE void * -alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign) +#if defined(ARCH_32) + +void* +erts_alcu_literal_32_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) +{ + void* res; + Uint sz = ERTS_SUPERALIGNED_CEILING(*size_p); + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && + allctr->t == 0); + ERTS_SMP_LC_ASSERT(allctr->thread_safe); + + res = erts_alcu_mseg_alloc(allctr, &sz, flags); + if (res) { + set_literal_range(res, sz); + *size_p = sz; + } + return res; +} + +void* +erts_alcu_literal_32_mseg_realloc(Allctr_t *allctr, void *seg, + Uint old_size, Uint *new_size_p) +{ + void* res; + Uint new_sz = ERTS_SUPERALIGNED_CEILING(*new_size_p); + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && + allctr->t == 0); + ERTS_SMP_LC_ASSERT(allctr->thread_safe); + + if (seg && old_size) + clear_literal_range(seg, old_size); + res = erts_alcu_mseg_realloc(allctr, seg, old_size, &new_sz); + if (res) { + set_literal_range(res, new_sz); + *new_size_p = new_sz; + } + return res; +} + +void +erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, + Uint flags) +{ + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && + allctr->t == 0); + ERTS_SMP_LC_ASSERT(allctr->thread_safe); + + erts_alcu_mseg_dealloc(allctr, seg, size, flags); + + clear_literal_range(seg, size); +} + +#elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) + +/* For allocators that have their own mmapper (super carrier), + * like literal_alloc and exec_alloc on amd64 + */ +void* +erts_alcu_mmapper_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) +{ + void* res; + UWord size = (UWord) *size_p; + Uint32 mmap_flags = ERTS_MMAPFLG_SUPERCARRIER_ONLY; + if (flags & ERTS_MSEG_FLG_2POW) + mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED; + + res = erts_mmap(allctr->mseg_mmapper, mmap_flags, &size); + *size_p = (Uint)size; + INC_CC(allctr->calls.mseg_alloc); + return res; +} + +void* +erts_alcu_mmapper_mseg_realloc(Allctr_t *allctr, void *seg, + Uint old_size, Uint *new_size_p) { void *res; + UWord new_size = (UWord) *new_size_p; + res = erts_mremap(allctr->mseg_mmapper, ERTS_MSEG_FLG_NONE, seg, old_size, &new_size); + *new_size_p = (Uint) new_size; + INC_CC(allctr->calls.mseg_realloc); + return res; +} + +void +erts_alcu_mmapper_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, + Uint flags) +{ + Uint32 mmap_flags = ERTS_MMAPFLG_SUPERCARRIER_ONLY; + if (flags & ERTS_MSEG_FLG_2POW) + mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED; + + erts_munmap(allctr->mseg_mmapper, mmap_flags, seg, (UWord)size); + INC_CC(allctr->calls.mseg_dealloc); +} +#endif /* ARCH_64 && ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */ + +#if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER) + +/* + * For exec_alloc on non-amd64 that just need memory with PROT_EXEC + */ +void* +erts_alcu_exec_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) +{ + void* res = erts_alcu_mseg_alloc(allctr, size_p, flags); + + if (res) { + int r = mprotect(res, *size_p, PROT_EXEC | PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + return res; +} + +void* +erts_alcu_exec_mseg_realloc(Allctr_t *allctr, void *seg, + Uint old_size, Uint *new_size_p) +{ + void *res; + + if (seg && old_size) { + int r = mprotect(seg, old_size, PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + res = erts_alcu_mseg_realloc(allctr, seg, old_size, new_size_p); + if (res) { + int r = mprotect(res, *new_size_p, PROT_EXEC | PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + return res; +} + +void +erts_alcu_exec_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) +{ + int r = mprotect(seg, size, PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + erts_alcu_mseg_dealloc(allctr, seg, size, flags); +} +#endif /* ERTS_ALC_A_EXEC && !ERTS_HAVE_EXEC_MMAPPER */ + +#endif /* HAVE_ERTS_MSEG */ + +static void* +erts_alcu_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign) +{ + void *res; + const Uint size = *size_p; #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC if (superalign) res = erts_sys_aligned_alloc(ERTS_SACRR_UNIT_SZ, size); @@ -803,10 +1013,11 @@ alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign) return res; } -static ERTS_INLINE void * -alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign) +static void* +erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign) { void *res; + const Uint size = *size_p; #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC if (superalign) @@ -824,8 +1035,8 @@ alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int supe return res; } -static ERTS_INLINE void -alcu_sys_free(Allctr_t *allctr, void *ptr, int superalign) +static void +erts_alcu_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign) { #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC if (superalign) @@ -838,6 +1049,59 @@ alcu_sys_free(Allctr_t *allctr, void *ptr, int superalign) erts_mtrace_crr_free(allctr->alloc_no, ERTS_ALC_A_SYSTEM, ptr); } +#ifdef ARCH_32 + +void* +erts_alcu_literal_32_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign) +{ + void* res; + Uint size = ERTS_SUPERALIGNED_CEILING(*size_p); + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && + allctr->t == 0); + ERTS_SMP_LC_ASSERT(allctr->thread_safe); + + res = erts_alcu_sys_alloc(allctr, &size, 1); + if (res) { + set_literal_range(res, size); + *size_p = size; + } + return res; +} + +void* +erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint* size_p, Uint old_size, int superalign) +{ + void* res; + Uint size = ERTS_SUPERALIGNED_CEILING(*size_p); + + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && + allctr->t == 0); + ERTS_SMP_LC_ASSERT(allctr->thread_safe); + + if (ptr && old_size) + clear_literal_range(ptr, old_size); + res = erts_alcu_sys_realloc(allctr, ptr, &size, old_size, 1); + if (res) { + set_literal_range(res, size); + *size_p = size; + } + return res; +} + +void +erts_alcu_literal_32_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign) +{ + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && + allctr->t == 0); + ERTS_SMP_LC_ASSERT(allctr->thread_safe); + + erts_alcu_sys_dealloc(allctr, ptr, size, 1); + + clear_literal_range(ptr, size); +} + +#endif /* ARCH_32 */ + static Uint get_next_mbc_size(Allctr_t *allctr) { @@ -1143,6 +1407,7 @@ fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + ASSERT(size == fix->type_size); res = fix->list; if (res) { @@ -1154,8 +1419,6 @@ fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) fix_cpool_check_shrink(allctr, type, fix, NULL); return res; } - if (size < 2*sizeof(UWord)) - size += sizeof(UWord); if (size >= allctr->sbc_threshold) { Block_t *blk; blk = create_carrier(allctr, size, CFLG_SBC); @@ -1275,6 +1538,7 @@ fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + ASSERT(size == fix->type_size); ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); fix->u.nocpool.used++; @@ -1297,8 +1561,6 @@ fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); return res; } - if (size < 2*sizeof(UWord)) - size += sizeof(UWord); if (fix->u.nocpool.limit < fix->u.nocpool.used) fix->u.nocpool.limit = fix->u.nocpool.used; if (fix->u.nocpool.max_used < fix->u.nocpool.used) @@ -1437,6 +1699,16 @@ erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) static void dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, int superaligned); +static ERTS_INLINE void +dealloc_mbc(Allctr_t *allctr, Carrier_t *crr) +{ + ASSERT(IS_MB_CARRIER(crr)); + if (allctr->destroying_mbc) + allctr->destroying_mbc(allctr, crr); + + dealloc_carrier(allctr, crr, 1); +} + #ifdef ERTS_SMP static ERTS_INLINE Allctr_t* @@ -2074,7 +2346,7 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp) if (!blk) { blk = create_carrier(allctr, get_blk_sz, CFLG_MBC); -#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY +#if !ERTS_SUPER_ALIGNED_MSEG_ONLY if (!blk) { /* Emergency! We couldn't create the carrier as we wanted. Try to place it in a sys_alloced sbc. */ @@ -2920,6 +3192,7 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) erts_smp_atomic_set_wb(&crr->allctr, ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); + LTTNG3(carrier_pool_put, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, CARRIER_SZ(crr)); } static void @@ -3035,6 +3308,7 @@ cpool_fetch(Allctr_t *allctr, UWord size) first_old_traitor = allctr->cpool.traitor_list.next; cpool_entrance = NULL; + LTTNG3(carrier_pool_get, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, (unsigned long)size); /* * Search my own pooled_list, * i.e my abandoned carriers that were in the pool last time I checked. @@ -3149,7 +3423,7 @@ cpool_fetch(Allctr_t *allctr, UWord size) cpool_entrance = sentinel; cpdp = cpool_aint2cpd(cpool_read(&cpool_entrance->prev)); if (cpdp == sentinel) - return NULL; + goto check_dc_list; } has_passed_sentinel = 0; @@ -3160,18 +3434,18 @@ cpool_fetch(Allctr_t *allctr, UWord size) if (cpool_entrance == sentinel) { cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); if (cpdp == sentinel) - return NULL; + break; } i = 0; /* Last one to inspect */ } else if (cpdp == sentinel) { if (has_passed_sentinel) { /* We been here before. cpool_entrance must have been removed */ - return NULL; + break; } cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); if (cpdp == sentinel) - return NULL; + break; has_passed_sentinel = 1; } crr = (Carrier_t *)(((char *)cpdp) - offsetof(Carrier_t, cpool)); @@ -3195,10 +3469,12 @@ cpool_fetch(Allctr_t *allctr, UWord size) return NULL; } +check_dc_list: /* Last; check our own pending dealloc carrier list... */ crr = allctr->cpool.dc_list.last; while (crr) { if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) { + Block_t* blk; unlink_carrier(&allctr->cpool.dc_list, crr); #ifdef ERTS_ALC_CPOOL_DEBUG ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr, @@ -3207,6 +3483,9 @@ cpool_fetch(Allctr_t *allctr, UWord size) #else erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); #endif + blk = MBC_TO_FIRST_BLK(allctr, crr); + ASSERT(FBLK_TO_MBC(blk) == crr); + allctr->link_free_block(allctr, blk); return crr; } crr = crr->prev; @@ -3237,7 +3516,7 @@ check_pending_dealloc_carrier(Allctr_t *allctr, dcrr = crr; crr = crr->next; - dealloc_carrier(allctr, dcrr, 1); + dealloc_mbc(allctr, dcrr); i++; } while (crr && i < ERTS_ALC_MAX_DEALLOC_CARRIER); @@ -3268,18 +3547,20 @@ static void schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) { Allctr_t *orig_allctr; + Block_t *blk; int check_pending_dealloc; erts_aint_t max_size; + ASSERT(IS_MB_CARRIER(crr)); + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { - dealloc_carrier(allctr, crr, 1); + dealloc_mbc(allctr, crr); return; } orig_allctr = crr->cpool.orig_allctr; if (allctr != orig_allctr) { - Block_t *blk = MBC_TO_FIRST_BLK(allctr, crr); int cinit = orig_allctr->dd.ix - allctr->dd.ix; /* @@ -3296,6 +3577,7 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) * since the block is an mbc block that is free and last * in the carrier. */ + blk = MBC_TO_FIRST_BLK(allctr, crr); ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(blk)); ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, blk)); @@ -3315,11 +3597,13 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) if (crr->cpool.thr_prgr == ERTS_THR_PRGR_INVALID || erts_thr_progress_has_reached(crr->cpool.thr_prgr)) { - dealloc_carrier(allctr, crr, 1); + dealloc_mbc(allctr, crr); return; } - max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); + blk = MBC_TO_FIRST_BLK(allctr, crr); + ASSERT(IS_FREE_LAST_MBC_BLK(blk)); + max_size = (erts_aint_t) MBC_FBLK_SZ(blk); erts_atomic_set_nob(&crr->cpool.max_size, max_size); crr->next = NULL; @@ -3511,8 +3795,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) int is_mseg = 0; #endif - if (HALFWORD_HEAP - || (ERTS_SUPER_ALIGNED_MSEG_ONLY && (flags & CFLG_MBC)) + if ((ERTS_SUPER_ALIGNED_MSEG_ONLY && (flags & CFLG_MBC)) || !allow_sys_alloc_carriers) { flags |= CFLG_FORCE_MSEG; flags &= ~CFLG_FORCE_SYS_ALLOC; @@ -3520,6 +3803,8 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) return NULL; #endif } + flags |= allctr->crr_set_flgs; + flags &= ~allctr->crr_clr_flgs; ASSERT((flags & CFLG_SBC && !(flags & CFLG_MBC)) || (flags & CFLG_MBC && !(flags & CFLG_SBC))); @@ -3535,7 +3820,21 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) return NULL; } - blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz); + if (flags & CFLG_MAIN_CARRIER) { + ASSERT(flags & CFLG_MBC); + ASSERT(flags & CFLG_NO_CPOOL); + ASSERT(umem_sz == allctr->main_carrier_size); + ERTS_UNDEF(blk_sz, 0); + + if (allctr->main_carrier_size < allctr->min_mbc_size) + allctr->main_carrier_size = allctr->min_mbc_size; + crr_sz = bcrr_sz = allctr->main_carrier_size; + } + else { + ERTS_UNDEF(bcrr_sz, 0); + ERTS_UNDEF(crr_sz, 0); + blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz); + } #ifdef ERTS_SMP allctr->cpool.disable_abandon = ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON; @@ -3581,13 +3880,15 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) mseg_flags = ERTS_MSEG_FLG_NONE; } else { - crr_sz = (*allctr->get_next_mbc_size)(allctr); - if (crr_sz < MBC_HEADER_SIZE(allctr) + blk_sz) - crr_sz = MBC_HEADER_SIZE(allctr) + blk_sz; - mseg_flags = ERTS_MSEG_FLG_2POW; + if (!(flags & CFLG_MAIN_CARRIER)) { + crr_sz = (*allctr->get_next_mbc_size)(allctr); + if (crr_sz < MBC_HEADER_SIZE(allctr) + blk_sz) + crr_sz = MBC_HEADER_SIZE(allctr) + blk_sz; + } + mseg_flags = ERTS_MSEG_FLG_2POW; } - crr = (Carrier_t *) alcu_mseg_alloc(allctr, &crr_sz, mseg_flags); + crr = (Carrier_t *) allctr->mseg_alloc(allctr, &crr_sz, mseg_flags); if (!crr) { have_tried_mseg = 1; if (!(have_tried_sys_alloc || flags & CFLG_FORCE_MSEG)) @@ -3619,23 +3920,22 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) if (flags & CFLG_SBC) { bcrr_sz = blk_sz + SBC_HEADER_SIZE; } - else { + else if (!(flags & CFLG_MAIN_CARRIER)) { bcrr_sz = MBC_HEADER_SIZE(allctr) + blk_sz; - if (!(flags & CFLG_MAIN_CARRIER) - && bcrr_sz < allctr->smallest_mbc_size) - bcrr_sz = allctr->smallest_mbc_size; + if (bcrr_sz < allctr->smallest_mbc_size) + bcrr_sz = allctr->smallest_mbc_size; } crr_sz = (flags & CFLG_FORCE_SIZE ? UNIT_CEILING(bcrr_sz) : SYS_ALLOC_CARRIER_CEILING(bcrr_sz)); - crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz, flags & CFLG_MBC); + crr = (Carrier_t *) allctr->sys_alloc(allctr, &crr_sz, flags & CFLG_MBC); if (!crr) { if (crr_sz > UNIT_CEILING(bcrr_sz)) { crr_sz = UNIT_CEILING(bcrr_sz); - crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz, flags & CFLG_MBC); + crr = (Carrier_t *) allctr->sys_alloc(allctr, &crr_sz, flags & CFLG_MBC); } if (!crr) { #if HAVE_ERTS_MSEG @@ -3694,6 +3994,21 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) } +#ifdef USE_LTTNG_VM_TRACEPOINTS + if (LTTNG_ENABLED(carrier_create)) { + lttng_decl_carrier_stats(mbc_stats); + lttng_decl_carrier_stats(sbc_stats); + LTTNG_CARRIER_STATS_TO_LTTNG_STATS(&(allctr->mbcs), mbc_stats); + LTTNG_CARRIER_STATS_TO_LTTNG_STATS(&(allctr->sbcs), sbc_stats); + LTTNG5(carrier_create, + ERTS_ALC_A2AD(allctr->alloc_no), + allctr->ix, + crr_sz, + mbc_stats, + sbc_stats); + } +#endif + DEBUG_SAVE_ALIGNMENT(crr); return blk; } @@ -3734,7 +4049,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) new_crr_sz = new_blk_sz + SBC_HEADER_SIZE; new_crr_sz = ERTS_SACRR_UNIT_CEILING(new_crr_sz); - new_crr = (Carrier_t *) alcu_mseg_realloc(allctr, + new_crr = (Carrier_t *) allctr->mseg_realloc(allctr, old_crr, old_crr_sz, &new_crr_sz); @@ -3749,11 +4064,6 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) DEBUG_SAVE_ALIGNMENT(new_crr); return new_blk; } -#if HALFWORD_HEAP - /* Old carrier unchanged; restore stat */ - STAT_MSEG_SBC_ALLOC(allctr, old_crr_sz, old_blk_sz); - return NULL; -#endif create_flags |= CFLG_FORCE_SYS_ALLOC; /* since mseg_realloc() failed */ } @@ -3764,7 +4074,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) (void *) BLK2UMEM(old_blk), MIN(new_blk_sz, old_blk_sz) - ABLK_HDR_SZ); unlink_carrier(&allctr->sbc_list, old_crr); - alcu_mseg_dealloc(allctr, old_crr, old_crr_sz, ERTS_MSEG_FLG_NONE); + allctr->mseg_dealloc(allctr, old_crr, old_crr_sz, ERTS_MSEG_FLG_NONE); } else { /* Old carrier unchanged; restore stat */ @@ -3781,9 +4091,9 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) ? UNIT_CEILING(new_bcrr_sz) : SYS_ALLOC_CARRIER_CEILING(new_bcrr_sz)); - new_crr = (Carrier_t *) alcu_sys_realloc(allctr, + new_crr = (Carrier_t *) allctr->sys_realloc(allctr, (void *) old_crr, - new_crr_sz, + &new_crr_sz, old_crr_sz, 0); if (new_crr) { @@ -3802,9 +4112,9 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) else if (new_crr_sz > UNIT_CEILING(new_bcrr_sz)) { new_crr_sz = new_blk_sz + SBC_HEADER_SIZE; new_crr_sz = UNIT_CEILING(new_crr_sz); - new_crr = (Carrier_t *) alcu_sys_realloc(allctr, + new_crr = (Carrier_t *) allctr->sys_realloc(allctr, (void *) old_crr, - new_crr_sz, + &new_crr_sz, old_crr_sz, 0); if (new_crr) @@ -3825,7 +4135,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) (void *) BLK2UMEM(old_blk), MIN(new_blk_sz, old_blk_sz) - ABLK_HDR_SZ); unlink_carrier(&allctr->sbc_list, old_crr); - alcu_sys_free(allctr, old_crr, 0); + allctr->sys_dealloc(allctr, old_crr, CARRIER_SZ(old_crr), 0); } else { /* Old carrier unchanged; restore... */ @@ -3841,13 +4151,13 @@ dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, int superaligned) { #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(crr)) - alcu_mseg_dealloc(allctr, crr, CARRIER_SZ(crr), + allctr->mseg_dealloc(allctr, crr, CARRIER_SZ(crr), (superaligned ? ERTS_MSEG_FLG_2POW : ERTS_MSEG_FLG_NONE)); else #endif - alcu_sys_free(allctr, crr, superaligned); + allctr->sys_dealloc(allctr, crr, CARRIER_SZ(crr), superaligned); } static void @@ -3887,16 +4197,13 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) ASSERT(IS_LAST_BLK(blk)); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - (*allctr->link_free_block)(allctr, blk, 0); + (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, blk, 0); + (*allctr->unlink_free_block)(allctr, blk); #endif } #endif - if (allctr->destroying_mbc) - (*allctr->destroying_mbc)(allctr, crr); - #ifdef ERTS_SMP if (busy_pcrr_pp && *busy_pcrr_pp) { ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr); @@ -3920,12 +4227,30 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) else #endif STAT_SYS_ALLOC_MBC_FREE(allctr, crr_sz); + + if (allctr->remove_mbc) + allctr->remove_mbc(allctr, crr); } +#ifdef USE_LTTNG_VM_TRACEPOINTS + if (LTTNG_ENABLED(carrier_destroy)) { + lttng_decl_carrier_stats(mbc_stats); + lttng_decl_carrier_stats(sbc_stats); + LTTNG_CARRIER_STATS_TO_LTTNG_STATS(&(allctr->mbcs), mbc_stats); + LTTNG_CARRIER_STATS_TO_LTTNG_STATS(&(allctr->sbcs), sbc_stats); + LTTNG5(carrier_destroy, + ERTS_ALC_A2AD(allctr->alloc_no), + allctr->ix, + crr_sz, + mbc_stats, + sbc_stats); + } +#endif + #ifdef ERTS_SMP schedule_dealloc_carrier(allctr, crr); #else - dealloc_carrier(allctr, crr, 1); + dealloc_mbc(allctr, crr); #endif } } @@ -3942,9 +4267,6 @@ static struct { Eterm e; Eterm t; Eterm ramv; -#if HALFWORD_HEAP - Eterm low; -#endif Eterm sbct; #if HAVE_ERTS_MSEG Eterm asbcst; @@ -4035,9 +4357,6 @@ init_atoms(Allctr_t *allctr) AM_INIT(e); AM_INIT(t); AM_INIT(ramv); -#if HALFWORD_HEAP - AM_INIT(low); -#endif AM_INIT(sbct); #if HAVE_ERTS_MSEG AM_INIT(asbcst); @@ -4198,7 +4517,7 @@ add_fix_types(Allctr_t *allctr, int internal, Uint **hpp, Uint *szp, static Eterm sz_info_fix(Allctr_t *allctr, int internal, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -4219,7 +4538,7 @@ sz_info_fix(Allctr_t *allctr, UWord used = fix->type_size * fix->u.cpool.used; if (print_to_p) { - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; erts_print(to, arg, @@ -4247,7 +4566,7 @@ sz_info_fix(Allctr_t *allctr, UWord used = fix->type_size*fix->u.nocpool.used; if (print_to_p) { - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; erts_print(to, arg, @@ -4273,7 +4592,7 @@ static Eterm sz_info_carriers(Allctr_t *allctr, CarriersStats_t *cs, char *prefix, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -4282,7 +4601,7 @@ sz_info_carriers(Allctr_t *allctr, UWord curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; if (print_to_p) { - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; erts_print(to, arg, @@ -4323,7 +4642,7 @@ static Eterm info_cpool(Allctr_t *allctr, int sz_only, char *prefix, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -4340,7 +4659,7 @@ info_cpool(Allctr_t *allctr, } if (print_to_p) { - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; if (!sz_only) erts_print(to, arg, "%sblocks: %bpu\n", prefix, nob); @@ -4377,7 +4696,7 @@ static Eterm info_carriers(Allctr_t *allctr, CarriersStats_t *cs, char *prefix, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -4389,7 +4708,7 @@ info_carriers(Allctr_t *allctr, curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; if (print_to_p) { - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; erts_print(to, arg, @@ -4498,7 +4817,7 @@ make_name_atoms(Allctr_t *allctr) size_t prefix_len = strlen(allctr->name_prefix); if (prefix_len > MAX_ATOM_CHARACTERS + sizeof(realloc) - 1) - erl_exit(1,"Too long allocator name: %salloc\n",allctr->name_prefix); + erts_exit(ERTS_ERROR_EXIT,"Too long allocator name: %salloc\n",allctr->name_prefix); memcpy((void *) buf, (void *) allctr->name_prefix, prefix_len); @@ -4515,7 +4834,7 @@ make_name_atoms(Allctr_t *allctr) static Eterm info_calls(Allctr_t *allctr, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -4532,7 +4851,7 @@ info_calls(Allctr_t *allctr, erts_print(TO, TOA, "%s%s calls: %b64u\n",PRFX,NAME,CC) char *prefix = allctr->name_prefix; - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; PRINT_CC_5(to, arg, prefix, "alloc", allctr->calls.this_alloc); @@ -4608,7 +4927,7 @@ info_calls(Allctr_t *allctr, static Eterm info_options(Allctr_t *allctr, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -4643,9 +4962,6 @@ info_options(Allctr_t *allctr, "option e: true\n" "option t: %s\n" "option ramv: %s\n" -#if HALFWORD_HEAP - "option low: %s\n" -#endif "option sbct: %beu\n" #if HAVE_ERTS_MSEG "option asbcst: %bpu\n" @@ -4664,9 +4980,6 @@ info_options(Allctr_t *allctr, "option acul: %d\n", topt, allctr->ramv ? "true" : "false", -#if HALFWORD_HEAP - allctr->mseg_opt.low_mem ? "true" : "false", -#endif allctr->sbc_threshold, #if HAVE_ERTS_MSEG allctr->mseg_opt.abs_shrink_th, @@ -4729,9 +5042,6 @@ info_options(Allctr_t *allctr, add_2tup(hpp, szp, &res, am.sbct, bld_uint(hpp, szp, allctr->sbc_threshold)); -#if HALFWORD_HEAP - add_2tup(hpp, szp, &res, am.low, allctr->mseg_opt.low_mem ? am_true : am_false); -#endif add_2tup(hpp, szp, &res, am.ramv, allctr->ramv ? am_true : am_false); add_2tup(hpp, szp, &res, am.t, (allctr->t ? am_true : am_false)); add_2tup(hpp, szp, &res, am.e, am_true); @@ -4769,7 +5079,7 @@ reset_max_values(CarriersStats_t *cs) \* */ Eterm -erts_alcu_au_info_options(int *print_to_p, void *print_to_arg, +erts_alcu_au_info_options(fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) { Eterm res = THE_NON_VALUE; @@ -4812,7 +5122,7 @@ erts_alcu_au_info_options(int *print_to_p, void *print_to_arg, Eterm erts_alcu_info_options(Allctr_t *allctr, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -4844,7 +5154,7 @@ Eterm erts_alcu_sz_info(Allctr_t *allctr, int internal, int begin_max_period, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -4876,7 +5186,7 @@ erts_alcu_sz_info(Allctr_t *allctr, ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); - /* Update sbc values not continously updated */ + /* Update sbc values not continuously updated */ allctr->sbcs.blocks.curr.no = allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no; allctr->sbcs.blocks.max.no = allctr->sbcs.max.no; @@ -4930,7 +5240,7 @@ Eterm erts_alcu_info(Allctr_t *allctr, int internal, int begin_max_period, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -4962,7 +5272,7 @@ erts_alcu_info(Allctr_t *allctr, ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); - /* Update sbc values not continously updated */ + /* Update sbc values not continuously updated */ allctr->sbcs.blocks.curr.no = allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no; allctr->sbcs.blocks.max.no = allctr->sbcs.max.no; @@ -5126,6 +5436,11 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) void *erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) { void *res; +#ifdef ERTS_SMP + ASSERT(!"This is not thread safe"); +#elif defined(USE_THREADS) + ASSERT(erts_equal_tids(erts_main_thread, erts_thr_self())); +#endif res = do_erts_alcu_alloc(type, extra, size); DEBUG_CHECK_ALIGNMENT(res); return res; @@ -5416,11 +5731,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, Block_t *new_blk; if(IS_SBC_BLK(blk)) { do_carrier_resize: -#if HALFWORD_HEAP - new_blk = resize_carrier(allctr, blk, size, CFLG_SBC | CFLG_FORCE_MSEG); -#else new_blk = resize_carrier(allctr, blk, size, CFLG_SBC); -#endif res = new_blk ? BLK2UMEM(new_blk) : NULL; } else if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) @@ -5700,7 +6011,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) /* erts_alcu_start assumes that allctr has been zeroed */ if (((UWord)allctr & ERTS_CRR_ALCTR_FLG_MASK) != 0) { - erl_exit(ERTS_ABORT_EXIT, "%s:%d:erts_alcu_start: Alignment error\n", + erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_alcu_start: Alignment error\n", __FILE__, __LINE__); } @@ -5714,11 +6025,8 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #ifdef ERTS_SMP if (init->tspec || init->tpref) allctr->mseg_opt.sched_spec = 1; -#endif -# if HALFWORD_HEAP - allctr->mseg_opt.low_mem = init->low_mem; -# endif -#endif +#endif /* ERTS_SMP */ +#endif /* HAVE_ERTS_MSEG */ allctr->name_prefix = init->name_prefix; if (!allctr->name_prefix) @@ -5775,7 +6083,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) goto error; allctr->min_block_size = UNIT_CEILING(allctr->min_block_size + sizeof(FreeBlkFtr_t)); -#if ERTS_SMP +#ifdef ERTS_SMP if (init->tpref) { Uint sz = ABLK_HDR_SZ; sz += (init->fix ? @@ -5826,18 +6134,10 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #ifdef USE_THREADS if (init->ts) { allctr->thread_safe = 1; - -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_mtx_init_x_opt(&allctr->mutex, - "alcu_allocator", - make_small(allctr->alloc_no), - ERTS_LCNT_LT_ALLOC,1); -#else - erts_mtx_init_x(&allctr->mutex, - "alcu_allocator", - make_small(allctr->alloc_no),1); -#endif /*ERTS_ENABLE_LOCK_COUNT*/ - + + erts_mtx_init(&allctr->mutex, "alcu_allocator", make_small(allctr->alloc_no), + ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR); + #ifdef DEBUG allctr->debug.saved_tid = 0; #endif @@ -5867,24 +6167,60 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) + ABLK_HDR_SZ) - ABLK_HDR_SZ); + if (init->sys_alloc) { + ASSERT(init->sys_realloc && init->sys_dealloc); + allctr->sys_alloc = init->sys_alloc; + allctr->sys_realloc = init->sys_realloc; + allctr->sys_dealloc = init->sys_dealloc; + } + else { + ASSERT(!init->sys_realloc && !init->sys_dealloc); + allctr->sys_alloc = &erts_alcu_sys_alloc; + allctr->sys_realloc = &erts_alcu_sys_realloc; + allctr->sys_dealloc = &erts_alcu_sys_dealloc; + } +#if HAVE_ERTS_MSEG + if (init->mseg_alloc) { + ASSERT(init->mseg_realloc && init->mseg_dealloc); + allctr->mseg_alloc = init->mseg_alloc; + allctr->mseg_realloc = init->mseg_realloc; + allctr->mseg_dealloc = init->mseg_dealloc; + allctr->mseg_mmapper = init->mseg_mmapper; + } + else { + ASSERT(!init->mseg_realloc && !init->mseg_dealloc); + allctr->mseg_alloc = &erts_alcu_mseg_alloc; + allctr->mseg_realloc = &erts_alcu_mseg_realloc; + allctr->mseg_dealloc = &erts_alcu_mseg_dealloc; + } + /* If a custom carrier alloc function is specified, make sure it's used */ + if (init->mseg_alloc && !init->sys_alloc) { + allctr->crr_set_flgs = CFLG_FORCE_MSEG; + allctr->crr_clr_flgs = CFLG_FORCE_SYS_ALLOC; + } + else if (!init->mseg_alloc && init->sys_alloc) { + allctr->crr_set_flgs = CFLG_FORCE_SYS_ALLOC; + allctr->crr_clr_flgs = CFLG_FORCE_MSEG; + } +#endif + if (allctr->main_carrier_size) { Block_t *blk; blk = create_carrier(allctr, allctr->main_carrier_size, - CFLG_MBC + (ERTS_SUPER_ALIGNED_MSEG_ONLY + ? CFLG_FORCE_MSEG : CFLG_FORCE_SYS_ALLOC) + | CFLG_MBC | CFLG_FORCE_SIZE | CFLG_NO_CPOOL -#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY - | CFLG_FORCE_SYS_ALLOC -#endif | CFLG_MAIN_CARRIER); if (!blk) { #ifdef USE_THREADS if (allctr->thread_safe) erts_mtx_destroy(&allctr->mutex); #endif - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Failed to create main carrier for %salloc\n", init->name_prefix); } @@ -5980,7 +6316,8 @@ erts_alcu_init(AlcUInit_t *init) carrier_alignment = sizeof(Unit_t); #endif - erts_mtx_init(&init_atoms_mtx, "alcu_init_atoms"); + erts_mtx_init(&init_atoms_mtx, "alcu_init_atoms", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR); atoms_initialized = 0; initialized = 1; @@ -6029,7 +6366,7 @@ erts_alcu_test(UWord op, UWord a1, UWord a2) case 0x019: return (UWord) PREV_BLK((Block_t *) a1); case 0x01a: return (UWord) IS_MBC_FIRST_BLK((Allctr_t*)a1, (Block_t *) a2); case 0x01b: return (UWord) sizeof(Unit_t); - case 0x01c: return (unsigned long) BLK_TO_MBC((Block_t*) a1); + case 0x01c: return (UWord) BLK_TO_MBC((Block_t*) a1); case 0x01d: ((Allctr_t*) a1)->add_mbc((Allctr_t*)a1, (Carrier_t*)a2); break; case 0x01e: ((Allctr_t*) a1)->remove_mbc((Allctr_t*)a1, (Carrier_t*)a2); break; #ifdef ERTS_SMP @@ -6054,6 +6391,16 @@ erts_alcu_test(UWord op, UWord a1, UWord a2) case 0x023: return (UWord) 0; case 0x024: return (UWord) 0; #endif + case 0x025: /* UMEM2BLK_TEST*/ +#ifdef DEBUG +# ifdef HARD_DEBUG + return (UWord)UMEM2BLK(a1-3*sizeof(UWord)); +# else + return (UWord)UMEM2BLK(a1-2*sizeof(UWord)); +# endif +#else + return (UWord)UMEM2BLK(a1); +#endif default: ASSERT(0); return ~((UWord) 0); } @@ -6090,7 +6437,7 @@ erts_alcu_verify_unused(Allctr_t *allctr) if (no) { UWord sz = allctr->sbcs.blocks.curr.size; sz += allctr->mbcs.blocks.curr.size; - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%salloc() used when expected to be unused!\n" "Total amount of blocks allocated: %bpu\n" "Total amount of bytes allocated: %bpu\n", @@ -6130,11 +6477,6 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) ASSERT(SBC2BLK(allctr, sbc) == iblk); ASSERT(CARRIER_SZ(sbc) - SBC_HEADER_SIZE >= SBC_BLK_SZ(iblk)); -#if HAVE_ERTS_MSEG - if (IS_MSEG_CARRIER(sbc)) { - ASSERT(CARRIER_SZ(sbc) % ERTS_SACRR_UNIT_SZ == 0); - } -#endif crr = sbc; cl = &allctr->sbc_list; } @@ -6243,3 +6585,45 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) #endif /* ERTS_ALLOC_UTIL_HARD_DEBUG */ +#ifdef ERTS_ENABLE_LOCK_COUNT + +static void lcnt_enable_allocator_lock_count(Allctr_t *allocator, int enable) { + if(!allocator->thread_safe) { + return; + } + + if(enable) { + erts_lcnt_install_new_lock_info(&allocator->mutex.lcnt, + "alcu_allocator", make_small(allocator->alloc_no), + ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR); + } else { + erts_lcnt_uninstall(&allocator->mutex.lcnt); + } +} + +static void lcnt_update_thread_spec_locks(ErtsAllocatorThrSpec_t *tspec, int enable) { + if(tspec->enabled) { + int i; + + for(i = 0; i < tspec->size; i++) { + lcnt_enable_allocator_lock_count(tspec->allctr[i], enable); + } + } +} + +void erts_lcnt_update_allocator_locks(int enable) { + int i; + + for(i = ERTS_ALC_A_MIN; i < ERTS_ALC_A_MAX; i++) { + ErtsAllocatorInfo_t *ai = &erts_allctrs_info[i]; + + if(ai->enabled && ai->alloc_util) { + if(ai->thr_spec) { + lcnt_update_thread_spec_locks((ErtsAllocatorThrSpec_t*)ai->extra, enable); + } else { + lcnt_enable_allocator_lock_count((Allctr_t*)ai->extra, enable); + } + } + } +} +#endif /* ERTS_ENABLE_LOCK_COUNT */ |