From cabfe2cdc0b8d879431f81233addd2a43ff1f742 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 11 Sep 2013 21:49:32 +0200 Subject: Implement platform specific aligned sys_alloc and use when supported erts_sys_aligned_alloc() is currently implemented using posix_memalign if it exist, or using _aligned_malloc on Windows. If erts_sys_aligned_alloc() exist allocators will create sys_alloc carriers similar to how this was done pre-R16. --- erts/configure.in | 4 +- erts/emulator/beam/erl_alloc.c | 21 ++----- erts/emulator/beam/erl_alloc.h | 10 ++++ erts/emulator/beam/erl_alloc_util.c | 103 ++++++++++++++++++++-------------- erts/emulator/beam/erl_alloc_util.h | 27 ++++++--- erts/emulator/beam/sys.h | 3 + erts/emulator/sys/common/erl_mseg.h | 6 +- erts/emulator/sys/unix/erl_unix_sys.h | 4 ++ erts/emulator/sys/unix/sys.c | 46 +++++++++++++++ erts/emulator/sys/win32/erl_win_sys.h | 2 + erts/emulator/sys/win32/sys.c | 26 ++++++++- 11 files changed, 181 insertions(+), 71 deletions(-) diff --git a/erts/configure.in b/erts/configure.in index 7257751068..c391aece14 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -1924,8 +1924,8 @@ AC_CHECK_FUNCS([getipnodebyname getipnodebyaddr gethostbyname2]) AC_CHECK_FUNCS([ieee_handler fpsetmask finite isnan isinf res_gethostbyname dlopen \ pread pwrite memmove strerror strerror_r strncasecmp \ - gethrtime localtime_r gmtime_r inet_pton mmap mremap memcpy mallopt \ - sbrk _sbrk __sbrk brk _brk __brk \ + gethrtime localtime_r gmtime_r inet_pton posix_memalign \ + mmap mremap memcpy mallopt sbrk _sbrk __sbrk brk _brk __brk \ flockfile fstat strlcpy strlcat setsid posix2time time2posix \ setlocale nl_langinfo poll]) diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index d748f86d75..13692edbfc 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -235,7 +235,6 @@ set_default_sbmbc_alloc_opts(struct au_init *ip) ip->init.bf.ao = 1; ip->init.util.ramv = 0; ip->init.util.mmsbc = 0; - ip->init.util.mmmbc = 500; ip->init.util.sbct = ~((UWord) 0); ip->init.util.name_prefix = "sbmbc_"; ip->init.util.alloc_no = ERTS_ALC_A_SBMBC; @@ -261,7 +260,6 @@ set_default_sl_alloc_opts(struct au_init *ip) ip->thr_spec = 1; ip->atype = GOODFIT; ip->init.util.name_prefix = "sl_"; - ip->init.util.mmmbc = 5; ip->init.util.alloc_no = ERTS_ALC_A_SHORT_LIVED; #ifndef SMALL_MEMORY ip->init.util.mmbcs = 128*1024; /* Main carrier size */ @@ -285,7 +283,6 @@ set_default_std_alloc_opts(struct au_init *ip) ip->thr_spec = 1; ip->atype = BESTFIT; ip->init.util.name_prefix = "std_"; - ip->init.util.mmmbc = 5; ip->init.util.alloc_no = ERTS_ALC_A_STANDARD; #ifndef SMALL_MEMORY ip->init.util.mmbcs = 128*1024; /* Main carrier size */ @@ -305,7 +302,6 @@ set_default_ll_alloc_opts(struct au_init *ip) ip->init.bf.ao = 1; ip->init.util.ramv = 0; ip->init.util.mmsbc = 0; - ip->init.util.mmmbc = 0; ip->init.util.sbct = ~((UWord) 0); ip->init.util.name_prefix = "ll_"; ip->init.util.alloc_no = ERTS_ALC_A_LONG_LIVED; @@ -353,7 +349,6 @@ set_default_eheap_alloc_opts(struct au_init *ip) ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; ip->atype = GOODFIT; - ip->init.util.mmmbc = 100; ip->init.util.name_prefix = "eheap_"; ip->init.util.alloc_no = ERTS_ALC_A_EHEAP; #ifndef SMALL_MEMORY @@ -376,7 +371,6 @@ set_default_binary_alloc_opts(struct au_init *ip) ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; ip->atype = BESTFIT; - ip->init.util.mmmbc = 50; ip->init.util.name_prefix = "binary_"; ip->init.util.alloc_no = ERTS_ALC_A_BINARY; #ifndef SMALL_MEMORY @@ -394,7 +388,6 @@ set_default_ets_alloc_opts(struct au_init *ip) ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; ip->atype = BESTFIT; - ip->init.util.mmmbc = 100; ip->init.util.name_prefix = "ets_"; ip->init.util.alloc_no = ERTS_ALC_A_ETS; #ifndef SMALL_MEMORY @@ -462,13 +455,6 @@ adjust_tpref(struct au_init *ip, int no_sched) /* ... shrink smallest multi-block carrier size */ if (ip->default_.smbcs) ip->init.util.smbcs /= ERTS_MIN(4, no_sched); - /* ... and more than three allocators shrink - max mseg multi-block carriers */ - if (ip->default_.mmmbc && no_sched > 2) { - ip->init.util.mmmbc /= ERTS_MIN(4, no_sched - 1); - if (ip->init.util.mmmbc < 3) - ip->init.util.mmmbc = 3; - } } } @@ -2609,8 +2595,8 @@ erts_allocator_options(void *proc) #endif Uint sz, *szp, *hp, **hpp; Eterm res, features, settings; - Eterm atoms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+5]; - Uint terms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+5]; + Eterm atoms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+6]; + Uint terms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+6]; int a, length; SysAllocStat sas; Uint *endp = NULL; @@ -2723,6 +2709,9 @@ erts_allocator_options(void *proc) if (use_mseg) terms[length++] = am_atom_put("mseg_alloc", 10); #endif +#if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC + terms[length++] = am_atom_put("sys_aligned_alloc", 17); +#endif features = length ? erts_bld_list(hpp, szp, length, terms) : NIL; diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index ba5ec9c367..7168241f76 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -54,6 +54,16 @@ void erts_sys_alloc_init(void); void *erts_sys_alloc(ErtsAlcType_t, void *, Uint); void *erts_sys_realloc(ErtsAlcType_t, void *, void *, Uint); void erts_sys_free(ErtsAlcType_t, void *, void *); +#if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC +/* + * Note 'alignment' must remain the same in calls to + * 'erts_sys_aligned_realloc()' and 'erts_sys_aligned_free()' + * as in the initial call to 'erts_sys_aligned_alloc()'. + */ +void *erts_sys_aligned_alloc(UWord alignment, UWord size); +void *erts_sys_aligned_realloc(UWord alignment, void *ptr, UWord size, UWord old_size); +void erts_sys_aligned_free(UWord alignment, void *ptr); +#endif Eterm erts_memory(int *, void *, void *, Eterm); Eterm erts_allocated_areas(int *, void *, void *); diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index ac7f420708..24f297dc58 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -76,14 +76,14 @@ static int initialized = 0; int erts_have_sbmbc_alloc; -#if HAVE_ERTS_MSEG +#if ERTS_SA_MB_CARRIERS -#define MSEG_UNIT_SHIFT MSEG_ALIGN_BITS -#define MSEG_UNIT_SZ (1 << MSEG_UNIT_SHIFT) -#define MSEG_UNIT_MASK ((~(UWord)0) << MSEG_UNIT_SHIFT) +#define ERTS_SACRR_UNIT_SHIFT ERTS_SUPER_ALIGN_BITS +#define ERTS_SACRR_UNIT_SZ (1 << ERTS_SACRR_UNIT_SHIFT) +#define ERTS_SACRR_UNIT_MASK ((~(UWord)0) << ERTS_SACRR_UNIT_SHIFT) -#define MSEG_UNIT_FLOOR(X) ((X) & MSEG_UNIT_MASK) -#define MSEG_UNIT_CEILING(X) MSEG_UNIT_FLOOR((X) + ~MSEG_UNIT_MASK) +#define ERTS_SACRR_UNIT_FLOOR(X) ((X) & ERTS_SACRR_UNIT_MASK) +#define ERTS_SACRR_UNIT_CEILING(X) ERTS_SACRR_UNIT_FLOOR((X) + ~ERTS_SACRR_UNIT_MASK) #endif @@ -208,9 +208,9 @@ MBC after deallocating first block: #if MBC_ABLK_OFFSET_BITS -# define MBC_SZ_MAX_LIMIT ((((UWord)1 << MBC_ABLK_OFFSET_BITS) - 1) << MSEG_ALIGN_BITS) +# define MBC_SZ_MAX_LIMIT ((((UWord)1 << MBC_ABLK_OFFSET_BITS) - 1) << ERTS_SUPER_ALIGN_BITS) -# define BLK_CARRIER_OFFSET(B, C) (((char*)(B) - (char*)(C)) >> MSEG_UNIT_SHIFT) +# define BLK_CARRIER_OFFSET(B, C) (((char*)(B) - (char*)(C)) >> ERTS_SACRR_UNIT_SHIFT) # define SET_MBC_ABLK_HDR(B, Sz, F, C) \ (ASSERT(((Sz) & ~MBC_ABLK_SZ_MASK) == 0), \ @@ -225,8 +225,8 @@ MBC after deallocating first block: # define ABLK_TO_MBC(B) \ (ASSERT(IS_MBC_BLK(B) && IS_ALLOCED_BLK(B)), \ - (Carrier_t*)((MSEG_UNIT_FLOOR((UWord)(B)) - \ - (((B)->bhdr >> MBC_ABLK_OFFSET_SHIFT) << MSEG_UNIT_SHIFT)))) + (Carrier_t*)((ERTS_SACRR_UNIT_FLOOR((UWord)(B)) - \ + (((B)->bhdr >> MBC_ABLK_OFFSET_SHIFT) << ERTS_SACRR_UNIT_SHIFT)))) # define FBLK_TO_MBC(B) \ (ASSERT(IS_MBC_BLK(B) && IS_FREE_BLK(B)), \ @@ -235,7 +235,7 @@ MBC after deallocating first block: # define BLK_TO_MBC(B) (IS_FREE_BLK(B) ? FBLK_TO_MBC(B) : ABLK_TO_MBC(B)) # define IS_MBC_FIRST_ABLK(AP,B) \ - ((((UWord)(B) & ~MSEG_UNIT_MASK) == MBC_HEADER_SIZE(AP)) \ + ((((UWord)(B) & ~ERTS_SACRR_UNIT_MASK) == MBC_HEADER_SIZE(AP)) \ && ((B)->bhdr & MBC_ABLK_OFFSET_MASK) == 0) # define IS_MBC_FIRST_FBLK(AP,B) \ @@ -652,11 +652,15 @@ alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) #endif static ERTS_INLINE void * -alcu_sys_alloc(Allctr_t *allctr, Uint size) +alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign) { void *res; - - res = erts_sys_alloc(0, NULL, size); +#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC + if (superalign) + res = erts_sys_aligned_alloc(ERTS_SACRR_UNIT_SZ, size); + else +#endif + res = erts_sys_alloc(0, NULL, size); INC_CC(allctr->calls.sys_alloc); if (erts_mtrace_enabled) erts_mtrace_crr_alloc(res, allctr->alloc_no, ERTS_ALC_A_SYSTEM, size); @@ -664,11 +668,16 @@ alcu_sys_alloc(Allctr_t *allctr, Uint size) } static ERTS_INLINE void * -alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size) +alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign) { void *res; - res = erts_sys_realloc(0, NULL, ptr, size); +#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC + if (superalign) + res = erts_sys_aligned_realloc(ERTS_SACRR_UNIT_SZ, ptr, size, old_size); + else +#endif + res = erts_sys_realloc(0, NULL, ptr, size); INC_CC(allctr->calls.sys_realloc); if (erts_mtrace_enabled) erts_mtrace_crr_realloc(res, @@ -680,9 +689,14 @@ alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size) } static ERTS_INLINE void -alcu_sys_free(Allctr_t *allctr, void *ptr) +alcu_sys_free(Allctr_t *allctr, void *ptr, int superalign) { - erts_sys_free(0, NULL, ptr); +#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC + if (superalign) + erts_sys_aligned_free(ERTS_SACRR_UNIT_SZ, ptr); + else +#endif + erts_sys_free(0, NULL, ptr); INC_CC(allctr->calls.sys_free); if (erts_mtrace_enabled) erts_mtrace_crr_free(allctr->alloc_no, ERTS_ALC_A_SYSTEM, ptr); @@ -1321,7 +1335,7 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp, Uint32 *alcu_flgsp) blk = create_sbmbc(allctr, get_blk_sz); else { blk = create_carrier(allctr, get_blk_sz, CFLG_MBC); -#if !HALFWORD_HEAP && !HAVE_SUPER_ALIGNED_MB_CARRIERS +#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY if (!blk) { /* Emergency! We couldn't create the carrier as we wanted. Try to place it in a sys_alloced sbc. */ @@ -1927,10 +1941,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) #ifdef DEBUG -#if HAVE_ERTS_MSEG -#define ASSERT_MSEG_UNIT_SIZE_MULTIPLE(CSZ) ASSERT((CSZ) % MSEG_UNIT_SZ == 0) +#if ERTS_SA_MB_CARRIERS +#define ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE(CSZ) ASSERT((CSZ) % ERTS_SACRR_UNIT_SZ == 0) #else -#define ASSERT_MSEG_UNIT_SIZE_MULTIPLE(CSZ) +#define ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE(CSZ) #endif static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C, @@ -1955,7 +1969,7 @@ static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C, } if ((MSEGED)) { ASSERT(IS_MSEG_CARRIER((C))); - ASSERT_MSEG_UNIT_SIZE_MULTIPLE((CSZ)); + ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE((CSZ)); } else { ASSERT(IS_SYS_ALLOC_CARRIER((C))); @@ -2058,7 +2072,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) #if HALFWORD_HEAP flags |= CFLG_FORCE_MSEG; -#elif HAVE_SUPER_ALIGNED_MB_CARRIERS +#elif ERTS_SUPER_ALIGNED_MSEG_ONLY if (flags & CFLG_MBC) { flags |= CFLG_FORCE_MSEG; } @@ -2083,7 +2097,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) if (allctr->sbcs.curr.norm.mseg.no >= allctr->max_mseg_sbcs) goto try_sys_alloc; } -#if !HAVE_SUPER_ALIGNED_MB_CARRIERS +#if !ERTS_SUPER_ALIGNED_MSEG_ONLY else { if (allctr->mbcs.curr.norm.mseg.no >= allctr->max_mseg_mbcs) goto try_sys_alloc; @@ -2146,12 +2160,12 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) ? UNIT_CEILING(bcrr_sz) : SYS_ALLOC_CARRIER_CEILING(bcrr_sz)); - crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz); + crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz, flags & CFLG_MBC); if (!crr) { if (crr_sz > UNIT_CEILING(bcrr_sz)) { crr_sz = UNIT_CEILING(bcrr_sz); - crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz); + crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz, flags & CFLG_MBC); } if (!crr) { #if HAVE_ERTS_MSEG @@ -2245,7 +2259,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) if (!(flags & CFLG_FORCE_SYS_ALLOC)) { new_crr_sz = new_blk_sz + SBC_HEADER_SIZE; - new_crr_sz = MSEG_UNIT_CEILING(new_crr_sz); + new_crr_sz = ERTS_SACRR_UNIT_CEILING(new_crr_sz); new_crr = (Carrier_t *) alcu_mseg_realloc(allctr, old_crr, old_crr_sz, @@ -2295,7 +2309,9 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) new_crr = (Carrier_t *) alcu_sys_realloc(allctr, (void *) old_crr, - new_crr_sz); + new_crr_sz, + old_crr_sz, + 0); if (new_crr) { sys_realloc_success: SET_CARRIER_SZ(new_crr, new_crr_sz); @@ -2314,7 +2330,9 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) new_crr_sz = UNIT_CEILING(new_crr_sz); new_crr = (Carrier_t *) alcu_sys_realloc(allctr, (void *) old_crr, - new_crr_sz); + new_crr_sz, + old_crr_sz, + 0); if (new_crr) goto sys_realloc_success; } @@ -2333,7 +2351,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) (void *) BLK2UMEM(old_blk), MIN(new_blk_sz, old_blk_sz) - ABLK_HDR_SZ); unlink_carrier(&allctr->sbc_list, old_crr); - alcu_sys_free(allctr, old_crr); + alcu_sys_free(allctr, old_crr, 0); } else { /* Old carrier unchanged; restore... */ @@ -2350,6 +2368,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) { Uint crr_sz; Carrier_t *crr; + int is_mbc; #if HAVE_ERTS_MSEG Uint is_mseg = 0; Uint mseg_flags = ERTS_MSEG_FLG_NONE; @@ -2357,6 +2376,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) if (IS_SBC_BLK(blk)) { Uint blk_sz = SBC_BLK_SZ(blk); + is_mbc = 0; crr = BLK_TO_SBC(blk); crr_sz = CARRIER_SZ(crr); @@ -2367,7 +2387,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(crr)) { is_mseg++; - ASSERT(crr_sz % MSEG_UNIT_SZ == 0); + ASSERT(crr_sz % ERTS_SACRR_UNIT_SZ == 0); STAT_MSEG_SBC_FREE(allctr, crr_sz, blk_sz); } else @@ -2378,6 +2398,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) } else { + is_mbc = 1; ASSERT(IS_MBC_FIRST_FBLK(allctr, blk)); crr = FIRST_BLK_TO_MBC(allctr, blk); crr_sz = CARRIER_SZ(crr); @@ -2397,7 +2418,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(crr)) { is_mseg++; - ASSERT(crr_sz % MSEG_UNIT_SZ == 0); + ASSERT(crr_sz % ERTS_SACRR_UNIT_SZ == 0); STAT_MSEG_MBC_FREE(allctr, crr_sz); mseg_flags = ERTS_MSEG_FLG_2POW; } @@ -2417,7 +2438,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) } else #endif - alcu_sys_free(allctr, crr); + alcu_sys_free(allctr, crr, is_mbc); } @@ -3833,7 +3854,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, crr_sz = SYS_ALLOC_CARRIER_CEILING(used_sz); #if HAVE_ERTS_MSEG else - crr_sz = MSEG_UNIT_CEILING(used_sz); + crr_sz = ERTS_SACRR_UNIT_CEILING(used_sz); #endif diff_sz_val = crr_sz - used_sz; if (diff_sz_val < (~((Uint) 0) / 100)) @@ -4175,7 +4196,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->mbc_move_threshold = init->rmbcmt; #if HAVE_ERTS_MSEG allctr->max_mseg_sbcs = init->mmsbc; -# if HAVE_SUPER_ALIGNED_MB_CARRIERS +# if ERTS_SUPER_ALIGNED_MSEG_ONLY allctr->max_mseg_mbcs = ~(Uint)0; # else allctr->max_mseg_mbcs = init->mmmbc; @@ -4300,7 +4321,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->main_carrier_size, CFLG_MBC | CFLG_FORCE_SIZE -#if !HALFWORD_HEAP && !HAVE_SUPER_ALIGNED_MB_CARRIERS +#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY | CFLG_FORCE_SYS_ALLOC #endif | CFLG_MAIN_CARRIER); @@ -4369,9 +4390,9 @@ erts_alcu_init(AlcUInit_t *init) { ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */ #if HAVE_ERTS_MSEG - ASSERT(erts_mseg_unit_size() == MSEG_UNIT_SZ); + ASSERT(erts_mseg_unit_size() == ERTS_SACRR_UNIT_SZ); max_mseg_carriers = init->mmc; - sys_alloc_carrier_size = MSEG_UNIT_CEILING(init->ycs); + sys_alloc_carrier_size = ERTS_SACRR_UNIT_CEILING(init->ycs); #else /* #if HAVE_ERTS_MSEG */ sys_alloc_carrier_size = ((init->ycs + 4095) / 4096) * 4096; #endif @@ -4493,7 +4514,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) ASSERT(CARRIER_SZ(sbc) - SBC_HEADER_SIZE >= SBC_BLK_SZ(iblk)); #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(sbc)) { - ASSERT(CARRIER_SZ(sbc) % MSEG_UNIT_SZ == 0); + ASSERT(CARRIER_SZ(sbc) % ERTS_SACRR_UNIT_SZ == 0); } #endif crr = sbc; @@ -4578,7 +4599,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(crr)) { - ASSERT(CARRIER_SZ(crr) % MSEG_UNIT_SZ == 0); + ASSERT(CARRIER_SZ(crr) % ERTS_SACRR_UNIT_SZ == 0); } #endif cl = &allctr->mbc_list; diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index e0754e7f69..e975dc34a7 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -76,7 +76,7 @@ typedef struct { #define ERTS_DEFAULT_ALCU_INIT { \ 1024*1024, /* (bytes) ycs: sys_alloc carrier size */\ - 1024 /* (amount) mmc: max mseg carriers */\ + ~((UWord) 0) /* (amount) mmc: max mseg carriers */ \ } #define ERTS_DEFAULT_ALLCTR_INIT { \ @@ -96,7 +96,7 @@ typedef struct { 50, /* (%) rmbcmt: rel mbc move threshold */\ 1024*1024, /* (bytes) mmbcs: main multiblock carrier size */\ 256, /* (amount) mmsbc: max mseg sbcs */\ - 10, /* (amount) mmmbc: max mseg mbcs */\ + ~((UWord) 0), /* (amount) mmmbc: max mseg mbcs */ \ 10*1024*1024, /* (bytes) lmbcs: largest mbc size */\ 1024*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ @@ -130,7 +130,7 @@ typedef struct { 80, /* (%) rsbcmt: rel sbc move threshold */\ 128*1024, /* (bytes) mmbcs: main multiblock carrier size */\ 256, /* (amount) mmsbc: max mseg sbcs */\ - 10, /* (amount) mmmbc: max mseg mbcs */\ + ~((UWord) 0), /* (amount) mmmbc: max mseg mbcs */ \ 1024*1024, /* (bytes) lmbcs: largest mbc size */\ 128*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ @@ -221,18 +221,29 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); #define MBC_FBLK_SZ_MASK UNIT_MASK #define CARRIER_SZ_MASK UNIT_MASK - -#if HAVE_ERTS_MSEG +#if ERTS_HAVE_MSEG_SUPER_ALIGNED \ + || (!HAVE_ERTS_MSEG && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC) +# ifndef MSEG_ALIGN_BITS +# define ERTS_SUPER_ALIGN_BITS MSEG_ALIGN_BITS +# else +# define ERTS_SUPER_ALIGN_BITS 18 +# endif # ifdef ARCH_64 # define MBC_ABLK_OFFSET_BITS 24 -# elif HAVE_SUPER_ALIGNED_MB_CARRIERS +# else # define MBC_ABLK_OFFSET_BITS 9 /* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */ # endif -#endif -#ifndef MBC_ABLK_OFFSET_BITS +# define ERTS_SA_MB_CARRIERS 1 +#else +# define ERTS_SA_MB_CARRIERS 0 # define MBC_ABLK_OFFSET_BITS 0 /* no carrier offset in block header */ #endif +#if ERTS_HAVE_MSEG_SUPER_ALIGNED && !ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC +# define ERTS_SUPER_ALIGNED_MSEG_ONLY 1 +#else +# define ERTS_SUPER_ALIGNED_MSEG_ONLY 0 +#endif #if MBC_ABLK_OFFSET_BITS # define MBC_ABLK_OFFSET_SHIFT (sizeof(UWord)*8 - MBC_ABLK_OFFSET_BITS) diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 05bff430e3..096394b878 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -1012,6 +1012,9 @@ void erl_bin_write(unsigned char *, int, int); #define ERTS_SMALL_ABS(Small) labs(Small) #endif +#ifndef ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC +# define ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC 0 +#endif #ifdef __WIN32__ void call_break_handler(void); diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h index 3cab9e18da..127576b850 100644 --- a/erts/emulator/sys/common/erl_mseg.h +++ b/erts/emulator/sys/common/erl_mseg.h @@ -32,13 +32,13 @@ #if HAVE_MMAP # define HAVE_ERTS_MSEG 1 -# define HAVE_SUPER_ALIGNED_MB_CARRIERS 1 +# define ERTS_HAVE_MSEG_SUPER_ALIGNED 1 #else # define HAVE_ERTS_MSEG 0 -# define HAVE_SUPER_ALIGNED_MB_CARRIERS 0 +# define ERTS_HAVE_MSEG_SUPER_ALIGNED 0 #endif -#if HAVE_SUPER_ALIGNED_MB_CARRIERS +#if ERTS_HAVE_MSEG_SUPER_ALIGNED # define MSEG_ALIGN_BITS (18) /* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */ #else diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h index c8fcec8547..2c47aa06c2 100644 --- a/erts/emulator/sys/unix/erl_unix_sys.h +++ b/erts/emulator/sys/unix/erl_unix_sys.h @@ -107,6 +107,10 @@ #endif #include +#ifdef HAVE_POSIX_MEMALIGN +# define ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC 1 +#endif + /* * Make sure that MAXPATHLEN is defined. */ diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index a7ea4b2490..fdc3167c62 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -2524,6 +2524,52 @@ void erts_sys_alloc_init(void) { } +#if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC +void *erts_sys_aligned_alloc(UWord alignment, UWord size) +{ +#ifdef HAVE_POSIX_MEMALIGN + void *ptr = NULL; + int error; + ASSERT(alignment && (alignment & ~alignment) == 0); /* power of 2 */ + error = posix_memalign(&ptr, (size_t) alignment, (size_t) size); +#if HAVE_ERTS_MSEG + if (error || !ptr) { + erts_mseg_clear_cache(); + error = posix_memalign(&ptr, (size_t) alignment, (size_t) size); + } +#endif + if (error) { + errno = error; + return NULL; + } + if (!ptr) + errno = ENOMEM; + ASSERT(!ptr || (((UWord) ptr) & (alignment - 1)) == 0); + return ptr; +#else +# error "Missing erts_sys_aligned_alloc() implementation" +#endif +} + +void erts_sys_aligned_free(UWord alignment, void *ptr) +{ + ASSERT(alignment && (alignment & ~alignment) == 0); /* power of 2 */ + free(ptr); +} + +void *erts_sys_aligned_realloc(UWord alignment, void *ptr, UWord size, UWord old_size) +{ + void *new_ptr = erts_sys_aligned_alloc(alignment, size); + if (new_ptr) { + UWord copy_size = old_size < size ? old_size : size; + sys_memcpy(new_ptr, ptr, (size_t) copy_size); + erts_sys_aligned_free(alignment, ptr); + } + return new_ptr; +} + +#endif + void *erts_sys_alloc(ErtsAlcType_t t, void *x, Uint sz) { void *res = malloc((size_t) sz); diff --git a/erts/emulator/sys/win32/erl_win_sys.h b/erts/emulator/sys/win32/erl_win_sys.h index 5ce1a61303..c44c5d9492 100644 --- a/erts/emulator/sys/win32/erl_win_sys.h +++ b/erts/emulator/sys/win32/erl_win_sys.h @@ -95,6 +95,8 @@ # define ERTS_I64_LITERAL(X) X##i64 #endif +#define ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC 1 + /* * Practial Windows specific macros. */ diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c index e0422de026..8244a43583 100755 --- a/erts/emulator/sys/win32/sys.c +++ b/erts/emulator/sys/win32/sys.c @@ -32,7 +32,7 @@ #include "erl_threads.h" #include "../../drivers/win32/win_con.h" #include "erl_cpu_topology.h" - +#include void erts_sys_init_float(void); @@ -2906,6 +2906,30 @@ void erts_sys_free(ErtsAlcType_t t, void *x, void *p) free(p); } +void *erts_sys_aligned_alloc(UWord alignment, UWord size) +{ + void *ptr; + ASSERT(alignment && (alignment & ~alignment) == 0); /* power of 2 */ + ptr = _aligned_malloc((size_t) size, (size_t) alignment); + ASSERT(!ptr || (((UWord) ptr) & (alignment - 1)) == 0); + return ptr; +} + +void erts_sys_aligned_free(UWord alignment, void *ptr) +{ + ASSERT(alignment && (alignment & ~alignment) == 0); /* power of 2 */ + _aligned_free(ptr); +} + +void *erts_sys_aligned_realloc(UWord alignment, void *ptr, UWord size, UWord old_size) +{ + void *new_ptr; + ASSERT(alignment && (alignment & ~alignment) == 0); /* power of 2 */ + new_ptr = _aligned_realloc(ptr, (size_t) size, (size_t) alignment); + ASSERT(!new_ptr || (((UWord) new_ptr) & (alignment - 1)) == 0); + return new_ptr; +} + static Preload* preloaded = NULL; static unsigned* res_name = NULL; static int num_preloaded = 0; -- cgit v1.2.3