diff options
author | John Högberg <[email protected]> | 2018-07-03 09:33:02 +0200 |
---|---|---|
committer | GitHub <[email protected]> | 2018-07-03 09:33:02 +0200 |
commit | 6cfa8af60a9c6fe9043e4992434d40d4a34019e3 (patch) | |
tree | 822993e0a4fb987014238369db045bf53eaafba3 | |
parent | bc22321f10c1ad71fb7e28275a8e6ed26a34d4ab (diff) | |
parent | 29ce7ab0e4403fa688d1e5e5e2170e88d7e8f299 (diff) | |
download | otp-6cfa8af60a9c6fe9043e4992434d40d4a34019e3.tar.gz otp-6cfa8af60a9c6fe9043e4992434d40d4a34019e3.tar.bz2 otp-6cfa8af60a9c6fe9043e4992434d40d4a34019e3.zip |
Merge pull request #1854 from jhogberg/john/erts/cross-type-carrier-migration/OTP-15063
Allow carrier migration between different allocator types
-rw-r--r-- | erts/doc/src/erts_alloc.xml | 11 | ||||
-rw-r--r-- | erts/emulator/beam/erl_alloc.c | 234 | ||||
-rw-r--r-- | erts/emulator/beam/erl_alloc.h | 33 | ||||
-rw-r--r-- | erts/emulator/beam/erl_alloc_util.c | 666 | ||||
-rw-r--r-- | erts/emulator/beam/erl_alloc_util.h | 81 | ||||
-rw-r--r-- | erts/emulator/beam/erl_ao_firstfit_alloc.c | 66 | ||||
-rw-r--r-- | erts/emulator/beam/erl_ao_firstfit_alloc.h | 7 | ||||
-rw-r--r-- | erts/emulator/internal_doc/CarrierMigration.md | 15 | ||||
-rw-r--r-- | erts/emulator/test/alloc_SUITE.erl | 3 | ||||
-rw-r--r-- | erts/emulator/test/driver_SUITE.erl | 19 | ||||
-rw-r--r-- | erts/emulator/test/erts_debug_SUITE.erl | 31 | ||||
-rw-r--r-- | erts/emulator/test/process_SUITE.erl | 8 | ||||
-rw-r--r-- | erts/emulator/test/system_info_SUITE.erl | 13 | ||||
-rw-r--r-- | erts/preloaded/ebin/erlang.beam | bin | 103300 -> 101860 bytes | |||
-rw-r--r-- | erts/preloaded/src/erlang.erl | 241 | ||||
-rw-r--r-- | lib/kernel/src/erts_debug.erl | 58 | ||||
-rw-r--r-- | lib/stdlib/test/ets_SUITE.erl | 35 | ||||
-rw-r--r-- | lib/tools/doc/src/instrument.xml | 13 | ||||
-rw-r--r-- | lib/tools/test/instrument_SUITE.erl | 30 |
19 files changed, 910 insertions, 654 deletions
diff --git a/erts/doc/src/erts_alloc.xml b/erts/doc/src/erts_alloc.xml index a094217959..962bc9a244 100644 --- a/erts/doc/src/erts_alloc.xml +++ b/erts/doc/src/erts_alloc.xml @@ -487,11 +487,10 @@ utilization value used. Once a carrier is abandoned, no new allocations are made in it. When an allocator instance gets an increased multiblock carrier need, it first tries to fetch an - abandoned carrier from an allocator instance of the same - allocator type. If no abandoned carrier can be fetched, it - creates a new empty carrier. When an abandoned carrier has been - fetched, it will function as an ordinary carrier. This feature has - special requirements on the + abandoned carrier from another allocator instance. If no abandoned + carrier can be fetched, it creates a new empty carrier. When an + abandoned carrier has been fetched, it will function as an ordinary + carrier. This feature has special requirements on the <seealso marker="#M_as">allocation strategy</seealso> used. Only the strategies <c>aoff</c>, <c>aoffcbf</c>, <c>aoffcaobf</c>, <c>ageffcaoff</c>m, <c>ageffcbf</c> and <c>ageffcaobf</c> @@ -584,7 +583,7 @@ carriers are decided in section <seealso marker="#mseg_mbc_sizes"> The alloc_util Framework</seealso>. On - 32-bit Unix style OS this limit cannot be set > 128 MB.</p> + 32-bit Unix style OS this limit cannot be set > 64 MB.</p> </item> <tag><marker id="M_mbcgs"/><c><![CDATA[+M<S>mbcgs <ratio>]]></c></tag> <item> diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 575d6ca867..493c3799c2 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -64,9 +64,6 @@ # error "Too many schedulers; cannot create that many pref alloc instances" #endif -#define ERTS_ALC_FIX_TYPE_IX(T) \ - (ERTS_ALC_T2N((T)) - ERTS_ALC_N_MIN_A_FIXED_SIZE) - #define ERTS_ALC_DEFAULT_MAX_THR_PREF ERTS_MAX_NO_OF_SCHEDULERS #if defined(SMALL_MEMORY) || defined(PURIFY) || defined(VALGRIND) @@ -156,20 +153,13 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(aireq, ErtsAlcType_t erts_fix_core_allocator_ix; -enum allctr_type { - GOODFIT, - BESTFIT, - AFIT, - FIRSTFIT -}; - struct au_init { int enable; int thr_spec; int disable_allowed; int thr_spec_allowed; int carrier_migration_allowed; - enum allctr_type atype; + ErtsAlcStrat_t astrat; struct { AllctrInit_t util; GFAllctrInit_t gf; @@ -219,7 +209,9 @@ typedef struct { struct au_init test_alloc; } erts_alc_hndl_args_init_t; -#define ERTS_AU_INIT__ {0, 0, 1, 1, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}} +#define ERTS_AU_INIT__ {0, 0, 1, 1, 1, \ + ERTS_ALC_S_GOODFIT, DEFAULT_ALLCTR_INIT, \ + {1,1,1,1}} #define SET_DEFAULT_ALLOC_OPTS(IP) \ do { \ @@ -233,7 +225,7 @@ set_default_sl_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; - ip->atype = GOODFIT; + ip->astrat = ERTS_ALC_S_GOODFIT; ip->init.util.name_prefix = "sl_"; ip->init.util.alloc_no = ERTS_ALC_A_SHORT_LIVED; #ifndef SMALL_MEMORY @@ -252,7 +244,7 @@ set_default_std_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; - ip->atype = BESTFIT; + ip->astrat = ERTS_ALC_S_BESTFIT; ip->init.util.name_prefix = "std_"; ip->init.util.alloc_no = ERTS_ALC_A_STANDARD; #ifndef SMALL_MEMORY @@ -270,7 +262,7 @@ set_default_ll_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 0; - ip->atype = BESTFIT; + ip->astrat = ERTS_ALC_S_BESTFIT; ip->init.bf.ao = 1; ip->init.util.ramv = 0; ip->init.util.mmsbc = 0; @@ -299,7 +291,7 @@ set_default_literal_alloc_opts(struct au_init *ip) ip->disable_allowed = 0; ip->thr_spec_allowed = 0; ip->carrier_migration_allowed = 0; - ip->atype = BESTFIT; + ip->astrat = ERTS_ALC_S_BESTFIT; ip->init.bf.ao = 1; ip->init.util.ramv = 0; ip->init.util.mmsbc = 0; @@ -349,7 +341,7 @@ set_default_exec_alloc_opts(struct au_init *ip) ip->disable_allowed = 0; ip->thr_spec_allowed = 0; ip->carrier_migration_allowed = 0; - ip->atype = BESTFIT; + ip->astrat = ERTS_ALC_S_BESTFIT; ip->init.bf.ao = 1; ip->init.util.ramv = 0; ip->init.util.mmsbc = 0; @@ -378,7 +370,7 @@ set_default_temp_alloc_opts(struct au_init *ip) ip->thr_spec = 1; ip->disable_allowed = 0; ip->carrier_migration_allowed = 0; - ip->atype = AFIT; + ip->astrat = ERTS_ALC_S_AFIT; ip->init.util.name_prefix = "temp_"; ip->init.util.alloc_no = ERTS_ALC_A_TEMPORARY; #ifndef SMALL_MEMORY @@ -397,7 +389,7 @@ set_default_eheap_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; - ip->atype = GOODFIT; + ip->astrat = ERTS_ALC_S_GOODFIT; ip->init.util.name_prefix = "eheap_"; ip->init.util.alloc_no = ERTS_ALC_A_EHEAP; #ifndef SMALL_MEMORY @@ -416,7 +408,7 @@ set_default_binary_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; - ip->atype = BESTFIT; + ip->astrat = ERTS_ALC_S_BESTFIT; ip->init.util.name_prefix = "binary_"; ip->init.util.alloc_no = ERTS_ALC_A_BINARY; #ifndef SMALL_MEMORY @@ -435,7 +427,7 @@ set_default_ets_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; - ip->atype = BESTFIT; + ip->astrat = ERTS_ALC_S_BESTFIT; ip->init.util.name_prefix = "ets_"; ip->init.util.alloc_no = ERTS_ALC_A_ETS; #ifndef SMALL_MEMORY @@ -453,7 +445,7 @@ set_default_driver_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; - ip->atype = BESTFIT; + ip->astrat = ERTS_ALC_S_BESTFIT; ip->init.util.name_prefix = "driver_"; ip->init.util.alloc_no = ERTS_ALC_A_DRIVER; #ifndef SMALL_MEMORY @@ -473,7 +465,7 @@ set_default_fix_alloc_opts(struct au_init *ip, SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; - ip->atype = BESTFIT; + ip->astrat = ERTS_ALC_S_BESTFIT; ip->init.bf.ao = 1; ip->init.util.name_prefix = "fix_"; ip->init.util.fix_type_size = fix_type_sizes; @@ -493,7 +485,7 @@ set_default_test_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = 0; /* Disabled by default */ ip->thr_spec = -1 * erts_no_schedulers; - ip->atype = FIRSTFIT; + ip->astrat = ERTS_ALC_S_FIRSTFIT; ip->init.aoff.crr_order = FF_AOFF; ip->init.aoff.blk_order = FF_BF; ip->init.util.name_prefix = "test_"; @@ -552,8 +544,8 @@ start_au_allocator(ErtsAlcType_t alctr_n, static void refuse_af_strategy(struct au_init *init) { - if (init->atype == AFIT) - init->atype = GOODFIT; + if (init->astrat == ERTS_ALC_S_AFIT) + init->astrat = ERTS_ALC_S_GOODFIT; } #ifdef HARD_DEBUG @@ -597,7 +589,7 @@ strategy_support_carrier_migration(struct au_init *auip) * Currently only aoff* and ageff* support carrier * migration, i.e, type AOFIRSTFIT. */ - return auip->atype == FIRSTFIT; + return auip->astrat == ERTS_ALC_S_FIRSTFIT; } static ERTS_INLINE void @@ -612,7 +604,7 @@ adjust_carrier_migration_support(struct au_init *auip) */ if (!strategy_support_carrier_migration(auip)) { /* Default to aoffcbf */ - auip->atype = FIRSTFIT; + auip->astrat = ERTS_ALC_S_FIRSTFIT; auip->init.aoff.crr_order = FF_AOFF; auip->init.aoff.blk_order = FF_BF; } @@ -1018,7 +1010,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, int i; int size = 1; void *as0; - enum allctr_type atype; + ErtsAlcStrat_t astrat; ErtsAllocatorFunctions_t *af = &erts_allctrs[alctr_n]; ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n]; ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n]; @@ -1077,7 +1069,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, for (i = 0; i < size; i++) { Allctr_t *as; - atype = init->atype; + astrat = init->astrat; if (!init->thr_spec) as0 = state; @@ -1094,8 +1086,8 @@ start_au_allocator(ErtsAlcType_t alctr_n, if (i != 0) init->init.util.ts = 0; else { - if (atype == AFIT) - atype = GOODFIT; + if (astrat == ERTS_ALC_S_AFIT) + astrat = ERTS_ALC_S_GOODFIT; init->init.util.ts = 1; } init->init.util.tspec = init->thr_spec + 1; @@ -1109,25 +1101,26 @@ start_au_allocator(ErtsAlcType_t alctr_n, (((char *) fix_lists) + fix_list_size)); } + init->init.util.alloc_strat = astrat; init->init.util.ix = i; - switch (atype) { - case GOODFIT: + switch (astrat) { + case ERTS_ALC_S_GOODFIT: as = erts_gfalc_start((GFAllctr_t *) as0, &init->init.gf, &init->init.util); break; - case BESTFIT: + case ERTS_ALC_S_BESTFIT: as = erts_bfalc_start((BFAllctr_t *) as0, &init->init.bf, &init->init.util); break; - case AFIT: + case ERTS_ALC_S_AFIT: as = erts_afalc_start((AFAllctr_t *) as0, &init->init.af, &init->init.util); break; - case FIRSTFIT: + case ERTS_ALC_S_FIRSTFIT: as = erts_aoffalc_start((AOFFAllctr_t *) as0, &init->init.aoff, &init->init.util); @@ -1363,51 +1356,59 @@ handle_au_arg(struct au_init *auip, else if(has_prefix("as", sub_param)) { char *alg = get_value(sub_param + 2, argv, ip); if (sys_strcmp("bf", alg) == 0) { - auip->atype = BESTFIT; + auip->astrat = ERTS_ALC_S_BESTFIT; auip->init.bf.ao = 0; } else if (sys_strcmp("aobf", alg) == 0) { - auip->atype = BESTFIT; + auip->astrat = ERTS_ALC_S_BESTFIT; auip->init.bf.ao = 1; } else if (sys_strcmp("gf", alg) == 0) { - auip->atype = GOODFIT; + auip->astrat = ERTS_ALC_S_GOODFIT; } else if (sys_strcmp("af", alg) == 0) { - auip->atype = AFIT; + auip->astrat = ERTS_ALC_S_AFIT; } else if (sys_strcmp("aoff", alg) == 0) { - auip->atype = FIRSTFIT; + auip->astrat = ERTS_ALC_S_FIRSTFIT; auip->init.aoff.crr_order = FF_AOFF; auip->init.aoff.blk_order = FF_AOFF; } else if (sys_strcmp("aoffcbf", alg) == 0) { - auip->atype = FIRSTFIT; + auip->astrat = ERTS_ALC_S_FIRSTFIT; auip->init.aoff.crr_order = FF_AOFF; auip->init.aoff.blk_order = FF_BF; } else if (sys_strcmp("aoffcaobf", alg) == 0) { - auip->atype = FIRSTFIT; + auip->astrat = ERTS_ALC_S_FIRSTFIT; auip->init.aoff.crr_order = FF_AOFF; auip->init.aoff.blk_order = FF_AOBF; } else if (sys_strcmp("ageffcaoff", alg) == 0) { - auip->atype = FIRSTFIT; + auip->astrat = ERTS_ALC_S_FIRSTFIT; auip->init.aoff.crr_order = FF_AGEFF; auip->init.aoff.blk_order = FF_AOFF; } else if (sys_strcmp("ageffcbf", alg) == 0) { - auip->atype = FIRSTFIT; + auip->astrat = ERTS_ALC_S_FIRSTFIT; auip->init.aoff.crr_order = FF_AGEFF; auip->init.aoff.blk_order = FF_BF; } else if (sys_strcmp("ageffcaobf", alg) == 0) { - auip->atype = FIRSTFIT; + auip->astrat = ERTS_ALC_S_FIRSTFIT; auip->init.aoff.crr_order = FF_AGEFF; auip->init.aoff.blk_order = FF_AOBF; } else { - bad_value(param, sub_param + 1, alg); + if (auip->init.util.alloc_no == ERTS_ALC_A_TEST + && sys_strcmp("chaosff", alg) == 0) { + auip->astrat = ERTS_ALC_S_FIRSTFIT; + auip->init.aoff.crr_order = FF_CHAOS; + auip->init.aoff.blk_order = FF_CHAOS; + } + else { + bad_value(param, sub_param + 1, alg); + } } if (!strategy_support_carrier_migration(auip)) auip->init.util.acul = 0; @@ -2030,33 +2031,55 @@ erts_realloc_n_enomem(ErtsAlcType_t n, void *ptr, Uint size) } static ERTS_INLINE UWord -alcu_size(ErtsAlcType_t ai, ErtsAlcUFixInfo_t *fi, int fisz) +alcu_size(ErtsAlcType_t alloc_no, ErtsAlcUFixInfo_t *fi, int fisz) { - UWord res = 0; + UWord res; + int ai; + + if (!erts_allctrs_info[alloc_no].thr_spec) { + AllctrSize_t size; + Allctr_t *allctr; - ASSERT(erts_allctrs_info[ai].enabled); - ASSERT(erts_allctrs_info[ai].alloc_util); + allctr = erts_allctrs_info[alloc_no].extra; + erts_alcu_current_size(allctr, &size, fi, fisz); - if (!erts_allctrs_info[ai].thr_spec) { - Allctr_t *allctr = erts_allctrs_info[ai].extra; - AllctrSize_t asize; - erts_alcu_current_size(allctr, &asize, fi, fisz); - res += asize.blocks; + return size.blocks; } - else { - ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ai]; - int i; - ASSERT(tspec->enabled); + res = 0; - for (i = tspec->size - 1; i >= 0; i--) { - Allctr_t *allctr = tspec->allctr[i]; - AllctrSize_t asize; - if (allctr) { - erts_alcu_current_size(allctr, &asize, fi, fisz); - res += asize.blocks; - } - } + /* Thread-specific allocators can migrate carriers across types, so we have + * to visit every allocator type to gather information on blocks that were + * allocated by us. */ + for (ai = ERTS_ALC_A_MIN; ai < ERTS_ALC_A_MAX; ai++) { + ErtsAllocatorThrSpec_t *tspec; + Allctr_t *allctr; + int i; + + if (!erts_allctrs_info[ai].thr_spec) { + continue; + } + + tspec = &erts_allctr_thr_spec[ai]; + ASSERT(tspec->enabled); + + for (i = tspec->size - 1; i >= 0; i--) { + allctr = tspec->allctr[i]; + + if (allctr) { + AllctrSize_t size; + + if (ai == alloc_no) { + erts_alcu_current_size(allctr, &size, fi, fisz); + } else { + erts_alcu_foreign_size(allctr, alloc_no, &size); + } + + ASSERT(((SWord)size.blocks) >= 0); + + res += size.blocks; + } + } } return res; @@ -2400,6 +2423,7 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) } if (want_tot_or_sys) { + ASSERT(size.total >= size.processes); size.system = size.total - size.processes; } @@ -3459,29 +3483,29 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) switch (op) { case 0xf00: if (((Allctr_t *) a1)->thread_safe) - return (UWord) erts_alcu_alloc_ts(ERTS_ALC_T_UNDEF, + return (UWord) erts_alcu_alloc_ts(ERTS_ALC_T_TEST, (void *) a1, (Uint) a2); else - return (UWord) erts_alcu_alloc(ERTS_ALC_T_UNDEF, + return (UWord) erts_alcu_alloc(ERTS_ALC_T_TEST, (void *) a1, (Uint) a2); case 0xf01: if (((Allctr_t *) a1)->thread_safe) - return (UWord) erts_alcu_realloc_ts(ERTS_ALC_T_UNDEF, + return (UWord) erts_alcu_realloc_ts(ERTS_ALC_T_TEST, (void *) a1, (void *) a2, (Uint) a3); else - return (UWord) erts_alcu_realloc(ERTS_ALC_T_UNDEF, + return (UWord) erts_alcu_realloc(ERTS_ALC_T_TEST, (void *) a1, (void *) a2, (Uint) a3); case 0xf02: if (((Allctr_t *) a1)->thread_safe) - erts_alcu_free_ts(ERTS_ALC_T_UNDEF, (void *) a1, (void *) a2); + erts_alcu_free_ts(ERTS_ALC_T_TEST, (void *) a1, (void *) a2); else - erts_alcu_free(ERTS_ALC_T_UNDEF, (void *) a1, (void *) a2); + erts_alcu_free(ERTS_ALC_T_TEST, (void *) a1, (void *) a2); return 0; case 0xf03: { Allctr_t *allctr; @@ -3489,8 +3513,10 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) SET_DEFAULT_ALLOC_OPTS(&init); init.enable = 1; - init.atype = GOODFIT; + init.astrat = ERTS_ALC_S_GOODFIT; init.init.util.name_prefix = (char *) a1; + init.init.util.alloc_no = ERTS_ALC_A_TEST; + init.init.util.alloc_strat = init.astrat; init.init.util.ts = 1; if ((char **) a3) { char **argv = (char **) a3; @@ -3504,31 +3530,31 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) } } - switch (init.atype) { - case GOODFIT: + switch (init.astrat) { + case ERTS_ALC_S_GOODFIT: allctr = erts_gfalc_start((GFAllctr_t *) - erts_alloc(ERTS_ALC_T_UNDEF, + erts_alloc(ERTS_ALC_T_TEST, sizeof(GFAllctr_t)), &init.init.gf, &init.init.util); break; - case BESTFIT: + case ERTS_ALC_S_BESTFIT: allctr = erts_bfalc_start((BFAllctr_t *) - erts_alloc(ERTS_ALC_T_UNDEF, + erts_alloc(ERTS_ALC_T_TEST, sizeof(BFAllctr_t)), &init.init.bf, &init.init.util); break; - case AFIT: + case ERTS_ALC_S_AFIT: allctr = erts_afalc_start((AFAllctr_t *) - erts_alloc(ERTS_ALC_T_UNDEF, + erts_alloc(ERTS_ALC_T_TEST, sizeof(AFAllctr_t)), &init.init.af, &init.init.util); break; - case FIRSTFIT: + case ERTS_ALC_S_FIRSTFIT: allctr = erts_aoffalc_start((AOFFAllctr_t *) - erts_alloc(ERTS_ALC_T_UNDEF, + erts_alloc(ERTS_ALC_T_TEST, sizeof(AOFFAllctr_t)), &init.init.aoff, &init.init.util); @@ -3544,7 +3570,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) } case 0xf04: erts_alcu_stop((Allctr_t *) a1); - erts_free(ERTS_ALC_T_UNDEF, (void *) a1); + erts_free(ERTS_ALC_T_TEST, (void *) a1); break; case 0xf05: return (UWord) 1; case 0xf06: return (UWord) ((Allctr_t *) a1)->thread_safe; @@ -3554,7 +3580,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) case 0xf07: return (UWord) ((Allctr_t *) a1)->thread_safe; #endif case 0xf08: { - ethr_mutex *mtx = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_mutex)); + ethr_mutex *mtx = erts_alloc(ERTS_ALC_T_TEST, sizeof(ethr_mutex)); if (ethr_mutex_init(mtx) != 0) ERTS_ALC_TEST_ABORT; return (UWord) mtx; @@ -3563,7 +3589,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) ethr_mutex *mtx = (ethr_mutex *) a1; if (ethr_mutex_destroy(mtx) != 0) ERTS_ALC_TEST_ABORT; - erts_free(ERTS_ALC_T_UNDEF, (void *) mtx); + erts_free(ERTS_ALC_T_TEST, (void *) mtx); break; } case 0xf0a: @@ -3573,7 +3599,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) ethr_mutex_unlock((ethr_mutex *) a1); break; case 0xf0c: { - ethr_cond *cnd = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_cond)); + ethr_cond *cnd = erts_alloc(ERTS_ALC_T_TEST, sizeof(ethr_cond)); if (ethr_cond_init(cnd) != 0) ERTS_ALC_TEST_ABORT; return (UWord) cnd; @@ -3582,7 +3608,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) ethr_cond *cnd = (ethr_cond *) a1; if (ethr_cond_destroy(cnd) != 0) ERTS_ALC_TEST_ABORT; - erts_free(ERTS_ALC_T_UNDEF, (void *) cnd); + erts_free(ERTS_ALC_T_TEST, (void *) cnd); break; } case 0xf0e: @@ -3596,7 +3622,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) break; } case 0xf10: { - ethr_tid *tid = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_tid)); + ethr_tid *tid = erts_alloc(ERTS_ALC_T_TEST, sizeof(ethr_tid)); if (ethr_thr_create(tid, (void * (*)(void *)) a1, (void *) a2, @@ -3608,7 +3634,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) ethr_tid *tid = (ethr_tid *) a1; if (ethr_thr_join(*tid, NULL) != 0) ERTS_ALC_TEST_ABORT; - erts_free(ERTS_ALC_T_UNDEF, (void *) tid); + erts_free(ERTS_ALC_T_TEST, (void *) tid); break; } case 0xf12: @@ -3960,9 +3986,10 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) static ErtsAllocatorFunctions_t real_allctrs[ERTS_ALC_A_MAX+1]; static void * -debug_alloc(ErtsAlcType_t n, void *extra, Uint size) +debug_alloc(ErtsAlcType_t type, void *extra, Uint size) { ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; + ErtsAlcType_t n; Uint dsize; void *res; @@ -3970,9 +3997,11 @@ debug_alloc(ErtsAlcType_t n, void *extra, Uint size) erts_hdbg_chk_blks(); #endif + n = ERTS_ALC_T2N(type); + ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); dsize = size + FENCE_SZ; - res = (*real_af->alloc)(n, real_af->extra, dsize); + res = (*real_af->alloc)(type, real_af->extra, dsize); res = set_memory_fence(res, size, n); @@ -3986,14 +4015,17 @@ debug_alloc(ErtsAlcType_t n, void *extra, Uint size) static void * -debug_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) +debug_realloc(ErtsAlcType_t type, void *extra, void *ptr, Uint size) { ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; + ErtsAlcType_t n; Uint dsize; Uint old_size; void *dptr; void *res; + n = ERTS_ALC_T2N(type); + ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); dsize = size + FENCE_SZ; @@ -4008,7 +4040,7 @@ debug_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) 0xf, sizeof(Uint) + old_size - size); - res = (*real_af->realloc)(n, real_af->extra, dptr, dsize); + res = (*real_af->realloc)(type, real_af->extra, dptr, dsize); res = set_memory_fence(res, size, n); @@ -4021,12 +4053,16 @@ debug_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) } static void -debug_free(ErtsAlcType_t n, void *extra, void *ptr) +debug_free(ErtsAlcType_t type, void *extra, void *ptr) { ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; + ErtsAlcType_t n; void *dptr; Uint size; - int free_pattern = n; + int free_pattern; + + n = ERTS_ALC_T2N(type); + free_pattern = n; ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); @@ -4041,7 +4077,7 @@ debug_free(ErtsAlcType_t n, void *extra, void *ptr) #endif sys_memset((void *) dptr, free_pattern, size + FENCE_SZ); - (*real_af->free)(n, real_af->extra, dptr); + (*real_af->free)(type, real_af->extra, dptr); #ifdef PRINT_OPS fprintf(stderr, "free(%s, 0x%lx)\r\n", ERTS_ALC_N2TD(n), (Uint) ptr); diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index fcb58ff58a..c13cf3f5b0 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -26,10 +26,23 @@ #define ERL_THR_PROGRESS_TSD_TYPE_ONLY #include "erl_thr_progress.h" #undef ERL_THR_PROGRESS_TSD_TYPE_ONLY -#include "erl_alloc_util.h" #include "erl_threads.h" #include "erl_mmap.h" +typedef enum { + ERTS_ALC_S_INVALID = 0, + + ERTS_ALC_S_GOODFIT, + ERTS_ALC_S_BESTFIT, + ERTS_ALC_S_AFIT, + ERTS_ALC_S_FIRSTFIT, + + ERTS_ALC_S_MIN = ERTS_ALC_S_GOODFIT, + ERTS_ALC_S_MAX = ERTS_ALC_S_FIRSTFIT +} ErtsAlcStrat_t; + +#include "erl_alloc_util.h" + #ifdef DEBUG # undef ERTS_ALC_WANT_INLINE # define ERTS_ALC_WANT_INLINE 0 @@ -52,6 +65,14 @@ #define ERTS_ALC_NO_FIXED_SIZES \ (ERTS_ALC_N_MAX_A_FIXED_SIZE - ERTS_ALC_N_MIN_A_FIXED_SIZE + 1) +#define ERTS_ALC_IS_FIX_TYPE(T) \ + (ERTS_ALC_T2N(T) >= ERTS_ALC_N_MIN_A_FIXED_SIZE && \ + ERTS_ALC_T2N(T) <= ERTS_ALC_N_MAX_A_FIXED_SIZE) + +#define ERTS_ALC_FIX_TYPE_IX(T) \ + (ASSERT(ERTS_ALC_IS_FIX_TYPE(T)), \ + ERTS_ALC_T2N((T)) - ERTS_ALC_N_MIN_A_FIXED_SIZE) + void erts_sys_alloc_init(void); void *erts_sys_alloc(ErtsAlcType_t, void *, Uint); void *erts_sys_realloc(ErtsAlcType_t, void *, void *, Uint); @@ -228,7 +249,7 @@ void *erts_alloc(ErtsAlcType_t type, Uint size) void *res; ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); res = (*erts_allctrs[ERTS_ALC_T2A(type)].alloc)( - ERTS_ALC_T2N(type), + type, erts_allctrs[ERTS_ALC_T2A(type)].extra, size); if (!res) @@ -243,7 +264,7 @@ void *erts_realloc(ErtsAlcType_t type, void *ptr, Uint size) void *res; ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); res = (*erts_allctrs[ERTS_ALC_T2A(type)].realloc)( - ERTS_ALC_T2N(type), + type, erts_allctrs[ERTS_ALC_T2A(type)].extra, ptr, size); @@ -258,7 +279,7 @@ void erts_free(ErtsAlcType_t type, void *ptr) { ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); (*erts_allctrs[ERTS_ALC_T2A(type)].free)( - ERTS_ALC_T2N(type), + type, erts_allctrs[ERTS_ALC_T2A(type)].extra, ptr); ERTS_MSACC_POP_STATE_X(); @@ -271,7 +292,7 @@ void *erts_alloc_fnf(ErtsAlcType_t type, Uint size) void *res; ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); res = (*erts_allctrs[ERTS_ALC_T2A(type)].alloc)( - ERTS_ALC_T2N(type), + type, erts_allctrs[ERTS_ALC_T2A(type)].extra, size); ERTS_MSACC_POP_STATE_X(); @@ -285,7 +306,7 @@ void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size) void *res; ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); res = (*erts_allctrs[ERTS_ALC_T2A(type)].realloc)( - ERTS_ALC_T2N(type), + type, erts_allctrs[ERTS_ALC_T2A(type)].extra, ptr, size); diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index a5740a08cf..1c95385649 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -96,7 +96,6 @@ static int initialized = 0; #define MBC_REALLOC_ALWAYS_MOVES #endif - /* alloc_util global parameters */ static Uint sys_alloc_carrier_size; #if HAVE_ERTS_MSEG @@ -113,32 +112,38 @@ static int allow_sys_alloc_carriers; #define DEC_CC(CC) ((CC)--) -/* Multi block carrier (MBC) memory layout in R16: +/* Multi block carrier (MBC) memory layout in OTP 22: Empty MBC: -[Carrier_t|pad|Block_t L0T|fhdr| free... ] +[Carrier_t|pad|Block_t L0T0|fhdr| free... ] MBC after allocating first block: -[Carrier_t|pad|Block_t 000| udata |pad|Block_t L0T|fhdr| free... ] +[Carrier_t|pad|Block_t 0000| udata |pad|Block_t L0T0|fhdr| free... ] MBC after allocating second block: -[Carrier_t|pad|Block_t 000| udata |pad|Block_t 000| udata |pad|Block_t L0T|fhdr| free... ] +[Carrier_t|pad|Block_t 0000| udata |pad|Block_t 0000| udata |pad|Block_t L0T0|fhdr| free... ] MBC after deallocating first block: -[Carrier_t|pad|Block_t 00T|fhdr| free |FreeBlkFtr_t|Block_t 0P0| udata |pad|Block_t L0T|fhdr| free... ] +[Carrier_t|pad|Block_t 00T0|fhdr| free |FreeBlkFtr_t|Block_t 0P00| udata |pad|Block_t L0T0|fhdr| free... ] +MBC after allocating first block, with allocation tagging enabled: +[Carrier_t|pad|Block_t 000A| udata |atag|pad|Block_t L0T0|fhdr| free... ] udata = Allocated user data + atag = A tag with basic metadata about this allocation pad = Padding to ensure correct alignment for user data fhdr = Allocator specific header to keep track of free block free = Unused free memory T = This block is free (THIS_FREE_BLK_HDR_FLG) P = Previous block is free (PREV_FREE_BLK_HDR_FLG) L = Last block in carrier (LAST_BLK_HDR_FLG) + A = Block has an allocation tag footer, only valid for allocated blocks + (ATAG_BLK_HDR_FLG) */ /* Single block carrier (SBC): -[Carrier_t|pad|Block_t 111| udata... ] +[Carrier_t|pad|Block_t 1110| udata... ] +[Carrier_t|pad|Block_t 111A| udata | atag] */ /* Allocation tags ... @@ -154,20 +159,20 @@ MBC after deallocating first block: typedef UWord alcu_atag_t; -#define MAKE_ATAG(IdAtom, Type) \ - (ASSERT((Type) >= ERTS_ALC_N_MIN && (Type) <= ERTS_ALC_N_MAX), \ +#define MAKE_ATAG(IdAtom, TypeNum) \ + (ASSERT((TypeNum) >= ERTS_ALC_N_MIN && (TypeNum) <= ERTS_ALC_N_MAX), \ ASSERT(atom_val(IdAtom) <= MAX_ATAG_ATOM_ID), \ - (atom_val(IdAtom) << ERTS_ALC_N_BITS) | (Type)) + (atom_val(IdAtom) << ERTS_ALC_N_BITS) | (TypeNum)) #define ATAG_ID(AT) (make_atom((AT) >> ERTS_ALC_N_BITS)) #define ATAG_TYPE(AT) ((AT) & ERTS_ALC_N_MASK) #define MAX_ATAG_ATOM_ID (ERTS_UWORD_MAX >> ERTS_ALC_N_BITS) -#define DBG_IS_VALID_ATAG(Allocator, AT) \ +#define DBG_IS_VALID_ATAG(AT) \ (ATAG_TYPE(AT) >= ERTS_ALC_N_MIN && \ ATAG_TYPE(AT) <= ERTS_ALC_N_MAX && \ - (Allocator)->alloc_no == ERTS_ALC_T2A(ERTS_ALC_N2T(ATAG_TYPE(AT)))) + ATAG_ID(AT) <= MAX_ATAG_ATOM_ID) /* Blocks ... */ @@ -182,10 +187,15 @@ typedef UWord alcu_atag_t; #endif #define FBLK_FTR_SZ (sizeof(FreeBlkFtr_t)) +#define BLK_HAS_ATAG(B) \ + (!!((B)->bhdr & ATAG_BLK_HDR_FLG)) + #define GET_BLK_ATAG(B) \ - (((alcu_atag_t *) (((char *) (B)) + (BLK_SZ(B))))[-1]) + (ASSERT(BLK_HAS_ATAG(B)), \ + ((alcu_atag_t *) (((char *) (B)) + (BLK_SZ(B))))[-1]) #define SET_BLK_ATAG(B, T) \ - (((alcu_atag_t *) (((char *) (B)) + (BLK_SZ(B))))[-1] = (T)) + ((B)->bhdr |= ATAG_BLK_HDR_FLG, \ + ((alcu_atag_t *) (((char *) (B)) + (BLK_SZ(B))))[-1] = (T)) #define BLK_ATAG_SZ(AP) ((AP)->atags ? sizeof(alcu_atag_t) : 0) @@ -203,13 +213,13 @@ typedef UWord alcu_atag_t; (((FreeBlkFtr_t *) (((char *) (B)) + (SZ)))[-1] = (SZ)) #define SET_MBC_ABLK_SZ(B, SZ) \ - (ASSERT(((SZ) & FLG_MASK) == 0), \ + (ASSERT(((SZ) & BLK_FLG_MASK) == 0), \ (B)->bhdr = (((B)->bhdr) & ~MBC_ABLK_SZ_MASK) | (SZ)) #define SET_MBC_FBLK_SZ(B, SZ) \ - (ASSERT(((SZ) & FLG_MASK) == 0), \ + (ASSERT(((SZ) & BLK_FLG_MASK) == 0), \ (B)->bhdr = (((B)->bhdr) & ~MBC_FBLK_SZ_MASK) | (SZ)) #define SET_SBC_BLK_SZ(B, SZ) \ - (ASSERT(((SZ) & FLG_MASK) == 0), \ + (ASSERT(((SZ) & BLK_FLG_MASK) == 0), \ (B)->bhdr = (((B)->bhdr) & ~SBC_BLK_SZ_MASK) | (SZ)) #define SET_PREV_BLK_FREE(AP,B) \ (ASSERT(!IS_MBC_FIRST_BLK(AP,B)), \ @@ -235,12 +245,12 @@ typedef UWord alcu_atag_t; # define SET_MBC_ABLK_HDR(B, Sz, F, C) \ (ASSERT(((Sz) & ~MBC_ABLK_SZ_MASK) == 0), \ - ASSERT(!((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG))), \ + ASSERT(!((UWord)(F) & (~BLK_FLG_MASK|THIS_FREE_BLK_HDR_FLG))), \ (B)->bhdr = ((Sz) | (F) | (BLK_CARRIER_OFFSET(B,C) << MBC_ABLK_OFFSET_SHIFT))) # define SET_MBC_FBLK_HDR(B, Sz, F, C) \ (ASSERT(((Sz) & ~MBC_FBLK_SZ_MASK) == 0), \ - ASSERT(((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ + ASSERT(((UWord)(F) & (~BLK_FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ (B)->bhdr = ((Sz) | (F)), \ (B)->u.carrier = (C)) @@ -257,8 +267,8 @@ typedef UWord alcu_atag_t; # define SET_BLK_FREE(B) \ (ASSERT(!IS_PREV_BLK_FREE(B)), \ (B)->u.carrier = ABLK_TO_MBC(B), \ - (B)->bhdr |= THIS_FREE_BLK_HDR_FLG, \ - (B)->bhdr &= (MBC_ABLK_SZ_MASK|FLG_MASK)) + (B)->bhdr &= (MBC_ABLK_SZ_MASK|LAST_BLK_HDR_FLG), \ + (B)->bhdr |= THIS_FREE_BLK_HDR_FLG) # define SET_BLK_ALLOCED(B) \ (ASSERT(((B)->bhdr & (MBC_ABLK_OFFSET_MASK|THIS_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ @@ -270,15 +280,16 @@ typedef UWord alcu_atag_t; # define MBC_SZ_MAX_LIMIT ((UWord)~0) # define SET_MBC_ABLK_HDR(B, Sz, F, C) \ - (ASSERT(((Sz) & FLG_MASK) == 0), \ - ASSERT(!((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG))), \ - ASSERT((UWord)(F) < SBC_BLK_HDR_FLG), \ + (ASSERT(((Sz) & BLK_FLG_MASK) == 0), \ + ASSERT(((F) & ~BLK_FLG_MASK) == 0), \ + ASSERT(!((UWord)(F) & (~BLK_FLG_MASK|THIS_FREE_BLK_HDR_FLG))), \ (B)->bhdr = ((Sz) | (F)), \ (B)->carrier = (C)) # define SET_MBC_FBLK_HDR(B, Sz, F, C) \ - (ASSERT(((Sz) & FLG_MASK) == 0), \ - ASSERT(((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ + (ASSERT(((Sz) & BLK_FLG_MASK) == 0), \ + ASSERT(((F) & ~BLK_FLG_MASK) == 0), \ + ASSERT(((UWord)(F) & (~BLK_FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ (B)->bhdr = ((Sz) | (F)), \ (B)->carrier = (C)) @@ -297,7 +308,7 @@ typedef UWord alcu_atag_t; #endif /* !MBC_ABLK_OFFSET_BITS */ #define SET_SBC_BLK_HDR(B, Sz) \ - (ASSERT(((Sz) & FLG_MASK) == 0), (B)->bhdr = ((Sz) | (SBC_BLK_HDR_FLG))) + (ASSERT(((Sz) & BLK_FLG_MASK) == 0), (B)->bhdr = ((Sz) | (SBC_BLK_HDR_FLG))) #define BLK_UMEM_SZ(B) \ @@ -320,7 +331,7 @@ typedef UWord alcu_atag_t; #define GET_PREV_FREE_BLK_HDR_FLG(B) \ ((B)->bhdr & PREV_FREE_BLK_HDR_FLG) #define GET_BLK_HDR_FLGS(B) \ - ((B)->bhdr & FLG_MASK) + ((B)->bhdr & BLK_FLG_MASK) #define NXT_BLK(B) \ (ASSERT(IS_MBC_BLK(B)), \ @@ -419,7 +430,7 @@ do { \ #define SCH_SBC SBC_CARRIER_HDR_FLAG #define SET_CARRIER_HDR(C, Sz, F, AP) \ - (ASSERT(((Sz) & FLG_MASK) == 0), (C)->chdr = ((Sz) | (F)), \ + (ASSERT(((Sz) & CRR_FLG_MASK) == 0), (C)->chdr = ((Sz) | (F)), \ erts_atomic_init_nob(&(C)->allctr, (erts_aint_t) (AP))) #define BLK_TO_SBC(B) \ @@ -444,8 +455,8 @@ do { \ (!IS_SB_CARRIER((C))) #define SET_CARRIER_SZ(C, SZ) \ - (ASSERT(((SZ) & FLG_MASK) == 0), \ - ((C)->chdr = ((C)->chdr & FLG_MASK) | (SZ))) + (ASSERT(((SZ) & CRR_FLG_MASK) == 0), \ + ((C)->chdr = ((C)->chdr & CRR_FLG_MASK) | (SZ))) #define CFLG_SBC (1 << 0) #define CFLG_MBC (1 << 1) @@ -575,10 +586,11 @@ do { \ STAT_MSEG_MBC_ALLOC((AP), csz__); \ else \ STAT_SYS_ALLOC_MBC_ALLOC((AP), csz__); \ - (AP)->mbcs.blocks.curr.no += (CRR)->cpool.blocks; \ + (AP)->mbcs.blocks.curr.no += (CRR)->cpool.blocks[(AP)->alloc_no]; \ if ((AP)->mbcs.blocks.max.no < (AP)->mbcs.blocks.curr.no) \ (AP)->mbcs.blocks.max.no = (AP)->mbcs.blocks.curr.no; \ - (AP)->mbcs.blocks.curr.size += (CRR)->cpool.blocks_size; \ + (AP)->mbcs.blocks.curr.size += \ + (CRR)->cpool.blocks_size[(AP)->alloc_no]; \ if ((AP)->mbcs.blocks.max.size < (AP)->mbcs.blocks.curr.size) \ (AP)->mbcs.blocks.max.size = (AP)->mbcs.blocks.curr.size; \ } while (0) @@ -601,25 +613,27 @@ do { \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) -#define STAT_MBC_ABANDON(AP, CRR) \ -do { \ - UWord csz__ = CARRIER_SZ((CRR)); \ - if (IS_MSEG_CARRIER((CRR))) \ - STAT_MSEG_MBC_FREE((AP), csz__); \ - else \ - STAT_SYS_ALLOC_MBC_FREE((AP), csz__); \ - ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.no \ - >= (CRR)->cpool.blocks); \ - (AP)->mbcs.blocks.curr.no -= (CRR)->cpool.blocks; \ - ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.size \ - >= (CRR)->cpool.blocks_size); \ - (AP)->mbcs.blocks.curr.size -= (CRR)->cpool.blocks_size; \ +#define STAT_MBC_ABANDON(AP, CRR) \ +do { \ + UWord csz__ = CARRIER_SZ((CRR)); \ + if (IS_MSEG_CARRIER((CRR))) { \ + STAT_MSEG_MBC_FREE((AP), csz__); \ + } else { \ + STAT_SYS_ALLOC_MBC_FREE((AP), csz__); \ + } \ + ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.no \ + >= (CRR)->cpool.blocks[(AP)->alloc_no]); \ + (AP)->mbcs.blocks.curr.no -= (CRR)->cpool.blocks[(AP)->alloc_no]; \ + ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.size \ + >= (CRR)->cpool.blocks_size[(AP)->alloc_no]); \ + (AP)->mbcs.blocks.curr.size -= (CRR)->cpool.blocks_size[(AP)->alloc_no]; \ } while (0) -#define STAT_MBC_BLK_ALLOC_CRR(CRR, BSZ) \ +#define STAT_MBC_BLK_ALLOC_CRR(AP, CRR, BSZ) \ do { \ - (CRR)->cpool.blocks++; \ - (CRR)->cpool.blocks_size += (BSZ); \ + (CRR)->cpool.blocks[(AP)->alloc_no]++; \ + (CRR)->cpool.blocks_size[(AP)->alloc_no] += (BSZ); \ + (CRR)->cpool.total_blocks_size += (BSZ); \ } while (0) #define STAT_MBC_BLK_ALLOC(AP, CRR, BSZ, FLGS) \ @@ -631,50 +645,67 @@ do { \ cstats__->blocks.curr.size += (BSZ); \ if (cstats__->blocks.max.size < cstats__->blocks.curr.size) \ cstats__->blocks.max.size = cstats__->blocks.curr.size; \ - STAT_MBC_BLK_ALLOC_CRR((CRR), (BSZ)); \ + STAT_MBC_BLK_ALLOC_CRR((AP), (CRR), (BSZ)); \ } while (0) static ERTS_INLINE int stat_cpool_mbc_blk_free(Allctr_t *allctr, + ErtsAlcType_t type, Carrier_t *crr, Carrier_t **busy_pcrr_pp, UWord blksz) { + Allctr_t *orig_allctr; + int alloc_no; - ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks > 0); - crr->cpool.blocks--; - ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks_size >= blksz); - crr->cpool.blocks_size -= blksz; + alloc_no = ERTS_ALC_T2A(type); - if (!busy_pcrr_pp || !*busy_pcrr_pp) - return 0; + ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks[alloc_no] > 0); + crr->cpool.blocks[alloc_no]--; + ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks_size[alloc_no] >= blksz); + crr->cpool.blocks_size[alloc_no] -= blksz; + ERTS_ALC_CPOOL_ASSERT(crr->cpool.total_blocks_size >= blksz); + crr->cpool.total_blocks_size -= blksz; + + if (allctr->alloc_no == alloc_no && (!busy_pcrr_pp || !*busy_pcrr_pp)) { + /* This is a local block, so we should not update the pool + * statistics. */ + return 0; + } - ERTS_ALC_CPOOL_ASSERT(crr == *busy_pcrr_pp); + /* This is either a foreign block that's been fetched from the pool, or any + * block that's in the pool. The carrier's owner keeps the statistics for + * both pooled and foreign blocks. */ + + orig_allctr = crr->cpool.orig_allctr; + + ERTS_ALC_CPOOL_ASSERT(alloc_no != allctr->alloc_no || + (crr == *busy_pcrr_pp && allctr == orig_allctr)); #ifdef ERTS_ALC_CPOOL_DEBUG ERTS_ALC_CPOOL_ASSERT( - erts_atomic_dec_read_nob(&allctr->cpool.stat.no_blocks) >= 0); + erts_atomic_dec_read_nob(&orig_allctr->cpool.stat.no_blocks[alloc_no]) >= 0); ERTS_ALC_CPOOL_ASSERT( - erts_atomic_add_read_nob(&allctr->cpool.stat.blocks_size, + erts_atomic_add_read_nob(&orig_allctr->cpool.stat.blocks_size[alloc_no], -((erts_aint_t) blksz)) >= 0); #else - erts_atomic_dec_nob(&allctr->cpool.stat.no_blocks); - erts_atomic_add_nob(&allctr->cpool.stat.blocks_size, + erts_atomic_dec_nob(&orig_allctr->cpool.stat.no_blocks[alloc_no]); + erts_atomic_add_nob(&orig_allctr->cpool.stat.blocks_size[alloc_no], -((erts_aint_t) blksz)); #endif return 1; } -#define STAT_MBC_BLK_FREE(AP, CRR, BPCRRPP, BSZ, FLGS) \ -do { \ - if (!stat_cpool_mbc_blk_free((AP), (CRR), (BPCRRPP), (BSZ))) { \ - CarriersStats_t *cstats__ = &(AP)->mbcs; \ - ASSERT(cstats__->blocks.curr.no > 0); \ - cstats__->blocks.curr.no--; \ - ASSERT(cstats__->blocks.curr.size >= (BSZ)); \ - cstats__->blocks.curr.size -= (BSZ); \ - } \ +#define STAT_MBC_BLK_FREE(AP, TYPE, CRR, BPCRRPP, BSZ, FLGS) \ +do { \ + if (!stat_cpool_mbc_blk_free((AP), (TYPE), (CRR), (BPCRRPP), (BSZ))) { \ + CarriersStats_t *cstats__ = &(AP)->mbcs; \ + ASSERT(cstats__->blocks.curr.no > 0); \ + cstats__->blocks.curr.no--; \ + ASSERT(cstats__->blocks.curr.size >= (BSZ)); \ + cstats__->blocks.curr.size -= (BSZ); \ + } \ } while (0) /* Debug stuff... */ @@ -721,8 +752,8 @@ static void make_name_atoms(Allctr_t *allctr); static Block_t *create_carrier(Allctr_t *, Uint, UWord); static void destroy_carrier(Allctr_t *, Block_t *, Carrier_t **); -static void mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp); -static void dealloc_block(Allctr_t *, void *, ErtsAlcFixList_t *, int); +static void mbc_free(Allctr_t *allctr, ErtsAlcType_t type, void *p, Carrier_t **busy_pcrr_pp); +static void dealloc_block(Allctr_t *, ErtsAlcType_t, Uint32, void *, ErtsAlcFixList_t *); static alcu_atag_t determine_alloc_tag(Allctr_t *allocator, ErtsAlcType_t type) { @@ -764,14 +795,14 @@ static alcu_atag_t determine_alloc_tag(Allctr_t *allocator, ErtsAlcType_t type) } } - return MAKE_ATAG(id, type); + return MAKE_ATAG(id, ERTS_ALC_T2N(type)); } static void set_alloc_tag(Allctr_t *allocator, void *p, alcu_atag_t tag) { Block_t *block; - ASSERT(DBG_IS_VALID_ATAG(allocator, tag)); + ASSERT(DBG_IS_VALID_ATAG(tag)); ASSERT(allocator->atags && p); (void)allocator; @@ -1308,28 +1339,9 @@ chk_fix_list(Allctr_t *allctr, ErtsAlcFixList_t *fix, int ix, int before) #define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) #endif +static ERTS_INLINE Allctr_t *get_pref_allctr(void *extra); static void *mbc_alloc(Allctr_t *allctr, Uint size); -typedef struct { - ErtsAllctrDDBlock_t ddblock__; /* must be first */ - ErtsAlcType_t fix_type; -} ErtsAllctrFixDDBlock_t; - -#define ERTS_ALC_FIX_NO_UNUSE (((ErtsAlcType_t) 1) << ERTS_ALC_N_BITS) - -static ERTS_INLINE void -dealloc_fix_block(Allctr_t *allctr, - ErtsAlcType_t type, - void *ptr, - ErtsAlcFixList_t *fix, - int dec_cc_on_redirect) -{ - /* May be redirected... */ - ASSERT((type & ERTS_ALC_FIX_NO_UNUSE) == 0); - ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type | ERTS_ALC_FIX_NO_UNUSE; - dealloc_block(allctr, ptr, fix, dec_cc_on_redirect); -} - static ERTS_INLINE void sched_fix_shrink(Allctr_t *allctr, int on) { @@ -1371,7 +1383,7 @@ fix_cpool_check_shrink(Allctr_t *allctr, if (fix->u.cpool.min_list_size > fix->list_size) fix->u.cpool.min_list_size = fix->list_size; - dealloc_fix_block(allctr, type, p, fix, 0); + dealloc_block(allctr, type, DEALLOC_FLG_FIX_SHRINK, p, fix); } } } @@ -1382,10 +1394,8 @@ fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) void *res; ErtsAlcFixList_t *fix; - ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type - && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); - - fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + fix = &allctr->fix[ERTS_ALC_FIX_TYPE_IX(type)]; + ASSERT(type == fix->type); ASSERT(size == fix->type_size); res = fix->list; @@ -1415,21 +1425,39 @@ fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) static ERTS_INLINE void fix_cpool_free(Allctr_t *allctr, ErtsAlcType_t type, + Uint32 flags, void *p, - Carrier_t **busy_pcrr_pp, - int unuse) + Carrier_t **busy_pcrr_pp) { ErtsAlcFixList_t *fix; + Allctr_t *fix_allctr; - ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type - && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + /* If this isn't a fix allocator we need to update the fix list of our + * neighboring fix_alloc to keep the statistics consistent. */ + if (!allctr->fix) { + ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ERTS_ALC_A_FIXED_SIZE]; + fix_allctr = get_pref_allctr(tspec); + ASSERT(!fix_allctr->thread_safe); + ASSERT(allctr != fix_allctr); + } + else { + fix_allctr = allctr; + } - fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(fix_allctr)); + ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr)); - if (unuse) - fix->u.cpool.used--; + fix = &fix_allctr->fix[ERTS_ALC_FIX_TYPE_IX(type)]; + ASSERT(type == fix->type); - if ((!busy_pcrr_pp || !*busy_pcrr_pp) + if (!(flags & DEALLOC_FLG_FIX_SHRINK)) { + fix->u.cpool.used--; + } + + /* We don't want foreign blocks to be long-lived, so we skip recycling if + * allctr != fix_allctr. */ + if (allctr == fix_allctr + && (!busy_pcrr_pp || !*busy_pcrr_pp) && !fix->u.cpool.shrink_list && fix->list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { *((void **) p) = fix->list; @@ -1442,7 +1470,7 @@ fix_cpool_free(Allctr_t *allctr, if (IS_SBC_BLK(blk)) destroy_carrier(allctr, blk, NULL); else - mbc_free(allctr, p, busy_pcrr_pp); + mbc_free(allctr, type, p, busy_pcrr_pp); fix->u.cpool.allocated--; fix_cpool_check_shrink(allctr, type, fix, busy_pcrr_pp); } @@ -1469,7 +1497,7 @@ fix_cpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) fix->u.cpool.shrink_list = fix->u.cpool.min_list_size; fix->u.cpool.min_list_size = fix->list_size; } - type = (ErtsAlcType_t) (ix + ERTS_ALC_N_MIN_A_FIXED_SIZE); + type = ERTS_ALC_N2T((ErtsAlcType_t) (ix + ERTS_ALC_N_MIN_A_FIXED_SIZE)); for (o = 0; o < ERTS_ALC_FIX_MAX_SHRINK_OPS || flush; o++) { void *ptr; @@ -1483,7 +1511,7 @@ fix_cpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) fix->list = *((void **) ptr); fix->list_size--; fix->u.cpool.shrink_list--; - dealloc_fix_block(allctr, type, ptr, fix, 0); + dealloc_block(allctr, type, DEALLOC_FLG_FIX_SHRINK, ptr, fix); } if (fix->u.cpool.min_list_size > fix->list_size) fix->u.cpool.min_list_size = fix->list_size; @@ -1509,11 +1537,8 @@ fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) ErtsAlcFixList_t *fix; void *res; - ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type - && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); - - fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; - ASSERT(size == fix->type_size); + fix = &allctr->fix[ERTS_ALC_FIX_TYPE_IX(type)]; + ASSERT(type == fix->type && size == fix->type_size); ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); fix->u.nocpool.used++; @@ -1530,7 +1555,7 @@ fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) if (IS_SBC_BLK(blk)) destroy_carrier(allctr, blk, NULL); else - mbc_free(allctr, p, NULL); + mbc_free(allctr, type, p, NULL); fix->u.nocpool.allocated--; } ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); @@ -1565,10 +1590,8 @@ fix_nocpool_free(Allctr_t *allctr, Block_t *blk; ErtsAlcFixList_t *fix; - ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type - && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); - - fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + fix = &allctr->fix[ERTS_ALC_T2N(type) - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + ASSERT(fix->type == type); ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); fix->u.nocpool.used--; @@ -1587,7 +1610,7 @@ fix_nocpool_free(Allctr_t *allctr, if (IS_SBC_BLK(blk)) destroy_carrier(allctr, blk, NULL); else - mbc_free(allctr, p, NULL); + mbc_free(allctr, type, p, NULL); p = fix->list; fix->list = *((void **) p); fix->list_size--; @@ -1598,7 +1621,7 @@ fix_nocpool_free(Allctr_t *allctr, if (IS_SBC_BLK(blk)) destroy_carrier(allctr, blk, NULL); else - mbc_free(allctr, p, NULL); + mbc_free(allctr, type, p, NULL); ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); } @@ -1639,7 +1662,7 @@ fix_nocpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) ptr = fix->list; fix->list = *((void **) ptr); fix->list_size--; - dealloc_block(allctr, ptr, NULL, 0); + dealloc_block(allctr, fix->type, 0, ptr, NULL); fix->u.nocpool.allocated--; } if (fix->list_size != 0) { @@ -1802,7 +1825,7 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep, static void init_dd_queue(ErtsAllctrDDQueue_t *ddq) { - erts_atomic_init_nob(&ddq->tail.data.marker.atmc_next, ERTS_AINT_NULL); + erts_atomic_init_nob(&ddq->tail.data.marker.u.atmc_next, ERTS_AINT_NULL); erts_atomic_init_nob(&ddq->tail.data.last, (erts_aint_t) &ddq->tail.data.marker); erts_atomic_init_nob(&ddq->tail.data.um_refc[0], 0); @@ -1823,17 +1846,17 @@ ddq_managed_thread_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr, int cinit) erts_aint_t itmp; ErtsAllctrDDBlock_t *enq, *this = ptr; - erts_atomic_init_nob(&this->atmc_next, ERTS_AINT_NULL); + erts_atomic_init_nob(&this->u.atmc_next, ERTS_AINT_NULL); /* Enqueue at end of list... */ enq = (ErtsAllctrDDBlock_t *) erts_atomic_read_nob(&ddq->tail.data.last); - itmp = erts_atomic_cmpxchg_relb(&enq->atmc_next, + itmp = erts_atomic_cmpxchg_relb(&enq->u.atmc_next, (erts_aint_t) this, ERTS_AINT_NULL); if (itmp == ERTS_AINT_NULL) { /* We are required to move last pointer */ #ifdef DEBUG - ASSERT(ERTS_AINT_NULL == erts_atomic_read_nob(&this->atmc_next)); + ASSERT(ERTS_AINT_NULL == erts_atomic_read_nob(&this->u.atmc_next)); ASSERT(((erts_aint_t) enq) == erts_atomic_xchg_relb(&ddq->tail.data.last, (erts_aint_t) this)); @@ -1851,8 +1874,8 @@ ddq_managed_thread_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr, int cinit) while (1) { erts_aint_t itmp2; - erts_atomic_set_nob(&this->atmc_next, itmp); - itmp2 = erts_atomic_cmpxchg_relb(&enq->atmc_next, + erts_atomic_set_nob(&this->u.atmc_next, itmp); + itmp2 = erts_atomic_cmpxchg_relb(&enq->u.atmc_next, (erts_aint_t) this, itmp); if (itmp == itmp2) @@ -1861,7 +1884,7 @@ ddq_managed_thread_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr, int cinit) itmp = itmp2; else { enq = (ErtsAllctrDDBlock_t *) itmp2; - itmp = erts_atomic_read_acqb(&enq->atmc_next); + itmp = erts_atomic_read_acqb(&enq->u.atmc_next); ASSERT(itmp != ERTS_AINT_NULL); } i++; @@ -1877,8 +1900,8 @@ check_insert_marker(ErtsAllctrDDQueue_t *ddq, erts_aint_t ilast) erts_aint_t itmp; ErtsAllctrDDBlock_t *last = (ErtsAllctrDDBlock_t *) ilast; - erts_atomic_init_nob(&ddq->tail.data.marker.atmc_next, ERTS_AINT_NULL); - itmp = erts_atomic_cmpxchg_relb(&last->atmc_next, + erts_atomic_init_nob(&ddq->tail.data.marker.u.atmc_next, ERTS_AINT_NULL); + itmp = erts_atomic_cmpxchg_relb(&last->u.atmc_next, (erts_aint_t) &ddq->tail.data.marker, ERTS_AINT_NULL); if (itmp == ERTS_AINT_NULL) { @@ -1929,7 +1952,7 @@ ddq_dequeue(ErtsAllctrDDQueue_t *ddq) ASSERT(ddq->head.used_marker); ddq->head.used_marker = 0; blk = ((ErtsAllctrDDBlock_t *) - erts_atomic_read_nob(&blk->atmc_next)); + erts_atomic_read_nob(&blk->u.atmc_next)); if (blk == ddq->head.unref_end) { ddq->head.first = blk; return NULL; @@ -1937,7 +1960,7 @@ ddq_dequeue(ErtsAllctrDDQueue_t *ddq) } ddq->head.first = ((ErtsAllctrDDBlock_t *) - erts_atomic_read_nob(&blk->atmc_next)); + erts_atomic_read_nob(&blk->u.atmc_next)); ASSERT(ddq->head.first); @@ -1999,19 +2022,13 @@ check_pending_dealloc_carrier(Allctr_t *allctr, int *need_more_work); static void -handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr) +handle_delayed_fix_dealloc(Allctr_t *allctr, ErtsAlcType_t type, Uint32 flags, + void *ptr) { - ErtsAlcType_t type; - - type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type; - - ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE - <= (type & ~ERTS_ALC_FIX_NO_UNUSE)); - ASSERT((type & ~ERTS_ALC_FIX_NO_UNUSE) - <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + ASSERT(ERTS_ALC_IS_FIX_TYPE(type)); if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) - fix_nocpool_free(allctr, (type & ~ERTS_ALC_FIX_NO_UNUSE), ptr); + fix_nocpool_free(allctr, type, ptr); else { Block_t *blk = UMEM2BLK(ptr); Carrier_t *busy_pcrr_p; @@ -2026,20 +2043,24 @@ handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr) NULL, &busy_pcrr_p); if (used_allctr == allctr) { doit: - fix_cpool_free(allctr, (type & ~ERTS_ALC_FIX_NO_UNUSE), - ptr, &busy_pcrr_p, - !(type & ERTS_ALC_FIX_NO_UNUSE)); + fix_cpool_free(allctr, type, flags, ptr, &busy_pcrr_p); clear_busy_pool_carrier(allctr, busy_pcrr_p); } else { /* Carrier migrated; need to redirect block to new owner... */ - int cinit = used_allctr->dd.ix - allctr->dd.ix; + ErtsAllctrDDBlock_t *dd_block; + int cinit; + + dd_block = (ErtsAllctrDDBlock_t*)ptr; + dd_block->flags = flags; + dd_block->type = type; ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); DEC_CC(allctr->calls.this_free); - ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type; + cinit = used_allctr->dd.ix - allctr->dd.ix; + if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit)) erts_alloc_notify_delayed_dealloc(used_allctr->ix); } @@ -2063,7 +2084,6 @@ handle_delayed_dealloc(Allctr_t *allctr, int need_mr_wrk = 0; int have_checked_incoming = 0; int ops = 0; - ErtsAlcFixList_t *fix; int res; ErtsAllctrDDQueue_t *ddq; @@ -2072,8 +2092,6 @@ handle_delayed_dealloc(Allctr_t *allctr, ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); - fix = allctr->fix; - ddq = &allctr->dd.q; res = 0; @@ -2162,16 +2180,27 @@ handle_delayed_dealloc(Allctr_t *allctr, } } else { + ErtsAllctrDDBlock_t *dd_block; + ErtsAlcType_t type; + Uint32 flags; + + dd_block = (ErtsAllctrDDBlock_t*)ptr; + flags = dd_block->flags; + type = dd_block->type; + + flags |= DEALLOC_FLG_REDIRECTED; + ASSERT(IS_SBC_BLK(blk) || (ABLK_TO_MBC(blk) != ErtsContainerStruct(blk, Carrier_t, cpool.homecoming_dd.blk))); INC_CC(allctr->calls.this_free); - if (fix) - handle_delayed_fix_dealloc(allctr, ptr); - else - dealloc_block(allctr, ptr, NULL, 1); + if (ERTS_ALC_IS_FIX_TYPE(type)) { + handle_delayed_fix_dealloc(allctr, type, flags, ptr); + } else { + dealloc_block(allctr, type, flags, ptr, NULL); + } } } @@ -2199,8 +2228,10 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type, void *ptr, int cinit) { - if (allctr->fix) - ((ErtsAllctrFixDDBlock_t*) ptr)->fix_type = type; + ErtsAllctrDDBlock_t *dd_block = ((ErtsAllctrDDBlock_t*)ptr); + + dd_block->type = type; + dd_block->flags = 0; if (ddq_enqueue(&allctr->dd.q, ptr, cinit)) erts_alloc_notify_delayed_dealloc(allctr->ix); @@ -2251,7 +2282,7 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) if (allctr->main_carrier == crr) return; - if (crr->cpool.blocks_size > crr->cpool.abandon_limit) + if (crr->cpool.total_blocks_size > crr->cpool.abandon_limit) return; if (crr->cpool.thr_prgr != ERTS_THR_PRGR_INVALID @@ -2287,24 +2318,26 @@ erts_alcu_check_delayed_dealloc(Allctr_t *allctr, ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL, NULL) static void -dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_redirect) +dealloc_block(Allctr_t *allctr, ErtsAlcType_t type, Uint32 flags, void *ptr, + ErtsAlcFixList_t *fix) { Block_t *blk = UMEM2BLK(ptr); + ASSERT(!fix || type == fix->type); + ERTS_LC_ASSERT(!allctr->thread_safe || erts_lc_mtx_is_locked(&allctr->mutex)); if (IS_SBC_BLK(blk)) { destroy_carrier(allctr, blk, NULL); if (fix && ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { - ErtsAlcType_t type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type; - if (!(type & ERTS_ALC_FIX_NO_UNUSE)) + if (!(flags & DEALLOC_FLG_FIX_SHRINK)) fix->u.cpool.used--; fix->u.cpool.allocated--; } } else if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) - mbc_free(allctr, ptr, NULL); + mbc_free(allctr, type, ptr, NULL); else { Carrier_t *busy_pcrr_p; Allctr_t *used_allctr; @@ -2313,22 +2346,29 @@ dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_ NULL, &busy_pcrr_p); if (used_allctr == allctr) { if (fix) { - ErtsAlcType_t type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type; - if (!(type & ERTS_ALC_FIX_NO_UNUSE)) + if (!(flags & DEALLOC_FLG_FIX_SHRINK)) fix->u.cpool.used--; fix->u.cpool.allocated--; } - mbc_free(allctr, ptr, &busy_pcrr_p); + mbc_free(allctr, type, ptr, &busy_pcrr_p); clear_busy_pool_carrier(allctr, busy_pcrr_p); } else { /* Carrier migrated; need to redirect block to new owner... */ - int cinit = used_allctr->dd.ix - allctr->dd.ix; + ErtsAllctrDDBlock_t *dd_block; + int cinit; + + dd_block = (ErtsAllctrDDBlock_t*)ptr; + dd_block->flags = flags; + dd_block->type = type; ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); - if (dec_cc_on_redirect) + if (flags & DEALLOC_FLG_REDIRECTED) DEC_CC(allctr->calls.this_free); + + cinit = used_allctr->dd.ix - allctr->dd.ix; + if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit)) erts_alloc_notify_delayed_dealloc(used_allctr->ix); } @@ -2495,7 +2535,7 @@ mbc_alloc(Allctr_t *allctr, Uint size) } static void -mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp) +mbc_free(Allctr_t *allctr, ErtsAlcType_t type, void *p, Carrier_t **busy_pcrr_pp) { Uint is_first_blk; Uint is_last_blk; @@ -2517,7 +2557,8 @@ mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp) crr = ABLK_TO_MBC(blk); ERTS_ALC_CPOOL_FREE_OP(allctr); - STAT_MBC_BLK_FREE(allctr, crr, busy_pcrr_pp, blk_sz, alcu_flgs); + + STAT_MBC_BLK_FREE(allctr, type, crr, busy_pcrr_pp, blk_sz, alcu_flgs); is_first_blk = IS_MBC_FIRST_ABLK(allctr, blk); is_last_blk = IS_LAST_BLK(blk); @@ -2586,8 +2627,8 @@ mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp) } static void * -mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, - Carrier_t **busy_pcrr_pp) +mbc_realloc(Allctr_t *allctr, ErtsAlcType_t type, void *p, Uint size, + Uint32 alcu_flgs, Carrier_t **busy_pcrr_pp) { void *new_p; Uint old_blk_sz; @@ -2625,7 +2666,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, new_blk = UMEM2BLK(new_p); ASSERT(!(IS_MBC_BLK(new_blk) && ABLK_TO_MBC(new_blk) == *busy_pcrr_pp)); sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ)); - mbc_free(allctr, p, busy_pcrr_pp); + mbc_free(allctr, type, p, busy_pcrr_pp); return new_p; } @@ -2702,7 +2743,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, crr = ABLK_TO_MBC(blk); ERTS_ALC_CPOOL_REALLOC_OP(allctr); - STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs); + STAT_MBC_BLK_FREE(allctr, type, crr, NULL, old_blk_sz, alcu_flgs); STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs); ASSERT(MBC_BLK_SZ(blk) >= allctr->min_block_size); @@ -2806,7 +2847,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, } ERTS_ALC_CPOOL_REALLOC_OP(allctr); - STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs); + STAT_MBC_BLK_FREE(allctr, type, crr, NULL, old_blk_sz, alcu_flgs); STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs); ASSERT(IS_ALLOCED_BLK(blk)); @@ -2867,7 +2908,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, if (!new_p) return NULL; sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ)); - mbc_free(allctr, p, busy_pcrr_pp); + mbc_free(allctr, type, p, busy_pcrr_pp); return new_p; @@ -2897,7 +2938,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, 1); new_p = BLK2UMEM(new_blk); sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ)); - mbc_free(allctr, p, NULL); + mbc_free(allctr, type, p, NULL); return new_p; } else { @@ -2954,7 +2995,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, 0); ERTS_ALC_CPOOL_FREE_OP(allctr); - STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs); + STAT_MBC_BLK_FREE(allctr, type, crr, NULL, old_blk_sz, alcu_flgs); return new_p; } @@ -2992,14 +3033,11 @@ typedef union { # error "Carrier pool implementation assumes ERTS_ALC_A_MIN > ERTS_ALC_A_INVALID" #endif -/* - * The pool is only allowed to be manipulated by managed - * threads except in the alloc_SUITE:cpool case. In this - * test case carrier_pool[ERTS_ALC_A_INVALID] will be - * used. - */ +/* The pools are only allowed to be manipulated by managed threads except in + * the alloc_SUITE:cpool test, where only test_carrier_pool is used. */ -static ErtsAlcCrrPool_t carrier_pool[ERTS_ALC_A_MAX+1] erts_align_attribute(ERTS_CACHE_LINE_SIZE); +static ErtsAlcCrrPool_t firstfit_carrier_pool; +static ErtsAlcCrrPool_t test_carrier_pool; #define ERTS_ALC_CPOOL_MAX_BACKOFF (1 << 8) @@ -3020,12 +3058,12 @@ backoff(int n) static int cpool_dbg_is_in_pool(Allctr_t *allctr, Carrier_t *crr) { - ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + ErtsAlcCPoolData_t *sentinel = allctr->cpool.sentinel; ErtsAlcCPoolData_t *cpdp = sentinel; Carrier_t *tmp_crr; while (1) { - cpdp = (ErtsAlcCPoolData_t *) (erts_atomic_read_ddrb(&cpdp->next) & ~FLG_MASK); + cpdp = (ErtsAlcCPoolData_t *) (erts_atomic_read_ddrb(&cpdp->next) & ~CRR_FLG_MASK); if (cpdp == sentinel) return 0; tmp_crr = (Carrier_t *) (((char *) cpdp) - offsetof(Carrier_t, cpool)); @@ -3037,7 +3075,7 @@ cpool_dbg_is_in_pool(Allctr_t *allctr, Carrier_t *crr) static int cpool_is_empty(Allctr_t *allctr) { - ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + ErtsAlcCPoolData_t *sentinel = allctr->cpool.sentinel; return ((erts_atomic_read_rb(&sentinel->next) == (erts_aint_t) sentinel) && (erts_atomic_read_rb(&sentinel->prev) == (erts_aint_t) sentinel)); } @@ -3127,16 +3165,31 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) { ErtsAlcCPoolData_t *cpd1p, *cpd2p; erts_aint_t val; - ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + ErtsAlcCPoolData_t *sentinel = allctr->cpool.sentinel; Allctr_t *orig_allctr = crr->cpool.orig_allctr; - ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ + ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_TEST /* testcase */ || erts_thr_progress_is_managed_thread()); - erts_atomic_add_nob(&orig_allctr->cpool.stat.blocks_size, - (erts_aint_t) crr->cpool.blocks_size); - erts_atomic_add_nob(&orig_allctr->cpool.stat.no_blocks, - (erts_aint_t) crr->cpool.blocks); + { + int alloc_no = allctr->alloc_no; + + ERTS_ALC_CPOOL_ASSERT( + erts_atomic_read_nob(&orig_allctr->cpool.stat.blocks_size[alloc_no]) >= 0 && + crr->cpool.blocks_size[alloc_no] >= 0); + + ERTS_ALC_CPOOL_ASSERT( + erts_atomic_read_nob(&orig_allctr->cpool.stat.no_blocks[alloc_no]) >= 0 && + crr->cpool.blocks[alloc_no] >= 0); + + /* We only modify the counter for our current type since the others are + * conceptually still in the pool. */ + erts_atomic_add_nob(&orig_allctr->cpool.stat.blocks_size[alloc_no], + ((erts_aint_t) crr->cpool.blocks_size[alloc_no])); + erts_atomic_add_nob(&orig_allctr->cpool.stat.no_blocks[alloc_no], + ((erts_aint_t) crr->cpool.blocks[alloc_no])); + } + erts_atomic_add_nob(&orig_allctr->cpool.stat.carriers_size, (erts_aint_t) CARRIER_SZ(crr)); erts_atomic_inc_nob(&orig_allctr->cpool.stat.no_carriers); @@ -3209,10 +3262,10 @@ cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr) ErtsAlcCPoolData_t *cpd1p, *cpd2p; erts_aint_t val; #ifdef ERTS_ALC_CPOOL_DEBUG - ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + ErtsAlcCPoolData_t *sentinel = allctr->cpool.sentinel; #endif - ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ + ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_TEST /* testcase */ || erts_thr_progress_is_managed_thread()); ERTS_ALC_CPOOL_ASSERT(sentinel != &crr->cpool); @@ -3288,13 +3341,29 @@ cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr) crr->cpool.thr_prgr = erts_thr_progress_later(NULL); - erts_atomic_add_nob(&prev_allctr->cpool.stat.blocks_size, - -((erts_aint_t) crr->cpool.blocks_size)); - erts_atomic_add_nob(&prev_allctr->cpool.stat.no_blocks, - -((erts_aint_t) crr->cpool.blocks)); - erts_atomic_add_nob(&prev_allctr->cpool.stat.carriers_size, + { + Allctr_t *orig_allctr = crr->cpool.orig_allctr; + int alloc_no = allctr->alloc_no; + + ERTS_ALC_CPOOL_ASSERT(orig_allctr == prev_allctr); + + ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks_size[alloc_no] <= + erts_atomic_read_nob(&orig_allctr->cpool.stat.blocks_size[alloc_no])); + + ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks[alloc_no] <= + erts_atomic_read_nob(&orig_allctr->cpool.stat.no_blocks[alloc_no])); + + /* We only modify the counters for our current type since the others + * were, conceptually, never taken out of the pool. */ + erts_atomic_add_nob(&orig_allctr->cpool.stat.blocks_size[alloc_no], + -((erts_aint_t) crr->cpool.blocks_size[alloc_no])); + erts_atomic_add_nob(&orig_allctr->cpool.stat.no_blocks[alloc_no], + -((erts_aint_t) crr->cpool.blocks[alloc_no])); + + erts_atomic_add_nob(&orig_allctr->cpool.stat.carriers_size, -((erts_aint_t) CARRIER_SZ(crr))); - erts_atomic_dec_wb(&prev_allctr->cpool.stat.no_carriers); + erts_atomic_dec_wb(&orig_allctr->cpool.stat.no_carriers); + } } @@ -3309,7 +3378,7 @@ cpool_fetch(Allctr_t *allctr, UWord size) ErtsAlcCPoolData_t *cpool_entrance = NULL; ErtsAlcCPoolData_t *sentinel; - ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ + ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_TEST /* testcase */ || erts_thr_progress_is_managed_thread()); i = ERTS_ALC_CPOOL_MAX_FETCH_INSPECT; @@ -3411,7 +3480,7 @@ cpool_fetch(Allctr_t *allctr, UWord size) /* * Finally search the shared pool and try employ foreign carriers */ - sentinel = &carrier_pool[allctr->alloc_no].sentinel; + sentinel = allctr->cpool.sentinel; if (cpool_entrance) { /* * We saw a pooled carried above, use it as entrance into the pool @@ -3664,8 +3733,9 @@ cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr) crr->cpool.orig_allctr = allctr; crr->cpool.thr_prgr = ERTS_THR_PRGR_INVALID; erts_atomic_init_nob(&crr->cpool.max_size, 0); - crr->cpool.blocks = 0; - crr->cpool.blocks_size = 0; + sys_memset(&crr->cpool.blocks_size, 0, sizeof(crr->cpool.blocks_size)); + sys_memset(&crr->cpool.blocks, 0, sizeof(crr->cpool.blocks)); + crr->cpool.total_blocks_size = 0; if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) crr->cpool.abandon_limit = 0; else { @@ -3762,7 +3832,8 @@ poolify_my_carrier(Allctr_t *allctr, Carrier_t *crr) } static void -cpool_read_stat(Allctr_t *allctr, UWord *nocp, UWord *cszp, UWord *nobp, UWord *bszp) +cpool_read_stat(Allctr_t *allctr, int alloc_no, + UWord *nocp, UWord *cszp, UWord *nobp, UWord *bszp) { int i; UWord noc = 0, csz = 0, nob = 0, bsz = 0; @@ -3782,10 +3853,10 @@ cpool_read_stat(Allctr_t *allctr, UWord *nocp, UWord *cszp, UWord *nobp, UWord * ? erts_atomic_read_nob(&allctr->cpool.stat.carriers_size) : 0); tnob = (UWord) (nobp - ? erts_atomic_read_nob(&allctr->cpool.stat.no_blocks) + ? erts_atomic_read_nob(&allctr->cpool.stat.no_blocks[alloc_no]) : 0); tbsz = (UWord) (bszp - ? erts_atomic_read_nob(&allctr->cpool.stat.blocks_size) + ? erts_atomic_read_nob(&allctr->cpool.stat.blocks_size[alloc_no]) : 0); if (tnoc == noc && tcsz == csz && tnob == nob && tbsz == bsz) break; @@ -4390,6 +4461,8 @@ static struct { Eterm blocks_size; Eterm blocks; + Eterm foreign_blocks; + Eterm calls; Eterm sys_alloc; Eterm sys_free; @@ -4490,6 +4563,7 @@ init_atoms(Allctr_t *allctr) AM_INIT(carriers); AM_INIT(blocks_size); AM_INIT(blocks); + AM_INIT(foreign_blocks); AM_INIT(calls); AM_INIT(sys_alloc); @@ -4625,7 +4699,6 @@ sz_info_fix(Allctr_t *allctr, ErtsAlcFixList_t *fix = &allctr->fix[ix]; UWord alloced = fix->type_size * fix->u.cpool.allocated; UWord used = fix->type_size * fix->u.cpool.used; - ErtsAlcType_t n = ERTS_ALC_N_MIN_A_FIXED_SIZE + ix; if (print_to_p) { fmtfn_t to = *print_to_p; @@ -4633,14 +4706,14 @@ sz_info_fix(Allctr_t *allctr, erts_print(to, arg, "fix type internal: %s %bpu %bpu\n", - (char *) ERTS_ALC_N2TD(n), + (char *) ERTS_ALC_T2TD(fix->type), alloced, used); } if (hpp || szp) { add_3tup(hpp, szp, &res, - alloc_type_atoms[n], + alloc_type_atoms[ERTS_ALC_T2N(fix->type)], bld_unstable_uint(hpp, szp, alloced), bld_unstable_uint(hpp, szp, used)); } @@ -4653,7 +4726,6 @@ sz_info_fix(Allctr_t *allctr, ErtsAlcFixList_t *fix = &allctr->fix[ix]; UWord alloced = fix->type_size * fix->u.nocpool.allocated; UWord used = fix->type_size*fix->u.nocpool.used; - ErtsAlcType_t n = ERTS_ALC_N_MIN_A_FIXED_SIZE + ix; if (print_to_p) { fmtfn_t to = *print_to_p; @@ -4661,14 +4733,14 @@ sz_info_fix(Allctr_t *allctr, erts_print(to, arg, "fix type: %s %bpu %bpu\n", - (char *) ERTS_ALC_N2TD(n), + (char *) ERTS_ALC_T2TD(fix->type), alloced, used); } if (hpp || szp) { add_3tup(hpp, szp, &res, - alloc_type_atoms[n], + alloc_type_atoms[ERTS_ALC_T2N(fix->type)], bld_unstable_uint(hpp, szp, alloced), bld_unstable_uint(hpp, szp, used)); } @@ -4741,9 +4813,9 @@ info_cpool(Allctr_t *allctr, noc = csz = nob = bsz = ~0; if (print_to_p || hpp) { if (sz_only) - cpool_read_stat(allctr, NULL, &csz, NULL, &bsz); + cpool_read_stat(allctr, allctr->alloc_no, NULL, &csz, NULL, &bsz); else - cpool_read_stat(allctr, &noc, &csz, &nob, &bsz); + cpool_read_stat(allctr, allctr->alloc_no, &noc, &csz, &nob, &bsz); } if (print_to_p) { @@ -4758,6 +4830,10 @@ info_cpool(Allctr_t *allctr, } if (hpp || szp) { + Eterm foreign_blocks; + int i; + + foreign_blocks = NIL; res = NIL; if (!sz_only) { @@ -4804,22 +4880,61 @@ info_cpool(Allctr_t *allctr, add_3tup(hpp, szp, &res, am.entrance_removed, bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.entrance_removed)), bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.entrance_removed))); + } add_2tup(hpp, szp, &res, am.carriers_size, bld_unstable_uint(hpp, szp, csz)); - } - if (!sz_only) - add_2tup(hpp, szp, &res, - am.carriers, - bld_unstable_uint(hpp, szp, noc)); + + if (!sz_only) { + add_2tup(hpp, szp, &res, + am.carriers, + bld_unstable_uint(hpp, szp, noc)); + } + add_2tup(hpp, szp, &res, am.blocks_size, bld_unstable_uint(hpp, szp, bsz)); - if (!sz_only) + + if (!sz_only) { add_2tup(hpp, szp, &res, am.blocks, bld_unstable_uint(hpp, szp, nob)); + } + + for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { + const char *name_str; + Eterm name, info; + + if (i == allctr->alloc_no) { + continue; + } + + cpool_read_stat(allctr, i, NULL, NULL, &nob, &bsz); + + if (bsz == 0 && (nob == 0 || sz_only)) { + continue; + } + + name_str = ERTS_ALC_A2AD(i); + info = NIL; + + add_2tup(hpp, szp, &info, + am.blocks_size, + bld_unstable_uint(hpp, szp, bsz)); + + if (!sz_only) { + add_2tup(hpp, szp, &info, + am.blocks, + bld_unstable_uint(hpp, szp, nob)); + } + + name = am_atom_put(name_str, sys_strlen(name_str)); + + add_2tup(hpp, szp, &foreign_blocks, name, info); + } + + add_2tup(hpp, szp, &res, am.foreign_blocks, foreign_blocks); } return res; @@ -5455,6 +5570,19 @@ erts_alcu_info(Allctr_t *allctr, return res; } +void +erts_alcu_foreign_size(Allctr_t *allctr, ErtsAlcType_t alloc_no, AllctrSize_t *size) +{ + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + UWord csz, bsz; + cpool_read_stat(allctr, alloc_no, NULL, &csz, NULL, &bsz); + size->carriers = csz; + size->blocks = bsz; + } else { + size->carriers = 0; + size->blocks = 0; + } +} void erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t *fi, int fisz) @@ -5473,7 +5601,7 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t * if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { UWord csz, bsz; - cpool_read_stat(allctr, NULL, &csz, NULL, &bsz); + cpool_read_stat(allctr, allctr->alloc_no, NULL, &csz, NULL, &bsz); size->blocks += bsz; size->carriers += csz; } @@ -5518,6 +5646,11 @@ do_erts_alcu_alloc(ErtsAlcType_t type, Allctr_t *allctr, Uint size) ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); + /* Reject sizes that can't fit into the header word. */ + if (size > ~BLK_FLG_MASK) { + return NULL; + } + #if ALLOC_ZERO_EQ_NULL if (!size) return NULL; @@ -5684,12 +5817,11 @@ do_erts_alcu_free(ErtsAlcType_t type, Allctr_t *allctr, void *p, ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); if (p) { - INC_CC(allctr->calls.this_free); - if (allctr->fix) { + if (ERTS_ALC_IS_FIX_TYPE(type)) { if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) - fix_cpool_free(allctr, type, p, busy_pcrr_pp, 1); + fix_cpool_free(allctr, type, 0, p, busy_pcrr_pp); else fix_nocpool_free(allctr, type, p); } @@ -5698,7 +5830,7 @@ do_erts_alcu_free(ErtsAlcType_t type, Allctr_t *allctr, void *p, if (IS_SBC_BLK(blk)) destroy_carrier(allctr, blk, NULL); else - mbc_free(allctr, p, busy_pcrr_pp); + mbc_free(allctr, type, p, busy_pcrr_pp); } } } @@ -5800,6 +5932,11 @@ do_erts_alcu_realloc(ErtsAlcType_t type, return res; } + /* Reject sizes that can't fit into the header word. */ + if (size > ~BLK_FLG_MASK) { + return NULL; + } + #if ALLOC_ZERO_EQ_NULL if (!size) { ASSERT(p); @@ -5816,7 +5953,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, if (size < allctr->sbc_threshold) { if (IS_MBC_BLK(blk)) - res = mbc_realloc(allctr, p, size, alcu_flgs, busy_pcrr_pp); + res = mbc_realloc(allctr, type, p, size, alcu_flgs, busy_pcrr_pp); else { Uint used_sz = SBC_HEADER_SIZE + ABLK_HDR_SZ + size; Uint crr_sz; @@ -5875,7 +6012,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, sys_memcpy((void *) res, (void *) p, MIN(MBC_ABLK_SZ(blk) - ABLK_HDR_SZ, size)); - mbc_free(allctr, p, busy_pcrr_pp); + mbc_free(allctr, type, p, busy_pcrr_pp); } else res = NULL; @@ -6243,6 +6380,7 @@ int erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) { /* erts_alcu_start assumes that allctr has been zeroed */ + int i; if (((UWord)allctr & ERTS_CRR_ALCTR_FLG_MASK) != 0) { erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_alcu_start: Alignment error\n", @@ -6266,6 +6404,11 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->ix = init->ix; allctr->alloc_no = init->alloc_no; + allctr->alloc_strat = init->alloc_strat; + + ASSERT(allctr->alloc_no >= ERTS_ALC_A_MIN && + allctr->alloc_no <= ERTS_ALC_A_MAX); + if (allctr->alloc_no < ERTS_ALC_A_MIN || ERTS_ALC_A_MAX < allctr->alloc_no) allctr->alloc_no = ERTS_ALC_A_INVALID; @@ -6318,8 +6461,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) + sizeof(FreeBlkFtr_t)); if (init->tpref) { Uint sz = ABLK_HDR_SZ; - sz += (init->fix ? - sizeof(ErtsAllctrFixDDBlock_t) : sizeof(ErtsAllctrDDBlock_t)); + sz += sizeof(ErtsAllctrDDBlock_t); sz = UNIT_CEILING(sz); if (sz > allctr->min_block_size) allctr->min_block_size = sz; @@ -6330,8 +6472,10 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->cpool.dc_list.last = NULL; allctr->cpool.abandon_limit = 0; allctr->cpool.disable_abandon = 0; - erts_atomic_init_nob(&allctr->cpool.stat.blocks_size, 0); - erts_atomic_init_nob(&allctr->cpool.stat.no_blocks, 0); + for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { + erts_atomic_init_nob(&allctr->cpool.stat.blocks_size[i], 0); + erts_atomic_init_nob(&allctr->cpool.stat.no_blocks[i], 0); + } erts_atomic_init_nob(&allctr->cpool.stat.carriers_size, 0); erts_atomic_init_nob(&allctr->cpool.stat.no_carriers, 0); allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT; @@ -6339,6 +6483,13 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->cpool.util_limit = init->acul; allctr->cpool.in_pool_limit = init->acnl; allctr->cpool.fblk_min_limit = init->acfml; + + if (allctr->alloc_strat == ERTS_ALC_S_FIRSTFIT) { + allctr->cpool.sentinel = &firstfit_carrier_pool.sentinel; + } + else if (allctr->alloc_no != ERTS_ALC_A_TEST) { + ERTS_INTERNAL_ERROR("Impossible carrier migration config."); + } } else { allctr->cpool.util_limit = 0; @@ -6346,6 +6497,12 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->cpool.fblk_min_limit = 0; } + /* The invasive tests don't really care whether the pool is enabled or not, + * so we need to set this unconditionally for this allocator type. */ + if (allctr->alloc_no == ERTS_ALC_A_TEST) { + allctr->cpool.sentinel = &test_carrier_pool.sentinel; + } + allctr->sbc_threshold = adjust_sbct(allctr, init->sbct); #if HAVE_ERTS_MSEG @@ -6457,9 +6614,10 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->fix_shrink_scheduled = 0; for (i = 0; i < ERTS_ALC_NO_FIXED_SIZES; i++) { allctr->fix[i].type_size = init->fix_type_size[i]; + allctr->fix[i].type = ERTS_ALC_N2T(i + ERTS_ALC_N_MIN_A_FIXED_SIZE); allctr->fix[i].list_size = 0; allctr->fix[i].list = NULL; - ASSERT(allctr->fix[i].type_size >= sizeof(ErtsAllctrFixDDBlock_t)); + ASSERT(allctr->fix[i].type_size >= sizeof(ErtsAllctrDDBlock_t)); if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { allctr->fix[i].u.cpool.min_list_size = 0; allctr->fix[i].u.cpool.shrink_list = 0; @@ -6508,12 +6666,16 @@ erts_alcu_stop(Allctr_t *allctr) void erts_alcu_init(AlcUInit_t *init) { - int i; - for (i = 0; i <= ERTS_ALC_A_MAX; i++) { - ErtsAlcCPoolData_t *sentinel = &carrier_pool[i].sentinel; - erts_atomic_init_nob(&sentinel->next, (erts_aint_t) sentinel); - erts_atomic_init_nob(&sentinel->prev, (erts_aint_t) sentinel); - } + ErtsAlcCPoolData_t *sentinel; + + sentinel = &firstfit_carrier_pool.sentinel; + erts_atomic_init_nob(&sentinel->next, (erts_aint_t) sentinel); + erts_atomic_init_nob(&sentinel->prev, (erts_aint_t) sentinel); + + sentinel = &test_carrier_pool.sentinel; + erts_atomic_init_nob(&sentinel->next, (erts_aint_t) sentinel); + erts_atomic_init_nob(&sentinel->prev, (erts_aint_t) sentinel); + ERTS_CT_ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */ #if HAVE_ERTS_MSEG ASSERT(erts_mseg_unit_size() == ERTS_SACRR_UNIT_SZ); @@ -6695,7 +6857,7 @@ static int blockscan_cpool_yielding(blockscan_t *state) { ErtsAlcCPoolData_t *sentinel, *cursor; - sentinel = &carrier_pool[(state->allocator)->alloc_no].sentinel; + sentinel = (state->allocator)->cpool.sentinel; cursor = blockscan_restore_cpool_cursor(state); if (ERTS_PROC_IS_EXITING(state->process)) { @@ -6827,11 +6989,8 @@ static int blockscan_sweep_mbcs(blockscan_t *state) static int blockscan_sweep_cpool(blockscan_t *state) { if (state->current_op != blockscan_sweep_cpool) { - ErtsAlcCPoolData_t *sentinel; - SET_CARRIER_HDR(&state->dummy_carrier, 0, SCH_MBC, state->allocator); - sentinel = &carrier_pool[(state->allocator)->alloc_no].sentinel; - state->cpool_cursor = sentinel; + state->cpool_cursor = (state->allocator)->cpool.sentinel; } state->current_op = blockscan_sweep_cpool; @@ -7115,11 +7274,14 @@ static int gather_ahist_scan(Allctr_t *allocator, alcu_atag_t tag; block = SBC2BLK(allocator, carrier); - tag = GET_BLK_ATAG(block); - ASSERT(DBG_IS_VALID_ATAG(allocator, tag)); + if (BLK_HAS_ATAG(block)) { + tag = GET_BLK_ATAG(block); + + ASSERT(DBG_IS_VALID_ATAG(tag)); - gather_ahist_update(state, tag, SBC_BLK_SZ(block)); + gather_ahist_update(state, tag, SBC_BLK_SZ(block)); + } } else { UWord scanned_bytes = MBC_HEADER_SIZE(allocator); @@ -7130,10 +7292,10 @@ static int gather_ahist_scan(Allctr_t *allocator, while (1) { UWord block_size = MBC_BLK_SZ(block); - if (IS_ALLOCED_BLK(block)) { + if (IS_ALLOCED_BLK(block) && BLK_HAS_ATAG(block)) { alcu_atag_t tag = GET_BLK_ATAG(block); - ASSERT(DBG_IS_VALID_ATAG(allocator, tag)); + ASSERT(DBG_IS_VALID_ATAG(tag)); gather_ahist_update(state, tag, block_size); } @@ -7293,8 +7455,6 @@ int erts_alcu_gather_alloc_histograms(Process *p, int allocator_num, sched_id, &allocator)) { return 0; - } else if (!allocator->atags) { - return 0; } ensure_atoms_initialized(allocator); diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index f26ace1534..9ab8589bf3 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -24,6 +24,7 @@ #define ERTS_ALCU_VSN_STR "3.0" #include "erl_alloc_types.h" +#include "erl_alloc.h" #define ERL_THREADS_EMU_INTERNAL__ #include "erl_threads.h" @@ -44,6 +45,7 @@ typedef struct { typedef struct { char *name_prefix; ErtsAlcType_t alloc_no; + ErtsAlcStrat_t alloc_strat; int force; int ix; int ts; @@ -101,6 +103,7 @@ typedef struct { #define ERTS_DEFAULT_ALLCTR_INIT { \ NULL, \ ERTS_ALC_A_INVALID, /* (number) alloc_no: allocator number */\ + ERTS_ALC_S_INVALID, /* (number) alloc_strat: allocator strategy */\ 0, /* (bool) force: force enabled */\ 0, /* (number) ix: instance index */\ 1, /* (bool) ts: thread safe */\ @@ -138,6 +141,7 @@ typedef struct { #define ERTS_DEFAULT_ALLCTR_INIT { \ NULL, \ ERTS_ALC_A_INVALID, /* (number) alloc_no: allocator number */\ + ERTS_ALC_S_INVALID, /* (number) alloc_strat: allocator strategy */\ 0, /* (bool) force: force enabled */\ 0, /* (number) ix: instance index */\ 1, /* (bool) ts: thread safe */\ @@ -188,6 +192,7 @@ Eterm erts_alcu_info(Allctr_t *, int, int, fmtfn_t *, void *, Uint **, Uint *); void erts_alcu_init(AlcUInit_t *); void erts_alcu_current_size(Allctr_t *, AllctrSize_t *, ErtsAlcUFixInfo_t *, int); +void erts_alcu_foreign_size(Allctr_t *, ErtsAlcType_t, AllctrSize_t *); void erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, ErtsThrPrgrVal *, int *); erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); @@ -286,10 +291,18 @@ void erts_alcu_sched_spec_data_init(struct ErtsSchedulerData_ *esdp); #define UNIT_FLOOR(X) ((X) & UNIT_MASK) #define UNIT_CEILING(X) UNIT_FLOOR((X) + INV_UNIT_MASK) -#define FLG_MASK INV_UNIT_MASK -#define SBC_BLK_SZ_MASK UNIT_MASK -#define MBC_FBLK_SZ_MASK UNIT_MASK -#define CARRIER_SZ_MASK UNIT_MASK +/* We store flags in the bits that no one will ever use. Generally these are + * the bits below the alignment size, but for blocks we also steal the highest + * bit since the header's a size and no one can expect to be able to allocate + * objects that large. */ +#define HIGHEST_WORD_BIT (((UWord) 1) << (sizeof(UWord) * CHAR_BIT - 1)) + +#define BLK_FLG_MASK (INV_UNIT_MASK | HIGHEST_WORD_BIT) +#define SBC_BLK_SZ_MASK (~BLK_FLG_MASK) +#define MBC_FBLK_SZ_MASK (~BLK_FLG_MASK) + +#define CRR_FLG_MASK INV_UNIT_MASK +#define CRR_SZ_MASK UNIT_MASK #if ERTS_HAVE_MSEG_SUPER_ALIGNED \ || (!HAVE_ERTS_MSEG && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC) @@ -299,9 +312,9 @@ void erts_alcu_sched_spec_data_init(struct ErtsSchedulerData_ *esdp); # define ERTS_SUPER_ALIGN_BITS 18 # endif # ifdef ARCH_64 -# define MBC_ABLK_OFFSET_BITS 24 +# define MBC_ABLK_OFFSET_BITS 23 # else -# define MBC_ABLK_OFFSET_BITS 9 +# define MBC_ABLK_OFFSET_BITS 8 /* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */ # endif # define ERTS_SACRR_UNIT_SHIFT ERTS_SUPER_ALIGN_BITS @@ -322,18 +335,17 @@ void erts_alcu_sched_spec_data_init(struct ErtsSchedulerData_ *esdp); #if MBC_ABLK_OFFSET_BITS # define MBC_ABLK_OFFSET_SHIFT (sizeof(UWord)*8 - MBC_ABLK_OFFSET_BITS) -# define MBC_ABLK_OFFSET_MASK (~((UWord)0) << MBC_ABLK_OFFSET_SHIFT) -# define MBC_ABLK_SZ_MASK (~MBC_ABLK_OFFSET_MASK & ~FLG_MASK) +# define MBC_ABLK_OFFSET_MASK ((~((UWord)0) << MBC_ABLK_OFFSET_SHIFT) & ~BLK_FLG_MASK) +# define MBC_ABLK_SZ_MASK (~MBC_ABLK_OFFSET_MASK & ~BLK_FLG_MASK) #else -# define MBC_ABLK_SZ_MASK (~FLG_MASK) +# define MBC_ABLK_SZ_MASK (~BLK_FLG_MASK) #endif #define MBC_ABLK_SZ(B) (ASSERT(!is_sbc_blk(B)), (B)->bhdr & MBC_ABLK_SZ_MASK) #define MBC_FBLK_SZ(B) (ASSERT(!is_sbc_blk(B)), (B)->bhdr & MBC_FBLK_SZ_MASK) #define SBC_BLK_SZ(B) (ASSERT(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK) -#define CARRIER_SZ(C) \ - ((C)->chdr & CARRIER_SZ_MASK) +#define CARRIER_SZ(C) ((C)->chdr & CRR_SZ_MASK) typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t; @@ -351,12 +363,20 @@ typedef struct { #endif } Block_t; -typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t; +typedef struct ErtsAllctrDDBlock__ { + union { + struct ErtsAllctrDDBlock__ *ptr_next; + erts_atomic_t atmc_next; + } u; + ErtsAlcType_t type; + Uint32 flags; +} ErtsAllctrDDBlock_t; -union ErtsAllctrDDBlock_t_ { - erts_atomic_t atmc_next; - ErtsAllctrDDBlock_t *ptr_next; -}; +/* Deallocation was caused by shrinking a fix-list, so usage statistics has + * already been updated. */ +#define DEALLOC_FLG_FIX_SHRINK (1 << 0) +/* Deallocation was redirected to another instance. */ +#define DEALLOC_FLG_REDIRECTED (1 << 1) typedef struct { Block_t blk; @@ -365,11 +385,10 @@ typedef struct { #endif } ErtsFakeDDBlock_t; - - #define THIS_FREE_BLK_HDR_FLG (((UWord) 1) << 0) #define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1) #define LAST_BLK_HDR_FLG (((UWord) 1) << 2) +#define ATAG_BLK_HDR_FLG HIGHEST_WORD_BIT #define SBC_BLK_HDR_FLG /* Special flag combo for (allocated) SBC blocks */\ (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) @@ -381,9 +400,9 @@ typedef struct { #define HOMECOMING_MBC_BLK_HDR (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) #define IS_FREE_LAST_MBC_BLK(B) \ - (((B)->bhdr & FLG_MASK) == (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)) + (((B)->bhdr & BLK_FLG_MASK) == (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)) -#define IS_SBC_BLK(B) (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG) +#define IS_SBC_BLK(B) (((B)->bhdr & SBC_BLK_HDR_FLG) == SBC_BLK_HDR_FLG) #define IS_MBC_BLK(B) (!IS_SBC_BLK((B))) #define IS_FREE_BLK(B) (ASSERT(IS_MBC_BLK(B)), \ (B)->bhdr & THIS_FREE_BLK_HDR_FLG) @@ -394,7 +413,8 @@ typedef struct { # define ABLK_TO_MBC(B) \ (ASSERT(IS_MBC_BLK(B) && !IS_FREE_BLK(B)), \ (Carrier_t*)((ERTS_SACRR_UNIT_FLOOR((UWord)(B)) - \ - (((B)->bhdr >> MBC_ABLK_OFFSET_SHIFT) << ERTS_SACRR_UNIT_SHIFT)))) + ((((B)->bhdr & ~BLK_FLG_MASK) >> MBC_ABLK_OFFSET_SHIFT) \ + << ERTS_SACRR_UNIT_SHIFT)))) # define BLK_TO_MBC(B) (IS_FREE_BLK(B) ? FBLK_TO_MBC(B) : ABLK_TO_MBC(B)) #else # define FBLK_TO_MBC(B) ((B)->carrier) @@ -433,8 +453,9 @@ typedef struct { ErtsThrPrgrVal thr_prgr; erts_atomic_t max_size; UWord abandon_limit; - UWord blocks; - UWord blocks_size; + UWord blocks[ERTS_ALC_A_MAX + 1]; + UWord blocks_size[ERTS_ALC_A_MAX + 1]; + UWord total_blocks_size; enum { ERTS_MBC_IS_HOME, ERTS_MBC_WAS_POOLED, @@ -452,7 +473,7 @@ struct Carrier_t_ { }; #define ERTS_ALC_CARRIER_TO_ALLCTR(C) \ - ((Allctr_t *) (erts_atomic_read_nob(&(C)->allctr) & ~FLG_MASK)) + ((Allctr_t *) (erts_atomic_read_nob(&(C)->allctr) & ~CRR_FLG_MASK)) typedef struct { Carrier_t *first; @@ -530,7 +551,6 @@ typedef struct { } head; } ErtsAllctrDDQueue_t; - typedef struct { size_t type_size; SWord list_size; @@ -549,6 +569,7 @@ typedef struct { UWord used; } cpool; } u; + ErtsAlcType_t type; } ErtsAlcFixList_t; struct Allctr_t_ { @@ -569,6 +590,9 @@ struct Allctr_t_ { /* Allocator number */ ErtsAlcType_t alloc_no; + /* Allocator strategy */ + ErtsAlcStrat_t alloc_strat; + /* Instance index */ int ix; @@ -617,6 +641,9 @@ struct Allctr_t_ { AOFF_RBTree_t* pooled_tree; CarrierList_t dc_list; + /* the sentinel of the cpool we're attached to */ + ErtsAlcCPoolData_t *sentinel; + UWord abandon_limit; int disable_abandon; int check_limit_count; @@ -624,8 +651,8 @@ struct Allctr_t_ { UWord in_pool_limit; /* acnl */ UWord fblk_min_limit; /* acmfl */ struct { - erts_atomic_t blocks_size; - erts_atomic_t no_blocks; + erts_atomic_t blocks_size[ERTS_ALC_A_MAX + 1]; + erts_atomic_t no_blocks[ERTS_ALC_A_MAX + 1]; erts_atomic_t carriers_size; erts_atomic_t no_carriers; CallCounter_t fail_pooled; diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 3f0ab33597..0e3e4c890a 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -107,9 +107,11 @@ typedef struct AOFF_Carrier_t_ AOFF_Carrier_t; struct AOFF_Carrier_t_ { Carrier_t crr; - AOFF_RBTree_t rbt_node; /* My node in the carrier tree */ - AOFF_RBTree_t* root; /* Root of my block tree */ + AOFF_RBTree_t rbt_node; /* My node in the carrier tree */ + AOFF_RBTree_t* root; /* Root of my block tree */ + enum AOFFSortOrder blk_order; }; + #define RBT_NODE_TO_MBC(PTR) ErtsContainerStruct((PTR), AOFF_Carrier_t, rbt_node) /* @@ -281,15 +283,28 @@ erts_aoffalc_start(AOFFAllctr_t *alc, sys_memcpy((void *) alc, (void *) &zero.allctr, sizeof(AOFFAllctr_t)); + if (aoffinit->blk_order == FF_CHAOS) { + const enum AOFFSortOrder orders[3] = {FF_AOFF, FF_AOBF, FF_BF}; + int index = init->ix % (sizeof(orders) / sizeof(orders[0])); + + ASSERT(init->alloc_no == ERTS_ALC_A_TEST); + aoffinit->blk_order = orders[index]; + } + + if (aoffinit->crr_order == FF_CHAOS) { + const enum AOFFSortOrder orders[2] = {FF_AGEFF, FF_AOFF}; + int index = init->ix % (sizeof(orders) / sizeof(orders[0])); + + ASSERT(init->alloc_no == ERTS_ALC_A_TEST); + aoffinit->crr_order = orders[index]; + } + alc->blk_order = aoffinit->blk_order; alc->crr_order = aoffinit->crr_order; allctr->mbc_header_size = sizeof(AOFF_Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; - allctr->min_block_size = (aoffinit->blk_order == FF_BF - ? (offsetof(AOFF_RBTree_t, u.next) - + ErtsSizeofMember(AOFF_RBTree_t, u.next)) - : offsetof(AOFF_RBTree_t, u)); + allctr->min_block_size = sizeof(AOFF_RBTree_t); allctr->vsn_str = ERTS_ALC_AOFF_ALLOC_VSN_STR; @@ -512,14 +527,15 @@ tree_insert_fixup(AOFF_RBTree_t** root, AOFF_RBTree_t *blk) static void aoff_unlink_free_block(Allctr_t *allctr, Block_t *blk) { - AOFFAllctr_t* alc = (AOFFAllctr_t*)allctr; AOFF_RBTree_t* del = (AOFF_RBTree_t*)blk; AOFF_Carrier_t *crr = (AOFF_Carrier_t*) FBLK_TO_MBC(&del->hdr); + (void)allctr; + ASSERT(crr->rbt_node.hdr.bhdr == crr->root->max_sz); - HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0); + HARD_CHECK_TREE(&crr->crr, crr->blk_order, crr->root, 0); - if (alc->blk_order == FF_BF) { + if (crr->blk_order == FF_BF) { ASSERT(del->flags & IS_BF_FLG); if (IS_LIST_ELEM(del)) { /* Remove from list */ @@ -540,14 +556,14 @@ aoff_unlink_free_block(Allctr_t *allctr, Block_t *blk) replace(&crr->root, (AOFF_RBTree_t*)del, LIST_NEXT(del)); - HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0); + HARD_CHECK_TREE(&crr->crr, crr->blk_order, crr->root, 0); return; } } rbt_delete(&crr->root, (AOFF_RBTree_t*)del); - HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0); + HARD_CHECK_TREE(&crr->crr, crr->blk_order, crr->root, 0); /* Update the carrier tree with a potentially new (lower) max_sz */ @@ -737,17 +753,18 @@ rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del) static void aoff_link_free_block(Allctr_t *allctr, Block_t *block) { - AOFFAllctr_t* alc = (AOFFAllctr_t*) allctr; AOFF_RBTree_t *blk = (AOFF_RBTree_t *) block; AOFF_RBTree_t *crr_node; AOFF_Carrier_t *blk_crr = (AOFF_Carrier_t*) FBLK_TO_MBC(block); Uint blk_sz = AOFF_BLK_SZ(blk); + (void)allctr; + ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(&blk_crr->crr)); ASSERT(blk_crr->rbt_node.hdr.bhdr == (blk_crr->root ? blk_crr->root->max_sz : 0)); - HARD_CHECK_TREE(&blk_crr->crr, alc->blk_order, blk_crr->root, 0); + HARD_CHECK_TREE(&blk_crr->crr, blk_crr->blk_order, blk_crr->root, 0); - rbt_insert(alc->blk_order, &blk_crr->root, blk); + rbt_insert(blk_crr->blk_order, &blk_crr->root, blk); /* * Update carrier tree with a potentially new (larger) max_sz @@ -891,7 +908,7 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, /* Get block within carrier tree */ #ifdef HARD_DEBUG - dbg_blk = HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, size); + dbg_blk = HARD_CHECK_TREE(&crr->crr, crr->blk_order, crr->root, size); #endif blk = rbt_search(crr->root, size); @@ -904,7 +921,7 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, if (!blk) return NULL; - if (cand_blk && cmp_cand_blk(alc->blk_order, cand_blk, blk) < 0) { + if (cand_blk && cmp_cand_blk(crr->blk_order, cand_blk, blk) < 0) { return NULL; /* cand_blk was better */ } @@ -927,21 +944,28 @@ static void aoff_creating_mbc(Allctr_t *allctr, Carrier_t *carrier) AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; AOFF_RBTree_t **root = &alc->mbc_root; + Sint64 bt = get_birth_time(); HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); crr->rbt_node.hdr.bhdr = 0; - if (alc->crr_order == FF_AGEFF || IS_DEBUG) { - Sint64 bt = get_birth_time(); - crr->rbt_node.u.birth_time = bt; - crr->crr.cpool.pooled.u.birth_time = bt; - } + + /* While birth time is only used for FF_AGEFF, we have to set it for all + * types as we can be migrated to an instance that uses it and we don't + * want to mess its order up. */ + crr->rbt_node.u.birth_time = bt; + crr->crr.cpool.pooled.u.birth_time = bt; + rbt_insert(alc->crr_order, root, &crr->rbt_node); /* aoff_link_free_block will add free block later */ crr->root = NULL; HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); + + /* When a carrier has been migrated, its block order may differ from that + * of the allocator it's been migrated to. */ + crr->blk_order = alc->blk_order; } #define IS_CRR_IN_TREE(CRR,ROOT) \ diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h index 68df9e0a49..9c9b98da86 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.h +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h @@ -32,7 +32,12 @@ enum AOFFSortOrder { FF_AGEFF = 0, /* carrier trees only */ FF_AOFF = 1, FF_AOBF = 2, /* block trees only */ - FF_BF = 3 /* block trees only */ + FF_BF = 3, /* block trees only */ + + FF_CHAOS = -1 /* A test-specific sort order that picks any of the above + * after instance id. Used to test that carriers created + * under one order will work fine after being migrated + * to another. */ }; typedef struct { diff --git a/erts/emulator/internal_doc/CarrierMigration.md b/erts/emulator/internal_doc/CarrierMigration.md index 3a796d11b7..6c79bd731c 100644 --- a/erts/emulator/internal_doc/CarrierMigration.md +++ b/erts/emulator/internal_doc/CarrierMigration.md @@ -34,8 +34,7 @@ Solution -------- In order to prevent scenarios like this we've implemented support for -migration of multi-block carriers between allocator instances of the -same type. +migration of multi-block carriers between allocator instances. ### Management of Free Blocks ### @@ -130,10 +129,6 @@ threads may have references to it via the pool. ### Migration ### -There exists one pool for each allocator type enabling migration of -carriers between scheduler specific allocator instances of the same -allocator type. - Each allocator instance keeps track of the current utilization of its multi-block carriers. When the total utilization falls below the "abandon carrier utilization limit" it starts to inspect the utilization of the @@ -287,11 +282,3 @@ reduced using the `aoffcbf` strategy. A trade off between memory consumption and performance is however inevitable, and it is up to the user to decide what is most important. -Further work ------------- - -It would be quite easy to extend this to allow migration of multi-block -carriers between all allocator types. More or less the only obstacle -is maintenance of the statistics information. - - diff --git a/erts/emulator/test/alloc_SUITE.erl b/erts/emulator/test/alloc_SUITE.erl index 343afe85e6..4e0243c1cd 100644 --- a/erts/emulator/test/alloc_SUITE.erl +++ b/erts/emulator/test/alloc_SUITE.erl @@ -71,7 +71,8 @@ migration(Cfg) -> %% Disable driver_alloc to avoid recursive alloc_util calls %% through enif_mutex_create() in my_creating_mbc(). drv_case(Cfg, concurrent, "+MZe true +MRe false"), - drv_case(Cfg, concurrent, "+MZe true +MRe false +MZas ageffcbf"). + drv_case(Cfg, concurrent, "+MZe true +MRe false +MZas ageffcbf"), + drv_case(Cfg, concurrent, "+MZe true +MRe false +MZas chaosff"). erts_mmap(Config) when is_list(Config) -> case {os:type(), mmsc_flags()} of diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl index 7aff3a6ea1..bb76e6f24b 100644 --- a/erts/emulator/test/driver_SUITE.erl +++ b/erts/emulator/test/driver_SUITE.erl @@ -2643,24 +2643,7 @@ wait_deallocations() -> driver_alloc_size() -> wait_deallocations(), - case erlang:system_info({allocator_sizes, driver_alloc}) of - false -> - undefined; - MemInfo -> - CS = lists:foldl( - fun ({instance, _, L}, Acc) -> - {value,{_,MBCS}} = lists:keysearch(mbcs, 1, L), - {value,{_,SBCS}} = lists:keysearch(sbcs, 1, L), - [MBCS,SBCS | Acc] - end, - [], - MemInfo), - lists:foldl( - fun(L, Sz0) -> - {value,{_,Sz,_,_}} = lists:keysearch(blocks_size, 1, L), - Sz0+Sz - end, 0, CS) - end. + erts_debug:alloc_blocks_size(driver_alloc). rpc(Config, Fun) -> case proplists:get_value(node, Config) of diff --git a/erts/emulator/test/erts_debug_SUITE.erl b/erts/emulator/test/erts_debug_SUITE.erl index 6aa7a445b5..f39dbedd8f 100644 --- a/erts/emulator/test/erts_debug_SUITE.erl +++ b/erts/emulator/test/erts_debug_SUITE.erl @@ -22,8 +22,10 @@ -include_lib("common_test/include/ct.hrl"). -export([all/0, suite/0, - test_size/1,flat_size_big/1,df/1,term_type/1, - instructions/1, stack_check/1]). + test_size/1,flat_size_big/1,df/1,term_type/1, + instructions/1, stack_check/1, alloc_blocks_size/1]). + +-export([do_alloc_blocks_size/0]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -31,7 +33,7 @@ suite() -> all() -> [test_size, flat_size_big, df, instructions, term_type, - stack_check]. + stack_check, alloc_blocks_size]. test_size(Config) when is_list(Config) -> ConsCell1 = id([a|b]), @@ -210,5 +212,28 @@ instructions(Config) when is_list(Config) -> _ = [list_to_atom(I) || I <- Is], ok. +alloc_blocks_size(Config) when is_list(Config) -> + F = fun(Args) -> + Node = start_slave(Args), + ok = rpc:call(Node, ?MODULE, do_alloc_blocks_size, []), + true = test_server:stop_node(Node) + end, + F("+Meamax"), + F("+Meamin"), + F(""), + ok. + +do_alloc_blocks_size() -> + _ = erts_debug:alloc_blocks_size(binary_alloc), + ok. + +start_slave(Args) -> + Name = ?MODULE_STRING ++ "_slave", + Pa = filename:dirname(code:which(?MODULE)), + {ok, Node} = test_server:start_node(list_to_atom(Name), + slave, + [{args, "-pa " ++ Pa ++ " " ++ Args}]), + Node. + id(I) -> I. diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl index 57eb082d64..f4b1d885fe 100644 --- a/erts/emulator/test/process_SUITE.erl +++ b/erts/emulator/test/process_SUITE.erl @@ -2233,8 +2233,8 @@ processes_term_proc_list(Config) when is_list(Config) -> %% We have to run this test case with +S1 since instrument:allocations() %% will report a free()'d block as present until it's actually deallocated %% by its employer. - Run("+MSe true +MSatags false +S1"), - Run("+MSe true +MSatags true +S1"), + Run("+MSe true +Muatags false +S1"), + Run("+MSe true +Muatags true +S1"), ok. @@ -2242,10 +2242,12 @@ processes_term_proc_list(Config) when is_list(Config) -> chk_term_proc_list(?LINE, MC, XB)). chk_term_proc_list(Line, MustChk, ExpectBlks) -> - Allocs = instrument:allocations(#{ allocator_types => [sl_alloc] }), + Allocs = instrument:allocations(), case {MustChk, Allocs} of {false, {error, not_enabled}} -> not_enabled; + {false, {ok, {_Shift, _Unscanned, ByOrigin}}} when ByOrigin =:= #{} -> + not_enabled; {_, {ok, {_Shift, _Unscanned, ByOrigin}}} -> ByType = maps:get(system, ByOrigin, #{}), Hist = maps:get(ptab_list_deleted_el, ByType, {}), diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl index 21ab6b378a..8ea2d88ec4 100644 --- a/erts/emulator/test/system_info_SUITE.erl +++ b/erts/emulator/test/system_info_SUITE.erl @@ -457,11 +457,16 @@ cmp_memory(MWs, Str) -> %% Total, processes, processes_used, and system will seldom %% give us exactly the same result since the two readings %% aren't taken atomically. + %% + %% Torerance is scaled according to the number of schedulers + %% to match spawn_mem_workers. + + Tolerance = 1.05 + 0.01 * erlang:system_info(schedulers_online), - cmp_memory(total, EM, EDM, 1.05), - cmp_memory(processes, EM, EDM, 1.05), - cmp_memory(processes_used, EM, EDM, 1.05), - cmp_memory(system, EM, EDM, 1.05), + cmp_memory(total, EM, EDM, Tolerance), + cmp_memory(processes, EM, EDM, Tolerance), + cmp_memory(processes_used, EM, EDM, Tolerance), + cmp_memory(system, EM, EDM, Tolerance), ok. diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam Binary files differindex bd8cc7d7e0..7a4167b0e9 100644 --- a/erts/preloaded/ebin/erlang.beam +++ b/erts/preloaded/ebin/erlang.beam diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index 75f6901258..fd8a35aabf 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -3654,90 +3654,28 @@ memory() -> -spec erlang:memory(Type :: memory_type()) -> non_neg_integer(); (TypeList :: [memory_type()]) -> [{memory_type(), non_neg_integer()}]. memory(Type) when erlang:is_atom(Type) -> - {AA, ALCU, ChkSup, BadArgZero} = need_mem_info(Type), - case get_mem_data(ChkSup, ALCU, AA) of - notsup -> - erlang:error(notsup, [Type]); - Mem -> - Value = get_memval(Type, Mem), - case {BadArgZero, Value} of - {true, 0} -> erlang:error(badarg, [Type]); - _ -> Value - end + try + case aa_mem_data(au_mem_data(?ALL_NEEDED_ALLOCS)) of + notsup -> erlang:error(notsup); + Mem -> get_memval(Type, Mem) + end + catch + error:badarg -> erlang:error(badarg) end; memory(Types) when erlang:is_list(Types) -> - {AA, ALCU, ChkSup, BadArgZeroList} = need_mem_info_list(Types), - case get_mem_data(ChkSup, ALCU, AA) of - notsup -> - erlang:error(notsup, [Types]); - Mem -> - case memory_result_list(Types, BadArgZeroList, Mem) of - badarg -> erlang:error(badarg, [Types]); - Result -> Result - end - end. - -memory_result_list([], [], _Mem) -> - []; -memory_result_list([T|Ts], [BAZ|BAZs], Mem) -> - case memory_result_list(Ts, BAZs, Mem) of - badarg -> badarg; - TVs -> - V = get_memval(T, Mem), - case {BAZ, V} of - {true, 0} -> badarg; - _ -> [{T, V}| TVs] - end - end. - -get_mem_data(true, AlcUAllocs, NeedAllocatedAreas) -> - case memory_is_supported() of - false -> notsup; - true -> get_mem_data(false, AlcUAllocs, NeedAllocatedAreas) - end; -get_mem_data(false, AlcUAllocs, NeedAllocatedAreas) -> - AlcUMem = case AlcUAllocs of - [] -> #memory{}; - _ -> - au_mem_data(AlcUAllocs) - end, - case NeedAllocatedAreas of - true -> aa_mem_data(AlcUMem); - false -> AlcUMem + try + case aa_mem_data(au_mem_data(?ALL_NEEDED_ALLOCS)) of + notsup -> erlang:error(notsup); + Mem -> memory_1(Types, Mem) + end + catch + error:badarg -> erlang:error(badarg) end. -need_mem_info_list([]) -> - {false, [], false, []}; -need_mem_info_list([T|Ts]) -> - {MAA, MALCU, MChkSup, MBadArgZero} = need_mem_info_list(Ts), - {AA, ALCU, ChkSup, BadArgZero} = need_mem_info(T), - {case AA of - true -> true; - _ -> MAA - end, - ALCU ++ (MALCU -- ALCU), - case ChkSup of - true -> true; - _ -> MChkSup - end, - [BadArgZero|MBadArgZero]}. - -need_mem_info(Type) when Type == total; - Type == system -> - {true, ?ALL_NEEDED_ALLOCS, false, false}; -need_mem_info(Type) when Type == processes; - Type == processes_used -> - {true, [eheap_alloc, fix_alloc], true, false}; -need_mem_info(Type) when Type == atom; - Type == atom_used; - Type == code -> - {true, [], true, false}; -need_mem_info(binary) -> - {false, [binary_alloc], true, false}; -need_mem_info(ets) -> - {true, [ets_alloc], true, false}; -need_mem_info(_) -> - {false, [], false, true}. +memory_1([Type | Types], Mem) -> + [{Type, get_memval(Type, Mem)} | memory_1(Types, Mem)]; +memory_1([], _Mem) -> + []. get_memval(total, #memory{total = V}) -> V; get_memval(processes, #memory{processes = V}) -> V; @@ -3748,16 +3686,7 @@ get_memval(atom_used, #memory{atom_used = V}) -> V; get_memval(binary, #memory{binary = V}) -> V; get_memval(code, #memory{code = V}) -> V; get_memval(ets, #memory{ets = V}) -> V; -get_memval(_, #memory{}) -> 0. - -memory_is_supported() -> - {_, _, FeatureList, _} = erlang:system_info(allocator), - case ((erlang:system_info(alloc_util_allocators) - -- ?CARRIER_ALLOCS) - -- FeatureList) of - [] -> true; - _ -> false - end. +get_memval(_, #memory{}) -> erlang:error(badarg). get_blocks_size([{blocks_size, Sz, _, _} | Rest], Acc) -> get_blocks_size(Rest, Acc+Sz); @@ -3768,16 +3697,6 @@ get_blocks_size([_ | Rest], Acc) -> get_blocks_size([], Acc) -> Acc. - -blocks_size([{Carriers, SizeList} | Rest], Acc) when Carriers == mbcs; - Carriers == mbcs_pool; - Carriers == sbcs -> - blocks_size(Rest, get_blocks_size(SizeList, Acc)); -blocks_size([_ | Rest], Acc) -> - blocks_size(Rest, Acc); -blocks_size([], Acc) -> - Acc. - get_fix_proc([{ProcType, A1, U1}| Rest], {A0, U0}) when ProcType == proc; ProcType == monitor; ProcType == link; @@ -3802,64 +3721,78 @@ fix_proc([_ | Rest], Acc) -> fix_proc([], Acc) -> Acc. +au_mem_fix(#memory{ processes = Proc, + processes_used = ProcU, + system = Sys } = Mem, Data) -> + case fix_proc(Data, {0, 0}) of + {A, U} -> + Mem#memory{ processes = Proc+A, + processes_used = ProcU+U, + system = Sys-A }; + {Mask, A, U} -> + Mem#memory{ processes = Mask band (Proc+A), + processes_used = Mask band (ProcU+U), + system = Mask band (Sys-A) } + end. + +au_mem_acc(#memory{ total = Tot, + processes = Proc, + processes_used = ProcU } = Mem, + eheap_alloc, Data) -> + Sz = get_blocks_size(Data, 0), + Mem#memory{ total = Tot+Sz, + processes = Proc+Sz, + processes_used = ProcU+Sz}; +au_mem_acc(#memory{ total = Tot, + system = Sys, + ets = Ets } = Mem, ets_alloc, Data) -> + Sz = get_blocks_size(Data, 0), + Mem#memory{ total = Tot+Sz, + system = Sys+Sz, + ets = Ets+Sz }; +au_mem_acc(#memory{total = Tot, + system = Sys, + binary = Bin } = Mem, + binary_alloc, Data) -> + Sz = get_blocks_size(Data, 0), + Mem#memory{ total = Tot+Sz, + system = Sys+Sz, + binary = Bin+Sz}; +au_mem_acc(#memory{ total = Tot, + system = Sys } = Mem, + _Type, Data) -> + Sz = get_blocks_size(Data, 0), + Mem#memory{ total = Tot+Sz, + system = Sys+Sz }. + +au_mem_foreign(Mem, [{Type, SizeList} | Rest]) -> + au_mem_foreign(au_mem_acc(Mem, Type, SizeList), Rest); +au_mem_foreign(Mem, []) -> + Mem. + +au_mem_current(Mem0, Type, [{mbcs_pool, MBCS} | Rest]) -> + [Foreign] = [Foreign || {foreign_blocks, Foreign} <- MBCS], + SizeList = MBCS -- [Foreign], + Mem = au_mem_foreign(Mem0, Foreign), + au_mem_current(au_mem_acc(Mem, Type, SizeList), Type, Rest); +au_mem_current(Mem, Type, [{mbcs, SizeList} | Rest]) -> + au_mem_current(au_mem_acc(Mem, Type, SizeList), Type, Rest); +au_mem_current(Mem, Type, [{sbcs, SizeList} | Rest]) -> + au_mem_current(au_mem_acc(Mem, Type, SizeList), Type, Rest); +au_mem_current(Mem, Type, [_ | Rest]) -> + au_mem_current(Mem, Type, Rest); +au_mem_current(Mem, _Type, []) -> + Mem. + au_mem_data(notsup, _) -> notsup; au_mem_data(_, [{_, false} | _]) -> notsup; -au_mem_data(#memory{total = Tot, - processes = Proc, - processes_used = ProcU} = Mem, - [{eheap_alloc, _, Data} | Rest]) -> - Sz = blocks_size(Data, 0), - au_mem_data(Mem#memory{total = Tot+Sz, - processes = Proc+Sz, - processes_used = ProcU+Sz}, - Rest); -au_mem_data(#memory{total = Tot, - system = Sys, - ets = Ets} = Mem, - [{ets_alloc, _, Data} | Rest]) -> - Sz = blocks_size(Data, 0), - au_mem_data(Mem#memory{total = Tot+Sz, - system = Sys+Sz, - ets = Ets+Sz}, - Rest); -au_mem_data(#memory{total = Tot, - system = Sys, - binary = Bin} = Mem, - [{binary_alloc, _, Data} | Rest]) -> - Sz = blocks_size(Data, 0), - au_mem_data(Mem#memory{total = Tot+Sz, - system = Sys+Sz, - binary = Bin+Sz}, - Rest); -au_mem_data(#memory{total = Tot, - processes = Proc, - processes_used = ProcU, - system = Sys} = Mem, - [{fix_alloc, _, Data} | Rest]) -> - Sz = blocks_size(Data, 0), - case fix_proc(Data, {0, 0}) of - {A, U} -> - au_mem_data(Mem#memory{total = Tot+Sz, - processes = Proc+A, - processes_used = ProcU+U, - system = Sys+Sz-A}, - Rest); - {Mask, A, U} -> - au_mem_data(Mem#memory{total = Tot+Sz, - processes = Mask band (Proc+A), - processes_used = Mask band (ProcU+U), - system = Mask band (Sys+Sz-A)}, - Rest) - end; -au_mem_data(#memory{total = Tot, - system = Sys} = Mem, - [{_, _, Data} | Rest]) -> - Sz = blocks_size(Data, 0), - au_mem_data(Mem#memory{total = Tot+Sz, - system = Sys+Sz}, - Rest); +au_mem_data(#memory{} = Mem0, [{fix_alloc, _, Data} | Rest]) -> + Mem = au_mem_fix(Mem0, Data), + au_mem_data(au_mem_current(Mem, fix_alloc, Data), Rest); +au_mem_data(#memory{} = Mem, [{Type, _, Data} | Rest]) -> + au_mem_data(au_mem_current(Mem, Type, Data), Rest); au_mem_data(EMD, []) -> EMD. diff --git a/lib/kernel/src/erts_debug.erl b/lib/kernel/src/erts_debug.erl index 1270de4144..c4d276f9e8 100644 --- a/lib/kernel/src/erts_debug.erl +++ b/lib/kernel/src/erts_debug.erl @@ -36,7 +36,8 @@ map_info/1, same/2, set_internal_state/2, size_shared/1, copy_shared/1, dirty_cpu/2, dirty_io/2, dirty/3, lcnt_control/1, lcnt_control/2, lcnt_collect/0, lcnt_clear/0, - lc_graph/0, lc_graph_to_dot/2, lc_graph_merge/2]). + lc_graph/0, lc_graph_to_dot/2, lc_graph_merge/2, + alloc_blocks_size/1]). -spec breakpoint(MFA, Flag) -> non_neg_integer() when MFA :: {Module :: module(), @@ -495,3 +496,58 @@ lcg_print_locks(Out, [LastLock]) -> lcg_print_locks(Out, [Lock | Rest]) -> io:format(Out, "~w,\n", [Lock]), lcg_print_locks(Out, Rest). + + +%% Returns the amount of memory allocated by the given allocator type. +-spec alloc_blocks_size(Type) -> non_neg_integer() | undefined when + Type :: atom(). + +alloc_blocks_size(Type) -> + Allocs = erlang:system_info(alloc_util_allocators), + Sizes = erlang:system_info({allocator_sizes, Allocs}), + alloc_blocks_size_1(Sizes, Type, 0). + +alloc_blocks_size_1([], _Type, 0) -> + undefined; +alloc_blocks_size_1([{_Type, false} | Rest], Type, Acc) -> + alloc_blocks_size_1(Rest, Type, Acc); +alloc_blocks_size_1([{Type, Instances} | Rest], Type, Acc0) -> + F = fun ({instance, _, L}, Acc) -> + MBCSPool = case lists:keyfind(mbcs_pool, 1, L) of + {_, Pool} -> Pool; + false -> [] + end, + {_,MBCS} = lists:keyfind(mbcs, 1, L), + {_,SBCS} = lists:keyfind(sbcs, 1, L), + Acc + + sum_block_sizes(MBCSPool) + + sum_block_sizes(MBCS) + + sum_block_sizes(SBCS) + end, + alloc_blocks_size_1(Rest, Type, lists:foldl(F, Acc0, Instances)); +alloc_blocks_size_1([{_Type, Instances} | Rest], Type, Acc0) -> + F = fun ({instance, _, L}, Acc) -> + Acc + sum_foreign_sizes(Type, L) + end, + alloc_blocks_size_1(Rest, Type, lists:foldl(F, Acc0, Instances)); +alloc_blocks_size_1([], _Type, Acc) -> + Acc. + +sum_foreign_sizes(Type, L) -> + case lists:keyfind(mbcs_pool, 1, L) of + {_,Pool} -> + {_,ForeignBlocks} = lists:keyfind(foreign_blocks, 1, Pool), + case lists:keyfind(Type, 1, ForeignBlocks) of + {_,TypeSizes} -> sum_block_sizes(TypeSizes); + false -> 0 + end; + _ -> + 0 + end. + +sum_block_sizes(Blocks) -> + lists:foldl( + fun({blocks_size, Sz,_,_}, Sz0) -> Sz0+Sz; + ({blocks_size, Sz}, Sz0) -> Sz0+Sz; + (_, Sz) -> Sz + end, 0, Blocks). diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 7a48d1d55e..fee8b204f4 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -6112,40 +6112,11 @@ etsmem() -> ets:info(T,memory),ets:info(T,type)} end, ets:all()), - EtsAllocInfo = erlang:system_info({allocator,ets_alloc}), + EtsAllocSize = erts_debug:alloc_blocks_size(ets_alloc), ErlangMemoryEts = try erlang:memory(ets) catch error:notsup -> notsup end, - Mem = - {ErlangMemoryEts, - case EtsAllocInfo of - false -> undefined; - MemInfo -> - CS = lists:foldl( - fun ({instance, _, L}, Acc) -> - {value,{mbcs,MBCS}} = lists:keysearch(mbcs, 1, L), - {value,{sbcs,SBCS}} = lists:keysearch(sbcs, 1, L), - NewAcc = [MBCS, SBCS | Acc], - case lists:keysearch(mbcs_pool, 1, L) of - {value,{mbcs_pool, MBCS_POOL}} -> - [MBCS_POOL|NewAcc]; - _ -> NewAcc - end - end, - [], - MemInfo), - lists:foldl( - fun(L, {Bl0,BlSz0}) -> - {value,BlTup} = lists:keysearch(blocks, 1, L), - blocks = element(1, BlTup), - Bl = element(2, BlTup), - {value,BlSzTup} = lists:keysearch(blocks_size, 1, L), - blocks_size = element(1, BlSzTup), - BlSz = element(2, BlSzTup), - {Bl0+Bl,BlSz0+BlSz} - end, {0,0}, CS) - end}, - {Mem,AllTabs}. - + Mem = {ErlangMemoryEts, EtsAllocSize}, + {Mem, AllTabs}. verify_etsmem(MI) -> wait_for_test_procs(), diff --git a/lib/tools/doc/src/instrument.xml b/lib/tools/doc/src/instrument.xml index 9fd9332373..79bacb2927 100644 --- a/lib/tools/doc/src/instrument.xml +++ b/lib/tools/doc/src/instrument.xml @@ -111,15 +111,18 @@ default, but this can be configured an a per-allocator basis with the <seealso marker="erts:erts_alloc#M_atags"><c>+M<S>atags</c> </seealso> emulator option.</p> - <p>If tagged allocations are not enabled on any of the specified - allocator types, the call will fail with - <c>{error, not_enabled}</c>.</p> + <p>If the specified allocator types are not enabled, the call will fail + with <c>{error, not_enabled}</c>.</p> <p>The following options can be used:</p> <taglist> <tag><c>allocator_types</c></tag> <item> - <p>The allocator types that will be searched. Defaults to all - <c>alloc_util</c> allocators.</p> + <p>The allocator types that will be searched. Note that blocks can + move freely between allocator types, so restricting the search to + certain allocators may return unexpected types (e.g. process + heaps when searching binary_alloc), or hide blocks that were + migrated out.</p> + <p>Defaults to all <c>alloc_util</c> allocators.</p> </item> <tag><c>scheduler_ids</c></tag> <item> diff --git a/lib/tools/test/instrument_SUITE.erl b/lib/tools/test/instrument_SUITE.erl index 8c521b2e1a..33800c5934 100644 --- a/lib/tools/test/instrument_SUITE.erl +++ b/lib/tools/test/instrument_SUITE.erl @@ -77,6 +77,8 @@ allocations_ramv(Config) when is_list(Config) -> verify_allocations_disabled(_AllocType, Result) -> verify_allocations_disabled(Result). +verify_allocations_disabled({ok, {_HistStart, _UnscannedBytes, Allocs}}) -> + true = Allocs =:= #{}; verify_allocations_disabled({error, not_enabled}) -> ok. @@ -91,6 +93,13 @@ verify_allocations_enabled(_AllocType, Result) -> verify_allocations_enabled({ok, {_HistStart, _UnscannedBytes, Allocs}}) -> true = Allocs =/= #{}. +verify_allocations_output(#{}, {ok, {_, _, Allocs}}) when Allocs =:= #{} -> + %% This happens when the allocator is enabled but tagging is disabled. If + %% there's an error that causes Allocs to always be empty when enabled it + %% will be caught by verify_allocations_enabled. + ok; +verify_allocations_output(#{}, {error, not_enabled}) -> + ok; verify_allocations_output(#{ histogram_start := HistStart, histogram_width := HistWidth }, {ok, {HistStart, _UnscannedBytes, ByOrigin}}) -> @@ -124,8 +133,6 @@ verify_allocations_output(#{ histogram_start := HistStart, [BlockCount, GenTotalBlockCount]) end, - ok; -verify_allocations_output(#{}, {error, not_enabled}) -> ok. %% %% %% %% %% %% @@ -214,7 +221,8 @@ verify_carriers_output(#{ histogram_start := HistStart, ct:fail("Carrier count is ~p, expected at least ~p (SBC).", [CarrierCount, GenSBCCount]); CarrierCount >= GenSBCCount -> - ok + ct:pal("Found ~p blocks, required at least ~p (SBC)." , + [CarrierCount, GenSBCCount]) end, ok; @@ -292,9 +300,19 @@ start_slave(Args) -> MicroSecs = erlang:monotonic_time(), Name = "instr" ++ integer_to_list(MicroSecs), Pa = filename:dirname(code:which(?MODULE)), - {ok, Node} = test_server:start_node(list_to_atom(Name), - slave, - [{args, "-pa " ++ Pa ++ " " ++ Args}]), + + %% We pass arguments through ZFLAGS as the nightly tests rotate + %% +Meamax/+Meamin which breaks the _enabled and _disabled tests unless + %% overridden. + ZFlags = os:getenv("ERL_ZFLAGS", ""), + {ok, Node} = try + os:putenv("ERL_ZFLAGS", ZFlags ++ [" " | Args]), + test_server:start_node(list_to_atom(Name), + slave, + [{args, "-pa " ++ Pa}]) + after + os:putenv("ERL_ZFLAGS", ZFlags) + end, Node. generate_test_blocks() -> |