From a67e91e658bdbba24fcc3c79b06fdf10ff830bc9 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 15 Sep 2010 22:14:51 +0200 Subject: Optimize memory allocation A number of memory allocation optimizations have been implemented. Most optimizations reduce contention caused by synchronization between threads during allocation and deallocation of memory. Most notably: * Synchronization of memory management in scheduler specific allocator instances has been rewritten to use lock-free synchronization. * Synchronization of memory management in scheduler specific pre-allocators has been rewritten to use lock-free synchronization. * The 'mseg_alloc' memory segment allocator now use scheduler specific instances instead of one instance. Apart from reducing contention this also ensures that memory allocators always create memory segments on the local NUMA node on a NUMA system. --- erts/emulator/beam/atom.names | 3 + erts/emulator/beam/bif.tab | 4 - erts/emulator/beam/erl_afit_alloc.c | 18 +- erts/emulator/beam/erl_alloc.c | 1282 +++++++++++++------------ erts/emulator/beam/erl_alloc.h | 186 +--- erts/emulator/beam/erl_alloc.types | 26 +- erts/emulator/beam/erl_alloc_util.c | 1017 ++++++++++++++++---- erts/emulator/beam/erl_alloc_util.h | 109 ++- erts/emulator/beam/erl_ao_firstfit_alloc.c | 16 +- erts/emulator/beam/erl_bestfit_alloc.c | 16 +- erts/emulator/beam/erl_bif_info.c | 81 +- erts/emulator/beam/erl_goodfit_alloc.c | 18 +- erts/emulator/beam/erl_init.c | 15 +- erts/emulator/beam/erl_lock_check.c | 5 +- erts/emulator/beam/erl_mtrace.c | 8 +- erts/emulator/beam/erl_process.c | 1157 ++++++++++++++++------ erts/emulator/beam/erl_process.h | 60 +- erts/emulator/beam/erl_process_lock.h | 17 +- erts/emulator/beam/erl_sched_spec_pre_alloc.c | 305 ++++++ erts/emulator/beam/erl_sched_spec_pre_alloc.h | 239 +++++ erts/emulator/beam/erl_thr_progress.c | 1010 +++++++++++++++++++ erts/emulator/beam/erl_thr_progress.h | 210 ++++ erts/emulator/beam/erl_threads.h | 2 + erts/emulator/beam/fix_alloc.c | 287 ------ erts/emulator/beam/global.h | 2 +- erts/emulator/beam/sys.h | 13 +- erts/emulator/beam/time.c | 2 +- erts/emulator/beam/utils.c | 16 +- 28 files changed, 4541 insertions(+), 1583 deletions(-) create mode 100644 erts/emulator/beam/erl_sched_spec_pre_alloc.c create mode 100644 erts/emulator/beam/erl_sched_spec_pre_alloc.h create mode 100644 erts/emulator/beam/erl_thr_progress.c create mode 100644 erts/emulator/beam/erl_thr_progress.h delete mode 100644 erts/emulator/beam/fix_alloc.c (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 68d64fb7b0..37738faae3 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -69,6 +69,8 @@ atom ac atom active atom all atom all_but_first +atom alloc_info +atom alloc_sizes atom allocated atom allocated_areas atom allocator @@ -553,5 +555,6 @@ atom warning_msg atom wordsize atom write_concurrency atom xor +atom x86 atom yes atom yield diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index ba30fa85b8..987008c937 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -160,10 +160,6 @@ bif erlang:md5_update/2 bif 'erl.util.crypt.md5':update/2 ebif_md5_update_2 bif erlang:md5_final/1 bif 'erl.util.crypt.md5':final/1 ebif_md5_final_1 -bif erlang:memory/0 -bif 'erl.lang':memory/0 ebif_memory_0 -bif erlang:memory/1 -bif 'erl.lang':memory/1 ebif_memory_1 bif erlang:module_loaded/1 bif 'erl.system.code':is_loaded/1 ebif_is_loaded_1 module_loaded_1 bif erlang:function_exported/3 diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c index bcc7ea04ae..570cc59be2 100644 --- a/erts/emulator/beam/erl_afit_alloc.c +++ b/erts/emulator/beam/erl_afit_alloc.c @@ -65,16 +65,20 @@ erts_afalc_start(AFAllctr_t *afallctr, AFAllctrInit_t *afinit, AllctrInit_t *init) { - AFAllctr_t nulled_state = {{0}}; - /* {{0}} is used instead of {0}, in order to avoid (an incorrect) gcc - warning. gcc warns if {0} is used as initializer of a struct when - the first member is a struct (not if, for example, the third member - is a struct). */ + struct { + int dummy; + AFAllctr_t allctr; + } zero = {0}; + /* The struct with a dummy element first is used in order to avoid (an + incorrect) gcc warning. gcc warns if {0} is used as initializer of + a struct when the first member is a struct (not if, for example, + the third member is a struct). */ + Allctr_t *allctr = (Allctr_t *) afallctr; - init->sbmbct = 0; /* Small mbc not supported by afit */ + sys_memcpy((void *) afallctr, (void *) &zero.allctr, sizeof(AFAllctr_t)); - sys_memcpy((void *) afallctr, (void *) &nulled_state, sizeof(AFAllctr_t)); + init->sbmbct = 0; /* Small mbc not supported by afit */ allctr->mbc_header_size = sizeof(Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 9af80dd7a9..80951c9b50 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -40,6 +40,7 @@ #include "erl_mseg.h" #include "erl_monitors.h" #include "erl_bif_timer.h" +#include "erl_cpu_topology.h" #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE) #include "erl_check_io.h" #endif @@ -54,7 +55,14 @@ #include "erl_ao_firstfit_alloc.h" -#define ERTS_ALC_DEFAULT_MAX_THR_PREF 16 +#if ERTS_MAX_NO_OF_SCHEDULERS > ERTS_AU_MAX_PREF_ALLOC_INSTANCES +# error "Too many schedulers; cannot create that many pref alloc instances" +#endif + +#define ERTS_ALC_FIX_TYPE_IX(T) \ + (ERTS_ALC_T2N((T)) - ERTS_ALC_N_MIN_A_FIXED_SIZE) + +#define ERTS_ALC_DEFAULT_MAX_THR_PREF ERTS_MAX_NO_OF_SCHEDULERS #if defined(SMALL_MEMORY) || defined(PURIFY) || defined(VALGRIND) #define AU_ALLOC_DEFAULT_ENABLE(X) 0 @@ -106,24 +114,43 @@ static ErtsAllocatorState_t eheap_alloc_state; static ErtsAllocatorState_t binary_alloc_state; static ErtsAllocatorState_t ets_alloc_state; static ErtsAllocatorState_t driver_alloc_state; +static ErtsAllocatorState_t fix_alloc_state; -ErtsAlcType_t erts_fix_core_allocator_ix; -#ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE -static void *(*fix_core_allocator)(ErtsAlcType_t, void *, Uint); -static void *fix_core_extra; -static void *fix_core_alloc(Uint size) +typedef struct { + erts_smp_atomic32_t refc; + int only_sz; + Uint req_sched; + Process *proc; + Eterm ref; + Eterm ref_heap[REF_THING_SIZE]; + int allocs[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+1+2]; +} ErtsAllocInfoReq; + +#define ERTS_ALC_INFO_A_ALLOC_UTIL (ERTS_ALC_A_MAX + 1) +#define ERTS_ALC_INFO_A_MSEG_ALLOC (ERTS_ALC_A_MAX + 2) +#define ERTS_ALC_INFO_A_MAX ERTS_ALC_INFO_A_MSEG_ALLOC + +#if !HALFWORD_HEAP +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(aireq, + ErtsAllocInfoReq, + 5, + ERTS_ALC_T_AINFO_REQ) +#else +static ERTS_INLINE ErtsAllocInfoReq * +aireq_alloc(void) { - void *res; - res = (*fix_core_allocator)(ERTS_ALC_T_UNDEF, fix_core_extra, size); - if (erts_mtrace_enabled) - erts_mtrace_crr_alloc(res, - ERTS_ALC_A_FIXED_SIZE, - erts_fix_core_allocator_ix, - size); - return res; + return erts_alloc(ERTS_ALC_T_AINFO_REQ, sizeof(ErtsAllocInfoReq)); +} + +static ERTS_INLINE void +aireq_free(ErtsAllocInfoReq *ptr) +{ + erts_free(ERTS_ALC_T_AINFO_REQ, ptr); } #endif +ErtsAlcType_t erts_fix_core_allocator_ix; + enum allctr_type { GOODFIT, BESTFIT, @@ -181,6 +208,7 @@ typedef struct { struct au_init binary_alloc; struct au_init ets_alloc; struct au_init driver_alloc; + struct au_init fix_alloc; #if HALFWORD_HEAP struct au_init sbmbc_low_alloc; struct au_init std_low_alloc; @@ -393,46 +421,52 @@ set_default_driver_alloc_opts(struct au_init *ip) ip->init.util.ts = ERTS_ALC_MTA_DRIVER; } +static void +set_default_fix_alloc_opts(struct au_init *ip, + size_t *fix_type_sizes) +{ + SET_DEFAULT_ALLOC_OPTS(ip); + ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); + ip->thr_spec = 1; + ip->atype = BESTFIT; + ip->init.bf.ao = 1; + ip->init.util.name_prefix = "fix_"; + ip->init.util.fix_type_size = fix_type_sizes; + ip->init.util.alloc_no = ERTS_ALC_A_FIXED_SIZE; +#ifndef SMALL_MEMORY + ip->init.util.mmbcs = 128*1024; /* Main carrier size */ +#else + ip->init.util.mmbcs = 128*1024; /* Main carrier size */ +#endif + ip->init.util.ts = ERTS_ALC_MTA_FIXED_SIZE; +} + #ifdef ERTS_SMP static void adjust_tpref(struct au_init *ip, int no_sched) { if (ip->thr_spec) { - Uint allocs; - if (ip->thr_spec < 0) {/* User specified amount */ - allocs = abs(ip->thr_spec); - if (allocs > no_sched) - allocs = no_sched; - } - else if (no_sched > ERTS_ALC_DEFAULT_MAX_THR_PREF) - allocs = ERTS_ALC_DEFAULT_MAX_THR_PREF; - else - allocs = no_sched; - if (allocs <= 1) - ip->thr_spec = 0; - else { - ip->thr_spec = (int) allocs; - ip->thr_spec *= -1; /* thread preferred */ - - /* If default ... */ - - /* ... shrink main multi-block carrier size */ - if (ip->default_.mmbcs) - ip->init.util.mmbcs /= ERTS_MIN(4, allocs); - /* ... shrink largest multi-block carrier size */ - if (ip->default_.lmbcs) - ip->init.util.lmbcs /= ERTS_MIN(2, allocs); - /* ... shrink smallest multi-block carrier size */ - if (ip->default_.smbcs) - ip->init.util.smbcs /= ERTS_MIN(4, allocs); - /* ... and more than three allocators shrink - max mseg multi-block carriers */ - if (ip->default_.mmmbc && allocs > 2) { - ip->init.util.mmmbc /= ERTS_MIN(4, allocs - 1); - if (ip->init.util.mmmbc < 3) - ip->init.util.mmmbc = 3; - } + ip->thr_spec = no_sched; + ip->thr_spec *= -1; /* thread preferred */ + + /* If default ... */ + + /* ... shrink main multi-block carrier size */ + if (ip->default_.mmbcs) + ip->init.util.mmbcs /= ERTS_MIN(4, no_sched); + /* ... shrink largest multi-block carrier size */ + if (ip->default_.lmbcs) + ip->init.util.lmbcs /= ERTS_MIN(2, no_sched); + /* ... shrink smallest multi-block carrier size */ + if (ip->default_.smbcs) + ip->init.util.smbcs /= ERTS_MIN(4, no_sched); + /* ... and more than three allocators shrink + max mseg multi-block carriers */ + if (ip->default_.mmmbc && no_sched > 2) { + ip->init.util.mmmbc /= ERTS_MIN(4, no_sched - 1); + if (ip->init.util.mmmbc < 3) + ip->init.util.mmmbc = 3; } } } @@ -442,7 +476,7 @@ adjust_tpref(struct au_init *ip, int no_sched) static void handle_args(int *, char **, erts_alc_hndl_args_init_t *); static void -set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init); +set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu); static void start_au_allocator(ErtsAlcType_t alctr_n, @@ -456,8 +490,6 @@ refuse_af_strategy(struct au_init *init) init->atype = GOODFIT; } -static void init_thr_ix(int static_ixs); - #ifdef HARD_DEBUG static void hdbg_init(void); #endif @@ -466,7 +498,7 @@ void erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) { UWord extra_block_size = 0; - int i; + int i, ncpu; erts_alc_hndl_args_init_t init = { 0, #if HAVE_ERTS_MSEG @@ -474,17 +506,34 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) #endif ERTS_DEFAULT_TRIM_THRESHOLD, ERTS_DEFAULT_TOP_PAD, - ERTS_DEFAULT_ALCU_INIT + ERTS_DEFAULT_ALCU_INIT, }; + size_t fix_type_sizes[ERTS_ALC_NO_FIXED_SIZES] = {0}; + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_PROC)] + = sizeof(Process); +#if !HALFWORD_HEAP + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MONITOR_SH)] + = ERTS_MONITOR_SH_SIZE; + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NLINK_SH)] + = ERTS_LINK_SH_SIZE; +#endif + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_EV_D_STATE)] + = sizeof(ErtsDrvEventDataState); + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)] + = sizeof(ErtsDrvSelectDataState); + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MSG_REF)] + = sizeof(ErlMessage); #ifdef HARD_DEBUG hdbg_init(); #endif erts_have_sbmbc_alloc = 0; + ncpu = eaiop->ncpu; + if (ncpu < 1) + ncpu = 1; erts_sys_alloc_init(); - init_thr_ix(erts_no_schedulers); erts_init_utils_mem(); set_default_sbmbc_alloc_opts(&init.sbmbc_alloc); @@ -496,20 +545,23 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) set_default_binary_alloc_opts(&init.binary_alloc); set_default_ets_alloc_opts(&init.ets_alloc); set_default_driver_alloc_opts(&init.driver_alloc); + set_default_fix_alloc_opts(&init.fix_alloc, + fix_type_sizes); if (argc && argv) handle_args(argc, argv, &init); - if (erts_no_schedulers <= 1) { - init.sbmbc_alloc.thr_spec = 0; - init.sl_alloc.thr_spec = 0; - init.std_alloc.thr_spec = 0; - init.ll_alloc.thr_spec = 0; - init.eheap_alloc.thr_spec = 0; - init.binary_alloc.thr_spec = 0; - init.ets_alloc.thr_spec = 0; - init.driver_alloc.thr_spec = 0; - } +#ifndef ERTS_SMP + init.sbmbc_alloc.thr_spec = 0; + init.sl_alloc.thr_spec = 0; + init.std_alloc.thr_spec = 0; + init.ll_alloc.thr_spec = 0; + init.eheap_alloc.thr_spec = 0; + init.binary_alloc.thr_spec = 0; + init.ets_alloc.thr_spec = 0; + init.driver_alloc.thr_spec = 0; + init.fix_alloc.thr_spec = 0; +#endif if (init.erts_alloc_config) { /* Adjust flags that erts_alloc_config won't like */ @@ -522,6 +574,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.binary_alloc.thr_spec = 0; init.ets_alloc.thr_spec = 0; init.driver_alloc.thr_spec = 0; + init.fix_alloc.thr_spec = 0; } #ifdef ERTS_SMP @@ -538,6 +591,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) adjust_tpref(&init.binary_alloc, erts_no_schedulers); adjust_tpref(&init.ets_alloc, erts_no_schedulers); adjust_tpref(&init.driver_alloc, erts_no_schedulers); + adjust_tpref(&init.fix_alloc, erts_no_schedulers); #else /* No thread specific if not smp */ @@ -556,6 +610,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) refuse_af_strategy(&init.binary_alloc); refuse_af_strategy(&init.ets_alloc); refuse_af_strategy(&init.driver_alloc); + refuse_af_strategy(&init.fix_alloc); #ifdef ERTS_SMP if (!init.temp_alloc.thr_spec) @@ -564,6 +619,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_mtrace_pre_init(); #if HAVE_ERTS_MSEG + init.mseg.nos = erts_no_schedulers; erts_mseg_init(&init.mseg); #endif erts_alcu_init(&init.alloc_util); @@ -583,20 +639,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_allctrs_info[i].extra = NULL; } -#ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE -#if !defined(PURIFY) && !defined(VALGRIND) - erts_allctrs[ERTS_ALC_A_FIXED_SIZE].alloc = erts_fix_alloc; - erts_allctrs[ERTS_ALC_A_FIXED_SIZE].realloc = erts_fix_realloc; - erts_allctrs[ERTS_ALC_A_FIXED_SIZE].free = erts_fix_free; - erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].enabled = 1; -#else - erts_allctrs[ERTS_ALC_A_FIXED_SIZE].alloc = erts_sys_alloc; - erts_allctrs[ERTS_ALC_A_FIXED_SIZE].realloc = erts_sys_realloc; - erts_allctrs[ERTS_ALC_A_FIXED_SIZE].free = erts_sys_free; - erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].enabled = 0; -#endif -#endif - erts_allctrs[ERTS_ALC_A_SYSTEM].alloc = erts_sys_alloc; erts_allctrs[ERTS_ALC_A_SYSTEM].realloc = erts_sys_realloc; erts_allctrs[ERTS_ALC_A_SYSTEM].free = erts_sys_free; @@ -621,20 +663,21 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.ll_low_alloc.init.util.force = 1; init.ll_low_alloc.init.util.low_mem = 1; - set_au_allocator(ERTS_ALC_A_SBMBC_LOW, &init.sbmbc_low_alloc); - set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc); - set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc); + set_au_allocator(ERTS_ALC_A_SBMBC_LOW, &init.sbmbc_low_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc, ncpu); #endif /* HALFWORD */ - set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc); - set_au_allocator(ERTS_ALC_A_SBMBC, &init.sbmbc_alloc); - set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc); - set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc); - set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc); - set_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc); - set_au_allocator(ERTS_ALC_A_BINARY, &init.binary_alloc); - set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc); - set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc); + set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_SBMBC, &init.sbmbc_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_BINARY, &init.binary_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu); for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { if (!erts_allctrs[i].alloc) @@ -650,10 +693,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) sys_alloc_opt(SYS_ALLOC_OPT_TRIM_THRESHOLD, init.trim_threshold); sys_alloc_opt(SYS_ALLOC_OPT_TOP_PAD, init.top_pad); - if (erts_allctrs_info[ERTS_FIX_CORE_ALLOCATOR].enabled) - erts_fix_core_allocator_ix = ERTS_FIX_CORE_ALLOCATOR; - else - erts_fix_core_allocator_ix = ERTS_ALC_A_SYSTEM; erts_mtrace_init(init.instr.mtrace, init.instr.nodename); @@ -710,49 +749,40 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) &init.driver_alloc, &driver_alloc_state); - fix_core_allocator = erts_allctrs[erts_fix_core_allocator_ix].alloc; - fix_core_extra = erts_allctrs[erts_fix_core_allocator_ix].extra; + start_au_allocator(ERTS_ALC_A_FIXED_SIZE, + &init.fix_alloc, + &fix_alloc_state); erts_mtrace_install_wrapper_functions(); extra_block_size += erts_instr_init(init.instr.stat, init.instr.map); +#if !HALFWORD_HEAP + init_aireq_alloc(); +#endif + #ifdef DEBUG extra_block_size += install_debug_functions(); #endif -#ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE - - erts_init_fix_alloc(extra_block_size, fix_core_alloc); - +} -#if !defined(PURIFY) && !defined(VALGRIND) - erts_set_fix_size(ERTS_ALC_T_PROC, sizeof(Process)); - erts_set_fix_size(ERTS_ALC_T_DB_TABLE, sizeof(DbTable)); - erts_set_fix_size(ERTS_ALC_T_ATOM, sizeof(Atom)); +void +erts_alloc_late_init(void) +{ - erts_set_fix_size(ERTS_ALC_T_MODULE, sizeof(Module)); - erts_set_fix_size(ERTS_ALC_T_REG_PROC, sizeof(RegProc)); - erts_set_fix_size(ERTS_ALC_T_FUN_ENTRY, sizeof(ErlFunEntry)); -#ifdef ERTS_ALC_T_DRV_EV_D_STATE - erts_set_fix_size(ERTS_ALC_T_DRV_EV_D_STATE, - sizeof(ErtsDrvEventDataState)); -#endif -#ifdef ERTS_ALC_T_DRV_SEL_D_STATE - erts_set_fix_size(ERTS_ALC_T_DRV_SEL_D_STATE, - sizeof(ErtsDrvSelectDataState)); -#endif -#if !HALFWORD_HEAP - erts_set_fix_size(ERTS_ALC_T_EXPORT, sizeof(Export)); - erts_set_fix_size(ERTS_ALC_T_MONITOR_SH, ERTS_MONITOR_SH_SIZE*sizeof(Uint)); - erts_set_fix_size(ERTS_ALC_T_NLINK_SH, ERTS_LINK_SH_SIZE*sizeof(Uint)); -#endif -#endif -#endif +} +static void * +erts_realloc_fixed_size(ErtsAlcType_t type, void *extra, void *p, Uint size) +{ + erl_exit(ERTS_ABORT_EXIT, + "Attempt to reallocate a block of the fixed size type %s\n", + ERTS_ALC_T2TD(type)); } + static void -set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init) +set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu) { ErtsAllocatorFunctions_t *af = &erts_allctrs[alctr_n]; ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n]; @@ -764,6 +794,12 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init) if (init->init.util.force) init->enable = 1; + tspec->enabled = 0; + tspec->dd = 0; + tspec->aix = alctr_n; + tspec->size = 0; + ai->thr_spec = 0; + if (!init->enable) { af->alloc = erts_sys_alloc; af->realloc = erts_sys_realloc; @@ -775,14 +811,14 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init) return; } - tspec->enabled = 0; - tspec->all_thr_safe = 0; - ai->thr_spec = 0; #ifdef USE_THREADS +#ifdef ERTS_SMP if (init->thr_spec) { if (init->thr_spec > 0) { af->alloc = erts_alcu_alloc_thr_spec; - if (init->init.util.ramv) + if (init->init.util.fix_type_size) + af->realloc = erts_realloc_fixed_size; + else if (init->init.util.ramv) af->realloc = erts_alcu_realloc_mv_thr_spec; else af->realloc = erts_alcu_realloc_thr_spec; @@ -790,12 +826,14 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init) } else { af->alloc = erts_alcu_alloc_thr_pref; - if (init->init.util.ramv) + if (init->init.util.fix_type_size) + af->realloc = erts_realloc_fixed_size; + else if (init->init.util.ramv) af->realloc = erts_alcu_realloc_mv_thr_pref; else af->realloc = erts_alcu_realloc_thr_pref; af->free = erts_alcu_free_thr_pref; - tspec->all_thr_safe = 1; + tspec->dd = 1; } tspec->enabled = 1; @@ -803,9 +841,13 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init) ai->thr_spec = tspec->size; } - else if (init->init.util.ts) { + else +#endif + if (init->init.util.ts) { af->alloc = erts_alcu_alloc_ts; - if (init->init.util.ramv) + if (init->init.util.fix_type_size) + af->realloc = erts_realloc_fixed_size; + else if (init->init.util.ramv) af->realloc = erts_alcu_realloc_mv_ts; else af->realloc = erts_alcu_realloc_ts; @@ -815,7 +857,9 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init) #endif { af->alloc = erts_alcu_alloc; - if (init->init.util.ramv) + if (init->init.util.fix_type_size) + af->realloc = erts_realloc_fixed_size; + else if (init->init.util.ramv) af->realloc = erts_alcu_realloc_mv; else af->realloc = erts_alcu_realloc; @@ -838,12 +882,14 @@ start_au_allocator(ErtsAlcType_t alctr_n, ErtsAllocatorFunctions_t *af = &erts_allctrs[alctr_n]; ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n]; ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n]; + ErtsAlcFixList_t *fix_lists = NULL; + size_t fix_list_size = 0; if (!init->enable) return; if (init->thr_spec) { - void *states = erts_sys_alloc(0, + char *states = erts_sys_alloc(0, NULL, ((sizeof(Allctr_t *) * (tspec->size + 1)) @@ -855,18 +901,40 @@ start_au_allocator(ErtsAlcType_t alctr_n, "Failed to allocate allocator states for %salloc\n", init->init.util.name_prefix); tspec->allctr = (Allctr_t **) states; - states = ((char *) states) + sizeof(Allctr_t *) * (tspec->size + 1); + states += sizeof(Allctr_t *) * (tspec->size + 1); states = ((((UWord) states) & ERTS_CACHE_LINE_MASK) - ? (void *) ((((UWord) states) & ~ERTS_CACHE_LINE_MASK) + ? (char *) ((((UWord) states) & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE) - : (void *) states); - tspec->allctr[0] = init->thr_spec > 0 ? (Allctr_t *) state : (Allctr_t *) NULL; + : (char *) states); + tspec->allctr[0] = (Allctr_t *) state; size = tspec->size; for (i = 1; i < size; i++) tspec->allctr[i] = (Allctr_t *) &((ErtsAllocatorState_t *) states)[i-1]; } + if (init->init.util.fix_type_size) { + size_t tot_fix_list_size; + fix_list_size = sizeof(ErtsAlcFixList_t)*ERTS_ALC_NO_FIXED_SIZES; + fix_list_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(fix_list_size); + tot_fix_list_size = fix_list_size; + if (init->thr_spec) + tot_fix_list_size *= tspec->size; + fix_lists = erts_sys_alloc(0, + NULL, + (tot_fix_list_size + + ERTS_CACHE_LINE_SIZE - 1)); + if (!fix_lists) + erl_exit(ERTS_ABORT_EXIT, + "Failed to allocate fix lists for %salloc\n", + init->init.util.name_prefix); + + if (((UWord) fix_lists) & ERTS_CACHE_LINE_MASK) + fix_lists = ((ErtsAlcFixList_t *) + ((((UWord) fix_lists) & ~ERTS_CACHE_LINE_MASK) + + ERTS_CACHE_LINE_SIZE)); + } + for (i = 0; i < size; i++) { void *as; atype = init->atype; @@ -877,25 +945,32 @@ start_au_allocator(ErtsAlcType_t alctr_n, as0 = (void *) tspec->allctr[i]; if (!as0) continue; - if (i == 0) { - if (atype == AFIT) - atype = GOODFIT; - init->init.util.ts = 1; + if (init->thr_spec < 0) { + init->init.util.ts = i == 0; + init->init.util.tspec = 0; + init->init.util.tpref = -1*init->thr_spec + 1; } else { - if (init->thr_spec < 0) { + if (i != 0) + init->init.util.ts = 0; + else { + if (atype == AFIT) + atype = GOODFIT; init->init.util.ts = 1; - init->init.util.tspec = 0; - init->init.util.tpref = -1*init->thr_spec; } - else { - init->init.util.ts = 0; - init->init.util.tspec = init->thr_spec + 1; - init->init.util.tpref = 0; - } - } + init->init.util.tspec = init->thr_spec + 1; + init->init.util.tpref = 0; + } + } + + if (fix_lists) { + init->init.util.fix = fix_lists; + fix_lists = ((ErtsAlcFixList_t *) + (((char *) fix_lists) + fix_list_size)); } + init->init.util.ix = i; + switch (atype) { case GOODFIT: as = (void *) erts_gfalc_start((GFAllctr_t *) as0, @@ -931,11 +1006,8 @@ start_au_allocator(ErtsAlcType_t alctr_n, af->extra = as; } - if (init->thr_spec) { + if (init->thr_spec) af->extra = tspec; - init->init.util.ts = 1; - } - ai->extra = af->extra; } @@ -1055,34 +1127,6 @@ get_amount_value(char *param_end, char** argv, int* ip) return (Uint) tmp; } -static int -get_bool_or_possitive_amount_value(int *bool, Uint *amount, - char *param_end, char** argv, int* ip) -{ - char *param = argv[*ip]+1; - char *value = get_value(param_end, argv, ip); - if (strcmp(value, "true") == 0) { - *bool = 1; - return 1; - } - else if (strcmp(value, "false") == 0) { - *bool = 0; - return 1; - } - else { - Sint tmp; - char *rest; - errno = 0; - tmp = (Sint) strtol(value, &rest, 10); - if (errno != 0 || rest == value || tmp <= 0) { - bad_value(param, param_end, value); - return -1; - } - *amount = (Uint) tmp; - return 0; - } -} - static void handle_au_arg(struct au_init *auip, char* sub_param, @@ -1197,25 +1241,16 @@ handle_au_arg(struct au_init *auip, goto bad_switch; break; case 't': { - Uint no; - int enable; - int res = get_bool_or_possitive_amount_value(&enable, - &no, - sub_param+1, - argv, - ip); - if (res > 0) - auip->thr_spec = enable ? 1 : 0; + int res = get_bool_value(sub_param+1, argv, ip); + if (res > 0) { + auip->thr_spec = 1; + break; + } else if (res == 0) { - int allocs = (int) no; - if (allocs < 0) - allocs = INT_MIN; - else { - allocs *= -1; - } - auip->thr_spec = allocs; + auip->thr_spec = 0; + break; } - break; + goto bad_switch; } default: bad_switch: @@ -1234,6 +1269,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) &init->eheap_alloc, &init->ll_alloc, &init->driver_alloc, + &init->fix_alloc, &init->sl_alloc, &init->temp_alloc }; @@ -1264,14 +1300,8 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) case 'E': handle_au_arg(&init->ets_alloc, &argv[i][3], argv, &i); break; - case 'F': /* fix_alloc */ - if (has_prefix("e", param+2)) { - arg = get_value(param+3, argv, &i); - if (strcmp("true", arg) != 0) - bad_value(param, param+3, arg); - } - else - bad_param(param, param+2); + case 'F': + handle_au_arg(&init->fix_alloc, &argv[i][3], argv, &i); break; case 'H': handle_au_arg(&init->eheap_alloc, &argv[i][3], argv, &i); @@ -1295,12 +1325,6 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) else if (has_prefix("mcs", argv[i]+3)) { #if HAVE_ERTS_MSEG init->mseg.mcs = -#endif - get_amount_value(argv[i]+6, argv, &i); - } - else if (has_prefix("cci", argv[i]+3)) { -#if HAVE_ERTS_MSEG - init->mseg.cci = #endif get_amount_value(argv[i]+6, argv, &i); } @@ -1389,6 +1413,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) set_default_binary_alloc_opts(&init->binary_alloc); set_default_ets_alloc_opts(&init->ets_alloc); set_default_driver_alloc_opts(&init->driver_alloc); + set_default_driver_alloc_opts(&init->fix_alloc); init->driver_alloc.enable = 0; if (strcmp("r9c", arg) == 0) { @@ -1523,43 +1548,74 @@ static char *type_no_str(ErtsAlcType_t n) #define type_str(T) type_no_str(ERTS_ALC_T2N((T))) -erts_tsd_key_t thr_ix_key; -erts_spinlock_t alloc_thr_ix_lock; -int last_thr_ix; -int first_dyn_thr_ix; - -static void -init_thr_ix(int static_ixs) +void +erts_alloc_register_scheduler(void *vesdp) { - erts_tsd_key_create(&thr_ix_key); - erts_spinlock_init(&alloc_thr_ix_lock, "alloc_thr_ix_lock"); - last_thr_ix = -4711; - first_dyn_thr_ix = static_ixs+1; + ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; + int ix = (int) esdp->no; + int aix; + + for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) { + ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix]; + esdp->alloc_data.deallctr[aix] = NULL; + esdp->alloc_data.pref_ix[aix] = -1; + if (tspec->enabled) { + if (!tspec->dd) + esdp->alloc_data.pref_ix[aix] = ix; + else { + Allctr_t *allctr = tspec->allctr[ix]; + ASSERT(allctr); + esdp->alloc_data.deallctr[aix] = allctr; + esdp->alloc_data.pref_ix[aix] = ix; + } + } + } } -int -erts_alc_get_thr_ix(void) +void +erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp, + int *need_thr_progress, + int *more_work) { - int ix = (int)(long) erts_tsd_get(thr_ix_key); - if (ix == 0) { - erts_spin_lock(&alloc_thr_ix_lock); - last_thr_ix++; - if (last_thr_ix < 0) - last_thr_ix = first_dyn_thr_ix; - ix = last_thr_ix; - erts_spin_unlock(&alloc_thr_ix_lock); - erts_tsd_set(thr_ix_key, (void *)(long) ix); + ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; + int aix; + for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) { + Allctr_t *allctr; + if (esdp) + allctr = esdp->alloc_data.deallctr[aix]; + else { + ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix]; + if (tspec->enabled && tspec->dd) + allctr = tspec->allctr[0]; + else + allctr = NULL; + } + if (allctr) { + erts_alcu_check_delayed_dealloc(allctr, + 1, + need_thr_progress, + more_work); + } } - ASSERT(ix > 0); - return ix; } -void erts_alloc_reg_scheduler_id(Uint id) +erts_aint32_t +erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs) { - int ix = (int) id; - ASSERT(0 < ix && ix <= first_dyn_thr_ix); - ASSERT(0 == (int) (long) erts_tsd_get(thr_ix_key)); - erts_tsd_set(thr_ix_key, (void *)(long) ix); +#ifdef ERTS_SMP + ErtsAllocatorThrSpec_t *tspec; + tspec = &erts_allctr_thr_spec[ERTS_ALC_A_FIXED_SIZE]; + if (erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].thr_spec && tspec->enabled) + return erts_alcu_fix_alloc_shrink(tspec->allctr[ix], flgs); + if (ix == 0 && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra) + return erts_alcu_fix_alloc_shrink( + erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra, flgs); +#else + if (ix == 1 && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra) + return erts_alcu_fix_alloc_shrink( + erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra, flgs); +#endif + return 0; } static void @@ -1574,14 +1630,12 @@ erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr) if (erts_allctrs_info[ERTS_ALC_A_TEMPORARY].alloc_util && erts_allctrs_info[ERTS_ALC_A_TEMPORARY].thr_spec) { ErtsAllocatorThrSpec_t *tspec; + int ix = ERTS_ALC_GET_THR_IX(); tspec = &erts_allctr_thr_spec[ERTS_ALC_A_TEMPORARY]; - if (!tspec->all_thr_safe) { - int ix = erts_alc_get_thr_ix(); - if (ix < tspec->size) { - *allctr = tspec->allctr[ix]; - return erts_alcu_verify_unused; - } + if (ix < tspec->size) { + *allctr = tspec->allctr[ix]; + return erts_alcu_verify_unused; } } @@ -1680,7 +1734,7 @@ erts_realloc_n_enomem(ErtsAlcType_t n, void *ptr, Uint size) } static ERTS_INLINE UWord -alcu_size(ErtsAlcType_t ai) +alcu_size(ErtsAlcType_t ai, ErtsAlcUFixInfo_t *fi, int fisz) { UWord res = 0; @@ -1690,22 +1744,20 @@ alcu_size(ErtsAlcType_t ai) if (!erts_allctrs_info[ai].thr_spec) { Allctr_t *allctr = erts_allctrs_info[ai].extra; AllctrSize_t asize; - erts_alcu_current_size(allctr, &asize); + erts_alcu_current_size(allctr, &asize, fi, fisz); res += asize.blocks; } else { ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ai]; int i; - ASSERT(tspec->all_thr_safe); - ASSERT(tspec->enabled); for (i = tspec->size - 1; i >= 0; i--) { Allctr_t *allctr = tspec->allctr[i]; AllctrSize_t asize; if (allctr) { - erts_alcu_current_size(allctr, &asize); + erts_alcu_current_size(allctr, &asize, fi, fisz); res += asize.blocks; } } @@ -1733,7 +1785,6 @@ alcu_is_low(ErtsAlcType_t ai) int found_one = 0; # endif - ASSERT(tspec->all_thr_safe); ASSERT(tspec->enabled); for (i = tspec->size - 1; i >= 0; i--) { @@ -1757,11 +1808,24 @@ alcu_is_low(ErtsAlcType_t ai) } #endif /* HALFWORD */ +static ERTS_INLINE void +add_fix_values(UWord *ap, UWord *up, ErtsAlcUFixInfo_t *fi, ErtsAlcType_t type) +{ + int ix = ERTS_ALC_T2N(type) - ERTS_ALC_N_MIN_A_FIXED_SIZE; + ASSERT(0 <= ix && ix < ERTS_ALC_NO_FIXED_SIZES); + + *ap += (UWord) fi[ix].allocated; + *up += (UWord) fi[ix].used; +} + Eterm erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) { +/* + * NOTE! When updating this function, make sure to also update + * erlang:memory/[0,1] in $ERL_TOP/erts/preloaded/src/erlang.erl + */ #define ERTS_MEM_NEED_ALL_ALCU (!erts_instr_stat && want_tot_or_sys) - ErtsFixInfo efi; struct { int total; int processes; @@ -1800,6 +1864,11 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) Eterm res = THE_NON_VALUE; ErtsAlcType_t ai; int only_one_value = 0; + ErtsAlcUFixInfo_t fi[ERTS_ALC_NO_FIXED_SIZES] = {{0,0}}; + + ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0) + || (ERTS_IS_CRASH_DUMPING + && erts_smp_is_system_blocked(ERTS_BS_FLG_ALLOW_GC))); /* Figure out whats wanted... */ @@ -1969,7 +2038,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) { switch (ai) { case ERTS_ALC_A_SYSTEM: - case ERTS_ALC_A_FIXED_SIZE: case ERTS_ALC_A_SBMBC: #if HALFWORD_HEAP case ERTS_ALC_A_SBMBC_LOW: @@ -2029,11 +2097,15 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) case ERTS_ALC_A_BINARY: save = &size.binary; break; + case ERTS_ALC_A_FIXED_SIZE: + asz = alcu_size(ai, fi, ERTS_ALC_NO_FIXED_SIZES); + size.total += asz; + continue; default: save = NULL; break; } - asz = alcu_size(ai); + asz = alcu_size(ai, NULL, 0); if (save) *save = asz; size.total += asz; @@ -2053,8 +2125,11 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (ERTS_MEM_NEED_ALL_ALCU) tmp = size.processes; - else - tmp = alcu_size(ERTS_ALC_A_EHEAP); + else { + alcu_size(ERTS_ALC_A_FIXED_SIZE, + fi, ERTS_ALC_NO_FIXED_SIZES); + tmp = alcu_size(ERTS_ALC_A_EHEAP, NULL, 0); + } tmp += erts_max_processes*sizeof(Process*); #ifdef HYBRID tmp += erts_max_processes*sizeof(Process*); @@ -2064,69 +2139,54 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) size.processes = size.processes_used = tmp; -#if HALFWORD_HEAP - /* BUG: We ignore link and monitor memory */ -#else - erts_fix_info(ERTS_ALC_T_NLINK_SH, &efi); - size.processes += efi.total; - size.processes_used += efi.used; + add_fix_values(&size.processes, + &size.processes_used, + fi, + ERTS_ALC_T_PROC); +#if !HALFWORD_HEAP + add_fix_values(&size.processes, + &size.processes_used, + fi, + ERTS_ALC_T_MONITOR_SH); - erts_fix_info(ERTS_ALC_T_MONITOR_SH, &efi); - size.processes += efi.total; - size.processes_used += efi.used; + add_fix_values(&size.processes, + &size.processes_used, + fi, + ERTS_ALC_T_NLINK_SH); #endif - - erts_fix_info(ERTS_ALC_T_PROC, &efi); - size.processes += efi.total; - size.processes_used += efi.used; - - erts_fix_info(ERTS_ALC_T_REG_PROC, &efi); - size.processes += efi.total; - size.processes_used += efi.used; - + add_fix_values(&size.processes, + &size.processes_used, + fi, + ERTS_ALC_T_MSG_REF); } if (want.atom || want.atom_used) { Uint reserved_atom_space, atom_space; erts_atom_get_text_space_sizes(&reserved_atom_space, &atom_space); size.atom = size.atom_used = atom_table_sz(); - erts_fix_info(ERTS_ALC_T_ATOM, &efi); - if (want.atom) { + if (want.atom) size.atom += reserved_atom_space; - size.atom += efi.total; - } - if (want.atom_used) { + if (want.atom_used) size.atom_used += atom_space; - size.atom_used += efi.used; - } } if (!ERTS_MEM_NEED_ALL_ALCU && want.binary) - size.binary = alcu_size(ERTS_ALC_A_BINARY); + size.binary = alcu_size(ERTS_ALC_A_BINARY, NULL, 0); if (want.code) { size.code = module_table_sz(); - erts_fix_info(ERTS_ALC_T_MODULE, &efi); - size.code += efi.used; size.code += export_table_sz(); -#if HALFWORD_HEAP size.code += export_list_size() * sizeof(Export); -#else - erts_fix_info(ERTS_ALC_T_EXPORT, &efi); - size.code += efi.used; -#endif size.code += erts_fun_table_sz(); - erts_fix_info(ERTS_ALC_T_FUN_ENTRY, &efi); - size.code += efi.used; size.code += allocated_modules*sizeof(Range); size.code += erts_total_code_size; } if (want.ets) { if (!ERTS_MEM_NEED_ALL_ALCU) - size.ets = alcu_size(ERTS_ALC_A_ETS); + size.ets = alcu_size(ERTS_ALC_A_ETS, NULL, 0); size.ets += erts_get_ets_misc_mem_size(); } @@ -2199,13 +2259,10 @@ struct aa_values { Eterm erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) { -#define MAX_AA_VALUES \ - (20 + (ERTS_ALC_N_MAX_A_FIXED_SIZE - ERTS_ALC_N_MIN_A_FIXED_SIZE + 1)) - +#define MAX_AA_VALUES (23) struct aa_values values[MAX_AA_VALUES]; Eterm res = THE_NON_VALUE; int i, length; - ErtsFixInfo efi; Uint reserved_atom_space, atom_space; if (proc) { @@ -2269,6 +2326,11 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) values[i].ui[0] = export_table_sz(); i++; + values[i].arity = 2; + values[i].name = "export_list"; + values[i].ui[0] = export_list_size() * sizeof(Export); + i++; + values[i].arity = 2; values[i].name = "register_table"; values[i].ui[0] = process_reg_sz(); @@ -2314,22 +2376,15 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) values[i].ui[0] = erts_tot_link_lh_size(); i++; - { - Uint n; - - for (n = ERTS_ALC_N_MIN_A_FIXED_SIZE; - n <= ERTS_ALC_N_MAX_A_FIXED_SIZE; - n++) { - erts_fix_info(ERTS_ALC_N2T(n), &efi); - - values[i].arity = 3; - values[i].name = ERTS_ALC_N2TD(n); - values[i].ui[0] = efi.total; - values[i].ui[1] = efi.used; - i++; - } + values[i].arity = 2; + values[i].name = "process_table"; + values[i].ui[0] = erts_max_processes*sizeof(Process*); + i++; - } + values[i].arity = 2; + values[i].name = "ets_misc"; + values[i].ui[0] = erts_get_ets_misc_mem_size(); + i++; length = i; ASSERT(length <= MAX_AA_VALUES); @@ -2423,17 +2478,16 @@ erts_alloc_util_allocators(void *proc) Uint sz; int i; /* - * Currently all allocators except sys_alloc and fix_alloc are + * Currently all allocators except sys_alloc are * alloc_util allocators. */ - sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 2)*2; + sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 1)*2; ASSERT(sz > 0); hp = HAlloc((Process *) proc, sz); res = NIL; for (i = ERTS_ALC_A_MAX; i >= ERTS_ALC_A_MIN; i--) { switch (i) { case ERTS_ALC_A_SYSTEM: - case ERTS_ALC_A_FIXED_SIZE: break; default: { char *alc_str = (char *) ERTS_ALC_A2AD(i); @@ -2447,259 +2501,6 @@ erts_alloc_util_allocators(void *proc) return res; } -Eterm -erts_allocator_info_term(void *proc, Eterm which_alloc, int only_sz) -{ -#define ERTS_AIT_RET(R) \ - do { res = (R); goto done; } while (0) -#define ERTS_AIT_HALLOC(P, S) \ - do { hp = HAlloc((P), (S)); hp_end = hp + (S); } while (0) - - ErtsAlcType_t i; - Uint sz = 0; - Uint *hp = NULL; - Uint *hp_end = NULL; - Eterm res = am_undefined; - - if (is_not_atom(which_alloc)) - goto done; - - for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { - if (erts_is_atom_str((char *) ERTS_ALC_A2AD(i), which_alloc)) { - if (!erts_allctrs_info[i].enabled) - ERTS_AIT_RET(am_false); - else { - if (erts_allctrs_info[i].alloc_util) { - Eterm ires, tmp; - Eterm **hpp; - Uint *szp; - Eterm (*info_func)(Allctr_t *, - int, - int *, - void *, - Uint **, - Uint *); - - info_func = (only_sz - ? erts_alcu_sz_info - : erts_alcu_info); - - if (erts_allctrs_info[i].thr_spec) { - ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[i]; - int j; - int block_system = !tspec->all_thr_safe; - - if (block_system) { - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); - } - ASSERT(tspec->enabled); - - szp = &sz; - hpp = NULL; - - while (1) { - ires = NIL; - for (j = tspec->size - 1; j >= 0; j--) { - Allctr_t *allctr = tspec->allctr[j]; - if (allctr) { - tmp = erts_bld_tuple(hpp, - szp, - 3, - erts_bld_atom(hpp, - szp, - "instance"), - make_small((Uint) j), - (*info_func)(allctr, - hpp != NULL, - NULL, - NULL, - hpp, - szp)); - ires = erts_bld_cons(hpp, szp, tmp, ires); - } - } - if (hpp) - break; - ERTS_AIT_HALLOC((Process *) proc, sz); - hpp = &hp; - szp = NULL; - } - - if (block_system) { - erts_smp_release_system(); - erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); - } - } - else { - Allctr_t *allctr = erts_allctrs_info[i].extra; - szp = &sz; - hpp = NULL; - while (1) { - ires = NIL; - tmp = erts_bld_tuple(hpp, - szp, - 3, - erts_bld_atom(hpp, - szp, - "instance"), - make_small((Uint) 0), - (*info_func)(allctr, - hpp != NULL, - NULL, - NULL, - hpp, - szp)); - ires = erts_bld_cons(hpp, szp, tmp, ires); - if (hpp) - break; - ERTS_AIT_HALLOC((Process *) proc, sz); - hpp = &hp; - szp = NULL; - } - } - ERTS_AIT_RET(ires); - } - else { - Eterm *szp, **hpp; - - switch (i) { - case ERTS_ALC_A_SYSTEM: { - SysAllocStat sas; - Eterm opts_am; - Eterm opts; - Eterm as[4]; /* Ok even if !HEAP_ON_C_STACK, not really heap data on stack */ - Eterm ts[4]; /* Ok even if !HEAP_ON_C_STACK, not really heap data on stack */ - int l; - - if (only_sz) - ERTS_AIT_RET(NIL); - - sys_alloc_stat(&sas); - opts_am = am_atom_put("options", 7); - - szp = &sz; - hpp = NULL; - - restart_sys_alloc: - l = 0; - as[l] = am_atom_put("e", 1); - ts[l++] = am_true; - as[l] = am_atom_put("m", 1); - ts[l++] = am_atom_put("libc", 4); - if(sas.trim_threshold >= 0) { - as[l] = am_atom_put("tt", 2); - ts[l++] = erts_bld_uint(hpp, szp, - (Uint) sas.trim_threshold); - } - if(sas.top_pad >= 0) { - as[l] = am_atom_put("tp", 2); - ts[l++] = erts_bld_uint(hpp, szp, (Uint) sas.top_pad); - } - - opts = erts_bld_2tup_list(hpp, szp, l, as, ts); - res = erts_bld_2tup_list(hpp, szp, 1, &opts_am, &opts); - - if (szp) { - ERTS_AIT_HALLOC((Process *) proc, sz); - szp = NULL; - hpp = &hp; - goto restart_sys_alloc; - } - ERTS_AIT_RET(res); - } - case ERTS_ALC_A_FIXED_SIZE: { - ErtsAlcType_t n; - Eterm as[2], vs[2]; - - if (only_sz) - ERTS_AIT_RET(NIL); - - as[0] = am_atom_put("options", 7); - as[1] = am_atom_put("pools", 5); - - szp = &sz; - hpp = NULL; - - restart_fix_alloc: - - vs[0] = erts_bld_cons(hpp, szp, - erts_bld_tuple(hpp, szp, 2, - am_atom_put("e", - 1), - am_true), - NIL); - - vs[1] = NIL; - for (n = ERTS_ALC_N_MIN_A_FIXED_SIZE; - n <= ERTS_ALC_N_MAX_A_FIXED_SIZE; - n++) { - ErtsFixInfo efi; - erts_fix_info(ERTS_ALC_N2T(n), &efi); - - vs[1] = erts_bld_cons( - hpp, szp, - erts_bld_tuple( - hpp, szp, 3, - am_atom_put((char *) ERTS_ALC_N2TD(n), - strlen(ERTS_ALC_N2TD(n))), - erts_bld_uint(hpp, szp, efi.total), - erts_bld_uint(hpp, szp, efi.used)), - vs[1]); - - } - - res = erts_bld_2tup_list(hpp, szp, 2, as, vs); - if (szp) { - ERTS_AIT_HALLOC((Process *) proc, sz); - szp = NULL; - hpp = &hp; - goto restart_fix_alloc; - } - ERTS_AIT_RET(res); - } - default: - ASSERT(0); - goto done; - } - } - } - } - } - - if (ERTS_IS_ATOM_STR("mseg_alloc", which_alloc)) { -#if HAVE_ERTS_MSEG - if (only_sz) - ERTS_AIT_RET(NIL); - erts_mseg_info(NULL, NULL, 0, NULL, &sz); - if (sz) - ERTS_AIT_HALLOC((Process *) proc, sz); - ERTS_AIT_RET(erts_mseg_info(NULL, NULL, 1, &hp, NULL)); -#else - ERTS_AIT_RET(am_false); -#endif - - } - else if (ERTS_IS_ATOM_STR("alloc_util", which_alloc)) { - if (only_sz) - ERTS_AIT_RET(NIL); - erts_alcu_au_info_options(NULL, NULL, NULL, &sz); - if (sz) - ERTS_AIT_HALLOC((Process *) proc, sz); - ERTS_AIT_RET(erts_alcu_au_info_options(NULL, NULL, &hp, NULL)); - } - - done: - if (hp) { - ASSERT(hp_end >= hp); - HRelease((Process *) proc, hp_end, hp); - } - return res; - -#undef ERTS_AIT_RET -#undef ERTS_AIT_HALLOC -} - void erts_allocator_info(int to, void *arg) { @@ -2748,22 +2549,6 @@ erts_allocator_info(int to, void *arg) erts_print(to, arg, "option tp: %d\n", sas.top_pad); break; } - case ERTS_ALC_A_FIXED_SIZE: { - ErtsAlcType_t n; - erts_print(to, arg, "option e: true\n"); - - for (n = ERTS_ALC_N_MIN_A_FIXED_SIZE; - n <= ERTS_ALC_N_MAX_A_FIXED_SIZE; - n++) { - ErtsFixInfo efi; - erts_fix_info(ERTS_ALC_N2T(n), &efi); - erts_print(to, arg, "%s: %lu %lu\n", - ERTS_ALC_N2TD(n), - efi.total, - efi.used); - } - break; - } default: ASSERT(0); break; @@ -2774,8 +2559,18 @@ erts_allocator_info(int to, void *arg) } #if HAVE_ERTS_MSEG - erts_print(to, arg, "=allocator:mseg_alloc\n"); - erts_mseg_info(&to, arg, 0, NULL, NULL); + { +#ifdef ERTS_SMP + int max = (int) erts_no_schedulers; +#else + int max = 0; +#endif + int i; + for (i = 0; i <= max; i++) { + erts_print(to, arg, "=allocator:mseg_alloc[%d]\n", i); + erts_mseg_info(i, &to, arg, 0, NULL, NULL); + } + } #endif erts_print(to, arg, "=allocator:alloc_util\n"); @@ -2829,7 +2624,7 @@ erts_allocator_options(void *proc) use_mseg++; #endif if (erts_allctr_thr_spec[a].enabled) - allctr = erts_allctr_thr_spec[a].allctr[1]; + allctr = erts_allctr_thr_spec[a].allctr[0]; else allctr = erts_allctrs_info[a].extra; tmp = erts_alcu_info_options(allctr, NULL, NULL, hpp, szp); @@ -2878,7 +2673,7 @@ erts_allocator_options(void *proc) #if HAVE_ERTS_MSEG if (use_mseg) { atoms[length] = am_atom_put("mseg_alloc", 10); - terms[length++] = erts_mseg_info_options(NULL, NULL, hpp, szp); + terms[length++] = erts_mseg_info_options(0, NULL, NULL, hpp, szp); } #endif @@ -2982,6 +2777,313 @@ void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size) return (void*)v; } +static void +reply_alloc_info(void *vair) +{ + ErtsAllocInfoReq *air = (ErtsAllocInfoReq *) vair; + Uint sched_id = erts_get_scheduler_id(); + int global_instances = air->req_sched == sched_id; + ErtsProcLocks rp_locks; + Process *rp = air->proc; + Eterm ref_copy = NIL, ai_list, msg; + Eterm *hp = NULL, *hp_end = NULL, *hp_start = NULL; + Eterm **hpp; + Uint sz, *szp; + ErlOffHeap *ohp = NULL; + ErlHeapFragment *bp = NULL; + int i; + Eterm (*info_func)(Allctr_t *, + int, + int *, + void *, + Uint **, + Uint *) = (air->only_sz + ? erts_alcu_sz_info + : erts_alcu_info); + + rp_locks = air->req_sched == sched_id ? ERTS_PROC_LOCK_MAIN : 0; + + sz = 0; + hpp = NULL; + szp = &sz; + + while (1) { + + if (hpp) + ref_copy = STORE_NC(hpp, ohp, air->ref); + else + *szp += REF_THING_SIZE; + + ai_list = NIL; + for (i = 0; air->allocs[i] != ERTS_ALC_A_INVALID; i++); + for (i--; i >= 0; i--) { + int ai = air->allocs[i]; + Allctr_t *allctr; + Eterm ainfo; + Eterm alloc_atom; + if (global_instances) { + switch (ai) { + case ERTS_ALC_A_SYSTEM: { + alloc_atom = erts_bld_atom(hpp, szp, "sys_alloc"); + ainfo = NIL; + if (!air->only_sz) { + SysAllocStat sas; + if (hpp) + sys_alloc_stat(&sas); + if (szp) { + /* ensure ehough heap */ + sas.top_pad = INT_MAX; + sas.trim_threshold = INT_MAX; + } + if (sas.top_pad >= 0) { + ainfo = erts_bld_cons( + hpp, szp, + erts_bld_tuple( + hpp, szp, 2, + erts_bld_atom(hpp, szp, "tp"), + erts_bld_uint( + hpp, szp, + (Uint) sas.top_pad)), + ainfo); + } + if (sas.trim_threshold >= 0) { + ainfo = erts_bld_cons( + hpp, szp, + erts_bld_tuple( + hpp, szp, 2, + erts_bld_atom(hpp, szp, "tt"), + erts_bld_uint( + hpp, szp, + (Uint) sas.trim_threshold)), + ainfo); + } + ainfo = erts_bld_cons(hpp, szp, + erts_bld_tuple( + hpp, szp, 2, + erts_bld_atom(hpp, szp, + "m"), + erts_bld_atom(hpp, szp, + "libc")), + ainfo); + ainfo = erts_bld_cons(hpp, szp, + erts_bld_tuple( + hpp, szp, 2, + erts_bld_atom(hpp, szp, + "e"), + am_true), + ainfo); + ainfo = erts_bld_tuple(hpp, szp, 2, + erts_bld_atom(hpp, szp, + "otps"), + ainfo); + } + ainfo = erts_bld_tuple(hpp, szp, 3, + alloc_atom, + make_small(0), + ainfo); + break; + } + case ERTS_ALC_INFO_A_ALLOC_UTIL: + alloc_atom = erts_bld_atom(hpp, szp, "alloc_util"); + ainfo = (air->only_sz + ? NIL + : erts_alcu_au_info_options(NULL, NULL, + hpp, szp)); + ainfo = erts_bld_tuple(hpp, szp, 3, + alloc_atom, + make_small(0), + ainfo); + break; + case ERTS_ALC_INFO_A_MSEG_ALLOC: + alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc"); +#if HAVE_ERTS_MSEG + ainfo = (air->only_sz + ? NIL + : erts_mseg_info(0, NULL, NULL, hpp != NULL, + hpp, szp)); + ainfo = erts_bld_tuple(hpp, szp, 3, + alloc_atom, + make_small(0), + ainfo); +#else + ainfo = erts_bld_tuple(hpp, szp, 2, alloc_atom, + am_false); +#endif + break; + default: + alloc_atom = erts_bld_atom(hpp, szp, + (char *) ERTS_ALC_A2AD(ai)); + if (!erts_allctrs_info[ai].enabled) + ainfo = erts_bld_tuple(hpp, szp, 2, alloc_atom, + am_false); + else if (erts_allctrs_info[ai].alloc_util) { + if (erts_allctrs_info[ai].thr_spec) + allctr = erts_allctr_thr_spec[ai].allctr[0]; + else + allctr = erts_allctrs_info[ai].extra; + ainfo = info_func(allctr, hpp != NULL, NULL, + NULL, hpp, szp); + ainfo = erts_bld_tuple(hpp, szp, 3, alloc_atom, + make_small(0), ainfo); + } + else { + erl_exit(ERTS_ABORT_EXIT, "%s:%d: internal error\n", + __FILE__, __LINE__); + } + } + ai_list = erts_bld_cons(hpp, szp, + ainfo, ai_list); + } + switch (ai) { + case ERTS_ALC_A_SYSTEM: + case ERTS_ALC_INFO_A_ALLOC_UTIL: + break; + case ERTS_ALC_INFO_A_MSEG_ALLOC: +#if HAVE_ERTS_MSEG && defined(ERTS_SMP) + alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc"); + ainfo = (air->only_sz + ? NIL + : erts_mseg_info(sched_id, NULL, NULL, + hpp != NULL, hpp, szp)); + ainfo = erts_bld_tuple(hpp, szp, 3, + alloc_atom, + make_small(sched_id), + ainfo); + ai_list = erts_bld_cons(hpp, szp, ainfo, ai_list); +#endif + break; + default: + if (erts_allctrs_info[ai].thr_spec) { + alloc_atom = erts_bld_atom(hpp, szp, + (char *) ERTS_ALC_A2AD(ai)); + allctr = erts_allctr_thr_spec[ai].allctr[sched_id]; + ainfo = info_func(allctr, hpp != NULL, NULL, + NULL, hpp, szp); + ai_list = erts_bld_cons(hpp, szp, + erts_bld_tuple( + hpp, szp, + 3, + alloc_atom, + make_small(sched_id), + ainfo), + ai_list); + } + break; + } + msg = erts_bld_tuple(hpp, szp, + 3, + ref_copy, + make_small(sched_id), + ai_list); + + } + if (hpp) + break; + + hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks); + hp_start = hp; + hp_end = hp + sz; + szp = NULL; + hpp = &hp; + } + if (bp) + bp = erts_resize_message_buffer(bp, hp - hp_start, &msg, 1); + else { + ASSERT(hp); + HRelease(rp, hp_end, hp); + } + + erts_queue_message(rp, &rp_locks, bp, msg, NIL); + + if (air->req_sched == sched_id) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + + erts_smp_proc_unlock(rp, rp_locks); + erts_smp_proc_dec_refc(rp); + + if (erts_smp_atomic32_dec_read_nob(&air->refc) == 0) + aireq_free(air); +} + +int +erts_request_alloc_info(struct process *c_p, + Eterm ref, + Eterm allocs, + int only_sz) +{ + ErtsAllocInfoReq *air = aireq_alloc(); + Eterm req_ai[ERTS_ALC_A_MAX+1+2] = {0}; + Eterm alist; + Eterm *hp; + int airix = 0, ai; + + air->req_sched = erts_get_scheduler_id(); + + air->only_sz = only_sz; + + air->proc = c_p; + + if (is_not_internal_ref(ref)) + return 0; + + hp = &air->ref_heap[0]; + air->ref = STORE_NC(&hp, NULL, ref); + + if (is_not_list(allocs)) + return 0; + + alist = allocs; + + while (is_list(alist)) { + int saved = 0; + Eterm* consp = list_val(alist); + Eterm alloc = CAR(consp); + + for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) + if (erts_is_atom_str((char *) erts_alc_a2ad[ai], alloc)) + goto save_alloc; + if (erts_is_atom_str("mseg_alloc", alloc)) { + ai = ERTS_ALC_INFO_A_MSEG_ALLOC; + goto save_alloc; + } + if (erts_is_atom_str("alloc_util", alloc)) { + ai = ERTS_ALC_INFO_A_ALLOC_UTIL; + save_alloc: + if (req_ai[ai]) + return 0; + air->allocs[airix++] = ai; + req_ai[ai] = 1; + saved = 1; + } + + if (!saved) + return 0; + + alist = CDR(consp); + } + + if (is_not_nil(alist)) + return 0; + + air->allocs[airix] = ERTS_ALC_A_INVALID; + + erts_smp_atomic32_init_nob(&air->refc, + (erts_aint32_t) erts_no_schedulers); + + erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers); + +#ifdef ERTS_SMP + if (erts_no_schedulers > 1) + erts_smp_schedule_misc_aux_work(1, + erts_no_schedulers, + reply_alloc_info, + (void *) air); +#endif + + reply_alloc_info((void *) air); + + return 1; +} /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * Deprecated functions * diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index 80cb82c393..f4133cdb1a 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -43,43 +43,40 @@ # define ERTS_ALC_INLINE #endif -#define ERTS_FIX_CORE_ALLOCATOR ERTS_ALC_A_LONG_LIVED -extern ErtsAlcType_t erts_fix_core_allocator_ix; - -typedef struct { - Uint total; - Uint used; -} ErtsFixInfo; +#define ERTS_ALC_NO_FIXED_SIZES \ + (ERTS_ALC_N_MAX_A_FIXED_SIZE - ERTS_ALC_N_MIN_A_FIXED_SIZE + 1) void erts_sys_alloc_init(void); void *erts_sys_alloc(ErtsAlcType_t, void *, Uint); void *erts_sys_realloc(ErtsAlcType_t, void *, void *, Uint); void erts_sys_free(ErtsAlcType_t, void *, void *); - -void erts_init_fix_alloc(Uint, void *(*)(Uint)); -Uint erts_get_fix_size(ErtsAlcType_t); -void erts_set_fix_size(ErtsAlcType_t, Uint); -void erts_fix_info(ErtsAlcType_t, ErtsFixInfo *); -void *erts_fix_alloc(ErtsAlcType_t, void *, Uint); -void *erts_fix_realloc(ErtsAlcType_t, void *, void*, Uint); -void erts_fix_free(ErtsAlcType_t, void *, void*); - - Eterm erts_memory(int *, void *, void *, Eterm); Eterm erts_allocated_areas(int *, void *, void *); Eterm erts_alloc_util_allocators(void *proc); void erts_allocator_info(int, void *); -Eterm erts_allocator_info_term(void *proc, Eterm which_alloc, int only_sz); Eterm erts_allocator_options(void *proc); +struct process; + +int erts_request_alloc_info(struct process *c_p, Eterm ref, Eterm allocs, + int only_sz); + #define ERTS_ALLOC_INIT_DEF_OPTS_INITER {0} typedef struct { - int dummy; + int ncpu; } ErtsAllocInitOpts; +typedef struct { + Allctr_t *deallctr[ERTS_ALC_A_MAX+1]; + int pref_ix[ERTS_ALC_A_MAX+1]; + int flist_ix[ERTS_ALC_A_MAX+1]; + int pre_alc_ix; +} ErtsSchedAllocData; + void erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop); +void erts_alloc_late_init(void); #if defined(GET_ERTS_ALC_TEST) || defined(ERTS_ALC_INTERNAL__) /* Only for testing */ @@ -126,15 +123,19 @@ extern ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1]; typedef struct { int enabled; - int all_thr_safe; + int dd; + int aix; int size; Allctr_t **allctr; } ErtsAllocatorThrSpec_t; extern ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1]; -int erts_alc_get_thr_ix(void); -void erts_alloc_reg_scheduler_id(Uint id); +void erts_alloc_register_scheduler(void *vesdp); +void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp, + int *need_thr_progress, + int *more_work); +erts_aint32_t erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs); __decl_noreturn void erts_alloc_enomem(ErtsAlcType_t,Uint) __noreturn; @@ -252,6 +253,8 @@ void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size) #endif /* #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) */ +#define ERTS_ALC_GET_THR_IX() ((int) erts_get_scheduler_id()) + typedef void (*erts_alloc_verify_func_t)(Allctr_t *); erts_alloc_verify_func_t @@ -436,136 +439,41 @@ NAME##_free(TYPE *p) \ } \ } -typedef struct { - void *start; - void *end; - int chunks_mem_size; -} erts_sched_pref_quick_alloc_data_t; - -#ifdef DEBUG -#define ERTS_SPPA_DBG_CHK_IN_CHNK(A, C, P) \ -do { \ - ASSERT((void *) (C) < (void *) (P)); \ - ASSERT((void *) (P) \ - < (void *) (((char *) (C)) + (A)->chunks_mem_size)); \ -} while (0) -#else -#define ERTS_SPPA_DBG_CHK_IN_CHNK(A, C, P) -#endif +#include "erl_sched_spec_pre_alloc.h" #define ERTS_SCHED_PREF_PRE_ALLOC_IMPL(NAME, TYPE, PASZ) \ -union erts_qa_##NAME##__ { \ +union erts_sspa_##NAME##__ { \ + erts_sspa_blk_t next; \ TYPE type; \ - union erts_qa_##NAME##__ *next; \ }; \ -typedef struct { \ - erts_smp_spinlock_t lock; \ - union erts_qa_##NAME##__ *freelist; \ - union erts_qa_##NAME##__ pre_alloced[1]; \ -} erts_qa_##NAME##_chunk__; \ -static erts_sched_pref_quick_alloc_data_t *qa_data_##NAME##__; \ -static ERTS_INLINE erts_qa_##NAME##_chunk__ * \ -get_##NAME##_chunk_ix(int cix) \ -{ \ - char *ptr = (char *) qa_data_##NAME##__->start; \ - ptr += cix*qa_data_##NAME##__->chunks_mem_size; \ - return (erts_qa_##NAME##_chunk__ *) ptr; \ -} \ -static ERTS_INLINE erts_qa_##NAME##_chunk__ * \ -get_##NAME##_chunk_ptr(void *ptr) \ -{ \ - int cix; \ - size_t diff; \ - if (ptr < qa_data_##NAME##__->start || qa_data_##NAME##__->end <= ptr)\ - return NULL; \ - diff = ((char *) ptr) - ((char *) qa_data_##NAME##__->start); \ - cix = diff / qa_data_##NAME##__->chunks_mem_size; \ - return get_##NAME##_chunk_ix(cix); \ -} \ + \ +static erts_sspa_data_t *sspa_data_##NAME##__; \ + \ static void \ init_##NAME##_alloc(void) \ { \ - size_t tot_size; \ - size_t chunk_mem_size; \ - char *chunk_start; \ - int cix; \ - int no_blocks = ERTS_PRE_ALLOC_SIZE((PASZ)); \ - int no_blocks_per_chunk = 2*((no_blocks-1)/erts_no_schedulers + 1); \ - no_blocks = no_blocks_per_chunk * erts_no_schedulers; \ - chunk_mem_size = sizeof(erts_qa_##NAME##_chunk__); \ - chunk_mem_size += (sizeof(union erts_qa_##NAME##__) \ - * (no_blocks_per_chunk - 1)); \ - chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(chunk_mem_size); \ - tot_size = sizeof(erts_sched_pref_quick_alloc_data_t); \ - tot_size += ERTS_CACHE_LINE_SIZE - 1; \ - tot_size += chunk_mem_size*erts_no_schedulers; \ - qa_data_##NAME##__ = erts_alloc(ERTS_ALC_T_PRE_ALLOC_DATA,tot_size);\ - chunk_start = (((char *) qa_data_##NAME##__) \ - + sizeof(erts_sched_pref_quick_alloc_data_t)); \ - if ((((UWord) chunk_start) & ERTS_CACHE_LINE_MASK) != ((UWord) 0)) \ - chunk_start = ((char *) \ - ((((UWord) chunk_start) & ~ERTS_CACHE_LINE_MASK) \ - + ERTS_CACHE_LINE_SIZE)); \ - qa_data_##NAME##__->chunks_mem_size = chunk_mem_size; \ - qa_data_##NAME##__->start = (void *) chunk_start; \ - qa_data_##NAME##__->end = (chunk_start \ - + chunk_mem_size*erts_no_schedulers); \ - for (cix = 0; cix < erts_no_schedulers; cix++) { \ - int i; \ - erts_qa_##NAME##_chunk__ *chunk = get_##NAME##_chunk_ix(cix); \ - erts_smp_spinlock_init(&chunk->lock, #NAME "_alloc_lock"); \ - chunk->freelist = &chunk->pre_alloced[0]; \ - for (i = 1; i < no_blocks_per_chunk; i++) { \ - ERTS_PRE_ALLOC_CLOBBER(&chunk->pre_alloced[i-1], \ - union erts_qa_##NAME##__); \ - chunk->pre_alloced[i-1].next = &chunk->pre_alloced[i]; \ - } \ - ERTS_PRE_ALLOC_CLOBBER(&chunk->pre_alloced[no_blocks_per_chunk-1],\ - union erts_qa_##NAME##__); \ - chunk->pre_alloced[no_blocks_per_chunk-1].next = NULL; \ - } \ + sspa_data_##NAME##__ = \ + erts_sspa_create(sizeof(union erts_sspa_##NAME##__), \ + ERTS_PRE_ALLOC_SIZE((PASZ))); \ } \ -static ERTS_INLINE TYPE * \ + \ +static TYPE * \ NAME##_alloc(void) \ { \ - int cix = ((int) erts_get_scheduler_id()) - 1; \ - TYPE *res; \ - if (cix < 0) \ - res = NULL; \ - else { \ - erts_qa_##NAME##_chunk__ *chunk = get_##NAME##_chunk_ix(cix); \ - erts_smp_spin_lock(&chunk->lock); \ - if (!chunk->freelist) \ - res = NULL; \ - else { \ - res = &chunk->freelist->type; \ - chunk->freelist = chunk->freelist->next; \ - ERTS_SPPA_DBG_CHK_IN_CHNK(qa_data_##NAME##__, chunk, res); \ - } \ - erts_smp_spin_unlock(&chunk->lock); \ - } \ - return res; \ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); \ + if (!esdp) \ + return NULL; \ + return (TYPE *) erts_sspa_alloc(sspa_data_##NAME##__, \ + (int) esdp->no - 1); \ } \ -static ERTS_INLINE int \ + \ +static int \ NAME##_free(TYPE *p) \ { \ - erts_qa_##NAME##_chunk__ *chunk; \ - chunk = get_##NAME##_chunk_ptr((void *) p); \ - if (!chunk) \ - return 0; \ - else { \ - union erts_qa_##NAME##__ *up; \ - ERTS_SPPA_DBG_CHK_IN_CHNK(qa_data_##NAME##__, chunk, p); \ - up = ((union erts_qa_##NAME##__ *) \ - (((char *) p) \ - - ((char *) &((union erts_qa_##NAME##__ *) 0)->type))); \ - erts_smp_spin_lock(&chunk->lock); \ - ERTS_PRE_ALLOC_CLOBBER(up, union erts_qa_##NAME##__); \ - up->next = chunk->freelist; \ - chunk->freelist = up; \ - erts_smp_spin_unlock(&chunk->lock); \ - return 1; \ - } \ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); \ + return erts_sspa_free(sspa_data_##NAME##__, \ + esdp ? (int) esdp->no - 1 : -1, \ + (char *) p); \ } #ifdef DEBUG diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index eda0831441..7a27ee28ec 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -133,29 +133,25 @@ class SYSTEM system_data # should be deallocated before the emulator starts executing Erlang # code again. # -# NOTE: When adding or removing a type which uses the FIXED_SIZE allocator, -# also add or remove initialization of the type in erts_alloc_init() -# (erl_alloc.c). -# # type SBMBC SBMBC SYSTEM small_block_mbc type PROC FIXED_SIZE PROCESSES proc -type ATOM FIXED_SIZE ATOM atom_entry -type MODULE FIXED_SIZE CODE module_entry -type REG_PROC FIXED_SIZE PROCESSES reg_proc +type ATOM LONG_LIVED ATOM atom_entry +type MODULE LONG_LIVED CODE module_entry +type REG_PROC STANDARD PROCESSES reg_proc type LINK_LH STANDARD PROCESSES link_lh type SUSPEND_MON STANDARD PROCESSES suspend_monitor type PEND_SUSPEND SHORT_LIVED PROCESSES pending_suspend type PROC_LIST SHORT_LIVED PROCESSES proc_list -type FUN_ENTRY FIXED_SIZE CODE fun_entry +type FUN_ENTRY LONG_LIVED CODE fun_entry type ATOM_TXT LONG_LIVED ATOM atom_text type BEAM_REGISTER EHEAP PROCESSES beam_register type HEAP EHEAP PROCESSES heap type OLD_HEAP EHEAP PROCESSES old_heap type HEAP_FRAG EHEAP PROCESSES heap_frag type TMP_HEAP TEMPORARY PROCESSES tmp_heap -type MSG_REF SHORT_LIVED PROCESSES msg_ref +type MSG_REF FIXED_SIZE PROCESSES msg_ref type MSG_ROOTS TEMPORARY PROCESSES msg_roots type ROOTSET TEMPORARY PROCESSES root_set type LOADER_TMP TEMPORARY CODE loader_tmp @@ -199,7 +195,7 @@ type TMP_DIST_BUF TEMPORARY SYSTEM tmp_dist_buf type ASYNC_Q LONG_LIVED SYSTEM async_queue type ESTACK TEMPORARY SYSTEM estack type PORT_CALL_BUF TEMPORARY SYSTEM port_call_buf -type DB_TABLE FIXED_SIZE ETS db_tab +type DB_TABLE ETS ETS db_tab type DB_FIXATION SHORT_LIVED ETS db_fixation type DB_FIX_DEL SHORT_LIVED ETS fixed_del type DB_TABLES LONG_LIVED ETS db_tabs @@ -256,6 +252,7 @@ type TMP_CPU_IDS SHORT_LIVED SYSTEM tmp_cpu_ids type EXT_TERM_DATA SHORT_LIVED PROCESSES external_term_data type ZLIB STANDARD SYSTEM zlib type CPU_GRPS_MAP LONG_LIVED SYSTEM cpu_groups_map +type AUX_WORK_TMO LONG_LIVED SYSTEM aux_work_timeouts +if smp type ASYNC SHORT_LIVED SYSTEM async @@ -273,6 +270,8 @@ type PROC_LCK_QS LONG_LIVED SYSTEM proc_lock_queues type RUNQ_BLNS LONG_LIVED SYSTEM run_queue_balancing type MISC_AUX_WORK_Q LONG_LIVED SYSTEM misc_aux_work_q type MISC_AUX_WORK SHORT_LIVED SYSTEM misc_aux_work +type THR_PRGR_IDATA LONG_LIVED SYSTEM thr_prgr_internal_data +type THR_PRGR_DATA LONG_LIVED SYSTEM thr_prgr_data +endif # @@ -346,10 +345,10 @@ type DB_MS_PSDO_PROC LONG_LIVED_LOW ETS db_match_pseudo_proc type SCHDLR_DATA LONG_LIVED_LOW SYSTEM scheduler_data type LL_TEMP_TERM LONG_LIVED_LOW SYSTEM ll_temp_term -# no FIXED_SIZE for low memory -type EXPORT STANDARD_LOW CODE export_entry +type EXPORT LONG_LIVED_LOW CODE export_entry type MONITOR_SH STANDARD_LOW PROCESSES monitor_sh type NLINK_SH STANDARD_LOW PROCESSES nlink_sh +type AINFO_REQ STANDARD_LOW SYSTEM alloc_info_request +else # "fullword" @@ -362,9 +361,10 @@ type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data type LL_TEMP_TERM LONG_LIVED SYSTEM ll_temp_term -type EXPORT FIXED_SIZE CODE export_entry +type EXPORT LONG_LIVED CODE export_entry type MONITOR_SH FIXED_SIZE PROCESSES monitor_sh type NLINK_SH FIXED_SIZE PROCESSES nlink_sh +type AINFO_REQ SHORT_LIVED SYSTEM alloc_info_request +endif diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index d51ed0c36d..daf8822564 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -46,6 +46,7 @@ #include "erl_alloc_util.h" #include "erl_mseg.h" #include "erl_threads.h" +#include "erl_thr_progress.h" #ifdef ERTS_ENABLE_LOCK_COUNT #include "erl_lock_count.h" @@ -61,6 +62,13 @@ #warning "* * * * * * * * * *" #endif +#define ERTS_ALCU_DD_OPS_LIM_HIGH 20 +#define ERTS_ALCU_DD_OPS_LIM_LOW 2 + +/* Fix alloc limit */ +#define ERTS_ALCU_FIX_MAX_LIST_SZ 1000 +#define ERTS_ALC_FIX_MAX_SHRINK_OPS 30 + #define ALLOC_ZERO_EQ_NULL 0 static int atoms_initialized = 0; @@ -269,7 +277,6 @@ static void check_blk_carrier(Allctr_t *, Block_t *); #define HARD_CHECK_BLK_CARRIER(A, B) #endif - /* Statistics updating ... */ #ifdef DEBUG @@ -465,26 +472,36 @@ do { \ #ifdef DEBUG #ifdef USE_THREADS -#define ERTS_ALCU_DBG_CHK_THR_SPEC(A) \ +#define ERTS_ALCU_DBG_CHK_THR_ACCESS(A) \ do { \ if (!(A)->thread_safe) { \ - if (!(A)->debug.saved_tid) \ + if (!(A)->debug.saved_tid) { \ (A)->debug.tid = erts_thr_self(); \ + (A)->debug.saved_tid = 1; \ + } \ else { \ - ASSERT(ethr_equal_tids((A)->debug.tid, erts_thr_self())); \ + ERTS_SMP_LC_ASSERT( \ + ethr_equal_tids((A)->debug.tid, erts_thr_self()) \ + || erts_is_system_blocked(ERTS_BS_FLG_ALLOW_GC) \ + || (ERTS_IS_CRASH_DUMPING \ + && erts_is_system_blocked(ERTS_BS_FLG_ALLOW_GC))); \ } \ } \ } while (0) #else -#define ERTS_ALCU_DBG_CHK_THR_SPEC(A) +#define ERTS_ALCU_DBG_CHK_THR_ACCESS(A) #endif #else -#define ERTS_ALCU_DBG_CHK_THR_SPEC(A) +#define ERTS_ALCU_DBG_CHK_THR_ACCESS(A) #endif static void make_name_atoms(Allctr_t *allctr); +static Block_t *create_carrier(Allctr_t *, Uint, UWord); +static void destroy_carrier(Allctr_t *, Block_t *); +static void mbc_free(Allctr_t *allctr, void *p); + /* mseg ... */ @@ -651,6 +668,446 @@ static void destroy_sbmbc(Allctr_t *allctr, Block_t *blk); static Block_t *create_carrier(Allctr_t *, Uint, UWord); static void destroy_carrier(Allctr_t *, Block_t *); +#if 0 +#define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) \ + do { if ((FIX)) chk_fix_list((A), (FIX), (IX), (B)); } while (0) +static void +chk_fix_list(Allctr_t *allctr, ErtsAlcFixList_t *fix, int ix, int before) +{ + void *p; + int n; + for (n = 0, p = fix[ix].list; p; p = *((void **) p)) + n++; + if (n != fix[ix].list_size) { + erts_fprintf(stderr, "FOUND IT ts=%d, sched=%d, ix=%d, n=%d, ls=%d %s!\n", + allctr->thread_safe, allctr->ix, ix, n, fix[ix].list_size, before ? "before" : "after"); + abort(); + } +} +#else +#define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) +#endif + +erts_aint32_t +erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) +{ + int all_empty = 1; + erts_aint32_t res = 0; + int ix, o; + ErtsAlcFixList_t *fix = allctr->fix; + int flush = flgs == 0; + +#ifdef USE_THREADS + if (allctr->thread_safe) + erts_mtx_lock(&allctr->mutex); +#endif + + for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) { + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); + if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) { + fix[ix].limit = fix[ix].max_used; + if (fix[ix].limit < fix[ix].used) + fix[ix].limit = fix[ix].used; + fix[ix].max_used = fix[ix].used; + ASSERT(fix[ix].limit >= 0); + + } + if (flush) { + fix[ix].limit = 0; + fix[ix].max_used = fix[ix].used; + ASSERT(fix[ix].limit >= 0); + } + for (o = 0; o < ERTS_ALC_FIX_MAX_SHRINK_OPS || flush; o++) { + Block_t *blk; + void *ptr; + + if (!flush && fix[ix].limit >= fix[ix].allocated) + break; + if (fix[ix].list_size == 0) + break; + ptr = fix[ix].list; + fix[ix].list = *((void **) ptr); + fix[ix].list_size--; + + blk = UMEM2BLK(ptr); + + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk); + else + mbc_free(allctr, ptr); + + fix[ix].allocated--; + } + if (fix[ix].list_size != 0) { + if (fix[ix].limit < fix[ix].allocated) + res |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC; + all_empty = 0; + } + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + } + + if (all_empty && allctr->fix_shrink_scheduled) { + allctr->fix_shrink_scheduled = 0; + erts_set_aux_work_timeout(allctr->ix, + (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), + 0); + } + +#ifdef USE_THREADS + if (allctr->thread_safe) + erts_mtx_unlock(&allctr->mutex); +#endif + + return res; +} + +#ifdef ERTS_SMP + +#define ERTS_ALCU_DD_FIX_TYPE_OFFS \ + ((sizeof(ErtsAllctrDDBlock_t)-1)/sizeof(UWord) + 1) + +#define ERTS_AU_PREF_ALLOC_IX_MASK \ + ((((UWord) 1) << ERTS_AU_PREF_ALLOC_BITS) - 1) +#define ERTS_AU_PREF_ALLOC_SIZE_MASK \ + ((((UWord) 1) << (sizeof(UWord)*8 - ERTS_AU_PREF_ALLOC_BITS)) - 1) + +static ERTS_INLINE int +get_pref_allctr(void *extra, Allctr_t **allctr) +{ + ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; + int pref_ix; + + pref_ix = ERTS_ALC_GET_THR_IX(); + + ASSERT(sizeof(UWord) == sizeof(Allctr_t *)); + ASSERT(0 <= pref_ix && pref_ix < tspec->size); + + *allctr = tspec->allctr[pref_ix]; + return pref_ix; +} + +static ERTS_INLINE void * +get_used_allctr(void *extra, void *p, Allctr_t **allctr, UWord *sizep) +{ + ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; + void *ptr = (void *) (((char *) p) - sizeof(UWord)); + UWord ainfo = *((UWord *) ptr); + int aix = (int) (ainfo & ERTS_AU_PREF_ALLOC_IX_MASK); + *allctr = tspec->allctr[aix]; + if (sizep) + *sizep = ((ainfo >> ERTS_AU_PREF_ALLOC_BITS) + & ERTS_AU_PREF_ALLOC_SIZE_MASK); + return ptr; +} + +static ERTS_INLINE void * +put_used_allctr(void *p, int ix, UWord size) +{ + UWord ainfo = (size >= ERTS_AU_PREF_ALLOC_SIZE_MASK + ? ERTS_AU_PREF_ALLOC_SIZE_MASK + : size); + ainfo <<= ERTS_AU_PREF_ALLOC_BITS; + ainfo |= (UWord) ix; + *((UWord *) p) = ainfo; + return (void *) (((char *) p) + sizeof(UWord)); +} + +static void +init_dd_queue(ErtsAllctrDDQueue_t *ddq) +{ + erts_atomic_init_nob(&ddq->tail.data.marker.atmc_next, ERTS_AINT_NULL); + erts_atomic_init_nob(&ddq->tail.data.last, + (erts_aint_t) &ddq->tail.data.marker); + erts_atomic_init_nob(&ddq->tail.data.um_refc[0], 0); + erts_atomic_init_nob(&ddq->tail.data.um_refc[1], 0); + erts_atomic32_init_nob(&ddq->tail.data.um_refc_ix, 0); + ddq->head.first = &ddq->tail.data.marker; + ddq->head.unref_end = &ddq->tail.data.marker; + ddq->head.next.thr_progress = erts_thr_progress_current(); + ddq->head.next.thr_progress_reached = 1; + ddq->head.next.um_refc_ix = 1; + ddq->head.next.unref_end = &ddq->tail.data.marker; + ddq->head.used_marker = 1; +} + +static ERTS_INLINE erts_aint_t +ddq_managed_thread_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr) +{ + erts_aint_t ilast, itmp; + ErtsAllctrDDBlock_t *this = ptr; + + erts_atomic_init_nob(&this->atmc_next, ERTS_AINT_NULL); + + /* Enqueue at end of list... */ + + ilast = erts_atomic_read_nob(&ddq->tail.data.last); + while (1) { + ErtsAllctrDDBlock_t *last = (ErtsAllctrDDBlock_t *) ilast; + itmp = erts_atomic_cmpxchg_mb(&last->atmc_next, + (erts_aint_t) this, + ERTS_AINT_NULL); + if (itmp == ERTS_AINT_NULL) + break; + ilast = itmp; + } + + /* Move last pointer forward... */ + while (1) { + if (erts_atomic_read_rb(&this->atmc_next) != ERTS_AINT_NULL) { + /* Someone else will move it forward */ + return erts_atomic_read_rb(&ddq->tail.data.last); + } + itmp = erts_atomic_cmpxchg_mb(&ddq->tail.data.last, + (erts_aint_t) this, + ilast); + if (ilast == itmp) + return (erts_aint_t) this; + ilast = itmp; + } +} + +static ERTS_INLINE int +ddq_enqueue(ErtsAlcType_t type, ErtsAllctrDDQueue_t *ddq, void *ptr) +{ + erts_aint_t ilast; + int um_refc_ix = 0; + int managed_thread = erts_thr_progress_is_managed_thread(); + if (!managed_thread) { + um_refc_ix = erts_atomic32_read_acqb(&ddq->tail.data.um_refc_ix); + while (1) { + int tmp_um_refc_ix; + erts_atomic_inc_acqb(&ddq->tail.data.um_refc[um_refc_ix]); + tmp_um_refc_ix = erts_atomic32_read_acqb(&ddq->tail.data.um_refc_ix); + if (tmp_um_refc_ix == um_refc_ix) + break; + erts_atomic_dec_relb(&ddq->tail.data.um_refc[um_refc_ix]); + um_refc_ix = tmp_um_refc_ix; + } + } + + ilast = ddq_managed_thread_enqueue(ddq, ptr); + + if (!managed_thread) + erts_atomic_dec_relb(&ddq->tail.data.um_refc[um_refc_ix]); + return ilast == (erts_aint_t) ptr; +} + +static ERTS_INLINE void * +ddq_dequeue(ErtsAllctrDDQueue_t *ddq) +{ + ErtsAllctrDDBlock_t *blk; + + if (ddq->head.first == ddq->head.unref_end) + return NULL; + + blk = ddq->head.first; + if (blk == &ddq->tail.data.marker) { + ASSERT(ddq->head.used_marker); + ddq->head.used_marker = 0; + blk = ((ErtsAllctrDDBlock_t *) + erts_atomic_read_nob(&blk->atmc_next)); + if (blk == ddq->head.unref_end) { + ddq->head.first = blk; + return NULL; + } + } + + ddq->head.first = ((ErtsAllctrDDBlock_t *) + erts_atomic_read_nob(&blk->atmc_next)); + + ASSERT(ddq->head.first); + + return (void *) blk; +} + +static int +ddq_check_incoming(ErtsAllctrDDQueue_t *ddq) +{ + erts_aint_t ilast = erts_atomic_read_nob(&ddq->tail.data.last); + if (((ErtsAllctrDDBlock_t *) ilast) == &ddq->tail.data.marker + && ddq->head.first == &ddq->tail.data.marker) { + /* Nothing more to do... */ + return 0; + } + + if (ddq->head.next.thr_progress_reached + || erts_thr_progress_has_reached(ddq->head.next.thr_progress)) { + int um_refc_ix; + ddq->head.next.thr_progress_reached = 1; + um_refc_ix = ddq->head.next.um_refc_ix; + if (erts_atomic_read_acqb(&ddq->tail.data.um_refc[um_refc_ix]) == 0) { + /* Move unreferenced end pointer forward... */ + + ddq->head.unref_end = ddq->head.next.unref_end; + + if (!ddq->head.used_marker + && ddq->head.unref_end == (ErtsAllctrDDBlock_t *) ilast) { + ddq->head.used_marker = 1; + ilast = ddq_managed_thread_enqueue(ddq, &ddq->tail.data.marker); + } + + if (ddq->head.unref_end == (ErtsAllctrDDBlock_t *) ilast) + ERTS_THR_MEMORY_BARRIER; + else { + ddq->head.next.unref_end = (ErtsAllctrDDBlock_t *) ilast; + ERTS_THR_MEMORY_BARRIER; + ddq->head.next.thr_progress = erts_thr_progress_later(); + erts_atomic32_set_relb(&ddq->tail.data.um_refc_ix, + um_refc_ix); + ddq->head.next.um_refc_ix = um_refc_ix == 0 ? 1 : 0; + ddq->head.next.thr_progress_reached = 0; + } + } + } + return 1; +} + +static ERTS_INLINE int +handle_delayed_dealloc(Allctr_t *allctr, + int allctr_locked, + int use_limit, + int ops_limit, + int *need_thr_progress, + int *need_more_work) +{ + int need_thr_prgr = 0; + int need_mr_wrk = 0; + int have_checked_incoming = 0; + int ops = 0; + ErtsAlcFixList_t *fix; + int res; + ErtsAllctrDDQueue_t *ddq; + + if (allctr->thread_safe && !allctr_locked) + erts_mtx_lock(&allctr->mutex); + + ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); + + fix = allctr->fix; + + ddq = &allctr->dd.q; + + res = 0; + + while (1) { + Block_t *blk; + void *ptr; + int ix; + + if (use_limit && ++ops > ops_limit) { + if (ddq->head.first != ddq->head.unref_end) { + need_mr_wrk = 1; + if (need_more_work) + *need_more_work |= 1; + } + break; + } + + dequeue: + ptr = ddq_dequeue(ddq); + if (!ptr) { + if (have_checked_incoming) + break; + need_thr_prgr = ddq_check_incoming(ddq); + if (need_thr_progress) + *need_thr_progress |= need_thr_prgr; + have_checked_incoming = 1; + goto dequeue; + } + + res = 1; + + INC_CC(allctr->calls.this_free); + + if (fix) { + ErtsAlcType_t type; + + type = (ErtsAlcType_t) ((UWord *) ptr)[ERTS_ALCU_DD_FIX_TYPE_OFFS]; + ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); + fix[ix].used--; + if (fix[ix].allocated < fix[ix].limit + && fix[ix].list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { + *((void **) ptr) = fix[ix].list; + fix[ix].list = ptr; + fix[ix].list_size++; + if (!allctr->fix_shrink_scheduled) { + allctr->fix_shrink_scheduled = 1; + erts_set_aux_work_timeout( + allctr->ix, + (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), + 1); + } + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + continue; + } + fix[ix].allocated--; + if (fix[ix].list && fix[ix].allocated > fix[ix].limit) { + blk = UMEM2BLK(ptr); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk); + else + mbc_free(allctr, ptr); + ptr = fix[ix].list; + fix[ix].list = *((void **) ptr); + fix[ix].list_size--; + fix[ix].allocated--; + } + } + + blk = UMEM2BLK(ptr); + + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk); + else + mbc_free(allctr, ptr); + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + } + + if (need_thr_progress && !(need_thr_prgr | need_mr_wrk)) { + need_thr_prgr = ddq_check_incoming(ddq); + *need_thr_progress |= need_thr_prgr; + } + + if (allctr->thread_safe && !allctr_locked) + erts_mtx_unlock(&allctr->mutex); + return res; +} + +static ERTS_INLINE void +enqueue_dealloc_other_instance(ErtsAlcType_t type, Allctr_t *allctr, void *ptr) +{ + if (allctr->fix) + ((UWord *) ptr)[ERTS_ALCU_DD_FIX_TYPE_OFFS] = (UWord) type; + + if (ddq_enqueue(type, &allctr->dd.q, ptr)) + erts_alloc_notify_delayed_dealloc(allctr->ix); +} + +#endif + +void +erts_alcu_check_delayed_dealloc(Allctr_t *allctr, + int limit, + int *need_thr_progress, + int *more_work) +{ +#ifdef ERTS_SMP + handle_delayed_dealloc(allctr, + 0, + limit, + ERTS_ALCU_DD_OPS_LIM_HIGH, + need_thr_progress, + more_work); +#endif +} + +#define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \ + handle_delayed_dealloc((Allctr), (Locked), 1, \ + ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL) + /* Multi block carrier alloc/realloc/free ... */ /* NOTE! mbc_alloc() may in case of memory shortage place the requested @@ -680,8 +1137,21 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp, Uint32 *alcu_flgsp) } } +#ifdef ERTS_SMP + if (allctr->dd.use) + ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); +#endif + blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0, *alcu_flgsp); +#ifdef ERTS_SMP + if (!blk && allctr->dd.use) { + if (ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1)) + blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0, + *alcu_flgsp); + } +#endif + if (!blk) { if ((*alcu_flgsp) & ERTS_ALCU_FLG_SBMBC) blk = create_sbmbc(allctr, get_blk_sz); @@ -939,6 +1409,11 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) Uint is_last_blk; #endif /* #ifndef MBC_REALLOC_ALWAYS_MOVES */ +#ifdef ERTS_SMP + if (allctr->dd.use) + ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); +#endif + ASSERT(p); ASSERT(size); ASSERT(size < allctr->sbc_threshold); @@ -1005,7 +1480,6 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) cand_blk, cand_blk_sz, alcu_flgs); - if (new_blk || cand_blk != blk) goto move_into_new_blk; } @@ -1441,7 +1915,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) goto try_sys_alloc; if (flags & CFLG_FORCE_MSEG) goto try_mseg; - if (erts_mseg_no() >= max_mseg_carriers) + if (erts_mseg_no(&allctr->mseg_opt) >= max_mseg_carriers) goto try_sys_alloc; if (flags & CFLG_SBC) { if (allctr->sbcs.curr.norm.mseg.no >= allctr->max_mseg_sbcs) @@ -1840,8 +2314,12 @@ static struct { Eterm ycs; /* Eterm sbmbcs; */ + + Eterm fix_types; + Eterm mbcs; Eterm sbcs; + Eterm sys_alloc_carriers_size; #if HAVE_ERTS_MSEG Eterm mseg_alloc_carriers_size; @@ -1871,6 +2349,8 @@ static struct { #endif } am; +static Eterm fix_type_atoms[ERTS_ALC_NO_FIXED_SIZES]; + static ERTS_INLINE void atom_init(Eterm *atom, char *name) { *atom = am_atom_put(name, strlen(name)); @@ -1891,6 +2371,7 @@ init_atoms(Allctr_t *allctr) erts_mtx_lock(&init_atoms_mtx); if (!atoms_initialized) { + int ix; #ifdef DEBUG Eterm *atom; @@ -1933,8 +2414,12 @@ init_atoms(Allctr_t *allctr) AM_INIT(ycs); /*AM_INIT(sbmbcs);*/ + + AM_INIT(fix_types); + AM_INIT(mbcs); AM_INIT(sbcs); + AM_INIT(sys_alloc_carriers_size); #if HAVE_ERTS_MSEG AM_INIT(mseg_alloc_carriers_size); @@ -1965,6 +2450,13 @@ init_atoms(Allctr_t *allctr) ASSERT(*atom != THE_NON_VALUE); } #endif + + for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) { + ErtsAlcType_t n = ERTS_ALC_N_MIN_A_FIXED_SIZE + ix; + char *name = (char *) ERTS_ALC_N2TD(n); + size_t len = strlen(name); + fix_type_atoms[ix] = am_atom_put(name, len); + } } @@ -2042,6 +2534,48 @@ add_4tup(Uint **hpp, Uint *szp, Eterm *lp, bld_cons(hpp, szp, bld_tuple(hpp, szp, 4, el1, el2, el3, el4), *lp); } +static Eterm +sz_info_fix(Allctr_t *allctr, + int *print_to_p, + void *print_to_arg, + Uint **hpp, + Uint *szp) +{ + Eterm res; + int ix; + ErtsAlcFixList_t *fix = allctr->fix; + + ASSERT(fix); + + res = NIL; + + for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) { + ErtsAlcType_t n = ix + ERTS_ALC_N_MIN_A_FIXED_SIZE; + Uint alloced = (fix[ix].type_size * fix[ix].allocated); + Uint used = fix[ix].type_size*fix[ix].used; + + if (print_to_p) { + int to = *print_to_p; + void *arg = print_to_arg; + erts_print(to, + arg, + "fix type: %s %bpu %bpu\n", + (char *) ERTS_ALC_N2TD(n), + alloced, + used); + } + + if (hpp || szp) { + add_3tup(hpp, szp, &res, + fix_type_atoms[ix], + bld_unstable_uint(hpp, szp, alloced), + bld_unstable_uint(hpp, szp, used)); + } + } + + return res; +} + static Eterm sz_info_carriers(Allctr_t *allctr, CarriersStats_t *cs, @@ -2590,7 +3124,7 @@ erts_alcu_sz_info(Allctr_t *allctr, Uint **hpp, Uint *szp) { - Eterm res, sbmbcs, mbcs, sbcs; + Eterm res, sbmbcs, mbcs, sbcs, fix = THE_NON_VALUE; res = THE_NON_VALUE; @@ -2607,6 +3141,8 @@ erts_alcu_sz_info(Allctr_t *allctr, erts_mtx_lock(&allctr->mutex); #endif + ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); + if (hpp || szp) ensure_atoms_initialized(allctr); @@ -2619,6 +3155,8 @@ erts_alcu_sz_info(Allctr_t *allctr, update_max_ever_values(&allctr->mbcs); update_max_ever_values(&allctr->sbcs); + if (allctr->fix) + fix = sz_info_fix(allctr, print_to_p, print_to_arg, hpp, szp); sbmbcs = sz_info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p, print_to_arg, hpp, szp); mbcs = sz_info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, @@ -2631,6 +3169,8 @@ erts_alcu_sz_info(Allctr_t *allctr, add_2tup(hpp, szp, &res, am.sbcs, sbcs); add_2tup(hpp, szp, &res, am.mbcs, mbcs); add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs); + if (allctr->fix) + add_2tup(hpp, szp, &res, am.fix_types, fix); } if (begin_max_period) { @@ -2656,7 +3196,7 @@ erts_alcu_info(Allctr_t *allctr, Uint **hpp, Uint *szp) { - Eterm res, sett, sbmbcs, mbcs, sbcs, calls; + Eterm res, sett, sbmbcs, mbcs, sbcs, calls, fix = THE_NON_VALUE; res = THE_NON_VALUE; @@ -2673,6 +3213,8 @@ erts_alcu_info(Allctr_t *allctr, erts_mtx_lock(&allctr->mutex); #endif + ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); + if (hpp || szp) ensure_atoms_initialized(allctr); @@ -2694,6 +3236,8 @@ erts_alcu_info(Allctr_t *allctr, } sett = info_options(allctr, print_to_p, print_to_arg, hpp, szp); + if (allctr->fix) + fix = sz_info_fix(allctr, print_to_p, print_to_arg, hpp, szp); sbmbcs = info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p, print_to_arg, hpp, szp); mbcs = info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, @@ -2709,6 +3253,8 @@ erts_alcu_info(Allctr_t *allctr, add_2tup(hpp, szp, &res, am.sbcs, sbcs); add_2tup(hpp, szp, &res, am.mbcs, mbcs); add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs); + if (allctr->fix) + add_2tup(hpp, szp, &res, am.fix_types, fix); add_2tup(hpp, szp, &res, am.options, sett); add_3tup(hpp, szp, &res, am.versions, @@ -2733,7 +3279,7 @@ erts_alcu_info(Allctr_t *allctr, void -erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size) +erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t *fi, int fisz) { #ifdef USE_THREADS @@ -2751,6 +3297,18 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size) size->blocks += allctr->sbmbcs.blocks.curr.size; size->blocks += allctr->sbcs.blocks.curr.size; + if (fi) { + int ix; + for (ix = 0; ix < fisz; ix++) { + if (allctr->fix) { + fi[ix].allocated += (allctr->fix[ix].type_size + * allctr->fix[ix].allocated); + fi[ix].used += (allctr->fix[ix].type_size + * allctr->fix[ix].used); + } + } + } + #ifdef USE_THREADS if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); @@ -2764,12 +3322,16 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) { Allctr_t *allctr = (Allctr_t *) extra; void *res; + ErtsAlcFixList_t *fix; ASSERT(initialized); ASSERT(allctr); - ERTS_ALCU_DBG_CHK_THR_SPEC(allctr); + ERTS_SMP_LC_ASSERT(!allctr->thread_safe + || erts_lc_mtx_is_locked(&allctr->mutex)); + + ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); #if ALLOC_ZERO_EQ_NULL if (!size) @@ -2778,18 +3340,61 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) INC_CC(allctr->calls.this_alloc); + fix = allctr->fix; + if (fix) { + int ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); + fix[ix].used++; + res = fix[ix].list; + if (res) { + fix[ix].list_size--; + fix[ix].list = *((void **) res); + if (fix[ix].list && fix[ix].allocated > fix[ix].limit) { + void *p = fix[ix].list; + Block_t *blk; + fix[ix].list = *((void **) p); + fix[ix].list_size--; + blk = UMEM2BLK(p); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk); + else + mbc_free(allctr, p); + fix[ix].allocated--; + } + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + return res; + } + if (size < 2*sizeof(UWord)) + size += sizeof(UWord); + if (fix[ix].limit < fix[ix].used) + fix[ix].limit = fix[ix].used; + if (fix[ix].max_used < fix[ix].used) + fix[ix].max_used = fix[ix].used; + fix[ix].allocated++; + } + if (size >= allctr->sbc_threshold) { + Block_t *blk; +#ifdef ERTS_SMP + if (allctr->dd.use) + ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); +#endif #if HALFWORD_HEAP - Block_t *blk = create_carrier(allctr, size, - CFLG_SBC | CFLG_FORCE_MSEG); + blk = create_carrier(allctr, size, + CFLG_SBC | CFLG_FORCE_MSEG); #else - Block_t *blk = create_carrier(allctr, size, CFLG_SBC); + blk = create_carrier(allctr, size, CFLG_SBC); #endif res = blk ? BLK2UMEM(blk) : NULL; } else res = mbc_alloc(allctr, size); + if (!res && fix) { + int ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; + fix[ix].allocated--; + fix[ix].used--; + } return res; } @@ -2818,29 +3423,28 @@ erts_alcu_alloc_ts(ErtsAlcType_t type, void *extra, Uint size) return res; } +#ifdef ERTS_SMP + void * erts_alcu_alloc_thr_spec(ErtsAlcType_t type, void *extra, Uint size) { ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; - int ix = erts_alc_get_thr_ix(); + int ix; Allctr_t *allctr; - int unlock; void *res; - ASSERT(ix > 0); - if (ix < tspec->size) { - allctr = tspec->allctr[ix]; - unlock = 0; - } - else { - allctr = tspec->allctr[0]; - unlock = 1; + ix = ERTS_ALC_GET_THR_IX(); + + ASSERT(0 <= ix && ix < tspec->size); + + allctr = tspec->allctr[ix]; + + if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); - } res = do_erts_alcu_alloc(type, allctr, size); - if (unlock) + if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); DEBUG_CHECK_ALIGNMENT(res); @@ -2851,51 +3455,96 @@ erts_alcu_alloc_thr_spec(ErtsAlcType_t type, void *extra, Uint size) void * erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size) { - ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; - int ix = erts_alc_get_thr_ix(); - Allctr_t *allctr; + int pref_ix; + Allctr_t *pref_allctr; void *res; - ASSERT(sizeof(UWord) == sizeof(Allctr_t *)); - ASSERT(ix > 0); - if (ix >= tspec->size) - ix = (ix % (tspec->size - 1)) + 1; - allctr = tspec->allctr[ix]; - erts_mtx_lock(&allctr->mutex); - res = do_erts_alcu_alloc(type, allctr, size + sizeof(UWord)); - if (res) { - *((Allctr_t **) res) = allctr; - res = (void *) (((char *) res) + sizeof(UWord)); - } - erts_mtx_unlock(&allctr->mutex); + pref_ix = get_pref_allctr(extra, &pref_allctr); + + if (pref_allctr->thread_safe) + erts_mtx_lock(&pref_allctr->mutex); + + ERTS_ALCU_DBG_CHK_THR_ACCESS(pref_allctr); + + res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(UWord)); + if (pref_allctr->thread_safe) + erts_mtx_unlock(&pref_allctr->mutex); + + if (res) + res = put_used_allctr(res, pref_ix, size); + DEBUG_CHECK_ALIGNMENT(res); + + return res; } #endif +#endif + /* ------------------------------------------------------------------------- */ static ERTS_INLINE void do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p) { + int ix; Allctr_t *allctr = (Allctr_t *) extra; ASSERT(initialized); ASSERT(allctr); - ERTS_ALCU_DBG_CHK_THR_SPEC(allctr); + ERTS_SMP_LC_ASSERT(!allctr->thread_safe + || erts_lc_mtx_is_locked(&allctr->mutex)); + + ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); if (p) { + ErtsAlcFixList_t *fix = allctr->fix; Block_t *blk; INC_CC(allctr->calls.this_free); + if (fix) { + ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); + fix[ix].used--; + if (fix[ix].allocated < fix[ix].limit + && fix[ix].list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { + *((void **) p) = fix[ix].list; + fix[ix].list = p; + fix[ix].list_size++; + if (!allctr->fix_shrink_scheduled) { + allctr->fix_shrink_scheduled = 1; + erts_set_aux_work_timeout( + allctr->ix, + (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), + 1); + } + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + return; + } + fix[ix].allocated--; + if (fix[ix].list && fix[ix].allocated > fix[ix].limit) { + blk = UMEM2BLK(p); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk); + else + mbc_free(allctr, p); + p = fix[ix].list; + fix[ix].list = *((void **) p); + fix[ix].list_size--; + fix[ix].allocated--; + } + } + blk = UMEM2BLK(p); if (IS_SBC_BLK(blk)) destroy_carrier(allctr, blk); else mbc_free(allctr, p); + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); } } @@ -2915,44 +3564,56 @@ erts_alcu_free_ts(ErtsAlcType_t type, void *extra, void *p) erts_mtx_unlock(&allctr->mutex); } +#ifdef ERTS_SMP + void erts_alcu_free_thr_spec(ErtsAlcType_t type, void *extra, void *p) { ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; - int ix = erts_alc_get_thr_ix(); - int unlock; + int ix; Allctr_t *allctr; - ASSERT(ix > 0); - if (ix < tspec->size) { - allctr = tspec->allctr[ix]; - unlock = 0; - } - else { - allctr = tspec->allctr[0]; - unlock = 1; + ix = ERTS_ALC_GET_THR_IX(); + + ASSERT(0 <= ix && ix < tspec->size); + + allctr = tspec->allctr[ix]; + + if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); - } do_erts_alcu_free(type, allctr, p); - if (unlock) + + if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); } void -erts_alcu_free_thr_pref(ErtsAlcType_t type, void *unused, void *p) +erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p) { if (p) { - void *ptr = (void *) (((char *) p) - sizeof(UWord)); - Allctr_t *allctr = *((Allctr_t **) ptr); - erts_mtx_lock(&allctr->mutex); - do_erts_alcu_free(type, allctr, ptr); - erts_mtx_unlock(&allctr->mutex); + Allctr_t *pref_allctr, *used_allctr; + void *ptr; + + get_pref_allctr(extra, &pref_allctr); + ptr = get_used_allctr(extra, p, &used_allctr, NULL); + if (pref_allctr != used_allctr) + enqueue_dealloc_other_instance(type, used_allctr, ptr); + else { + if (used_allctr->thread_safe) + erts_mtx_lock(&used_allctr->mutex); + ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr); + do_erts_alcu_free(type, used_allctr, ptr); + if (used_allctr->thread_safe) + erts_mtx_unlock(&used_allctr->mutex); + } } } #endif +#endif + /* ------------------------------------------------------------------------- */ static ERTS_INLINE void * @@ -2970,7 +3631,10 @@ do_erts_alcu_realloc(ErtsAlcType_t type, ASSERT(allctr); - ERTS_ALCU_DBG_CHK_THR_SPEC(allctr); + ERTS_SMP_LC_ASSERT(!allctr->thread_safe + || erts_lc_mtx_is_locked(&allctr->mutex)); + + ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); if (!p) { res = do_erts_alcu_alloc(type, extra, size); @@ -3063,6 +3727,10 @@ do_erts_alcu_realloc(ErtsAlcType_t type, } else { Block_t *new_blk; +#ifdef ERTS_SMP + if (allctr->dd.use) + ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); +#endif if(IS_SBC_BLK(blk)) { do_carrier_resize: #if HALFWORD_HEAP @@ -3166,30 +3834,29 @@ erts_alcu_realloc_mv_ts(ErtsAlcType_t type, void *extra, void *p, Uint size) return res; } +#ifdef ERTS_SMP + void * erts_alcu_realloc_thr_spec(ErtsAlcType_t type, void *extra, void *ptr, Uint size) { ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; - int ix = erts_alc_get_thr_ix(); + int ix; Allctr_t *allctr; - int unlock; void *res; - ASSERT(ix > 0); - if (ix < tspec->size) { - allctr = tspec->allctr[ix]; - unlock = 0; - } - else { - allctr = tspec->allctr[0]; - unlock = 1; + ix = ERTS_ALC_GET_THR_IX(); + + ASSERT(0 <= ix && ix < tspec->size); + + allctr = tspec->allctr[ix]; + + if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); - } res = do_erts_alcu_realloc(type, allctr, ptr, size, 0); - if (unlock) + if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); DEBUG_CHECK_ALIGNMENT(res); @@ -3202,26 +3869,22 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra, void *ptr, Uint size) { ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; - int ix = erts_alc_get_thr_ix(); + int ix; Allctr_t *allctr; - int unlock; void *res; - ASSERT(ix > 0); - if (ix < tspec->size) { - allctr = tspec->allctr[ix]; - unlock = 0; - } - else { - allctr = tspec->allctr[0]; - unlock = 1; - erts_mtx_lock(&allctr->mutex); - } + ix = ERTS_ALC_GET_THR_IX(); + ASSERT(0 <= ix && ix < tspec->size); + + allctr = tspec->allctr[ix]; + + if (allctr->thread_safe) + erts_mtx_lock(&allctr->mutex); res = do_erts_alcu_alloc(type, allctr, size); if (!res) { - if (unlock) + if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); res = erts_alcu_realloc_thr_spec(type, allctr, ptr, size); } @@ -3235,7 +3898,7 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra, cpy_size = size; sys_memcpy(res, ptr, cpy_size); do_erts_alcu_free(type, allctr, ptr); - if (unlock) + if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); } @@ -3244,129 +3907,101 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra, return res; } -void * -erts_alcu_realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size) +static ERTS_INLINE void * +realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size, + int force_move) { - ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; - int ix; + int pref_ix; void *ptr, *res; Allctr_t *pref_allctr, *used_allctr; + UWord old_user_size; if (!p) return erts_alcu_alloc_thr_pref(type, extra, size); - ptr = (void *) (((char *) p) - sizeof(UWord)); - used_allctr = *((Allctr_t **) ptr); + pref_ix = get_pref_allctr(extra, &pref_allctr); + ptr = get_used_allctr(extra, p, &used_allctr, &old_user_size); - ix = erts_alc_get_thr_ix(); - ASSERT(ix > 0); - if (ix >= tspec->size) - ix = (ix % (tspec->size - 1)) + 1; - pref_allctr = tspec->allctr[ix]; ASSERT(used_allctr && pref_allctr); - erts_mtx_lock(&used_allctr->mutex); - res = do_erts_alcu_realloc(type, - used_allctr, - ptr, - size + sizeof(UWord), - (pref_allctr != used_allctr - ? ERTS_ALCU_FLG_FAIL_REALLOC_MOVE - : 0)); - erts_mtx_unlock(&used_allctr->mutex); - if (res) { - ASSERT(used_allctr == *((Allctr_t **) res)); - res = (void *) (((char *) res) + sizeof(UWord)); - DEBUG_CHECK_ALIGNMENT(res); + if (!force_move && used_allctr == pref_allctr) { + if (used_allctr->thread_safe) + erts_mtx_lock(&used_allctr->mutex); + ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr); + res = do_erts_alcu_realloc(type, + used_allctr, + ptr, + size + sizeof(UWord), + 0); + if (used_allctr->thread_safe) + erts_mtx_unlock(&used_allctr->mutex); + if (res) + res = put_used_allctr(res, pref_ix, size); } else { - erts_mtx_lock(&pref_allctr->mutex); + if (pref_allctr->thread_safe) + erts_mtx_lock(&pref_allctr->mutex); res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(UWord)); - erts_mtx_unlock(&pref_allctr->mutex); + if (pref_allctr->thread_safe && (!force_move + || used_allctr != pref_allctr)) + erts_mtx_unlock(&pref_allctr->mutex); if (res) { Block_t *blk; size_t cpy_size; - *((Allctr_t **) res) = pref_allctr; - res = (void *) (((char *) res) + sizeof(UWord)); + res = put_used_allctr(res, pref_ix, size); DEBUG_CHECK_ALIGNMENT(res); - erts_mtx_lock(&used_allctr->mutex); blk = UMEM2BLK(ptr); - cpy_size = BLK_SZ(blk) - ABLK_HDR_SZ - sizeof(UWord); + if (old_user_size != ERTS_AU_PREF_ALLOC_SIZE_MASK) + cpy_size = old_user_size; + else { + if (used_allctr->thread_safe && (!force_move + || used_allctr != pref_allctr)) + erts_mtx_lock(&used_allctr->mutex); + ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&used_allctr->mutex)); + cpy_size = BLK_SZ(blk); + if (used_allctr->thread_safe && (!force_move + || used_allctr != pref_allctr)) + erts_mtx_unlock(&used_allctr->mutex); + cpy_size -= ABLK_HDR_SZ + sizeof(UWord); + } if (cpy_size > size) cpy_size = size; sys_memcpy(res, p, cpy_size); - do_erts_alcu_free(type, used_allctr, ptr); - erts_mtx_unlock(&used_allctr->mutex); + + if (!force_move || used_allctr != pref_allctr) + enqueue_dealloc_other_instance(type, used_allctr, ptr); + else { + do_erts_alcu_free(type, used_allctr, ptr); + ASSERT(pref_allctr == used_allctr); + if (pref_allctr->thread_safe) + erts_mtx_unlock(&pref_allctr->mutex); + } } } return res; } +void * +erts_alcu_realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size) +{ + return realloc_thr_pref(type, extra, p, size, 0); +} void * erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size) { - ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; - int ix; - void *ptr, *res; - Allctr_t *pref_allctr, *used_allctr; - - if (!p) - return erts_alcu_alloc_thr_pref(type, extra, size); - - ptr = (void *) (((char *) p) - sizeof(UWord)); - used_allctr = *((Allctr_t **) ptr); - - ix = erts_alc_get_thr_ix(); - ASSERT(ix > 0); - if (ix >= tspec->size) - ix = (ix % (tspec->size - 1)) + 1; - pref_allctr = tspec->allctr[ix]; - ASSERT(used_allctr && pref_allctr); - - erts_mtx_lock(&pref_allctr->mutex); - res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(UWord)); - if (!res) { - erts_mtx_unlock(&pref_allctr->mutex); - res = erts_alcu_realloc_thr_pref(type, extra, p, size); - } - else { - Block_t *blk; - size_t cpy_size; - Allctr_t *allctr; - - *((Allctr_t **) res) = pref_allctr; - res = (void *) (((char *) res) + sizeof(UWord)); - - DEBUG_CHECK_ALIGNMENT(res); - - if (used_allctr == pref_allctr) - allctr = pref_allctr; - else { - erts_mtx_unlock(&pref_allctr->mutex); - allctr = used_allctr; - erts_mtx_lock(&allctr->mutex); - } - - blk = UMEM2BLK(ptr); - cpy_size = BLK_SZ(blk) - ABLK_HDR_SZ - sizeof(UWord); - if (cpy_size > size) - cpy_size = size; - sys_memcpy(res, p, cpy_size); - do_erts_alcu_free(type, allctr, ptr); - erts_mtx_unlock(&allctr->mutex); - } - - return res; + return realloc_thr_pref(type, extra, p, size, 1); } #endif +#endif + /* ------------------------------------------------------------------------- */ int @@ -3381,6 +4016,10 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) sys_memcpy((void *) &allctr->mseg_opt, (void *) &erts_mseg_default_opt, sizeof(ErtsMsegOpt_t)); +#ifdef ERTS_SMP + if (init->tspec || init->tpref) + allctr->mseg_opt.sched_spec = 1; +#endif # if HALFWORD_HEAP allctr->mseg_opt.low_mem = init->low_mem; # endif @@ -3390,6 +4029,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (!allctr->name_prefix) goto error; + allctr->ix = init->ix; allctr->alloc_no = init->alloc_no; if (allctr->alloc_no < ERTS_ALC_A_MIN || ERTS_ALC_A_MAX < allctr->alloc_no) @@ -3431,6 +4071,18 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) goto error; allctr->min_block_size = UNIT_CEILING(allctr->min_block_size + sizeof(UWord)); +#if ERTS_SMP + if (init->tpref) { + Uint sz = sizeof(Block_t); + sz += ERTS_ALCU_DD_FIX_TYPE_OFFS*sizeof(UWord); + if (init->fix) + sz += sizeof(UWord); + sz = UNIT_CEILING(sz); + if (sz > allctr->min_block_size) + allctr->min_block_size = sz; + } +#endif + allctr->sbmbc_threshold = init->sbmbct; @@ -3493,7 +4145,8 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (allctr->mbc_header_size < sizeof(Carrier_t)) goto error; -#ifdef USE_THREADS +#ifdef ERTS_SMP + allctr->dd.use = 0; if (init->tpref) { allctr->mbc_header_size = (UNIT_CEILING(allctr->mbc_header_size + FBLK_FTR_SZ @@ -3507,6 +4160,9 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) + sizeof(UWord)) - ABLK_HDR_SZ - sizeof(UWord)); + + allctr->dd.use = 1; + init_dd_queue(&allctr->dd.q); } else #endif @@ -3548,6 +4204,21 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) } + if (init->fix) { + int i; + allctr->fix = init->fix; + allctr->fix_shrink_scheduled = 0; + for (i = 0; i < ERTS_ALC_NO_FIXED_SIZES; i++) { + allctr->fix[i].max_used = 0; + allctr->fix[i].limit = 0; + allctr->fix[i].type_size = init->fix_type_size[i]; + allctr->fix[i].list_size = 0; + allctr->fix[i].list = NULL; + allctr->fix[i].allocated = 0; + allctr->fix[i].used = 0; + } + } + return 1; error: diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index fed4d3dbe6..df560a0de2 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -20,10 +20,13 @@ #ifndef ERL_ALLOC_UTIL__ #define ERL_ALLOC_UTIL__ -#define ERTS_ALCU_VSN_STR "2.2" +#define ERTS_ALCU_VSN_STR "3.0" #include "erl_alloc_types.h" +#define ERTS_AU_PREF_ALLOC_BITS 11 +#define ERTS_AU_MAX_PREF_ALLOC_INSTANCES (1 << ERTS_AU_PREF_ALLOC_BITS) + typedef struct Allctr_t_ Allctr_t; typedef struct { @@ -35,6 +38,7 @@ typedef struct { char *name_prefix; ErtsAlcType_t alloc_no; int force; + int ix; int ts; int tspec; int tpref; @@ -53,6 +57,9 @@ typedef struct { UWord mbcgs; UWord sbmbct; UWord sbmbcs; + + void *fix; + size_t *fix_type_size; } AllctrInit_t; typedef struct { @@ -60,6 +67,11 @@ typedef struct { UWord carriers; } AllctrSize_t; +typedef struct { + UWord allocated; + UWord used; +} ErtsAlcUFixInfo_t; + #ifndef SMALL_MEMORY #define ERTS_DEFAULT_ALCU_INIT { \ @@ -71,6 +83,7 @@ typedef struct { NULL, \ ERTS_ALC_A_INVALID, /* (number) alloc_no: allocator number */\ 0, /* (bool) force: force enabled */\ + 0, /* (number) ix: instance index */\ 1, /* (bool) ts: thread safe */\ 0, /* (bool) tspec: thread specific */\ 0, /* (bool) tpref: thread preferred */\ @@ -88,7 +101,10 @@ typedef struct { 1024*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ 256, /* (bytes) sbmbct: small block mbc threshold */\ - 8*1024 /* (bytes) sbmbcs: small block mbc size */\ + 8*1024, /* (bytes) sbmbcs: small block mbc size */ \ + /* --- Data not options -------------------------------------------- */\ + NULL, /* (ptr) fix */\ + NULL /* (ptr) fix_type_size */\ } #else /* if SMALL_MEMORY */ @@ -102,6 +118,7 @@ typedef struct { NULL, \ ERTS_ALC_A_INVALID, /* (number) alloc_no: allocator number */\ 0, /* (bool) force: force enabled */\ + 0, /* (number) ix: instance index */\ 1, /* (bool) ts: thread safe */\ 0, /* (bool) tspec: thread specific */\ 0, /* (bool) tpref: thread preferred */\ @@ -118,7 +135,10 @@ typedef struct { 128*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ 256, /* (bytes) sbmbct: small block mbc threshold */\ - 8*1024 /* (bytes) sbmbcs: small block mbc size */\ + 8*1024, /* (bytes) sbmbcs: small block mbc size */ \ + /* --- Data not options -------------------------------------------- */\ + NULL, /* (ptr) fix */\ + NULL /* (ptr) fix_type_size */\ } #endif @@ -132,6 +152,7 @@ void * erts_alcu_alloc_ts(ErtsAlcType_t, void *, Uint); void * erts_alcu_realloc_ts(ErtsAlcType_t, void *, void *, Uint); void * erts_alcu_realloc_mv_ts(ErtsAlcType_t, void *, void *, Uint); void erts_alcu_free_ts(ErtsAlcType_t, void *, void *); +#ifdef ERTS_SMP void * erts_alcu_alloc_thr_spec(ErtsAlcType_t, void *, Uint); void * erts_alcu_realloc_thr_spec(ErtsAlcType_t, void *, void *, Uint); void * erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t, void *, void *, Uint); @@ -141,12 +162,16 @@ void * erts_alcu_realloc_thr_pref(ErtsAlcType_t, void *, void *, Uint); void * erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t, void *, void *, Uint); void erts_alcu_free_thr_pref(ErtsAlcType_t, void *, void *); #endif +#endif Eterm erts_alcu_au_info_options(int *, void *, Uint **, Uint *); Eterm erts_alcu_info_options(Allctr_t *, int *, void *, Uint **, Uint *); Eterm erts_alcu_sz_info(Allctr_t *, int, int *, void *, Uint **, Uint *); Eterm erts_alcu_info(Allctr_t *, int, int *, void *, Uint **, Uint *); void erts_alcu_init(AlcUInit_t *); -void erts_alcu_current_size(Allctr_t *, AllctrSize_t *); +void erts_alcu_current_size(Allctr_t *, AllctrSize_t *, + ErtsAlcUFixInfo_t *, int); +void erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, int *); +erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); #endif @@ -246,7 +271,74 @@ typedef struct { } blocks; } CarriersStats_t; +#ifdef ERTS_SMP + +typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t; + +union ErtsAllctrDDBlock_t_ { + erts_atomic_t atmc_next; + ErtsAllctrDDBlock_t *ptr_next; +}; + +typedef struct { + ErtsAllctrDDBlock_t marker; + erts_atomic_t last; + erts_atomic_t um_refc[2]; + erts_atomic32_t um_refc_ix; +} ErtsDDTail_t; + +typedef struct { + /* + * This structure needs to be cache line aligned for best + * performance. + */ + union { + /* Modified by threads returning memory to this allocator */ + ErtsDDTail_t data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsDDTail_t))]; + } tail; + /* + * Everything below this point is *only* accessed by the + * thread owning the allocator. + */ + struct { + ErtsAllctrDDBlock_t *first; + ErtsAllctrDDBlock_t *unref_end; + struct { + ErtsThrPrgrVal thr_progress; + int thr_progress_reached; + int um_refc_ix; + ErtsAllctrDDBlock_t *unref_end; + } next; + int used_marker; + } head; +} ErtsAllctrDDQueue_t; + +#endif + +typedef struct { + size_t type_size; + SWord list_size; + void *list; + SWord max_used; + SWord limit; + SWord allocated; + SWord used; +} ErtsAlcFixList_t; + struct Allctr_t_ { +#ifdef ERTS_SMP + struct { + /* + * We want the queue at the beginning of + * the Allctr_t struct, due to cache line + * alignment reasons. + */ + ErtsAllctrDDQueue_t q; + int use; + int ix; + } dd; +#endif /* Allocator name prefix */ char * name_prefix; @@ -254,6 +346,9 @@ struct Allctr_t_ { /* Allocator number */ ErtsAlcType_t alloc_no; + /* Instance index */ + int ix; + /* Alloc, realloc and free names as atoms */ struct { Eterm alloc; @@ -278,6 +373,7 @@ struct Allctr_t_ { Uint mbc_growth_stages; Uint sbmbc_threshold; Uint sbmbc_size; + #if HAVE_ERTS_MSEG ErtsMsegOpt_t mseg_opt; #endif @@ -315,6 +411,10 @@ struct Allctr_t_ { void (*check_mbc) (Allctr_t *, Carrier_t *); #endif + int fix_n_base; + int fix_shrink_scheduled; + ErtsAlcFixList_t *fix; + #ifdef USE_THREADS /* Mutex for this allocator */ erts_mtx_t mutex; @@ -323,6 +423,7 @@ struct Allctr_t_ { Allctr_t *prev; Allctr_t *next; } ts_list; + #endif int atoms_initialized; diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 90d8ea7300..5bdb752d3a 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -170,14 +170,18 @@ erts_aoffalc_start(AOFFAllctr_t *alc, AOFFAllctrInit_t* aoffinit, AllctrInit_t *init) { - AOFFAllctr_t nulled_state = {{0}}; - /* {{0}} is used instead of {0}, in order to avoid (an incorrect) gcc - warning. gcc warns if {0} is used as initializer of a struct when - the first member is a struct (not if, for example, the third member - is a struct). */ + struct { + int dummy; + AOFFAllctr_t allctr; + } zero = {0}; + /* The struct with a dummy element first is used in order to avoid (an + incorrect) gcc warning. gcc warns if {0} is used as initializer of + a struct when the first member is a struct (not if, for example, + the third member is a struct). */ + Allctr_t *allctr = (Allctr_t *) alc; - sys_memcpy((void *) alc, (void *) &nulled_state, sizeof(AOFFAllctr_t)); + sys_memcpy((void *) alc, (void *) &zero.allctr, sizeof(AOFFAllctr_t)); allctr->mbc_header_size = sizeof(Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c index f2199d41a1..c50fdeb4e8 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.c +++ b/erts/emulator/beam/erl_bestfit_alloc.c @@ -161,14 +161,18 @@ erts_bfalc_start(BFAllctr_t *bfallctr, BFAllctrInit_t *bfinit, AllctrInit_t *init) { - BFAllctr_t nulled_state = {{0}}; - /* {{0}} is used instead of {0}, in order to avoid (an incorrect) gcc - warning. gcc warns if {0} is used as initializer of a struct when - the first member is a struct (not if, for example, the third member - is a struct). */ + struct { + int dummy; + BFAllctr_t allctr; + } zero = {0}; + /* The struct with a dummy element first is used in order to avoid (an + incorrect) gcc warning. gcc warns if {0} is used as initializer of + a struct when the first member is a struct (not if, for example, + the third member is a struct). */ + Allctr_t *allctr = (Allctr_t *) bfallctr; - sys_memcpy((void *) bfallctr, (void *) &nulled_state, sizeof(BFAllctr_t)); + sys_memcpy((void *) bfallctr, (void *) &zero.allctr, sizeof(BFAllctr_t)); bfallctr->address_order = bfinit->ao; diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 6a74596f76..70d728340a 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -52,6 +52,9 @@ #include #endif +static Export* alloc_info_trap = NULL; +static Export* alloc_sizes_trap = NULL; + #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) /* Keep erts_system_version as a global variable for easy access from a core */ @@ -1633,9 +1636,19 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ sel = *tp++; - if (sel == am_allocator_sizes && arity == 2) { - return erts_allocator_info_term(BIF_P, *tp, 1); - } else if (sel == am_wordsize && arity == 2) { + if (sel == am_allocator_sizes) { + switch (arity) { + case 2: + ERTS_BIF_PREP_TRAP1(ret, alloc_sizes_trap, BIF_P, *tp); + return ret; + case 3: + if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 1)) + return am_true; + default: + goto badarg; + } + } + else if (sel == am_wordsize && arity == 2) { if (tp[0] == am_internal) { return make_small(sizeof(Eterm)); } @@ -1682,8 +1695,17 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ } else goto badarg; - } else if (sel == am_allocator && arity == 2) { - return erts_allocator_info_term(BIF_P, *tp, 0); + } else if (sel == am_allocator) { + switch (arity) { + case 2: + ERTS_BIF_PREP_TRAP1(ret, alloc_info_trap, BIF_P, *tp); + return ret; + case 3: + if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 0)) + return am_true; + default: + goto badarg; + } } else if (ERTS_IS_ATOM_STR("internal_cpu_topology", sel) && arity == 2) { return erts_get_cpu_topology_term(BIF_P, *tp); } else if (ERTS_IS_ATOM_STR("cpu_topology", sel) && arity == 2) { @@ -2546,8 +2568,12 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) res = erts_bld_uint(&hp, NULL, erts_dist_buf_busy_limit); BIF_RET(res); } else if (ERTS_IS_ATOM_STR("print_ethread_info", BIF_ARG_1)) { +#if defined(ETHR_NATIVE_ATOMIC32_IMPL) \ + || defined(ETHR_NATIVE_ATOMIC64_IMPL) \ + || defined(ETHR_NATIVE_DW_ATOMIC_IMPL) int i; char **str; +#endif #ifdef ETHR_NATIVE_ATOMIC32_IMPL erts_printf("32-bit native atomics: %s\n", ETHR_NATIVE_ATOMIC32_IMPL); @@ -2610,6 +2636,12 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) #endif BIF_RET(am_true); } +#ifdef ERTS_SMP + else if (ERTS_IS_ATOM_STR("thread_progress", BIF_ARG_1)) { + erts_thr_progress_dbg_print_state(); + BIF_RET(am_true); + } +#endif BIF_ERROR(BIF_P, BADARG); } @@ -3170,26 +3202,6 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } -BIF_RETTYPE memory_0(BIF_ALIST_0) -{ - BIF_RETTYPE res = erts_memory(NULL, NULL, BIF_P, THE_NON_VALUE); - switch (res) { - case am_badarg: BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); /* never... */ - case am_notsup: BIF_ERROR(BIF_P, EXC_NOTSUP); - default: BIF_RET(res); - } -} - -BIF_RETTYPE memory_1(BIF_ALIST_1) -{ - BIF_RETTYPE res = erts_memory(NULL, NULL, BIF_P, BIF_ARG_1); - switch (res) { - case am_badarg: BIF_ERROR(BIF_P, BADARG); - case am_notsup: BIF_ERROR(BIF_P, EXC_NOTSUP); - default: BIF_RET(res); - } -} - BIF_RETTYPE error_logger_warning_map_0(BIF_ALIST_0) { BIF_RET(erts_error_logger_warnings); @@ -3291,6 +3303,15 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(am_false); #endif } + else if (ERTS_IS_ATOM_STR("memory", BIF_ARG_1)) { + Eterm res; + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_block_system(0); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + res = erts_memory(NULL, NULL, BIF_P, THE_NON_VALUE); + erts_smp_release_system(); + BIF_RET(res); + } } else if (is_tuple(BIF_ARG_1)) { Eterm* tp = tuple_val(BIF_ARG_1); @@ -3493,6 +3514,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) static erts_smp_atomic_t hipe_test_reschedule_flag; + BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) { /* @@ -3766,6 +3788,13 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_ERROR(BIF_P, EXC_NOTSUP); #endif } + else if (ERTS_IS_ATOM_STR("wait", BIF_ARG_1)) { + if (ERTS_IS_ATOM_STR("deallocations", BIF_ARG_2)) { + if (erts_debug_wait_deallocations(BIF_P)) { + ERTS_BIF_YIELD_RETURN(BIF_P, am_ok); + } + } + } } BIF_ERROR(BIF_P, BADARG); @@ -4018,5 +4047,7 @@ erts_bif_info_init(void) erts_smp_atomic_init_nob(&available_internal_state, 0); erts_smp_atomic_init_nob(&hipe_test_reschedule_flag, 0); + alloc_info_trap = erts_export_put(am_erlang, am_alloc_info, 1); + alloc_sizes_trap = erts_export_put(am_erlang, am_alloc_sizes, 1); process_info_init(); } diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index 8322b233ac..e7d4ac2b67 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -190,16 +190,20 @@ erts_gfalc_start(GFAllctr_t *gfallctr, GFAllctrInit_t *gfinit, AllctrInit_t *init) { - GFAllctr_t nulled_state = {{0}}; - /* {{0}} is used instead of {0}, in order to avoid (an incorrect) gcc - warning. gcc warns if {0} is used as initializer of a struct when - the first member is a struct (not if, for example, the third member - is a struct). */ + struct { + int dummy; + GFAllctr_t allctr; + } zero = {0}; + /* The struct with a dummy element first is used in order to avoid (an + incorrect) gcc warning. gcc warns if {0} is used as initializer of + a struct when the first member is a struct (not if, for example, + the third member is a struct). */ + Allctr_t *allctr = (Allctr_t *) gfallctr; - init->sbmbct = 0; /* Small mbc not yet supported by goodfit */ + sys_memcpy((void *) gfallctr, (void *) &zero.allctr, sizeof(GFAllctr_t)); - sys_memcpy((void *) gfallctr, (void *) &nulled_state, sizeof(GFAllctr_t)); + init->sbmbct = 0; /* Small mbc not yet supported by goodfit */ allctr->mbc_header_size = sizeof(Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index eaa5f9eefc..ef86e6db5e 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -42,6 +42,7 @@ #include "erl_misc_utils.h" #include "packet_parser.h" #include "erl_cpu_topology.h" +#include "erl_thr_progress.h" #ifdef HIPE #include "hipe_mode_switch.h" /* for hipe_mode_switch_init() */ @@ -258,6 +259,8 @@ erl_init(int ncpu) no_schedulers, no_schedulers_online); erts_init_cpu_topology(); /* Must be after init_scheduling */ + erts_alloc_late_init(); + H_MIN_SIZE = erts_next_heap_size(H_MIN_SIZE, 0); BIN_VH_MIN_SIZE = erts_next_heap_size(BIN_VH_MIN_SIZE, 0); @@ -642,6 +645,9 @@ early_init(int *argc, char **argv) /* erts_use_r9_pids_ports = 0; erts_sys_pre_init(); +#ifdef ERTS_SMP + erts_thr_progress_pre_init(); +#endif #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_init(); @@ -765,11 +771,15 @@ early_init(int *argc, char **argv) /* erts_no_schedulers = (Uint) no_schedulers; #endif + erts_early_init_scheduling(no_schedulers); + alloc_opts.ncpu = ncpu; erts_alloc_init(argc, argv, &alloc_opts); /* Handles (and removes) -M flags. */ /* Require allocators */ - erts_early_init_scheduling(); +#ifdef ERTS_SMP + erts_thr_progress_init(no_schedulers, no_schedulers+1, 0); +#endif erts_init_utils(); erts_early_init_cpu_topology(no_schedulers, &max_main_threads, @@ -1521,9 +1531,6 @@ system_cleanup(int exit_code) #if defined(USE_THREADS) exit_async(); #endif -#if HAVE_ERTS_MSEG - erts_mseg_exit(); -#endif /* * A lot more cleaning could/should have been done... diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 587d82f2bb..02d1407a2d 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -125,7 +125,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "drv_ev_state", "address" }, { "safe_hash", "address" }, { "pollset_rm_list", NULL }, - { "removed_fd_pre_alloc_lock", NULL }, + { "removed_fd_pre_alloc_lock", "address" }, { "state_prealloc", NULL }, { "schdlr_sspnd", NULL }, { "run_queue", "address" }, @@ -151,10 +151,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "mtrace_op", NULL }, { "instr_x", NULL }, { "instr", NULL }, - { "fix_alloc", "index" }, { "alcu_allocator", "index" }, { "sbmbc_alloc", "index" }, - { "alcu_delayed_free", "index" }, { "mseg", NULL }, #if HALFWORD_HEAP { "pmmap", NULL }, @@ -183,7 +181,6 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "sched_stat", NULL }, { "run_queue_sleep_list", "address" }, #endif - { "alloc_thr_ix_lock", NULL }, #ifdef ERTS_SMP { "proc_lck_qs_alloc", NULL }, #endif diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c index b1478758a1..358c67bf20 100644 --- a/erts/emulator/beam/erl_mtrace.c +++ b/erts/emulator/beam/erl_mtrace.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2010. All Rights Reserved. + * Copyright Ericsson AB 2003-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -503,12 +503,6 @@ write_trace_header(char *nodename, char *pid, char *hostname) case ERTS_ALC_A_SYSTEM: PUT_UI16(tracep, ERTS_MTRACE_SEGMENT_ID); break; - case ERTS_ALC_A_FIXED_SIZE: - if (erts_allctrs_info[ERTS_FIX_CORE_ALLOCATOR].enabled) - PUT_UI16(tracep, ERTS_FIX_CORE_ALLOCATOR); - else - PUT_UI16(tracep, ERTS_ALC_A_SYSTEM); - break; default: PUT_UI16(tracep, ERTS_MTRACE_SEGMENT_ID); break; diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 9574435415..4dda17f559 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -39,6 +39,7 @@ #include "erl_binary.h" #include "beam_bp.h" #include "erl_cpu_topology.h" +#include "erl_thr_progress.h" #define ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED (2000*CONTEXT_REDS) #define ERTS_RUNQ_CALL_CHECK_BALANCE_REDS \ @@ -127,6 +128,8 @@ ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE]; int erts_disable_proc_not_running_opt; +static ErtsAuxWorkData *aux_thread_aux_work_data; + #define ERTS_SCHDLR_SSPND_CHNG_WAITER (((erts_aint32_t) 1) << 0) #define ERTS_SCHDLR_SSPND_CHNG_MSB (((erts_aint32_t) 1) << 1) #define ERTS_SCHDLR_SSPND_CHNG_ONLN (((erts_aint32_t) 1) << 2) @@ -152,6 +155,7 @@ do { \ static struct { erts_smp_mtx_t mtx; erts_smp_cnd_t cnd; + int aux_thread; int online; int curr_online; int wait_curr_online; @@ -213,8 +217,6 @@ Uint erts_no_run_queues; ErtsAlignedSchedulerData *erts_aligned_scheduler_data; -#ifdef ERTS_SMP - typedef union { ErtsSchedulerSleepInfo ssi; char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerSleepInfo))]; @@ -222,8 +224,6 @@ typedef union { static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info; -#endif - #ifndef BM_COUNTERS static int processes_busy; #endif @@ -285,8 +285,9 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist, ERTS_ALC_T_PROC_LIST) #define ERTS_SCHED_SLEEP_INFO_IX(IX) \ - (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers), \ - &aligned_sched_sleep_info[(IX)].ssi) + (ASSERT_EXPR(-1 <= ((int) (IX)) \ + && ((int) (IX)) < ((int) erts_no_schedulers)), \ + &aligned_sched_sleep_info[(IX)].ssi) #define ERTS_FOREACH_RUNQ(RQVAR, DO) \ do { \ @@ -339,6 +340,57 @@ static void exec_misc_ops(ErtsRunQueue *); static void print_function_from_pc(int to, void *to_arg, BeamInstr* x); static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg); + +static void aux_work_timeout(void *unused); +static void aux_work_timeout_early_init(int no_schedulers); +static void aux_work_timeout_late_init(void); +static void setup_aux_work_timer(void); + +#if defined(DEBUG) || 0 +#define ERTS_DBG_CHK_AUX_WORK_VAL(V) dbg_chk_aux_work_val((V)) +static void +dbg_chk_aux_work_val(erts_aint32_t value) +{ + erts_aint32_t valid = 0; + +#ifdef ERTS_SSI_AUX_WORK_SET_TMO + valid |= ERTS_SSI_AUX_WORK_SET_TMO; +#endif +#ifdef ERTS_SSI_AUX_WORK_CHECK_CHILDREN + valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN; +#endif +#ifdef ERTS_SSI_AUX_WORK_MISC + valid |= ERTS_SSI_AUX_WORK_MISC; +#endif + +#ifdef ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM; +#endif +#ifdef ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC + valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC; +#endif +#ifdef ERTS_SSI_AUX_WORK_DD + valid |= ERTS_SSI_AUX_WORK_DD; +#endif +#ifdef ERTS_SSI_AUX_WORK_DD + valid |= ERTS_SSI_AUX_WORK_DD_THR_PRGR; +#endif +#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK + valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK; +#endif + + if (~valid & value) + erl_exit(ERTS_ABORT_EXIT, + "Invalid aux_work value found: 0x%x\n", + ~valid & value); +} +#define ERTS_DBG_CHK_SSI_AUX_WORK(SSI) \ + ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&(SSI)->aux_work)) +#else +#define ERTS_DBG_CHK_AUX_WORK_VAL(V) +#define ERTS_DBG_CHK_SSI_AUX_WORK(SSI) +#endif + #ifdef ERTS_SMP static void handle_pending_exiters(ErtsProcList *); @@ -570,6 +622,18 @@ erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data) #ifdef ERTS_SMP +static void +prepare_for_block(void *vrq) +{ + erts_smp_runq_unlock((ErtsRunQueue *) vrq); +} + +static void +resume_after_block(void *vrq) +{ + erts_smp_runq_lock((ErtsRunQueue *) vrq); +} + void erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags) { @@ -589,6 +653,68 @@ erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags) } } +#endif + +static ERTS_INLINE void +set_aux_work_flags_wakeup_nob(ErtsSchedulerSleepInfo *ssi, + erts_aint32_t flgs) +{ + erts_aint32_t old_flgs; + + ERTS_DBG_CHK_SSI_AUX_WORK(ssi); + + old_flgs = erts_atomic32_read_nob(&ssi->aux_work); + if ((old_flgs & flgs) == 0) { + + old_flgs = erts_atomic32_read_bor_nob(&ssi->aux_work, flgs); + + if ((old_flgs & flgs) == 0) { +#ifdef ERTS_SMP + erts_sched_poke(ssi); +#else + erts_sys_schedule_interrupt(1); +#endif + } + } +} + +#if 0 /* Currently not used */ + +static ERTS_INLINE void +set_aux_work_flags_wakeup_relb(ErtsSchedulerSleepInfo *ssi, + erts_aint32_t flgs) +{ + erts_aint32_t old_flgs; + + ERTS_DBG_CHK_SSI_AUX_WORK(ssi); + + old_flgs = erts_atomic32_read_bor_relb(&ssi->aux_work, flgs); + + if ((old_flgs & flgs) == 0) { +#ifdef ERTS_SMP + erts_sched_poke(ssi); +#else + erts_sys_schedule_interrupt(1); +#endif + } +} + +#endif + +static ERTS_INLINE erts_aint32_t +set_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs) +{ + return erts_atomic32_read_bor_nob(&ssi->aux_work, flgs); +} + +static ERTS_INLINE erts_aint32_t +unset_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs) +{ + return erts_atomic32_read_band_nob(&ssi->aux_work, ~flgs); +} + +#ifdef ERTS_SMP + typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t; struct erts_misc_aux_work_t_ { erts_misc_aux_work_t *next; @@ -623,24 +749,27 @@ init_misc_aux_work(void) misc_aux_work_queues = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_MISC_AUX_WORK_Q, - erts_no_schedulers * - sizeof(erts_algnd_misc_aux_work_q_t)); + sizeof(erts_algnd_misc_aux_work_q_t) + * (erts_no_schedulers+1)); - for (ix = 0; ix < erts_no_schedulers; ix++) { + for (ix = 0; ix <= erts_no_schedulers; ix++) { erts_smp_mtx_init_x(&misc_aux_work_queues[ix].data.mtx, "misc_aux_work_queue", - make_small(ix + 1)); + make_small(ix)); misc_aux_work_queues[ix].data.first = NULL; misc_aux_work_queues[ix].data.last = NULL; } } -static void -handle_misc_aux_work(ErtsSchedulerData *esdp) +static erts_aint32_t +handle_misc_aux_work(ErtsAuxWorkData *awdp, + erts_aint32_t aux_work) { - int ix = (int) esdp->no - 1; + int ix = (int) awdp->sched_id; erts_misc_aux_work_t *mawp; + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC); + erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx); mawp = misc_aux_work_queues[ix].data.first; misc_aux_work_queues[ix].data.first = NULL; @@ -654,6 +783,35 @@ handle_misc_aux_work(ErtsSchedulerData *esdp) mawp = mawp->next; misc_aux_work_free(free_mawp); } + + return aux_work & ~ERTS_SSI_AUX_WORK_MISC; +} + +static void +smp_schedule_misc_aux_work(int ix, + void (*func)(void *), + void *arg) +{ + erts_aint32_t aux_work; + erts_misc_aux_work_t *mawp; + ErtsSchedulerSleepInfo *ssi; + + mawp = misc_aux_work_alloc(); + + mawp->func = func; + mawp->arg = arg; + mawp->next = NULL; + + erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx); + if (!misc_aux_work_queues[ix].data.last) + misc_aux_work_queues[ix].data.first = mawp; + else + misc_aux_work_queues[ix].data.last->next = mawp; + misc_aux_work_queues[ix].data.last = mawp; + erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx); + + set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(ix-1), + ERTS_SSI_AUX_WORK_MISC); } void @@ -667,111 +825,425 @@ erts_smp_schedule_misc_aux_work(int ignore_self, if (ignore_self) { ErtsSchedulerData *esdp = erts_get_scheduler_data(); if (esdp) - ignore_ix = (int) esdp->no - 1; + ignore_ix = (int) esdp->no; } - ASSERT(0 <= max_sched && max_sched <= erts_no_schedulers); + ASSERT(0 < max_sched && max_sched <= erts_no_schedulers); - for (ix = 0; ix < max_sched; ix++) { - erts_aint32_t aux_work; - erts_misc_aux_work_t *mawp; - ErtsSchedulerSleepInfo *ssi; + for (ix = 1; ix <= max_sched; ix++) { if (ix == ignore_ix) continue; + smp_schedule_misc_aux_work(ix, func, arg); + } +} - mawp = misc_aux_work_alloc(); +#endif - mawp->func = func; - mawp->arg = arg; - mawp->next = NULL; +static erts_aint32_t +handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + ErtsSchedulerSleepInfo *ssi = awdp->ssi; + erts_aint32_t res; - erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx); - if (!misc_aux_work_queues[ix].data.last) - misc_aux_work_queues[ix].data.first = mawp; - else - misc_aux_work_queues[ix].data.last->next = mawp; - misc_aux_work_queues[ix].data.last = mawp; - erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx); - - ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); - aux_work = erts_smp_atomic32_read_bor_nob(&ssi->aux_work, - ERTS_SSI_AUX_WORK_MISC); - if ((aux_work & ERTS_SSI_AUX_WORK_MISC) == 0) - erts_sched_poke(ssi); - } + unset_aux_work_flags(ssi, (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC)); + aux_work &= ~(ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC); + res = erts_alloc_fix_alloc_shrink(awdp->sched_id, aux_work); + if (res) { + set_aux_work_flags(ssi, res); + aux_work |= res; + } + + return aux_work; +} + +#ifdef ERTS_SMP + +void +erts_alloc_notify_delayed_dealloc(int ix) +{ + set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(ix-1), + ERTS_SSI_AUX_WORK_DD); +} + +static erts_aint32_t +handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + ErtsSchedulerSleepInfo *ssi = awdp->ssi; + int need_thr_progress = 0; + int more_work = 0; + + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD); + erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp, + &need_thr_progress, + &more_work); + if (more_work) { + if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD) + & ERTS_SSI_AUX_WORK_DD_THR_PRGR) { + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); + aux_work &= ~ERTS_SSI_AUX_WORK_DD_THR_PRGR; + } + return aux_work; + } + + if (need_thr_progress) { + set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); + awdp->dd.thr_prgr = erts_thr_progress_later(); + erts_thr_progress_wakeup(awdp->esdp, awdp->dd.thr_prgr); + } + else if (awdp->dd.completed_callback) { + awdp->dd.completed_callback(awdp->dd.completed_arg); + awdp->dd.completed_callback = NULL; + awdp->dd.completed_arg = NULL; + } + return aux_work & ~ERTS_SSI_AUX_WORK_DD; } +static erts_aint32_t +handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + ErtsSchedulerSleepInfo *ssi; + int need_thr_progress; + int more_work; + + if (!erts_thr_progress_has_reached(awdp->dd.thr_prgr)) + return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR; + + ssi = awdp->ssi; + need_thr_progress = 0; + more_work = 0; + + erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp, + &need_thr_progress, + &more_work); + if (more_work) { + set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD); + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); + return ((aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR) + | ERTS_SSI_AUX_WORK_DD); + } + + if (need_thr_progress) { + awdp->dd.thr_prgr = erts_thr_progress_later(); + erts_thr_progress_wakeup(awdp->esdp, awdp->dd.thr_prgr); + } + else { + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); + if (awdp->dd.completed_callback) { + awdp->dd.completed_callback(awdp->dd.completed_arg); + awdp->dd.completed_callback = NULL; + awdp->dd.completed_arg = NULL; + } + } + + return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR; +} + +static erts_atomic32_t completed_dealloc_count; + +static void +completed_dealloc(void *vproc) +{ + if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == 0) { + erts_resume((Process *) vproc, (ErtsProcLocks) 0); + erts_smp_proc_dec_refc((Process *) vproc); + } +} + +static void +setup_completed_dealloc(void *vproc) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsAuxWorkData *awdp = (esdp + ? &esdp->aux_work_data + : aux_thread_aux_work_data); + erts_alloc_fix_alloc_shrink(awdp->sched_id, 0); + set_aux_work_flags_wakeup_nob(awdp->ssi, ERTS_SSI_AUX_WORK_DD); + awdp->dd.completed_callback = completed_dealloc; + awdp->dd.completed_arg = vproc; +} + +static void +prep_setup_completed_dealloc(void *vproc) +{ + erts_aint32_t count = (erts_aint32_t) (erts_no_schedulers+1); + if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == count) { + /* scheduler threads */ + erts_smp_schedule_misc_aux_work(0, + erts_no_schedulers, + setup_completed_dealloc, + vproc); + /* aux_thread */ + smp_schedule_misc_aux_work(0, + setup_completed_dealloc, + vproc); + } +} + +#endif /* ERTS_SMP */ + +int +erts_debug_wait_deallocations(Process *c_p) +{ +#ifndef ERTS_SMP + erts_alloc_fix_alloc_shrink(1, 0); + return 1; +#else + /* Only one process at a time can do this */ + erts_aint32_t count = (erts_aint32_t) (2*(erts_no_schedulers+1)); + if (0 == erts_atomic32_cmpxchg_mb(&completed_dealloc_count, + count, + 0)) { + erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); + erts_smp_proc_inc_refc(c_p); + /* scheduler threads */ + erts_smp_schedule_misc_aux_work(0, + erts_no_schedulers, + prep_setup_completed_dealloc, + (void *) c_p); + /* aux_thread */ + smp_schedule_misc_aux_work(0, + prep_setup_completed_dealloc, + (void *) c_p); + return 1; + } + return 0; +#endif +} + + #ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN void erts_smp_notify_check_children_needed(void) { int i; + for (i = 0; i < erts_no_schedulers; i++) + set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(i), + ERTS_SSI_AUX_WORK_CHECK_CHILDREN); +} - for (i = 0; i < erts_no_schedulers; i++) { - erts_aint32_t aux_work; - ErtsSchedulerSleepInfo *ssi; - ssi = ERTS_SCHED_SLEEP_INFO_IX(i); - aux_work = erts_smp_atomic32_read_bor_nob(&ssi->aux_work, - ERTS_SSI_AUX_WORK_CHECK_CHILDREN); - if (!(aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN)) - erts_sched_poke(ssi); - } +static erts_aint32_t +handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_CHECK_CHILDREN); + erts_check_children(); + return aux_work & ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN; } + #endif -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK +#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK + +static erts_aint32_t +handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK); + erts_mseg_cache_check(); + return aux_work & ~ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK; +} + +#endif + +static erts_aint32_t +handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_SET_TMO); + setup_aux_work_timer(); + return aux_work & ~ERTS_SSI_AUX_WORK_SET_TMO; +} + static ERTS_INLINE erts_aint32_t -blockable_aux_work(ErtsSchedulerData *esdp, - ErtsSchedulerSleepInfo *ssi, - erts_aint32_t aux_work) +handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { - if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { - if (aux_work & ERTS_SSI_AUX_WORK_MISC) { - aux_work = erts_smp_atomic32_read_band_nob(&ssi->aux_work, - ~ERTS_SSI_AUX_WORK_MISC); - aux_work &= ~ERTS_SSI_AUX_WORK_MISC; - handle_misc_aux_work(esdp); - } + /* + * Handlers are *only* allowed to modify flags in return value + * and ssi flags that are explicity handled by the handler. + * Handlers are, e.g., not allowed to read the ssi flag field and + * then unconditionally return that value. + */ + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + if (aux_work & ERTS_SSI_AUX_WORK_SET_TMO) { + aux_work = handle_setup_aux_work_timer(awdp, aux_work); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + } +#ifdef ERTS_SMP + if (aux_work & ERTS_SSI_AUX_WORK_MISC) { + aux_work = handle_misc_aux_work(awdp, aux_work); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + } +#endif #ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) { - aux_work = erts_smp_atomic32_band_nob(&ssi->aux_work, - ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN); - aux_work &= ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN; - erts_check_children(); - } + if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) { + aux_work = handle_check_children(awdp, aux_work); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + } +#endif + if (aux_work & (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC)) { + aux_work = handle_fix_alloc(awdp, aux_work); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + } +#ifdef ERTS_SMP + if (aux_work & ERTS_SSI_AUX_WORK_DD) { + aux_work = handle_delayed_dealloc(awdp, aux_work); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + } + if (aux_work & ERTS_SSI_AUX_WORK_DD_THR_PRGR) { + aux_work = handle_delayed_dealloc_thr_prgr(awdp, aux_work); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + } #endif +#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK + if (aux_work & ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK) { + aux_work = handle_mseg_cache_check(awdp, aux_work); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); } +#endif + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); return aux_work; } -#endif +typedef struct { + union { + ErlTimer data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErlTimer))]; + } timer; -#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK -static ERTS_INLINE erts_aint32_t -nonblockable_aux_work(ErtsSchedulerData *esdp, - ErtsSchedulerSleepInfo *ssi, - erts_aint32_t aux_work) + int initialized; + erts_atomic32_t refc; + erts_atomic32_t type[1]; +} ErtsAuxWorkTmo; + +static ErtsAuxWorkTmo *aux_work_tmo; + +static void +aux_work_timeout_early_init(int no_schedulers) { - if (aux_work & ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK) { + int i; + UWord p; + + /* + * This is done really early. Our own allocators have + * not been started yet. + */ + + p = (UWord) malloc((sizeof(ErtsAuxWorkTmo) + + sizeof(erts_atomic32_t)*(no_schedulers+1)) + + ERTS_CACHE_LINE_SIZE-1); + if (p & ERTS_CACHE_LINE_MASK) + p = (p & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE; + ASSERT((p & ERTS_CACHE_LINE_MASK) == 0); + + aux_work_tmo = (ErtsAuxWorkTmo *) p; + aux_work_tmo->initialized = 0; + erts_atomic32_init_nob(&aux_work_tmo->refc, 0); + for (i = 0; i <= no_schedulers; i++) + erts_atomic32_init_nob(&aux_work_tmo->type[i], 0); +} +void +aux_work_timeout_late_init(void) +{ + aux_work_tmo->initialized = 1; + if (erts_atomic32_read_nob(&aux_work_tmo->refc)) { + aux_work_tmo->timer.data.active = 0; + erts_set_timer(&aux_work_tmo->timer.data, + aux_work_timeout, + NULL, + NULL, + 1000); } } -#endif static void -prepare_for_block(void *vrq) +aux_work_timeout(void *unused) { - erts_smp_runq_unlock((ErtsRunQueue *) vrq); + erts_aint32_t refc; + int i; +#ifdef ERTS_SMP + i = 0; +#else + i = 1; +#endif + + for (; i <= erts_no_schedulers; i++) { + erts_aint32_t type; + type = erts_atomic32_read_acqb(&aux_work_tmo->type[i]); + if (type) + set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(i-1), + type); + } + + refc = erts_atomic32_read_nob(&aux_work_tmo->refc); + ASSERT(refc >= 1); + if (refc != 1 + || 1 != erts_atomic32_cmpxchg_relb(&aux_work_tmo->refc, 0, 1)) { + /* Setup next timeout... */ + aux_work_tmo->timer.data.active = 0; + erts_set_timer(&aux_work_tmo->timer.data, + aux_work_timeout, + NULL, + NULL, + 1000); + } } static void -resume_after_block(void *vrq) +setup_aux_work_timer(void) { - erts_smp_runq_lock((ErtsRunQueue *) vrq); +#ifndef ERTS_SMP + if (!erts_get_scheduler_data()) + set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(0), + ERTS_SSI_AUX_WORK_SET_TMO); + else +#endif + { + aux_work_tmo->timer.data.active = 0; + erts_set_timer(&aux_work_tmo->timer.data, + aux_work_timeout, + NULL, + NULL, + 1000); + } } +erts_aint32_t +erts_set_aux_work_timeout(int ix, erts_aint32_t type, int enable) +{ + erts_aint32_t old, refc; + +#ifndef ERTS_SMP + ix = 1; #endif + ERTS_DBG_CHK_AUX_WORK_VAL(type); + ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&aux_work_tmo->type[ix])); +// erts_fprintf(stderr, "t(%d, 0x%x, %d)\n", ix, type, enable); + + if (!enable) { + old = erts_atomic32_read_band_mb(&aux_work_tmo->type[ix], ~type); + ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&aux_work_tmo->type[ix])); + if (old != 0 && (old & ~type) == 0) + erts_atomic32_dec_relb(&aux_work_tmo->refc); + return old; + } + + old = erts_atomic32_read_bor_mb(&aux_work_tmo->type[ix], type); + ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&aux_work_tmo->type[ix])); + if (old == 0 && type != 0) { + refc = erts_atomic32_inc_read_acqb(&aux_work_tmo->refc); + if (refc == 1) { + erts_atomic32_inc_acqb(&aux_work_tmo->refc); + if (aux_work_tmo->initialized) + setup_aux_work_timer(); + } + } + return old; +} + + + static ERTS_INLINE void sched_waiting_sys(Uint no, ErtsRunQueue *rq) { @@ -800,8 +1272,6 @@ sched_active_sys(Uint no, ErtsRunQueue *rq) Uint erts_active_schedulers(void) { - /* RRRRRRRRR */ - Uint as = erts_no_schedulers; ERTS_ATOMIC_FOREACH_RUNQ(rq, as -= abs(rq->waiting)); @@ -988,6 +1458,10 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type) if (sleep_type == ERTS_SSI_FLG_TSE_SLEEPING) erts_tse_reset(ssi->event); + else { + ASSERT(sleep_type == ERTS_SSI_FLG_POLL_SLEEPING); + erts_sys_schedule_interrupt(0); + } while (1) { oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); @@ -1006,16 +1480,96 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type) (((FLGS) & (ERTS_SSI_FLG_WAITING|ERTS_SSI_FLG_SUSPENDED)) \ != ERTS_SSI_FLG_WAITING) + +static void +poke_ssi(void *vssi) +{ + erts_sched_poke((ErtsSchedulerSleepInfo *) vssi); +} + +static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp); + +static void * +aux_thread(void *unused) +{ + ErtsAuxWorkData *awdp = aux_thread_aux_work_data; + ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(-1); + erts_aint32_t aux_work; + ErtsThrPrgrWakeupCallback wake_me; + int thr_prgr_active = 1; + + wake_me.wakeup = poke_ssi; + wake_me.arg = (void *) ssi; + + erts_thr_progress_register_managed_thread(NULL, &wake_me, 1); + init_aux_work_data(awdp, NULL); + awdp->ssi = ssi; + + erts_register_blockable_thread(); + + ssi->event = erts_tse_fetch(); + + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + schdlr_sspnd.aux_thread = 1; + erts_smp_cnd_signal(&schdlr_sspnd.cnd); + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + + sched_prep_spin_wait(ssi); + + while (1) { + erts_aint32_t flgs; + + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + if (aux_work) { + if (!thr_prgr_active) + erts_thr_progress_active(NULL, thr_prgr_active = 1); + aux_work = handle_aux_work(awdp, aux_work); + if (aux_work && erts_thr_progress_update(NULL)) + erts_thr_progress_leader_update(NULL); + } + + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + + if (!aux_work) { + if (thr_prgr_active) + erts_thr_progress_active(NULL, thr_prgr_active = 0); + erts_thr_progress_prepare_wait(NULL); + + flgs = sched_spin_wait(ssi, 0); + + if (flgs & ERTS_SSI_FLG_SLEEPING) { + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + int res; + ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } + } + erts_thr_progress_finalize_wait(NULL); + } + + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + + flgs = sched_prep_spin_wait(ssi); + } + return NULL; +} + +#endif /* ERTS_SMP */ + static void scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) { ErtsSchedulerSleepInfo *ssi = esdp->ssi; int spincount; + erts_aint32_t aux_work = 0; +#ifdef ERTS_SMP + int thr_prgr_active = 1; erts_aint32_t flgs; -#if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \ - || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) - erts_aint32_t aux_work; -#endif ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); @@ -1049,36 +1603,44 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) tse_wait: -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); - tse_blockable_aux_work: - aux_work = blockable_aux_work(esdp, ssi, aux_work); -#endif - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - while (1) { -#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK -#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); -#endif - nonblockable_aux_work(esdp, ssi, aux_work); -#endif + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + if (aux_work) { + if (!thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 1); + aux_work = handle_aux_work(&esdp->aux_work_data, aux_work); + if (aux_work && erts_thr_progress_update(esdp)) + erts_thr_progress_leader_update(esdp); + } - flgs = sched_spin_wait(ssi, spincount); - if (flgs & ERTS_SSI_FLG_SLEEPING) { - ASSERT(flgs & ERTS_SSI_FLG_WAITING); - flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + + if (aux_work) + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + else { + if (thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 0); + erts_thr_progress_prepare_wait(esdp); + + flgs = sched_spin_wait(ssi, spincount); if (flgs & ERTS_SSI_FLG_SLEEPING) { - int res; - ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); ASSERT(flgs & ERTS_SSI_FLG_WAITING); - do { - res = erts_tse_wait(ssi->event); - } while (res == EINTR); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + int res; + ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } } + erts_thr_progress_finalize_wait(esdp); } + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); break; @@ -1092,18 +1654,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) break; } -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); - if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - goto tse_blockable_aux_work; - } -#endif - } - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - if (flgs & ~ERTS_SSI_FLG_SUSPENDED) erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); @@ -1111,7 +1663,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) sched_active(esdp->no, rq); } - else { + else +#endif + { erts_aint_t dt; erts_smp_atomic32_set_relb(&function_calls, 0); @@ -1135,18 +1689,27 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) if (dt) erts_bump_timer(dt); sys_aux_work: - -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); - aux_work = blockable_aux_work(esdp, ssi, aux_work); +#ifndef ERTS_SMP + erts_sys_schedule_interrupt(0); #endif -#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK -#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); + + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + if (aux_work) { +#ifdef ERTS_SMP + if (!thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 1); #endif - nonblockable_aux_work(esdp, ssi, aux_work); + aux_work = handle_aux_work(&esdp->aux_work_data, aux_work); +#ifdef ERTS_SMP + if (aux_work && erts_thr_progress_update(esdp)) + erts_thr_progress_leader_update(esdp); #endif + } +#ifndef ERTS_SMP + if (rq->len != 0 || rq->misc.start) + goto sys_woken; +#else flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if (!(flgs & ERTS_SSI_FLG_WAITING)) { ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); @@ -1168,10 +1731,12 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) goto tse_wait; } } +#endif } erts_smp_runq_lock(rq); +#ifdef ERTS_SMP /* * If we got new I/O tasks we aren't allowed to * sleep in erl_sys_schedule(). @@ -1183,64 +1748,81 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * Got to check that we still got I/O tasks; otherwise * we have to wait in erl_sys_schedule() after all... */ - if (prepare_for_sys_schedule()) - goto do_sys_schedule; - - /* - * Not allowed to wait in erl_sys_schedule; - * do tse wait instead... - */ - sched_change_waiting_sys_to_waiting(esdp->no, rq); + if (!prepare_for_sys_schedule()) { + /* + * Not allowed to wait in erl_sys_schedule; + * do tse wait instead... + */ + sched_change_waiting_sys_to_waiting(esdp->no, rq); + erts_smp_runq_unlock(rq); + spincount = 0; + goto tse_wait; + } + } +#endif + if (aux_work) { erts_smp_runq_unlock(rq); - spincount = 0; - goto tse_wait; + goto sys_poll_aux_work; } - else { - do_sys_schedule: - erts_sys_schedule_interrupt(0); - flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING); - if (!(flgs & ERTS_SSI_FLG_SLEEPING)) { - if (!(flgs & ERTS_SSI_FLG_WAITING)) - goto sys_locked_woken; - erts_smp_runq_unlock(rq); - flgs = sched_prep_cont_spin_wait(ssi); - if (!(flgs & ERTS_SSI_FLG_WAITING)) { - ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); - goto sys_woken; - } - ASSERT(!erts_port_task_have_outstanding_io_tasks()); - goto sys_poll_aux_work; +#ifdef ERTS_SMP + if (thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 0); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING); + if (!(flgs & ERTS_SSI_FLG_SLEEPING)) { + if (!(flgs & ERTS_SSI_FLG_WAITING)) + goto sys_locked_woken; + erts_smp_runq_unlock(rq); + flgs = sched_prep_cont_spin_wait(ssi); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + goto sys_woken; } + ASSERT(!erts_port_task_have_outstanding_io_tasks()); + goto sys_poll_aux_work; + } - ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING); - ASSERT(flgs & ERTS_SSI_FLG_WAITING); + ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); +#endif - erts_smp_runq_unlock(rq); + erts_smp_runq_unlock(rq); - ASSERT(!erts_port_task_have_outstanding_io_tasks()); + ASSERT(!erts_port_task_have_outstanding_io_tasks()); - erl_sys_schedule(0); + erl_sys_schedule(0); - dt = erts_do_time_read_and_reset(); - if (dt) erts_bump_timer(dt); + dt = erts_do_time_read_and_reset(); + if (dt) erts_bump_timer(dt); - flgs = sched_prep_cont_spin_wait(ssi); - if (flgs & ERTS_SSI_FLG_WAITING) - goto sys_aux_work; +#ifndef ERTS_SMP + if (rq->len == 0 && !rq->misc.start) + goto sys_aux_work; + sys_woken: +#else + flgs = sched_prep_cont_spin_wait(ssi); + if (flgs & ERTS_SSI_FLG_WAITING) + goto sys_aux_work; - sys_woken: - erts_smp_runq_lock(rq); - sys_locked_woken: - clear_sys_scheduling(); - if (flgs & ~ERTS_SSI_FLG_SUSPENDED) - erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); - sched_active_sys(esdp->no, rq); - } + sys_woken: + erts_smp_runq_lock(rq); + sys_locked_woken: + clear_sys_scheduling(); + if (flgs & ~ERTS_SSI_FLG_SUSPENDED) + erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); +#endif + sched_active_sys(esdp->no, rq); } +#ifdef ERTS_SMP + if (!thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 1); + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); +#endif } +#ifdef ERTS_SMP + static ERTS_INLINE erts_aint32_t ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) { @@ -2547,8 +3129,9 @@ erts_debug_nbalance(void) } void -erts_early_init_scheduling(void) +erts_early_init_scheduling(int no_schedulers) { + aux_work_timeout_early_init(no_schedulers); wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM; } @@ -2569,12 +3152,24 @@ erts_sched_set_wakeup_limit(char *str) return EINVAL; return 0; } - + +static void +init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp) +{ + awdp->sched_id = esdp ? (int) esdp->no : 0; + awdp->esdp = esdp; + awdp->ssi = esdp ? esdp->ssi : NULL; +#ifdef ERTS_SMP + awdp->dd.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; + awdp->dd.completed_callback = NULL; + awdp->dd.completed_arg = NULL; +#endif +} void erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) { - int ix, n; + int ix, n, no_ssi; #ifndef ERTS_SMP mrq = 0; @@ -2684,23 +3279,31 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) n = (int) no_schedulers; erts_no_schedulers = n; -#ifdef ERTS_SMP /* Create and initialize scheduler sleep info */ - +#ifdef ERTS_SMP + no_ssi = n+1; +#else + no_ssi = 1; +#endif aligned_sched_sleep_info = - erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_SLP_INFO, - n * sizeof(ErtsAlignedSchedulerSleepInfo)); - - for (ix = 0; ix < n; ix++) { - ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); + erts_alloc_permanent_cache_aligned( + ERTS_ALC_T_SCHDLR_SLP_INFO, + no_ssi*sizeof(ErtsAlignedSchedulerSleepInfo)); + for (ix = 0; ix < no_ssi; ix++) { + ErtsSchedulerSleepInfo *ssi = &aligned_sched_sleep_info[ix].ssi; +#ifdef ERTS_SMP #if 0 /* no need to initialize these... */ ssi->next = NULL; ssi->prev = NULL; #endif erts_smp_atomic32_init_nob(&ssi->flags, 0); ssi->event = NULL; /* initialized in sched_thread_func */ - erts_smp_atomic32_init_nob(&ssi->aux_work, 0); +#endif + erts_atomic32_init_nob(&ssi->aux_work, 0); } + +#ifdef ERTS_SMP + aligned_sched_sleep_info++; #endif /* Create and initialize scheduler specific data */ @@ -2714,7 +3317,6 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) #ifdef ERTS_SMP erts_bits_init_state(&esdp->erl_bits_state); esdp->match_pseudo_process = NULL; - esdp->ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); esdp->free_process = NULL; #if HALFWORD_HEAP /* Registers need to be heap allocated (correct memory range) for tracing to work */ @@ -2725,6 +3327,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) esdp->num_tmp_heap_used = 0; #endif esdp->no = (Uint) ix+1; + esdp->ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); esdp->current_process = NULL; esdp->current_port = NULL; @@ -2745,13 +3348,23 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) #ifdef ERTS_SMP erts_smp_atomic32_init_nob(&esdp->chk_cpu_bind, 0); #endif + init_aux_work_data(&esdp->aux_work_data, esdp); } #ifdef ERTS_SMP + init_misc_aux_work(); + + erts_atomic32_init_nob(&completed_dealloc_count, 0); /* debug only */ + + aux_thread_aux_work_data = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, + sizeof(ErtsAuxWorkData)); + erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); erts_smp_cnd_init(&schdlr_sspnd.cnd); erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0); + schdlr_sspnd.aux_thread = 0; schdlr_sspnd.online = no_schedulers_online; schdlr_sspnd.curr_online = no_schedulers; schdlr_sspnd.msb.ongoing = 0; @@ -2809,6 +3422,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) /* init port tasks */ erts_port_task_init(); + aux_work_timeout_late_init(); + #ifndef ERTS_SMP #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC erts_scheduler_data->verify_unused_temp_alloc @@ -3058,10 +3673,8 @@ suspend_scheduler(ErtsSchedulerData *esdp) long active_schedulers; int curr_online = 1; int wake = 0; -#if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \ - || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) erts_aint32_t aux_work; -#endif + int thr_prgr_active = 1; /* * Schedulers may be suspended in two different ways: @@ -3134,40 +3747,46 @@ suspend_scheduler(ErtsSchedulerData *esdp) break; erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); - blockable_aux_work: - blockable_aux_work(esdp, ssi, aux_work); -#endif - - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); while (1) { erts_aint32_t flgs; -#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK -#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read(&ssi->aux_work); -#endif - nonblockable_aux_work(esdp, ssi, aux_work); -#endif - flgs = sched_spin_suspended(ssi, - ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); - if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED)) { - flgs = sched_set_suspended_sleeptype(ssi); + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + if (aux_work) { + if (!thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 1); + aux_work = handle_aux_work(&esdp->aux_work_data, aux_work); + if (aux_work && erts_thr_progress_update(esdp)) + erts_thr_progress_leader_update(esdp); + } + + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + + if (!aux_work) { + if (thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 0); + erts_thr_progress_prepare_wait(esdp); + flgs = sched_spin_suspended(ssi, + ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_TSE_SLEEPING | ERTS_SSI_FLG_WAITING | ERTS_SSI_FLG_SUSPENDED)) { - int res; - do { - res = erts_tse_wait(ssi->event); - } while (res == EINTR); + flgs = sched_set_suspended_sleeptype(ssi); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + int res; + + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } } + erts_thr_progress_finalize_wait(esdp); } + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING | ERTS_SSI_FLG_SUSPENDED)); if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) @@ -3175,19 +3794,8 @@ suspend_scheduler(ErtsSchedulerData *esdp) changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER) break; - - -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); - if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - goto blockable_aux_work; - } -#endif - } - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); erts_smp_mtx_lock(&schdlr_sspnd.mtx); changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); @@ -3213,6 +3821,9 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_active); + if (!thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 1); + erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); @@ -3720,8 +4331,14 @@ erts_multi_scheduling_blockers(Process *p) static void * sched_thread_func(void *vesdp) { + ErtsThrPrgrWakeupCallback wake_me; + ErtsSchedulerData *esdp = vesdp; + Uint no = esdp->no; #ifdef ERTS_SMP - Uint no = ((ErtsSchedulerData *) vesdp)->no; + wake_me.wakeup = poke_ssi; + wake_me.arg = (void *) esdp->ssi; + erts_thr_progress_register_managed_thread(esdp, &wake_me, 0); + erts_alloc_register_scheduler(vesdp); #endif #ifdef ERTS_ENABLE_LOCK_CHECK { @@ -3730,11 +4347,13 @@ sched_thread_func(void *vesdp) erts_lc_set_thread_name(&buf[0]); } #endif - erts_alloc_reg_scheduler_id(no); erts_tsd_set(sched_data_key, vesdp); #ifdef ERTS_SMP +#if HAVE_ERTS_MSEG + erts_mseg_late_init(); +#endif - erts_sched_init_check_cpu_bind((ErtsSchedulerData *) vesdp); + erts_sched_init_check_cpu_bind(esdp); erts_proc_lock_prepare_proc_lock_waiter(); ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch(); @@ -3754,17 +4373,18 @@ sched_thread_func(void *vesdp) if (--schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) { erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, ~ERTS_SCHDLR_SSPND_CHNG_ONLN); - if (((ErtsSchedulerData *) vesdp)->no != 1) + if (no != 1) erts_smp_cnd_signal(&schdlr_sspnd.cnd); } - if (((ErtsSchedulerData *) vesdp)->no == 1) { + if (no == 1) { if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) { erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, susp_sched_prep_block, susp_sched_resume_block, NULL); - while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) + while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online + || !schdlr_sspnd.aux_thread) erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); erts_smp_activity_end(ERTS_ACTIVITY_WAIT, susp_sched_prep_block, @@ -3776,19 +4396,22 @@ sched_thread_func(void *vesdp) erts_smp_mtx_unlock(&schdlr_sspnd.mtx); #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC - ((ErtsSchedulerData *) vesdp)->verify_unused_temp_alloc + esdp->verify_unused_temp_alloc = erts_alloc_get_verify_unused_temp_alloc( - &((ErtsSchedulerData *) vesdp)->verify_unused_temp_alloc_data); + &esdp->verify_unused_temp_alloc_data); ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL); #endif process_main(); /* No schedulers should *ever* terminate */ - erl_exit(ERTS_ABORT_EXIT, "Scheduler thread number %beu terminated\n", - ((ErtsSchedulerData *) vesdp)->no); + erl_exit(ERTS_ABORT_EXIT, + "Scheduler thread number %beu terminated\n", + no); return NULL; } +static ethr_tid aux_tid; + void erts_start_schedulers(void) { @@ -3810,6 +4433,10 @@ erts_start_schedulers(void) erts_block_system(0); + res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts); + if (res != 0) + erl_exit(1, "Failed to create aux thread\n"); + while (actual < wanted) { ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(actual); actual++; @@ -5355,23 +5982,19 @@ Process *schedule(Process *p, int calls) } } -#if defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) \ - || defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) { - ErtsSchedulerSleepInfo *ssi = esdp->ssi; - erts_aint32_t aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); - if (aux_work) { + erts_aint32_t aux_work; + int leader_update = erts_thr_progress_update(esdp); + aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work); + if (aux_work | leader_update) { erts_smp_runq_unlock(rq); -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = blockable_aux_work(esdp, ssi, aux_work); -#endif -#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK - nonblockable_aux_work(esdp, ssi, aux_work); -#endif + if (leader_update) + erts_thr_progress_leader_update(esdp); + if (aux_work) + handle_aux_work(&esdp->aux_work_data, aux_work); erts_smp_runq_lock(rq); } } -#endif erts_smp_chk_system_block(prepare_for_block, resume_after_block, @@ -5380,17 +6003,21 @@ Process *schedule(Process *p, int calls) ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); -#endif +#else /* ERTS_SMP */ + { + erts_aint32_t aux_work; + aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work); + if (aux_work) + handle_aux_work(&esdp->aux_work_data, aux_work); + } +#endif /* ERTS_SMP */ ASSERT(rq->len == rq->procs.len + rq->ports.info.len); -#ifndef ERTS_SMP + if (rq->len == 0 && !rq->misc.start) { - if (rq->len == 0 && !rq->misc.start) - goto do_sys_schedule; +#ifdef ERTS_SMP -#else /* ERTS_SMP */ - if (rq->len == 0 && !rq->misc.start) { ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); rq->wakeup_other = 0; @@ -5421,26 +6048,17 @@ Process *schedule(Process *p, int calls) } } +#endif + scheduler_wait(&fcalls, esdp, rq); +#ifdef ERTS_SMP non_empty_runq(rq); +#endif goto check_activities_to_run; } - else -#endif /* ERTS_SMP */ - if (fcalls > input_reductions && prepare_for_sys_schedule()) { - int runnable; - -#ifdef ERTS_SMP - runnable = 1; -#else - do_sys_schedule: - runnable = rq->len != 0; - if (!runnable) - sched_waiting_sys(esdp->no, rq); -#endif - + else if (fcalls > input_reductions && prepare_for_sys_schedule()) { /* * Schedule system-level activities. */ @@ -5450,11 +6068,11 @@ Process *schedule(Process *p, int calls) ASSERT(!erts_port_task_have_outstanding_io_tasks()); -#ifdef ERTS_SMP - /* erts_sys_schedule_interrupt(0); */ +#if 0 /* Not needed since we wont wait in sys schedule */ + erts_sys_schedule_interrupt(0); #endif erts_smp_runq_unlock(rq); - erl_sys_schedule(runnable); + erl_sys_schedule(1); dt = erts_do_time_read_and_reset(); if (dt) erts_bump_timer(dt); #ifdef ERTS_SMP @@ -5462,8 +6080,6 @@ Process *schedule(Process *p, int calls) clear_sys_scheduling(); goto continue_check_activities_to_run; #else - if (!runnable) - sched_active_sys(esdp->no, rq); goto check_activities_to_run; #endif } @@ -5778,20 +6394,9 @@ erts_sched_stat_term(Process *p, int total) void erts_schedule_misc_op(void (*func)(void *), void *arg) { - ErtsRunQueue *rq; - ErtsMiscOpList *molp = misc_op_list_alloc(); ErtsSchedulerData *esdp = erts_get_scheduler_data(); - - if (esdp) { - rq = esdp->run_queue; - } else { - /* - * This can only happen when the sys msg dispatcher - * thread schedules misc ops (this happens *very* - * seldom; only when trace drivers are unloaded). - */ - rq = ERTS_RUNQ_IX(0); - } + ErtsRunQueue *rq = esdp ? esdp->run_queue : ERTS_RUNQ_IX(0); + ErtsMiscOpList *molp = misc_op_list_alloc(); erts_smp_runq_lock(rq); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 739aef3130..43593f32d9 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -53,11 +53,17 @@ typedef struct process Process; #include "erl_time.h" #include "erl_atom_table.h" #include "external.h" +#include "erl_mseg.h" #ifdef HIPE #include "hipe_process.h" #endif +#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY +#define ERL_THR_PROGRESS_TSD_TYPE_ONLY +#include "erl_thr_progress.h" +#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY + struct ErtsNodesMonitor_; struct port; @@ -242,16 +248,20 @@ typedef enum { | ERTS_SSI_FLG_WAITING \ | ERTS_SSI_FLG_SUSPENDED) -#define ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - -#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 0) -#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 1) +#define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 0) +#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 1) +#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 2) +#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM (((erts_aint32_t) 1) << 3) +#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC (((erts_aint32_t) 1) << 4) +#ifdef ERTS_SMP +#define ERTS_SSI_AUX_WORK_DD (((erts_aint32_t) 1) << 5) +#define ERTS_SSI_AUX_WORK_DD_THR_PRGR (((erts_aint32_t) 1) << 6) +#endif +#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 7) -#define ERTS_SSI_BLOCKABLE_AUX_WORK_MASK \ - (ERTS_SSI_AUX_WORK_CHECK_CHILDREN \ - | ERTS_SSI_AUX_WORK_MISC) -#define ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK \ - (0) +#if !HAVE_ERTS_MSEG +# undef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK +#endif typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo; @@ -261,11 +271,13 @@ typedef struct { } ErtsSchedulerSleepList; struct ErtsSchedulerSleepInfo_ { +#ifdef ERTS_SMP ErtsSchedulerSleepInfo *next; ErtsSchedulerSleepInfo *prev; erts_smp_atomic32_t flags; erts_tse_t *event; - erts_smp_atomic32_t aux_work; +#endif + erts_atomic32_t aux_work; }; /* times to reschedule low prio process before running */ @@ -386,6 +398,22 @@ do { \ (RQ)->wakeup_other_reds += (REDS); \ } while (0) +typedef struct { + int sched_id; + ErtsSchedulerData *esdp; + ErtsSchedulerSleepInfo *ssi; + struct { + int ix; + } misc; +#ifdef ERTS_SMP + struct { + ErtsThrPrgrVal thr_prgr; + void (*completed_callback)(void *); + void (*completed_arg)(void *); + } dd; +#endif +} ErtsAuxWorkData; + struct ErtsSchedulerData_ { #ifdef ERTS_SMP @@ -403,8 +431,8 @@ struct ErtsSchedulerData_ { ethr_tid tid; /* Thread id */ struct erl_bits_state erl_bits_state; /* erl_bits.c state */ void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */ - ErtsSchedulerSleepInfo *ssi; Process *free_process; + ErtsThrPrgrData thr_progress_data; #endif #if !HEAP_ON_C_STACK Eterm tmp_heap[TMP_HEAP_SIZE]; @@ -413,16 +441,19 @@ struct ErtsSchedulerData_ { Eterm cmp_tmp_heap[CMP_TMP_HEAP_SIZE]; Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE]; #endif - + ErtsSchedulerSleepInfo *ssi; Process *current_process; Uint no; /* Scheduler number */ struct port *current_port; ErtsRunQueue *run_queue; int virtual_reds; int cpu_id; /* >= 0 when bound */ + ErtsAuxWorkData aux_work_data; ErtsAtomCacheMap atom_cache_map; + ErtsSchedAllocData alloc_data; + #ifdef ERTS_SMP /* NOTE: These fields are modified under held mutexes by other threads */ erts_smp_atomic32_t chk_cpu_bind; /* Only used when common run queue */ @@ -1032,7 +1063,7 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; void erts_pre_init_process(void); void erts_late_init_process(void); -void erts_early_init_scheduling(void); +void erts_early_init_scheduling(int); void erts_init_scheduling(int, int, int); ErtsProcList *erts_proclist_create(Process *); @@ -1058,6 +1089,7 @@ erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int); int erts_is_multi_scheduling_blocked(void); Eterm erts_multi_scheduling_blockers(Process *); void erts_start_schedulers(void); +void erts_alloc_notify_delayed_dealloc(int); void erts_smp_notify_check_children_needed(void); void erts_smp_schedule_misc_aux_work(int ignore_self, @@ -1065,6 +1097,7 @@ erts_smp_schedule_misc_aux_work(int ignore_self, void (*func)(void *), void *arg); #endif +erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int); void erts_sched_notify_check_cpu_bind(void); Uint erts_active_schedulers(void); void erts_init_process(int); @@ -1148,6 +1181,7 @@ Sint erts_test_next_pid(int, Uint); Eterm erts_debug_processes(Process *c_p); Eterm erts_debug_processes_bif_info(Process *c_p); Uint erts_debug_nbalance(void); +int erts_debug_wait_deallocations(Process *c_p); #ifdef ERTS_SMP # define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) ((PROC)->scheduler_data) diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index cd3b2182fd..97f250138e 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -651,7 +651,7 @@ ERTS_GLB_INLINE int erts_smp_proc_trylock(Process *, ErtsProcLocks); ERTS_GLB_INLINE void erts_smp_proc_inc_refc(Process *); ERTS_GLB_INLINE void erts_smp_proc_dec_refc(Process *); - +ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *, Sint32); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -737,6 +737,21 @@ ERTS_GLB_INLINE void erts_smp_proc_dec_refc(Process *p) #endif } +ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *p, Sint32 refc) +{ +#ifdef ERTS_SMP + Process *fp; + erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id); + erts_pix_lock(pixlck); + ERTS_LC_ASSERT(p->lock.refc > 0); + p->lock.refc += refc; + fp = p->lock.refc == 0 ? p : NULL; + erts_pix_unlock(pixlck); + if (fp) + erts_free_proc(fp); +#endif +} + #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ #ifdef ERTS_SMP diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c new file mode 100644 index 0000000000..a7ccea7403 --- /dev/null +++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c @@ -0,0 +1,305 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Scheduler specific pre-allocators. Each scheduler + * thread allocates memory in its own private chunk of + * memory. Memory blocks deallocated by remote + * schedulers (or other threads) are passed back to + * the chunk owner via a lock-free data structure. + * + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#ifdef ERTS_SMP + +#include "erl_process.h" +#include "erl_thr_progress.h" + +erts_sspa_data_t * +erts_sspa_create(size_t blk_sz, int pa_size) +{ + erts_sspa_data_t *data; + size_t tot_size; + size_t chunk_mem_size; + char *p; + char *chunk_start; + int cix; + int no_blocks = pa_size; + int no_blocks_per_chunk; + + if (erts_no_schedulers == 1) + no_blocks_per_chunk = no_blocks; + else { + int extra = (no_blocks - 1)/4 + 1; + if (extra == 0) + extra = 1; + no_blocks_per_chunk = no_blocks; + no_blocks_per_chunk += extra*erts_no_schedulers; + no_blocks_per_chunk /= erts_no_schedulers; + } + no_blocks = no_blocks_per_chunk * erts_no_schedulers; + chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_chunk_header_t)); + chunk_mem_size += blk_sz * no_blocks_per_chunk; + chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(chunk_mem_size); + tot_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t)); + tot_size += chunk_mem_size*erts_no_schedulers; + + p = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_PRE_ALLOC_DATA, tot_size); + data = (erts_sspa_data_t *) p; + p += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t)); + chunk_start = p; + + data->chunks_mem_size = chunk_mem_size; + data->start = chunk_start; + data->end = chunk_start + chunk_mem_size*erts_no_schedulers; + + /* Initialize all chunks */ + for (cix = 0; cix < erts_no_schedulers; cix++) { + erts_sspa_chunk_t *chnk = erts_sspa_cix2chunk(data, cix); + erts_sspa_chunk_header_t *chdr = &chnk->aligned.header; + erts_sspa_blk_t *blk; + int i; + + erts_atomic_init_nob(&chdr->tail.data.last, (erts_aint_t) &chdr->tail.data.marker); + erts_atomic_init_nob(&chdr->tail.data.marker.next_atmc, ERTS_AINT_NULL); + erts_atomic_init_nob(&chdr->tail.data.um_refc[0], 0); + erts_atomic_init_nob(&chdr->tail.data.um_refc[1], 0); + erts_atomic32_init_nob(&chdr->tail.data.um_refc_ix, 0); + + chdr->head.no_thr_progress_check = 0; + chdr->head.used_marker = 1; + chdr->head.first = &chdr->tail.data.marker; + chdr->head.unref_end = &chdr->tail.data.marker; + chdr->head.next.thr_progress = erts_thr_progress_current(); + chdr->head.next.thr_progress_reached = 1; + chdr->head.next.um_refc_ix = 1; + chdr->head.next.unref_end = &chdr->tail.data.marker; + + p = &chnk->data[0]; + chdr->local.first = (erts_sspa_blk_t *) p; + blk = (erts_sspa_blk_t *) p; + for (i = 0; i < no_blocks_per_chunk; i++) { + blk = (erts_sspa_blk_t *) p; + p += blk_sz; + blk->next_ptr = (erts_sspa_blk_t *) p; + } + + blk->next_ptr = NULL; + chdr->local.last = blk; + chdr->local.cnt = no_blocks_per_chunk; + chdr->local.lim = no_blocks_per_chunk / 3; + + ERTS_SSPA_DBG_CHK_LCL(chdr); + } + + return data; +} + +static ERTS_INLINE erts_aint_t +enqueue_remote_managed_thread(erts_sspa_chunk_header_t *chdr, + erts_sspa_blk_t *this, + int want_last) +{ + erts_aint_t ilast, itmp; + + erts_atomic_init_nob(&this->next_atmc, ERTS_AINT_NULL); + + /* Enqueue at end of list... */ + + ilast = erts_atomic_read_nob(&chdr->tail.data.last); + while (1) { + erts_sspa_blk_t *last = (erts_sspa_blk_t *) ilast; + itmp = erts_atomic_cmpxchg_mb(&last->next_atmc, + (erts_aint_t) this, + ERTS_AINT_NULL); + if (itmp == ERTS_AINT_NULL) + break; + ilast = itmp; + } + + /* Move last pointer forward... */ + while (1) { + erts_aint_t itmp; + if (want_last) { + if (erts_atomic_read_rb(&this->next_atmc) != ERTS_AINT_NULL) { + /* Someone else will move it forward */ + return erts_atomic_read_nob(&chdr->tail.data.last); + } + } + else { + if (erts_atomic_read_nob(&this->next_atmc) != ERTS_AINT_NULL) { + /* Someone else will move it forward */ + return ERTS_AINT_NULL; + } + } + itmp = erts_atomic_cmpxchg_mb(&chdr->tail.data.last, + (erts_aint_t) this, + ilast); + if (ilast == itmp) + return want_last ? (erts_aint_t) this : ERTS_AINT_NULL; + ilast = itmp; + } +} + +void +erts_sspa_remote_free(erts_sspa_chunk_header_t *chdr, erts_sspa_blk_t *blk) +{ + int um_refc_ix = 0; + int managed_thread = erts_thr_progress_is_managed_thread(); + if (!managed_thread) { + um_refc_ix = erts_atomic32_read_acqb(&chdr->tail.data.um_refc_ix); + while (1) { + int tmp_um_refc_ix; + erts_atomic_inc_acqb(&chdr->tail.data.um_refc[um_refc_ix]); + tmp_um_refc_ix = erts_atomic32_read_acqb(&chdr->tail.data.um_refc_ix); + if (tmp_um_refc_ix == um_refc_ix) + break; + erts_atomic_dec_relb(&chdr->tail.data.um_refc[um_refc_ix]); + um_refc_ix = tmp_um_refc_ix; + } + } + + (void) enqueue_remote_managed_thread(chdr, blk, 0); + + if (!managed_thread) + erts_atomic_dec_relb(&chdr->tail.data.um_refc[um_refc_ix]); +} + +static ERTS_INLINE void +fetch_remote(erts_sspa_chunk_header_t *chdr, int max) +{ + int new_local = 0; + + if (chdr->head.no_thr_progress_check < ERTS_SSPA_FORCE_THR_CHECK_PROGRESS) + chdr->head.no_thr_progress_check++; + else { + erts_aint_t ilast; + + chdr->head.no_thr_progress_check = 0; + + ilast = erts_atomic_read_nob(&chdr->tail.data.last); + if (((erts_sspa_blk_t *) ilast) == &chdr->tail.data.marker + && chdr->head.first == &chdr->tail.data.marker) + return; + + if (chdr->head.next.thr_progress_reached + || erts_thr_progress_has_reached(chdr->head.next.thr_progress)) { + int um_refc_ix; + chdr->head.next.thr_progress_reached = 1; + um_refc_ix = chdr->head.next.um_refc_ix; + if (erts_atomic_read_acqb(&chdr->tail.data.um_refc[um_refc_ix]) == 0) { + + /* Move unreferenced end pointer forward... */ + + chdr->head.unref_end = chdr->head.next.unref_end; + + if (!chdr->head.used_marker + && chdr->head.unref_end == (erts_sspa_blk_t *) ilast) { + /* Need to equeue marker */ + chdr->head.used_marker = 1; + ilast = enqueue_remote_managed_thread(chdr, + &chdr->tail.data.marker, + 1); + } + + if (chdr->head.unref_end == (erts_sspa_blk_t *) ilast) + ERTS_THR_MEMORY_BARRIER; + else { + chdr->head.next.unref_end = (erts_sspa_blk_t *) ilast; + ERTS_THR_MEMORY_BARRIER; + chdr->head.next.thr_progress = erts_thr_progress_later(); + erts_atomic32_set_relb(&chdr->tail.data.um_refc_ix, + um_refc_ix); + chdr->head.next.um_refc_ix = um_refc_ix == 0 ? 1 : 0; + chdr->head.next.thr_progress_reached = 0; + } + } + } + } + + if (new_local < max && chdr->head.first != chdr->head.unref_end) { + erts_sspa_blk_t *first, *this, *next, *last; + first = chdr->head.first; + if (first == &chdr->tail.data.marker) { + chdr->head.used_marker = 0; + first = ((erts_sspa_blk_t *) + erts_atomic_read_nob(&first->next_atmc)); + chdr->head.first = first; + } + if (first != chdr->head.unref_end) { + + ERTS_SSPA_DBG_CHK_LCL(chdr); + + this = last = first; + do { + next = (erts_sspa_blk_t *) erts_atomic_read_nob(&this->next_atmc); + if (this == &chdr->tail.data.marker) + chdr->head.used_marker = 0; + else { + last->next_ptr = this; + last = this; + new_local++; + } + this = next; + } while (new_local < max && this != chdr->head.unref_end); + chdr->head.first = this; + if (!chdr->local.last) + chdr->local.first = first; + else + chdr->local.last->next_ptr = first; + chdr->local.last = last; + last->next_ptr = NULL; + chdr->local.cnt += new_local; + + ERTS_SSPA_DBG_CHK_LCL(chdr); + } + } + +} + +erts_sspa_blk_t * +erts_sspa_process_remote_frees(erts_sspa_chunk_header_t *chdr, + erts_sspa_blk_t *old_res) +{ + erts_sspa_blk_t *res = old_res; + + fetch_remote(chdr, ERTS_SSPA_MAX_GET_NEW_LOCAL); + + if (!res && chdr->local.first) { + + ERTS_SSPA_DBG_CHK_LCL(chdr); + + res = chdr->local.first; + chdr->local.first = res->next_ptr; + chdr->local.cnt--; + if (!chdr->local.first) + chdr->local.last = NULL; + + ERTS_SSPA_DBG_CHK_LCL(chdr); + } + + return res; +} + +#endif /* ERTS_SMP */ diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.h b/erts/emulator/beam/erl_sched_spec_pre_alloc.h new file mode 100644 index 0000000000..d36066c399 --- /dev/null +++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.h @@ -0,0 +1,239 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Scheduler specific pre-allocators. Each scheduler + * thread allocates memory in its own private chunk of + * memory. Memory blocks deallocated by remote + * schedulers (or other threads) are passed back to + * the chunk owner via a lock-free data structure. + * + * Author: Rickard Green + */ + +#ifndef ERTS_SCHED_SPEC_PRE_ALLOC_H__ +#define ERTS_SCHED_SPEC_PRE_ALLOC_H__ + +#ifdef ERTS_SMP + +#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY +#define ERL_THR_PROGRESS_TSD_TYPE_ONLY +#include "erl_thr_progress.h" +#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY + +#ifdef DEBUG +#define ERTS_SPPA_DBG_CHK_IN_CHNK(A, C, P) \ +do { \ + ASSERT((void *) (C) < (void *) (P)); \ + ASSERT((void *) (P) \ + < (void *) (((char *) (C)) + (A)->chunks_mem_size)); \ +} while (0) +#else +#define ERTS_SPPA_DBG_CHK_IN_CHNK(A, C, P) +#endif + +#ifdef DEBUG +extern Uint erts_no_schedulers; +#endif + +#define ERTS_SSPA_FORCE_THR_CHECK_PROGRESS 10 +#define ERTS_SSPA_MAX_GET_NEW_LOCAL 5 + +typedef struct { + char *start; + char *end; + int chunks_mem_size; +} erts_sspa_data_t; + +typedef union erts_sspa_blk_t_ erts_sspa_blk_t; +union erts_sspa_blk_t_ { + erts_atomic_t next_atmc; + erts_sspa_blk_t *next_ptr; +}; + +typedef struct { + erts_sspa_blk_t *first; + erts_sspa_blk_t *last; + int cnt; + int lim; +} erts_sspa_local_freelist_t; + +typedef struct { + erts_sspa_blk_t marker; + erts_atomic_t last; + erts_atomic_t um_refc[2]; + erts_atomic32_t um_refc_ix; +} erts_sspa_tail_t; + +typedef struct { + /* + * This structure needs to be cache line aligned for best + * performance. + */ + union { + /* Modified by threads returning memory to this chunk */ + erts_sspa_tail_t data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_tail_t))]; + } tail; + /* + * Everything below this point is *only* accessed by the + * thread owning this chunk. + */ + struct { + int no_thr_progress_check; + int used_marker; + erts_sspa_blk_t *first; + erts_sspa_blk_t *unref_end; + struct { + ErtsThrPrgrVal thr_progress; + int thr_progress_reached; + int um_refc_ix; + erts_sspa_blk_t *unref_end; + } next; + } head; + erts_sspa_local_freelist_t local; +} erts_sspa_chunk_header_t; + +typedef struct { + union { + erts_sspa_chunk_header_t header; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE( + sizeof(erts_sspa_chunk_header_t))]; + } aligned; + char data[1]; +} erts_sspa_chunk_t; + +#ifdef DEBUG +ERTS_GLB_INLINE void +check_local_list(erts_sspa_chunk_header_t *chdr); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE void +check_local_list(erts_sspa_chunk_header_t *chdr) +{ + erts_sspa_blk_t *blk; + int n = 0; + for (blk = chdr->local.first; blk; blk = blk->next_ptr) + n++; + ASSERT(n == chdr->local.cnt); +} +#endif +#define ERTS_SSPA_DBG_CHK_LCL(CHDR) check_local_list((CHDR)) +#else +#define ERTS_SSPA_DBG_CHK_LCL(CHDR) +#endif + +erts_sspa_data_t *erts_sspa_create(size_t blk_sz, + int pa_size); +void erts_sspa_remote_free(erts_sspa_chunk_header_t *chdr, + erts_sspa_blk_t *blk); +erts_sspa_blk_t *erts_sspa_process_remote_frees(erts_sspa_chunk_header_t *chdr, + erts_sspa_blk_t *old_res); + +ERTS_GLB_INLINE erts_sspa_chunk_t *erts_sspa_cix2chunk(erts_sspa_data_t *data, + int cix); +ERTS_GLB_INLINE int erts_sspa_ptr2cix(erts_sspa_data_t *data, void *ptr); +ERTS_GLB_INLINE char *erts_sspa_alloc(erts_sspa_data_t *data, int cix); +ERTS_GLB_INLINE int erts_sspa_free(erts_sspa_data_t *data, int cix, char *blk); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE erts_sspa_chunk_t * +erts_sspa_cix2chunk(erts_sspa_data_t *data, int cix) +{ + ASSERT(0 <= cix && cix < erts_no_schedulers); + return (erts_sspa_chunk_t *) (data->start + cix*data->chunks_mem_size); +} + +ERTS_GLB_INLINE int +erts_sspa_ptr2cix(erts_sspa_data_t *data, void *ptr) +{ + int cix; + size_t diff; + if ((char *) ptr < data->start || data->end <= (char *) ptr) + return -1; + diff = ((char *) ptr) - data->start; + cix = (int) diff / data->chunks_mem_size; + ASSERT(0 <= cix && cix < erts_no_schedulers); + return cix; +} + +ERTS_GLB_INLINE char * +erts_sspa_alloc(erts_sspa_data_t *data, int cix) +{ + erts_sspa_chunk_t *chnk; + erts_sspa_chunk_header_t *chdr; + erts_sspa_blk_t *res; + + chnk = erts_sspa_cix2chunk(data, cix); + chdr = &chnk->aligned.header; + res = chdr->local.first; + ERTS_SSPA_DBG_CHK_LCL(chdr); + if (res) { + ERTS_SSPA_DBG_CHK_LCL(chdr); + chdr->local.first = res->next_ptr; + chdr->local.cnt--; + if (!chdr->local.first) + chdr->local.last = NULL; + ERTS_SSPA_DBG_CHK_LCL(chdr); + } + if (chdr->local.cnt <= chdr->local.lim) + return (char *) erts_sspa_process_remote_frees(chdr, res); + else if (chdr->head.no_thr_progress_check < ERTS_SSPA_FORCE_THR_CHECK_PROGRESS) + chdr->head.no_thr_progress_check++; + ASSERT(res); + return (char *) res; +} + +ERTS_GLB_INLINE int +erts_sspa_free(erts_sspa_data_t *data, int cix, char *cblk) +{ + erts_sspa_chunk_t *chnk; + erts_sspa_chunk_header_t *chdr; + erts_sspa_blk_t *blk = (erts_sspa_blk_t *) cblk; + int chnk_cix = erts_sspa_ptr2cix(data, blk); + + if (chnk_cix < 0) + return 0; + + chnk = erts_sspa_cix2chunk(data, chnk_cix); + chdr = &chnk->aligned.header; + if (chnk_cix != cix) { + /* Remote chunk */ + erts_sspa_remote_free(chdr, blk); + } + else { + /* Local chunk */ + ERTS_SSPA_DBG_CHK_LCL(chdr); + blk->next_ptr = chdr->local.first; + chdr->local.first = blk; + if (!chdr->local.last) + chdr->local.last = blk; + chdr->local.cnt++; + ERTS_SSPA_DBG_CHK_LCL(chdr); + } + + return 1; +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* ERTS_SMP */ + +#endif /* ERTS_SCHED_SPEC_PRE_ALLOC_H__ */ diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c new file mode 100644 index 0000000000..f96ae4b70d --- /dev/null +++ b/erts/emulator/beam/erl_thr_progress.c @@ -0,0 +1,1010 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Thread progress information. Used by lock free algorithms + * to determine when all involved threads are guaranteed to + * have passed a specific point of execution. + * + * Usage instructions below. + * + * Author: Rickard Green + */ + +/* + * ------ Usage instructions ----------------------------------------------- + * + * This module keeps track of the progress of a set of managed threads. Only + * threads that behave well can be allowed to be managed. A managed thread + * should update its thread progress frequently. Currently only scheduler + * threads and the aux_thread are managed threads. We typically do not want + * any async threads as managed threads since they cannot guarantee a + * frequent update of thread progress, since they execute user implemented + * driver code. + * + * erts_thr_progress_current() returns the global current thread progress + * value of managed threads. I.e., the latest progress value that all + * managed threads have reached. Thread progress values are opaque. + * + * erts_thr_progress_has_reached(VAL) returns a value != 0 if current + * global thread progress has reached or passed VAL. + * + * erts_thr_progress_later() returns a thread progress value in the future + * which no managed thread have yet reached. + * + * All threads issue a full memory barrier when reaching a new thread + * progress value. They only reach new thread progress values in specific + * controlled states when calling erts_thr_progress_update(). Schedulers + * call erts_thr_progress_update() in between execution of processes, + * when going to sleep and when waking up. + * + * Sleeping managed threads are considered to have reached next thread + * progress value immediately. They are not woken and do therefore not + * issue any memory barriers when reaching a new thread progress value. + * A sleeping thread do however immediately issue a memory barrier upon + * wakeup. + * + * Both managed and registered unmanaged threads may request wakeup when + * the global thread progress reach a certain value using + * erts_thr_progress_wakeup(). + * + * Note that thread progress values are opaque, and that you are only + * allowed to use thread progress values retrieved from this API! + * + * ------------------------------------------------------------------------- + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include /* offsetof() */ +#include "erl_thr_progress.h" + +#ifdef ERTS_SMP + +/* + * We use a 64-bit value for thread progress. By this wrapping of + * the thread progress will more or less never occur. + * + * On 32-bit systems we therefore need a double word atomic. + */ + +#define ERTS_THR_PRGR_PRINT_LEADER 0 +#define ERTS_THR_PRGR_PRINT_VAL 0 + +#define ERTS_THR_PRGR_LFLG_NO_LEADER (((erts_aint32_t) 1) << 31) +#define ERTS_THR_PRGR_LFLG_ACTIVE_MASK (~ERTS_THR_PRGR_LFLG_NO_LEADER) + +#define ERTS_THR_PRGR_LFLGS_ACTIVE(LFLGS) \ + ((LFLGS) & ERTS_THR_PRGR_LFLG_ACTIVE_MASK) + +#define ERTS_THR_PRGR_LFLGS_ALL_WAITING(LFLGS) \ + (((LFLGS) & (ERTS_THR_PRGR_LFLG_NO_LEADER \ + |ERTS_THR_PRGR_LFLG_ACTIVE_MASK)) \ + == ERTS_THR_PRGR_LFLG_NO_LEADER) + +#define read_acqb erts_thr_prgr_read_acqb__ + +#ifdef ARCH_64 + +static ERTS_INLINE void +set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) +{ + erts_atomic_set_mb(atmc, val); +} + +static ERTS_INLINE void +set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) +{ + erts_atomic_set_nob(atmc, val); +} + +static ERTS_INLINE ErtsThrPrgrVal +read_nob(ERTS_THR_PRGR_ATOMIC *atmc) +{ + return (ErtsThrPrgrVal) erts_atomic_read_nob(atmc); +} + +static ERTS_INLINE void +init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) +{ + erts_atomic_init_nob(atmc, val); +} + +#else + +#undef dw_sint_to_val +#define dw_sint_to_val erts_thr_prgr_dw_sint_to_val__ + +static void +val_to_dw_sint(ethr_dw_sint_t *dw_sint, ErtsThrPrgrVal val) +{ +#ifdef ETHR_SU_DW_NAINT_T__ + dw_sint->dw_sint = (ETHR_SU_DW_NAINT_T__) val; +#else + dw_sint->sint[ETHR_DW_SINT_LOW_WORD] + = (ethr_sint_t) (val & 0xffffffff); + dw_sint->sint[ETHR_DW_SINT_HIGH_WORD] + = (ethr_sint_t) ((val >> 32) & 0xffffffff); +#endif +} + +static ERTS_INLINE void +set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) +{ + ethr_dw_sint_t dw_sint; + val_to_dw_sint(&dw_sint, val); + erts_dw_atomic_set_mb(atmc, &dw_sint); +} + +static ERTS_INLINE void +set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) +{ + ethr_dw_sint_t dw_sint; + val_to_dw_sint(&dw_sint, val); + erts_dw_atomic_set_nob(atmc, &dw_sint); +} + +static ERTS_INLINE ErtsThrPrgrVal +read_nob(ERTS_THR_PRGR_ATOMIC *atmc) +{ + ethr_dw_sint_t dw_sint; + erts_dw_atomic_read_nob(atmc, &dw_sint); + return erts_thr_prgr_dw_sint_to_val__(&dw_sint); +} + +static ERTS_INLINE void +init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) +{ + ethr_dw_sint_t dw_sint; + val_to_dw_sint(&dw_sint, val); + erts_dw_atomic_init_nob(atmc, &dw_sint); +} + +#endif + +/* #define ERTS_THR_PROGRESS_STATE_DEBUG */ + +#ifdef ERTS_THR_PROGRESS_STATE_DEBUG + +#ifdef __GNUC__ +#warning "Thread progress state debug is on" +#endif + +#define ERTS_THR_PROGRESS_STATE_DEBUG_LEADER (((erts_aint32_t) 1) << 0) +#define ERTS_THR_PROGRESS_STATE_DEBUG_ACTIVE (((erts_aint32_t) 1) << 1) + +#define ERTS_THR_PROGRESS_STATE_DEBUG_INIT(ID) \ + erts_atomic32_init_nob(&intrnl->thr[(ID)].data.state_debug, \ + ERTS_THR_PROGRESS_STATE_DEBUG_ACTIVE) + +#define ERTS_THR_PROGRESS_STATE_DEBUG_SET_ACTIVE(ID, ON) \ +do { \ + erts_aint32_t state_debug__; \ + state_debug__ = erts_atomic32_read_nob(&intrnl->thr[(ID)].data.state_debug); \ + if ((ON)) \ + state_debug__ |= ERTS_THR_PROGRESS_STATE_DEBUG_ACTIVE; \ + else \ + state_debug__ &= ~ERTS_THR_PROGRESS_STATE_DEBUG_ACTIVE; \ + erts_atomic32_set_nob(&intrnl->thr[(ID)].data.state_debug, state_debug__); \ +} while (0) + +#define ERTS_THR_PROGRESS_STATE_DEBUG_SET_LEADER(ID, ON) \ +do { \ + erts_aint32_t state_debug__; \ + state_debug__ = erts_atomic32_read_nob(&intrnl->thr[(ID)].data.state_debug); \ + if ((ON)) \ + state_debug__ |= ERTS_THR_PROGRESS_STATE_DEBUG_LEADER; \ + else \ + state_debug__ &= ~ERTS_THR_PROGRESS_STATE_DEBUG_LEADER; \ + erts_atomic32_set_nob(&intrnl->thr[(ID)].data.state_debug, state_debug__); \ +} while (0) + +#else + +#define ERTS_THR_PROGRESS_STATE_DEBUG_INIT(ID) +#define ERTS_THR_PROGRESS_STATE_DEBUG_SET_ACTIVE(ID, ON) +#define ERTS_THR_PROGRESS_STATE_DEBUG_SET_LEADER(ID, ON) + +#endif /* ERTS_THR_PROGRESS_STATE_DEBUG */ + +#define ERTS_THR_PRGR_BM_BITS 32 +#define ERTS_THR_PRGR_BM_SHIFT 5 +#define ERTS_THR_PRGR_BM_MASK 0x1f + +#define ERTS_THR_PRGR_WAKEUP_DATA_MASK (ERTS_THR_PRGR_WAKEUP_DATA_SIZE - 1) + +#define ERTS_THR_PRGR_WAKEUP_IX(V) \ + ((int) ((V) & ERTS_THR_PRGR_WAKEUP_DATA_MASK)) + +typedef struct { + erts_atomic32_t len; + int id[1]; +} ErtsThrPrgrManagedWakeupData; + +typedef struct { + erts_atomic32_t len; + int high_sz; + int low_sz; + erts_atomic32_t *high; + erts_atomic32_t *low; +} ErtsThrPrgrUnmanagedWakeupData; + +typedef struct { + erts_atomic32_t lflgs; + + erts_atomic32_t pref_wakeup_used; + erts_atomic32_t managed_id; + erts_atomic32_t unmanaged_id; +} ErtsThrPrgrMiscVolatile; + +typedef struct { + ERTS_THR_PRGR_ATOMIC current; +#ifdef ERTS_THR_PROGRESS_STATE_DEBUG + erts_atomic32_t state_debug; +#endif +} ErtsThrPrgrElement; + +typedef union { + ErtsThrPrgrElement data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsThrPrgrElement))]; +} ErtsThrPrgrArray; + +typedef struct { + union { + ErtsThrPrgrMiscVolatile tile; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE( + sizeof(ErtsThrPrgrMiscVolatile))]; + } vola; + ErtsThrPrgrArray *thr; + struct { + int no; + ErtsThrPrgrWakeupCallback *callback; + ErtsThrPrgrManagedWakeupData *data[ERTS_THR_PRGR_WAKEUP_DATA_SIZE]; + } managed; + struct { + int no; + ErtsThrPrgrWakeupCallback *callback; + ErtsThrPrgrUnmanagedWakeupData *data[ERTS_THR_PRGR_WAKEUP_DATA_SIZE]; + } unmanaged; +} ErtsThrPrgrInternalData; + +static ErtsThrPrgrInternalData *intrnl; + +ErtsThrPrgr erts_thr_prgr__; + +erts_tsd_key_t erts_thr_prgr_data_key__; + +static void handle_wakeup_requests(ErtsThrPrgrVal current); +static int got_sched_wakeups(void); + +static ERTS_INLINE void +wakeup_managed(int id) +{ + ErtsThrPrgrWakeupCallback *wdp = &intrnl->managed.callback[id]; + ASSERT(0 <= id && id < intrnl->managed.no); + wdp->wakeup(wdp->arg); +} + + +static ERTS_INLINE void +wakeup_unmanaged(int id) +{ + ErtsThrPrgrWakeupCallback *wdp = &intrnl->unmanaged.callback[id]; + ASSERT(0 <= id && id < intrnl->unmanaged.no); + wdp->wakeup(wdp->arg); +} + +static ERTS_INLINE ErtsThrPrgrData * +thr_prgr_data(ErtsSchedulerData *esdp) +{ + ErtsThrPrgrData *tpd; + if (esdp) + tpd = &esdp->thr_progress_data; + else + tpd = erts_tsd_get(erts_thr_prgr_data_key__); + ASSERT(tpd); + return tpd; +} + +void +erts_thr_progress_pre_init(void) +{ + intrnl = NULL; + erts_tsd_key_create(&erts_thr_prgr_data_key__); + init_nob(&erts_thr_prgr__.current, 0); +} + +void +erts_thr_progress_init(int no_schedulers, int managed, int unmanaged) +{ + int i, j, um_low, um_high; + char *ptr; + size_t cb_sz, intrnl_sz, thr_arr_sz, m_wakeup_size, um_wakeup_size, + tot_size; + + intrnl_sz = sizeof(ErtsThrPrgrInternalData); + intrnl_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(intrnl_sz); + + cb_sz = sizeof(ErtsThrPrgrWakeupCallback)*(managed+unmanaged); + cb_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(cb_sz); + + thr_arr_sz = sizeof(ErtsThrPrgrArray)*managed; + ASSERT(thr_arr_sz == ERTS_ALC_CACHE_LINE_ALIGN_SIZE(thr_arr_sz)); + + m_wakeup_size = sizeof(ErtsThrPrgrManagedWakeupData); + m_wakeup_size += (managed - 1)*sizeof(int); + m_wakeup_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(m_wakeup_size); + + um_low = (unmanaged - 1)/ERTS_THR_PRGR_BM_BITS + 1; + um_high = (um_low - 1)/ERTS_THR_PRGR_BM_BITS + 1; + + um_wakeup_size = sizeof(ErtsThrPrgrUnmanagedWakeupData); + um_wakeup_size += (um_high + um_low)*sizeof(erts_atomic32_t); + um_wakeup_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(um_wakeup_size); + + tot_size = intrnl_sz; + tot_size += cb_sz; + tot_size += thr_arr_sz; + tot_size += m_wakeup_size*ERTS_THR_PRGR_WAKEUP_DATA_SIZE; + tot_size += um_wakeup_size*ERTS_THR_PRGR_WAKEUP_DATA_SIZE; + + ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_THR_PRGR_IDATA, + tot_size); + + intrnl = (ErtsThrPrgrInternalData *) ptr; + ptr += intrnl_sz; + + erts_atomic32_init_nob(&intrnl->vola.tile.lflgs, + ERTS_THR_PRGR_LFLG_NO_LEADER); + erts_atomic32_init_nob(&intrnl->vola.tile.pref_wakeup_used, 0); + erts_atomic32_init_nob(&intrnl->vola.tile.managed_id, no_schedulers); + erts_atomic32_init_nob(&intrnl->vola.tile.unmanaged_id, -1); + + intrnl->thr = (ErtsThrPrgrArray *) ptr; + ptr += thr_arr_sz; + for (i = 0; i < managed; i++) + init_nob(&intrnl->thr[i].data.current, 0); + + intrnl->managed.callback = (ErtsThrPrgrWakeupCallback *) ptr; + intrnl->unmanaged.callback = &intrnl->managed.callback[managed]; + ptr += cb_sz; + + intrnl->managed.no = managed; + for (i = 0; i < managed; i++) { + intrnl->managed.callback[i].arg = NULL; + intrnl->managed.callback[i].wakeup = NULL; + } + + intrnl->unmanaged.no = unmanaged; + for (i = 0; i < unmanaged; i++) { + intrnl->unmanaged.callback[i].arg = NULL; + intrnl->unmanaged.callback[i].wakeup = NULL; + } + + for (i = 0; i < ERTS_THR_PRGR_WAKEUP_DATA_SIZE; i++) { + intrnl->managed.data[i] = (ErtsThrPrgrManagedWakeupData *) ptr; + erts_atomic32_init_nob(&intrnl->managed.data[i]->len, 0); + ptr += m_wakeup_size; + } + + for (i = 0; i < ERTS_THR_PRGR_WAKEUP_DATA_SIZE; i++) { + erts_atomic32_t *bm; + intrnl->unmanaged.data[i] = (ErtsThrPrgrUnmanagedWakeupData *) ptr; + erts_atomic32_init_nob(&intrnl->unmanaged.data[i]->len, 0); + bm = (erts_atomic32_t *) (ptr + sizeof(ErtsThrPrgrUnmanagedWakeupData)); + intrnl->unmanaged.data[i]->high = bm; + intrnl->unmanaged.data[i]->high_sz = um_high; + for (j = 0; j < um_high; j++) + erts_atomic32_init_nob(&intrnl->unmanaged.data[i]->high[j], 0); + intrnl->unmanaged.data[i]->low + = &intrnl->unmanaged.data[i]->high[um_high]; + intrnl->unmanaged.data[i]->low_sz = um_low; + for (j = 0; j < um_low; j++) + erts_atomic32_init_nob(&intrnl->unmanaged.data[i]->low[j], 0); + ptr += um_wakeup_size; + } + ERTS_THR_MEMORY_BARRIER; +} + +static void +init_wakeup_request_array(ErtsThrPrgrVal *w) +{ + int i; + ErtsThrPrgrVal current; + + current = read_acqb(&erts_thr_prgr__.current); + for (i = 0; i < ERTS_THR_PRGR_WAKEUP_DATA_SIZE; i++) { + w[i] = current - ((ErtsThrPrgrVal) (ERTS_THR_PRGR_WAKEUP_DATA_SIZE + i)); + if (w[i] > current) + w[i]--; + } +} + +void +erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrWakeupCallback *callback) +{ + ErtsThrPrgrData *tpd; + if (erts_tsd_get(erts_thr_prgr_data_key__)) + erl_exit(ERTS_ABORT_EXIT, + "%s:%d:%s(): Double register of thread\n", + __FILE__, __LINE__, __func__); + /* + * We only allocate the part up to the leader field + * which is the first field only used by managed threads + */ + tpd = erts_alloc(ERTS_ALC_T_THR_PRGR_DATA, + offsetof(ErtsThrPrgrData, leader)); + tpd->id = (int) erts_atomic32_inc_read_nob(&intrnl->vola.tile.unmanaged_id); + tpd->is_managed = 0; + ASSERT(tpd->id >= 0); + if (tpd->id >= intrnl->unmanaged.no) + erl_exit(ERTS_ABORT_EXIT, + "%s:%d:%s(): Too many unmanaged registered threads\n", + __FILE__, __LINE__, __func__); + + init_wakeup_request_array(&tpd->wakeup_request[0]); + erts_tsd_set(erts_thr_prgr_data_key__, (void *) tpd); + + intrnl->unmanaged.callback[tpd->id] = *callback; +} + + +void +erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp, + ErtsThrPrgrWakeupCallback *callback, + int pref_wakeup) +{ + ErtsThrPrgrData *tpd; + if (erts_tsd_get(erts_thr_prgr_data_key__)) + erl_exit(ERTS_ABORT_EXIT, + "%s:%d:%s(): Double register of thread\n", + __FILE__, __LINE__, __func__); + if (esdp) + tpd = &esdp->thr_progress_data; + else + tpd = erts_alloc(ERTS_ALC_T_THR_PRGR_DATA, sizeof(ErtsThrPrgrData)); + + if (pref_wakeup + && !erts_atomic32_xchg_nob(&intrnl->vola.tile.pref_wakeup_used, 1)) + tpd->id = 0; + else if (esdp) + tpd->id = (int) esdp->no; + else + tpd->id = erts_atomic32_inc_read_nob(&intrnl->vola.tile.managed_id); + ASSERT(tpd->id >= 0); + if (tpd->id >= intrnl->managed.no) + erl_exit(ERTS_ABORT_EXIT, + "%s:%d:%s(): Too many managed registered threads\n", + __FILE__, __LINE__, __func__); + + tpd->is_managed = 1; + + init_wakeup_request_array(&tpd->wakeup_request[0]); + + ERTS_THR_PROGRESS_STATE_DEBUG_INIT(tpd->id); + + tpd->leader = 0; + tpd->active = 1; + tpd->previous.local = 0; + tpd->previous.current = ERTS_THR_PRGR_VAL_WAITING; + erts_tsd_set(erts_thr_prgr_data_key__, (void *) tpd); + + erts_atomic32_inc_nob(&intrnl->vola.tile.lflgs); + intrnl->managed.callback[tpd->id] = *callback; +} + +static ERTS_INLINE int +leader_update(ErtsThrPrgrData *tpd) +{ + if (tpd->leader) { + erts_aint32_t lflgs; + ErtsThrPrgrVal next; + int ix, sz, make_progress; + + if (tpd->previous.current == ERTS_THR_PRGR_VAL_WAITING) { + /* Took over as leader from another thread */ + tpd->previous.current = read_acqb(&erts_thr_prgr__.current); + tpd->previous.next = tpd->previous.current; + tpd->previous.next++; + if (tpd->previous.next == ERTS_THR_PRGR_VAL_WAITING) + tpd->previous.next = 0; + } + + if (tpd->previous.local == tpd->previous.current) { + ErtsThrPrgrVal val = tpd->previous.current + 1; + if (val == ERTS_THR_PRGR_VAL_WAITING) + val = 0; + tpd->previous.local = val; + set_mb(&intrnl->thr[tpd->id].data.current, val); + } + + next = tpd->previous.next; + + make_progress = 1; + sz = intrnl->managed.no; + for (ix = 0; ix < sz; ix++) { + ErtsThrPrgrVal tmp; + tmp = read_nob(&intrnl->thr[ix].data.current); + if (tmp != next && tmp != ERTS_THR_PRGR_VAL_WAITING) { + make_progress = 0; + ASSERT(erts_thr_progress_has_passed__(next, tmp)); + break; + } + } + + if (make_progress) { + ErtsThrPrgrVal current = next; + + next++; + if (next == ERTS_THR_PRGR_VAL_WAITING) + next = 0; + + set_nob(&intrnl->thr[tpd->id].data.current, next); + set_mb(&erts_thr_prgr__.current, current); + tpd->previous.local = next; + tpd->previous.next = next; + tpd->previous.current = current; + +#if ERTS_THR_PRGR_PRINT_VAL + if (current % 1000 == 0) + erts_fprintf(stderr, "%b64u\n", current); +#endif + handle_wakeup_requests(current); + } + + if (!tpd->active) { + tpd->leader = 0; + tpd->previous.current = ERTS_THR_PRGR_VAL_WAITING; +#if ERTS_THR_PRGR_PRINT_LEADER + erts_fprintf(stderr, "L <- %d\n", tpd->id); +#endif + ERTS_THR_PROGRESS_STATE_DEBUG_SET_LEADER(tpd->id, 0); + + lflgs = erts_atomic32_read_bor_relb(&intrnl->vola.tile.lflgs, + ERTS_THR_PRGR_LFLG_NO_LEADER); + if (ERTS_THR_PRGR_LFLGS_ACTIVE(lflgs) == 0 && got_sched_wakeups()) + wakeup_managed(0); + } + } + + return tpd->leader; +} + +static int +update(ErtsThrPrgrData *tpd) +{ + ErtsThrPrgrVal val; + + if (!tpd->leader) { + erts_aint32_t lflgs; + val = read_acqb(&erts_thr_prgr__.current); + if (tpd->previous.local == val) { + val++; + if (val == ERTS_THR_PRGR_VAL_WAITING) + val = 0; + tpd->previous.local = val; + set_mb(&intrnl->thr[tpd->id].data.current, val); + } + + lflgs = erts_atomic32_read_nob(&intrnl->vola.tile.lflgs); + if ((lflgs & ERTS_THR_PRGR_LFLG_NO_LEADER) + && (tpd->active || ERTS_THR_PRGR_LFLGS_ACTIVE(lflgs) == 0)) { + /* Try to take over leadership... */ + erts_aint32_t olflgs; + olflgs = erts_atomic32_read_band_acqb( + &intrnl->vola.tile.lflgs, + ~ERTS_THR_PRGR_LFLG_NO_LEADER); + if (olflgs & ERTS_THR_PRGR_LFLG_NO_LEADER) { + tpd->leader = 1; +#if ERTS_THR_PRGR_PRINT_LEADER + erts_fprintf(stderr, "L -> %d\n", tpd->id); +#endif + ERTS_THR_PROGRESS_STATE_DEBUG_SET_LEADER(tpd->id, 1); + } + } + } + return tpd->leader; +} + +int +erts_thr_progress_update(ErtsSchedulerData *esdp) +{ + return update(thr_prgr_data(esdp)); +} + + +int +erts_thr_progress_leader_update(ErtsSchedulerData *esdp) +{ + return leader_update(thr_prgr_data(esdp)); +} + +void +erts_thr_progress_prepare_wait(ErtsSchedulerData *esdp) +{ + erts_aint32_t lflgs; + ErtsThrPrgrData *tpd = thr_prgr_data(esdp); + + tpd->previous.local = ERTS_THR_PRGR_VAL_WAITING; + set_mb(&intrnl->thr[tpd->id].data.current, ERTS_THR_PRGR_VAL_WAITING); + + lflgs = erts_atomic32_read_nob(&intrnl->vola.tile.lflgs); + if (ERTS_THR_PRGR_LFLGS_ALL_WAITING(lflgs) && got_sched_wakeups()) + wakeup_managed(0); /* Someone need to make progress */ +} + +void +erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp) +{ + ErtsThrPrgrData *tpd = thr_prgr_data(esdp); + ErtsThrPrgrVal current, val; + + /* + * We aren't allowed to continue until our thread + * progress is past global current. + */ + val = current = read_acqb(&erts_thr_prgr__.current); + while (1) { + val++; + if (val == ERTS_THR_PRGR_VAL_WAITING) + val = 0; + tpd->previous.local = val; + set_mb(&intrnl->thr[tpd->id].data.current, val); + val = read_acqb(&erts_thr_prgr__.current); + if (current == val) + break; + current = val; + } + if (update(tpd)) + leader_update(tpd); +} + +void +erts_thr_progress_active(ErtsSchedulerData *esdp, int on) +{ + ErtsThrPrgrData *tpd = thr_prgr_data(esdp); + + ERTS_THR_PROGRESS_STATE_DEBUG_SET_ACTIVE(tpd->id, on); + + if (on) { + ASSERT(!tpd->active); + tpd->active = 1; + erts_atomic32_inc_nob(&intrnl->vola.tile.lflgs); + } + else { + ASSERT(tpd->active); + tpd->active = 0; + erts_atomic32_dec_nob(&intrnl->vola.tile.lflgs); + + if (update(tpd)) + leader_update(tpd); + } + +#ifdef DEBUG + { + erts_aint32_t n = erts_atomic32_read_nob(&intrnl->vola.tile.lflgs); + n &= ERTS_THR_PRGR_LFLG_ACTIVE_MASK; + ASSERT(tpd->active <= n && n <= intrnl->managed.no); + } +#endif + +} + +static ERTS_INLINE int +has_reached_wakeup(ErtsThrPrgrVal wakeup) +{ + /* + * Exactly the same as erts_thr_progress_has_reached(), but + * also verify valid wakeup requests in debug mode. + */ + ErtsThrPrgrVal current; + + current = read_acqb(&erts_thr_prgr__.current); + +#if ERTS_THR_PRGR_DBG_CHK_WAKEUP_REQUEST_VALUE + { + ErtsThrPrgrVal limit; + /* + * erts_thr_progress_later() returns values which are + * equal to 'current + 2'. That is, users should never + * get a hold of values larger than that. + * + * That is, valid values are values less than 'current + 3'. + * + * Values larger than this won't work with the wakeup + * algorithm. + */ + + limit = current + 3; + if (limit == ERTS_THR_PRGR_VAL_WAITING) + limit = 0; + else if (limit < current) /* Wrapped */ + limit + 1; + + if (!erts_thr_progress_has_passed__(limit, wakeup)) + erl_exit(ERTS_ABORT_EXIT, + "Invalid wakeup request value found:" + " current=%b64u, wakeup=%b64u, limit=%b64u", + current, wakeup, limit); + } +#endif + + if (current == wakeup) + return 1; + return erts_thr_progress_has_passed__(current, wakeup); +} + +static void +request_wakeup_managed(ErtsThrPrgrData *tpd, ErtsThrPrgrVal value) +{ + ErtsThrPrgrManagedWakeupData *mwd; + int ix, wix; + + /* + * Only managed threads that aren't in waiting state + * are allowed to call this function. + */ + + ASSERT(tpd->is_managed); + ASSERT(tpd->previous.local != ERTS_THR_PRGR_VAL_WAITING); + + if (has_reached_wakeup(value)) + wakeup_managed(tpd->id); + + wix = ERTS_THR_PRGR_WAKEUP_IX(value); + if (tpd->wakeup_request[wix] == value) + return; /* Already got a request registered */ + + ASSERT(erts_thr_progress_has_passed__(value, + tpd->wakeup_request[wix])); + + + if (tpd->previous.local == value) { + /* + * We have already confirmed this value. We need to request + * wakeup for a value later than our latest confirmed value in + * order to prevent progress from reaching the requested value + * while we are writing the request. + * + * It is ok to move the wakeup request forward since the only + * guarantee we make (and can make) is that the thread will be + * woken some time *after* the requested value has been reached. + */ + value++; + if (value == ERTS_THR_PRGR_VAL_WAITING) + value = 0; + + wix = ERTS_THR_PRGR_WAKEUP_IX(value); + if (tpd->wakeup_request[wix] == value) + return; /* Already got a request registered */ + + ASSERT(erts_thr_progress_has_passed__(value, + tpd->wakeup_request[wix])); + } + + tpd->wakeup_request[wix] = value; + + mwd = intrnl->managed.data[wix]; + + ix = erts_atomic32_inc_read_nob(&mwd->len) - 1; + mwd->id[ix] = tpd->id; + + ASSERT(!erts_thr_progress_has_reached(value)); + + /* + * This thread is guarranteed to issue a full memory barrier: + * - after the request has been written, but + * - before the global thread progress reach the (possibly + * increased) requested wakeup value. + */ +} + +static void +request_wakeup_unmanaged(ErtsThrPrgrData *tpd, ErtsThrPrgrVal value) +{ + int wix, ix, id, bit; + ErtsThrPrgrUnmanagedWakeupData *umwd; + + ASSERT(!tpd->is_managed); + + /* + * Thread progress *can* reach and pass our requested value while + * we are writing the request. + */ + + if (has_reached_wakeup(value)) + wakeup_unmanaged(tpd->id); + + wix = ERTS_THR_PRGR_WAKEUP_IX(value); + + if (tpd->wakeup_request[wix] == value) + return; /* Already got a request registered */ + + ASSERT(erts_thr_progress_has_passed__(value, + tpd->wakeup_request[wix])); + + umwd = intrnl->unmanaged.data[wix]; + + id = tpd->id; + + bit = id & ERTS_THR_PRGR_BM_MASK; + ix = id >> ERTS_THR_PRGR_BM_SHIFT; + ASSERT(0 <= ix && ix < umwd->low_sz); + erts_atomic32_read_bor_nob(&umwd->low[ix], 1 << bit); + + bit = ix & ERTS_THR_PRGR_BM_MASK; + ix >>= ERTS_THR_PRGR_BM_SHIFT; + ASSERT(0 <= ix && ix < umwd->high_sz); + erts_atomic32_read_bor_nob(&umwd->high[ix], 1 << bit); + + erts_atomic32_inc_mb(&umwd->len); + + if (erts_thr_progress_has_reached(value)) + wakeup_unmanaged(tpd->id); + else + tpd->wakeup_request[wix] = value; +} + +void +erts_thr_progress_wakeup(ErtsSchedulerData *esdp, + ErtsThrPrgrVal value) +{ + ErtsThrPrgrData *tpd = thr_prgr_data(esdp); + ASSERT(tpd); + if (tpd->is_managed) + request_wakeup_managed(tpd, value); + else + request_wakeup_unmanaged(tpd, value); +} + +static void +wakeup_unmanaged_threads(ErtsThrPrgrUnmanagedWakeupData *umwd) +{ + int hix; + for (hix = 0; hix < umwd->high_sz; hix++) { + erts_aint32_t hmask = erts_atomic32_read_nob(&umwd->high[hix]); + if (hmask) { + int hbase = hix << ERTS_THR_PRGR_BM_SHIFT; + int hbit; + for (hbit = 0; hbit < ERTS_THR_PRGR_BM_BITS; hbit++) { + if (hmask & (1 << hbit)) { + erts_aint_t lmask; + int lix = hbase + hbit; + ASSERT(0 <= lix && lix < umwd->low_sz); + lmask = erts_atomic32_read_nob(&umwd->low[lix]); + if (lmask) { + int lbase = lix << ERTS_THR_PRGR_BM_SHIFT; + int lbit; + for (lbit = 0; lbit < ERTS_THR_PRGR_BM_BITS; lbit++) { + if (lmask & (1 << lbit)) { + int id = lbase + lbit; + wakeup_unmanaged(id); + } + } + erts_atomic32_set_nob(&umwd->low[lix], 0); + } + } + } + erts_atomic32_set_nob(&umwd->high[hix], 0); + } + } +} + + +static void +handle_wakeup_requests(ErtsThrPrgrVal current) +{ + ErtsThrPrgrManagedWakeupData *mwd; + ErtsThrPrgrUnmanagedWakeupData *umwd; + int wix, len, i; + + wix = ERTS_THR_PRGR_WAKEUP_IX(current); + + mwd = intrnl->managed.data[wix]; + len = erts_atomic32_read_nob(&mwd->len); + ASSERT(len >= 0); + if (len) { + for (i = 0; i < len; i++) + wakeup_managed(mwd->id[i]); + erts_atomic32_set_nob(&mwd->len, 0); + } + + umwd = intrnl->unmanaged.data[wix]; + len = erts_atomic32_read_nob(&umwd->len); + ASSERT(len >= 0); + if (len) { + wakeup_unmanaged_threads(umwd); + erts_atomic32_set_nob(&umwd->len, 0); + } + +} + +static int +got_sched_wakeups(void) +{ + int wix; + + ERTS_THR_MEMORY_BARRIER; + + for (wix = 0; wix < ERTS_THR_PRGR_WAKEUP_DATA_SIZE; wix++) { + ErtsThrPrgrManagedWakeupData **mwd = intrnl->managed.data; + if (erts_atomic32_read_nob(&mwd[wix]->len)) + return 1; + } + for (wix = 0; wix < ERTS_THR_PRGR_WAKEUP_DATA_SIZE; wix++) { + ErtsThrPrgrUnmanagedWakeupData **umwd = intrnl->unmanaged.data; + if (erts_atomic32_read_nob(&umwd[wix]->len)) + return 1; + } + return 0; +} + + +void erts_thr_progress_dbg_print_state(void) +{ + int id; + int sz = intrnl->managed.no; + + erts_fprintf(stderr, "--- thread progress ---\n"); + erts_fprintf(stderr,"current=%b64u\n", erts_thr_progress_current()); + for (id = 0; id < sz; id++) { + ErtsThrPrgrVal current = read_nob(&intrnl->thr[id].data.current); +#ifdef ERTS_THR_PROGRESS_STATE_DEBUG + erts_aint32_t state_debug; + char *active, *leader; + + state_debug = erts_atomic32_read_nob(&intrnl->thr[id].data.state_debug); + active = (state_debug & ERTS_THR_PROGRESS_STATE_DEBUG_ACTIVE + ? "true" + : "false"); + leader = (state_debug & ERTS_THR_PROGRESS_STATE_DEBUG_LEADER + ? "true" + : "false"); +#endif + if (current == ERTS_THR_PRGR_VAL_WAITING) + erts_fprintf(stderr, + " id=%d, current=WAITING" +#ifdef ERTS_THR_PROGRESS_STATE_DEBUG + ", active=%s, leader=%s" +#endif + "\n", id +#ifdef ERTS_THR_PROGRESS_STATE_DEBUG + , active, leader +#endif + ); + else + erts_fprintf(stderr, + " id=%d, current=%b64u" +#ifdef ERTS_THR_PROGRESS_STATE_DEBUG + ", active=%s, leader=%s" +#endif + "\n", id, current +#ifdef ERTS_THR_PROGRESS_STATE_DEBUG + , active, leader +#endif + ); + } + erts_fprintf(stderr, "-----------------------\n"); + + +} + +#endif diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h new file mode 100644 index 0000000000..36880c10f0 --- /dev/null +++ b/erts/emulator/beam/erl_thr_progress.h @@ -0,0 +1,210 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Thread progress information. Used by lock free algorithms + * to determine when all involved threads are guaranteed to + * have passed a specific point of execution. + * + * Usage instructions can be found in ert_thr_progress.c + * + * Author: Rickard Green + */ + +#if !defined(ERL_THR_PROGRESS_H__TSD_TYPE__) +#define ERL_THR_PROGRESS_H__TSD_TYPE__ + +#include "sys.h" + +#ifdef ERTS_SMP + +typedef Uint64 ErtsThrPrgrVal; + +#define ERTS_THR_PRGR_WAKEUP_DATA_SIZE 4 /* Need to be an even power of 2. */ + +typedef struct { + int id; + int is_managed; + ErtsThrPrgrVal wakeup_request[ERTS_THR_PRGR_WAKEUP_DATA_SIZE]; + + /* --- Part below only for managed threads */ + + int leader; /* Needs to be first in the managed threads part */ + int active; + struct { + ErtsThrPrgrVal local; + ErtsThrPrgrVal next; + ErtsThrPrgrVal current; + } previous; +} ErtsThrPrgrData; +#endif /* ERTS_SMP */ + +#endif + +#if !defined(ERL_THR_PROGRESS_H__) && !defined(ERL_THR_PROGRESS_TSD_TYPE_ONLY) +#define ERL_THR_PROGRESS_H__ + +#include "erl_threads.h" +#include "erl_process.h" + +#ifdef ERTS_SMP + +#define ERTS_THR_PRGR_VAL_WAITING (~((ErtsThrPrgrVal) 0)) + +extern erts_tsd_key_t erts_thr_prgr_data_key__; + +#ifdef ARCH_64 +# define ERTS_THR_PRGR_ATOMIC erts_atomic_t +#else /* ARCH_32 */ +# define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t +#endif + +typedef struct { + void *arg; + void (*wakeup)(void *arg); +} ErtsThrPrgrWakeupCallback; + +typedef struct { + ERTS_THR_PRGR_ATOMIC current; +} ErtsThrPrgr; + +extern ErtsThrPrgr erts_thr_prgr__; + +void erts_thr_progress_pre_init(void); +void erts_thr_progress_init(int no_schedulers, int managed, int unmanaged); +void erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp, + ErtsThrPrgrWakeupCallback *, + int); +void erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrWakeupCallback *); +void erts_thr_progress_active(ErtsSchedulerData *esdp, int on); +void erts_thr_progress_wakeup(ErtsSchedulerData *esdp, + ErtsThrPrgrVal value); +int erts_thr_progress_update(ErtsSchedulerData *esdp); +int erts_thr_progress_leader_update(ErtsSchedulerData *esdp); +void erts_thr_progress_prepare_wait(ErtsSchedulerData *esdp); +void erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp); + +void erts_thr_progress_dbg_print_state(void); + +#ifdef ARCH_32 +#define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_dw_sint_to_val__(ethr_dw_sint_t *dw_sint); +#endif +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc); + +ERTS_GLB_INLINE int erts_thr_progress_is_managed_thread(void); +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later(void); +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current(void); +ERTS_GLB_INLINE int erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2); +ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +#ifdef ARCH_64 + +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc) +{ + return (ErtsThrPrgrVal) erts_atomic_read_acqb(atmc); +} + +#else /* ARCH_32 */ + +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_prgr_dw_sint_to_val__(ethr_dw_sint_t *dw_sint) +{ +#ifdef ETHR_SU_DW_NAINT_T__ + return (ErtsThrPrgrVal) dw_sint->dw_sint; +#else + ErtsThrPrgrVal res; + res = (ErtsThrPrgrVal) ((Uint32) dw_sint->sint[ETHR_DW_SINT_HIGH_WORD]); + res <<= 32; + res |= (ErtsThrPrgrVal) ((Uint32) dw_sint->sint[ETHR_DW_SINT_LOW_WORD]); + return res; +#endif +} + +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc) +{ + ethr_dw_sint_t dw_sint; + erts_dw_atomic_read_acqb(atmc, &dw_sint); + return erts_thr_prgr_dw_sint_to_val__(&dw_sint); +} + +#endif + +ERTS_GLB_INLINE int +erts_thr_progress_is_managed_thread(void) +{ + ErtsThrPrgrData *tpd = erts_tsd_get(erts_thr_prgr_data_key__); + return tpd && tpd->is_managed; +} + +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_progress_later(void) +{ + ErtsThrPrgrVal val = erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current); + if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)2))) + return ((ErtsThrPrgrVal) 0); + else if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)1))) + return ((ErtsThrPrgrVal) 1); + else + return val + ((ErtsThrPrgrVal) 2); +} + +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_progress_current(void) +{ + return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current); +} + +ERTS_GLB_INLINE int +erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val0) +{ + if ((((((ErtsThrPrgrVal) 1) << 63) & val1) + ^ ((((ErtsThrPrgrVal) 1) << 63) & val0)) != 0) { + /* May have wrapped... */ + if (val1 < (((ErtsThrPrgrVal) 1) << 62) + && val0 > (((ErtsThrPrgrVal) 3) << 62)) { + /* + * 'val1' has wrapped but 'val0' has not yet wrapped. While in + * these ranges 'current' is considered later than 'val0'. + */ + return 1; + } + } + return val1 > val0; +} + +ERTS_GLB_INLINE int +erts_thr_progress_has_reached(ErtsThrPrgrVal val) +{ + ErtsThrPrgrVal current; + current = erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current); + if (current == val) + return 1; + return erts_thr_progress_has_passed__(current, val); +} + +#endif + +#endif /* ERTS_SMP */ + +#endif diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index 12eaf39ec7..b4b6d0dfd5 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -193,6 +193,8 @@ typedef struct { int gcc_is_buggy; } erts_rwlock_t; #endif /* #ifdef USE_THREADS */ +#define ERTS_AINT_NULL ((erts_aint_t) NULL) + #define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1))) #define ERTS_AINT_T_MIN ((((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1))) #define ERTS_AINT32_T_MAX (~(((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1))) diff --git a/erts/emulator/beam/fix_alloc.c b/erts/emulator/beam/fix_alloc.c deleted file mode 100644 index 5637281597..0000000000 --- a/erts/emulator/beam/fix_alloc.c +++ /dev/null @@ -1,287 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ -/* General purpose Memory allocator for fixed block size objects */ -/* This allocater is at least an order of magnitude faster than malloc() */ - - -#define NOPERBLOCK 20 -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "sys.h" -#include "erl_vm.h" -#include "global.h" -#include "erl_db.h" - -#ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE - -#if ERTS_ALC_MTA_FIXED_SIZE -#include "erl_threads.h" -#include "erl_smp.h" -# ifdef ERTS_SMP -# define FA_LOCK(FA) erts_smp_spin_lock(&(FA)->slck) -# define FA_UNLOCK(FA) erts_smp_spin_unlock(&(FA)->slck) -# else -# define FA_LOCK(FA) erts_mtx_lock(&(FA)->mtx) -# define FA_UNLOCK(FA) erts_mtx_unlock(&(FA)->mtx) -# endif -#else -# define FA_LOCK(FA) -# define FA_UNLOCK(FA) -#endif - -typedef union {double d; long l;} align_t; - -typedef struct fix_alloc_block { - struct fix_alloc_block *next; - align_t mem[1]; -} FixAllocBlock; - -typedef struct fix_alloc { - Uint item_size; - void *freelist; - Uint no_free; - Uint no_blocks; - FixAllocBlock *blocks; -#if ERTS_ALC_MTA_FIXED_SIZE -# ifdef ERTS_SMP - erts_smp_spinlock_t slck; -# else - erts_mtx_t mtx; -# endif -#endif -} FixAlloc; - -static void *(*core_alloc)(Uint); -static Uint xblk_sz; - -static FixAlloc **fa; -#define FA_SZ (1 + ERTS_ALC_N_MAX_A_FIXED_SIZE - ERTS_ALC_N_MIN_A_FIXED_SIZE) - -#define FIX_IX(N) ((N) - ERTS_ALC_N_MIN_A_FIXED_SIZE) - -#define FIX_POOL_SZ(I_SZ) \ - ((I_SZ)*NOPERBLOCK + sizeof(FixAllocBlock) - sizeof(align_t)) - -#if defined(DEBUG) && !ERTS_ALC_MTA_FIXED_SIZE -static int first_time; -#endif - -void erts_init_fix_alloc(Uint extra_block_size, - void *(*alloc)(Uint)) -{ - int i; - - xblk_sz = extra_block_size; - core_alloc = alloc; - - fa = (FixAlloc **) (*core_alloc)(FA_SZ * sizeof(FixAlloc *)); - if (!fa) - erts_alloc_enomem(ERTS_ALC_T_UNDEF, FA_SZ * sizeof(FixAlloc *)); - - for (i = 0; i < FA_SZ; i++) - fa[i] = NULL; -#if defined(DEBUG) && !ERTS_ALC_MTA_FIXED_SIZE - first_time = 1; -#endif -} - -Uint -erts_get_fix_size(ErtsAlcType_t type) -{ - Uint i = FIX_IX(ERTS_ALC_T2N(type)); - return i < FA_SZ && fa[i] ? fa[i]->item_size : 0; -} - -void -erts_set_fix_size(ErtsAlcType_t type, Uint size) -{ - Uint sz; - Uint i; - FixAlloc *fs; - ErtsAlcType_t t_no = ERTS_ALC_T2N(type); - sz = xblk_sz + size; - -#ifdef DEBUG - ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= t_no); - ASSERT(t_no <= ERTS_ALC_N_MAX_A_FIXED_SIZE); -#endif - - while (sz % sizeof(align_t) != 0) /* Alignment */ - sz++; - - i = FIX_IX(t_no); - fs = (FixAlloc *) (*core_alloc)(sizeof(FixAlloc)); - if (!fs) - erts_alloc_n_enomem(t_no, sizeof(FixAlloc)); - - fs->item_size = sz; - fs->no_blocks = 0; - fs->no_free = 0; - fs->blocks = NULL; - fs->freelist = NULL; - if (fa[i]) - erl_exit(-1, "Attempt to overwrite existing fix size (%d)", i); - fa[i] = fs; - -#if ERTS_ALC_MTA_FIXED_SIZE -#ifdef ERTS_SMP - erts_smp_spinlock_init_x(&fs->slck, "fix_alloc", make_small(i)); -#else - erts_mtx_init_x(&fs->mtx, "fix_alloc", make_small(i)); -#endif -#endif - -} - -void -erts_fix_info(ErtsAlcType_t type, ErtsFixInfo *efip) -{ - Uint i; - FixAlloc *f; -#ifdef DEBUG - FixAllocBlock *b; - void *fp; -#endif - Uint real_item_size; - ErtsAlcType_t t_no = ERTS_ALC_T2N(type); - - ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= t_no); - ASSERT(t_no <= ERTS_ALC_N_MAX_A_FIXED_SIZE); - - i = FIX_IX(t_no); - f = fa[i]; - - efip->total = sizeof(FixAlloc *); - efip->used = 0; - if (!f) - return; - - real_item_size = f->item_size - xblk_sz; - - FA_LOCK(f); - - efip->total += sizeof(FixAlloc); - efip->total += f->no_blocks*FIX_POOL_SZ(real_item_size); - efip->used = efip->total - f->no_free*real_item_size; - -#ifdef DEBUG - ASSERT(efip->total >= efip->used); - for(i = 0, b = f->blocks; b; i++, b = b->next); - ASSERT(f->no_blocks == i); - for (i = 0, fp = f->freelist; fp; i++, fp = *((void **) fp)); - ASSERT(f->no_free == i); -#endif - - FA_UNLOCK(f); - -} - -void -erts_fix_free(ErtsAlcType_t t_no, void *extra, void* ptr) -{ - Uint i; - FixAlloc *f; - - ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= t_no); - ASSERT(t_no <= ERTS_ALC_N_MAX_A_FIXED_SIZE); - - i = FIX_IX(t_no); - f = fa[i]; - - FA_LOCK(f); - *((void **) ptr) = f->freelist; - f->freelist = ptr; - f->no_free++; - FA_UNLOCK(f); -} - - -void *erts_fix_realloc(ErtsAlcType_t t_no, void *extra, void* ptr, Uint size) -{ - erts_alc_fatal_error(ERTS_ALC_E_NOTSUP, ERTS_ALC_O_REALLOC, t_no); - return NULL; -} - -void *erts_fix_alloc(ErtsAlcType_t t_no, void *extra, Uint size) -{ - void *ret; - int i; - FixAlloc *f; - -#if defined(DEBUG) && !ERTS_ALC_MTA_FIXED_SIZE - ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= t_no); - ASSERT(t_no <= ERTS_ALC_N_MAX_A_FIXED_SIZE); - if (first_time) { /* Check that all sizes have been initialized */ - int i; - for (i = 0; i < FA_SZ; i++) - ASSERT(fa[i]); - first_time = 0; - } -#endif - - - i = FIX_IX(t_no); - f = fa[i]; - - ASSERT(f); - ASSERT(f->item_size >= size); - - FA_LOCK(f); - if (f->freelist == NULL) { /* Gotta alloc some more mem */ - char *ptr; - FixAllocBlock *bl; - Uint n; - - - FA_UNLOCK(f); - bl = (*core_alloc)(FIX_POOL_SZ(f->item_size)); - if (!bl) - return NULL; - - FA_LOCK(f); - bl->next = f->blocks; /* link in first */ - f->blocks = bl; - - n = NOPERBLOCK; - ptr = (char *) &f->blocks->mem[0]; - while(n--) { - *((void **) ptr) = f->freelist; - f->freelist = (void *) ptr; - ptr += f->item_size; - } -#if !ERTS_ALC_MTA_FIXED_SIZE - ASSERT(f->no_free == 0); -#endif - f->no_free += NOPERBLOCK; - f->no_blocks++; - } - - ret = f->freelist; - f->freelist = *((void **) f->freelist); - ASSERT(f->no_free > 0); - f->no_free--; - - FA_UNLOCK(f); - - return ret; -} - -#endif /* #ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE */ diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 249df54015..6687e02485 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -546,7 +546,7 @@ ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt) tombstone = (Eterm*) erts_smp_atomic_add_read_nob(&erts_dead_ports_ptr, -(erts_aint_t)sizeof(Eterm)); ASSERT(tombstone+1 != NULL); - ASSERT(prt->snapshot == erts_smp_atomic_read_nob(&erts_ports_snapshot) - 1); + ASSERT(prt->snapshot == erts_smp_atomic32_read_nob(&erts_ports_snapshot) - 1); *tombstone = prt->id; } /*else no ongoing snapshot or port was already included or created after snapshot */ diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 669a601b35..9f5747205d 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -616,13 +616,10 @@ extern char *erts_sys_ddll_error(int code); * System interfaces for startup. */ - -#ifdef ERTS_SMP void erts_sys_schedule_interrupt(int set); +#ifdef ERTS_SMP void erts_sys_schedule_interrupt_timed(int set, long msec); void erts_sys_main_thread(void); -#else -#define erts_sys_schedule_interrupt(Set) #endif extern void erts_sys_prepare_crash_dump(void); @@ -697,10 +694,10 @@ int erts_write_env(char *key, char *value); int sys_alloc_opt(int, int); typedef struct { - Sint trim_threshold; - Sint top_pad; - Sint mmap_threshold; - Sint mmap_max; + int trim_threshold; + int top_pad; + int mmap_threshold; + int mmap_max; } SysAllocStat; void sys_alloc_stat(SysAllocStat *); diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 8fa8c1cfe0..db9a24e0a3 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -444,7 +444,7 @@ erts_time_left(ErlTimer *p) } #ifdef DEBUG -void erts_p_slpq() +void erts_p_slpq(void) { int i; ErlTimer* p; diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 3f6accba2d..3fa84bd13c 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -42,6 +42,8 @@ #include "erl_threads.h" #include "erl_smp.h" #include "erl_time.h" +#include "erl_thr_progress.h" +#include "erl_sched_spec_pre_alloc.h" #undef M_TRIM_THRESHOLD #undef M_TOP_PAD @@ -3250,10 +3252,10 @@ erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer) #endif -static Sint trim_threshold; -static Sint top_pad; -static Sint mmap_threshold; -static Sint mmap_max; +static int trim_threshold; +static int top_pad; +static int mmap_threshold; +static int mmap_max; Uint tot_bin_allocated; @@ -3276,8 +3278,8 @@ int sys_alloc_opt(int opt, int value) { #if HAVE_MALLOPT - Sint m_opt; - Sint *curr_val; + int m_opt; + int *curr_val; switch(opt) { case SYS_ALLOC_OPT_TRIM_THRESHOLD: @@ -3317,7 +3319,7 @@ sys_alloc_opt(int opt, int value) } if(mallopt(m_opt, value)) { - *curr_val = (Sint) value; + *curr_val = value; return 1; } -- cgit v1.2.3