/* * %CopyrightBegin% * * Copyright Ericsson AB 2002-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. * * %CopyrightEnd% */ /* * Description: Management of memory allocators. * * Author: Rickard Green */ #ifdef HAVE_CONFIG_H # include "config.h" #endif #define ERTS_ALLOC_C__ #define ERTS_ALC_INTERNAL__ #include "sys.h" #define ERL_THREADS_EMU_INTERNAL__ #include "erl_threads.h" #include "global.h" #include "erl_db.h" #include "erl_binary.h" #include "erl_bits.h" #include "erl_instrument.h" #include "erl_mseg.h" #include "erl_monitors.h" #include "erl_bif_timer.h" #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE) #include "erl_check_io.h" #endif #define GET_ERL_GF_ALLOC_IMPL #include "erl_goodfit_alloc.h" #define GET_ERL_BF_ALLOC_IMPL #include "erl_bestfit_alloc.h" #define GET_ERL_AF_ALLOC_IMPL #include "erl_afit_alloc.h" #define ERTS_ALC_DEFAULT_MAX_THR_PREF 16 #if defined(SMALL_MEMORY) || defined(PURIFY) || defined(VALGRIND) #define AU_ALLOC_DEFAULT_ENABLE(X) 0 #else #define AU_ALLOC_DEFAULT_ENABLE(X) (X) #endif #ifdef DEBUG static Uint install_debug_functions(void); #if 0 #define HARD_DEBUG #ifdef __GNUC__ #warning "* * * * * * * * * * * * * *" #warning "* HARD DEBUG IS ENABLED! *" #warning "* * * * * * * * * * * * * *" #endif #endif #endif ErtsAllocatorFunctions_t erts_allctrs[ERTS_ALC_A_MAX+1]; ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1]; ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1]; #define ERTS_MIN(A, B) ((A) < (B) ? (A) : (B)) #define ERTS_MAX(A, B) ((A) > (B) ? (A) : (B)) typedef union { GFAllctr_t gfa; char align_gfa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(GFAllctr_t))]; BFAllctr_t bfa; char align_bfa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(BFAllctr_t))]; AFAllctr_t afa; char align_afa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AFAllctr_t))]; } ErtsAllocatorState_t; static ErtsAllocatorState_t sl_alloc_state; static ErtsAllocatorState_t std_alloc_state; static ErtsAllocatorState_t ll_alloc_state; #if HALFWORD_HEAP static ErtsAllocatorState_t std_alloc_low_state; static ErtsAllocatorState_t ll_alloc_low_state; #endif static ErtsAllocatorState_t temp_alloc_state; static ErtsAllocatorState_t eheap_alloc_state; static ErtsAllocatorState_t binary_alloc_state; static ErtsAllocatorState_t ets_alloc_state; static ErtsAllocatorState_t driver_alloc_state; ErtsAlcType_t erts_fix_core_allocator_ix; #ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE static void *(*fix_core_allocator)(ErtsAlcType_t, void *, Uint); static void *fix_core_extra; static void *fix_core_alloc(Uint size) { void *res; res = (*fix_core_allocator)(ERTS_ALC_T_UNDEF, fix_core_extra, size); if (erts_mtrace_enabled) erts_mtrace_crr_alloc(res, ERTS_ALC_A_FIXED_SIZE, erts_fix_core_allocator_ix, size); return res; } #endif enum allctr_type { GOODFIT, BESTFIT, AFIT }; struct au_init { int enable; int thr_spec; enum allctr_type atype; struct { AllctrInit_t util; GFAllctrInit_t gf; BFAllctrInit_t bf; AFAllctrInit_t af; } init; struct { int mmbcs; int lmbcs; int smbcs; int mmmbc; } default_; }; #define DEFAULT_ALLCTR_INIT { \ ERTS_DEFAULT_ALLCTR_INIT, \ ERTS_DEFAULT_GF_ALLCTR_INIT, \ ERTS_DEFAULT_BF_ALLCTR_INIT, \ ERTS_DEFAULT_AF_ALLCTR_INIT \ } typedef struct { int erts_alloc_config; #if HAVE_ERTS_MSEG ErtsMsegInit_t mseg; #endif int trim_threshold; int top_pad; AlcUInit_t alloc_util; struct { int stat; int map; char *mtrace; char *nodename; } instr; struct au_init sl_alloc; struct au_init std_alloc; struct au_init ll_alloc; struct au_init temp_alloc; struct au_init eheap_alloc; struct au_init binary_alloc; struct au_init ets_alloc; struct au_init driver_alloc; #if HALFWORD_HEAP struct au_init std_alloc_low; struct au_init ll_alloc_low; #endif } erts_alc_hndl_args_init_t; #define ERTS_AU_INIT__ {0, 0, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}} #define SET_DEFAULT_ALLOC_OPTS(IP) \ do { \ struct au_init aui__ = ERTS_AU_INIT__; \ sys_memcpy((void *) (IP), (void *) &aui__, sizeof(struct au_init)); \ } while (0) static void set_default_sl_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; ip->atype = GOODFIT; ip->init.util.name_prefix = "sl_"; ip->init.util.mmmbc = 5; ip->init.util.alloc_no = ERTS_ALC_A_SHORT_LIVED; #ifndef SMALL_MEMORY ip->init.util.mmbcs = 128*1024; /* Main carrier size */ #else ip->init.util.mmbcs = 32*1024; /* Main carrier size */ #endif ip->init.util.ts = ERTS_ALC_MTA_SHORT_LIVED; ip->init.util.rsbcst = 80; #if HALFWORD_HEAP ip->init.util.low_mem = 1; #endif } static void set_default_std_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; ip->atype = BESTFIT; ip->init.util.name_prefix = "std_"; ip->init.util.mmmbc = 5; ip->init.util.alloc_no = ERTS_ALC_A_STANDARD; #ifndef SMALL_MEMORY ip->init.util.mmbcs = 128*1024; /* Main carrier size */ #else ip->init.util.mmbcs = 32*1024; /* Main carrier size */ #endif ip->init.util.ts = ERTS_ALC_MTA_STANDARD; } static void set_default_ll_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 0; ip->atype = BESTFIT; ip->init.bf.ao = 1; ip->init.util.ramv = 0; ip->init.util.mmsbc = 0; ip->init.util.mmmbc = 0; ip->init.util.sbct = ~((UWord) 0); ip->init.util.name_prefix = "ll_"; ip->init.util.alloc_no = ERTS_ALC_A_LONG_LIVED; #ifndef SMALL_MEMORY ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */ #else ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */ #endif ip->init.util.ts = ERTS_ALC_MTA_LONG_LIVED; ip->init.util.asbcst = 0; ip->init.util.rsbcst = 0; ip->init.util.rsbcmt = 0; ip->init.util.rmbcmt = 0; } static void set_default_temp_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; ip->atype = AFIT; ip->init.util.name_prefix = "temp_"; ip->init.util.alloc_no = ERTS_ALC_A_TEMPORARY; #ifndef SMALL_MEMORY ip->init.util.mmbcs = 128*1024; /* Main carrier size */ #else ip->init.util.mmbcs = 32*1024; /* Main carrier size */ #endif ip->init.util.ts = ERTS_ALC_MTA_TEMPORARY; ip->init.util.rsbcst = 90; ip->init.util.rmbcmt = 100; #if HALFWORD_HEAP ip->init.util.low_mem = 1; #endif } static void set_default_eheap_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; ip->atype = GOODFIT; ip->init.util.mmmbc = 100; ip->init.util.name_prefix = "eheap_"; ip->init.util.alloc_no = ERTS_ALC_A_EHEAP; #ifndef SMALL_MEMORY ip->init.util.mmbcs = 512*1024; /* Main carrier size */ #else ip->init.util.mmbcs = 256*1024; /* Main carrier size */ #endif ip->init.util.ts = ERTS_ALC_MTA_EHEAP; ip->init.util.rsbcst = 50; #if HALFWORD_HEAP ip->init.util.low_mem = 1; #endif } static void set_default_binary_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; ip->atype = BESTFIT; ip->init.util.mmmbc = 50; ip->init.util.name_prefix = "binary_"; ip->init.util.alloc_no = ERTS_ALC_A_BINARY; #ifndef SMALL_MEMORY ip->init.util.mmbcs = 128*1024; /* Main carrier size */ #else ip->init.util.mmbcs = 32*1024; /* Main carrier size */ #endif ip->init.util.ts = ERTS_ALC_MTA_BINARY; } static void set_default_ets_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; ip->atype = BESTFIT; ip->init.util.mmmbc = 100; ip->init.util.name_prefix = "ets_"; ip->init.util.alloc_no = ERTS_ALC_A_ETS; #ifndef SMALL_MEMORY ip->init.util.mmbcs = 128*1024; /* Main carrier size */ #else ip->init.util.mmbcs = 32*1024; /* Main carrier size */ #endif ip->init.util.ts = ERTS_ALC_MTA_ETS; } static void set_default_driver_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; ip->atype = BESTFIT; ip->init.util.name_prefix = "driver_"; ip->init.util.alloc_no = ERTS_ALC_A_DRIVER; #ifndef SMALL_MEMORY ip->init.util.mmbcs = 128*1024; /* Main carrier size */ #else ip->init.util.mmbcs = 32*1024; /* Main carrier size */ #endif ip->init.util.ts = ERTS_ALC_MTA_DRIVER; } #ifdef ERTS_SMP static void adjust_tpref(struct au_init *ip, int no_sched) { if (ip->thr_spec) { Uint allocs; if (ip->thr_spec < 0) {/* User specified amount */ allocs = abs(ip->thr_spec); if (allocs > no_sched) allocs = no_sched; } else if (no_sched > ERTS_ALC_DEFAULT_MAX_THR_PREF) allocs = ERTS_ALC_DEFAULT_MAX_THR_PREF; else allocs = no_sched; if (allocs <= 1) ip->thr_spec = 0; else { ip->thr_spec = (int) allocs; ip->thr_spec *= -1; /* thread preferred */ /* If default ... */ /* ... shrink main multi-block carrier size */ if (ip->default_.mmbcs) ip->init.util.mmbcs /= ERTS_MIN(4, allocs); /* ... shrink largest multi-block carrier size */ if (ip->default_.lmbcs) ip->init.util.lmbcs /= ERTS_MIN(2, allocs); /* ... shrink smallest multi-block carrier size */ if (ip->default_.smbcs) ip->init.util.smbcs /= ERTS_MIN(4, allocs); /* ... and more than three allocators shrink max mseg multi-block carriers */ if (ip->default_.mmmbc && allocs > 2) { ip->init.util.mmmbc /= ERTS_MIN(4, allocs - 1); if (ip->init.util.mmmbc < 3) ip->init.util.mmmbc = 3; } } } } #endif static void handle_args(int *, char **, erts_alc_hndl_args_init_t *); static void set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init); static void start_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, ErtsAllocatorState_t *state); static void refuse_af_strategy(struct au_init *init) { if (init->atype == AFIT) init->atype = GOODFIT; } static void init_thr_ix(int static_ixs); #ifdef HARD_DEBUG static void hdbg_init(void); #endif void erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) { UWord extra_block_size = 0; int i; erts_alc_hndl_args_init_t init = { 0, #if HAVE_ERTS_MSEG ERTS_MSEG_INIT_DEFAULT_INITIALIZER, #endif ERTS_DEFAULT_TRIM_THRESHOLD, ERTS_DEFAULT_TOP_PAD, ERTS_DEFAULT_ALCU_INIT }; #ifdef HARD_DEBUG hdbg_init(); #endif erts_sys_alloc_init(); init_thr_ix(erts_no_schedulers); erts_init_utils_mem(); set_default_sl_alloc_opts(&init.sl_alloc); set_default_std_alloc_opts(&init.std_alloc); set_default_ll_alloc_opts(&init.ll_alloc); set_default_temp_alloc_opts(&init.temp_alloc); set_default_eheap_alloc_opts(&init.eheap_alloc); set_default_binary_alloc_opts(&init.binary_alloc); set_default_ets_alloc_opts(&init.ets_alloc); set_default_driver_alloc_opts(&init.driver_alloc); if (argc && argv) handle_args(argc, argv, &init); if (erts_no_schedulers <= 1) { init.sl_alloc.thr_spec = 0; init.std_alloc.thr_spec = 0; init.ll_alloc.thr_spec = 0; init.eheap_alloc.thr_spec = 0; init.binary_alloc.thr_spec = 0; init.ets_alloc.thr_spec = 0; init.driver_alloc.thr_spec = 0; } if (init.erts_alloc_config) { /* Adjust flags that erts_alloc_config won't like */ init.temp_alloc.thr_spec = 0; init.sl_alloc.thr_spec = 0; init.std_alloc.thr_spec = 0; init.ll_alloc.thr_spec = 0; init.eheap_alloc.thr_spec = 0; init.binary_alloc.thr_spec = 0; init.ets_alloc.thr_spec = 0; init.driver_alloc.thr_spec = 0; } #ifdef ERTS_SMP /* Only temp_alloc can use thread specific interface */ if (init.temp_alloc.thr_spec) init.temp_alloc.thr_spec = erts_no_schedulers; /* Others must use thread preferred interface */ adjust_tpref(&init.sl_alloc, erts_no_schedulers); adjust_tpref(&init.std_alloc, erts_no_schedulers); adjust_tpref(&init.ll_alloc, erts_no_schedulers); adjust_tpref(&init.eheap_alloc, erts_no_schedulers); adjust_tpref(&init.binary_alloc, erts_no_schedulers); adjust_tpref(&init.ets_alloc, erts_no_schedulers); adjust_tpref(&init.driver_alloc, erts_no_schedulers); #else /* No thread specific if not smp */ init.temp_alloc.thr_spec = 0; #endif /* * The following allocators cannot be run with afit strategy. * Make sure they don't... */ refuse_af_strategy(&init.sl_alloc); refuse_af_strategy(&init.std_alloc); refuse_af_strategy(&init.ll_alloc); refuse_af_strategy(&init.eheap_alloc); refuse_af_strategy(&init.binary_alloc); refuse_af_strategy(&init.ets_alloc); refuse_af_strategy(&init.driver_alloc); #ifdef ERTS_SMP if (!init.temp_alloc.thr_spec) refuse_af_strategy(&init.temp_alloc); #endif erts_mtrace_pre_init(); #if HAVE_ERTS_MSEG erts_mseg_init(&init.mseg); #endif erts_alcu_init(&init.alloc_util); erts_afalc_init(); erts_bfalc_init(); erts_gfalc_init(); for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { erts_allctrs[i].alloc = NULL; erts_allctrs[i].realloc = NULL; erts_allctrs[i].free = NULL; erts_allctrs[i].extra = NULL; erts_allctrs_info[i].alloc_util = 0; erts_allctrs_info[i].enabled = 0; erts_allctrs_info[i].thr_spec = 0; erts_allctrs_info[i].extra = NULL; } #ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE #if !defined(PURIFY) && !defined(VALGRIND) erts_allctrs[ERTS_ALC_A_FIXED_SIZE].alloc = erts_fix_alloc; erts_allctrs[ERTS_ALC_A_FIXED_SIZE].realloc = erts_fix_realloc; erts_allctrs[ERTS_ALC_A_FIXED_SIZE].free = erts_fix_free; erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].enabled = 1; #else erts_allctrs[ERTS_ALC_A_FIXED_SIZE].alloc = erts_sys_alloc; erts_allctrs[ERTS_ALC_A_FIXED_SIZE].realloc = erts_sys_realloc; erts_allctrs[ERTS_ALC_A_FIXED_SIZE].free = erts_sys_free; erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].enabled = 0; #endif #endif erts_allctrs[ERTS_ALC_A_SYSTEM].alloc = erts_sys_alloc; erts_allctrs[ERTS_ALC_A_SYSTEM].realloc = erts_sys_realloc; erts_allctrs[ERTS_ALC_A_SYSTEM].free = erts_sys_free; erts_allctrs_info[ERTS_ALC_A_SYSTEM].enabled = 1; #if HALFWORD_HEAP init.std_alloc_low = init.std_alloc; init.std_alloc_low.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW; init.std_alloc_low.init.util.low_mem = 1; init.ll_alloc_low = init.ll_alloc; init.ll_alloc_low.init.util.alloc_no = ERTS_ALC_A_LONG_LIVED_LOW; init.ll_alloc_low.init.util.low_mem = 1; set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_alloc_low); set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_alloc_low); #endif /* HALFWORD */ set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc); set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc); set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc); set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc); set_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc); set_au_allocator(ERTS_ALC_A_BINARY, &init.binary_alloc); set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc); set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc); for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { if (!erts_allctrs[i].alloc) erl_exit(ERTS_ABORT_EXIT, "Missing alloc function for %s\n", ERTS_ALC_A2AD(i)); if (!erts_allctrs[i].realloc) erl_exit(ERTS_ABORT_EXIT, "Missing realloc function for %s\n", ERTS_ALC_A2AD(i)); if (!erts_allctrs[i].free) erl_exit(ERTS_ABORT_EXIT, "Missing free function for %s\n", ERTS_ALC_A2AD(i)); } sys_alloc_opt(SYS_ALLOC_OPT_TRIM_THRESHOLD, init.trim_threshold); sys_alloc_opt(SYS_ALLOC_OPT_TOP_PAD, init.top_pad); if (erts_allctrs_info[ERTS_FIX_CORE_ALLOCATOR].enabled) erts_fix_core_allocator_ix = ERTS_FIX_CORE_ALLOCATOR; else erts_fix_core_allocator_ix = ERTS_ALC_A_SYSTEM; erts_mtrace_init(init.instr.mtrace, init.instr.nodename); start_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, &temp_alloc_state); start_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc, &sl_alloc_state); start_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc, &std_alloc_state); start_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc, &ll_alloc_state); #if HALFWORD_HEAP start_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_alloc_low, &ll_alloc_low_state); start_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_alloc_low, &std_alloc_low_state); #endif start_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc, &eheap_alloc_state); start_au_allocator(ERTS_ALC_A_BINARY, &init.binary_alloc, &binary_alloc_state); start_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc, &ets_alloc_state); start_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, &driver_alloc_state); fix_core_allocator = erts_allctrs[erts_fix_core_allocator_ix].alloc; fix_core_extra = erts_allctrs[erts_fix_core_allocator_ix].extra; erts_mtrace_install_wrapper_functions(); extra_block_size += erts_instr_init(init.instr.stat, init.instr.map); #ifdef DEBUG extra_block_size += install_debug_functions(); #endif #ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE erts_init_fix_alloc(extra_block_size, fix_core_alloc); #if !defined(PURIFY) && !defined(VALGRIND) erts_set_fix_size(ERTS_ALC_T_PROC, sizeof(Process)); erts_set_fix_size(ERTS_ALC_T_DB_TABLE, sizeof(DbTable)); erts_set_fix_size(ERTS_ALC_T_ATOM, sizeof(Atom)); erts_set_fix_size(ERTS_ALC_T_MODULE, sizeof(Module)); erts_set_fix_size(ERTS_ALC_T_REG_PROC, sizeof(RegProc)); erts_set_fix_size(ERTS_ALC_T_FUN_ENTRY, sizeof(ErlFunEntry)); #ifdef ERTS_ALC_T_DRV_EV_D_STATE erts_set_fix_size(ERTS_ALC_T_DRV_EV_D_STATE, sizeof(ErtsDrvEventDataState)); #endif #ifdef ERTS_ALC_T_DRV_SEL_D_STATE erts_set_fix_size(ERTS_ALC_T_DRV_SEL_D_STATE, sizeof(ErtsDrvSelectDataState)); #endif #if !HALFWORD_HEAP erts_set_fix_size(ERTS_ALC_T_EXPORT, sizeof(Export)); erts_set_fix_size(ERTS_ALC_T_MONITOR_SH, ERTS_MONITOR_SH_SIZE*sizeof(Uint)); erts_set_fix_size(ERTS_ALC_T_NLINK_SH, ERTS_LINK_SH_SIZE*sizeof(Uint)); #endif #endif #endif } static void set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init) { ErtsAllocatorFunctions_t *af = &erts_allctrs[alctr_n]; ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n]; ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n]; #if HALFWORD_HEAP /* If halfword heap, silently ignore any disabling of internal * allocators for low memory */ if (init->init.util.low_mem) { init->enable = 1; } #endif if (!init->enable) { af->alloc = erts_sys_alloc; af->realloc = erts_sys_realloc; af->free = erts_sys_free; af->extra = NULL; ai->alloc_util = 0; ai->enabled = 0; ai->extra = NULL; return; } tspec->enabled = 0; tspec->all_thr_safe = 0; ai->thr_spec = 0; #ifdef USE_THREADS if (init->thr_spec) { if (init->thr_spec > 0) { af->alloc = erts_alcu_alloc_thr_spec; if (init->init.util.ramv) af->realloc = erts_alcu_realloc_mv_thr_spec; else af->realloc = erts_alcu_realloc_thr_spec; af->free = erts_alcu_free_thr_spec; } else { af->alloc = erts_alcu_alloc_thr_pref; if (init->init.util.ramv) af->realloc = erts_alcu_realloc_mv_thr_pref; else af->realloc = erts_alcu_realloc_thr_pref; af->free = erts_alcu_free_thr_pref; tspec->all_thr_safe = 1; } tspec->enabled = 1; tspec->size = abs(init->thr_spec) + 1; ai->thr_spec = tspec->size; } else if (init->init.util.ts) { af->alloc = erts_alcu_alloc_ts; if (init->init.util.ramv) af->realloc = erts_alcu_realloc_mv_ts; else af->realloc = erts_alcu_realloc_ts; af->free = erts_alcu_free_ts; } else #endif { af->alloc = erts_alcu_alloc; if (init->init.util.ramv) af->realloc = erts_alcu_realloc_mv; else af->realloc = erts_alcu_realloc; af->free = erts_alcu_free; } af->extra = NULL; ai->alloc_util = 1; ai->enabled = 1; } static void start_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, ErtsAllocatorState_t *state) { int i; int size = 1; void *as0; enum allctr_type atype; ErtsAllocatorFunctions_t *af = &erts_allctrs[alctr_n]; ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n]; ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n]; if (!init->enable) return; if (init->thr_spec) { void *states = erts_sys_alloc(0, NULL, ((sizeof(Allctr_t *) * (tspec->size + 1)) + (sizeof(ErtsAllocatorState_t) * tspec->size) + ERTS_CACHE_LINE_SIZE - 1)); if (!states) erl_exit(ERTS_ABORT_EXIT, "Failed to allocate allocator states for %salloc\n", init->init.util.name_prefix); tspec->allctr = (Allctr_t **) states; states = ((char *) states) + sizeof(Allctr_t *) * (tspec->size + 1); states = ((((UWord) states) & ERTS_CACHE_LINE_MASK) ? (void *) ((((UWord) states) & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE) : (void *) states); tspec->allctr[0] = init->thr_spec > 0 ? (Allctr_t *) state : (Allctr_t *) NULL; size = tspec->size; for (i = 1; i < size; i++) tspec->allctr[i] = (Allctr_t *) &((ErtsAllocatorState_t *) states)[i-1]; } for (i = 0; i < size; i++) { void *as; atype = init->atype; if (!init->thr_spec) as0 = state; else { as0 = (void *) tspec->allctr[i]; if (!as0) continue; if (i == 0) { if (atype == AFIT) atype = GOODFIT; init->init.util.ts = 1; } else { if (init->thr_spec < 0) { init->init.util.ts = 1; init->init.util.tspec = 0; init->init.util.tpref = -1*init->thr_spec; } else { init->init.util.ts = 0; init->init.util.tspec = init->thr_spec + 1; init->init.util.tpref = 0; } } } switch (atype) { case GOODFIT: as = (void *) erts_gfalc_start((GFAllctr_t *) as0, &init->init.gf, &init->init.util); break; case BESTFIT: as = (void *) erts_bfalc_start((BFAllctr_t *) as0, &init->init.bf, &init->init.util); break; case AFIT: as = (void *) erts_afalc_start((AFAllctr_t *) as0, &init->init.af, &init->init.util); break; default: as = NULL; ASSERT(0); } if (!as) erl_exit(ERTS_ABORT_EXIT, "Failed to start %salloc\n", init->init.util.name_prefix); ASSERT(as == (void *) as0); af->extra = as; } if (init->thr_spec) { af->extra = tspec; init->init.util.ts = 1; } ai->extra = af->extra; } static void bad_param(char *param_start, char *param_end) { size_t len = param_end - param_start; char param[100]; if (len > 99) len = 99; sys_memcpy((void *) param, (void *) param_start, len); param[len] = '\0'; erts_fprintf(stderr, "bad \"%s\" parameter\n", param); erts_usage(); } static void bad_value(char *param_start, char *param_end, char *value) { size_t len = param_end - param_start; char param[100]; if (len > 99) len = 99; sys_memcpy((void *) param, (void *) param_start, len); param[len] = '\0'; erts_fprintf(stderr, "bad \"%s\" value: %s\n", param, value); erts_usage(); } /* Get arg marks argument as handled by putting NULL in argv */ static char * get_value(char* rest, char** argv, int* ip) { char *param = argv[*ip]+1; argv[*ip] = NULL; if (*rest == '\0') { char *next = argv[*ip + 1]; if (next[0] == '-' && next[1] == '-' && next[2] == '\0') { bad_value(param, rest, ""); } (*ip)++; argv[*ip] = NULL; return next; } return rest; } static ERTS_INLINE int has_prefix(const char *prefix, const char *string) { int i; for (i = 0; prefix[i]; i++) if (prefix[i] != string[i]) return 0; return 1; } static int get_bool_value(char *param_end, char** argv, int* ip) { char *param = argv[*ip]+1; char *value = get_value(param_end, argv, ip); if (strcmp(value, "true") == 0) return 1; else if (strcmp(value, "false") == 0) return 0; else bad_value(param, param_end, value); return -1; } static Uint get_kb_value(char *param_end, char** argv, int* ip) { Sint tmp; Uint max = ((~((Uint) 0))/1024) + 1; char *rest; char *param = argv[*ip]+1; char *value = get_value(param_end, argv, ip); errno = 0; tmp = (Sint) strtol(value, &rest, 10); if (errno != 0 || rest == value || tmp < 0 || max < ((Uint) tmp)) bad_value(param, param_end, value); if (max == (Uint) tmp) return ~((Uint) 0); else return ((Uint) tmp)*1024; } static Uint get_amount_value(char *param_end, char** argv, int* ip) { Sint tmp; char *rest; char *param = argv[*ip]+1; char *value = get_value(param_end, argv, ip); errno = 0; tmp = (Sint) strtol(value, &rest, 10); if (errno != 0 || rest == value || tmp < 0) bad_value(param, param_end, value); return (Uint) tmp; } static int get_bool_or_possitive_amount_value(int *bool, Uint *amount, char *param_end, char** argv, int* ip) { char *param = argv[*ip]+1; char *value = get_value(param_end, argv, ip); if (strcmp(value, "true") == 0) { *bool = 1; return 1; } else if (strcmp(value, "false") == 0) { *bool = 0; return 1; } else { Sint tmp; char *rest; errno = 0; tmp = (Sint) strtol(value, &rest, 10); if (errno != 0 || rest == value || tmp <= 0) { bad_value(param, param_end, value); return -1; } *amount = (Uint) tmp; return 0; } } static void handle_au_arg(struct au_init *auip, char* sub_param, char** argv, int* ip) { char *param = argv[*ip]+1; switch (sub_param[0]) { case 'a': if(has_prefix("asbcst", sub_param)) { auip->init.util.asbcst = get_kb_value(sub_param + 6, argv, ip); } else if(has_prefix("as", sub_param)) { char *alg = get_value(sub_param + 2, argv, ip); if (strcmp("bf", alg) == 0) { auip->atype = BESTFIT; auip->init.bf.ao = 0; } else if (strcmp("aobf", alg) == 0) { auip->atype = BESTFIT; auip->init.bf.ao = 1; } else if (strcmp("gf", alg) == 0) { auip->atype = GOODFIT; } else if (strcmp("af", alg) == 0) { auip->atype = AFIT; } else { bad_value(param, sub_param + 1, alg); } } else goto bad_switch; break; case 'e': auip->enable = get_bool_value(sub_param+1, argv, ip); break; case 'l': if (has_prefix("lmbcs", sub_param)) { auip->default_.lmbcs = 0; auip->init.util.lmbcs = get_kb_value(sub_param + 5, argv, ip); } else goto bad_switch; break; case 'm': if (has_prefix("mbcgs", sub_param)) { auip->init.util.mbcgs = get_amount_value(sub_param + 5, argv, ip); } else if (has_prefix("mbsd", sub_param)) { auip->init.gf.mbsd = get_amount_value(sub_param + 4, argv, ip); if (auip->init.gf.mbsd < 1) auip->init.gf.mbsd = 1; } else if (has_prefix("mmbcs", sub_param)) { auip->default_.mmbcs = 0; auip->init.util.mmbcs = get_kb_value(sub_param + 5, argv, ip); } else if (has_prefix("mmmbc", sub_param)) { auip->default_.mmmbc = 0; auip->init.util.mmmbc = get_amount_value(sub_param + 5, argv, ip); } else if (has_prefix("mmsbc", sub_param)) { auip->init.util.mmsbc = get_amount_value(sub_param + 5, argv, ip); } else goto bad_switch; break; case 'r': if(has_prefix("rsbcmt", sub_param)) { auip->init.util.rsbcmt = get_amount_value(sub_param + 6, argv, ip); if (auip->init.util.rsbcmt > 100) auip->init.util.rsbcmt = 100; } else if(has_prefix("rsbcst", sub_param)) { auip->init.util.rsbcst = get_amount_value(sub_param + 6, argv, ip); if (auip->init.util.rsbcst > 100) auip->init.util.rsbcst = 100; } else if (has_prefix("rmbcmt", sub_param)) { auip->init.util.rmbcmt = get_amount_value(sub_param + 6, argv, ip); if (auip->init.util.rmbcmt > 100) auip->init.util.rmbcmt = 100; } else if (has_prefix("ramv", sub_param)) { auip->init.util.ramv = get_bool_value(sub_param + 4, argv, ip); } else goto bad_switch; break; case 's': if(has_prefix("sbct", sub_param)) { auip->init.util.sbct = get_kb_value(sub_param + 4, argv, ip); } else if (has_prefix("smbcs", sub_param)) { auip->default_.smbcs = 0; auip->init.util.smbcs = get_kb_value(sub_param + 5, argv, ip); } else goto bad_switch; break; case 't': { Uint no; int enable; int res = get_bool_or_possitive_amount_value(&enable, &no, sub_param+1, argv, ip); if (res > 0) auip->thr_spec = enable ? 1 : 0; else if (res == 0) { int allocs = (int) no; if (allocs < 0) allocs = INT_MIN; else { allocs *= -1; } auip->thr_spec = allocs; } break; } default: bad_switch: bad_param(param, sub_param); } } static void handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) { struct au_init *aui[] = { &init->binary_alloc, &init->std_alloc, &init->ets_alloc, &init->eheap_alloc, &init->ll_alloc, &init->driver_alloc, &init->sl_alloc, &init->temp_alloc }; int aui_sz = (int) sizeof(aui)/sizeof(aui[0]); char *arg; char *rest; int i, j; i = 1; ASSERT(argc && argv && init); while (i < *argc) { if(argv[i][0] == '-') { char *param = argv[i]+1; switch (argv[i][1]) { case 'M': switch (argv[i][2]) { case 'B': handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i); break; case 'D': handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i); break; case 'E': handle_au_arg(&init->ets_alloc, &argv[i][3], argv, &i); break; case 'F': /* fix_alloc */ if (has_prefix("e", param+2)) { arg = get_value(param+3, argv, &i); if (strcmp("true", arg) != 0) bad_value(param, param+3, arg); } else bad_param(param, param+2); break; case 'H': handle_au_arg(&init->eheap_alloc, &argv[i][3], argv, &i); break; case 'L': handle_au_arg(&init->ll_alloc, &argv[i][3], argv, &i); break; case 'M': if (has_prefix("amcbf", argv[i]+3)) { #if HAVE_ERTS_MSEG init->mseg.amcbf = #endif get_kb_value(argv[i]+8, argv, &i); } else if (has_prefix("rmcbf", argv[i]+3)) { #if HAVE_ERTS_MSEG init->mseg.rmcbf = #endif get_amount_value(argv[i]+8, argv, &i); } else if (has_prefix("mcs", argv[i]+3)) { #if HAVE_ERTS_MSEG init->mseg.mcs = #endif get_amount_value(argv[i]+6, argv, &i); } else if (has_prefix("cci", argv[i]+3)) { #if HAVE_ERTS_MSEG init->mseg.cci = #endif get_amount_value(argv[i]+6, argv, &i); } else { bad_param(param, param+2); } break; case 'R': handle_au_arg(&init->driver_alloc, &argv[i][3], argv, &i); break; case 'S': handle_au_arg(&init->sl_alloc, &argv[i][3], argv, &i); break; case 'T': handle_au_arg(&init->temp_alloc, &argv[i][3], argv, &i); break; case 'Y': { /* sys_alloc */ if (has_prefix("tt", param+2)) { /* set trim threshold */ arg = get_value(param+4, argv, &i); errno = 0; init->trim_threshold = (int) strtol(arg, &rest, 10); if (errno != 0 || rest == arg || init->trim_threshold < 0 || (INT_MAX/1024) < init->trim_threshold) { bad_value(param, param+4, arg); } VERBOSE(DEBUG_SYSTEM, ("using trim threshold: %d\n", init->trim_threshold)); init->trim_threshold *= 1024; } else if (has_prefix("tp", param+2)) { /* set top pad */ arg = get_value(param+4, argv, &i); errno = 0; init->top_pad = (int) strtol(arg, &rest, 10); if (errno != 0 || rest == arg || init->top_pad < 0 || (INT_MAX/1024) < init->top_pad) { bad_value(param, param+4, arg); } VERBOSE(DEBUG_SYSTEM, ("using top pad: %d\n",init->top_pad)); init->top_pad *= 1024; } else if (has_prefix("m", param+2)) { /* Has been handled by erlexec */ (void) get_value(param+3, argv, &i); } else if (has_prefix("e", param+2)) { arg = get_value(param+3, argv, &i); if (strcmp("true", arg) != 0) bad_value(param, param+3, arg); } else bad_param(param, param+2); break; } case 'e': switch (argv[i][3]) { case 'a': { int a; arg = get_value(argv[i]+4, argv, &i); if (strcmp("min", arg) == 0) { for (a = 0; a < aui_sz; a++) aui[a]->enable = 0; } else if (strcmp("max", arg) == 0) { for (a = 0; a < aui_sz; a++) aui[a]->enable = 1; } else if (strcmp("config", arg) == 0) { init->erts_alloc_config = 1; } else if (strcmp("r9c", arg) == 0 || strcmp("r10b", arg) == 0 || strcmp("r11b", arg) == 0) { set_default_sl_alloc_opts(&init->sl_alloc); set_default_std_alloc_opts(&init->std_alloc); set_default_ll_alloc_opts(&init->ll_alloc); set_default_temp_alloc_opts(&init->temp_alloc); set_default_eheap_alloc_opts(&init->eheap_alloc); set_default_binary_alloc_opts(&init->binary_alloc); set_default_ets_alloc_opts(&init->ets_alloc); set_default_driver_alloc_opts(&init->driver_alloc); init->driver_alloc.enable = 0; if (strcmp("r9c", arg) == 0) { init->sl_alloc.enable = 0; init->std_alloc.enable = 0; init->binary_alloc.enable = 0; init->ets_alloc.enable = 0; } for (a = 0; a < aui_sz; a++) { aui[a]->thr_spec = 0; aui[a]->init.util.ramv = 0; aui[a]->init.util.mmmbc = 10; aui[a]->init.util.lmbcs = 5*1024*1024; } } else { bad_param(param, param+3); } break; } default: bad_param(param, param+1); } break; case 'i': switch (argv[i][3]) { case 's': arg = get_value(argv[i]+4, argv, &i); if (strcmp("true", arg) == 0) init->instr.stat = 1; else if (strcmp("false", arg) == 0) init->instr.stat = 0; else bad_value(param, param+3, arg); break; case 'm': arg = get_value(argv[i]+4, argv, &i); if (strcmp("true", arg) == 0) init->instr.map = 1; else if (strcmp("false", arg) == 0) init->instr.map = 0; else bad_value(param, param+3, arg); break; case 't': init->instr.mtrace = get_value(argv[i]+4, argv, &i); break; default: bad_param(param, param+2); } break; case 'u': if (has_prefix("ycs", argv[i]+3)) { init->alloc_util.ycs = get_kb_value(argv[i]+6, argv, &i); } else if (has_prefix("mmc", argv[i]+3)) { init->alloc_util.mmc = get_amount_value(argv[i]+6, argv, &i); } else { int a; int start = i; char *param = argv[i]; char *val = i+1 < *argc ? argv[i+1] : NULL; for (a = 0; a < aui_sz; a++) { if (a > 0) { ASSERT(i == start || i == start+1); argv[start] = param; if (i != start) argv[start + 1] = val; i = start; } handle_au_arg(aui[a], &argv[i][3], argv, &i); } } break; default: bad_param(param, param+1); } break; case '-': if (argv[i][2] == '\0') { /* End of system flags reached */ if (init->instr.mtrace /* || init->instr.stat || init->instr.map */) { while (i < *argc) { if(strcmp(argv[i], "-sname") == 0 || strcmp(argv[i], "-name") == 0) { if (i + 1 <*argc) { init->instr.nodename = argv[i+1]; break; } } i++; } } goto args_parsed; } break; default: break; } } i++; } args_parsed: /* Handled arguments have been marked with NULL. Slide arguments not handled towards the beginning of argv. */ for (i = 0, j = 0; i < *argc; i++) { if (argv[i]) argv[j++] = argv[i]; } *argc = j; } static char *type_no_str(ErtsAlcType_t n) { #if ERTS_ALC_N_MIN != 0 if (n < ERTS_ALC_N_MIN) return NULL; #endif if (n > ERTS_ALC_N_MAX) return NULL; return (char *) ERTS_ALC_N2TD(n); } #define type_str(T) type_no_str(ERTS_ALC_T2N((T))) erts_tsd_key_t thr_ix_key; erts_spinlock_t alloc_thr_ix_lock; int last_thr_ix; int first_dyn_thr_ix; static void init_thr_ix(int static_ixs) { erts_tsd_key_create(&thr_ix_key); erts_spinlock_init(&alloc_thr_ix_lock, "alloc_thr_ix_lock"); last_thr_ix = -4711; first_dyn_thr_ix = static_ixs+1; } int erts_alc_get_thr_ix(void) { int ix = (int)(long) erts_tsd_get(thr_ix_key); if (ix == 0) { erts_spin_lock(&alloc_thr_ix_lock); last_thr_ix++; if (last_thr_ix < 0) last_thr_ix = first_dyn_thr_ix; ix = last_thr_ix; erts_spin_unlock(&alloc_thr_ix_lock); erts_tsd_set(thr_ix_key, (void *)(long) ix); } ASSERT(ix > 0); return ix; } void erts_alloc_reg_scheduler_id(Uint id) { int ix = (int) id; ASSERT(0 < ix && ix <= first_dyn_thr_ix); ASSERT(0 == (int) (long) erts_tsd_get(thr_ix_key)); erts_tsd_set(thr_ix_key, (void *)(long) ix); } static void no_verify(Allctr_t *allctr) { } erts_alloc_verify_func_t erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr) { if (erts_allctrs_info[ERTS_ALC_A_TEMPORARY].alloc_util && erts_allctrs_info[ERTS_ALC_A_TEMPORARY].thr_spec) { ErtsAllocatorThrSpec_t *tspec; tspec = &erts_allctr_thr_spec[ERTS_ALC_A_TEMPORARY]; if (!tspec->all_thr_safe) { int ix = erts_alc_get_thr_ix(); if (ix < tspec->size) { *allctr = tspec->allctr[ix]; return erts_alcu_verify_unused; } } } *allctr = NULL; return no_verify; } __decl_noreturn void erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...) { char buf[10]; char *t_str; char *allctr_str; ASSERT(n >= ERTS_ALC_N_MIN); ASSERT(n <= ERTS_ALC_N_MAX); if (n < ERTS_ALC_N_MIN || ERTS_ALC_N_MAX < n) allctr_str = "UNKNOWN"; else { ErtsAlcType_t a = ERTS_ALC_T2A(ERTS_ALC_N2T(n)); if (erts_allctrs_info[a].enabled) allctr_str = (char *) ERTS_ALC_A2AD(a); else allctr_str = (char *) ERTS_ALC_A2AD(ERTS_ALC_A_SYSTEM); } t_str = type_no_str(n); if (!t_str) { sprintf(buf, "%d", (int) n); t_str = buf; } switch (error) { case ERTS_ALC_E_NOTSUP: { char *op_str; switch (func) { case ERTS_ALC_O_ALLOC: op_str = "alloc"; break; case ERTS_ALC_O_REALLOC: op_str = "realloc"; break; case ERTS_ALC_O_FREE: op_str = "free"; break; default: op_str = "UNKNOWN"; break; } erl_exit(ERTS_ABORT_EXIT, "%s: %s operation not supported (memory type: \"%s\")\n", allctr_str, op_str, t_str); break; } case ERTS_ALC_E_NOMEM: { Uint size; va_list argp; char *op = func == ERTS_ALC_O_REALLOC ? "reallocate" : "allocate"; va_start(argp, n); size = va_arg(argp, Uint); va_end(argp); erl_exit(1, "%s: Cannot %s %lu bytes of memory (of type \"%s\").\n", allctr_str, op, size, t_str); break; } case ERTS_ALC_E_NOALLCTR: erl_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown allocator type: %d\n", ERTS_ALC_T2A(ERTS_ALC_N2T(n))); break; default: erl_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown error: %d\n", error); break; } } __decl_noreturn void erts_alloc_enomem(ErtsAlcType_t type, Uint size) { erts_alloc_n_enomem(ERTS_ALC_T2N(type), size); } __decl_noreturn void erts_alloc_n_enomem(ErtsAlcType_t n, Uint size) { erts_alc_fatal_error(ERTS_ALC_E_NOMEM, ERTS_ALC_O_ALLOC, n, size); } __decl_noreturn void erts_realloc_enomem(ErtsAlcType_t type, void *ptr, Uint size) { erts_realloc_n_enomem(ERTS_ALC_T2N(type), ptr, size); } __decl_noreturn void erts_realloc_n_enomem(ErtsAlcType_t n, void *ptr, Uint size) { erts_alc_fatal_error(ERTS_ALC_E_NOMEM, ERTS_ALC_O_REALLOC, n, size); } static ERTS_INLINE UWord alcu_size(ErtsAlcType_t ai) { UWord res = 0; ASSERT(erts_allctrs_info[ai].enabled); ASSERT(erts_allctrs_info[ai].alloc_util); if (!erts_allctrs_info[ai].thr_spec) { Allctr_t *allctr = erts_allctrs_info[ai].extra; AllctrSize_t asize; erts_alcu_current_size(allctr, &asize); res += asize.blocks; } else { ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ai]; int i; ASSERT(tspec->all_thr_safe); ASSERT(tspec->enabled); for (i = tspec->size - 1; i >= 0; i--) { Allctr_t *allctr = tspec->allctr[i]; AllctrSize_t asize; if (allctr) { erts_alcu_current_size(allctr, &asize); res += asize.blocks; } } } return res; } Eterm erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) { #define ERTS_MEM_NEED_ALL_ALCU (!erts_instr_stat && want_tot_or_sys) ErtsFixInfo efi; struct { int total; int processes; int processes_used; int system; int atom; int atom_used; int binary; int code; int ets; int maximum; } want = {0}; struct { UWord total; UWord processes; UWord processes_used; UWord system; UWord atom; UWord atom_used; UWord binary; UWord code; UWord ets; UWord maximum; } size = {0}; Eterm atoms[sizeof(size)/sizeof(UWord)]; UWord *uintps[sizeof(size)/sizeof(UWord)]; Eterm euints[sizeof(size)/sizeof(UWord)]; int want_tot_or_sys; int length; Eterm res = THE_NON_VALUE; ErtsAlcType_t ai; int only_one_value = 0; /* Figure out whats wanted... */ length = 0; if (is_non_value(earg)) { /* i.e. wants all */ want.total = 1; atoms[length] = am_total; uintps[length++] = &size.total; want.processes = 1; atoms[length] = am_processes; uintps[length++] = &size.processes; want.processes_used = 1; atoms[length] = am_processes_used; uintps[length++] = &size.processes_used; want.system = 1; atoms[length] = am_system; uintps[length++] = &size.system; want.atom = 1; atoms[length] = am_atom; uintps[length++] = &size.atom; want.atom_used = 1; atoms[length] = am_atom_used; uintps[length++] = &size.atom_used; want.binary = 1; atoms[length] = am_binary; uintps[length++] = &size.binary; want.code = 1; atoms[length] = am_code; uintps[length++] = &size.code; want.ets = 1; atoms[length] = am_ets; uintps[length++] = &size.ets; want.maximum = erts_instr_stat; if (want.maximum) { atoms[length] = am_maximum; uintps[length++] = &size.maximum; } } else { DeclareTmpHeapNoproc(tmp_heap,2); Eterm wanted_list; if (is_nil(earg)) return NIL; UseTmpHeapNoproc(2); if (is_not_atom(earg)) wanted_list = earg; else { wanted_list = CONS(&tmp_heap[0], earg, NIL); only_one_value = 1; } while (is_list(wanted_list)) { switch (CAR(list_val(wanted_list))) { case am_total: if (!want.total) { want.total = 1; atoms[length] = am_total; uintps[length++] = &size.total; } break; case am_processes: if (!want.processes) { want.processes = 1; atoms[length] = am_processes; uintps[length++] = &size.processes; } break; case am_processes_used: if (!want.processes_used) { want.processes_used = 1; atoms[length] = am_processes_used; uintps[length++] = &size.processes_used; } break; case am_system: if (!want.system) { want.system = 1; atoms[length] = am_system; uintps[length++] = &size.system; } break; case am_atom: if (!want.atom) { want.atom = 1; atoms[length] = am_atom; uintps[length++] = &size.atom; } break; case am_atom_used: if (!want.atom_used) { want.atom_used = 1; atoms[length] = am_atom_used; uintps[length++] = &size.atom_used; } break; case am_binary: if (!want.binary) { want.binary = 1; atoms[length] = am_binary; uintps[length++] = &size.binary; } break; case am_code: if (!want.code) { want.code = 1; atoms[length] = am_code; uintps[length++] = &size.code; } break; case am_ets: if (!want.ets) { want.ets = 1; atoms[length] = am_ets; uintps[length++] = &size.ets; } break; case am_maximum: if (erts_instr_stat) { if (!want.maximum) { want.maximum = 1; atoms[length] = am_maximum; uintps[length++] = &size.maximum; } } else { UnUseTmpHeapNoproc(2); return am_badarg; } break; default: UnUseTmpHeapNoproc(2); return am_badarg; } wanted_list = CDR(list_val(wanted_list)); } UnUseTmpHeapNoproc(2); if (is_not_nil(wanted_list)) return am_badarg; } /* All alloc_util allocators *have* to be enabled */ for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) { switch (ai) { case ERTS_ALC_A_SYSTEM: case ERTS_ALC_A_FIXED_SIZE: break; default: if (!erts_allctrs_info[ai].enabled || !erts_allctrs_info[ai].alloc_util) { return am_notsup; } break; } } ASSERT(length <= sizeof(atoms)/sizeof(Eterm)); ASSERT(length <= sizeof(euints)/sizeof(Eterm)); ASSERT(length <= sizeof(uintps)/sizeof(UWord)); if (proc) { ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(proc)); /* We'll need locks early in the lock order */ erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); } /* Calculate values needed... */ want_tot_or_sys = want.total || want.system; if (ERTS_MEM_NEED_ALL_ALCU) { size.total = 0; for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) { if (erts_allctrs_info[ai].alloc_util) { UWord *save; UWord asz; switch (ai) { case ERTS_ALC_A_TEMPORARY: /* * Often not thread safe and usually never * contain any allocated memory. */ continue; case ERTS_ALC_A_EHEAP: save = &size.processes; break; case ERTS_ALC_A_ETS: save = &size.ets; break; case ERTS_ALC_A_BINARY: save = &size.binary; break; default: save = NULL; break; } asz = alcu_size(ai); if (save) *save = asz; size.total += asz; } } } if (want_tot_or_sys || want.processes || want.processes_used) { UWord tmp; if (ERTS_MEM_NEED_ALL_ALCU) tmp = size.processes; else tmp = alcu_size(ERTS_ALC_A_EHEAP); tmp += erts_max_processes*sizeof(Process*); #ifdef HYBRID tmp += erts_max_processes*sizeof(Process*); #endif tmp += erts_bif_timer_memory_size(); tmp += erts_tot_link_lh_size(); size.processes = size.processes_used = tmp; #if HALFWORD_HEAP /* BUG: We ignore link and monitor memory */ #else erts_fix_info(ERTS_ALC_T_NLINK_SH, &efi); size.processes += efi.total; size.processes_used += efi.used; erts_fix_info(ERTS_ALC_T_MONITOR_SH, &efi); size.processes += efi.total; size.processes_used += efi.used; #endif erts_fix_info(ERTS_ALC_T_PROC, &efi); size.processes += efi.total; size.processes_used += efi.used; erts_fix_info(ERTS_ALC_T_REG_PROC, &efi); size.processes += efi.total; size.processes_used += efi.used; } if (want.atom || want.atom_used) { Uint reserved_atom_space, atom_space; erts_atom_get_text_space_sizes(&reserved_atom_space, &atom_space); size.atom = size.atom_used = atom_table_sz(); erts_fix_info(ERTS_ALC_T_ATOM, &efi); if (want.atom) { size.atom += reserved_atom_space; size.atom += efi.total; } if (want.atom_used) { size.atom_used += atom_space; size.atom_used += efi.used; } } if (!ERTS_MEM_NEED_ALL_ALCU && want.binary) size.binary = alcu_size(ERTS_ALC_A_BINARY); if (want.code) { size.code = module_table_sz(); erts_fix_info(ERTS_ALC_T_MODULE, &efi); size.code += efi.used; size.code += export_table_sz(); #if HALFWORD_HEAP size.code += export_list_size() * sizeof(Export); #else erts_fix_info(ERTS_ALC_T_EXPORT, &efi); size.code += efi.used; #endif size.code += erts_fun_table_sz(); erts_fix_info(ERTS_ALC_T_FUN_ENTRY, &efi); size.code += efi.used; size.code += allocated_modules*sizeof(Range); size.code += erts_total_code_size; } if (want.ets) { if (!ERTS_MEM_NEED_ALL_ALCU) size.ets = alcu_size(ERTS_ALC_A_ETS); size.ets += erts_get_ets_misc_mem_size(); } if (erts_instr_stat && (want_tot_or_sys || want.maximum)) { if (want_tot_or_sys) { size.total = erts_instr_get_total(); size.system = size.total - size.processes; } size.maximum = erts_instr_get_max_total(); } else if (want_tot_or_sys) { size.system = size.total - size.processes; } if (print_to_p) { int i; int to = *print_to_p; void *arg = print_to_arg; /* Print result... */ erts_print(to, arg, "=memory\n"); for (i = 0; i < length; i++) erts_print(to, arg, "%T: %bpu\n", atoms[i], *uintps[i]); } if (proc) { /* Build erlang term result... */ Uint *hp; Uint hsz; erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); if (only_one_value) { ASSERT(length == 1); hsz = 0; erts_bld_uword(NULL, &hsz, *uintps[0]); hp = hsz ? HAlloc((Process *) proc, hsz) : NULL; res = erts_bld_uword(&hp, NULL, *uintps[0]); } else { Uint **hpp = NULL; Uint *hszp = &hsz; hsz = 0; while (1) { int i; for (i = 0; i < length; i++) euints[i] = erts_bld_uword(hpp, hszp, *uintps[i]); res = erts_bld_2tup_list(hpp, hszp, length, atoms, euints); if (hpp) break; hp = HAlloc((Process *) proc, hsz); hpp = &hp; hszp = NULL; } } } return res; #undef ERTS_MEM_NEED_ALL_ALCU } struct aa_values { Uint arity; const char *name; Uint ui[2]; }; Eterm erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) { #define MAX_AA_VALUES \ (20 + (ERTS_ALC_N_MAX_A_FIXED_SIZE - ERTS_ALC_N_MIN_A_FIXED_SIZE + 1)) struct aa_values values[MAX_AA_VALUES]; Eterm res = THE_NON_VALUE; int i, length; ErtsFixInfo efi; Uint reserved_atom_space, atom_space; if (proc) { ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(proc)); /* We'll need locks early in the lock order */ erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); } i = 0; if (erts_instr_stat) { values[i].arity = 2; values[i].name = "total"; values[i].ui[0] = erts_instr_get_total(); i++; values[i].arity = 2; values[i].name = "maximum"; values[i].ui[0] = erts_instr_get_max_total(); i++; } values[i].arity = 2; values[i].name = "sys_misc"; values[i].ui[0] = erts_sys_misc_mem_sz(); i++; values[i].arity = 2; values[i].name = "static"; values[i].ui[0] = erts_max_ports*sizeof(Port) /* Port table */ + erts_timer_wheel_memory_size() /* Timer wheel */ #ifdef SYS_TMP_BUF_SIZE + SYS_TMP_BUF_SIZE /* tmp_buf in sys on vxworks & ose */ #endif ; i++; erts_atom_get_text_space_sizes(&reserved_atom_space, &atom_space); values[i].arity = 3; values[i].name = "atom_space"; values[i].ui[0] = reserved_atom_space; values[i].ui[1] = atom_space; i++; values[i].arity = 2; values[i].name = "atom_table"; values[i].ui[0] = atom_table_sz(); i++; values[i].arity = 2; values[i].name = "module_table"; values[i].ui[0] = module_table_sz(); i++; values[i].arity = 2; values[i].name = "export_table"; values[i].ui[0] = export_table_sz(); i++; values[i].arity = 2; values[i].name = "register_table"; values[i].ui[0] = process_reg_sz(); i++; values[i].arity = 2; values[i].name = "fun_table"; values[i].ui[0] = erts_fun_table_sz(); i++; values[i].arity = 2; values[i].name = "module_refs"; values[i].ui[0] = allocated_modules*sizeof(Range); i++; values[i].arity = 2; values[i].name = "loaded_code"; values[i].ui[0] = erts_total_code_size; i++; values[i].arity = 2; values[i].name = "dist_table"; values[i].ui[0] = erts_dist_table_size(); i++; values[i].arity = 2; values[i].name = "node_table"; values[i].ui[0] = erts_node_table_size(); i++; values[i].arity = 2; values[i].name = "bits_bufs_size"; values[i].ui[0] = erts_bits_bufs_size(); i++; values[i].arity = 2; values[i].name = "bif_timer"; values[i].ui[0] = erts_bif_timer_memory_size(); i++; values[i].arity = 2; values[i].name = "link_lh"; values[i].ui[0] = erts_tot_link_lh_size(); i++; { Uint n; for (n = ERTS_ALC_N_MIN_A_FIXED_SIZE; n <= ERTS_ALC_N_MAX_A_FIXED_SIZE; n++) { erts_fix_info(ERTS_ALC_N2T(n), &efi); values[i].arity = 3; values[i].name = ERTS_ALC_N2TD(n); values[i].ui[0] = efi.total; values[i].ui[1] = efi.used; i++; } } length = i; ASSERT(length <= MAX_AA_VALUES); if (print_to_p) { /* Print result... */ int to = *print_to_p; void *arg = print_to_arg; erts_print(to, arg, "=allocated_areas\n"); for (i = 0; i < length; i++) { switch (values[i].arity) { case 2: erts_print(to, arg, "%s: %beu\n", values[i].name, values[i].ui[0]); break; case 3: erts_print(to, arg, "%s: %beu %beu\n", values[i].name, values[i].ui[0], values[i].ui[1]); break; default: erts_print(to, arg, "ERROR: internal_error\n"); ASSERT(0); return am_internal_error; } } } if (proc) { /* Build erlang term result... */ Eterm tuples[MAX_AA_VALUES]; Uint *hp; Uint **hpp; Uint hsz; Uint *hszp; erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); hpp = NULL; hsz = 0; hszp = &hsz; while (1) { int i; for (i = 0; i < length; i++) { Eterm atom; if (hpp) atom = am_atom_put(values[i].name, (int) strlen(values[i].name)); else atom = am_true; switch (values[i].arity) { case 2: tuples[i] = erts_bld_tuple(hpp, hszp, 2, atom, erts_bld_uint(hpp, hszp, values[i].ui[0])); break; case 3: tuples[i] = erts_bld_tuple(hpp, hszp, 3, atom, erts_bld_uint(hpp, hszp, values[i].ui[0]), erts_bld_uint(hpp, hszp, values[i].ui[1])); break; default: ASSERT(0); return am_internal_error; } } res = erts_bld_list(hpp, hszp, length, tuples); if (hpp) break; hp = HAlloc((Process *) proc, hsz); hpp = &hp; hszp = NULL; } } return res; #undef MAX_AA_VALUES } Eterm erts_alloc_util_allocators(void *proc) { Eterm res; Uint *hp; Uint sz; int i; /* * Currently all allocators except sys_alloc and fix_alloc are * alloc_util allocators. */ sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 2)*2; ASSERT(sz > 0); hp = HAlloc((Process *) proc, sz); res = NIL; for (i = ERTS_ALC_A_MAX; i >= ERTS_ALC_A_MIN; i--) { switch (i) { case ERTS_ALC_A_SYSTEM: case ERTS_ALC_A_FIXED_SIZE: break; default: { char *alc_str = (char *) ERTS_ALC_A2AD(i); Eterm alc = am_atom_put(alc_str, sys_strlen(alc_str)); res = CONS(hp, alc, res); hp += 2; break; } } } return res; } Eterm erts_allocator_info_term(void *proc, Eterm which_alloc, int only_sz) { #define ERTS_AIT_RET(R) \ do { res = (R); goto done; } while (0) #define ERTS_AIT_HALLOC(P, S) \ do { hp = HAlloc((P), (S)); hp_end = hp + (S); } while (0) ErtsAlcType_t i; Uint sz = 0; Uint *hp = NULL; Uint *hp_end = NULL; Eterm res = am_undefined; if (is_not_atom(which_alloc)) goto done; for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { if (erts_is_atom_str((char *) ERTS_ALC_A2AD(i), which_alloc)) { if (!erts_allctrs_info[i].enabled) ERTS_AIT_RET(am_false); else { if (erts_allctrs_info[i].alloc_util) { Eterm ires, tmp; Eterm **hpp; Uint *szp; Eterm (*info_func)(Allctr_t *, int, int *, void *, Uint **, Uint *); info_func = (only_sz ? erts_alcu_sz_info : erts_alcu_info); if (erts_allctrs_info[i].thr_spec) { ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[i]; int j; int block_system = !tspec->all_thr_safe; if (block_system) { erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); erts_smp_block_system(0); } ASSERT(tspec->enabled); szp = &sz; hpp = NULL; while (1) { ires = NIL; for (j = tspec->size - 1; j >= 0; j--) { Allctr_t *allctr = tspec->allctr[j]; if (allctr) { tmp = erts_bld_tuple(hpp, szp, 3, erts_bld_atom(hpp, szp, "instance"), make_small((Uint) j), (*info_func)(allctr, hpp != NULL, NULL, NULL, hpp, szp)); ires = erts_bld_cons(hpp, szp, tmp, ires); } } if (hpp) break; ERTS_AIT_HALLOC((Process *) proc, sz); hpp = &hp; szp = NULL; } if (block_system) { erts_smp_release_system(); erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); } } else { Allctr_t *allctr = erts_allctrs_info[i].extra; szp = &sz; hpp = NULL; while (1) { ires = NIL; tmp = erts_bld_tuple(hpp, szp, 3, erts_bld_atom(hpp, szp, "instance"), make_small((Uint) 0), (*info_func)(allctr, hpp != NULL, NULL, NULL, hpp, szp)); ires = erts_bld_cons(hpp, szp, tmp, ires); if (hpp) break; ERTS_AIT_HALLOC((Process *) proc, sz); hpp = &hp; szp = NULL; } } ERTS_AIT_RET(ires); } else { Eterm *szp, **hpp; switch (i) { case ERTS_ALC_A_SYSTEM: { SysAllocStat sas; Eterm opts_am; Eterm opts; Eterm as[4]; /* Ok even if !HEAP_ON_C_STACK, not really heap data on stack */ Eterm ts[4]; /* Ok even if !HEAP_ON_C_STACK, not really heap data on stack */ int l; if (only_sz) ERTS_AIT_RET(NIL); sys_alloc_stat(&sas); opts_am = am_atom_put("options", 7); szp = &sz; hpp = NULL; restart_sys_alloc: l = 0; as[l] = am_atom_put("e", 1); ts[l++] = am_true; as[l] = am_atom_put("m", 1); ts[l++] = am_atom_put("libc", 4); if(sas.trim_threshold >= 0) { as[l] = am_atom_put("tt", 2); ts[l++] = erts_bld_uint(hpp, szp, (Uint) sas.trim_threshold); } if(sas.top_pad >= 0) { as[l] = am_atom_put("tp", 2); ts[l++] = erts_bld_uint(hpp, szp, (Uint) sas.top_pad); } opts = erts_bld_2tup_list(hpp, szp, l, as, ts); res = erts_bld_2tup_list(hpp, szp, 1, &opts_am, &opts); if (szp) { ERTS_AIT_HALLOC((Process *) proc, sz); szp = NULL; hpp = &hp; goto restart_sys_alloc; } ERTS_AIT_RET(res); } case ERTS_ALC_A_FIXED_SIZE: { ErtsAlcType_t n; Eterm as[2], vs[2]; if (only_sz) ERTS_AIT_RET(NIL); as[0] = am_atom_put("options", 7); as[1] = am_atom_put("pools", 5); szp = &sz; hpp = NULL; restart_fix_alloc: vs[0] = erts_bld_cons(hpp, szp, erts_bld_tuple(hpp, szp, 2, am_atom_put("e", 1), am_true), NIL); vs[1] = NIL; for (n = ERTS_ALC_N_MIN_A_FIXED_SIZE; n <= ERTS_ALC_N_MAX_A_FIXED_SIZE; n++) { ErtsFixInfo efi; erts_fix_info(ERTS_ALC_N2T(n), &efi); vs[1] = erts_bld_cons( hpp, szp, erts_bld_tuple( hpp, szp, 3, am_atom_put((char *) ERTS_ALC_N2TD(n), strlen(ERTS_ALC_N2TD(n))), erts_bld_uint(hpp, szp, efi.total), erts_bld_uint(hpp, szp, efi.used)), vs[1]); } res = erts_bld_2tup_list(hpp, szp, 2, as, vs); if (szp) { ERTS_AIT_HALLOC((Process *) proc, sz); szp = NULL; hpp = &hp; goto restart_fix_alloc; } ERTS_AIT_RET(res); } default: ASSERT(0); goto done; } } } } } if (ERTS_IS_ATOM_STR("mseg_alloc", which_alloc)) { #if HAVE_ERTS_MSEG if (only_sz) ERTS_AIT_RET(NIL); erts_mseg_info(NULL, NULL, 0, NULL, &sz); if (sz) ERTS_AIT_HALLOC((Process *) proc, sz); ERTS_AIT_RET(erts_mseg_info(NULL, NULL, 1, &hp, NULL)); #else ERTS_AIT_RET(am_false); #endif } else if (ERTS_IS_ATOM_STR("alloc_util", which_alloc)) { if (only_sz) ERTS_AIT_RET(NIL); erts_alcu_au_info_options(NULL, NULL, NULL, &sz); if (sz) ERTS_AIT_HALLOC((Process *) proc, sz); ERTS_AIT_RET(erts_alcu_au_info_options(NULL, NULL, &hp, NULL)); } done: if (hp) { ASSERT(hp_end >= hp); HRelease((Process *) proc, hp_end, hp); } return res; #undef ERTS_AIT_RET #undef ERTS_AIT_HALLOC } void erts_allocator_info(int to, void *arg) { ErtsAlcType_t a; ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0) || (ERTS_IS_CRASH_DUMPING && erts_smp_is_system_blocked(ERTS_BS_FLG_ALLOW_GC))); for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) { int ai; for (ai = 0; ai == 0 || ai < erts_allctrs_info[a].thr_spec; ai++) { if (erts_allctrs_info[a].thr_spec) { if (!erts_allctr_thr_spec[a].allctr[ai]) continue; erts_print(to, arg, "=allocator:%s[%d]\n", ERTS_ALC_A2AD(a), ai); } else { erts_print(to, arg, "=allocator:%s\n", ERTS_ALC_A2AD(a)); } if (!erts_allctrs_info[a].enabled) erts_print(to, arg, "option e: false\n"); else { if (erts_allctrs_info[a].alloc_util) { void *as; if (!erts_allctrs_info[a].thr_spec) as = erts_allctrs_info[a].extra; else { ASSERT(erts_allctr_thr_spec[a].enabled); as = erts_allctr_thr_spec[a].allctr[ai]; } /* Binary alloc has its own thread safety... */ erts_alcu_info(as, 0, &to, arg, NULL, NULL); } else { switch (a) { case ERTS_ALC_A_SYSTEM: { SysAllocStat sas; erts_print(to, arg, "option e: true\n"); erts_print(to, arg, "option m: libc\n"); sys_alloc_stat(&sas); if(sas.trim_threshold >= 0) erts_print(to, arg, "option tt: %d\n", sas.trim_threshold); if(sas.top_pad >= 0) erts_print(to, arg, "option tp: %d\n", sas.top_pad); break; } case ERTS_ALC_A_FIXED_SIZE: { ErtsAlcType_t n; erts_print(to, arg, "option e: true\n"); for (n = ERTS_ALC_N_MIN_A_FIXED_SIZE; n <= ERTS_ALC_N_MAX_A_FIXED_SIZE; n++) { ErtsFixInfo efi; erts_fix_info(ERTS_ALC_N2T(n), &efi); erts_print(to, arg, "%s: %lu %lu\n", ERTS_ALC_N2TD(n), efi.total, efi.used); } break; } default: ASSERT(0); break; } } } } } #if HAVE_ERTS_MSEG erts_print(to, arg, "=allocator:mseg_alloc\n"); erts_mseg_info(&to, arg, 0, NULL, NULL); #endif erts_print(to, arg, "=allocator:alloc_util\n"); erts_alcu_au_info_options(&to, arg, NULL, NULL); erts_print(to, arg, "=allocator:instr\n"); erts_print(to, arg, "option m: %s\n", erts_instr_memory_map ? "true" : "false"); erts_print(to, arg, "option s: %s\n", erts_instr_stat ? "true" : "false"); erts_print(to, arg, "option t: %s\n", erts_mtrace_enabled ? "true" : "false"); } Eterm erts_allocator_options(void *proc) { #if HAVE_ERTS_MSEG int use_mseg = 0; #endif Uint sz, *szp, *hp, **hpp; Eterm res, features, settings; Eterm atoms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+5]; Uint terms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+5]; int a, length; SysAllocStat sas; Uint *endp = NULL; sys_alloc_stat(&sas); /* First find out the heap size needed ... */ hpp = NULL; szp = &sz; sz = 0; bld_term: length = 0; features = NIL; settings = NIL; for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) { Eterm tmp = NIL; atoms[length] = am_atom_put((char *) ERTS_ALC_A2AD(a), strlen(ERTS_ALC_A2AD(a))); if (erts_allctrs_info[a].enabled) { if (erts_allctrs_info[a].alloc_util) { Allctr_t *allctr; #if HAVE_ERTS_MSEG use_mseg++; #endif if (erts_allctr_thr_spec[a].enabled) allctr = erts_allctr_thr_spec[a].allctr[1]; else allctr = erts_allctrs_info[a].extra; tmp = erts_alcu_info_options(allctr, NULL, NULL, hpp, szp); } else { int l = 0; Eterm as[4]; Eterm ts[4]; as[l] = am_atom_put("e", 1); ts[l++] = am_true; switch (a) { case ERTS_ALC_A_SYSTEM: as[l] = am_atom_put("m", 1); ts[l++] = am_atom_put("libc", 4); if(sas.trim_threshold >= 0) { as[l] = am_atom_put("tt", 2); ts[l++] = erts_bld_uint(hpp, szp, (Uint) sas.trim_threshold); } if(sas.top_pad >= 0) { as[l] = am_atom_put("tp", 2); ts[l++] = erts_bld_uint(hpp, szp, (Uint) sas.top_pad); } break; default: break; } tmp = erts_bld_2tup_list(hpp, szp, l, as, ts); } } else { Eterm atom = am_atom_put("e", 1); Eterm term = am_false; tmp = erts_bld_2tup_list(hpp, szp, 1, &atom, &term); } terms[length++] = tmp; } #if HAVE_ERTS_MSEG if (use_mseg) { atoms[length] = am_atom_put("mseg_alloc", 10); terms[length++] = erts_mseg_info_options(NULL, NULL, hpp, szp); } #endif atoms[length] = am_atom_put("alloc_util", 10); terms[length++] = erts_alcu_au_info_options(NULL, NULL, hpp, szp); { Eterm o[3], v[3]; o[0] = am_atom_put("m", 1); v[0] = erts_instr_memory_map ? am_true : am_false; o[1] = am_atom_put("s", 1); v[1] = erts_instr_stat ? am_true : am_false; o[2] = am_atom_put("t", 1); v[2] = erts_mtrace_enabled ? am_true : am_false; atoms[length] = am_atom_put("instr", 5); terms[length++] = erts_bld_2tup_list(hpp, szp, 3, o, v); } settings = erts_bld_2tup_list(hpp, szp, length, atoms, terms); length = 0; for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) { if (erts_allctrs_info[a].enabled) { terms[length++] = am_atom_put((char *) ERTS_ALC_A2AD(a), strlen(ERTS_ALC_A2AD(a))); } } #if HAVE_ERTS_MSEG if (use_mseg) terms[length++] = am_atom_put("mseg_alloc", 10); #endif features = length ? erts_bld_list(hpp, szp, length, terms) : NIL; #if defined(__GLIBC__) { Eterm AM_glibc = am_atom_put("glibc", 5); Eterm version; version = erts_bld_cons(hpp, szp, make_small(__GLIBC__), #ifdef __GLIBC_MINOR__ erts_bld_cons(hpp, szp, make_small(__GLIBC_MINOR__), NIL) #else NIL #endif ); res = erts_bld_tuple(hpp, szp, 4, AM_glibc, version, features, settings); } #else /* unknown allocator */ res = erts_bld_tuple(hpp, szp, 4, am_undefined, NIL, features, settings); #endif if (szp) { /* ... and then build the term */ hp = HAlloc((Process *) proc, sz); endp = hp + sz; hpp = &hp; szp = NULL; goto bld_term; } ASSERT(endp >= hp); HRelease((Process *) proc, endp, hp); return res; } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * Deprecated functions * * * * These functions are still defined since "non-OTP linked in drivers" may * * contain (illegal) calls to them. * \* */ /* --- DO *NOT* USE THESE FUNCTIONS --- */ void *sys_alloc(Uint sz) { return erts_alloc_fnf(ERTS_ALC_T_UNDEF, sz); } void *sys_realloc(void *ptr, Uint sz) { return erts_realloc_fnf(ERTS_ALC_T_UNDEF, ptr, sz); } void sys_free(void *ptr) { erts_free(ERTS_ALC_T_UNDEF, ptr); } void *safe_alloc(Uint sz) { return erts_alloc(ERTS_ALC_T_UNDEF, sz); } void *safe_realloc(void *ptr, Uint sz) { return erts_realloc(ERTS_ALC_T_UNDEF, ptr, sz); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * NOTE: erts_alc_test() is only supposed to be used for testing. * * * * Keep alloc_SUITE_data/allocator_test.h updated if changes are made * * to erts_alc_test() * \* */ #define ERTS_ALC_TEST_ABORT erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n") unsigned long erts_alc_test(unsigned long op, unsigned long a1, unsigned long a2, unsigned long a3) { switch (op >> 8) { case 0x0: return erts_alcu_test(op, a1, a2); case 0x1: return erts_gfalc_test(op, a1, a2); case 0x2: return erts_bfalc_test(op, a1, a2); case 0x3: return erts_afalc_test(op, a1, a2); case 0x4: return erts_mseg_test(op, a1, a2, a3); case 0xf: switch (op) { case 0xf00: #ifdef USE_THREADS if (((Allctr_t *) a1)->thread_safe) return (unsigned long) erts_alcu_alloc_ts(ERTS_ALC_T_UNDEF, (void *) a1, (Uint) a2); else #endif return (unsigned long) erts_alcu_alloc(ERTS_ALC_T_UNDEF, (void *) a1, (Uint) a2); case 0xf01: #ifdef USE_THREADS if (((Allctr_t *) a1)->thread_safe) return (unsigned long) erts_alcu_realloc_ts(ERTS_ALC_T_UNDEF, (void *) a1, (void *) a2, (Uint) a3); else #endif return (unsigned long) erts_alcu_realloc(ERTS_ALC_T_UNDEF, (void *) a1, (void *) a2, (Uint) a3); case 0xf02: #ifdef USE_THREADS if (((Allctr_t *) a1)->thread_safe) erts_alcu_free_ts(ERTS_ALC_T_UNDEF, (void *) a1, (void *) a2); else #endif erts_alcu_free(ERTS_ALC_T_UNDEF, (void *) a1, (void *) a2); return 0; case 0xf03: { Allctr_t *allctr; struct au_init init; SET_DEFAULT_ALLOC_OPTS(&init); init.enable = 1; init.atype = GOODFIT; init.init.util.name_prefix = (char *) a1; init.init.util.ts = a2 ? 1 : 0; if ((char **) a3) { char **argv = (char **) a3; int i = 0; while (argv[i]) { if (argv[i][0] == '-' && argv[i][1] == 't') handle_au_arg(&init, &argv[i][2], argv, &i); else return (unsigned long) NULL; i++; } } switch (init.atype) { case GOODFIT: allctr = erts_gfalc_start((GFAllctr_t *) erts_alloc(ERTS_ALC_T_UNDEF, sizeof(GFAllctr_t)), &init.init.gf, &init.init.util); break; case BESTFIT: allctr = erts_bfalc_start((BFAllctr_t *) erts_alloc(ERTS_ALC_T_UNDEF, sizeof(BFAllctr_t)), &init.init.bf, &init.init.util); break; case AFIT: allctr = erts_afalc_start((AFAllctr_t *) erts_alloc(ERTS_ALC_T_UNDEF, sizeof(AFAllctr_t)), &init.init.af, &init.init.util); break; default: ASSERT(0); allctr = NULL; break; } return (unsigned long) allctr; } case 0xf04: erts_alcu_stop((Allctr_t *) a1); erts_free(ERTS_ALC_T_UNDEF, (void *) a1); break; #ifdef USE_THREADS case 0xf05: return (unsigned long) 1; case 0xf06: return (unsigned long) ((Allctr_t *) a1)->thread_safe; #ifdef ETHR_NO_FORKSAFETY case 0xf07: return (unsigned long) 0; #else case 0xf07: return (unsigned long) ((Allctr_t *) a1)->thread_safe; #endif case 0xf08: { ethr_mutex *mtx = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_mutex)); if (ethr_mutex_init(mtx) != 0) ERTS_ALC_TEST_ABORT; return (unsigned long) mtx; } case 0xf09: { ethr_mutex *mtx = (ethr_mutex *) a1; if (ethr_mutex_destroy(mtx) != 0) ERTS_ALC_TEST_ABORT; erts_free(ERTS_ALC_T_UNDEF, (void *) mtx); break; } case 0xf0a: ethr_mutex_lock((ethr_mutex *) a1); break; case 0xf0b: ethr_mutex_unlock((ethr_mutex *) a1); break; case 0xf0c: { ethr_cond *cnd = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_cond)); if (ethr_cond_init(cnd) != 0) ERTS_ALC_TEST_ABORT; return (unsigned long) cnd; } case 0xf0d: { ethr_cond *cnd = (ethr_cond *) a1; if (ethr_cond_destroy(cnd) != 0) ERTS_ALC_TEST_ABORT; erts_free(ERTS_ALC_T_UNDEF, (void *) cnd); break; } case 0xf0e: ethr_cond_broadcast((ethr_cond *) a1); break; case 0xf0f: { int res; do { res = ethr_cond_wait((ethr_cond *) a1, (ethr_mutex *) a2); } while (res == EINTR); break; } case 0xf10: { ethr_tid *tid = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_tid)); if (ethr_thr_create(tid, (void * (*)(void *)) a1, (void *) a2, NULL) != 0) ERTS_ALC_TEST_ABORT; return (unsigned long) tid; } case 0xf11: { ethr_tid *tid = (ethr_tid *) a1; if (ethr_thr_join(*tid, NULL) != 0) ERTS_ALC_TEST_ABORT; erts_free(ERTS_ALC_T_UNDEF, (void *) tid); break; } case 0xf12: ethr_thr_exit((void *) a1); ERTS_ALC_TEST_ABORT; break; #endif /* #ifdef USE_THREADS */ default: break; } return (unsigned long) 0; default: break; } ASSERT(0); return ~((unsigned long) 0); } #ifdef DEBUG /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * Debug stuff * \* */ #if 0 #define PRINT_OPS #else #undef PRINT_OPS #endif #ifdef HARD_DEBUG #define FENCE_SZ (4*sizeof(UWord)) #else #define FENCE_SZ (3*sizeof(UWord)) #endif #if defined(ARCH_64) #define FENCE_PATTERN 0xABCDEF97ABCDEF97 #else #define FENCE_PATTERN 0xABCDEF97 #endif #define TYPE_PATTERN_MASK ERTS_ALC_N_MASK #define TYPE_PATTERN_SHIFT 16 #define FIXED_FENCE_PATTERN_MASK \ (~((UWord) (TYPE_PATTERN_MASK << TYPE_PATTERN_SHIFT))) #define FIXED_FENCE_PATTERN \ (FENCE_PATTERN & FIXED_FENCE_PATTERN_MASK) #define MK_PATTERN(T) \ (FIXED_FENCE_PATTERN | (((T) & TYPE_PATTERN_MASK) << TYPE_PATTERN_SHIFT)) #define GET_TYPE_OF_PATTERN(P) \ (((P) >> TYPE_PATTERN_SHIFT) & TYPE_PATTERN_MASK) #ifdef HARD_DEBUG #define ERL_ALC_HDBG_MAX_MBLK 100000 #define ERTS_ALC_O_CHECK -1 typedef struct hdbg_mblk_ hdbg_mblk; struct hdbg_mblk_ { hdbg_mblk *next; hdbg_mblk *prev; void *p; Uint s; ErtsAlcType_t n; }; static hdbg_mblk hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK]; static hdbg_mblk *free_hdbg_mblks; static hdbg_mblk *used_hdbg_mblks; static erts_mtx_t hdbg_mblk_mtx; static void hdbg_init(void) { int i; for (i = 0; i < ERL_ALC_HDBG_MAX_MBLK-1; i++) hdbg_mblks[i].next = &hdbg_mblks[i+1]; hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK-1].next = NULL; free_hdbg_mblks = &hdbg_mblks[0]; used_hdbg_mblks = NULL; erts_mtx_init(&hdbg_mblk_mtx, "erts_alloc_hard_debug"); } static void *check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func); void erts_hdbg_chk_blks(void); void erts_hdbg_chk_blks(void) { hdbg_mblk *mblk; erts_mtx_lock(&hdbg_mblk_mtx); for (mblk = used_hdbg_mblks; mblk; mblk = mblk->next) { Uint sz; check_memory_fence(mblk->p, &sz, mblk->n, ERTS_ALC_O_CHECK); ASSERT(sz == mblk->s); } erts_mtx_unlock(&hdbg_mblk_mtx); } static hdbg_mblk * hdbg_alloc(void *p, Uint s, ErtsAlcType_t n) { hdbg_mblk *mblk; erts_mtx_lock(&hdbg_mblk_mtx); mblk = free_hdbg_mblks; if (!mblk) { erts_fprintf(stderr, "Ran out of debug blocks; please increase " "ERL_ALC_HDBG_MAX_MBLK=%d and recompile!\n", ERL_ALC_HDBG_MAX_MBLK); abort(); } free_hdbg_mblks = mblk->next; mblk->p = p; mblk->s = s; mblk->n = n; mblk->next = used_hdbg_mblks; mblk->prev = NULL; if (used_hdbg_mblks) used_hdbg_mblks->prev = mblk; used_hdbg_mblks = mblk; erts_mtx_unlock(&hdbg_mblk_mtx); return (void *) mblk; } static void hdbg_free(hdbg_mblk *mblk) { erts_mtx_lock(&hdbg_mblk_mtx); if (mblk->next) mblk->next->prev = mblk->prev; if (mblk->prev) mblk->prev->next = mblk->next; else used_hdbg_mblks = mblk->next; mblk->next = free_hdbg_mblks; free_hdbg_mblks = mblk; erts_mtx_unlock(&hdbg_mblk_mtx); } #endif #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG static void *check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func); void check_allocated_block( Uint type, void *blk) { Uint dummy; check_memory_fence(blk, &dummy, ERTS_ALC_T2N(type), ERTS_ALC_O_FREE); } void check_allocators(void) { int i; if (!erts_initialized) return; for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; ++i) { if (erts_allctrs_info[i].alloc_util) { ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) erts_allctrs[i].extra; Allctr_t *allctr = real_af->extra; Carrier_t *ct; #ifdef USE_THREADS if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); #endif if (allctr->check_mbc) { for (ct = allctr->mbc_list.first; ct; ct = ct->next) { fprintf(stderr,"Checking allocator %d\r\n",i); allctr->check_mbc(allctr,ct); } } #ifdef USE_THREADS if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); #endif } } } #endif static void * set_memory_fence(void *ptr, Uint sz, ErtsAlcType_t n) { UWord *ui_ptr; UWord pattern; #ifdef HARD_DEBUG hdbg_mblk **mblkpp; #endif if (!ptr) return NULL; ui_ptr = (UWord *) ptr; pattern = MK_PATTERN(n); #ifdef HARD_DEBUG mblkpp = (hdbg_mblk **) ui_ptr++; #endif *(ui_ptr++) = sz; *(ui_ptr++) = pattern; memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(UWord)); #ifdef HARD_DEBUG *mblkpp = hdbg_alloc((void *) ui_ptr, sz, n); #endif return (void *) ui_ptr; } static void * check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) { Uint sz; Uint found_type; UWord pre_pattern; UWord post_pattern; UWord *ui_ptr; #ifdef HARD_DEBUG hdbg_mblk *mblk; #endif if (!ptr) return NULL; ui_ptr = (UWord *) ptr; pre_pattern = *(--ui_ptr); *size = sz = *(--ui_ptr); #ifdef HARD_DEBUG mblk = (hdbg_mblk *) *(--ui_ptr); #endif found_type = GET_TYPE_OF_PATTERN(pre_pattern); if (pre_pattern != MK_PATTERN(n)) { if ((FIXED_FENCE_PATTERN_MASK & pre_pattern) != FIXED_FENCE_PATTERN) erl_exit(ERTS_ABORT_EXIT, "ERROR: Fence at beginning of memory block (p=0x%u) " "clobbered.\n", (unsigned long) ptr); } memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(UWord)); if (post_pattern != MK_PATTERN(n) || pre_pattern != post_pattern) { char fbuf[10]; char obuf[10]; char *ftype; char *otype; char *op_str; if ((FIXED_FENCE_PATTERN_MASK & post_pattern) != FIXED_FENCE_PATTERN) erl_exit(ERTS_ABORT_EXIT, "ERROR: Fence at end of memory block (p=0x%u, sz=%u) " "clobbered.\n", (unsigned long) ptr, (unsigned long) sz); if (found_type != GET_TYPE_OF_PATTERN(post_pattern)) erl_exit(ERTS_ABORT_EXIT, "ERROR: Fence around memory block (p=0x%u, sz=%u) " "clobbered.\n", (unsigned long) ptr, (unsigned long) sz); ftype = type_no_str(found_type); if (!ftype) { sprintf(fbuf, "%d", (int) found_type); ftype = fbuf; } otype = type_no_str(n); if (!otype) { sprintf(obuf, "%d", (int) n); otype = obuf; } switch (func) { case ERTS_ALC_O_ALLOC: op_str = "allocated"; break; case ERTS_ALC_O_REALLOC: op_str = "reallocated"; break; case ERTS_ALC_O_FREE: op_str = "freed"; break; default: op_str = "???"; break; } erl_exit(ERTS_ABORT_EXIT, "ERROR: Memory block (p=0x%u, sz=%u) allocated as type \"%s\"," " but %s as type \"%s\".\n", (unsigned long) ptr, (unsigned long) sz, ftype, op_str, otype); } #ifdef HARD_DEBUG switch (func) { case ERTS_ALC_O_REALLOC: case ERTS_ALC_O_FREE: hdbg_free(mblk); break; default: break; } #endif return (void *) ui_ptr; } static ErtsAllocatorFunctions_t real_allctrs[ERTS_ALC_A_MAX+1]; static void * debug_alloc(ErtsAlcType_t n, void *extra, Uint size) { ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; Uint dsize; void *res; #ifdef HARD_DEBUG erts_hdbg_chk_blks(); #endif ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); dsize = size + FENCE_SZ; res = (*real_af->alloc)(n, real_af->extra, dsize); res = set_memory_fence(res, size, n); #ifdef PRINT_OPS fprintf(stderr, "0x%lx = alloc(%s, %lu)\r\n", (Uint) res, ERTS_ALC_N2TD(n), size); #endif return res; } static void * debug_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) { ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; Uint dsize; Uint old_size; void *dptr; void *res; ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); dsize = size + FENCE_SZ; dptr = check_memory_fence(ptr, &old_size, n, ERTS_ALC_O_REALLOC); #ifdef HARD_DEBUG erts_hdbg_chk_blks(); #endif if (old_size > size) sys_memset((void *) (((char *) ptr) + size), 0xf, sizeof(Uint) + old_size - size); res = (*real_af->realloc)(n, real_af->extra, dptr, dsize); res = set_memory_fence(res, size, n); #ifdef PRINT_OPS fprintf(stderr, "0x%lx = realloc(%s, 0x%lx, %lu)\r\n", (Uint) res, ERTS_ALC_N2TD(n), (Uint) ptr, size); #endif return res; } static void debug_free(ErtsAlcType_t n, void *extra, void *ptr) { ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; void *dptr; Uint size; ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); dptr = check_memory_fence(ptr, &size, n, ERTS_ALC_O_FREE); sys_memset((void *) dptr, n, size + FENCE_SZ); (*real_af->free)(n, real_af->extra, dptr); #ifdef PRINT_OPS fprintf(stderr, "free(%s, 0x%lx)\r\n", ERTS_ALC_N2TD(n), (Uint) ptr); #endif #ifdef HARD_DEBUG erts_hdbg_chk_blks(); #endif } static Uint install_debug_functions(void) { int i; ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs)); sys_memcpy((void *)real_allctrs,(void *)erts_allctrs,sizeof(erts_allctrs)); for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { erts_allctrs[i].alloc = debug_alloc; erts_allctrs[i].realloc = debug_realloc; erts_allctrs[i].free = debug_free; erts_allctrs[i].extra = (void *) &real_allctrs[i]; } return FENCE_SZ; } #endif /* #ifdef DEBUG */