aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/erl_alloc.c
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam/erl_alloc.c')
-rw-r--r--erts/emulator/beam/erl_alloc.c548
1 files changed, 397 insertions, 151 deletions
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index d748f86d75..b5ba9bb94a 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -71,6 +71,23 @@
#define AU_ALLOC_DEFAULT_ENABLE(X) (X)
#endif
+#define ERTS_ALC_DEFAULT_ENABLED_ACUL 60
+#define ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC 45
+#define ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC 85
+
+#define ERTS_ALC_DEFAULT_ACUL 0
+#define ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC 0
+#define ERTS_ALC_DEFAULT_ACUL_LL_ALLOC 0
+
+#ifndef ERTS_SMP
+# undef ERTS_ALC_DEFAULT_ACUL
+# define ERTS_ALC_DEFAULT_ACUL 0
+# undef ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC
+# define ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC 0
+# undef ERTS_ALC_DEFAULT_ACUL_LL_ALLOC
+# define ERTS_ALC_DEFAULT_ACUL_LL_ALLOC 0
+#endif
+
#ifdef DEBUG
static Uint install_debug_functions(void);
#if 0
@@ -83,6 +100,8 @@ static Uint install_debug_functions(void);
#endif
#endif
+static int lock_all_physical_memory = 0;
+
ErtsAllocatorFunctions_t erts_allctrs[ERTS_ALC_A_MAX+1];
ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1];
ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1];
@@ -101,11 +120,9 @@ typedef union {
char align_aoffa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AOFFAllctr_t))];
} ErtsAllocatorState_t;
-static ErtsAllocatorState_t sbmbc_alloc_state;
static ErtsAllocatorState_t std_alloc_state;
static ErtsAllocatorState_t ll_alloc_state;
#if HALFWORD_HEAP
-static ErtsAllocatorState_t sbmbc_low_alloc_state;
static ErtsAllocatorState_t std_low_alloc_state;
static ErtsAllocatorState_t ll_low_alloc_state;
#endif
@@ -120,6 +137,7 @@ static ErtsAllocatorState_t fix_alloc_state;
typedef struct {
erts_smp_atomic32_t refc;
int only_sz;
+ int internal;
Uint req_sched;
Process *proc;
Eterm ref;
@@ -162,6 +180,7 @@ enum allctr_type {
struct au_init {
int enable;
int thr_spec;
+ int carrier_migration_allowed;
enum allctr_type atype;
struct {
AllctrInit_t util;
@@ -200,7 +219,6 @@ typedef struct {
char *mtrace;
char *nodename;
} instr;
- struct au_init sbmbc_alloc;
struct au_init sl_alloc;
struct au_init std_alloc;
struct au_init ll_alloc;
@@ -211,13 +229,12 @@ typedef struct {
struct au_init driver_alloc;
struct au_init fix_alloc;
#if HALFWORD_HEAP
- struct au_init sbmbc_low_alloc;
struct au_init std_low_alloc;
struct au_init ll_low_alloc;
#endif
} erts_alc_hndl_args_init_t;
-#define ERTS_AU_INIT__ {0, 0, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}}
+#define ERTS_AU_INIT__ {0, 0, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}}
#define SET_DEFAULT_ALLOC_OPTS(IP) \
do { \
@@ -225,43 +242,33 @@ do { \
sys_memcpy((void *) (IP), (void *) &aui__, sizeof(struct au_init)); \
} while (0)
-static void
-set_default_sbmbc_alloc_opts(struct au_init *ip)
+#if ERTS_ALC_DEFAULT_ACUL \
+ || ERTS_ALC_DEFAULT_ACUL_LL_ALLOC \
+ || ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC
+
+static ERTS_INLINE void
+set_default_acul(struct au_init *ip, int acul)
{
- SET_DEFAULT_ALLOC_OPTS(ip);
- ip->enable = 0;
- ip->thr_spec = 0;
- ip->atype = BESTFIT;
- ip->init.bf.ao = 1;
- ip->init.util.ramv = 0;
- ip->init.util.mmsbc = 0;
- ip->init.util.mmmbc = 500;
- ip->init.util.sbct = ~((UWord) 0);
- ip->init.util.name_prefix = "sbmbc_";
- ip->init.util.alloc_no = ERTS_ALC_A_SBMBC;
-#ifndef SMALL_MEMORY
- ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */
-#else
- ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */
-#endif
- ip->init.util.ts = ERTS_ALC_MTA_SBMBC;
- ip->init.util.asbcst = 0;
- ip->init.util.rsbcst = 0;
- ip->init.util.rsbcmt = 0;
- ip->init.util.rmbcmt = 0;
- ip->init.util.sbmbct = 0;
- ip->init.util.sbmbcs = 0;
+ ip->thr_spec = 1;
+ ip->atype = AOFIRSTFIT;
+ ip->init.aoff.flavor = AOFF_BF;
+ ip->init.util.acul = acul;
}
+#endif
+
static void
set_default_sl_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
+#if ERTS_ALC_DEFAULT_ACUL
+ set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL);
+#else
ip->thr_spec = 1;
ip->atype = GOODFIT;
+#endif
ip->init.util.name_prefix = "sl_";
- ip->init.util.mmmbc = 5;
ip->init.util.alloc_no = ERTS_ALC_A_SHORT_LIVED;
#ifndef SMALL_MEMORY
ip->init.util.mmbcs = 128*1024; /* Main carrier size */
@@ -282,10 +289,13 @@ set_default_std_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
+#if ERTS_ALC_DEFAULT_ACUL
+ set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL);
+#else
ip->thr_spec = 1;
ip->atype = BESTFIT;
+#endif
ip->init.util.name_prefix = "std_";
- ip->init.util.mmmbc = 5;
ip->init.util.alloc_no = ERTS_ALC_A_STANDARD;
#ifndef SMALL_MEMORY
ip->init.util.mmbcs = 128*1024; /* Main carrier size */
@@ -300,12 +310,15 @@ set_default_ll_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
+#if ERTS_ALC_DEFAULT_ACUL_LL_ALLOC
+ set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL_LL_ALLOC);
+#else
ip->thr_spec = 0;
ip->atype = BESTFIT;
ip->init.bf.ao = 1;
+#endif
ip->init.util.ramv = 0;
ip->init.util.mmsbc = 0;
- ip->init.util.mmmbc = 0;
ip->init.util.sbct = ~((UWord) 0);
ip->init.util.name_prefix = "ll_";
ip->init.util.alloc_no = ERTS_ALC_A_LONG_LIVED;
@@ -319,8 +332,6 @@ set_default_ll_alloc_opts(struct au_init *ip)
ip->init.util.rsbcst = 0;
ip->init.util.rsbcmt = 0;
ip->init.util.rmbcmt = 0;
- ip->init.util.sbmbct = 0;
- ip->init.util.sbmbcs = 0;
}
static void
@@ -329,6 +340,7 @@ set_default_temp_alloc_opts(struct au_init *ip)
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
ip->thr_spec = 1;
+ ip->carrier_migration_allowed = 0;
ip->atype = AFIT;
ip->init.util.name_prefix = "temp_";
ip->init.util.alloc_no = ERTS_ALC_A_TEMPORARY;
@@ -351,9 +363,12 @@ set_default_eheap_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
+#if ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC
+ set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC);
+#else
ip->thr_spec = 1;
ip->atype = GOODFIT;
- ip->init.util.mmmbc = 100;
+#endif
ip->init.util.name_prefix = "eheap_";
ip->init.util.alloc_no = ERTS_ALC_A_EHEAP;
#ifndef SMALL_MEMORY
@@ -374,9 +389,12 @@ set_default_binary_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
+#if ERTS_ALC_DEFAULT_ACUL
+ set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL);
+#else
ip->thr_spec = 1;
ip->atype = BESTFIT;
- ip->init.util.mmmbc = 50;
+#endif
ip->init.util.name_prefix = "binary_";
ip->init.util.alloc_no = ERTS_ALC_A_BINARY;
#ifndef SMALL_MEMORY
@@ -392,9 +410,12 @@ set_default_ets_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
+#if ERTS_ALC_DEFAULT_ACUL
+ set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL);
+#else
ip->thr_spec = 1;
ip->atype = BESTFIT;
- ip->init.util.mmmbc = 100;
+#endif
ip->init.util.name_prefix = "ets_";
ip->init.util.alloc_no = ERTS_ALC_A_ETS;
#ifndef SMALL_MEMORY
@@ -410,8 +431,12 @@ set_default_driver_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
+#if ERTS_ALC_DEFAULT_ACUL
+ set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL);
+#else
ip->thr_spec = 1;
ip->atype = BESTFIT;
+#endif
ip->init.util.name_prefix = "driver_";
ip->init.util.alloc_no = ERTS_ALC_A_DRIVER;
#ifndef SMALL_MEMORY
@@ -428,8 +453,12 @@ set_default_fix_alloc_opts(struct au_init *ip,
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
+#if ERTS_ALC_DEFAULT_ACUL
+ set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL);
+#else
ip->thr_spec = 1;
ip->atype = BESTFIT;
+#endif
ip->init.bf.ao = 1;
ip->init.util.name_prefix = "fix_";
ip->init.util.fix_type_size = fix_type_sizes;
@@ -462,13 +491,6 @@ adjust_tpref(struct au_init *ip, int no_sched)
/* ... shrink smallest multi-block carrier size */
if (ip->default_.smbcs)
ip->init.util.smbcs /= ERTS_MIN(4, no_sched);
- /* ... and more than three allocators shrink
- max mseg multi-block carriers */
- if (ip->default_.mmmbc && no_sched > 2) {
- ip->init.util.mmmbc /= ERTS_MIN(4, no_sched - 1);
- if (ip->init.util.mmmbc < 3)
- ip->init.util.mmmbc = 3;
- }
}
}
@@ -495,6 +517,71 @@ refuse_af_strategy(struct au_init *init)
static void hdbg_init(void);
#endif
+static void adjust_fix_alloc_sizes(UWord extra_block_size)
+{
+
+ if (extra_block_size && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].enabled) {
+ int j;
+
+#ifdef ERTS_SMP
+ if (erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].thr_spec) {
+ int i;
+ ErtsAllocatorThrSpec_t* tspec;
+
+ tspec = &erts_allctr_thr_spec[ERTS_ALC_A_FIXED_SIZE];
+ ASSERT(tspec->enabled);
+
+ for (i=0; i < tspec->size; i++) {
+ Allctr_t* allctr = tspec->allctr[i];
+ for (j=0; j < ERTS_ALC_NO_FIXED_SIZES; ++j) {
+ allctr->fix[j].type_size += extra_block_size;
+ }
+ }
+ }
+ else
+#endif
+ {
+ Allctr_t* allctr = erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra;
+ for (j=0; j < ERTS_ALC_NO_FIXED_SIZES; ++j) {
+ allctr->fix[j].type_size += extra_block_size;
+ }
+ }
+ }
+}
+
+static ERTS_INLINE int
+strategy_support_carrier_migration(struct au_init *auip)
+{
+ /*
+ * Currently only aoff, aoffcbf and aoffcaobf support carrier
+ * migration, i.e, type AOFIRSTFIT.
+ */
+ return auip->atype == AOFIRSTFIT;
+}
+
+static ERTS_INLINE void
+check_disable_carrier_migration(struct au_init *auip)
+{
+ if (!strategy_support_carrier_migration(auip) || !auip->thr_spec)
+ auip->init.util.acul = 0;
+}
+
+static ERTS_INLINE void
+ensure_carrier_migration_support(struct au_init *auip)
+{
+ auip->thr_spec = 1; /* Need thread preferred */
+
+ /*
+ * If strategy cannot handle carrier migration,
+ * default to a strategy that can...
+ */
+ if (!strategy_support_carrier_migration(auip)) {
+ /* Default to aoffcbf */
+ auip->atype = AOFIRSTFIT;
+ auip->init.aoff.flavor = AOFF_BF;
+ }
+}
+
void
erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
{
@@ -515,9 +602,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
= sizeof(Process);
#if !HALFWORD_HEAP
fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MONITOR_SH)]
- = ERTS_MONITOR_SH_SIZE;
+ = ERTS_MONITOR_SH_SIZE * sizeof(Uint);
fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NLINK_SH)]
- = ERTS_LINK_SH_SIZE;
+ = ERTS_LINK_SH_SIZE * sizeof(Uint);
#endif
fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_EV_D_STATE)]
= sizeof(ErtsDrvEventDataState);
@@ -533,15 +620,17 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
hdbg_init();
#endif
- erts_have_sbmbc_alloc = 0;
+ lock_all_physical_memory = 0;
+
ncpu = eaiop->ncpu;
if (ncpu < 1)
ncpu = 1;
+ erts_tsd_key_create(&erts_allctr_prelock_tsd_key);
+
erts_sys_alloc_init();
erts_init_utils_mem();
- set_default_sbmbc_alloc_opts(&init.sbmbc_alloc);
set_default_sl_alloc_opts(&init.sl_alloc);
set_default_std_alloc_opts(&init.std_alloc);
set_default_ll_alloc_opts(&init.ll_alloc);
@@ -556,8 +645,21 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
if (argc && argv)
handle_args(argc, argv, &init);
+ if (lock_all_physical_memory) {
+#ifdef HAVE_MLOCKALL
+ errno = 0;
+ if (mlockall(MCL_CURRENT|MCL_FUTURE) != 0) {
+ int err = errno;
+ char *errstr = err ? strerror(err) : "unknown";
+ erl_exit(-1, "Failed to lock physical memory: %s (%d)\n",
+ errstr, err);
+ }
+#else
+ erl_exit(-1, "Failed to lock physical memory: Not supported\n");
+#endif
+ }
+
#ifndef ERTS_SMP
- init.sbmbc_alloc.thr_spec = 0;
init.sl_alloc.thr_spec = 0;
init.std_alloc.thr_spec = 0;
init.ll_alloc.thr_spec = 0;
@@ -570,7 +672,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
if (init.erts_alloc_config) {
/* Adjust flags that erts_alloc_config won't like */
- init.sbmbc_alloc.thr_spec = 0;
init.temp_alloc.thr_spec = 0;
init.sl_alloc.thr_spec = 0;
init.std_alloc.thr_spec = 0;
@@ -582,13 +683,22 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
init.fix_alloc.thr_spec = 0;
}
+ check_disable_carrier_migration(&init.sl_alloc);
+ check_disable_carrier_migration(&init.std_alloc);
+ check_disable_carrier_migration(&init.ll_alloc);
+ check_disable_carrier_migration(&init.eheap_alloc);
+ check_disable_carrier_migration(&init.binary_alloc);
+ check_disable_carrier_migration(&init.ets_alloc);
+ check_disable_carrier_migration(&init.driver_alloc);
+ check_disable_carrier_migration(&init.fix_alloc);
+
+
#ifdef ERTS_SMP
/* Only temp_alloc can use thread specific interface */
if (init.temp_alloc.thr_spec)
init.temp_alloc.thr_spec = erts_no_schedulers;
/* Others must use thread preferred interface */
- adjust_tpref(&init.sbmbc_alloc, erts_no_schedulers);
adjust_tpref(&init.sl_alloc, erts_no_schedulers);
adjust_tpref(&init.std_alloc, erts_no_schedulers);
adjust_tpref(&init.ll_alloc, erts_no_schedulers);
@@ -607,7 +717,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
* The following allocators cannot be run with afit strategy.
* Make sure they don't...
*/
- refuse_af_strategy(&init.sbmbc_alloc);
refuse_af_strategy(&init.sl_alloc);
refuse_af_strategy(&init.std_alloc);
refuse_af_strategy(&init.ll_alloc);
@@ -627,6 +736,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
init.mseg.nos = erts_no_schedulers;
erts_mseg_init(&init.mseg);
#endif
+
erts_alcu_init(&init.alloc_util);
erts_afalc_init();
erts_bfalc_init();
@@ -651,11 +761,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
#if HALFWORD_HEAP
/* Init low memory variants by cloning */
- init.sbmbc_low_alloc = init.sbmbc_alloc;
- init.sbmbc_low_alloc.init.util.name_prefix = "sbmbc_low_";
- init.sbmbc_low_alloc.init.util.alloc_no = ERTS_ALC_A_SBMBC_LOW;
- init.sbmbc_low_alloc.init.util.low_mem = 1;
-
init.std_low_alloc = init.std_alloc;
init.std_low_alloc.init.util.name_prefix = "std_low_";
init.std_low_alloc.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW;
@@ -668,13 +773,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
init.ll_low_alloc.init.util.force = 1;
init.ll_low_alloc.init.util.low_mem = 1;
- set_au_allocator(ERTS_ALC_A_SBMBC_LOW, &init.sbmbc_low_alloc, ncpu);
set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc, ncpu);
set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc, ncpu);
#endif /* HALFWORD */
set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, ncpu);
- set_au_allocator(ERTS_ALC_A_SBMBC, &init.sbmbc_alloc, ncpu);
set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc, ncpu);
set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc, ncpu);
set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc, ncpu);
@@ -701,20 +804,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
erts_mtrace_init(init.instr.mtrace, init.instr.nodename);
- /* sbmbc_alloc() needs to be started first */
- start_au_allocator(ERTS_ALC_A_SBMBC,
- &init.sbmbc_alloc,
- &sbmbc_alloc_state);
-#if HALFWORD_HEAP
- start_au_allocator(ERTS_ALC_A_SBMBC_LOW,
- &init.sbmbc_low_alloc,
- &sbmbc_low_alloc_state);
- erts_have_sbmbc_alloc = (init.sbmbc_alloc.enable
- && init.sbmbc_low_alloc.enable);
-#else
- erts_have_sbmbc_alloc = init.sbmbc_alloc.enable;
-#endif
-
start_au_allocator(ERTS_ALC_A_TEMPORARY,
&init.temp_alloc,
&temp_alloc_state);
@@ -768,7 +857,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
#ifdef DEBUG
extra_block_size += install_debug_functions();
#endif
-
+ adjust_fix_alloc_sizes(extra_block_size);
}
void
@@ -1104,6 +1193,26 @@ get_kb_value(char *param_end, char** argv, int* ip)
return ((Uint) tmp)*1024;
}
+static UWord
+get_mb_value(char *param_end, char** argv, int* ip)
+{
+ SWord tmp;
+ UWord max = ((~((UWord) 0))/(1024*1024)) + 1;
+ char *rest;
+ char *param = argv[*ip]+1;
+ char *value = get_value(param_end, argv, ip);
+ errno = 0;
+ tmp = (SWord) ErtsStrToSint(value, &rest, 10);
+ if (errno != 0 || rest == value || tmp < 0 || max < ((UWord) tmp))
+ bad_value(param, param_end, value);
+ if (max == (UWord) tmp)
+ return ~((UWord) 0);
+ else
+ return ((UWord) tmp)*1024*1024;
+}
+
+
+#if 0
static Uint
get_byte_value(char *param_end, char** argv, int* ip)
{
@@ -1117,6 +1226,7 @@ get_byte_value(char *param_end, char** argv, int* ip)
bad_value(param, param_end, value);
return (Uint) tmp;
}
+#endif
static Uint
get_amount_value(char *param_end, char** argv, int* ip)
@@ -1132,17 +1242,59 @@ get_amount_value(char *param_end, char** argv, int* ip)
return (Uint) tmp;
}
+static Uint
+get_acul_value(struct au_init *auip, char *param_end, char** argv, int* ip)
+{
+ Sint tmp;
+ char *rest;
+ char *param = argv[*ip]+1;
+ char *value = get_value(param_end, argv, ip);
+ if (sys_strcmp(value, "de") == 0) {
+ switch (auip->init.util.alloc_no) {
+ case ERTS_ALC_A_LONG_LIVED:
+#if HALFWORD_HEAP
+ case ERTS_ALC_A_LONG_LIVED_LOW:
+#endif
+ return ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC;
+ case ERTS_ALC_A_EHEAP:
+ return ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC;
+ default:
+ return ERTS_ALC_DEFAULT_ENABLED_ACUL;
+ }
+ }
+ errno = 0;
+ tmp = (Sint) ErtsStrToSint(value, &rest, 10);
+ if (errno != 0 || rest == value || tmp < 0 || 100 < tmp)
+ bad_value(param, param_end, value);
+ return (Uint) tmp;
+}
+
static void
handle_au_arg(struct au_init *auip,
char* sub_param,
char** argv,
- int* ip)
+ int* ip,
+ int u_switch)
{
char *param = argv[*ip]+1;
switch (sub_param[0]) {
case 'a':
- if(has_prefix("asbcst", sub_param)) {
+ if (has_prefix("acul", sub_param)) {
+ if (!auip->carrier_migration_allowed) {
+ if (!u_switch)
+ goto bad_switch;
+ else {
+ /* ignore */
+ (void) get_acul_value(auip, sub_param + 4, argv, ip);
+ break;
+ }
+ }
+ ensure_carrier_migration_support(auip);
+
+ auip->init.util.acul = get_acul_value(auip, sub_param + 4, argv, ip);
+ }
+ else if(has_prefix("asbcst", sub_param)) {
auip->init.util.asbcst = get_kb_value(sub_param + 6, argv, ip);
}
else if(has_prefix("as", sub_param)) {
@@ -1163,21 +1315,26 @@ handle_au_arg(struct au_init *auip,
}
else if (strcmp("aoff", alg) == 0) {
auip->atype = AOFIRSTFIT;
+ auip->init.aoff.flavor = AOFF_AOFF;
+ }
+ else if (strcmp("aoffcbf", alg) == 0) {
+ auip->atype = AOFIRSTFIT;
+ auip->init.aoff.flavor = AOFF_BF;
+ }
+ else if (strcmp("aoffcaobf", alg) == 0) {
+ auip->atype = AOFIRSTFIT;
+ auip->init.aoff.flavor = AOFF_AOBF;
}
else {
bad_value(param, sub_param + 1, alg);
}
+ check_disable_carrier_migration(auip);
}
else
goto bad_switch;
break;
case 'e':
auip->enable = get_bool_value(sub_param+1, argv, ip);
-#if !HAVE_ERTS_SBMBC
- if (auip->init.util.alloc_no == ERTS_ALC_A_SBMBC) {
- auip->enable = 0;
- }
-#endif
break;
case 'l':
if (has_prefix("lmbcs", sub_param)) {
@@ -1237,18 +1394,6 @@ handle_au_arg(struct au_init *auip,
if(has_prefix("sbct", sub_param)) {
auip->init.util.sbct = get_kb_value(sub_param + 4, argv, ip);
}
- else if (has_prefix("sbmbcs", sub_param)) {
-#if HAVE_ERTS_SBMBC
- auip->init.util.sbmbcs =
-#endif
- get_byte_value(sub_param + 6, argv, ip);
- }
- else if (has_prefix("sbmbct", sub_param)) {
-#if HAVE_ERTS_SBMBC
- auip->init.util.sbmbct =
-#endif
- get_byte_value(sub_param + 6, argv, ip);
- }
else if (has_prefix("smbcs", sub_param)) {
auip->default_.smbcs = 0;
auip->init.util.smbcs = get_kb_value(sub_param + 5, argv, ip);
@@ -1264,6 +1409,7 @@ handle_au_arg(struct au_init *auip,
}
else if (res == 0) {
auip->thr_spec = 0;
+ check_disable_carrier_migration(auip);
break;
}
goto bad_switch;
@@ -1278,7 +1424,6 @@ static void
handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
{
struct au_init *aui[] = {
- &init->sbmbc_alloc,
&init->binary_alloc,
&init->std_alloc,
&init->ets_alloc,
@@ -1305,25 +1450,22 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
case 'M':
switch (argv[i][2]) {
case 'B':
- handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i);
- break;
- case 'C':
- handle_au_arg(&init->sbmbc_alloc, &argv[i][3], argv, &i);
+ handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i, 0);
break;
case 'D':
- handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i);
+ handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i, 0);
break;
case 'E':
- handle_au_arg(&init->ets_alloc, &argv[i][3], argv, &i);
+ handle_au_arg(&init->ets_alloc, &argv[i][3], argv, &i, 0);
break;
case 'F':
- handle_au_arg(&init->fix_alloc, &argv[i][3], argv, &i);
+ handle_au_arg(&init->fix_alloc, &argv[i][3], argv, &i, 0);
break;
case 'H':
- handle_au_arg(&init->eheap_alloc, &argv[i][3], argv, &i);
+ handle_au_arg(&init->eheap_alloc, &argv[i][3], argv, &i, 0);
break;
case 'L':
- handle_au_arg(&init->ll_alloc, &argv[i][3], argv, &i);
+ handle_au_arg(&init->ll_alloc, &argv[i][3], argv, &i, 0);
break;
case 'M':
if (has_prefix("amcbf", argv[i]+3)) {
@@ -1344,18 +1486,42 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
#endif
get_amount_value(argv[i]+6, argv, &i);
}
+ else if (has_prefix("scs", argv[i]+3)) {
+#if HAVE_ERTS_MSEG
+ init->mseg.mmap.scs =
+#endif
+ get_mb_value(argv[i]+6, argv, &i);
+ }
+ else if (has_prefix("sco", argv[i]+3)) {
+#if HAVE_ERTS_MSEG
+ init->mseg.mmap.sco =
+#endif
+ get_bool_value(argv[i]+6, argv, &i);
+ }
+ else if (has_prefix("scrpm", argv[i]+3)) {
+#if HAVE_ERTS_MSEG
+ init->mseg.mmap.scrpm =
+#endif
+ get_bool_value(argv[i]+8, argv, &i);
+ }
+ else if (has_prefix("scrfsd", argv[i]+3)) {
+#if HAVE_ERTS_MSEG
+ init->mseg.mmap.scrfsd =
+#endif
+ get_amount_value(argv[i]+9, argv, &i);
+ }
else {
bad_param(param, param+2);
}
break;
case 'R':
- handle_au_arg(&init->driver_alloc, &argv[i][3], argv, &i);
+ handle_au_arg(&init->driver_alloc, &argv[i][3], argv, &i, 0);
break;
case 'S':
- handle_au_arg(&init->sl_alloc, &argv[i][3], argv, &i);
+ handle_au_arg(&init->sl_alloc, &argv[i][3], argv, &i, 0);
break;
case 'T':
- handle_au_arg(&init->temp_alloc, &argv[i][3], argv, &i);
+ handle_au_arg(&init->temp_alloc, &argv[i][3], argv, &i, 0);
break;
case 'Y': { /* sys_alloc */
if (has_prefix("tt", param+2)) {
@@ -1414,9 +1580,6 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
else if (strcmp("max", arg) == 0) {
for (a = 0; a < aui_sz; a++)
aui[a]->enable = 1;
-#if !HAVE_ERTS_SBMBC
- init->sbmbc_alloc.enable = 0;
-#endif
}
else if (strcmp("config", arg) == 0) {
init->erts_alloc_config = 1;
@@ -1444,8 +1607,8 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
for (a = 0; a < aui_sz; a++) {
aui[a]->thr_spec = 0;
+ check_disable_carrier_migration(aui[a]);
aui[a]->init.util.ramv = 0;
- aui[a]->init.util.mmmbc = 10;
aui[a]->init.util.lmbcs = 5*1024*1024;
}
}
@@ -1485,6 +1648,19 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
bad_param(param, param+2);
}
break;
+ case 'l':
+ if (has_prefix("pm", param+2)) {
+ arg = get_value(argv[i]+5, argv, &i);
+ if (strcmp("all", arg) == 0)
+ lock_all_physical_memory = 1;
+ else if (strcmp("no", arg) == 0)
+ lock_all_physical_memory = 0;
+ else
+ bad_value(param, param+4, arg);
+ break;
+ }
+ bad_param(param, param+2);
+ break;
case 'u':
if (has_prefix("ycs", argv[i]+3)) {
init->alloc_util.ycs
@@ -1494,6 +1670,10 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
init->alloc_util.mmc
= get_amount_value(argv[i]+6, argv, &i);
}
+ else if (has_prefix("sac", argv[i]+3)) {
+ init->alloc_util.sac
+ = get_bool_value(argv[i]+6, argv, &i);
+ }
else {
int a;
int start = i;
@@ -1508,7 +1688,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
argv[start + 1] = val;
i = start;
}
- handle_au_arg(aui[a], &argv[i][3], argv, &i);
+ handle_au_arg(aui[a], &argv[i][3], argv, &i, 1);
}
}
break;
@@ -2054,15 +2234,11 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
return am_badarg;
}
- /* All alloc_util allocators except sbmbc_alloc *have* to be enabled */
+ /* All alloc_util allocators *have* to be enabled */
for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) {
switch (ai) {
case ERTS_ALC_A_SYSTEM:
- case ERTS_ALC_A_SBMBC:
-#if HALFWORD_HEAP
- case ERTS_ALC_A_SBMBC_LOW:
-#endif
break;
default:
if (!erts_allctrs_info[ai].enabled
@@ -2102,12 +2278,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
* Often not thread safe and usually never
* contain any allocated memory.
*/
- case ERTS_ALC_A_SBMBC:
- /* Included in other allocators */
-#if HALFWORD_HEAP
- case ERTS_ALC_A_SBMBC_LOW:
- /* Included in other allocators */
-#endif
continue;
case ERTS_ALC_A_EHEAP:
save = &size.processes;
@@ -2142,7 +2312,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
if (want_tot_or_sys || want.processes || want.processes_used) {
- int max_processes = erts_ptab_max(&erts_proc);
UWord tmp;
if (ERTS_MEM_NEED_ALL_ALCU)
@@ -2152,7 +2321,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
fi, ERTS_ALC_NO_FIXED_SIZES);
tmp = alcu_size(ERTS_ALC_A_EHEAP, NULL, 0);
}
- tmp += max_processes*sizeof(erts_smp_atomic_t);
+ tmp += erts_ptab_mem_size(&erts_proc);
tmp += erts_bif_timer_memory_size();
tmp += erts_tot_link_lh_size();
@@ -2278,13 +2447,11 @@ struct aa_values {
Eterm
erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc)
{
-#define MAX_AA_VALUES (23)
+#define MAX_AA_VALUES (24)
struct aa_values values[MAX_AA_VALUES];
Eterm res = THE_NON_VALUE;
int i, length;
Uint reserved_atom_space, atom_space;
- int max_processes = erts_ptab_max(&erts_proc);
- int max_ports = erts_ptab_max(&erts_port);
if (proc) {
ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN
@@ -2315,8 +2482,8 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc)
values[i].arity = 2;
values[i].name = "static";
- values[i].ui[0] =
- max_ports*sizeof(erts_smp_atomic_t) /* Port table */
+ values[i].ui[0] =
+ sizeof(ErtsPTab)*2 /* proc & port tables */
+ erts_timer_wheel_memory_size(); /* Timer wheel */
i++;
@@ -2395,7 +2562,12 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc)
values[i].arity = 2;
values[i].name = "process_table";
- values[i].ui[0] = max_processes*sizeof(Process*);
+ values[i].ui[0] = erts_ptab_mem_size(&erts_proc);
+ i++;
+
+ values[i].arity = 2;
+ values[i].name = "port_table";
+ values[i].ui[0] = erts_ptab_mem_size(&erts_port);
i++;
values[i].arity = 2;
@@ -2549,7 +2721,7 @@ erts_allocator_info(int to, void *arg)
as = erts_allctr_thr_spec[a].allctr[ai];
}
/* Binary alloc has its own thread safety... */
- erts_alcu_info(as, 0, &to, arg, NULL, NULL);
+ erts_alcu_info(as, 0, 0, &to, arg, NULL, NULL);
}
else {
switch (a) {
@@ -2575,6 +2747,7 @@ erts_allocator_info(int to, void *arg)
#if HAVE_ERTS_MSEG
{
+ struct erts_mmap_info_struct emis;
#ifdef ERTS_SMP
int max = (int) erts_no_schedulers;
#else
@@ -2585,6 +2758,8 @@ erts_allocator_info(int to, void *arg)
erts_print(to, arg, "=allocator:mseg_alloc[%d]\n", i);
erts_mseg_info(i, &to, arg, 0, NULL, NULL);
}
+ erts_print(to, arg, "=allocator:mseg_alloc.erts_mmap\n");
+ erts_mmap_info(&to, arg, NULL, NULL, &emis);
}
#endif
@@ -2609,8 +2784,8 @@ erts_allocator_options(void *proc)
#endif
Uint sz, *szp, *hp, **hpp;
Eterm res, features, settings;
- Eterm atoms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+5];
- Uint terms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+5];
+ Eterm atoms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+7];
+ Uint terms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+7];
int a, length;
SysAllocStat sas;
Uint *endp = NULL;
@@ -2708,6 +2883,11 @@ erts_allocator_options(void *proc)
terms[length++] = erts_bld_2tup_list(hpp, szp, 3, o, v);
}
+ atoms[length] = am_atom_put("lock_physical_memory", 20);
+ terms[length++] = (lock_all_physical_memory
+ ? am_atom_put("all", 3)
+ : am_atom_put("no", 2));
+
settings = erts_bld_2tup_list(hpp, szp, length, atoms, terms);
length = 0;
@@ -2723,6 +2903,9 @@ erts_allocator_options(void *proc)
if (use_mseg)
terms[length++] = am_atom_put("mseg_alloc", 10);
#endif
+#if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
+ terms[length++] = am_atom_put("sys_aligned_alloc", 17);
+#endif
features = length ? erts_bld_list(hpp, szp, length, terms) : NIL;
@@ -2808,9 +2991,11 @@ reply_alloc_info(void *vair)
Uint sz, *szp;
ErlOffHeap *ohp = NULL;
ErlHeapFragment *bp = NULL;
+ struct erts_mmap_info_struct emis;
int i;
Eterm (*info_func)(Allctr_t *,
int,
+ int,
int *,
void *,
Uint **,
@@ -2919,15 +3104,23 @@ reply_alloc_info(void *vair)
? NIL
: erts_mseg_info(0, NULL, NULL, hpp != NULL,
hpp, szp));
- ainfo = erts_bld_tuple(hpp, szp, 3,
- alloc_atom,
- make_small(0),
- ainfo);
+ ainfo = erts_bld_tuple3(hpp, szp,
+ alloc_atom,
+ make_small(0),
+ ainfo);
+
+ ai_list = erts_bld_cons(hpp, szp,
+ ainfo, ai_list);
+ ainfo = (air->only_sz ? NIL : erts_mmap_info(NULL, NULL, hpp, szp, &emis));
+ ainfo = erts_bld_tuple3(hpp, szp,
+ alloc_atom,
+ erts_bld_atom(hpp,szp,"erts_mmap"),
+ ainfo);
#else
- ainfo = erts_bld_tuple(hpp, szp, 2, alloc_atom,
- am_false);
+ ainfo = erts_bld_tuple2(hpp, szp, alloc_atom,
+ am_false);
#endif
- break;
+ break;
default:
alloc_atom = erts_bld_atom(hpp, szp,
(char *) ERTS_ALC_A2AD(ai));
@@ -2939,8 +3132,8 @@ reply_alloc_info(void *vair)
allctr = erts_allctr_thr_spec[ai].allctr[0];
else
allctr = erts_allctrs_info[ai].extra;
- ainfo = info_func(allctr, hpp != NULL, NULL,
- NULL, hpp, szp);
+ ainfo = info_func(allctr, air->internal, hpp != NULL,
+ NULL, NULL, hpp, szp);
ainfo = erts_bld_tuple(hpp, szp, 3, alloc_atom,
make_small(0), ainfo);
}
@@ -2975,7 +3168,7 @@ reply_alloc_info(void *vair)
alloc_atom = erts_bld_atom(hpp, szp,
(char *) ERTS_ALC_A2AD(ai));
allctr = erts_allctr_thr_spec[ai].allctr[sched_id];
- ainfo = info_func(allctr, hpp != NULL, NULL,
+ ainfo = info_func(allctr, air->internal, hpp != NULL, NULL,
NULL, hpp, szp);
ai_list = erts_bld_cons(hpp, szp,
erts_bld_tuple(
@@ -3031,7 +3224,8 @@ int
erts_request_alloc_info(struct process *c_p,
Eterm ref,
Eterm allocs,
- int only_sz)
+ int only_sz,
+ int internal)
{
ErtsAllocInfoReq *air = aireq_alloc();
Eterm req_ai[ERTS_ALC_A_MAX+1+2] = {0};
@@ -3043,6 +3237,8 @@ erts_request_alloc_info(struct process *c_p,
air->only_sz = only_sz;
+ air->internal = internal;
+
air->proc = c_p;
if (is_not_internal_ref(ref))
@@ -3107,6 +3303,55 @@ erts_request_alloc_info(struct process *c_p,
return 1;
}
+/*
+ * The allocator wrapper prelocking stuff below is about the locking order.
+ * It only affects wrappers (erl_mtrace.c and erl_instrument.c) that keep locks
+ * during alloc/realloc/free.
+ *
+ * Some query functions in erl_alloc_util.c lock the allocator mutex and then
+ * use erts_printf that in turn may call the sys allocator through the wrappers.
+ * To avoid breaking locking order these query functions first "pre-locks" all
+ * allocator wrappers.
+ */
+ErtsAllocatorWrapper_t *erts_allctr_wrappers;
+int erts_allctr_wrapper_prelocked = 0;
+erts_tsd_key_t erts_allctr_prelock_tsd_key;
+
+void erts_allctr_wrapper_prelock_init(ErtsAllocatorWrapper_t* wrapper)
+{
+ ASSERT(wrapper->lock && wrapper->unlock);
+ wrapper->next = erts_allctr_wrappers;
+ erts_allctr_wrappers = wrapper;
+}
+
+void erts_allctr_wrapper_pre_lock(void)
+{
+ if (erts_allctr_wrappers) {
+ ErtsAllocatorWrapper_t* wrapper = erts_allctr_wrappers;
+ for ( ; wrapper; wrapper = wrapper->next) {
+ wrapper->lock();
+ }
+ ASSERT(!erts_allctr_wrapper_prelocked);
+ erts_allctr_wrapper_prelocked = 1;
+ erts_tsd_set(erts_allctr_prelock_tsd_key, (void*)1);
+ }
+}
+
+void erts_allctr_wrapper_pre_unlock(void)
+{
+ if (erts_allctr_wrappers) {
+ ErtsAllocatorWrapper_t* wrapper = erts_allctr_wrappers;
+
+ erts_allctr_wrapper_prelocked = 0;
+ erts_tsd_set(erts_allctr_prelock_tsd_key, (void*)0);
+ for ( ; wrapper; wrapper = wrapper->next) {
+ wrapper->unlock();
+ }
+ }
+}
+
+
+
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
* Deprecated functions *
* *
@@ -3136,10 +3381,7 @@ void *safe_realloc(void *ptr, Uint sz)
\* */
#define ERTS_ALC_TEST_ABORT erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n")
-UWord erts_alc_test(UWord op,
- UWord a1,
- UWord a2,
- UWord a3)
+UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3)
{
switch (op >> 8) {
case 0x0: return erts_alcu_test(op, a1, a2);
@@ -3191,14 +3433,13 @@ UWord erts_alc_test(UWord op,
init.atype = GOODFIT;
init.init.util.name_prefix = (char *) a1;
init.init.util.ts = a2 ? 1 : 0;
- init.init.util.sbmbct = 0;
if ((char **) a3) {
char **argv = (char **) a3;
int i = 0;
while (argv[i]) {
if (argv[i][0] == '-' && argv[i][1] == 't')
- handle_au_arg(&init, &argv[i][2], argv, &i);
+ handle_au_arg(&init, &argv[i][2], argv, &i, 0);
else
return (UWord) NULL;
i++;
@@ -3318,6 +3559,11 @@ UWord erts_alc_test(UWord op,
ERTS_ALC_TEST_ABORT;
break;
#endif /* #ifdef USE_THREADS */
+#ifdef ERTS_SMP
+ case 0xf13: return (UWord) 1;
+#else
+ case 0xf13: return (UWord) 0;
+#endif
default:
break;
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-26 21:35:38 +0000