aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/erl_alloc.c
diff options
context:
space:
mode:
authorRickard Green <[email protected]>2010-09-15 22:14:51 +0200
committerRickard Green <[email protected]>2011-11-13 20:39:30 +0100
commita67e91e658bdbba24fcc3c79b06fdf10ff830bc9 (patch)
tree07f9e6b1fd715d516d2571521307fe1b9d7c3948 /erts/emulator/beam/erl_alloc.c
parent55358c54778ead444e51f565d00175ba887ef182 (diff)
downloadotp-a67e91e658bdbba24fcc3c79b06fdf10ff830bc9.tar.gz
otp-a67e91e658bdbba24fcc3c79b06fdf10ff830bc9.tar.bz2
otp-a67e91e658bdbba24fcc3c79b06fdf10ff830bc9.zip
Optimize memory allocation
A number of memory allocation optimizations have been implemented. Most optimizations reduce contention caused by synchronization between threads during allocation and deallocation of memory. Most notably: * Synchronization of memory management in scheduler specific allocator instances has been rewritten to use lock-free synchronization. * Synchronization of memory management in scheduler specific pre-allocators has been rewritten to use lock-free synchronization. * The 'mseg_alloc' memory segment allocator now use scheduler specific instances instead of one instance. Apart from reducing contention this also ensures that memory allocators always create memory segments on the local NUMA node on a NUMA system.
Diffstat (limited to 'erts/emulator/beam/erl_alloc.c')
-rw-r--r--erts/emulator/beam/erl_alloc.c1282
1 files changed, 692 insertions, 590 deletions
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 9af80dd7a9..80951c9b50 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -40,6 +40,7 @@
#include "erl_mseg.h"
#include "erl_monitors.h"
#include "erl_bif_timer.h"
+#include "erl_cpu_topology.h"
#if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE)
#include "erl_check_io.h"
#endif
@@ -54,7 +55,14 @@
#include "erl_ao_firstfit_alloc.h"
-#define ERTS_ALC_DEFAULT_MAX_THR_PREF 16
+#if ERTS_MAX_NO_OF_SCHEDULERS > ERTS_AU_MAX_PREF_ALLOC_INSTANCES
+# error "Too many schedulers; cannot create that many pref alloc instances"
+#endif
+
+#define ERTS_ALC_FIX_TYPE_IX(T) \
+ (ERTS_ALC_T2N((T)) - ERTS_ALC_N_MIN_A_FIXED_SIZE)
+
+#define ERTS_ALC_DEFAULT_MAX_THR_PREF ERTS_MAX_NO_OF_SCHEDULERS
#if defined(SMALL_MEMORY) || defined(PURIFY) || defined(VALGRIND)
#define AU_ALLOC_DEFAULT_ENABLE(X) 0
@@ -106,24 +114,43 @@ static ErtsAllocatorState_t eheap_alloc_state;
static ErtsAllocatorState_t binary_alloc_state;
static ErtsAllocatorState_t ets_alloc_state;
static ErtsAllocatorState_t driver_alloc_state;
+static ErtsAllocatorState_t fix_alloc_state;
-ErtsAlcType_t erts_fix_core_allocator_ix;
-#ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE
-static void *(*fix_core_allocator)(ErtsAlcType_t, void *, Uint);
-static void *fix_core_extra;
-static void *fix_core_alloc(Uint size)
+typedef struct {
+ erts_smp_atomic32_t refc;
+ int only_sz;
+ Uint req_sched;
+ Process *proc;
+ Eterm ref;
+ Eterm ref_heap[REF_THING_SIZE];
+ int allocs[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+1+2];
+} ErtsAllocInfoReq;
+
+#define ERTS_ALC_INFO_A_ALLOC_UTIL (ERTS_ALC_A_MAX + 1)
+#define ERTS_ALC_INFO_A_MSEG_ALLOC (ERTS_ALC_A_MAX + 2)
+#define ERTS_ALC_INFO_A_MAX ERTS_ALC_INFO_A_MSEG_ALLOC
+
+#if !HALFWORD_HEAP
+ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(aireq,
+ ErtsAllocInfoReq,
+ 5,
+ ERTS_ALC_T_AINFO_REQ)
+#else
+static ERTS_INLINE ErtsAllocInfoReq *
+aireq_alloc(void)
{
- void *res;
- res = (*fix_core_allocator)(ERTS_ALC_T_UNDEF, fix_core_extra, size);
- if (erts_mtrace_enabled)
- erts_mtrace_crr_alloc(res,
- ERTS_ALC_A_FIXED_SIZE,
- erts_fix_core_allocator_ix,
- size);
- return res;
+ return erts_alloc(ERTS_ALC_T_AINFO_REQ, sizeof(ErtsAllocInfoReq));
+}
+
+static ERTS_INLINE void
+aireq_free(ErtsAllocInfoReq *ptr)
+{
+ erts_free(ERTS_ALC_T_AINFO_REQ, ptr);
}
#endif
+ErtsAlcType_t erts_fix_core_allocator_ix;
+
enum allctr_type {
GOODFIT,
BESTFIT,
@@ -181,6 +208,7 @@ typedef struct {
struct au_init binary_alloc;
struct au_init ets_alloc;
struct au_init driver_alloc;
+ struct au_init fix_alloc;
#if HALFWORD_HEAP
struct au_init sbmbc_low_alloc;
struct au_init std_low_alloc;
@@ -393,46 +421,52 @@ set_default_driver_alloc_opts(struct au_init *ip)
ip->init.util.ts = ERTS_ALC_MTA_DRIVER;
}
+static void
+set_default_fix_alloc_opts(struct au_init *ip,
+ size_t *fix_type_sizes)
+{
+ SET_DEFAULT_ALLOC_OPTS(ip);
+ ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
+ ip->thr_spec = 1;
+ ip->atype = BESTFIT;
+ ip->init.bf.ao = 1;
+ ip->init.util.name_prefix = "fix_";
+ ip->init.util.fix_type_size = fix_type_sizes;
+ ip->init.util.alloc_no = ERTS_ALC_A_FIXED_SIZE;
+#ifndef SMALL_MEMORY
+ ip->init.util.mmbcs = 128*1024; /* Main carrier size */
+#else
+ ip->init.util.mmbcs = 128*1024; /* Main carrier size */
+#endif
+ ip->init.util.ts = ERTS_ALC_MTA_FIXED_SIZE;
+}
+
#ifdef ERTS_SMP
static void
adjust_tpref(struct au_init *ip, int no_sched)
{
if (ip->thr_spec) {
- Uint allocs;
- if (ip->thr_spec < 0) {/* User specified amount */
- allocs = abs(ip->thr_spec);
- if (allocs > no_sched)
- allocs = no_sched;
- }
- else if (no_sched > ERTS_ALC_DEFAULT_MAX_THR_PREF)
- allocs = ERTS_ALC_DEFAULT_MAX_THR_PREF;
- else
- allocs = no_sched;
- if (allocs <= 1)
- ip->thr_spec = 0;
- else {
- ip->thr_spec = (int) allocs;
- ip->thr_spec *= -1; /* thread preferred */
-
- /* If default ... */
-
- /* ... shrink main multi-block carrier size */
- if (ip->default_.mmbcs)
- ip->init.util.mmbcs /= ERTS_MIN(4, allocs);
- /* ... shrink largest multi-block carrier size */
- if (ip->default_.lmbcs)
- ip->init.util.lmbcs /= ERTS_MIN(2, allocs);
- /* ... shrink smallest multi-block carrier size */
- if (ip->default_.smbcs)
- ip->init.util.smbcs /= ERTS_MIN(4, allocs);
- /* ... and more than three allocators shrink
- max mseg multi-block carriers */
- if (ip->default_.mmmbc && allocs > 2) {
- ip->init.util.mmmbc /= ERTS_MIN(4, allocs - 1);
- if (ip->init.util.mmmbc < 3)
- ip->init.util.mmmbc = 3;
- }
+ ip->thr_spec = no_sched;
+ ip->thr_spec *= -1; /* thread preferred */
+
+ /* If default ... */
+
+ /* ... shrink main multi-block carrier size */
+ if (ip->default_.mmbcs)
+ ip->init.util.mmbcs /= ERTS_MIN(4, no_sched);
+ /* ... shrink largest multi-block carrier size */
+ if (ip->default_.lmbcs)
+ ip->init.util.lmbcs /= ERTS_MIN(2, no_sched);
+ /* ... shrink smallest multi-block carrier size */
+ if (ip->default_.smbcs)
+ ip->init.util.smbcs /= ERTS_MIN(4, no_sched);
+ /* ... and more than three allocators shrink
+ max mseg multi-block carriers */
+ if (ip->default_.mmmbc && no_sched > 2) {
+ ip->init.util.mmmbc /= ERTS_MIN(4, no_sched - 1);
+ if (ip->init.util.mmmbc < 3)
+ ip->init.util.mmmbc = 3;
}
}
}
@@ -442,7 +476,7 @@ adjust_tpref(struct au_init *ip, int no_sched)
static void handle_args(int *, char **, erts_alc_hndl_args_init_t *);
static void
-set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init);
+set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu);
static void
start_au_allocator(ErtsAlcType_t alctr_n,
@@ -456,8 +490,6 @@ refuse_af_strategy(struct au_init *init)
init->atype = GOODFIT;
}
-static void init_thr_ix(int static_ixs);
-
#ifdef HARD_DEBUG
static void hdbg_init(void);
#endif
@@ -466,7 +498,7 @@ void
erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
{
UWord extra_block_size = 0;
- int i;
+ int i, ncpu;
erts_alc_hndl_args_init_t init = {
0,
#if HAVE_ERTS_MSEG
@@ -474,17 +506,34 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
#endif
ERTS_DEFAULT_TRIM_THRESHOLD,
ERTS_DEFAULT_TOP_PAD,
- ERTS_DEFAULT_ALCU_INIT
+ ERTS_DEFAULT_ALCU_INIT,
};
+ size_t fix_type_sizes[ERTS_ALC_NO_FIXED_SIZES] = {0};
+ fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_PROC)]
+ = sizeof(Process);
+#if !HALFWORD_HEAP
+ fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MONITOR_SH)]
+ = ERTS_MONITOR_SH_SIZE;
+ fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NLINK_SH)]
+ = ERTS_LINK_SH_SIZE;
+#endif
+ fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_EV_D_STATE)]
+ = sizeof(ErtsDrvEventDataState);
+ fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)]
+ = sizeof(ErtsDrvSelectDataState);
+ fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MSG_REF)]
+ = sizeof(ErlMessage);
#ifdef HARD_DEBUG
hdbg_init();
#endif
erts_have_sbmbc_alloc = 0;
+ ncpu = eaiop->ncpu;
+ if (ncpu < 1)
+ ncpu = 1;
erts_sys_alloc_init();
- init_thr_ix(erts_no_schedulers);
erts_init_utils_mem();
set_default_sbmbc_alloc_opts(&init.sbmbc_alloc);
@@ -496,20 +545,23 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
set_default_binary_alloc_opts(&init.binary_alloc);
set_default_ets_alloc_opts(&init.ets_alloc);
set_default_driver_alloc_opts(&init.driver_alloc);
+ set_default_fix_alloc_opts(&init.fix_alloc,
+ fix_type_sizes);
if (argc && argv)
handle_args(argc, argv, &init);
- if (erts_no_schedulers <= 1) {
- init.sbmbc_alloc.thr_spec = 0;
- init.sl_alloc.thr_spec = 0;
- init.std_alloc.thr_spec = 0;
- init.ll_alloc.thr_spec = 0;
- init.eheap_alloc.thr_spec = 0;
- init.binary_alloc.thr_spec = 0;
- init.ets_alloc.thr_spec = 0;
- init.driver_alloc.thr_spec = 0;
- }
+#ifndef ERTS_SMP
+ init.sbmbc_alloc.thr_spec = 0;
+ init.sl_alloc.thr_spec = 0;
+ init.std_alloc.thr_spec = 0;
+ init.ll_alloc.thr_spec = 0;
+ init.eheap_alloc.thr_spec = 0;
+ init.binary_alloc.thr_spec = 0;
+ init.ets_alloc.thr_spec = 0;
+ init.driver_alloc.thr_spec = 0;
+ init.fix_alloc.thr_spec = 0;
+#endif
if (init.erts_alloc_config) {
/* Adjust flags that erts_alloc_config won't like */
@@ -522,6 +574,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
init.binary_alloc.thr_spec = 0;
init.ets_alloc.thr_spec = 0;
init.driver_alloc.thr_spec = 0;
+ init.fix_alloc.thr_spec = 0;
}
#ifdef ERTS_SMP
@@ -538,6 +591,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
adjust_tpref(&init.binary_alloc, erts_no_schedulers);
adjust_tpref(&init.ets_alloc, erts_no_schedulers);
adjust_tpref(&init.driver_alloc, erts_no_schedulers);
+ adjust_tpref(&init.fix_alloc, erts_no_schedulers);
#else
/* No thread specific if not smp */
@@ -556,6 +610,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
refuse_af_strategy(&init.binary_alloc);
refuse_af_strategy(&init.ets_alloc);
refuse_af_strategy(&init.driver_alloc);
+ refuse_af_strategy(&init.fix_alloc);
#ifdef ERTS_SMP
if (!init.temp_alloc.thr_spec)
@@ -564,6 +619,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
erts_mtrace_pre_init();
#if HAVE_ERTS_MSEG
+ init.mseg.nos = erts_no_schedulers;
erts_mseg_init(&init.mseg);
#endif
erts_alcu_init(&init.alloc_util);
@@ -583,20 +639,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
erts_allctrs_info[i].extra = NULL;
}
-#ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE
-#if !defined(PURIFY) && !defined(VALGRIND)
- erts_allctrs[ERTS_ALC_A_FIXED_SIZE].alloc = erts_fix_alloc;
- erts_allctrs[ERTS_ALC_A_FIXED_SIZE].realloc = erts_fix_realloc;
- erts_allctrs[ERTS_ALC_A_FIXED_SIZE].free = erts_fix_free;
- erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].enabled = 1;
-#else
- erts_allctrs[ERTS_ALC_A_FIXED_SIZE].alloc = erts_sys_alloc;
- erts_allctrs[ERTS_ALC_A_FIXED_SIZE].realloc = erts_sys_realloc;
- erts_allctrs[ERTS_ALC_A_FIXED_SIZE].free = erts_sys_free;
- erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].enabled = 0;
-#endif
-#endif
-
erts_allctrs[ERTS_ALC_A_SYSTEM].alloc = erts_sys_alloc;
erts_allctrs[ERTS_ALC_A_SYSTEM].realloc = erts_sys_realloc;
erts_allctrs[ERTS_ALC_A_SYSTEM].free = erts_sys_free;
@@ -621,20 +663,21 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
init.ll_low_alloc.init.util.force = 1;
init.ll_low_alloc.init.util.low_mem = 1;
- set_au_allocator(ERTS_ALC_A_SBMBC_LOW, &init.sbmbc_low_alloc);
- set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc);
- set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc);
+ set_au_allocator(ERTS_ALC_A_SBMBC_LOW, &init.sbmbc_low_alloc, ncpu);
+ set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc, ncpu);
+ set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc, ncpu);
#endif /* HALFWORD */
- set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc);
- set_au_allocator(ERTS_ALC_A_SBMBC, &init.sbmbc_alloc);
- set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc);
- set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc);
- set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc);
- set_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc);
- set_au_allocator(ERTS_ALC_A_BINARY, &init.binary_alloc);
- set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc);
- set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc);
+ set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, ncpu);
+ set_au_allocator(ERTS_ALC_A_SBMBC, &init.sbmbc_alloc, ncpu);
+ set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc, ncpu);
+ set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc, ncpu);
+ set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc, ncpu);
+ set_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc, ncpu);
+ set_au_allocator(ERTS_ALC_A_BINARY, &init.binary_alloc, ncpu);
+ set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc, ncpu);
+ set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu);
+ set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu);
for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
if (!erts_allctrs[i].alloc)
@@ -650,10 +693,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
sys_alloc_opt(SYS_ALLOC_OPT_TRIM_THRESHOLD, init.trim_threshold);
sys_alloc_opt(SYS_ALLOC_OPT_TOP_PAD, init.top_pad);
- if (erts_allctrs_info[ERTS_FIX_CORE_ALLOCATOR].enabled)
- erts_fix_core_allocator_ix = ERTS_FIX_CORE_ALLOCATOR;
- else
- erts_fix_core_allocator_ix = ERTS_ALC_A_SYSTEM;
erts_mtrace_init(init.instr.mtrace, init.instr.nodename);
@@ -710,49 +749,40 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
&init.driver_alloc,
&driver_alloc_state);
- fix_core_allocator = erts_allctrs[erts_fix_core_allocator_ix].alloc;
- fix_core_extra = erts_allctrs[erts_fix_core_allocator_ix].extra;
+ start_au_allocator(ERTS_ALC_A_FIXED_SIZE,
+ &init.fix_alloc,
+ &fix_alloc_state);
erts_mtrace_install_wrapper_functions();
extra_block_size += erts_instr_init(init.instr.stat, init.instr.map);
+#if !HALFWORD_HEAP
+ init_aireq_alloc();
+#endif
+
#ifdef DEBUG
extra_block_size += install_debug_functions();
#endif
-#ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE
-
- erts_init_fix_alloc(extra_block_size, fix_core_alloc);
-
+}
-#if !defined(PURIFY) && !defined(VALGRIND)
- erts_set_fix_size(ERTS_ALC_T_PROC, sizeof(Process));
- erts_set_fix_size(ERTS_ALC_T_DB_TABLE, sizeof(DbTable));
- erts_set_fix_size(ERTS_ALC_T_ATOM, sizeof(Atom));
+void
+erts_alloc_late_init(void)
+{
- erts_set_fix_size(ERTS_ALC_T_MODULE, sizeof(Module));
- erts_set_fix_size(ERTS_ALC_T_REG_PROC, sizeof(RegProc));
- erts_set_fix_size(ERTS_ALC_T_FUN_ENTRY, sizeof(ErlFunEntry));
-#ifdef ERTS_ALC_T_DRV_EV_D_STATE
- erts_set_fix_size(ERTS_ALC_T_DRV_EV_D_STATE,
- sizeof(ErtsDrvEventDataState));
-#endif
-#ifdef ERTS_ALC_T_DRV_SEL_D_STATE
- erts_set_fix_size(ERTS_ALC_T_DRV_SEL_D_STATE,
- sizeof(ErtsDrvSelectDataState));
-#endif
-#if !HALFWORD_HEAP
- erts_set_fix_size(ERTS_ALC_T_EXPORT, sizeof(Export));
- erts_set_fix_size(ERTS_ALC_T_MONITOR_SH, ERTS_MONITOR_SH_SIZE*sizeof(Uint));
- erts_set_fix_size(ERTS_ALC_T_NLINK_SH, ERTS_LINK_SH_SIZE*sizeof(Uint));
-#endif
-#endif
-#endif
+}
+static void *
+erts_realloc_fixed_size(ErtsAlcType_t type, void *extra, void *p, Uint size)
+{
+ erl_exit(ERTS_ABORT_EXIT,
+ "Attempt to reallocate a block of the fixed size type %s\n",
+ ERTS_ALC_T2TD(type));
}
+
static void
-set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init)
+set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu)
{
ErtsAllocatorFunctions_t *af = &erts_allctrs[alctr_n];
ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n];
@@ -764,6 +794,12 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init)
if (init->init.util.force)
init->enable = 1;
+ tspec->enabled = 0;
+ tspec->dd = 0;
+ tspec->aix = alctr_n;
+ tspec->size = 0;
+ ai->thr_spec = 0;
+
if (!init->enable) {
af->alloc = erts_sys_alloc;
af->realloc = erts_sys_realloc;
@@ -775,14 +811,14 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init)
return;
}
- tspec->enabled = 0;
- tspec->all_thr_safe = 0;
- ai->thr_spec = 0;
#ifdef USE_THREADS
+#ifdef ERTS_SMP
if (init->thr_spec) {
if (init->thr_spec > 0) {
af->alloc = erts_alcu_alloc_thr_spec;
- if (init->init.util.ramv)
+ if (init->init.util.fix_type_size)
+ af->realloc = erts_realloc_fixed_size;
+ else if (init->init.util.ramv)
af->realloc = erts_alcu_realloc_mv_thr_spec;
else
af->realloc = erts_alcu_realloc_thr_spec;
@@ -790,12 +826,14 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init)
}
else {
af->alloc = erts_alcu_alloc_thr_pref;
- if (init->init.util.ramv)
+ if (init->init.util.fix_type_size)
+ af->realloc = erts_realloc_fixed_size;
+ else if (init->init.util.ramv)
af->realloc = erts_alcu_realloc_mv_thr_pref;
else
af->realloc = erts_alcu_realloc_thr_pref;
af->free = erts_alcu_free_thr_pref;
- tspec->all_thr_safe = 1;
+ tspec->dd = 1;
}
tspec->enabled = 1;
@@ -803,9 +841,13 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init)
ai->thr_spec = tspec->size;
}
- else if (init->init.util.ts) {
+ else
+#endif
+ if (init->init.util.ts) {
af->alloc = erts_alcu_alloc_ts;
- if (init->init.util.ramv)
+ if (init->init.util.fix_type_size)
+ af->realloc = erts_realloc_fixed_size;
+ else if (init->init.util.ramv)
af->realloc = erts_alcu_realloc_mv_ts;
else
af->realloc = erts_alcu_realloc_ts;
@@ -815,7 +857,9 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init)
#endif
{
af->alloc = erts_alcu_alloc;
- if (init->init.util.ramv)
+ if (init->init.util.fix_type_size)
+ af->realloc = erts_realloc_fixed_size;
+ else if (init->init.util.ramv)
af->realloc = erts_alcu_realloc_mv;
else
af->realloc = erts_alcu_realloc;
@@ -838,12 +882,14 @@ start_au_allocator(ErtsAlcType_t alctr_n,
ErtsAllocatorFunctions_t *af = &erts_allctrs[alctr_n];
ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n];
ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n];
+ ErtsAlcFixList_t *fix_lists = NULL;
+ size_t fix_list_size = 0;
if (!init->enable)
return;
if (init->thr_spec) {
- void *states = erts_sys_alloc(0,
+ char *states = erts_sys_alloc(0,
NULL,
((sizeof(Allctr_t *)
* (tspec->size + 1))
@@ -855,18 +901,40 @@ start_au_allocator(ErtsAlcType_t alctr_n,
"Failed to allocate allocator states for %salloc\n",
init->init.util.name_prefix);
tspec->allctr = (Allctr_t **) states;
- states = ((char *) states) + sizeof(Allctr_t *) * (tspec->size + 1);
+ states += sizeof(Allctr_t *) * (tspec->size + 1);
states = ((((UWord) states) & ERTS_CACHE_LINE_MASK)
- ? (void *) ((((UWord) states) & ~ERTS_CACHE_LINE_MASK)
+ ? (char *) ((((UWord) states) & ~ERTS_CACHE_LINE_MASK)
+ ERTS_CACHE_LINE_SIZE)
- : (void *) states);
- tspec->allctr[0] = init->thr_spec > 0 ? (Allctr_t *) state : (Allctr_t *) NULL;
+ : (char *) states);
+ tspec->allctr[0] = (Allctr_t *) state;
size = tspec->size;
for (i = 1; i < size; i++)
tspec->allctr[i] = (Allctr_t *)
&((ErtsAllocatorState_t *) states)[i-1];
}
+ if (init->init.util.fix_type_size) {
+ size_t tot_fix_list_size;
+ fix_list_size = sizeof(ErtsAlcFixList_t)*ERTS_ALC_NO_FIXED_SIZES;
+ fix_list_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(fix_list_size);
+ tot_fix_list_size = fix_list_size;
+ if (init->thr_spec)
+ tot_fix_list_size *= tspec->size;
+ fix_lists = erts_sys_alloc(0,
+ NULL,
+ (tot_fix_list_size
+ + ERTS_CACHE_LINE_SIZE - 1));
+ if (!fix_lists)
+ erl_exit(ERTS_ABORT_EXIT,
+ "Failed to allocate fix lists for %salloc\n",
+ init->init.util.name_prefix);
+
+ if (((UWord) fix_lists) & ERTS_CACHE_LINE_MASK)
+ fix_lists = ((ErtsAlcFixList_t *)
+ ((((UWord) fix_lists) & ~ERTS_CACHE_LINE_MASK)
+ + ERTS_CACHE_LINE_SIZE));
+ }
+
for (i = 0; i < size; i++) {
void *as;
atype = init->atype;
@@ -877,25 +945,32 @@ start_au_allocator(ErtsAlcType_t alctr_n,
as0 = (void *) tspec->allctr[i];
if (!as0)
continue;
- if (i == 0) {
- if (atype == AFIT)
- atype = GOODFIT;
- init->init.util.ts = 1;
+ if (init->thr_spec < 0) {
+ init->init.util.ts = i == 0;
+ init->init.util.tspec = 0;
+ init->init.util.tpref = -1*init->thr_spec + 1;
}
else {
- if (init->thr_spec < 0) {
+ if (i != 0)
+ init->init.util.ts = 0;
+ else {
+ if (atype == AFIT)
+ atype = GOODFIT;
init->init.util.ts = 1;
- init->init.util.tspec = 0;
- init->init.util.tpref = -1*init->thr_spec;
}
- else {
- init->init.util.ts = 0;
- init->init.util.tspec = init->thr_spec + 1;
- init->init.util.tpref = 0;
- }
- }
+ init->init.util.tspec = init->thr_spec + 1;
+ init->init.util.tpref = 0;
+ }
+ }
+
+ if (fix_lists) {
+ init->init.util.fix = fix_lists;
+ fix_lists = ((ErtsAlcFixList_t *)
+ (((char *) fix_lists) + fix_list_size));
}
+ init->init.util.ix = i;
+
switch (atype) {
case GOODFIT:
as = (void *) erts_gfalc_start((GFAllctr_t *) as0,
@@ -931,11 +1006,8 @@ start_au_allocator(ErtsAlcType_t alctr_n,
af->extra = as;
}
- if (init->thr_spec) {
+ if (init->thr_spec)
af->extra = tspec;
- init->init.util.ts = 1;
- }
-
ai->extra = af->extra;
}
@@ -1055,34 +1127,6 @@ get_amount_value(char *param_end, char** argv, int* ip)
return (Uint) tmp;
}
-static int
-get_bool_or_possitive_amount_value(int *bool, Uint *amount,
- char *param_end, char** argv, int* ip)
-{
- char *param = argv[*ip]+1;
- char *value = get_value(param_end, argv, ip);
- if (strcmp(value, "true") == 0) {
- *bool = 1;
- return 1;
- }
- else if (strcmp(value, "false") == 0) {
- *bool = 0;
- return 1;
- }
- else {
- Sint tmp;
- char *rest;
- errno = 0;
- tmp = (Sint) strtol(value, &rest, 10);
- if (errno != 0 || rest == value || tmp <= 0) {
- bad_value(param, param_end, value);
- return -1;
- }
- *amount = (Uint) tmp;
- return 0;
- }
-}
-
static void
handle_au_arg(struct au_init *auip,
char* sub_param,
@@ -1197,25 +1241,16 @@ handle_au_arg(struct au_init *auip,
goto bad_switch;
break;
case 't': {
- Uint no;
- int enable;
- int res = get_bool_or_possitive_amount_value(&enable,
- &no,
- sub_param+1,
- argv,
- ip);
- if (res > 0)
- auip->thr_spec = enable ? 1 : 0;
+ int res = get_bool_value(sub_param+1, argv, ip);
+ if (res > 0) {
+ auip->thr_spec = 1;
+ break;
+ }
else if (res == 0) {
- int allocs = (int) no;
- if (allocs < 0)
- allocs = INT_MIN;
- else {
- allocs *= -1;
- }
- auip->thr_spec = allocs;
+ auip->thr_spec = 0;
+ break;
}
- break;
+ goto bad_switch;
}
default:
bad_switch:
@@ -1234,6 +1269,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
&init->eheap_alloc,
&init->ll_alloc,
&init->driver_alloc,
+ &init->fix_alloc,
&init->sl_alloc,
&init->temp_alloc
};
@@ -1264,14 +1300,8 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
case 'E':
handle_au_arg(&init->ets_alloc, &argv[i][3], argv, &i);
break;
- case 'F': /* fix_alloc */
- if (has_prefix("e", param+2)) {
- arg = get_value(param+3, argv, &i);
- if (strcmp("true", arg) != 0)
- bad_value(param, param+3, arg);
- }
- else
- bad_param(param, param+2);
+ case 'F':
+ handle_au_arg(&init->fix_alloc, &argv[i][3], argv, &i);
break;
case 'H':
handle_au_arg(&init->eheap_alloc, &argv[i][3], argv, &i);
@@ -1298,12 +1328,6 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
#endif
get_amount_value(argv[i]+6, argv, &i);
}
- else if (has_prefix("cci", argv[i]+3)) {
-#if HAVE_ERTS_MSEG
- init->mseg.cci =
-#endif
- get_amount_value(argv[i]+6, argv, &i);
- }
else {
bad_param(param, param+2);
}
@@ -1389,6 +1413,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
set_default_binary_alloc_opts(&init->binary_alloc);
set_default_ets_alloc_opts(&init->ets_alloc);
set_default_driver_alloc_opts(&init->driver_alloc);
+ set_default_driver_alloc_opts(&init->fix_alloc);
init->driver_alloc.enable = 0;
if (strcmp("r9c", arg) == 0) {
@@ -1523,43 +1548,74 @@ static char *type_no_str(ErtsAlcType_t n)
#define type_str(T) type_no_str(ERTS_ALC_T2N((T)))
-erts_tsd_key_t thr_ix_key;
-erts_spinlock_t alloc_thr_ix_lock;
-int last_thr_ix;
-int first_dyn_thr_ix;
-
-static void
-init_thr_ix(int static_ixs)
+void
+erts_alloc_register_scheduler(void *vesdp)
{
- erts_tsd_key_create(&thr_ix_key);
- erts_spinlock_init(&alloc_thr_ix_lock, "alloc_thr_ix_lock");
- last_thr_ix = -4711;
- first_dyn_thr_ix = static_ixs+1;
+ ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
+ int ix = (int) esdp->no;
+ int aix;
+
+ for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) {
+ ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix];
+ esdp->alloc_data.deallctr[aix] = NULL;
+ esdp->alloc_data.pref_ix[aix] = -1;
+ if (tspec->enabled) {
+ if (!tspec->dd)
+ esdp->alloc_data.pref_ix[aix] = ix;
+ else {
+ Allctr_t *allctr = tspec->allctr[ix];
+ ASSERT(allctr);
+ esdp->alloc_data.deallctr[aix] = allctr;
+ esdp->alloc_data.pref_ix[aix] = ix;
+ }
+ }
+ }
}
-int
-erts_alc_get_thr_ix(void)
+void
+erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
+ int *need_thr_progress,
+ int *more_work)
{
- int ix = (int)(long) erts_tsd_get(thr_ix_key);
- if (ix == 0) {
- erts_spin_lock(&alloc_thr_ix_lock);
- last_thr_ix++;
- if (last_thr_ix < 0)
- last_thr_ix = first_dyn_thr_ix;
- ix = last_thr_ix;
- erts_spin_unlock(&alloc_thr_ix_lock);
- erts_tsd_set(thr_ix_key, (void *)(long) ix);
+ ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
+ int aix;
+ for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) {
+ Allctr_t *allctr;
+ if (esdp)
+ allctr = esdp->alloc_data.deallctr[aix];
+ else {
+ ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix];
+ if (tspec->enabled && tspec->dd)
+ allctr = tspec->allctr[0];
+ else
+ allctr = NULL;
+ }
+ if (allctr) {
+ erts_alcu_check_delayed_dealloc(allctr,
+ 1,
+ need_thr_progress,
+ more_work);
+ }
}
- ASSERT(ix > 0);
- return ix;
}
-void erts_alloc_reg_scheduler_id(Uint id)
+erts_aint32_t
+erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs)
{
- int ix = (int) id;
- ASSERT(0 < ix && ix <= first_dyn_thr_ix);
- ASSERT(0 == (int) (long) erts_tsd_get(thr_ix_key));
- erts_tsd_set(thr_ix_key, (void *)(long) ix);
+#ifdef ERTS_SMP
+ ErtsAllocatorThrSpec_t *tspec;
+ tspec = &erts_allctr_thr_spec[ERTS_ALC_A_FIXED_SIZE];
+ if (erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].thr_spec && tspec->enabled)
+ return erts_alcu_fix_alloc_shrink(tspec->allctr[ix], flgs);
+ if (ix == 0 && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra)
+ return erts_alcu_fix_alloc_shrink(
+ erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra, flgs);
+#else
+ if (ix == 1 && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra)
+ return erts_alcu_fix_alloc_shrink(
+ erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra, flgs);
+#endif
+ return 0;
}
static void
@@ -1574,14 +1630,12 @@ erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr)
if (erts_allctrs_info[ERTS_ALC_A_TEMPORARY].alloc_util
&& erts_allctrs_info[ERTS_ALC_A_TEMPORARY].thr_spec) {
ErtsAllocatorThrSpec_t *tspec;
+ int ix = ERTS_ALC_GET_THR_IX();
tspec = &erts_allctr_thr_spec[ERTS_ALC_A_TEMPORARY];
- if (!tspec->all_thr_safe) {
- int ix = erts_alc_get_thr_ix();
- if (ix < tspec->size) {
- *allctr = tspec->allctr[ix];
- return erts_alcu_verify_unused;
- }
+ if (ix < tspec->size) {
+ *allctr = tspec->allctr[ix];
+ return erts_alcu_verify_unused;
}
}
@@ -1680,7 +1734,7 @@ erts_realloc_n_enomem(ErtsAlcType_t n, void *ptr, Uint size)
}
static ERTS_INLINE UWord
-alcu_size(ErtsAlcType_t ai)
+alcu_size(ErtsAlcType_t ai, ErtsAlcUFixInfo_t *fi, int fisz)
{
UWord res = 0;
@@ -1690,22 +1744,20 @@ alcu_size(ErtsAlcType_t ai)
if (!erts_allctrs_info[ai].thr_spec) {
Allctr_t *allctr = erts_allctrs_info[ai].extra;
AllctrSize_t asize;
- erts_alcu_current_size(allctr, &asize);
+ erts_alcu_current_size(allctr, &asize, fi, fisz);
res += asize.blocks;
}
else {
ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ai];
int i;
- ASSERT(tspec->all_thr_safe);
-
ASSERT(tspec->enabled);
for (i = tspec->size - 1; i >= 0; i--) {
Allctr_t *allctr = tspec->allctr[i];
AllctrSize_t asize;
if (allctr) {
- erts_alcu_current_size(allctr, &asize);
+ erts_alcu_current_size(allctr, &asize, fi, fisz);
res += asize.blocks;
}
}
@@ -1733,7 +1785,6 @@ alcu_is_low(ErtsAlcType_t ai)
int found_one = 0;
# endif
- ASSERT(tspec->all_thr_safe);
ASSERT(tspec->enabled);
for (i = tspec->size - 1; i >= 0; i--) {
@@ -1757,11 +1808,24 @@ alcu_is_low(ErtsAlcType_t ai)
}
#endif /* HALFWORD */
+static ERTS_INLINE void
+add_fix_values(UWord *ap, UWord *up, ErtsAlcUFixInfo_t *fi, ErtsAlcType_t type)
+{
+ int ix = ERTS_ALC_T2N(type) - ERTS_ALC_N_MIN_A_FIXED_SIZE;
+ ASSERT(0 <= ix && ix < ERTS_ALC_NO_FIXED_SIZES);
+
+ *ap += (UWord) fi[ix].allocated;
+ *up += (UWord) fi[ix].used;
+}
+
Eterm
erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
{
+/*
+ * NOTE! When updating this function, make sure to also update
+ * erlang:memory/[0,1] in $ERL_TOP/erts/preloaded/src/erlang.erl
+ */
#define ERTS_MEM_NEED_ALL_ALCU (!erts_instr_stat && want_tot_or_sys)
- ErtsFixInfo efi;
struct {
int total;
int processes;
@@ -1800,6 +1864,11 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
Eterm res = THE_NON_VALUE;
ErtsAlcType_t ai;
int only_one_value = 0;
+ ErtsAlcUFixInfo_t fi[ERTS_ALC_NO_FIXED_SIZES] = {{0,0}};
+
+ ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)
+ || (ERTS_IS_CRASH_DUMPING
+ && erts_smp_is_system_blocked(ERTS_BS_FLG_ALLOW_GC)));
/* Figure out whats wanted... */
@@ -1969,7 +2038,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) {
switch (ai) {
case ERTS_ALC_A_SYSTEM:
- case ERTS_ALC_A_FIXED_SIZE:
case ERTS_ALC_A_SBMBC:
#if HALFWORD_HEAP
case ERTS_ALC_A_SBMBC_LOW:
@@ -2029,11 +2097,15 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
case ERTS_ALC_A_BINARY:
save = &size.binary;
break;
+ case ERTS_ALC_A_FIXED_SIZE:
+ asz = alcu_size(ai, fi, ERTS_ALC_NO_FIXED_SIZES);
+ size.total += asz;
+ continue;
default:
save = NULL;
break;
}
- asz = alcu_size(ai);
+ asz = alcu_size(ai, NULL, 0);
if (save)
*save = asz;
size.total += asz;
@@ -2053,8 +2125,11 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
if (ERTS_MEM_NEED_ALL_ALCU)
tmp = size.processes;
- else
- tmp = alcu_size(ERTS_ALC_A_EHEAP);
+ else {
+ alcu_size(ERTS_ALC_A_FIXED_SIZE,
+ fi, ERTS_ALC_NO_FIXED_SIZES);
+ tmp = alcu_size(ERTS_ALC_A_EHEAP, NULL, 0);
+ }
tmp += erts_max_processes*sizeof(Process*);
#ifdef HYBRID
tmp += erts_max_processes*sizeof(Process*);
@@ -2064,69 +2139,54 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
size.processes = size.processes_used = tmp;
-#if HALFWORD_HEAP
- /* BUG: We ignore link and monitor memory */
-#else
- erts_fix_info(ERTS_ALC_T_NLINK_SH, &efi);
- size.processes += efi.total;
- size.processes_used += efi.used;
+ add_fix_values(&size.processes,
+ &size.processes_used,
+ fi,
+ ERTS_ALC_T_PROC);
+#if !HALFWORD_HEAP
+ add_fix_values(&size.processes,
+ &size.processes_used,
+ fi,
+ ERTS_ALC_T_MONITOR_SH);
- erts_fix_info(ERTS_ALC_T_MONITOR_SH, &efi);
- size.processes += efi.total;
- size.processes_used += efi.used;
+ add_fix_values(&size.processes,
+ &size.processes_used,
+ fi,
+ ERTS_ALC_T_NLINK_SH);
#endif
-
- erts_fix_info(ERTS_ALC_T_PROC, &efi);
- size.processes += efi.total;
- size.processes_used += efi.used;
-
- erts_fix_info(ERTS_ALC_T_REG_PROC, &efi);
- size.processes += efi.total;
- size.processes_used += efi.used;
-
+ add_fix_values(&size.processes,
+ &size.processes_used,
+ fi,
+ ERTS_ALC_T_MSG_REF);
}
if (want.atom || want.atom_used) {
Uint reserved_atom_space, atom_space;
erts_atom_get_text_space_sizes(&reserved_atom_space, &atom_space);
size.atom = size.atom_used = atom_table_sz();
- erts_fix_info(ERTS_ALC_T_ATOM, &efi);
- if (want.atom) {
+ if (want.atom)
size.atom += reserved_atom_space;
- size.atom += efi.total;
- }
- if (want.atom_used) {
+ if (want.atom_used)
size.atom_used += atom_space;
- size.atom_used += efi.used;
- }
}
if (!ERTS_MEM_NEED_ALL_ALCU && want.binary)
- size.binary = alcu_size(ERTS_ALC_A_BINARY);
+ size.binary = alcu_size(ERTS_ALC_A_BINARY, NULL, 0);
if (want.code) {
size.code = module_table_sz();
- erts_fix_info(ERTS_ALC_T_MODULE, &efi);
- size.code += efi.used;
size.code += export_table_sz();
-#if HALFWORD_HEAP
size.code += export_list_size() * sizeof(Export);
-#else
- erts_fix_info(ERTS_ALC_T_EXPORT, &efi);
- size.code += efi.used;
-#endif
size.code += erts_fun_table_sz();
- erts_fix_info(ERTS_ALC_T_FUN_ENTRY, &efi);
- size.code += efi.used;
size.code += allocated_modules*sizeof(Range);
size.code += erts_total_code_size;
}
if (want.ets) {
if (!ERTS_MEM_NEED_ALL_ALCU)
- size.ets = alcu_size(ERTS_ALC_A_ETS);
+ size.ets = alcu_size(ERTS_ALC_A_ETS, NULL, 0);
size.ets += erts_get_ets_misc_mem_size();
}
@@ -2199,13 +2259,10 @@ struct aa_values {
Eterm
erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc)
{
-#define MAX_AA_VALUES \
- (20 + (ERTS_ALC_N_MAX_A_FIXED_SIZE - ERTS_ALC_N_MIN_A_FIXED_SIZE + 1))
-
+#define MAX_AA_VALUES (23)
struct aa_values values[MAX_AA_VALUES];
Eterm res = THE_NON_VALUE;
int i, length;
- ErtsFixInfo efi;
Uint reserved_atom_space, atom_space;
if (proc) {
@@ -2270,6 +2327,11 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc)
i++;
values[i].arity = 2;
+ values[i].name = "export_list";
+ values[i].ui[0] = export_list_size() * sizeof(Export);
+ i++;
+
+ values[i].arity = 2;
values[i].name = "register_table";
values[i].ui[0] = process_reg_sz();
i++;
@@ -2314,22 +2376,15 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc)
values[i].ui[0] = erts_tot_link_lh_size();
i++;
- {
- Uint n;
-
- for (n = ERTS_ALC_N_MIN_A_FIXED_SIZE;
- n <= ERTS_ALC_N_MAX_A_FIXED_SIZE;
- n++) {
- erts_fix_info(ERTS_ALC_N2T(n), &efi);
-
- values[i].arity = 3;
- values[i].name = ERTS_ALC_N2TD(n);
- values[i].ui[0] = efi.total;
- values[i].ui[1] = efi.used;
- i++;
- }
+ values[i].arity = 2;
+ values[i].name = "process_table";
+ values[i].ui[0] = erts_max_processes*sizeof(Process*);
+ i++;
- }
+ values[i].arity = 2;
+ values[i].name = "ets_misc";
+ values[i].ui[0] = erts_get_ets_misc_mem_size();
+ i++;
length = i;
ASSERT(length <= MAX_AA_VALUES);
@@ -2423,17 +2478,16 @@ erts_alloc_util_allocators(void *proc)
Uint sz;
int i;
/*
- * Currently all allocators except sys_alloc and fix_alloc are
+ * Currently all allocators except sys_alloc are
* alloc_util allocators.
*/
- sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 2)*2;
+ sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 1)*2;
ASSERT(sz > 0);
hp = HAlloc((Process *) proc, sz);
res = NIL;
for (i = ERTS_ALC_A_MAX; i >= ERTS_ALC_A_MIN; i--) {
switch (i) {
case ERTS_ALC_A_SYSTEM:
- case ERTS_ALC_A_FIXED_SIZE:
break;
default: {
char *alc_str = (char *) ERTS_ALC_A2AD(i);
@@ -2447,259 +2501,6 @@ erts_alloc_util_allocators(void *proc)
return res;
}
-Eterm
-erts_allocator_info_term(void *proc, Eterm which_alloc, int only_sz)
-{
-#define ERTS_AIT_RET(R) \
- do { res = (R); goto done; } while (0)
-#define ERTS_AIT_HALLOC(P, S) \
- do { hp = HAlloc((P), (S)); hp_end = hp + (S); } while (0)
-
- ErtsAlcType_t i;
- Uint sz = 0;
- Uint *hp = NULL;
- Uint *hp_end = NULL;
- Eterm res = am_undefined;
-
- if (is_not_atom(which_alloc))
- goto done;
-
- for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
- if (erts_is_atom_str((char *) ERTS_ALC_A2AD(i), which_alloc)) {
- if (!erts_allctrs_info[i].enabled)
- ERTS_AIT_RET(am_false);
- else {
- if (erts_allctrs_info[i].alloc_util) {
- Eterm ires, tmp;
- Eterm **hpp;
- Uint *szp;
- Eterm (*info_func)(Allctr_t *,
- int,
- int *,
- void *,
- Uint **,
- Uint *);
-
- info_func = (only_sz
- ? erts_alcu_sz_info
- : erts_alcu_info);
-
- if (erts_allctrs_info[i].thr_spec) {
- ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[i];
- int j;
- int block_system = !tspec->all_thr_safe;
-
- if (block_system) {
- erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
- erts_smp_block_system(0);
- }
- ASSERT(tspec->enabled);
-
- szp = &sz;
- hpp = NULL;
-
- while (1) {
- ires = NIL;
- for (j = tspec->size - 1; j >= 0; j--) {
- Allctr_t *allctr = tspec->allctr[j];
- if (allctr) {
- tmp = erts_bld_tuple(hpp,
- szp,
- 3,
- erts_bld_atom(hpp,
- szp,
- "instance"),
- make_small((Uint) j),
- (*info_func)(allctr,
- hpp != NULL,
- NULL,
- NULL,
- hpp,
- szp));
- ires = erts_bld_cons(hpp, szp, tmp, ires);
- }
- }
- if (hpp)
- break;
- ERTS_AIT_HALLOC((Process *) proc, sz);
- hpp = &hp;
- szp = NULL;
- }
-
- if (block_system) {
- erts_smp_release_system();
- erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
- }
- }
- else {
- Allctr_t *allctr = erts_allctrs_info[i].extra;
- szp = &sz;
- hpp = NULL;
- while (1) {
- ires = NIL;
- tmp = erts_bld_tuple(hpp,
- szp,
- 3,
- erts_bld_atom(hpp,
- szp,
- "instance"),
- make_small((Uint) 0),
- (*info_func)(allctr,
- hpp != NULL,
- NULL,
- NULL,
- hpp,
- szp));
- ires = erts_bld_cons(hpp, szp, tmp, ires);
- if (hpp)
- break;
- ERTS_AIT_HALLOC((Process *) proc, sz);
- hpp = &hp;
- szp = NULL;
- }
- }
- ERTS_AIT_RET(ires);
- }
- else {
- Eterm *szp, **hpp;
-
- switch (i) {
- case ERTS_ALC_A_SYSTEM: {
- SysAllocStat sas;
- Eterm opts_am;
- Eterm opts;
- Eterm as[4]; /* Ok even if !HEAP_ON_C_STACK, not really heap data on stack */
- Eterm ts[4]; /* Ok even if !HEAP_ON_C_STACK, not really heap data on stack */
- int l;
-
- if (only_sz)
- ERTS_AIT_RET(NIL);
-
- sys_alloc_stat(&sas);
- opts_am = am_atom_put("options", 7);
-
- szp = &sz;
- hpp = NULL;
-
- restart_sys_alloc:
- l = 0;
- as[l] = am_atom_put("e", 1);
- ts[l++] = am_true;
- as[l] = am_atom_put("m", 1);
- ts[l++] = am_atom_put("libc", 4);
- if(sas.trim_threshold >= 0) {
- as[l] = am_atom_put("tt", 2);
- ts[l++] = erts_bld_uint(hpp, szp,
- (Uint) sas.trim_threshold);
- }
- if(sas.top_pad >= 0) {
- as[l] = am_atom_put("tp", 2);
- ts[l++] = erts_bld_uint(hpp, szp, (Uint) sas.top_pad);
- }
-
- opts = erts_bld_2tup_list(hpp, szp, l, as, ts);
- res = erts_bld_2tup_list(hpp, szp, 1, &opts_am, &opts);
-
- if (szp) {
- ERTS_AIT_HALLOC((Process *) proc, sz);
- szp = NULL;
- hpp = &hp;
- goto restart_sys_alloc;
- }
- ERTS_AIT_RET(res);
- }
- case ERTS_ALC_A_FIXED_SIZE: {
- ErtsAlcType_t n;
- Eterm as[2], vs[2];
-
- if (only_sz)
- ERTS_AIT_RET(NIL);
-
- as[0] = am_atom_put("options", 7);
- as[1] = am_atom_put("pools", 5);
-
- szp = &sz;
- hpp = NULL;
-
- restart_fix_alloc:
-
- vs[0] = erts_bld_cons(hpp, szp,
- erts_bld_tuple(hpp, szp, 2,
- am_atom_put("e",
- 1),
- am_true),
- NIL);
-
- vs[1] = NIL;
- for (n = ERTS_ALC_N_MIN_A_FIXED_SIZE;
- n <= ERTS_ALC_N_MAX_A_FIXED_SIZE;
- n++) {
- ErtsFixInfo efi;
- erts_fix_info(ERTS_ALC_N2T(n), &efi);
-
- vs[1] = erts_bld_cons(
- hpp, szp,
- erts_bld_tuple(
- hpp, szp, 3,
- am_atom_put((char *) ERTS_ALC_N2TD(n),
- strlen(ERTS_ALC_N2TD(n))),
- erts_bld_uint(hpp, szp, efi.total),
- erts_bld_uint(hpp, szp, efi.used)),
- vs[1]);
-
- }
-
- res = erts_bld_2tup_list(hpp, szp, 2, as, vs);
- if (szp) {
- ERTS_AIT_HALLOC((Process *) proc, sz);
- szp = NULL;
- hpp = &hp;
- goto restart_fix_alloc;
- }
- ERTS_AIT_RET(res);
- }
- default:
- ASSERT(0);
- goto done;
- }
- }
- }
- }
- }
-
- if (ERTS_IS_ATOM_STR("mseg_alloc", which_alloc)) {
-#if HAVE_ERTS_MSEG
- if (only_sz)
- ERTS_AIT_RET(NIL);
- erts_mseg_info(NULL, NULL, 0, NULL, &sz);
- if (sz)
- ERTS_AIT_HALLOC((Process *) proc, sz);
- ERTS_AIT_RET(erts_mseg_info(NULL, NULL, 1, &hp, NULL));
-#else
- ERTS_AIT_RET(am_false);
-#endif
-
- }
- else if (ERTS_IS_ATOM_STR("alloc_util", which_alloc)) {
- if (only_sz)
- ERTS_AIT_RET(NIL);
- erts_alcu_au_info_options(NULL, NULL, NULL, &sz);
- if (sz)
- ERTS_AIT_HALLOC((Process *) proc, sz);
- ERTS_AIT_RET(erts_alcu_au_info_options(NULL, NULL, &hp, NULL));
- }
-
- done:
- if (hp) {
- ASSERT(hp_end >= hp);
- HRelease((Process *) proc, hp_end, hp);
- }
- return res;
-
-#undef ERTS_AIT_RET
-#undef ERTS_AIT_HALLOC
-}
-
void
erts_allocator_info(int to, void *arg)
{
@@ -2748,22 +2549,6 @@ erts_allocator_info(int to, void *arg)
erts_print(to, arg, "option tp: %d\n", sas.top_pad);
break;
}
- case ERTS_ALC_A_FIXED_SIZE: {
- ErtsAlcType_t n;
- erts_print(to, arg, "option e: true\n");
-
- for (n = ERTS_ALC_N_MIN_A_FIXED_SIZE;
- n <= ERTS_ALC_N_MAX_A_FIXED_SIZE;
- n++) {
- ErtsFixInfo efi;
- erts_fix_info(ERTS_ALC_N2T(n), &efi);
- erts_print(to, arg, "%s: %lu %lu\n",
- ERTS_ALC_N2TD(n),
- efi.total,
- efi.used);
- }
- break;
- }
default:
ASSERT(0);
break;
@@ -2774,8 +2559,18 @@ erts_allocator_info(int to, void *arg)
}
#if HAVE_ERTS_MSEG
- erts_print(to, arg, "=allocator:mseg_alloc\n");
- erts_mseg_info(&to, arg, 0, NULL, NULL);
+ {
+#ifdef ERTS_SMP
+ int max = (int) erts_no_schedulers;
+#else
+ int max = 0;
+#endif
+ int i;
+ for (i = 0; i <= max; i++) {
+ erts_print(to, arg, "=allocator:mseg_alloc[%d]\n", i);
+ erts_mseg_info(i, &to, arg, 0, NULL, NULL);
+ }
+ }
#endif
erts_print(to, arg, "=allocator:alloc_util\n");
@@ -2829,7 +2624,7 @@ erts_allocator_options(void *proc)
use_mseg++;
#endif
if (erts_allctr_thr_spec[a].enabled)
- allctr = erts_allctr_thr_spec[a].allctr[1];
+ allctr = erts_allctr_thr_spec[a].allctr[0];
else
allctr = erts_allctrs_info[a].extra;
tmp = erts_alcu_info_options(allctr, NULL, NULL, hpp, szp);
@@ -2878,7 +2673,7 @@ erts_allocator_options(void *proc)
#if HAVE_ERTS_MSEG
if (use_mseg) {
atoms[length] = am_atom_put("mseg_alloc", 10);
- terms[length++] = erts_mseg_info_options(NULL, NULL, hpp, szp);
+ terms[length++] = erts_mseg_info_options(0, NULL, NULL, hpp, szp);
}
#endif
@@ -2982,6 +2777,313 @@ void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size)
return (void*)v;
}
+static void
+reply_alloc_info(void *vair)
+{
+ ErtsAllocInfoReq *air = (ErtsAllocInfoReq *) vair;
+ Uint sched_id = erts_get_scheduler_id();
+ int global_instances = air->req_sched == sched_id;
+ ErtsProcLocks rp_locks;
+ Process *rp = air->proc;
+ Eterm ref_copy = NIL, ai_list, msg;
+ Eterm *hp = NULL, *hp_end = NULL, *hp_start = NULL;
+ Eterm **hpp;
+ Uint sz, *szp;
+ ErlOffHeap *ohp = NULL;
+ ErlHeapFragment *bp = NULL;
+ int i;
+ Eterm (*info_func)(Allctr_t *,
+ int,
+ int *,
+ void *,
+ Uint **,
+ Uint *) = (air->only_sz
+ ? erts_alcu_sz_info
+ : erts_alcu_info);
+
+ rp_locks = air->req_sched == sched_id ? ERTS_PROC_LOCK_MAIN : 0;
+
+ sz = 0;
+ hpp = NULL;
+ szp = &sz;
+
+ while (1) {
+
+ if (hpp)
+ ref_copy = STORE_NC(hpp, ohp, air->ref);
+ else
+ *szp += REF_THING_SIZE;
+
+ ai_list = NIL;
+ for (i = 0; air->allocs[i] != ERTS_ALC_A_INVALID; i++);
+ for (i--; i >= 0; i--) {
+ int ai = air->allocs[i];
+ Allctr_t *allctr;
+ Eterm ainfo;
+ Eterm alloc_atom;
+ if (global_instances) {
+ switch (ai) {
+ case ERTS_ALC_A_SYSTEM: {
+ alloc_atom = erts_bld_atom(hpp, szp, "sys_alloc");
+ ainfo = NIL;
+ if (!air->only_sz) {
+ SysAllocStat sas;
+ if (hpp)
+ sys_alloc_stat(&sas);
+ if (szp) {
+ /* ensure ehough heap */
+ sas.top_pad = INT_MAX;
+ sas.trim_threshold = INT_MAX;
+ }
+ if (sas.top_pad >= 0) {
+ ainfo = erts_bld_cons(
+ hpp, szp,
+ erts_bld_tuple(
+ hpp, szp, 2,
+ erts_bld_atom(hpp, szp, "tp"),
+ erts_bld_uint(
+ hpp, szp,
+ (Uint) sas.top_pad)),
+ ainfo);
+ }
+ if (sas.trim_threshold >= 0) {
+ ainfo = erts_bld_cons(
+ hpp, szp,
+ erts_bld_tuple(
+ hpp, szp, 2,
+ erts_bld_atom(hpp, szp, "tt"),
+ erts_bld_uint(
+ hpp, szp,
+ (Uint) sas.trim_threshold)),
+ ainfo);
+ }
+ ainfo = erts_bld_cons(hpp, szp,
+ erts_bld_tuple(
+ hpp, szp, 2,
+ erts_bld_atom(hpp, szp,
+ "m"),
+ erts_bld_atom(hpp, szp,
+ "libc")),
+ ainfo);
+ ainfo = erts_bld_cons(hpp, szp,
+ erts_bld_tuple(
+ hpp, szp, 2,
+ erts_bld_atom(hpp, szp,
+ "e"),
+ am_true),
+ ainfo);
+ ainfo = erts_bld_tuple(hpp, szp, 2,
+ erts_bld_atom(hpp, szp,
+ "otps"),
+ ainfo);
+ }
+ ainfo = erts_bld_tuple(hpp, szp, 3,
+ alloc_atom,
+ make_small(0),
+ ainfo);
+ break;
+ }
+ case ERTS_ALC_INFO_A_ALLOC_UTIL:
+ alloc_atom = erts_bld_atom(hpp, szp, "alloc_util");
+ ainfo = (air->only_sz
+ ? NIL
+ : erts_alcu_au_info_options(NULL, NULL,
+ hpp, szp));
+ ainfo = erts_bld_tuple(hpp, szp, 3,
+ alloc_atom,
+ make_small(0),
+ ainfo);
+ break;
+ case ERTS_ALC_INFO_A_MSEG_ALLOC:
+ alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc");
+#if HAVE_ERTS_MSEG
+ ainfo = (air->only_sz
+ ? NIL
+ : erts_mseg_info(0, NULL, NULL, hpp != NULL,
+ hpp, szp));
+ ainfo = erts_bld_tuple(hpp, szp, 3,
+ alloc_atom,
+ make_small(0),
+ ainfo);
+#else
+ ainfo = erts_bld_tuple(hpp, szp, 2, alloc_atom,
+ am_false);
+#endif
+ break;
+ default:
+ alloc_atom = erts_bld_atom(hpp, szp,
+ (char *) ERTS_ALC_A2AD(ai));
+ if (!erts_allctrs_info[ai].enabled)
+ ainfo = erts_bld_tuple(hpp, szp, 2, alloc_atom,
+ am_false);
+ else if (erts_allctrs_info[ai].alloc_util) {
+ if (erts_allctrs_info[ai].thr_spec)
+ allctr = erts_allctr_thr_spec[ai].allctr[0];
+ else
+ allctr = erts_allctrs_info[ai].extra;
+ ainfo = info_func(allctr, hpp != NULL, NULL,
+ NULL, hpp, szp);
+ ainfo = erts_bld_tuple(hpp, szp, 3, alloc_atom,
+ make_small(0), ainfo);
+ }
+ else {
+ erl_exit(ERTS_ABORT_EXIT, "%s:%d: internal error\n",
+ __FILE__, __LINE__);
+ }
+ }
+ ai_list = erts_bld_cons(hpp, szp,
+ ainfo, ai_list);
+ }
+ switch (ai) {
+ case ERTS_ALC_A_SYSTEM:
+ case ERTS_ALC_INFO_A_ALLOC_UTIL:
+ break;
+ case ERTS_ALC_INFO_A_MSEG_ALLOC:
+#if HAVE_ERTS_MSEG && defined(ERTS_SMP)
+ alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc");
+ ainfo = (air->only_sz
+ ? NIL
+ : erts_mseg_info(sched_id, NULL, NULL,
+ hpp != NULL, hpp, szp));
+ ainfo = erts_bld_tuple(hpp, szp, 3,
+ alloc_atom,
+ make_small(sched_id),
+ ainfo);
+ ai_list = erts_bld_cons(hpp, szp, ainfo, ai_list);
+#endif
+ break;
+ default:
+ if (erts_allctrs_info[ai].thr_spec) {
+ alloc_atom = erts_bld_atom(hpp, szp,
+ (char *) ERTS_ALC_A2AD(ai));
+ allctr = erts_allctr_thr_spec[ai].allctr[sched_id];
+ ainfo = info_func(allctr, hpp != NULL, NULL,
+ NULL, hpp, szp);
+ ai_list = erts_bld_cons(hpp, szp,
+ erts_bld_tuple(
+ hpp, szp,
+ 3,
+ alloc_atom,
+ make_small(sched_id),
+ ainfo),
+ ai_list);
+ }
+ break;
+ }
+ msg = erts_bld_tuple(hpp, szp,
+ 3,
+ ref_copy,
+ make_small(sched_id),
+ ai_list);
+
+ }
+ if (hpp)
+ break;
+
+ hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks);
+ hp_start = hp;
+ hp_end = hp + sz;
+ szp = NULL;
+ hpp = &hp;
+ }
+ if (bp)
+ bp = erts_resize_message_buffer(bp, hp - hp_start, &msg, 1);
+ else {
+ ASSERT(hp);
+ HRelease(rp, hp_end, hp);
+ }
+
+ erts_queue_message(rp, &rp_locks, bp, msg, NIL);
+
+ if (air->req_sched == sched_id)
+ rp_locks &= ~ERTS_PROC_LOCK_MAIN;
+
+ erts_smp_proc_unlock(rp, rp_locks);
+ erts_smp_proc_dec_refc(rp);
+
+ if (erts_smp_atomic32_dec_read_nob(&air->refc) == 0)
+ aireq_free(air);
+}
+
+int
+erts_request_alloc_info(struct process *c_p,
+ Eterm ref,
+ Eterm allocs,
+ int only_sz)
+{
+ ErtsAllocInfoReq *air = aireq_alloc();
+ Eterm req_ai[ERTS_ALC_A_MAX+1+2] = {0};
+ Eterm alist;
+ Eterm *hp;
+ int airix = 0, ai;
+
+ air->req_sched = erts_get_scheduler_id();
+
+ air->only_sz = only_sz;
+
+ air->proc = c_p;
+
+ if (is_not_internal_ref(ref))
+ return 0;
+
+ hp = &air->ref_heap[0];
+ air->ref = STORE_NC(&hp, NULL, ref);
+
+ if (is_not_list(allocs))
+ return 0;
+
+ alist = allocs;
+
+ while (is_list(alist)) {
+ int saved = 0;
+ Eterm* consp = list_val(alist);
+ Eterm alloc = CAR(consp);
+
+ for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++)
+ if (erts_is_atom_str((char *) erts_alc_a2ad[ai], alloc))
+ goto save_alloc;
+ if (erts_is_atom_str("mseg_alloc", alloc)) {
+ ai = ERTS_ALC_INFO_A_MSEG_ALLOC;
+ goto save_alloc;
+ }
+ if (erts_is_atom_str("alloc_util", alloc)) {
+ ai = ERTS_ALC_INFO_A_ALLOC_UTIL;
+ save_alloc:
+ if (req_ai[ai])
+ return 0;
+ air->allocs[airix++] = ai;
+ req_ai[ai] = 1;
+ saved = 1;
+ }
+
+ if (!saved)
+ return 0;
+
+ alist = CDR(consp);
+ }
+
+ if (is_not_nil(alist))
+ return 0;
+
+ air->allocs[airix] = ERTS_ALC_A_INVALID;
+
+ erts_smp_atomic32_init_nob(&air->refc,
+ (erts_aint32_t) erts_no_schedulers);
+
+ erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers);
+
+#ifdef ERTS_SMP
+ if (erts_no_schedulers > 1)
+ erts_smp_schedule_misc_aux_work(1,
+ erts_no_schedulers,
+ reply_alloc_info,
+ (void *) air);
+#endif
+
+ reply_alloc_info((void *) air);
+
+ return 1;
+}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
* Deprecated functions *