aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRickard Green <[email protected]>2013-05-17 09:36:44 +0200
committerRickard Green <[email protected]>2013-06-04 11:34:51 +0200
commit32a73196379a8c3e6083104e2d235f35b5bcbbbf (patch)
tree4b5e51280b8b6b68b341cce9ee794e4f409206a8
parent6aa140d2e9c0a810e057a140a3fdc252acf658d1 (diff)
downloadotp-32a73196379a8c3e6083104e2d235f35b5bcbbbf.tar.gz
otp-32a73196379a8c3e6083104e2d235f35b5bcbbbf.tar.bz2
otp-32a73196379a8c3e6083104e2d235f35b5bcbbbf.zip
erts: Use carrier pool for migration of carriers
Diffstat
-rw-r--r--erts/emulator/beam/atom.names1
-rw-r--r--erts/emulator/beam/erl_alloc.c191
-rw-r--r--erts/emulator/beam/erl_alloc.h2
-rw-r--r--erts/emulator/beam/erl_alloc_util.c1542
-rw-r--r--erts/emulator/beam/erl_alloc_util.h42
-rw-r--r--erts/emulator/beam/erl_ao_firstfit_alloc.c6
-rwxr-xr-xerts/emulator/beam/erl_bif_info.c15
-rw-r--r--erts/emulator/beam/erl_process.c11
-rw-r--r--erts/emulator/beam/erl_process.h1
-rw-r--r--erts/etc/common/erlexec.c1
-rw-r--r--erts/preloaded/ebin/erlang.beambin93444 -> 93976 bytes
-rw-r--r--erts/preloaded/src/erlang.erl25
12 files changed, 1465 insertions, 372 deletions
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 9e12080732..7d86e486f1 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -323,6 +323,7 @@ atom maximum
atom max_tables max_processes
atom mbuf_size
atom memory
+atom memory_internal
atom memory_types
atom message
atom message_binary
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index e85d770daa..a547191d6d 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -71,6 +71,23 @@
#define AU_ALLOC_DEFAULT_ENABLE(X) (X)
#endif
+#define ERTS_ALC_DEFAULT_ENABLED_ACUL 60
+#define ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC 45
+#define ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC 85
+
+#define ERTS_ALC_DEFAULT_ACUL 0
+#define ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC 0
+#define ERTS_ALC_DEFAULT_ACUL_LL_ALLOC 0
+
+#ifndef ERTS_SMP
+# undef ERTS_ALC_DEFAULT_ACUL
+# define ERTS_ALC_DEFAULT_ACUL 0
+# undef ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC
+# define ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC 0
+# undef ERTS_ALC_DEFAULT_ACUL_LL_ALLOC
+# define ERTS_ALC_DEFAULT_ACUL_LL_ALLOC 0
+#endif
+
#ifdef DEBUG
static Uint install_debug_functions(void);
#if 0
@@ -118,6 +135,7 @@ static ErtsAllocatorState_t fix_alloc_state;
typedef struct {
erts_smp_atomic32_t refc;
int only_sz;
+ int internal;
Uint req_sched;
Process *proc;
Eterm ref;
@@ -160,6 +178,7 @@ enum allctr_type {
struct au_init {
int enable;
int thr_spec;
+ int carrier_migration_allowed;
enum allctr_type atype;
struct {
AllctrInit_t util;
@@ -213,7 +232,7 @@ typedef struct {
#endif
} erts_alc_hndl_args_init_t;
-#define ERTS_AU_INIT__ {0, 0, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}}
+#define ERTS_AU_INIT__ {0, 0, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}}
#define SET_DEFAULT_ALLOC_OPTS(IP) \
do { \
@@ -221,13 +240,26 @@ do { \
sys_memcpy((void *) (IP), (void *) &aui__, sizeof(struct au_init)); \
} while (0)
+static ERTS_INLINE void
+set_default_acul(struct au_init *ip, int acul)
+{
+ ip->thr_spec = 1;
+ ip->atype = AOFIRSTFIT;
+ ip->init.aoff.bf_within_carrier = 1;
+ ip->init.util.acul = acul;
+}
+
static void
set_default_sl_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
+#if ERTS_ALC_DEFAULT_ACUL
+ set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL);
+#else
ip->thr_spec = 1;
ip->atype = GOODFIT;
+#endif
ip->init.util.name_prefix = "sl_";
ip->init.util.mmmbc = 5;
ip->init.util.alloc_no = ERTS_ALC_A_SHORT_LIVED;
@@ -250,8 +282,12 @@ set_default_std_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
+#if ERTS_ALC_DEFAULT_ACUL
+ set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL);
+#else
ip->thr_spec = 1;
ip->atype = BESTFIT;
+#endif
ip->init.util.name_prefix = "std_";
ip->init.util.mmmbc = 5;
ip->init.util.alloc_no = ERTS_ALC_A_STANDARD;
@@ -268,9 +304,13 @@ set_default_ll_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
+#if ERTS_ALC_DEFAULT_ACUL_LL_ALLOC
+ set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL_LL_ALLOC);
+#else
ip->thr_spec = 0;
ip->atype = BESTFIT;
ip->init.bf.ao = 1;
+#endif
ip->init.util.ramv = 0;
ip->init.util.mmsbc = 0;
ip->init.util.mmmbc = 0;
@@ -295,6 +335,7 @@ set_default_temp_alloc_opts(struct au_init *ip)
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
ip->thr_spec = 1;
+ ip->carrier_migration_allowed = 0;
ip->atype = AFIT;
ip->init.util.name_prefix = "temp_";
ip->init.util.alloc_no = ERTS_ALC_A_TEMPORARY;
@@ -317,8 +358,12 @@ set_default_eheap_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
+#if ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC
+ set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC);
+#else
ip->thr_spec = 1;
ip->atype = GOODFIT;
+#endif
ip->init.util.mmmbc = 100;
ip->init.util.name_prefix = "eheap_";
ip->init.util.alloc_no = ERTS_ALC_A_EHEAP;
@@ -340,8 +385,12 @@ set_default_binary_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
+#if ERTS_ALC_DEFAULT_ACUL
+ set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL);
+#else
ip->thr_spec = 1;
ip->atype = BESTFIT;
+#endif
ip->init.util.mmmbc = 50;
ip->init.util.name_prefix = "binary_";
ip->init.util.alloc_no = ERTS_ALC_A_BINARY;
@@ -358,8 +407,12 @@ set_default_ets_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
+#if ERTS_ALC_DEFAULT_ACUL
+ set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL);
+#else
ip->thr_spec = 1;
ip->atype = BESTFIT;
+#endif
ip->init.util.mmmbc = 100;
ip->init.util.name_prefix = "ets_";
ip->init.util.alloc_no = ERTS_ALC_A_ETS;
@@ -376,8 +429,12 @@ set_default_driver_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
+#if ERTS_ALC_DEFAULT_ACUL
+ set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL);
+#else
ip->thr_spec = 1;
ip->atype = BESTFIT;
+#endif
ip->init.util.name_prefix = "driver_";
ip->init.util.alloc_no = ERTS_ALC_A_DRIVER;
#ifndef SMALL_MEMORY
@@ -394,8 +451,12 @@ set_default_fix_alloc_opts(struct au_init *ip,
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
+#if ERTS_ALC_DEFAULT_ACUL
+ set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL);
+#else
ip->thr_spec = 1;
ip->atype = BESTFIT;
+#endif
ip->init.bf.ao = 1;
ip->init.util.name_prefix = "fix_";
ip->init.util.fix_type_size = fix_type_sizes;
@@ -493,6 +554,39 @@ static void adjust_fix_alloc_sizes(UWord extra_block_size)
}
}
+static ERTS_INLINE int
+strategy_support_carrier_migration(struct au_init *auip)
+{
+ /*
+ * Currently only aoff and aoffcaobf support carrier
+ * migration, i.e, type AOFIRSTFIT.
+ */
+ return auip->atype == AOFIRSTFIT;
+}
+
+static ERTS_INLINE void
+check_disable_carrier_migration(struct au_init *auip)
+{
+ if (!strategy_support_carrier_migration(auip) || !auip->thr_spec)
+ auip->init.util.acul = 0;
+}
+
+static ERTS_INLINE void
+ensure_carrier_migration_support(struct au_init *auip)
+{
+ auip->thr_spec = 1; /* Need thread preferred */
+
+ /*
+ * If strategy cannot handle carrier migration,
+ * default to a strategy that can...
+ */
+ if (!strategy_support_carrier_migration(auip)) {
+ /* Default to aoffcaobf */
+ auip->atype = AOFIRSTFIT;
+ auip->init.aoff.bf_within_carrier = 1;
+ }
+}
+
void
erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
{
@@ -578,6 +672,16 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
init.fix_alloc.thr_spec = 0;
}
+ check_disable_carrier_migration(&init.sl_alloc);
+ check_disable_carrier_migration(&init.std_alloc);
+ check_disable_carrier_migration(&init.ll_alloc);
+ check_disable_carrier_migration(&init.eheap_alloc);
+ check_disable_carrier_migration(&init.binary_alloc);
+ check_disable_carrier_migration(&init.ets_alloc);
+ check_disable_carrier_migration(&init.driver_alloc);
+ check_disable_carrier_migration(&init.fix_alloc);
+
+
#ifdef ERTS_SMP
/* Only temp_alloc can use thread specific interface */
if (init.temp_alloc.thr_spec)
@@ -1107,17 +1211,59 @@ get_amount_value(char *param_end, char** argv, int* ip)
return (Uint) tmp;
}
+static Uint
+get_acul_value(struct au_init *auip, char *param_end, char** argv, int* ip)
+{
+ Sint tmp;
+ char *rest;
+ char *param = argv[*ip]+1;
+ char *value = get_value(param_end, argv, ip);
+ if (sys_strcmp(value, "de") == 0) {
+ switch (auip->init.util.alloc_no) {
+ case ERTS_ALC_A_LONG_LIVED:
+#if HALFWORD_HEAP
+ case ERTS_ALC_A_LONG_LIVED_LOW:
+#endif
+ return ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC;
+ case ERTS_ALC_A_EHEAP:
+ return ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC;
+ default:
+ return ERTS_ALC_DEFAULT_ENABLED_ACUL;
+ }
+ }
+ errno = 0;
+ tmp = (Sint) ErtsStrToSint(value, &rest, 10);
+ if (errno != 0 || rest == value || tmp < 0 || 100 < tmp)
+ bad_value(param, param_end, value);
+ return (Uint) tmp;
+}
+
static void
handle_au_arg(struct au_init *auip,
char* sub_param,
char** argv,
- int* ip)
+ int* ip,
+ int u_switch)
{
char *param = argv[*ip]+1;
switch (sub_param[0]) {
case 'a':
- if(has_prefix("asbcst", sub_param)) {
+ if (has_prefix("acul", sub_param)) {
+ if (!auip->carrier_migration_allowed) {
+ if (!u_switch)
+ goto bad_switch;
+ else {
+ /* ignore */
+ (void) get_acul_value(auip, sub_param + 4, argv, ip);
+ break;
+ }
+ }
+ ensure_carrier_migration_support(auip);
+
+ auip->init.util.acul = get_acul_value(auip, sub_param + 4, argv, ip);
+ }
+ else if(has_prefix("asbcst", sub_param)) {
auip->init.util.asbcst = get_kb_value(sub_param + 6, argv, ip);
}
else if(has_prefix("as", sub_param)) {
@@ -1147,6 +1293,7 @@ handle_au_arg(struct au_init *auip,
else {
bad_value(param, sub_param + 1, alg);
}
+ check_disable_carrier_migration(auip);
}
else
goto bad_switch;
@@ -1227,6 +1374,7 @@ handle_au_arg(struct au_init *auip,
}
else if (res == 0) {
auip->thr_spec = 0;
+ check_disable_carrier_migration(auip);
break;
}
goto bad_switch;
@@ -1267,22 +1415,22 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
case 'M':
switch (argv[i][2]) {
case 'B':
- handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i);
+ handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i, 0);
break;
case 'D':
- handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i);
+ handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i, 0);
break;
case 'E':
- handle_au_arg(&init->ets_alloc, &argv[i][3], argv, &i);
+ handle_au_arg(&init->ets_alloc, &argv[i][3], argv, &i, 0);
break;
case 'F':
- handle_au_arg(&init->fix_alloc, &argv[i][3], argv, &i);
+ handle_au_arg(&init->fix_alloc, &argv[i][3], argv, &i, 0);
break;
case 'H':
- handle_au_arg(&init->eheap_alloc, &argv[i][3], argv, &i);
+ handle_au_arg(&init->eheap_alloc, &argv[i][3], argv, &i, 0);
break;
case 'L':
- handle_au_arg(&init->ll_alloc, &argv[i][3], argv, &i);
+ handle_au_arg(&init->ll_alloc, &argv[i][3], argv, &i, 0);
break;
case 'M':
if (has_prefix("amcbf", argv[i]+3)) {
@@ -1308,13 +1456,13 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
}
break;
case 'R':
- handle_au_arg(&init->driver_alloc, &argv[i][3], argv, &i);
+ handle_au_arg(&init->driver_alloc, &argv[i][3], argv, &i, 0);
break;
case 'S':
- handle_au_arg(&init->sl_alloc, &argv[i][3], argv, &i);
+ handle_au_arg(&init->sl_alloc, &argv[i][3], argv, &i, 0);
break;
case 'T':
- handle_au_arg(&init->temp_alloc, &argv[i][3], argv, &i);
+ handle_au_arg(&init->temp_alloc, &argv[i][3], argv, &i, 0);
break;
case 'Y': { /* sys_alloc */
if (has_prefix("tt", param+2)) {
@@ -1400,6 +1548,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
for (a = 0; a < aui_sz; a++) {
aui[a]->thr_spec = 0;
+ check_disable_carrier_migration(aui[a]);
aui[a]->init.util.ramv = 0;
aui[a]->init.util.mmmbc = 10;
aui[a]->init.util.lmbcs = 5*1024*1024;
@@ -1464,7 +1613,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
argv[start + 1] = val;
i = start;
}
- handle_au_arg(aui[a], &argv[i][3], argv, &i);
+ handle_au_arg(aui[a], &argv[i][3], argv, &i, 1);
}
}
break;
@@ -2497,7 +2646,7 @@ erts_allocator_info(int to, void *arg)
as = erts_allctr_thr_spec[a].allctr[ai];
}
/* Binary alloc has its own thread safety... */
- erts_alcu_info(as, 0, &to, arg, NULL, NULL);
+ erts_alcu_info(as, 0, 0, &to, arg, NULL, NULL);
}
else {
switch (a) {
@@ -2759,6 +2908,7 @@ reply_alloc_info(void *vair)
int i;
Eterm (*info_func)(Allctr_t *,
int,
+ int,
int *,
void *,
Uint **,
@@ -2887,8 +3037,8 @@ reply_alloc_info(void *vair)
allctr = erts_allctr_thr_spec[ai].allctr[0];
else
allctr = erts_allctrs_info[ai].extra;
- ainfo = info_func(allctr, hpp != NULL, NULL,
- NULL, hpp, szp);
+ ainfo = info_func(allctr, air->internal, hpp != NULL,
+ NULL, NULL, hpp, szp);
ainfo = erts_bld_tuple(hpp, szp, 3, alloc_atom,
make_small(0), ainfo);
}
@@ -2923,7 +3073,7 @@ reply_alloc_info(void *vair)
alloc_atom = erts_bld_atom(hpp, szp,
(char *) ERTS_ALC_A2AD(ai));
allctr = erts_allctr_thr_spec[ai].allctr[sched_id];
- ainfo = info_func(allctr, hpp != NULL, NULL,
+ ainfo = info_func(allctr, air->internal, hpp != NULL, NULL,
NULL, hpp, szp);
ai_list = erts_bld_cons(hpp, szp,
erts_bld_tuple(
@@ -2979,7 +3129,8 @@ int
erts_request_alloc_info(struct process *c_p,
Eterm ref,
Eterm allocs,
- int only_sz)
+ int only_sz,
+ int internal)
{
ErtsAllocInfoReq *air = aireq_alloc();
Eterm req_ai[ERTS_ALC_A_MAX+1+2] = {0};
@@ -2991,6 +3142,8 @@ erts_request_alloc_info(struct process *c_p,
air->only_sz = only_sz;
+ air->internal = internal;
+
air->proc = c_p;
if (is_not_internal_ref(ref))
@@ -3191,7 +3344,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3)
int i = 0;
while (argv[i]) {
if (argv[i][0] == '-' && argv[i][1] == 't')
- handle_au_arg(&init, &argv[i][2], argv, &i);
+ handle_au_arg(&init, &argv[i][2], argv, &i, 0);
else
return (UWord) NULL;
i++;
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index 8c80cbb4cc..d9fdfc6f58 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -65,7 +65,7 @@ Eterm erts_allocator_options(void *proc);
struct process;
int erts_request_alloc_info(struct process *c_p, Eterm ref, Eterm allocs,
- int only_sz);
+ int only_sz, int internal);
#define ERTS_ALLOC_INIT_DEF_OPTS_INITER {0}
typedef struct {
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 142fcab981..a52298c759 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -71,6 +71,13 @@
#define ALLOC_ZERO_EQ_NULL 0
+#ifndef ERTS_MSEG_FLG_2POW
+# define ERTS_MSEG_FLG_2POW 0
+#endif
+#ifndef ERTS_MSEG_FLG_NONE
+# define ERTS_MSEG_FLG_NONE 0
+#endif
+
static int atoms_initialized = 0;
static int initialized = 0;
@@ -293,7 +300,7 @@ MBC after deallocating first block:
/* Carriers ... */
-#define ERTS_ALC_CPOOL_DEBUG
+/* #define ERTS_ALC_CPOOL_DEBUG */
#if defined(DEBUG) && !defined(ERTS_ALC_CPOOL_DEBUG)
# define ERTS_ALC_CPOOL_DEBUG
@@ -316,7 +323,56 @@ MBC after deallocating first block:
# define ERTS_ALC_CPOOL_ASSERT(A) ((void) 1)
#endif
+#ifdef ERTS_SMP
#define ERTS_ALC_IS_CPOOL_ENABLED(A) ((A)->cpool.util_limit)
+#else
+#define ERTS_ALC_IS_CPOOL_ENABLED(A) (0)
+#endif
+
+#ifdef ERTS_SMP
+
+#define ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON 1000
+#define ERTS_ALC_CPOOL_ALLOC_OP_INC 8
+#define ERTS_ALC_CPOOL_FREE_OP_DEC 10
+
+#define ERTS_ALC_CPOOL_ALLOC_OP(A) \
+do { \
+ if ((A)->cpool.disable_abandon < ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON) { \
+ (A)->cpool.disable_abandon += ERTS_ALC_CPOOL_ALLOC_OP_INC; \
+ if ((A)->cpool.disable_abandon > ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON) \
+ (A)->cpool.disable_abandon = ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON; \
+ } \
+} while (0)
+
+
+#if ERTS_ALC_CPOOL_ALLOC_OP_INC >= ERTS_ALC_CPOOL_FREE_OP_DEC
+# error "Implementation assume ERTS_ALC_CPOOL_ALLOC_OP_INC < ERTS_ALC_CPOOL_FREE_OP_DEC"
+#endif
+
+#define ERTS_ALC_CPOOL_REALLOC_OP(A) \
+do { \
+ if ((A)->cpool.disable_abandon) { \
+ (A)->cpool.disable_abandon -= (ERTS_ALC_CPOOL_FREE_OP_DEC \
+ - ERTS_ALC_CPOOL_ALLOC_OP_INC); \
+ if ((A)->cpool.disable_abandon < 0) \
+ (A)->cpool.disable_abandon = 0; \
+ } \
+} while (0)
+
+#define ERTS_ALC_CPOOL_FREE_OP(A) \
+do { \
+ if ((A)->cpool.disable_abandon) { \
+ (A)->cpool.disable_abandon -= ERTS_ALC_CPOOL_FREE_OP_DEC; \
+ if ((A)->cpool.disable_abandon < 0) \
+ (A)->cpool.disable_abandon = 0; \
+ } \
+} while (0)
+
+#else
+#define ERTS_ALC_CPOOL_ALLOC_OP(A)
+#define ERTS_ALC_CPOOL_REALLOC_OP(A)
+#define ERTS_ALC_CPOOL_FREE_OP(A)
+#endif
#define ERTS_CRR_ALCTR_FLG_IN_POOL (((erts_aint_t) 1) << 0)
#define ERTS_CRR_ALCTR_FLG_BUSY (((erts_aint_t) 1) << 1)
@@ -379,6 +435,7 @@ MBC after deallocating first block:
#define CFLG_FORCE_SYS_ALLOC (1 << 3)
#define CFLG_FORCE_SIZE (1 << 4)
#define CFLG_MAIN_CARRIER (1 << 5)
+#define CFLG_NO_CPOOL (1 << 6)
#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
static void check_blk_carrier(Allctr_t *, Block_t *);
@@ -493,6 +550,21 @@ do { \
DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
} while (0)
+#define STAT_MBC_CPOOL_FETCH(AP, CRR) \
+do { \
+ UWord csz__ = CARRIER_SZ((CRR)); \
+ if (IS_MSEG_CARRIER((CRR))) \
+ STAT_MSEG_MBC_ALLOC((AP), csz__); \
+ else \
+ STAT_SYS_ALLOC_MBC_ALLOC((AP), csz__); \
+ (AP)->mbcs.blocks.curr.no += (CRR)->cpool.blocks; \
+ if ((AP)->mbcs.blocks.max.no < (AP)->mbcs.blocks.curr.no) \
+ (AP)->mbcs.blocks.max.no = (AP)->mbcs.blocks.curr.no; \
+ (AP)->mbcs.blocks.curr.size += (CRR)->cpool.blocks_size; \
+ if ((AP)->mbcs.blocks.max.size < (AP)->mbcs.blocks.curr.size) \
+ (AP)->mbcs.blocks.max.size = (AP)->mbcs.blocks.curr.size; \
+} while (0)
+
#define STAT_MSEG_MBC_FREE(AP, CSZ) \
do { \
ASSERT((AP)->mbcs.curr.norm.mseg.no > 0); \
@@ -511,7 +583,32 @@ do { \
DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
} while (0)
-#define STAT_MBC_BLK_ALLOC(AP, BSZ) \
+#define STAT_MBC_CPOOL_INSERT(AP, CRR) \
+do { \
+ UWord csz__ = CARRIER_SZ((CRR)); \
+ if (IS_MSEG_CARRIER((CRR))) \
+ STAT_MSEG_MBC_FREE((AP), csz__); \
+ else \
+ STAT_SYS_ALLOC_MBC_FREE((AP), csz__); \
+ ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.no \
+ >= (CRR)->cpool.blocks); \
+ (AP)->mbcs.blocks.curr.no -= (CRR)->cpool.blocks; \
+ ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.size \
+ >= (CRR)->cpool.blocks_size); \
+ (AP)->mbcs.blocks.curr.size -= (CRR)->cpool.blocks_size; \
+} while (0)
+
+#ifdef ERTS_SMP
+#define STAT_MBC_BLK_ALLOC_CRR(CRR, BSZ) \
+do { \
+ (CRR)->cpool.blocks++; \
+ (CRR)->cpool.blocks_size += (BSZ); \
+} while (0)
+#else
+#define STAT_MBC_BLK_ALLOC_CRR(CRR, BSZ) ((void) (CRR)) /* Get rid of warning */
+#endif
+
+#define STAT_MBC_BLK_ALLOC(AP, CRR, BSZ, FLGS) \
do { \
CarriersStats_t *cstats__ = &(AP)->mbcs; \
cstats__->blocks.curr.no++; \
@@ -520,15 +617,54 @@ do { \
cstats__->blocks.curr.size += (BSZ); \
if (cstats__->blocks.max.size < cstats__->blocks.curr.size) \
cstats__->blocks.max.size = cstats__->blocks.curr.size; \
+ STAT_MBC_BLK_ALLOC_CRR((CRR), (BSZ)); \
} while (0)
-#define STAT_MBC_BLK_FREE(AP, BSZ) \
+static ERTS_INLINE int
+stat_cpool_mbc_blk_free(Allctr_t *allctr,
+ Carrier_t *crr,
+ Carrier_t **busy_pcrr_pp,
+ UWord blksz)
+{
+#ifdef ERTS_SMP
+
+ ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks > 0);
+ crr->cpool.blocks--;
+ ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks_size >= blksz);
+ crr->cpool.blocks_size -= blksz;
+
+ if (!busy_pcrr_pp || !*busy_pcrr_pp)
+ return 0;
+
+ ERTS_ALC_CPOOL_ASSERT(crr == *busy_pcrr_pp);
+
+#ifdef ERTS_ALC_CPOOL_DEBUG
+ ERTS_ALC_CPOOL_ASSERT(
+ erts_atomic_dec_read_nob(&allctr->cpool.stat.no_blocks) >= 0);
+ ERTS_ALC_CPOOL_ASSERT(
+ erts_atomic_add_read_nob(&allctr->cpool.stat.blocks_size,
+ -((erts_aint_t) blksz)) >= 0);
+#else
+ erts_atomic_dec_nob(&allctr->cpool.stat.no_blocks);
+ erts_atomic_add_nob(&allctr->cpool.stat.blocks_size,
+ -((erts_aint_t) blksz));
+#endif
+
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+#define STAT_MBC_BLK_FREE(AP, CRR, BPCRRPP, BSZ, FLGS) \
do { \
- CarriersStats_t *cstats__ = &(AP)->mbcs; \
- ASSERT(cstats__->blocks.curr.no > 0); \
- cstats__->blocks.curr.no--; \
- ASSERT(cstats__->blocks.curr.size >= (BSZ)); \
- cstats__->blocks.curr.size -= (BSZ); \
+ if (!stat_cpool_mbc_blk_free((AP), (CRR), (BPCRRPP), (BSZ))) { \
+ CarriersStats_t *cstats__ = &(AP)->mbcs; \
+ ASSERT(cstats__->blocks.curr.no > 0); \
+ cstats__->blocks.curr.no--; \
+ ASSERT(cstats__->blocks.curr.size >= (BSZ)); \
+ cstats__->blocks.curr.size -= (BSZ); \
+ } \
} while (0)
/* Debug stuff... */
@@ -579,12 +715,12 @@ do { \
#define ERTS_ALCU_DBG_CHK_THR_ACCESS(A)
#endif
-
static void make_name_atoms(Allctr_t *allctr);
static Block_t *create_carrier(Allctr_t *, Uint, UWord);
-static void destroy_carrier(Allctr_t *, Block_t *);
-static void mbc_free(Allctr_t *allctr, void *p);
+static void destroy_carrier(Allctr_t *, Block_t *, Carrier_t **);
+static void mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp);
+static void dealloc_block(Allctr_t *, void *, int);
/* internal data... */
@@ -776,8 +912,35 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr)
}
}
-static Block_t *create_carrier(Allctr_t *, Uint, UWord);
-static void destroy_carrier(Allctr_t *, Block_t *);
+#ifdef ERTS_SMP
+
+static ERTS_INLINE void
+clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr)
+{
+ if (crr) {
+ erts_aint_t max_size;
+ erts_aint_t new_val;
+
+ max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr);
+ erts_atomic_set_nob(&crr->cpool.max_size, max_size);
+
+ new_val = (((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL);
+
+#ifdef ERTS_ALC_CPOOL_DEBUG
+ {
+ erts_aint_t old_val = new_val|ERTS_CRR_ALCTR_FLG_BUSY;
+
+ ERTS_ALC_CPOOL_ASSERT(old_val
+ == erts_smp_atomic_xchg_relb(&crr->allctr,
+ new_val));
+ }
+#else
+ erts_smp_atomic_set_relb(&crr->allctr, new_val);
+#endif
+ }
+}
+
+#endif
#if 0
#define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) \
@@ -799,13 +962,154 @@ chk_fix_list(Allctr_t *allctr, ErtsAlcFixList_t *fix, int ix, int before)
#define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B)
#endif
-erts_aint32_t
-erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
+static void *mbc_alloc(Allctr_t *allctr, Uint size);
+
+#ifdef ERTS_SMP
+typedef struct {
+ ErtsAllctrDDBlock_t ddblock__; /* must be first */
+ ErtsAlcType_t fix_type;
+} ErtsAllctrFixDDBlock_t;
+#endif
+
+static ERTS_INLINE void
+dealloc_fix_block(Allctr_t *allctr,
+ ErtsAlcType_t type,
+ void *ptr,
+ int dec_cc_on_redirect)
+{
+#ifdef ERTS_SMP
+ /* May be redirected... */
+ ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type;
+#endif
+ dealloc_block(allctr, ptr, dec_cc_on_redirect);
+}
+
+static ERTS_INLINE void
+sched_fix_shrink(Allctr_t *allctr, int on)
+{
+ if (on && !allctr->fix_shrink_scheduled) {
+ allctr->fix_shrink_scheduled = 1;
+ erts_set_aux_work_timeout(allctr->ix,
+ (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
+ | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
+ 1);
+ }
+ else if (!on && allctr->fix_shrink_scheduled) {
+ allctr->fix_shrink_scheduled = 0;
+ erts_set_aux_work_timeout(allctr->ix,
+ (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
+ | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
+ 0);
+ }
+}
+
+static ERTS_INLINE void
+fix_cpool_check_shrink(Allctr_t *allctr,
+ ErtsAlcType_t type,
+ ErtsAlcFixList_t *fix,
+ Carrier_t **busy_pcrr_pp)
+{
+ if (fix->u.cpool.shrink_list > 0) {
+ if (fix->list_size == 0)
+ fix->u.cpool.shrink_list = 0;
+ else {
+ void *p;
+#ifdef ERTS_SMP
+ if (busy_pcrr_pp) {
+ clear_busy_pool_carrier(allctr, *busy_pcrr_pp);
+ *busy_pcrr_pp = NULL;
+ }
+#endif
+ fix->u.cpool.shrink_list--;
+ p = fix->list;
+ fix->list = *((void **) p);
+ fix->list_size--;
+ if (fix->u.cpool.min_list_size > fix->list_size)
+ fix->u.cpool.min_list_size = fix->list_size;
+
+ fix->u.cpool.allocated--;
+ dealloc_fix_block(allctr, type, p, 0);
+ }
+ }
+}
+
+static ERTS_INLINE void *
+fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size)
+{
+ void *res;
+ ErtsAlcFixList_t *fix;
+
+ ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type
+ && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE);
+
+ fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE];
+
+ res = fix->list;
+ if (res) {
+ fix->list = *((void **) res);
+ fix->list_size--;
+ if (fix->u.cpool.min_list_size > fix->list_size)
+ fix->u.cpool.min_list_size = fix->list_size;
+ fix->u.cpool.used++;
+ fix_cpool_check_shrink(allctr, type, fix, NULL);
+ return res;
+ }
+ if (size < 2*sizeof(UWord))
+ size += sizeof(UWord);
+ if (size >= allctr->sbc_threshold) {
+ Block_t *blk;
+ blk = create_carrier(allctr, size, CFLG_SBC);
+ res = blk ? BLK2UMEM(blk) : NULL;
+ }
+ else
+ res = mbc_alloc(allctr, size);
+ if (res) {
+ fix->u.cpool.used++;
+ fix->u.cpool.allocated++;
+ }
+ return res;
+}
+
+static ERTS_INLINE void
+fix_cpool_free(Allctr_t *allctr,
+ ErtsAlcType_t type,
+ void *p,
+ Carrier_t **busy_pcrr_pp)
+{
+ ErtsAlcFixList_t *fix;
+
+ ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type
+ && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE);
+
+ fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE];
+
+ fix->u.cpool.used--;
+
+ if ((!busy_pcrr_pp || !*busy_pcrr_pp)
+ && !fix->u.cpool.shrink_list
+ && fix->list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) {
+ *((void **) p) = fix->list;
+ fix->list = p;
+ fix->list_size++;
+ sched_fix_shrink(allctr, 1);
+ }
+ else {
+ Block_t *blk = UMEM2BLK(p);
+ if (IS_SBC_BLK(blk))
+ destroy_carrier(allctr, blk, NULL);
+ else
+ mbc_free(allctr, p, busy_pcrr_pp);
+ fix->u.cpool.allocated--;
+ fix_cpool_check_shrink(allctr, type, fix, busy_pcrr_pp);
+ }
+}
+
+static ERTS_INLINE erts_aint32_t
+fix_cpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
{
int all_empty = 1;
erts_aint32_t res = 0;
int ix, o;
- ErtsAlcFixList_t *fix = allctr->fix;
int flush = flgs == 0;
#ifdef USE_THREADS
@@ -814,56 +1118,204 @@ erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
#endif
for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) {
+ ErtsAlcFixList_t *fix = &allctr->fix[ix];
+ ErtsAlcType_t type;
ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
- if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) {
- fix[ix].limit = fix[ix].max_used;
- if (fix[ix].limit < fix[ix].used)
- fix[ix].limit = fix[ix].used;
- fix[ix].max_used = fix[ix].used;
- ASSERT(fix[ix].limit >= 0);
-
- }
- if (flush) {
- fix[ix].limit = 0;
- fix[ix].max_used = fix[ix].used;
- ASSERT(fix[ix].limit >= 0);
+ if (flush)
+ fix->u.cpool.shrink_list = fix->list_size;
+ else if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) {
+ fix->u.cpool.shrink_list = fix->u.cpool.min_list_size;
+ fix->u.cpool.min_list_size = fix->list_size;
}
+ type = (ErtsAlcType_t) (ix + ERTS_ALC_N_MIN_A_FIXED_SIZE);
for (o = 0; o < ERTS_ALC_FIX_MAX_SHRINK_OPS || flush; o++) {
- Block_t *blk;
void *ptr;
- if (!flush && fix[ix].limit >= fix[ix].allocated)
+ if (fix->u.cpool.shrink_list == 0)
break;
- if (fix[ix].list_size == 0)
+ if (fix->list_size == 0) {
+ fix->u.cpool.shrink_list = 0;
break;
- ptr = fix[ix].list;
- fix[ix].list = *((void **) ptr);
- fix[ix].list_size--;
+ }
+ ptr = fix->list;
+ fix->list = *((void **) ptr);
+ fix->list_size--;
+ fix->u.cpool.shrink_list--;
+ fix->u.cpool.allocated--;
+ dealloc_fix_block(allctr, type, ptr, 0);
+ }
+ if (fix->u.cpool.min_list_size > fix->list_size)
+ fix->u.cpool.min_list_size = fix->list_size;
+ if (fix->list_size != 0) {
+ if (fix->u.cpool.shrink_list > 0)
+ res |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC;
+ all_empty = 0;
+ }
+ }
+
+ if (all_empty)
+ sched_fix_shrink(allctr, 0);
+
+#ifdef USE_THREADS
+ if (allctr->thread_safe)
+ erts_mtx_unlock(&allctr->mutex);
+#endif
- blk = UMEM2BLK(ptr);
+ return res;
+}
+
+static ERTS_INLINE void *
+fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size)
+{
+ ErtsAlcFixList_t *fix;
+ void *res;
+
+ ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type
+ && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE);
+
+ fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE];
+ ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
+ fix->u.nocpool.used++;
+ res = fix->list;
+ if (res) {
+ fix->list_size--;
+ fix->list = *((void **) res);
+ if (fix->list && fix->u.nocpool.allocated > fix->u.nocpool.limit) {
+ Block_t *blk;
+ void *p = fix->list;
+ fix->list = *((void **) p);
+ fix->list_size--;
+ blk = UMEM2BLK(p);
if (IS_SBC_BLK(blk))
- destroy_carrier(allctr, blk);
+ destroy_carrier(allctr, blk, NULL);
else
- mbc_free(allctr, ptr);
+ mbc_free(allctr, p, NULL);
+ fix->u.nocpool.allocated--;
+ }
+ ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
+ return res;
+ }
+ if (size < 2*sizeof(UWord))
+ size += sizeof(UWord);
+ if (fix->u.nocpool.limit < fix->u.nocpool.used)
+ fix->u.nocpool.limit = fix->u.nocpool.used;
+ if (fix->u.nocpool.max_used < fix->u.nocpool.used)
+ fix->u.nocpool.max_used = fix->u.nocpool.used;
+ fix->u.nocpool.allocated++;
+
+ if (size >= allctr->sbc_threshold) {
+ Block_t *blk;
+ blk = create_carrier(allctr, size, CFLG_SBC);
+ res = blk ? BLK2UMEM(blk) : NULL;
+ }
+ else
+ res = mbc_alloc(allctr, size);
+
+ if (!res) {
+ fix->u.nocpool.allocated--;
+ fix->u.nocpool.used--;
+ }
+ return res;
+}
+
+static ERTS_INLINE void
+fix_nocpool_free(Allctr_t *allctr,
+ ErtsAlcType_t type,
+ void *p)
+{
+ Block_t *blk;
+ ErtsAlcFixList_t *fix;
+
+ ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type
+ && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE);
+
+ fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE];
+
+ ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
+ fix->u.nocpool.used--;
+ if (fix->u.nocpool.allocated < fix->u.nocpool.limit
+ && fix->list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) {
+ *((void **) p) = fix->list;
+ fix->list = p;
+ fix->list_size++;
+ sched_fix_shrink(allctr, 1);
+ ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
+ return;
+ }
+ fix->u.nocpool.allocated--;
+ if (fix->list && fix->u.nocpool.allocated > fix->u.nocpool.limit) {
+ blk = UMEM2BLK(p);
+ if (IS_SBC_BLK(blk))
+ destroy_carrier(allctr, blk, NULL);
+ else
+ mbc_free(allctr, p, NULL);
+ p = fix->list;
+ fix->list = *((void **) p);
+ fix->list_size--;
+ fix->u.nocpool.allocated--;
+ }
+
+ blk = UMEM2BLK(p);
+ if (IS_SBC_BLK(blk))
+ destroy_carrier(allctr, blk, NULL);
+ else
+ mbc_free(allctr, p, NULL);
+ ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
+}
- fix[ix].allocated--;
+static ERTS_INLINE erts_aint32_t
+fix_nocpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
+{
+ int all_empty = 1;
+ erts_aint32_t res = 0;
+ int ix, o;
+ int flush = flgs == 0;
+
+#ifdef USE_THREADS
+ if (allctr->thread_safe)
+ erts_mtx_lock(&allctr->mutex);
+#endif
+
+ for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) {
+ ErtsAlcFixList_t *fix = &allctr->fix[ix];
+ ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
+ if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) {
+ fix->u.nocpool.limit = fix->u.nocpool.max_used;
+ if (fix->u.nocpool.limit < fix->u.nocpool.used)
+ fix->u.nocpool.limit = fix->u.nocpool.used;
+ fix->u.nocpool.max_used = fix->u.nocpool.used;
+ ASSERT(fix->u.nocpool.limit >= 0);
+
+ }
+ if (flush) {
+ fix->u.nocpool.limit = 0;
+ fix->u.nocpool.max_used = fix->u.nocpool.used;
+ ASSERT(fix->u.nocpool.limit >= 0);
+ }
+ for (o = 0; o < ERTS_ALC_FIX_MAX_SHRINK_OPS || flush; o++) {
+ void *ptr;
+
+ if (!flush && fix->u.nocpool.limit >= fix->u.nocpool.allocated)
+ break;
+ if (fix->list_size == 0)
+ break;
+ ptr = fix->list;
+ fix->list = *((void **) ptr);
+ fix->list_size--;
+ dealloc_block(allctr, ptr, 0);
+ fix->u.nocpool.allocated--;
}
- if (fix[ix].list_size != 0) {
- if (fix[ix].limit < fix[ix].allocated)
+ if (fix->list_size != 0) {
+ if (fix->u.nocpool.limit < fix->u.nocpool.allocated)
res |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC;
all_empty = 0;
}
ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
}
- if (all_empty && allctr->fix_shrink_scheduled) {
- allctr->fix_shrink_scheduled = 0;
- erts_set_aux_work_timeout(allctr->ix,
- (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
- | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
- 0);
- }
+ if (all_empty)
+ sched_fix_shrink(allctr, 0);
#ifdef USE_THREADS
if (allctr->thread_safe)
@@ -873,13 +1325,18 @@ erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
return res;
}
-#ifdef ERTS_SMP
+erts_aint32_t
+erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
+{
+ if (ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ return fix_cpool_alloc_shrink(allctr, flgs);
+ else
+ return fix_nocpool_alloc_shrink(allctr, flgs);
+}
-typedef struct {
- ErtsAllctrDDBlock_t ddblock__; /* must be first */
- ErtsAlcType_t fix_type;
-}ErtsAllctrFixDDBlock_t;
+static void dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, Uint mseg_flags);
+#ifdef ERTS_SMP
static ERTS_INLINE Allctr_t*
get_pref_allctr(void *extra)
@@ -895,6 +1352,9 @@ get_pref_allctr(void *extra)
return tspec->allctr[pref_ix];
}
+#define ERTS_ALC_TS_PREF_LOCK_IF_USED (1)
+#define ERTS_ALC_TS_PREF_LOCK_NO (0)
+
/* SMP note:
* get_used_allctr() must be safe WITHOUT locking the allocator while
* concurrent threads may be updating adjacent blocks.
@@ -903,11 +1363,15 @@ get_pref_allctr(void *extra)
* the "PREV_FREE" flag bit.
*/
static ERTS_INLINE Allctr_t*
-get_used_allctr(Allctr_t *pref_allctr, void *extra, void *p, UWord *sizep)
+get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep,
+ Carrier_t **busy_pcrr_pp)
{
Block_t* blk = UMEM2BLK(p);
- Carrier_t* crr;
+ Carrier_t *crr;
erts_aint_t iallctr;
+ Allctr_t *used_allctr;
+
+ *busy_pcrr_pp = NULL;
if (IS_SBC_BLK(blk)) {
crr = BLK_TO_SBC(blk);
@@ -917,14 +1381,67 @@ get_used_allctr(Allctr_t *pref_allctr, void *extra, void *p, UWord *sizep)
}
else {
crr = ABLK_TO_MBC(blk);
+
if (sizep)
*sizep = MBC_ABLK_SZ(blk) - ABLK_HDR_SZ;
if (!ERTS_ALC_IS_CPOOL_ENABLED(pref_allctr))
iallctr = erts_smp_atomic_read_dirty(&crr->allctr);
- else
+ else {
+ int locked_pref_allctr = 0;
iallctr = erts_smp_atomic_read_ddrb(&crr->allctr);
+
+ if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock
+ && pref_allctr->thread_safe) {
+ used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK);
+ if (pref_allctr == used_allctr) {
+ erts_mtx_lock(&pref_allctr->mutex);
+ locked_pref_allctr = 1;
+ }
+ }
+
+ while ((iallctr & ((~FLG_MASK)|ERTS_CRR_ALCTR_FLG_IN_POOL))
+ == (((erts_aint_t) pref_allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL)) {
+ erts_aint_t act;
+
+ ERTS_ALC_CPOOL_ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY));
+ act = erts_smp_atomic_cmpxchg_ddrb(&crr->allctr,
+ iallctr|ERTS_CRR_ALCTR_FLG_BUSY,
+ iallctr);
+ if (act == iallctr) {
+ *busy_pcrr_pp = crr;
+ break;
+ }
+ iallctr = act;
+ }
+
+ used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK);
+
+ if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock) {
+ if (locked_pref_allctr && used_allctr != pref_allctr) {
+ /* Was taken out of pool; now owned by someone else */
+ erts_mtx_unlock(&pref_allctr->mutex);
+ }
+ }
+
+ ERTS_ALC_CPOOL_ASSERT(
+ (((iallctr & ~FLG_MASK) == (erts_aint_t) pref_allctr)
+ ? (((iallctr & FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL)
+ || ((iallctr & FLG_MASK) == 0))
+ : 1));
+
+ return used_allctr;
+ }
}
- return (Allctr_t *) (iallctr & ~FLG_MASK);
+
+ used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK);
+
+ if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock
+ && used_allctr == pref_allctr
+ && pref_allctr->thread_safe) {
+ erts_mtx_lock(&pref_allctr->mutex);
+ }
+
+ return used_allctr;
}
static void
@@ -1019,7 +1536,7 @@ check_insert_marker(ErtsAllctrDDQueue_t *ddq, erts_aint_t ilast)
}
static ERTS_INLINE int
-ddq_enqueue(ErtsAlcType_t type, ErtsAllctrDDQueue_t *ddq, void *ptr, int cinit)
+ddq_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr, int cinit)
{
int last_elem;
int um_refc_ix = 0;
@@ -1120,6 +1637,59 @@ store_earliest_thr_prgr(ErtsThrPrgrVal *prev_val, ErtsAllctrDDQueue_t *ddq)
}
}
+static void
+check_pending_dealloc_carrier(Allctr_t *allctr,
+ int *need_thr_progress,
+ ErtsThrPrgrVal *thr_prgr_p,
+ int *need_more_work);
+
+static void
+handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr)
+{
+ ErtsAlcType_t type;
+
+ type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type;
+
+ ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type
+ && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE);
+
+ if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ fix_nocpool_free(allctr, type, ptr);
+ else {
+ Block_t *blk = UMEM2BLK(ptr);
+ Carrier_t *busy_pcrr_p;
+ Allctr_t *used_allctr;
+
+ if (IS_SBC_BLK(blk)) {
+ busy_pcrr_p = NULL;
+ goto doit;
+ }
+
+ used_allctr = get_used_allctr(allctr, ERTS_ALC_TS_PREF_LOCK_NO, ptr,
+ NULL, &busy_pcrr_p);
+ if (used_allctr == allctr) {
+ doit:
+ fix_cpool_free(allctr, type, ptr, &busy_pcrr_p);
+ clear_busy_pool_carrier(allctr, busy_pcrr_p);
+ }
+ else {
+ /* Carrier migrated; need to redirect block to new owner... */
+ int cinit = used_allctr->dd.ix - allctr->dd.ix;
+
+ ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p);
+
+ DEC_CC(allctr->calls.this_free);
+
+ ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type;
+ if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit))
+ erts_alloc_notify_delayed_dealloc(used_allctr->ix);
+ }
+ }
+}
+
+static void
+schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr);
+
static ERTS_INLINE int
handle_delayed_dealloc(Allctr_t *allctr,
int allctr_locked,
@@ -1151,7 +1721,6 @@ handle_delayed_dealloc(Allctr_t *allctr,
while (1) {
Block_t *blk;
void *ptr;
- int ix;
if (use_limit && ++ops > ops_limit) {
if (ddq->head.first != ddq->head.unref_end) {
@@ -1180,52 +1749,29 @@ handle_delayed_dealloc(Allctr_t *allctr,
res = 1;
- INC_CC(allctr->calls.this_free);
-
- if (fix) {
- ErtsAlcType_t type;
-
- type = ((ErtsAllctrFixDDBlock_t*) ptr)->fix_type;
- ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE;
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
- fix[ix].used--;
- if (fix[ix].allocated < fix[ix].limit
- && fix[ix].list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) {
- *((void **) ptr) = fix[ix].list;
- fix[ix].list = ptr;
- fix[ix].list_size++;
- if (!allctr->fix_shrink_scheduled) {
- allctr->fix_shrink_scheduled = 1;
- erts_set_aux_work_timeout(
- allctr->ix,
- (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
- | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
- 1);
- }
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
- continue;
- }
- fix[ix].allocated--;
- if (fix[ix].list && fix[ix].allocated > fix[ix].limit) {
- blk = UMEM2BLK(ptr);
- if (IS_SBC_BLK(blk))
- destroy_carrier(allctr, blk);
- else
- mbc_free(allctr, ptr);
- ptr = fix[ix].list;
- fix[ix].list = *((void **) ptr);
- fix[ix].list_size--;
- fix[ix].allocated--;
- }
+ blk = UMEM2BLK(ptr);
+ if (IS_FREE_LAST_MBC_BLK(blk)) {
+ /*
+ * A multiblock carrier that previously has been migrated away
+ * from us and now is back to be deallocated...
+ *
+ * Note that we cannot use FBLK_TO_MBC(blk) since it
+ * data has been overwritten by the queue.
+ */
+ Carrier_t *crr = FIRST_BLK_TO_MBC(allctr, blk);
+ ERTS_ALC_CPOOL_ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr));
+ ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr);
+ schedule_dealloc_carrier(allctr, crr);
}
+ else {
- blk = UMEM2BLK(ptr);
+ INC_CC(allctr->calls.this_free);
- if (IS_SBC_BLK(blk))
- destroy_carrier(allctr, blk);
- else
- mbc_free(allctr, ptr);
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
+ if (fix)
+ handle_delayed_fix_dealloc(allctr, ptr);
+ else
+ dealloc_block(allctr, ptr, 1);
+ }
}
if (need_thr_progress && !(need_thr_prgr | need_mr_wrk)) {
@@ -1235,6 +1781,12 @@ handle_delayed_dealloc(Allctr_t *allctr,
store_earliest_thr_prgr(thr_prgr_p, ddq);
}
+ if (ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ check_pending_dealloc_carrier(allctr,
+ need_thr_progress,
+ thr_prgr_p,
+ need_more_work);
+
if (allctr->thread_safe && !allctr_locked)
erts_mtx_unlock(&allctr->mutex);
return res;
@@ -1249,13 +1801,59 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type,
if (allctr->fix)
((ErtsAllctrFixDDBlock_t*) ptr)->fix_type = type;
- if (ddq_enqueue(type, &allctr->dd.q, ptr, cinit))
+ if (ddq_enqueue(&allctr->dd.q, ptr, cinit))
erts_alloc_notify_delayed_dealloc(allctr->ix);
}
#endif
#ifdef ERTS_SMP
+static void
+set_new_allctr_abandon_limit(Allctr_t *allctr);
+static void
+abandon_carrier(Allctr_t *allctr, Carrier_t *crr);
+
+
+static ERTS_INLINE void
+check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp)
+{
+ Carrier_t *crr;
+
+ if (busy_pcrr_pp && *busy_pcrr_pp)
+ return;
+
+ if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ return;
+
+ allctr->cpool.check_limit_count--;
+ if (--allctr->cpool.check_limit_count <= 0)
+ set_new_allctr_abandon_limit(allctr);
+
+ if (!erts_thr_progress_is_managed_thread())
+ return;
+
+ if (allctr->cpool.disable_abandon)
+ return;
+
+ if (allctr->mbcs.blocks.curr.size > allctr->cpool.abandon_limit)
+ return;
+
+
+ crr = FBLK_TO_MBC(fblk);
+
+ if (allctr->main_carrier == crr)
+ return;
+
+ if (crr->cpool.blocks_size > crr->cpool.abandon_limit)
+ return;
+
+ if (crr->cpool.thr_prgr != ERTS_THR_PRGR_INVALID
+ && !erts_thr_progress_has_reached(crr->cpool.thr_prgr))
+ return;
+
+ abandon_carrier(allctr, crr);
+}
+
void
erts_alcu_check_delayed_dealloc(Allctr_t *allctr,
int limit,
@@ -1273,10 +1871,50 @@ erts_alcu_check_delayed_dealloc(Allctr_t *allctr,
}
#endif
-#define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \
- handle_delayed_dealloc((Allctr), (Locked), 1, \
+#define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \
+ handle_delayed_dealloc((Allctr), (Locked), 1, \
ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL, NULL)
+static void
+dealloc_block(Allctr_t *allctr, void *ptr, int dec_cc_on_redirect)
+{
+ Block_t *blk = UMEM2BLK(ptr);
+
+ ERTS_SMP_LC_ASSERT(!allctr->thread_safe
+ || erts_lc_mtx_is_locked(&allctr->mutex));
+
+ if (IS_SBC_BLK(blk))
+ destroy_carrier(allctr, blk, NULL);
+#ifndef ERTS_SMP
+ else
+ mbc_free(allctr, ptr, NULL);
+#else
+ else if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ mbc_free(allctr, ptr, NULL);
+ else {
+ Carrier_t *busy_pcrr_p;
+ Allctr_t *used_allctr;
+ used_allctr = get_used_allctr(allctr, ERTS_ALC_TS_PREF_LOCK_NO, ptr,
+ NULL, &busy_pcrr_p);
+ if (used_allctr == allctr) {
+ mbc_free(allctr, ptr, &busy_pcrr_p);
+ clear_busy_pool_carrier(allctr, busy_pcrr_p);
+ }
+ else {
+ /* Carrier migrated; need to redirect block to new owner... */
+ int cinit = used_allctr->dd.ix - allctr->dd.ix;
+
+ ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p);
+
+ if (dec_cc_on_redirect)
+ DEC_CC(allctr->calls.this_free);
+ if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit))
+ erts_alloc_notify_delayed_dealloc(used_allctr->ix);
+ }
+ }
+#endif
+}
+
/* Multi block carrier alloc/realloc/free ... */
/* NOTE! mbc_alloc() may in case of memory shortage place the requested
@@ -1293,20 +1931,8 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp)
*blk_szp = get_blk_sz = UMEMSZ2BLKSZ(allctr, size);
-#ifdef ERTS_SMP
- if (allctr->dd.use)
- ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1);
-#endif
-
blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0);
-#ifdef ERTS_SMP
- if (!blk && allctr->dd.use) {
- if (ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1))
- blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0);
- }
-#endif
-
if (!blk) {
blk = create_carrier(allctr, get_blk_sz, CFLG_MBC);
#if !HALFWORD_HEAP && !HAVE_SUPER_ALIGNED_MB_CARRIERS
@@ -1413,7 +2039,8 @@ mbc_alloc_finalize(Allctr_t *allctr,
ASSERT(ABLK_TO_MBC(blk) == crr);
}
- STAT_MBC_BLK_ALLOC(allctr, blk_sz);
+ ERTS_ALC_CPOOL_ALLOC_OP(allctr);
+ STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs);
ASSERT(IS_ALLOCED_BLK(blk));
ASSERT(blk_sz == MBC_BLK_SZ(blk));
@@ -1448,14 +2075,14 @@ mbc_alloc(Allctr_t *allctr, Uint size)
}
static void
-mbc_free(Allctr_t *allctr, void *p)
+mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp)
{
Uint is_first_blk;
Uint is_last_blk;
Uint blk_sz;
Block_t *blk;
Block_t *nxt_blk;
-
+ Carrier_t *crr;
ASSERT(p);
@@ -1467,7 +2094,10 @@ mbc_free(Allctr_t *allctr, void *p)
HARD_CHECK_BLK_CARRIER(allctr, blk);
- STAT_MBC_BLK_FREE(allctr, blk_sz);
+ crr = ABLK_TO_MBC(blk);
+
+ ERTS_ALC_CPOOL_FREE_OP(allctr);
+ STAT_MBC_BLK_FREE(allctr, crr, busy_pcrr_pp, blk_sz, alcu_flgs);
is_first_blk = IS_MBC_FIRST_ABLK(allctr, blk);
is_last_blk = IS_LAST_BLK(blk);
@@ -1525,16 +2155,20 @@ mbc_free(Allctr_t *allctr, void *p)
if (is_first_blk
&& is_last_blk
&& allctr->main_carrier != FIRST_BLK_TO_MBC(allctr, blk)) {
- destroy_carrier(allctr, blk);
+ destroy_carrier(allctr, blk, busy_pcrr_pp);
}
else {
(*allctr->link_free_block)(allctr, blk);
HARD_CHECK_BLK_CARRIER(allctr, blk);
+#ifdef ERTS_SMP
+ check_abandon_carrier(allctr, blk, busy_pcrr_pp);
+#endif
}
}
static void *
-mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
+mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs,
+ Carrier_t **busy_pcrr_pp)
{
void *new_p;
Uint old_blk_sz;
@@ -1548,11 +2182,6 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
Uint is_last_blk;
#endif /* #ifndef MBC_REALLOC_ALWAYS_MOVES */
-#ifdef ERTS_SMP
- if (allctr->dd.use)
- ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1);
-#endif
-
ASSERT(p);
ASSERT(size);
ASSERT(size < allctr->sbc_threshold);
@@ -1566,6 +2195,12 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
return NULL;
#else /* !MBC_REALLOC_ALWAYS_MOVES */
+
+#ifdef ERTS_SMP
+ if (busy_pcrr_pp && *busy_pcrr_pp)
+ goto realloc_move; /* Don't want to use carrier in pool */
+#endif
+
get_blk_sz = blk_sz = UMEMSZ2BLKSZ(allctr, size);
ASSERT(IS_ALLOCED_BLK(blk));
@@ -1636,8 +2271,11 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
nxt_blk = BLK_AFTER(blk, blk_sz);
- STAT_MBC_BLK_FREE(allctr, old_blk_sz);
- STAT_MBC_BLK_ALLOC(allctr, blk_sz);
+ crr = ABLK_TO_MBC(blk);
+
+ ERTS_ALC_CPOOL_REALLOC_OP(allctr);
+ STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs);
+ STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs);
ASSERT(MBC_BLK_SZ(blk) >= allctr->min_block_size);
@@ -1655,7 +2293,6 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz);
}
- crr = ABLK_TO_MBC(blk);
SET_MBC_FBLK_HDR(nxt_blk, nxt_blk_sz,
SBH_THIS_FREE | (is_last_blk ? SBH_LAST_BLK : 0),
crr);
@@ -1683,6 +2320,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
HARD_CHECK_BLK_CARRIER(allctr, blk);
+#ifdef ERTS_SMP
+ check_abandon_carrier(allctr, nxt_blk, NULL);
+#endif
+
return p;
}
@@ -1692,6 +2333,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
nxt_blk = BLK_AFTER(blk, old_blk_sz);
nxt_blk_sz = MBC_BLK_SZ(nxt_blk);
if (IS_FREE_BLK(nxt_blk) && get_blk_sz <= old_blk_sz + nxt_blk_sz) {
+ Carrier_t* crr = ABLK_TO_MBC(blk);
/* Grow into next block... */
HARD_CHECK_BLK_CARRIER(allctr, blk);
@@ -1721,7 +2363,6 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
}
}
else {
- Carrier_t* crr = ABLK_TO_MBC(blk);
SET_MBC_ABLK_SZ(blk, blk_sz);
nxt_blk = BLK_AFTER(blk, blk_sz);
@@ -1738,9 +2379,9 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
ASSERT(FBLK_TO_MBC(nxt_blk) == crr);
}
- STAT_MBC_BLK_FREE(allctr, old_blk_sz);
- STAT_MBC_BLK_ALLOC(allctr, blk_sz);
-
+ ERTS_ALC_CPOOL_REALLOC_OP(allctr);
+ STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs);
+ STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs);
ASSERT(IS_ALLOCED_BLK(blk));
ASSERT(blk_sz == MBC_BLK_SZ(blk));
@@ -1793,13 +2434,16 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
if (cand_blk_sz < get_blk_sz) {
/* We wont fit in cand_blk get a new one */
+#ifdef ERTS_SMP
+ realloc_move:
+#endif
#endif /* !MBC_REALLOC_ALWAYS_MOVES */
new_p = mbc_alloc(allctr, size);
if (!new_p)
return NULL;
sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ));
- mbc_free(allctr, p);
+ mbc_free(allctr, p, busy_pcrr_pp);
return new_p;
@@ -1829,7 +2473,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
1);
new_p = BLK2UMEM(new_blk);
sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ));
- mbc_free(allctr, p);
+ mbc_free(allctr, p, NULL);
return new_p;
}
else {
@@ -1885,7 +2529,8 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
blk_sz,
0);
- STAT_MBC_BLK_FREE(allctr, old_blk_sz);
+ ERTS_ALC_CPOOL_FREE_OP(allctr);
+ STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs);
return new_p;
}
@@ -1895,7 +2540,11 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
#ifdef ERTS_SMP
+#define ERTS_ALC_MAX_DEALLOC_CARRIER 10
#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 10
+#define ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT 100
+#define ERTS_ALC_CPOOL_MAX_NO_CARRIERS 5
+#define ERTS_ALC_CPOOL_INSERT_ALLOWED_OFFSET 100
#define ERTS_ALC_CPOOL_PTR_MOD_MRK (((erts_aint_t) 1) << 0)
#define ERTS_ALC_CPOOL_PTR_DEL_MRK (((erts_aint_t) 1) << 1)
@@ -2063,7 +2712,6 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr)
ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_read_nob(&crr->allctr)
== (erts_aint_t) allctr);
-
erts_atomic_add_nob(&allctr->cpool.stat.blocks_size,
(erts_aint_t) crr->cpool.blocks_size);
erts_atomic_add_nob(&allctr->cpool.stat.no_blocks,
@@ -2221,7 +2869,7 @@ cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr)
crr->cpool.thr_prgr = erts_thr_progress_later(NULL);
erts_atomic_add_nob(&prev_allctr->cpool.stat.blocks_size,
- -((erts_aint_t) -crr->cpool.blocks_size));
+ -((erts_aint_t) crr->cpool.blocks_size));
erts_atomic_add_nob(&prev_allctr->cpool.stat.no_blocks,
-((erts_aint_t) crr->cpool.blocks));
erts_atomic_add_nob(&prev_allctr->cpool.stat.carriers_size,
@@ -2248,6 +2896,13 @@ cpool_fetch(Allctr_t *allctr, UWord size)
while (crr && i < ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) {
if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) {
unlink_carrier(&allctr->cpool.dc_list, crr);
+#ifdef ERTS_ALC_CPOOL_DEBUG
+ ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr,
+ ((erts_aint_t) allctr))
+ == (((erts_aint_t) allctr) & ~FLG_MASK));
+#else
+ erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr));
+#endif
return crr;
}
crr = crr->prev;
@@ -2295,21 +2950,172 @@ cpool_fetch(Allctr_t *allctr, UWord size)
return NULL;
}
+static void
+check_pending_dealloc_carrier(Allctr_t *allctr,
+ int *need_thr_progress,
+ ErtsThrPrgrVal *thr_prgr_p,
+ int *need_more_work)
+{
+ Carrier_t *crr = allctr->cpool.dc_list.first;
+
+ if (crr) {
+ ErtsThrPrgrVal current = erts_thr_progress_current();
+ int i = 0;
+
+ do {
+ Carrier_t *dcrr;
+
+ if (!erts_thr_progress_has_reached_this(current, crr->cpool.thr_prgr))
+ break;
+
+ dcrr = crr;
+ crr = crr->next;
+ dealloc_carrier(allctr, dcrr, ERTS_MSEG_FLG_2POW);
+ i++;
+ } while (crr && i < ERTS_ALC_MAX_DEALLOC_CARRIER);
+
+ allctr->cpool.dc_list.first = crr;
+ if (!crr)
+ allctr->cpool.dc_list.last = NULL;
+ else {
+ crr->prev = NULL;
+
+ if (need_more_work) {
+ ERTS_ALC_CPOOL_ASSERT(need_thr_progress && thr_prgr_p);
+ if (erts_thr_progress_has_reached_this(current, crr->cpool.thr_prgr))
+ *need_more_work = 1;
+ else {
+ *need_thr_progress = 1;
+ if (*thr_prgr_p == ERTS_THR_PRGR_INVALID
+ || erts_thr_progress_cmp(crr->cpool.thr_prgr,
+ *thr_prgr_p) < 0) {
+ *thr_prgr_p = crr->cpool.thr_prgr;
+ }
+ }
+ }
+ }
+ }
+}
+
+static void
+schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr)
+{
+ Allctr_t *used_allctr;
+ int check_pending_dealloc;
+ erts_aint_t max_size;
+
+ if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) {
+ dealloc_carrier(allctr, crr, ERTS_MSEG_FLG_2POW);
+ return;
+ }
+
+ used_allctr = crr->cpool.orig_allctr;
+
+ if (allctr != used_allctr) {
+ Block_t *blk = MBC_TO_FIRST_BLK(allctr, crr);
+ int cinit = used_allctr->dd.ix - allctr->dd.ix;
+
+ /*
+ * Receiver will recognize that this is a carrier to
+ * deallocate since the block is an mbc block that
+ * is free and last in carrier...
+ */
+ ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(blk));
+
+ ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, blk));
+ ERTS_ALC_CPOOL_ASSERT(crr == FBLK_TO_MBC(blk));
+ ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(used_allctr, blk));
+
+ if (ddq_enqueue(&used_allctr->dd.q, BLK2UMEM(blk), cinit))
+ erts_alloc_notify_delayed_dealloc(used_allctr->ix);
+ return;
+ }
+
+ if (crr->cpool.thr_prgr == ERTS_THR_PRGR_INVALID
+ || erts_thr_progress_has_reached(crr->cpool.thr_prgr)) {
+ dealloc_carrier(allctr, crr, ERTS_MSEG_FLG_2POW);
+ return;
+ }
+
+ max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr);
+ erts_atomic_set_nob(&crr->cpool.max_size, max_size);
+
+ crr->next = NULL;
+ crr->prev = allctr->cpool.dc_list.last;
+ if (allctr->cpool.dc_list.last) {
+ check_pending_dealloc = 1;
+ allctr->cpool.dc_list.last->next = crr;
+ }
+ else {
+ check_pending_dealloc = 0;
+ allctr->cpool.dc_list.first = crr;
+ }
+ allctr->cpool.dc_list.last = crr;
+ if (check_pending_dealloc)
+ check_pending_dealloc_carrier(allctr, NULL, NULL, NULL);
+ erts_alloc_ensure_handle_delayed_dealloc_call(allctr->ix);
+}
+
static ERTS_INLINE void
cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr)
{
- UWord limit, csz;
erts_atomic_init_nob(&crr->cpool.next, ERTS_AINT_NULL);
erts_atomic_init_nob(&crr->cpool.prev, ERTS_AINT_NULL);
+ crr->cpool.orig_allctr = allctr;
crr->cpool.thr_prgr = ERTS_THR_PRGR_INVALID;
erts_atomic_init_nob(&crr->cpool.max_size, 0);
- csz = CARRIER_SZ(crr);
+ crr->cpool.blocks = 0;
+ crr->cpool.blocks_size = 0;
+ if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ crr->cpool.abandon_limit = 0;
+ else {
+ UWord csz = CARRIER_SZ(crr);
+ UWord limit = csz*allctr->cpool.util_limit;
+ if (limit > csz)
+ limit /= 100;
+ else
+ limit = (csz/100)*allctr->cpool.util_limit;
+ crr->cpool.abandon_limit = limit;
+ }
+}
+
+static void
+set_new_allctr_abandon_limit(Allctr_t *allctr)
+{
+ UWord limit;
+ UWord csz;
+
+ allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT;
+
+ csz = allctr->mbcs.curr.norm.mseg.size;
+ csz += allctr->mbcs.curr.norm.sys_alloc.size;
+
limit = csz*allctr->cpool.util_limit;
if (limit > csz)
limit /= 100;
else
limit = (csz/100)*allctr->cpool.util_limit;
- crr->cpool.abandon_limit = limit;
+
+ allctr->cpool.abandon_limit = limit;
+}
+
+static void
+abandon_carrier(Allctr_t *allctr, Carrier_t *crr)
+{
+ erts_aint_t max_size;
+
+ STAT_MBC_CPOOL_INSERT(allctr, crr);
+
+ unlink_carrier(&allctr->mbc_list, crr);
+
+ allctr->remove_mbc(allctr, crr);
+
+ max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr);
+ erts_atomic_set_nob(&crr->cpool.max_size, max_size);
+
+ cpool_insert(allctr, crr);
+
+ set_new_allctr_abandon_limit(allctr);
}
#endif /* ERTS_SMP */
@@ -2385,6 +3191,24 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz);
+#ifdef ERTS_SMP
+ allctr->cpool.disable_abandon = ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON;
+
+ if ((flags & (CFLG_MBC|CFLG_NO_CPOOL)) == CFLG_MBC
+ && ERTS_ALC_IS_CPOOL_ENABLED(allctr)
+ && erts_thr_progress_is_managed_thread()) {
+ crr = cpool_fetch(allctr, blk_sz);
+ if (crr) {
+ STAT_MBC_CPOOL_FETCH(allctr, crr);
+ link_carrier(&allctr->mbc_list, crr);
+ (*allctr->add_mbc)(allctr, crr);
+ blk = (*allctr->get_free_block)(allctr, blk_sz, NULL, 0);
+ ASSERT(blk);
+ return blk;
+ }
+ }
+#endif
+
#if HAVE_ERTS_MSEG
if (flags & CFLG_FORCE_SYS_ALLOC)
@@ -2512,6 +3336,10 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
allctr->main_carrier = crr;
}
+#ifdef ERTS_SMP
+ cpool_init_carrier_data(allctr, crr);
+#endif
+
link_carrier(&allctr->mbc_list, crr);
CHECK_1BLK_CARRIER(allctr, 0, is_mseg, crr, crr_sz, blk, blk_sz);
@@ -2660,14 +3488,21 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
}
static void
-destroy_carrier(Allctr_t *allctr, Block_t *blk)
+dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, Uint mseg_flags)
{
- Uint crr_sz;
- Carrier_t *crr;
#if HAVE_ERTS_MSEG
- Uint is_mseg = 0;
- Uint mseg_flags = ERTS_MSEG_FLG_NONE;
+ if (IS_MSEG_CARRIER(crr))
+ alcu_mseg_dealloc(allctr, crr, CARRIER_SZ(crr), mseg_flags);
+ else
#endif
+ alcu_sys_free(allctr, crr);
+}
+
+static void
+destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp)
+{
+ Uint crr_sz;
+ Carrier_t *crr;
if (IS_SBC_BLK(blk)) {
Uint blk_sz = SBC_BLK_SZ(blk);
@@ -2680,7 +3515,6 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
#if HAVE_ERTS_MSEG
if (IS_MSEG_CARRIER(crr)) {
- is_mseg++;
ASSERT(crr_sz % MSEG_UNIT_SZ == 0);
STAT_MSEG_SBC_FREE(allctr, crr_sz, blk_sz);
}
@@ -2690,6 +3524,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
unlink_carrier(&allctr->sbc_list, crr);
+ dealloc_carrier(allctr, crr, ERTS_MSEG_FLG_NONE);
}
else {
ASSERT(IS_MBC_FIRST_FBLK(allctr, blk));
@@ -2708,30 +3543,36 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
}
#endif
-#if HAVE_ERTS_MSEG
- if (IS_MSEG_CARRIER(crr)) {
- is_mseg++;
- ASSERT(crr_sz % MSEG_UNIT_SZ == 0);
- STAT_MSEG_MBC_FREE(allctr, crr_sz);
- mseg_flags = ERTS_MSEG_FLG_2POW;
+ if (allctr->destroying_mbc)
+ (*allctr->destroying_mbc)(allctr, crr);
+
+#ifdef ERTS_SMP
+ if (busy_pcrr_pp && *busy_pcrr_pp) {
+ ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr);
+ *busy_pcrr_pp = NULL;
+ cpool_delete(allctr, allctr, crr);
}
else
#endif
- STAT_SYS_ALLOC_MBC_FREE(allctr, crr_sz);
+ {
+ unlink_carrier(&allctr->mbc_list, crr);
+#if HAVE_ERTS_MSEG
+ if (IS_MSEG_CARRIER(crr)) {
+ ASSERT(crr_sz % MSEG_UNIT_SZ == 0);
+ STAT_MSEG_MBC_FREE(allctr, crr_sz);
+ }
+ else
+#endif
+ STAT_SYS_ALLOC_MBC_FREE(allctr, crr_sz);
+ }
- unlink_carrier(&allctr->mbc_list, crr);
- if (allctr->destroying_mbc)
- (*allctr->destroying_mbc)(allctr, crr);
+#ifdef ERTS_SMP
+ schedule_dealloc_carrier(allctr, crr);
+#else
+ dealloc_carrier(allctr, crr, ERTS_MSEG_FLG_2POW);
+#endif
}
-
-#if HAVE_ERTS_MSEG
- if (is_mseg) {
- alcu_mseg_dealloc(allctr, crr, crr_sz, mseg_flags);
- }
- else
-#endif
- alcu_sys_free(allctr, crr);
}
@@ -2974,8 +3815,24 @@ add_4tup(Uint **hpp, Uint *szp, Eterm *lp,
bld_cons(hpp, szp, bld_tuple(hpp, szp, 4, el1, el2, el3, el4), *lp);
}
+static ERTS_INLINE void
+add_fix_types(Allctr_t *allctr, int internal, Uint **hpp, Uint *szp,
+ Eterm *lp, Eterm fix)
+{
+ if (allctr->fix) {
+ if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ add_2tup(hpp, szp, lp, am.fix_types, fix);
+ else if (internal)
+ add_3tup(hpp, szp, lp,
+ am.fix_types,
+ erts_bld_uword(hpp, szp, ~((UWord) 0)),
+ fix);
+ }
+}
+
static Eterm
sz_info_fix(Allctr_t *allctr,
+ int internal,
int *print_to_p,
void *print_to_arg,
Uint **hpp,
@@ -2983,36 +3840,67 @@ sz_info_fix(Allctr_t *allctr,
{
Eterm res;
int ix;
- ErtsAlcFixList_t *fix = allctr->fix;
- ASSERT(fix);
+ ASSERT(allctr->fix);
res = NIL;
- for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) {
- ErtsAlcType_t n = ix + ERTS_ALC_N_MIN_A_FIXED_SIZE;
- Uint alloced = (fix[ix].type_size * fix[ix].allocated);
- Uint used = fix[ix].type_size*fix[ix].used;
-
- if (print_to_p) {
- int to = *print_to_p;
- void *arg = print_to_arg;
- erts_print(to,
- arg,
- "fix type: %s %bpu %bpu\n",
- (char *) ERTS_ALC_N2TD(n),
- alloced,
- used);
- }
+ if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) {
+
+ if (internal) {
+ for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) {
+ ErtsAlcFixList_t *fix = &allctr->fix[ix];
+ UWord alloced = fix->type_size * fix->u.cpool.allocated;
+ UWord used = fix->type_size * fix->u.cpool.used;
+
+ if (print_to_p) {
+ int to = *print_to_p;
+ void *arg = print_to_arg;
+ erts_print(to,
+ arg,
+ "fix type internal: %s %bpu %bpu\n",
+ (char *) ERTS_ALC_N2TD(ERTS_ALC_N_MIN_A_FIXED_SIZE
+ + ix),
+ alloced,
+ used);
+ }
- if (hpp || szp) {
- add_3tup(hpp, szp, &res,
- fix_type_atoms[ix],
- bld_unstable_uint(hpp, szp, alloced),
- bld_unstable_uint(hpp, szp, used));
+ if (hpp || szp) {
+ add_3tup(hpp, szp, &res,
+ fix_type_atoms[ix],
+ bld_unstable_uint(hpp, szp, alloced),
+ bld_unstable_uint(hpp, szp, used));
+ }
+ }
}
}
+ else {
+ for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) {
+ ErtsAlcFixList_t *fix = &allctr->fix[ix];
+ UWord alloced = fix->type_size * fix->u.nocpool.allocated;
+ UWord used = fix->type_size*fix->u.nocpool.used;
+
+ if (print_to_p) {
+ int to = *print_to_p;
+ void *arg = print_to_arg;
+ erts_print(to,
+ arg,
+ "fix type: %s %bpu %bpu\n",
+ (char *) ERTS_ALC_N2TD(ERTS_ALC_N_MIN_A_FIXED_SIZE
+ + ix),
+ alloced,
+ used);
+ }
+
+ if (hpp || szp) {
+ add_3tup(hpp, szp, &res,
+ fix_type_atoms[ix],
+ bld_unstable_uint(hpp, szp, alloced),
+ bld_unstable_uint(hpp, szp, used));
+ }
+ }
+ }
return res;
}
@@ -3516,6 +4404,7 @@ erts_alcu_info_options(Allctr_t *allctr,
Eterm
erts_alcu_sz_info(Allctr_t *allctr,
+ int internal,
int begin_max_period,
int *print_to_p,
void *print_to_arg,
@@ -3555,7 +4444,7 @@ erts_alcu_sz_info(Allctr_t *allctr,
update_max_ever_values(&allctr->sbcs);
if (allctr->fix)
- fix = sz_info_fix(allctr, print_to_p, print_to_arg, hpp, szp);
+ fix = sz_info_fix(allctr, internal, print_to_p, print_to_arg, hpp, szp);
mbcs = sz_info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p,
print_to_arg, hpp, szp);
sbcs = sz_info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p,
@@ -3565,8 +4454,7 @@ erts_alcu_sz_info(Allctr_t *allctr,
res = NIL;
add_2tup(hpp, szp, &res, am.sbcs, sbcs);
add_2tup(hpp, szp, &res, am.mbcs, mbcs);
- if (allctr->fix)
- add_2tup(hpp, szp, &res, am.fix_types, fix);
+ add_fix_types(allctr, internal, hpp, szp, &res, fix);
}
if (begin_max_period) {
@@ -3588,6 +4476,7 @@ erts_alcu_sz_info(Allctr_t *allctr,
Eterm
erts_alcu_info(Allctr_t *allctr,
+ int internal,
int begin_max_period,
int *print_to_p,
void *print_to_arg,
@@ -3636,7 +4525,7 @@ erts_alcu_info(Allctr_t *allctr,
sett = info_options(allctr, print_to_p, print_to_arg, hpp, szp);
if (allctr->fix)
- fix = sz_info_fix(allctr, print_to_p, print_to_arg, hpp, szp);
+ fix = sz_info_fix(allctr, internal, print_to_p, print_to_arg, hpp, szp);
mbcs = info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p,
print_to_arg, hpp, szp);
sbcs = info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p,
@@ -3649,8 +4538,7 @@ erts_alcu_info(Allctr_t *allctr,
add_2tup(hpp, szp, &res, am.calls, calls);
add_2tup(hpp, szp, &res, am.sbcs, sbcs);
add_2tup(hpp, szp, &res, am.mbcs, mbcs);
- if (allctr->fix)
- add_2tup(hpp, szp, &res, am.fix_types, fix);
+ add_fix_types(allctr, internal, hpp, szp, &res, fix);
add_2tup(hpp, szp, &res, am.options, sett);
add_3tup(hpp, szp, &res,
am.versions,
@@ -3696,10 +4584,18 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t *
int ix;
for (ix = 0; ix < fisz; ix++) {
if (allctr->fix) {
- fi[ix].allocated += (allctr->fix[ix].type_size
- * allctr->fix[ix].allocated);
- fi[ix].used += (allctr->fix[ix].type_size
- * allctr->fix[ix].used);
+ if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) {
+ fi[ix].allocated += (allctr->fix[ix].type_size
+ * allctr->fix[ix].u.cpool.allocated);
+ fi[ix].used += (allctr->fix[ix].type_size
+ * allctr->fix[ix].u.cpool.used);
+ }
+ else {
+ fi[ix].allocated += (allctr->fix[ix].type_size
+ * allctr->fix[ix].u.nocpool.allocated);
+ fi[ix].used += (allctr->fix[ix].type_size
+ * allctr->fix[ix].u.nocpool.used);
+ }
}
}
}
@@ -3717,7 +4613,6 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size)
{
Allctr_t *allctr = (Allctr_t *) extra;
void *res;
- ErtsAlcFixList_t *fix;
ASSERT(initialized);
@@ -3735,57 +4630,21 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size)
INC_CC(allctr->calls.this_alloc);
- fix = allctr->fix;
- if (fix) {
- int ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE;
- ASSERT((unsigned)ix < ERTS_ALC_NO_FIXED_SIZES);
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
- ASSERT(size <= fix[ix].type_size);
- fix[ix].used++;
- res = fix[ix].list;
- if (res) {
- fix[ix].list_size--;
- fix[ix].list = *((void **) res);
- if (fix[ix].list && fix[ix].allocated > fix[ix].limit) {
- void *p = fix[ix].list;
- Block_t *blk;
- fix[ix].list = *((void **) p);
- fix[ix].list_size--;
- blk = UMEM2BLK(p);
- if (IS_SBC_BLK(blk))
- destroy_carrier(allctr, blk);
- else
- mbc_free(allctr, p);
- fix[ix].allocated--;
- }
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
- return res;
- }
- size = fix[ix].type_size;
- if (fix[ix].limit < fix[ix].used)
- fix[ix].limit = fix[ix].used;
- if (fix[ix].max_used < fix[ix].used)
- fix[ix].max_used = fix[ix].used;
- fix[ix].allocated++;
+ if (allctr->fix) {
+ if (ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ return fix_cpool_alloc(allctr, type, size);
+ else
+ return fix_nocpool_alloc(allctr, type, size);
}
if (size >= allctr->sbc_threshold) {
Block_t *blk;
-#ifdef ERTS_SMP
- if (allctr->dd.use)
- ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1);
-#endif
blk = create_carrier(allctr, size, CFLG_SBC);
res = blk ? BLK2UMEM(blk) : NULL;
}
else
res = mbc_alloc(allctr, size);
- if (!res && fix) {
- int ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE;
- fix[ix].allocated--;
- fix[ix].used--;
- }
return res;
}
@@ -3854,9 +4713,22 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size)
if (pref_allctr->thread_safe)
erts_mtx_lock(&pref_allctr->mutex);
+#ifdef ERTS_SMP
+ ASSERT(pref_allctr->dd.use);
+ ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1);
+#endif
+
ERTS_ALCU_DBG_CHK_THR_ACCESS(pref_allctr);
res = do_erts_alcu_alloc(type, pref_allctr, size);
+
+#ifdef ERTS_SMP
+ if (!res && ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1)) {
+ /* Cleaned up a bit more; try one more time... */
+ res = do_erts_alcu_alloc(type, pref_allctr, size);
+ }
+#endif
+
if (pref_allctr->thread_safe)
erts_mtx_unlock(&pref_allctr->mutex);
@@ -3873,9 +4745,9 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size)
/* ------------------------------------------------------------------------- */
static ERTS_INLINE void
-do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p)
+do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p,
+ Carrier_t **busy_pcrr_pp)
{
- int ix;
Allctr_t *allctr = (Allctr_t *) extra;
ASSERT(initialized);
@@ -3887,57 +4759,28 @@ do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p)
ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr);
if (p) {
- ErtsAlcFixList_t *fix = allctr->fix;
- Block_t *blk;
INC_CC(allctr->calls.this_free);
- if (fix) {
- ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE;
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
- fix[ix].used--;
- if (fix[ix].allocated < fix[ix].limit
- && fix[ix].list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) {
- *((void **) p) = fix[ix].list;
- fix[ix].list = p;
- fix[ix].list_size++;
- if (!allctr->fix_shrink_scheduled) {
- allctr->fix_shrink_scheduled = 1;
- erts_set_aux_work_timeout(
- allctr->ix,
- (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
- | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
- 1);
- }
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
- return;
- }
- fix[ix].allocated--;
- if (fix[ix].list && fix[ix].allocated > fix[ix].limit) {
- blk = UMEM2BLK(p);
- if (IS_SBC_BLK(blk))
- destroy_carrier(allctr, blk);
- else
- mbc_free(allctr, p);
- p = fix[ix].list;
- fix[ix].list = *((void **) p);
- fix[ix].list_size--;
- fix[ix].allocated--;
- }
+ if (allctr->fix) {
+ if (ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ fix_cpool_free(allctr, type, p, busy_pcrr_pp);
+ else
+ fix_nocpool_free(allctr, type, p);
+ }
+ else {
+ Block_t *blk = UMEM2BLK(p);
+ if (IS_SBC_BLK(blk))
+ destroy_carrier(allctr, blk, NULL);
+ else
+ mbc_free(allctr, p, busy_pcrr_pp);
}
-
- blk = UMEM2BLK(p);
- if (IS_SBC_BLK(blk))
- destroy_carrier(allctr, blk);
- else
- mbc_free(allctr, p);
- ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
}
}
void erts_alcu_free(ErtsAlcType_t type, void *extra, void *p)
{
- do_erts_alcu_free(type, extra, p);
+ do_erts_alcu_free(type, extra, p, NULL);
}
#ifdef USE_THREADS
@@ -3947,7 +4790,7 @@ erts_alcu_free_ts(ErtsAlcType_t type, void *extra, void *p)
{
Allctr_t *allctr = (Allctr_t *) extra;
erts_mtx_lock(&allctr->mutex);
- do_erts_alcu_free(type, extra, p);
+ do_erts_alcu_free(type, extra, p, NULL);
erts_mtx_unlock(&allctr->mutex);
}
@@ -3969,7 +4812,7 @@ erts_alcu_free_thr_spec(ErtsAlcType_t type, void *extra, void *p)
if (allctr->thread_safe)
erts_mtx_lock(&allctr->mutex);
- do_erts_alcu_free(type, allctr, p);
+ do_erts_alcu_free(type, allctr, p, NULL);
if (allctr->thread_safe)
erts_mtx_unlock(&allctr->mutex);
@@ -3979,10 +4822,12 @@ void
erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p)
{
if (p) {
+ Carrier_t *busy_pcrr_p;
Allctr_t *pref_allctr, *used_allctr;
pref_allctr = get_pref_allctr(extra);
- used_allctr = get_used_allctr(pref_allctr, extra, p, NULL);
+ used_allctr = get_used_allctr(pref_allctr, ERTS_ALC_TS_PREF_LOCK_IF_USED,
+ p, NULL, &busy_pcrr_p);
if (pref_allctr != used_allctr)
enqueue_dealloc_other_instance(type,
used_allctr,
@@ -3990,12 +4835,11 @@ erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p)
(used_allctr->dd.ix
- pref_allctr->dd.ix));
else {
- if (used_allctr->thread_safe)
- erts_mtx_lock(&used_allctr->mutex);
ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr);
- do_erts_alcu_free(type, used_allctr, p);
- if (used_allctr->thread_safe)
- erts_mtx_unlock(&used_allctr->mutex);
+ do_erts_alcu_free(type, used_allctr, p, &busy_pcrr_p);
+ clear_busy_pool_carrier(used_allctr, busy_pcrr_p);
+ if (pref_allctr->thread_safe)
+ erts_mtx_unlock(&pref_allctr->mutex);
}
}
}
@@ -4011,7 +4855,8 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
void *extra,
void *p,
Uint size,
- Uint32 alcu_flgs)
+ Uint32 alcu_flgs,
+ Carrier_t **busy_pcrr_pp)
{
Allctr_t *allctr = (Allctr_t *) extra;
Block_t *blk;
@@ -4036,7 +4881,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
#if ALLOC_ZERO_EQ_NULL
if (!size) {
ASSERT(p);
- do_erts_alcu_free(type, extra, p);
+ do_erts_alcu_free(type, extra, p, busy_pcrr_pp);
INC_CC(allctr->calls.this_realloc);
DEC_CC(allctr->calls.this_free);
return NULL;
@@ -4049,7 +4894,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
if (size < allctr->sbc_threshold) {
if (IS_MBC_BLK(blk))
- res = mbc_realloc(allctr, p, size, alcu_flgs);
+ res = mbc_realloc(allctr, p, size, alcu_flgs, busy_pcrr_pp);
else {
Uint used_sz = SBC_HEADER_SIZE + ABLK_HDR_SZ + size;
Uint crr_sz;
@@ -4088,16 +4933,12 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
sys_memcpy((void*) res,
(void*) p,
MIN(SBC_BLK_SZ(blk) - ABLK_HDR_SZ, size));
- destroy_carrier(allctr, blk);
+ destroy_carrier(allctr, blk, NULL);
}
}
}
else {
Block_t *new_blk;
-#ifdef ERTS_SMP
- if (allctr->dd.use)
- ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1);
-#endif
if(IS_SBC_BLK(blk)) {
do_carrier_resize:
#if HALFWORD_HEAP
@@ -4116,7 +4957,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
sys_memcpy((void *) res,
(void *) p,
MIN(MBC_ABLK_SZ(blk) - ABLK_HDR_SZ, size));
- mbc_free(allctr, p);
+ mbc_free(allctr, p, busy_pcrr_pp);
}
else
res = NULL;
@@ -4130,7 +4971,7 @@ void *
erts_alcu_realloc(ErtsAlcType_t type, void *extra, void *p, Uint size)
{
void *res;
- res = do_erts_alcu_realloc(type, extra, p, size, 0);
+ res = do_erts_alcu_realloc(type, extra, p, size, 0, NULL);
DEBUG_CHECK_ALIGNMENT(res);
return res;
}
@@ -4151,7 +4992,7 @@ erts_alcu_realloc_mv(ErtsAlcType_t type, void *extra, void *p, Uint size)
if (cpy_size > size)
cpy_size = size;
sys_memcpy(res, p, cpy_size);
- do_erts_alcu_free(type, extra, p);
+ do_erts_alcu_free(type, extra, p, NULL);
}
DEBUG_CHECK_ALIGNMENT(res);
return res;
@@ -4166,7 +5007,7 @@ erts_alcu_realloc_ts(ErtsAlcType_t type, void *extra, void *ptr, Uint size)
Allctr_t *allctr = (Allctr_t *) extra;
void *res;
erts_mtx_lock(&allctr->mutex);
- res = do_erts_alcu_realloc(type, extra, ptr, size, 0);
+ res = do_erts_alcu_realloc(type, extra, ptr, size, 0, NULL);
erts_mtx_unlock(&allctr->mutex);
DEBUG_CHECK_ALIGNMENT(res);
return res;
@@ -4190,7 +5031,7 @@ erts_alcu_realloc_mv_ts(ErtsAlcType_t type, void *extra, void *p, Uint size)
if (cpy_size > size)
cpy_size = size;
sys_memcpy(res, p, cpy_size);
- do_erts_alcu_free(type, extra, p);
+ do_erts_alcu_free(type, extra, p, NULL);
}
erts_mtx_unlock(&allctr->mutex);
DEBUG_CHECK_ALIGNMENT(res);
@@ -4217,7 +5058,7 @@ erts_alcu_realloc_thr_spec(ErtsAlcType_t type, void *extra,
if (allctr->thread_safe)
erts_mtx_lock(&allctr->mutex);
- res = do_erts_alcu_realloc(type, allctr, ptr, size, 0);
+ res = do_erts_alcu_realloc(type, allctr, ptr, size, 0, NULL);
if (allctr->thread_safe)
erts_mtx_unlock(&allctr->mutex);
@@ -4260,7 +5101,7 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra,
if (cpy_size > size)
cpy_size = size;
sys_memcpy(res, ptr, cpy_size);
- do_erts_alcu_free(type, allctr, ptr);
+ do_erts_alcu_free(type, allctr, ptr, NULL);
if (allctr->thread_safe)
erts_mtx_unlock(&allctr->mutex);
}
@@ -4277,40 +5118,64 @@ realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size,
void *res;
Allctr_t *pref_allctr, *used_allctr;
UWord old_user_size;
+ Carrier_t *busy_pcrr_p;
+#ifdef ERTS_SMP
+ int retried;
+#endif
if (!p)
return erts_alcu_alloc_thr_pref(type, extra, size);
pref_allctr = get_pref_allctr(extra);
- used_allctr = get_used_allctr(pref_allctr, extra, p, &old_user_size);
+
+ if (pref_allctr->thread_safe)
+ erts_mtx_lock(&pref_allctr->mutex);
+
+#ifdef ERTS_SMP
+ ASSERT(pref_allctr->dd.use);
+ ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1);
+ retried = 0;
+restart:
+#endif
+
+ used_allctr = get_used_allctr(pref_allctr, ERTS_ALC_TS_PREF_LOCK_NO,
+ p, &old_user_size, &busy_pcrr_p);
ASSERT(used_allctr && pref_allctr);
if (!force_move && used_allctr == pref_allctr) {
- if (used_allctr->thread_safe)
- erts_mtx_lock(&used_allctr->mutex);
ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr);
res = do_erts_alcu_realloc(type,
used_allctr,
p,
size,
- 0);
- if (used_allctr->thread_safe)
- erts_mtx_unlock(&used_allctr->mutex);
+ 0,
+ &busy_pcrr_p);
+ clear_busy_pool_carrier(used_allctr, busy_pcrr_p);
+#ifdef ERTS_SMP
+ if (!res && !retried && ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1)) {
+ /* Cleaned up a bit more; try one more time... */
+ retried = 1;
+ goto restart;
+ }
+#endif
+ if (pref_allctr->thread_safe)
+ erts_mtx_unlock(&pref_allctr->mutex);
}
else {
- if (pref_allctr->thread_safe)
- erts_mtx_lock(&pref_allctr->mutex);
res = do_erts_alcu_alloc(type, pref_allctr, size);
- if (pref_allctr->thread_safe && used_allctr != pref_allctr) {
- erts_mtx_unlock(&pref_allctr->mutex);
- }
- if (res) {
- DEBUG_CHECK_ALIGNMENT(res);
+ if (!res)
+ goto unlock_ts_return;
+ else {
- sys_memcpy(res, p, MIN(size,old_user_size));
+ DEBUG_CHECK_ALIGNMENT(res);
if (used_allctr != pref_allctr) {
+ if (pref_allctr->thread_safe)
+ erts_mtx_unlock(&pref_allctr->mutex);
+
+ sys_memcpy(res, p, MIN(size, old_user_size));
+
enqueue_dealloc_other_instance(type,
used_allctr,
p,
@@ -4318,8 +5183,14 @@ realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size,
- pref_allctr->dd.ix));
}
else {
- do_erts_alcu_free(type, used_allctr, p);
+
+ sys_memcpy(res, p, MIN(size, old_user_size));
+
+ do_erts_alcu_free(type, used_allctr, p, &busy_pcrr_p);
ASSERT(pref_allctr == used_allctr);
+ clear_busy_pool_carrier(used_allctr, busy_pcrr_p);
+
+ unlock_ts_return:
if (pref_allctr->thread_safe)
erts_mtx_unlock(&pref_allctr->mutex);
}
@@ -4437,13 +5308,13 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
allctr->cpool.dc_list.first = NULL;
allctr->cpool.dc_list.last = NULL;
allctr->cpool.abandon_limit = 0;
- allctr->cpool.insert_allowed_cc = 0;
+ allctr->cpool.disable_abandon = 0;
erts_atomic_init_nob(&allctr->cpool.stat.blocks_size, 0);
erts_atomic_init_nob(&allctr->cpool.stat.no_blocks, 0);
erts_atomic_init_nob(&allctr->cpool.stat.carriers_size, 0);
erts_atomic_init_nob(&allctr->cpool.stat.no_carriers, 0);
- allctr->cpool.check_limit_count = 0;
- allctr->cpool.util_limit = 0;
+ allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT;
+ allctr->cpool.util_limit = init->acul;
#endif
allctr->sbc_threshold = init->sbct;
@@ -4519,6 +5390,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
allctr->main_carrier_size,
CFLG_MBC
| CFLG_FORCE_SIZE
+ | CFLG_NO_CPOOL
#if !HALFWORD_HEAP && !HAVE_SUPER_ALIGNED_MB_CARRIERS
| CFLG_FORCE_SYS_ALLOC
#endif
@@ -4537,16 +5409,24 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
allctr->fix = init->fix;
allctr->fix_shrink_scheduled = 0;
for (i = 0; i < ERTS_ALC_NO_FIXED_SIZES; i++) {
- allctr->fix[i].max_used = 0;
- allctr->fix[i].limit = 0;
allctr->fix[i].type_size = init->fix_type_size[i];
allctr->fix[i].list_size = 0;
allctr->fix[i].list = NULL;
- allctr->fix[i].allocated = 0;
- allctr->fix[i].used = 0;
#ifdef ERTS_SMP
ASSERT(allctr->fix[i].type_size >= sizeof(ErtsAllctrFixDDBlock_t));
#endif
+ if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) {
+ allctr->fix[i].u.cpool.min_list_size = 0;
+ allctr->fix[i].u.cpool.shrink_list = 0;
+ allctr->fix[i].u.cpool.allocated = 0;
+ allctr->fix[i].u.cpool.used = 0;
+ }
+ else {
+ allctr->fix[i].u.nocpool.max_used = 0;
+ allctr->fix[i].u.nocpool.limit = 0;
+ allctr->fix[i].u.nocpool.allocated = 0;
+ allctr->fix[i].u.nocpool.used = 0;
+ }
}
}
@@ -4571,9 +5451,9 @@ erts_alcu_stop(Allctr_t *allctr)
allctr->stopped = 1;
while (allctr->sbc_list.first)
- destroy_carrier(allctr, SBC2BLK(allctr, allctr->sbc_list.first));
+ destroy_carrier(allctr, SBC2BLK(allctr, allctr->sbc_list.first), NULL);
while (allctr->mbc_list.first)
- destroy_carrier(allctr, MBC_TO_FIRST_BLK(allctr, allctr->mbc_list.first));
+ destroy_carrier(allctr, MBC_TO_FIRST_BLK(allctr, allctr->mbc_list.first), NULL);
#ifdef USE_THREADS
if (allctr->thread_safe)
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 14fe289f7f..5e52b9b733 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -55,6 +55,7 @@ typedef struct {
UWord lmbcs;
UWord smbcs;
UWord mbcgs;
+ int acul;
void *fix;
size_t *fix_type_size;
@@ -98,6 +99,7 @@ typedef struct {
10*1024*1024, /* (bytes) lmbcs: largest mbc size */\
1024*1024, /* (bytes) smbcs: smallest mbc size */\
10, /* (amount) mbcgs: mbc growth stages */\
+ 0, /* (%) acul: abandon carrier utilization limit */\
/* --- Data not options -------------------------------------------- */\
NULL, /* (ptr) fix */\
NULL /* (ptr) fix_type_size */\
@@ -130,6 +132,7 @@ typedef struct {
1024*1024, /* (bytes) lmbcs: largest mbc size */\
128*1024, /* (bytes) smbcs: smallest mbc size */\
10, /* (amount) mbcgs: mbc growth stages */\
+ 0, /* (%) acul: abandon carrier utilization limit */\
/* --- Data not options -------------------------------------------- */\
NULL, /* (ptr) fix */\
NULL /* (ptr) fix_type_size */\
@@ -159,8 +162,8 @@ void erts_alcu_free_thr_pref(ErtsAlcType_t, void *, void *);
#endif
Eterm erts_alcu_au_info_options(int *, void *, Uint **, Uint *);
Eterm erts_alcu_info_options(Allctr_t *, int *, void *, Uint **, Uint *);
-Eterm erts_alcu_sz_info(Allctr_t *, int, int *, void *, Uint **, Uint *);
-Eterm erts_alcu_info(Allctr_t *, int, int *, void *, Uint **, Uint *);
+Eterm erts_alcu_sz_info(Allctr_t *, int, int, int *, void *, Uint **, Uint *);
+Eterm erts_alcu_info(Allctr_t *, int, int, int *, void *, Uint **, Uint *);
void erts_alcu_init(AlcUInit_t *);
void erts_alcu_current_size(Allctr_t *, AllctrSize_t *,
ErtsAlcUFixInfo_t *, int);
@@ -256,10 +259,10 @@ typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t;
typedef struct {
erts_atomic_t next;
erts_atomic_t prev;
+ Allctr_t *orig_allctr;
ErtsThrPrgrVal thr_prgr;
erts_atomic_t max_size;
UWord abandon_limit;
- /* Statistics */
UWord blocks;
UWord blocks_size;
} ErtsAlcCPoolData_t;
@@ -277,6 +280,9 @@ struct Carrier_t_ {
#endif
};
+#define ERTS_ALC_CARRIER_TO_ALLCTR(C) \
+ ((Allctr_t *) (erts_smp_atomic_read_nob(&(C)->allctr) & ~FLG_MASK))
+
typedef struct {
Carrier_t *first;
Carrier_t *last;
@@ -301,6 +307,16 @@ typedef struct {
#define SBC_BLK_HDR_FLG /* Special flag combo for (allocated) SBC blocks */\
(THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)
+/*
+ * FREE_LAST_MBC_BLK_HDR_FLGS is a special flag combo used for
+ * distinguishing empty mbc's from allocated blocks in
+ * handle_delayed_dealloc().
+ */
+#define FREE_LAST_MBC_BLK_HDR_FLGS (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)
+
+#define IS_FREE_LAST_MBC_BLK(B) \
+ (((B)->bhdr & FLG_MASK) == FREE_LAST_MBC_BLK_HDR_FLGS)
+
#define IS_SBC_BLK(B) (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG)
#define IS_MBC_BLK(B) (!IS_SBC_BLK((B)))
#define IS_FREE_BLK(B) (ASSERT(IS_MBC_BLK(B)), \
@@ -395,10 +411,20 @@ typedef struct {
size_t type_size;
SWord list_size;
void *list;
- SWord max_used;
- SWord limit;
- SWord allocated;
- SWord used;
+ union {
+ struct {
+ SWord max_used;
+ SWord limit;
+ SWord allocated;
+ SWord used;
+ } nocpool;
+ struct {
+ int min_list_size;
+ int shrink_list;
+ UWord allocated;
+ UWord used;
+ } cpool;
+ } u;
} ErtsAlcFixList_t;
struct Allctr_t_ {
@@ -464,7 +490,7 @@ struct Allctr_t_ {
struct {
CarrierList_t dc_list;
UWord abandon_limit;
- CallCounter_t insert_allowed_cc;
+ int disable_abandon;
int check_limit_count;
int util_limit;
struct {
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
index 3bb0bf146e..953569f437 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.c
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -664,7 +664,7 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block)
AOFF_Carrier_t *blk_crr = (AOFF_Carrier_t*) FBLK_TO_MBC(block);
Uint blk_sz = AOFF_BLK_SZ(blk);
- ASSERT(allctr == blk_crr->crr.allctr);
+ ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(&blk_crr->crr));
ASSERT(blk_crr->rbt_node.hdr.bhdr == (blk_crr->root ? blk_crr->root->max_sz : 0));
HARD_CHECK_IS_MEMBER(alc->mbc_root, &blk_crr->rbt_node);
HARD_CHECK_TREE(&blk_crr->crr, alc->bf_within_carrier, blk_crr->root, 0);
@@ -854,7 +854,7 @@ static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier)
AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier;
AOFF_RBTree_t **root = &alc->mbc_root;
- ASSERT(allctr == carrier->allctr);
+ ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(carrier));
HARD_CHECK_TREE(NULL, 0, *root, 0);
rbt_delete(root, &crr->rbt_node);
@@ -870,7 +870,7 @@ static UWord aoff_largest_fblk_in_mbc(Allctr_t* allctr, Carrier_t* carrier)
{
AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier;
- ASSERT(allctr == carrier->allctr);
+ ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(carrier));
ASSERT(crr->rbt_node.hdr.bhdr == (crr->root ? crr->root->max_sz : 0));
return crr->rbt_node.hdr.bhdr;
}
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 1744afbae1..74a37f374d 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1671,13 +1671,22 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */
sel = *tp++;
- if (sel == am_allocator_sizes) {
+ if (sel == am_memory_internal) {
+ switch (arity) {
+ case 3:
+ if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 1, 1))
+ return am_true;
+ default:
+ goto badarg;
+ }
+ }
+ else if (sel == am_allocator_sizes) {
switch (arity) {
case 2:
ERTS_BIF_PREP_TRAP1(ret, alloc_sizes_trap, BIF_P, *tp);
return ret;
case 3:
- if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 1))
+ if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 1, 0))
return am_true;
default:
goto badarg;
@@ -1736,7 +1745,7 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */
ERTS_BIF_PREP_TRAP1(ret, alloc_info_trap, BIF_P, *tp);
return ret;
case 3:
- if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 0))
+ if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 0, 0))
return am_true;
default:
goto badarg;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 81799ddbb1..88eb224f84 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1327,6 +1327,17 @@ erts_alloc_notify_delayed_dealloc(int ix)
ERTS_SSI_AUX_WORK_DD);
}
+void
+erts_alloc_ensure_handle_delayed_dealloc_call(int ix)
+{
+#ifdef DEBUG
+ ErtsSchedulerData *esdp = erts_get_scheduler_data();
+ ASSERT(!esdp || ix == (int) esdp->no);
+#endif
+ set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(ix-1),
+ ERTS_SSI_AUX_WORK_DD);
+}
+
static ERTS_INLINE erts_aint32_t
handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
{
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 7dd8116857..865ac6c43f 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1340,6 +1340,7 @@ int erts_is_multi_scheduling_blocked(void);
Eterm erts_multi_scheduling_blockers(Process *);
void erts_start_schedulers(void);
void erts_alloc_notify_delayed_dealloc(int);
+void erts_alloc_ensure_handle_delayed_dealloc_call(int);
void erts_smp_notify_check_children_needed(void);
#endif
#if ERTS_USE_ASYNC_READY_Q
diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c
index aaaeb0eaa8..01134dd718 100644
--- a/erts/etc/common/erlexec.c
+++ b/erts/etc/common/erlexec.c
@@ -79,6 +79,7 @@ static const char plusM_au_allocs[]= {
static char *plusM_au_alloc_switches[] = {
"as",
"asbcst",
+ "acul",
"e",
"t",
"lmbcs",
diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam
index a792d667c8..cfbe777633 100644
--- a/erts/preloaded/ebin/erlang.beam
+++ b/erts/preloaded/ebin/erlang.beam
Binary files differ
diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl
index 3186ce6e4c..27dfeac4d6 100644
--- a/erts/preloaded/src/erlang.erl
+++ b/erts/preloaded/src/erlang.erl
@@ -3305,6 +3305,9 @@ get_fix_proc([], Acc) ->
fix_proc([{fix_types, SizeList} | _Rest], Acc) ->
get_fix_proc(SizeList, Acc);
+fix_proc([{fix_types, Mask, SizeList} | _Rest], Acc) ->
+ {A, U} = get_fix_proc(SizeList, Acc),
+ {Mask, A, U};
fix_proc([_ | Rest], Acc) ->
fix_proc(Rest, Acc);
fix_proc([], Acc) ->
@@ -3356,13 +3359,21 @@ au_mem_data(#memory{total = Tot,
processes_used = ProcU,
system = Sys} = Mem,
[{fix_alloc, _, Data} | Rest]) ->
- {A, U} = fix_proc(Data, {0, 0}),
Sz = blocks_size(Data, 0),
- au_mem_data(Mem#memory{total = Tot+Sz,
- processes = Proc+A,
- processes_used = ProcU+U,
- system = Sys+Sz-A},
- Rest);
+ case fix_proc(Data, {0, 0}) of
+ {A, U} ->
+ au_mem_data(Mem#memory{total = Tot+Sz,
+ processes = Proc+A,
+ processes_used = ProcU+U,
+ system = Sys+Sz-A},
+ Rest);
+ {Mask, A, U} ->
+ au_mem_data(Mem#memory{total = Tot+Sz,
+ processes = Mask band (Proc+A),
+ processes_used = Mask band (ProcU+U),
+ system = Mask band (Sys+Sz-A)},
+ Rest)
+ end;
au_mem_data(#memory{total = Tot,
system = Sys,
low = Low} = Mem,
@@ -3380,7 +3391,7 @@ au_mem_data(EMD, []) ->
au_mem_data(Allocs) ->
Ref = erlang:make_ref(),
- erlang:system_info({allocator_sizes, Ref, Allocs}),
+ erlang:system_info({memory_internal, Ref, Allocs}),
receive_emd(Ref).
receive_emd(_Ref, EMD, 0) ->
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-29 21:25:14 +0000