aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/erl_alloc.c
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam/erl_alloc.c')
-rw-r--r--erts/emulator/beam/erl_alloc.c688
1 files changed, 570 insertions, 118 deletions
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index b853ec0f01..9af80dd7a9 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1,19 +1,19 @@
/*
* %CopyrightBegin%
- *
- * Copyright Ericsson AB 2002-2009. All Rights Reserved.
- *
+ *
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ *
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
* compliance with the License. You should have received a copy of the
* Erlang Public License along with this software. If not, it can be
* retrieved online at http://www.erlang.org/.
- *
+ *
* Software distributed under the License is distributed on an "AS IS"
* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
* the License for the specific language governing rights and limitations
* under the License.
- *
+ *
* %CopyrightEnd%
*/
@@ -38,9 +38,6 @@
#include "erl_bits.h"
#include "erl_instrument.h"
#include "erl_mseg.h"
-#ifdef ELIB_ALLOC_IS_CLIB
-#include "erl_version.h"
-#endif
#include "erl_monitors.h"
#include "erl_bif_timer.h"
#if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE)
@@ -53,6 +50,9 @@
#include "erl_bestfit_alloc.h"
#define GET_ERL_AF_ALLOC_IMPL
#include "erl_afit_alloc.h"
+#define GET_ERL_AOFF_ALLOC_IMPL
+#include "erl_ao_firstfit_alloc.h"
+
#define ERTS_ALC_DEFAULT_MAX_THR_PREF 16
@@ -64,8 +64,15 @@
#ifdef DEBUG
static Uint install_debug_functions(void);
+#if 0
+#define HARD_DEBUG
+#ifdef __GNUC__
+#warning "* * * * * * * * * * * * * *"
+#warning "* HARD DEBUG IS ENABLED! *"
+#warning "* * * * * * * * * * * * * *"
+#endif
+#endif
#endif
-extern void elib_ensure_initialized(void);
ErtsAllocatorFunctions_t erts_allctrs[ERTS_ALC_A_MAX+1];
ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1];
@@ -81,11 +88,19 @@ typedef union {
char align_bfa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(BFAllctr_t))];
AFAllctr_t afa;
char align_afa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AFAllctr_t))];
+ AOFFAllctr_t aoffa;
+ char align_aoffa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AOFFAllctr_t))];
} ErtsAllocatorState_t;
-static ErtsAllocatorState_t sl_alloc_state;
+static ErtsAllocatorState_t sbmbc_alloc_state;
static ErtsAllocatorState_t std_alloc_state;
static ErtsAllocatorState_t ll_alloc_state;
+#if HALFWORD_HEAP
+static ErtsAllocatorState_t sbmbc_low_alloc_state;
+static ErtsAllocatorState_t std_low_alloc_state;
+static ErtsAllocatorState_t ll_low_alloc_state;
+#endif
+static ErtsAllocatorState_t sl_alloc_state;
static ErtsAllocatorState_t temp_alloc_state;
static ErtsAllocatorState_t eheap_alloc_state;
static ErtsAllocatorState_t binary_alloc_state;
@@ -112,7 +127,8 @@ static void *fix_core_alloc(Uint size)
enum allctr_type {
GOODFIT,
BESTFIT,
- AFIT
+ AFIT,
+ AOFIRSTFIT
};
struct au_init {
@@ -124,6 +140,7 @@ struct au_init {
GFAllctrInit_t gf;
BFAllctrInit_t bf;
AFAllctrInit_t af;
+ AOFFAllctrInit_t aoff;
} init;
struct {
int mmbcs;
@@ -137,7 +154,8 @@ struct au_init {
ERTS_DEFAULT_ALLCTR_INIT, \
ERTS_DEFAULT_GF_ALLCTR_INIT, \
ERTS_DEFAULT_BF_ALLCTR_INIT, \
- ERTS_DEFAULT_AF_ALLCTR_INIT \
+ ERTS_DEFAULT_AF_ALLCTR_INIT, \
+ ERTS_DEFAULT_AOFF_ALLCTR_INIT \
}
typedef struct {
@@ -154,6 +172,7 @@ typedef struct {
char *mtrace;
char *nodename;
} instr;
+ struct au_init sbmbc_alloc;
struct au_init sl_alloc;
struct au_init std_alloc;
struct au_init ll_alloc;
@@ -162,6 +181,11 @@ typedef struct {
struct au_init binary_alloc;
struct au_init ets_alloc;
struct au_init driver_alloc;
+#if HALFWORD_HEAP
+ struct au_init sbmbc_low_alloc;
+ struct au_init std_low_alloc;
+ struct au_init ll_low_alloc;
+#endif
} erts_alc_hndl_args_init_t;
#define ERTS_AU_INIT__ {0, 0, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}}
@@ -173,6 +197,34 @@ do { \
} while (0)
static void
+set_default_sbmbc_alloc_opts(struct au_init *ip)
+{
+ SET_DEFAULT_ALLOC_OPTS(ip);
+ ip->enable = 0;
+ ip->thr_spec = 0;
+ ip->atype = BESTFIT;
+ ip->init.bf.ao = 1;
+ ip->init.util.ramv = 0;
+ ip->init.util.mmsbc = 0;
+ ip->init.util.mmmbc = 500;
+ ip->init.util.sbct = ~((UWord) 0);
+ ip->init.util.name_prefix = "sbmbc_";
+ ip->init.util.alloc_no = ERTS_ALC_A_SBMBC;
+#ifndef SMALL_MEMORY
+ ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */
+#else
+ ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */
+#endif
+ ip->init.util.ts = ERTS_ALC_MTA_SBMBC;
+ ip->init.util.asbcst = 0;
+ ip->init.util.rsbcst = 0;
+ ip->init.util.rsbcmt = 0;
+ ip->init.util.rmbcmt = 0;
+ ip->init.util.sbmbct = 0;
+ ip->init.util.sbmbcs = 0;
+}
+
+static void
set_default_sl_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
@@ -189,6 +241,11 @@ set_default_sl_alloc_opts(struct au_init *ip)
#endif
ip->init.util.ts = ERTS_ALC_MTA_SHORT_LIVED;
ip->init.util.rsbcst = 80;
+#if HALFWORD_HEAP
+ ip->init.util.force = 1;
+ ip->init.util.low_mem = 1;
+#endif
+
}
static void
@@ -220,7 +277,7 @@ set_default_ll_alloc_opts(struct au_init *ip)
ip->init.util.ramv = 0;
ip->init.util.mmsbc = 0;
ip->init.util.mmmbc = 0;
- ip->init.util.sbct = ~((Uint) 0);
+ ip->init.util.sbct = ~((UWord) 0);
ip->init.util.name_prefix = "ll_";
ip->init.util.alloc_no = ERTS_ALC_A_LONG_LIVED;
#ifndef SMALL_MEMORY
@@ -233,6 +290,8 @@ set_default_ll_alloc_opts(struct au_init *ip)
ip->init.util.rsbcst = 0;
ip->init.util.rsbcmt = 0;
ip->init.util.rmbcmt = 0;
+ ip->init.util.sbmbct = 0;
+ ip->init.util.sbmbcs = 0;
}
static void
@@ -252,6 +311,10 @@ set_default_temp_alloc_opts(struct au_init *ip)
ip->init.util.ts = ERTS_ALC_MTA_TEMPORARY;
ip->init.util.rsbcst = 90;
ip->init.util.rmbcmt = 100;
+#if HALFWORD_HEAP
+ ip->init.util.force = 1;
+ ip->init.util.low_mem = 1;
+#endif
}
static void
@@ -271,6 +334,10 @@ set_default_eheap_alloc_opts(struct au_init *ip)
#endif
ip->init.util.ts = ERTS_ALC_MTA_EHEAP;
ip->init.util.rsbcst = 50;
+#if HALFWORD_HEAP
+ ip->init.util.force = 1;
+ ip->init.util.low_mem = 1;
+#endif
}
static void
@@ -391,10 +458,14 @@ refuse_af_strategy(struct au_init *init)
static void init_thr_ix(int static_ixs);
+#ifdef HARD_DEBUG
+static void hdbg_init(void);
+#endif
+
void
erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
{
- Uint extra_block_size = 0;
+ UWord extra_block_size = 0;
int i;
erts_alc_hndl_args_init_t init = {
0,
@@ -406,10 +477,17 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
ERTS_DEFAULT_ALCU_INIT
};
+#ifdef HARD_DEBUG
+ hdbg_init();
+#endif
+
+ erts_have_sbmbc_alloc = 0;
+
erts_sys_alloc_init();
init_thr_ix(erts_no_schedulers);
erts_init_utils_mem();
+ set_default_sbmbc_alloc_opts(&init.sbmbc_alloc);
set_default_sl_alloc_opts(&init.sl_alloc);
set_default_std_alloc_opts(&init.std_alloc);
set_default_ll_alloc_opts(&init.ll_alloc);
@@ -423,6 +501,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
handle_args(argc, argv, &init);
if (erts_no_schedulers <= 1) {
+ init.sbmbc_alloc.thr_spec = 0;
init.sl_alloc.thr_spec = 0;
init.std_alloc.thr_spec = 0;
init.ll_alloc.thr_spec = 0;
@@ -434,6 +513,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
if (init.erts_alloc_config) {
/* Adjust flags that erts_alloc_config won't like */
+ init.sbmbc_alloc.thr_spec = 0;
init.temp_alloc.thr_spec = 0;
init.sl_alloc.thr_spec = 0;
init.std_alloc.thr_spec = 0;
@@ -450,6 +530,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
init.temp_alloc.thr_spec = erts_no_schedulers;
/* Others must use thread preferred interface */
+ adjust_tpref(&init.sbmbc_alloc, erts_no_schedulers);
adjust_tpref(&init.sl_alloc, erts_no_schedulers);
adjust_tpref(&init.std_alloc, erts_no_schedulers);
adjust_tpref(&init.ll_alloc, erts_no_schedulers);
@@ -467,6 +548,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
* The following allocators cannot be run with afit strategy.
* Make sure they don't...
*/
+ refuse_af_strategy(&init.sbmbc_alloc);
refuse_af_strategy(&init.sl_alloc);
refuse_af_strategy(&init.std_alloc);
refuse_af_strategy(&init.ll_alloc);
@@ -488,6 +570,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
erts_afalc_init();
erts_bfalc_init();
erts_gfalc_init();
+ erts_aoffalc_init();
for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
erts_allctrs[i].alloc = NULL;
@@ -519,7 +602,32 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
erts_allctrs[ERTS_ALC_A_SYSTEM].free = erts_sys_free;
erts_allctrs_info[ERTS_ALC_A_SYSTEM].enabled = 1;
+#if HALFWORD_HEAP
+ /* Init low memory variants by cloning */
+ init.sbmbc_low_alloc = init.sbmbc_alloc;
+ init.sbmbc_low_alloc.init.util.name_prefix = "sbmbc_low_";
+ init.sbmbc_low_alloc.init.util.alloc_no = ERTS_ALC_A_SBMBC_LOW;
+ init.sbmbc_low_alloc.init.util.low_mem = 1;
+
+ init.std_low_alloc = init.std_alloc;
+ init.std_low_alloc.init.util.name_prefix = "std_low_";
+ init.std_low_alloc.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW;
+ init.std_low_alloc.init.util.force = 1;
+ init.std_low_alloc.init.util.low_mem = 1;
+
+ init.ll_low_alloc = init.ll_alloc;
+ init.ll_low_alloc.init.util.name_prefix = "ll_low_";
+ init.ll_low_alloc.init.util.alloc_no = ERTS_ALC_A_LONG_LIVED_LOW;
+ init.ll_low_alloc.init.util.force = 1;
+ init.ll_low_alloc.init.util.low_mem = 1;
+
+ set_au_allocator(ERTS_ALC_A_SBMBC_LOW, &init.sbmbc_low_alloc);
+ set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc);
+ set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc);
+#endif /* HALFWORD */
+
set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc);
+ set_au_allocator(ERTS_ALC_A_SBMBC, &init.sbmbc_alloc);
set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc);
set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc);
set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc);
@@ -542,7 +650,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
sys_alloc_opt(SYS_ALLOC_OPT_TRIM_THRESHOLD, init.trim_threshold);
sys_alloc_opt(SYS_ALLOC_OPT_TOP_PAD, init.top_pad);
-
if (erts_allctrs_info[ERTS_FIX_CORE_ALLOCATOR].enabled)
erts_fix_core_allocator_ix = ERTS_FIX_CORE_ALLOCATOR;
else
@@ -550,6 +657,20 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
erts_mtrace_init(init.instr.mtrace, init.instr.nodename);
+ /* sbmbc_alloc() needs to be started first */
+ start_au_allocator(ERTS_ALC_A_SBMBC,
+ &init.sbmbc_alloc,
+ &sbmbc_alloc_state);
+#if HALFWORD_HEAP
+ start_au_allocator(ERTS_ALC_A_SBMBC_LOW,
+ &init.sbmbc_low_alloc,
+ &sbmbc_low_alloc_state);
+ erts_have_sbmbc_alloc = (init.sbmbc_alloc.enable
+ && init.sbmbc_low_alloc.enable);
+#else
+ erts_have_sbmbc_alloc = init.sbmbc_alloc.enable;
+#endif
+
start_au_allocator(ERTS_ALC_A_TEMPORARY,
&init.temp_alloc,
&temp_alloc_state);
@@ -565,7 +686,14 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
start_au_allocator(ERTS_ALC_A_LONG_LIVED,
&init.ll_alloc,
&ll_alloc_state);
-
+#if HALFWORD_HEAP
+ start_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW,
+ &init.ll_low_alloc,
+ &ll_low_alloc_state);
+ start_au_allocator(ERTS_ALC_A_STANDARD_LOW,
+ &init.std_low_alloc,
+ &std_low_alloc_state);
+#endif
start_au_allocator(ERTS_ALC_A_EHEAP,
&init.eheap_alloc,
&eheap_alloc_state);
@@ -601,11 +729,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
erts_set_fix_size(ERTS_ALC_T_PROC, sizeof(Process));
erts_set_fix_size(ERTS_ALC_T_DB_TABLE, sizeof(DbTable));
erts_set_fix_size(ERTS_ALC_T_ATOM, sizeof(Atom));
- erts_set_fix_size(ERTS_ALC_T_EXPORT, sizeof(Export));
+
erts_set_fix_size(ERTS_ALC_T_MODULE, sizeof(Module));
erts_set_fix_size(ERTS_ALC_T_REG_PROC, sizeof(RegProc));
- erts_set_fix_size(ERTS_ALC_T_MONITOR_SH, ERTS_MONITOR_SH_SIZE*sizeof(Uint));
- erts_set_fix_size(ERTS_ALC_T_NLINK_SH, ERTS_LINK_SH_SIZE*sizeof(Uint));
erts_set_fix_size(ERTS_ALC_T_FUN_ENTRY, sizeof(ErlFunEntry));
#ifdef ERTS_ALC_T_DRV_EV_D_STATE
erts_set_fix_size(ERTS_ALC_T_DRV_EV_D_STATE,
@@ -615,6 +741,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
erts_set_fix_size(ERTS_ALC_T_DRV_SEL_D_STATE,
sizeof(ErtsDrvSelectDataState));
#endif
+#if !HALFWORD_HEAP
+ erts_set_fix_size(ERTS_ALC_T_EXPORT, sizeof(Export));
+ erts_set_fix_size(ERTS_ALC_T_MONITOR_SH, ERTS_MONITOR_SH_SIZE*sizeof(Uint));
+ erts_set_fix_size(ERTS_ALC_T_NLINK_SH, ERTS_LINK_SH_SIZE*sizeof(Uint));
+#endif
#endif
#endif
@@ -627,6 +758,12 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init)
ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n];
ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n];
+ /*
+ * Some allocators are forced on if halfword heap is used.
+ */
+ if (init->init.util.force)
+ init->enable = 1;
+
if (!init->enable) {
af->alloc = erts_sys_alloc;
af->realloc = erts_sys_realloc;
@@ -719,8 +856,8 @@ start_au_allocator(ErtsAlcType_t alctr_n,
init->init.util.name_prefix);
tspec->allctr = (Allctr_t **) states;
states = ((char *) states) + sizeof(Allctr_t *) * (tspec->size + 1);
- states = ((((Uint) states) & ERTS_CACHE_LINE_MASK)
- ? (void *) ((((Uint) states) & ~ERTS_CACHE_LINE_MASK)
+ states = ((((UWord) states) & ERTS_CACHE_LINE_MASK)
+ ? (void *) ((((UWord) states) & ~ERTS_CACHE_LINE_MASK)
+ ERTS_CACHE_LINE_SIZE)
: (void *) states);
tspec->allctr[0] = init->thr_spec > 0 ? (Allctr_t *) state : (Allctr_t *) NULL;
@@ -775,6 +912,12 @@ start_au_allocator(ErtsAlcType_t alctr_n,
&init->init.af,
&init->init.util);
break;
+ case AOFIRSTFIT:
+ as = (void *) erts_aoffalc_start((AOFFAllctr_t *) as0,
+ &init->init.aoff,
+ &init->init.util);
+ break;
+
default:
as = NULL;
ASSERT(0);
@@ -885,6 +1028,20 @@ get_kb_value(char *param_end, char** argv, int* ip)
}
static Uint
+get_byte_value(char *param_end, char** argv, int* ip)
+{
+ Sint tmp;
+ char *rest;
+ char *param = argv[*ip]+1;
+ char *value = get_value(param_end, argv, ip);
+ errno = 0;
+ tmp = (Sint) strtol(value, &rest, 10);
+ if (errno != 0 || rest == value || tmp < 0)
+ bad_value(param, param_end, value);
+ return (Uint) tmp;
+}
+
+static Uint
get_amount_value(char *param_end, char** argv, int* ip)
{
Sint tmp;
@@ -955,6 +1112,9 @@ handle_au_arg(struct au_init *auip,
else if (strcmp("af", alg) == 0) {
auip->atype = AFIT;
}
+ else if (strcmp("aoff", alg) == 0) {
+ auip->atype = AOFIRSTFIT;
+ }
else {
bad_value(param, sub_param + 1, alg);
}
@@ -1023,6 +1183,12 @@ handle_au_arg(struct au_init *auip,
if(has_prefix("sbct", sub_param)) {
auip->init.util.sbct = get_kb_value(sub_param + 4, argv, ip);
}
+ else if (has_prefix("sbmbcs", sub_param)) {
+ auip->init.util.sbmbcs = get_byte_value(sub_param + 6, argv, ip);
+ }
+ else if (has_prefix("sbmbct", sub_param)) {
+ auip->init.util.sbmbct = get_byte_value(sub_param + 6, argv, ip);
+ }
else if (has_prefix("smbcs", sub_param)) {
auip->default_.smbcs = 0;
auip->init.util.smbcs = get_kb_value(sub_param + 5, argv, ip);
@@ -1061,6 +1227,7 @@ static void
handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
{
struct au_init *aui[] = {
+ &init->sbmbc_alloc,
&init->binary_alloc,
&init->std_alloc,
&init->ets_alloc,
@@ -1088,6 +1255,9 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
case 'B':
handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i);
break;
+ case 'C':
+ handle_au_arg(&init->sbmbc_alloc, &argv[i][3], argv, &i);
+ break;
case 'D':
handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i);
break;
@@ -1337,7 +1507,6 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
argv[j++] = argv[i];
}
*argc = j;
-
}
static char *type_no_str(ErtsAlcType_t n)
@@ -1393,6 +1562,33 @@ void erts_alloc_reg_scheduler_id(Uint id)
erts_tsd_set(thr_ix_key, (void *)(long) ix);
}
+static void
+no_verify(Allctr_t *allctr)
+{
+
+}
+
+erts_alloc_verify_func_t
+erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr)
+{
+ if (erts_allctrs_info[ERTS_ALC_A_TEMPORARY].alloc_util
+ && erts_allctrs_info[ERTS_ALC_A_TEMPORARY].thr_spec) {
+ ErtsAllocatorThrSpec_t *tspec;
+ tspec = &erts_allctr_thr_spec[ERTS_ALC_A_TEMPORARY];
+ if (!tspec->all_thr_safe) {
+ int ix = erts_alc_get_thr_ix();
+
+ if (ix < tspec->size) {
+ *allctr = tspec->allctr[ix];
+ return erts_alcu_verify_unused;
+ }
+ }
+ }
+
+ *allctr = NULL;
+ return no_verify;
+}
+
__decl_noreturn void
erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...)
{
@@ -1483,10 +1679,10 @@ erts_realloc_n_enomem(ErtsAlcType_t n, void *ptr, Uint size)
erts_alc_fatal_error(ERTS_ALC_E_NOMEM, ERTS_ALC_O_REALLOC, n, size);
}
-static ERTS_INLINE Uint
+static ERTS_INLINE UWord
alcu_size(ErtsAlcType_t ai)
{
- Uint res = 0;
+ UWord res = 0;
ASSERT(erts_allctrs_info[ai].enabled);
ASSERT(erts_allctrs_info[ai].alloc_util);
@@ -1518,6 +1714,49 @@ alcu_size(ErtsAlcType_t ai)
return res;
}
+#if HALFWORD_HEAP
+static ERTS_INLINE int
+alcu_is_low(ErtsAlcType_t ai)
+{
+ int is_low = 0;
+ ASSERT(erts_allctrs_info[ai].enabled);
+ ASSERT(erts_allctrs_info[ai].alloc_util);
+
+ if (!erts_allctrs_info[ai].thr_spec) {
+ Allctr_t *allctr = erts_allctrs_info[ai].extra;
+ is_low = allctr->mseg_opt.low_mem;
+ }
+ else {
+ ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ai];
+ int i;
+# ifdef DEBUG
+ int found_one = 0;
+# endif
+
+ ASSERT(tspec->all_thr_safe);
+ ASSERT(tspec->enabled);
+
+ for (i = tspec->size - 1; i >= 0; i--) {
+ Allctr_t *allctr = tspec->allctr[i];
+ if (allctr) {
+# ifdef DEBUG
+ if (!found_one) {
+ is_low = allctr->mseg_opt.low_mem;
+ found_one = 1;
+ }
+ else ASSERT(is_low == allctr->mseg_opt.low_mem);
+# else
+ is_low = allctr->mseg_opt.low_mem;
+ break;
+# endif
+ }
+ }
+ ASSERT(found_one);
+ }
+ return is_low;
+}
+#endif /* HALFWORD */
+
Eterm
erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
{
@@ -1534,23 +1773,28 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
int code;
int ets;
int maximum;
+#if HALFWORD_HEAP
+ int low;
+#endif
} want = {0};
struct {
- Uint total;
- Uint processes;
- Uint processes_used;
- Uint system;
- Uint atom;
- Uint atom_used;
- Uint binary;
- Uint code;
- Uint ets;
- Uint maximum;
+ UWord total;
+ UWord processes;
+ UWord processes_used;
+ UWord system;
+ UWord atom;
+ UWord atom_used;
+ UWord binary;
+ UWord code;
+ UWord ets;
+ UWord maximum;
+#if HALFWORD_HEAP
+ UWord low;
+#endif
} size = {0};
- Eterm atoms[sizeof(size)/sizeof(Uint)];
- Uint *uintps[sizeof(size)/sizeof(Uint)];
- Eterm euints[sizeof(size)/sizeof(Uint)];
- int need_atom;
+ Eterm atoms[sizeof(size)/sizeof(UWord)];
+ UWord *uintps[sizeof(size)/sizeof(UWord)];
+ Eterm euints[sizeof(size)/sizeof(UWord)];
int want_tot_or_sys;
int length;
Eterm res = THE_NON_VALUE;
@@ -1602,15 +1846,20 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
atoms[length] = am_maximum;
uintps[length++] = &size.maximum;
}
-
+#if HALFWORD_HEAP
+ want.low = 1;
+ atoms[length] = am_low;
+ uintps[length++] = &size.low;
+#endif
}
else {
- Eterm tmp_heap[2];
+ DeclareTmpHeapNoproc(tmp_heap,2);
Eterm wanted_list;
if (is_nil(earg))
return NIL;
+ UseTmpHeapNoproc(2);
if (is_not_atom(earg))
wanted_list = earg;
else {
@@ -1690,25 +1939,41 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
atoms[length] = am_maximum;
uintps[length++] = &size.maximum;
}
- }
- else
+ } else {
+ UnUseTmpHeapNoproc(2);
return am_badarg;
+ }
break;
+#if HALFWORD_HEAP
+ case am_low:
+ if (!want.low) {
+ want.low = 1;
+ atoms[length] = am_low;
+ uintps[length++] = &size.low;
+ }
+ break;
+#endif
default:
+ UnUseTmpHeapNoproc(2);
return am_badarg;
}
wanted_list = CDR(list_val(wanted_list));
}
+ UnUseTmpHeapNoproc(2);
if (is_not_nil(wanted_list))
return am_badarg;
}
- /* All alloc_util allocators *have* to be enabled */
+ /* All alloc_util allocators except sbmbc_alloc *have* to be enabled */
for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) {
switch (ai) {
case ERTS_ALC_A_SYSTEM:
case ERTS_ALC_A_FIXED_SIZE:
+ case ERTS_ALC_A_SBMBC:
+#if HALFWORD_HEAP
+ case ERTS_ALC_A_SBMBC_LOW:
+#endif
break;
default:
if (!erts_allctrs_info[ai].enabled
@@ -1721,7 +1986,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
ASSERT(length <= sizeof(atoms)/sizeof(Eterm));
ASSERT(length <= sizeof(euints)/sizeof(Eterm));
- ASSERT(length <= sizeof(uintps)/sizeof(Uint));
+ ASSERT(length <= sizeof(uintps)/sizeof(UWord));
if (proc) {
@@ -1734,21 +1999,26 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
/* Calculate values needed... */
want_tot_or_sys = want.total || want.system;
- need_atom = ERTS_MEM_NEED_ALL_ALCU || want.atom;
if (ERTS_MEM_NEED_ALL_ALCU) {
size.total = 0;
for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) {
if (erts_allctrs_info[ai].alloc_util) {
- Uint *save;
- Uint asz;
+ UWord *save;
+ UWord asz;
switch (ai) {
case ERTS_ALC_A_TEMPORARY:
/*
* Often not thread safe and usually never
* contain any allocated memory.
*/
+ case ERTS_ALC_A_SBMBC:
+ /* Included in other allocators */
+#if HALFWORD_HEAP
+ case ERTS_ALC_A_SBMBC_LOW:
+ /* Included in other allocators */
+#endif
continue;
case ERTS_ALC_A_EHEAP:
save = &size.processes;
@@ -1767,6 +2037,11 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
if (save)
*save = asz;
size.total += asz;
+#if HALFWORD_HEAP
+ if (alcu_is_low(ai)) {
+ size.low += asz;
+ }
+#endif
}
}
}
@@ -1774,7 +2049,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
if (want_tot_or_sys || want.processes || want.processes_used) {
- Uint tmp;
+ UWord tmp;
if (ERTS_MEM_NEED_ALL_ALCU)
tmp = size.processes;
@@ -1789,6 +2064,9 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
size.processes = size.processes_used = tmp;
+#if HALFWORD_HEAP
+ /* BUG: We ignore link and monitor memory */
+#else
erts_fix_info(ERTS_ALC_T_NLINK_SH, &efi);
size.processes += efi.total;
size.processes_used += efi.used;
@@ -1796,6 +2074,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
erts_fix_info(ERTS_ALC_T_MONITOR_SH, &efi);
size.processes += efi.total;
size.processes_used += efi.used;
+#endif
erts_fix_info(ERTS_ALC_T_PROC, &efi);
size.processes += efi.total;
@@ -1832,8 +2111,12 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
erts_fix_info(ERTS_ALC_T_MODULE, &efi);
size.code += efi.used;
size.code += export_table_sz();
+#if HALFWORD_HEAP
+ size.code += export_list_size() * sizeof(Export);
+#else
erts_fix_info(ERTS_ALC_T_EXPORT, &efi);
size.code += efi.used;
+#endif
size.code += erts_fun_table_sz();
erts_fix_info(ERTS_ALC_T_FUN_ENTRY, &efi);
size.code += efi.used;
@@ -1879,9 +2162,9 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
if (only_one_value) {
ASSERT(length == 1);
hsz = 0;
- erts_bld_uint(NULL, &hsz, *uintps[0]);
+ erts_bld_uword(NULL, &hsz, *uintps[0]);
hp = hsz ? HAlloc((Process *) proc, hsz) : NULL;
- res = erts_bld_uint(&hp, NULL, *uintps[0]);
+ res = erts_bld_uword(&hp, NULL, *uintps[0]);
}
else {
Uint **hpp = NULL;
@@ -1891,7 +2174,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
while (1) {
int i;
for (i = 0; i < length; i++)
- euints[i] = erts_bld_uint(hpp, hszp, *uintps[i]);
+ euints[i] = erts_bld_uword(hpp, hszp, *uintps[i]);
res = erts_bld_2tup_list(hpp, hszp, length, atoms, euints);
if (hpp)
break;
@@ -2060,11 +2343,11 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc)
for (i = 0; i < length; i++) {
switch (values[i].arity) {
case 2:
- erts_print(to, arg, "%s: %bpu\n",
+ erts_print(to, arg, "%s: %beu\n",
values[i].name, values[i].ui[0]);
break;
case 3:
- erts_print(to, arg, "%s: %bpu %bpu\n",
+ erts_print(to, arg, "%s: %beu %beu\n",
values[i].name, values[i].ui[0], values[i].ui[1]);
break;
default:
@@ -2285,8 +2568,8 @@ erts_allocator_info_term(void *proc, Eterm which_alloc, int only_sz)
SysAllocStat sas;
Eterm opts_am;
Eterm opts;
- Eterm as[4];
- Eterm ts[4];
+ Eterm as[4]; /* Ok even if !HEAP_ON_C_STACK, not really heap data on stack */
+ Eterm ts[4]; /* Ok even if !HEAP_ON_C_STACK, not really heap data on stack */
int l;
if (only_sz)
@@ -2302,13 +2585,8 @@ erts_allocator_info_term(void *proc, Eterm which_alloc, int only_sz)
l = 0;
as[l] = am_atom_put("e", 1);
ts[l++] = am_true;
-#ifdef ELIB_ALLOC_IS_CLIB
- as[l] = am_atom_put("m", 1);
- ts[l++] = am_atom_put("elib", 4);
-#else
as[l] = am_atom_put("m", 1);
ts[l++] = am_atom_put("libc", 4);
-#endif
if(sas.trim_threshold >= 0) {
as[l] = am_atom_put("tt", 2);
ts[l++] = erts_bld_uint(hpp, szp,
@@ -2462,11 +2740,7 @@ erts_allocator_info(int to, void *arg)
case ERTS_ALC_A_SYSTEM: {
SysAllocStat sas;
erts_print(to, arg, "option e: true\n");
-#ifdef ELIB_ALLOC_IS_CLIB
- erts_print(to, arg, "option m: elib\n");
-#else
erts_print(to, arg, "option m: libc\n");
-#endif
sys_alloc_stat(&sas);
if(sas.trim_threshold >= 0)
erts_print(to, arg, "option tt: %d\n", sas.trim_threshold);
@@ -2570,13 +2844,8 @@ erts_allocator_options(void *proc)
switch (a) {
case ERTS_ALC_A_SYSTEM:
-#ifdef ELIB_ALLOC_IS_CLIB
- as[l] = am_atom_put("m", 1);
- ts[l++] = am_atom_put("elib", 4);
-#else
as[l] = am_atom_put("m", 1);
ts[l++] = am_atom_put("libc", 4);
-#endif
if(sas.trim_threshold >= 0) {
as[l] = am_atom_put("tt", 2);
ts[l++] = erts_bld_uint(hpp, szp,
@@ -2647,23 +2916,7 @@ erts_allocator_options(void *proc)
features = length ? erts_bld_list(hpp, szp, length, terms) : NIL;
-#if defined(ELIB_ALLOC_IS_CLIB)
- {
- Eterm version;
- int i;
- int ver[5];
- i = sscanf(ERLANG_VERSION,
- "%d.%d.%d.%d.%d",
- &ver[0], &ver[1], &ver[2], &ver[3], &ver[4]);
-
- version = NIL;
- for(i--; i >= 0; i--)
- version = erts_bld_cons(hpp, szp, make_small(ver[i]), version);
-
- res = erts_bld_tuple(hpp, szp, 4,
- am_elib_malloc, version, features, settings);
- }
-#elif defined(__GLIBC__)
+#if defined(__GLIBC__)
{
Eterm AM_glibc = am_atom_put("glibc", 5);
Eterm version;
@@ -2707,6 +2960,29 @@ erts_allocator_options(void *proc)
return res;
}
+void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size)
+{
+ UWord v = (UWord) erts_alloc(type, size + (ERTS_CACHE_LINE_SIZE-1));
+
+#ifdef VALGRIND
+ { /* Avoid Leak_PossiblyLost */
+ static UWord vg_root_set[10];
+ static unsigned ix = 0;
+ if (ix >= sizeof(vg_root_set) / sizeof(*vg_root_set)) {
+ erl_exit(ERTS_ABORT_EXIT, "Too many erts_alloc_permanent_cache_aligned's\n");
+ }
+ vg_root_set[ix++] = v; /* not thread safe */
+ }
+#endif
+
+ if (v & ERTS_CACHE_LINE_MASK) {
+ v = (v & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE;
+ }
+ ASSERT((v & ERTS_CACHE_LINE_MASK) == 0);
+ return (void*)v;
+}
+
+
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
* Deprecated functions *
* *
@@ -2747,6 +3023,7 @@ unsigned long erts_alc_test(unsigned long op,
case 0x2: return erts_bfalc_test(op, a1, a2);
case 0x3: return erts_afalc_test(op, a1, a2);
case 0x4: return erts_mseg_test(op, a1, a2, a3);
+ case 0x5: return erts_aoffalc_test(op, a1, a2);
case 0xf:
switch (op) {
case 0xf00:
@@ -2790,6 +3067,7 @@ unsigned long erts_alc_test(unsigned long op,
init.atype = GOODFIT;
init.init.util.name_prefix = (char *) a1;
init.init.util.ts = a2 ? 1 : 0;
+ init.init.util.sbmbct = 0;
if ((char **) a3) {
char **argv = (char **) a3;
@@ -2825,6 +3103,14 @@ unsigned long erts_alc_test(unsigned long op,
&init.init.af,
&init.init.util);
break;
+ case AOFIRSTFIT:
+ allctr = erts_aoffalc_start((AOFFAllctr_t *)
+ erts_alloc(ERTS_ALC_T_UNDEF,
+ sizeof(AOFFAllctr_t)),
+ &init.init.aoff,
+ &init.init.util);
+ break;
+
default:
ASSERT(0);
allctr = NULL;
@@ -2859,12 +3145,10 @@ unsigned long erts_alc_test(unsigned long op,
break;
}
case 0xf0a:
- if (ethr_mutex_lock((ethr_mutex *) a1) != 0)
- ERTS_ALC_TEST_ABORT;
+ ethr_mutex_lock((ethr_mutex *) a1);
break;
case 0xf0b:
- if (ethr_mutex_unlock((ethr_mutex *) a1) != 0)
- ERTS_ALC_TEST_ABORT;
+ ethr_mutex_unlock((ethr_mutex *) a1);
break;
case 0xf0c: {
ethr_cond *cnd = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_cond));
@@ -2880,31 +3164,21 @@ unsigned long erts_alc_test(unsigned long op,
break;
}
case 0xf0e:
- if (ethr_cond_broadcast((ethr_cond *) a1) != 0)
- ERTS_ALC_TEST_ABORT;
+ ethr_cond_broadcast((ethr_cond *) a1);
break;
case 0xf0f: {
int res;
do {
res = ethr_cond_wait((ethr_cond *) a1, (ethr_mutex *) a2);
} while (res == EINTR);
- if (res != 0)
- ERTS_ALC_TEST_ABORT;
break;
}
case 0xf10: {
ethr_tid *tid = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_tid));
-#ifdef ERTS_ENABLE_LOCK_COUNT
- if (erts_lcnt_thr_create(tid,
- (void * (*)(void *)) a1,
- (void *) a2,
- NULL) != 0)
-#else
if (ethr_thr_create(tid,
(void * (*)(void *)) a1,
(void *) a2,
NULL) != 0)
-#endif
ERTS_ALC_TEST_ABORT;
return (unsigned long) tid;
}
@@ -2943,10 +3217,13 @@ unsigned long erts_alc_test(unsigned long op,
#undef PRINT_OPS
#endif
+#ifdef HARD_DEBUG
+#define FENCE_SZ (4*sizeof(UWord))
+#else
+#define FENCE_SZ (3*sizeof(UWord))
+#endif
-#define FENCE_SZ (3*sizeof(Uint))
-
-#ifdef ARCH_64
+#if defined(ARCH_64)
#define FENCE_PATTERN 0xABCDEF97ABCDEF97
#else
#define FENCE_PATTERN 0xABCDEF97
@@ -2956,7 +3233,7 @@ unsigned long erts_alc_test(unsigned long op,
#define TYPE_PATTERN_SHIFT 16
#define FIXED_FENCE_PATTERN_MASK \
- (~((Uint) (TYPE_PATTERN_MASK << TYPE_PATTERN_SHIFT)))
+ (~((UWord) (TYPE_PATTERN_MASK << TYPE_PATTERN_SHIFT)))
#define FIXED_FENCE_PATTERN \
(FENCE_PATTERN & FIXED_FENCE_PATTERN_MASK)
@@ -2966,22 +3243,170 @@ unsigned long erts_alc_test(unsigned long op,
#define GET_TYPE_OF_PATTERN(P) \
(((P) >> TYPE_PATTERN_SHIFT) & TYPE_PATTERN_MASK)
+#ifdef HARD_DEBUG
+
+#define ERL_ALC_HDBG_MAX_MBLK 100000
+#define ERTS_ALC_O_CHECK -1
+
+typedef struct hdbg_mblk_ hdbg_mblk;
+struct hdbg_mblk_ {
+ hdbg_mblk *next;
+ hdbg_mblk *prev;
+ void *p;
+ Uint s;
+ ErtsAlcType_t n;
+};
+
+static hdbg_mblk hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK];
+
+static hdbg_mblk *free_hdbg_mblks;
+static hdbg_mblk *used_hdbg_mblks;
+static erts_mtx_t hdbg_mblk_mtx;
+
+static void
+hdbg_init(void)
+{
+ int i;
+ for (i = 0; i < ERL_ALC_HDBG_MAX_MBLK-1; i++)
+ hdbg_mblks[i].next = &hdbg_mblks[i+1];
+ hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK-1].next = NULL;
+ free_hdbg_mblks = &hdbg_mblks[0];
+ used_hdbg_mblks = NULL;
+ erts_mtx_init(&hdbg_mblk_mtx, "erts_alloc_hard_debug");
+}
+
+static void *check_memory_fence(void *ptr,
+ Uint *size,
+ ErtsAlcType_t n,
+ int func);
+void erts_hdbg_chk_blks(void);
+
+void
+erts_hdbg_chk_blks(void)
+{
+ hdbg_mblk *mblk;
+
+ erts_mtx_lock(&hdbg_mblk_mtx);
+ for (mblk = used_hdbg_mblks; mblk; mblk = mblk->next) {
+ Uint sz;
+ check_memory_fence(mblk->p, &sz, mblk->n, ERTS_ALC_O_CHECK);
+ ASSERT(sz == mblk->s);
+ }
+ erts_mtx_unlock(&hdbg_mblk_mtx);
+}
+
+static hdbg_mblk *
+hdbg_alloc(void *p, Uint s, ErtsAlcType_t n)
+{
+ hdbg_mblk *mblk;
+
+ erts_mtx_lock(&hdbg_mblk_mtx);
+ mblk = free_hdbg_mblks;
+ if (!mblk) {
+ erts_fprintf(stderr,
+ "Ran out of debug blocks; please increase "
+ "ERL_ALC_HDBG_MAX_MBLK=%d and recompile!\n",
+ ERL_ALC_HDBG_MAX_MBLK);
+ abort();
+ }
+ free_hdbg_mblks = mblk->next;
+
+ mblk->p = p;
+ mblk->s = s;
+ mblk->n = n;
+
+ mblk->next = used_hdbg_mblks;
+ mblk->prev = NULL;
+ if (used_hdbg_mblks)
+ used_hdbg_mblks->prev = mblk;
+ used_hdbg_mblks = mblk;
+ erts_mtx_unlock(&hdbg_mblk_mtx);
+ return (void *) mblk;
+}
+
+static void
+hdbg_free(hdbg_mblk *mblk)
+{
+ erts_mtx_lock(&hdbg_mblk_mtx);
+ if (mblk->next)
+ mblk->next->prev = mblk->prev;
+ if (mblk->prev)
+ mblk->prev->next = mblk->next;
+ else
+ used_hdbg_mblks = mblk->next;
+
+ mblk->next = free_hdbg_mblks;
+ free_hdbg_mblks = mblk;
+ erts_mtx_unlock(&hdbg_mblk_mtx);
+}
+
+#endif
+
+#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
+static void *check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func);
+
+void check_allocated_block( Uint type, void *blk)
+{
+ Uint dummy;
+ check_memory_fence(blk, &dummy, ERTS_ALC_T2N(type), ERTS_ALC_O_FREE);
+}
+
+void check_allocators(void)
+{
+ int i;
+ if (!erts_initialized)
+ return;
+ for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; ++i) {
+ if (erts_allctrs_info[i].alloc_util) {
+ ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) erts_allctrs[i].extra;
+ Allctr_t *allctr = real_af->extra;
+ Carrier_t *ct;
+#ifdef USE_THREADS
+ if (allctr->thread_safe)
+ erts_mtx_lock(&allctr->mutex);
+#endif
+
+ if (allctr->check_mbc) {
+ for (ct = allctr->mbc_list.first; ct; ct = ct->next) {
+ fprintf(stderr,"Checking allocator %d\r\n",i);
+ allctr->check_mbc(allctr,ct);
+ }
+ }
+#ifdef USE_THREADS
+ if (allctr->thread_safe)
+ erts_mtx_unlock(&allctr->mutex);
+#endif
+ }
+ }
+}
+#endif
static void *
set_memory_fence(void *ptr, Uint sz, ErtsAlcType_t n)
{
- Uint *ui_ptr;
- Uint pattern;
+ UWord *ui_ptr;
+ UWord pattern;
+#ifdef HARD_DEBUG
+ hdbg_mblk **mblkpp;
+#endif
if (!ptr)
return NULL;
- ui_ptr = (Uint *) ptr;
+ ui_ptr = (UWord *) ptr;
pattern = MK_PATTERN(n);
-
+
+#ifdef HARD_DEBUG
+ mblkpp = (hdbg_mblk **) ui_ptr++;
+#endif
+
*(ui_ptr++) = sz;
*(ui_ptr++) = pattern;
- memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(Uint));
+ memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(UWord));
+
+#ifdef HARD_DEBUG
+ *mblkpp = hdbg_alloc((void *) ui_ptr, sz, n);
+#endif
return (void *) ui_ptr;
}
@@ -2991,16 +3416,22 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
{
Uint sz;
Uint found_type;
- Uint pre_pattern;
- Uint post_pattern;
- Uint *ui_ptr;
+ UWord pre_pattern;
+ UWord post_pattern;
+ UWord *ui_ptr;
+#ifdef HARD_DEBUG
+ hdbg_mblk *mblk;
+#endif
if (!ptr)
return NULL;
- ui_ptr = (Uint *) ptr;
+ ui_ptr = (UWord *) ptr;
pre_pattern = *(--ui_ptr);
*size = sz = *(--ui_ptr);
+#ifdef HARD_DEBUG
+ mblk = (hdbg_mblk *) *(--ui_ptr);
+#endif
found_type = GET_TYPE_OF_PATTERN(pre_pattern);
if (pre_pattern != MK_PATTERN(n)) {
@@ -3011,7 +3442,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
(unsigned long) ptr);
}
- memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(Uint));
+ memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(UWord));
if (post_pattern != MK_PATTERN(n)
|| pre_pattern != post_pattern) {
@@ -3056,6 +3487,17 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
(unsigned long) ptr, (unsigned long) sz, ftype, op_str, otype);
}
+#ifdef HARD_DEBUG
+ switch (func) {
+ case ERTS_ALC_O_REALLOC:
+ case ERTS_ALC_O_FREE:
+ hdbg_free(mblk);
+ break;
+ default:
+ break;
+ }
+#endif
+
return (void *) ui_ptr;
}
@@ -3068,6 +3510,10 @@ debug_alloc(ErtsAlcType_t n, void *extra, Uint size)
Uint dsize;
void *res;
+#ifdef HARD_DEBUG
+ erts_hdbg_chk_blks();
+#endif
+
ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX);
dsize = size + FENCE_SZ;
res = (*real_af->alloc)(n, real_af->extra, dsize);
@@ -3097,13 +3543,17 @@ debug_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size)
dsize = size + FENCE_SZ;
dptr = check_memory_fence(ptr, &old_size, n, ERTS_ALC_O_REALLOC);
+#ifdef HARD_DEBUG
+ erts_hdbg_chk_blks();
+#endif
+
if (old_size > size)
sys_memset((void *) (((char *) ptr) + size),
0xf,
sizeof(Uint) + old_size - size);
res = (*real_af->realloc)(n, real_af->extra, dptr, dsize);
-
+
res = set_memory_fence(res, size, n);
#ifdef PRINT_OPS
@@ -3133,6 +3583,10 @@ debug_free(ErtsAlcType_t n, void *extra, void *ptr)
fprintf(stderr, "free(%s, 0x%lx)\r\n", ERTS_ALC_N2TD(n), (Uint) ptr);
#endif
+#ifdef HARD_DEBUG
+ erts_hdbg_chk_blks();
+#endif
+
}
static Uint
@@ -3152,6 +3606,4 @@ install_debug_functions(void)
return FENCE_SZ;
}
-
-
#endif /* #ifdef DEBUG */
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-01 13:11:24 +0000