/*
* %CopyrightBegin%
*
* Copyright Ericsson AB 2002-2012. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
* compliance with the License. You should have received a copy of the
* Erlang Public License along with this software. If not, it can be
* retrieved online at http://www.erlang.org/.
*
* Software distributed under the License is distributed on an "AS IS"
* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
* the License for the specific language governing rights and limitations
* under the License.
*
* %CopyrightEnd%
*/
/*
* Description: Management of memory allocators.
*
* Author: Rickard Green
*/
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif
#define ERTS_ALLOC_C__
#define ERTS_ALC_INTERNAL__
#include "sys.h"
#define ERL_THREADS_EMU_INTERNAL__
#include "erl_threads.h"
#include "global.h"
#include "erl_db.h"
#include "erl_binary.h"
#include "erl_bits.h"
#include "erl_instrument.h"
#include "erl_mseg.h"
#include "erl_monitors.h"
#include "erl_bif_timer.h"
#include "erl_cpu_topology.h"
#include "erl_thr_queue.h"
#if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE)
#include "erl_check_io.h"
#endif
#define GET_ERL_GF_ALLOC_IMPL
#include "erl_goodfit_alloc.h"
#define GET_ERL_BF_ALLOC_IMPL
#include "erl_bestfit_alloc.h"
#define GET_ERL_AF_ALLOC_IMPL
#include "erl_afit_alloc.h"
#define GET_ERL_AOFF_ALLOC_IMPL
#include "erl_ao_firstfit_alloc.h"
#if ERTS_MAX_NO_OF_SCHEDULERS > ERTS_AU_MAX_PREF_ALLOC_INSTANCES
# error "Too many schedulers; cannot create that many pref alloc instances"
#endif
#define ERTS_ALC_FIX_TYPE_IX(T) \
(ERTS_ALC_T2N((T)) - ERTS_ALC_N_MIN_A_FIXED_SIZE)
#define ERTS_ALC_DEFAULT_MAX_THR_PREF ERTS_MAX_NO_OF_SCHEDULERS
#if defined(SMALL_MEMORY) || defined(PURIFY) || defined(VALGRIND)
#define AU_ALLOC_DEFAULT_ENABLE(X) 0
#else
#define AU_ALLOC_DEFAULT_ENABLE(X) (X)
#endif
#ifdef DEBUG
static Uint install_debug_functions(void);
#if 0
#define HARD_DEBUG
#ifdef __GNUC__
#warning "* * * * * * * * * * * * * *"
#warning "* HARD DEBUG IS ENABLED! *"
#warning "* * * * * * * * * * * * * *"
#endif
#endif
#endif
ErtsAllocatorFunctions_t erts_allctrs[ERTS_ALC_A_MAX+1];
ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1];
ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1];
#define ERTS_MIN(A, B) ((A) < (B) ? (A) : (B))
#define ERTS_MAX(A, B) ((A) > (B) ? (A) : (B))
typedef union {
GFAllctr_t gfa;
char align_gfa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(GFAllctr_t))];
BFAllctr_t bfa;
char align_bfa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(BFAllctr_t))];
AFAllctr_t afa;
char align_afa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AFAllctr_t))];
AOFFAllctr_t aoffa;
char align_aoffa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AOFFAllctr_t))];
} ErtsAllocatorState_t;
static ErtsAllocatorState_t sbmbc_alloc_state;
static ErtsAllocatorState_t std_alloc_state;
static ErtsAllocatorState_t ll_alloc_state;
#if HALFWORD_HEAP
static ErtsAllocatorState_t sbmbc_low_alloc_state;
static ErtsAllocatorState_t std_low_alloc_state;
static ErtsAllocatorState_t ll_low_alloc_state;
#endif
static ErtsAllocatorState_t sl_alloc_state;
static ErtsAllocatorState_t temp_alloc_state;
static ErtsAllocatorState_t eheap_alloc_state;
static ErtsAllocatorState_t binary_alloc_state;
static ErtsAllocatorState_t ets_alloc_state;
static ErtsAllocatorState_t driver_alloc_state;
static ErtsAllocatorState_t fix_alloc_state;
typedef struct {
erts_smp_atomic32_t refc;
int only_sz;
Uint req_sched;
Process *proc;
Eterm ref;
Eterm ref_heap[REF_THING_SIZE];
int allocs[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+1+2];
} ErtsAllocInfoReq;
#define ERTS_ALC_INFO_A_ALLOC_UTIL (ERTS_ALC_A_MAX + 1)
#define ERTS_ALC_INFO_A_MSEG_ALLOC (ERTS_ALC_A_MAX + 2)
#define ERTS_ALC_INFO_A_MAX ERTS_ALC_INFO_A_MSEG_ALLOC
#if !HALFWORD_HEAP
ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(aireq,
ErtsAllocInfoReq,
5,
ERTS_ALC_T_AINFO_REQ)
#else
static ERTS_INLINE ErtsAllocInfoReq *
aireq_alloc(void)
{
return erts_alloc(ERTS_ALC_T_AINFO_REQ, sizeof(ErtsAllocInfoReq));
}
static ERTS_INLINE void
aireq_free(ErtsAllocInfoReq *ptr)
{
erts_free(ERTS_ALC_T_AINFO_REQ, ptr);
}
#endif
ErtsAlcType_t erts_fix_core_allocator_ix;
enum allctr_type {
GOODFIT,
BESTFIT,
AFIT,
AOFIRSTFIT
};
struct au_init {
int enable;
int thr_spec;
enum allctr_type atype;
struct {
AllctrInit_t util;
GFAllctrInit_t gf;
BFAllctrInit_t bf;
AFAllctrInit_t af;
AOFFAllctrInit_t aoff;
} init;
struct {
int mmbcs;
int lmbcs;
int smbcs;
int mmmbc;
} default_;
};
#define DEFAULT_ALLCTR_INIT { \
ERTS_DEFAULT_ALLCTR_INIT, \
ERTS_DEFAULT_GF_ALLCTR_INIT, \
ERTS_DEFAULT_BF_ALLCTR_INIT, \
ERTS_DEFAULT_AF_ALLCTR_INIT, \
ERTS_DEFAULT_AOFF_ALLCTR_INIT \
}
typedef struct {
int erts_alloc_config;
#if HAVE_ERTS_MSEG
ErtsMsegInit_t mseg;
#endif
int trim_threshold;
int top_pad;
AlcUInit_t alloc_util;
struct {
int stat;
int map;
char *mtrace;
char *nodename;
} instr;
struct au_init sbmbc_alloc;
struct au_init sl_alloc;
struct au_init std_alloc;
struct au_init ll_alloc;
struct au_init temp_alloc;
struct au_init eheap_alloc;
struct au_init binary_alloc;
struct au_init ets_alloc;
struct au_init driver_alloc;
struct au_init fix_alloc;
#if HALFWORD_HEAP
struct au_init sbmbc_low_alloc;
struct au_init std_low_alloc;
struct au_init ll_low_alloc;
#endif
} erts_alc_hndl_args_init_t;
#define ERTS_AU_INIT__ {0, 0, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}}
#define SET_DEFAULT_ALLOC_OPTS(IP) \
do { \
struct au_init aui__ = ERTS_AU_INIT__; \
sys_memcpy((void *) (IP), (void *) &aui__, sizeof(struct au_init)); \
} while (0)
static void
set_default_sbmbc_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = 0;
ip->thr_spec = 0;
ip->atype = BESTFIT;
ip->init.bf.ao = 1;
ip->init.util.ramv = 0;
ip->init.util.mmsbc = 0;
ip->init.util.mmmbc = 500;
ip->init.util.sbct = ~((UWord) 0);
ip->init.util.name_prefix = "sbmbc_";
ip->init.util.alloc_no = ERTS_ALC_A_SBMBC;
#ifndef SMALL_MEMORY
ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */
#else
ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */
#endif
ip->init.util.ts = ERTS_ALC_MTA_SBMBC;
ip->init.util.asbcst = 0;
ip->init.util.rsbcst = 0;
ip->init.util.rsbcmt = 0;
ip->init.util.rmbcmt = 0;
ip->init.util.sbmbct = 0;
ip->init.util.sbmbcs = 0;
}
static void
set_default_sl_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
ip->thr_spec = 1;
ip->atype = GOODFIT;
ip->init.util.name_prefix = "sl_";
ip->init.util.mmmbc = 5;
ip->init.util.alloc_no = ERTS_ALC_A_SHORT_LIVED;
#ifndef SMALL_MEMORY
ip->init.util.mmbcs = 128*1024; /* Main carrier size */
#else
ip->init.util.mmbcs = 32*1024; /* Main carrier size */
#endif
ip->init.util.ts = ERTS_ALC_MTA_SHORT_LIVED;
ip->init.util.rsbcst = 80;
#if HALFWORD_HEAP
ip->init.util.force = 1;
ip->init.util.low_mem = 1;
#endif
}
static void
set_default_std_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
ip->thr_spec = 1;
ip->atype = BESTFIT;
ip->init.util.name_prefix = "std_";
ip->init.util.mmmbc = 5;
ip->init.util.alloc_no = ERTS_ALC_A_STANDARD;
#ifndef SMALL_MEMORY
ip->init.util.mmbcs = 128*1024; /* Main carrier size */
#else
ip->init.util.mmbcs = 32*1024; /* Main carrier size */
#endif
ip->init.util.ts = ERTS_ALC_MTA_STANDARD;
}
static void
set_default_ll_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
ip->thr_spec = 0;
ip->atype = BESTFIT;
ip->init.bf.ao = 1;
ip->init.util.ramv = 0;
ip->init.util.mmsbc = 0;
ip->init.util.mmmbc = 0;
ip->init.util.sbct = ~((UWord) 0);
ip->init.util.name_prefix = "ll_";
ip->init.util.alloc_no = ERTS_ALC_A_LONG_LIVED;
#ifndef SMALL_MEMORY
ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */
#else
ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */
#endif
ip->init.util.ts = ERTS_ALC_MTA_LONG_LIVED;
ip->init.util.asbcst = 0;
ip->init.util.rsbcst = 0;
ip->init.util.rsbcmt = 0;
ip->init.util.rmbcmt = 0;
ip->init.util.sbmbct = 0;
ip->init.util.sbmbcs = 0;
}
static void
set_default_temp_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
ip->thr_spec = 1;
ip->atype = AFIT;
ip->init.util.name_prefix = "temp_";
ip->init.util.alloc_no = ERTS_ALC_A_TEMPORARY;
#ifndef SMALL_MEMORY
ip->init.util.mmbcs = 128*1024; /* Main carrier size */
#else
ip->init.util.mmbcs = 32*1024; /* Main carrier size */
#endif
ip->init.util.ts = ERTS_ALC_MTA_TEMPORARY;
ip->init.util.rsbcst = 90;
ip->init.util.rmbcmt = 100;
#if HALFWORD_HEAP
ip->init.util.force = 1;
ip->init.util.low_mem = 1;
#endif
}
static void
set_default_eheap_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
ip->thr_spec = 1;
ip->atype = GOODFIT;
ip->init.util.mmmbc = 100;
ip->init.util.name_prefix = "eheap_";
ip->init.util.alloc_no = ERTS_ALC_A_EHEAP;
#ifndef SMALL_MEMORY
ip->init.util.mmbcs = 512*1024; /* Main carrier size */
#else
ip->init.util.mmbcs = 256*1024; /* Main carrier size */
#endif
ip->init.util.ts = ERTS_ALC_MTA_EHEAP;
ip->init.util.rsbcst = 50;
#if HALFWORD_HEAP
ip->init.util.force = 1;
ip->init.util.low_mem = 1;
#endif
}
static void
set_default_binary_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
ip->thr_spec = 1;
ip->atype = BESTFIT;
ip->init.util.mmmbc = 50;
ip->init.util.name_prefix = "binary_";
ip->init.util.alloc_no = ERTS_ALC_A_BINARY;
#ifndef SMALL_MEMORY
ip->init.util.mmbcs = 128*1024; /* Main carrier size */
#else
ip->init.util.mmbcs = 32*1024; /* Main carrier size */
#endif
ip->init.util.ts = ERTS_ALC_MTA_BINARY;
}
static void
set_default_ets_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
ip->thr_spec = 1;
ip->atype = BESTFIT;
ip->init.util.mmmbc = 100;
ip->init.util.name_prefix = "ets_";
ip->init.util.alloc_no = ERTS_ALC_A_ETS;
#ifndef SMALL_MEMORY
ip->init.util.mmbcs = 128*1024; /* Main carrier size */
#else
ip->init.util.mmbcs = 32*1024; /* Main carrier size */
#endif
ip->init.util.ts = ERTS_ALC_MTA_ETS;
}
static void
set_default_driver_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
ip->thr_spec = 1;
ip->atype = BESTFIT;
ip->init.util.name_prefix = "driver_";
ip->init.util.alloc_no = ERTS_ALC_A_DRIVER;
#ifndef SMALL_MEMORY
ip->init.util.mmbcs = 128*1024; /* Main carrier size */
#else
ip->init.util.mmbcs = 32*1024; /* Main carrier size */
#endif
ip->init.util.ts = ERTS_ALC_MTA_DRIVER;
}
static void
set_default_fix_alloc_opts(struct au_init *ip,
size_t *fix_type_sizes)
{
SET_DEFAULT_ALLOC_OPTS(ip);
ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
ip->thr_spec = 1;
ip->atype = BESTFIT;
ip->init.bf.ao = 1;
ip->init.util.name_prefix = "fix_";
ip->init.util.fix_type_size = fix_type_sizes;
ip->init.util.alloc_no = ERTS_ALC_A_FIXED_SIZE;
#ifndef SMALL_MEMORY
ip->init.util.mmbcs = 128*1024; /* Main carrier size */
#else
ip->init.util.mmbcs = 128*1024; /* Main carrier size */
#endif
ip->init.util.ts = ERTS_ALC_MTA_FIXED_SIZE;
}
#ifdef ERTS_SMP
static void
adjust_tpref(struct au_init *ip, int no_sched)
{
if (ip->thr_spec) {
ip->thr_spec = no_sched;
ip->thr_spec *= -1; /* thread preferred */
/* If default ... */
/* ... shrink main multi-block carrier size */
if (ip->default_.mmbcs)
ip->init.util.mmbcs /= ERTS_MIN(4, no_sched);
/* ... shrink largest multi-block carrier size */
if (ip->default_.lmbcs)
ip->init.util.lmbcs /= ERTS_MIN(2, no_sched);
/* ... shrink smallest multi-block carrier size */
if (ip->default_.smbcs)
ip->init.util.smbcs /= ERTS_MIN(4, no_sched);
/* ... and more than three allocators shrink
max mseg multi-block carriers */
if (ip->default_.mmmbc && no_sched > 2) {
ip->init.util.mmmbc /= ERTS_MIN(4, no_sched - 1);
if (ip->init.util.mmmbc < 3)
ip->init.util.mmmbc = 3;
}
}
}
#endif
static void handle_args(int *, char **, erts_alc_hndl_args_init_t *);
static void
set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu);
static void
start_au_allocator(ErtsAlcType_t alctr_n,
struct au_init *init,
ErtsAllocatorState_t *state);
static void
refuse_af_strategy(struct au_init *init)
{
if (init->atype == AFIT)
init->atype = GOODFIT;
}
#ifdef HARD_DEBUG
static void hdbg_init(void);
#endif
void
erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
{
UWord extra_block_size = 0;
int i, ncpu;
erts_alc_hndl_args_init_t init = {
0,
#if HAVE_ERTS_MSEG
ERTS_MSEG_INIT_DEFAULT_INITIALIZER,
#endif
ERTS_DEFAULT_TRIM_THRESHOLD,
ERTS_DEFAULT_TOP_PAD,
ERTS_DEFAULT_ALCU_INIT,
};
size_t fix_type_sizes[ERTS_ALC_NO_FIXED_SIZES] = {0};
fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_PROC)]
= sizeof(Process);
#if !HALFWORD_HEAP
fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MONITOR_SH)]
= ERTS_MONITOR_SH_SIZE;
fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NLINK_SH)]
= ERTS_LINK_SH_SIZE;
#endif
fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_EV_D_STATE)]
= sizeof(ErtsDrvEventDataState);
fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)]
= sizeof(ErtsDrvSelectDataState);
fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MSG_REF)]
= sizeof(ErlMessage);
#ifdef ERTS_SMP
fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_THR_Q_EL_SL)]
= sizeof(ErtsThrQElement_t);
#endif
#ifdef HARD_DEBUG
hdbg_init();
#endif
erts_have_sbmbc_alloc = 0;
ncpu = eaiop->ncpu;
if (ncpu < 1)
ncpu = 1;
erts_sys_alloc_init();
erts_init_utils_mem();
set_default_sbmbc_alloc_opts(&init.sbmbc_alloc);
set_default_sl_alloc_opts(&init.sl_alloc);
set_default_std_alloc_opts(&init.std_alloc);
set_default_ll_alloc_opts(&init.ll_alloc);
set_default_temp_alloc_opts(&init.temp_alloc);
set_default_eheap_alloc_opts(&init.eheap_alloc);
set_default_binary_alloc_opts(&init.binary_alloc);
set_default_ets_alloc_opts(&init.ets_alloc);
set_default_driver_alloc_opts(&init.driver_alloc);
set_default_fix_alloc_opts(&init.fix_alloc,
fix_type_sizes);
if (argc && argv)
handle_args(argc, argv, &init);
#ifndef ERTS_SMP
init.sbmbc_alloc.thr_spec = 0;
init.sl_alloc.thr_spec = 0;
init.std_alloc.thr_spec = 0;
init.ll_alloc.thr_spec = 0;
init.eheap_alloc.thr_spec = 0;
init.binary_alloc.thr_spec = 0;
init.ets_alloc.thr_spec = 0;
init.driver_alloc.thr_spec = 0;
init.fix_alloc.thr_spec = 0;
#endif
if (init.erts_alloc_config) {
/* Adjust flags that erts_alloc_config won't like */
init.sbmbc_alloc.thr_spec = 0;
init.temp_alloc.thr_spec = 0;
init.sl_alloc.thr_spec = 0;
init.std_alloc.thr_spec = 0;
init.ll_alloc.thr_spec = 0;
init.eheap_alloc.thr_spec = 0;
init.binary_alloc.thr_spec = 0;
init.ets_alloc.thr_spec = 0;
init.driver_alloc.thr_spec = 0;
init.fix_alloc.thr_spec = 0;
}
#ifdef ERTS_SMP
/* Only temp_alloc can use thread specific interface */
if (init.temp_alloc.thr_spec)
init.temp_alloc.thr_spec = erts_no_schedulers;
/* Others must use thread preferred interface */
adjust_tpref(&init.sbmbc_alloc, erts_no_schedulers);
adjust_tpref(&init.sl_alloc, erts_no_schedulers);
adjust_tpref(&init.std_alloc, erts_no_schedulers);
adjust_tpref(&init.ll_alloc, erts_no_schedulers);
adjust_tpref(&init.eheap_alloc, erts_no_schedulers);
adjust_tpref(&init.binary_alloc, erts_no_schedulers);
adjust_tpref(&init.ets_alloc, erts_no_schedulers);
adjust_tpref(&init.driver_alloc, erts_no_schedulers);
adjust_tpref(&init.fix_alloc, erts_no_schedulers);
#else
/* No thread specific if not smp */
init.temp_alloc.thr_spec = 0;
#endif
/*
* The following allocators cannot be run with afit strategy.
* Make sure they don't...
*/
refuse_af_strategy(&init.sbmbc_alloc);
refuse_af_strategy(&init.sl_alloc);
refuse_af_strategy(&init.std_alloc);
refuse_af_strategy(&init.ll_alloc);
refuse_af_strategy(&init.eheap_alloc);
refuse_af_strategy(&init.binary_alloc);
refuse_af_strategy(&init.ets_alloc);
refuse_af_strategy(&init.driver_alloc);
refuse_af_strategy(&init.fix_alloc);
#ifdef ERTS_SMP
if (!init.temp_alloc.thr_spec)
refuse_af_strategy(&init.temp_alloc);
#endif
erts_mtrace_pre_init();
#if HAVE_ERTS_MSEG
init.mseg.nos = erts_no_schedulers;
erts_mseg_init(&init.mseg);
#endif
erts_alcu_init(&init.alloc_util);
erts_afalc_init();
erts_bfalc_init();
erts_gfalc_init();
erts_aoffalc_init();
for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
erts_allctrs[i].alloc = NULL;
erts_allctrs[i].realloc = NULL;
erts_allctrs[i].free = NULL;
erts_allctrs[i].extra = NULL;
erts_allctrs_info[i].alloc_util = 0;
erts_allctrs_info[i].enabled = 0;
erts_allctrs_info[i].thr_spec = 0;
erts_allctrs_info[i].extra = NULL;
}
erts_allctrs[ERTS_ALC_A_SYSTEM].alloc = erts_sys_alloc;
erts_allctrs[ERTS_ALC_A_SYSTEM].realloc = erts_sys_realloc;
erts_allctrs[ERTS_ALC_A_SYSTEM].free = erts_sys_free;
erts_allctrs_info[ERTS_ALC_A_SYSTEM].enabled = 1;
#if HALFWORD_HEAP
/* Init low memory variants by cloning */
init.sbmbc_low_alloc = init.sbmbc_alloc;
init.sbmbc_low_alloc.init.util.name_prefix = "sbmbc_low_";
init.sbmbc_low_alloc.init.util.alloc_no = ERTS_ALC_A_SBMBC_LOW;
init.sbmbc_low_alloc.init.util.low_mem = 1;
init.std_low_alloc = init.std_alloc;
init.std_low_alloc.init.util.name_prefix = "std_low_";
init.std_low_alloc.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW;
init.std_low_alloc.init.util.force = 1;
init.std_low_alloc.init.util.low_mem = 1;
init.ll_low_alloc = init.ll_alloc;
init.ll_low_alloc.init.util.name_prefix = "ll_low_";
init.ll_low_alloc.init.util.alloc_no = ERTS_ALC_A_LONG_LIVED_LOW;
init.ll_low_alloc.init.util.force = 1;
init.ll_low_alloc.init.util.low_mem = 1;
set_au_allocator(ERTS_ALC_A_SBMBC_LOW, &init.sbmbc_low_alloc, ncpu);
set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc, ncpu);
set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc, ncpu);
#endif /* HALFWORD */
set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, ncpu);
set_au_allocator(ERTS_ALC_A_SBMBC, &init.sbmbc_alloc, ncpu);
set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc, ncpu);
set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc, ncpu);
set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc, ncpu);
set_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc, ncpu);
set_au_allocator(ERTS_ALC_A_BINARY, &init.binary_alloc, ncpu);
set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc, ncpu);
set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu);
set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu);
for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
if (!erts_allctrs[i].alloc)
erl_exit(ERTS_ABORT_EXIT,
"Missing alloc function for %s\n", ERTS_ALC_A2AD(i));
if (!erts_allctrs[i].realloc)
erl_exit(ERTS_ABORT_EXIT,
"Missing realloc function for %s\n", ERTS_ALC_A2AD(i));
if (!erts_allctrs[i].free)
erl_exit(ERTS_ABORT_EXIT,
"Missing free function for %s\n", ERTS_ALC_A2AD(i));
}
sys_alloc_opt(SYS_ALLOC_OPT_TRIM_THRESHOLD, init.trim_threshold);
sys_alloc_opt(SYS_ALLOC_OPT_TOP_PAD, init.top_pad);
erts_mtrace_init(init.instr.mtrace, init.instr.nodename);
/* sbmbc_alloc() needs to be started first */
start_au_allocator(ERTS_ALC_A_SBMBC,
&init.sbmbc_alloc,
&sbmbc_alloc_state);
#if HALFWORD_HEAP
start_au_allocator(ERTS_ALC_A_SBMBC_LOW,
&init.sbmbc_low_alloc,
&sbmbc_low_alloc_state);
erts_have_sbmbc_alloc = (init.sbmbc_alloc.enable
&& init.sbmbc_low_alloc.enable);
#else
erts_have_sbmbc_alloc = init.sbmbc_alloc.enable;
#endif
start_au_allocator(ERTS_ALC_A_TEMPORARY,
&init.temp_alloc,
&temp_alloc_state);
start_au_allocator(ERTS_ALC_A_SHORT_LIVED,
&init.sl_alloc,
&sl_alloc_state);
start_au_allocator(ERTS_ALC_A_STANDARD,
&init.std_alloc,
&std_alloc_state);
start_au_allocator(ERTS_ALC_A_LONG_LIVED,
&init.ll_alloc,
&ll_alloc_state);
#if HALFWORD_HEAP
start_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW,
&init.ll_low_alloc,
&ll_low_alloc_state);
start_au_allocator(ERTS_ALC_A_STANDARD_LOW,
&init.std_low_alloc,
&std_low_alloc_state);
#endif
start_au_allocator(ERTS_ALC_A_EHEAP,
&init.eheap_alloc,
&eheap_alloc_state);
start_au_allocator(ERTS_ALC_A_BINARY,
&init.binary_alloc,
&binary_alloc_state);
start_au_allocator(ERTS_ALC_A_ETS,
&init.ets_alloc,
&ets_alloc_state);
start_au_allocator(ERTS_ALC_A_DRIVER,
&init.driver_alloc,
&driver_alloc_state);
start_au_allocator(ERTS_ALC_A_FIXED_SIZE,
&init.fix_alloc,
&fix_alloc_state);
erts_mtrace_install_wrapper_functions();
extra_block_size += erts_instr_init(init.instr.stat, init.instr.map);
#if !HALFWORD_HEAP
init_aireq_alloc();
#endif
#ifdef DEBUG
extra_block_size += install_debug_functions();
#endif
}
void
erts_alloc_late_init(void)
{
}
static void *
erts_realloc_fixed_size(ErtsAlcType_t type, void *extra, void *p, Uint size)
{
erl_exit(ERTS_ABORT_EXIT,
"Attempt to reallocate a block of the fixed size type %s\n",
ERTS_ALC_T2TD(type));
}
static void
set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu)
{
ErtsAllocatorFunctions_t *af = &erts_allctrs[alctr_n];
ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n];
ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n];
/*
* Some allocators are forced on if halfword heap is used.
*/
if (init->init.util.force)
init->enable = 1;
tspec->enabled = 0;
tspec->dd = 0;
tspec->aix = alctr_n;
tspec->size = 0;
ai->thr_spec = 0;
if (!init->enable) {
af->alloc = erts_sys_alloc;
af->realloc = erts_sys_realloc;
af->free = erts_sys_free;
af->extra = NULL;
ai->alloc_util = 0;
ai->enabled = 0;
ai->extra = NULL;
return;
}
#ifdef USE_THREADS
#ifdef ERTS_SMP
if (init->thr_spec) {
if (init->thr_spec > 0) {
af->alloc = erts_alcu_alloc_thr_spec;
if (init->init.util.fix_type_size)
af->realloc = erts_realloc_fixed_size;
else if (init->init.util.ramv)
af->realloc = erts_alcu_realloc_mv_thr_spec;
else
af->realloc = erts_alcu_realloc_thr_spec;
af->free = erts_alcu_free_thr_spec;
}
else {
af->alloc = erts_alcu_alloc_thr_pref;
if (init->init.util.fix_type_size)
af->realloc = erts_realloc_fixed_size;
else if (init->init.util.ramv)
af->realloc = erts_alcu_realloc_mv_thr_pref;
else
af->realloc = erts_alcu_realloc_thr_pref;
af->free = erts_alcu_free_thr_pref;
tspec->dd = 1;
}
tspec->enabled = 1;
tspec->size = abs(init->thr_spec) + 1;
ai->thr_spec = tspec->size;
}
else
#endif
if (init->init.util.ts) {
af->alloc = erts_alcu_alloc_ts;
if (init->init.util.fix_type_size)
af->realloc = erts_realloc_fixed_size;
else if (init->init.util.ramv)
af->realloc = erts_alcu_realloc_mv_ts;
else
af->realloc = erts_alcu_realloc_ts;
af->free = erts_alcu_free_ts;
}
else
#endif
{
af->alloc = erts_alcu_alloc;
if (init->init.util.fix_type_size)
af->realloc = erts_realloc_fixed_size;
else if (init->init.util.ramv)
af->realloc = erts_alcu_realloc_mv;
else
af->realloc = erts_alcu_realloc;
af->free = erts_alcu_free;
}
af->extra = NULL;
ai->alloc_util = 1;
ai->enabled = 1;
}
static void
start_au_allocator(ErtsAlcType_t alctr_n,
struct au_init *init,
ErtsAllocatorState_t *state)
{
int i;
int size = 1;
void *as0;
enum allctr_type atype;
ErtsAllocatorFunctions_t *af = &erts_allctrs[alctr_n];
ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n];
ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n];
ErtsAlcFixList_t *fix_lists = NULL;
size_t fix_list_size = 0;
if (!init->enable)
return;
if (init->thr_spec) {
char *states = erts_sys_alloc(0,
NULL,
((sizeof(Allctr_t *)
* (tspec->size + 1))
+ (sizeof(ErtsAllocatorState_t)
* tspec->size)
+ ERTS_CACHE_LINE_SIZE - 1));
if (!states)
erl_exit(ERTS_ABORT_EXIT,
"Failed to allocate allocator states for %salloc\n",
init->init.util.name_prefix);
tspec->allctr = (Allctr_t **) states;
states += sizeof(Allctr_t *) * (tspec->size + 1);
states = ((((UWord) states) & ERTS_CACHE_LINE_MASK)
? (char *) ((((UWord) states) & ~ERTS_CACHE_LINE_MASK)
+ ERTS_CACHE_LINE_SIZE)
: (char *) states);
tspec->allctr[0] = (Allctr_t *) state;
size = tspec->size;
for (i = 1; i < size; i++)
tspec->allctr[i] = (Allctr_t *)
&((ErtsAllocatorState_t *) states)[i-1];
}
if (init->init.util.fix_type_size) {
size_t tot_fix_list_size;
fix_list_size = sizeof(ErtsAlcFixList_t)*ERTS_ALC_NO_FIXED_SIZES;
fix_list_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(fix_list_size);
tot_fix_list_size = fix_list_size;
if (init->thr_spec)
tot_fix_list_size *= tspec->size;
fix_lists = erts_sys_alloc(0,
NULL,
(tot_fix_list_size
+ ERTS_CACHE_LINE_SIZE - 1));
if (!fix_lists)
erl_exit(ERTS_ABORT_EXIT,
"Failed to allocate fix lists for %salloc\n",
init->init.util.name_prefix);
if (((UWord) fix_lists) & ERTS_CACHE_LINE_MASK)
fix_lists = ((ErtsAlcFixList_t *)
((((UWord) fix_lists) & ~ERTS_CACHE_LINE_MASK)
+ ERTS_CACHE_LINE_SIZE));
}
for (i = 0; i < size; i++) {
void *as;
atype = init->atype;
if (!init->thr_spec)
as0 = state;
else {
as0 = (void *) tspec->allctr[i];
if (!as0)
continue;
if (init->thr_spec < 0) {
init->init.util.ts = i == 0;
init->init.util.tspec = 0;
init->init.util.tpref = -1*init->thr_spec + 1;
}
else {
if (i != 0)
init->init.util.ts = 0;
else {
if (atype == AFIT)
atype = GOODFIT;
init->init.util.ts = 1;
}
init->init.util.tspec = init->thr_spec + 1;
init->init.util.tpref = 0;
}
}
if (fix_lists) {
init->init.util.fix = fix_lists;
fix_lists = ((ErtsAlcFixList_t *)
(((char *) fix_lists) + fix_list_size));
}
init->init.util.ix = i;
switch (atype) {
case GOODFIT:
as = (void *) erts_gfalc_start((GFAllctr_t *) as0,
&init->init.gf,
&init->init.util);
break;
case BESTFIT:
as = (void *) erts_bfalc_start((BFAllctr_t *) as0,
&init->init.bf,
&init->init.util);
break;
case AFIT:
as = (void *) erts_afalc_start((AFAllctr_t *) as0,
&init->init.af,
&init->init.util);
break;
case AOFIRSTFIT:
as = (void *) erts_aoffalc_start((AOFFAllctr_t *) as0,
&init->init.aoff,
&init->init.util);
break;
default:
as = NULL;
ASSERT(0);
}
if (!as)
erl_exit(ERTS_ABORT_EXIT,
"Failed to start %salloc\n", init->init.util.name_prefix);
ASSERT(as == (void *) as0);
af->extra = as;
}
if (init->thr_spec)
af->extra = tspec;
ai->extra = af->extra;
}
static void bad_param(char *param_start, char *param_end)
{
size_t len = param_end - param_start;
char param[100];
if (len > 99)
len = 99;
sys_memcpy((void *) param, (void *) param_start, len);
param[len] = '\0';
erts_fprintf(stderr, "bad \"%s\" parameter\n", param);
erts_usage();
}
static void bad_value(char *param_start, char *param_end, char *value)
{
size_t len = param_end - param_start;
char param[100];
if (len > 99)
len = 99;
sys_memcpy((void *) param, (void *) param_start, len);
param[len] = '\0';
erts_fprintf(stderr, "bad \"%s\" value: %s\n", param, value);
erts_usage();
}
/* Get arg marks argument as handled by
putting NULL in argv */
static char *
get_value(char* rest, char** argv, int* ip)
{
char *param = argv[*ip]+1;
argv[*ip] = NULL;
if (*rest == '\0') {
char *next = argv[*ip + 1];
if (next[0] == '-'
&& next[1] == '-'
&& next[2] == '\0') {
bad_value(param, rest, "");
}
(*ip)++;
argv[*ip] = NULL;
return next;
}
return rest;
}
static ERTS_INLINE int
has_prefix(const char *prefix, const char *string)
{
int i;
for (i = 0; prefix[i]; i++)
if (prefix[i] != string[i])
return 0;
return 1;
}
static int
get_bool_value(char *param_end, char** argv, int* ip)
{
char *param = argv[*ip]+1;
char *value = get_value(param_end, argv, ip);
if (strcmp(value, "true") == 0)
return 1;
else if (strcmp(value, "false") == 0)
return 0;
else
bad_value(param, param_end, value);
return -1;
}
static Uint
get_kb_value(char *param_end, char** argv, int* ip)
{
Sint tmp;
Uint max = ((~((Uint) 0))/1024) + 1;
char *rest;
char *param = argv[*ip]+1;
char *value = get_value(param_end, argv, ip);
errno = 0;
tmp = (Sint) ErtsStrToSint(value, &rest, 10);
if (errno != 0 || rest == value || tmp < 0 || max < ((Uint) tmp))
bad_value(param, param_end, value);
if (max == (Uint) tmp)
return ~((Uint) 0);
else
return ((Uint) tmp)*1024;
}
static Uint
get_byte_value(char *param_end, char** argv, int* ip)
{
Sint tmp;
char *rest;
char *param = argv[*ip]+1;
char *value = get_value(param_end, argv, ip);
errno = 0;
tmp = (Sint) ErtsStrToSint(value, &rest, 10);
if (errno != 0 || rest == value || tmp < 0)
bad_value(param, param_end, value);
return (Uint) tmp;
}
static Uint
get_amount_value(char *param_end, char** argv, int* ip)
{
Sint tmp;
char *rest;
char *param = argv[*ip]+1;
char *value = get_value(param_end, argv, ip);
errno = 0;
tmp = (Sint) ErtsStrToSint(value, &rest, 10);
if (errno != 0 || rest == value || tmp < 0)
bad_value(param, param_end, value);
return (Uint) tmp;
}
static void
handle_au_arg(struct au_init *auip,
char* sub_param,
char** argv,
int* ip)
{
char *param = argv[*ip]+1;
switch (sub_param[0]) {
case 'a':
if(has_prefix("asbcst", sub_param)) {
auip->init.util.asbcst = get_kb_value(sub_param + 6, argv, ip);
}
else if(has_prefix("as", sub_param)) {
char *alg = get_value(sub_param + 2, argv, ip);
if (strcmp("bf", alg) == 0) {
auip->atype = BESTFIT;
auip->init.bf.ao = 0;
}
else if (strcmp("aobf", alg) == 0) {
auip->atype = BESTFIT;
auip->init.bf.ao = 1;
}
else if (strcmp("gf", alg) == 0) {
auip->atype = GOODFIT;
}
else if (strcmp("af", alg) == 0) {
auip->atype = AFIT;
}
else if (strcmp("aoff", alg) == 0) {
auip->atype = AOFIRSTFIT;
}
else {
bad_value(param, sub_param + 1, alg);
}
}
else
goto bad_switch;
break;
case 'e':
auip->enable = get_bool_value(sub_param+1, argv, ip);
break;
case 'l':
if (has_prefix("lmbcs", sub_param)) {
auip->default_.lmbcs = 0;
auip->init.util.lmbcs = get_kb_value(sub_param + 5, argv, ip);
}
else
goto bad_switch;
break;
case 'm':
if (has_prefix("mbcgs", sub_param)) {
auip->init.util.mbcgs = get_amount_value(sub_param + 5, argv, ip);
}
else if (has_prefix("mbsd", sub_param)) {
auip->init.gf.mbsd = get_amount_value(sub_param + 4, argv, ip);
if (auip->init.gf.mbsd < 1)
auip->init.gf.mbsd = 1;
}
else if (has_prefix("mmbcs", sub_param)) {
auip->default_.mmbcs = 0;
auip->init.util.mmbcs = get_kb_value(sub_param + 5, argv, ip);
}
else if (has_prefix("mmmbc", sub_param)) {
auip->default_.mmmbc = 0;
auip->init.util.mmmbc = get_amount_value(sub_param + 5, argv, ip);
}
else if (has_prefix("mmsbc", sub_param)) {
auip->init.util.mmsbc = get_amount_value(sub_param + 5, argv, ip);
}
else
goto bad_switch;
break;
case 'r':
if(has_prefix("rsbcmt", sub_param)) {
auip->init.util.rsbcmt = get_amount_value(sub_param + 6, argv, ip);
if (auip->init.util.rsbcmt > 100)
auip->init.util.rsbcmt = 100;
}
else if(has_prefix("rsbcst", sub_param)) {
auip->init.util.rsbcst = get_amount_value(sub_param + 6, argv, ip);
if (auip->init.util.rsbcst > 100)
auip->init.util.rsbcst = 100;
}
else if (has_prefix("rmbcmt", sub_param)) {
auip->init.util.rmbcmt = get_amount_value(sub_param + 6, argv, ip);
if (auip->init.util.rmbcmt > 100)
auip->init.util.rmbcmt = 100;
}
else if (has_prefix("ramv", sub_param)) {
auip->init.util.ramv = get_bool_value(sub_param + 4, argv, ip);
}
else
goto bad_switch;
break;
case 's':
if(has_prefix("sbct", sub_param)) {
auip->init.util.sbct = get_kb_value(sub_param + 4, argv, ip);
}
else if (has_prefix("sbmbcs", sub_param)) {
auip->init.util.sbmbcs = get_byte_value(sub_param + 6, argv, ip);
}
else if (has_prefix("sbmbct", sub_param)) {
auip->init.util.sbmbct = get_byte_value(sub_param + 6, argv, ip);
}
else if (has_prefix("smbcs", sub_param)) {
auip->default_.smbcs = 0;
auip->init.util.smbcs = get_kb_value(sub_param + 5, argv, ip);
}
else
goto bad_switch;
break;
case 't': {
int res = get_bool_value(sub_param+1, argv, ip);
if (res > 0) {
auip->thr_spec = 1;
break;
}
else if (res == 0) {
auip->thr_spec = 0;
break;
}
goto bad_switch;
}
default:
bad_switch:
bad_param(param, sub_param);
}
}
static void
handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
{
struct au_init *aui[] = {
&init->sbmbc_alloc,
&init->binary_alloc,
&init->std_alloc,
&init->ets_alloc,
&init->eheap_alloc,
&init->ll_alloc,
&init->driver_alloc,
&init->fix_alloc,
&init->sl_alloc,
&init->temp_alloc
};
int aui_sz = (int) sizeof(aui)/sizeof(aui[0]);
char *arg;
char *rest;
int i, j;
i = 1;
ASSERT(argc && argv && init);
while (i < *argc) {
if(argv[i][0] == '-') {
char *param = argv[i]+1;
switch (argv[i][1]) {
case 'M':
switch (argv[i][2]) {
case 'B':
handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i);
break;
case 'C':
handle_au_arg(&init->sbmbc_alloc, &argv[i][3], argv, &i);
break;
case 'D':
handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i);
break;
case 'E':
handle_au_arg(&init->ets_alloc, &argv[i][3], argv, &i);
break;
case 'F':
handle_au_arg(&init->fix_alloc, &argv[i][3], argv, &i);
break;
case 'H':
handle_au_arg(&init->eheap_alloc, &argv[i][3], argv, &i);
break;
case 'L':
handle_au_arg(&init->ll_alloc, &argv[i][3], argv, &i);
break;
case 'M':
if (has_prefix("amcbf", argv[i]+3)) {
#if HAVE_ERTS_MSEG
init->mseg.amcbf =
#endif
get_kb_value(argv[i]+8, argv, &i);
}
else if (has_prefix("rmcbf", argv[i]+3)) {
#if HAVE_ERTS_MSEG
init->mseg.rmcbf =
#endif
get_amount_value(argv[i]+8, argv, &i);
}
else if (has_prefix("mcs", argv[i]+3)) {
#if HAVE_ERTS_MSEG
init->mseg.mcs =
#endif
get_amount_value(argv[i]+6, argv, &i);
}
else {
bad_param(param, param+2);
}
break;
case 'R':
handle_au_arg(&init->driver_alloc, &argv[i][3], argv, &i);
break;
case 'S':
handle_au_arg(&init->sl_alloc, &argv[i][3], argv, &i);
break;
case 'T':
handle_au_arg(&init->temp_alloc, &argv[i][3], argv, &i);
break;
case 'Y': { /* sys_alloc */
if (has_prefix("tt", param+2)) {
/* set trim threshold */
arg = get_value(param+4, argv, &i);
errno = 0;
init->trim_threshold = (int) strtol(arg, &rest, 10);
if (errno != 0
|| rest == arg
|| init->trim_threshold < 0
|| (INT_MAX/1024) < init->trim_threshold) {
bad_value(param, param+4, arg);
}
VERBOSE(DEBUG_SYSTEM,
("using trim threshold: %d\n",
init->trim_threshold));
init->trim_threshold *= 1024;
}
else if (has_prefix("tp", param+2)) {
/* set top pad */
arg = get_value(param+4, argv, &i);
errno = 0;
init->top_pad = (int) strtol(arg, &rest, 10);
if (errno != 0
|| rest == arg
|| init->top_pad < 0
|| (INT_MAX/1024) < init->top_pad) {
bad_value(param, param+4, arg);
}
VERBOSE(DEBUG_SYSTEM,
("using top pad: %d\n",init->top_pad));
init->top_pad *= 1024;
}
else if (has_prefix("m", param+2)) {
/* Has been handled by erlexec */
(void) get_value(param+3, argv, &i);
}
else if (has_prefix("e", param+2)) {
arg = get_value(param+3, argv, &i);
if (strcmp("true", arg) != 0)
bad_value(param, param+3, arg);
}
else
bad_param(param, param+2);
break;
}
case 'e':
switch (argv[i][3]) {
case 'a': {
int a;
arg = get_value(argv[i]+4, argv, &i);
if (strcmp("min", arg) == 0) {
for (a = 0; a < aui_sz; a++)
aui[a]->enable = 0;
}
else if (strcmp("max", arg) == 0) {
for (a = 0; a < aui_sz; a++)
aui[a]->enable = 1;
}
else if (strcmp("config", arg) == 0) {
init->erts_alloc_config = 1;
}
else if (strcmp("r9c", arg) == 0
|| strcmp("r10b", arg) == 0
|| strcmp("r11b", arg) == 0) {
set_default_sl_alloc_opts(&init->sl_alloc);
set_default_std_alloc_opts(&init->std_alloc);
set_default_ll_alloc_opts(&init->ll_alloc);
set_default_temp_alloc_opts(&init->temp_alloc);
set_default_eheap_alloc_opts(&init->eheap_alloc);
set_default_binary_alloc_opts(&init->binary_alloc);
set_default_ets_alloc_opts(&init->ets_alloc);
set_default_driver_alloc_opts(&init->driver_alloc);
set_default_driver_alloc_opts(&init->fix_alloc);
init->driver_alloc.enable = 0;
if (strcmp("r9c", arg) == 0) {
init->sl_alloc.enable = 0;
init->std_alloc.enable = 0;
init->binary_alloc.enable = 0;
init->ets_alloc.enable = 0;
}
for (a = 0; a < aui_sz; a++) {
aui[a]->thr_spec = 0;
aui[a]->init.util.ramv = 0;
aui[a]->init.util.mmmbc = 10;
aui[a]->init.util.lmbcs = 5*1024*1024;
}
}
else {
bad_param(param, param+3);
}
break;
}
default:
bad_param(param, param+1);
}
break;
case 'i':
switch (argv[i][3]) {
case 's':
arg = get_value(argv[i]+4, argv, &i);
if (strcmp("true", arg) == 0)
init->instr.stat = 1;
else if (strcmp("false", arg) == 0)
init->instr.stat = 0;
else
bad_value(param, param+3, arg);
break;
case 'm':
arg = get_value(argv[i]+4, argv, &i);
if (strcmp("true", arg) == 0)
init->instr.map = 1;
else if (strcmp("false", arg) == 0)
init->instr.map = 0;
else
bad_value(param, param+3, arg);
break;
case 't':
init->instr.mtrace = get_value(argv[i]+4, argv, &i);
break;
default:
bad_param(param, param+2);
}
break;
case 'u':
if (has_prefix("ycs", argv[i]+3)) {
init->alloc_util.ycs
= get_kb_value(argv[i]+6, argv, &i);
}
else if (has_prefix("mmc", argv[i]+3)) {
init->alloc_util.mmc
= get_amount_value(argv[i]+6, argv, &i);
}
else {
int a;
int start = i;
char *param = argv[i];
char *val = i+1 < *argc ? argv[i+1] : NULL;
for (a = 0; a < aui_sz; a++) {
if (a > 0) {
ASSERT(i == start || i == start+1);
argv[start] = param;
if (i != start)
argv[start + 1] = val;
i = start;
}
handle_au_arg(aui[a], &argv[i][3], argv, &i);
}
}
break;
default:
bad_param(param, param+1);
}
break;
case '-':
if (argv[i][2] == '\0') {
/* End of system flags reached */
if (init->instr.mtrace
/* || init->instr.stat
|| init->instr.map */) {
while (i < *argc) {
if(strcmp(argv[i], "-sname") == 0
|| strcmp(argv[i], "-name") == 0) {
if (i + 1 <*argc) {
init->instr.nodename = argv[i+1];
break;
}
}
i++;
}
}
goto args_parsed;
}
break;
default:
break;
}
}
i++;
}
args_parsed:
/* Handled arguments have been marked with NULL. Slide arguments
not handled towards the beginning of argv. */
for (i = 0, j = 0; i < *argc; i++) {
if (argv[i])
argv[j++] = argv[i];
}
*argc = j;
}
static char *type_no_str(ErtsAlcType_t n)
{
#if ERTS_ALC_N_MIN != 0
if (n < ERTS_ALC_N_MIN)
return NULL;
#endif
if (n > ERTS_ALC_N_MAX)
return NULL;
return (char *) ERTS_ALC_N2TD(n);
}
#define type_str(T) type_no_str(ERTS_ALC_T2N((T)))
void
erts_alloc_register_scheduler(void *vesdp)
{
ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
int ix = (int) esdp->no;
int aix;
for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) {
ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix];
esdp->alloc_data.deallctr[aix] = NULL;
esdp->alloc_data.pref_ix[aix] = -1;
if (tspec->enabled) {
if (!tspec->dd)
esdp->alloc_data.pref_ix[aix] = ix;
else {
Allctr_t *allctr = tspec->allctr[ix];
ASSERT(allctr);
esdp->alloc_data.deallctr[aix] = allctr;
esdp->alloc_data.pref_ix[aix] = ix;
}
}
}
}
#ifdef ERTS_SMP
void
erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
int *need_thr_progress,
ErtsThrPrgrVal *thr_prgr_p,
int *more_work)
{
ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
int aix;
for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) {
Allctr_t *allctr;
if (esdp)
allctr = esdp->alloc_data.deallctr[aix];
else {
ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix];
if (tspec->enabled && tspec->dd)
allctr = tspec->allctr[0];
else
allctr = NULL;
}
if (allctr) {
erts_alcu_check_delayed_dealloc(allctr,
1,
need_thr_progress,
thr_prgr_p,
more_work);
}
}
}
#endif
erts_aint32_t
erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs)
{
#ifdef ERTS_SMP
ErtsAllocatorThrSpec_t *tspec;
tspec = &erts_allctr_thr_spec[ERTS_ALC_A_FIXED_SIZE];
if (erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].thr_spec && tspec->enabled)
return erts_alcu_fix_alloc_shrink(tspec->allctr[ix], flgs);
if (ix == 0 && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra)
return erts_alcu_fix_alloc_shrink(
erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra, flgs);
#else
if (ix == 1 && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra)
return erts_alcu_fix_alloc_shrink(
erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra, flgs);
#endif
return 0;
}
static void
no_verify(Allctr_t *allctr)
{
}
erts_alloc_verify_func_t
erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr)
{
if (erts_allctrs_info[ERTS_ALC_A_TEMPORARY].alloc_util
&& erts_allctrs_info[ERTS_ALC_A_TEMPORARY].thr_spec) {
ErtsAllocatorThrSpec_t *tspec;
int ix = ERTS_ALC_GET_THR_IX();
tspec = &erts_allctr_thr_spec[ERTS_ALC_A_TEMPORARY];
if (ix < tspec->size) {
*allctr = tspec->allctr[ix];
return erts_alcu_verify_unused;
}
}
*allctr = NULL;
return no_verify;
}
__decl_noreturn void
erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...)
{
char buf[10];
char *t_str;
char *allctr_str;
ASSERT(n >= ERTS_ALC_N_MIN);
ASSERT(n <= ERTS_ALC_N_MAX);
if (n < ERTS_ALC_N_MIN || ERTS_ALC_N_MAX < n)
allctr_str = "UNKNOWN";
else {
ErtsAlcType_t a = ERTS_ALC_T2A(ERTS_ALC_N2T(n));
if (erts_allctrs_info[a].enabled)
allctr_str = (char *) ERTS_ALC_A2AD(a);
else
allctr_str = (char *) ERTS_ALC_A2AD(ERTS_ALC_A_SYSTEM);
}
t_str = type_no_str(n);
if (!t_str) {
sprintf(buf, "%d", (int) n);
t_str = buf;
}
switch (error) {
case ERTS_ALC_E_NOTSUP: {
char *op_str;
switch (func) {
case ERTS_ALC_O_ALLOC: op_str = "alloc"; break;
case ERTS_ALC_O_REALLOC: op_str = "realloc"; break;
case ERTS_ALC_O_FREE: op_str = "free"; break;
default: op_str = "UNKNOWN"; break;
}
erl_exit(ERTS_ABORT_EXIT,
"%s: %s operation not supported (memory type: \"%s\")\n",
allctr_str, op_str, t_str);
break;
}
case ERTS_ALC_E_NOMEM: {
Uint size;
va_list argp;
char *op = func == ERTS_ALC_O_REALLOC ? "reallocate" : "allocate";
va_start(argp, n);
size = va_arg(argp, Uint);
va_end(argp);
erl_exit(1,
"%s: Cannot %s %lu bytes of memory (of type \"%s\").\n",
allctr_str, op, size, t_str);
break;
}
case ERTS_ALC_E_NOALLCTR:
erl_exit(ERTS_ABORT_EXIT,
"erts_alloc: Unknown allocator type: %d\n",
ERTS_ALC_T2A(ERTS_ALC_N2T(n)));
break;
default:
erl_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown error: %d\n", error);
break;
}
}
__decl_noreturn void
erts_alloc_enomem(ErtsAlcType_t type, Uint size)
{
erts_alloc_n_enomem(ERTS_ALC_T2N(type), size);
}
__decl_noreturn void
erts_alloc_n_enomem(ErtsAlcType_t n, Uint size)
{
erts_alc_fatal_error(ERTS_ALC_E_NOMEM, ERTS_ALC_O_ALLOC, n, size);
}
__decl_noreturn void
erts_realloc_enomem(ErtsAlcType_t type, void *ptr, Uint size)
{
erts_realloc_n_enomem(ERTS_ALC_T2N(type), ptr, size);
}
__decl_noreturn void
erts_realloc_n_enomem(ErtsAlcType_t n, void *ptr, Uint size)
{
erts_alc_fatal_error(ERTS_ALC_E_NOMEM, ERTS_ALC_O_REALLOC, n, size);
}
static ERTS_INLINE UWord
alcu_size(ErtsAlcType_t ai, ErtsAlcUFixInfo_t *fi, int fisz)
{
UWord res = 0;
ASSERT(erts_allctrs_info[ai].enabled);
ASSERT(erts_allctrs_info[ai].alloc_util);
if (!erts_allctrs_info[ai].thr_spec) {
Allctr_t *allctr = erts_allctrs_info[ai].extra;
AllctrSize_t asize;
erts_alcu_current_size(allctr, &asize, fi, fisz);
res += asize.blocks;
}
else {
ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ai];
int i;
ASSERT(tspec->enabled);
for (i = tspec->size - 1; i >= 0; i--) {
Allctr_t *allctr = tspec->allctr[i];
AllctrSize_t asize;
if (allctr) {
erts_alcu_current_size(allctr, &asize, fi, fisz);
res += asize.blocks;
}
}
}
return res;
}
#if HALFWORD_HEAP
static ERTS_INLINE int
alcu_is_low(ErtsAlcType_t ai)
{
int is_low = 0;
ASSERT(erts_allctrs_info[ai].enabled);
ASSERT(erts_allctrs_info[ai].alloc_util);
if (!erts_allctrs_info[ai].thr_spec) {
Allctr_t *allctr = erts_allctrs_info[ai].extra;
is_low = allctr->mseg_opt.low_mem;
}
else {
ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ai];
int i;
# ifdef DEBUG
int found_one = 0;
# endif
ASSERT(tspec->enabled);
for (i = tspec->size - 1; i >= 0; i--) {
Allctr_t *allctr = tspec->allctr[i];
if (allctr) {
# ifdef DEBUG
if (!found_one) {
is_low = allctr->mseg_opt.low_mem;
found_one = 1;
}
else ASSERT(is_low == allctr->mseg_opt.low_mem);
# else
is_low = allctr->mseg_opt.low_mem;
break;
# endif
}
}
ASSERT(found_one);
}
return is_low;
}
#endif /* HALFWORD */
static ERTS_INLINE void
add_fix_values(UWord *ap, UWord *up, ErtsAlcUFixInfo_t *fi, ErtsAlcType_t type)
{
int ix = ERTS_ALC_T2N(type) - ERTS_ALC_N_MIN_A_FIXED_SIZE;
ASSERT(0 <= ix && ix < ERTS_ALC_NO_FIXED_SIZES);
*ap += (UWord) fi[ix].allocated;
*up += (UWord) fi[ix].used;
}
Eterm
erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
{
/*
* NOTE! When updating this function, make sure to also update
* erlang:memory/[0,1] in $ERL_TOP/erts/preloaded/src/erlang.erl
*/
#define ERTS_MEM_NEED_ALL_ALCU (!erts_instr_stat && want_tot_or_sys)
struct {
int total;
int processes;
int processes_used;
int system;
int atom;
int atom_used;
int binary;
int code;
int ets;
int maximum;
#if HALFWORD_HEAP
int low;
#endif
} want = {0};
struct {
UWord total;
UWord processes;
UWord processes_used;
UWord system;
UWord atom;
UWord atom_used;
UWord binary;
UWord code;
UWord ets;
UWord maximum;
#if HALFWORD_HEAP
UWord low;
#endif
} size = {0};
Eterm atoms[sizeof(size)/sizeof(UWord)];
UWord *uintps[sizeof(size)/sizeof(UWord)];
Eterm euints[sizeof(size)/sizeof(UWord)];
int want_tot_or_sys;
int length;
Eterm res = THE_NON_VALUE;
ErtsAlcType_t ai;
int only_one_value = 0;
ErtsAlcUFixInfo_t fi[ERTS_ALC_NO_FIXED_SIZES] = {{0,0}};
ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
/* Figure out whats wanted... */
length = 0;
if (is_non_value(earg)) { /* i.e. wants all */
want.total = 1;
atoms[length] = am_total;
uintps[length++] = &size.total;
want.processes = 1;
atoms[length] = am_processes;
uintps[length++] = &size.processes;
want.processes_used = 1;
atoms[length] = am_processes_used;
uintps[length++] = &size.processes_used;
want.system = 1;
atoms[length] = am_system;
uintps[length++] = &size.system;
want.atom = 1;
atoms[length] = am_atom;
uintps[length++] = &size.atom;
want.atom_used = 1;
atoms[length] = am_atom_used;
uintps[length++] = &size.atom_used;
want.binary = 1;
atoms[length] = am_binary;
uintps[length++] = &size.binary;
want.code = 1;
atoms[length] = am_code;
uintps[length++] = &size.code;
want.ets = 1;
atoms[length] = am_ets;
uintps[length++] = &size.ets;
want.maximum = erts_instr_stat;
if (want.maximum) {
atoms[length] = am_maximum;
uintps[length++] = &size.maximum;
}
#if HALFWORD_HEAP
want.low = 1;
atoms[length] = am_low;
uintps[length++] = &size.low;
#endif
}
else {
DeclareTmpHeapNoproc(tmp_heap,2);
Eterm wanted_list;
if (is_nil(earg))
return NIL;
UseTmpHeapNoproc(2);
if (is_not_atom(earg))
wanted_list = earg;
else {
wanted_list = CONS(&tmp_heap[0], earg, NIL);
only_one_value = 1;
}
while (is_list(wanted_list)) {
switch (CAR(list_val(wanted_list))) {
case am_total:
if (!want.total) {
want.total = 1;
atoms[length] = am_total;
uintps[length++] = &size.total;
}
break;
case am_processes:
if (!want.processes) {
want.processes = 1;
atoms[length] = am_processes;
uintps[length++] = &size.processes;
}
break;
case am_processes_used:
if (!want.processes_used) {
want.processes_used = 1;
atoms[length] = am_processes_used;
uintps[length++] = &size.processes_used;
}
break;
case am_system:
if (!want.system) {
want.system = 1;
atoms[length] = am_system;
uintps[length++] = &size.system;
}
break;
case am_atom:
if (!want.atom) {
want.atom = 1;
atoms[length] = am_atom;
uintps[length++] = &size.atom;
}
break;
case am_atom_used:
if (!want.atom_used) {
want.atom_used = 1;
atoms[length] = am_atom_used;
uintps[length++] = &size.atom_used;
}
break;
case am_binary:
if (!want.binary) {
want.binary = 1;
atoms[length] = am_binary;
uintps[length++] = &size.binary;
}
break;
case am_code:
if (!want.code) {
want.code = 1;
atoms[length] = am_code;
uintps[length++] = &size.code;
}
break;
case am_ets:
if (!want.ets) {
want.ets = 1;
atoms[length] = am_ets;
uintps[length++] = &size.ets;
}
break;
case am_maximum:
if (erts_instr_stat) {
if (!want.maximum) {
want.maximum = 1;
atoms[length] = am_maximum;
uintps[length++] = &size.maximum;
}
} else {
UnUseTmpHeapNoproc(2);
return am_badarg;
}
break;
#if HALFWORD_HEAP
case am_low:
if (!want.low) {
want.low = 1;
atoms[length] = am_low;
uintps[length++] = &size.low;
}
break;
#endif
default:
UnUseTmpHeapNoproc(2);
return am_badarg;
}
wanted_list = CDR(list_val(wanted_list));
}
UnUseTmpHeapNoproc(2);
if (is_not_nil(wanted_list))
return am_badarg;
}
/* All alloc_util allocators except sbmbc_alloc *have* to be enabled */
for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) {
switch (ai) {
case ERTS_ALC_A_SYSTEM:
case ERTS_ALC_A_SBMBC:
#if HALFWORD_HEAP
case ERTS_ALC_A_SBMBC_LOW:
#endif
break;
default:
if (!erts_allctrs_info[ai].enabled
|| !erts_allctrs_info[ai].alloc_util) {
return am_notsup;
}
break;
}
}
ASSERT(length <= sizeof(atoms)/sizeof(Eterm));
ASSERT(length <= sizeof(euints)/sizeof(Eterm));
ASSERT(length <= sizeof(uintps)/sizeof(UWord));
if (proc) {
ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN
== erts_proc_lc_my_proc_locks(proc));
/* We'll need locks early in the lock order */
erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
}
/* Calculate values needed... */
want_tot_or_sys = want.total || want.system;
if (ERTS_MEM_NEED_ALL_ALCU) {
size.total = 0;
for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) {
if (erts_allctrs_info[ai].alloc_util) {
UWord *save;
UWord asz;
switch (ai) {
case ERTS_ALC_A_TEMPORARY:
/*
* Often not thread safe and usually never
* contain any allocated memory.
*/
case ERTS_ALC_A_SBMBC:
/* Included in other allocators */
#if HALFWORD_HEAP
case ERTS_ALC_A_SBMBC_LOW:
/* Included in other allocators */
#endif
continue;
case ERTS_ALC_A_EHEAP:
save = &size.processes;
break;
case ERTS_ALC_A_ETS:
save = &size.ets;
break;
case ERTS_ALC_A_BINARY:
save = &size.binary;
break;
case ERTS_ALC_A_FIXED_SIZE:
asz = alcu_size(ai, fi, ERTS_ALC_NO_FIXED_SIZES);
size.total += asz;
continue;
default:
save = NULL;
break;
}
asz = alcu_size(ai, NULL, 0);
if (save)
*save = asz;
size.total += asz;
#if HALFWORD_HEAP
if (alcu_is_low(ai)) {
size.low += asz;
}
#endif
}
}
}
if (want_tot_or_sys || want.processes || want.processes_used) {
UWord tmp;
if (ERTS_MEM_NEED_ALL_ALCU)
tmp = size.processes;
else {
alcu_size(ERTS_ALC_A_FIXED_SIZE,
fi, ERTS_ALC_NO_FIXED_SIZES);
tmp = alcu_size(ERTS_ALC_A_EHEAP, NULL, 0);
}
tmp += erts_max_processes*sizeof(Process*);
#ifdef HYBRID
tmp += erts_max_processes*sizeof(Process*);
#endif
tmp += erts_bif_timer_memory_size();
tmp += erts_tot_link_lh_size();
size.processes = size.processes_used = tmp;
add_fix_values(&size.processes,
&size.processes_used,
fi,
ERTS_ALC_T_PROC);
#if !HALFWORD_HEAP
add_fix_values(&size.processes,
&size.processes_used,
fi,
ERTS_ALC_T_MONITOR_SH);
add_fix_values(&size.processes,
&size.processes_used,
fi,
ERTS_ALC_T_NLINK_SH);
#endif
add_fix_values(&size.processes,
&size.processes_used,
fi,
ERTS_ALC_T_MSG_REF);
}
if (want.atom || want.atom_used) {
Uint reserved_atom_space, atom_space;
erts_atom_get_text_space_sizes(&reserved_atom_space, &atom_space);
size.atom = size.atom_used = atom_table_sz();
if (want.atom)
size.atom += reserved_atom_space;
if (want.atom_used)
size.atom_used += atom_space;
}
if (!ERTS_MEM_NEED_ALL_ALCU && want.binary)
size.binary = alcu_size(ERTS_ALC_A_BINARY, NULL, 0);
if (want.code) {
size.code = module_table_sz();
size.code += export_table_sz();
size.code += export_list_size() * sizeof(Export);
size.code += erts_fun_table_sz();
size.code += allocated_modules*sizeof(Range);
size.code += erts_total_code_size;
}
if (want.ets) {
if (!ERTS_MEM_NEED_ALL_ALCU)
size.ets = alcu_size(ERTS_ALC_A_ETS, NULL, 0);
size.ets += erts_get_ets_misc_mem_size();
}
if (erts_instr_stat && (want_tot_or_sys || want.maximum)) {
if (want_tot_or_sys) {
size.total = erts_instr_get_total();
size.system = size.total - size.processes;
}
size.maximum = erts_instr_get_max_total();
}
else if (want_tot_or_sys) {
size.system = size.total - size.processes;
}
if (print_to_p) {
int i;
int to = *print_to_p;
void *arg = print_to_arg;
/* Print result... */
erts_print(to, arg, "=memory\n");
for (i = 0; i < length; i++)
erts_print(to, arg, "%T: %bpu\n", atoms[i], *uintps[i]);
}
if (proc) {
/* Build erlang term result... */
Uint *hp;
Uint hsz;
erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
if (only_one_value) {
ASSERT(length == 1);
hsz = 0;
erts_bld_uword(NULL, &hsz, *uintps[0]);
hp = hsz ? HAlloc((Process *) proc, hsz) : NULL;
res = erts_bld_uword(&hp, NULL, *uintps[0]);
}
else {
Uint **hpp = NULL;
Uint *hszp = &hsz;
hsz = 0;
while (1) {
int i;
for (i = 0; i < length; i++)
euints[i] = erts_bld_uword(hpp, hszp, *uintps[i]);
res = erts_bld_2tup_list(hpp, hszp, length, atoms, euints);
if (hpp)
break;
hp = HAlloc((Process *) proc, hsz);
hpp = &hp;
hszp = NULL;
}
}
}
return res;
#undef ERTS_MEM_NEED_ALL_ALCU
}
struct aa_values {
Uint arity;
const char *name;
Uint ui[2];
};
Eterm
erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc)
{
#define MAX_AA_VALUES (23)
struct aa_values values[MAX_AA_VALUES];
Eterm res = THE_NON_VALUE;
int i, length;
Uint reserved_atom_space, atom_space;
if (proc) {
ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN
== erts_proc_lc_my_proc_locks(proc));
/* We'll need locks early in the lock order */
erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
}
i = 0;
if (erts_instr_stat) {
values[i].arity = 2;
values[i].name = "total";
values[i].ui[0] = erts_instr_get_total();
i++;
values[i].arity = 2;
values[i].name = "maximum";
values[i].ui[0] = erts_instr_get_max_total();
i++;
}
values[i].arity = 2;
values[i].name = "sys_misc";
values[i].ui[0] = erts_sys_misc_mem_sz();
i++;
values[i].arity = 2;
values[i].name = "static";
values[i].ui[0] =
erts_max_ports*sizeof(Port) /* Port table */
+ erts_timer_wheel_memory_size() /* Timer wheel */
#ifdef SYS_TMP_BUF_SIZE
+ SYS_TMP_BUF_SIZE /* tmp_buf in sys on vxworks & ose */
#endif
;
i++;
erts_atom_get_text_space_sizes(&reserved_atom_space, &atom_space);
values[i].arity = 3;
values[i].name = "atom_space";
values[i].ui[0] = reserved_atom_space;
values[i].ui[1] = atom_space;
i++;
values[i].arity = 2;
values[i].name = "atom_table";
values[i].ui[0] = atom_table_sz();
i++;
values[i].arity = 2;
values[i].name = "module_table";
values[i].ui[0] = module_table_sz();
i++;
values[i].arity = 2;
values[i].name = "export_table";
values[i].ui[0] = export_table_sz();
i++;
values[i].arity = 2;
values[i].name = "export_list";
values[i].ui[0] = export_list_size() * sizeof(Export);
i++;
values[i].arity = 2;
values[i].name = "register_table";
values[i].ui[0] = process_reg_sz();
i++;
values[i].arity = 2;
values[i].name = "fun_table";
values[i].ui[0] = erts_fun_table_sz();
i++;
values[i].arity = 2;
values[i].name = "module_refs";
values[i].ui[0] = allocated_modules*sizeof(Range);
i++;
values[i].arity = 2;
values[i].name = "loaded_code";
values[i].ui[0] = erts_total_code_size;
i++;
values[i].arity = 2;
values[i].name = "dist_table";
values[i].ui[0] = erts_dist_table_size();
i++;
values[i].arity = 2;
values[i].name = "node_table";
values[i].ui[0] = erts_node_table_size();
i++;
values[i].arity = 2;
values[i].name = "bits_bufs_size";
values[i].ui[0] = erts_bits_bufs_size();
i++;
values[i].arity = 2;
values[i].name = "bif_timer";
values[i].ui[0] = erts_bif_timer_memory_size();
i++;
values[i].arity = 2;
values[i].name = "link_lh";
values[i].ui[0] = erts_tot_link_lh_size();
i++;
values[i].arity = 2;
values[i].name = "process_table";
values[i].ui[0] = erts_max_processes*sizeof(Process*);
i++;
values[i].arity = 2;
values[i].name = "ets_misc";
values[i].ui[0] = erts_get_ets_misc_mem_size();
i++;
length = i;
ASSERT(length <= MAX_AA_VALUES);
if (print_to_p) {
/* Print result... */
int to = *print_to_p;
void *arg = print_to_arg;
erts_print(to, arg, "=allocated_areas\n");
for (i = 0; i < length; i++) {
switch (values[i].arity) {
case 2:
erts_print(to, arg, "%s: %beu\n",
values[i].name, values[i].ui[0]);
break;
case 3:
erts_print(to, arg, "%s: %beu %beu\n",
values[i].name, values[i].ui[0], values[i].ui[1]);
break;
default:
erts_print(to, arg, "ERROR: internal_error\n");
ASSERT(0);
return am_internal_error;
}
}
}
if (proc) {
/* Build erlang term result... */
Eterm tuples[MAX_AA_VALUES];
Uint *hp;
Uint **hpp;
Uint hsz;
Uint *hszp;
erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
hpp = NULL;
hsz = 0;
hszp = &hsz;
while (1) {
int i;
for (i = 0; i < length; i++) {
Eterm atom;
if (hpp)
atom = am_atom_put(values[i].name,
(int) strlen(values[i].name));
else
atom = am_true;
switch (values[i].arity) {
case 2:
tuples[i] = erts_bld_tuple(hpp, hszp, 2,
atom,
erts_bld_uint(hpp, hszp,
values[i].ui[0]));
break;
case 3:
tuples[i] = erts_bld_tuple(hpp, hszp, 3,
atom,
erts_bld_uint(hpp, hszp,
values[i].ui[0]),
erts_bld_uint(hpp, hszp,
values[i].ui[1]));
break;
default:
ASSERT(0);
return am_internal_error;
}
}
res = erts_bld_list(hpp, hszp, length, tuples);
if (hpp)
break;
hp = HAlloc((Process *) proc, hsz);
hpp = &hp;
hszp = NULL;
}
}
return res;
#undef MAX_AA_VALUES
}
Eterm
erts_alloc_util_allocators(void *proc)
{
Eterm res;
Uint *hp;
Uint sz;
int i;
/*
* Currently all allocators except sys_alloc are
* alloc_util allocators.
*/
sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 1)*2;
ASSERT(sz > 0);
hp = HAlloc((Process *) proc, sz);
res = NIL;
for (i = ERTS_ALC_A_MAX; i >= ERTS_ALC_A_MIN; i--) {
switch (i) {
case ERTS_ALC_A_SYSTEM:
break;
default: {
char *alc_str = (char *) ERTS_ALC_A2AD(i);
Eterm alc = am_atom_put(alc_str, sys_strlen(alc_str));
res = CONS(hp, alc, res);
hp += 2;
break;
}
}
}
return res;
}
void
erts_allocator_info(int to, void *arg)
{
ErtsAlcType_t a;
ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) {
int ai;
for (ai = 0; ai == 0 || ai < erts_allctrs_info[a].thr_spec; ai++) {
if (erts_allctrs_info[a].thr_spec) {
if (!erts_allctr_thr_spec[a].allctr[ai])
continue;
erts_print(to, arg, "=allocator:%s[%d]\n",
ERTS_ALC_A2AD(a), ai);
}
else {
erts_print(to, arg, "=allocator:%s\n", ERTS_ALC_A2AD(a));
}
if (!erts_allctrs_info[a].enabled)
erts_print(to, arg, "option e: false\n");
else {
if (erts_allctrs_info[a].alloc_util) {
void *as;
if (!erts_allctrs_info[a].thr_spec)
as = erts_allctrs_info[a].extra;
else {
ASSERT(erts_allctr_thr_spec[a].enabled);
as = erts_allctr_thr_spec[a].allctr[ai];
}
/* Binary alloc has its own thread safety... */
erts_alcu_info(as, 0, &to, arg, NULL, NULL);
}
else {
switch (a) {
case ERTS_ALC_A_SYSTEM: {
SysAllocStat sas;
erts_print(to, arg, "option e: true\n");
erts_print(to, arg, "option m: libc\n");
sys_alloc_stat(&sas);
if(sas.trim_threshold >= 0)
erts_print(to, arg, "option tt: %d\n", sas.trim_threshold);
if(sas.top_pad >= 0)
erts_print(to, arg, "option tp: %d\n", sas.top_pad);
break;
}
default:
ASSERT(0);
break;
}
}
}
}
}
#if HAVE_ERTS_MSEG
{
#ifdef ERTS_SMP
int max = (int) erts_no_schedulers;
#else
int max = 0;
#endif
int i;
for (i = 0; i <= max; i++) {
erts_print(to, arg, "=allocator:mseg_alloc[%d]\n", i);
erts_mseg_info(i, &to, arg, 0, NULL, NULL);
}
}
#endif
erts_print(to, arg, "=allocator:alloc_util\n");
erts_alcu_au_info_options(&to, arg, NULL, NULL);
erts_print(to, arg, "=allocator:instr\n");
erts_print(to, arg, "option m: %s\n",
erts_instr_memory_map ? "true" : "false");
erts_print(to, arg, "option s: %s\n",
erts_instr_stat ? "true" : "false");
erts_print(to, arg, "option t: %s\n",
erts_mtrace_enabled ? "true" : "false");
}
Eterm
erts_allocator_options(void *proc)
{
#if HAVE_ERTS_MSEG
int use_mseg = 0;
#endif
Uint sz, *szp, *hp, **hpp;
Eterm res, features, settings;
Eterm atoms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+5];
Uint terms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+5];
int a, length;
SysAllocStat sas;
Uint *endp = NULL;
sys_alloc_stat(&sas);
/* First find out the heap size needed ... */
hpp = NULL;
szp = &sz;
sz = 0;
bld_term:
length = 0;
features = NIL;
settings = NIL;
for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) {
Eterm tmp = NIL;
atoms[length] = am_atom_put((char *) ERTS_ALC_A2AD(a),
strlen(ERTS_ALC_A2AD(a)));
if (erts_allctrs_info[a].enabled) {
if (erts_allctrs_info[a].alloc_util) {
Allctr_t *allctr;
#if HAVE_ERTS_MSEG
use_mseg++;
#endif
if (erts_allctr_thr_spec[a].enabled)
allctr = erts_allctr_thr_spec[a].allctr[0];
else
allctr = erts_allctrs_info[a].extra;
tmp = erts_alcu_info_options(allctr, NULL, NULL, hpp, szp);
}
else {
int l = 0;
Eterm as[4];
Eterm ts[4];
as[l] = am_atom_put("e", 1);
ts[l++] = am_true;
switch (a) {
case ERTS_ALC_A_SYSTEM:
as[l] = am_atom_put("m", 1);
ts[l++] = am_atom_put("libc", 4);
if(sas.trim_threshold >= 0) {
as[l] = am_atom_put("tt", 2);
ts[l++] = erts_bld_uint(hpp, szp,
(Uint) sas.trim_threshold);
}
if(sas.top_pad >= 0) {
as[l] = am_atom_put("tp", 2);
ts[l++] = erts_bld_uint(hpp, szp, (Uint) sas.top_pad);
}
break;
default:
break;
}
tmp = erts_bld_2tup_list(hpp, szp, l, as, ts);
}
}
else {
Eterm atom = am_atom_put("e", 1);
Eterm term = am_false;
tmp = erts_bld_2tup_list(hpp, szp, 1, &atom, &term);
}
terms[length++] = tmp;
}
#if HAVE_ERTS_MSEG
if (use_mseg) {
atoms[length] = am_atom_put("mseg_alloc", 10);
terms[length++] = erts_mseg_info_options(0, NULL, NULL, hpp, szp);
}
#endif
atoms[length] = am_atom_put("alloc_util", 10);
terms[length++] = erts_alcu_au_info_options(NULL, NULL, hpp, szp);
{
Eterm o[3], v[3];
o[0] = am_atom_put("m", 1);
v[0] = erts_instr_memory_map ? am_true : am_false;
o[1] = am_atom_put("s", 1);
v[1] = erts_instr_stat ? am_true : am_false;
o[2] = am_atom_put("t", 1);
v[2] = erts_mtrace_enabled ? am_true : am_false;
atoms[length] = am_atom_put("instr", 5);
terms[length++] = erts_bld_2tup_list(hpp, szp, 3, o, v);
}
settings = erts_bld_2tup_list(hpp, szp, length, atoms, terms);
length = 0;
for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) {
if (erts_allctrs_info[a].enabled) {
terms[length++] = am_atom_put((char *) ERTS_ALC_A2AD(a),
strlen(ERTS_ALC_A2AD(a)));
}
}
#if HAVE_ERTS_MSEG
if (use_mseg)
terms[length++] = am_atom_put("mseg_alloc", 10);
#endif
features = length ? erts_bld_list(hpp, szp, length, terms) : NIL;
#if defined(__GLIBC__)
{
Eterm AM_glibc = am_atom_put("glibc", 5);
Eterm version;
version = erts_bld_cons(hpp,
szp,
make_small(__GLIBC__),
#ifdef __GLIBC_MINOR__
erts_bld_cons(hpp,
szp,
make_small(__GLIBC_MINOR__),
NIL)
#else
NIL
#endif
);
res = erts_bld_tuple(hpp, szp, 4,
AM_glibc, version, features, settings);
}
#else /* unknown allocator */
res = erts_bld_tuple(hpp, szp, 4,
am_undefined, NIL, features, settings);
#endif
if (szp) {
/* ... and then build the term */
hp = HAlloc((Process *) proc, sz);
endp = hp + sz;
hpp = &hp;
szp = NULL;
goto bld_term;
}
ASSERT(endp >= hp);
HRelease((Process *) proc, endp, hp);
return res;
}
void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size)
{
UWord v = (UWord) erts_alloc(type, size + (ERTS_CACHE_LINE_SIZE-1)
#ifdef VALGRIND
+ sizeof(UWord)
#endif
);
#ifdef VALGRIND
{ /* Link them to avoid Leak_PossiblyLost */
static UWord* first_in_list = NULL;
*(UWord**)v = first_in_list;
first_in_list = (UWord*) v;
v += sizeof(UWord);
}
#endif
if (v & ERTS_CACHE_LINE_MASK) {
v = (v & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE;
}
ASSERT((v & ERTS_CACHE_LINE_MASK) == 0);
return (void*)v;
}
static void
reply_alloc_info(void *vair)
{
ErtsAllocInfoReq *air = (ErtsAllocInfoReq *) vair;
Uint sched_id = erts_get_scheduler_id();
int global_instances = air->req_sched == sched_id;
ErtsProcLocks rp_locks;
Process *rp = air->proc;
Eterm ref_copy = NIL, ai_list, msg;
Eterm *hp = NULL, *hp_end = NULL, *hp_start = NULL;
Eterm **hpp;
Uint sz, *szp;
ErlOffHeap *ohp = NULL;
ErlHeapFragment *bp = NULL;
int i;
Eterm (*info_func)(Allctr_t *,
int,
int *,
void *,
Uint **,
Uint *) = (air->only_sz
? erts_alcu_sz_info
: erts_alcu_info);
rp_locks = air->req_sched == sched_id ? ERTS_PROC_LOCK_MAIN : 0;
sz = 0;
hpp = NULL;
szp = &sz;
while (1) {
if (hpp)
ref_copy = STORE_NC(hpp, ohp, air->ref);
else
*szp += REF_THING_SIZE;
ai_list = NIL;
for (i = 0; air->allocs[i] != ERTS_ALC_A_INVALID; i++);
for (i--; i >= 0; i--) {
int ai = air->allocs[i];
Allctr_t *allctr;
Eterm ainfo;
Eterm alloc_atom;
if (global_instances) {
switch (ai) {
case ERTS_ALC_A_SYSTEM: {
alloc_atom = erts_bld_atom(hpp, szp, "sys_alloc");
ainfo = NIL;
if (!air->only_sz) {
SysAllocStat sas;
if (hpp)
sys_alloc_stat(&sas);
if (szp) {
/* ensure ehough heap */
sas.top_pad = INT_MAX;
sas.trim_threshold = INT_MAX;
}
if (sas.top_pad >= 0) {
ainfo = erts_bld_cons(
hpp, szp,
erts_bld_tuple(
hpp, szp, 2,
erts_bld_atom(hpp, szp, "tp"),
erts_bld_uint(
hpp, szp,
(Uint) sas.top_pad)),
ainfo);
}
if (sas.trim_threshold >= 0) {
ainfo = erts_bld_cons(
hpp, szp,
erts_bld_tuple(
hpp, szp, 2,
erts_bld_atom(hpp, szp, "tt"),
erts_bld_uint(
hpp, szp,
(Uint) sas.trim_threshold)),
ainfo);
}
ainfo = erts_bld_cons(hpp, szp,
erts_bld_tuple(
hpp, szp, 2,
erts_bld_atom(hpp, szp,
"m"),
erts_bld_atom(hpp, szp,
"libc")),
ainfo);
ainfo = erts_bld_cons(hpp, szp,
erts_bld_tuple(
hpp, szp, 2,
erts_bld_atom(hpp, szp,
"e"),
am_true),
ainfo);
ainfo = erts_bld_tuple(hpp, szp, 2,
erts_bld_atom(hpp, szp,
"options"),
ainfo);
ainfo = erts_bld_cons(hpp, szp,ainfo,NIL);
}
ainfo = erts_bld_tuple(hpp, szp, 3,
alloc_atom,
make_small(0),
ainfo);
break;
}
case ERTS_ALC_INFO_A_ALLOC_UTIL:
alloc_atom = erts_bld_atom(hpp, szp, "alloc_util");
ainfo = (air->only_sz
? NIL
: erts_alcu_au_info_options(NULL, NULL,
hpp, szp));
ainfo = erts_bld_tuple(hpp, szp, 3,
alloc_atom,
make_small(0),
ainfo);
break;
case ERTS_ALC_INFO_A_MSEG_ALLOC:
alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc");
#if HAVE_ERTS_MSEG
ainfo = (air->only_sz
? NIL
: erts_mseg_info(0, NULL, NULL, hpp != NULL,
hpp, szp));
ainfo = erts_bld_tuple(hpp, szp, 3,
alloc_atom,
make_small(0),
ainfo);
#else
ainfo = erts_bld_tuple(hpp, szp, 2, alloc_atom,
am_false);
#endif
break;
default:
alloc_atom = erts_bld_atom(hpp, szp,
(char *) ERTS_ALC_A2AD(ai));
if (!erts_allctrs_info[ai].enabled)
ainfo = erts_bld_tuple(hpp, szp, 2, alloc_atom,
am_false);
else if (erts_allctrs_info[ai].alloc_util) {
if (erts_allctrs_info[ai].thr_spec)
allctr = erts_allctr_thr_spec[ai].allctr[0];
else
allctr = erts_allctrs_info[ai].extra;
ainfo = info_func(allctr, hpp != NULL, NULL,
NULL, hpp, szp);
ainfo = erts_bld_tuple(hpp, szp, 3, alloc_atom,
make_small(0), ainfo);
}
else {
erl_exit(ERTS_ABORT_EXIT, "%s:%d: internal error\n",
__FILE__, __LINE__);
}
}
ai_list = erts_bld_cons(hpp, szp,
ainfo, ai_list);
}
switch (ai) {
case ERTS_ALC_A_SYSTEM:
case ERTS_ALC_INFO_A_ALLOC_UTIL:
break;
case ERTS_ALC_INFO_A_MSEG_ALLOC:
#if HAVE_ERTS_MSEG && defined(ERTS_SMP)
alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc");
ainfo = (air->only_sz
? NIL
: erts_mseg_info(sched_id, NULL, NULL,
hpp != NULL, hpp, szp));
ainfo = erts_bld_tuple(hpp, szp, 3,
alloc_atom,
make_small(sched_id),
ainfo);
ai_list = erts_bld_cons(hpp, szp, ainfo, ai_list);
#endif
break;
default:
if (erts_allctrs_info[ai].thr_spec) {
alloc_atom = erts_bld_atom(hpp, szp,
(char *) ERTS_ALC_A2AD(ai));
allctr = erts_allctr_thr_spec[ai].allctr[sched_id];
ainfo = info_func(allctr, hpp != NULL, NULL,
NULL, hpp, szp);
ai_list = erts_bld_cons(hpp, szp,
erts_bld_tuple(
hpp, szp,
3,
alloc_atom,
make_small(sched_id),
ainfo),
ai_list);
}
break;
}
msg = erts_bld_tuple(hpp, szp,
3,
ref_copy,
make_small(sched_id),
ai_list);
}
if (hpp)
break;
hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks);
hp_start = hp;
hp_end = hp + sz;
szp = NULL;
hpp = &hp;
}
if (bp)
bp = erts_resize_message_buffer(bp, hp - hp_start, &msg, 1);
else {
ASSERT(hp);
HRelease(rp, hp_end, hp);
}
erts_queue_message(rp, &rp_locks, bp, msg, NIL);
if (air->req_sched == sched_id)
rp_locks &= ~ERTS_PROC_LOCK_MAIN;
erts_smp_proc_unlock(rp, rp_locks);
erts_smp_proc_dec_refc(rp);
if (erts_smp_atomic32_dec_read_nob(&air->refc) == 0)
aireq_free(air);
}
int
erts_request_alloc_info(struct process *c_p,
Eterm ref,
Eterm allocs,
int only_sz)
{
ErtsAllocInfoReq *air = aireq_alloc();
Eterm req_ai[ERTS_ALC_A_MAX+1+2] = {0};
Eterm alist;
Eterm *hp;
int airix = 0, ai;
air->req_sched = erts_get_scheduler_id();
air->only_sz = only_sz;
air->proc = c_p;
if (is_not_internal_ref(ref))
return 0;
hp = &air->ref_heap[0];
air->ref = STORE_NC(&hp, NULL, ref);
if (is_not_list(allocs))
return 0;
alist = allocs;
while (is_list(alist)) {
int saved = 0;
Eterm* consp = list_val(alist);
Eterm alloc = CAR(consp);
for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++)
if (erts_is_atom_str((char *) erts_alc_a2ad[ai], alloc))
goto save_alloc;
if (erts_is_atom_str("mseg_alloc", alloc)) {
ai = ERTS_ALC_INFO_A_MSEG_ALLOC;
goto save_alloc;
}
if (erts_is_atom_str("alloc_util", alloc)) {
ai = ERTS_ALC_INFO_A_ALLOC_UTIL;
save_alloc:
if (req_ai[ai])
return 0;
air->allocs[airix++] = ai;
req_ai[ai] = 1;
saved = 1;
}
if (!saved)
return 0;
alist = CDR(consp);
}
if (is_not_nil(alist))
return 0;
air->allocs[airix] = ERTS_ALC_A_INVALID;
erts_smp_atomic32_init_nob(&air->refc,
(erts_aint32_t) erts_no_schedulers);
erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers);
#ifdef ERTS_SMP
if (erts_no_schedulers > 1)
erts_schedule_multi_misc_aux_work(1,
erts_no_schedulers,
reply_alloc_info,
(void *) air);
#endif
reply_alloc_info((void *) air);
return 1;
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
* Deprecated functions *
* *
* These functions are still defined since "non-OTP linked in drivers" may *
* contain (illegal) calls to them. *
\* */
/* --- DO *NOT* USE THESE FUNCTIONS --- */
void *sys_alloc(Uint sz)
{ return erts_alloc_fnf(ERTS_ALC_T_UNDEF, sz); }
void *sys_realloc(void *ptr, Uint sz)
{ return erts_realloc_fnf(ERTS_ALC_T_UNDEF, ptr, sz); }
void sys_free(void *ptr)
{ erts_free(ERTS_ALC_T_UNDEF, ptr); }
void *safe_alloc(Uint sz)
{ return erts_alloc(ERTS_ALC_T_UNDEF, sz); }
void *safe_realloc(void *ptr, Uint sz)
{ return erts_realloc(ERTS_ALC_T_UNDEF, ptr, sz); }
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
* NOTE: erts_alc_test() is only supposed to be used for testing. *
* *
* Keep alloc_SUITE_data/allocator_test.h updated if changes are made *
* to erts_alc_test() *
\* */
#define ERTS_ALC_TEST_ABORT erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n")
UWord erts_alc_test(UWord op,
UWord a1,
UWord a2,
UWord a3)
{
switch (op >> 8) {
case 0x0: return erts_alcu_test(op, a1, a2);
case 0x1: return erts_gfalc_test(op, a1, a2);
case 0x2: return erts_bfalc_test(op, a1, a2);
case 0x3: return erts_afalc_test(op, a1, a2);
case 0x4: return erts_mseg_test(op, a1, a2, a3);
case 0x5: return erts_aoffalc_test(op, a1, a2);
case 0xf:
switch (op) {
case 0xf00:
#ifdef USE_THREADS
if (((Allctr_t *) a1)->thread_safe)
return (UWord) erts_alcu_alloc_ts(ERTS_ALC_T_UNDEF,
(void *) a1,
(Uint) a2);
else
#endif
return (UWord) erts_alcu_alloc(ERTS_ALC_T_UNDEF,
(void *) a1,
(Uint) a2);
case 0xf01:
#ifdef USE_THREADS
if (((Allctr_t *) a1)->thread_safe)
return (UWord) erts_alcu_realloc_ts(ERTS_ALC_T_UNDEF,
(void *) a1,
(void *) a2,
(Uint) a3);
else
#endif
return (UWord) erts_alcu_realloc(ERTS_ALC_T_UNDEF,
(void *) a1,
(void *) a2,
(Uint) a3);
case 0xf02:
#ifdef USE_THREADS
if (((Allctr_t *) a1)->thread_safe)
erts_alcu_free_ts(ERTS_ALC_T_UNDEF, (void *) a1, (void *) a2);
else
#endif
erts_alcu_free(ERTS_ALC_T_UNDEF, (void *) a1, (void *) a2);
return 0;
case 0xf03: {
Allctr_t *allctr;
struct au_init init;
SET_DEFAULT_ALLOC_OPTS(&init);
init.enable = 1;
init.atype = GOODFIT;
init.init.util.name_prefix = (char *) a1;
init.init.util.ts = a2 ? 1 : 0;
init.init.util.sbmbct = 0;
if ((char **) a3) {
char **argv = (char **) a3;
int i = 0;
while (argv[i]) {
if (argv[i][0] == '-' && argv[i][1] == 't')
handle_au_arg(&init, &argv[i][2], argv, &i);
else
return (UWord) NULL;
i++;
}
}
switch (init.atype) {
case GOODFIT:
allctr = erts_gfalc_start((GFAllctr_t *)
erts_alloc(ERTS_ALC_T_UNDEF,
sizeof(GFAllctr_t)),
&init.init.gf,
&init.init.util);
break;
case BESTFIT:
allctr = erts_bfalc_start((BFAllctr_t *)
erts_alloc(ERTS_ALC_T_UNDEF,
sizeof(BFAllctr_t)),
&init.init.bf,
&init.init.util);
break;
case AFIT:
allctr = erts_afalc_start((AFAllctr_t *)
erts_alloc(ERTS_ALC_T_UNDEF,
sizeof(AFAllctr_t)),
&init.init.af,
&init.init.util);
break;
case AOFIRSTFIT:
allctr = erts_aoffalc_start((AOFFAllctr_t *)
erts_alloc(ERTS_ALC_T_UNDEF,
sizeof(AOFFAllctr_t)),
&init.init.aoff,
&init.init.util);
break;
default:
ASSERT(0);
allctr = NULL;
break;
}
return (UWord) allctr;
}
case 0xf04:
erts_alcu_stop((Allctr_t *) a1);
erts_free(ERTS_ALC_T_UNDEF, (void *) a1);
break;
#ifdef USE_THREADS
case 0xf05: return (UWord) 1;
case 0xf06: return (UWord) ((Allctr_t *) a1)->thread_safe;
#ifdef ETHR_NO_FORKSAFETY
case 0xf07: return (UWord) 0;
#else
case 0xf07: return (UWord) ((Allctr_t *) a1)->thread_safe;
#endif
case 0xf08: {
ethr_mutex *mtx = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_mutex));
if (ethr_mutex_init(mtx) != 0)
ERTS_ALC_TEST_ABORT;
return (UWord) mtx;
}
case 0xf09: {
ethr_mutex *mtx = (ethr_mutex *) a1;
if (ethr_mutex_destroy(mtx) != 0)
ERTS_ALC_TEST_ABORT;
erts_free(ERTS_ALC_T_UNDEF, (void *) mtx);
break;
}
case 0xf0a:
ethr_mutex_lock((ethr_mutex *) a1);
break;
case 0xf0b:
ethr_mutex_unlock((ethr_mutex *) a1);
break;
case 0xf0c: {
ethr_cond *cnd = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_cond));
if (ethr_cond_init(cnd) != 0)
ERTS_ALC_TEST_ABORT;
return (UWord) cnd;
}
case 0xf0d: {
ethr_cond *cnd = (ethr_cond *) a1;
if (ethr_cond_destroy(cnd) != 0)
ERTS_ALC_TEST_ABORT;
erts_free(ERTS_ALC_T_UNDEF, (void *) cnd);
break;
}
case 0xf0e:
ethr_cond_broadcast((ethr_cond *) a1);
break;
case 0xf0f: {
int res;
do {
res = ethr_cond_wait((ethr_cond *) a1, (ethr_mutex *) a2);
} while (res == EINTR);
break;
}
case 0xf10: {
ethr_tid *tid = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_tid));
if (ethr_thr_create(tid,
(void * (*)(void *)) a1,
(void *) a2,
NULL) != 0)
ERTS_ALC_TEST_ABORT;
return (UWord) tid;
}
case 0xf11: {
ethr_tid *tid = (ethr_tid *) a1;
if (ethr_thr_join(*tid, NULL) != 0)
ERTS_ALC_TEST_ABORT;
erts_free(ERTS_ALC_T_UNDEF, (void *) tid);
break;
}
case 0xf12:
ethr_thr_exit((void *) a1);
ERTS_ALC_TEST_ABORT;
break;
#endif /* #ifdef USE_THREADS */
default:
break;
}
return (UWord) 0;
default:
break;
}
ASSERT(0);
return ~((UWord) 0);
}
#ifdef DEBUG
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
* Debug stuff *
\* */
#if 0
#define PRINT_OPS
#else
#undef PRINT_OPS
#endif
#ifdef HARD_DEBUG
#define FENCE_SZ (4*sizeof(UWord))
#else
#define FENCE_SZ (3*sizeof(UWord))
#endif
#if defined(ARCH_64)
#define FENCE_PATTERN 0xABCDEF97ABCDEF97
#else
#define FENCE_PATTERN 0xABCDEF97
#endif
#define TYPE_PATTERN_MASK ERTS_ALC_N_MASK
#define TYPE_PATTERN_SHIFT 16
#define FIXED_FENCE_PATTERN_MASK \
(~((UWord) (TYPE_PATTERN_MASK << TYPE_PATTERN_SHIFT)))
#define FIXED_FENCE_PATTERN \
(FENCE_PATTERN & FIXED_FENCE_PATTERN_MASK)
#define MK_PATTERN(T) \
(FIXED_FENCE_PATTERN | (((T) & TYPE_PATTERN_MASK) << TYPE_PATTERN_SHIFT))
#define GET_TYPE_OF_PATTERN(P) \
(((P) >> TYPE_PATTERN_SHIFT) & TYPE_PATTERN_MASK)
#ifdef HARD_DEBUG
#define ERL_ALC_HDBG_MAX_MBLK 100000
#define ERTS_ALC_O_CHECK -1
typedef struct hdbg_mblk_ hdbg_mblk;
struct hdbg_mblk_ {
hdbg_mblk *next;
hdbg_mblk *prev;
void *p;
Uint s;
ErtsAlcType_t n;
};
static hdbg_mblk hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK];
static hdbg_mblk *free_hdbg_mblks;
static hdbg_mblk *used_hdbg_mblks;
static erts_mtx_t hdbg_mblk_mtx;
static void
hdbg_init(void)
{
int i;
for (i = 0; i < ERL_ALC_HDBG_MAX_MBLK-1; i++)
hdbg_mblks[i].next = &hdbg_mblks[i+1];
hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK-1].next = NULL;
free_hdbg_mblks = &hdbg_mblks[0];
used_hdbg_mblks = NULL;
erts_mtx_init(&hdbg_mblk_mtx, "erts_alloc_hard_debug");
}
static void *check_memory_fence(void *ptr,
Uint *size,
ErtsAlcType_t n,
int func);
void erts_hdbg_chk_blks(void);
void
erts_hdbg_chk_blks(void)
{
hdbg_mblk *mblk;
erts_mtx_lock(&hdbg_mblk_mtx);
for (mblk = used_hdbg_mblks; mblk; mblk = mblk->next) {
Uint sz;
check_memory_fence(mblk->p, &sz, mblk->n, ERTS_ALC_O_CHECK);
ASSERT(sz == mblk->s);
}
erts_mtx_unlock(&hdbg_mblk_mtx);
}
static hdbg_mblk *
hdbg_alloc(void *p, Uint s, ErtsAlcType_t n)
{
hdbg_mblk *mblk;
erts_mtx_lock(&hdbg_mblk_mtx);
mblk = free_hdbg_mblks;
if (!mblk) {
erts_fprintf(stderr,
"Ran out of debug blocks; please increase "
"ERL_ALC_HDBG_MAX_MBLK=%d and recompile!\n",
ERL_ALC_HDBG_MAX_MBLK);
abort();
}
free_hdbg_mblks = mblk->next;
mblk->p = p;
mblk->s = s;
mblk->n = n;
mblk->next = used_hdbg_mblks;
mblk->prev = NULL;
if (used_hdbg_mblks)
used_hdbg_mblks->prev = mblk;
used_hdbg_mblks = mblk;
erts_mtx_unlock(&hdbg_mblk_mtx);
return (void *) mblk;
}
static void
hdbg_free(hdbg_mblk *mblk)
{
erts_mtx_lock(&hdbg_mblk_mtx);
if (mblk->next)
mblk->next->prev = mblk->prev;
if (mblk->prev)
mblk->prev->next = mblk->next;
else
used_hdbg_mblks = mblk->next;
mblk->next = free_hdbg_mblks;
free_hdbg_mblks = mblk;
erts_mtx_unlock(&hdbg_mblk_mtx);
}
#endif
#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
static void *check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func);
void check_allocated_block( Uint type, void *blk)
{
Uint dummy;
check_memory_fence(blk, &dummy, ERTS_ALC_T2N(type), ERTS_ALC_O_FREE);
}
void check_allocators(void)
{
int i;
if (!erts_initialized)
return;
for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; ++i) {
if (erts_allctrs_info[i].alloc_util) {
ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) erts_allctrs[i].extra;
Allctr_t *allctr = real_af->extra;
Carrier_t *ct;
#ifdef USE_THREADS
if (allctr->thread_safe)
erts_mtx_lock(&allctr->mutex);
#endif
if (allctr->check_mbc) {
for (ct = allctr->mbc_list.first; ct; ct = ct->next) {
fprintf(stderr,"Checking allocator %d\r\n",i);
allctr->check_mbc(allctr,ct);
}
}
#ifdef USE_THREADS
if (allctr->thread_safe)
erts_mtx_unlock(&allctr->mutex);
#endif
}
}
}
#endif
static void *
set_memory_fence(void *ptr, Uint sz, ErtsAlcType_t n)
{
UWord *ui_ptr;
UWord pattern;
#ifdef HARD_DEBUG
hdbg_mblk **mblkpp;
#endif
if (!ptr)
return NULL;
ui_ptr = (UWord *) ptr;
pattern = MK_PATTERN(n);
#ifdef HARD_DEBUG
mblkpp = (hdbg_mblk **) ui_ptr++;
#endif
*(ui_ptr++) = sz;
*(ui_ptr++) = pattern;
memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(UWord));
#ifdef HARD_DEBUG
*mblkpp = hdbg_alloc((void *) ui_ptr, sz, n);
#endif
return (void *) ui_ptr;
}
static void *
check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
{
Uint sz;
Uint found_type;
UWord pre_pattern;
UWord post_pattern;
UWord *ui_ptr;
#ifdef HARD_DEBUG
hdbg_mblk *mblk;
#endif
if (!ptr)
return NULL;
ui_ptr = (UWord *) ptr;
pre_pattern = *(--ui_ptr);
*size = sz = *(--ui_ptr);
#ifdef HARD_DEBUG
mblk = (hdbg_mblk *) *(--ui_ptr);
#endif
found_type = GET_TYPE_OF_PATTERN(pre_pattern);
if (pre_pattern != MK_PATTERN(n)) {
if ((FIXED_FENCE_PATTERN_MASK & pre_pattern) != FIXED_FENCE_PATTERN)
erl_exit(ERTS_ABORT_EXIT,
"ERROR: Fence at beginning of memory block (p=0x%u) "
"clobbered.\n",
(UWord) ptr);
}
memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(UWord));
if (post_pattern != MK_PATTERN(n)
|| pre_pattern != post_pattern) {
char fbuf[10];
char obuf[10];
char *ftype;
char *otype;
char *op_str;
if ((FIXED_FENCE_PATTERN_MASK & post_pattern) != FIXED_FENCE_PATTERN)
erl_exit(ERTS_ABORT_EXIT,
"ERROR: Fence at end of memory block (p=0x%u, sz=%u) "
"clobbered.\n",
(UWord) ptr, (UWord) sz);
if (found_type != GET_TYPE_OF_PATTERN(post_pattern))
erl_exit(ERTS_ABORT_EXIT,
"ERROR: Fence around memory block (p=0x%u, sz=%u) "
"clobbered.\n",
(UWord) ptr, (UWord) sz);
ftype = type_no_str(found_type);
if (!ftype) {
sprintf(fbuf, "%d", (int) found_type);
ftype = fbuf;
}
otype = type_no_str(n);
if (!otype) {
sprintf(obuf, "%d", (int) n);
otype = obuf;
}
switch (func) {
case ERTS_ALC_O_ALLOC: op_str = "allocated"; break;
case ERTS_ALC_O_REALLOC: op_str = "reallocated"; break;
case ERTS_ALC_O_FREE: op_str = "freed"; break;
default: op_str = "???"; break;
}
erl_exit(ERTS_ABORT_EXIT,
"ERROR: Memory block (p=0x%u, sz=%u) allocated as type \"%s\","
" but %s as type \"%s\".\n",
(UWord) ptr, (UWord) sz, ftype, op_str, otype);
}
#ifdef HARD_DEBUG
switch (func) {
case ERTS_ALC_O_REALLOC:
case ERTS_ALC_O_FREE:
hdbg_free(mblk);
break;
default:
break;
}
#endif
return (void *) ui_ptr;
}
static ErtsAllocatorFunctions_t real_allctrs[ERTS_ALC_A_MAX+1];
static void *
debug_alloc(ErtsAlcType_t n, void *extra, Uint size)
{
ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
Uint dsize;
void *res;
#ifdef HARD_DEBUG
erts_hdbg_chk_blks();
#endif
ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX);
dsize = size + FENCE_SZ;
res = (*real_af->alloc)(n, real_af->extra, dsize);
res = set_memory_fence(res, size, n);
#ifdef PRINT_OPS
fprintf(stderr, "0x%lx = alloc(%s, %lu)\r\n",
(Uint) res, ERTS_ALC_N2TD(n), size);
#endif
return res;
}
static void *
debug_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size)
{
ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
Uint dsize;
Uint old_size;
void *dptr;
void *res;
ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX);
dsize = size + FENCE_SZ;
dptr = check_memory_fence(ptr, &old_size, n, ERTS_ALC_O_REALLOC);
#ifdef HARD_DEBUG
erts_hdbg_chk_blks();
#endif
if (old_size > size)
sys_memset((void *) (((char *) ptr) + size),
0xf,
sizeof(Uint) + old_size - size);
res = (*real_af->realloc)(n, real_af->extra, dptr, dsize);
res = set_memory_fence(res, size, n);
#ifdef PRINT_OPS
fprintf(stderr, "0x%lx = realloc(%s, 0x%lx, %lu)\r\n",
(Uint) res, ERTS_ALC_N2TD(n), (Uint) ptr, size);
#endif
return res;
}
static void
debug_free(ErtsAlcType_t n, void *extra, void *ptr)
{
ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
void *dptr;
Uint size;
ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX);
dptr = check_memory_fence(ptr, &size, n, ERTS_ALC_O_FREE);
sys_memset((void *) dptr, n, size + FENCE_SZ);
(*real_af->free)(n, real_af->extra, dptr);
#ifdef PRINT_OPS
fprintf(stderr, "free(%s, 0x%lx)\r\n", ERTS_ALC_N2TD(n), (Uint) ptr);
#endif
#ifdef HARD_DEBUG
erts_hdbg_chk_blks();
#endif
}
static Uint
install_debug_functions(void)
{
int i;
ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs));
sys_memcpy((void *)real_allctrs,(void *)erts_allctrs,sizeof(erts_allctrs));
for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
erts_allctrs[i].alloc = debug_alloc;
erts_allctrs[i].realloc = debug_realloc;
erts_allctrs[i].free = debug_free;
erts_allctrs[i].extra = (void *) &real_allctrs[i];
}
return FENCE_SZ;
}
#endif /* #ifdef DEBUG */
