/*
* %CopyrightBegin%
*
* Copyright Ericsson AB 2002-2009. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
* compliance with the License. You should have received a copy of the
* Erlang Public License along with this software. If not, it can be
* retrieved online at http://www.erlang.org/.
*
* Software distributed under the License is distributed on an "AS IS"
* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
* the License for the specific language governing rights and limitations
* under the License.
*
* %CopyrightEnd%
*/
/*
* Description: A memory segment allocator. Segments that are deallocated
* are kept for a while in a segment "cache" before they are
* destroyed. When segments are allocated, cached segments
* are used if possible instead of creating new segments.
*
* Author: Rickard Green
*/
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif
#include "sys.h"
#include "erl_mseg.h"
#include "global.h"
#include "erl_threads.h"
#include "erl_mtrace.h"
#include "big.h"
#if HAVE_ERTS_MSEG
#if defined(USE_THREADS) && !defined(ERTS_SMP)
# define ERTS_THREADS_NO_SMP
#endif
#define SEGTYPE ERTS_MTRACE_SEGMENT_ID
#ifndef HAVE_GETPAGESIZE
#define HAVE_GETPAGESIZE 0
#endif
#ifdef _SC_PAGESIZE
# define GET_PAGE_SIZE sysconf(_SC_PAGESIZE)
#elif HAVE_GETPAGESIZE
# define GET_PAGE_SIZE getpagesize()
#else
# error "Page size unknown"
/* Implement some other way to get the real page size if needed! */
#endif
#define MAX_CACHE_SIZE 30
#undef MIN
#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
#undef MAX
#define MAX(X, Y) ((X) > (Y) ? (X) : (Y))
#undef PAGE_MASK
#define INV_PAGE_MASK ((Uint) (page_size - 1))
#define PAGE_MASK (~INV_PAGE_MASK)
#define PAGE_FLOOR(X) ((X) & PAGE_MASK)
#define PAGE_CEILING(X) PAGE_FLOOR((X) + INV_PAGE_MASK)
#define PAGES(X) ((X) >> page_shift)
static int atoms_initialized;
static Uint cache_check_interval;
static void check_cache(void *unused);
static void mseg_clear_cache(void);
static int is_cache_check_scheduled;
#ifdef ERTS_THREADS_NO_SMP
static int is_cache_check_requested;
#endif
#if HAVE_MMAP
/* Mmap ... */
#define MMAP_PROT (PROT_READ|PROT_WRITE)
#ifdef MAP_ANON
# define MMAP_FLAGS (MAP_ANON|MAP_PRIVATE)
# define MMAP_FD (-1)
#else
# define MMAP_FLAGS (MAP_PRIVATE)
# define MMAP_FD mmap_fd
static int mmap_fd;
#endif
#if HAVE_MREMAP
# define HAVE_MSEG_RECREATE 1
#else
# define HAVE_MSEG_RECREATE 0
#endif
#define CAN_PARTLY_DESTROY 1
#else /* #if HAVE_MMAP */
#define CAN_PARTLY_DESTROY 0
#error "Not supported"
#endif /* #if HAVE_MMAP */
#if defined(ERTS_MSEG_FAKE_SEGMENTS)
#undef CAN_PARTLY_DESTROY
#define CAN_PARTLY_DESTROY 0
#endif
static const ErtsMsegOpt_t default_opt = ERTS_MSEG_DEFAULT_OPT_INITIALIZER;
typedef struct cache_desc_t_ {
void *seg;
Uint size;
struct cache_desc_t_ *next;
struct cache_desc_t_ *prev;
} cache_desc_t;
typedef struct {
Uint32 giga_no;
Uint32 no;
} CallCounter;
static int is_init_done;
static Uint page_size;
static Uint page_shift;
static struct {
CallCounter alloc;
CallCounter dealloc;
CallCounter realloc;
CallCounter create;
CallCounter destroy;
#if HAVE_MSEG_RECREATE
CallCounter recreate;
#endif
CallCounter clear_cache;
CallCounter check_cache;
} calls;
static cache_desc_t cache_descs[MAX_CACHE_SIZE];
static cache_desc_t *free_cache_descs;
static cache_desc_t *cache;
static cache_desc_t *cache_end;
static Uint cache_hits;
static Uint cache_size;
static Uint min_cached_seg_size;
static Uint max_cached_seg_size;
static Uint max_cache_size;
static Uint abs_max_cache_bad_fit;
static Uint rel_max_cache_bad_fit;
#if CAN_PARTLY_DESTROY
static Uint min_seg_size;
#endif
struct {
struct {
Uint watermark;
Uint no;
Uint sz;
} current;
struct {
Uint no;
Uint sz;
} max;
struct {
Uint no;
Uint sz;
} max_ever;
} segments;
#define ERTS_MSEG_ALLOC_STAT(SZ) \
do { \
segments.current.no++; \
if (segments.max.no < segments.current.no) \
segments.max.no = segments.current.no; \
if (segments.current.watermark < segments.current.no) \
segments.current.watermark = segments.current.no; \
segments.current.sz += (SZ); \
if (segments.max.sz < segments.current.sz) \
segments.max.sz = segments.current.sz; \
} while (0)
#define ERTS_MSEG_DEALLOC_STAT(SZ) \
do { \
ASSERT(segments.current.no > 0); \
segments.current.no--; \
ASSERT(segments.current.sz >= (SZ)); \
segments.current.sz -= (SZ); \
} while (0)
#define ERTS_MSEG_REALLOC_STAT(OSZ, NSZ) \
do { \
ASSERT(segments.current.sz >= (OSZ)); \
segments.current.sz -= (OSZ); \
segments.current.sz += (NSZ); \
} while (0)
#define ONE_GIGA (1000000000)
#define ZERO_CC(CC) (calls.CC.no = 0, calls.CC.giga_no = 0)
#define INC_CC(CC) (calls.CC.no == ONE_GIGA - 1 \
? (calls.CC.giga_no++, calls.CC.no = 0) \
: calls.CC.no++)
#define DEC_CC(CC) (calls.CC.no == 0 \
? (calls.CC.giga_no--, \
calls.CC.no = ONE_GIGA - 1) \
: calls.CC.no--)
static erts_mtx_t mseg_mutex; /* Also needed when !USE_THREADS */
static erts_mtx_t init_atoms_mutex; /* Also needed when !USE_THREADS */
#ifdef USE_THREADS
#ifdef ERTS_THREADS_NO_SMP
static erts_tid_t main_tid;
static int async_handle = -1;
#endif
static void thread_safe_init(void)
{
erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms");
erts_mtx_init(&mseg_mutex, "mseg");
#ifdef ERTS_THREADS_NO_SMP
main_tid = erts_thr_self();
#endif
}
#endif
static ErlTimer cache_check_timer;
static ERTS_INLINE void
schedule_cache_check(void)
{
if (!is_cache_check_scheduled && is_init_done) {
#ifdef ERTS_THREADS_NO_SMP
if (!erts_equal_tids(erts_thr_self(), main_tid)) {
if (!is_cache_check_requested) {
is_cache_check_requested = 1;
sys_async_ready(async_handle);
}
}
else
#endif
{
cache_check_timer.active = 0;
erl_set_timer(&cache_check_timer,
check_cache,
NULL,
NULL,
cache_check_interval);
is_cache_check_scheduled = 1;
#ifdef ERTS_THREADS_NO_SMP
is_cache_check_requested = 0;
#endif
}
}
}
#ifdef ERTS_THREADS_NO_SMP
static void
check_schedule_cache_check(void)
{
erts_mtx_lock(&mseg_mutex);
if (is_cache_check_requested
&& !is_cache_check_scheduled) {
schedule_cache_check();
}
erts_mtx_unlock(&mseg_mutex);
}
#endif
static void
mseg_shutdown(void)
{
#ifdef ERTS_SMP
erts_mtx_lock(&mseg_mutex);
#endif
mseg_clear_cache();
#ifdef ERTS_SMP
erts_mtx_unlock(&mseg_mutex);
#endif
}
static ERTS_INLINE void *
mseg_create(Uint size)
{
void *seg;
ASSERT(size % page_size == 0);
#if defined(ERTS_MSEG_FAKE_SEGMENTS)
seg = erts_sys_alloc(ERTS_ALC_N_INVALID, NULL, size);
#elif HAVE_MMAP
seg = (void *) mmap((void *) 0, (size_t) size,
MMAP_PROT, MMAP_FLAGS, MMAP_FD, 0);
if (seg == (void *) MAP_FAILED)
seg = NULL;
#else
#error "Missing mseg_create() implementation"
#endif
INC_CC(create);
return seg;
}
static ERTS_INLINE void
mseg_destroy(void *seg, Uint size)
{
#if defined(ERTS_MSEG_FAKE_SEGMENTS)
erts_sys_free(ERTS_ALC_N_INVALID, NULL, seg);
#elif HAVE_MMAP
#ifdef DEBUG
int res =
#endif
munmap((void *) seg, size);
ASSERT(size % page_size == 0);
ASSERT(res == 0);
#else
#error "Missing mseg_destroy() implementation"
#endif
INC_CC(destroy);
}
#if HAVE_MSEG_RECREATE
static ERTS_INLINE void *
mseg_recreate(void *old_seg, Uint old_size, Uint new_size)
{
void *new_seg;
ASSERT(old_size % page_size == 0);
ASSERT(new_size % page_size == 0);
#if defined(ERTS_MSEG_FAKE_SEGMENTS)
new_seg = erts_sys_realloc(ERTS_ALC_N_INVALID, NULL, old_seg, new_size);
#elif HAVE_MREMAP
new_seg = (void *) mremap((void *) old_seg,
(size_t) old_size,
(size_t) new_size,
MREMAP_MAYMOVE);
if (new_seg == (void *) MAP_FAILED)
new_seg = NULL;
#else
#error "Missing mseg_recreate() implementation"
#endif
INC_CC(recreate);
return new_seg;
}
#endif /* #if HAVE_MSEG_RECREATE */
static ERTS_INLINE cache_desc_t *
alloc_cd(void)
{
cache_desc_t *cd = free_cache_descs;
if (cd)
free_cache_descs = cd->next;
return cd;
}
static ERTS_INLINE void
free_cd(cache_desc_t *cd)
{
cd->next = free_cache_descs;
free_cache_descs = cd;
}
static ERTS_INLINE void
link_cd(cache_desc_t *cd)
{
if (cache)
cache->prev = cd;
cd->next = cache;
cd->prev = NULL;
cache = cd;
if (!cache_end) {
ASSERT(!cd->next);
cache_end = cd;
}
cache_size++;
}
static ERTS_INLINE void
end_link_cd(cache_desc_t *cd)
{
if (cache_end)
cache_end->next = cd;
cd->next = NULL;
cd->prev = cache_end;
cache_end = cd;
if (!cache) {
ASSERT(!cd->prev);
cache = cd;
}
cache_size++;
}
static ERTS_INLINE void
unlink_cd(cache_desc_t *cd)
{
if (cd->next)
cd->next->prev = cd->prev;
else
cache_end = cd->prev;
if (cd->prev)
cd->prev->next = cd->next;
else
cache = cd->next;
ASSERT(cache_size > 0);
cache_size--;
}
static ERTS_INLINE void
check_cache_limits(void)
{
cache_desc_t *cd;
max_cached_seg_size = 0;
min_cached_seg_size = ~((Uint) 0);
for (cd = cache; cd; cd = cd->next) {
if (cd->size < min_cached_seg_size)
min_cached_seg_size = cd->size;
if (cd->size > max_cached_seg_size)
max_cached_seg_size = cd->size;
}
}
static ERTS_INLINE void
adjust_cache_size(int force_check_limits)
{
cache_desc_t *cd;
int check_limits = force_check_limits;
Sint max_cached = ((Sint) segments.current.watermark
- (Sint) segments.current.no);
while (((Sint) cache_size) > max_cached && ((Sint) cache_size) > 0) {
ASSERT(cache_end);
cd = cache_end;
if (!check_limits &&
!(min_cached_seg_size < cd->size
&& cd->size < max_cached_seg_size)) {
check_limits = 1;
}
if (erts_mtrace_enabled)
erts_mtrace_crr_free(SEGTYPE, SEGTYPE, cd->seg);
mseg_destroy(cd->seg, cd->size);
unlink_cd(cd);
free_cd(cd);
}
if (check_limits)
check_cache_limits();
}
static void
check_cache(void *unused)
{
#ifdef ERTS_SMP
erts_mtx_lock(&mseg_mutex);
#endif
is_cache_check_scheduled = 0;
if (segments.current.watermark > segments.current.no)
segments.current.watermark--;
adjust_cache_size(0);
if (cache_size)
schedule_cache_check();
INC_CC(check_cache);
#ifdef ERTS_SMP
erts_mtx_unlock(&mseg_mutex);
#endif
}
static void
mseg_clear_cache(void)
{
segments.current.watermark = 0;
adjust_cache_size(1);
ASSERT(!cache);
ASSERT(!cache_end);
ASSERT(!cache_size);
segments.current.watermark = segments.current.no;
INC_CC(clear_cache);
}
static void *
mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt)
{
Uint max, min, diff_size, size;
cache_desc_t *cd, *cand_cd;
void *seg;
INC_CC(alloc);
size = PAGE_CEILING(*size_p);
#if CAN_PARTLY_DESTROY
if (size < min_seg_size)
min_seg_size = size;
#endif
if (!opt->cache) {
create_seg:
adjust_cache_size(0);
seg = mseg_create(size);
if (!seg) {
mseg_clear_cache();
seg = mseg_create(size);
if (!seg)
size = 0;
}
*size_p = size;
if (seg) {
if (erts_mtrace_enabled)
erts_mtrace_crr_alloc(seg, atype, ERTS_MTRACE_SEGMENT_ID, size);
ERTS_MSEG_ALLOC_STAT(size);
}
return seg;
}
if (size > max_cached_seg_size)
goto create_seg;
if (size < min_cached_seg_size) {
diff_size = min_cached_seg_size - size;
if (diff_size > abs_max_cache_bad_fit)
goto create_seg;
if (100*PAGES(diff_size) > rel_max_cache_bad_fit*PAGES(size))
goto create_seg;
}
max = 0;
min = ~((Uint) 0);
cand_cd = NULL;
for (cd = cache; cd; cd = cd->next) {
if (cd->size >= size) {
if (!cand_cd) {
cand_cd = cd;
continue;
}
else if (cd->size < cand_cd->size) {
if (max < cand_cd->size)
max = cand_cd->size;
if (min > cand_cd->size)
min = cand_cd->size;
cand_cd = cd;
continue;
}
}
if (max < cd->size)
max = cd->size;
if (min > cd->size)
min = cd->size;
}
min_cached_seg_size = min;
max_cached_seg_size = max;
if (!cand_cd)
goto create_seg;
diff_size = cand_cd->size - size;
if (diff_size > abs_max_cache_bad_fit
|| 100*PAGES(diff_size) > rel_max_cache_bad_fit*PAGES(size)) {
if (max_cached_seg_size < cand_cd->size)
max_cached_seg_size = cand_cd->size;
if (min_cached_seg_size > cand_cd->size)
min_cached_seg_size = cand_cd->size;
goto create_seg;
}
cache_hits++;
size = cand_cd->size;
seg = cand_cd->seg;
unlink_cd(cand_cd);
free_cd(cand_cd);
*size_p = size;
if (erts_mtrace_enabled) {
erts_mtrace_crr_free(SEGTYPE, SEGTYPE, seg);
erts_mtrace_crr_alloc(seg, atype, SEGTYPE, size);
}
if (seg)
ERTS_MSEG_ALLOC_STAT(size);
return seg;
}
static void
mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size,
const ErtsMsegOpt_t *opt)
{
cache_desc_t *cd;
ERTS_MSEG_DEALLOC_STAT(size);
if (!opt->cache || max_cache_size == 0) {
if (erts_mtrace_enabled)
erts_mtrace_crr_free(atype, SEGTYPE, seg);
mseg_destroy(seg, size);
}
else {
int check_limits = 0;
if (size < min_cached_seg_size)
min_cached_seg_size = size;
if (size > max_cached_seg_size)
max_cached_seg_size = size;
if (!free_cache_descs) {
cd = cache_end;
if (!(min_cached_seg_size < cd->size
&& cd->size < max_cached_seg_size)) {
check_limits = 1;
}
if (erts_mtrace_enabled)
erts_mtrace_crr_free(SEGTYPE, SEGTYPE, cd->seg);
mseg_destroy(cd->seg, cd->size);
unlink_cd(cd);
free_cd(cd);
}
cd = alloc_cd();
ASSERT(cd);
cd->seg = seg;
cd->size = size;
link_cd(cd);
if (erts_mtrace_enabled) {
erts_mtrace_crr_free(atype, SEGTYPE, seg);
erts_mtrace_crr_alloc(seg, SEGTYPE, SEGTYPE, size);
}
/* ASSERT(segments.current.watermark >= segments.current.no + cache_size); */
if (check_limits)
check_cache_limits();
schedule_cache_check();
}
INC_CC(dealloc);
}
static void *
mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p,
const ErtsMsegOpt_t *opt)
{
void *new_seg;
Uint new_size;
if (!seg || !old_size) {
new_seg = mseg_alloc(atype, new_size_p, opt);
DEC_CC(alloc);
return new_seg;
}
if (!(*new_size_p)) {
mseg_dealloc(atype, seg, old_size, opt);
DEC_CC(dealloc);
return NULL;
}
new_seg = seg;
new_size = PAGE_CEILING(*new_size_p);
if (new_size == old_size)
;
else if (new_size < old_size) {
Uint shrink_sz = old_size - new_size;
#if CAN_PARTLY_DESTROY
if (new_size < min_seg_size)
min_seg_size = new_size;
#endif
if (shrink_sz < opt->abs_shrink_th
&& 100*PAGES(shrink_sz) < opt->rel_shrink_th*PAGES(old_size)) {
new_size = old_size;
}
else {
#if CAN_PARTLY_DESTROY
if (shrink_sz > min_seg_size
&& free_cache_descs
&& opt->cache) {
cache_desc_t *cd;
cd = alloc_cd();
ASSERT(cd);
cd->seg = ((char *) seg) + new_size;
cd->size = shrink_sz;
end_link_cd(cd);
if (erts_mtrace_enabled) {
erts_mtrace_crr_realloc(new_seg,
atype,
SEGTYPE,
seg,
new_size);
erts_mtrace_crr_alloc(cd->seg, SEGTYPE, SEGTYPE, cd->size);
}
schedule_cache_check();
}
else {
if (erts_mtrace_enabled)
erts_mtrace_crr_realloc(new_seg,
atype,
SEGTYPE,
seg,
new_size);
mseg_destroy(((char *) seg) + new_size, shrink_sz);
}
#elif HAVE_MSEG_RECREATE
goto do_recreate;
#else
new_seg = mseg_alloc(atype, &new_size, opt);
if (!new_seg)
new_size = old_size;
else {
sys_memcpy(((char *) new_seg),
((char *) seg),
MIN(new_size, old_size));
mseg_dealloc(atype, seg, old_size, opt);
}
#endif
}
}
else {
if (!opt->preserv) {
mseg_dealloc(atype, seg, old_size, opt);
new_seg = mseg_alloc(atype, &new_size, opt);
}
else {
#if HAVE_MSEG_RECREATE
#if !CAN_PARTLY_DESTROY
do_recreate:
#endif
new_seg = mseg_recreate((void *) seg, old_size, new_size);
if (erts_mtrace_enabled)
erts_mtrace_crr_realloc(new_seg, atype, SEGTYPE, seg, new_size);
if (!new_seg)
new_size = old_size;
#else
new_seg = mseg_alloc(atype, &new_size, opt);
if (!new_seg)
new_size = old_size;
else {
sys_memcpy(((char *) new_seg),
((char *) seg),
MIN(new_size, old_size));
mseg_dealloc(atype, seg, old_size, opt);
}
#endif
}
}
INC_CC(realloc);
*new_size_p = new_size;
ERTS_MSEG_REALLOC_STAT(old_size, new_size);
return new_seg;
}
/* --- Info stuff ---------------------------------------------------------- */
static struct {
Eterm version;
Eterm options;
Eterm amcbf;
Eterm rmcbf;
Eterm mcs;
Eterm cci;
Eterm status;
Eterm cached_segments;
Eterm cache_hits;
Eterm segments;
Eterm segments_size;
Eterm segments_watermark;
Eterm calls;
Eterm mseg_alloc;
Eterm mseg_dealloc;
Eterm mseg_realloc;
Eterm mseg_create;
Eterm mseg_destroy;
#if HAVE_MSEG_RECREATE
Eterm mseg_recreate;
#endif
Eterm mseg_clear_cache;
Eterm mseg_check_cache;
#ifdef DEBUG
Eterm end_of_atoms;
#endif
} am;
static void ERTS_INLINE atom_init(Eterm *atom, char *name)
{
*atom = am_atom_put(name, strlen(name));
}
#define AM_INIT(AM) atom_init(&am.AM, #AM)
static void
init_atoms(void)
{
#ifdef DEBUG
Eterm *atom;
#endif
erts_mtx_unlock(&mseg_mutex);
erts_mtx_lock(&init_atoms_mutex);
if (!atoms_initialized) {
#ifdef DEBUG
for (atom = (Eterm *) &am; atom <= &am.end_of_atoms; atom++) {
*atom = THE_NON_VALUE;
}
#endif
AM_INIT(version);
AM_INIT(options);
AM_INIT(amcbf);
AM_INIT(rmcbf);
AM_INIT(mcs);
AM_INIT(cci);
AM_INIT(status);
AM_INIT(cached_segments);
AM_INIT(cache_hits);
AM_INIT(segments);
AM_INIT(segments_size);
AM_INIT(segments_watermark);
AM_INIT(calls);
AM_INIT(mseg_alloc);
AM_INIT(mseg_dealloc);
AM_INIT(mseg_realloc);
AM_INIT(mseg_create);
AM_INIT(mseg_destroy);
#if HAVE_MSEG_RECREATE
AM_INIT(mseg_recreate);
#endif
AM_INIT(mseg_clear_cache);
AM_INIT(mseg_check_cache);
#ifdef DEBUG
for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) {
ASSERT(*atom != THE_NON_VALUE);
}
#endif
}
erts_mtx_lock(&mseg_mutex);
atoms_initialized = 1;
erts_mtx_unlock(&init_atoms_mutex);
}
#define bld_uint erts_bld_uint
#define bld_cons erts_bld_cons
#define bld_tuple erts_bld_tuple
#define bld_string erts_bld_string
#define bld_2tup_list erts_bld_2tup_list
/*
* bld_unstable_uint() (instead of bld_uint()) is used when values may
* change between size check and actual build. This because a value
* that would fit a small when size check is done may need to be built
* as a big when the actual build is performed. Caller is required to
* HRelease after build.
*/
static ERTS_INLINE Eterm
bld_unstable_uint(Uint **hpp, Uint *szp, Uint ui)
{
Eterm res = THE_NON_VALUE;
if (szp)
*szp += BIG_UINT_HEAP_SIZE;
if (hpp) {
if (IS_USMALL(0, ui))
res = make_small(ui);
else {
res = uint_to_big(ui, *hpp);
*hpp += BIG_UINT_HEAP_SIZE;
}
}
return res;
}
static ERTS_INLINE void
add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2)
{
*lp = bld_cons(hpp, szp, bld_tuple(hpp, szp, 2, el1, el2), *lp);
}
static ERTS_INLINE void
add_3tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2, Eterm el3)
{
*lp = bld_cons(hpp, szp, bld_tuple(hpp, szp, 3, el1, el2, el3), *lp);
}
static ERTS_INLINE void
add_4tup(Uint **hpp, Uint *szp, Eterm *lp,
Eterm el1, Eterm el2, Eterm el3, Eterm el4)
{
*lp = bld_cons(hpp, szp, bld_tuple(hpp, szp, 4, el1, el2, el3, el4), *lp);
}
static Eterm
info_options(char *prefix,
int *print_to_p,
void *print_to_arg,
Uint **hpp,
Uint *szp)
{
Eterm res = THE_NON_VALUE;
if (print_to_p) {
int to = *print_to_p;
void *arg = print_to_arg;
erts_print(to, arg, "%samcbf: %bpu\n", prefix, abs_max_cache_bad_fit);
erts_print(to, arg, "%srmcbf: %bpu\n", prefix, rel_max_cache_bad_fit);
erts_print(to, arg, "%smcs: %bpu\n", prefix, max_cache_size);
erts_print(to, arg, "%scci: %bpu\n", prefix, cache_check_interval);
}
if (hpp || szp) {
if (!atoms_initialized)
init_atoms();
res = NIL;
add_2tup(hpp, szp, &res,
am.cci,
bld_uint(hpp, szp, cache_check_interval));
add_2tup(hpp, szp, &res,
am.mcs,
bld_uint(hpp, szp, max_cache_size));
add_2tup(hpp, szp, &res,
am.rmcbf,
bld_uint(hpp, szp, rel_max_cache_bad_fit));
add_2tup(hpp, szp, &res,
am.amcbf,
bld_uint(hpp, szp, abs_max_cache_bad_fit));
}
return res;
}
static Eterm
info_calls(int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp)
{
Eterm res = THE_NON_VALUE;
if (print_to_p) {
#define PRINT_CC(TO, TOA, CC) \
if (calls.CC.giga_no == 0) \
erts_print(TO, TOA, "mseg_%s calls: %bpu\n", #CC, calls.CC.no); \
else \
erts_print(TO, TOA, "mseg_%s calls: %bpu%09bpu\n", #CC, \
calls.CC.giga_no, calls.CC.no)
int to = *print_to_p;
void *arg = print_to_arg;
PRINT_CC(to, arg, alloc);
PRINT_CC(to, arg, dealloc);
PRINT_CC(to, arg, realloc);
PRINT_CC(to, arg, create);
PRINT_CC(to, arg, destroy);
#if HAVE_MSEG_RECREATE
PRINT_CC(to, arg, recreate);
#endif
PRINT_CC(to, arg, clear_cache);
PRINT_CC(to, arg, check_cache);
#undef PRINT_CC
}
if (hpp || szp) {
res = NIL;
add_3tup(hpp, szp, &res,
am.mseg_check_cache,
bld_unstable_uint(hpp, szp, calls.check_cache.giga_no),
bld_unstable_uint(hpp, szp, calls.check_cache.no));
add_3tup(hpp, szp, &res,
am.mseg_clear_cache,
bld_unstable_uint(hpp, szp, calls.clear_cache.giga_no),
bld_unstable_uint(hpp, szp, calls.clear_cache.no));
#if HAVE_MSEG_RECREATE
add_3tup(hpp, szp, &res,
am.mseg_recreate,
bld_unstable_uint(hpp, szp, calls.recreate.giga_no),
bld_unstable_uint(hpp, szp, calls.recreate.no));
#endif
add_3tup(hpp, szp, &res,
am.mseg_destroy,
bld_unstable_uint(hpp, szp, calls.destroy.giga_no),
bld_unstable_uint(hpp, szp, calls.destroy.no));
add_3tup(hpp, szp, &res,
am.mseg_create,
bld_unstable_uint(hpp, szp, calls.create.giga_no),
bld_unstable_uint(hpp, szp, calls.create.no));
add_3tup(hpp, szp, &res,
am.mseg_realloc,
bld_unstable_uint(hpp, szp, calls.realloc.giga_no),
bld_unstable_uint(hpp, szp, calls.realloc.no));
add_3tup(hpp, szp, &res,
am.mseg_dealloc,
bld_unstable_uint(hpp, szp, calls.dealloc.giga_no),
bld_unstable_uint(hpp, szp, calls.dealloc.no));
add_3tup(hpp, szp, &res,
am.mseg_alloc,
bld_unstable_uint(hpp, szp, calls.alloc.giga_no),
bld_unstable_uint(hpp, szp, calls.alloc.no));
}
return res;
}
static Eterm
info_status(int *print_to_p,
void *print_to_arg,
int begin_new_max_period,
Uint **hpp,
Uint *szp)
{
Eterm res = THE_NON_VALUE;
if (segments.max_ever.no < segments.max.no)
segments.max_ever.no = segments.max.no;
if (segments.max_ever.sz < segments.max.sz)
segments.max_ever.sz = segments.max.sz;
if (print_to_p) {
int to = *print_to_p;
void *arg = print_to_arg;
erts_print(to, arg, "cached_segments: %bpu\n", cache_size);
erts_print(to, arg, "cache_hits: %bpu\n", cache_hits);
erts_print(to, arg, "segments: %bpu %bpu %bpu\n",
segments.current.no, segments.max.no, segments.max_ever.no);
erts_print(to, arg, "segments_size: %bpu %bpu %bpu\n",
segments.current.sz, segments.max.sz, segments.max_ever.sz);
erts_print(to, arg, "segments_watermark: %bpu\n",
segments.current.watermark);
}
if (hpp || szp) {
res = NIL;
add_2tup(hpp, szp, &res,
am.segments_watermark,
bld_unstable_uint(hpp, szp, segments.current.watermark));
add_4tup(hpp, szp, &res,
am.segments_size,
bld_unstable_uint(hpp, szp, segments.current.sz),
bld_unstable_uint(hpp, szp, segments.max.sz),
bld_unstable_uint(hpp, szp, segments.max_ever.sz));
add_4tup(hpp, szp, &res,
am.segments,
bld_unstable_uint(hpp, szp, segments.current.no),
bld_unstable_uint(hpp, szp, segments.max.no),
bld_unstable_uint(hpp, szp, segments.max_ever.no));
add_2tup(hpp, szp, &res,
am.cache_hits,
bld_unstable_uint(hpp, szp, cache_hits));
add_2tup(hpp, szp, &res,
am.cached_segments,
bld_unstable_uint(hpp, szp, cache_size));
}
if (begin_new_max_period) {
segments.max.no = segments.current.no;
segments.max.sz = segments.current.sz;
}
return res;
}
static Eterm
info_version(int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp)
{
Eterm res = THE_NON_VALUE;
if (print_to_p) {
erts_print(*print_to_p, print_to_arg, "version: %s\n",
ERTS_MSEG_VSN_STR);
}
if (hpp || szp) {
res = bld_string(hpp, szp, ERTS_MSEG_VSN_STR);
}
return res;
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
* Exported functions *
\* */
Eterm
erts_mseg_info_options(int *print_to_p, void *print_to_arg,
Uint **hpp, Uint *szp)
{
Eterm res;
erts_mtx_lock(&mseg_mutex);
res = info_options("option ", print_to_p, print_to_arg, hpp, szp);
erts_mtx_unlock(&mseg_mutex);
return res;
}
Eterm
erts_mseg_info(int *print_to_p,
void *print_to_arg,
int begin_max_per,
Uint **hpp,
Uint *szp)
{
Eterm res = THE_NON_VALUE;
Eterm atoms[4];
Eterm values[4];
erts_mtx_lock(&mseg_mutex);
if (hpp || szp) {
if (!atoms_initialized)
init_atoms();
atoms[0] = am.version;
atoms[1] = am.options;
atoms[2] = am.status;
atoms[3] = am.calls;
}
values[0] = info_version(print_to_p, print_to_arg, hpp, szp);
values[1] = info_options("option ", print_to_p, print_to_arg, hpp, szp);
values[2] = info_status(print_to_p, print_to_arg, begin_max_per, hpp, szp);
values[3] = info_calls(print_to_p, print_to_arg, hpp, szp);
if (hpp || szp)
res = bld_2tup_list(hpp, szp, 4, atoms, values);
erts_mtx_unlock(&mseg_mutex);
return res;
}
void *
erts_mseg_alloc_opt(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt)
{
void *seg;
erts_mtx_lock(&mseg_mutex);
seg = mseg_alloc(atype, size_p, opt);
erts_mtx_unlock(&mseg_mutex);
return seg;
}
void *
erts_mseg_alloc(ErtsAlcType_t atype, Uint *size_p)
{
return erts_mseg_alloc_opt(atype, size_p, &default_opt);
}
void
erts_mseg_dealloc_opt(ErtsAlcType_t atype, void *seg, Uint size,
const ErtsMsegOpt_t *opt)
{
erts_mtx_lock(&mseg_mutex);
mseg_dealloc(atype, seg, size, opt);
erts_mtx_unlock(&mseg_mutex);
}
void
erts_mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size)
{
erts_mseg_dealloc_opt(atype, seg, size, &default_opt);
}
void *
erts_mseg_realloc_opt(ErtsAlcType_t atype, void *seg, Uint old_size,
Uint *new_size_p, const ErtsMsegOpt_t *opt)
{
void *new_seg;
erts_mtx_lock(&mseg_mutex);
new_seg = mseg_realloc(atype, seg, old_size, new_size_p, opt);
erts_mtx_unlock(&mseg_mutex);
return new_seg;
}
void *
erts_mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size,
Uint *new_size_p)
{
return erts_mseg_realloc_opt(atype, seg, old_size, new_size_p, &default_opt);
}
void
erts_mseg_clear_cache(void)
{
erts_mtx_lock(&mseg_mutex);
mseg_clear_cache();
erts_mtx_unlock(&mseg_mutex);
}
Uint
erts_mseg_no(void)
{
Uint n;
erts_mtx_lock(&mseg_mutex);
n = segments.current.no;
erts_mtx_unlock(&mseg_mutex);
return n;
}
Uint
erts_mseg_unit_size(void)
{
return page_size;
}
void
erts_mseg_init(ErtsMsegInit_t *init)
{
unsigned i;
atoms_initialized = 0;
is_init_done = 0;
/* Options ... */
abs_max_cache_bad_fit = init->amcbf;
rel_max_cache_bad_fit = init->rmcbf;
max_cache_size = init->mcs;
cache_check_interval = init->cci;
/* */
#ifdef USE_THREADS
thread_safe_init();
#endif
#if HAVE_MMAP && !defined(MAP_ANON)
mmap_fd = open("/dev/zero", O_RDWR);
if (mmap_fd < 0)
erl_exit(ERTS_ABORT_EXIT, "erts_mseg: unable to open /dev/zero\n");
#endif
page_size = GET_PAGE_SIZE;
page_shift = 1;
while ((page_size >> page_shift) != 1) {
if ((page_size & (1 << (page_shift - 1))) != 0)
erl_exit(ERTS_ABORT_EXIT,
"erts_mseg: Unexpected page_size %bpu\n", page_size);
page_shift++;
}
sys_memzero((void *) &calls, sizeof(calls));
#if CAN_PARTLY_DESTROY
min_seg_size = ~((Uint) 0);
#endif
cache = NULL;
cache_end = NULL;
cache_hits = 0;
max_cached_seg_size = 0;
min_cached_seg_size = ~((Uint) 0);
cache_size = 0;
is_cache_check_scheduled = 0;
#ifdef ERTS_THREADS_NO_SMP
is_cache_check_requested = 0;
#endif
if (max_cache_size > MAX_CACHE_SIZE)
max_cache_size = MAX_CACHE_SIZE;
if (max_cache_size > 0) {
for (i = 0; i < max_cache_size - 1; i++)
cache_descs[i].next = &cache_descs[i + 1];
cache_descs[max_cache_size - 1].next = NULL;
free_cache_descs = &cache_descs[0];
}
else
free_cache_descs = NULL;
segments.current.watermark = 0;
segments.current.no = 0;
segments.current.sz = 0;
segments.max.no = 0;
segments.max.sz = 0;
segments.max_ever.no = 0;
segments.max_ever.sz = 0;
}
/*
* erts_mseg_late_init() have to be called after all allocators,
* threads and timers have been initialized.
*/
void
erts_mseg_late_init(void)
{
#ifdef ERTS_THREADS_NO_SMP
int handle =
erts_register_async_ready_callback(
check_schedule_cache_check);
#endif
erts_mtx_lock(&mseg_mutex);
is_init_done = 1;
#ifdef ERTS_THREADS_NO_SMP
async_handle = handle;
#endif
if (cache_size)
schedule_cache_check();
erts_mtx_unlock(&mseg_mutex);
}
void
erts_mseg_exit(void)
{
mseg_shutdown();
}
#endif /* #if HAVE_ERTS_MSEG */
unsigned long
erts_mseg_test(unsigned long op,
unsigned long a1,
unsigned long a2,
unsigned long a3)
{
switch (op) {
#if HAVE_ERTS_MSEG
case 0x400: /* Have erts_mseg */
return (unsigned long) 1;
case 0x401:
return (unsigned long) erts_mseg_alloc(ERTS_ALC_A_INVALID, (Uint *) a1);
case 0x402:
erts_mseg_dealloc(ERTS_ALC_A_INVALID, (void *) a1, (Uint) a2);
return (unsigned long) 0;
case 0x403:
return (unsigned long) erts_mseg_realloc(ERTS_ALC_A_INVALID,
(void *) a1,
(Uint) a2,
(Uint *) a3);
case 0x404:
erts_mseg_clear_cache();
return (unsigned long) 0;
case 0x405:
return (unsigned long) erts_mseg_no();
case 0x406: {
unsigned long res;
erts_mtx_lock(&mseg_mutex);
res = (unsigned long) cache_size;
erts_mtx_unlock(&mseg_mutex);
return res;
}
#else /* #if HAVE_ERTS_MSEG */
case 0x400: /* Have erts_mseg */
return (unsigned long) 0;
#endif /* #if HAVE_ERTS_MSEG */
default: ASSERT(0); return ~((unsigned long) 0);
}
}