aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/sys/common/erl_mseg.c
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/sys/common/erl_mseg.c')
-rw-r--r--erts/emulator/sys/common/erl_mseg.c1452
1 files changed, 1452 insertions, 0 deletions
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
new file mode 100644
index 0000000000..f4e21bc05f
--- /dev/null
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -0,0 +1,1452 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2002-2009. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: A memory segment allocator. Segments that are deallocated
+ * are kept for a while in a segment "cache" before they are
+ * destroyed. When segments are allocated, cached segments
+ * are used if possible instead of creating new segments.
+ *
+ * Author: Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include "sys.h"
+#include "erl_mseg.h"
+#include "global.h"
+#include "erl_threads.h"
+#include "erl_mtrace.h"
+#include "big.h"
+
+#if HAVE_ERTS_MSEG
+
+#if defined(USE_THREADS) && !defined(ERTS_SMP)
+# define ERTS_THREADS_NO_SMP
+#endif
+
+#define SEGTYPE ERTS_MTRACE_SEGMENT_ID
+
+#ifndef HAVE_GETPAGESIZE
+#define HAVE_GETPAGESIZE 0
+#endif
+
+#ifdef _SC_PAGESIZE
+# define GET_PAGE_SIZE sysconf(_SC_PAGESIZE)
+#elif HAVE_GETPAGESIZE
+# define GET_PAGE_SIZE getpagesize()
+#else
+# error "Page size unknown"
+ /* Implement some other way to get the real page size if needed! */
+#endif
+
+#define MAX_CACHE_SIZE 30
+
+#undef MIN
+#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
+#undef MAX
+#define MAX(X, Y) ((X) > (Y) ? (X) : (Y))
+
+#undef PAGE_MASK
+#define INV_PAGE_MASK ((Uint) (page_size - 1))
+#define PAGE_MASK (~INV_PAGE_MASK)
+#define PAGE_FLOOR(X) ((X) & PAGE_MASK)
+#define PAGE_CEILING(X) PAGE_FLOOR((X) + INV_PAGE_MASK)
+#define PAGES(X) ((X) >> page_shift)
+
+static int atoms_initialized;
+
+static Uint cache_check_interval;
+
+static void check_cache(void *unused);
+static void mseg_clear_cache(void);
+static int is_cache_check_scheduled;
+#ifdef ERTS_THREADS_NO_SMP
+static int is_cache_check_requested;
+#endif
+
+#if HAVE_MMAP
+/* Mmap ... */
+
+#define MMAP_PROT (PROT_READ|PROT_WRITE)
+#ifdef MAP_ANON
+# define MMAP_FLAGS (MAP_ANON|MAP_PRIVATE)
+# define MMAP_FD (-1)
+#else
+# define MMAP_FLAGS (MAP_PRIVATE)
+# define MMAP_FD mmap_fd
+static int mmap_fd;
+#endif
+
+#if HAVE_MREMAP
+# define HAVE_MSEG_RECREATE 1
+#else
+# define HAVE_MSEG_RECREATE 0
+#endif
+
+#define CAN_PARTLY_DESTROY 1
+#else /* #if HAVE_MMAP */
+#define CAN_PARTLY_DESTROY 0
+#error "Not supported"
+#endif /* #if HAVE_MMAP */
+
+
+#if defined(ERTS_MSEG_FAKE_SEGMENTS)
+#undef CAN_PARTLY_DESTROY
+#define CAN_PARTLY_DESTROY 0
+#endif
+
+static const ErtsMsegOpt_t default_opt = ERTS_MSEG_DEFAULT_OPT_INITIALIZER;
+
+typedef struct cache_desc_t_ {
+ void *seg;
+ Uint size;
+ struct cache_desc_t_ *next;
+ struct cache_desc_t_ *prev;
+} cache_desc_t;
+
+typedef struct {
+ Uint32 giga_no;
+ Uint32 no;
+} CallCounter;
+
+static int is_init_done;
+static Uint page_size;
+static Uint page_shift;
+
+static struct {
+ CallCounter alloc;
+ CallCounter dealloc;
+ CallCounter realloc;
+ CallCounter create;
+ CallCounter destroy;
+#if HAVE_MSEG_RECREATE
+ CallCounter recreate;
+#endif
+ CallCounter clear_cache;
+ CallCounter check_cache;
+} calls;
+
+static cache_desc_t cache_descs[MAX_CACHE_SIZE];
+static cache_desc_t *free_cache_descs;
+static cache_desc_t *cache;
+static cache_desc_t *cache_end;
+static Uint cache_hits;
+static Uint cache_size;
+static Uint min_cached_seg_size;
+static Uint max_cached_seg_size;
+
+static Uint max_cache_size;
+static Uint abs_max_cache_bad_fit;
+static Uint rel_max_cache_bad_fit;
+
+#if CAN_PARTLY_DESTROY
+static Uint min_seg_size;
+#endif
+
+struct {
+ struct {
+ Uint watermark;
+ Uint no;
+ Uint sz;
+ } current;
+ struct {
+ Uint no;
+ Uint sz;
+ } max;
+ struct {
+ Uint no;
+ Uint sz;
+ } max_ever;
+} segments;
+
+#define ERTS_MSEG_ALLOC_STAT(SZ) \
+do { \
+ segments.current.no++; \
+ if (segments.max.no < segments.current.no) \
+ segments.max.no = segments.current.no; \
+ if (segments.current.watermark < segments.current.no) \
+ segments.current.watermark = segments.current.no; \
+ segments.current.sz += (SZ); \
+ if (segments.max.sz < segments.current.sz) \
+ segments.max.sz = segments.current.sz; \
+} while (0)
+
+#define ERTS_MSEG_DEALLOC_STAT(SZ) \
+do { \
+ ASSERT(segments.current.no > 0); \
+ segments.current.no--; \
+ ASSERT(segments.current.sz >= (SZ)); \
+ segments.current.sz -= (SZ); \
+} while (0)
+
+#define ERTS_MSEG_REALLOC_STAT(OSZ, NSZ) \
+do { \
+ ASSERT(segments.current.sz >= (OSZ)); \
+ segments.current.sz -= (OSZ); \
+ segments.current.sz += (NSZ); \
+} while (0)
+
+#define ONE_GIGA (1000000000)
+
+#define ZERO_CC(CC) (calls.CC.no = 0, calls.CC.giga_no = 0)
+
+#define INC_CC(CC) (calls.CC.no == ONE_GIGA - 1 \
+ ? (calls.CC.giga_no++, calls.CC.no = 0) \
+ : calls.CC.no++)
+
+#define DEC_CC(CC) (calls.CC.no == 0 \
+ ? (calls.CC.giga_no--, \
+ calls.CC.no = ONE_GIGA - 1) \
+ : calls.CC.no--)
+
+
+static erts_mtx_t mseg_mutex; /* Also needed when !USE_THREADS */
+static erts_mtx_t init_atoms_mutex; /* Also needed when !USE_THREADS */
+
+#ifdef USE_THREADS
+#ifdef ERTS_THREADS_NO_SMP
+static erts_tid_t main_tid;
+static int async_handle = -1;
+#endif
+
+static void thread_safe_init(void)
+{
+ erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms");
+ erts_mtx_init(&mseg_mutex, "mseg");
+#ifdef ERTS_THREADS_NO_SMP
+ main_tid = erts_thr_self();
+#endif
+}
+
+#endif
+
+static ErlTimer cache_check_timer;
+
+static ERTS_INLINE void
+schedule_cache_check(void)
+{
+ if (!is_cache_check_scheduled && is_init_done) {
+#ifdef ERTS_THREADS_NO_SMP
+ if (!erts_equal_tids(erts_thr_self(), main_tid)) {
+ if (!is_cache_check_requested) {
+ is_cache_check_requested = 1;
+ sys_async_ready(async_handle);
+ }
+ }
+ else
+#endif
+ {
+ cache_check_timer.active = 0;
+ erl_set_timer(&cache_check_timer,
+ check_cache,
+ NULL,
+ NULL,
+ cache_check_interval);
+ is_cache_check_scheduled = 1;
+#ifdef ERTS_THREADS_NO_SMP
+ is_cache_check_requested = 0;
+#endif
+ }
+ }
+}
+
+#ifdef ERTS_THREADS_NO_SMP
+
+static void
+check_schedule_cache_check(void)
+{
+ erts_mtx_lock(&mseg_mutex);
+ if (is_cache_check_requested
+ && !is_cache_check_scheduled) {
+ schedule_cache_check();
+ }
+ erts_mtx_unlock(&mseg_mutex);
+}
+
+#endif
+
+static void
+mseg_shutdown(void)
+{
+#ifdef ERTS_SMP
+ erts_mtx_lock(&mseg_mutex);
+#endif
+ mseg_clear_cache();
+#ifdef ERTS_SMP
+ erts_mtx_unlock(&mseg_mutex);
+#endif
+}
+
+static ERTS_INLINE void *
+mseg_create(Uint size)
+{
+ void *seg;
+
+ ASSERT(size % page_size == 0);
+
+#if defined(ERTS_MSEG_FAKE_SEGMENTS)
+ seg = erts_sys_alloc(ERTS_ALC_N_INVALID, NULL, size);
+#elif HAVE_MMAP
+ seg = (void *) mmap((void *) 0, (size_t) size,
+ MMAP_PROT, MMAP_FLAGS, MMAP_FD, 0);
+ if (seg == (void *) MAP_FAILED)
+ seg = NULL;
+#else
+#error "Missing mseg_create() implementation"
+#endif
+
+ INC_CC(create);
+
+ return seg;
+}
+
+static ERTS_INLINE void
+mseg_destroy(void *seg, Uint size)
+{
+#if defined(ERTS_MSEG_FAKE_SEGMENTS)
+ erts_sys_free(ERTS_ALC_N_INVALID, NULL, seg);
+#elif HAVE_MMAP
+
+#ifdef DEBUG
+ int res =
+#endif
+
+ munmap((void *) seg, size);
+
+ ASSERT(size % page_size == 0);
+ ASSERT(res == 0);
+#else
+#error "Missing mseg_destroy() implementation"
+#endif
+
+ INC_CC(destroy);
+
+}
+
+#if HAVE_MSEG_RECREATE
+
+static ERTS_INLINE void *
+mseg_recreate(void *old_seg, Uint old_size, Uint new_size)
+{
+ void *new_seg;
+
+ ASSERT(old_size % page_size == 0);
+ ASSERT(new_size % page_size == 0);
+
+#if defined(ERTS_MSEG_FAKE_SEGMENTS)
+ new_seg = erts_sys_realloc(ERTS_ALC_N_INVALID, NULL, old_seg, new_size);
+#elif HAVE_MREMAP
+ new_seg = (void *) mremap((void *) old_seg,
+ (size_t) old_size,
+ (size_t) new_size,
+ MREMAP_MAYMOVE);
+ if (new_seg == (void *) MAP_FAILED)
+ new_seg = NULL;
+#else
+#error "Missing mseg_recreate() implementation"
+#endif
+
+ INC_CC(recreate);
+
+ return new_seg;
+}
+
+#endif /* #if HAVE_MSEG_RECREATE */
+
+
+static ERTS_INLINE cache_desc_t *
+alloc_cd(void)
+{
+ cache_desc_t *cd = free_cache_descs;
+ if (cd)
+ free_cache_descs = cd->next;
+ return cd;
+}
+
+static ERTS_INLINE void
+free_cd(cache_desc_t *cd)
+{
+ cd->next = free_cache_descs;
+ free_cache_descs = cd;
+}
+
+
+static ERTS_INLINE void
+link_cd(cache_desc_t *cd)
+{
+ if (cache)
+ cache->prev = cd;
+ cd->next = cache;
+ cd->prev = NULL;
+ cache = cd;
+
+ if (!cache_end) {
+ ASSERT(!cd->next);
+ cache_end = cd;
+ }
+
+ cache_size++;
+}
+
+static ERTS_INLINE void
+end_link_cd(cache_desc_t *cd)
+{
+ if (cache_end)
+ cache_end->next = cd;
+ cd->next = NULL;
+ cd->prev = cache_end;
+ cache_end = cd;
+
+ if (!cache) {
+ ASSERT(!cd->prev);
+ cache = cd;
+ }
+
+ cache_size++;
+}
+
+static ERTS_INLINE void
+unlink_cd(cache_desc_t *cd)
+{
+
+ if (cd->next)
+ cd->next->prev = cd->prev;
+ else
+ cache_end = cd->prev;
+
+ if (cd->prev)
+ cd->prev->next = cd->next;
+ else
+ cache = cd->next;
+ ASSERT(cache_size > 0);
+ cache_size--;
+}
+
+static ERTS_INLINE void
+check_cache_limits(void)
+{
+ cache_desc_t *cd;
+ max_cached_seg_size = 0;
+ min_cached_seg_size = ~((Uint) 0);
+ for (cd = cache; cd; cd = cd->next) {
+ if (cd->size < min_cached_seg_size)
+ min_cached_seg_size = cd->size;
+ if (cd->size > max_cached_seg_size)
+ max_cached_seg_size = cd->size;
+ }
+
+}
+
+static ERTS_INLINE void
+adjust_cache_size(int force_check_limits)
+{
+ cache_desc_t *cd;
+ int check_limits = force_check_limits;
+ Sint max_cached = ((Sint) segments.current.watermark
+ - (Sint) segments.current.no);
+
+ while (((Sint) cache_size) > max_cached && ((Sint) cache_size) > 0) {
+ ASSERT(cache_end);
+ cd = cache_end;
+ if (!check_limits &&
+ !(min_cached_seg_size < cd->size
+ && cd->size < max_cached_seg_size)) {
+ check_limits = 1;
+ }
+ if (erts_mtrace_enabled)
+ erts_mtrace_crr_free(SEGTYPE, SEGTYPE, cd->seg);
+ mseg_destroy(cd->seg, cd->size);
+ unlink_cd(cd);
+ free_cd(cd);
+ }
+
+ if (check_limits)
+ check_cache_limits();
+
+}
+
+static void
+check_cache(void *unused)
+{
+#ifdef ERTS_SMP
+ erts_mtx_lock(&mseg_mutex);
+#endif
+
+ is_cache_check_scheduled = 0;
+
+ if (segments.current.watermark > segments.current.no)
+ segments.current.watermark--;
+ adjust_cache_size(0);
+
+ if (cache_size)
+ schedule_cache_check();
+
+ INC_CC(check_cache);
+
+#ifdef ERTS_SMP
+ erts_mtx_unlock(&mseg_mutex);
+#endif
+
+}
+
+static void
+mseg_clear_cache(void)
+{
+ segments.current.watermark = 0;
+
+ adjust_cache_size(1);
+
+ ASSERT(!cache);
+ ASSERT(!cache_end);
+ ASSERT(!cache_size);
+
+ segments.current.watermark = segments.current.no;
+
+ INC_CC(clear_cache);
+}
+
+static void *
+mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt)
+{
+
+ Uint max, min, diff_size, size;
+ cache_desc_t *cd, *cand_cd;
+ void *seg;
+
+ INC_CC(alloc);
+
+ size = PAGE_CEILING(*size_p);
+
+#if CAN_PARTLY_DESTROY
+ if (size < min_seg_size)
+ min_seg_size = size;
+#endif
+
+ if (!opt->cache) {
+ create_seg:
+ adjust_cache_size(0);
+ seg = mseg_create(size);
+ if (!seg) {
+ mseg_clear_cache();
+ seg = mseg_create(size);
+ if (!seg)
+ size = 0;
+ }
+
+ *size_p = size;
+ if (seg) {
+ if (erts_mtrace_enabled)
+ erts_mtrace_crr_alloc(seg, atype, ERTS_MTRACE_SEGMENT_ID, size);
+ ERTS_MSEG_ALLOC_STAT(size);
+ }
+ return seg;
+ }
+
+ if (size > max_cached_seg_size)
+ goto create_seg;
+
+ if (size < min_cached_seg_size) {
+
+ diff_size = min_cached_seg_size - size;
+
+ if (diff_size > abs_max_cache_bad_fit)
+ goto create_seg;
+
+ if (100*PAGES(diff_size) > rel_max_cache_bad_fit*PAGES(size))
+ goto create_seg;
+
+ }
+
+ max = 0;
+ min = ~((Uint) 0);
+ cand_cd = NULL;
+
+ for (cd = cache; cd; cd = cd->next) {
+ if (cd->size >= size) {
+ if (!cand_cd) {
+ cand_cd = cd;
+ continue;
+ }
+ else if (cd->size < cand_cd->size) {
+ if (max < cand_cd->size)
+ max = cand_cd->size;
+ if (min > cand_cd->size)
+ min = cand_cd->size;
+ cand_cd = cd;
+ continue;
+ }
+ }
+ if (max < cd->size)
+ max = cd->size;
+ if (min > cd->size)
+ min = cd->size;
+ }
+
+ min_cached_seg_size = min;
+ max_cached_seg_size = max;
+
+ if (!cand_cd)
+ goto create_seg;
+
+ diff_size = cand_cd->size - size;
+
+ if (diff_size > abs_max_cache_bad_fit
+ || 100*PAGES(diff_size) > rel_max_cache_bad_fit*PAGES(size)) {
+ if (max_cached_seg_size < cand_cd->size)
+ max_cached_seg_size = cand_cd->size;
+ if (min_cached_seg_size > cand_cd->size)
+ min_cached_seg_size = cand_cd->size;
+ goto create_seg;
+ }
+
+ cache_hits++;
+
+ size = cand_cd->size;
+ seg = cand_cd->seg;
+
+ unlink_cd(cand_cd);
+ free_cd(cand_cd);
+
+ *size_p = size;
+
+ if (erts_mtrace_enabled) {
+ erts_mtrace_crr_free(SEGTYPE, SEGTYPE, seg);
+ erts_mtrace_crr_alloc(seg, atype, SEGTYPE, size);
+ }
+
+ if (seg)
+ ERTS_MSEG_ALLOC_STAT(size);
+ return seg;
+}
+
+
+static void
+mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size,
+ const ErtsMsegOpt_t *opt)
+{
+ cache_desc_t *cd;
+
+ ERTS_MSEG_DEALLOC_STAT(size);
+
+ if (!opt->cache || max_cache_size == 0) {
+ if (erts_mtrace_enabled)
+ erts_mtrace_crr_free(atype, SEGTYPE, seg);
+ mseg_destroy(seg, size);
+ }
+ else {
+ int check_limits = 0;
+
+ if (size < min_cached_seg_size)
+ min_cached_seg_size = size;
+ if (size > max_cached_seg_size)
+ max_cached_seg_size = size;
+
+ if (!free_cache_descs) {
+ cd = cache_end;
+ if (!(min_cached_seg_size < cd->size
+ && cd->size < max_cached_seg_size)) {
+ check_limits = 1;
+ }
+ if (erts_mtrace_enabled)
+ erts_mtrace_crr_free(SEGTYPE, SEGTYPE, cd->seg);
+ mseg_destroy(cd->seg, cd->size);
+ unlink_cd(cd);
+ free_cd(cd);
+ }
+
+ cd = alloc_cd();
+ ASSERT(cd);
+ cd->seg = seg;
+ cd->size = size;
+ link_cd(cd);
+
+ if (erts_mtrace_enabled) {
+ erts_mtrace_crr_free(atype, SEGTYPE, seg);
+ erts_mtrace_crr_alloc(seg, SEGTYPE, SEGTYPE, size);
+ }
+
+ /* ASSERT(segments.current.watermark >= segments.current.no + cache_size); */
+
+ if (check_limits)
+ check_cache_limits();
+
+ schedule_cache_check();
+
+ }
+
+ INC_CC(dealloc);
+}
+
+static void *
+mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p,
+ const ErtsMsegOpt_t *opt)
+{
+ void *new_seg;
+ Uint new_size;
+
+ if (!seg || !old_size) {
+ new_seg = mseg_alloc(atype, new_size_p, opt);
+ DEC_CC(alloc);
+ return new_seg;
+ }
+
+ if (!(*new_size_p)) {
+ mseg_dealloc(atype, seg, old_size, opt);
+ DEC_CC(dealloc);
+ return NULL;
+ }
+
+ new_seg = seg;
+ new_size = PAGE_CEILING(*new_size_p);
+
+ if (new_size == old_size)
+ ;
+ else if (new_size < old_size) {
+ Uint shrink_sz = old_size - new_size;
+
+#if CAN_PARTLY_DESTROY
+ if (new_size < min_seg_size)
+ min_seg_size = new_size;
+#endif
+
+ if (shrink_sz < opt->abs_shrink_th
+ && 100*PAGES(shrink_sz) < opt->rel_shrink_th*PAGES(old_size)) {
+ new_size = old_size;
+ }
+ else {
+
+#if CAN_PARTLY_DESTROY
+
+ if (shrink_sz > min_seg_size
+ && free_cache_descs
+ && opt->cache) {
+ cache_desc_t *cd;
+
+ cd = alloc_cd();
+ ASSERT(cd);
+ cd->seg = ((char *) seg) + new_size;
+ cd->size = shrink_sz;
+ end_link_cd(cd);
+
+ if (erts_mtrace_enabled) {
+ erts_mtrace_crr_realloc(new_seg,
+ atype,
+ SEGTYPE,
+ seg,
+ new_size);
+ erts_mtrace_crr_alloc(cd->seg, SEGTYPE, SEGTYPE, cd->size);
+ }
+ schedule_cache_check();
+ }
+ else {
+ if (erts_mtrace_enabled)
+ erts_mtrace_crr_realloc(new_seg,
+ atype,
+ SEGTYPE,
+ seg,
+ new_size);
+ mseg_destroy(((char *) seg) + new_size, shrink_sz);
+ }
+
+#elif HAVE_MSEG_RECREATE
+
+ goto do_recreate;
+
+#else
+
+ new_seg = mseg_alloc(atype, &new_size, opt);
+ if (!new_seg)
+ new_size = old_size;
+ else {
+ sys_memcpy(((char *) new_seg),
+ ((char *) seg),
+ MIN(new_size, old_size));
+ mseg_dealloc(atype, seg, old_size, opt);
+ }
+
+#endif
+
+ }
+ }
+ else {
+
+ if (!opt->preserv) {
+ mseg_dealloc(atype, seg, old_size, opt);
+ new_seg = mseg_alloc(atype, &new_size, opt);
+ }
+ else {
+#if HAVE_MSEG_RECREATE
+#if !CAN_PARTLY_DESTROY
+ do_recreate:
+#endif
+ new_seg = mseg_recreate((void *) seg, old_size, new_size);
+ if (erts_mtrace_enabled)
+ erts_mtrace_crr_realloc(new_seg, atype, SEGTYPE, seg, new_size);
+ if (!new_seg)
+ new_size = old_size;
+#else
+ new_seg = mseg_alloc(atype, &new_size, opt);
+ if (!new_seg)
+ new_size = old_size;
+ else {
+ sys_memcpy(((char *) new_seg),
+ ((char *) seg),
+ MIN(new_size, old_size));
+ mseg_dealloc(atype, seg, old_size, opt);
+ }
+#endif
+ }
+ }
+
+ INC_CC(realloc);
+
+ *new_size_p = new_size;
+
+ ERTS_MSEG_REALLOC_STAT(old_size, new_size);
+
+ return new_seg;
+}
+
+/* --- Info stuff ---------------------------------------------------------- */
+
+static struct {
+ Eterm version;
+
+ Eterm options;
+ Eterm amcbf;
+ Eterm rmcbf;
+ Eterm mcs;
+ Eterm cci;
+
+ Eterm status;
+ Eterm cached_segments;
+ Eterm cache_hits;
+ Eterm segments;
+ Eterm segments_size;
+ Eterm segments_watermark;
+
+
+ Eterm calls;
+ Eterm mseg_alloc;
+ Eterm mseg_dealloc;
+ Eterm mseg_realloc;
+ Eterm mseg_create;
+ Eterm mseg_destroy;
+#if HAVE_MSEG_RECREATE
+ Eterm mseg_recreate;
+#endif
+ Eterm mseg_clear_cache;
+ Eterm mseg_check_cache;
+
+#ifdef DEBUG
+ Eterm end_of_atoms;
+#endif
+} am;
+
+static void ERTS_INLINE atom_init(Eterm *atom, char *name)
+{
+ *atom = am_atom_put(name, strlen(name));
+}
+#define AM_INIT(AM) atom_init(&am.AM, #AM)
+
+static void
+init_atoms(void)
+{
+#ifdef DEBUG
+ Eterm *atom;
+#endif
+
+ erts_mtx_unlock(&mseg_mutex);
+ erts_mtx_lock(&init_atoms_mutex);
+
+ if (!atoms_initialized) {
+#ifdef DEBUG
+ for (atom = (Eterm *) &am; atom <= &am.end_of_atoms; atom++) {
+ *atom = THE_NON_VALUE;
+ }
+#endif
+
+ AM_INIT(version);
+
+ AM_INIT(options);
+ AM_INIT(amcbf);
+ AM_INIT(rmcbf);
+ AM_INIT(mcs);
+ AM_INIT(cci);
+
+ AM_INIT(status);
+ AM_INIT(cached_segments);
+ AM_INIT(cache_hits);
+ AM_INIT(segments);
+ AM_INIT(segments_size);
+ AM_INIT(segments_watermark);
+
+ AM_INIT(calls);
+ AM_INIT(mseg_alloc);
+ AM_INIT(mseg_dealloc);
+ AM_INIT(mseg_realloc);
+ AM_INIT(mseg_create);
+ AM_INIT(mseg_destroy);
+#if HAVE_MSEG_RECREATE
+ AM_INIT(mseg_recreate);
+#endif
+ AM_INIT(mseg_clear_cache);
+ AM_INIT(mseg_check_cache);
+
+#ifdef DEBUG
+ for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) {
+ ASSERT(*atom != THE_NON_VALUE);
+ }
+#endif
+ }
+
+ erts_mtx_lock(&mseg_mutex);
+ atoms_initialized = 1;
+ erts_mtx_unlock(&init_atoms_mutex);
+}
+
+
+#define bld_uint erts_bld_uint
+#define bld_cons erts_bld_cons
+#define bld_tuple erts_bld_tuple
+#define bld_string erts_bld_string
+#define bld_2tup_list erts_bld_2tup_list
+
+
+/*
+ * bld_unstable_uint() (instead of bld_uint()) is used when values may
+ * change between size check and actual build. This because a value
+ * that would fit a small when size check is done may need to be built
+ * as a big when the actual build is performed. Caller is required to
+ * HRelease after build.
+ */
+static ERTS_INLINE Eterm
+bld_unstable_uint(Uint **hpp, Uint *szp, Uint ui)
+{
+ Eterm res = THE_NON_VALUE;
+ if (szp)
+ *szp += BIG_UINT_HEAP_SIZE;
+ if (hpp) {
+ if (IS_USMALL(0, ui))
+ res = make_small(ui);
+ else {
+ res = uint_to_big(ui, *hpp);
+ *hpp += BIG_UINT_HEAP_SIZE;
+ }
+ }
+ return res;
+}
+
+static ERTS_INLINE void
+add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2)
+{
+ *lp = bld_cons(hpp, szp, bld_tuple(hpp, szp, 2, el1, el2), *lp);
+}
+
+static ERTS_INLINE void
+add_3tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2, Eterm el3)
+{
+ *lp = bld_cons(hpp, szp, bld_tuple(hpp, szp, 3, el1, el2, el3), *lp);
+}
+
+static ERTS_INLINE void
+add_4tup(Uint **hpp, Uint *szp, Eterm *lp,
+ Eterm el1, Eterm el2, Eterm el3, Eterm el4)
+{
+ *lp = bld_cons(hpp, szp, bld_tuple(hpp, szp, 4, el1, el2, el3, el4), *lp);
+}
+
+static Eterm
+info_options(char *prefix,
+ int *print_to_p,
+ void *print_to_arg,
+ Uint **hpp,
+ Uint *szp)
+{
+ Eterm res = THE_NON_VALUE;
+
+ if (print_to_p) {
+ int to = *print_to_p;
+ void *arg = print_to_arg;
+ erts_print(to, arg, "%samcbf: %bpu\n", prefix, abs_max_cache_bad_fit);
+ erts_print(to, arg, "%srmcbf: %bpu\n", prefix, rel_max_cache_bad_fit);
+ erts_print(to, arg, "%smcs: %bpu\n", prefix, max_cache_size);
+ erts_print(to, arg, "%scci: %bpu\n", prefix, cache_check_interval);
+ }
+
+ if (hpp || szp) {
+
+ if (!atoms_initialized)
+ init_atoms();
+
+ res = NIL;
+ add_2tup(hpp, szp, &res,
+ am.cci,
+ bld_uint(hpp, szp, cache_check_interval));
+ add_2tup(hpp, szp, &res,
+ am.mcs,
+ bld_uint(hpp, szp, max_cache_size));
+ add_2tup(hpp, szp, &res,
+ am.rmcbf,
+ bld_uint(hpp, szp, rel_max_cache_bad_fit));
+ add_2tup(hpp, szp, &res,
+ am.amcbf,
+ bld_uint(hpp, szp, abs_max_cache_bad_fit));
+
+ }
+
+ return res;
+}
+
+static Eterm
+info_calls(int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp)
+{
+ Eterm res = THE_NON_VALUE;
+
+ if (print_to_p) {
+
+#define PRINT_CC(TO, TOA, CC) \
+ if (calls.CC.giga_no == 0) \
+ erts_print(TO, TOA, "mseg_%s calls: %bpu\n", #CC, calls.CC.no); \
+ else \
+ erts_print(TO, TOA, "mseg_%s calls: %bpu%09bpu\n", #CC, \
+ calls.CC.giga_no, calls.CC.no)
+
+ int to = *print_to_p;
+ void *arg = print_to_arg;
+
+ PRINT_CC(to, arg, alloc);
+ PRINT_CC(to, arg, dealloc);
+ PRINT_CC(to, arg, realloc);
+ PRINT_CC(to, arg, create);
+ PRINT_CC(to, arg, destroy);
+#if HAVE_MSEG_RECREATE
+ PRINT_CC(to, arg, recreate);
+#endif
+ PRINT_CC(to, arg, clear_cache);
+ PRINT_CC(to, arg, check_cache);
+
+#undef PRINT_CC
+
+ }
+
+ if (hpp || szp) {
+
+ res = NIL;
+
+ add_3tup(hpp, szp, &res,
+ am.mseg_check_cache,
+ bld_unstable_uint(hpp, szp, calls.check_cache.giga_no),
+ bld_unstable_uint(hpp, szp, calls.check_cache.no));
+ add_3tup(hpp, szp, &res,
+ am.mseg_clear_cache,
+ bld_unstable_uint(hpp, szp, calls.clear_cache.giga_no),
+ bld_unstable_uint(hpp, szp, calls.clear_cache.no));
+
+#if HAVE_MSEG_RECREATE
+ add_3tup(hpp, szp, &res,
+ am.mseg_recreate,
+ bld_unstable_uint(hpp, szp, calls.recreate.giga_no),
+ bld_unstable_uint(hpp, szp, calls.recreate.no));
+#endif
+ add_3tup(hpp, szp, &res,
+ am.mseg_destroy,
+ bld_unstable_uint(hpp, szp, calls.destroy.giga_no),
+ bld_unstable_uint(hpp, szp, calls.destroy.no));
+ add_3tup(hpp, szp, &res,
+ am.mseg_create,
+ bld_unstable_uint(hpp, szp, calls.create.giga_no),
+ bld_unstable_uint(hpp, szp, calls.create.no));
+
+
+ add_3tup(hpp, szp, &res,
+ am.mseg_realloc,
+ bld_unstable_uint(hpp, szp, calls.realloc.giga_no),
+ bld_unstable_uint(hpp, szp, calls.realloc.no));
+ add_3tup(hpp, szp, &res,
+ am.mseg_dealloc,
+ bld_unstable_uint(hpp, szp, calls.dealloc.giga_no),
+ bld_unstable_uint(hpp, szp, calls.dealloc.no));
+ add_3tup(hpp, szp, &res,
+ am.mseg_alloc,
+ bld_unstable_uint(hpp, szp, calls.alloc.giga_no),
+ bld_unstable_uint(hpp, szp, calls.alloc.no));
+ }
+
+ return res;
+}
+
+static Eterm
+info_status(int *print_to_p,
+ void *print_to_arg,
+ int begin_new_max_period,
+ Uint **hpp,
+ Uint *szp)
+{
+ Eterm res = THE_NON_VALUE;
+
+ if (segments.max_ever.no < segments.max.no)
+ segments.max_ever.no = segments.max.no;
+ if (segments.max_ever.sz < segments.max.sz)
+ segments.max_ever.sz = segments.max.sz;
+
+ if (print_to_p) {
+ int to = *print_to_p;
+ void *arg = print_to_arg;
+
+ erts_print(to, arg, "cached_segments: %bpu\n", cache_size);
+ erts_print(to, arg, "cache_hits: %bpu\n", cache_hits);
+ erts_print(to, arg, "segments: %bpu %bpu %bpu\n",
+ segments.current.no, segments.max.no, segments.max_ever.no);
+ erts_print(to, arg, "segments_size: %bpu %bpu %bpu\n",
+ segments.current.sz, segments.max.sz, segments.max_ever.sz);
+ erts_print(to, arg, "segments_watermark: %bpu\n",
+ segments.current.watermark);
+ }
+
+ if (hpp || szp) {
+ res = NIL;
+ add_2tup(hpp, szp, &res,
+ am.segments_watermark,
+ bld_unstable_uint(hpp, szp, segments.current.watermark));
+ add_4tup(hpp, szp, &res,
+ am.segments_size,
+ bld_unstable_uint(hpp, szp, segments.current.sz),
+ bld_unstable_uint(hpp, szp, segments.max.sz),
+ bld_unstable_uint(hpp, szp, segments.max_ever.sz));
+ add_4tup(hpp, szp, &res,
+ am.segments,
+ bld_unstable_uint(hpp, szp, segments.current.no),
+ bld_unstable_uint(hpp, szp, segments.max.no),
+ bld_unstable_uint(hpp, szp, segments.max_ever.no));
+ add_2tup(hpp, szp, &res,
+ am.cache_hits,
+ bld_unstable_uint(hpp, szp, cache_hits));
+ add_2tup(hpp, szp, &res,
+ am.cached_segments,
+ bld_unstable_uint(hpp, szp, cache_size));
+
+ }
+
+ if (begin_new_max_period) {
+ segments.max.no = segments.current.no;
+ segments.max.sz = segments.current.sz;
+ }
+
+ return res;
+}
+
+static Eterm
+info_version(int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp)
+{
+ Eterm res = THE_NON_VALUE;
+
+ if (print_to_p) {
+ erts_print(*print_to_p, print_to_arg, "version: %s\n",
+ ERTS_MSEG_VSN_STR);
+ }
+
+ if (hpp || szp) {
+ res = bld_string(hpp, szp, ERTS_MSEG_VSN_STR);
+ }
+
+ return res;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Exported functions *
+\* */
+
+Eterm
+erts_mseg_info_options(int *print_to_p, void *print_to_arg,
+ Uint **hpp, Uint *szp)
+{
+ Eterm res;
+
+ erts_mtx_lock(&mseg_mutex);
+
+ res = info_options("option ", print_to_p, print_to_arg, hpp, szp);
+
+ erts_mtx_unlock(&mseg_mutex);
+
+ return res;
+}
+
+Eterm
+erts_mseg_info(int *print_to_p,
+ void *print_to_arg,
+ int begin_max_per,
+ Uint **hpp,
+ Uint *szp)
+{
+ Eterm res = THE_NON_VALUE;
+ Eterm atoms[4];
+ Eterm values[4];
+
+ erts_mtx_lock(&mseg_mutex);
+
+ if (hpp || szp) {
+
+ if (!atoms_initialized)
+ init_atoms();
+
+ atoms[0] = am.version;
+ atoms[1] = am.options;
+ atoms[2] = am.status;
+ atoms[3] = am.calls;
+ }
+
+ values[0] = info_version(print_to_p, print_to_arg, hpp, szp);
+ values[1] = info_options("option ", print_to_p, print_to_arg, hpp, szp);
+ values[2] = info_status(print_to_p, print_to_arg, begin_max_per, hpp, szp);
+ values[3] = info_calls(print_to_p, print_to_arg, hpp, szp);
+
+ if (hpp || szp)
+ res = bld_2tup_list(hpp, szp, 4, atoms, values);
+
+ erts_mtx_unlock(&mseg_mutex);
+
+ return res;
+}
+
+void *
+erts_mseg_alloc_opt(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt)
+{
+ void *seg;
+ erts_mtx_lock(&mseg_mutex);
+ seg = mseg_alloc(atype, size_p, opt);
+ erts_mtx_unlock(&mseg_mutex);
+ return seg;
+}
+
+void *
+erts_mseg_alloc(ErtsAlcType_t atype, Uint *size_p)
+{
+ return erts_mseg_alloc_opt(atype, size_p, &default_opt);
+}
+
+void
+erts_mseg_dealloc_opt(ErtsAlcType_t atype, void *seg, Uint size,
+ const ErtsMsegOpt_t *opt)
+{
+ erts_mtx_lock(&mseg_mutex);
+ mseg_dealloc(atype, seg, size, opt);
+ erts_mtx_unlock(&mseg_mutex);
+}
+
+void
+erts_mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size)
+{
+ erts_mseg_dealloc_opt(atype, seg, size, &default_opt);
+}
+
+void *
+erts_mseg_realloc_opt(ErtsAlcType_t atype, void *seg, Uint old_size,
+ Uint *new_size_p, const ErtsMsegOpt_t *opt)
+{
+ void *new_seg;
+ erts_mtx_lock(&mseg_mutex);
+ new_seg = mseg_realloc(atype, seg, old_size, new_size_p, opt);
+ erts_mtx_unlock(&mseg_mutex);
+ return new_seg;
+}
+
+void *
+erts_mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size,
+ Uint *new_size_p)
+{
+ return erts_mseg_realloc_opt(atype, seg, old_size, new_size_p, &default_opt);
+}
+
+void
+erts_mseg_clear_cache(void)
+{
+ erts_mtx_lock(&mseg_mutex);
+ mseg_clear_cache();
+ erts_mtx_unlock(&mseg_mutex);
+}
+
+Uint
+erts_mseg_no(void)
+{
+ Uint n;
+ erts_mtx_lock(&mseg_mutex);
+ n = segments.current.no;
+ erts_mtx_unlock(&mseg_mutex);
+ return n;
+}
+
+Uint
+erts_mseg_unit_size(void)
+{
+ return page_size;
+}
+
+void
+erts_mseg_init(ErtsMsegInit_t *init)
+{
+ unsigned i;
+
+ atoms_initialized = 0;
+ is_init_done = 0;
+
+ /* Options ... */
+
+ abs_max_cache_bad_fit = init->amcbf;
+ rel_max_cache_bad_fit = init->rmcbf;
+ max_cache_size = init->mcs;
+ cache_check_interval = init->cci;
+
+ /* */
+
+#ifdef USE_THREADS
+ thread_safe_init();
+#endif
+
+#if HAVE_MMAP && !defined(MAP_ANON)
+ mmap_fd = open("/dev/zero", O_RDWR);
+ if (mmap_fd < 0)
+ erl_exit(ERTS_ABORT_EXIT, "erts_mseg: unable to open /dev/zero\n");
+#endif
+
+ page_size = GET_PAGE_SIZE;
+
+ page_shift = 1;
+ while ((page_size >> page_shift) != 1) {
+ if ((page_size & (1 << (page_shift - 1))) != 0)
+ erl_exit(ERTS_ABORT_EXIT,
+ "erts_mseg: Unexpected page_size %bpu\n", page_size);
+ page_shift++;
+ }
+
+ sys_memzero((void *) &calls, sizeof(calls));
+
+#if CAN_PARTLY_DESTROY
+ min_seg_size = ~((Uint) 0);
+#endif
+
+ cache = NULL;
+ cache_end = NULL;
+ cache_hits = 0;
+ max_cached_seg_size = 0;
+ min_cached_seg_size = ~((Uint) 0);
+ cache_size = 0;
+
+ is_cache_check_scheduled = 0;
+#ifdef ERTS_THREADS_NO_SMP
+ is_cache_check_requested = 0;
+#endif
+
+ if (max_cache_size > MAX_CACHE_SIZE)
+ max_cache_size = MAX_CACHE_SIZE;
+
+ if (max_cache_size > 0) {
+ for (i = 0; i < max_cache_size - 1; i++)
+ cache_descs[i].next = &cache_descs[i + 1];
+ cache_descs[max_cache_size - 1].next = NULL;
+ free_cache_descs = &cache_descs[0];
+ }
+ else
+ free_cache_descs = NULL;
+
+ segments.current.watermark = 0;
+ segments.current.no = 0;
+ segments.current.sz = 0;
+ segments.max.no = 0;
+ segments.max.sz = 0;
+ segments.max_ever.no = 0;
+ segments.max_ever.sz = 0;
+}
+
+
+/*
+ * erts_mseg_late_init() have to be called after all allocators,
+ * threads and timers have been initialized.
+ */
+void
+erts_mseg_late_init(void)
+{
+#ifdef ERTS_THREADS_NO_SMP
+ int handle =
+ erts_register_async_ready_callback(
+ check_schedule_cache_check);
+#endif
+ erts_mtx_lock(&mseg_mutex);
+ is_init_done = 1;
+#ifdef ERTS_THREADS_NO_SMP
+ async_handle = handle;
+#endif
+ if (cache_size)
+ schedule_cache_check();
+ erts_mtx_unlock(&mseg_mutex);
+}
+
+void
+erts_mseg_exit(void)
+{
+ mseg_shutdown();
+}
+
+#endif /* #if HAVE_ERTS_MSEG */
+
+unsigned long
+erts_mseg_test(unsigned long op,
+ unsigned long a1,
+ unsigned long a2,
+ unsigned long a3)
+{
+ switch (op) {
+#if HAVE_ERTS_MSEG
+ case 0x400: /* Have erts_mseg */
+ return (unsigned long) 1;
+ case 0x401:
+ return (unsigned long) erts_mseg_alloc(ERTS_ALC_A_INVALID, (Uint *) a1);
+ case 0x402:
+ erts_mseg_dealloc(ERTS_ALC_A_INVALID, (void *) a1, (Uint) a2);
+ return (unsigned long) 0;
+ case 0x403:
+ return (unsigned long) erts_mseg_realloc(ERTS_ALC_A_INVALID,
+ (void *) a1,
+ (Uint) a2,
+ (Uint *) a3);
+ case 0x404:
+ erts_mseg_clear_cache();
+ return (unsigned long) 0;
+ case 0x405:
+ return (unsigned long) erts_mseg_no();
+ case 0x406: {
+ unsigned long res;
+ erts_mtx_lock(&mseg_mutex);
+ res = (unsigned long) cache_size;
+ erts_mtx_unlock(&mseg_mutex);
+ return res;
+ }
+#else /* #if HAVE_ERTS_MSEG */
+ case 0x400: /* Have erts_mseg */
+ return (unsigned long) 0;
+#endif /* #if HAVE_ERTS_MSEG */
+ default: ASSERT(0); return ~((unsigned long) 0);
+ }
+
+}
+
+