aboutsummaryrefslogtreecommitdiffstats
path: root/erts
diff options
context:
space:
mode:
Diffstat (limited to 'erts')
-rw-r--r--erts/emulator/sys/common/erl_mmap.c389
-rw-r--r--erts/emulator/sys/common/erl_mseg.c22
2 files changed, 349 insertions, 62 deletions
diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c
index 73874759f5..1f9168df28 100644
--- a/erts/emulator/sys/common/erl_mmap.c
+++ b/erts/emulator/sys/common/erl_mmap.c
@@ -27,9 +27,18 @@
#include "erl_mmap.h"
#include <stddef.h>
+/* #define ERTS_MMAP_OP_RINGBUF_SZ 100 */
+
#if defined(DEBUG) || 0
# undef ERTS_MMAP_DEBUG
# define ERTS_MMAP_DEBUG
+# ifndef ERTS_MMAP_OP_RINGBUF_SZ
+# define ERTS_MMAP_OP_RINGBUF_SZ 100
+# endif
+#endif
+
+#ifndef ERTS_MMAP_OP_RINGBUF_SZ
+# define ERTS_MMAP_OP_RINGBUF_SZ 0
#endif
/* #define ERTS_MMAP_DEBUG_FILL_AREAS */
@@ -128,6 +137,148 @@ static RBTNode* check_tree(RBTree* tree, Uint);
# define HARD_CHECK_TREE(TREE,SZ)
#endif
+#if ERTS_MMAP_OP_RINGBUF_SZ
+
+static int mmap_op_ix;
+
+typedef enum {
+ ERTS_OP_TYPE_NONE,
+ ERTS_OP_TYPE_MMAP,
+ ERTS_OP_TYPE_MUNMAP,
+ ERTS_OP_TYPE_MREMAP
+} ErtsMMapOpType;
+
+typedef struct {
+ ErtsMMapOpType type;
+ void *result;
+ UWord in_size;
+ UWord out_size;
+ void *old_ptr;
+ UWord old_size;
+} ErtsMMapOp;
+
+static ErtsMMapOp mmap_ops[ERTS_MMAP_OP_RINGBUF_SZ];
+
+#define ERTS_MMAP_OP_RINGBUF_INIT() \
+ do { \
+ int ix__; \
+ for (ix__ = 0; ix__ < ERTS_MMAP_OP_RINGBUF_SZ; ix__++) {\
+ mmap_ops[ix__].type = ERTS_OP_TYPE_NONE; \
+ mmap_ops[ix__].result = NULL; \
+ mmap_ops[ix__].in_size = 0; \
+ mmap_ops[ix__].out_size = 0; \
+ mmap_ops[ix__].old_ptr = NULL; \
+ mmap_ops[ix__].old_size = 0; \
+ } \
+ mmap_op_ix = ERTS_MMAP_OP_RINGBUF_SZ-1; \
+ } while (0)
+
+#define ERTS_MMAP_OP_START(SZ) \
+ do { \
+ int ix__; \
+ if (++mmap_op_ix >= ERTS_MMAP_OP_RINGBUF_SZ) \
+ mmap_op_ix = 0; \
+ ix__ = mmap_op_ix; \
+ mmap_ops[ix__].type = ERTS_OP_TYPE_MMAP; \
+ mmap_ops[ix__].result = NULL; \
+ mmap_ops[ix__].in_size = (SZ); \
+ mmap_ops[ix__].out_size = 0; \
+ mmap_ops[ix__].old_ptr = NULL; \
+ mmap_ops[ix__].old_size = 0; \
+ } while (0)
+
+#define ERTS_MMAP_OP_END(PTR, SZ) \
+ do { \
+ int ix__ = mmap_op_ix; \
+ mmap_ops[ix__].result = (PTR); \
+ mmap_ops[ix__].out_size = (SZ); \
+ } while (0)
+
+#define ERTS_MMAP_OP_LCK(RES, IN_SZ, OUT_SZ) \
+ do { \
+ erts_smp_mtx_lock(&mmap_state.mtx); \
+ ERTS_MMAP_OP_START((IN_SZ)); \
+ ERTS_MMAP_OP_END((RES), (OUT_SZ)); \
+ erts_smp_mtx_unlock(&mmap_state.mtx); \
+ } while (0)
+
+#define ERTS_MUNMAP_OP(PTR, SZ) \
+ do { \
+ int ix__; \
+ if (++mmap_op_ix >= ERTS_MMAP_OP_RINGBUF_SZ) \
+ mmap_op_ix = 0; \
+ ix__ = mmap_op_ix; \
+ mmap_ops[ix__].type = ERTS_OP_TYPE_MUNMAP; \
+ mmap_ops[ix__].result = NULL; \
+ mmap_ops[ix__].in_size = 0; \
+ mmap_ops[ix__].out_size = 0; \
+ mmap_ops[ix__].old_ptr = (PTR); \
+ mmap_ops[ix__].old_size = (SZ); \
+ } while (0)
+
+#define ERTS_MUNMAP_OP_LCK(PTR, SZ) \
+ do { \
+ erts_smp_mtx_lock(&mmap_state.mtx); \
+ ERTS_MUNMAP_OP((PTR), (SZ)); \
+ erts_smp_mtx_unlock(&mmap_state.mtx); \
+ } while (0)
+
+#define ERTS_MREMAP_OP_START(OLD_PTR, OLD_SZ, IN_SZ) \
+ do { \
+ int ix__; \
+ if (++mmap_op_ix >= ERTS_MMAP_OP_RINGBUF_SZ) \
+ mmap_op_ix = 0; \
+ ix__ = mmap_op_ix; \
+ mmap_ops[ix__].type = ERTS_OP_TYPE_MREMAP; \
+ mmap_ops[ix__].result = NULL; \
+ mmap_ops[ix__].in_size = (IN_SZ); \
+ mmap_ops[ix__].out_size = (OLD_SZ); \
+ mmap_ops[ix__].old_ptr = (OLD_PTR); \
+ mmap_ops[ix__].old_size = (OLD_SZ); \
+ } while (0)
+
+#define ERTS_MREMAP_OP_END(PTR, SZ) \
+ do { \
+ int ix__ = mmap_op_ix; \
+ mmap_ops[ix__].result = (PTR); \
+ mmap_ops[mmap_op_ix].out_size = (SZ); \
+ } while (0)
+
+#define ERTS_MREMAP_OP_LCK(RES, OLD_PTR, OLD_SZ, IN_SZ, OUT_SZ) \
+ do { \
+ erts_smp_mtx_lock(&mmap_state.mtx); \
+ ERTS_MREMAP_OP_START((OLD_PTR), (OLD_SZ), (IN_SZ)); \
+ ERTS_MREMAP_OP_END((RES), (OUT_SZ)); \
+ erts_smp_mtx_unlock(&mmap_state.mtx); \
+ } while (0)
+
+#define ERTS_MMAP_OP_ABORT() \
+ do { \
+ int ix__ = mmap_op_ix; \
+ mmap_ops[ix__].type = ERTS_OP_TYPE_NONE; \
+ mmap_ops[ix__].result = NULL; \
+ mmap_ops[ix__].in_size = 0; \
+ mmap_ops[ix__].out_size = 0; \
+ mmap_ops[ix__].old_ptr = NULL; \
+ mmap_ops[ix__].old_size = 0; \
+ if (--mmap_op_ix < 0) \
+ mmap_op_ix = ERTS_MMAP_OP_RINGBUF_SZ-1; \
+ } while (0)
+
+#else
+
+#define ERTS_MMAP_OP_RINGBUF_INIT()
+#define ERTS_MMAP_OP_START(SZ)
+#define ERTS_MMAP_OP_END(PTR, SZ)
+#define ERTS_MMAP_OP_LCK(RES, IN_SZ, OUT_SZ)
+#define ERTS_MUNMAP_OP(PTR, SZ)
+#define ERTS_MUNMAP_OP_LCK(PTR, SZ)
+#define ERTS_MREMAP_OP_START(OLD_PTR, OLD_SZ, IN_SZ)
+#define ERTS_MREMAP_OP_END(PTR, SZ)
+#define ERTS_MREMAP_OP_LCK(RES, OLD_PTR, OLD_SZ, IN_SZ, OUT_SZ)
+#define ERTS_MMAP_OP_ABORT()
+
+#endif
typedef struct {
RBTNode snode; /* node in 'stree' */
@@ -167,7 +318,19 @@ static struct {
int mmap_fd;
#endif
erts_smp_mtx_t mtx;
- char *desc_free_list;
+ struct {
+ char *free_list;
+ char *unused_start;
+ char *unused_end;
+ char *new_area_hint;
+ } desc;
+ struct {
+ UWord free_seg_descs;
+ struct {
+ UWord curr;
+ UWord max;
+ } free_segs;
+ } no;
struct {
struct {
UWord total;
@@ -230,12 +393,15 @@ static struct {
static void
add_free_desc_area(char *start, char *end)
{
- if (end > start && sizeof(ErtsFreeSegDesc) <= end - start) {
+ ERTS_MMAP_ASSERT(end == (void *) 0 || end > start);
+ if (sizeof(ErtsFreeSegDesc) <= ((UWord) end) - ((UWord) start)) {
+ UWord no;
ErtsFreeSegDesc *prev_desc, *desc;
char *desc_end;
+ no = 1;
prev_desc = (ErtsFreeSegDesc *) start;
- prev_desc->start = mmap_state.desc_free_list;
+ prev_desc->start = mmap_state.desc.free_list;
desc = (ErtsFreeSegDesc *) (start + sizeof(ErtsFreeSegDesc));
desc_end = start + 2*sizeof(ErtsFreeSegDesc);
@@ -244,26 +410,61 @@ add_free_desc_area(char *start, char *end)
prev_desc = desc;
desc = (ErtsFreeSegDesc *) desc_end;
desc_end += sizeof(ErtsFreeSegDesc);
+ no++;
}
- mmap_state.desc_free_list = (char *) prev_desc;
+ mmap_state.desc.free_list = (char *) prev_desc;
+ mmap_state.no.free_seg_descs += no;
}
}
+static ErtsFreeSegDesc *
+add_unused_free_desc_area(void)
+{
+ char *ptr;
+ if (!mmap_state.desc.unused_start)
+ return NULL;
+
+ ERTS_MMAP_ASSERT(mmap_state.desc.unused_end);
+ ERTS_MMAP_ASSERT(ERTS_PAGEALIGNED_SIZE
+ <= mmap_state.desc.unused_end - mmap_state.desc.unused_start);
+
+ ptr = mmap_state.desc.unused_start + ERTS_PAGEALIGNED_SIZE;
+ add_free_desc_area(mmap_state.desc.unused_start, ptr);
+
+ if ((mmap_state.desc.unused_end - ptr) >= ERTS_PAGEALIGNED_SIZE)
+ mmap_state.desc.unused_start = ptr;
+ else
+ mmap_state.desc.unused_end = mmap_state.desc.unused_start = NULL;
+
+ ERTS_MMAP_ASSERT(mmap_state.desc.free_list);
+ return (ErtsFreeSegDesc *) mmap_state.desc.free_list;
+}
+
static ERTS_INLINE ErtsFreeSegDesc *
alloc_desc(void)
{
ErtsFreeSegDesc *res;
- res = (ErtsFreeSegDesc *) mmap_state.desc_free_list;
- if (res)
- mmap_state.desc_free_list = res->start;
+ res = (ErtsFreeSegDesc *) mmap_state.desc.free_list;
+ if (!res) {
+ res = add_unused_free_desc_area();
+ if (!res)
+ return NULL;
+ }
+ mmap_state.desc.free_list = res->start;
+ ASSERT(mmap_state.no.free_segs.curr < mmap_state.no.free_seg_descs);
+ mmap_state.no.free_segs.curr++;
+ if (mmap_state.no.free_segs.max < mmap_state.no.free_segs.curr)
+ mmap_state.no.free_segs.max = mmap_state.no.free_segs.curr;
return res;
}
static ERTS_INLINE void
free_desc(ErtsFreeSegDesc *desc)
{
- desc->start = mmap_state.desc_free_list;
- mmap_state.desc_free_list = (char *) desc;
+ desc->start = mmap_state.desc.free_list;
+ mmap_state.desc.free_list = (char *) desc;
+ ERTS_MMAP_ASSERT(mmap_state.no.free_segs.curr > 0);
+ mmap_state.no.free_segs.curr--;
}
static ERTS_INLINE ErtsFreeSegDesc* anode_to_desc(RBTNode* anode)
@@ -1040,7 +1241,7 @@ Eterm build_free_seg_list(Process* p, ErtsFreeSegMap* map)
#endif
static ERTS_INLINE void *
-os_mmap(UWord size, int try_superalign)
+os_mmap(void *hint_ptr, UWord size, int try_superalign)
{
#if HAVE_MMAP
void *res;
@@ -1050,7 +1251,7 @@ os_mmap(UWord size, int try_superalign)
ERTS_MMAP_FLAGS|MAP_ALIGN, ERTS_MMAP_FD, 0);
else
#endif
- res = mmap((void *) 0, size, ERTS_MMAP_PROT,
+ res = mmap((void *) hint_ptr, size, ERTS_MMAP_PROT,
ERTS_MMAP_FLAGS, ERTS_MMAP_FD, 0);
if (res == MAP_FAILED)
return NULL;
@@ -1179,37 +1380,90 @@ static void unreserve_noop(char *ptr, UWord size)
#endif
}
-static ERTS_INLINE UWord
+static UWord
alloc_desc_insert_free_seg(ErtsFreeSegMap *map, char* start, char* end)
{
- UWord ad_sz;
- char *new_end;
+ char *ptr;
+ ErtsFreeSegMap *da_map;
ErtsFreeSegDesc *desc = alloc_desc();
if (desc) {
insert_free_seg(map, desc, start, end);
return 0;
}
- /* Use part of the free segment for descriptors */
+ /*
+ * Ahh; ran out of free segment descriptors.
+ *
+ * First try to map a new page...
+ */
- if (map == &mmap_state.sa.map) {
- ERTS_MMAP_SIZE_SC_SA_INC(ERTS_SUPERALIGNED_SIZE);
- ad_sz = ERTS_SUPERALIGNED_SIZE;
+#if ERTS_HAVE_OS_MMAP
+ ptr = os_mmap(mmap_state.desc.new_area_hint, ERTS_PAGEALIGNED_SIZE, 0);
+ if (ptr) {
+ mmap_state.desc.new_area_hint = ptr+ERTS_PAGEALIGNED_SIZE;
+ ERTS_MMAP_SIZE_OS_INC(ERTS_PAGEALIGNED_SIZE);
+ add_free_desc_area(ptr, ptr+ERTS_PAGEALIGNED_SIZE);
+ desc = alloc_desc();
+ ERTS_MMAP_ASSERT(desc);
+ insert_free_seg(map, desc, start, end);
+ return 0;
+ }
+#endif
+
+ /*
+ * ...then try to find a good place in the supercarrier...
+ */
+ da_map = &mmap_state.sua.map;
+ desc = lookup_free_seg(da_map, ERTS_PAGEALIGNED_SIZE);
+ if (desc) {
+ if (mmap_state.reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE))
+ ERTS_MMAP_SIZE_SC_SUA_INC(ERTS_PAGEALIGNED_SIZE);
+ else
+ desc = NULL;
+
}
else {
- ERTS_MMAP_SIZE_SC_SUA_INC(ERTS_PAGEALIGNED_SIZE);
- ad_sz = ERTS_PAGEALIGNED_SIZE;
+ da_map = &mmap_state.sa.map;
+ desc = lookup_free_seg(da_map, ERTS_PAGEALIGNED_SIZE);
+ if (desc) {
+ if (mmap_state.reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE))
+ ERTS_MMAP_SIZE_SC_SA_INC(ERTS_PAGEALIGNED_SIZE);
+ else
+ desc = NULL;
+ }
}
+ if (desc) {
+ char *da_end = desc->start + ERTS_PAGEALIGNED_SIZE;
+ add_free_desc_area(desc->start, da_end);
+ if (da_end != desc->end)
+ resize_free_seg(da_map, desc, da_end, desc->end);
+ else {
+ delete_free_seg(da_map, desc);
+ free_desc(desc);
+ }
- new_end = end - ad_sz;
- ERTS_MMAP_ASSERT(start <= new_end);
- if (start != new_end) {
desc = alloc_desc();
ERTS_MMAP_ASSERT(desc);
- insert_free_seg(map, desc, start, new_end);
+ insert_free_seg(map, desc, start, end);
+ return 0;
}
- return ad_sz;
+ /*
+ * ... and then as last resort use the first page of the
+ * free segment we are trying to insert for free descriptors.
+ */
+ ptr = start + ERTS_PAGEALIGNED_SIZE;
+ ERTS_MMAP_ASSERT(ptr <= end);
+
+ add_free_desc_area(start, ptr);
+
+ if (ptr != end) {
+ desc = alloc_desc();
+ ERTS_MMAP_ASSERT(desc);
+ insert_free_seg(map, desc, ptr, end);
+ }
+
+ return ERTS_PAGEALIGNED_SIZE;
}
void *
@@ -1226,6 +1480,8 @@ erts_mmap(Uint32 flags, UWord *sizep)
erts_smp_mtx_lock(&mmap_state.mtx);
+ ERTS_MMAP_OP_START(*sizep);
+
if (!superaligned) {
desc = lookup_free_seg(&mmap_state.sua.map, asize);
if (desc) {
@@ -1290,10 +1546,10 @@ erts_mmap(Uint32 flags, UWord *sizep)
}
if (superaligned) {
- seg = (char *) ERTS_SUPERALIGNED_CEILING(mmap_state.sa.top);
+ char *start = mmap_state.sa.top;
+ seg = (char *) ERTS_SUPERALIGNED_CEILING(start);
- if (asize <= mmap_state.sua.bot - seg) {
- char *start = mmap_state.sa.top;
+ if (asize + (seg - start) <= mmap_state.sua.bot - start) {
end = seg + asize;
if (!mmap_state.reserve_physical(start, (UWord) (end - start)))
goto supercarrier_reserve_failure;
@@ -1342,6 +1598,7 @@ erts_mmap(Uint32 flags, UWord *sizep)
}
}
+ ERTS_MMAP_OP_ABORT();
erts_smp_mtx_unlock(&mmap_state.mtx);
}
@@ -1349,15 +1606,15 @@ erts_mmap(Uint32 flags, UWord *sizep)
/* Map using OS primitives */
if (!(ERTS_MMAPFLG_SUPERCARRIER_ONLY & flags) && !mmap_state.no_os_mmap) {
if (!(ERTS_MMAPFLG_SUPERALIGNED & flags)) {
- seg = os_mmap(asize, 0);
+ seg = os_mmap(NULL, asize, 0);
if (!seg)
- return NULL;
+ goto failure;
}
else {
asize = ERTS_SUPERALIGNED_CEILING(*sizep);
- seg = os_mmap(asize, 1);
+ seg = os_mmap(NULL, asize, 1);
if (!seg)
- return NULL;
+ goto failure;
if (!ERTS_IS_SUPERALIGNED(seg)) {
char *ptr;
@@ -1365,9 +1622,9 @@ erts_mmap(Uint32 flags, UWord *sizep)
os_munmap(seg, asize);
- ptr = os_mmap(asize + ERTS_SUPERALIGNED_SIZE, 1);
+ ptr = os_mmap(NULL, asize + ERTS_SUPERALIGNED_SIZE, 1);
if (!ptr)
- return NULL;
+ goto failure;
seg = (char *) ERTS_SUPERALIGNED_CEILING(ptr);
sz = (UWord) (seg - ptr);
@@ -1380,11 +1637,14 @@ erts_mmap(Uint32 flags, UWord *sizep)
}
}
+ ERTS_MMAP_OP_LCK(seg, *sizep, asize);
ERTS_MMAP_SIZE_OS_INC(asize);
*sizep = asize;
return (void *) seg;
}
+failure:
#endif
+ ERTS_MMAP_OP_LCK(NULL, *sizep, 0);
*sizep = 0;
return NULL;
@@ -1401,6 +1661,7 @@ supercarrier_success:
}
#endif
+ ERTS_MMAP_OP_END(seg, asize);
erts_smp_mtx_unlock(&mmap_state.mtx);
*sizep = asize;
@@ -1408,10 +1669,8 @@ supercarrier_success:
supercarrier_reserve_failure:
erts_smp_mtx_unlock(&mmap_state.mtx);
-
*sizep = 0;
return NULL;
-
}
void
@@ -1419,9 +1678,11 @@ erts_munmap(Uint32 flags, void *ptr, UWord size)
{
ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr));
ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(size));
+
if (!ERTS_MMAP_IN_SUPERCARRIER(ptr)) {
ERTS_MMAP_ASSERT(!mmap_state.no_os_mmap);
#if ERTS_HAVE_OS_MMAP
+ ERTS_MUNMAP_OP_LCK(ptr, size);
ERTS_MMAP_SIZE_OS_DEC(size);
os_munmap(ptr, size);
#endif
@@ -1439,6 +1700,8 @@ erts_munmap(Uint32 flags, void *ptr, UWord size)
erts_smp_mtx_lock(&mmap_state.mtx);
+ ERTS_MUNMAP_OP(ptr, size);
+
if (ERTS_MMAP_IN_SUPERALIGNED_AREA(ptr)) {
map = &mmap_state.sa.map;
@@ -1504,7 +1767,7 @@ erts_munmap(Uint32 flags, void *ptr, UWord size)
ERTS_MMAP_ASSERT(size >= ad_sz);
unres_sz = size - ad_sz;
if (unres_sz)
- mmap_state.unreserve_physical((char *) ptr, unres_sz);
+ mmap_state.unreserve_physical(((char *) ptr) + ad_sz, unres_sz);
}
}
}
@@ -1546,8 +1809,10 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep)
return new_ptr;
}
- if (ERTS_MMAPFLG_SUPERCARRIER_ONLY & flags)
+ if (ERTS_MMAPFLG_SUPERCARRIER_ONLY & flags) {
+ ERTS_MREMAP_OP_LCK(NULL, ptr, old_size, *sizep, old_size);
return NULL;
+ }
#if ERTS_HAVE_OS_MREMAP || ERTS_HAVE_GENUINE_OS_MMAP
superaligned = (ERTS_MMAPFLG_SUPERALIGNED & flags);
@@ -1555,6 +1820,7 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep)
if (superaligned) {
asize = ERTS_SUPERALIGNED_CEILING(*sizep);
if (asize == old_size && ERTS_IS_SUPERALIGNED(ptr)) {
+ ERTS_MREMAP_OP_LCK(ptr, ptr, old_size, *sizep, asize);
*sizep = asize;
return ptr;
}
@@ -1562,6 +1828,7 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep)
else {
asize = ERTS_PAGEALIGNED_CEILING(*sizep);
if (asize == old_size) {
+ ERTS_MREMAP_OP_LCK(ptr, ptr, old_size, *sizep, asize);
*sizep = asize;
return ptr;
}
@@ -1577,6 +1844,7 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep)
um_sz = (UWord) ((((char *) ptr) + old_size) - (char *) new_ptr);
ERTS_MMAP_SIZE_OS_DEC(um_sz);
os_munmap(new_ptr, um_sz);
+ ERTS_MREMAP_OP_LCK(ptr, ptr, old_size, *sizep, asize);
*sizep = asize;
return ptr;
}
@@ -1592,6 +1860,7 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep)
ERTS_MMAP_SIZE_OS_INC(asize - old_size);
else
ERTS_MMAP_SIZE_OS_DEC(old_size - asize);
+ ERTS_MREMAP_OP_LCK(new_ptr, ptr, old_size, *sizep, asize);
*sizep = asize;
return new_ptr;
}
@@ -1630,6 +1899,8 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep)
old_size, sizep);
}
+ ERTS_MREMAP_OP_START(ptr, old_size, *sizep);
+
if (asize == old_size) {
new_ptr = ptr;
goto supercarrier_resize_success;
@@ -1670,7 +1941,7 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep)
ERTS_MMAP_ASSERT(old_size - asize >= ad_sz);
unres_sz = old_size - asize - ad_sz;
if (unres_sz)
- mmap_state.unreserve_physical(((char *) ptr) + asize,
+ mmap_state.unreserve_physical(((char *) ptr) + asize + ad_sz,
unres_sz);
goto supercarrier_resize_success;
}
@@ -1728,6 +1999,8 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep)
}
}
}
+
+ ERTS_MMAP_OP_ABORT();
erts_smp_mtx_unlock(&mmap_state.mtx);
/* Failed to resize... */
@@ -1749,15 +2022,16 @@ supercarrier_resize_success:
}
#endif
+ ERTS_MREMAP_OP_END(new_ptr, asize);
erts_smp_mtx_unlock(&mmap_state.mtx);
*sizep = asize;
return new_ptr;
supercarrier_reserve_failure:
-
+ ERTS_MREMAP_OP_END(NULL, old_size);
erts_smp_mtx_unlock(&mmap_state.mtx);
- *sizep = 0;
+ *sizep = old_size;
return NULL;
}
@@ -1794,8 +2068,11 @@ erts_mmap_init(ErtsMMapInit *init)
erl_exit(-1, "erts_mmap: Invalid pagesize: %bpu\n",
pagesize);
+ ERTS_MMAP_OP_RINGBUF_INIT();
+
erts_have_erts_mmap = 0;
+ mmap_state.supercarrier = 0;
mmap_state.reserve_physical = reserve_noop;
mmap_state.unreserve_physical = unreserve_noop;
@@ -1851,7 +2128,7 @@ erts_mmap_init(ErtsMMapInit *init)
* The whole supercarrier will by physically
* reserved all the time.
*/
- start = os_mmap(sz, 1);
+ start = os_mmap(NULL, sz, 1);
}
if (!start)
erl_exit(-1,
@@ -1871,12 +2148,18 @@ erts_mmap_init(ErtsMMapInit *init)
erts_have_erts_mmap |= ERTS_HAVE_ERTS_OS_MMAP;
#endif
+ mmap_state.no.free_seg_descs = 0;
+ mmap_state.no.free_segs.curr = 0;
+ mmap_state.no.free_segs.max = 0;
+
mmap_state.size.supercarrier.total = 0;
mmap_state.size.supercarrier.used.total = 0;
mmap_state.size.supercarrier.used.sa = 0;
mmap_state.size.supercarrier.used.sua = 0;
mmap_state.size.os.used = 0;
+ mmap_state.desc.new_area_hint = NULL;
+
if (!start) {
mmap_state.sa.bot = NULL;
mmap_state.sua.top = NULL;
@@ -1907,6 +2190,8 @@ erts_mmap_init(ErtsMMapInit *init)
mmap_state.size.os.used += (UWord) (mmap_state.sa.bot - start);
+ mmap_state.desc.free_list = NULL;
+
if (end == (void *) 0) {
/*
* Very unlikely, but we need a guarantee
@@ -1916,6 +2201,10 @@ erts_mmap_init(ErtsMMapInit *init)
*/
mmap_state.sua.top -= ERTS_PAGEALIGNED_SIZE;
mmap_state.size.os.used += ERTS_PAGEALIGNED_SIZE;
+#ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION
+ if (!virtual_map || os_reserve_physical(mmap_state.sua.top, ERTS_PAGEALIGNED_SIZE))
+#endif
+ add_free_desc_area(mmap_state.sua.top, end);
}
mmap_state.size.supercarrier.total = (UWord) (mmap_state.sua.top - mmap_state.sa.bot);
@@ -1924,23 +2213,21 @@ erts_mmap_init(ErtsMMapInit *init)
* Area before (and after) super carrier
* will be used for free segment descritors.
*/
- mmap_state.desc_free_list = NULL;
-#ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION
- if (virtual_map && mmap_state.sa.bot - start > 0)
- os_reserve_physical(start, mmap_state.sa.bot - start);
-#endif
- add_free_desc_area(start, mmap_state.sa.bot);
#ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION
- if (virtual_map && end - mmap_state.sua.top > 0)
- os_reserve_physical(mmap_state.sua.top, end - mmap_state.sua.top);
+ if (virtual_map && !os_reserve_physical(start, mmap_state.sa.bot - start))
+ erl_exit(-1, "erts_mmap: Failed to reserve physical memory for descriptors\n");
#endif
- add_free_desc_area(mmap_state.sua.top, end);
+ mmap_state.desc.unused_start = start;
+ mmap_state.desc.unused_end = mmap_state.sa.bot;
init_free_seg_map(&mmap_state.sa.map, SA_SZ_ADDR_ORDER);
init_free_seg_map(&mmap_state.sua.map, SZ_REVERSE_ADDR_ORDER);
mmap_state.supercarrier = 1;
erts_have_erts_mmap |= ERTS_HAVE_ERTS_SUPERCARRIER_MMAP;
+
+ mmap_state.desc.new_area_hint = end;
+
}
#if !ERTS_HAVE_OS_MMAP
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index 2474015bcb..c65973b3be 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -132,7 +132,7 @@ typedef struct {
typedef struct cache_t_ cache_t;
struct cache_t_ {
- Uint size;
+ UWord size;
void *seg;
cache_t *next;
cache_t *prev;
@@ -408,7 +408,7 @@ static ERTS_INLINE void mseg_cache_clear_node(cache_t *c) {
c->prev = c;
}
-static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size, Uint flags) {
+static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, UWord size, Uint flags) {
cache_t *c;
ERTS_DBG_MK_CHK_THR_ACCESS(mk);
@@ -496,7 +496,7 @@ static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size, Ui
static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flags) {
- Uint size = (UWord) *size_p;
+ UWord size = *size_p;
ERTS_DBG_MK_CHK_THR_ACCESS(mk);
@@ -505,7 +505,7 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag
int i, ix = SIZE_TO_CACHE_AREA_IDX(size);
char *seg;
cache_t *c;
- Uint csize;
+ UWord csize;
ASSERT(IS_2POW(size));
@@ -542,10 +542,10 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag
void *seg;
cache_t *c;
cache_t *best = NULL;
- Uint bdiff = 0;
- Uint csize;
- Uint bad_max_abs = mk->ma->abs_max_cache_bad_fit;
- Uint bad_max_rel = mk->ma->rel_max_cache_bad_fit;
+ UWord bdiff = 0;
+ UWord csize;
+ UWord bad_max_abs = mk->ma->abs_max_cache_bad_fit;
+ UWord bad_max_rel = mk->ma->rel_max_cache_bad_fit;
erts_circleq_foreach(c, &(mk->cache_unpowered_node)) {
csize = c->size;
@@ -562,7 +562,7 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag
mseg_cache_clear_node(c);
erts_circleq_push_head(&(mk->cache_free), c);
- *size_p = (UWord) csize;
+ *size_p = csize;
return seg;
@@ -593,7 +593,7 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag
ASSERT((size % GET_PAGE_SIZE) == 0);
ASSERT((best->size % GET_PAGE_SIZE) == 0);
- *size_p = (UWord) size;
+ *size_p = size;
return seg;
@@ -822,7 +822,7 @@ mseg_dealloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, UWord size,
ERTS_MSEG_DEALLOC_STAT(mk,size);
- if (opt->cache && cache_bless_segment(mk, seg, (Uint) size, flags)) {
+ if (opt->cache && cache_bless_segment(mk, seg, size, flags)) {
schedule_cache_check(ma);
goto done;
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-26 08:36:23 +0000