diff options
| -rw-r--r-- | erts/emulator/sys/common/erl_mmap.c | 389 | ||||
| -rw-r--r-- | erts/emulator/sys/common/erl_mseg.c | 22 |
2 files changed, 349 insertions, 62 deletions
diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 73874759f5..1f9168df28 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -27,9 +27,18 @@ #include "erl_mmap.h" #include <stddef.h> +/* #define ERTS_MMAP_OP_RINGBUF_SZ 100 */ + #if defined(DEBUG) || 0 # undef ERTS_MMAP_DEBUG # define ERTS_MMAP_DEBUG +# ifndef ERTS_MMAP_OP_RINGBUF_SZ +# define ERTS_MMAP_OP_RINGBUF_SZ 100 +# endif +#endif + +#ifndef ERTS_MMAP_OP_RINGBUF_SZ +# define ERTS_MMAP_OP_RINGBUF_SZ 0 #endif /* #define ERTS_MMAP_DEBUG_FILL_AREAS */ @@ -128,6 +137,148 @@ static RBTNode* check_tree(RBTree* tree, Uint); # define HARD_CHECK_TREE(TREE,SZ) #endif +#if ERTS_MMAP_OP_RINGBUF_SZ + +static int mmap_op_ix; + +typedef enum { + ERTS_OP_TYPE_NONE, + ERTS_OP_TYPE_MMAP, + ERTS_OP_TYPE_MUNMAP, + ERTS_OP_TYPE_MREMAP +} ErtsMMapOpType; + +typedef struct { + ErtsMMapOpType type; + void *result; + UWord in_size; + UWord out_size; + void *old_ptr; + UWord old_size; +} ErtsMMapOp; + +static ErtsMMapOp mmap_ops[ERTS_MMAP_OP_RINGBUF_SZ]; + +#define ERTS_MMAP_OP_RINGBUF_INIT() \ + do { \ + int ix__; \ + for (ix__ = 0; ix__ < ERTS_MMAP_OP_RINGBUF_SZ; ix__++) {\ + mmap_ops[ix__].type = ERTS_OP_TYPE_NONE; \ + mmap_ops[ix__].result = NULL; \ + mmap_ops[ix__].in_size = 0; \ + mmap_ops[ix__].out_size = 0; \ + mmap_ops[ix__].old_ptr = NULL; \ + mmap_ops[ix__].old_size = 0; \ + } \ + mmap_op_ix = ERTS_MMAP_OP_RINGBUF_SZ-1; \ + } while (0) + +#define ERTS_MMAP_OP_START(SZ) \ + do { \ + int ix__; \ + if (++mmap_op_ix >= ERTS_MMAP_OP_RINGBUF_SZ) \ + mmap_op_ix = 0; \ + ix__ = mmap_op_ix; \ + mmap_ops[ix__].type = ERTS_OP_TYPE_MMAP; \ + mmap_ops[ix__].result = NULL; \ + mmap_ops[ix__].in_size = (SZ); \ + mmap_ops[ix__].out_size = 0; \ + mmap_ops[ix__].old_ptr = NULL; \ + mmap_ops[ix__].old_size = 0; \ + } while (0) + +#define ERTS_MMAP_OP_END(PTR, SZ) \ + do { \ + int ix__ = mmap_op_ix; \ + mmap_ops[ix__].result = (PTR); \ + mmap_ops[ix__].out_size = (SZ); \ + } while (0) + +#define ERTS_MMAP_OP_LCK(RES, IN_SZ, OUT_SZ) \ + do { \ + erts_smp_mtx_lock(&mmap_state.mtx); \ + ERTS_MMAP_OP_START((IN_SZ)); \ + ERTS_MMAP_OP_END((RES), (OUT_SZ)); \ + erts_smp_mtx_unlock(&mmap_state.mtx); \ + } while (0) + +#define ERTS_MUNMAP_OP(PTR, SZ) \ + do { \ + int ix__; \ + if (++mmap_op_ix >= ERTS_MMAP_OP_RINGBUF_SZ) \ + mmap_op_ix = 0; \ + ix__ = mmap_op_ix; \ + mmap_ops[ix__].type = ERTS_OP_TYPE_MUNMAP; \ + mmap_ops[ix__].result = NULL; \ + mmap_ops[ix__].in_size = 0; \ + mmap_ops[ix__].out_size = 0; \ + mmap_ops[ix__].old_ptr = (PTR); \ + mmap_ops[ix__].old_size = (SZ); \ + } while (0) + +#define ERTS_MUNMAP_OP_LCK(PTR, SZ) \ + do { \ + erts_smp_mtx_lock(&mmap_state.mtx); \ + ERTS_MUNMAP_OP((PTR), (SZ)); \ + erts_smp_mtx_unlock(&mmap_state.mtx); \ + } while (0) + +#define ERTS_MREMAP_OP_START(OLD_PTR, OLD_SZ, IN_SZ) \ + do { \ + int ix__; \ + if (++mmap_op_ix >= ERTS_MMAP_OP_RINGBUF_SZ) \ + mmap_op_ix = 0; \ + ix__ = mmap_op_ix; \ + mmap_ops[ix__].type = ERTS_OP_TYPE_MREMAP; \ + mmap_ops[ix__].result = NULL; \ + mmap_ops[ix__].in_size = (IN_SZ); \ + mmap_ops[ix__].out_size = (OLD_SZ); \ + mmap_ops[ix__].old_ptr = (OLD_PTR); \ + mmap_ops[ix__].old_size = (OLD_SZ); \ + } while (0) + +#define ERTS_MREMAP_OP_END(PTR, SZ) \ + do { \ + int ix__ = mmap_op_ix; \ + mmap_ops[ix__].result = (PTR); \ + mmap_ops[mmap_op_ix].out_size = (SZ); \ + } while (0) + +#define ERTS_MREMAP_OP_LCK(RES, OLD_PTR, OLD_SZ, IN_SZ, OUT_SZ) \ + do { \ + erts_smp_mtx_lock(&mmap_state.mtx); \ + ERTS_MREMAP_OP_START((OLD_PTR), (OLD_SZ), (IN_SZ)); \ + ERTS_MREMAP_OP_END((RES), (OUT_SZ)); \ + erts_smp_mtx_unlock(&mmap_state.mtx); \ + } while (0) + +#define ERTS_MMAP_OP_ABORT() \ + do { \ + int ix__ = mmap_op_ix; \ + mmap_ops[ix__].type = ERTS_OP_TYPE_NONE; \ + mmap_ops[ix__].result = NULL; \ + mmap_ops[ix__].in_size = 0; \ + mmap_ops[ix__].out_size = 0; \ + mmap_ops[ix__].old_ptr = NULL; \ + mmap_ops[ix__].old_size = 0; \ + if (--mmap_op_ix < 0) \ + mmap_op_ix = ERTS_MMAP_OP_RINGBUF_SZ-1; \ + } while (0) + +#else + +#define ERTS_MMAP_OP_RINGBUF_INIT() +#define ERTS_MMAP_OP_START(SZ) +#define ERTS_MMAP_OP_END(PTR, SZ) +#define ERTS_MMAP_OP_LCK(RES, IN_SZ, OUT_SZ) +#define ERTS_MUNMAP_OP(PTR, SZ) +#define ERTS_MUNMAP_OP_LCK(PTR, SZ) +#define ERTS_MREMAP_OP_START(OLD_PTR, OLD_SZ, IN_SZ) +#define ERTS_MREMAP_OP_END(PTR, SZ) +#define ERTS_MREMAP_OP_LCK(RES, OLD_PTR, OLD_SZ, IN_SZ, OUT_SZ) +#define ERTS_MMAP_OP_ABORT() + +#endif typedef struct { RBTNode snode; /* node in 'stree' */ @@ -167,7 +318,19 @@ static struct { int mmap_fd; #endif erts_smp_mtx_t mtx; - char *desc_free_list; + struct { + char *free_list; + char *unused_start; + char *unused_end; + char *new_area_hint; + } desc; + struct { + UWord free_seg_descs; + struct { + UWord curr; + UWord max; + } free_segs; + } no; struct { struct { UWord total; @@ -230,12 +393,15 @@ static struct { static void add_free_desc_area(char *start, char *end) { - if (end > start && sizeof(ErtsFreeSegDesc) <= end - start) { + ERTS_MMAP_ASSERT(end == (void *) 0 || end > start); + if (sizeof(ErtsFreeSegDesc) <= ((UWord) end) - ((UWord) start)) { + UWord no; ErtsFreeSegDesc *prev_desc, *desc; char *desc_end; + no = 1; prev_desc = (ErtsFreeSegDesc *) start; - prev_desc->start = mmap_state.desc_free_list; + prev_desc->start = mmap_state.desc.free_list; desc = (ErtsFreeSegDesc *) (start + sizeof(ErtsFreeSegDesc)); desc_end = start + 2*sizeof(ErtsFreeSegDesc); @@ -244,26 +410,61 @@ add_free_desc_area(char *start, char *end) prev_desc = desc; desc = (ErtsFreeSegDesc *) desc_end; desc_end += sizeof(ErtsFreeSegDesc); + no++; } - mmap_state.desc_free_list = (char *) prev_desc; + mmap_state.desc.free_list = (char *) prev_desc; + mmap_state.no.free_seg_descs += no; } } +static ErtsFreeSegDesc * +add_unused_free_desc_area(void) +{ + char *ptr; + if (!mmap_state.desc.unused_start) + return NULL; + + ERTS_MMAP_ASSERT(mmap_state.desc.unused_end); + ERTS_MMAP_ASSERT(ERTS_PAGEALIGNED_SIZE + <= mmap_state.desc.unused_end - mmap_state.desc.unused_start); + + ptr = mmap_state.desc.unused_start + ERTS_PAGEALIGNED_SIZE; + add_free_desc_area(mmap_state.desc.unused_start, ptr); + + if ((mmap_state.desc.unused_end - ptr) >= ERTS_PAGEALIGNED_SIZE) + mmap_state.desc.unused_start = ptr; + else + mmap_state.desc.unused_end = mmap_state.desc.unused_start = NULL; + + ERTS_MMAP_ASSERT(mmap_state.desc.free_list); + return (ErtsFreeSegDesc *) mmap_state.desc.free_list; +} + static ERTS_INLINE ErtsFreeSegDesc * alloc_desc(void) { ErtsFreeSegDesc *res; - res = (ErtsFreeSegDesc *) mmap_state.desc_free_list; - if (res) - mmap_state.desc_free_list = res->start; + res = (ErtsFreeSegDesc *) mmap_state.desc.free_list; + if (!res) { + res = add_unused_free_desc_area(); + if (!res) + return NULL; + } + mmap_state.desc.free_list = res->start; + ASSERT(mmap_state.no.free_segs.curr < mmap_state.no.free_seg_descs); + mmap_state.no.free_segs.curr++; + if (mmap_state.no.free_segs.max < mmap_state.no.free_segs.curr) + mmap_state.no.free_segs.max = mmap_state.no.free_segs.curr; return res; } static ERTS_INLINE void free_desc(ErtsFreeSegDesc *desc) { - desc->start = mmap_state.desc_free_list; - mmap_state.desc_free_list = (char *) desc; + desc->start = mmap_state.desc.free_list; + mmap_state.desc.free_list = (char *) desc; + ERTS_MMAP_ASSERT(mmap_state.no.free_segs.curr > 0); + mmap_state.no.free_segs.curr--; } static ERTS_INLINE ErtsFreeSegDesc* anode_to_desc(RBTNode* anode) @@ -1040,7 +1241,7 @@ Eterm build_free_seg_list(Process* p, ErtsFreeSegMap* map) #endif static ERTS_INLINE void * -os_mmap(UWord size, int try_superalign) +os_mmap(void *hint_ptr, UWord size, int try_superalign) { #if HAVE_MMAP void *res; @@ -1050,7 +1251,7 @@ os_mmap(UWord size, int try_superalign) ERTS_MMAP_FLAGS|MAP_ALIGN, ERTS_MMAP_FD, 0); else #endif - res = mmap((void *) 0, size, ERTS_MMAP_PROT, + res = mmap((void *) hint_ptr, size, ERTS_MMAP_PROT, ERTS_MMAP_FLAGS, ERTS_MMAP_FD, 0); if (res == MAP_FAILED) return NULL; @@ -1179,37 +1380,90 @@ static void unreserve_noop(char *ptr, UWord size) #endif } -static ERTS_INLINE UWord +static UWord alloc_desc_insert_free_seg(ErtsFreeSegMap *map, char* start, char* end) { - UWord ad_sz; - char *new_end; + char *ptr; + ErtsFreeSegMap *da_map; ErtsFreeSegDesc *desc = alloc_desc(); if (desc) { insert_free_seg(map, desc, start, end); return 0; } - /* Use part of the free segment for descriptors */ + /* + * Ahh; ran out of free segment descriptors. + * + * First try to map a new page... + */ - if (map == &mmap_state.sa.map) { - ERTS_MMAP_SIZE_SC_SA_INC(ERTS_SUPERALIGNED_SIZE); - ad_sz = ERTS_SUPERALIGNED_SIZE; +#if ERTS_HAVE_OS_MMAP + ptr = os_mmap(mmap_state.desc.new_area_hint, ERTS_PAGEALIGNED_SIZE, 0); + if (ptr) { + mmap_state.desc.new_area_hint = ptr+ERTS_PAGEALIGNED_SIZE; + ERTS_MMAP_SIZE_OS_INC(ERTS_PAGEALIGNED_SIZE); + add_free_desc_area(ptr, ptr+ERTS_PAGEALIGNED_SIZE); + desc = alloc_desc(); + ERTS_MMAP_ASSERT(desc); + insert_free_seg(map, desc, start, end); + return 0; + } +#endif + + /* + * ...then try to find a good place in the supercarrier... + */ + da_map = &mmap_state.sua.map; + desc = lookup_free_seg(da_map, ERTS_PAGEALIGNED_SIZE); + if (desc) { + if (mmap_state.reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE)) + ERTS_MMAP_SIZE_SC_SUA_INC(ERTS_PAGEALIGNED_SIZE); + else + desc = NULL; + } else { - ERTS_MMAP_SIZE_SC_SUA_INC(ERTS_PAGEALIGNED_SIZE); - ad_sz = ERTS_PAGEALIGNED_SIZE; + da_map = &mmap_state.sa.map; + desc = lookup_free_seg(da_map, ERTS_PAGEALIGNED_SIZE); + if (desc) { + if (mmap_state.reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE)) + ERTS_MMAP_SIZE_SC_SA_INC(ERTS_PAGEALIGNED_SIZE); + else + desc = NULL; + } } + if (desc) { + char *da_end = desc->start + ERTS_PAGEALIGNED_SIZE; + add_free_desc_area(desc->start, da_end); + if (da_end != desc->end) + resize_free_seg(da_map, desc, da_end, desc->end); + else { + delete_free_seg(da_map, desc); + free_desc(desc); + } - new_end = end - ad_sz; - ERTS_MMAP_ASSERT(start <= new_end); - if (start != new_end) { desc = alloc_desc(); ERTS_MMAP_ASSERT(desc); - insert_free_seg(map, desc, start, new_end); + insert_free_seg(map, desc, start, end); + return 0; } - return ad_sz; + /* + * ... and then as last resort use the first page of the + * free segment we are trying to insert for free descriptors. + */ + ptr = start + ERTS_PAGEALIGNED_SIZE; + ERTS_MMAP_ASSERT(ptr <= end); + + add_free_desc_area(start, ptr); + + if (ptr != end) { + desc = alloc_desc(); + ERTS_MMAP_ASSERT(desc); + insert_free_seg(map, desc, ptr, end); + } + + return ERTS_PAGEALIGNED_SIZE; } void * @@ -1226,6 +1480,8 @@ erts_mmap(Uint32 flags, UWord *sizep) erts_smp_mtx_lock(&mmap_state.mtx); + ERTS_MMAP_OP_START(*sizep); + if (!superaligned) { desc = lookup_free_seg(&mmap_state.sua.map, asize); if (desc) { @@ -1290,10 +1546,10 @@ erts_mmap(Uint32 flags, UWord *sizep) } if (superaligned) { - seg = (char *) ERTS_SUPERALIGNED_CEILING(mmap_state.sa.top); + char *start = mmap_state.sa.top; + seg = (char *) ERTS_SUPERALIGNED_CEILING(start); - if (asize <= mmap_state.sua.bot - seg) { - char *start = mmap_state.sa.top; + if (asize + (seg - start) <= mmap_state.sua.bot - start) { end = seg + asize; if (!mmap_state.reserve_physical(start, (UWord) (end - start))) goto supercarrier_reserve_failure; @@ -1342,6 +1598,7 @@ erts_mmap(Uint32 flags, UWord *sizep) } } + ERTS_MMAP_OP_ABORT(); erts_smp_mtx_unlock(&mmap_state.mtx); } @@ -1349,15 +1606,15 @@ erts_mmap(Uint32 flags, UWord *sizep) /* Map using OS primitives */ if (!(ERTS_MMAPFLG_SUPERCARRIER_ONLY & flags) && !mmap_state.no_os_mmap) { if (!(ERTS_MMAPFLG_SUPERALIGNED & flags)) { - seg = os_mmap(asize, 0); + seg = os_mmap(NULL, asize, 0); if (!seg) - return NULL; + goto failure; } else { asize = ERTS_SUPERALIGNED_CEILING(*sizep); - seg = os_mmap(asize, 1); + seg = os_mmap(NULL, asize, 1); if (!seg) - return NULL; + goto failure; if (!ERTS_IS_SUPERALIGNED(seg)) { char *ptr; @@ -1365,9 +1622,9 @@ erts_mmap(Uint32 flags, UWord *sizep) os_munmap(seg, asize); - ptr = os_mmap(asize + ERTS_SUPERALIGNED_SIZE, 1); + ptr = os_mmap(NULL, asize + ERTS_SUPERALIGNED_SIZE, 1); if (!ptr) - return NULL; + goto failure; seg = (char *) ERTS_SUPERALIGNED_CEILING(ptr); sz = (UWord) (seg - ptr); @@ -1380,11 +1637,14 @@ erts_mmap(Uint32 flags, UWord *sizep) } } + ERTS_MMAP_OP_LCK(seg, *sizep, asize); ERTS_MMAP_SIZE_OS_INC(asize); *sizep = asize; return (void *) seg; } +failure: #endif + ERTS_MMAP_OP_LCK(NULL, *sizep, 0); *sizep = 0; return NULL; @@ -1401,6 +1661,7 @@ supercarrier_success: } #endif + ERTS_MMAP_OP_END(seg, asize); erts_smp_mtx_unlock(&mmap_state.mtx); *sizep = asize; @@ -1408,10 +1669,8 @@ supercarrier_success: supercarrier_reserve_failure: erts_smp_mtx_unlock(&mmap_state.mtx); - *sizep = 0; return NULL; - } void @@ -1419,9 +1678,11 @@ erts_munmap(Uint32 flags, void *ptr, UWord size) { ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr)); ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(size)); + if (!ERTS_MMAP_IN_SUPERCARRIER(ptr)) { ERTS_MMAP_ASSERT(!mmap_state.no_os_mmap); #if ERTS_HAVE_OS_MMAP + ERTS_MUNMAP_OP_LCK(ptr, size); ERTS_MMAP_SIZE_OS_DEC(size); os_munmap(ptr, size); #endif @@ -1439,6 +1700,8 @@ erts_munmap(Uint32 flags, void *ptr, UWord size) erts_smp_mtx_lock(&mmap_state.mtx); + ERTS_MUNMAP_OP(ptr, size); + if (ERTS_MMAP_IN_SUPERALIGNED_AREA(ptr)) { map = &mmap_state.sa.map; @@ -1504,7 +1767,7 @@ erts_munmap(Uint32 flags, void *ptr, UWord size) ERTS_MMAP_ASSERT(size >= ad_sz); unres_sz = size - ad_sz; if (unres_sz) - mmap_state.unreserve_physical((char *) ptr, unres_sz); + mmap_state.unreserve_physical(((char *) ptr) + ad_sz, unres_sz); } } } @@ -1546,8 +1809,10 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) return new_ptr; } - if (ERTS_MMAPFLG_SUPERCARRIER_ONLY & flags) + if (ERTS_MMAPFLG_SUPERCARRIER_ONLY & flags) { + ERTS_MREMAP_OP_LCK(NULL, ptr, old_size, *sizep, old_size); return NULL; + } #if ERTS_HAVE_OS_MREMAP || ERTS_HAVE_GENUINE_OS_MMAP superaligned = (ERTS_MMAPFLG_SUPERALIGNED & flags); @@ -1555,6 +1820,7 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) if (superaligned) { asize = ERTS_SUPERALIGNED_CEILING(*sizep); if (asize == old_size && ERTS_IS_SUPERALIGNED(ptr)) { + ERTS_MREMAP_OP_LCK(ptr, ptr, old_size, *sizep, asize); *sizep = asize; return ptr; } @@ -1562,6 +1828,7 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) else { asize = ERTS_PAGEALIGNED_CEILING(*sizep); if (asize == old_size) { + ERTS_MREMAP_OP_LCK(ptr, ptr, old_size, *sizep, asize); *sizep = asize; return ptr; } @@ -1577,6 +1844,7 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) um_sz = (UWord) ((((char *) ptr) + old_size) - (char *) new_ptr); ERTS_MMAP_SIZE_OS_DEC(um_sz); os_munmap(new_ptr, um_sz); + ERTS_MREMAP_OP_LCK(ptr, ptr, old_size, *sizep, asize); *sizep = asize; return ptr; } @@ -1592,6 +1860,7 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) ERTS_MMAP_SIZE_OS_INC(asize - old_size); else ERTS_MMAP_SIZE_OS_DEC(old_size - asize); + ERTS_MREMAP_OP_LCK(new_ptr, ptr, old_size, *sizep, asize); *sizep = asize; return new_ptr; } @@ -1630,6 +1899,8 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) old_size, sizep); } + ERTS_MREMAP_OP_START(ptr, old_size, *sizep); + if (asize == old_size) { new_ptr = ptr; goto supercarrier_resize_success; @@ -1670,7 +1941,7 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) ERTS_MMAP_ASSERT(old_size - asize >= ad_sz); unres_sz = old_size - asize - ad_sz; if (unres_sz) - mmap_state.unreserve_physical(((char *) ptr) + asize, + mmap_state.unreserve_physical(((char *) ptr) + asize + ad_sz, unres_sz); goto supercarrier_resize_success; } @@ -1728,6 +1999,8 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) } } } + + ERTS_MMAP_OP_ABORT(); erts_smp_mtx_unlock(&mmap_state.mtx); /* Failed to resize... */ @@ -1749,15 +2022,16 @@ supercarrier_resize_success: } #endif + ERTS_MREMAP_OP_END(new_ptr, asize); erts_smp_mtx_unlock(&mmap_state.mtx); *sizep = asize; return new_ptr; supercarrier_reserve_failure: - + ERTS_MREMAP_OP_END(NULL, old_size); erts_smp_mtx_unlock(&mmap_state.mtx); - *sizep = 0; + *sizep = old_size; return NULL; } @@ -1794,8 +2068,11 @@ erts_mmap_init(ErtsMMapInit *init) erl_exit(-1, "erts_mmap: Invalid pagesize: %bpu\n", pagesize); + ERTS_MMAP_OP_RINGBUF_INIT(); + erts_have_erts_mmap = 0; + mmap_state.supercarrier = 0; mmap_state.reserve_physical = reserve_noop; mmap_state.unreserve_physical = unreserve_noop; @@ -1851,7 +2128,7 @@ erts_mmap_init(ErtsMMapInit *init) * The whole supercarrier will by physically * reserved all the time. */ - start = os_mmap(sz, 1); + start = os_mmap(NULL, sz, 1); } if (!start) erl_exit(-1, @@ -1871,12 +2148,18 @@ erts_mmap_init(ErtsMMapInit *init) erts_have_erts_mmap |= ERTS_HAVE_ERTS_OS_MMAP; #endif + mmap_state.no.free_seg_descs = 0; + mmap_state.no.free_segs.curr = 0; + mmap_state.no.free_segs.max = 0; + mmap_state.size.supercarrier.total = 0; mmap_state.size.supercarrier.used.total = 0; mmap_state.size.supercarrier.used.sa = 0; mmap_state.size.supercarrier.used.sua = 0; mmap_state.size.os.used = 0; + mmap_state.desc.new_area_hint = NULL; + if (!start) { mmap_state.sa.bot = NULL; mmap_state.sua.top = NULL; @@ -1907,6 +2190,8 @@ erts_mmap_init(ErtsMMapInit *init) mmap_state.size.os.used += (UWord) (mmap_state.sa.bot - start); + mmap_state.desc.free_list = NULL; + if (end == (void *) 0) { /* * Very unlikely, but we need a guarantee @@ -1916,6 +2201,10 @@ erts_mmap_init(ErtsMMapInit *init) */ mmap_state.sua.top -= ERTS_PAGEALIGNED_SIZE; mmap_state.size.os.used += ERTS_PAGEALIGNED_SIZE; +#ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION + if (!virtual_map || os_reserve_physical(mmap_state.sua.top, ERTS_PAGEALIGNED_SIZE)) +#endif + add_free_desc_area(mmap_state.sua.top, end); } mmap_state.size.supercarrier.total = (UWord) (mmap_state.sua.top - mmap_state.sa.bot); @@ -1924,23 +2213,21 @@ erts_mmap_init(ErtsMMapInit *init) * Area before (and after) super carrier * will be used for free segment descritors. */ - mmap_state.desc_free_list = NULL; -#ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION - if (virtual_map && mmap_state.sa.bot - start > 0) - os_reserve_physical(start, mmap_state.sa.bot - start); -#endif - add_free_desc_area(start, mmap_state.sa.bot); #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION - if (virtual_map && end - mmap_state.sua.top > 0) - os_reserve_physical(mmap_state.sua.top, end - mmap_state.sua.top); + if (virtual_map && !os_reserve_physical(start, mmap_state.sa.bot - start)) + erl_exit(-1, "erts_mmap: Failed to reserve physical memory for descriptors\n"); #endif - add_free_desc_area(mmap_state.sua.top, end); + mmap_state.desc.unused_start = start; + mmap_state.desc.unused_end = mmap_state.sa.bot; init_free_seg_map(&mmap_state.sa.map, SA_SZ_ADDR_ORDER); init_free_seg_map(&mmap_state.sua.map, SZ_REVERSE_ADDR_ORDER); mmap_state.supercarrier = 1; erts_have_erts_mmap |= ERTS_HAVE_ERTS_SUPERCARRIER_MMAP; + + mmap_state.desc.new_area_hint = end; + } #if !ERTS_HAVE_OS_MMAP diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index 2474015bcb..c65973b3be 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -132,7 +132,7 @@ typedef struct { typedef struct cache_t_ cache_t; struct cache_t_ { - Uint size; + UWord size; void *seg; cache_t *next; cache_t *prev; @@ -408,7 +408,7 @@ static ERTS_INLINE void mseg_cache_clear_node(cache_t *c) { c->prev = c; } -static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size, Uint flags) { +static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, UWord size, Uint flags) { cache_t *c; ERTS_DBG_MK_CHK_THR_ACCESS(mk); @@ -496,7 +496,7 @@ static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size, Ui static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flags) { - Uint size = (UWord) *size_p; + UWord size = *size_p; ERTS_DBG_MK_CHK_THR_ACCESS(mk); @@ -505,7 +505,7 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag int i, ix = SIZE_TO_CACHE_AREA_IDX(size); char *seg; cache_t *c; - Uint csize; + UWord csize; ASSERT(IS_2POW(size)); @@ -542,10 +542,10 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag void *seg; cache_t *c; cache_t *best = NULL; - Uint bdiff = 0; - Uint csize; - Uint bad_max_abs = mk->ma->abs_max_cache_bad_fit; - Uint bad_max_rel = mk->ma->rel_max_cache_bad_fit; + UWord bdiff = 0; + UWord csize; + UWord bad_max_abs = mk->ma->abs_max_cache_bad_fit; + UWord bad_max_rel = mk->ma->rel_max_cache_bad_fit; erts_circleq_foreach(c, &(mk->cache_unpowered_node)) { csize = c->size; @@ -562,7 +562,7 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag mseg_cache_clear_node(c); erts_circleq_push_head(&(mk->cache_free), c); - *size_p = (UWord) csize; + *size_p = csize; return seg; @@ -593,7 +593,7 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag ASSERT((size % GET_PAGE_SIZE) == 0); ASSERT((best->size % GET_PAGE_SIZE) == 0); - *size_p = (UWord) size; + *size_p = size; return seg; @@ -822,7 +822,7 @@ mseg_dealloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, UWord size, ERTS_MSEG_DEALLOC_STAT(mk,size); - if (opt->cache && cache_bless_segment(mk, seg, (Uint) size, flags)) { + if (opt->cache && cache_bless_segment(mk, seg, size, flags)) { schedule_cache_check(ma); goto done; } |