From c8f87b4ec91b67c9d3373c8466a07db638e32cc2 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 9 Sep 2013 16:07:29 +0200 Subject: erts: Allow page aligned erts_munmap() --- erts/emulator/sys/common/erl_mmap.c | 102 +++++++++++++++++------------------- erts/emulator/sys/common/erl_mmap.h | 2 +- erts/emulator/sys/common/erl_mseg.c | 70 +++++++------------------ 3 files changed, 67 insertions(+), 107 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index eae1a1a410..73874759f5 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -1260,30 +1260,53 @@ erts_mmap(Uint32 flags, UWord *sizep) desc = lookup_free_seg(&mmap_state.sa.map, asize); if (desc) { - seg = desc->start; + char *start = seg = desc->start; + seg = (char *) ERTS_SUPERALIGNED_CEILING(seg); end = seg+asize; - if (!mmap_state.reserve_physical(seg, asize)) + if (!mmap_state.reserve_physical(start, (UWord) (end - start))) goto supercarrier_reserve_failure; + ERTS_MMAP_SIZE_SC_SA_INC(asize); if (desc->end == end) { - delete_free_seg(&mmap_state.sa.map, desc); - free_desc(desc); + if (start != seg) + resize_free_seg(&mmap_state.sa.map, desc, start, seg); + else { + delete_free_seg(&mmap_state.sa.map, desc); + free_desc(desc); + } } else { ERTS_MMAP_ASSERT(end < desc->end); resize_free_seg(&mmap_state.sa.map, desc, end, desc->end); + if (start != seg) { + UWord ad_sz; + ad_sz = alloc_desc_insert_free_seg(&mmap_state.sua.map, + start, seg); + start += ad_sz; + if (start != seg) + mmap_state.unreserve_physical(start, (UWord) (seg - start)); + } } - ERTS_MMAP_SIZE_SC_SA_INC(asize); goto supercarrier_success; } if (superaligned) { + seg = (char *) ERTS_SUPERALIGNED_CEILING(mmap_state.sa.top); - if (asize <= mmap_state.sua.bot - mmap_state.sa.top) { - seg = (void *) mmap_state.sa.top; - if (!mmap_state.reserve_physical(seg, asize)) + if (asize <= mmap_state.sua.bot - seg) { + char *start = mmap_state.sa.top; + end = seg + asize; + if (!mmap_state.reserve_physical(start, (UWord) (end - start))) goto supercarrier_reserve_failure; - mmap_state.sa.top += asize; + mmap_state.sa.top = end; ERTS_MMAP_SIZE_SC_SA_INC(asize); + if (start != seg) { + UWord ad_sz; + ad_sz = alloc_desc_insert_free_seg(&mmap_state.sua.map, + start, seg); + start += ad_sz; + if (start != seg) + mmap_state.unreserve_physical(start, (UWord) (seg - start)); + } goto supercarrier_success; } @@ -1294,7 +1317,7 @@ erts_mmap(Uint32 flags, UWord *sizep) seg = (char *) ERTS_SUPERALIGNED_CEILING(org_start); end = seg + asize; - if (!mmap_state.reserve_physical(seg, asize)) + if (!mmap_state.reserve_physical(seg, (UWord) (org_end - seg))) goto supercarrier_reserve_failure; ERTS_MMAP_SIZE_SC_SUA_INC(asize); if (org_start != seg) { @@ -1303,12 +1326,17 @@ erts_mmap(Uint32 flags, UWord *sizep) desc = NULL; } if (end != org_end) { + UWord ad_sz = 0; ERTS_MMAP_ASSERT(end < org_end); if (desc) resize_free_seg(&mmap_state.sua.map, desc, end, org_end); else - alloc_desc_insert_free_seg(&mmap_state.sua.map, - end, org_end); + ad_sz = alloc_desc_insert_free_seg(&mmap_state.sua.map, + end, org_end); + end += ad_sz; + if (end != org_end) + mmap_state.unreserve_physical(end, + (UWord) (org_end - end)); } goto supercarrier_success; } @@ -1363,8 +1391,7 @@ erts_mmap(Uint32 flags, UWord *sizep) supercarrier_success: #ifdef ERTS_MMAP_DEBUG - if ((ERTS_MMAPFLG_SUPERALIGNED & flags) - || ERTS_MMAP_IN_SUPERALIGNED_AREA(seg)) { + if (ERTS_MMAPFLG_SUPERALIGNED & flags) { ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(seg)); ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(asize)); } @@ -1388,10 +1415,8 @@ supercarrier_reserve_failure: } void -erts_munmap(Uint32 flags, void **ptrp, UWord *sizep) +erts_munmap(Uint32 flags, void *ptr, UWord size) { - void *ptr = *ptrp; - UWord size = *sizep; ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr)); ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(size)); if (!ERTS_MMAP_IN_SUPERCARRIER(ptr)) { @@ -1416,13 +1441,6 @@ erts_munmap(Uint32 flags, void **ptrp, UWord *sizep) if (ERTS_MMAP_IN_SUPERALIGNED_AREA(ptr)) { - start = (char *) ERTS_SUPERALIGNED_CEILING(start); - end = (char *) ERTS_SUPERALIGNED_FLOOR(end); - - size = (UWord) (end - start); - *ptrp = start; - *sizep = size; - map = &mmap_state.sa.map; adjacent_free_seg(map, start, end, &prev, &next); @@ -1439,8 +1457,6 @@ erts_munmap(Uint32 flags, void **ptrp, UWord *sizep) } } else { - ERTS_MMAP_ASSERT(ERTS_MMAP_IN_SUPERUNALIGNED_AREA(ptr)); - map = &mmap_state.sua.map; adjacent_free_seg(map, start, end, &prev, &next); @@ -1497,8 +1513,6 @@ static void * remap_move(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) { UWord size = *sizep; - UWord um_size = old_size; - void *um_ptr = ptr; void *new_ptr = erts_mmap(flags, &size); if (!new_ptr) return NULL; @@ -1506,9 +1520,7 @@ remap_move(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) if (old_size < size) size = old_size; sys_memcpy(new_ptr, ptr, (size_t) size); - erts_munmap(flags, &um_ptr, &um_size); - ERTS_MMAP_ASSERT(um_ptr == ptr); - ERTS_MMAP_ASSERT(um_size == old_size); + erts_munmap(flags, ptr, old_size); return new_ptr; } @@ -1627,28 +1639,18 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) end = start + old_size; new_end = start+asize; + ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr)); + ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(old_size)); + ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(asize)); + if (asize < old_size) { UWord unres_sz; new_ptr = ptr; if (!ERTS_MMAP_IN_SUPERALIGNED_AREA(ptr)) { - ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr)); - ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(old_size)); - ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(asize)); map = &mmap_state.sua.map; ERTS_MMAP_SIZE_SC_SUA_DEC(old_size - asize); } else { - ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(ptr)); - ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(old_size)); - if (!superaligned) { - /* must be a superaligned size in this area */ - asize = ERTS_SUPERALIGNED_CEILING(asize); - ERTS_MMAP_ASSERT(asize <= old_size); - if (asize == old_size) - goto supercarrier_resize_success; - new_end = start+asize; - } - ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(asize)); if (end == mmap_state.sa.top) { mmap_state.sa.top = new_end; mmap_state.unreserve_physical(((char *) ptr) + asize, @@ -1696,16 +1698,6 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) } } else { /* Superaligned area */ - ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(ptr)); - ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(old_size)); - - if (!superaligned) { - /* must be a superaligned size in this area */ - asize = ERTS_PAGEALIGNED_CEILING(asize); - new_end = start+asize; - } - - ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(asize)); if (end == mmap_state.sa.top) { if (new_end <= mmap_state.sua.bot) { diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h index b75200f4e9..6cb51fb0b4 100644 --- a/erts/emulator/sys/common/erl_mmap.h +++ b/erts/emulator/sys/common/erl_mmap.h @@ -53,7 +53,7 @@ typedef struct { {{NULL, NULL}, {NULL, NULL}, 0, 1, (1 << 16), 1} void *erts_mmap(Uint32 flags, UWord *sizep); -void erts_munmap(Uint32 flags, void **ptrp, UWord *sizep); +void erts_munmap(Uint32 flags, void *ptr, UWord size); void *erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep); int erts_mmap_in_supercarrier(void *ptr); void erts_mmap_init(ErtsMMapInit*); diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index 09035ca73e..2474015bcb 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -330,7 +330,7 @@ mseg_create(ErtsMsegAllctr_t *ma, Uint flags, MemKind* mk, UWord *sizep) } static ERTS_INLINE void -mseg_destroy(ErtsMsegAllctr_t *ma, Uint flags, MemKind* mk, void **seg_pp, UWord *size_p) { +mseg_destroy(ErtsMsegAllctr_t *ma, Uint flags, MemKind* mk, void *seg_p, UWord size) { Uint32 mmap_flags = 0; #if HALFWORD_HEAP @@ -341,11 +341,11 @@ mseg_destroy(ErtsMsegAllctr_t *ma, Uint flags, MemKind* mk, void **seg_pp, UWord if (MSEG_FLG_IS_2POW(flags)) mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED; - erts_munmap(mmap_flags, seg_pp, size_p); + erts_munmap(mmap_flags, seg_p, size); #ifdef ERTS_PRINT_ERTS_MMAP erts_fprintf(stderr, "erts_munmap(%s, %p, %bpu);\n", (mmap_flags & ERTS_MMAPFLG_SUPERALIGNED) ? "sa" : "sua", - *seg_pp, *size_p); + seg_p, *size); #endif INC_CC(ma, destroy); @@ -446,19 +446,13 @@ static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size, Ui return 1; } else if (!MSEG_FLG_IS_2POW(flags) && !erts_circleq_is_empty(&(mk->cache_unpowered_node))) { - void *destr_seg; - UWord destr_size; /* No free slots. * Evict oldest slot from unpowered cache so we can cache an unpowered (sbc) segment */ c = erts_circleq_tail(&(mk->cache_unpowered_node)); erts_circleq_remove(c); - destr_seg = c->seg; - destr_size = c->size; - mseg_destroy(mk->ma, ERTS_MSEG_FLG_NONE, mk, &destr_seg, &destr_size); - ASSERT(destr_seg == c->seg); - ASSERT(destr_size == c->size); + mseg_destroy(mk->ma, ERTS_MSEG_FLG_NONE, mk, c->seg, c->size); mseg_cache_clear_node(c); c->seg = seg; @@ -478,19 +472,13 @@ static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size, Ui int i; for( i = 0; i < CACHE_AREAS; i++) { - void *destr_seg; - UWord destr_size; if (erts_circleq_is_empty(&(mk->cache_powered_node[i]))) continue; c = erts_circleq_tail(&(mk->cache_powered_node[i])); erts_circleq_remove(c); - destr_seg = seg; - destr_size = c->size; - mseg_destroy(mk->ma, ERTS_MSEG_FLG_2POW, mk, &destr_seg, &destr_size); - ASSERT(destr_seg == c->seg); - ASSERT(destr_size == c->size); + mseg_destroy(mk->ma, ERTS_MSEG_FLG_2POW, mk, c->seg, c->size); mseg_cache_clear_node(c); @@ -544,13 +532,8 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag ASSERT(!(mk->cache_size < 0)); - if (csize != size) { - char *destr_seg = seg + size; - UWord destr_size = csize - size; - mseg_destroy(mk->ma, ERTS_MSEG_FLG_2POW, mk, (void**)&destr_seg, &destr_size); - *size_p = (UWord) (destr_seg - seg); - ASSERT(seg + csize == destr_seg + destr_size); - } + if (csize != size) + mseg_destroy(mk->ma, ERTS_MSEG_FLG_2POW, mk, seg + size, csize - size); return seg; } @@ -625,8 +608,6 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag */ static ERTS_INLINE Uint mseg_drop_one_memkind_cache_size(MemKind *mk, Uint flags, cache_t *head) { - void *destr_seg; - UWord destr_size; cache_t *c = NULL; c = erts_circleq_tail(head); @@ -635,11 +616,7 @@ static ERTS_INLINE Uint mseg_drop_one_memkind_cache_size(MemKind *mk, Uint flags if (erts_mtrace_enabled) erts_mtrace_crr_free(SEGTYPE, SEGTYPE, c->seg); - destr_seg = c->seg; - destr_size = c->size; - mseg_destroy(mk->ma, flags, mk, &destr_seg, &destr_size); - ASSERT(destr_seg == c->seg); - ASSERT(destr_size == c->size); + mseg_destroy(mk->ma, flags, mk, c->seg, c->size); mseg_cache_clear_node(c); erts_circleq_push_head(&(mk->cache_free), c); @@ -655,8 +632,6 @@ static ERTS_INLINE Uint mseg_drop_memkind_cache_size(MemKind *mk, Uint flags, ca cache_t *c = NULL; while (!erts_circleq_is_empty(head)) { - void *destr_seg; - UWord destr_size; c = erts_circleq_tail(head); erts_circleq_remove(c); @@ -664,11 +639,7 @@ static ERTS_INLINE Uint mseg_drop_memkind_cache_size(MemKind *mk, Uint flags, ca if (erts_mtrace_enabled) erts_mtrace_crr_free(SEGTYPE, SEGTYPE, c->seg); - destr_seg = c->seg; - destr_size = c->size; - mseg_destroy(mk->ma, flags, mk, &destr_seg, &destr_size); - ASSERT(destr_seg == c->seg); - ASSERT(destr_size == c->size); + mseg_destroy(mk->ma, flags, mk, c->seg, c->size); mseg_cache_clear_node(c); erts_circleq_push_head(&(mk->cache_free), c); @@ -813,12 +784,14 @@ mseg_alloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, UWord *size_p, INC_CC(ma, alloc); - if (!MSEG_FLG_IS_2POW(flags) && !IS_2POW(size)) + if (!MSEG_FLG_IS_2POW(flags)) size = ERTS_PAGEALIGNED_CEILING(*size_p); else { size = ALIGNED_CEILING(*size_p); - /* Cache optim (if applicable) */ - size = ceil_2pow(size); + if (!IS_2POW(size)) { + /* Cache optim (if applicable) */ + size = ceil_2pow(size); + } } if (opt->cache && mk->cache_size > 0 && (seg = cache_get_segment(mk, &size, flags)) != NULL) @@ -845,8 +818,6 @@ static void mseg_dealloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, UWord size, Uint flags, const ErtsMsegOpt_t *opt) { - void *destr_seg; - UWord destr_size; MemKind* mk = memkind(ma, opt); ERTS_MSEG_DEALLOC_STAT(mk,size); @@ -859,11 +830,7 @@ mseg_dealloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, UWord size, if (erts_mtrace_enabled) erts_mtrace_crr_free(atype, SEGTYPE, seg); - destr_seg = seg; - destr_size = size; - mseg_destroy(ma, flags, mk, &destr_seg, &destr_size); - ASSERT(destr_seg == seg); - ASSERT(destr_size == size); + mseg_destroy(ma, flags, mk, seg, size); done: @@ -896,13 +863,14 @@ mseg_realloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, mk = memkind(ma, opt); new_seg = seg; - if (!MSEG_FLG_IS_2POW(flags)) new_size = ERTS_PAGEALIGNED_CEILING(*new_size_p); else { new_size = ALIGNED_CEILING(*new_size_p); - /* Cache optim (if applicable) */ - new_size = ceil_2pow(new_size); + if (!IS_2POW(new_size)) { + /* Cache optim (if applicable) */ + new_size = ceil_2pow(new_size); + } } if (new_size > old_size) { -- cgit v1.2.3