erts: Allow page aligned erts_munmap()

3 files changed, 67 insertions, 107 deletions

diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c
index eae1a1a410..73874759f5 100644
--- a/erts/emulator/sys/common/erl_mmap.c
+++ b/erts/emulator/sys/common/erl_mmap.c
@@ -1260,30 +1260,53 @@ erts_mmap(Uint32 flags, UWord *sizep)
 
 	desc = lookup_free_seg(&mmap_state.sa.map, asize);
 	if (desc) {
-	    seg = desc->start;
+	    char *start = seg = desc->start;
+	    seg = (char *) ERTS_SUPERALIGNED_CEILING(seg);
 	    end = seg+asize;
-	    if (!mmap_state.reserve_physical(seg, asize))
+	    if (!mmap_state.reserve_physical(start, (UWord) (end - start)))
 		goto supercarrier_reserve_failure;
+	    ERTS_MMAP_SIZE_SC_SA_INC(asize);
 	    if (desc->end == end) {
-		delete_free_seg(&mmap_state.sa.map, desc);
-		free_desc(desc);
+		if (start != seg)
+		    resize_free_seg(&mmap_state.sa.map, desc, start, seg);
+		else {
+		    delete_free_seg(&mmap_state.sa.map, desc);
+		    free_desc(desc);
+		}
 	    }
 	    else {
 		ERTS_MMAP_ASSERT(end < desc->end);
 		resize_free_seg(&mmap_state.sa.map, desc, end, desc->end);
+		if (start != seg) {
+		    UWord ad_sz;
+		    ad_sz = alloc_desc_insert_free_seg(&mmap_state.sua.map,
+						       start, seg);
+		    start += ad_sz;
+		    if (start != seg)
+			mmap_state.unreserve_physical(start, (UWord) (seg - start));
+		}
 	    }
-	    ERTS_MMAP_SIZE_SC_SA_INC(asize);
 	    goto supercarrier_success;
 	}
 
 	if (superaligned) {
+	    seg = (char *) ERTS_SUPERALIGNED_CEILING(mmap_state.sa.top);
 
-	    if (asize <= mmap_state.sua.bot - mmap_state.sa.top) {
-		seg = (void *) mmap_state.sa.top;
-		if (!mmap_state.reserve_physical(seg, asize))
+	    if (asize <= mmap_state.sua.bot - seg) {
+		char *start = mmap_state.sa.top;
+		end = seg + asize;
+		if (!mmap_state.reserve_physical(start, (UWord) (end - start)))
 		    goto supercarrier_reserve_failure;
-		mmap_state.sa.top += asize;
+		mmap_state.sa.top = end;
 		ERTS_MMAP_SIZE_SC_SA_INC(asize);
+		if (start != seg) {
+		    UWord ad_sz;
+		    ad_sz = alloc_desc_insert_free_seg(&mmap_state.sua.map,
+						       start, seg);
+		    start += ad_sz;
+		    if (start != seg)
+			mmap_state.unreserve_physical(start, (UWord) (seg - start));
+		}
 		goto supercarrier_success;
 	    }
 
@@ -1294,7 +1317,7 @@ erts_mmap(Uint32 flags, UWord *sizep)
 
 		seg = (char *) ERTS_SUPERALIGNED_CEILING(org_start);
 		end = seg + asize;
-		if (!mmap_state.reserve_physical(seg, asize))
+		if (!mmap_state.reserve_physical(seg, (UWord) (org_end - seg)))
 		    goto supercarrier_reserve_failure;
 		ERTS_MMAP_SIZE_SC_SUA_INC(asize);
 		if (org_start != seg) {
@@ -1303,12 +1326,17 @@ erts_mmap(Uint32 flags, UWord *sizep)
 		    desc = NULL;
 		}
 		if (end != org_end) {
+		    UWord ad_sz = 0;
 		    ERTS_MMAP_ASSERT(end < org_end);
 		    if (desc)
 			resize_free_seg(&mmap_state.sua.map, desc, end, org_end);
 		    else
-			alloc_desc_insert_free_seg(&mmap_state.sua.map,
-						   end, org_end);
+			ad_sz = alloc_desc_insert_free_seg(&mmap_state.sua.map,
+							   end, org_end);
+		    end += ad_sz;
+		    if (end != org_end)
+			mmap_state.unreserve_physical(end,
+						      (UWord) (org_end - end));
 		}
 		goto supercarrier_success;
 	    }
@@ -1363,8 +1391,7 @@ erts_mmap(Uint32 flags, UWord *sizep)
 supercarrier_success:
 
 #ifdef ERTS_MMAP_DEBUG
-    if ((ERTS_MMAPFLG_SUPERALIGNED & flags)
-	|| ERTS_MMAP_IN_SUPERALIGNED_AREA(seg)) {
+    if (ERTS_MMAPFLG_SUPERALIGNED & flags) {
 	ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(seg));
 	ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(asize));
     }
@@ -1388,10 +1415,8 @@ supercarrier_reserve_failure:
 }
 
 void
-erts_munmap(Uint32 flags, void **ptrp, UWord *sizep)
+erts_munmap(Uint32 flags, void *ptr, UWord size)
 {
-    void *ptr = *ptrp;
-    UWord size = *sizep;
     ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr));
     ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(size));
     if (!ERTS_MMAP_IN_SUPERCARRIER(ptr)) {
@@ -1416,13 +1441,6 @@ erts_munmap(Uint32 flags, void **ptrp, UWord *sizep)
 
 	if (ERTS_MMAP_IN_SUPERALIGNED_AREA(ptr)) {
 
-	    start = (char *) ERTS_SUPERALIGNED_CEILING(start);
-	    end = (char *) ERTS_SUPERALIGNED_FLOOR(end);
-
-	    size = (UWord) (end - start);
-	    *ptrp = start;
-	    *sizep = size;
-
 	    map = &mmap_state.sa.map;
 	    adjacent_free_seg(map, start, end, &prev, &next);
 
@@ -1439,8 +1457,6 @@ erts_munmap(Uint32 flags, void **ptrp, UWord *sizep)
 	    }
 	}
 	else {
-	    ERTS_MMAP_ASSERT(ERTS_MMAP_IN_SUPERUNALIGNED_AREA(ptr));
-
 	    map = &mmap_state.sua.map;
 	    adjacent_free_seg(map, start, end, &prev, &next);
 
@@ -1497,8 +1513,6 @@ static void *
 remap_move(Uint32 flags, void *ptr, UWord old_size, UWord *sizep)
 {
     UWord size = *sizep;
-    UWord um_size = old_size;
-    void *um_ptr = ptr;
     void *new_ptr = erts_mmap(flags, &size);
     if (!new_ptr)
 	return NULL;
@@ -1506,9 +1520,7 @@ remap_move(Uint32 flags, void *ptr, UWord old_size, UWord *sizep)
     if (old_size < size)
 	size = old_size;
     sys_memcpy(new_ptr, ptr, (size_t) size);
-    erts_munmap(flags, &um_ptr, &um_size);
-    ERTS_MMAP_ASSERT(um_ptr == ptr);
-    ERTS_MMAP_ASSERT(um_size == old_size);
+    erts_munmap(flags, ptr, old_size);
     return new_ptr;
 }
 
@@ -1627,28 +1639,18 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep)
 	end = start + old_size;
 	new_end = start+asize;
 
+	ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr));
+	ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(old_size));
+	ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(asize));
+
 	if (asize < old_size) {
 	    UWord unres_sz;
 	    new_ptr = ptr;
 	    if (!ERTS_MMAP_IN_SUPERALIGNED_AREA(ptr)) {
-		ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr));
-		ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(old_size));
-		ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(asize));
 		map = &mmap_state.sua.map;
 		ERTS_MMAP_SIZE_SC_SUA_DEC(old_size - asize);
 	    }
 	    else {
-		ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(ptr));
-		ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(old_size));
-		if (!superaligned) {
-		    /* must be a superaligned size in this area */
-		    asize = ERTS_SUPERALIGNED_CEILING(asize);
-		    ERTS_MMAP_ASSERT(asize <= old_size);
-		    if (asize == old_size)
-			goto supercarrier_resize_success;
-		    new_end = start+asize;
-		}
-		ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(asize));
 		if (end == mmap_state.sa.top) {
 		    mmap_state.sa.top = new_end;
 		    mmap_state.unreserve_physical(((char *) ptr) + asize,
@@ -1696,16 +1698,6 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep)
 	    }
 	}
 	else { /* Superaligned area */
-	    ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(ptr));
-	    ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(old_size));
-
-	    if (!superaligned) {
-		/* must be a superaligned size in this area */
-		asize = ERTS_PAGEALIGNED_CEILING(asize);
-		new_end = start+asize;
-	    }
-
-	    ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(asize));
 
 	    if (end == mmap_state.sa.top) {
 		if (new_end <= mmap_state.sua.bot) {
diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h
index b75200f4e9..6cb51fb0b4 100644
--- a/erts/emulator/sys/common/erl_mmap.h
+++ b/erts/emulator/sys/common/erl_mmap.h
@@ -53,7 +53,7 @@ typedef struct {
     {{NULL, NULL}, {NULL, NULL}, 0, 1, (1 << 16), 1}
 
 void *erts_mmap(Uint32 flags, UWord *sizep);
-void erts_munmap(Uint32 flags, void **ptrp, UWord *sizep);
+void erts_munmap(Uint32 flags, void *ptr, UWord size);
 void *erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep);
 int erts_mmap_in_supercarrier(void *ptr);
 void erts_mmap_init(ErtsMMapInit*);
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index 09035ca73e..2474015bcb 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -330,7 +330,7 @@ mseg_create(ErtsMsegAllctr_t *ma, Uint flags, MemKind* mk, UWord *sizep)
 }
 
 static ERTS_INLINE void
-mseg_destroy(ErtsMsegAllctr_t *ma, Uint flags, MemKind* mk, void **seg_pp, UWord *size_p) {
+mseg_destroy(ErtsMsegAllctr_t *ma, Uint flags, MemKind* mk, void *seg_p, UWord size) {
     
     Uint32 mmap_flags = 0;
 #if HALFWORD_HEAP
@@ -341,11 +341,11 @@ mseg_destroy(ErtsMsegAllctr_t *ma, Uint flags, MemKind* mk, void **seg_pp, UWord
     if (MSEG_FLG_IS_2POW(flags))
 	 mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED;
 
-    erts_munmap(mmap_flags, seg_pp, size_p);
+    erts_munmap(mmap_flags, seg_p, size);
 #ifdef ERTS_PRINT_ERTS_MMAP
     erts_fprintf(stderr, "erts_munmap(%s, %p, %bpu);\n",
 		 (mmap_flags & ERTS_MMAPFLG_SUPERALIGNED) ? "sa" : "sua",
-		 *seg_pp, *size_p);
+		 seg_p, *size);
 #endif
     INC_CC(ma, destroy);
 
@@ -446,19 +446,13 @@ static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size, Ui
 
 	return 1;
     } else if (!MSEG_FLG_IS_2POW(flags) && !erts_circleq_is_empty(&(mk->cache_unpowered_node))) {
-	void *destr_seg;
-	UWord destr_size;
 	/* No free slots.
 	 * Evict oldest slot from unpowered cache so we can cache an unpowered (sbc) segment  */
 
 	c = erts_circleq_tail(&(mk->cache_unpowered_node));
 	erts_circleq_remove(c);
 
-	destr_seg = c->seg;
-	destr_size = c->size;
-	mseg_destroy(mk->ma, ERTS_MSEG_FLG_NONE, mk, &destr_seg, &destr_size);
-	ASSERT(destr_seg == c->seg);
-	ASSERT(destr_size == c->size);
+	mseg_destroy(mk->ma, ERTS_MSEG_FLG_NONE, mk, c->seg, c->size);
 	mseg_cache_clear_node(c);
 
 	c->seg  = seg;
@@ -478,19 +472,13 @@ static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size, Ui
 	int i;
 
 	for( i = 0; i < CACHE_AREAS; i++) {
-	    void *destr_seg;
-	    UWord destr_size;
 	    if (erts_circleq_is_empty(&(mk->cache_powered_node[i])))
 		continue;
 
 	    c = erts_circleq_tail(&(mk->cache_powered_node[i]));
 	    erts_circleq_remove(c);
 
-	    destr_seg = seg;
-	    destr_size = c->size;
-	    mseg_destroy(mk->ma, ERTS_MSEG_FLG_2POW, mk, &destr_seg, &destr_size);
-	    ASSERT(destr_seg == c->seg);
-	    ASSERT(destr_size == c->size);
+	    mseg_destroy(mk->ma, ERTS_MSEG_FLG_2POW, mk, c->seg, c->size);
 
 	    mseg_cache_clear_node(c);
 
@@ -544,13 +532,8 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag
 
 	    ASSERT(!(mk->cache_size < 0));
 
-	    if (csize != size) {
-		char *destr_seg = seg + size;
-		UWord destr_size = csize - size;
-		mseg_destroy(mk->ma, ERTS_MSEG_FLG_2POW, mk, (void**)&destr_seg, &destr_size);
-		*size_p = (UWord) (destr_seg - seg);
-		ASSERT(seg + csize == destr_seg + destr_size);
-	    }
+	    if (csize != size)
+		mseg_destroy(mk->ma, ERTS_MSEG_FLG_2POW, mk, seg + size, csize - size);
 
 	    return seg;
 	}
@@ -625,8 +608,6 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag
  */
 
 static ERTS_INLINE Uint mseg_drop_one_memkind_cache_size(MemKind *mk, Uint flags, cache_t *head) {
-    void *destr_seg;
-    UWord destr_size;
     cache_t *c = NULL;
 
     c = erts_circleq_tail(head);
@@ -635,11 +616,7 @@ static ERTS_INLINE Uint mseg_drop_one_memkind_cache_size(MemKind *mk, Uint flags
     if (erts_mtrace_enabled)
 	erts_mtrace_crr_free(SEGTYPE, SEGTYPE, c->seg);
 
-    destr_seg = c->seg;
-    destr_size = c->size;
-    mseg_destroy(mk->ma, flags, mk, &destr_seg, &destr_size);
-    ASSERT(destr_seg == c->seg);
-    ASSERT(destr_size == c->size);
+    mseg_destroy(mk->ma, flags, mk, c->seg, c->size);
     mseg_cache_clear_node(c);
     erts_circleq_push_head(&(mk->cache_free), c);
 
@@ -655,8 +632,6 @@ static ERTS_INLINE Uint mseg_drop_memkind_cache_size(MemKind *mk, Uint flags, ca
     cache_t *c = NULL;
 
     while (!erts_circleq_is_empty(head)) {
-	void *destr_seg;
-	UWord destr_size;
 
 	c = erts_circleq_tail(head);
 	erts_circleq_remove(c);
@@ -664,11 +639,7 @@ static ERTS_INLINE Uint mseg_drop_memkind_cache_size(MemKind *mk, Uint flags, ca
 	if (erts_mtrace_enabled)
 	    erts_mtrace_crr_free(SEGTYPE, SEGTYPE, c->seg);
 
-	destr_seg = c->seg;
-	destr_size = c->size;
-	mseg_destroy(mk->ma, flags, mk, &destr_seg, &destr_size);
-	ASSERT(destr_seg == c->seg);
-	ASSERT(destr_size == c->size);
+	mseg_destroy(mk->ma, flags, mk, c->seg, c->size);
 
 	mseg_cache_clear_node(c);
 	erts_circleq_push_head(&(mk->cache_free), c);
@@ -813,12 +784,14 @@ mseg_alloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, UWord *size_p,
 
     INC_CC(ma, alloc);
 
-    if (!MSEG_FLG_IS_2POW(flags) && !IS_2POW(size))
+    if (!MSEG_FLG_IS_2POW(flags))
 	size = ERTS_PAGEALIGNED_CEILING(*size_p);
     else {
 	size = ALIGNED_CEILING(*size_p);
-	/* Cache optim (if applicable) */
-	size = ceil_2pow(size);
+	if (!IS_2POW(size)) {
+	    /* Cache optim (if applicable) */
+	    size = ceil_2pow(size);
+	}
     }
 
     if (opt->cache && mk->cache_size > 0 && (seg = cache_get_segment(mk, &size, flags)) != NULL)
@@ -845,8 +818,6 @@ static void
 mseg_dealloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, UWord size,
 	     Uint flags, const ErtsMsegOpt_t *opt)
 {
-    void *destr_seg;
-    UWord destr_size;
     MemKind* mk = memkind(ma, opt);
 
     ERTS_MSEG_DEALLOC_STAT(mk,size);
@@ -859,11 +830,7 @@ mseg_dealloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, UWord size,
     if (erts_mtrace_enabled)
 	erts_mtrace_crr_free(atype, SEGTYPE, seg);
 
-    destr_seg = seg;
-    destr_size = size;
-    mseg_destroy(ma, flags, mk, &destr_seg, &destr_size);
-    ASSERT(destr_seg == seg);
-    ASSERT(destr_size == size);
+    mseg_destroy(ma, flags, mk, seg, size);
 
 done:
 
@@ -896,13 +863,14 @@ mseg_realloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg,
     mk = memkind(ma, opt);
     new_seg = seg;
 
-
     if (!MSEG_FLG_IS_2POW(flags))
 	new_size = ERTS_PAGEALIGNED_CEILING(*new_size_p);
     else {
 	new_size = ALIGNED_CEILING(*new_size_p);
-	/* Cache optim (if applicable) */
-	new_size = ceil_2pow(new_size);
+	if (!IS_2POW(new_size)) {
+	    /* Cache optim (if applicable) */
+	    new_size = ceil_2pow(new_size);
+	}
     }
 
     if (new_size > old_size) {