erts: Improve erts_mmap out of free descriptor management

1 files changed, 60 insertions, 40 deletions

diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c
index 723efa772f..0ac08a0004 100644
--- a/erts/emulator/sys/common/erl_mmap.c
+++ b/erts/emulator/sys/common/erl_mmap.c
@@ -222,6 +222,7 @@ static struct {
 	mmap_state.size.os.used -= (SZ);					\
     } while (0)
 
+
 static void
 add_free_desc_area(char *start, char *end)
 {
@@ -1164,6 +1165,39 @@ static void unreserve_noop(char *ptr, UWord size)
 #endif
 }
 
+static ERTS_INLINE UWord
+alloc_desc_insert_free_seg(ErtsFreeSegMap *map, char* start, char* end)
+{
+    UWord ad_sz;
+    char *new_end;
+    ErtsFreeSegDesc *desc = alloc_desc();
+    if (desc) {
+	insert_free_seg(map, desc, start, end);
+	return 0;
+    }
+
+    /* Use part of the free segment for descriptors */
+
+    if (map == &mmap_state.sa.map) {
+	ERTS_MMAP_SIZE_SC_SA_INC(ERTS_SUPERALIGNED_SIZE);
+	ad_sz = ERTS_SUPERALIGNED_SIZE;
+    }
+    else {
+	ERTS_MMAP_SIZE_SC_SUA_INC(ERTS_PAGEALIGNED_SIZE);
+	ad_sz = ERTS_PAGEALIGNED_SIZE;
+    }
+
+    new_end = end - ad_sz;
+    ERTS_MMAP_ASSERT(start <= new_end);
+    if (start != new_end) {
+	desc = alloc_desc();
+	ERTS_MMAP_ASSERT(desc);
+	insert_free_seg(map, desc, start, new_end);
+    }
+
+    return ad_sz;
+}
+
 void *
 erts_mmap(Uint32 flags, UWord *sizep)
 {
@@ -1248,6 +1282,7 @@ erts_mmap(Uint32 flags, UWord *sizep)
 		end = seg + asize;
 		if (!mmap_state.reserve_physical(seg, asize))
 		    goto supercarrier_reserve_failure;
+		ERTS_MMAP_SIZE_SC_SUA_INC(asize);
 		if (org_start != seg) {
 		    ERTS_MMAP_ASSERT(org_start < seg);
 		    resize_free_seg(&mmap_state.sua.map, desc, org_start, seg);
@@ -1257,15 +1292,10 @@ erts_mmap(Uint32 flags, UWord *sizep)
 		    ERTS_MMAP_ASSERT(end < org_end);
 		    if (desc)
 			resize_free_seg(&mmap_state.sua.map, desc, end, org_end);
-		    else {
-			desc = alloc_desc();
-			if (!desc)
-			    add_free_desc_area(end, org_end);
-			else
-			    insert_free_seg(&mmap_state.sua.map, desc, end, org_end);
-		    }
+		    else
+			alloc_desc_insert_free_seg(&mmap_state.sua.map,
+						   end, org_end);
 		}
-		ERTS_MMAP_SIZE_SC_SA_INC(asize);
 		goto supercarrier_success;
 	    }
 	}
@@ -1361,6 +1391,7 @@ erts_munmap(Uint32 flags, void **ptrp, UWord *sizep)
 	char *start, *end;
 	ErtsFreeSegMap *map;
 	ErtsFreeSegDesc *prev, *next, *desc;
+	UWord ad_sz = 0;
 
 	ERTS_MMAP_ASSERT(mmap_state.supercarrier);
 
@@ -1432,23 +1463,19 @@ erts_munmap(Uint32 flags, void **ptrp, UWord *sizep)
 
 	if (desc)
 	    resize_free_seg(map, desc, start, end);
-	else {
-	    desc = alloc_desc();
-	    if (desc)
-		insert_free_seg(map, desc, start, end);
-	    else {
-		if (map == &mmap_state.sa.map)
-		    ERTS_MMAP_SIZE_SC_SA_INC(size);
-		else
-		    ERTS_MMAP_SIZE_SC_SUA_INC(size);
-		add_free_desc_area(start, end);
-	    }
-	}
+	else
+	    ad_sz = alloc_desc_insert_free_seg(map, start, end);
+	
+    supercarrier_success: {
+	    UWord unres_sz;
 
-    supercarrier_success:
-	erts_smp_mtx_unlock(&mmap_state.mtx);
+	    erts_smp_mtx_unlock(&mmap_state.mtx);
 
-	mmap_state.unreserve_physical((char *) ptr, size);
+	    ERTS_MMAP_ASSERT(size >= ad_sz);
+	    unres_sz = size - ad_sz;
+	    if (unres_sz)
+		mmap_state.unreserve_physical((char *) ptr, unres_sz);
+	}
     }
 }
 
@@ -1548,7 +1575,8 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep)
     else { /* In super carrier */
 	char *start, *end, *new_end;
 	ErtsFreeSegMap *map;
-	ErtsFreeSegDesc *prev, *next, *desc;
+	ErtsFreeSegDesc *prev, *next;
+	UWord ad_sz = 0;
 
 	ERTS_MMAP_ASSERT(mmap_state.supercarrier);
 
@@ -1586,6 +1614,7 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep)
 	new_end = start+asize;
 
 	if (asize < old_size) {
+	    UWord unres_sz;
 	    new_ptr = ptr;
 	    if (!ERTS_MMAP_IN_SUPERALIGNED_AREA(ptr)) {
 		ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr));
@@ -1620,21 +1649,13 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep)
 
 	    if (next)
 		resize_free_seg(map, next, new_end, next->end);
-	    else {
-		desc = alloc_desc();
-		if (desc)
-		    insert_free_seg(map, desc, new_end, end);
-		else {
-		    if (map == &mmap_state.sa.map)
-			ERTS_MMAP_SIZE_SC_SA_INC(old_size - asize);
-		    else
-			ERTS_MMAP_SIZE_SC_SUA_INC(old_size - asize);
-		    add_free_desc_area(new_end, end);
-		    goto supercarrier_resize_success;
-		}
-	    }
-	    mmap_state.unreserve_physical(((char *) ptr) + asize,
-					  old_size - asize);
+	    else
+		ad_sz = alloc_desc_insert_free_seg(map, new_end, end);
+	    ERTS_MMAP_ASSERT(old_size - asize >= ad_sz);
+	    unres_sz = old_size - asize - ad_sz;
+	    if (unres_sz)
+		mmap_state.unreserve_physical(((char *) ptr) + asize,
+					      unres_sz);
 	    goto supercarrier_resize_success;
 	}
 
@@ -1921,7 +1942,6 @@ erts_mmap_init(ErtsMMapInit *init)
 #endif
 }
 
-
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
  * Debug functions                                                           *
 \*                                                                           */