erts: Make ErtsMemMapper able to map memory with PROT_EXEC

to prepare for hipe native code allocation.

3 files changed, 44 insertions, 30 deletions

diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c
index 756d038da4..f5f40b35a8 100644
--- a/erts/emulator/sys/common/erl_mmap.c
+++ b/erts/emulator/sys/common/erl_mmap.c
@@ -294,10 +294,11 @@ typedef struct {
 }ErtsFreeSegMap;
 
 struct ErtsMemMapper_ {
-    int (*reserve_physical)(char *, UWord);
+    int (*reserve_physical)(char *, UWord, int exec);
     void (*unreserve_physical)(char *, UWord);
     int supercarrier;
     int no_os_mmap;
+    int executable;   /* is client a native code allocator? */
     /*
      * Super unaligned area is located above super aligned
      * area. That is, `sa.bot` is beginning of the super
@@ -1232,6 +1233,7 @@ Eterm build_free_seg_list(Process* p, ErtsFreeSegMap* map)
 
 #if HAVE_MMAP
 #  define ERTS_MMAP_PROT		(PROT_READ|PROT_WRITE)
+#  define ERTS_MMAP_PROT_EXEC		(PROT_READ|PROT_WRITE|PROT_EXEC)
 #  if defined(MAP_ANONYMOUS)
 #    define ERTS_MMAP_FLAGS		(MAP_ANON|MAP_PRIVATE)
 #    define ERTS_MMAP_FD		(-1)
@@ -1245,24 +1247,26 @@ Eterm build_free_seg_list(Process* p, ErtsFreeSegMap* map)
 #endif
 
 static ERTS_INLINE void *
-os_mmap(void *hint_ptr, UWord size, int try_superalign)
+os_mmap(void *hint_ptr, UWord size, int try_superalign, int executable)
 {
 #if HAVE_MMAP
+    const int prot = executable ? ERTS_MMAP_PROT_EXEC : ERTS_MMAP_PROT;
     void *res;
 #ifdef MAP_ALIGN
     if (try_superalign)
-	res = mmap((void *) ERTS_SUPERALIGNED_SIZE, size, ERTS_MMAP_PROT,
+	res = mmap((void *) ERTS_SUPERALIGNED_SIZE, size, prot,
 		   ERTS_MMAP_FLAGS|MAP_ALIGN, ERTS_MMAP_FD, 0);
     else
 #endif
-	res = mmap((void *) hint_ptr, size, ERTS_MMAP_PROT,
+	res = mmap((void *) hint_ptr, size, prot,
 		   ERTS_MMAP_FLAGS, ERTS_MMAP_FD, 0);
     if (res == MAP_FAILED)
 	return NULL;
     return res;
 #elif HAVE_VIRTUALALLOC
+    const DWORD prot = executable ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE;
     return (void *) VirtualAlloc(NULL, (SIZE_T) size,
-				 MEM_COMMIT|MEM_RESERVE, PAGE_READWRITE);
+				 MEM_COMMIT|MEM_RESERVE, prot);
 #else
 #  error "missing mmap() or similar"
 #endif
@@ -1319,6 +1323,7 @@ os_mremap(void *ptr, UWord old_size, UWord new_size, int try_superalign)
 #if HAVE_MMAP
 
 #define ERTS_MMAP_RESERVE_PROT		(ERTS_MMAP_PROT)
+#define ERTS_MMAP_RESERVE_PROT_EXEC	(ERTS_MMAP_PROT_EXEC)
 #define ERTS_MMAP_RESERVE_FLAGS		(ERTS_MMAP_FLAGS|MAP_FIXED)
 #define ERTS_MMAP_UNRESERVE_PROT	(PROT_NONE)
 #define ERTS_MMAP_UNRESERVE_FLAGS	(ERTS_MMAP_FLAGS|MAP_NORESERVE|MAP_FIXED)
@@ -1326,9 +1331,10 @@ os_mremap(void *ptr, UWord old_size, UWord new_size, int try_superalign)
 #define ERTS_MMAP_VIRTUAL_FLAGS		(ERTS_MMAP_FLAGS|MAP_NORESERVE)
 
 static int
-os_reserve_physical(char *ptr, UWord size)
+os_reserve_physical(char *ptr, UWord size, int exec)
 {
-    void *res = mmap((void *) ptr, (size_t) size, ERTS_MMAP_RESERVE_PROT,
+    const int prot = exec ? ERTS_MMAP_RESERVE_PROT_EXEC : ERTS_MMAP_RESERVE_PROT;
+    void *res = mmap((void *) ptr, (size_t) size, prot,
 		     ERTS_MMAP_RESERVE_FLAGS, ERTS_MMAP_FD, 0);
     if (res == (void *) MAP_FAILED)
 	return 0;
@@ -1361,7 +1367,7 @@ os_mmap_virtual(char *ptr, UWord size)
 
 #endif /* ERTS_HAVE_OS_MMAP */
 
-static int reserve_noop(char *ptr, UWord size)
+static int reserve_noop(char *ptr, UWord size, int exec)
 {
 #ifdef ERTS_MMAP_DEBUG_FILL_AREAS
     Uint32 *uip, *end = (Uint32 *) (ptr + size);
@@ -1404,7 +1410,7 @@ alloc_desc_insert_free_seg(ErtsMemMapper* mm,
 
 #if ERTS_HAVE_OS_MMAP
     if (!mm->no_os_mmap) {
-        ptr = os_mmap(mm->desc.new_area_hint, ERTS_PAGEALIGNED_SIZE, 0);
+        ptr = os_mmap(mm->desc.new_area_hint, ERTS_PAGEALIGNED_SIZE, 0, 0);
         if (ptr) {
             mm->desc.new_area_hint = ptr+ERTS_PAGEALIGNED_SIZE;
             ERTS_MMAP_SIZE_OS_INC(ERTS_PAGEALIGNED_SIZE);
@@ -1423,7 +1429,7 @@ alloc_desc_insert_free_seg(ErtsMemMapper* mm,
     da_map = &mm->sua.map;
     desc = lookup_free_seg(da_map, ERTS_PAGEALIGNED_SIZE);
     if (desc) {
-	if (mm->reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE))
+	if (mm->reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE, 0))
 	    ERTS_MMAP_SIZE_SC_SUA_INC(ERTS_PAGEALIGNED_SIZE);
 	else
 	    desc = NULL;
@@ -1433,7 +1439,7 @@ alloc_desc_insert_free_seg(ErtsMemMapper* mm,
 	da_map = &mm->sa.map;
 	desc = lookup_free_seg(da_map, ERTS_PAGEALIGNED_SIZE);
 	if (desc) {
-	    if (mm->reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE))
+	    if (mm->reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE, 0))
 		ERTS_MMAP_SIZE_SC_SA_INC(ERTS_PAGEALIGNED_SIZE);
 	    else
 		desc = NULL;
@@ -1494,7 +1500,7 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep)
 	    if (desc) {
 		seg = desc->start;
 		end = seg+asize;
-		if (!mm->reserve_physical(seg, asize))
+		if (!mm->reserve_physical(seg, asize, mm->executable))
 		    goto supercarrier_reserve_failure;
 		if (desc->end == end) {
 		    delete_free_seg(&mm->sua.map, desc);
@@ -1509,8 +1515,8 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep)
 	    }
 
 	    if (asize <= mm->sua.bot - mm->sa.top) {
-		if (!mm->reserve_physical(mm->sua.bot - asize,
-						 asize))
+		if (!mm->reserve_physical(mm->sua.bot - asize, asize,
+                                          mm->executable))
 		    goto supercarrier_reserve_failure;
 		mm->sua.bot -= asize;
 		seg = mm->sua.bot;
@@ -1526,7 +1532,8 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep)
 	    char *start = seg = desc->start;
 	    seg = (char *) ERTS_SUPERALIGNED_CEILING(seg);
 	    end = seg+asize;
-	    if (!mm->reserve_physical(start, (UWord) (end - start)))
+	    if (!mm->reserve_physical(start, (UWord) (end - start),
+                                      mm->executable))
 		goto supercarrier_reserve_failure;
 	    ERTS_MMAP_SIZE_SC_SA_INC(asize);
 	    if (desc->end == end) {
@@ -1558,7 +1565,8 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep)
 
 	    if (asize + (seg - start) <= mm->sua.bot - start) {
 		end = seg + asize;
-		if (!mm->reserve_physical(start, (UWord) (end - start)))
+		if (!mm->reserve_physical(start, (UWord) (end - start),
+                                          mm->executable))
 		    goto supercarrier_reserve_failure;
 		mm->sa.top = end;
 		ERTS_MMAP_SIZE_SC_SA_INC(asize);
@@ -1580,7 +1588,8 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep)
 
 		seg = (char *) ERTS_SUPERALIGNED_CEILING(org_start);
 		end = seg + asize;
-		if (!mm->reserve_physical(seg, (UWord) (org_end - seg)))
+		if (!mm->reserve_physical(seg, (UWord) (org_end - seg),
+                                          mm->executable))
 		    goto supercarrier_reserve_failure;
 		ERTS_MMAP_SIZE_SC_SUA_INC(asize);
 		if (org_start != seg) {
@@ -1613,13 +1622,13 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep)
     /* Map using OS primitives */
     if (!(ERTS_MMAPFLG_SUPERCARRIER_ONLY & flags) && !mm->no_os_mmap) {
 	if (!(ERTS_MMAPFLG_SUPERALIGNED & flags)) {
-	    seg = os_mmap(NULL, asize, 0);
+	    seg = os_mmap(NULL, asize, 0, mm->executable);
 	    if (!seg)
 		goto failure;
 	}
 	else {
 	    asize = ERTS_SUPERALIGNED_CEILING(*sizep);
-	    seg = os_mmap(NULL, asize, 1);
+	    seg = os_mmap(NULL, asize, 1, mm->executable);
 	    if (!seg)
 		goto failure;
 
@@ -1629,7 +1638,8 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep)
 
 		os_munmap(seg, asize);
 
-		ptr = os_mmap(NULL, asize + ERTS_SUPERALIGNED_SIZE, 1);
+		ptr = os_mmap(NULL, asize + ERTS_SUPERALIGNED_SIZE, 1,
+                              mm->executable);
 		if (!ptr)
 		    goto failure;
 
@@ -1964,7 +1974,8 @@ erts_mremap(ErtsMemMapper* mm,
 
 	    if (next && new_end <= next->end) {
 		if (!mm->reserve_physical(((char *) ptr) + old_size,
-						 asize - old_size))
+                                          asize - old_size,
+                                          mm->executable))
 		    goto supercarrier_reserve_failure;
 		if (new_end < next->end)
 		    resize_free_seg(&mm->sua.map, next, new_end, next->end);
@@ -1982,7 +1993,8 @@ erts_mremap(ErtsMemMapper* mm,
 	    if (end == mm->sa.top) {
 		if (new_end <= mm->sua.bot) {
 		    if (!mm->reserve_physical(((char *) ptr) + old_size,
-						     asize - old_size))
+                                              asize - old_size,
+                                              mm->executable))
 			goto supercarrier_reserve_failure;
 		    mm->sa.top = new_end;
 		    new_ptr = ptr;
@@ -1994,7 +2006,8 @@ erts_mremap(ErtsMemMapper* mm,
 		adjacent_free_seg(&mm->sa.map, start, end, &prev, &next);
 		if (next && new_end <= next->end) {
 		    if (!mm->reserve_physical(((char *) ptr) + old_size,
-						     asize - old_size))
+                                              asize - old_size,
+                                              mm->executable))
 			goto supercarrier_reserve_failure;
 		    if (new_end < next->end)
 			resize_free_seg(&mm->sa.map, next, new_end, next->end);
@@ -2112,7 +2125,7 @@ static void hard_dbg_mseg_init(void);
 #endif
 
 void
-erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init)
+erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable)
 {
     static int is_first_call = 1;
     int virtual_map = 0;
@@ -2144,6 +2157,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init)
     mm->supercarrier = 0;
     mm->reserve_physical = reserve_noop;
     mm->unreserve_physical = unreserve_noop;
+    mm->executable = executable;
 
 #if HAVE_MMAP && !defined(MAP_ANON)
     mm->mmap_fd = open("/dev/zero", O_RDWR);
@@ -2200,7 +2214,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init)
 	     * The whole supercarrier will by physically
 	     * reserved all the time.
 	     */
-	    start = os_mmap(NULL, sz, 1);
+	    start = os_mmap(NULL, sz, 1, executable);
 	}
 	if (!start)
 	    erts_exit(1,
@@ -2274,7 +2288,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init)
 	    mm->sua.top -= ERTS_PAGEALIGNED_SIZE;
 	    mm->size.supercarrier.used.total += ERTS_PAGEALIGNED_SIZE;
 #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION
-	    if (!virtual_map || os_reserve_physical(mm->sua.top, ERTS_PAGEALIGNED_SIZE))
+	    if (!virtual_map || os_reserve_physical(mm->sua.top, ERTS_PAGEALIGNED_SIZE, 0))
 #endif
 		add_free_desc_area(mm, mm->sua.top, end);
             mm->desc.reserved += (end - mm->sua.top) / sizeof(ErtsFreeSegDesc);
@@ -2287,7 +2301,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init)
 	 * will be used for free segment descritors.
 	 */
 #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION
-	if (virtual_map && !os_reserve_physical(start, mm->sa.bot - start))
+	if (virtual_map && !os_reserve_physical(start, mm->sa.bot - start, 0))
 	    erts_exit(1, "erts_mmap: Failed to reserve physical memory for descriptors\n");
 #endif
 	mm->desc.unused_start = start;
diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h
index 7c5ddbec0b..30548bd0b0 100644
--- a/erts/emulator/sys/common/erl_mmap.h
+++ b/erts/emulator/sys/common/erl_mmap.h
@@ -61,7 +61,7 @@ void *erts_mmap(ErtsMemMapper*, Uint32 flags, UWord *sizep);
 void erts_munmap(ErtsMemMapper*, Uint32 flags, void *ptr, UWord size);
 void *erts_mremap(ErtsMemMapper*, Uint32 flags, void *ptr, UWord old_size, UWord *sizep);
 int erts_mmap_in_supercarrier(ErtsMemMapper*, void *ptr);
-void erts_mmap_init(ErtsMemMapper*, ErtsMMapInit*);
+void erts_mmap_init(ErtsMemMapper*, ErtsMMapInit*, int executable);
 struct erts_mmap_info_struct
 {
     UWord sizes[6];
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index 286130bece..f0fbb682b0 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -1402,9 +1402,9 @@ erts_mseg_init(ErtsMsegInit_t *init)
 
     erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms");
 
-    erts_mmap_init(&erts_dflt_mmapper, &init->dflt_mmap);
+    erts_mmap_init(&erts_dflt_mmapper, &init->dflt_mmap, 0);
 #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
-    erts_mmap_init(&erts_literal_mmapper, &init->literal_mmap);
+    erts_mmap_init(&erts_literal_mmapper, &init->literal_mmap, 0);
 #endif
 
     if (!IS_2POW(GET_PAGE_SIZE))