hipe: improve error handling at code allocation failure

6 files changed, 48 insertions, 23 deletions

diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c
index 627ba7603d..63646825b2 100644
--- a/erts/emulator/hipe/hipe_amd64.c
+++ b/erts/emulator/hipe/hipe_amd64.c
@@ -125,7 +125,7 @@ static void atexit_alloc_code_stats(void)
 #define MAP_ANONYMOUS MAP_ANON
 #endif
 
-static void morecore(unsigned int alloc_bytes)
+static int morecore(unsigned int alloc_bytes)
 {
     unsigned int map_bytes;
     char *map_hint, *map_start;
@@ -174,10 +174,9 @@ static void morecore(unsigned int alloc_bytes)
 	abort();
     }
 #endif
-    if (map_start == MAP_FAILED) {
-	perror("mmap");
-	abort();
-    }
+    if (map_start == MAP_FAILED)
+	return -1;
+
     ALLOC_CODE_STATS(total_mapped += map_bytes);
 
     /* Merge adjacent mappings, so the trailing portion of the previous
@@ -197,6 +196,8 @@ static void morecore(unsigned int alloc_bytes)
     }
 
     ALLOC_CODE_STATS(atexit_alloc_code_stats());
+
+    return 0;
 }
 
 static void *alloc_code(unsigned int alloc_bytes)
@@ -206,8 +207,8 @@ static void *alloc_code(unsigned int alloc_bytes)
     /* Align function entries. */
     alloc_bytes = (alloc_bytes + 3) & ~3;
 
-    if (code_bytes < alloc_bytes)
-	morecore(alloc_bytes);
+    if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0)
+	return NULL;
     ALLOC_CODE_STATS(++nr_allocs);
     ALLOC_CODE_STATS(total_alloc += alloc_bytes);
     res = code_next;
@@ -252,6 +253,8 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
       ((P_CALLEE_EXP + 4) >= 128 ? 3 : 0) +
       (P_ARITY >= 128 ? 3 : 0);
     codep = code = alloc_code(codeSize);
+    if (!code)
+	return NULL;
 
     /* movl $callee_exp, P_CALLEE_EXP(%ebp); 3 or 6 bytes, plus 4 */
     codep[0] = 0xc7;
diff --git a/erts/emulator/hipe/hipe_arm.c b/erts/emulator/hipe/hipe_arm.c
index 165eb543c8..c0c6305c68 100644
--- a/erts/emulator/hipe/hipe_arm.c
+++ b/erts/emulator/hipe/hipe_arm.c
@@ -283,6 +283,8 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
      */
 
     code = alloc_stub(4, &tramp_callemu);
+    if (!code)
+	return NULL;
     callemu_offset = ((int)&nbif_callemu - ((int)&code[2] + 8)) >> 2;
     if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF)) {
 	callemu_offset = ((int)tramp_callemu - ((int)&code[2] + 8)) >> 2;
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index dc45e60411..9e5830f345 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -398,8 +398,16 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2)
     ASSERT(bitsize == 0);
     trampolines = NIL;
     address = hipe_alloc_code(nrbytes, BIF_ARG_2, &trampolines, BIF_P);
-    if (!address)
-	BIF_ERROR(BIF_P, BADARG);
+    if (!address) {
+	Uint nrcallees;
+
+	if (is_tuple(BIF_ARG_2))
+	    nrcallees = arityval(tuple_val(BIF_ARG_2)[0]);
+	else
+	    nrcallees = 0;
+	erl_exit(1, "%s: failed to allocate %lu bytes and %lu trampolines\r\n",
+		 __func__, (unsigned long)nrbytes, (unsigned long)nrcallees);
+    }
     memcpy(address, bytes, nrbytes);
     hipe_flush_icache_range(address, nrbytes);
     hp = HAlloc(BIF_P, 3);
@@ -1274,6 +1282,8 @@ static void *hipe_make_stub(Eterm m, Eterm f, unsigned int arity, int is_remote)
 
     export_entry = erts_export_get_or_make_stub(m, f, arity);
     StubAddress = hipe_make_native_stub(export_entry, arity);
+    if (!StubAddress)
+	erl_exit(1, "hipe_make_stub: code allocation failed\r\n");
     return StubAddress;
 }
 
diff --git a/erts/emulator/hipe/hipe_ppc.c b/erts/emulator/hipe/hipe_ppc.c
index 4dc26cdbc8..1eaa9f6855 100644
--- a/erts/emulator/hipe/hipe_ppc.c
+++ b/erts/emulator/hipe/hipe_ppc.c
@@ -293,6 +293,8 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
 	abort();
 
     code = alloc_stub(7);
+    if (!code)
+	return NULL;
 
     /* addis r12,0,callee_exp@highest */
     code[0] = 0x3d800000 | (((unsigned long)callee_exp >> 48) & 0xffff);
@@ -381,6 +383,8 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
 	abort();
 
     code = alloc_stub(4);
+    if (!code)
+	return NULL;
 
     /* addi r12,0,callee_exp@l */
     code[0] = 0x39800000 | ((unsigned long)callee_exp & 0xFFFF);
diff --git a/erts/emulator/hipe/hipe_sparc.c b/erts/emulator/hipe/hipe_sparc.c
index 2052aa8498..fea3b623a9 100644
--- a/erts/emulator/hipe/hipe_sparc.c
+++ b/erts/emulator/hipe/hipe_sparc.c
@@ -130,7 +130,7 @@ static void atexit_alloc_code_stats(void)
 #define ALLOC_CODE_STATS(X)	do{}while(0)
 #endif
 
-static void morecore(unsigned int alloc_bytes)
+static int morecore(unsigned int alloc_bytes)
 {
     unsigned int map_bytes;
     char *map_hint, *map_start;
@@ -158,10 +158,9 @@ static void morecore(unsigned int alloc_bytes)
 #endif
 		     ,
 		     -1, 0);
-    if (map_start == MAP_FAILED) {
-	perror("mmap");
-	abort();
-    }
+    if (map_start == MAP_FAILED)
+	return -1;
+
     ALLOC_CODE_STATS(total_mapped += map_bytes);
 
     /* Merge adjacent mappings, so the trailing portion of the previous
@@ -177,6 +176,8 @@ static void morecore(unsigned int alloc_bytes)
     }
 
     ALLOC_CODE_STATS(atexit_alloc_code_stats());
+
+    return 0;
 }
 
 static void *alloc_code(unsigned int alloc_bytes)
@@ -186,8 +187,8 @@ static void *alloc_code(unsigned int alloc_bytes)
     /* Align function entries. */
     alloc_bytes = (alloc_bytes + 3) & ~3;
 
-    if (code_bytes < alloc_bytes)
-	morecore(alloc_bytes);
+    if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0)
+	return NULL;
     ALLOC_CODE_STATS(++nr_allocs);
     ALLOC_CODE_STATS(total_alloc += alloc_bytes);
     res = code_next;
@@ -211,6 +212,8 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
     int i;
 
     code = alloc_code(5*sizeof(int));
+    if (!code)
+	return NULL;
 
     /* sethi %hi(Address), %i4 */
     code[0] = 0x39000000 | (((unsigned int)callee_exp >> 10) & 0x3FFFFF);
diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c
index 314f6b597c..998905ea63 100644
--- a/erts/emulator/hipe/hipe_x86.c
+++ b/erts/emulator/hipe/hipe_x86.c
@@ -108,7 +108,7 @@ static void atexit_alloc_code_stats(void)
 #define MAP_ANONYMOUS MAP_ANON
 #endif
 
-static void morecore(unsigned int alloc_bytes)
+static int morecore(unsigned int alloc_bytes)
 {
     unsigned int map_bytes;
     char *map_hint, *map_start;
@@ -136,10 +136,9 @@ static void morecore(unsigned int alloc_bytes)
 #endif
 		     ,
 		     -1, 0);
-    if (map_start == MAP_FAILED) {
-	perror("mmap");
-	abort();
-    }
+    if (map_start == MAP_FAILED)
+	return -1;
+
     ALLOC_CODE_STATS(total_mapped += map_bytes);
 
     /* Merge adjacent mappings, so the trailing portion of the previous
@@ -155,6 +154,8 @@ static void morecore(unsigned int alloc_bytes)
     }
 
     ALLOC_CODE_STATS(atexit_alloc_code_stats());
+
+    return 0;
 }
 
 static void *alloc_code(unsigned int alloc_bytes)
@@ -164,8 +165,8 @@ static void *alloc_code(unsigned int alloc_bytes)
     /* Align function entries. */
     alloc_bytes = (alloc_bytes + 3) & ~3;
 
-    if (code_bytes < alloc_bytes)
-	morecore(alloc_bytes);
+    if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0)
+	return NULL;
     ALLOC_CODE_STATS(++nr_allocs);
     ALLOC_CODE_STATS(total_alloc += alloc_bytes);
     res = code_next;
@@ -207,6 +208,8 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
 	(P_CALLEE_EXP >= 128 ? 3 : 0) +
 	(P_ARITY >= 128 ? 3 : 0);
     codep = code = alloc_code(codeSize);
+    if (!code)
+	return NULL;
 
     /* movl $beamAddress, P_CALLEE_EXP(%ebp); 3 or 6 bytes, plus 4 */
     codep[0] = 0xc7;