erts: Fix heap overwrite by hipe "trap frames" when GC is disabled

by trapping BIFs like term_to_binary and binary_to_term.

4 files changed, 107 insertions, 6 deletions

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 9fb2dbd8bf..2ca52c8025 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1068,7 +1068,7 @@ static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1)
             BIF_RET(res);
     }
 }
-    
+
 BIF_RETTYPE term_to_binary_1(BIF_ALIST_1)
 {
     Eterm res = erts_term_to_binary_int(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS, NULL);
@@ -4459,3 +4459,66 @@ error:
 #undef SKIP2
 #undef CHKSIZE
 }
+
+
+#ifdef HIPE
+BIF_RETTYPE hipe_wrapper_term_to_binary_1(BIF_ALIST_1);
+BIF_RETTYPE hipe_wrapper_term_to_binary_2(BIF_ALIST_2);
+BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_1(BIF_ALIST_1);
+BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_2(BIF_ALIST_2);
+
+/* Hipe wrappers used by native code for BIFs that disable GC while trapping.
+ *
+ * Problem:
+ * When native code calls a BIF that traps, hipe_mode_switch will push a
+ * "trap frame" on the Erlang stack in order to find its way back from beam_emu
+ * back to native caller when finally done. If GC is disabled and stack/heap
+ * is full there is no place to push the "trap frame".
+ *
+ * Solution:
+ * We reserve space on stack for the "trap frame" here before the BIF is called.
+ * If the BIF does not trap, the space is reclaimed here before returning.
+ * If the BIF traps, hipe_push_beam_trap_frame() will detect that a "trap frame"
+ * already is reserved and use it.
+ */
+BIF_RETTYPE hipe_wrapper_term_to_binary_1(BIF_ALIST_1)
+{
+    Eterm res;
+    hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 1);
+    res = term_to_binary_1(BIF_P, BIF__ARGS);
+    if (is_value(res) || BIF_P->freason != TRAP) {
+	hipe_unreserve_beam_trap_frame(BIF_P);
+    }
+    return res;
+}
+BIF_RETTYPE hipe_wrapper_term_to_binary_2(BIF_ALIST_2)
+{
+    Eterm res;
+    hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 2);
+    res = term_to_binary_2(BIF_P, BIF__ARGS);
+    if (is_value(res) || BIF_P->freason != TRAP) {
+	hipe_unreserve_beam_trap_frame(BIF_P);
+    }
+    return res;
+}
+BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_1(BIF_ALIST_1)
+{
+    Eterm res;
+    hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 1);
+    res = erts_internal_binary_to_term_1(BIF_P, BIF__ARGS);
+    if (is_value(res) || BIF_P->freason != TRAP) {
+	hipe_unreserve_beam_trap_frame(BIF_P);
+    }
+    return res;
+}
+BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_2(BIF_ALIST_2)
+{
+    Eterm res;
+    hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 2);
+    res = erts_internal_binary_to_term_2(BIF_P, BIF__ARGS);
+    if (is_value(res) || BIF_P->freason != TRAP) {
+	hipe_unreserve_beam_trap_frame(BIF_P);
+    }
+    return res;
+}
+#endif /*HIPE*/
diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4
index b1fedf4838..0997d81b2f 100644
--- a/erts/emulator/hipe/hipe_bif_list.m4
+++ b/erts/emulator/hipe/hipe_bif_list.m4
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2004-2012. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -262,7 +262,17 @@ noproc_primop_interface_1(nbif_atomic_inc, hipe_atomic_inc)
  * Standard BIFs.
  * BIF_LIST(ModuleAtom,FunctionAtom,Arity,CFun,Index)
  */
-define(BIF_LIST,`standard_bif_interface_$3(nbif_$4, $4)')
+
+/* BIFs that disable GC while trapping are called via a wrapper
+ * to reserve stack space for the "trap frame".
+ */
+define(CFUN,`ifelse($1,term_to_binary_1,hipe_wrapper_term_to_binary_1,
+ifelse($1,term_to_binary_2,hipe_wrapper_term_to_binary_2,
+ifelse($1,erts_internal_binary_to_term_1,hipe_wrapper_erts_internal_binary_to_term_1,
+ifelse($1,erts_internal_binary_to_term_2,hipe_wrapper_erts_internal_binary_to_term_2,
+$1))))')
+
+define(BIF_LIST,`standard_bif_interface_$3(nbif_$4, CFUN($4))')
 include(TARGET/`erl_bif_list.h')
 
 /*
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index adc8793469..15cdb231a2 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -184,21 +184,46 @@ void hipe_set_call_trap(Uint *bfun, void *nfun, int is_closure)
     bfun[-4] = (Uint)nfun;
 }
 
-static __inline__ void
-hipe_push_beam_trap_frame(Process *p, Eterm reg[], unsigned arity)
+void hipe_reserve_beam_trap_frame(Process *p, Eterm reg[], unsigned arity)
 {
     /* ensure that at least 2 words are available on the BEAM stack */
     if ((p->stop - 2) < p->htop) {
-	DPRINTF("calling gc to increase BEAM stack size");
+	DPRINTF("calling gc to reserve BEAM stack size");
 	p->fcalls -= erts_garbage_collect(p, 2, reg, arity);
+	ASSERT(!((p->stop - 2) < p->htop));
     }
     p->stop -= 2;
+    p->stop[0] = NIL;
+    p->stop[1] = NIL;
+}
+
+static __inline__ void
+hipe_push_beam_trap_frame(Process *p, Eterm reg[], unsigned arity)
+{
+    if (p->flags & F_DISABLE_GC) {
+	/* Trap frame already reserved */
+	ASSERT(p->stop[0] == NIL && p->stop[1] == NIL);
+    }
+    else {
+	if ((p->stop - 2) < p->htop) {
+	    DPRINTF("calling gc to increase BEAM stack size");
+	    p->fcalls -= erts_garbage_collect(p, 2, reg, arity);
+	    ASSERT(!((p->stop - 2) < p->htop));
+	}
+	p->stop -= 2;
+    }
     p->stop[1] = hipe_beam_catch_throw;
     p->stop[0] = make_cp(p->cp);
     ++p->catches;
     p->cp = hipe_beam_pc_return;
 }
 
+void hipe_unreserve_beam_trap_frame(Process *p)
+{
+    ASSERT(p->stop[0] == NIL && p->stop[1] == NIL);
+    p->stop += 2;
+}
+
 static __inline__ void hipe_pop_beam_trap_frame(Process *p)
 {
     p->cp = cp_val(p->stop[0]);
diff --git a/erts/emulator/hipe/hipe_mode_switch.h b/erts/emulator/hipe/hipe_mode_switch.h
index a3e908a3b3..06721e3c04 100644
--- a/erts/emulator/hipe/hipe_mode_switch.h
+++ b/erts/emulator/hipe/hipe_mode_switch.h
@@ -59,6 +59,9 @@ void hipe_empty_nstack(Process *p);
 void hipe_set_closure_stub(ErlFunEntry *fe, unsigned num_free);
 Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s);
 
+void hipe_reserve_beam_trap_frame(Process*, Eterm reg[], unsigned arity);
+void hipe_unreserve_beam_trap_frame(Process*);
+
 extern Uint hipe_beam_pc_return[];
 extern Uint hipe_beam_pc_throw[];
 extern Uint hipe_beam_pc_resume[];