erts: Fix heap overwrite by hipe "trap frames" when GC is disabled

by trapping BIFs like term_to_binary and binary_to_term.

3 files changed, 43 insertions, 5 deletions

diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4
index b1fedf4838..0997d81b2f 100644
--- a/erts/emulator/hipe/hipe_bif_list.m4
+++ b/erts/emulator/hipe/hipe_bif_list.m4
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2004-2012. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -262,7 +262,17 @@ noproc_primop_interface_1(nbif_atomic_inc, hipe_atomic_inc)
  * Standard BIFs.
  * BIF_LIST(ModuleAtom,FunctionAtom,Arity,CFun,Index)
  */
-define(BIF_LIST,`standard_bif_interface_$3(nbif_$4, $4)')
+
+/* BIFs that disable GC while trapping are called via a wrapper
+ * to reserve stack space for the "trap frame".
+ */
+define(CFUN,`ifelse($1,term_to_binary_1,hipe_wrapper_term_to_binary_1,
+ifelse($1,term_to_binary_2,hipe_wrapper_term_to_binary_2,
+ifelse($1,erts_internal_binary_to_term_1,hipe_wrapper_erts_internal_binary_to_term_1,
+ifelse($1,erts_internal_binary_to_term_2,hipe_wrapper_erts_internal_binary_to_term_2,
+$1))))')
+
+define(BIF_LIST,`standard_bif_interface_$3(nbif_$4, CFUN($4))')
 include(TARGET/`erl_bif_list.h')
 
 /*
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index adc8793469..15cdb231a2 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -184,21 +184,46 @@ void hipe_set_call_trap(Uint *bfun, void *nfun, int is_closure)
     bfun[-4] = (Uint)nfun;
 }
 
-static __inline__ void
-hipe_push_beam_trap_frame(Process *p, Eterm reg[], unsigned arity)
+void hipe_reserve_beam_trap_frame(Process *p, Eterm reg[], unsigned arity)
 {
     /* ensure that at least 2 words are available on the BEAM stack */
     if ((p->stop - 2) < p->htop) {
-	DPRINTF("calling gc to increase BEAM stack size");
+	DPRINTF("calling gc to reserve BEAM stack size");
 	p->fcalls -= erts_garbage_collect(p, 2, reg, arity);
+	ASSERT(!((p->stop - 2) < p->htop));
     }
     p->stop -= 2;
+    p->stop[0] = NIL;
+    p->stop[1] = NIL;
+}
+
+static __inline__ void
+hipe_push_beam_trap_frame(Process *p, Eterm reg[], unsigned arity)
+{
+    if (p->flags & F_DISABLE_GC) {
+	/* Trap frame already reserved */
+	ASSERT(p->stop[0] == NIL && p->stop[1] == NIL);
+    }
+    else {
+	if ((p->stop - 2) < p->htop) {
+	    DPRINTF("calling gc to increase BEAM stack size");
+	    p->fcalls -= erts_garbage_collect(p, 2, reg, arity);
+	    ASSERT(!((p->stop - 2) < p->htop));
+	}
+	p->stop -= 2;
+    }
     p->stop[1] = hipe_beam_catch_throw;
     p->stop[0] = make_cp(p->cp);
     ++p->catches;
     p->cp = hipe_beam_pc_return;
 }
 
+void hipe_unreserve_beam_trap_frame(Process *p)
+{
+    ASSERT(p->stop[0] == NIL && p->stop[1] == NIL);
+    p->stop += 2;
+}
+
 static __inline__ void hipe_pop_beam_trap_frame(Process *p)
 {
     p->cp = cp_val(p->stop[0]);
diff --git a/erts/emulator/hipe/hipe_mode_switch.h b/erts/emulator/hipe/hipe_mode_switch.h
index a3e908a3b3..06721e3c04 100644
--- a/erts/emulator/hipe/hipe_mode_switch.h
+++ b/erts/emulator/hipe/hipe_mode_switch.h
@@ -59,6 +59,9 @@ void hipe_empty_nstack(Process *p);
 void hipe_set_closure_stub(ErlFunEntry *fe, unsigned num_free);
 Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s);
 
+void hipe_reserve_beam_trap_frame(Process*, Eterm reg[], unsigned arity);
+void hipe_unreserve_beam_trap_frame(Process*);
+
 extern Uint hipe_beam_pc_return[];
 extern Uint hipe_beam_pc_throw[];
 extern Uint hipe_beam_pc_resume[];