erts: Fix bug when hipe tailcalls trapping BIF that disables GC

Symptom: base64_SUITE:roundtrip crashes with hipe compiled stdlib.

Problem: HIPE_WRAPPER_BIF_DISABLE_GC pushed a "trap frame", but the frame
was only popped if the call was recursive.

Solution: Only reserve "trap frame" if BIF call is recursive.

3 files changed, 22 insertions, 0 deletions

diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index 38d58ef744..1ae1d17b7f 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -187,6 +187,9 @@ void hipe_set_call_trap(Uint *bfun, void *nfun, int is_closure)
 
 void hipe_reserve_beam_trap_frame(Process *p, Eterm reg[], unsigned arity)
 {
+    if (!hipe_bifcall_from_native_is_recursive(p))
+	return;
+
     /* ensure that at least 2 words are available on the BEAM stack */
     if ((p->stop - 2) < p->htop) {
 	DPRINTF("calling gc to reserve BEAM stack size");
@@ -222,6 +225,9 @@ hipe_push_beam_trap_frame(Process *p, Eterm reg[], unsigned arity)
 
 void hipe_unreserve_beam_trap_frame(Process *p)
 {
+    if (!hipe_bifcall_from_native_is_recursive(p))
+	return;
+
     ASSERT(p->stop[0] == NIL && p->stop[1] == hipe_beam_catch_throw);
     p->stop += 2;
 }
diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h
index cc2671c016..dbb7086dae 100644
--- a/erts/emulator/hipe/hipe_risc_glue.h
+++ b/erts/emulator/hipe/hipe_risc_glue.h
@@ -214,6 +214,14 @@ hipe_trap_from_native_is_recursive(Process *p)
     return 0;
 }
 
+/* Native called BIF. Is it a recursive call?
+   i.e should we return back to native when BIF is done? */
+static __inline__ int
+hipe_bifcall_from_native_is_recursive(Process *p)
+{
+    return (p->hipe.nra != (void(*)(void))&nbif_return);
+}
+
 
 /* Native makes a call which needs to unload the parameters.
    This differs from hipe_call_from_native_is_recursive() in
diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h
index 63ad250d60..4b6e495b9a 100644
--- a/erts/emulator/hipe/hipe_x86_glue.h
+++ b/erts/emulator/hipe/hipe_x86_glue.h
@@ -207,6 +207,14 @@ hipe_trap_from_native_is_recursive(Process *p)
     return 0;
 }
 
+/* Native called BIF. Is it a recursive call?
+   i.e should we return back to native when BIF is done? */
+static __inline__ int
+hipe_bifcall_from_native_is_recursive(Process *p)
+{
+    return (*p->hipe.nsp != (Eterm)nbif_return);
+}
+
 
 /* Native makes a call which needs to unload the parameters.
    This differs from hipe_call_from_native_is_recursive() in