Merge branch 'sverk/hipe-inline-reserve-trap-frame'

* sverk/hipe-inline-reserve-trap-frame:
  erts: Extend usage of ASM macro to avoid including asm macros in C code
  erts: Make hipe_{un}reserve_beam_trap_frame INLINE

31 files changed, 213 insertions, 108 deletions

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 5370b592f3..12a1ecd50e 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -28,7 +28,9 @@
 #include "global.h"
 #include "erl_process.h"
 #include "error.h"
+#define ERL_WANT_HIPE_BIF_WRAPPER__
 #include "bif.h"
+#undef ERL_WANT_HIPE_BIF_WRAPPER__
 #include "big.h"
 #include "dist.h"
 #include "erl_version.h"
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index 72c55ccb55..7b69b39511 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -465,6 +465,8 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p,
 					 Eterm args[],
 					 int nargs);
 
+#ifdef ERL_WANT_HIPE_BIF_WRAPPER__
+
 #ifndef HIPE
 
 #define HIPE_WRAPPER_BIF_DISABLE_GC(BIF_NAME, ARITY)
@@ -509,6 +511,7 @@ BIF_RETTYPE hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (Process* c_p,	\
 
 #endif
 
+#endif /* ERL_WANT_HIPE_BIF_WRAPPER__ */
 
 #include "erl_bif_table.h"
 
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index b014bca108..cc0b3b9b6c 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -26,7 +26,9 @@
 #include "global.h"
 #include "erl_process.h"
 #include "error.h"
+#define ERL_WANT_HIPE_BIF_WRAPPER__
 #include "bif.h"
+#undef ERL_WANT_HIPE_BIF_WRAPPER__
 #include "big.h"
 #include "erl_binary.h"
 #include "erl_bits.h"
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 50548850eb..0010f6a440 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -21,6 +21,8 @@
 #  include "config.h"
 #endif
 
+#define ERL_WANT_GC_INTERNALS__
+
 #include "sys.h"
 #include "erl_vm.h"
 #include "global.h"
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 85bc2daf5d..bd0d7c71cc 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -32,7 +32,9 @@
 #include "global.h"
 #include "erl_process.h"
 #include "error.h"
+#define ERL_WANT_HIPE_BIF_WRAPPER__
 #include "bif.h"
+#undef ERL_WANT_HIPE_BIF_WRAPPER__
 #include "big.h"
 #include "erl_binary.h"
 #include "erl_bits.h"
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 1dc9e8a786..5f78a7b532 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -20,6 +20,8 @@
 #  include "config.h"
 #endif
 
+#define ERL_WANT_GC_INTERNALS__
+
 #include "sys.h"
 #include "erl_vm.h"
 #include "global.h"
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 5203dda263..bf0496c112 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -20,10 +20,12 @@
 #ifndef __ERL_GC_H__
 #define __ERL_GC_H__
 
-#include "erl_map.h"
+#if defined(ERL_WANT_GC_INTERNALS__) || defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF)
 
 /* GC declarations shared by beam/erl_gc.c and hipe/hipe_gc.c */
 
+#include "erl_map.h"
+
 #if defined(DEBUG) && !ERTS_GLB_INLINE_INCL_FUNC_DEF
 #  define HARDDEBUG 1
 #endif
@@ -67,8 +69,6 @@ do {                                                                    \
 #define in_area(ptr,start,nbytes) \
  ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
 
-extern Uint erts_test_long_gc_sleep;
-
 #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG)
 int within(Eterm *ptr, Process *p);
 #endif
@@ -97,4 +97,33 @@ ERTS_GLB_INLINE Eterm follow_moved(Eterm term)
 }
 #endif
 
+#endif /* ERL_GC_C__ || HIPE_GC_C__ */
+
+/*
+ * Global exported
+ */
+
+extern Uint erts_test_long_gc_sleep;
+
+typedef struct {
+  Uint64 reclaimed;
+  Uint64 garbage_cols;
+} ErtsGCInfo;
+
+void erts_gc_info(ErtsGCInfo *gcip);
+void erts_init_gc(void);
+int erts_garbage_collect(struct process*, int, Eterm*, int);
+void erts_garbage_collect_hibernate(struct process* p);
+Eterm erts_gc_after_bif_call(struct process* p, Eterm result, Eterm* regs, Uint arity);
+void erts_garbage_collect_literals(struct process* p, Eterm* literals,
+				   Uint lit_size,
+				   struct erl_off_heap_header* oh);
+Uint erts_next_heap_size(Uint, Uint);
+Eterm erts_heap_sizes(struct process* p);
+
+void erts_offset_off_heap(struct erl_off_heap*, Sint, Eterm*, Eterm*);
+void erts_offset_heap_ptr(Eterm*, Uint, Sint, Eterm*, Eterm*);
+void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*);
+void erts_free_heap_frags(struct process* p);
+
 #endif /* __ERL_GC_H__ */
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 27a3a3553b..3d08be25ff 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -58,6 +58,7 @@ typedef struct process Process;
 #include "external.h"
 #include "erl_mseg.h"
 #include "erl_async.h"
+#include "erl_gc.h"
 
 #ifdef HIPE
 #include "hipe_process.h"
@@ -489,11 +490,6 @@ typedef struct {
 } ErtsSchedWallTime;
 
 typedef struct {
-  Uint64 reclaimed;
-  Uint64 garbage_cols;
-} ErtsGCInfo;
-
-typedef struct {
     int sched;
     erts_aint32_t aux_work;
 } ErtsDelayedAuxWorkWakeupJob;
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 196913a741..48e717bb6f 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -36,7 +36,9 @@
 #include "erl_process.h"
 #include "error.h"
 #include "external.h"
+#define ERL_WANT_HIPE_BIF_WRAPPER__
 #include "bif.h"
+#undef ERL_WANT_HIPE_BIF_WRAPPER__
 #include "big.h"
 #include "dist.h"
 #include "erl_binary.h"
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 891046a8b5..da9f029a9f 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -41,6 +41,7 @@
 #include "error.h"
 #include "erl_utils.h"
 #include "erl_port.h"
+#include "erl_gc.h"
 
 struct enif_environment_t /* ErlNifEnv */
 {
@@ -809,23 +810,6 @@ void MD5Init(MD5_CTX *);
 void MD5Update(MD5_CTX *, unsigned char *, unsigned int);
 void MD5Final(unsigned char [16], MD5_CTX *);
 
-/* ggc.c */
-
-void erts_gc_info(ErtsGCInfo *gcip);
-void erts_init_gc(void);
-int erts_garbage_collect(Process*, int, Eterm*, int);
-void erts_garbage_collect_hibernate(Process* p);
-Eterm erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity);
-void erts_garbage_collect_literals(Process* p, Eterm* literals,
-				   Uint lit_size,
-				   struct erl_off_heap_header* oh);
-Uint erts_next_heap_size(Uint, Uint);
-Eterm erts_heap_sizes(Process* p);
-
-void erts_offset_off_heap(ErlOffHeap *, Sint, Eterm*, Eterm*);
-void erts_offset_heap_ptr(Eterm*, Uint, Sint, Eterm*, Eterm*);
-void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*);
-void erts_free_heap_frags(Process* p);
 
 /* io.c */
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index f20e6e5665..9b3ee5cc65 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -49,6 +49,9 @@
 #include "beam_bp.h"
 #include "erl_ptab.h"
 #include "erl_check_io.h"
+#ifdef HIPE
+#  include "hipe_mode_switch.h"
+#endif
 
 #undef M_TRIM_THRESHOLD
 #undef M_TOP_PAD
diff --git a/erts/emulator/hipe/hipe_amd64_asm.m4 b/erts/emulator/hipe/hipe_amd64_asm.m4
index 7c81040b8b..b4b3c073ab 100644
--- a/erts/emulator/hipe/hipe_amd64_asm.m4
+++ b/erts/emulator/hipe/hipe_amd64_asm.m4
@@ -33,7 +33,35 @@ define(HEAP_LIMIT_IN_REGISTER,0)dnl global for HL
 define(SIMULATE_NSP,0)dnl change to 1 to simulate call/ret insns
 
 `#define AMD64_LEAF_WORDS	'LEAF_WORDS
-`#define LEAF_WORDS	'LEAF_WORDS
+`#define LEAF_WORDS		'LEAF_WORDS
+`#define AMD64_NR_ARG_REGS	'NR_ARG_REGS
+`#define NR_ARG_REGS		'NR_ARG_REGS
+
+`#define AMD64_HP_IN_REGISTER	'HP_IN_REGISTER
+`#if AMD64_HP_IN_REGISTER'
+`#define AMD64_HEAP_POINTER 15'
+define(HP,%r15)dnl Only change this together with above
+`#endif'
+
+`#define AMD64_FCALLS_IN_REGISTER 'FCALLS_IN_REGISTER
+`#if AMD64_FCALLS_IN_REGISTER'
+`#define AMD64_FCALLS_REGISTER 11'
+define(FCALLS,%r11)dnl This goes together with line above
+`#endif'
+
+`#define AMD64_HEAP_LIMIT_IN_REGISTER 'HEAP_LIMIT_IN_REGISTER
+`#if AMD64_HEAP_LIMIT_IN_REGISTER'
+`#define AMD64_HEAP_LIMIT_REGISTER 12'
+define(HEAP_LIMIT,%r12)dnl Change this together with line above
+`#endif'
+
+`#define AMD64_SIMULATE_NSP	'SIMULATE_NSP
+
+
+`#ifdef ASM'
+/*
+ * Only assembler stuff from here on (when included from *.S)
+ */
 
 /*
  * Workarounds for Darwin.
@@ -63,33 +91,24 @@ ifelse(OPSYS,darwin,``
  */
 `#define P		%rbp'
 
-`#define AMD64_HP_IN_REGISTER	'HP_IN_REGISTER
 `#if AMD64_HP_IN_REGISTER
-#define AMD64_HEAP_POINTER 15'
-define(HP,%r15)dnl Only change this together with above
-`#define SAVE_HP	movq 'HP`, P_HP(P)
+#define SAVE_HP	        movq 'HP`, P_HP(P)
 #define RESTORE_HP	movq P_HP(P), 'HP`
 #else
 #define SAVE_HP		/*empty*/
 #define RESTORE_HP	/*empty*/
 #endif'
 
-`#define AMD64_FCALLS_IN_REGISTER 'FCALLS_IN_REGISTER
 `#if AMD64_FCALLS_IN_REGISTER
-#define AMD64_FCALLS_REGISTER 11'
-define(FCALLS,%r11)dnl This goes together with line above
-`#define SAVE_FCALLS	movq 'FCALLS`, P_FCALLS(P)
+#define SAVE_FCALLS	movq 'FCALLS`, P_FCALLS(P)
 #define RESTORE_FCALLS	movq P_FCALLS(P), 'FCALLS`
 #else
 #define SAVE_FCALLS	/*empty*/
 #define RESTORE_FCALLS	/*empty*/
 #endif'
 
-`#define AMD64_HEAP_LIMIT_IN_REGISTER 'HEAP_LIMIT_IN_REGISTER
 `#if AMD64_HEAP_LIMIT_IN_REGISTER
-#define AMD64_HEAP_LIMIT_REGISTER 12'
-define(HEAP_LIMIT,%r12)dnl Change this together with line above
-`#define RESTORE_HEAP_LIMIT	movq P_HP_LIMIT(P), 'HEAP_LIMIT`
+#define RESTORE_HEAP_LIMIT	movq P_HP_LIMIT(P), 'HEAP_LIMIT`
 #else
 #define RESTORE_HEAP_LIMIT	/*empty*/
 #endif'
@@ -99,7 +118,6 @@ define(NSP,%rsp)dnl
 `#define SAVE_CSP	movq	%rsp, P_CSP(P)
 #define RESTORE_CSP	movq	P_CSP(P), %rsp'
 
-`#define AMD64_SIMULATE_NSP	'SIMULATE_NSP
 
 /*
  * Context switching macros.
@@ -132,8 +150,6 @@ define(NSP,%rsp)dnl
 /*
  * Argument (parameter) registers.
  */
-`#define AMD64_NR_ARG_REGS	'NR_ARG_REGS
-`#define NR_ARG_REGS		'NR_ARG_REGS
 
 define(defarg,`define(ARG$1,`$2')dnl
 #`define ARG'$1	$2'
@@ -263,4 +279,6 @@ define(NBIF_RET,`NBIF_RET_N(eval(RET_POP($1)))')dnl
 `/* #define NBIF_RET_3	'NBIF_RET(3)` */'
 `/* #define NBIF_RET_5	'NBIF_RET(5)` */'
 
+`#endif /* ASM */'
+
 `#endif /* HIPE_AMD64_ASM_H */'
diff --git a/erts/emulator/hipe/hipe_amd64_bifs.m4 b/erts/emulator/hipe/hipe_amd64_bifs.m4
index a3219c7586..7a4bb30447 100644
--- a/erts/emulator/hipe/hipe_amd64_bifs.m4
+++ b/erts/emulator/hipe/hipe_amd64_bifs.m4
@@ -18,7 +18,7 @@ changecom(`/*', `*/')dnl
  * %CopyrightEnd%
  */
 
-
+#`define ASM'
 include(`hipe/hipe_amd64_asm.m4')
 #`include' "config.h"
 #`include' "hipe_literals.h"
diff --git a/erts/emulator/hipe/hipe_amd64_glue.S b/erts/emulator/hipe/hipe_amd64_glue.S
index bebe0a8fd1..955f7362b4 100644
--- a/erts/emulator/hipe/hipe_amd64_glue.S
+++ b/erts/emulator/hipe/hipe_amd64_glue.S
@@ -17,10 +17,9 @@
  * %CopyrightEnd%
  */
 
-
+#define ASM
 #include "hipe_amd64_asm.h"
 #include "hipe_literals.h"
-#define ASM
 #include "hipe_mode_switch.h"
 
 /*
diff --git a/erts/emulator/hipe/hipe_arm_asm.m4 b/erts/emulator/hipe/hipe_arm_asm.m4
index 85dc84973d..b2e3f83d1e 100644
--- a/erts/emulator/hipe/hipe_arm_asm.m4
+++ b/erts/emulator/hipe/hipe_arm_asm.m4
@@ -29,6 +29,14 @@ define(LEAF_WORDS,16)dnl number of stack words for leaf functions
 define(NR_ARG_REGS,3)dnl admissible values are 0 to 6, inclusive
 
 `#define ARM_LEAF_WORDS	'LEAF_WORDS
+`#define ARM_NR_ARG_REGS	'NR_ARG_REGS
+`#define NR_ARG_REGS	'NR_ARG_REGS
+
+
+`#ifdef ASM'
+/*
+ * Only assembler stuff from here on (when included from *.S)
+ */
 
 /*
  * Reserved registers.
@@ -77,8 +85,6 @@ define(NR_ARG_REGS,3)dnl admissible values are 0 to 6, inclusive
 /*
  * Argument (parameter) registers.
  */
-`#define ARM_NR_ARG_REGS	'NR_ARG_REGS
-`#define NR_ARG_REGS	'NR_ARG_REGS
 
 define(defarg,`define(ARG$1,`$2')dnl
 #`define ARG'$1	$2'
@@ -195,4 +201,6 @@ define(QUICK_CALL_RET,`NBIF_POP_N(eval(RET_POP($2)))b $1')dnl
 `/* #define QUICK_CALL_RET_F_3 'QUICK_CALL_RET(F,3)` */'
 `/* #define QUICK_CALL_RET_F_5 'QUICK_CALL_RET(F,5)` */'
 
+`#endif /* ASM */'
+
 `#endif /* HIPE_ARM_ASM_H */'
diff --git a/erts/emulator/hipe/hipe_arm_bifs.m4 b/erts/emulator/hipe/hipe_arm_bifs.m4
index bd8bc5ab6b..57e51bb8b1 100644
--- a/erts/emulator/hipe/hipe_arm_bifs.m4
+++ b/erts/emulator/hipe/hipe_arm_bifs.m4
@@ -19,6 +19,7 @@ changecom(`/*', `*/')dnl
  */
 
 
+#`define ASM'
 include(`hipe/hipe_arm_asm.m4')
 #`include' "config.h"
 #`include' "hipe_literals.h"
diff --git a/erts/emulator/hipe/hipe_arm_glue.S b/erts/emulator/hipe/hipe_arm_glue.S
index e58e112ca7..069cb4512e 100644
--- a/erts/emulator/hipe/hipe_arm_glue.S
+++ b/erts/emulator/hipe/hipe_arm_glue.S
@@ -17,10 +17,9 @@
  * %CopyrightEnd%
  */
 
-
+#define ASM
 #include "hipe_arm_asm.h"
 #include "hipe_literals.h"
-#define ASM
 #include "hipe_mode_switch.h"
 
 	.text
diff --git a/erts/emulator/hipe/hipe_debug.c b/erts/emulator/hipe/hipe_debug.c
index 7f82252308..61406b92af 100644
--- a/erts/emulator/hipe/hipe_debug.c
+++ b/erts/emulator/hipe/hipe_debug.c
@@ -172,8 +172,10 @@ void hipe_print_pcb(Process *p)
     printf("P: 0x%0*lx\r\n", 2*(int)sizeof(long), (unsigned long)p);
     printf("-----------------------------------------------\r\n");
     printf("Offset| Name        | Value      | *Value     |\r\n");
+#undef U
 #define U(n,x) \
     printf(" % 4d | %s | 0x%0*lx |            |\r\n", (int)offsetof(Process,x), n, 2*(int)sizeof(long), (unsigned long)p->x)
+#undef P
 #define P(n,x) \
     printf(" % 4d | %s | 0x%0*lx | 0x%0*lx |\r\n", (int)offsetof(Process,x), n, 2*(int)sizeof(long), (unsigned long)p->x, 2*(int)sizeof(long), p->x ? (unsigned long)*(p->x) : -1UL)
 
diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c
index 86c4068072..b10263f6e2 100644
--- a/erts/emulator/hipe/hipe_gc.c
+++ b/erts/emulator/hipe/hipe_gc.c
@@ -22,6 +22,9 @@
 #ifdef HAVE_CONFIG_H
 #include "config.h"
 #endif
+
+#define ERL_WANT_GC_INTERNALS__
+
 #include "global.h"
 
 #include "erl_gc.h"
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index a9dec05e2d..8c73312d45 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -140,7 +140,6 @@ void hipe_check_pcb(Process *p, const char *file, unsigned line)
 #endif	/* HIPE_DEBUG > 0 */
 
 /* ensure that at least nwords words are available on the native stack */
-static void hipe_check_nstack(Process *p, unsigned nwords);
 
 #if defined(__sparc__)
 #include "hipe_sparc_glue.h"
@@ -159,7 +158,7 @@ static void hipe_check_nstack(Process *p, unsigned nwords);
 Uint hipe_beam_pc_return[1];	/* needed in hipe_debug.c */
 Uint hipe_beam_pc_throw[1];	/* needed in hipe_debug.c */
 Uint hipe_beam_pc_resume[1];	/* needed by hipe_set_timeout() */
-static Eterm hipe_beam_catch_throw;
+Eterm hipe_beam_catch_throw;
 
 void hipe_mode_switch_init(void)
 {
@@ -185,22 +184,6 @@ void hipe_set_call_trap(Uint *bfun, void *nfun, int is_closure)
     bfun[-4] = (Uint)nfun;
 }
 
-void hipe_reserve_beam_trap_frame(Process *p, Eterm reg[], unsigned arity)
-{
-    if (!hipe_bifcall_from_native_is_recursive(p))
-	return;
-
-    /* ensure that at least 2 words are available on the BEAM stack */
-    if ((p->stop - 2) < p->htop) {
-	DPRINTF("calling gc to reserve BEAM stack size");
-	p->fcalls -= erts_garbage_collect(p, 2, reg, arity);
-	ASSERT(!((p->stop - 2) < p->htop));
-    }
-    p->stop -= 2;
-    p->stop[0] = NIL;
-    p->stop[1] = hipe_beam_catch_throw;
-}
-
 static __inline__ void
 hipe_push_beam_trap_frame(Process *p, Eterm reg[], unsigned arity)
 {
@@ -223,15 +206,6 @@ hipe_push_beam_trap_frame(Process *p, Eterm reg[], unsigned arity)
     p->cp = hipe_beam_pc_return;
 }
 
-void hipe_unreserve_beam_trap_frame(Process *p)
-{
-    if (!hipe_bifcall_from_native_is_recursive(p))
-	return;
-
-    ASSERT(p->stop[0] == NIL && p->stop[1] == hipe_beam_catch_throw);
-    p->stop += 2;
-}
-
 static __inline__ void hipe_pop_beam_trap_frame(Process *p)
 {
     ASSERT(p->stop[1] == hipe_beam_catch_throw);
@@ -607,7 +581,6 @@ static unsigned hipe_next_nstack_size(unsigned size)
 }
 
 #if 0 && defined(HIPE_NSTACK_GROWS_UP)
-#define hipe_nstack_avail(p)	((p)->hipe.nstend - (p)->hipe.nsp)
 void hipe_inc_nstack(Process *p)
 {
     Eterm *old_nstack = p->hipe.nstack;
@@ -631,7 +604,6 @@ void hipe_inc_nstack(Process *p)
 #endif
 
 #if defined(HIPE_NSTACK_GROWS_DOWN)
-#define hipe_nstack_avail(p)	((unsigned)((p)->hipe.nsp - (p)->hipe.nstack))
 void hipe_inc_nstack(Process *p)
 {
     unsigned old_size = p->hipe.nstend - p->hipe.nstack;
@@ -663,12 +635,6 @@ void hipe_empty_nstack(Process *p)
     p->hipe.nstend = NULL;
 }
 
-static void hipe_check_nstack(Process *p, unsigned nwords)
-{
-    while (hipe_nstack_avail(p) < nwords)
-	hipe_inc_nstack(p);
-}
-
 void hipe_set_closure_stub(ErlFunEntry *fe, unsigned num_free)
 {
     unsigned arity;
diff --git a/erts/emulator/hipe/hipe_mode_switch.h b/erts/emulator/hipe/hipe_mode_switch.h
index 6ec5da1ae9..b8de12fcbb 100644
--- a/erts/emulator/hipe/hipe_mode_switch.h
+++ b/erts/emulator/hipe/hipe_mode_switch.h
@@ -61,13 +61,58 @@ void hipe_empty_nstack(Process *p);
 void hipe_set_closure_stub(ErlFunEntry *fe, unsigned num_free);
 Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s);
 
-void hipe_reserve_beam_trap_frame(Process*, Eterm reg[], unsigned arity);
-void hipe_unreserve_beam_trap_frame(Process*);
+ERTS_GLB_INLINE void hipe_reserve_beam_trap_frame(Process*, Eterm reg[], unsigned arity);
+ERTS_GLB_INLINE void hipe_unreserve_beam_trap_frame(Process*);
 
 extern Uint hipe_beam_pc_return[];
 extern Uint hipe_beam_pc_throw[];
 extern Uint hipe_beam_pc_resume[];
 
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+#include "erl_gc.h"
+#include "hipe_stack.h"
+
+#if defined(__sparc__)
+#include "hipe_sparc_glue.h"
+#elif defined(__i386__)
+#include "hipe_x86_glue.h"
+#elif defined(__x86_64__)
+#include "hipe_amd64_glue.h"
+#elif defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
+#include "hipe_ppc_glue.h"
+#elif defined(__arm__)
+#include "hipe_arm_glue.h"
+#endif
+
+extern Eterm hipe_beam_catch_throw;
+
+ERTS_GLB_INLINE void hipe_reserve_beam_trap_frame(Process *p, Eterm reg[], unsigned arity)
+{
+    if (!hipe_bifcall_from_native_is_recursive(p))
+	return;
+
+    /* ensure that at least 2 words are available on the BEAM stack */
+    if ((p->stop - 2) < p->htop) {
+	p->fcalls -= erts_garbage_collect(p, 2, reg, arity);
+	ASSERT(!((p->stop - 2) < p->htop));
+    }
+    p->stop -= 2;
+    p->stop[0] = NIL;
+    p->stop[1] = hipe_beam_catch_throw;
+}
+
+ERTS_GLB_INLINE void hipe_unreserve_beam_trap_frame(Process *p)
+{
+    if (!hipe_bifcall_from_native_is_recursive(p))
+	return;
+
+    ASSERT(p->stop[0] == NIL && p->stop[1] == hipe_beam_catch_throw);
+    p->stop += 2;
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
 #endif	/* ASM */
 
 #endif /* HIPE_MODE_SWITCH_H */
diff --git a/erts/emulator/hipe/hipe_ppc_asm.m4 b/erts/emulator/hipe/hipe_ppc_asm.m4
index 343402f9f0..4a1caa1543 100644
--- a/erts/emulator/hipe/hipe_ppc_asm.m4
+++ b/erts/emulator/hipe/hipe_ppc_asm.m4
@@ -23,6 +23,22 @@ changecom(`/*', `*/')dnl
 #define HIPE_PPC_ASM_H'
 
 /*
+ * Tunables.
+ */
+define(LEAF_WORDS,16)dnl number of stack words for leaf functions
+define(NR_ARG_REGS,4)dnl admissible values are 0 to 6, inclusive
+
+`#define PPC_LEAF_WORDS	'LEAF_WORDS
+`#define PPC_NR_ARG_REGS	'NR_ARG_REGS
+`#define NR_ARG_REGS	'NR_ARG_REGS
+
+
+`#ifdef ASM'
+/*
+ * Only assembler stuff from here on (when included from *.S)
+ */
+
+/*
  * Handle 32 vs 64-bit.
  */
 ifelse(ARCH,ppc64,`
@@ -53,13 +69,6 @@ define(WSIZE,4)dnl
 `#define STORE	'STORE
 `#define CMPI	'CMPI
 
-/*
- * Tunables.
- */
-define(LEAF_WORDS,16)dnl number of stack words for leaf functions
-define(NR_ARG_REGS,4)dnl admissible values are 0 to 6, inclusive
-
-`#define PPC_LEAF_WORDS	'LEAF_WORDS
 
 /*
  * Workarounds for Darwin.
@@ -193,8 +202,6 @@ NAME:						\
 /*
  * Argument (parameter) registers.
  */
-`#define PPC_NR_ARG_REGS	'NR_ARG_REGS
-`#define NR_ARG_REGS	'NR_ARG_REGS
 
 define(defarg,`define(ARG$1,`$2')dnl
 #`define ARG'$1	$2'
@@ -309,4 +316,6 @@ define(QUICK_CALL_RET,`NBIF_POP_N(eval(RET_POP($2)))b $1')dnl
 `/* #define QUICK_CALL_RET_F_3 'QUICK_CALL_RET(F,3)` */'
 `/* #define QUICK_CALL_RET_F_5 'QUICK_CALL_RET(F,5)` */'
 
+`#endif /* ASM */'
+
 `#endif /* HIPE_PPC_ASM_H */'
diff --git a/erts/emulator/hipe/hipe_ppc_bifs.m4 b/erts/emulator/hipe/hipe_ppc_bifs.m4
index 7cc2b5c7b6..f53b79b52e 100644
--- a/erts/emulator/hipe/hipe_ppc_bifs.m4
+++ b/erts/emulator/hipe/hipe_ppc_bifs.m4
@@ -19,6 +19,7 @@ changecom(`/*', `*/')dnl
  */
 
 
+#`define ASM'
 include(`hipe/hipe_ppc_asm.m4')
 #`include' "config.h"
 #`include' "hipe_literals.h"
diff --git a/erts/emulator/hipe/hipe_ppc_glue.S b/erts/emulator/hipe/hipe_ppc_glue.S
index 0c337a14df..c48fb150af 100644
--- a/erts/emulator/hipe/hipe_ppc_glue.S
+++ b/erts/emulator/hipe/hipe_ppc_glue.S
@@ -17,10 +17,9 @@
  * %CopyrightEnd%
  */
 
-
+#define ASM
 #include "hipe_ppc_asm.h"
 #include "hipe_literals.h"
-#define ASM
 #include "hipe_mode_switch.h"
 
 	.text
diff --git a/erts/emulator/hipe/hipe_sparc_asm.m4 b/erts/emulator/hipe/hipe_sparc_asm.m4
index 227d10ed80..c3c3bcb74a 100644
--- a/erts/emulator/hipe/hipe_sparc_asm.m4
+++ b/erts/emulator/hipe/hipe_sparc_asm.m4
@@ -29,6 +29,14 @@ define(LEAF_WORDS,16)dnl number of stack words for leaf functions
 define(NR_ARG_REGS,4)dnl admissible values are 0 to 6, inclusive
 
 `#define SPARC_LEAF_WORDS	'LEAF_WORDS
+`#define SPARC_NR_ARG_REGS	'NR_ARG_REGS
+`#define NR_ARG_REGS	'NR_ARG_REGS
+
+
+`#ifdef ASM'
+/*
+ * Only assembler stuff from here on (when included from *.S)
+ */
 
 /*
  * Reserved registers.
@@ -80,9 +88,6 @@ define(NR_ARG_REGS,4)dnl admissible values are 0 to 6, inclusive
 /*
  * Argument (parameter) registers.
  */
-`#define SPARC_NR_ARG_REGS	'NR_ARG_REGS
-`#define NR_ARG_REGS	'NR_ARG_REGS
-
 define(defarg,`define(ARG$1,`$2')dnl
 #`define ARG'$1	$2'
 )dnl
@@ -210,4 +215,6 @@ define(QUICK_CALL_RET,`ba $1; NBIF_POP_N(eval(RET_POP($2)))')dnl
 `/* #define QUICK_CALL_RET_F_3 'QUICK_CALL_RET(F,3)` */'
 `/* #define QUICK_CALL_RET_F_5 'QUICK_CALL_RET(F,5)` */'
 
+`#endif /* ASM */'
+
 `#endif /* HIPE_SPARC_ASM_H */'
diff --git a/erts/emulator/hipe/hipe_sparc_bifs.m4 b/erts/emulator/hipe/hipe_sparc_bifs.m4
index ca5af45d58..2bfe3a4646 100644
--- a/erts/emulator/hipe/hipe_sparc_bifs.m4
+++ b/erts/emulator/hipe/hipe_sparc_bifs.m4
@@ -19,6 +19,7 @@ changecom(`/*', `*/')dnl
  */
 
 
+#`define ASM'
 include(`hipe/hipe_sparc_asm.m4')
 #`include' "config.h"
 #`include' "hipe_literals.h"
diff --git a/erts/emulator/hipe/hipe_sparc_glue.S b/erts/emulator/hipe/hipe_sparc_glue.S
index ab40a48ee7..6c8c841194 100644
--- a/erts/emulator/hipe/hipe_sparc_glue.S
+++ b/erts/emulator/hipe/hipe_sparc_glue.S
@@ -18,10 +18,9 @@
  * %CopyrightEnd%
  */
 
-
+#define ASM
 #include "hipe_sparc_asm.h"
 #include "hipe_literals.h"
-#define ASM
 #include "hipe_mode_switch.h"
 
 	.section ".text"
diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h
index 66f9f04c73..4cfdb54dd8 100644
--- a/erts/emulator/hipe/hipe_stack.h
+++ b/erts/emulator/hipe/hipe_stack.h
@@ -108,12 +108,23 @@ extern int hipe_fill_stacktrace(Process*, int, Eterm**);
 #if 0 && defined(HIPE_NSTACK_GROWS_UP)
 #define hipe_nstack_start(p)	((p)->hipe.nstack)
 #define hipe_nstack_used(p)	((p)->hipe.nsp - (p)->hipe.nstack)
+#define hipe_nstack_avail(p)	((p)->hipe.nstend - (p)->hipe.nsp)
 #endif
 #if defined(HIPE_NSTACK_GROWS_DOWN)
 #define hipe_nstack_start(p)	((p)->hipe.nsp)
 #define hipe_nstack_used(p)	((p)->hipe.nstend - (p)->hipe.nsp)
+#define hipe_nstack_avail(p)	((unsigned)((p)->hipe.nsp - (p)->hipe.nstack))
 #endif
 
+/* ensure that at least nwords words are available on the native stack */
+static __inline__ void hipe_check_nstack(Process *p, unsigned nwords)
+{
+    extern void hipe_inc_nstack(Process *p);
+
+    while (hipe_nstack_avail(p) < nwords)
+	hipe_inc_nstack(p);
+}
+
 /*
  * GC support procedures
  */
diff --git a/erts/emulator/hipe/hipe_x86_asm.m4 b/erts/emulator/hipe/hipe_x86_asm.m4
index 020ccf8d4b..39c5cb1044 100644
--- a/erts/emulator/hipe/hipe_x86_asm.m4
+++ b/erts/emulator/hipe/hipe_x86_asm.m4
@@ -33,6 +33,18 @@ define(SIMULATE_NSP,0)dnl change to 1 to simulate call/ret insns
 `#define X86_LEAF_WORDS	'LEAF_WORDS
 `#define LEAF_WORDS	'LEAF_WORDS
 
+`#define X86_NR_ARG_REGS	'NR_ARG_REGS
+`#define NR_ARG_REGS	'NR_ARG_REGS
+
+`#define X86_HP_IN_ESI	'HP_IN_ESI
+`#define X86_SIMULATE_NSP	'SIMULATE_NSP
+
+
+`#ifdef ASM'
+/*
+ * Only assembler stuff from here on (when included from *.S)
+ */
+
 /*
  * Workarounds for Darwin.
  */
@@ -60,7 +72,6 @@ ifelse(OPSYS,darwin,``
  */
 `#define P	%ebp'
 
-`#define X86_HP_IN_ESI	'HP_IN_ESI
 `#if X86_HP_IN_ESI
 #define SAVE_HP		movl %esi, P_HP(P)
 #define RESTORE_HP	movl P_HP(P), %esi
@@ -73,7 +84,6 @@ ifelse(OPSYS,darwin,``
 #define SAVE_CSP	movl %esp, P_CSP(P)
 #define RESTORE_CSP	movl P_CSP(P), %esp'
 
-`#define X86_SIMULATE_NSP	'SIMULATE_NSP
 
 /*
  * Context switching macros.
@@ -100,12 +110,10 @@ ifelse(OPSYS,darwin,``
 	SAVE_CACHED_STATE;	\
 	SWITCH_ERLANG_TO_C_QUICK'
 
+
 /*
  * Argument (parameter) registers.
  */
-`#define X86_NR_ARG_REGS	'NR_ARG_REGS
-`#define NR_ARG_REGS	'NR_ARG_REGS
-
 ifelse(eval(NR_ARG_REGS >= 1),0,,
 ``#define ARG0	%eax
 '')dnl
@@ -282,4 +290,6 @@ define(LOAD_CALLER_SAVE,`LAR_N(eval(NR_CALLER_SAVE-1))')dnl
 `#define STORE_CALLER_SAVE	'STORE_CALLER_SAVE
 `#define LOAD_CALLER_SAVE	'LOAD_CALLER_SAVE
 
+`#endif /* ASM */'
+
 `#endif /* HIPE_X86_ASM_H */'
diff --git a/erts/emulator/hipe/hipe_x86_bifs.m4 b/erts/emulator/hipe/hipe_x86_bifs.m4
index dd6980f555..a0f16efa33 100644
--- a/erts/emulator/hipe/hipe_x86_bifs.m4
+++ b/erts/emulator/hipe/hipe_x86_bifs.m4
@@ -19,6 +19,7 @@ changecom(`/*', `*/')dnl
  */
 
 
+#`define ASM'
 include(`hipe/hipe_x86_asm.m4')
 #`include' "config.h"
 #`include' "hipe_literals.h"
diff --git a/erts/emulator/hipe/hipe_x86_glue.S b/erts/emulator/hipe/hipe_x86_glue.S
index 638780156a..9d38eaaafd 100644
--- a/erts/emulator/hipe/hipe_x86_glue.S
+++ b/erts/emulator/hipe/hipe_x86_glue.S
@@ -18,10 +18,9 @@
  * %CopyrightEnd%
  */
 
-
+#define ASM
 #include "hipe_x86_asm.h"
 #include "hipe_literals.h"
-#define ASM
 #include "hipe_mode_switch.h"
 
 /*