1 files changed, 156 insertions, 417 deletions

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 32ea8588d2..9c5450bd48 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -36,6 +36,7 @@
 #include "dist.h"
 #include "beam_bp.h"
 #include "beam_catches.h"
+#include "erl_thr_progress.h"
 #ifdef HIPE
 #include "hipe_mode_switch.h"
 #include "hipe_bif1.h"
@@ -70,7 +71,7 @@ do {									\
     }									\
     else								\
 	erts_lc_check_exact(NULL, 0);					\
-    ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING);				\
+    	ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());		\
 } while (0)
 #    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
         if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN)
@@ -303,44 +304,6 @@ extern int count_instructions;
      PROCESS_MAIN_CHK_LOCKS((P));					\
      ERTS_SMP_UNREQ_PROC_MAIN_LOCK((P))
 
-#if defined(HYBRID)
-#  define POST_BIF_GC_SWAPIN_0(_p, _res)				\
-     if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) {	\
-       _res = erts_gc_after_bif_call((_p), (_res), NULL, 0);		\
-     }									\
-     SWAPIN
-
-#  define POST_BIF_GC_SWAPIN(_p, _res, _regs, _arity)			\
-     if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) {	\
-       _regs[0] = r(0);							\
-       _res = erts_gc_after_bif_call((_p), (_res), _regs, (_arity));	\
-       r(0) = _regs[0];							\
-     }									\
-     SWAPIN
-#else
-#  define POST_BIF_GC_SWAPIN_0(_p, _res)				\
-     ERTS_SMP_REQ_PROC_MAIN_LOCK((_p));					\
-     PROCESS_MAIN_CHK_LOCKS((_p));					\
-     ERTS_VERIFY_UNUSED_TEMP_ALLOC((_p));				\
-     if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) {	\
-       _res = erts_gc_after_bif_call((_p), (_res), NULL, 0);		\
-       E = (_p)->stop;							\
-     }									\
-     HTOP = HEAP_TOP((_p))
-
-#  define POST_BIF_GC_SWAPIN(_p, _res, _regs, _arity)			\
-     ERTS_VERIFY_UNUSED_TEMP_ALLOC((_p));				\
-     ERTS_SMP_REQ_PROC_MAIN_LOCK((_p));					\
-     PROCESS_MAIN_CHK_LOCKS((_p));					\
-     if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) {	\
-       _regs[0] = r(0);							\
-       _res = erts_gc_after_bif_call((_p), (_res), _regs, (_arity));	\
-       r(0) = _regs[0];							\
-       E = (_p)->stop;							\
-     }									\
-     HTOP = HEAP_TOP((_p))
-#endif
-
 #define db(N) (N)
 #define tb(N) (N)
 #define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N)))
@@ -794,11 +757,11 @@ extern int count_instructions;
      }						\
   } while (0)
 
-#define IsFunction2(F, A, Action)		\
-  do {						\
-     if (is_function_2(c_p, F, A) != am_true ) {\
-          Action;				\
-     }						\
+#define IsFunction2(F, A, Action)			\
+  do {							\
+     if (erl_is_function(c_p, F, A) != am_true ) {	\
+          Action;					\
+     }							\
   } while (0)
 
 #define IsTupleOfArity(Src, Arity, Fail)				      \
@@ -1145,26 +1108,11 @@ void process_main(void)
     Eterm *tmp_big;		/* Temporary buffer for small bignums if !HEAP_ON_C_STACK. */
 #endif
 
-#ifndef ERTS_SMP
-#if !HALFWORD_HEAP
-    static Eterm save_reg[ERTS_X_REGS_ALLOCATED];
-    /* X registers -- not used directly, but
-     * through 'reg', because using it directly
-     * needs two instructions on a SPARC,
-     * while using it through reg needs only
-     * one.
-     */
-#endif
     /*
-     * Floating point registers.
-     */
-    static FloatDef freg[MAX_REG];
-#else
-    /* X regisers and floating point registers are located in
+     * X registers and floating point registers are located in
      * scheduler specific data.
      */
     register FloatDef *freg;
-#endif
 
     /*
      * For keeping the negative old value of 'reds' when call saving is active.
@@ -1201,14 +1149,6 @@ void process_main(void)
 	init_done = 1;
 	goto init_emulator;
     }
-#ifndef ERTS_SMP
-#if !HALFWORD_HEAP
-    reg = save_reg;	/* XXX: probably wastes a register on x86 */
-#else
-    /* Registers need to be heap allocated (correct memory range) for tracing to work */
-    reg = erts_alloc(ERTS_ALC_T_BEAM_REGISTER, ERTS_X_REGS_ALLOCATED * sizeof(Eterm));
-#endif
-#endif
     c_p = NULL;
     reds_used = 0;
     goto do_schedule1;
@@ -1229,10 +1169,8 @@ void process_main(void)
 #endif
     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
     PROCESS_MAIN_CHK_LOCKS(c_p);
-#ifdef ERTS_SMP
-    reg = c_p->scheduler_data->save_reg;
-    freg = c_p->scheduler_data->freg;
-#endif
+    reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array;
+    freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array;
 #if !HEAP_ON_C_STACK
     tmp_big = ERTS_PROC_GET_SCHDATA(c_p)->beam_emu_tmp_heap;
 #endif
@@ -1566,9 +1504,17 @@ void process_main(void)
 
      PRE_BIF_SWAPOUT(c_p);
      c_p->fcalls = FCALLS - 1;
-     result = send_2(c_p, r(0), x(1));
+     reg[0] = r(0);
+     result = erl_send(c_p, r(0), x(1));
      PreFetch(0, next);
-     POST_BIF_GC_SWAPIN(c_p, result, reg, 2);
+     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+     PROCESS_MAIN_CHK_LOCKS(c_p);
+     if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) {
+	 result = erts_gc_after_bif_call(c_p, result, reg, 2);
+	 r(0) = reg[0];
+	 E = c_p->stop;
+     }
+     HTOP = HEAP_TOP(c_p);
      FCALLS = c_p->fcalls;
      if (is_value(result)) {
 	 r(0) = result;
@@ -1576,10 +1522,9 @@ void process_main(void)
 	 NextPF(0, next);
      } else if (c_p->freason == TRAP) {
 	 SET_CP(c_p, I+1);
-	 SET_I(*((BeamInstr **) (BeamInstr) ((c_p)->def_arg_reg + 3)));
+	 SET_I(c_p->i);
 	 SWAPIN;
-	 r(0) = c_p->def_arg_reg[0];
-	 x(1) = c_p->def_arg_reg[1];
+	 r(0) = reg[0];
 	 Dispatch();
      }
      goto find_func_info;
@@ -1911,13 +1856,15 @@ void process_main(void)
 	      * Note that for the halfword emulator, the two first elements
 	      * of the array are used.
 	      */
-	     *((BeamInstr **) (UWord) c_p->def_arg_reg) = I+3;
+	     BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg;
+	     *pi = I+3;
 	     set_timer(c_p, unsigned_val(timeout_value));
 	 } else if (timeout_value == am_infinity) {
 	     c_p->flags |= F_TIMO;
 #if !defined(ARCH_64) || HALFWORD_HEAP
 	 } else if (term_to_Uint(timeout_value, &time_val)) {
-	     *((BeamInstr **) (UWord) c_p->def_arg_reg) = I+3;
+	     BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg;
+	     *pi = I+3;
 	     set_timer(c_p, time_val);
 #endif
 	 } else {		/* Wrong time */
@@ -1974,7 +1921,8 @@ void process_main(void)
       * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag.
       */
      if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) {
-	 *((BeamInstr **) (UWord) c_p->def_arg_reg) = I+3;
+	 BeamInstr** p = (BeamInstr **) c_p->def_arg_reg;
+	 *p = I+3;
 	 set_timer(c_p, Arg(1));
      }
      goto wait2;
@@ -2231,16 +2179,16 @@ void process_main(void)
 
  OpCase(bif1_fbsd):
     {
-	Eterm (*bf)(Process*, Eterm);
-	Eterm arg;
+	Eterm (*bf)(Process*, Eterm*);
+	Eterm tmp_reg[1];
 	Eterm result;
 
-	GetArg1(2, arg);
+	GetArg1(2, tmp_reg[0]);
 	bf = (BifFunction) Arg(1);
 	c_p->fcalls = FCALLS;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	result = (*bf)(c_p, arg);
+	result = (*bf)(c_p, tmp_reg);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2259,17 +2207,17 @@ void process_main(void)
 
  OpCase(bif1_body_bsd):
     {
-	Eterm (*bf)(Process*, Eterm);
+	Eterm (*bf)(Process*, Eterm*);
 
-	Eterm arg;
+	Eterm tmp_reg[1];
 	Eterm result;
 
-	GetArg1(1, arg);
+	GetArg1(1, tmp_reg[0]);
 	bf = (BifFunction) Arg(0);
 	c_p->fcalls = FCALLS;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	result = (*bf)(c_p, arg);
+	result = (*bf)(c_p, tmp_reg);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2278,7 +2226,7 @@ void process_main(void)
 	if (is_value(result)) {
 	    StoreBifResult(2, result);
 	}
-	reg[0] = arg;
+	reg[0] = tmp_reg[0];
 	SWAPOUT;
 	I = handle_error(c_p, I, reg, bf);
 	goto post_error_handling;
@@ -2402,14 +2350,15 @@ void process_main(void)
   */
  OpCase(i_bif2_fbd):
     {
-	Eterm (*bf)(Process*, Eterm, Eterm);
+	Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2};
+	Eterm (*bf)(Process*, Eterm*);
 	Eterm result;
 
 	bf = (BifFunction) Arg(1);
 	c_p->fcalls = FCALLS;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	result = (*bf)(c_p, tmp_arg1, tmp_arg2);
+	result = (*bf)(c_p, tmp_reg);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2427,13 +2376,14 @@ void process_main(void)
   */
  OpCase(i_bif2_body_bd):
     {
-	Eterm (*bf)(Process*, Eterm, Eterm);
+	Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2};
+	Eterm (*bf)(Process*, Eterm*);
 	Eterm result;
 
 	bf = (BifFunction) Arg(0);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	result = (*bf)(c_p, tmp_arg1, tmp_arg2);
+	result = (*bf)(c_p, tmp_reg);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2453,77 +2403,9 @@ void process_main(void)
      * The most general BIF call.  The BIF may build any amount of data
      * on the heap.  The result is always returned in r(0).
      */
- OpCase(call_bif0_e):
-    {
-	Eterm (*bf)(Process*, BeamInstr*) = GET_BIF_ADDRESS(Arg(0));
-
-	PRE_BIF_SWAPOUT(c_p);
-	c_p->fcalls = FCALLS - 1;
-	if (FCALLS <= 0) {
-	    save_calls(c_p, (Export *) Arg(0));
-	}
-
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	r(0) = (*bf)(c_p, I);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(r(0)));
-	ERTS_HOLE_CHECK(c_p);
-	POST_BIF_GC_SWAPIN_0(c_p, r(0));
-	FCALLS = c_p->fcalls;
-	if (is_value(r(0))) {
-	    CHECK_TERM(r(0));
-	    Next(1);
-	}
-	else if (c_p->freason == TRAP) {
-	    goto call_bif_trap3;
-	}
-
-	/*
-	 * Error handling.  SWAPOUT is not needed because it was done above.
-	 */
-	ASSERT(c_p->stop == E);
-	reg[0] = r(0);
-	I = handle_error(c_p, I, reg, bf);
-	goto post_error_handling;
-    }
-
- OpCase(call_bif1_e):
-    {
-	Eterm (*bf)(Process*, Eterm, BeamInstr*) = GET_BIF_ADDRESS(Arg(0));
-	Eterm result;
-	BeamInstr *next;
-
-	c_p->fcalls = FCALLS - 1;
-	if (FCALLS <= 0) {
-	    save_calls(c_p, (Export *) Arg(0));
-	}
-	PreFetch(1, next);
-	PRE_BIF_SWAPOUT(c_p);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	result = (*bf)(c_p, r(0), I);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
-	ERTS_HOLE_CHECK(c_p);
-	POST_BIF_GC_SWAPIN(c_p, result, reg, 1);
-	FCALLS = c_p->fcalls;
-        if (is_value(result)) {
-	    r(0) = result;
-	    CHECK_TERM(r(0));
-	    NextPF(1, next);
-	} else if (c_p->freason == TRAP) {
-	    goto call_bif_trap3;
-	}
-
-	/*
-	 * Error handling.  SWAPOUT is not needed because it was done above.
-	 */
-	ASSERT(c_p->stop == E);
-	reg[0] = r(0);
-	I = handle_error(c_p, I, reg, bf);
-	goto post_error_handling;
-    }
-
- OpCase(call_bif2_e):
+ OpCase(call_bif_e):
     {
-	Eterm (*bf)(Process*, Eterm, Eterm, BeamInstr*) = GET_BIF_ADDRESS(Arg(0));
+	Eterm (*bf)(Process*, Eterm*, BeamInstr*) = GET_BIF_ADDRESS(Arg(0));
 	Eterm result;
 	BeamInstr *next;
 
@@ -2533,61 +2415,29 @@ void process_main(void)
 	   save_calls(c_p, (Export *) Arg(0));
 	}
 	PreFetch(1, next);
-	CHECK_TERM(r(0));
-	CHECK_TERM(x(1));
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	result = (*bf)(c_p, r(0), x(1), I);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
-	ERTS_HOLE_CHECK(c_p);
-	POST_BIF_GC_SWAPIN(c_p, result, reg, 2);
-	FCALLS = c_p->fcalls;
-	if (is_value(result)) {
-	    r(0) = result;
-	    CHECK_TERM(r(0));
-	    NextPF(1, next);
-	} else if (c_p->freason == TRAP) {
-	    goto call_bif_trap3;
-	}
-
-	/*
-	 * Error handling.  SWAPOUT is not needed because it was done above.
-	 */
-	ASSERT(c_p->stop == E);
 	reg[0] = r(0);
-	I = handle_error(c_p, I, reg, bf);
-	goto post_error_handling;
-    }
-
- OpCase(call_bif3_e):
-    {
-	Eterm (*bf)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = GET_BIF_ADDRESS(Arg(0));
-	Eterm result;
-	BeamInstr *next;
-
-	PRE_BIF_SWAPOUT(c_p);
-	c_p->fcalls = FCALLS - 1;
-	if (FCALLS <= 0) {
-	   save_calls(c_p, (Export *) Arg(0));
-	}
-	PreFetch(1, next);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	result = (*bf)(c_p, r(0), x(1), x(2), I);
+	result = (*bf)(c_p, reg, I);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_HOLE_CHECK(c_p);
-	POST_BIF_GC_SWAPIN(c_p, result, reg, 3);
+	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+	PROCESS_MAIN_CHK_LOCKS(c_p);
+	if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) {
+	    Uint arity = ((Export *)Arg(0))->code[2];
+	    result = erts_gc_after_bif_call(c_p, result, reg, arity);
+	    E = c_p->stop;
+	}
+	HTOP = HEAP_TOP(c_p);
 	FCALLS = c_p->fcalls;
 	if (is_value(result)) {
 	    r(0) = result;
 	    CHECK_TERM(r(0));
 	    NextPF(1, next);
 	} else if (c_p->freason == TRAP) {
-	call_bif_trap3:
 	    SET_CP(c_p, I+2);
-	    SET_I(*((BeamInstr **) (UWord) ((c_p)->def_arg_reg + 3)));
+	    SET_I(c_p->i);
 	    SWAPIN;
-	    r(0) = c_p->def_arg_reg[0];
-	    x(1) = c_p->def_arg_reg[1];
-	    x(2) = c_p->def_arg_reg[2];
+	    r(0) = reg[0];
 	    Dispatch();
 	}
 
@@ -2595,7 +2445,6 @@ void process_main(void)
 	 * Error handling.  SWAPOUT is not needed because it was done above.
 	 */
 	ASSERT(c_p->stop == E);
-	reg[0] = r(0);
 	I = handle_error(c_p, I, reg, bf);
 	goto post_error_handling;
     }
@@ -3348,64 +3197,23 @@ void process_main(void)
 	    ASSERT(bif_nif_arity <= 3);
 	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
 	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-	    switch (bif_nif_arity) {
-	    case 3:
-		{
-		    Eterm (*bf)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = vbf;
-		    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-		    nif_bif_result = (*bf)(c_p, r(0), x(1), x(2), I);
-		    ASSERT(!ERTS_PROC_IS_EXITING(c_p) ||
-			   is_non_value(nif_bif_result));
-		    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-		    PROCESS_MAIN_CHK_LOCKS(c_p);
-		}
-		break;
-	    case 2:
-		{
-		    Eterm (*bf)(Process*, Eterm, Eterm, BeamInstr*) = vbf;
-		    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-		    nif_bif_result = (*bf)(c_p, r(0), x(1), I);
-		    ASSERT(!ERTS_PROC_IS_EXITING(c_p) ||
-			   is_non_value(nif_bif_result));
-		    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-		    PROCESS_MAIN_CHK_LOCKS(c_p);
-		}
-		break;
-	    case 1:
-		{
-		    Eterm (*bf)(Process*, Eterm, BeamInstr*) = vbf;
-		    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-		    nif_bif_result = (*bf)(c_p, r(0), I);
-		    ASSERT(!ERTS_PROC_IS_EXITING(c_p) ||
-			   is_non_value(nif_bif_result));
-		    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-		    PROCESS_MAIN_CHK_LOCKS(c_p);
-		}
-		break;
-	    case 0:
-		{
-		    Eterm (*bf)(Process*, BeamInstr*) = vbf;
-		    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-		    nif_bif_result = (*bf)(c_p, I);
-		    ASSERT(!ERTS_PROC_IS_EXITING(c_p) ||
-			   is_non_value(nif_bif_result));
-		    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-		    PROCESS_MAIN_CHK_LOCKS(c_p);
-		    break;
-		}
-	    default:
-		erl_exit(1, "apply_bif: invalid arity: %u\n",
-			 bif_nif_arity);
+	    reg[0] = r(0);
+	    {
+		Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf;
+		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+		nif_bif_result = (*bf)(c_p, reg, I);
+		ASSERT(!ERTS_PROC_IS_EXITING(c_p) ||
+		       is_non_value(nif_bif_result));
+		ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+		PROCESS_MAIN_CHK_LOCKS(c_p);
 	    }
 
 	apply_bif_or_nif_epilogue:
 	    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
 	    ERTS_HOLE_CHECK(c_p);
 	    if (c_p->mbuf) {
-		reg[0] = r(0);
 		nif_bif_result = erts_gc_after_bif_call(c_p, nif_bif_result,
 						  reg, bif_nif_arity);
-		r(0) = reg[0];
 	    }
 	    SWAPIN;  /* There might have been a garbage collection. */
 	    FCALLS = c_p->fcalls;
@@ -3416,17 +3224,14 @@ void process_main(void)
 		c_p->cp = 0;
 		Goto(*I);
 	    } else if (c_p->freason == TRAP) {
-		SET_I(*((BeamInstr **) (UWord) ((c_p)->def_arg_reg + 3)));
-		r(0) = c_p->def_arg_reg[0];
-		x(1) = c_p->def_arg_reg[1];
-		x(2) = c_p->def_arg_reg[2];
+		SET_I(c_p->i);
+		r(0) = reg[0];
 		if (c_p->flags & F_HIBERNATE_SCHED) {
 		    c_p->flags &= ~F_HIBERNATE_SCHED;
 		    goto do_schedule;
 		}
 		Dispatch();
 	    }
-	    reg[0] = r(0);
 	    I = handle_error(c_p, c_p->cp, reg, vbf);
 	    goto post_error_handling;
 	}
@@ -3558,7 +3363,7 @@ void process_main(void)
       * Operands: NotUsed Live Dst
       */
  do_bs_init_bits_known:
-     num_bytes = (num_bits+7) >> 3;
+     num_bytes = ((Uint64)num_bits+(Uint64)7) >> 3;
      if (num_bits & 7) {
 	 alloc += ERL_SUB_BIN_SIZE;
      }
@@ -3989,8 +3794,7 @@ void process_main(void)
       * too big numbers).
       */
      if (is_not_small(val) || val > make_small(0x10FFFFUL) ||
-	 (make_small(0xD800UL) <= val && val <= make_small(0xDFFFUL)) ||
-	 val == make_small(0xFFFEUL) || val == make_small(0xFFFFUL)) {
+	 (make_small(0xD800UL) <= val && val <= make_small(0xDFFFUL))) {
 	 goto badarg;
      }
      Next(2);
@@ -4009,8 +3813,8 @@ void process_main(void)
       * the valid range).
       */
      if (is_not_small(tmp_arg1) || tmp_arg1 > make_small(0x10FFFFUL) ||
-	 (make_small(0xD800UL) <= tmp_arg1 && tmp_arg1 <= make_small(0xDFFFUL)) ||
-	 tmp_arg1 == make_small(0xFFFEUL) || tmp_arg1 == make_small(0xFFFFUL)) {
+	 (make_small(0xD800UL) <= tmp_arg1 &&
+	  tmp_arg1 <= make_small(0xDFFFUL))) {
 	 ErlBinMatchBuffer *mb = ms_matchbuffer(tmp_arg2);
 
 	 mb->offset -= 32;
@@ -4885,92 +4689,6 @@ void process_main(void)
  }
 
  /*
-  * Instructions for allocating on the message area.
-  */
-
- OpCase(i_global_cons):
- {
-     BeamInstr *next;
-#ifdef HYBRID
-     Eterm *hp;
-
-     PreFetch(0,next);
-     TestGlobalHeap(2,2,hp);
-     hp[0] = r(0);
-     hp[1] = x(1);
-     r(0) = make_list(hp);
-#ifndef INCREMENTAL
-     global_htop += 2;
-#endif
-     NextPF(0,next);
-#else
-     PreFetch(0,next);
-     c_p->freason = EXC_INTERNAL_ERROR;
-     goto find_func_info;
-#endif
- }
-
- OpCase(i_global_tuple):
- {
-     BeamInstr *next;
-     int len;
-#ifdef HYBRID
-     Eterm list;
-     Eterm *hp;
-#endif
-
-     if ((len = list_length(r(0))) < 0) {
-         goto badarg;
-     }
-
-     PreFetch(0,next);
-#ifdef HYBRID
-     TestGlobalHeap(len + 1,1,hp);
-     list = r(0);
-     r(0) = make_tuple(hp);
-     *hp++ = make_arityval(len);
-     while(is_list(list))
-     {
-         Eterm* cons = list_val(list);
-         *hp++ = CAR(cons);
-         list = CDR(cons);
-     }
-#ifndef INCREMENTAL
-     global_htop += len + 1;
-#endif
-     NextPF(0,next);
-#else
-     c_p->freason = EXC_INTERNAL_ERROR;
-     goto find_func_info;
-#endif
- }
-
- OpCase(i_global_copy):
- {
-     BeamInstr *next;
-     PreFetch(0,next);
-#ifdef HYBRID
-     if (!IS_CONST(r(0)))
-     {
-         BM_SWAP_TIMER(system,copy);
-         SWAPOUT;
-         reg[0] = r(0);
-         reg[1] = NIL;
-         r(0) = copy_struct_lazy(c_p,r(0),0);
-         ASSERT(ma_src_top == 0);
-         ASSERT(ma_dst_top == 0);
-         ASSERT(ma_offset_top == 0);
-         SWAPIN;
-         BM_SWAP_TIMER(copy,system);
-     }
-     NextPF(0,next);
-#else
-     c_p->freason = EXC_INTERNAL_ERROR;
-     goto find_func_info;
-#endif
- }
-
- /*
   * New floating point instructions.
   */
 
@@ -5148,10 +4866,8 @@ void process_main(void)
      c_p->def_arg_reg[4] = -neg_o_reds;
      reg[0] = r(0);
      c_p = hipe_mode_switch(c_p, cmd, reg);
-#ifdef ERTS_SMP
-     reg = c_p->scheduler_data->save_reg;
-     freg = c_p->scheduler_data->freg;
-#endif
+     reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array;
+     freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array;
      ERL_BITS_RELOAD_STATEP(c_p);
      neg_o_reds = -c_p->def_arg_reg[4];
      FCALLS = c_p->fcalls;
@@ -5265,8 +4981,8 @@ void process_main(void)
 
  OpCase(int_code_end):
  OpCase(label_L):
- OpCase(too_old_compiler):
  OpCase(on_load):
+ OpCase(line_I):
     erl_exit(1, "meta op\n");
 
     /*
@@ -5322,7 +5038,7 @@ void process_main(void)
 	 ep->code[3] = (BeamInstr) OpCode(apply_bif);
 	 ep->code[4] = (BeamInstr) bif_table[i].f;
 	 /* XXX: set func info for bifs */
-	 ((BeamInstr*)ep->code + 3)[-5] = (BeamInstr) BeamOp(op_i_func_info_IaaI);
+	 ep->fake_op_func_info_for_hipe[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI);
      }
 
      return;
@@ -5683,6 +5399,25 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) {
  * that c_p->ftrace will point to a cons cell which holds the given args
  * and the saved data (encoded as a bignum).
  *
+ * There is an issue with line number information. Line number
+ * information is associated with the address *before* an operation
+ * that may fail or be stored stored on the stack. But continuation
+ * pointers point after its call instruction, not before. To avoid
+ * finding the wrong line number, we'll need to adjust them so that
+ * they point at the beginning of the call instruction or inside the
+ * call instruction. Since its impractical to point at the beginning,
+ * we'll do the simplest thing and decrement the continuation pointers
+ * by one.
+ *
+ * Here is an example of what can go wrong. Without the adjustment
+ * of continuation pointers, the call at line 42 below would seem to
+ * be at line 43:
+ *
+ * line 42
+ * call ...
+ * line 43
+ * gc_bif ...
+ *
  * (It would be much better to put the arglist - when it exists - in the
  * error value instead of in the actual trace; e.g. '{badarg, Args}'
  * instead of using 'badarg' with Args in the trace. The arglist may
@@ -5749,7 +5484,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 	}
 	/* Save second stack entry if CP is valid and different from pc */
 	if (depth > 0 && c_p->cp != 0 && c_p->cp != pc) {
-	    s->trace[s->depth++] = c_p->cp;
+	    s->trace[s->depth++] = c_p->cp - 1;
 	    depth--;
 	}
 	s->pc = NULL;
@@ -5769,13 +5504,13 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 	    /* Save first stack entry */
 	    ASSERT(c_p->cp);
 	    if (depth > 0) {
-		s->trace[s->depth++] = c_p->cp;
+		s->trace[s->depth++] = c_p->cp - 1;
 		depth--;
 	    }
 	    s->pc = NULL; /* Ignore pc */
 	} else {
 	    if (depth > 0 && c_p->cp != 0 && c_p->cp != pc) {
-		s->trace[s->depth++] = c_p->cp;
+		s->trace[s->depth++] = c_p->cp - 1;
 		depth--;
 	    }
 	    s->pc = pc;
@@ -5790,24 +5525,31 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
     }
 
     /* Save the actual stack trace */
+    erts_save_stacktrace(c_p, s, depth);
+}
+
+void
+erts_save_stacktrace(Process* p, struct StackTrace* s, int depth)
+{
     if (depth > 0) {
 	Eterm *ptr;
 	BeamInstr *prev = s->depth ? s->trace[s->depth-1] : NULL;
 	BeamInstr i_return_trace = beam_return_trace[0];
 	BeamInstr i_return_to_trace = beam_return_to_trace[0];
+
 	/*
 	 * Traverse the stack backwards and add all unique continuation
 	 * pointers to the buffer, up to the maximum stack trace size.
 	 * 
 	 * Skip trace stack frames.
 	 */
-	ptr = c_p->stop;
-	if (ptr < STACK_START(c_p) 
-	    && (is_not_CP(*ptr)|| (*cp_val(*ptr) != i_return_trace &&
-				   *cp_val(*ptr) != i_return_to_trace))
-	    && c_p->cp) {
-	    /* Can not follow cp here - code may be unloaded */
-	    BeamInstr *cpp = c_p->cp;
+	ptr = p->stop;
+	if (ptr < STACK_START(p) &&
+	    (is_not_CP(*ptr)|| (*cp_val(*ptr) != i_return_trace &&
+				*cp_val(*ptr) != i_return_to_trace)) &&
+	    p->cp) {
+	    /* Cannot follow cp here - code may be unloaded */
+	    BeamInstr *cpp = p->cp;
 	    if (cpp == beam_exception_trace || cpp == beam_return_trace) {
 		/* Skip return_trace parameters */
 		ptr += 2;
@@ -5816,7 +5558,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 		ptr += 1;
 	    }
 	}
-	while (ptr < STACK_START(c_p) && depth > 0) {
+	while (ptr < STACK_START(p) && depth > 0) {
 	    if (is_CP(*ptr)) {
 		if (*cp_val(*ptr) == i_return_trace) {
 		    /* Skip stack frame variables */
@@ -5831,7 +5573,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 		    if (cp != prev) {
 			/* Record non-duplicates only */
 			prev = cp;
-			s->trace[s->depth++] = cp;
+			s->trace[s->depth++] = cp - 1;
 			depth--;
 		    }
 		    ptr++;
@@ -5899,9 +5641,14 @@ build_stacktrace(Process* c_p, Eterm exc) {
     struct StackTrace* s;
     Eterm  args;
     int    depth;
-    BeamInstr* current;
-    Eterm  Where = NIL;
-    Eterm *next_p = &Where;
+    FunctionInfo fi;
+    FunctionInfo* stk;
+    FunctionInfo* stkp;
+    Eterm res = NIL;
+    Uint heap_size;
+    Eterm* hp;
+    Eterm mfa;
+    int i;
 
     if (! (s = get_trace_from_exc(exc))) {
         return NIL;
@@ -5920,64 +5667,56 @@ build_stacktrace(Process* c_p, Eterm exc) {
      * saved s->current should already contain the proper value.
      */
     if (s->pc != NULL) {
-	current = find_function_from_pc(s->pc);
+	erts_lookup_function_info(&fi, s->pc, 1);
+    } else if (GET_EXC_INDEX(s->freason) ==
+	       GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) {
+	erts_lookup_function_info(&fi, s->current, 1);
     } else {
-	current = s->current;
+	erts_set_current_function(&fi, s->current);
     }
+
     /*
-     * If current is still NULL, default to the initial function
+     * If fi.current is still NULL, default to the initial function
      * (e.g. spawn_link(erlang, abs, [1])).
      */
-    if (current == NULL) {
-	current = c_p->initial;
+    if (fi.current == NULL) {
+	erts_set_current_function(&fi, c_p->initial);
 	args = am_true; /* Just in case */
     } else {
 	args = get_args_from_exc(exc);
     }
 
-    depth = s->depth;
-    
     /*
-     * Add the {M,F,A} for the current function 
-     * (where A is arity or [Argument]).
+     * Look up all saved continuation pointers and calculate
+     * needed heap space.
      */
-    {
-	int i;
-	Eterm mfa;
-	Uint heap_size = 6*(depth+1);
-	Eterm* hp = HAlloc(c_p, heap_size);
-	Eterm* hp_end = hp + heap_size;
-
-	if (args != am_true) {
-	    /* We have an arglist - use it */
-	    mfa = TUPLE3(hp, current[0], current[1], args);
-	} else {
-	    Eterm arity = make_small(current[2]);
-	    mfa = TUPLE3(hp, current[0], current[1], arity);
+    depth = s->depth;
+    stk = stkp = (FunctionInfo *) erts_alloc(ERTS_ALC_T_TMP,
+				      depth*sizeof(FunctionInfo));
+    heap_size = fi.needed + 2;
+    for (i = 0; i < depth; i++) {
+	erts_lookup_function_info(stkp, s->trace[i], 1);
+	if (stkp->current) {
+	    heap_size += stkp->needed + 2;
+	    stkp++;
 	}
-	hp += 4;
-	ASSERT(*next_p == NIL);
-	*next_p = CONS(hp, mfa, NIL);
-	next_p = &CDR(list_val(*next_p));
-	hp += 2;
+    }
 
-	/* 
-	 * Finally, we go through the saved continuation pointers.
-	 */
-	for (i = 0; i < depth; i++) {
-	    BeamInstr *fi = find_function_from_pc((BeamInstr *) s->trace[i]);
-	    if (fi == NULL) continue;
-	    mfa = TUPLE3(hp, fi[0], fi[1], make_small(fi[2]));
-	    hp += 4;
-	    ASSERT(*next_p == NIL);
-	    *next_p = CONS(hp, mfa, NIL);
-	    next_p = &CDR(list_val(*next_p));
-	    hp += 2;
-	}
-	ASSERT(hp <= hp_end);
-	HRelease(c_p, hp_end, hp);
+    /*
+     * Allocate heap space and build the stacktrace.
+     */
+    hp = HAlloc(c_p, heap_size);
+    while (stkp > stk) {
+	stkp--;
+	hp = erts_build_mfa_item(stkp, hp, am_true, &mfa);
+	res = CONS(hp, mfa, res);
+	hp += 2;
     }
-    return Where;
+    hp = erts_build_mfa_item(&fi, hp, args, &mfa);
+    res = CONS(hp, mfa, res);
+
+    erts_free(ERTS_ALC_T_TMP, (void *) stk);
+    return res;
 }