Merge branch 'bjorn/erts/improve-beam-ops'

* bjorn/erts/improve-beam-ops:
  Move out variables from the head of combined instructions
  Change operand from 'P' to 'Q' for i_apply_last and i_apply_fun_last
  Add CHECK_ALIGNED() for testing storage destinations
  instrs.tab: Add missing -no_next directives
  beam_load.c: Generalize the 'P' operator in the packing engine
  Break out most of the initialization from process_main()
  Eliminate the OpCode() macro
  Eliminate unnecessary and inconsistent casts
  Refactor macros for accessing Beam instructions
  beam_emu: Make order of macros consistent
  beam_SUITE: Strengthen test of packed registers

25 files changed, 237 insertions, 229 deletions

diff --git a/erts/emulator/beam/arith_instrs.tab b/erts/emulator/beam/arith_instrs.tab
index 3db19df3c4..b828e86788 100644
--- a/erts/emulator/beam/arith_instrs.tab
+++ b/erts/emulator/beam/arith_instrs.tab
@@ -85,9 +85,6 @@ i_increment := increment.fetch.execute;
 
 increment.head() {
     Eterm increment_reg_val;
-    Eterm increment_val;
-    Uint live;
-    Eterm result;
 }
 
 increment.fetch(Src) {
@@ -95,7 +92,10 @@ increment.fetch(Src) {
 }
 
 increment.execute(IncrementVal, Live, Dst) {
-    increment_val = $IncrementVal;
+    Eterm increment_val = $IncrementVal;
+    Uint live;
+    Eterm result;
+
     if (ERTS_LIKELY(is_small(increment_reg_val))) {
         Sint i = signed_val(increment_reg_val) + increment_val;
         if (ERTS_LIKELY(IS_SSMALL(i))) {
@@ -366,10 +366,10 @@ shift.execute(Fail, Live, Dst) {
         reg[0] = Op1;
         reg[1] = Op2;
         SWAPOUT;
-        if (I[0] == (BeamInstr) OpCode(i_bsl_ssjtd)) {
+        if (IsOpCode(I[0], i_bsl_ssjtd)) {
             I = handle_error(c_p, I, reg, &bif_export[BIF_bsl_2]->info.mfa);
         } else {
-            ASSERT(I[0] == (BeamInstr) OpCode(i_bsr_ssjtd));
+            ASSERT(IsOpCode(I[0], i_bsr_ssjtd));
             I = handle_error(c_p, I, reg, &bif_export[BIF_bsr_2]->info.mfa);
         }
         goto post_error_handling;
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index e48415ecc4..00822d1415 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -823,11 +823,11 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 		ep->beam[1] = 0;
 	    } else {
 		if (ep->addressv[code_ix] == ep->beam &&
-		    ep->beam[0] == (BeamInstr) em_apply_bif) {
+		    BeamIsOpCode(ep->beam[0], op_apply_bif)) {
 		    continue;
 		}
-		ep->addressv[code_ix] = ep->beam;
-		ep->beam[0] = (BeamInstr) em_call_error_handler;
+                ep->addressv[code_ix] = ep->beam;
+                ep->beam[0] = BeamOpCodeAddr(op_call_error_handler);
 	    }
 	}
 	modp->curr.code_hdr->on_load_function_ptr = NULL;
@@ -849,7 +849,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 	    if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) {
 		continue;
 	    }
-	    if (ep->beam[0] == (BeamInstr) em_apply_bif) {
+	    if (BeamIsOpCode(ep->beam[0], op_apply_bif)) {
 		continue;
 	    }
 	    ep->beam[1] = 0;
@@ -1766,22 +1766,23 @@ delete_code(Module* modp)
 	Export *ep = export_list(i, code_ix);
         if (ep != NULL && (ep->info.mfa.module == module)) {
 	    if (ep->addressv[code_ix] == ep->beam) {
-		if (ep->beam[0] == (BeamInstr) em_apply_bif) {
+		if (BeamIsOpCode(ep->beam[0], op_apply_bif)) {
 		    continue;
 		}
-		else if (ep->beam[0] ==
-			 (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
+		else if (BeamIsOpCode(ep->beam[0], op_i_generic_breakpoint)) {
 		    ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
 		    ASSERT(modp->curr.num_traced_exports > 0);
 		    DBG_TRACE_MFA_P(&ep->info.mfa,
 				  "export trace cleared, code_ix=%d", code_ix);
 		    erts_clear_export_break(modp, &ep->info);
 		}
-		else ASSERT(ep->beam[0] == (BeamInstr) em_call_error_handler
-			    || !erts_initialized);
-	    }
+		else {
+                    ASSERT(BeamIsOpCode(ep->beam[0], op_call_error_handler) ||
+                           !erts_initialized);
+                }
+            }
 	    ep->addressv[code_ix] = ep->beam;
-	    ep->beam[0] = (BeamInstr) em_call_error_handler;
+	    ep->beam[0] = BeamOpCodeAddr(op_call_error_handler);
 	    ep->beam[1] = 0;
 	    DBG_TRACE_MFA_P(&ep->info.mfa,
 			    "export invalidation, code_ix=%d", code_ix);
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 49ec59c989..c81380c14d 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -203,7 +203,7 @@ erts_bp_match_functions(BpFunctions* f, ErtsCodeMFA *mfa, int specified)
 	for (fi = 0; fi < num_functions; fi++) {
 
 	    ci = code_hdr->functions[fi];
-	    ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+	    ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
 	    if (erts_is_function_native(ci)) {
 		continue;
 	    }
@@ -265,11 +265,11 @@ erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified)
 
 	pc = ep->beam;
 	if (ep->addressv[code_ix] == pc) {
-	    if ((*pc == (BeamInstr) em_apply_bif ||
-		 *pc == (BeamInstr) em_call_error_handler)) {
-		continue;
+	    if (BeamIsOpCode(*pc, op_apply_bif) ||
+                BeamIsOpCode(*pc, op_call_error_handler)) {
+                continue;
 	    }
-	    ASSERT(*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint));
+	    ASSERT(BeamIsOpCode(*pc, op_i_generic_breakpoint));
 	} else if (erts_is_function_native(erts_code_to_codeinfo(ep->addressv[code_ix]))) {
 	    continue;
 	}
@@ -366,8 +366,8 @@ consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local)
 	    }
 	    ASSERT(modp->curr.num_breakpoints >= 0);
 	    ASSERT(modp->curr.num_traced_exports >= 0);
-	    ASSERT(*erts_codeinfo_to_code(ci) !=
-                   (BeamInstr) BeamOp(op_i_generic_breakpoint));
+	    ASSERT(! BeamIsOpCode(*erts_codeinfo_to_code(ci),
+                                  op_i_generic_breakpoint));
 	}
 	ci->u.gen_bp = NULL;
 	Free(g);
@@ -415,7 +415,7 @@ erts_install_breakpoints(BpFunctions* f)
 {
     Uint i;
     Uint n = f->matched;
-    BeamInstr br = (BeamInstr) BeamOp(op_i_generic_breakpoint);
+    BeamInstr br = BeamOpCodeAddr(op_i_generic_breakpoint);
 
     for (i = 0; i < n; i++) {
 	ErtsCodeInfo* ci = f->matching[i].ci;
@@ -460,7 +460,7 @@ static void
 uninstall_breakpoint(ErtsCodeInfo *ci)
 {
     BeamInstr *pc = erts_codeinfo_to_code(ci);
-    if (*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
+    if (BeamIsOpCode(*pc, op_i_generic_breakpoint)) {
 	GenericBp* g = ci->u.gen_bp;
 	if (g->data[erts_active_bp_ix()].flags == 0) {
 	    /*
@@ -642,7 +642,7 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg)
     Uint bp_flags;
     ErtsBpIndex ix = erts_active_bp_ix();
 
-    ASSERT(info->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+    ASSERT(BeamIsOpCode(info->op, op_i_func_info_IaaI));
 
     g = info->u.gen_bp;
     bp = &g->data[ix];
@@ -695,9 +695,9 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg)
 	Eterm w;
 	erts_trace_time_call(c_p, info, bp->time);
 	w = (BeamInstr) *c_p->cp;
-	if (! (w == (BeamInstr) BeamOp(op_i_return_time_trace) ||
-	       w == (BeamInstr) BeamOp(op_return_trace) ||
-	       w == (BeamInstr) BeamOp(op_i_return_to_trace)) ) {
+	if (! (BeamIsOpCode(w, op_i_return_time_trace) ||
+	       BeamIsOpCode(w, op_return_trace) ||
+               BeamIsOpCode(w, op_i_return_to_trace)) ) {
 	    Eterm* E = c_p->stop;
 	    ASSERT(c_p->htop <= E && E <= c_p->hend);
 	    if (E - 2 < c_p->htop) {
@@ -717,7 +717,7 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg)
     }
 
     if (bp_flags & ERTS_BPF_DEBUG) {
-	return (BeamInstr) BeamOp(op_i_debug_breakpoint);
+	return BeamOpCodeAddr(op_i_debug_breakpoint);
     } else {
 	return g->orig_instr;
     }
@@ -946,12 +946,12 @@ do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg,
     Eterm* E = c_p->stop;
 
     w = *c_p->cp;
-    if (w == (BeamInstr) BeamOp(op_return_trace)) {
+    if (BeamIsOpCode(w, op_return_trace)) {
 	cpp = &E[2];
-    } else if (w == (BeamInstr) BeamOp(op_i_return_to_trace)) {
+    } else if (BeamIsOpCode(w, op_i_return_to_trace)) {
 	return_to_trace = 1;
 	cpp = &E[0];
-    } else if (w == (BeamInstr) BeamOp(op_i_return_time_trace)) {
+    } else if (BeamIsOpCode(w, op_i_return_time_trace)) {
 	cpp = &E[0];
     } else {
 	cpp = NULL;
@@ -959,12 +959,12 @@ do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg,
     if (cpp) {
 	for (;;) {
 	    BeamInstr w = *cp_val(*cpp);
-	    if (w == (BeamInstr) BeamOp(op_return_trace)) {
+	    if (BeamIsOpCode(w, op_return_trace)) {
 		cpp += 3;
-	    } else if (w == (BeamInstr) BeamOp(op_i_return_to_trace)) {
+	    } else if (BeamIsOpCode(w, op_i_return_to_trace)) {
 		return_to_trace = 1;
 		cpp += 1;
-	    } else if (w == (BeamInstr) BeamOp(op_i_return_time_trace)) {
+            } else if (BeamIsOpCode(w, op_i_return_time_trace)) {
 		cpp += 2;
 	    } else {
 		break;
@@ -1293,7 +1293,7 @@ erts_find_local_func(ErtsCodeMFA *mfa) {
     n = (BeamInstr) code_hdr->num_functions;
     for (i = 0; i < n; ++i) {
 	ci = code_hdr->functions[i];
-	ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == ci->op);
+	ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
 	ASSERT(mfa->module == ci->mfa.module || is_nil(ci->mfa.module));
 	if (mfa->function == ci->mfa.function &&
 	    mfa->arity == ci->mfa.arity) {
@@ -1708,7 +1708,7 @@ check_break(ErtsCodeInfo *ci, Uint break_flags)
 {
     GenericBp* g = ci->u.gen_bp;
 
-    ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+    ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
     if (erts_is_function_native(ci)) {
 	return 0;
     }
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 7819e9907d..0f332da63f 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -199,7 +199,7 @@ void debug_dump_code(BeamInstr *I, int num)
 	erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr);
 	instr = (BeamInstr) code_ptr[0];
 	for (i = 0; i < NUM_SPECIFIC_OPS; i++) {
-	    if (instr == (BeamInstr) BeamOp(i) && opc[i].name[0] != '\0') {
+	    if (BeamIsOpCode(instr, i) && opc[i].name[0] != '\0') {
 		code_ptr += print_op(ERTS_PRINT_DSBUF, (void *) dsbufp,
 				     i, opc[i].sz-1, code_ptr+1) + 1;
 		break;
@@ -319,7 +319,7 @@ erts_debug_disassemble_1(BIF_ALIST_1)
     erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr);
     instr = (BeamInstr) code_ptr[0];
     for (i = 0; i < NUM_SPECIFIC_OPS; i++) {
-	if (instr == (BeamInstr) BeamOp(i) && opc[i].name[0] != '\0') {
+	if (BeamIsOpCode(instr, i) && opc[i].name[0] != '\0') {
 	    code_ptr += print_op(ERTS_PRINT_DSBUF, (void *) dsbufp,
 				 i, opc[i].sz-1, code_ptr+1) + 1;
 	    break;
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 81c4417b1e..9a77c63390 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -50,15 +50,14 @@
 #if defined(NO_JUMP_TABLE)
 #  define OpCase(OpCode)    case op_##OpCode
 #  define CountCase(OpCode) case op_count_##OpCode
-#  define OpCode(OpCode)    ((Uint*)op_##OpCode)
-#  define Goto(Rel) {Go = (int)(UWord)(Rel); goto emulator_loop;}
-#  define LabelAddr(Addr) &&##Addr
+#  define IsOpCode(InstrWord, OpCode)  ((InstrWord) == (BeamInstr)op_##OpCode)
+#  define Goto(Rel) {Go = (Rel); goto emulator_loop;}
 #else
 #  define OpCase(OpCode)    lb_##OpCode
 #  define CountCase(OpCode) lb_count_##OpCode
+#  define IsOpCode(InstrWord, OpCode)  ((InstrWord) == (BeamInstr)&&lb_##OpCode)
 #  define Goto(Rel) goto *((void *)Rel)
 #  define LabelAddr(Label) &&Label
-#  define OpCode(OpCode)  (&&lb_##OpCode)
 #endif
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -107,6 +106,8 @@ do {                                     \
 #  define CHECK_ARGS(T)
 #endif
 
+#define CHECK_ALIGNED(Dst) ASSERT((((Uint)&Dst) & (sizeof(Uint)-1)) == 0)
+
 #define GET_BIF_MODULE(p)  (p->info.mfa.module)
 #define GET_BIF_FUNCTION(p)  (p->info.mfa.function)
 #define GET_BIF_ARITY(p)  (p->info.mfa.arity)
@@ -159,11 +160,6 @@ BeamInstr beam_apply[2];
 BeamInstr beam_exit[1];
 BeamInstr beam_continue_exit[1];
 
-BeamInstr* em_call_error_handler;
-BeamInstr* em_apply_bif;
-BeamInstr* em_call_nif;
-BeamInstr* em_call_bif_e;
-
 
 /* NOTE These should be the only variables containing trace instructions.
 **      Sometimes tests are form the instruction value, and sometimes
@@ -255,8 +251,8 @@ void** beam_ops;
 
 #define DispatchMacro()				\
   do {						\
-     BeamInstr* dis_next;				\
-     dis_next = (BeamInstr *) *I;			\
+     BeamInstr dis_next;                        \
+     dis_next = *I;                             \
      CHECK_ARGS(I);				\
      if (FCALLS > 0 || FCALLS > neg_o_reds) {	\
         FCALLS--;				\
@@ -264,12 +260,12 @@ void** beam_ops;
      } else {					\
 	goto context_switch;			\
      }						\
- } while (0)
+ } while (0)                                    \
 
 #define DispatchMacroFun()			\
   do {						\
-     BeamInstr* dis_next;				\
-     dis_next = (BeamInstr *) *I;			\
+     BeamInstr dis_next;                        \
+     dis_next = *I;                             \
      CHECK_ARGS(I);				\
      if (FCALLS > 0 || FCALLS > neg_o_reds) {	\
         FCALLS--;				\
@@ -279,23 +275,23 @@ void** beam_ops;
      }						\
  } while (0)
 
-#define DispatchMacrox()					\
-  do {								\
-     if (FCALLS > 0) {						\
-        Eterm* dis_next;					\
-        SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \
-        dis_next = (Eterm *) *I;				\
-        FCALLS--;						\
-        CHECK_ARGS(I);						\
-        Goto(dis_next);						\
-     } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)		\
-		&& FCALLS > neg_o_reds) {			\
-        goto save_calls1;					\
-     } else {							\
-        SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \
-        CHECK_ARGS(I);						\
-	goto context_switch;					\
-     }								\
+#define DispatchMacrox()                                                \
+  do {                                                                  \
+     if (FCALLS > 0) {                                                  \
+        BeamInstr dis_next;                                             \
+        SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]);    \
+        dis_next = *I;                                                  \
+        FCALLS--;                                                       \
+        CHECK_ARGS(I);                                                  \
+        Goto(dis_next);                                                 \
+     } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)                      \
+		&& FCALLS > neg_o_reds) {                               \
+        goto save_calls1;                                               \
+     } else {                                                           \
+        SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]);    \
+        CHECK_ARGS(I);                                                  \
+	goto context_switch;                                            \
+     }                                                                  \
  } while (0)
 
 #ifdef DEBUG
@@ -382,6 +378,7 @@ do {                                            \
  * The following functions are called directly by process_main().
  * Don't inline them.
  */
+static void init_emulator_finish(void) NOINLINE;
 static ErtsCodeMFA *ubif2mfa(void* uf) NOINLINE;
 static ErtsCodeMFA *gcbif2mfa(void* gcf) NOINLINE;
 static BeamInstr* handle_error(Process* c_p, BeamInstr* pc,
@@ -627,7 +624,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 #ifndef NO_JUMP_TABLE
     static void* opcodes[] = { DEFINE_OPCODES };
 #else
-    int Go;
+    register BeamInstr Go;
 #endif
 #endif
 
@@ -703,7 +700,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
     {
 	int reds;
 	Eterm* argp;
-	BeamInstr *next;
+	BeamInstr next;
 	int i;
 
 	argp = c_p->arg_reg;
@@ -735,7 +732,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 
 	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 
-	next = (BeamInstr *) *I;
+	next = *I;
 	SWAPIN;
 	ASSERT(VALID_INSTR(next));
 
@@ -963,9 +960,6 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 
  init_emulator:
  {
-     int i;
-     Export* ep;
-
 #ifndef NO_JUMP_TABLE
 #ifdef ERTS_OPCODE_COUNTER_SUPPORT
 #ifdef DEBUG
@@ -977,35 +971,8 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
      beam_ops = opcodes;
 #endif /* ERTS_OPCODE_COUNTER_SUPPORT */
 #endif /* NO_JUMP_TABLE */
-     
-     em_call_error_handler = OpCode(call_error_handler);
-     em_apply_bif = OpCode(apply_bif);
-     em_call_nif = OpCode(call_nif);
-     em_call_bif_e = OpCode(call_bif_e);
-
-     beam_apply[0]             = (BeamInstr) OpCode(i_apply);
-     beam_apply[1]             = (BeamInstr) OpCode(normal_exit);
-     beam_exit[0]              = (BeamInstr) OpCode(error_action_code);
-     beam_continue_exit[0]     = (BeamInstr) OpCode(continue_exit);
-     beam_return_to_trace[0]   = (BeamInstr) OpCode(i_return_to_trace);
-     beam_return_trace[0]      = (BeamInstr) OpCode(return_trace);
-     beam_exception_trace[0]   = (BeamInstr) OpCode(return_trace); /* UGLY */
-     beam_return_time_trace[0] = (BeamInstr) OpCode(i_return_time_trace);
-
-     /*
-      * Enter all BIFs into the export table.
-      */
-     for (i = 0; i < BIF_SIZE; i++) {
-	 ep = erts_export_put(bif_table[i].module,
-			      bif_table[i].name,
-			      bif_table[i].arity);
-	 bif_export[i] = ep;
-	 ep->beam[0] = (BeamInstr) OpCode(apply_bif);
-	 ep->beam[1] = (BeamInstr) bif_table[i].f;
-	 /* XXX: set func info for bifs */
-	 ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI);
-     }
 
+     init_emulator_finish();
      return;
  }
 #ifdef NO_JUMP_TABLE
@@ -1017,19 +984,53 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 
   save_calls1:
     {
-	Eterm* dis_next;
+	BeamInstr dis_next;
 
 	save_calls(c_p, (Export *) Arg(0));
 
 	SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]);
 
-	dis_next = (Eterm *) *I;
+	dis_next = *I;
 	FCALLS--;
 	Goto(dis_next);
     }
 }
 
 /*
+ * One-time initialization of emulator. Does not need to be
+ * in process_main().
+ */
+static void
+init_emulator_finish(void)
+{
+     int i;
+     Export* ep;
+
+     beam_apply[0]             = BeamOpCodeAddr(op_i_apply);
+     beam_apply[1]             = BeamOpCodeAddr(op_normal_exit);
+     beam_exit[0]              = BeamOpCodeAddr(op_error_action_code);
+     beam_continue_exit[0]     = BeamOpCodeAddr(op_continue_exit);
+     beam_return_to_trace[0]   = BeamOpCodeAddr(op_i_return_to_trace);
+     beam_return_trace[0]      = BeamOpCodeAddr(op_return_trace);
+     beam_exception_trace[0]   = BeamOpCodeAddr(op_return_trace); /* UGLY */
+     beam_return_time_trace[0] = BeamOpCodeAddr(op_i_return_time_trace);
+
+     /*
+      * Enter all BIFs into the export table.
+      */
+     for (i = 0; i < BIF_SIZE; i++) {
+	 ep = erts_export_put(bif_table[i].module,
+			      bif_table[i].name,
+			      bif_table[i].arity);
+	 bif_export[i] = ep;
+	 ep->beam[0] = BeamOpCodeAddr(op_apply_bif);
+	 ep->beam[1] = (BeamInstr) bif_table[i].f;
+	 /* XXX: set func info for bifs */
+	 ep->info.op = BeamOpCodeAddr(op_i_func_info_IaaI);
+     }
+}
+
+/*
  * erts_dirty_process_main() is what dirty schedulers execute. Since they handle
  * only NIF calls they do not need to be able to execute all BEAM
  * instructions.
@@ -1250,12 +1251,12 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 	ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
 
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	if (em_apply_bif == (BeamInstr *) *I) {
+	if (BeamIsOpCode(*I, op_apply_bif)) {
 	    exiting = erts_call_dirty_bif(esdp, c_p, I, reg);
 	}
 	else {
-	    ASSERT(em_call_nif == (BeamInstr *) *I);
-	    exiting = erts_call_dirty_nif(esdp, c_p, I, reg);
+	    ASSERT(BeamIsOpCode(*I, op_call_nif));
+            exiting = erts_call_dirty_nif(esdp, c_p, I, reg);
 	}
 
 	ASSERT(!(c_p->flags & F_HIBERNATE_SCHED));
@@ -2090,8 +2091,8 @@ apply_bif_error_adjustment(Process *p, Export *ep,
      * and apply_last_IP.
      */
     if (I
-	&& ep->beam[0] == (BeamInstr) em_apply_bif
-	&& (ep == bif_export[BIF_error_1]
+	&& BeamIsOpCode(ep->beam[0], op_apply_bif)
+        && (ep == bif_export[BIF_error_1]
 	    || ep == bif_export[BIF_error_2]
 	    || ep == bif_export[BIF_exit_1]
 	    || ep == bif_export[BIF_throw_1])) {
@@ -3197,8 +3198,8 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity)
     if ((ep = export_get(&e)) == NULL) {
 	return 0;
     }
-    return ep->addressv[erts_active_code_ix()] == ep->beam
-	&& (ep->beam[0] == (BeamInstr) em_apply_bif);
+    return ep->addressv[erts_active_code_ix()] == ep->beam &&
+	BeamIsOpCode(ep->beam[0], op_apply_bif);
 }
 
 
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 3f9dc2c1aa..9835b1c096 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -55,12 +55,6 @@ ErlDrvBinary* erts_gzinflate_buffer(char*, int);
 #define DEFINED   1
 #define EXPORTED  2
 
-#ifdef NO_JUMP_TABLE
-#  define BeamOpCode(Op) ((BeamInstr)(Op))
-#else
-#  define BeamOpCode(Op) ((BeamInstr)beam_ops[Op])
-#endif
-
 #if defined(WORDS_BIGENDIAN)
 # define NATIVE_ENDIAN(F)			\
   if ((F).val & BSF_NATIVE) {			\
@@ -849,10 +843,9 @@ erts_finish_loading(Binary* magic, Process* c_p,
 		continue;
 	    }
 	    if (ep->addressv[code_ix] == ep->beam) {
-		if (ep->beam[0] == (BeamInstr) em_apply_bif) {
+		if (BeamIsOpCode(ep->beam[0], op_apply_bif)) {
 		    continue;
-		} else if (ep->beam[0] ==
-			   (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
+		} else if (BeamIsOpCode(ep->beam[0], op_i_generic_breakpoint)) {
 		    ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
 		    ASSERT(mod_tab_p->curr.num_traced_exports > 0);
 		    erts_clear_export_break(mod_tab_p, &ep->info);
@@ -1482,7 +1475,7 @@ load_import_table(LoaderState* stp)
 	 * the BIF function.
 	 */
 	if ((e = erts_active_export_entry(mod, func, arity)) != NULL) {
-	    if (e->beam[0] == (BeamInstr) em_apply_bif) {
+	    if (BeamIsOpCode(e->beam[0], op_apply_bif)) {
 		stp->import[i].bf = (BifFunction) e->beam[1];
 		if (func == am_load_nif && mod == am_erlang && arity == 2) {
 		    stp->may_load_nif = 1;
@@ -1576,7 +1569,7 @@ is_bif(Eterm mod, Eterm func, unsigned arity)
     if (e == NULL) {
 	return 0;
     }
-    if (e->beam[0] != (BeamInstr) em_apply_bif) {
+    if (! BeamIsOpCode(e->beam[0], op_apply_bif)) {
 	return 0;
     }
     if (mod == am_erlang && func == am_apply && arity == 3) {
@@ -2323,7 +2316,7 @@ load_code(LoaderState* stp)
 	    stp->specific_op = specific;
 	    CodeNeed(opc[stp->specific_op].sz+16); /* Extra margin for packing */
             last_instr_start = ci + opc[stp->specific_op].adjust;
-	    code[ci++] = BeamOpCode(stp->specific_op);
+	    code[ci++] = BeamOpCodeAddr(stp->specific_op);
 	}
 	
 	/*
@@ -2643,15 +2636,15 @@ load_code(LoaderState* stp)
                     }
                     ci++;
 		    break;
-		case 'P':	/* Put packed operands. */
+		case 'P':	/* Put packed operands (on the stack). */
                     sp->instr = packed;
                     sp->patch_pos = 0;
-                    sp++;
-		    packed = 0;
                     if (packed_label) {
-                        packed_label->pos = ci;
+                        sp->patch_pos = &packed_label->pos;
                         packed_label = 0;
                     }
+                    sp++;
+		    packed = 0;
 		    break;
 		default:
 		    ASSERT(0);
@@ -5079,7 +5072,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
     while (index != 0) {
 	BeamInstr next = codev[index];
         BeamInstr* abs_addr;
-	codev[index] = BeamOpCode(op_catch_yf);
+	codev[index] = BeamOpCodeAddr(op_catch_yf);
         /* We must make the address of the label absolute again. */
         abs_addr = (BeamInstr *)codev + index + codev[index+2];
 	catches = beam_catches_cons(abs_addr, catches);
@@ -6033,9 +6026,9 @@ int
 erts_is_function_native(ErtsCodeInfo *ci)
 {
 #ifdef HIPE
-    ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
-    return erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call)
-	|| erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call_closure);
+    ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
+    return BeamIsOpCode(erts_codeinfo_to_code(ci)[0], op_hipe_trap_call) ||
+	BeamIsOpCode(erts_codeinfo_to_code(ci)[0], op_hipe_trap_call_closure);
 #else
     return 0;
 #endif
@@ -6104,7 +6097,7 @@ exported_from_module(Process* p, /* Process whose heap to use. */
 	    Eterm tuple;
 	    
 	    if (ep->addressv[code_ix] == ep->beam &&
-		ep->beam[0] == (BeamInstr) em_call_error_handler) {
+		BeamIsOpCode(ep->beam[0], op_call_error_handler)) {
 		/* There is a call to the function, but it does not exist. */ 
 		continue;
 	    }
@@ -6375,7 +6368,7 @@ make_stub(ErtsCodeInfo* info, Eterm mod, Eterm func, Uint arity, Uint native, Be
 {
     DBG_TRACE_MFA(mod,func,arity,"make beam stub at %p", erts_codeinfo_to_code(info));
     ASSERT(WORDS_PER_FUNCTION == 6);
-    info->op = (BeamInstr) BeamOp(op_i_func_info_IaaI);
+    info->op = BeamOpCodeAddr(op_i_func_info_IaaI);
     info->u.ncallee = (void (*)(void)) native;
     info->mfa.module = mod;
     info->mfa.function = func;
@@ -6480,7 +6473,7 @@ stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci)
     for (i = 0, lp = stp->lambdas; i < n; i++, lp++) {
         ErlFunEntry* fe = stp->lambdas[i].fe;
 	if (lp->function == ci->mfa.function && lp->arity == ci->mfa.arity) {
-	    *erts_codeinfo_to_code(ci) = (Eterm) BeamOpCode(op_hipe_trap_call_closure);
+	    *erts_codeinfo_to_code(ci) = BeamOpCodeAddr(op_hipe_trap_call_closure);
             fe->address = erts_codeinfo_to_code(ci);
 	}
     }
@@ -6804,7 +6797,7 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info)
 	 * as the body until we know what kind of trap we should put there.
 	 */
 	code_hdr->functions[i] = (ErtsCodeInfo*)fp;
-	op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */
+	op = BeamOpCodeAddr(op_hipe_trap_call); /* Might be changed later. */
 	fp = make_stub((ErtsCodeInfo*)fp, hipe_stp->module, func, arity,
                        (Uint)native_address, op);
     }
@@ -6814,7 +6807,7 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info)
      */
 
     code_hdr->functions[i] = (ErtsCodeInfo*)fp;
-    *fp++ = (BeamInstr) BeamOp(op_int_code_end);
+    *fp++ = BeamOpCodeAddr(op_int_code_end);
 
     /*
      * Copy attributes and compilation information.
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index c4a90d3f3a..156c3c45e2 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -37,11 +37,6 @@ typedef struct gen_op_entry {
 
 extern const GenOpEntry gen_opc[];
 
-extern BeamInstr beam_debug_apply[];
-extern BeamInstr* em_call_error_handler;
-extern BeamInstr* em_apply_bif;
-extern BeamInstr* em_call_nif;
-
 struct ErtsLiteralArea_;
 
 /*
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 80f391e91e..ad555bb195 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -5031,7 +5031,7 @@ void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a,
     ep->info.mfa.module = m;
     ep->info.mfa.function = f;
     ep->info.mfa.arity = a;
-    ep->beam[0] = (BeamInstr) em_apply_bif;
+    ep->beam[0] = BeamOpCodeAddr(op_apply_bif);
     ep->beam[1] = (BeamInstr) bif;
 }
 
@@ -5096,7 +5096,7 @@ schedule(Process *c_p, Process *dirty_shadow_proc,
 {
     ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p));
     (void) erts_nif_export_schedule(c_p, dirty_shadow_proc,
-				    mfa, pc, (BeamInstr) em_apply_bif,
+				    mfa, pc, BeamOpCodeAddr(op_apply_bif),
 				    dfunc, ifunc,
 				    module, function,
 				    argc, argv);
@@ -5145,7 +5145,6 @@ static BIF_RETTYPE dirty_bif_exception(BIF_ALIST_2)
 }
 
 
-extern BeamInstr* em_call_bif_e;
 static BIF_RETTYPE call_bif(Process *c_p, Eterm *reg, BeamInstr *I);
 
 BIF_RETTYPE
@@ -5218,13 +5217,13 @@ erts_schedule_bif(Process *proc,
 	    mfa = &exp->info.mfa;
 	}
 #endif
-	else if (em_call_bif_e == (BeamInstr *) *i) {
+	else if (BeamIsOpCode(*i, op_call_bif_e)) {
 	    /* Pointer to bif export in i+1 */
 	    exp = (Export *) i[1];
 	    pc = i;
 	    mfa = &exp->info.mfa;
 	}
-	else if (em_apply_bif == (BeamInstr *) *i) {
+	else if (BeamIsOpCode(*i, op_apply_bif)) {
 	    /* Pointer to bif in i+1, and mfa in i-3 */	    
 	    pc = c_p->cp;
 	    mfa = erts_code_to_codemfa(i);
diff --git a/erts/emulator/beam/bs_instrs.tab b/erts/emulator/beam/bs_instrs.tab
index a4d4afe7d4..9f03b19731 100644
--- a/erts/emulator/beam/bs_instrs.tab
+++ b/erts/emulator/beam/bs_instrs.tab
@@ -364,11 +364,9 @@ i_bs_init_bits_fail      := bs_init_bits.fail.verify.execute;
 i_bs_init_bits_fail_heap := bs_init_bits.fail_heap.verify.execute;
 
 bs_init_bits.head() {
-    Eterm new_binary;
     Eterm num_bits_term;
     Uint num_bits;
     Uint alloc;
-    Uint num_bytes;
 }
 
 bs_init_bits.plain(NumBits) {
@@ -410,7 +408,9 @@ bs_init_bits.verify(Fail) {
 }
 
 bs_init_bits.execute(Live, Dst) {
-     num_bytes = ((Uint64)num_bits+(Uint64)7) >> 3;
+     Eterm new_binary;
+     Uint num_bytes = ((Uint64)num_bits+(Uint64)7) >> 3;
+
      if (num_bits & 7) {
 	 alloc += ERL_SUB_BIN_SIZE;
      }
@@ -709,9 +709,6 @@ i_bs_validate_unicode_retract(Fail, Src, Ms) {
 i_bs_start_match2 := bs_start_match.fetch.execute;
 
 bs_start_match.head() {
-    Uint slots;
-    Uint live;
-    Eterm header;
     Eterm context;
 }
 
@@ -720,6 +717,9 @@ bs_start_match.fetch(Src) {
 }
 
 bs_start_match.execute(Fail, Live, Slots, Dst) {
+    Uint slots;
+    Uint live;
+    Eterm header;
     if (!is_boxed(context)) {
         $FAIL($Fail);
     }
@@ -942,13 +942,10 @@ bs_context_to_binary := ctx_to_bin.fetch.execute;
 i_bs_get_binary_all_reuse := ctx_to_bin.fetch_bin.execute;
 
 ctx_to_bin.head() {
-     Eterm context;
-     ErlBinMatchBuffer* mb;
-     ErlSubBin* sb;
-     Uint size;
-     Uint offs;
-     Uint orig;
-     Uint hole_size;
+    Eterm context;
+    ErlBinMatchBuffer* mb;
+    Uint size;
+    Uint offs;
 }
 
 ctx_to_bin.fetch(Src) {
@@ -976,8 +973,9 @@ ctx_to_bin.fetch_bin(Src, Fail, Unit) {
 }
 
 ctx_to_bin.execute() {
-    orig = mb->orig;
-    sb = (ErlSubBin *) boxed_val(context);
+    Uint hole_size;
+    Uint orig = mb->orig;
+    ErlSubBin* sb = (ErlSubBin *) boxed_val(context);
     hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE;
     sb->thing_word = HEADER_SUB_BIN;
     sb->size = BYTE_OFFSET(size);
diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h
index 9e3280cd98..42976d2301 100644
--- a/erts/emulator/beam/code_ix.h
+++ b/erts/emulator/beam/code_ix.h
@@ -176,7 +176,7 @@ int erts_has_code_write_permission(void);
 ERTS_GLB_INLINE
 BeamInstr *erts_codeinfo_to_code(ErtsCodeInfo *ci)
 {
-    ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI) || !ci->op);
+    ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI) || !ci->op);
     ASSERT_MFA(&ci->mfa);
     return (BeamInstr*)(ci + 1);
 }
@@ -185,7 +185,7 @@ ERTS_GLB_INLINE
 ErtsCodeInfo *erts_code_to_codeinfo(BeamInstr *I)
 {
     ErtsCodeInfo *ci = ((ErtsCodeInfo *)(((char *)(I)) - sizeof(ErtsCodeInfo)));
-    ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI) || !ci->op);
+    ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI) || !ci->op);
     ASSERT_MFA(&ci->mfa);
     return ci;
 }
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 3fe089a00e..22942b40c4 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -966,12 +966,12 @@ static int function_is_traced(Process *p,
     if ((ep = export_get(&e)) != NULL) {
 	pc = ep->beam;
 	if (ep->addressv[erts_active_code_ix()] == pc &&
-	    *pc != (BeamInstr) em_call_error_handler) {
+	    ! BeamIsOpCode(*pc, op_call_error_handler)) {
 
 	    int r = 0;
 
-	    ASSERT(*pc == (BeamInstr) em_apply_bif ||
-		   *pc == (BeamInstr) BeamOp(op_i_generic_breakpoint));
+	    ASSERT(BeamIsOpCode(*pc, op_apply_bif) ||
+		   BeamIsOpCode(*pc, op_i_generic_breakpoint));
 
 	    if (erts_is_trace_break(&ep->info, ms, 0)) {
 		return FUNC_TRACE_GLOBAL_TRACE;
@@ -1361,14 +1361,14 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified,
 	    if (ep->addressv[code_ix] != pc) {
 		fp[i].mod->curr.num_traced_exports++;
 #ifdef DEBUG
-		ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI);
+		ep->info.op = BeamOpCodeAddr(op_i_func_info_IaaI);
 #endif
-                ep->beam[0] = (BeamInstr) BeamOp(op_trace_jump_W);
+                ep->beam[0] = BeamOpCodeAddr(op_trace_jump_W);
 		ep->beam[1] = (BeamInstr) ep->addressv[code_ix];
 	    }
 	    erts_set_call_trace_bif(ci, match_prog_set, 0);
 	    if (ep->addressv[code_ix] != pc) {
-		ep->beam[0] = (BeamInstr) BeamOp(op_i_generic_breakpoint);
+		ep->beam[0] = BeamOpCodeAddr(op_i_generic_breakpoint);
 	    }
 	} else if (!on && flags.breakpoint) {
 	    /* Turn off breakpoint tracing -- nothing to do here. */
@@ -1378,8 +1378,8 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified,
 	     * before turning on breakpoint tracing.
 	     */
 	    erts_clear_call_trace_bif(ci, 0);
-	    if (ep->beam[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
-		ep->beam[0] = (BeamInstr) BeamOp(op_trace_jump_W);
+	    if (BeamIsOpCode(ep->beam[0], op_i_generic_breakpoint)) {
+		ep->beam[0] = BeamOpCodeAddr(op_trace_jump_W);
 	    }
 	}
     }
@@ -1671,7 +1671,7 @@ uninstall_exp_breakpoints(BpFunctions* f)
 	if (ep->addressv[code_ix] != ep->beam) {
 	    continue;
 	}
-	ASSERT(ep->beam[0] == (BeamInstr) BeamOp(op_trace_jump_W));
+	ASSERT(BeamIsOpCode(ep->beam[0], op_trace_jump_W));
 	ep->addressv[code_ix] = (BeamInstr *) ep->beam[1];
     }
 }
@@ -1690,7 +1690,7 @@ clean_export_entries(BpFunctions* f)
 	if (ep->addressv[code_ix] == ep->beam) {
 	    continue;
 	}
-	if (ep->beam[0] == (BeamInstr) BeamOp(op_trace_jump_W)) {
+	if (BeamIsOpCode(ep->beam[0], op_trace_jump_W)) {
 	    ep->beam[0] = (BeamInstr) 0;
 	    ep->beam[1] = (BeamInstr) 0;
 	}
diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h
index 9be0e6f7c7..b8a4e4ebc3 100644
--- a/erts/emulator/beam/erl_nfunc_sched.h
+++ b/erts/emulator/beam/erl_nfunc_sched.h
@@ -211,8 +211,8 @@ erts_proc_shadow2real(Process *c_p)
 #define ERTS_NFUNC_SCHED_INTERNALS__
 
 #define ERTS_I_BEAM_OP_TO_NIF_EXPORT(I)					\
-    (ASSERT(BeamOp(op_apply_bif) == (BeamInstr *) (*(I))		\
-	    || BeamOp(op_call_nif) == (BeamInstr *) (*(I))),		\
+    (ASSERT(BeamIsOpCode(*(I), op_apply_bif) ||                         \
+            BeamIsOpCode(*(I), op_call_nif)),                           \
      ((NifExport *) (((char *) (I)) - offsetof(NifExport, exp.beam[0]))))
 
 
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 43441e0228..f7f12efe28 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -292,7 +292,7 @@ schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp,
     ep = erts_nif_export_schedule(c_p, dirty_shadow_proc,
 				  c_p->current,
 				  c_p->cp,
-				  (BeamInstr) em_call_nif,
+				  BeamOpCodeAddr(op_call_nif),
 				  direct_fp, indirect_fp,
 				  mod, func_name,
 				  argc, (const Eterm *) argv);
@@ -3613,7 +3613,7 @@ static ErtsCodeInfo** get_func_pp(BeamCodeHeader* mod_code, Eterm f_atom, unsign
     int j;
     for (j = 0; j < n; ++j) {
 	ErtsCodeInfo* ci = mod_code->functions[j];
-	ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+	ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
 	if (f_atom == ci->mfa.function
 	    && arity == ci->mfa.arity) {
 	    return mod_code->functions+j;
@@ -3982,13 +3982,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
             code_ptr = erts_codeinfo_to_code(ci);
 
 	    if (ci->u.gen_bp == NULL) {
-		code_ptr[0] = (BeamInstr) BeamOp(op_call_nif);
+		code_ptr[0] = BeamOpCodeAddr(op_call_nif);
 	    }
 	    else { /* Function traced, patch the original instruction word */
 		GenericBp* g = ci->u.gen_bp;
-		ASSERT(code_ptr[0] ==
-		       (BeamInstr) BeamOp(op_i_generic_breakpoint));
-		g->orig_instr = (BeamInstr) BeamOp(op_call_nif);
+		ASSERT(BeamIsOpCode(code_ptr[0], op_i_generic_breakpoint));
+		g->orig_instr = BeamOpCodeAddr(op_call_nif);
 	    }
 	    if (f->flags) {
 		code_ptr[3] = (BeamInstr) f->fptr;
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 295130f60c..661538eadd 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -200,11 +200,15 @@ extern int erts_pd_initial_size;/* Initial Process dictionary table size */
 
 #include "erl_term.h"
 
-#ifdef NO_JUMP_TABLE 
-#define BeamOp(Op) (Op)
+#if defined(NO_JUMP_TABLE)
+#  define BeamOpsAreInitialized() (1)
+#  define BeamOpCodeAddr(OpCode) ((BeamInstr)(OpCode))
 #else
 extern void** beam_ops;
-#define BeamOp(Op) beam_ops[(Op)]
+#  define BeamOpsAreInitialized() (beam_ops != 0)
+#  define BeamOpCodeAddr(OpCode) ((BeamInstr)beam_ops[(OpCode)])
 #endif
 
+#define BeamIsOpCode(InstrWord, OpCode) ((InstrWord) == BeamOpCodeAddr(OpCode))
+
 #endif	/* __ERL_VM_H__ */
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index c81503f722..946ffeffb8 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -48,7 +48,6 @@ static erts_atomic_t total_entries_bytes;
  */
 erts_mtx_t export_staging_lock;
 
-extern BeamInstr* em_call_error_handler;
 extern BeamInstr* em_call_traced_function;
 
 struct export_entry
@@ -130,7 +129,10 @@ export_alloc(struct export_entry* tmpl_e)
 	obj->info.mfa.module = tmpl->info.mfa.module;
 	obj->info.mfa.function = tmpl->info.mfa.function;
 	obj->info.mfa.arity = tmpl->info.mfa.arity;
-	obj->beam[0] = (BeamInstr) em_call_error_handler;
+        obj->beam[0] = 0;
+        if (BeamOpsAreInitialized()) {
+            obj->beam[0] = BeamOpCodeAddr(op_call_error_handler);
+        }
 	obj->beam[1] = 0;
 
 	for (ix=0; ix<ERTS_NUM_CODE_IX; ix++) {
@@ -267,7 +269,7 @@ erts_find_function(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix)
     ee = hash_get(&export_tables[code_ix].htable, init_template(&templ, m, f, a));
     if (ee == NULL ||
 	(ee->ep->addressv[code_ix] == ee->ep->beam &&
-	 ee->ep->beam[0] != (BeamInstr) BeamOp(op_i_generic_breakpoint))) {
+	 ! BeamIsOpCode(ee->ep->beam[0], op_i_generic_breakpoint))) {
 	return NULL;
     }
     return ee->ep;
diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h
index be6cce07bf..194e514b12 100644
--- a/erts/emulator/beam/export.h
+++ b/erts/emulator/beam/export.h
@@ -73,7 +73,7 @@ extern erts_mtx_t export_staging_lock;
 #include "beam_load.h" /* For em_* extern declarations */ 
 #define ExportIsBuiltIn(EntryPtr) 			\
 (((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->beam) && \
- ((EntryPtr)->beam[0] == (BeamInstr) em_apply_bif))
+ (BeamIsOpCode((EntryPtr)->beam[0], op_apply_bif)))
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index 7ea9dee299..07cc4bd527 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -375,7 +375,6 @@ i_element := element_group.fetch.execute;
 
 
 element_group.head() {
-    Eterm element_index;
     Eterm element_tuple;
 }
 
@@ -384,7 +383,7 @@ element_group.fetch(Src) {
 }
 
 element_group.execute(Fail, Index, Dst) {
-    element_index = $Index;
+    Eterm element_index = $Index;
     if (ERTS_LIKELY(is_small(element_index) && is_tuple(element_tuple))) {
         Eterm* tp = tuple_val(element_tuple);
 
@@ -831,12 +830,14 @@ badmatch(Src) {
     c_p->fvalue = $Src;
     c_p->freason = BADMATCH;
     goto find_func_info;
+    //| -no_next;
 }
 
 case_end(Src) {
     c_p->fvalue = $Src;
     c_p->freason = EXC_CASE_CLAUSE;
     goto find_func_info;
+    //| -no_next;
 }
 
 if_end() {
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 87ff92d354..75ff40606b 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -961,11 +961,11 @@ call_ext_last Ar Func D => i_call_ext_last Func D
 call_ext_only Ar Func   => i_call_ext_only Func
 
 i_apply
-i_apply_last P
+i_apply_last Q
 i_apply_only
 
 i_apply_fun
-i_apply_fun_last P
+i_apply_fun_last Q
 i_apply_fun_only
 
 %cold
diff --git a/erts/emulator/beam/trace_instrs.tab b/erts/emulator/beam/trace_instrs.tab
index b10442c5e7..3eee81c053 100644
--- a/erts/emulator/beam/trace_instrs.tab
+++ b/erts/emulator/beam/trace_instrs.tab
@@ -61,12 +61,12 @@ i_return_to_trace() {
         Uint *cpp = (Uint*) E;
         for(;;) {
             ASSERT(is_CP(*cpp));
-            if (*cp_val(*cpp) == (BeamInstr) OpCode(return_trace)) {
+            if (IsOpCode(*cp_val(*cpp), return_trace)) {
                 do
                     ++cpp;
                 while (is_not_CP(*cpp));
                 cpp += 2;
-            } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_to_trace)) {
+            } else if (IsOpCode(*cp_val(*cpp), i_return_to_trace)) {
                 do
                     ++cpp;
                 while (is_not_CP(*cpp));
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 05663648e9..380031bf13 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -53,8 +53,6 @@
 #include "hipe_literals.h"
 #endif
 
-#define BeamOpCode(Op)	((Uint)BeamOp(Op))
-
 
 int term_to_Sint32(Eterm term, Sint *sp)
 {
@@ -615,7 +613,7 @@ static ErtsCodeInfo* hipe_find_emu_address(Eterm mod, Eterm name, unsigned int a
     n = code_hdr->num_functions;
     for (i = 0; i < n; ++i) {
 	ErtsCodeInfo *ci = code_hdr->functions[i];
-	ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI));
+	ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
 	if (ci->mfa.function == name && ci->mfa.arity == arity)
 	    return ci;
     }
diff --git a/erts/emulator/hipe/hipe_bif1.c b/erts/emulator/hipe/hipe_bif1.c
index 3d3df4fd48..73d07f0ce5 100644
--- a/erts/emulator/hipe/hipe_bif1.c
+++ b/erts/emulator/hipe/hipe_bif1.c
@@ -32,11 +32,10 @@
 #include "big.h"
 #include "error.h"
 #include "beam_load.h"
+#include "erl_vm.h"
 #include "hipe_bif0.h"
 #include "hipe_bif1.h"
 
-#define BeamOpCode(Op)	((Uint)BeamOp(Op))
-
 BIF_RETTYPE hipe_bifs_call_count_on_1(BIF_ALIST_1)
 {
     ErtsCodeInfo *ci;
@@ -46,17 +45,17 @@ BIF_RETTYPE hipe_bifs_call_count_on_1(BIF_ALIST_1)
     ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
     if (!ci)
 	BIF_ERROR(BIF_P, BADARG);
-    ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI));
+    ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
     pc = erts_codeinfo_to_code(ci);
-    if (pc[0] == BeamOpCode(op_hipe_trap_call))
+    if (BeamIsOpCode(pc[0], op_hipe_trap_call))
 	BIF_ERROR(BIF_P, BADARG);
-    if (pc[0] == BeamOpCode(op_hipe_call_count))
+    if (BeamIsOpCode(pc[0], op_hipe_call_count))
 	BIF_RET(NIL);
     hcc = erts_alloc(ERTS_ALC_T_HIPE_SL, sizeof(*hcc));
     hcc->count = 0;
     hcc->opcode = pc[0];
     ci->u.hcc = hcc;
-    pc[0] = BeamOpCode(op_hipe_call_count);
+    pc[0] = BeamOpCodeAddr(op_hipe_call_count);
     BIF_RET(am_true);
 }
 
@@ -70,9 +69,9 @@ BIF_RETTYPE hipe_bifs_call_count_off_1(BIF_ALIST_1)
     ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
     if (!ci)
 	BIF_ERROR(BIF_P, BADARG);
-    ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI));
+    ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
     pc = erts_codeinfo_to_code(ci);
-    if (pc[0] != BeamOpCode(op_hipe_call_count))
+    if (! BeamIsOpCode(pc[0], op_hipe_call_count))
 	BIF_RET(am_false);
     hcc = ci->u.hcc;
     count = hcc->count;
@@ -91,9 +90,9 @@ BIF_RETTYPE hipe_bifs_call_count_get_1(BIF_ALIST_1)
     ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
     if (!ci)
 	BIF_ERROR(BIF_P, BADARG);
-    ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI));
+    ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
     pc = erts_codeinfo_to_code(ci);
-    if (pc[0] != BeamOpCode(op_hipe_call_count))
+    if (! BeamIsOpCode(pc[0], op_hipe_call_count))
 	BIF_RET(am_false);
     hcc = ci->u.hcc;
     BIF_RET(make_small(hcc->count));
@@ -109,9 +108,9 @@ BIF_RETTYPE hipe_bifs_call_count_clear_1(BIF_ALIST_1)
     ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
     if (!ci)
 	BIF_ERROR(BIF_P, BADARG);
-    ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI));
+    ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
     pc = erts_codeinfo_to_code(ci);
-    if (pc[0] != BeamOpCode(op_hipe_call_count))
+    if (! BeamIsOpCode(pc[0], op_hipe_call_count))
 	BIF_RET(am_false);
     hcc = ci->u.hcc;
     count = hcc->count;
diff --git a/erts/emulator/hipe/hipe_instrs.tab b/erts/emulator/hipe/hipe_instrs.tab
index bcce196a1d..a01baebddf 100644
--- a/erts/emulator/hipe/hipe_instrs.tab
+++ b/erts/emulator/hipe/hipe_instrs.tab
@@ -45,7 +45,7 @@ hipe_trap.call() {
      * ... remainder of original BEAM code
      */
     ErtsCodeInfo *ci = erts_code_to_codeinfo(I);
-    ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI));
+    ASSERT(IsOpCode(ci->op, i_func_info_IaaI));
     c_p->hipe.u.ncallee = ci->u.ncallee;
     ++hipe_trap_count;
     $HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (ci->mfa.arity << 8));
@@ -53,7 +53,7 @@ hipe_trap.call() {
 
 hipe_trap.call_closure() {
     ErtsCodeInfo *ci = erts_code_to_codeinfo(I);
-    ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI));
+    ASSERT(IsOpCode(ci->op, i_func_info_IaaI));
     c_p->hipe.u.ncallee = ci->u.ncallee;
     ++hipe_trap_count;
     $HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (ci->mfa.arity << 8));
@@ -132,7 +132,7 @@ hipe_call_count() {
      */
     ErtsCodeInfo *ci = erts_code_to_codeinfo(I);
     struct hipe_call_count *hcc = ci->u.hcc;
-    ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI));
+    ASSERT(IsOpCode(ci->op, i_func_info_IaaI));
     ASSERT(hcc != NULL);
     ASSERT(VALID_INSTR(hcc->opcode));
     ++(hcc->count);
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index b7f81fc4a6..8b497c9970 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -155,8 +155,6 @@ void hipe_check_pcb(Process *p, const char *file, unsigned line)
 #include "hipe_arm_glue.h"
 #endif
 
-#define BeamOpCode(Op)		((Uint)BeamOp(Op))
-
 Uint hipe_beam_pc_return[1];	/* needed in hipe_debug.c */
 Uint hipe_beam_pc_throw[1];	/* needed in hipe_debug.c */
 Uint hipe_beam_pc_resume[1];	/* needed by hipe_set_timeout() */
@@ -166,9 +164,9 @@ void hipe_mode_switch_init(void)
 {
     hipe_arch_glue_init();
 
-    hipe_beam_pc_return[0] = BeamOpCode(op_hipe_trap_return);
-    hipe_beam_pc_throw[0] = BeamOpCode(op_hipe_trap_throw);
-    hipe_beam_pc_resume[0] = BeamOpCode(op_hipe_trap_resume);
+    hipe_beam_pc_return[0] = BeamOpCodeAddr(op_hipe_trap_return);
+    hipe_beam_pc_throw[0] = BeamOpCodeAddr(op_hipe_trap_throw);
+    hipe_beam_pc_resume[0] = BeamOpCodeAddr(op_hipe_trap_resume);
 
     hipe_beam_catch_throw =
 	make_catch(beam_catches_cons(hipe_beam_pc_throw, BEAM_CATCHES_NIL));
@@ -182,8 +180,8 @@ void hipe_set_call_trap(ErtsCodeInfo* ci, void *nfun, int is_closure)
     HIPE_ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI));
     bfun[0] =
 	is_closure
-	? BeamOpCode(op_hipe_trap_call_closure)
-	: BeamOpCode(op_hipe_trap_call);
+	? BeamOpCodeAddr(op_hipe_trap_call_closure)
+	: BeamOpCodeAddr(op_hipe_trap_call);
     ci->u.ncallee = (void (*)(void)) nfun;
 }
 
diff --git a/erts/emulator/test/beam_SUITE.erl b/erts/emulator/test/beam_SUITE.erl
index 6a54fa87e0..bdf8f6c34e 100644
--- a/erts/emulator/test/beam_SUITE.erl
+++ b/erts/emulator/test/beam_SUITE.erl
@@ -113,20 +113,41 @@ packed_registers(Config) when is_list(Config) ->
 		   VarName = list_to_atom("M"++integer_to_list(V)),
 		   merl:var(VarName)
 	       end || V <- Seq],
+    MoreNewVars = [begin
+                       VarName = list_to_atom("MM"++integer_to_list(V)),
+                       merl:var(VarName)
+                   end || V <- Seq],
+    TupleEls = [?Q("id(_@Value@)") || {_,Value} <- S0],
     S = [?Q("_@Var = id(_@Value@)") || {Var,Value} <- S0],
     Code = ?Q(["-module('@Mod@').\n"
 	       "-export([f/0]).\n"
 	       "f() ->\n"
+               "Tuple = id({_@TupleEls}),\n"
+               "{_@MoreNewVars} = Tuple,\n"
 	       "_@S,\n"
 	       "_ = id(0),\n"
 	       "L = [_@Vars],\n"
 	       "_ = id(1),\n"
 	       "[_@NewVars] = L,\n"		%Test get_list/3.
 	       "_ = id(2),\n"
-	       "id([_@Vars,_@NewVars]).\n"
+	       "id([_@Vars,_@NewVars,_@MoreNewVars]).\n"
 	       "id(I) -> I.\n"]),
     merl:compile_and_load(Code),
-    CombinedSeq = Seq ++ Seq,
+
+    %% Optionally print the generated code.
+    PrintCode = false,                          %Change to true to print code.
+
+    case PrintCode of
+        false ->
+            ok;
+        true ->
+            merl:print(Code),
+            erts_debug:df(Mod),
+            {ok,Dis} = file:read_file(atom_to_list(Mod)++".dis"),
+            io:put_chars(Dis)
+    end,
+
+    CombinedSeq = Seq ++ Seq ++ Seq,
     CombinedSeq = Mod:f(),
 
     %% Clean up.
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index e4c91f1685..64a9a49ac8 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -1441,11 +1441,10 @@ sub code_gen {
             "ASSERT(VALID_INSTR(*I));\n" .
             "Goto(*I);";
     } else {
-        $var_decls .= "BeamInstr* _nextpf = " .
-            "(BeamInstr *) I[$instr_offset];\n";
+        $var_decls .= "BeamInstr next_pf = I[$instr_offset];\n";
         $dispatch_next = "\nI += $instr_offset;\n" .
-            "ASSERT(VALID_INSTR(_nextpf));\n" .
-            "Goto(_nextpf);";
+            "ASSERT(VALID_INSTR(next_pf));\n" .
+            "Goto(next_pf);";
     }
 
     #