Merge branch 'bjorn/erts/relative-jumps'

* bjorn/erts/relative-jumps:
  Pack failure labels in i_select_val2 and i_select_tuple_arity2
  Optimize i_select_tuple_arity2 and is_select_lins
  Rewrite select_val_bins so that its labels can be packed
  Pack sequences of trailing 'f' operands
  Implement packing of 'f' and 'j'
  Make sure that mask literals are 64 bits
  Use relative failure labels
  Add information about offset to common group start position
  Remove JUMP_OFFSET
  Refactor instructions to support relative jumps
  Introduce a new trace_jump/1 instruction for tracing
  Avoid using $Src more than once

13 files changed, 510 insertions, 212 deletions

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 21e295c63a..afde45ba71 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -53,6 +53,8 @@ void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg);
 
 static int print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr);
 static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif);
+static BeamInstr* f_to_addr(BeamInstr* base, int op, BeamInstr* ap);
+static BeamInstr* f_to_addr_packed(BeamInstr* base, int op, Sint32* ap);
 
 BIF_RETTYPE
 erts_debug_same_2(BIF_ALIST_2)
@@ -424,7 +426,9 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	while (start_prog < prog) {
 	    prog--;
 	    switch (*prog) {
+	    case 'f':
 	    case 'g':
+	    case 'q':
 		*ap++ = *--sp;
 		break;
 	    case 'i':		/* Initialize packing accumulator. */
@@ -558,9 +562,10 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	    break;
 	case 'f':		/* Destination label */
 	    {
-		ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap);
-		if (!cmfa || erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) {
-		    erts_print(to, to_arg, "f(" HEXF ")", *ap);
+                BeamInstr* target = f_to_addr(addr, op, ap);
+		ErtsCodeMFA* cmfa = find_function_from_pc(target);
+		if (!cmfa || erts_codemfa_to_code(cmfa) != target) {
+		    erts_print(to, to_arg, "f(" HEXF ")", target);
 		} else {
 		    erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module,
                                cmfa->function, cmfa->arity);
@@ -570,18 +575,18 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	    break;
 	case 'p':		/* Pointer (to label) */
 	    {
-		ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap);
-		if (!cmfa || erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) {
-		    erts_print(to, to_arg, "p(" HEXF ")", *ap);
-		} else {
-		    erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module,
-                               cmfa->function, cmfa->arity);
-		}
+                BeamInstr* target = f_to_addr(addr, op, ap);
+                erts_print(to, to_arg, "p(" HEXF ")", target);
 		ap++;
 	    }
 	    break;
 	case 'j':		/* Pointer (to label) */
-	    erts_print(to, to_arg, "j(" HEXF ")", *ap);
+            if (*ap == 0) {
+                erts_print(to, to_arg, "j(0)");
+            } else {
+                BeamInstr* target = f_to_addr(addr, op, ap);
+                erts_print(to, to_arg, "j(" HEXF ")", target);
+            }
 	    ap++;
 	    break;
 	case 'e':		/* Export entry */
@@ -627,9 +632,12 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
     switch (op) {
     case op_i_select_val_lins_xfI:
     case op_i_select_val_lins_yfI:
+    case op_i_select_val_bins_xfI:
+    case op_i_select_val_bins_yfI:
 	{
 	    int n = ap[-1];
 	    int ix = n;
+            Sint32* jump_tab = (Sint32 *)(ap + n);
 
 	    while (ix--) {
 		erts_print(to, to_arg, "%T ", (Eterm) ap[0]);
@@ -638,23 +646,11 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	    }
 	    ix = n;
 	    while (ix--) {
-		erts_print(to, to_arg, "f(" HEXF ") ", (Eterm) ap[0]);
-		ap++;
-		size++;
-	    }
-	}
-	break;
-    case op_i_select_val_bins_xfI:
-    case op_i_select_val_bins_yfI:
-	{
-	    int n = ap[-1];
-
-	    while (n > 0) {
-		erts_print(to, to_arg, "%T f(" HEXF ") ", (Eterm) ap[0], ap[1]);
-		ap += 2;
-		size += 2;
-		n--;
+                BeamInstr* target = f_to_addr_packed(addr, op, jump_tab);
+		erts_print(to, to_arg, "f(" HEXF ") ", target);
+                jump_tab++;
 	    }
+            size += (n+1) / 2;
 	}
 	break;
     case op_i_select_tuple_arity_xfI:
@@ -662,6 +658,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
         {
             int n = ap[-1];
             int ix = n - 1; /* without sentinel */
+            Sint32* jump_tab = (Sint32 *)(ap + n);
 
             while (ix--) {
                 Uint arity = arityval(ap[0]);
@@ -675,31 +672,54 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
             size++;
             ix = n;
             while (ix--) {
-                erts_print(to, to_arg, "f(" HEXF ") ", ap[0]);
-                ap++;
-                size++;
+                BeamInstr* target = f_to_addr_packed(addr, op, jump_tab);
+                erts_print(to, to_arg, "f(" HEXF ") ", target);
+                jump_tab++;
+            }
+            size += (n+1) / 2;
+        }
+        break;
+    case op_i_select_val2_xfcc:
+    case op_i_select_val2_yfcc:
+    case op_i_select_tuple_arity2_xfAA:
+    case op_i_select_tuple_arity2_yfAA:
+        {
+            Sint32* jump_tab = (Sint32 *) ap;
+            BeamInstr* target;
+            int i;
+
+            for (i = 0; i < 2; i++) {
+                target = f_to_addr_packed(addr, op, jump_tab++);
+                erts_print(to, to_arg, "f(" HEXF ") ", target);
             }
+            size += 1;
         }
         break;
     case op_i_jump_on_val_xfIW:
     case op_i_jump_on_val_yfIW:
 	{
-	    int n;
-	    for (n = ap[-2]; n > 0; n--) {
-		erts_print(to, to_arg, "f(" HEXF ") ", ap[0]);
-		ap++;
-		size++;
+	    int n = ap[-2];
+            Sint32* jump_tab = (Sint32 *) ap;
+
+            size += (n+1) / 2;
+            while (n-- > 0) {
+                BeamInstr* target = f_to_addr_packed(addr, op, jump_tab);
+		erts_print(to, to_arg, "f(" HEXF ") ", target);
+                jump_tab++;
 	    }
 	}
 	break;
     case op_i_jump_on_val_zero_xfI:
     case op_i_jump_on_val_zero_yfI:
 	{
-	    int n;
-	    for (n = ap[-1]; n > 0; n--) {
-		erts_print(to, to_arg, "f(" HEXF ") ", ap[0]);
-		ap++;
-		size++;
+	    int n = ap[-1];
+            Sint32* jump_tab = (Sint32 *) ap;
+
+            size += (n+1) / 2;
+            while (n-- > 0) {
+                BeamInstr* target = f_to_addr_packed(addr, op, jump_tab);
+		erts_print(to, to_arg, "f(" HEXF ") ", target);
+                jump_tab++;
 	    }
 	}
 	break;
@@ -796,6 +816,17 @@ static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif)
     }
 }
 
+static BeamInstr* f_to_addr(BeamInstr* base, int op, BeamInstr* ap)
+{
+    return base - 1 + opc[op].adjust + (Sint32) *ap;
+}
+
+static BeamInstr* f_to_addr_packed(BeamInstr* base, int op, Sint32* ap)
+{
+    return base - 1 + opc[op].adjust + *ap;
+}
+
+
 /*
  * Dirty BIF testing.
  *
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 55990362ff..1baf3ff7ab 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -235,6 +235,8 @@ void** beam_ops;
      ERTS_UNREQ_PROC_MAIN_LOCK((P))
 
 #define db(N) (N)
+#define fb(N) ((Sint)(Sint32)(N))
+#define jb(N) ((Sint)(Sint32)(N))
 #define tb(N) (N)
 #define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N)))
 #define yb(N) (*(Eterm *) (((unsigned char *)E) + (N)))
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 7d3a19ff86..3f9dc2c1aa 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -81,15 +81,28 @@ ErlDrvBinary* erts_gzinflate_buffer(char*, int);
 #define TE_FAIL (-1)
 #define TE_SHORT_WINDOW (-2)
 
+/*
+ * Type for a reference to a label that must be patched.
+ */
+
 typedef struct {
-    Uint value;			/* Value of label (NULL if not known yet). */
-    Sint patches;		/* Index (into code buffer) to first
-				 * location which must be patched with
-				 * the value of this label.
-				 */
+    Uint pos;                   /* Position of label reference to patch. */
+    Uint offset;                /* Offset from patch location.  */
+    int packed;                 /* 0 (not packed), 1 (lsw), 2 (msw) */
+} LabelPatch;
+
+/*
+ * Type for a label.
+ */
+
+typedef struct {
+    Uint value;			/* Value of label (0 if not known yet). */
     Uint looprec_targeted;	/* Non-zero if this label is the target of a loop_rec
 				 * instruction.
 				 */
+    LabelPatch* patches;       /* Array of label patches. */
+    Uint num_patches;           /* Number of patches in array. */
+    Uint num_allocated;         /* Number of allocated patches. */
 } Label;
 
 /*
@@ -226,7 +239,7 @@ typedef struct {
 
 typedef struct literal_patch LiteralPatch;
 struct literal_patch {
-    int pos;			/* Position in code */
+    Uint pos;			/* Position in code */
     LiteralPatch* next;
 };
 
@@ -507,6 +520,7 @@ static int read_lambda_table(LoaderState* stp);
 static int read_literal_table(LoaderState* stp);
 static int read_line_table(LoaderState* stp);
 static int read_code_header(LoaderState* stp);
+static void init_label(Label* lp);
 static int load_code(LoaderState* stp);
 static GenOp* gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index,
 			  GenOpArg Tuple, GenOpArg Dst);
@@ -1051,6 +1065,10 @@ loader_state_dtor(Binary* magic)
         stp->codev = 0;
     }
     if (stp->labels != 0) {
+        Uint num;
+        for (num = 0; num < stp->num_labels; num++) {
+            erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->labels[num].patches);
+        }
 	erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->labels);
 	stp->labels = 0;
     }
@@ -1534,7 +1552,7 @@ read_export_table(LoaderState* stp)
 	 * any other functions that walk through all local functions.
 	 */
 
-	if (stp->labels[n].patches >= 0) {
+	if (stp->labels[n].num_patches > 0) {
 	    LoadError3(stp, "there are local calls to the stub for "
 		       "the BIF %T:%T/%d",
 		       stp->module, func, arity);
@@ -1880,9 +1898,7 @@ read_code_header(LoaderState* stp)
     stp->labels = (Label *) erts_alloc(ERTS_ALC_T_PREPARED_CODE,
 				       stp->num_labels * sizeof(Label));
     for (i = 0; i < stp->num_labels; i++) {
-	stp->labels[i].value = 0;
-	stp->labels[i].patches = -1;
-	stp->labels[i].looprec_targeted = 0;
+        init_label(&stp->labels[i]);
     }
 
     stp->catches = 0;
@@ -1911,12 +1927,43 @@ read_code_header(LoaderState* stp)
     
 #define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm))))
 
+static void init_label(Label* lp)
+{
+    lp->value = 0;
+    lp->looprec_targeted = 0;
+    lp->num_patches = 0;
+    lp->num_allocated = 4;
+    lp->patches = erts_alloc(ERTS_ALC_T_PREPARED_CODE,
+                             lp->num_allocated * sizeof(LabelPatch));
+}
+
+static void
+register_label_patch(LoaderState* stp, Uint label, Uint ci, Uint offset)
+{
+    Label* lp;
+
+    ASSERT(label < stp->num_labels);
+    lp = &stp->labels[label];
+    if (lp->num_allocated <= lp->num_patches) {
+        lp->num_allocated *= 2;
+        lp->patches = erts_realloc(ERTS_ALC_T_PREPARED_CODE,
+                                   (void *) lp->patches,
+                                   lp->num_allocated * sizeof(LabelPatch));
+    }
+    lp->patches[lp->num_patches].pos = ci;
+    lp->patches[lp->num_patches].offset = offset;
+    lp->patches[lp->num_patches].packed = 0;
+    lp->num_patches++;
+    stp->codev[ci] = label;
+}
+
 static int
 load_code(LoaderState* stp)
 {
     int i;
-    int ci;
-    int last_func_start = 0;	/* Needed by nif loading and line instructions */
+    Uint ci;
+    Uint last_instr_start;       /* Needed for relative jumps */
+    Uint last_func_start = 0;	/* Needed by nif loading and line instructions */
     char* sign;
     int arg;			/* Number of current argument. */
     int num_specific;		/* Number of specific ops for current. */
@@ -1929,6 +1976,9 @@ load_code(LoaderState* stp)
     GenOp** last_op_next = NULL;
     int arity;
     int retval = 1;
+#if defined(BEAM_WIDE_SHIFT)
+    int num_trailing_f;     /* Number of extra 'f' arguments in a list */
+#endif
 
     /*
      * The size of the loaded func_info instruction is needed
@@ -2272,6 +2322,7 @@ load_code(LoaderState* stp)
 
 	    stp->specific_op = specific;
 	    CodeNeed(opc[stp->specific_op].sz+16); /* Extra margin for packing */
+            last_instr_start = ci + opc[stp->specific_op].adjust;
 	    code[ci++] = BeamOpCode(stp->specific_op);
 	}
 	
@@ -2401,16 +2452,14 @@ load_code(LoaderState* stp)
 		break;
 	    case 'f':		/* Destination label */
 		VerifyTag(stp, tag_to_letter[tag], *sign);
-		code[ci] = stp->labels[tmp_op->a[arg].val].patches;
-		stp->labels[tmp_op->a[arg].val].patches = ci;
+                register_label_patch(stp, tmp_op->a[arg].val, ci, -last_instr_start);
 		ci++;
 		break;
 	    case 'j':		/* 'f' or 'p' */
 		if (tag == TAG_p) {
 		    code[ci] = 0;
 		} else if (tag == TAG_f) {
-		    code[ci] = stp->labels[tmp_op->a[arg].val].patches;
-		    stp->labels[tmp_op->a[arg].val].patches = ci;
+                    register_label_patch(stp, tmp_op->a[arg].val, ci, -last_instr_start);
 		} else {
 		    LoadError3(stp, "bad tag %d; expected %d or %d",
 			       tag, TAG_f, TAG_p);
@@ -2430,7 +2479,6 @@ load_code(LoaderState* stp)
 		    LoadError1(stp, "label %d defined more than once", last_label);
 		}
 		stp->labels[last_label].value = ci;
-		ASSERT(stp->labels[last_label].patches < ci);
 		break;
 	    case 'e':		/* Export entry */
 		VerifyTag(stp, tag, TAG_u);
@@ -2479,23 +2527,58 @@ load_code(LoaderState* stp)
 	    char* prog;            /* Program for packing engine. */
 	    struct pack_stack {
                 BeamInstr instr;
-                LiteralPatch* patch;
+                Uint* patch_pos;
             } stack[8];            /* Stack. */
 	    struct pack_stack* sp = stack; /* Points to next free position. */
 	    BeamInstr packed = 0; /* Accumulator for packed operations. */
+            LabelPatch* packed_label = 0;
 
 	    for (prog = opc[stp->specific_op].pack; *prog; prog++) {
 		switch (*prog) {
-		case 'g':	/* Get instruction; push on stack. */
+		case 'g':	/* Get operand and push on stack. */
+                    ci--;
+                    sp->instr = code[ci];
+                    sp->patch_pos = 0;
+                    sp++;
+                    break;
+		case 'f':	/* Get possible 'f' operand and push on stack. */
+                    {
+                        Uint w = code[--ci];
+                        sp->instr = w;
+                        sp->patch_pos = 0;
+
+                        if (w != 0) {
+                            LabelPatch* lbl_p;
+                            int num_patches;
+                            int patch;
+
+                            ASSERT(w < stp->num_labels);
+                            lbl_p = stp->labels[w].patches;
+                            num_patches = stp->labels[w].num_patches;
+                            for (patch = num_patches - 1; patch >= 0; patch--) {
+                                if (lbl_p[patch].pos == ci) {
+                                    sp->patch_pos = &lbl_p[patch].pos;
+                                    break;
+                                }
+                            }
+                            ASSERT(sp->patch_pos);
+                        }
+                        sp++;
+                    }
+                    break;
+		case 'q':	/* Get possible 'q' operand and push on stack. */
                     {
                         LiteralPatch* lp;
 
                         ci--;
                         sp->instr = code[ci];
-                        sp->patch = 0;
-                        for (lp = stp->literal_patches; lp && lp->pos > ci-MAX_OPARGS; lp = lp->next) {
+                        sp->patch_pos = 0;
+
+                        for (lp = stp->literal_patches;
+                             lp && lp->pos > ci-MAX_OPARGS;
+                             lp = lp->next) {
                             if (lp->pos == ci) {
-                                sp->patch = lp;
+                                sp->patch_pos = &lp->pos;
                                 break;
                             }
                         }
@@ -2507,28 +2590,68 @@ load_code(LoaderState* stp)
 		    break;
 		case '0':	/* Tight shift */
 		    packed = (packed << BEAM_TIGHT_SHIFT) | code[--ci];
+                    if (packed_label) {
+                        packed_label->packed++;
+                    }
 		    break;
 		case '6':	/* Shift 16 steps */
 		    packed = (packed << BEAM_LOOSE_SHIFT) | code[--ci];
+                    if (packed_label) {
+                        packed_label->packed++;
+                    }
 		    break;
 #ifdef ARCH_64
 		case 'w':	/* Shift 32 steps */
-		    packed = (packed << BEAM_WIDE_SHIFT) | code[--ci];
-		    break;
+                    {
+                        Uint w = code[--ci];
+
+                        if (packed_label) {
+                            packed_label->packed++;
+                        }
+
+                        /*
+                         * 'w' can handle both labels ('f' and 'j'), as well
+                         * as 'I'. Test whether this is a label.
+                         */
+
+                        if (w < stp->num_labels) {
+                            /*
+                             * Probably a label. Look for patch pointing to this
+                             * position.
+                             */
+                            LabelPatch* lp = stp->labels[w].patches;
+                            int num_patches = stp->labels[w].num_patches;
+                            int patch;
+                            for (patch = num_patches - 1; patch >= 0; patch--) {
+                                if (lp[patch].pos == ci) {
+                                    lp[patch].packed = 1;
+                                    packed_label = &lp[patch];
+                                    break;
+                                }
+                            }
+                        }
+                        packed = (packed << BEAM_WIDE_SHIFT) |
+                            (code[ci] & BEAM_WIDE_MASK);
+                    }
+                    break;
 #endif
 		case 'p':	/* Put instruction (from stack). */
                     --sp;
                     code[ci] = sp->instr;
-                    if (sp->patch) {
-                        sp->patch->pos = ci;
+                    if (sp->patch_pos) {
+                        *sp->patch_pos = ci;
                     }
                     ci++;
 		    break;
 		case 'P':	/* Put packed operands. */
                     sp->instr = packed;
-                    sp->patch = 0;
+                    sp->patch_pos = 0;
                     sp++;
 		    packed = 0;
+                    if (packed_label) {
+                        packed_label->pos = ci;
+                        packed_label = 0;
+                    }
 		    break;
 		default:
 		    ASSERT(0);
@@ -2541,7 +2664,17 @@ load_code(LoaderState* stp)
 	 * Load any list arguments using the primitive tags.
 	 */
 
+#if defined(BEAM_WIDE_SHIFT)
+        num_trailing_f = 0;
+#endif
 	for ( ; arg < tmp_op->arity; arg++) {
+#if defined(BEAM_WIDE_SHIFT)
+	    if (tmp_op->a[arg].type == TAG_f) {
+                num_trailing_f++;
+            } else {
+                num_trailing_f = 0;
+            }
+#endif
 	    switch (tmp_op->a[arg].type) {
 	    case TAG_i:
 		CodeNeed(1);
@@ -2555,8 +2688,7 @@ load_code(LoaderState* stp)
 		break;
 	    case TAG_f:
 		CodeNeed(1);
-		code[ci] = stp->labels[tmp_op->a[arg].val].patches;
-		stp->labels[tmp_op->a[arg].val].patches = ci;
+                register_label_patch(stp, tmp_op->a[arg].val, ci, -last_instr_start);
 		ci++;
 		break;
 	    case TAG_x:
@@ -2582,6 +2714,61 @@ load_code(LoaderState* stp)
 	    }
 	}
 
+        /*
+         * If all the extra arguments were 'f' operands,
+         * and the wordsize is 64 bits, pack two 'f' operands
+         * into each word.
+         */
+
+#if defined(BEAM_WIDE_SHIFT)
+        if (num_trailing_f >= 1) {
+            Uint src_index = ci - num_trailing_f;
+            Uint src_limit = ci;
+            Uint dst_limit = src_index + (num_trailing_f+1)/2;
+
+            ci = src_index;
+            while (ci < dst_limit) {
+                Uint w[2];
+                BeamInstr packed = 0;
+                int wi;
+
+                w[0] = code[src_index];
+                if (src_index+1 < src_limit) {
+                    w[1] = code[src_index+1];
+                } else {
+                    w[1] = 0;
+                }
+                for (wi = 0; wi < 2; wi++) {
+                    Uint lbl = w[wi];
+                    LabelPatch* lp = stp->labels[lbl].patches;
+                    int num_patches = stp->labels[lbl].num_patches;
+
+#if defined(WORDS_BIGENDIAN)
+                    packed <<= BEAM_WIDE_SHIFT;
+                    packed |= lbl & BEAM_WIDE_MASK;
+#else
+                    packed >>= BEAM_WIDE_SHIFT;
+                    packed |= lbl << BEAM_WIDE_SHIFT;
+#endif
+                    while (num_patches-- > 0) {
+                        if (lp->pos == src_index + wi) {
+                            lp->pos = ci;
+#if defined(WORDS_BIGENDIAN)
+                            lp->packed = 2 - wi;
+#else
+                            lp->packed = wi + 1;
+#endif
+                            break;
+                        }
+                        lp++;
+                    }
+                }
+                code[ci++] = packed;
+                src_index += 2;
+            }
+        }
+#endif
+
 	/*
 	 * Handle a few special cases.
 	 */
@@ -2628,17 +2815,16 @@ load_code(LoaderState* stp)
                    the size of the ops.tab i_func_info instruction is not
                    the same as FUNC_INFO_SZ */
 		ASSERT(stp->labels[last_label].value == ci - FUNC_INFO_SZ);
-		stp->hdr->functions[function_number] = (ErtsCodeInfo*) stp->labels[last_label].patches;
 		offset = function_number;
-		stp->labels[last_label].patches = offset;
+                register_label_patch(stp, last_label, offset, 0);
 		function_number++;
 		if (stp->arity > MAX_ARG) {
 		    LoadError1(stp, "too many arguments: %d", stp->arity);
 		}
 #ifdef DEBUG
-		ASSERT(stp->labels[0].patches < 0); /* Should not be referenced. */
+		ASSERT(stp->labels[0].num_patches == 0); /* Should not be referenced. */
 		for (i = 1; i < stp->num_labels; i++) {
-		    ASSERT(stp->labels[i].patches < ci);
+                    ASSERT(stp->labels[i].num_patches <= stp->labels[i].num_allocated);
 		}
 #endif
 	    }
@@ -3563,7 +3749,7 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail,
     if (size == 2) {
 	NEW_GENOP(stp, op);
 	op->next = NULL;
-	op->op = genop_i_select_tuple_arity2_6;
+	op->op = genop_i_select_tuple_arity2_4;
 	GENOP_ARITY(op, arity - 1);
 	op->a[0] = S;
 	op->a[1] = Fail;
@@ -3853,14 +4039,13 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
     int i, j, align = 0;
 
     if (size == 2) {
-
 	/*
 	 * Use a special-cased instruction if there are only two values.
 	 */
 
 	NEW_GENOP(stp, op);
 	op->next = NULL;
-	op->op = genop_i_select_val2_6;
+	op->op = genop_i_select_val2_4;
 	GENOP_ARITY(op, arity - 1);
 	op->a[0] = S;
 	op->a[1] = Fail;
@@ -3870,47 +4055,19 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 	op->a[5] = Rest[3];
 
 	return op;
-
-    } else if (size > 10) {
-
-	/* binary search instruction */
-
-	NEW_GENOP(stp, op);
-	op->next = NULL;
-	op->op = genop_i_select_val_bins_3;
-	GENOP_ARITY(op, arity);
-	op->a[0] = S;
-	op->a[1] = Fail;
-	op->a[2].type = TAG_u;
-	op->a[2].val = size;
-	for (i = 3; i < arity; i++) {
-	    op->a[i] = Rest[i-3];
-	}
-
-	/*
-	 * Sort the values to make them useful for a binary search.
-	 */
-
-	qsort(op->a+3, size, 2*sizeof(GenOpArg),
-		(int (*)(const void *, const void *)) genopargcompare);
-#ifdef DEBUG
-	for (i = 3; i < arity-2; i += 2) {
-	    ASSERT(op->a[i].val < op->a[i+2].val);
-	}
-#endif
-	return op;
     }
 
-    /* linear search instruction */
-
-    align = 1;
+    if (size <= 10) {
+        /* Use linear search. Reserve place for a sentinel. */
+        align = 1;
+    }
 
     arity += 2*align;
     size  += align;
 
     NEW_GENOP(stp, op);
     op->next = NULL;
-    op->op = genop_i_select_val_lins_3;
+    op->op = (align == 0) ? genop_i_select_val_bins_3 : genop_i_select_val_lins_3;
     GENOP_ARITY(op, arity);
     op->a[0] = S;
     op->a[1] = Fail;
@@ -3924,7 +4081,7 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
     }
 
     /*
-     * Sort the values to make them useful for a sentinel search
+     * Sort the values to make them useful for a binary or sentinel search.
      */
 
     qsort(tmp, size - align, 2*sizeof(GenOpArg),
@@ -3939,11 +4096,12 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 
     erts_free(ERTS_ALC_T_LOADER_TMP, (void *) tmp);
 
-    /* add sentinel */
-
-    op->a[j].type = TAG_u;
-    op->a[j].val  = ~((BeamInstr)0);
-    op->a[j+size] = Fail;
+    if (align) {
+        /* Add sentinel for linear search. */
+        op->a[j].type = TAG_u;
+        op->a[j].val  = ~((BeamInstr)0);
+        op->a[j+size] = Fail;
+    }
 
 #ifdef DEBUG
     for (i = 0; i < size - 1; i++) {
@@ -4827,21 +4985,57 @@ freeze_code(LoaderState* stp)
      */
 
     for (i = 0; i < stp->num_labels; i++) {
-	Sint this_patch;
-	Sint next_patch;
+	Uint patch;
 	Uint value = stp->labels[i].value;
-	
-	if (value == 0 && stp->labels[i].patches >= 0) {
+
+	if (value == 0 && stp->labels[i].num_patches != 0) {
 	    LoadError1(stp, "label %d not resolved", i);
 	}
 	ASSERT(value < stp->ci);
-	this_patch = stp->labels[i].patches;
-	while (this_patch >= 0) {
-	    ASSERT(this_patch < stp->ci);
-	    next_patch = codev[this_patch];
-	    ASSERT(next_patch < stp->ci);
-	    codev[this_patch] = (BeamInstr) (codev + value);
-	    this_patch = next_patch;
+        for (patch = 0; patch < stp->labels[i].num_patches; patch++) {
+            LabelPatch* lp = &stp->labels[i].patches[patch];
+            Uint pos = lp->pos;
+	    ASSERT(pos < stp->ci);
+            if (pos < stp->num_functions) {
+                /*
+                 * This is the array of pointers to the beginning of
+                 * each function. The pointers must remain absolute.
+                 */
+                codev[pos] = (BeamInstr) (codev + value);
+            } else {
+#ifdef DEBUG
+                Uint w;
+#endif
+                Sint32 rel = lp->offset + value;
+                switch (lp->packed) {
+                case 0:         /* Not packed */
+                    ASSERT(codev[pos] == i);
+                    codev[pos] = rel;
+                    break;
+#ifdef BEAM_WIDE_MASK
+                case 1:         /* Least significant word. */
+#ifdef DEBUG
+                    w = codev[pos] & BEAM_WIDE_MASK;
+                    /* Correct label in least significant word? */
+                    ASSERT(w == i);
+#endif
+                    codev[pos] = (codev[pos] & ~BEAM_WIDE_MASK) |
+                        (rel & BEAM_WIDE_MASK);
+                    break;
+                case 2:         /* Most significant word */
+#ifdef DEBUG
+                    w = (codev[pos] >> BEAM_WIDE_SHIFT) & BEAM_WIDE_MASK;
+                    /* Correct label in most significant word? */
+                    ASSERT(w == i);
+#endif
+                    codev[pos] = ((Uint)rel << BEAM_WIDE_SHIFT) |
+                        (codev[pos] & BEAM_WIDE_MASK);
+                    break;
+#endif
+                default:
+                    ASSERT(0);
+                }
+            }
 	}
     }
     CHKBLK(ERTS_ALC_T_CODE,code_hdr);
@@ -4884,8 +5078,11 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
     catches = BEAM_CATCHES_NIL;
     while (index != 0) {
 	BeamInstr next = codev[index];
+        BeamInstr* abs_addr;
 	codev[index] = BeamOpCode(op_catch_yf);
-	catches = beam_catches_cons((BeamInstr *)codev[index+2], catches);
+        /* We must make the address of the label absolute again. */
+        abs_addr = (BeamInstr *)codev + index + codev[index+2];
+	catches = beam_catches_cons(abs_addr, catches);
 	codev[index+2] = make_catch(catches);
 	index = next;
     }
@@ -5573,8 +5770,7 @@ new_label(LoaderState* stp)
     stp->labels = (Label *) erts_realloc(ERTS_ALC_T_PREPARED_CODE,
 					 (void *) stp->labels,
 					 stp->num_labels * sizeof(Label));
-    stp->labels[num].value = 0;
-    stp->labels[num].patches = -1;
+    init_label(&stp->labels[num]);
     return num;
 }
 
diff --git a/erts/emulator/beam/bif_instrs.tab b/erts/emulator/beam/bif_instrs.tab
index 3c95113907..0932b8b985 100644
--- a/erts/emulator/beam/bif_instrs.tab
+++ b/erts/emulator/beam/bif_instrs.tab
@@ -151,7 +151,7 @@ i_gc_bif1(Fail, Bif, Src, Live, Dst) {
         $NEXT0();
     }
     if (ERTS_LIKELY($Fail != 0)) { /* Handle error in guard. */
-        $NEXT($Fail);
+        $JUMP($Fail);
     }
 
     /* Handle error in body. */
@@ -202,7 +202,7 @@ i_gc_bif2(Fail, Bif, Live, Src1, Src2, Dst) {
     }
 
     if (ERTS_LIKELY($Fail != 0)) { /* Handle error in guard. */
-        $NEXT($Fail);
+        $JUMP($Fail);
     }
 
     /* Handle error in body. */
@@ -257,7 +257,7 @@ i_gc_bif3(Fail, Bif, Live, Src2, Src3, Dst) {
 
     /* Handle error in guard. */
     if (ERTS_LIKELY($Fail != 0)) {
-        $NEXT($Fail);
+        $JUMP($Fail);
     }
 
     /* Handle error in body. */
@@ -473,10 +473,10 @@ nif_bif.apply_bif() {
 
     /* In case we apply process_info/1,2 or load_nif/1 */
     c_p->current = codemfa;
-    c_p->i = I;		/* In case we apply check_process_code/2. */
-    c_p->arity = 0;		/* To allow garbage collection on ourselves
-                                 * (check_process_code/2).
-                                 */
+    $SET_CP_I_ABS(I);     /* In case we apply check_process_code/2. */
+    c_p->arity = 0;       /* To allow garbage collection on ourselves
+                           * (check_process_code/2).
+                           */
     DTRACE_BIF_ENTRY(c_p, codemfa);
 
     SWAPOUT;
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index b02f966558..2859cdb0f1 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -1367,7 +1367,7 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified,
 #ifdef DEBUG
 		ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI);
 #endif
-                ep->beam[0] = (BeamInstr) BeamOp(op_jump_f);
+                ep->beam[0] = (BeamInstr) BeamOp(op_trace_jump_W);
 		ep->beam[1] = (BeamInstr) ep->addressv[code_ix];
 	    }
 	    erts_set_call_trace_bif(ci, match_prog_set, 0);
@@ -1383,7 +1383,7 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified,
 	     */
 	    erts_clear_call_trace_bif(ci, 0);
 	    if (ep->beam[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
-		ep->beam[0] = (BeamInstr) BeamOp(op_jump_f);
+		ep->beam[0] = (BeamInstr) BeamOp(op_trace_jump_W);
 	    }
 	}
     }
@@ -1675,7 +1675,7 @@ uninstall_exp_breakpoints(BpFunctions* f)
 	if (ep->addressv[code_ix] != ep->beam) {
 	    continue;
 	}
-	ASSERT(ep->beam[0] == (BeamInstr) BeamOp(op_jump_f));
+	ASSERT(ep->beam[0] == (BeamInstr) BeamOp(op_trace_jump_W));
 	ep->addressv[code_ix] = (BeamInstr *) ep->beam[1];
     }
 }
@@ -1694,7 +1694,7 @@ clean_export_entries(BpFunctions* f)
 	if (ep->addressv[code_ix] == ep->beam) {
 	    continue;
 	}
-	if (ep->beam[0] == (BeamInstr) BeamOp(op_jump_f)) {
+	if (ep->beam[0] == (BeamInstr) BeamOp(op_trace_jump_W)) {
 	    ep->beam[0] = (BeamInstr) 0;
 	    ep->beam[1] = (BeamInstr) 0;
 	}
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index f2d0af64df..076767c7cd 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -159,6 +159,7 @@ typedef struct op_entry {
    Uint32 mask[3];		/* Signature mask. */
    unsigned involves_r;		/* Needs special attention when matching. */
    int sz;			/* Number of loaded words. */
+   int adjust;                  /* Adjustment for start of instruction. */
    char* pack;			/* Instructions for packing engine. */
    char* sign;			/* Signature string. */
 } OpEntry;
diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index b79b960fd7..7ea9dee299 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -78,7 +78,14 @@ move_deallocate_return(Src, Deallocate) {
 
 // Call instructions
 
-DISPATCH(CallDest) {
+DISPATCH_REL(CallDest) {
+    //| -no_next
+    $SET_I_REL($CallDest);
+    DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I));
+    Dispatch();
+}
+
+DISPATCH_ABS(CallDest) {
     //| -no_next
     SET_I((BeamInstr *) $CallDest);
     DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I));
@@ -87,18 +94,18 @@ DISPATCH(CallDest) {
 
 i_call(CallDest) {
     SET_CP(c_p, $NEXT_INSTRUCTION);
-    $DISPATCH($CallDest);
+    $DISPATCH_REL($CallDest);
 }
 
 move_call(Src, CallDest) {
     x(0) = $Src;
     SET_CP(c_p, $NEXT_INSTRUCTION);
-    $DISPATCH($CallDest);
+    $DISPATCH_REL($CallDest);
 }
 
 i_call_last(CallDest, Deallocate) {
     $deallocate($Deallocate);
-    $DISPATCH($CallDest);
+    $DISPATCH_REL($CallDest);
 }
 
 move_call_last(Src, CallDest, Deallocate) {
@@ -107,7 +114,7 @@ move_call_last(Src, CallDest, Deallocate) {
 }
 
 i_call_only(CallDest) {
-    $DISPATCH($CallDest);
+    $DISPATCH_REL($CallDest);
 }
 
 move_call_only(Src, CallDest) {
@@ -168,7 +175,8 @@ i_apply() {
     BeamInstr *next;
     $APPLY(NULL, 0, next);
     if (ERTS_LIKELY(next != NULL)) {
-        $i_call(next);
+        SET_CP(c_p, $NEXT_INSTRUCTION);
+        $DISPATCH_ABS(next);
     }
     $HANDLE_APPLY_ERROR();
 }
@@ -177,7 +185,8 @@ i_apply_last(Deallocate) {
     BeamInstr *next;
     $APPLY(I, $Deallocate, next);
     if (ERTS_LIKELY(next != NULL)) {
-        $i_call_last(next, $Deallocate);
+        $deallocate($Deallocate);
+        $DISPATCH_ABS(next);
     }
     $HANDLE_APPLY_ERROR();
 }
@@ -186,7 +195,7 @@ i_apply_only() {
     BeamInstr *next;
     $APPLY(I, 0, next);
     if (ERTS_LIKELY(next != NULL)) {
-        $i_call_only(next);
+        $DISPATCH_ABS(next);
     }
     $HANDLE_APPLY_ERROR();
 }
@@ -202,7 +211,8 @@ apply(Arity) {
     BeamInstr *next;
     $FIXED_APPLY($Arity, NULL, 0, next);
     if (ERTS_LIKELY(next != NULL)) {
-        $i_call(next);
+        SET_CP(c_p, $NEXT_INSTRUCTION);
+        $DISPATCH_ABS(next);
     }
     $HANDLE_APPLY_ERROR();
 }
@@ -211,7 +221,8 @@ apply_last(Arity, Deallocate) {
     BeamInstr *next;
     $FIXED_APPLY($Arity, I, $Deallocate, next);
     if (ERTS_LIKELY(next != NULL)) {
-        $i_call_last(next, $Deallocate);
+        $deallocate($Deallocate);
+        $DISPATCH_ABS(next);
     }
     $HANDLE_APPLY_ERROR();
 }
@@ -560,6 +571,7 @@ i_put_tuple.fill(Arity) {
         }
     } while (--arity != 0);
     HTOP = hp;
+    ASSERT(VALID_INSTR(* (Eterm *)I));
     Goto(*I);
 }
 
@@ -731,9 +743,10 @@ is_reference(Fail, Src) {
 }
 
 is_tagged_tuple(Fail, Src, Arityval, Tag) {
-    if (!(BEAM_IS_TUPLE($Src) &&
-          (tuple_val($Src))[0] == $Arityval &&
-          (tuple_val($Src))[1] == $Tag)) {
+    Eterm term = $Src;
+    if (!(BEAM_IS_TUPLE(term) &&
+          (tuple_val(term))[0] == $Arityval &&
+          (tuple_val(term))[1] == $Tag)) {
         $FAIL($Fail);
     }
 }
@@ -745,7 +758,8 @@ is_tuple(Fail, Src) {
 }
 
 is_tuple_of_arity(Fail, Src, Arityval) {
-    if (!(BEAM_IS_TUPLE($Src) && *tuple_val($Src) == $Arityval)) {
+    Eterm term = $Src;
+    if (!(BEAM_IS_TUPLE(term) && *tuple_val(term) == $Arityval)) {
         $FAIL($Fail);
     }
 }
diff --git a/erts/emulator/beam/macros.tab b/erts/emulator/beam/macros.tab
index 35bbc92fea..6f9b78af6f 100644
--- a/erts/emulator/beam/macros.tab
+++ b/erts/emulator/beam/macros.tab
@@ -28,15 +28,30 @@ REFRESH_GEN_DEST() {
     dst_ptr = REG_TARGET_PTR(dst);
 }
 
+SET_I_REL(Offset) {
+    ASSERT(VALID_INSTR(*(I + ($Offset))));
+    I += $Offset;
+}
+
+SET_CP_I_ABS(Target) {
+    c_p->i = $Target;
+    ASSERT(VALID_INSTR(*c_p->i));
+}
+
+SET_REL_I(Dst, Offset) {
+    $Dst = I + ($Offset);
+    ASSERT(VALID_INSTR(*$Dst));
+}
+
 FAIL(Fail) {
     //| -no_prefetch
-    SET_I((BeamInstr *) $Fail);
+    $SET_I_REL($Fail);
     Goto(*I);
 }
 
 JUMP(Fail) {
     //| -no_next
-    SET_I((BeamInstr *) $Fail);
+    $SET_I_REL($Fail);
     Goto(*I);
 }
 
diff --git a/erts/emulator/beam/msg_instrs.tab b/erts/emulator/beam/msg_instrs.tab
index 093d48c64c..8055a8616f 100644
--- a/erts/emulator/beam/msg_instrs.tab
+++ b/erts/emulator/beam/msg_instrs.tab
@@ -49,7 +49,7 @@ recv_mark(Dest) {
      * the label for the loop_rec/2 instruction for the
      * the receive statement.
      */
-    c_p->msg.mark = (BeamInstr *) $Dest;
+    $SET_REL_I(c_p->msg.mark, $Dest);
     c_p->msg.saved_last = c_p->msg.last;
 }
 
@@ -116,7 +116,7 @@ i_loop_rec(Dest) {
             erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
         } else {
             c_p->flags &= ~F_DELAY_GC;
-            SET_I((BeamInstr *) $Dest);
+            $SET_I_REL($Dest);
             Goto(*I);		/* Jump to a wait or wait_timeout instruction */
         }
     }
@@ -253,7 +253,7 @@ loop_rec_end(Dest) {
 
     ASSERT(c_p->flags & F_DELAY_GC);
 
-    SET_I((BeamInstr *) $Dest);
+    $SET_I_REL($Dest);
     SAVE_MESSAGE(c_p);
     if (FCALLS > 0 || FCALLS > neg_o_reds) {
         FCALLS--;
@@ -261,7 +261,7 @@ loop_rec_end(Dest) {
     }
 
     c_p->flags &= ~F_DELAY_GC;
-    c_p->i = I;
+    $SET_CP_I_ABS(I);
     SWAPOUT;
     c_p->arity = 0;
     c_p->current = NULL;
@@ -374,7 +374,7 @@ wait.src(Src) {
 //
 
 wait.execute(JumpTarget) {
-    c_p->i = (BeamInstr *) $JumpTarget; /* L1 */
+    $SET_REL_I(c_p->i, $JumpTarget); /* L1 */
     SWAPOUT;
     c_p->arity = 0;
 
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 3c9f5a4073..87ff92d354 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -69,6 +69,7 @@ i_debug_breakpoint
 i_return_time_trace
 i_return_to_trace
 i_yield
+trace_jump W
 %hot
 
 return
@@ -161,11 +162,11 @@ i_select_val_bins xy f I
 
 i_select_val_lins xy f I
 
-i_select_val2 xy f c c f f
+i_select_val2 xy f c c
 
 i_select_tuple_arity xy f I
 
-i_select_tuple_arity2 xy f A A f f
+i_select_tuple_arity2 xy f A A
 
 i_jump_on_val_zero xy f I
 
diff --git a/erts/emulator/beam/select_instrs.tab b/erts/emulator/beam/select_instrs.tab
index e85ed2c304..2951949d38 100644
--- a/erts/emulator/beam/select_instrs.tab
+++ b/erts/emulator/beam/select_instrs.tab
@@ -30,16 +30,15 @@ select_val_bins.fetch(Src) {
 }
 
 select_val_bins.select(Fail, NumElements) {
-    struct Pairs {
+    struct Singleton {
         BeamInstr val;
-        BeamInstr* addr;
     };
-    struct Pairs* low;
-    struct Pairs* high;
-    struct Pairs* mid;
+    struct Singleton* low;
+    struct Singleton* high;
+    struct Singleton* mid;
     int bdiff; /* int not long because the arrays aren't that large */
 
-    low = (struct Pairs *) (&$NumElements + 1);
+    low = (struct Singleton *) ($NEXT_INSTRUCTION);
     high = low + $NumElements;
 
     /* The pointer subtraction (high-low) below must produce
@@ -60,80 +59,73 @@ select_val_bins.select(Fail, NumElements) {
      *
      */
     while ((bdiff = (int)((char*)high - (char*)low)) > 0) {
-        unsigned int boffset = ((unsigned int)bdiff >> 1) & ~(sizeof(struct Pairs)-1);
+        unsigned int boffset = ((unsigned int)bdiff >> 1) & ~(sizeof(struct Singleton)-1);
 
-        mid = (struct Pairs*)((char*)low + boffset);
+        mid = (struct Singleton*)((char*)low + boffset);
         if (select_val < mid->val) {
             high = mid;
         } else if (select_val > mid->val) {
             low = mid + 1;
         } else {
-            $NEXT(mid->addr);
+            Sint32* jump_tab = (Sint32 *) ($NEXT_INSTRUCTION + $NumElements);
+            Sint32 offset = jump_tab[mid - (struct Singleton *)($NEXT_INSTRUCTION)];
+            $JUMP(offset);
         }
     }
-    $NEXT($Fail);
+    $JUMP($Fail);
 }
 
-i_select_tuple_arity2 := select_val2.src.ta_fail.execute;
-i_select_val2 := select_val2.src.fail.execute;
+i_select_tuple_arity2 := select_val2.src.get_arity.execute;
+i_select_val2 := select_val2.src.execute;
 
 select_val2.head() {
     Eterm select_val2;
-    BeamInstr* select_fail;
 }
 
 select_val2.src(Src) {
     select_val2 = $Src;
 }
 
-select_val2.ta_fail(Fail) {
-    select_fail = &$Fail;
-    if (is_not_tuple(select_val2)) {
-        $FAIL(*select_fail);
+select_val2.get_arity() {
+    if (ERTS_LIKELY(is_tuple(select_val2))) {
+        select_val2 = *tuple_val(select_val2);
+    } else {
+        select_val2 = NIL;
     }
-    select_val2 = *tuple_val(select_val2);
 }
 
-select_val2.fail(Fail) {
-    select_fail = &$Fail;
-}
+select_val2.execute(Fail, T1, T2) {
+    Sint32* jump_tab = (Sint32 *) ($NEXT_INSTRUCTION);
 
-select_val2.execute(T1, T2, D1, D2) {
     if (select_val2 == $T1) {
-        $JUMP($D1);
+        $JUMP(jump_tab[0]);
     } else if (select_val2 == $T2) {
-        $JUMP($D2);
+        $JUMP(jump_tab[1]);
     } else {
-        $FAIL(*select_fail);
+        $FAIL($Fail);
     }
 }
 
-i_select_tuple_arity := select_val_lin.fetch.ta_fail.execute;
-i_select_val_lins := select_val_lin.fetch.fail.execute;
+i_select_tuple_arity := select_val_lin.fetch.get_arity.execute;
+i_select_val_lins := select_val_lin.fetch.execute;
 
 select_val_lin.head() {
     Eterm select_val;
-    BeamInstr* select_fail;
 }
 
 select_val_lin.fetch(Src) {
     select_val = $Src;
 }
 
-select_val_lin.ta_fail(Fail) {
-    select_fail = &$Fail;
-    if (is_tuple(select_val)) {
+select_val_lin.get_arity() {
+    if (ERTS_LIKELY(is_tuple(select_val))) {
         select_val = *tuple_val(select_val);
     } else {
-        $JUMP(*select_fail);
+        select_val = NIL;
     }
 }
 
-select_val_lin.fail(Fail) {
-    select_fail = &$Fail;
-}
-
-select_val_lin.execute(N) {
+select_val_lin.execute(Fail, N) {
     BeamInstr* vs = $NEXT_INSTRUCTION;
     int ix = 0;
 
@@ -150,10 +142,11 @@ select_val_lin.execute(N) {
     }
 
     if (vs[ix] == select_val) {
-        I = $NEXT_INSTRUCTION + $N + ix;
-        $JUMP(*I);
+        Sint32* jump_tab = (Sint32 *) ($NEXT_INSTRUCTION + $N);
+        Eterm offset = jump_tab[ix];
+        $JUMP(offset);
     } else {
-        $JUMP(*select_fail);
+        $JUMP($Fail);
     }
 }
 
@@ -161,7 +154,8 @@ JUMP_ON_VAL(Fail, Index, N, Base) {
     if (is_small($Index)) {
         $Index = (Uint) (signed_val($Index) - $Base);
         if ($Index < $N) {
-            $JUMP((($NEXT_INSTRUCTION)[$Index]));
+            Sint32* jump_tab = (Sint32 *) ($NEXT_INSTRUCTION);
+            $JUMP(jump_tab[$Index]);
         }
     }
     $FAIL($Fail);
diff --git a/erts/emulator/beam/trace_instrs.tab b/erts/emulator/beam/trace_instrs.tab
index c71f2ef003..b10442c5e7 100644
--- a/erts/emulator/beam/trace_instrs.tab
+++ b/erts/emulator/beam/trace_instrs.tab
@@ -94,7 +94,7 @@ i_yield() {
     c_p->arg_reg[0] = am_true;
     c_p->arity = 1; /* One living register (the 'true' return value) */
     SWAPOUT;
-    c_p->i = $NEXT_INSTRUCTION;
+    $SET_CP_I_ABS($NEXT_INSTRUCTION);
     c_p->current = NULL;
     goto do_schedule;
     //| -no_next
@@ -153,3 +153,16 @@ i_debug_breakpoint() {
     goto handle_error;
     //| -no_next
 }
+
+
+
+//
+// Special jump instruction used for tracing. Takes an absolute
+// failure address.
+//
+
+trace_jump(Fail) {
+    //| -no_next
+    SET_I((BeamInstr *) $Fail);
+    Goto(*I);
+}
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 8e34e07035..bb31db7eb5 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -80,6 +80,7 @@ my %gen_opnum;
 my %num_specific;
 my %gen_to_spec;
 my %specific_op;
+my %group_size;                 # Group size for specific operators.
 
 my %gen_arity;
 my @gen_arity;
@@ -633,7 +634,11 @@ sub emulator_output {
 		$sep = ",";
 	    }
 	    $init .= "}";
-	    init_item($print_name, $init, $involves_r, $size, $pack, $sign);
+            my $adj = 0;
+            if (defined $group_size{$print_name}) {
+                $adj = $size - $group_size{$print_name};
+            }
+	    init_item($print_name, $init, $involves_r, $size, $adj, $pack, $sign);
 	    $op_to_name[$spec_opnum] = $instr;
 	    $spec_opnum++;
 	}
@@ -713,9 +718,9 @@ sub emulator_output {
 	print "#if !defined(ARCH_64)\n";
 	print qq[  #error "64-bit architecture assumed, but ARCH_64 not defined"\n];
 	print "#endif\n";
-	print "#define BEAM_WIDE_MASK 0xFFFFFFFFUL\n";
-	print "#define BEAM_LOOSE_MASK 0xFFFFUL\n";
-	print "#define BEAM_TIGHT_MASK 0xFFFFUL\n";
+	print "#define BEAM_WIDE_MASK 0xFFFFFFFFull\n";
+	print "#define BEAM_LOOSE_MASK 0xFFFFull\n";
+	print "#define BEAM_TIGHT_MASK 0xFFFFull\n";
 	print "#define BEAM_WIDE_SHIFT 32\n";
 	print "#define BEAM_LOOSE_SHIFT 16\n";
 	print "#define BEAM_TIGHT_SHIFT 16\n";
@@ -1181,6 +1186,7 @@ sub combine_instruction_group {
             if ($opcase ne '') {
                 $gcode .= "OpCase($opcase):\n";
                 push @opcase_labels, $opcase;
+                $group_size{$opcase} = $group_size + 1;
             }
             if ($num_references{$label}) {
                 $gcode .= "$label:\n";
@@ -1236,7 +1242,7 @@ sub basic_generator {
     #
 
     my $c_code_ref = $c_code{$name};
-    if ($hot and defined $c_code_ref) {
+    if ($hot and defined $c_code_ref and $name ne 'catch') {
         ($var_decls, $pack_spec, @args) = do_pack(@args);
     }
 
@@ -1544,6 +1550,23 @@ sub do_pack {
     }
 
     #
+    # Try to pack 'f' and 'j', but not at expense at worse packing
+    # for other operands. For example, given the arguments "f x x", we
+    # want the 'x' operands to be packed, not 'f' and 'x' packed and
+    # the final 'x' not packed.
+    #
+
+    if ($wordsize == 64 and $packable_args == 1) {
+        for (my $i = 0; $i < @args; $i++) {
+            if ($args[$i] =~ /^[fj]$/) {
+                $bits_needed[$i] = 32;
+                $packable_args++;
+                last;
+            }
+        }
+    }
+
+    #
     # Nothing to pack unless there are at least 2 packable arguments.
     #
     return ('', '', @args) if $packable_args < 2;
@@ -1629,7 +1652,15 @@ sub do_pack {
 
         if ($arg_size{$arg} and $did_some_packing) {
             # Save the argument on the pack engine's stack.
-            $down = "g${down}";
+            my $push = 'g';
+            if ($type_bit{$arg} & $type_bit{'q'}) {
+                # The operand may be a literal.
+                $push = 'q';
+            } elsif ($type_bit{$arg} & $type_bit{'f'}) {
+                # The operand may be a failure label.
+                $push = 'f';
+            }
+            $down = "$push${down}";
             $up = "${up}p";
         } else {
             # The argument has either zero size (e.g. r(0)),
@@ -1657,7 +1688,7 @@ sub do_pack {
         if ($need_wide_mask[$word]) {
             @shift = ('0', 'BEAM_WIDE_SHIFT');
             @mask = ('BEAM_WIDE_MASK', $WHOLE_WORD);
-            @instr = ('w', 'i');
+            @instr = ('w', 'w');
         } else {
             @shift = @{$pack_shift[$args_per_word]};
             @mask = @{$pack_mask[$args_per_word]};