Merge pull request #2174 from bjorng/bjorn/tune-beam-2

Tune BEAM instructions for the new compiler (part 2)

9 files changed, 495 insertions, 190 deletions

diff --git a/erts/emulator/beam/arith_instrs.tab b/erts/emulator/beam/arith_instrs.tab
index 574fceec5b..5f23b2c168 100644
--- a/erts/emulator/beam/arith_instrs.tab
+++ b/erts/emulator/beam/arith_instrs.tab
@@ -116,6 +116,17 @@ increment.execute(IncrementVal, Dst) {
 i_times(Fail, Op1, Op2, Dst) {
     Eterm op1 = $Op1;
     Eterm op2 = $Op2;
+#ifdef HAVE_OVERFLOW_CHECK_BUILTINS
+    if (ERTS_LIKELY(is_both_small(op1, op2))) {
+        Sint a = signed_val(op1);
+        Sint b = signed_val(op2);
+        Sint res;
+        if (ERTS_LIKELY(!__builtin_mul_overflow(a, b, &res) && IS_SSMALL(res))) {
+            $Dst = make_small(res);
+            $NEXT0();
+        }
+    }
+#endif
     $OUTLINED_ARITH_2($Fail, mixed_times, BIF_stimes_2, op1, op2, $Dst);
 }
 
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 90162a6543..04a2a83123 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -322,19 +322,19 @@ void** beam_ops;
 
 #define Arg(N)       I[(N)+1]
 
-#define GetR(pos, tr)				\
+#define GetSource(raw, dst)			\
    do {						\
-     tr = Arg(pos);				\
-     switch (loader_tag(tr)) {			\
+     dst = raw;                                 \
+     switch (loader_tag(dst)) {			\
      case LOADER_X_REG:				\
-        tr = x(loader_x_reg_index(tr));		\
+        dst = x(loader_x_reg_index(dst));       \
         break;					\
      case LOADER_Y_REG:				\
-        ASSERT(loader_y_reg_index(tr) >= 1);	\
-        tr = y(loader_y_reg_index(tr));		\
+        ASSERT(loader_y_reg_index(dst) >= 1);	\
+        dst = y(loader_y_reg_index(dst));       \
         break;					\
      }						\
-     CHECK_TERM(tr);				\
+     CHECK_TERM(dst);				\
    } while (0)
 
 #define PUT_TERM_REG(term, desc)		\
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index f4eeb54a1b..43adf7a5e0 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -3295,11 +3295,11 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live,
     } else {
 	op->op = genop_i_bs_get_integer_6;
 	op->arity = 6;
-	op->a[0] = Fail;
-	op->a[1] = Live;
-	op->a[2].type = TAG_u;
-	op->a[2].val = (Unit.val << 3) | Flags.val;
-	op->a[3] = Ms;
+	op->a[0] = Ms;
+	op->a[1] = Fail;
+	op->a[2] = Live;
+	op->a[3].type = TAG_u;
+	op->a[3].val = (Unit.val << 3) | Flags.val;
 	op->a[4] = Size;
 	op->a[5] = Dst;
 	op->next = NULL;
@@ -3332,8 +3332,8 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live,
 	} else {
 	    op->op = genop_i_bs_get_binary_all2_5;
 	    op->arity = 5;
-	    op->a[0] = Fail;
-	    op->a[1] = Ms;
+	    op->a[0] = Ms;
+	    op->a[1] = Fail;
 	    op->a[2] = Live;	
 	    op->a[3] = Unit;
 	    op->a[4] = Dst;
@@ -3341,8 +3341,8 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live,
     } else if (Size.type == TAG_i) {
 	op->op = genop_i_bs_get_binary_imm2_6;
 	op->arity = 6;
-	op->a[0] = Fail;
-	op->a[1] = Ms;
+	op->a[0] = Ms;
+	op->a[1] = Fail;
 	op->a[2] = Live;
 	op->a[3].type = TAG_u;
 	if (!safe_mul(Size.val, Unit.val, &op->a[3].val)) {
@@ -3362,8 +3362,8 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live,
 	} else {
 	    op->op = genop_i_bs_get_binary_imm2_6;
 	    op->arity = 6;
-	    op->a[0] = Fail;
-	    op->a[1] = Ms;
+	    op->a[0] = Ms;
+	    op->a[1] = Fail;
 	    op->a[2] = Live;
 	    op->a[3].type = TAG_u;
 	    if (!safe_mul(bigval, Unit.val, &op->a[3].val)) {
@@ -3375,8 +3375,8 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live,
     } else {
 	op->op = genop_i_bs_get_binary2_6;
 	op->arity = 6;
-	op->a[0] = Fail;
-	op->a[1] = Ms;
+	op->a[0] = Ms;
+	op->a[1] = Fail;
 	op->a[2] = Live;
 	op->a[3] = Size;
 	op->a[4].type = TAG_u;
@@ -3409,8 +3409,8 @@ gen_put_binary(LoaderState* stp, GenOpArg Fail,GenOpArg Size,
     if (Size.type == TAG_a && Size.val == am_all) {
 	op->op = genop_i_new_bs_put_binary_all_3;
 	op->arity = 3;
-	op->a[0] = Fail;
-	op->a[1] = Src;
+	op->a[0] = Src;
+	op->a[1] = Fail;
 	op->a[2] = Unit;
     } else if (Size.type == TAG_i) {
 	op->op = genop_i_new_bs_put_binary_imm_3;
@@ -3420,10 +3420,33 @@ gen_put_binary(LoaderState* stp, GenOpArg Fail,GenOpArg Size,
 	if (safe_mul(Size.val, Unit.val, &op->a[1].val)) {
 	    op->a[2] = Src;
 	} else {
+        error:
 	    op->op = genop_badarg_1;
 	    op->arity = 1;
 	    op->a[0] = Fail;
 	}
+    } else if (Size.type == TAG_q) {
+#ifdef ARCH_64
+        /*
+         * There is no way that this binary would fit in memory.
+         */
+        goto error;
+#else
+	Eterm big = stp->literals[Size.val].term;
+	Uint bigval;
+        Uint size;
+
+	if (!term_to_Uint(big, &bigval) ||
+            !safe_mul(bigval, Unit.val, &size)) {
+	    goto error;
+	}
+        op->op = genop_i_new_bs_put_binary_imm_3;
+        op->arity = 3;
+        op->a[0] = Fail;
+        op->a[1].type = TAG_u;
+        op->a[1].val = size;
+        op->a[2] = Src;
+#endif
     } else {
 	op->op = genop_i_new_bs_put_binary_4;
 	op->arity = 4;
@@ -3448,11 +3471,8 @@ gen_put_integer(LoaderState* stp, GenOpArg Fail, GenOpArg Size,
     NATIVE_ENDIAN(Flags);
 	/* Negative size must fail */
     if (Size.type == TAG_i) {
-	op->op = genop_i_new_bs_put_integer_imm_4;
-	op->arity = 4;
-	op->a[0] = Fail;
-	op->a[1].type = TAG_u;
-	if (!safe_mul(Size.val, Unit.val, &op->a[1].val)) {
+        Uint size;
+	if (!safe_mul(Size.val, Unit.val, &size)) {
         error:
             op->op = genop_badarg_1;
             op->arity = 1;
@@ -3460,26 +3480,31 @@ gen_put_integer(LoaderState* stp, GenOpArg Fail, GenOpArg Size,
             op->next = NULL;
             return op;
 	}
-	op->a[1].val = Size.val * Unit.val;
-	op->a[2].type = Flags.type;
-	op->a[2].val = (Flags.val & 7);
-	op->a[3] = Src;
+	op->op = genop_i_new_bs_put_integer_imm_4;
+	op->arity = 4;
+	op->a[0] = Src;
+	op->a[1] = Fail;
+	op->a[2].type = TAG_u;
+	op->a[2].val = size;
+	op->a[3].type = Flags.type;
+	op->a[3].val = (Flags.val & 7);
     } else if (Size.type == TAG_q) {
 	Eterm big = stp->literals[Size.val].term;
 	Uint bigval;
+        Uint size;
 
-	if (!term_to_Uint(big, &bigval)) {
+	if (!term_to_Uint(big, &bigval) ||
+            !safe_mul(bigval, Unit.val, &size)) {
 	    goto error;
-	} else {
-	    op->op = genop_i_new_bs_put_integer_imm_4;
-	    op->arity = 4;
-	    op->a[0] = Fail;
-	    op->a[1].type = TAG_u;
-	    op->a[1].val = bigval * Unit.val;
-	    op->a[2].type = Flags.type;
-	    op->a[2].val = (Flags.val & 7);
-	    op->a[3] = Src;
 	}
+	op->op = genop_i_new_bs_put_integer_imm_4;
+	op->arity = 4;
+	op->a[0] = Src;
+	op->a[1] = Fail;
+	op->a[2].type = TAG_u;
+	op->a[2].val = size;
+	op->a[3].type = Flags.type;
+	op->a[3].val = (Flags.val & 7);
     } else {
 	op->op = genop_i_new_bs_put_integer_4;
 	op->arity = 4;
@@ -3541,8 +3566,8 @@ gen_get_float2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live,
     NATIVE_ENDIAN(Flags);
     op->op = genop_i_bs_get_float2_6;
     op->arity = 6;
-    op->a[0] = Fail;
-    op->a[1] = Ms;
+    op->a[0] = Ms;
+    op->a[1] = Fail;
     op->a[2] = Live;
     op->a[3] = Size;
     op->a[4].type = TAG_u;
@@ -3565,10 +3590,22 @@ gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms,
     NATIVE_ENDIAN(Flags);
     NEW_GENOP(stp, op);
     if (Size.type == TAG_a && Size.val == am_all) {
-	op->op = genop_i_bs_skip_bits_all2_3;
+        /*
+         * This kind of skip instruction will only be found in modules
+         * compiled before OTP 19. From OTP 19, the compiler generates
+         * a test_unit instruction of a bs_skip at the end of a
+         * binary.
+         *
+         * It is safe to replace the skip instruction with a test_unit
+         * instruction, because the position will never be used again.
+         * If the match context itself is used again, it will be used by
+         * a bs_restore2 instruction which will overwrite the position
+         * by one of the stored positions.
+         */
+	op->op = genop_bs_test_unit_3;
 	op->arity = 3;
 	op->a[0] = Fail;
-	op->a[1] = Ms; 
+	op->a[1] = Ms;
 	op->a[2] = Unit;
     } else if (Size.type == TAG_i) {
 	op->op = genop_i_bs_skip_bits_imm2_3;
@@ -3601,9 +3638,9 @@ gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms,
     } else {
 	op->op = genop_i_bs_skip_bits2_4;
 	op->arity = 4;
-	op->a[0] = Fail;
-	op->a[1] = Ms; 
-	op->a[2] = Size;
+	op->a[0] = Ms;
+	op->a[1] = Size;
+	op->a[2] = Fail;
 	op->a[3] = Unit;
     }
     op->next = NULL;
diff --git a/erts/emulator/beam/bif_instrs.tab b/erts/emulator/beam/bif_instrs.tab
index 418bbe2b23..8499f61114 100644
--- a/erts/emulator/beam/bif_instrs.tab
+++ b/erts/emulator/beam/bif_instrs.tab
@@ -269,7 +269,7 @@ call_bif(Exp) {
         CHECK_TERM(r(0));
         $NEXT0();
     } else if (c_p->freason == TRAP) {
-        SET_CP(c_p, I+2);
+        SET_CP(c_p, $NEXT_INSTRUCTION);
         SET_I(c_p->i);
         SWAPIN;
         Dispatch();
@@ -313,7 +313,7 @@ send() {
         r(0) = result;
         CHECK_TERM(r(0));
     } else if (c_p->freason == TRAP) {
-        SET_CP(c_p, I+1);
+        SET_CP(c_p, $NEXT_INSTRUCTION);
         SET_I(c_p->i);
         SWAPIN;
         Dispatch();
diff --git a/erts/emulator/beam/bs_instrs.tab b/erts/emulator/beam/bs_instrs.tab
index 2dde70c2e1..652460a66d 100644
--- a/erts/emulator/beam/bs_instrs.tab
+++ b/erts/emulator/beam/bs_instrs.tab
@@ -21,12 +21,57 @@
 
 %if ARCH_64
 BS_SAFE_MUL(A, B, Fail, Dst) {
-    Uint64 res = ($A) * ($B);
-    if (res / $B != $A) {
+    Uint a = $A;
+    Uint b = $B;
+    Uint res;
+#ifdef HAVE_OVERFLOW_CHECK_BUILTINS
+    if (__builtin_mul_overflow(a, b, &res)) {
+        $Fail;
+    }
+#else
+    res = a * b;
+    if (res / b != a) {
         $Fail;
     }
+#endif
     $Dst = res;
 }
+
+BS_GET_FIELD_SIZE(Bits, Unit, Fail, Dst) {
+    if (is_small($Bits)) {
+        Uint uint_size;
+        Sint signed_size = signed_val($Bits);
+        if (signed_size < 0) {
+            $Fail;
+        }
+        uint_size = (Uint) signed_size;
+        $BS_SAFE_MUL(uint_size, $Unit, $Fail, $Dst);
+    } else {
+        /*
+         * On a 64-bit architecture, the size of any binary
+         * that would fit in the memory fits in a small.
+         */
+        $Fail;
+    }
+}
+
+BS_GET_UNCHECKED_FIELD_SIZE(Bits, Unit, Fail, Dst) {
+    if (is_small($Bits)) {
+        Uint uint_size;
+        Sint signed_size = signed_val($Bits);
+        if (signed_size < 0) {
+            $Fail;
+        }
+        uint_size = (Uint) signed_size;
+        $Dst = uint_size * $Unit;
+    } else {
+        /*
+         * On a 64-bit architecture, the size of any binary
+         * that would fit in the memory fits in a small.
+         */
+        $Fail;
+    }
+}
 %else
 BS_SAFE_MUL(A, B, Fail, Dst) {
     Uint64 res = (Uint64)($A) * (Uint64)($B);
@@ -35,7 +80,6 @@ BS_SAFE_MUL(A, B, Fail, Dst) {
     }
     $Dst = res;
 }
-%endif
 
 BS_GET_FIELD_SIZE(Bits, Unit, Fail, Dst) {
     Sint signed_size;
@@ -76,6 +120,7 @@ BS_GET_UNCHECKED_FIELD_SIZE(Bits, Unit, Fail, Dst) {
     }
     $Dst = uint_size * $Unit;
 }
+%endif
 
 TEST_BIN_VHEAP(VNh, Nh, Live) {
     Uint need = $Nh;
@@ -90,12 +135,22 @@ TEST_BIN_VHEAP(VNh, Nh, Live) {
     HEAP_SPACE_VERIFIED(need);
 }
 
-i_bs_get_binary_all2(Fail, Ms, Live, Unit, Dst) {
+i_bs_get_binary_all2 := i_bs_get_binary_all2.fetch.execute;
+
+i_bs_get_binary_all2.head() {
+    Eterm context;
+}
+
+i_bs_get_binary_all2.fetch(Ctx) {
+    context = $Ctx;
+}
+
+i_bs_get_binary_all2.execute(Fail, Live, Unit, Dst) {
     ErlBinMatchBuffer *_mb;
     Eterm _result;
 
-    $GC_TEST(0, ERL_SUB_BIN_SIZE, $Live);
-    _mb = ms_matchbuffer($Ms);
+    $GC_TEST_PRESERVE(ERL_SUB_BIN_SIZE, $Live, context);
+    _mb = ms_matchbuffer(context);
     if (((_mb->size - _mb->offset) % $Unit) == 0) {
         LIGHT_SWAPOUT;
         _result = erts_bs_get_binary_all_2(c_p, _mb);
@@ -109,14 +164,23 @@ i_bs_get_binary_all2(Fail, Ms, Live, Unit, Dst) {
 	$FAIL($Fail);
     }
 }
+i_bs_get_binary2 := i_bs_get_binary2.fetch.execute;
+
+i_bs_get_binary2.head() {
+    Eterm context;
+}
+
+i_bs_get_binary2.fetch(Ctx) {
+    context = $Ctx;
+}
 
-i_bs_get_binary2(Fail, Ms, Live, Sz, Flags, Dst) {
+i_bs_get_binary2.execute(Fail, Live, Sz, Flags, Dst) {
     ErlBinMatchBuffer *_mb;
     Eterm _result;
     Uint _size;
     $BS_GET_FIELD_SIZE($Sz, (($Flags) >> 3), $FAIL($Fail), _size);
-    $GC_TEST(0, ERL_SUB_BIN_SIZE, $Live);
-    _mb = ms_matchbuffer($Ms);
+    $GC_TEST_PRESERVE(ERL_SUB_BIN_SIZE, $Live, context);
+    _mb = ms_matchbuffer(context);
     LIGHT_SWAPOUT;
     _result = erts_bs_get_binary_2(c_p, _size, $Flags, _mb);
     LIGHT_SWAPIN;
@@ -129,11 +193,22 @@ i_bs_get_binary2(Fail, Ms, Live, Sz, Flags, Dst) {
     }
 }
 
-i_bs_get_binary_imm2(Fail, Ms, Live, Sz, Flags, Dst) {
+i_bs_get_binary_imm2 := i_bs_get_binary_imm2.fetch.execute;
+
+i_bs_get_binary_imm2.head() {
+    Eterm context;
+}
+
+i_bs_get_binary_imm2.fetch(Ctx) {
+    context = $Ctx;
+}
+
+i_bs_get_binary_imm2.execute(Fail, Live, Sz, Flags, Dst) {
     ErlBinMatchBuffer *_mb;
     Eterm _result;
-    $GC_TEST(0, heap_bin_size(ERL_ONHEAP_BIN_LIMIT), $Live);
-    _mb = ms_matchbuffer($Ms);
+    $GC_TEST_PRESERVE(heap_bin_size(ERL_ONHEAP_BIN_LIMIT),
+                      $Live, context);
+    _mb = ms_matchbuffer(context);
     LIGHT_SWAPOUT;
     _result = erts_bs_get_binary_2(c_p, $Sz, $Flags, _mb);
     LIGHT_SWAPIN;
@@ -145,8 +220,17 @@ i_bs_get_binary_imm2(Fail, Ms, Live, Sz, Flags, Dst) {
         $Dst = _result;
     }
 }
+i_bs_get_float2 := i_bs_get_float2.fetch.execute;
+
+i_bs_get_float2.head() {
+    Eterm context;
+}
+
+i_bs_get_float2.fetch(Ctx) {
+    context = $Ctx;
+}
 
-i_bs_get_float2(Fail, Ms, Live, Sz, Flags, Dst) {
+i_bs_get_float2.execute(Fail, Live, Sz, Flags, Dst) {
     ErlBinMatchBuffer *_mb;
     Eterm _result;
     Sint _size;
@@ -155,8 +239,8 @@ i_bs_get_float2(Fail, Ms, Live, Sz, Flags, Dst) {
         $FAIL($Fail);
     }
     _size *= (($Flags) >> 3);
-    $GC_TEST(0, FLOAT_SIZE_OBJECT, $Live);
-    _mb = ms_matchbuffer($Ms);
+    $GC_TEST_PRESERVE(FLOAT_SIZE_OBJECT, $Live, context);
+    _mb = ms_matchbuffer(context);
     LIGHT_SWAPOUT;
     _result = erts_bs_get_float_2(c_p, _size, ($Flags), _mb);
     LIGHT_SWAPIN;
@@ -169,13 +253,24 @@ i_bs_get_float2(Fail, Ms, Live, Sz, Flags, Dst) {
     }
 }
 
-i_bs_skip_bits2(Fail, Ms, Bits, Unit) {
+i_bs_skip_bits2 := i_bs_skip_bits2.fetch.execute;
+
+i_bs_skip_bits2.head() {
+    Eterm context, bits;
+}
+
+i_bs_skip_bits2.fetch(Ctx, Bits) {
+    context = $Ctx;
+    bits = $Bits;
+}
+
+i_bs_skip_bits2.execute(Fail, Unit) {
     ErlBinMatchBuffer *_mb;
     size_t new_offset;
     Uint _size;
 
-    _mb = ms_matchbuffer($Ms);
-    $BS_GET_FIELD_SIZE($Bits, $Unit, $FAIL($Fail), _size);
+    _mb = ms_matchbuffer(context);
+    $BS_GET_FIELD_SIZE(bits, $Unit, $FAIL($Fail), _size);
     new_offset = _mb->offset + _size;
     if (new_offset <= _mb->size) {
         _mb->offset = new_offset;
@@ -184,16 +279,6 @@ i_bs_skip_bits2(Fail, Ms, Bits, Unit) {
     }
 }
 
-i_bs_skip_bits_all2(Fail, Ms, Unit) {
-    ErlBinMatchBuffer *_mb;
-    _mb = ms_matchbuffer($Ms);
-    if (((_mb->size - _mb->offset) % $Unit) == 0) {
-        _mb->offset = _mb->size;
-    } else {
-        $FAIL($Fail);
-    }
-}
-
 i_bs_skip_bits_imm2(Fail, Ms, Bits) {
     ErlBinMatchBuffer *_mb;
     size_t new_offset;
@@ -207,15 +292,25 @@ i_bs_skip_bits_imm2(Fail, Ms, Bits) {
 }
 
 i_new_bs_put_binary(Fail, Sz, Flags, Src) {
+    Eterm sz = $Sz;
     Sint _size;
-    $BS_GET_UNCHECKED_FIELD_SIZE($Sz, (($Flags) >> 3), $BADARG($Fail), _size);
+    $BS_GET_UNCHECKED_FIELD_SIZE(sz, (($Flags) >> 3), $BADARG($Fail), _size);
     if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2(($Src), _size))) {
         $BADARG($Fail);
     }
 }
+i_new_bs_put_binary_all := i_new_bs_put_binary_all.fetch.execute;
+
+i_new_bs_put_binary_all.head() {
+    Eterm src;
+}
+
+i_new_bs_put_binary_all.fetch(Src) {
+    src = $Src;
+}
 
-i_new_bs_put_binary_all(Fail, Src, Unit) {
-    if (!erts_new_bs_put_binary_all(ERL_BITS_ARGS_2(($Src), ($Unit)))) {
+i_new_bs_put_binary_all.execute(Fail, Unit) {
+    if (!erts_new_bs_put_binary_all(ERL_BITS_ARGS_2(src, ($Unit)))) {
         $BADARG($Fail);
     }
 }
@@ -227,9 +322,11 @@ i_new_bs_put_binary_imm(Fail, Sz, Src) {
 }
 
 i_new_bs_put_float(Fail, Sz, Flags, Src) {
+    Eterm sz = $Sz;
+    Eterm flags = $Flags;
     Sint _size;
-    $BS_GET_UNCHECKED_FIELD_SIZE($Sz, (($Flags) >> 3), $BADARG($Fail), _size);
-    if (!erts_new_bs_put_float(c_p, ($Src), _size, ($Flags))) {
+    $BS_GET_UNCHECKED_FIELD_SIZE(sz, (flags >> 3), $BADARG($Fail), _size);
+    if (!erts_new_bs_put_float(c_p, ($Src), _size, flags)) {
         $BADARG($Fail);
     }
 }
@@ -241,15 +338,27 @@ i_new_bs_put_float_imm(Fail, Sz, Flags, Src) {
 }
 
 i_new_bs_put_integer(Fail, Sz, Flags, Src) {
-   Sint _size;
-   $BS_GET_UNCHECKED_FIELD_SIZE($Sz, (($Flags) >> 3), $BADARG($Fail), _size);
-   if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(($Src), _size, ($Flags)))) {
-       $BADARG($Fail);
-   }
+    Eterm sz = $Sz;
+    Eterm flags = $Flags;
+    Sint _size;
+    $BS_GET_UNCHECKED_FIELD_SIZE(sz, (flags >> 3), $BADARG($Fail), _size);
+    if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(($Src), _size, flags))) {
+        $BADARG($Fail);
+    }
+}
+
+i_new_bs_put_integer_imm := i_new_bs_put_integer_imm.fetch.execute;
+
+i_new_bs_put_integer_imm.head() {
+    Eterm src;
 }
 
-i_new_bs_put_integer_imm(Fail, Sz, Flags, Src) {
-    if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(($Src), ($Sz), ($Flags)))) {
+i_new_bs_put_integer_imm.fetch(Src) {
+    src = $Src;
+}
+
+i_new_bs_put_integer_imm.execute(Fail, Sz, Flags) {
+    if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(src, ($Sz), ($Flags)))) {
         $BADARG($Fail);
     }
 }
@@ -809,9 +918,19 @@ bs_test_unit8(Fail, Ctx) {
     }
 }
 
-i_bs_get_integer_8(Ctx, Fail, Dst) {
+i_bs_get_integer_8 := i_bs_get_integer_8.fetch.execute;
+
+i_bs_get_integer_8.head() {
+    Eterm context;
+}
+
+i_bs_get_integer_8.fetch(Ctx) {
+    context = $Ctx;
+}
+
+i_bs_get_integer_8.execute(Fail, Dst) {
     Eterm _result;
-    ErlBinMatchBuffer* _mb = ms_matchbuffer($Ctx);
+    ErlBinMatchBuffer* _mb = ms_matchbuffer(context);
 
     if (_mb->size - _mb->offset < 8) {
         $FAIL($Fail);
@@ -825,9 +944,19 @@ i_bs_get_integer_8(Ctx, Fail, Dst) {
     $Dst = _result;
 }
 
-i_bs_get_integer_16(Ctx, Fail, Dst) {
+i_bs_get_integer_16 := i_bs_get_integer_16.fetch.execute;
+
+i_bs_get_integer_16.head() {
+    Eterm context;
+}
+
+i_bs_get_integer_16.fetch(Ctx) {
+    context = $Ctx;
+}
+
+i_bs_get_integer_16.execute(Fail, Dst) {
     Eterm _result;
-    ErlBinMatchBuffer* _mb = ms_matchbuffer($Ctx);
+    ErlBinMatchBuffer* _mb = ms_matchbuffer(context);
 
     if (_mb->size - _mb->offset < 16) {
         $FAIL($Fail);
@@ -842,9 +971,19 @@ i_bs_get_integer_16(Ctx, Fail, Dst) {
 }
 
 %if ARCH_64
-i_bs_get_integer_32(Ctx, Fail, Dst) {
+i_bs_get_integer_32 := i_bs_get_integer_32.fetch.execute;
+
+i_bs_get_integer_32.head() {
+    Eterm context;
+}
+
+i_bs_get_integer_32.fetch(Ctx) {
+    context = $Ctx;
+}
+
+i_bs_get_integer_32.execute(Fail, Dst) {
     Uint32 _integer;
-    ErlBinMatchBuffer* _mb = ms_matchbuffer($Ctx);
+    ErlBinMatchBuffer* _mb = ms_matchbuffer(context);
 
     if (_mb->size - _mb->offset < 32) {
         $FAIL($Fail);
@@ -894,15 +1033,23 @@ bs_get_integer.execute(Fail, Flags, Dst) {
     $Dst = result;
 }
 
-i_bs_get_integer(Fail, Live, FlagsAndUnit, Ms, Sz, Dst) {
+i_bs_get_integer := i_bs_get_integer.fetch.execute;
+
+i_bs_get_integer.head() {
+    Eterm context;
+}
+
+i_bs_get_integer.fetch(Ctx) {
+    context = $Ctx;
+}
+
+i_bs_get_integer.execute(Fail, Live, FlagsAndUnit, Sz, Dst) {
     Uint flags;
     Uint size;
-    Eterm ms;
     ErlBinMatchBuffer* mb;
     Eterm result;
 
     flags = $FlagsAndUnit;
-    ms = $Ms;
     $BS_GET_FIELD_SIZE($Sz, (flags >> 3), $FAIL($Fail), size);
     if (size >= SMALL_BITS) {
         Uint wordsneeded;
@@ -913,15 +1060,15 @@ i_bs_get_integer(Fail, Live, FlagsAndUnit, Ms, Sz, Dst) {
          * Remember to re-acquire the matchbuffer after gc.
          */
 
-        mb = ms_matchbuffer(ms);
+        mb = ms_matchbuffer(context);
         if (mb->size - mb->offset < size) {
             $FAIL($Fail);
         }
         wordsneeded = 1+WSIZE(NBYTES((Uint) size));
-        $GC_TEST_PRESERVE(wordsneeded, $Live, ms);
+        $GC_TEST_PRESERVE(wordsneeded, $Live, context);
         $REFRESH_GEN_DEST();
     }
-    mb = ms_matchbuffer(ms);
+    mb = ms_matchbuffer(context);
     LIGHT_SWAPOUT;
     result = erts_bs_get_integer_2(c_p, size, flags, mb);
     LIGHT_SWAPIN;
@@ -932,9 +1079,19 @@ i_bs_get_integer(Fail, Live, FlagsAndUnit, Ms, Sz, Dst) {
     $Dst = result;
 }
 
-i_bs_get_utf8(Ctx, Fail, Dst) {
+i_bs_get_utf8 := i_bs_get_utf8.fetch.execute;
+
+i_bs_get_utf8.head() {
+    Eterm context;
+}
+
+i_bs_get_utf8.fetch(Ctx) {
+    context = $Ctx;
+}
+
+i_bs_get_utf8.execute(Fail, Dst) {
     Eterm result;
-    ErlBinMatchBuffer* mb = ms_matchbuffer($Ctx);
+    ErlBinMatchBuffer* mb = ms_matchbuffer(context);
 
     if (mb->size - mb->offset < 8) {
         $FAIL($Fail);
@@ -957,8 +1114,18 @@ i_bs_get_utf8(Ctx, Fail, Dst) {
     $Dst = result;
 }
 
-i_bs_get_utf16(Ctx, Fail, Flags, Dst) {
-    ErlBinMatchBuffer* mb = ms_matchbuffer($Ctx);
+i_bs_get_utf16 := i_bs_get_utf16.fetch.execute;
+
+i_bs_get_utf16.head() {
+    Eterm context;
+}
+
+i_bs_get_utf16.fetch(Ctx) {
+    context = $Ctx;
+}
+
+i_bs_get_utf16.execute(Fail, Flags, Dst) {
+    ErlBinMatchBuffer* mb = ms_matchbuffer(context);
     Eterm result = erts_bs_get_utf16(mb, $Flags);
 
     if (is_non_value(result)) {
@@ -1055,13 +1222,20 @@ i_bs_restore2(Src, Slot) {
     _ms->mb.offset = _ms->save_offset[$Slot];
 }
 
-bs_get_tail(Src, Dst, Live) {
-    ErlBinMatchBuffer* mb;
-    Uint size, offs;
-    ErlSubBin* sb;
+bs_get_tail := bs_get_tail.fetch.execute;
+
+bs_get_tail.head() {
     Eterm context;
+}
 
+bs_get_tail.fetch(Src) {
     context = $Src;
+}
+
+bs_get_tail.execute(Dst, Live) {
+    ErlBinMatchBuffer* mb;
+    Uint size, offs;
+    ErlSubBin* sb;
 
     ASSERT(header_is_bin_matchstate(*boxed_val(context)));
 
@@ -1090,11 +1264,20 @@ bs_get_tail(Src, Dst, Live) {
 
 %if ARCH_64
 
-i_bs_start_match3_gp(Src, Live, Fail, Dst, Pos) {
-    Eterm context, header;
-    Uint position, live;
+i_bs_start_match3_gp := i_bs_start_match3_gp.fetch.execute;
 
+i_bs_start_match3_gp.head() {
+    Eterm context;
+}
+
+i_bs_start_match3_gp.fetch(Src) {
     context = $Src;
+}
+
+i_bs_start_match3_gp.execute(Live, Fail, Dst, Pos) {
+    Eterm header;
+    Uint position, live;
+
     live = $Live;
 
     if (!is_boxed(context)) {
@@ -1139,11 +1322,20 @@ i_bs_start_match3_gp(Src, Live, Fail, Dst, Pos) {
     $Pos = make_small(position);
 }
 
-i_bs_start_match3(Src, Live, Fail, Dst) {
-    Eterm context, header;
-    Uint live;
+i_bs_start_match3 := i_bs_start_match3.fetch.execute;
+
+i_bs_start_match3.head() {
+    Eterm context;
+}
 
+i_bs_start_match3.fetch(Src) {
     context = $Src;
+}
+
+i_bs_start_match3.execute(Live, Fail, Dst) {
+    Eterm header;
+    Uint live;
+
     live = $Live;
 
     if (!is_boxed(context)) {
@@ -1213,12 +1405,19 @@ i_bs_get_position(Ctx, Dst) {
 # match at a position beyond 16MB.
 #
 
-bs_set_position(Ctx, Pos) {
+bs_set_position := bs_set_position.fetch.execute;
+
+bs_set_position.head() {
     Eterm context, position;
-    ErlBinMatchState *ms;
+}
 
+bs_set_position.fetch(Ctx, Pos) {
     context = $Ctx;
     position = $Pos;
+}
+
+bs_set_position.execute() {
+    ErlBinMatchState *ms;
 
     ASSERT(header_is_bin_matchstate(*boxed_val(context)));
     ms = (ErlBinMatchState*)boxed_val(context);
@@ -1231,12 +1430,19 @@ bs_set_position(Ctx, Pos) {
     }
 }
 
-bs_get_position(Ctx, Dst, Live) {
-    ErlBinMatchState *ms;
+bs_get_position := bs_get_position.fetch.execute;
+
+bs_get_position.head() {
     Eterm context;
-    Uint position;
+}
 
+bs_get_position.fetch(Ctx) {
     context = $Ctx;
+}
+
+bs_get_position.execute(Dst, Live) {
+    ErlBinMatchState *ms;
+    Uint position;
 
     ASSERT(header_is_bin_matchstate(*boxed_val(context)));
     ms = (ErlBinMatchState*)boxed_val(context);
@@ -1261,11 +1467,20 @@ bs_get_position(Ctx, Dst, Live) {
     }
 }
 
-i_bs_start_match3(Src, Live, Fail, Dst) {
-    Eterm context, header;
-    Uint live;
+i_bs_start_match3 := i_bs_start_match3.fetch.execute;
+
+i_bs_start_match3.head() {
+    Eterm context;
+}
 
+i_bs_start_match3.fetch(Src) {
     context = $Src;
+}
+
+i_bs_start_match3.execute(Live, Fail, Dst) {
+    Eterm header;
+    Uint live;
+
     live = $Live;
 
     if (!is_boxed(context)) {
diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index fc88cab22f..1eb83b61f2 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -512,6 +512,15 @@ move_shift(Src, SD, D) {
     $SD = V;
 }
 
+move_window2(S1, S2, D) {
+    Eterm xt0, xt1;
+    Eterm* y = &$D;
+    xt0  = $S1;
+    xt1  = $S2;
+    y[0] = xt0;
+    y[1] = xt1;
+}
+
 move_window3(S1, S2, S3, D) {
     Eterm xt0, xt1, xt2;
     Eterm* y = &$D;
@@ -875,12 +884,16 @@ i_is_ne_exact_literal(Fail, Src, Literal) {
     }
 }
 
-is_eq(Fail, X, Y) {
-    CMP_EQ_ACTION($X, $Y, $FAIL($Fail));
+is_eq(Fail, A, B) {
+    Eterm a = $A;
+    Eterm b = $B;
+    CMP_EQ_ACTION(a, b, $FAIL($Fail));
 }
 
-is_ne(Fail, X, Y) {
-    CMP_NE_ACTION($X, $Y, $FAIL($Fail));
+is_ne(Fail, A, B) {
+    Eterm a = $A;
+    Eterm b = $B;
+    CMP_NE_ACTION(a, b, $FAIL($Fail));
 }
 
 is_lt(Fail, X, Y) {
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 3cfc685336..e688c6996b 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -215,11 +215,7 @@ set_tuple_element s S P
 
 # Get tuple element
 
-i_get_tuple_element xy P x
-
-%cold
-i_get_tuple_element xy P y
-%hot
+i_get_tuple_element xy P xy
 
 i_get_tuple_element2 x P x
 i_get_tuple_element2_dst x P x x
@@ -285,6 +281,9 @@ move_window/6
 move X1=x Y1=y | move X2=x Y2=y | move X3=x Y3=y | succ(Y1,Y2) | succ(Y2,Y3) => \
     move_window X1 X2 X3 Y1 Y3
 
+move X1=x Y1=y | move X2=x Y2=y | succ(Y1,Y2) => \
+    move_window2 X1 X2 Y1
+
 move_window X1=x X2=x X3=x Y1=y Y3=y | move X4=x Y4=y | succ(Y3,Y4) => \
     move_window X1 X2 X3 X4 Y1 Y4
 
@@ -294,6 +293,7 @@ move_window X1=x X2=x X3=x X4=x Y1=y Y4=y | move X5=x Y5=y | succ(Y4,Y5) => \
 move_window X1=x X2=x X3=x Y1=y Y3=y => move_window3 X1 X2 X3 Y1
 move_window X1=x X2=x X3=x X4=x Y1=y Y4=y => move_window4 X1 X2 X3 X4 Y1
 
+move_window2 x x y
 move_window3 x x x y
 move_window4 x x x x y
 move_window5 x x x x x y
@@ -346,11 +346,6 @@ move X1=x X2=x | move X3=x X4=x | independent_moves(X1, X2, X3, X4) => \
      move2_par X1 X2 X3 X4
 move2_par x x x x
 
-# move2_par x y x y
-
-move X1=x Y1=y | move X2=x Y2=y => move2_par X1 Y1 X2 Y2
-move2_par x y x y
-
 # move2_par x x x y
 
 move X1=x X2=x | move X3=x Y1=y | independent_moves(X1, X2, X3, Y1) => \
@@ -393,20 +388,14 @@ move3 x x x x x x
 move C=aiq X=x==1 => move_x1 C
 move C=aiq X=x==2 => move_x2 C
 
+move n D=y => init D
+
 move_x1 c
 move_x2 c
 
-# The compiler almost never generates a "move Literal y(Y)" instruction,
-# so let's cheat if we encounter one.
-move S=n D=y => init D
-move S=c D=y => move S x | move x D
-
-move x x
-move x y
-move y x
-move c x
+move xy xy
+move c xy
 move n x
-move y y
 
 # The following move instructions using x(0) are frequently used.
 
@@ -500,9 +489,13 @@ is_ge f? c x
 is_ge f? s s
 %hot
 
-is_eq f? s s
+is_eq Fail=f Const=c Reg=xy => is_eq Fail Reg Const
+is_eq Fail=f C1=c C2=c => move C1 x | is_eq Fail x C2
+is_eq f? S s
 
-is_ne f? s s
+is_ne Fail=f Const=c Reg=xy => is_ne Fail Reg Const
+is_ne Fail=f C1=c C2=c => move C1 x | is_ne Fail x C2
+is_ne f? S s
 
 #
 # Putting tuples.
@@ -1035,6 +1028,9 @@ call_bif e
 bif0 u$bif:erlang:self/0 Dst=d => self Dst
 bif0 u$bif:erlang:node/0 Dst=d => node Dst
 
+bif1 Fail=f Bif=u$bif:erlang:hd/1 Src=x Dst=x => is_nonempty_list_get_hd Fail Src Dst
+bif1 Fail=f Bif=u$bif:erlang:tl/1 Src=x Dst=x => is_nonempty_list_get_tl Fail Src Dst
+
 bif1 Fail Bif=u$bif:erlang:get/1 Src=s Dst=d => gen_get(Src, Dst)
 
 bif2 Jump=j u$bif:erlang:element/2 S1=s S2=xy Dst=d => gen_element(Jump, S1, S2, Dst)
@@ -1123,19 +1119,33 @@ is_function Fail=f c => jump Fail
 func_info M F A => i_func_info u M F A
 
 # ================================================================
-# New bit syntax matching (R11B).
+# Bit syntax matching obsoleted in OTP 22.
 # ================================================================
 
-%warm
+%cold
 bs_start_match2 Fail=f ica X Y D => jump Fail
 bs_start_match2 Fail Bin X Y D => i_bs_start_match2 Bin Fail X Y D
 i_bs_start_match2 xy f t t d
 
+bs_save2 Y=y Index => move Y x | bs_save2 x Index
 bs_save2 Reg Index => gen_bs_save(Reg, Index)
-i_bs_save2 xy t
+i_bs_save2 x t
 
+bs_restore2 Y=y Index => move Y x | bs_restore2 x Index
 bs_restore2 Reg Index => gen_bs_restore(Reg, Index)
-i_bs_restore2 xy t
+i_bs_restore2 x t
+
+bs_context_to_binary Y=y | line L | badmatch Y => \
+    move Y x | bs_context_to_binary x | line L | badmatch x
+bs_context_to_binary Y=y => move Y x | bs_context_to_binary x
+bs_context_to_binary x
+%warm
+
+# ================================================================
+# New bit syntax matching (R11B).
+# ================================================================
+
+%warm
 
 # Matching integers
 bs_match_string Fail Ms Bits Val => i_bs_match_string Ms Fail Bits Val
@@ -1154,7 +1164,7 @@ i_bs_get_integer_imm Ms Bits Live Fail Flags Y=y => \
 
 i_bs_get_integer_small_imm xy W f? t x
 i_bs_get_integer_imm xy W t f? t x
-i_bs_get_integer f? t t xy s d
+i_bs_get_integer xy f? t t s d
 i_bs_get_integer_8 xy f? d
 i_bs_get_integer_16 xy f? d
 
@@ -1166,9 +1176,9 @@ i_bs_get_integer_32 xy f? d
 bs_get_binary2 Fail=f Ms=xy Live=u Sz=sq Unit=u Flags=u Dst=d => \
 			gen_get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
 
-i_bs_get_binary_imm2 f? xy t W t d
-i_bs_get_binary2 f xy t? s t d
-i_bs_get_binary_all2 f? xy t t d
+i_bs_get_binary_imm2 xy f? t W t d
+i_bs_get_binary2 xy f t? s t d
+i_bs_get_binary_all2 xy f? t t d
 i_bs_get_binary_all_reuse xy f? t
 
 # Fetching float from binaries.
@@ -1177,7 +1187,7 @@ bs_get_float2 Fail=f Ms=xy Live=u Sz=s Unit=u Flags=u Dst=d => \
 
 bs_get_float2 Fail=f Ms=x Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail
 
-i_bs_get_float2 f? xy t s t d
+i_bs_get_float2 xy f? t s t d
 
 # Miscellanous
 
@@ -1185,8 +1195,7 @@ bs_skip_bits2 Fail=f Ms=xy Sz=sq Unit=u Flags=u => \
 			gen_skip_bits2(Fail, Ms, Sz, Unit, Flags)
 
 i_bs_skip_bits_imm2 f? xy W
-i_bs_skip_bits2 f? xy xy t
-i_bs_skip_bits_all2 f? xy t
+i_bs_skip_bits2 xy xy f? t
 
 bs_test_tail2 Fail=f Ms=xy Bits=u==0 => bs_test_zero_tail2 Fail Ms
 bs_test_tail2 Fail=f Ms=xy Bits=u => bs_test_tail_imm2 Fail Ms Bits
@@ -1197,16 +1206,6 @@ bs_test_unit F Ms Unit=u==8 => bs_test_unit8 F Ms
 bs_test_unit f? xy t
 bs_test_unit8 f? xy
 
-# An y register operand for bs_context_to_binary is rare,
-# but can happen because of inlining.
-
-bs_context_to_binary Y=y | line L | badmatch Y => \
-    move Y x | bs_context_to_binary x | line L | badmatch x
-
-bs_context_to_binary Y=y => move Y x | bs_context_to_binary x
-
-bs_context_to_binary x
-
 # Gets a bitstring from the tail of a context.
 bs_get_tail xy d t
 
@@ -1332,31 +1331,35 @@ i_bs_private_append j? t s S x
 bs_put_integer Fail=j Sz=sq Unit=u Flags=u Src=s => \
 			gen_put_integer(Fail, Sz, Unit, Flags, Src)
 
-i_new_bs_put_integer j? s t s
-i_new_bs_put_integer_imm j? W t s
+i_new_bs_put_integer j? S t s
+i_new_bs_put_integer_imm xyc j? W t
 
 #
 # Utf8/utf16/utf32 support. (R12B-5)
 #
 
-bs_utf8_size j Src=s Dst=d => i_bs_utf8_size Src Dst
-
-i_bs_utf8_size s x
-
-bs_utf16_size j Src=s Dst=d => i_bs_utf16_size Src Dst
+bs_utf8_size j Src Dst=d => i_bs_utf8_size Src Dst
+bs_utf16_size j Src Dst=d => i_bs_utf16_size Src Dst
 
-i_bs_utf16_size s x
+bs_put_utf8 Fail u Src => i_bs_put_utf8 Fail Src
 
-bs_put_utf8 Fail u Src=s => i_bs_put_utf8 Fail Src
+bs_put_utf32 Fail=j Flags=u Src=s => \
+   i_bs_validate_unicode Fail Src | bs_put_integer Fail i=32 u=1 Flags Src
 
-i_bs_put_utf8 j? s
+i_bs_utf8_size S x
+i_bs_utf16_size S x
 
-bs_put_utf16 j? t s
+i_bs_put_utf8 j? S
+bs_put_utf16 j? t S
 
-bs_put_utf32 Fail=j Flags=u Src=s => \
-   i_bs_validate_unicode Fail Src | bs_put_integer Fail i=32 u=1 Flags Src
+i_bs_validate_unicode j? S
 
-i_bs_validate_unicode j? s
+# Handle unoptimized code.
+i_bs_utf8_size Src=c Dst => move Src x | i_bs_utf8_size x Dst
+i_bs_utf16_size Src=c Dst => move Src x | i_bs_utf16_size x Dst
+i_bs_put_utf8 Fail Src=c => move Src x | i_bs_put_utf8 Fail x
+bs_put_utf16 Fail Flags Src=c => move Src x | bs_put_utf16 Fail Flags x
+i_bs_validate_unicode Fail Src=c => move Src x | i_bs_validate_unicode Fail x
 
 #
 # Storing floats into binaries.
@@ -1366,7 +1369,7 @@ bs_put_float Fail Sz=q Unit Flags Val => badarg Fail
 bs_put_float Fail=j Sz=s Unit=u Flags=u Src=s => \
 			gen_put_float(Fail, Sz, Unit, Flags, Src)
 
-i_new_bs_put_float j? s t s
+i_new_bs_put_float j? S t s
 i_new_bs_put_float_imm j? W t s
 
 #
@@ -1376,9 +1379,18 @@ i_new_bs_put_float_imm j? W t s
 bs_put_binary Fail=j Sz=s Unit=u Flags=u Src=s => \
 			gen_put_binary(Fail, Sz, Unit, Flags, Src)
 
-i_new_bs_put_binary j? s t s
-i_new_bs_put_binary_imm j? W s
-i_new_bs_put_binary_all j? s t
+# In unoptimized code, the binary argument could be a literal. (In optimized code,
+# there would be a bs_put_string instruction.)
+i_new_bs_put_binary Fail Size Unit Lit=c => \
+   move Lit x | i_new_bs_put_binary Fail Size Unit x
+i_new_bs_put_binary_imm Fail Size Lit=c => \
+   move Lit x | i_new_bs_put_binary_imm Fail Size x
+i_new_bs_put_binary_all Lit=c Fail Unit => \
+   move Lit x | i_new_bs_put_binary_all x Fail Unit
+
+i_new_bs_put_binary j? S t S
+i_new_bs_put_binary_imm j? W S
+i_new_bs_put_binary_all xy j? t
 
 #
 # Warning: The i_bs_put_string and i_new_bs_put_string instructions
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index a69da4d762..a6312293cc 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -111,6 +111,23 @@
 #endif
 #endif
 
+/*
+ * Test for clang's convenient __has_builtin feature checking macro.
+ */
+#ifndef __has_builtin
+  #define __has_builtin(x) 0
+#endif
+
+/*
+ * Define HAVE_OVERFLOW_CHECK_BUILTINS if the overflow checking arithmetic
+ * builtins are available.
+ */
+#if ERTS_AT_LEAST_GCC_VSN__(5, 1, 0)
+#  define HAVE_OVERFLOW_CHECK_BUILTINS 1
+#elif __has_builtin(__builtin_mul_overflow)
+#  define HAVE_OVERFLOW_CHECK_BUILTINS 1
+#endif
+
 #include "erl_misc_utils.h"
 
 /*
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index f73e2362bf..1625b2cc65 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -1488,7 +1488,7 @@ sub code_gen {
                 $var_decls .= "Eterm $tmp;\n";
                 $tmp_arg_num++;
                 push(@f, $tmp);
-                $prefix .= "GetR($arg_offset, $tmp);\n";
+                $prefix .= "GetSource(" . arg_offset($arg_offset) . ", $tmp);\n";
                 $need_block = 1;
                 last SWITCH;
             };