1 files changed, 179 insertions, 150 deletions

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index d8cf3fb8e2..0ad5329b2f 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -2868,6 +2868,7 @@ load_code(LoaderState* stp)
 	    break;
 	case op_bs_put_string_WW:
 	case op_i_bs_match_string_xfWW:
+	case op_i_bs_match_string_yfWW:
 	    new_string_patch(stp, ci-1);
 	    break;
 
@@ -2969,6 +2970,8 @@ load_code(LoaderState* stp)
 #define succ(St, X, Y) ((X).type == (Y).type && (X).val + 1 == (Y).val)
 #define succ2(St, X, Y) ((X).type == (Y).type && (X).val + 2 == (Y).val)
 #define succ3(St, X, Y) ((X).type == (Y).type && (X).val + 3 == (Y).val)
+#define succ4(St, X, Y) ((X).type == (Y).type && (X).val + 4 == (Y).val)
+
 
 #ifdef NO_FPE_SIGNALS 
 #define no_fpe_signals(St) 1
@@ -2985,6 +2988,35 @@ compiled_with_otp_20_or_higher(LoaderState* stp)
 }
 
 /*
+ * Predicate that tests whether the following two moves are independent:
+ *
+ *    move Src1 Dst1
+ *    move Src2 Dst2
+ *
+ */
+static int
+independent_moves(LoaderState* stp, GenOpArg Src1, GenOpArg Dst1,
+                  GenOpArg Src2, GenOpArg Dst2)
+{
+    return (Src1.type != Dst2.type || Src1.val != Dst2.val) &&
+        (Src2.type != Dst1.type || Src2.val != Dst1.val) &&
+        (Dst1.type != Dst2.type ||Dst1.val != Dst2.val);
+}
+
+/*
+ * Predicate that tests that two registers are distinct.
+ *
+ *    move Src1 Dst1
+ *    move Src2 Dst2
+ *
+ */
+static int
+distinct(LoaderState* stp, GenOpArg Reg1, GenOpArg Reg2)
+{
+    return Reg1.type != Reg2.type || Reg1.val != Reg2.val;
+}
+
+/*
  * Predicate that tests whether a jump table can be used.
  */
 
@@ -3263,11 +3295,11 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live,
     } else {
 	op->op = genop_i_bs_get_integer_6;
 	op->arity = 6;
-	op->a[0] = Fail;
-	op->a[1] = Live;
-	op->a[2].type = TAG_u;
-	op->a[2].val = (Unit.val << 3) | Flags.val;
-	op->a[3] = Ms;
+	op->a[0] = Ms;
+	op->a[1] = Fail;
+	op->a[2] = Live;
+	op->a[3].type = TAG_u;
+	op->a[3].val = (Unit.val << 3) | Flags.val;
 	op->a[4] = Size;
 	op->a[5] = Dst;
 	op->next = NULL;
@@ -3300,8 +3332,8 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live,
 	} else {
 	    op->op = genop_i_bs_get_binary_all2_5;
 	    op->arity = 5;
-	    op->a[0] = Fail;
-	    op->a[1] = Ms;
+	    op->a[0] = Ms;
+	    op->a[1] = Fail;
 	    op->a[2] = Live;	
 	    op->a[3] = Unit;
 	    op->a[4] = Dst;
@@ -3309,8 +3341,8 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live,
     } else if (Size.type == TAG_i) {
 	op->op = genop_i_bs_get_binary_imm2_6;
 	op->arity = 6;
-	op->a[0] = Fail;
-	op->a[1] = Ms;
+	op->a[0] = Ms;
+	op->a[1] = Fail;
 	op->a[2] = Live;
 	op->a[3].type = TAG_u;
 	if (!safe_mul(Size.val, Unit.val, &op->a[3].val)) {
@@ -3330,8 +3362,8 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live,
 	} else {
 	    op->op = genop_i_bs_get_binary_imm2_6;
 	    op->arity = 6;
-	    op->a[0] = Fail;
-	    op->a[1] = Ms;
+	    op->a[0] = Ms;
+	    op->a[1] = Fail;
 	    op->a[2] = Live;
 	    op->a[3].type = TAG_u;
 	    if (!safe_mul(bigval, Unit.val, &op->a[3].val)) {
@@ -3343,8 +3375,8 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live,
     } else {
 	op->op = genop_i_bs_get_binary2_6;
 	op->arity = 6;
-	op->a[0] = Fail;
-	op->a[1] = Ms;
+	op->a[0] = Ms;
+	op->a[1] = Fail;
 	op->a[2] = Live;
 	op->a[3] = Size;
 	op->a[4].type = TAG_u;
@@ -3377,8 +3409,8 @@ gen_put_binary(LoaderState* stp, GenOpArg Fail,GenOpArg Size,
     if (Size.type == TAG_a && Size.val == am_all) {
 	op->op = genop_i_new_bs_put_binary_all_3;
 	op->arity = 3;
-	op->a[0] = Fail;
-	op->a[1] = Src;
+	op->a[0] = Src;
+	op->a[1] = Fail;
 	op->a[2] = Unit;
     } else if (Size.type == TAG_i) {
 	op->op = genop_i_new_bs_put_binary_imm_3;
@@ -3388,10 +3420,33 @@ gen_put_binary(LoaderState* stp, GenOpArg Fail,GenOpArg Size,
 	if (safe_mul(Size.val, Unit.val, &op->a[1].val)) {
 	    op->a[2] = Src;
 	} else {
+        error:
 	    op->op = genop_badarg_1;
 	    op->arity = 1;
 	    op->a[0] = Fail;
 	}
+    } else if (Size.type == TAG_q) {
+#ifdef ARCH_64
+        /*
+         * There is no way that this binary would fit in memory.
+         */
+        goto error;
+#else
+	Eterm big = stp->literals[Size.val].term;
+	Uint bigval;
+        Uint size;
+
+	if (!term_to_Uint(big, &bigval) ||
+            !safe_mul(bigval, Unit.val, &size)) {
+	    goto error;
+	}
+        op->op = genop_i_new_bs_put_binary_imm_3;
+        op->arity = 3;
+        op->a[0] = Fail;
+        op->a[1].type = TAG_u;
+        op->a[1].val = size;
+        op->a[2] = Src;
+#endif
     } else {
 	op->op = genop_i_new_bs_put_binary_4;
 	op->arity = 4;
@@ -3416,11 +3471,8 @@ gen_put_integer(LoaderState* stp, GenOpArg Fail, GenOpArg Size,
     NATIVE_ENDIAN(Flags);
 	/* Negative size must fail */
     if (Size.type == TAG_i) {
-	op->op = genop_i_new_bs_put_integer_imm_4;
-	op->arity = 4;
-	op->a[0] = Fail;
-	op->a[1].type = TAG_u;
-	if (!safe_mul(Size.val, Unit.val, &op->a[1].val)) {
+        Uint size;
+	if (!safe_mul(Size.val, Unit.val, &size)) {
         error:
             op->op = genop_badarg_1;
             op->arity = 1;
@@ -3428,26 +3480,31 @@ gen_put_integer(LoaderState* stp, GenOpArg Fail, GenOpArg Size,
             op->next = NULL;
             return op;
 	}
-	op->a[1].val = Size.val * Unit.val;
-	op->a[2].type = Flags.type;
-	op->a[2].val = (Flags.val & 7);
-	op->a[3] = Src;
+	op->op = genop_i_new_bs_put_integer_imm_4;
+	op->arity = 4;
+	op->a[0] = Src;
+	op->a[1] = Fail;
+	op->a[2].type = TAG_u;
+	op->a[2].val = size;
+	op->a[3].type = Flags.type;
+	op->a[3].val = (Flags.val & 7);
     } else if (Size.type == TAG_q) {
 	Eterm big = stp->literals[Size.val].term;
 	Uint bigval;
+        Uint size;
 
-	if (!term_to_Uint(big, &bigval)) {
+	if (!term_to_Uint(big, &bigval) ||
+            !safe_mul(bigval, Unit.val, &size)) {
 	    goto error;
-	} else {
-	    op->op = genop_i_new_bs_put_integer_imm_4;
-	    op->arity = 4;
-	    op->a[0] = Fail;
-	    op->a[1].type = TAG_u;
-	    op->a[1].val = bigval * Unit.val;
-	    op->a[2].type = Flags.type;
-	    op->a[2].val = (Flags.val & 7);
-	    op->a[3] = Src;
 	}
+	op->op = genop_i_new_bs_put_integer_imm_4;
+	op->arity = 4;
+	op->a[0] = Src;
+	op->a[1] = Fail;
+	op->a[2].type = TAG_u;
+	op->a[2].val = size;
+	op->a[3].type = Flags.type;
+	op->a[3].val = (Flags.val & 7);
     } else {
 	op->op = genop_i_new_bs_put_integer_4;
 	op->arity = 4;
@@ -3509,8 +3566,8 @@ gen_get_float2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live,
     NATIVE_ENDIAN(Flags);
     op->op = genop_i_bs_get_float2_6;
     op->arity = 6;
-    op->a[0] = Fail;
-    op->a[1] = Ms;
+    op->a[0] = Ms;
+    op->a[1] = Fail;
     op->a[2] = Live;
     op->a[3] = Size;
     op->a[4].type = TAG_u;
@@ -3533,10 +3590,22 @@ gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms,
     NATIVE_ENDIAN(Flags);
     NEW_GENOP(stp, op);
     if (Size.type == TAG_a && Size.val == am_all) {
-	op->op = genop_i_bs_skip_bits_all2_3;
+        /*
+         * This kind of skip instruction will only be found in modules
+         * compiled before OTP 19. From OTP 19, the compiler generates
+         * a test_unit instruction of a bs_skip at the end of a
+         * binary.
+         *
+         * It is safe to replace the skip instruction with a test_unit
+         * instruction, because the position will never be used again.
+         * If the match context itself is used again, it will be used by
+         * a bs_restore2 instruction which will overwrite the position
+         * by one of the stored positions.
+         */
+	op->op = genop_bs_test_unit_3;
 	op->arity = 3;
 	op->a[0] = Fail;
-	op->a[1] = Ms; 
+	op->a[1] = Ms;
 	op->a[2] = Unit;
     } else if (Size.type == TAG_i) {
 	op->op = genop_i_bs_skip_bits_imm2_3;
@@ -3569,9 +3638,9 @@ gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms,
     } else {
 	op->op = genop_i_bs_skip_bits2_4;
 	op->arity = 4;
-	op->a[0] = Fail;
-	op->a[1] = Ms; 
-	op->a[2] = Size;
+	op->a[0] = Ms;
+	op->a[1] = Size;
+	op->a[2] = Fail;
 	op->a[3] = Unit;
     }
     op->next = NULL;
@@ -3579,38 +3648,36 @@ gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms,
 }
 
 static GenOp*
-gen_increment(LoaderState* stp, GenOpArg Reg, GenOpArg Integer,
-	      GenOpArg Live, GenOpArg Dst)
+gen_increment(LoaderState* stp, GenOpArg Reg,
+              GenOpArg Integer, GenOpArg Dst)
 {
     GenOp* op;
 
     NEW_GENOP(stp, op);
-    op->op = genop_i_increment_4;
-    op->arity = 4;
+    op->op = genop_i_increment_3;
+    op->arity = 3;
     op->next = NULL;
     op->a[0] = Reg;
     op->a[1].type = TAG_u;
     op->a[1].val = Integer.val;
-    op->a[2] = Live;
-    op->a[3] = Dst;
+    op->a[2] = Dst;
     return op;
 }
 
 static GenOp*
-gen_increment_from_minus(LoaderState* stp, GenOpArg Reg, GenOpArg Integer,
-			 GenOpArg Live, GenOpArg Dst)
+gen_increment_from_minus(LoaderState* stp, GenOpArg Reg,
+                         GenOpArg Integer, GenOpArg Dst)
 {
     GenOp* op;
 
     NEW_GENOP(stp, op);
-    op->op = genop_i_increment_4;
-    op->arity = 4;
+    op->op = genop_i_increment_3;
+    op->arity = 3;
     op->next = NULL;
     op->a[0] = Reg;
     op->a[1].type = TAG_u;
     op->a[1].val = -Integer.val;
-    op->a[2] = Live;
-    op->a[3] = Dst;
+    op->a[2] = Dst;
     return op;
 }
 
@@ -4299,95 +4366,69 @@ gen_make_fun2(LoaderState* stp, GenOpArg idx)
 }
 
 static GenOp*
-translate_gc_bif(LoaderState* stp, GenOp* op, GenOpArg Bif)
-{
-    const ErtsGcBif* p;
-    BifFunction bf;
-
-    bf = stp->import[Bif.val].bf;
-    for (p = erts_gc_bifs; p->bif != 0; p++) {
-	if (p->bif == bf) {
-	    op->a[1].type = TAG_u;
-	    op->a[1].val = (BeamInstr) p->gc_bif;
-	    return op;
-	}
-    }
-
-    op->op = genop_unsupported_guard_bif_3;
-    op->arity = 3;
-    op->a[0].type = TAG_a;
-    op->a[0].val = stp->import[Bif.val].module;
-    op->a[1].type = TAG_a;
-    op->a[1].val = stp->import[Bif.val].function;
-    op->a[2].type = TAG_u;
-    op->a[2].val = stp->import[Bif.val].arity;
-    return op;
-}
-
-/*
- * Rewrite gc_bifs with one parameter (the common case).
- */
-static GenOp*
-gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
-	      GenOpArg Src, GenOpArg Dst)
+gen_is_function2(LoaderState* stp, GenOpArg Fail, GenOpArg Fun, GenOpArg Arity)
 {
     GenOp* op;
+    int literal_arity =  Arity.type == TAG_i;
+    int fun_is_reg = Fun.type == TAG_x || Fun.type == TAG_y;
 
     NEW_GENOP(stp, op);
     op->next = NULL;
-    op->op = genop_i_gc_bif1_5;
-    op->arity = 5;
-    op->a[0] = Fail;
-    /* op->a[1] is set by translate_gc_bif() */
-    op->a[2] = Src;
-    op->a[3] = Live;
-    op->a[4] = Dst;
-    return translate_gc_bif(stp, op, Bif);
-}
-
-/*
- * This is used by the ops.tab rule that rewrites gc_bifs with two parameters.
- */
-static GenOp*
-gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
-	      GenOpArg S1, GenOpArg S2, GenOpArg Dst)
-{
-    GenOp* op;
 
-    NEW_GENOP(stp, op);
-    op->next = NULL;
-    op->op = genop_i_gc_bif2_6;
-    op->arity = 6;
-    op->a[0] = Fail;
-    /* op->a[1] is set by translate_gc_bif() */
-    op->a[2] = Live;
-    op->a[3] = S1;
-    op->a[4] = S2;
-    op->a[5] = Dst;
-    return translate_gc_bif(stp, op, Bif);
-}
-
-/*
- * This is used by the ops.tab rule that rewrites gc_bifs with three parameters.
- */
-static GenOp*
-gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
-	      GenOpArg S1, GenOpArg S2, GenOpArg S3, GenOpArg Dst)
-{
-    GenOp* op;
-
-    NEW_GENOP(stp, op);
-    op->next = NULL;
-    op->op = genop_ii_gc_bif3_7;
-    op->arity = 7;
-    op->a[0] = Fail;
-    /* op->a[1] is set by translate_gc_bif() */
-    op->a[2] = Live;
-    op->a[3] = S1;
-    op->a[4] = S2;
-    op->a[5] = S3;
-    op->a[6] = Dst;
-    return translate_gc_bif(stp, op, Bif);
+    if (fun_is_reg &&literal_arity) {
+        /*
+         * Most common case. Fun in a register and arity
+         * is an integer literal.
+         */
+        if (Arity.val > MAX_ARG) {
+            /* Arity is negative or too big. */
+            op->op = genop_jump_1;
+            op->arity = 1;
+            op->a[0] = Fail;
+            return op;
+        } else {
+            op->op = genop_hot_is_function2_3;
+            op->arity = 3;
+            op->a[0] = Fail;
+            op->a[1] = Fun;
+            op->a[2].type = TAG_u;
+            op->a[2].val = Arity.val;
+            return op;
+        }
+    } else {
+        /*
+         * Handle extremely uncommon cases by a slower sequence.
+         */
+        GenOp* move_fun;
+        GenOp* move_arity;
+
+        NEW_GENOP(stp, move_fun);
+        NEW_GENOP(stp, move_arity);
+
+        move_fun->next = move_arity;
+        move_arity->next = op;
+
+        move_fun->arity = 2;
+        move_fun->op = genop_move_2;
+        move_fun->a[0] = Fun;
+        move_fun->a[1].type = TAG_x;
+        move_fun->a[1].val = 1022;
+
+        move_arity->arity = 2;
+        move_arity->op = genop_move_2;
+        move_arity->a[0] = Arity;
+        move_arity->a[1].type = TAG_x;
+        move_arity->a[1].val = 1023;
+
+        op->op = genop_cold_is_function2_3;
+        op->arity = 3;
+        op->a[0] = Fail;
+        op->a[1].type = TAG_x;
+        op->a[1].val = 1022;
+        op->a[2].type = TAG_x;
+        op->a[2].val = 1023;
+        return move_fun;
+    }
 }
 
 static GenOp*
@@ -4556,19 +4597,6 @@ is_empty_map(LoaderState* stp, GenOpArg Lit)
 }
 
 /*
- * Predicate to test whether the given literal is an export.
- */
-static int
-literal_is_export(LoaderState* stp, GenOpArg Lit)
-{
-    Eterm term;
-
-    ASSERT(Lit.type == TAG_q);
-    term = stp->literals[Lit.val].term;
-    return is_export(term);
-}
-
-/*
  * Pseudo predicate map_key_sort that will sort the Rest operand for
  * map instructions as a side effect.
  */
@@ -6148,7 +6176,8 @@ erts_release_literal_area(ErtsLiteralArea* literal_area)
             }
         default:
             ASSERT(is_external_header(oh->thing_word));
-            erts_deref_node_entry(((ExternalThing*)oh)->node);
+            erts_deref_node_entry(((ExternalThing*)oh)->node,
+                                  make_boxed(&oh->thing_word));
         }
         oh = oh->next;
     }