diff options
Diffstat (limited to 'erts/emulator/beam/beam_load.c')
| -rw-r--r-- | erts/emulator/beam/beam_load.c | 634 |
1 files changed, 352 insertions, 282 deletions
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index e61199a8fd..35f2ea6688 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -492,6 +492,11 @@ typedef struct LoaderState { (Genop)->arity * sizeof(GenOpArg)); \ } while (0) +#define GENOP_NAME_ARITY(Genop, Name, Arity) \ + do { \ + (Genop)->op = genop_##Name##_##Arity; \ + (Genop)->arity = Arity; \ + } while (0) static void free_loader_state(Binary* magic); static ErlHeapFragment* new_literal_fragment(Uint size); @@ -1425,7 +1430,7 @@ load_atom_table(LoaderState* stp, ErtsAtomEncoding enc) ap = atom_tab(atom_val(stp->atom[1])); sys_memcpy(sbuf, ap->name, ap->len); sbuf[ap->len] = '\0'; - LoadError1(stp, "module name in object code is %s", sbuf); + LoadError1(stp, "BEAM file exists but it defines a module named %s", sbuf); } return 1; @@ -2868,6 +2873,7 @@ load_code(LoaderState* stp) break; case op_bs_put_string_WW: case op_i_bs_match_string_xfWW: + case op_i_bs_match_string_yfWW: new_string_patch(stp, ci-1); break; @@ -2969,6 +2975,8 @@ load_code(LoaderState* stp) #define succ(St, X, Y) ((X).type == (Y).type && (X).val + 1 == (Y).val) #define succ2(St, X, Y) ((X).type == (Y).type && (X).val + 2 == (Y).val) #define succ3(St, X, Y) ((X).type == (Y).type && (X).val + 3 == (Y).val) +#define succ4(St, X, Y) ((X).type == (Y).type && (X).val + 4 == (Y).val) + #ifdef NO_FPE_SIGNALS #define no_fpe_signals(St) 1 @@ -2985,6 +2993,35 @@ compiled_with_otp_20_or_higher(LoaderState* stp) } /* + * Predicate that tests whether the following two moves are independent: + * + * move Src1 Dst1 + * move Src2 Dst2 + * + */ +static int +independent_moves(LoaderState* stp, GenOpArg Src1, GenOpArg Dst1, + GenOpArg Src2, GenOpArg Dst2) +{ + return (Src1.type != Dst2.type || Src1.val != Dst2.val) && + (Src2.type != Dst1.type || Src2.val != Dst1.val) && + (Dst1.type != Dst2.type ||Dst1.val != Dst2.val); +} + +/* + * Predicate that tests that two registers are distinct. + * + * move Src1 Dst1 + * move Src2 Dst2 + * + */ +static int +distinct(LoaderState* stp, GenOpArg Reg1, GenOpArg Reg2) +{ + return Reg1.type != Reg2.type || Reg1.val != Reg2.val; +} + +/* * Predicate that tests whether a jump table can be used. */ @@ -3108,6 +3145,42 @@ is_killed(LoaderState* stp, GenOpArg Reg, GenOpArg Live) Live.val <= Reg.val; } +static int +is_killed_by_call_fun(LoaderState* stp, GenOpArg Reg, GenOpArg Live) +{ + return Reg.type == TAG_x && Live.type == TAG_u && + Live.val+1 <= Reg.val; +} + +/* + * Test whether register Reg is killed by make_fun instruction that + * creates the fun given by index idx. + */ + +static int +is_killed_by_make_fun(LoaderState* stp, GenOpArg Reg, GenOpArg idx) +{ + Uint num_free; + + if (idx.val >= stp->num_lambdas) { + /* Invalid index. Ignore the error for now. */ + return 0; + } else { + num_free = stp->lambdas[idx.val].num_free; + return Reg.type == TAG_x && num_free <= Reg.val; + } +} + +/* + * Test whether register Reg is killed by the send instruction that follows. + */ + +static int +is_killed_by_send(LoaderState* stp, GenOpArg Reg) +{ + return Reg.type == TAG_x && 2 <= Reg.val; +} + /* * Generate an instruction for element/2. */ @@ -3119,20 +3192,19 @@ gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index, GenOp* op; NEW_GENOP(stp, op); - op->arity = 4; op->next = NULL; if (Index.type == TAG_i && Index.val > 0 && Index.val <= ERTS_MAX_TUPLE_SIZE && (Tuple.type == TAG_x || Tuple.type == TAG_y)) { - op->op = genop_i_fast_element_4; + GENOP_NAME_ARITY(op, i_fast_element, 4); op->a[0] = Tuple; op->a[1] = Fail; op->a[2].type = TAG_u; op->a[2].val = Index.val; op->a[3] = Dst; } else { - op->op = genop_i_element_4; + GENOP_NAME_ARITY(op, i_element, 4); op->a[0] = Tuple; op->a[1] = Fail; op->a[2] = Index; @@ -3148,8 +3220,7 @@ gen_bs_save(LoaderState* stp, GenOpArg Reg, GenOpArg Index) GenOp* op; NEW_GENOP(stp, op); - op->op = genop_i_bs_save2_2; - op->arity = 2; + GENOP_NAME_ARITY(op, i_bs_save2, 2); op->a[0] = Reg; op->a[1] = Index; if (Index.type == TAG_u) { @@ -3168,8 +3239,7 @@ gen_bs_restore(LoaderState* stp, GenOpArg Reg, GenOpArg Index) GenOp* op; NEW_GENOP(stp, op); - op->op = genop_i_bs_restore2_2; - op->arity = 2; + GENOP_NAME_ARITY(op, i_bs_restore2, 2); op->a[0] = Reg; op->a[1] = Index; if (Index.type == TAG_u) { @@ -3203,21 +3273,18 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, } else if ((Flags.val & BSF_SIGNED) != 0) { goto generic; } else if (bits == 8) { - op->op = genop_i_bs_get_integer_8_3; - op->arity = 3; - op->a[0] = Ms; + GENOP_NAME_ARITY(op, i_bs_get_integer_8, 3); + op->a[0] = Ms; op->a[1] = Fail; op->a[2] = Dst; } else if (bits == 16 && (Flags.val & BSF_LITTLE) == 0) { - op->op = genop_i_bs_get_integer_16_3; - op->arity = 3; + GENOP_NAME_ARITY(op, i_bs_get_integer_16, 3); op->a[0] = Ms; op->a[1] = Fail; op->a[2] = Dst; #ifdef ARCH_64 } else if (bits == 32 && (Flags.val & BSF_LITTLE) == 0) { - op->op = genop_i_bs_get_integer_32_3; - op->arity = 3; + GENOP_NAME_ARITY(op, i_bs_get_integer_32, 3); op->a[0] = Ms; op->a[1] = Fail; op->a[2] = Dst; @@ -3225,8 +3292,7 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, } else { generic: if (bits < SMALL_BITS) { - op->op = genop_i_bs_get_integer_small_imm_5; - op->arity = 5; + GENOP_NAME_ARITY(op, i_bs_get_integer_small_imm, 5); op->a[0] = Ms; op->a[1].type = TAG_u; op->a[1].val = bits; @@ -3234,8 +3300,7 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, op->a[3] = Flags; op->a[4] = Dst; } else { - op->op = genop_i_bs_get_integer_imm_6; - op->arity = 6; + GENOP_NAME_ARITY(op, i_bs_get_integer_imm, 6); op->a[0] = Ms; op->a[1].type = TAG_u; op->a[1].val = bits; @@ -3251,8 +3316,7 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, if (!term_to_Uint(big, &bigval)) { error: - op->op = genop_jump_1; - op->arity = 1; + GENOP_NAME_ARITY(op, jump, 1); op->a[0] = Fail; } else { if (!safe_mul(bigval, Unit.val, &bits)) { @@ -3261,13 +3325,12 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, goto generic; } } else { - op->op = genop_i_bs_get_integer_6; - op->arity = 6; - op->a[0] = Fail; - op->a[1] = Live; - op->a[2].type = TAG_u; - op->a[2].val = (Unit.val << 3) | Flags.val; - op->a[3] = Ms; + GENOP_NAME_ARITY(op, i_bs_get_integer, 6); + op->a[0] = Ms; + op->a[1] = Fail; + op->a[2] = Live; + op->a[3].type = TAG_u; + op->a[3].val = (Unit.val << 3) | Flags.val; op->a[4] = Size; op->a[5] = Dst; op->next = NULL; @@ -3291,26 +3354,16 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, NATIVE_ENDIAN(Flags); if (Size.type == TAG_a && Size.val == am_all) { - if (Ms.type == Dst.type && Ms.val == Dst.val) { - op->op = genop_i_bs_get_binary_all_reuse_3; - op->arity = 3; - op->a[0] = Ms; - op->a[1] = Fail; - op->a[2] = Unit; - } else { - op->op = genop_i_bs_get_binary_all2_5; - op->arity = 5; - op->a[0] = Fail; - op->a[1] = Ms; - op->a[2] = Live; - op->a[3] = Unit; - op->a[4] = Dst; - } + GENOP_NAME_ARITY(op, i_bs_get_binary_all2, 5); + op->a[0] = Ms; + op->a[1] = Fail; + op->a[2] = Live; + op->a[3] = Unit; + op->a[4] = Dst; } else if (Size.type == TAG_i) { - op->op = genop_i_bs_get_binary_imm2_6; - op->arity = 6; - op->a[0] = Fail; - op->a[1] = Ms; + GENOP_NAME_ARITY(op, i_bs_get_binary_imm2, 6); + op->a[0] = Ms; + op->a[1] = Fail; op->a[2] = Live; op->a[3].type = TAG_u; if (!safe_mul(Size.val, Unit.val, &op->a[3].val)) { @@ -3324,14 +3377,12 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, if (!term_to_Uint(big, &bigval)) { error: - op->op = genop_jump_1; - op->arity = 1; + GENOP_NAME_ARITY(op, jump, 1); op->a[0] = Fail; } else { - op->op = genop_i_bs_get_binary_imm2_6; - op->arity = 6; - op->a[0] = Fail; - op->a[1] = Ms; + GENOP_NAME_ARITY(op, i_bs_get_binary_imm2, 6); + op->a[0] = Ms; + op->a[1] = Fail; op->a[2] = Live; op->a[3].type = TAG_u; if (!safe_mul(bigval, Unit.val, &op->a[3].val)) { @@ -3341,10 +3392,9 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, op->a[5] = Dst; } } else { - op->op = genop_i_bs_get_binary2_6; - op->arity = 6; - op->a[0] = Fail; - op->a[1] = Ms; + GENOP_NAME_ARITY(op, i_bs_get_binary2, 6); + op->a[0] = Ms; + op->a[1] = Fail; op->a[2] = Live; op->a[3] = Size; op->a[4].type = TAG_u; @@ -3375,26 +3425,44 @@ gen_put_binary(LoaderState* stp, GenOpArg Fail,GenOpArg Size, NATIVE_ENDIAN(Flags); if (Size.type == TAG_a && Size.val == am_all) { - op->op = genop_i_new_bs_put_binary_all_3; - op->arity = 3; - op->a[0] = Fail; - op->a[1] = Src; + GENOP_NAME_ARITY(op, i_new_bs_put_binary_all, 3); + op->a[0] = Src; + op->a[1] = Fail; op->a[2] = Unit; } else if (Size.type == TAG_i) { - op->op = genop_i_new_bs_put_binary_imm_3; - op->arity = 3; + GENOP_NAME_ARITY(op, i_new_bs_put_binary_imm, 3); op->a[0] = Fail; op->a[1].type = TAG_u; if (safe_mul(Size.val, Unit.val, &op->a[1].val)) { op->a[2] = Src; } else { - op->op = genop_badarg_1; - op->arity = 1; + error: + GENOP_NAME_ARITY(op, badarg, 1); op->a[0] = Fail; } + } else if (Size.type == TAG_q) { +#ifdef ARCH_64 + /* + * There is no way that this binary would fit in memory. + */ + goto error; +#else + Eterm big = stp->literals[Size.val].term; + Uint bigval; + Uint size; + + if (!term_to_Uint(big, &bigval) || + !safe_mul(bigval, Unit.val, &size)) { + goto error; + } + GENOP_NAME_ARITY(op, i_new_bs_put_binary_imm, 3); + op->a[0] = Fail; + op->a[1].type = TAG_u; + op->a[1].val = size; + op->a[2] = Src; +#endif } else { - op->op = genop_i_new_bs_put_binary_4; - op->arity = 4; + GENOP_NAME_ARITY(op, i_new_bs_put_binary, 4); op->a[0] = Fail; op->a[1] = Size; op->a[2].type = TAG_u; @@ -3416,41 +3484,39 @@ gen_put_integer(LoaderState* stp, GenOpArg Fail, GenOpArg Size, NATIVE_ENDIAN(Flags); /* Negative size must fail */ if (Size.type == TAG_i) { - op->op = genop_i_new_bs_put_integer_imm_4; - op->arity = 4; - op->a[0] = Fail; - op->a[1].type = TAG_u; - if (!safe_mul(Size.val, Unit.val, &op->a[1].val)) { + Uint size; + if (!safe_mul(Size.val, Unit.val, &size)) { error: - op->op = genop_badarg_1; - op->arity = 1; + GENOP_NAME_ARITY(op, badarg, 1); op->a[0] = Fail; op->next = NULL; return op; } - op->a[1].val = Size.val * Unit.val; - op->a[2].type = Flags.type; - op->a[2].val = (Flags.val & 7); - op->a[3] = Src; + GENOP_NAME_ARITY(op, i_new_bs_put_integer_imm, 4); + op->a[0] = Src; + op->a[1] = Fail; + op->a[2].type = TAG_u; + op->a[2].val = size; + op->a[3].type = Flags.type; + op->a[3].val = (Flags.val & 7); } else if (Size.type == TAG_q) { Eterm big = stp->literals[Size.val].term; Uint bigval; + Uint size; - if (!term_to_Uint(big, &bigval)) { + if (!term_to_Uint(big, &bigval) || + !safe_mul(bigval, Unit.val, &size)) { goto error; - } else { - op->op = genop_i_new_bs_put_integer_imm_4; - op->arity = 4; - op->a[0] = Fail; - op->a[1].type = TAG_u; - op->a[1].val = bigval * Unit.val; - op->a[2].type = Flags.type; - op->a[2].val = (Flags.val & 7); - op->a[3] = Src; } + GENOP_NAME_ARITY(op, i_new_bs_put_integer_imm, 4); + op->a[0] = Src; + op->a[1] = Fail; + op->a[2].type = TAG_u; + op->a[2].val = size; + op->a[3].type = Flags.type; + op->a[3].val = (Flags.val & 7); } else { - op->op = genop_i_new_bs_put_integer_4; - op->arity = 4; + GENOP_NAME_ARITY(op, i_new_bs_put_integer, 4); op->a[0] = Fail; op->a[1] = Size; op->a[2].type = TAG_u; @@ -3470,21 +3536,18 @@ gen_put_float(LoaderState* stp, GenOpArg Fail, GenOpArg Size, NATIVE_ENDIAN(Flags); if (Size.type == TAG_i) { - op->op = genop_i_new_bs_put_float_imm_4; - op->arity = 4; + GENOP_NAME_ARITY(op, i_new_bs_put_float_imm, 4); op->a[0] = Fail; op->a[1].type = TAG_u; if (!safe_mul(Size.val, Unit.val, &op->a[1].val)) { - op->op = genop_badarg_1; - op->arity = 1; + GENOP_NAME_ARITY(op, badarg, 1); op->a[0] = Fail; } else { op->a[2] = Flags; op->a[3] = Src; } } else { - op->op = genop_i_new_bs_put_float_4; - op->arity = 4; + GENOP_NAME_ARITY(op, i_new_bs_put_float, 4); op->a[0] = Fail; op->a[1] = Size; op->a[2].type = TAG_u; @@ -3507,10 +3570,9 @@ gen_get_float2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, NEW_GENOP(stp, op); NATIVE_ENDIAN(Flags); - op->op = genop_i_bs_get_float2_6; - op->arity = 6; - op->a[0] = Fail; - op->a[1] = Ms; + GENOP_NAME_ARITY(op, i_bs_get_float2, 6); + op->a[0] = Ms; + op->a[1] = Fail; op->a[2] = Live; op->a[3] = Size; op->a[4].type = TAG_u; @@ -3525,7 +3587,7 @@ gen_get_float2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, */ static GenOp* -gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, +gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Size, GenOpArg Unit, GenOpArg Flags) { GenOp* op; @@ -3533,16 +3595,26 @@ gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, NATIVE_ENDIAN(Flags); NEW_GENOP(stp, op); if (Size.type == TAG_a && Size.val == am_all) { - op->op = genop_i_bs_skip_bits_all2_3; - op->arity = 3; + /* + * This kind of skip instruction will only be found in modules + * compiled before OTP 19. From OTP 19, the compiler generates + * a test_unit instruction of a bs_skip at the end of a + * binary. + * + * It is safe to replace the skip instruction with a test_unit + * instruction, because the position will never be used again. + * If the match context itself is used again, it will be used by + * a bs_restore2 instruction which will overwrite the position + * by one of the stored positions. + */ + GENOP_NAME_ARITY(op, bs_test_unit, 3); op->a[0] = Fail; - op->a[1] = Ms; + op->a[1] = Ms; op->a[2] = Unit; } else if (Size.type == TAG_i) { - op->op = genop_i_bs_skip_bits_imm2_3; - op->arity = 3; + GENOP_NAME_ARITY(op, i_bs_skip_bits_imm2, 3); op->a[0] = Fail; - op->a[1] = Ms; + op->a[1] = Ms; op->a[2].type = TAG_u; if (!safe_mul(Size.val, Unit.val, &op->a[2].val)) { goto error; @@ -3553,25 +3625,22 @@ gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, if (!term_to_Uint(big, &bigval)) { error: - op->op = genop_jump_1; - op->arity = 1; + GENOP_NAME_ARITY(op, jump, 1); op->a[0] = Fail; } else { - op->op = genop_i_bs_skip_bits_imm2_3; - op->arity = 3; + GENOP_NAME_ARITY(op, i_bs_skip_bits_imm2, 3); op->a[0] = Fail; - op->a[1] = Ms; + op->a[1] = Ms; op->a[2].type = TAG_u; if (!safe_mul(bigval, Unit.val, &op->a[2].val)) { goto error; } } } else { - op->op = genop_i_bs_skip_bits2_4; - op->arity = 4; - op->a[0] = Fail; - op->a[1] = Ms; - op->a[2] = Size; + GENOP_NAME_ARITY(op, i_bs_skip_bits2, 4); + op->a[0] = Ms; + op->a[1] = Size; + op->a[2] = Fail; op->a[3] = Unit; } op->next = NULL; @@ -3579,38 +3648,52 @@ gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, } static GenOp* -gen_increment(LoaderState* stp, GenOpArg Reg, GenOpArg Integer, - GenOpArg Live, GenOpArg Dst) +gen_increment(LoaderState* stp, GenOpArg Reg, + GenOpArg Integer, GenOpArg Dst) { GenOp* op; NEW_GENOP(stp, op); - op->op = genop_i_increment_4; - op->arity = 4; + GENOP_NAME_ARITY(op, i_increment, 3); op->next = NULL; op->a[0] = Reg; op->a[1].type = TAG_u; op->a[1].val = Integer.val; - op->a[2] = Live; - op->a[3] = Dst; + op->a[2] = Dst; return op; } static GenOp* -gen_increment_from_minus(LoaderState* stp, GenOpArg Reg, GenOpArg Integer, - GenOpArg Live, GenOpArg Dst) +gen_increment_from_minus(LoaderState* stp, GenOpArg Reg, + GenOpArg Integer, GenOpArg Dst) { GenOp* op; NEW_GENOP(stp, op); - op->op = genop_i_increment_4; - op->arity = 4; + GENOP_NAME_ARITY(op, i_increment, 3); op->next = NULL; op->a[0] = Reg; op->a[1].type = TAG_u; op->a[1].val = -Integer.val; - op->a[2] = Live; - op->a[3] = Dst; + op->a[2] = Dst; + return op; +} + +static GenOp* +gen_plus_from_minus(LoaderState* stp, GenOpArg Fail, GenOpArg Live, + GenOpArg Src, GenOpArg Integer, GenOpArg Dst) +{ + GenOp* op; + + NEW_GENOP(stp, op); + GENOP_NAME_ARITY(op, gen_plus, 5); + op->next = NULL; + op->a[0] = Fail; + op->a[1] = Live; + op->a[2] = Src; + op->a[3].type = TAG_i; + op->a[3].val = -Integer.val; + op->a[4] = Dst; return op; } @@ -3660,12 +3743,11 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time) Sint timeout; NEW_GENOP(stp, op); - op->op = genop_wait_timeout_unlocked_int_2; + GENOP_NAME_ARITY(op, wait_timeout_unlocked_int, 2); op->next = NULL; - op->arity = 2; op->a[0].type = TAG_u; op->a[1] = Fail; - + if (Time.type == TAG_i && (timeout = Time.val) >= 0 && #if defined(ARCH_64) (timeout >> 32) == 0 @@ -3694,8 +3776,7 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time) #if !defined(ARCH_64) error: #endif - op->op = genop_i_wait_error_0; - op->arity = 0; + GENOP_NAME_ARITY(op, i_wait_error, 0); } return op; } @@ -3707,9 +3788,8 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time) Sint timeout; NEW_GENOP(stp, op); - op->op = genop_wait_timeout_locked_int_2; + GENOP_NAME_ARITY(op, wait_timeout_locked_int, 2); op->next = NULL; - op->arity = 2; op->a[0].type = TAG_u; op->a[1] = Fail; @@ -3741,8 +3821,7 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time) #if !defined(ARCH_64) error: #endif - op->op = genop_i_wait_error_locked_0; - op->arity = 0; + GENOP_NAME_ARITY(op, i_wait_error_locked, 0); } return op; } @@ -3779,9 +3858,9 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail, */ if (size == 2) { NEW_GENOP(stp, op); - op->next = NULL; - op->op = genop_i_select_tuple_arity2_4; + GENOP_NAME_ARITY(op, i_select_tuple_arity2, 4); GENOP_ARITY(op, arity - 1); + op->next = NULL; op->a[0] = S; op->a[1] = Fail; op->a[2].type = TAG_u; @@ -3807,9 +3886,9 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail, size += align; NEW_GENOP(stp, op); - op->next = NULL; - op->op = genop_i_select_tuple_arity_3; + GENOP_NAME_ARITY(op, i_select_tuple_arity, 3); GENOP_ARITY(op, arity); + op->next = NULL; op->a[0] = S; op->a[1] = Fail; op->a[2].type = TAG_u; @@ -3865,21 +3944,18 @@ gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg TypeFail, ASSERT(Size.val >= 2 && Size.val % 2 == 0); NEW_GENOP(stp, is_integer); - is_integer->op = genop_is_integer_2; - is_integer->arity = 2; + GENOP_NAME_ARITY(is_integer, is_integer, 2); is_integer->a[0] = TypeFail; is_integer->a[1] = S; NEW_GENOP(stp, label); - label->op = genop_label_1; - label->arity = 1; + GENOP_NAME_ARITY(label, label, 1); label->a[0].type = TAG_u; label->a[0].val = new_label(stp); NEW_GENOP(stp, op1); - op1->op = genop_select_val_3; + GENOP_NAME_ARITY(op1, select_val, 3); GENOP_ARITY(op1, 3 + Size.val); - op1->arity = 3; op1->a[0] = S; op1->a[1].type = TAG_f; op1->a[1].val = label->a[0].val; @@ -3887,9 +3963,8 @@ gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg TypeFail, op1->a[2].val = 0; NEW_GENOP(stp, op2); - op2->op = genop_select_val_3; + GENOP_NAME_ARITY(op2, select_val, 3); GENOP_ARITY(op2, 3 + Size.val); - op2->arity = 3; op2->a[0] = S; op2->a[1] = Fail; op2->a[2].type = TAG_u; @@ -3967,19 +4042,17 @@ gen_jump_tab(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpAr GenOp* jump; NEW_GENOP(stp, op); - op->arity = 3; - op->op = genop_is_ne_exact_3; + GENOP_NAME_ARITY(op, is_ne_exact, 3); op->a[0] = Rest[1]; op->a[1] = S; op->a[2] = Rest[0]; NEW_GENOP(stp, jump); - jump->next = NULL; - jump->arity = 1; - jump->op = genop_jump_1; + GENOP_NAME_ARITY(jump, jump, 1); jump->a[0] = Fail; op->next = jump; + jump->next = NULL; return op; } @@ -4006,12 +4079,11 @@ gen_jump_tab(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpAr NEW_GENOP(stp, op); op->next = NULL; if (min == 0) { - op->op = genop_i_jump_on_val_zero_3; - fixed_args = 3; + GENOP_NAME_ARITY(op, i_jump_on_val_zero, 3); } else { - op->op = genop_i_jump_on_val_4; - fixed_args = 4; + GENOP_NAME_ARITY(op, i_jump_on_val, 4); } + fixed_args = op->arity; arity = fixed_args + size; GENOP_ARITY(op, arity); op->a[0] = S; @@ -4076,7 +4148,7 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, NEW_GENOP(stp, op); op->next = NULL; - op->op = genop_i_select_val2_4; + GENOP_NAME_ARITY(op, i_select_val2, 4); GENOP_ARITY(op, arity - 1); op->a[0] = S; op->a[1] = Fail; @@ -4098,7 +4170,11 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, NEW_GENOP(stp, op); op->next = NULL; - op->op = (align == 0) ? genop_i_select_val_bins_3 : genop_i_select_val_lins_3; + if (align == 0) { + GENOP_NAME_ARITY(op, i_select_val_bins, 3); + } else { + GENOP_NAME_ARITY(op, i_select_val_lins, 3); + } GENOP_ARITY(op, arity); op->a[0] = S; op->a[1] = Fail; @@ -4162,8 +4238,7 @@ gen_select_literals(LoaderState* stp, GenOpArg S, GenOpArg Fail, ASSERT(Rest[i].type == TAG_q); NEW_GENOP(stp, op); - op->op = genop_is_ne_exact_3; - op->arity = 3; + GENOP_NAME_ARITY(op, is_ne_exact, 3); op->a[0] = Rest[i+1]; op->a[1] = S; op->a[2] = Rest[i]; @@ -4172,9 +4247,8 @@ gen_select_literals(LoaderState* stp, GenOpArg S, GenOpArg Fail, } NEW_GENOP(stp, jump); + GENOP_NAME_ARITY(jump, jump, 1); jump->next = NULL; - jump->op = genop_jump_1; - jump->arity = 1; jump->a[0] = Fail; *prev_next = jump; return op; @@ -4196,9 +4270,8 @@ const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, ASSERT(Size.type == TAG_u); NEW_GENOP(stp, op); + GENOP_NAME_ARITY(op, jump, 1); op->next = NULL; - op->op = genop_jump_1; - op->arity = 1; /* * Search for a literal matching the controlling expression. @@ -4245,115 +4318,120 @@ gen_make_fun2(LoaderState* stp, GenOpArg idx) { ErlFunEntry* fe; GenOp* op; + Uint arity, num_free; if (idx.val >= stp->num_lambdas) { - stp->lambda_error = "missing or short chunk 'FunT'"; - fe = 0; + stp->lambda_error = "missing or short chunk 'FunT'"; + fe = 0; + num_free = 0; + arity = 0; } else { - fe = stp->lambdas[idx.val].fe; + fe = stp->lambdas[idx.val].fe; + num_free = stp->lambdas[idx.val].num_free; + arity = fe->arity; } NEW_GENOP(stp, op); - op->op = genop_i_make_fun_2; - op->arity = 2; - op->a[0].type = TAG_u; - op->a[0].val = (BeamInstr) fe; - op->a[1].type = TAG_u; - op->a[1].val = stp->lambdas[idx.val].num_free; - op->next = NULL; - return op; -} -static GenOp* -translate_gc_bif(LoaderState* stp, GenOp* op, GenOpArg Bif) -{ - const ErtsGcBif* p; - BifFunction bf; - - bf = stp->import[Bif.val].bf; - for (p = erts_gc_bifs; p->bif != 0; p++) { - if (p->bif == bf) { - op->a[1].type = TAG_u; - op->a[1].val = (BeamInstr) p->gc_bif; - return op; - } + /* + * It's possible this is called before init process is started, + * skip the optimisation in such case. + */ + if (num_free == 0 && erts_init_process_id != ERTS_INVALID_PID) { + Uint lit; + Eterm* hp; + ErlFunThing* funp; + + lit = new_literal(stp, &hp, ERL_FUN_SIZE); + funp = (ErlFunThing *) hp; + erts_refc_inc(&fe->refc, 2); + funp->thing_word = HEADER_FUN; + funp->next = NULL; + funp->fe = fe; + funp->num_free = 0; + funp->creator = erts_init_process_id; + funp->arity = arity; + + /* + * Use a move_fun/2 instruction to load the fun to enable + * further optimizations. + */ + GENOP_NAME_ARITY(op, move_fun, 2); + op->a[0].type = TAG_q; + op->a[0].val = lit; + op->a[1].type = TAG_x; + op->a[1].val = 0; + } else { + GENOP_NAME_ARITY(op, i_make_fun, 2); + op->a[0].type = TAG_u; + op->a[0].val = (BeamInstr) fe; + op->a[1].type = TAG_u; + op->a[1].val = num_free; } - op->op = genop_unsupported_guard_bif_3; - op->arity = 3; - op->a[0].type = TAG_a; - op->a[0].val = stp->import[Bif.val].module; - op->a[1].type = TAG_a; - op->a[1].val = stp->import[Bif.val].function; - op->a[2].type = TAG_u; - op->a[2].val = stp->import[Bif.val].arity; + op->next = NULL; return op; } -/* - * Rewrite gc_bifs with one parameter (the common case). - */ static GenOp* -gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, - GenOpArg Src, GenOpArg Dst) +gen_is_function2(LoaderState* stp, GenOpArg Fail, GenOpArg Fun, GenOpArg Arity) { GenOp* op; + int literal_arity = Arity.type == TAG_i; + int fun_is_reg = Fun.type == TAG_x || Fun.type == TAG_y; NEW_GENOP(stp, op); op->next = NULL; - op->op = genop_i_gc_bif1_5; - op->arity = 5; - op->a[0] = Fail; - /* op->a[1] is set by translate_gc_bif() */ - op->a[2] = Src; - op->a[3] = Live; - op->a[4] = Dst; - return translate_gc_bif(stp, op, Bif); -} -/* - * This is used by the ops.tab rule that rewrites gc_bifs with two parameters. - */ -static GenOp* -gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, - GenOpArg S1, GenOpArg S2, GenOpArg Dst) -{ - GenOp* op; + if (fun_is_reg &&literal_arity) { + /* + * Most common case. Fun in a register and arity + * is an integer literal. + */ + if (Arity.val > MAX_ARG) { + /* Arity is negative or too big. */ + GENOP_NAME_ARITY(op, jump, 1); + op->a[0] = Fail; + return op; + } else { + GENOP_NAME_ARITY(op, hot_is_function2, 3); + op->a[0] = Fail; + op->a[1] = Fun; + op->a[2].type = TAG_u; + op->a[2].val = Arity.val; + return op; + } + } else { + /* + * Handle extremely uncommon cases by a slower sequence. + */ + GenOp* move_fun; + GenOp* move_arity; - NEW_GENOP(stp, op); - op->next = NULL; - op->op = genop_i_gc_bif2_6; - op->arity = 6; - op->a[0] = Fail; - /* op->a[1] is set by translate_gc_bif() */ - op->a[2] = Live; - op->a[3] = S1; - op->a[4] = S2; - op->a[5] = Dst; - return translate_gc_bif(stp, op, Bif); -} + NEW_GENOP(stp, move_fun); + NEW_GENOP(stp, move_arity); -/* - * This is used by the ops.tab rule that rewrites gc_bifs with three parameters. - */ -static GenOp* -gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, - GenOpArg S1, GenOpArg S2, GenOpArg S3, GenOpArg Dst) -{ - GenOp* op; + move_fun->next = move_arity; + move_arity->next = op; - NEW_GENOP(stp, op); - op->next = NULL; - op->op = genop_ii_gc_bif3_7; - op->arity = 7; - op->a[0] = Fail; - /* op->a[1] is set by translate_gc_bif() */ - op->a[2] = Live; - op->a[3] = S1; - op->a[4] = S2; - op->a[5] = S3; - op->a[6] = Dst; - return translate_gc_bif(stp, op, Bif); + GENOP_NAME_ARITY(move_fun, move, 2); + move_fun->a[0] = Fun; + move_fun->a[1].type = TAG_x; + move_fun->a[1].val = 1022; + + GENOP_NAME_ARITY(move_arity, move, 2); + move_arity->a[0] = Arity; + move_arity->a[1].type = TAG_x; + move_arity->a[1].val = 1023; + + GENOP_NAME_ARITY(op, cold_is_function2, 3); + op->a[0] = Fail; + op->a[1].type = TAG_x; + op->a[1].val = 1022; + op->a[2].type = TAG_x; + op->a[2].val = 1023; + return move_fun; + } } static GenOp* @@ -4367,8 +4445,8 @@ tuple_append_put5(LoaderState* stp, GenOpArg Arity, GenOpArg Dst, NEW_GENOP(stp, op); op->next = NULL; + GENOP_NAME_ARITY(op, i_put_tuple, 2); GENOP_ARITY(op, arity+2+5); - op->op = genop_i_put_tuple_2; op->a[0] = Dst; op->a[1].type = TAG_u; op->a[1].val = arity + 5; @@ -4393,8 +4471,8 @@ tuple_append_put(LoaderState* stp, GenOpArg Arity, GenOpArg Dst, NEW_GENOP(stp, op); op->next = NULL; + GENOP_NAME_ARITY(op, i_put_tuple, 2); GENOP_ARITY(op, arity+2+1); - op->op = genop_i_put_tuple_2; op->a[0] = Dst; op->a[1].type = TAG_u; op->a[1].val = arity + 1; @@ -4462,9 +4540,9 @@ gen_new_small_map_lit(LoaderState* stp, GenOpArg Dst, GenOpArg Live, Eterm keys; NEW_GENOP(stp, op); + GENOP_NAME_ARITY(op, i_new_small_map_lit, 3); GENOP_ARITY(op, 3 + size/2); op->next = NULL; - op->op = genop_i_new_small_map_lit_3; tmp = thp = erts_alloc(ERTS_ALC_T_LOADER_TMP, (1 + size/2) * sizeof(*tmp)); keys = make_tuple(thp); @@ -4522,19 +4600,6 @@ is_empty_map(LoaderState* stp, GenOpArg Lit) } /* - * Predicate to test whether the given literal is an export. - */ -static int -literal_is_export(LoaderState* stp, GenOpArg Lit) -{ - Eterm term; - - ASSERT(Lit.type == TAG_q); - term = stp->literals[Lit.val].term; - return is_export(term); -} - -/* * Pseudo predicate map_key_sort that will sort the Rest operand for * map instructions as a side effect. */ @@ -4547,7 +4612,15 @@ typedef struct SortGenOpArg { static int genopargtermcompare(SortGenOpArg* a, SortGenOpArg* b) { - return CMP_TERM(a->term, b->term); + Sint res = CMP_TERM(a->term, b->term); + + if (res < 0) { + return -1; + } else if (res > 0) { + return 1; + } + + return 0; } static int @@ -4658,14 +4731,12 @@ gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src, Key = Rest[0]; if (hash_genop_arg(stp, Key, &hx)) { - op->arity = 5; - op->op = genop_i_get_map_element_hash_5; + GENOP_NAME_ARITY(op, i_get_map_element_hash, 5); op->a[3].type = TAG_u; op->a[3].val = (BeamInstr) hx; op->a[4] = Rest[1]; } else { - op->arity = 4; - op->op = genop_i_get_map_element_4; + GENOP_NAME_ARITY(op, i_get_map_element, 4); op->a[3] = Rest[1]; } return op; @@ -4705,15 +4776,13 @@ gen_get(LoaderState* stp, GenOpArg Src, GenOpArg Dst) NEW_GENOP(stp, op); op->next = NULL; if (hash_internal_genop_arg(stp, Src, &hx)) { - op->arity = 3; - op->op = genop_i_get_hash_3; + GENOP_NAME_ARITY(op, i_get_hash, 3); op->a[0] = Src; op->a[1].type = TAG_u; op->a[1].val = (BeamInstr) hx; op->a[2] = Dst; } else { - op->arity = 2; - op->op = genop_i_get_2; + GENOP_NAME_ARITY(op, i_get, 2); op->a[0] = Src; op->a[1] = Dst; } @@ -4737,7 +4806,7 @@ gen_get_map_elements(LoaderState* stp, GenOpArg Fail, GenOpArg Src, ASSERT(Size.type == TAG_u); NEW_GENOP(stp, op); - op->op = genop_i_get_map_elements_3; + GENOP_NAME_ARITY(op, i_get_map_elements, 3); GENOP_ARITY(op, 3 + 3*(Size.val/2)); op->next = NULL; op->a[0] = Fail; @@ -4775,9 +4844,9 @@ gen_has_map_fields(LoaderState* stp, GenOpArg Fail, GenOpArg Src, n = Size.val; NEW_GENOP(stp, op); + GENOP_NAME_ARITY(op, get_map_elements, 3); GENOP_ARITY(op, 3 + 2*n); op->next = NULL; - op->op = genop_get_map_elements_3; op->a[0] = Fail; op->a[1] = Src; @@ -6114,7 +6183,8 @@ erts_release_literal_area(ErtsLiteralArea* literal_area) } default: ASSERT(is_external_header(oh->thing_word)); - erts_deref_node_entry(((ExternalThing*)oh)->node); + erts_deref_node_entry(((ExternalThing*)oh)->node, + make_boxed(&oh->thing_word)); } oh = oh->next; } |