diff options
Diffstat (limited to 'erts/emulator')
26 files changed, 1241 insertions, 603 deletions
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 3e9cfc990a..21351df656 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -267,7 +267,6 @@ DEXPORT = @DEXPORT@ RANLIB = @RANLIB@ STRIP = strip PERL = @PERL@ -RM = @RM@ MKDIR = @MKDIR@ USING_MINGW=@MIXED_CYGWIN_MINGW@ @@ -468,13 +467,13 @@ $(ERTS_LIB): .PHONY: clean clean: - $(RM) -f $(GENERATE) - $(RM) -rf $(TARGET)/*.c $(TARGET)/*.h $(TARGET)/*-GENERATED - $(RM) -rf $(TARGET)/*/* - $(RM) -rf obj/$(TARGET) - $(RM) -rf pcre/obj/$(TARGET) $(PCRE_GENINC) - $(RM) -rf zlib/obj/$(TARGET) - $(RM) -rf bin/$(TARGET) + $(RM) $(GENERATE) + $(RM) -r $(TARGET)/*.c $(TARGET)/*.h $(TARGET)/*-GENERATED + $(RM) -r $(TARGET)/*/* + $(RM) -r obj/$(TARGET) + $(RM) -r pcre/obj/$(TARGET) $(PCRE_GENINC) + $(RM) -r zlib/obj/$(TARGET) + $(RM) -r bin/$(TARGET) cd $(ERTS_LIB_DIR) && $(MAKE) clean .PHONY: docs diff --git a/erts/emulator/beam/arith_instrs.tab b/erts/emulator/beam/arith_instrs.tab index 574fceec5b..5f23b2c168 100644 --- a/erts/emulator/beam/arith_instrs.tab +++ b/erts/emulator/beam/arith_instrs.tab @@ -116,6 +116,17 @@ increment.execute(IncrementVal, Dst) { i_times(Fail, Op1, Op2, Dst) { Eterm op1 = $Op1; Eterm op2 = $Op2; +#ifdef HAVE_OVERFLOW_CHECK_BUILTINS + if (ERTS_LIKELY(is_both_small(op1, op2))) { + Sint a = signed_val(op1); + Sint b = signed_val(op2); + Sint res; + if (ERTS_LIKELY(!__builtin_mul_overflow(a, b, &res) && IS_SSMALL(res))) { + $Dst = make_small(res); + $NEXT0(); + } + } +#endif $OUTLINED_ARITH_2($Fail, mixed_times, BIF_stimes_2, op1, op2, $Dst); } diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 90162a6543..04a2a83123 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -322,19 +322,19 @@ void** beam_ops; #define Arg(N) I[(N)+1] -#define GetR(pos, tr) \ +#define GetSource(raw, dst) \ do { \ - tr = Arg(pos); \ - switch (loader_tag(tr)) { \ + dst = raw; \ + switch (loader_tag(dst)) { \ case LOADER_X_REG: \ - tr = x(loader_x_reg_index(tr)); \ + dst = x(loader_x_reg_index(dst)); \ break; \ case LOADER_Y_REG: \ - ASSERT(loader_y_reg_index(tr) >= 1); \ - tr = y(loader_y_reg_index(tr)); \ + ASSERT(loader_y_reg_index(dst) >= 1); \ + dst = y(loader_y_reg_index(dst)); \ break; \ } \ - CHECK_TERM(tr); \ + CHECK_TERM(dst); \ } while (0) #define PUT_TERM_REG(term, desc) \ diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index f4eeb54a1b..0ad5329b2f 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -1425,7 +1425,7 @@ load_atom_table(LoaderState* stp, ErtsAtomEncoding enc) ap = atom_tab(atom_val(stp->atom[1])); sys_memcpy(sbuf, ap->name, ap->len); sbuf[ap->len] = '\0'; - LoadError1(stp, "module name in object code is %s", sbuf); + LoadError1(stp, "BEAM file exists but it defines a module named %s", sbuf); } return 1; @@ -3295,11 +3295,11 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, } else { op->op = genop_i_bs_get_integer_6; op->arity = 6; - op->a[0] = Fail; - op->a[1] = Live; - op->a[2].type = TAG_u; - op->a[2].val = (Unit.val << 3) | Flags.val; - op->a[3] = Ms; + op->a[0] = Ms; + op->a[1] = Fail; + op->a[2] = Live; + op->a[3].type = TAG_u; + op->a[3].val = (Unit.val << 3) | Flags.val; op->a[4] = Size; op->a[5] = Dst; op->next = NULL; @@ -3332,8 +3332,8 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, } else { op->op = genop_i_bs_get_binary_all2_5; op->arity = 5; - op->a[0] = Fail; - op->a[1] = Ms; + op->a[0] = Ms; + op->a[1] = Fail; op->a[2] = Live; op->a[3] = Unit; op->a[4] = Dst; @@ -3341,8 +3341,8 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, } else if (Size.type == TAG_i) { op->op = genop_i_bs_get_binary_imm2_6; op->arity = 6; - op->a[0] = Fail; - op->a[1] = Ms; + op->a[0] = Ms; + op->a[1] = Fail; op->a[2] = Live; op->a[3].type = TAG_u; if (!safe_mul(Size.val, Unit.val, &op->a[3].val)) { @@ -3362,8 +3362,8 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, } else { op->op = genop_i_bs_get_binary_imm2_6; op->arity = 6; - op->a[0] = Fail; - op->a[1] = Ms; + op->a[0] = Ms; + op->a[1] = Fail; op->a[2] = Live; op->a[3].type = TAG_u; if (!safe_mul(bigval, Unit.val, &op->a[3].val)) { @@ -3375,8 +3375,8 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, } else { op->op = genop_i_bs_get_binary2_6; op->arity = 6; - op->a[0] = Fail; - op->a[1] = Ms; + op->a[0] = Ms; + op->a[1] = Fail; op->a[2] = Live; op->a[3] = Size; op->a[4].type = TAG_u; @@ -3409,8 +3409,8 @@ gen_put_binary(LoaderState* stp, GenOpArg Fail,GenOpArg Size, if (Size.type == TAG_a && Size.val == am_all) { op->op = genop_i_new_bs_put_binary_all_3; op->arity = 3; - op->a[0] = Fail; - op->a[1] = Src; + op->a[0] = Src; + op->a[1] = Fail; op->a[2] = Unit; } else if (Size.type == TAG_i) { op->op = genop_i_new_bs_put_binary_imm_3; @@ -3420,10 +3420,33 @@ gen_put_binary(LoaderState* stp, GenOpArg Fail,GenOpArg Size, if (safe_mul(Size.val, Unit.val, &op->a[1].val)) { op->a[2] = Src; } else { + error: op->op = genop_badarg_1; op->arity = 1; op->a[0] = Fail; } + } else if (Size.type == TAG_q) { +#ifdef ARCH_64 + /* + * There is no way that this binary would fit in memory. + */ + goto error; +#else + Eterm big = stp->literals[Size.val].term; + Uint bigval; + Uint size; + + if (!term_to_Uint(big, &bigval) || + !safe_mul(bigval, Unit.val, &size)) { + goto error; + } + op->op = genop_i_new_bs_put_binary_imm_3; + op->arity = 3; + op->a[0] = Fail; + op->a[1].type = TAG_u; + op->a[1].val = size; + op->a[2] = Src; +#endif } else { op->op = genop_i_new_bs_put_binary_4; op->arity = 4; @@ -3448,11 +3471,8 @@ gen_put_integer(LoaderState* stp, GenOpArg Fail, GenOpArg Size, NATIVE_ENDIAN(Flags); /* Negative size must fail */ if (Size.type == TAG_i) { - op->op = genop_i_new_bs_put_integer_imm_4; - op->arity = 4; - op->a[0] = Fail; - op->a[1].type = TAG_u; - if (!safe_mul(Size.val, Unit.val, &op->a[1].val)) { + Uint size; + if (!safe_mul(Size.val, Unit.val, &size)) { error: op->op = genop_badarg_1; op->arity = 1; @@ -3460,26 +3480,31 @@ gen_put_integer(LoaderState* stp, GenOpArg Fail, GenOpArg Size, op->next = NULL; return op; } - op->a[1].val = Size.val * Unit.val; - op->a[2].type = Flags.type; - op->a[2].val = (Flags.val & 7); - op->a[3] = Src; + op->op = genop_i_new_bs_put_integer_imm_4; + op->arity = 4; + op->a[0] = Src; + op->a[1] = Fail; + op->a[2].type = TAG_u; + op->a[2].val = size; + op->a[3].type = Flags.type; + op->a[3].val = (Flags.val & 7); } else if (Size.type == TAG_q) { Eterm big = stp->literals[Size.val].term; Uint bigval; + Uint size; - if (!term_to_Uint(big, &bigval)) { + if (!term_to_Uint(big, &bigval) || + !safe_mul(bigval, Unit.val, &size)) { goto error; - } else { - op->op = genop_i_new_bs_put_integer_imm_4; - op->arity = 4; - op->a[0] = Fail; - op->a[1].type = TAG_u; - op->a[1].val = bigval * Unit.val; - op->a[2].type = Flags.type; - op->a[2].val = (Flags.val & 7); - op->a[3] = Src; } + op->op = genop_i_new_bs_put_integer_imm_4; + op->arity = 4; + op->a[0] = Src; + op->a[1] = Fail; + op->a[2].type = TAG_u; + op->a[2].val = size; + op->a[3].type = Flags.type; + op->a[3].val = (Flags.val & 7); } else { op->op = genop_i_new_bs_put_integer_4; op->arity = 4; @@ -3541,8 +3566,8 @@ gen_get_float2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, NATIVE_ENDIAN(Flags); op->op = genop_i_bs_get_float2_6; op->arity = 6; - op->a[0] = Fail; - op->a[1] = Ms; + op->a[0] = Ms; + op->a[1] = Fail; op->a[2] = Live; op->a[3] = Size; op->a[4].type = TAG_u; @@ -3565,10 +3590,22 @@ gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, NATIVE_ENDIAN(Flags); NEW_GENOP(stp, op); if (Size.type == TAG_a && Size.val == am_all) { - op->op = genop_i_bs_skip_bits_all2_3; + /* + * This kind of skip instruction will only be found in modules + * compiled before OTP 19. From OTP 19, the compiler generates + * a test_unit instruction of a bs_skip at the end of a + * binary. + * + * It is safe to replace the skip instruction with a test_unit + * instruction, because the position will never be used again. + * If the match context itself is used again, it will be used by + * a bs_restore2 instruction which will overwrite the position + * by one of the stored positions. + */ + op->op = genop_bs_test_unit_3; op->arity = 3; op->a[0] = Fail; - op->a[1] = Ms; + op->a[1] = Ms; op->a[2] = Unit; } else if (Size.type == TAG_i) { op->op = genop_i_bs_skip_bits_imm2_3; @@ -3601,9 +3638,9 @@ gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, } else { op->op = genop_i_bs_skip_bits2_4; op->arity = 4; - op->a[0] = Fail; - op->a[1] = Ms; - op->a[2] = Size; + op->a[0] = Ms; + op->a[1] = Size; + op->a[2] = Fail; op->a[3] = Unit; } op->next = NULL; diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 7faba35e1c..c102ddbee6 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -5165,6 +5165,12 @@ erts_schedule_bif(Process *proc, pc = i; mfa = &exp->info.mfa; } + else if (BeamIsOpCode(*i, op_call_bif_only_e)) { + /* Pointer to bif export in i+1 */ + exp = (Export *) i[1]; + pc = i; + mfa = &exp->info.mfa; + } else if (BeamIsOpCode(*i, op_apply_bif)) { /* Pointer to bif in i+1, and mfa in i-3 */ pc = c_p->cp; diff --git a/erts/emulator/beam/bif_instrs.tab b/erts/emulator/beam/bif_instrs.tab index 418bbe2b23..8e0caa38a3 100644 --- a/erts/emulator/beam/bif_instrs.tab +++ b/erts/emulator/beam/bif_instrs.tab @@ -209,8 +209,8 @@ i_length.execute(Fail, Live, Dst) { } // -// The most general BIF call. The BIF may build any amount of data -// on the heap. The result is always returned in r(0). +// Call a BIF, store the result in x(0) and transfer control to the +// next instruction. // call_bif(Exp) { ErtsBifFunc bf; @@ -219,8 +219,10 @@ call_bif(Exp) { Export *export = (Export*) $Exp; if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) { - /* If we have run out of reductions, we do a context - switch before calling the bif */ + /* + * If we have run out of reductions, do a context + * switch before calling the BIF. + */ c_p->arity = GET_BIF_ARITY(export); c_p->current = &export->info.mfa; goto context_switch3; @@ -257,9 +259,12 @@ call_bif(Exp) { HTOP = HEAP_TOP(c_p); FCALLS = c_p->fcalls; ERTS_DBG_CHK_REDS(c_p, FCALLS); - /* We have to update the cache if we are enabled in order - to make sure no book keeping is done after we disabled - msacc. We don't always do this as it is quite expensive. */ + + /* + * We have to update the cache if we are enabled in order + * to make sure no bookkeeping is done after we disabled + * msacc. We don't always do this as it is quite expensive. + */ if (ERTS_MSACC_IS_ENABLED_CACHED_X()) { ERTS_MSACC_UPDATE_CACHE_X(); } @@ -269,7 +274,13 @@ call_bif(Exp) { CHECK_TERM(r(0)); $NEXT0(); } else if (c_p->freason == TRAP) { - SET_CP(c_p, I+2); + /* + * Set the continuation pointer to return to next + * instruction after the trap (either by a return from + * erlang code or by nif_bif.epilogue() when the BIF + * is done). + */ + SET_CP(c_p, $NEXT_INSTRUCTION); SET_I(c_p->i); SWAPIN; Dispatch(); @@ -281,6 +292,95 @@ call_bif(Exp) { ASSERT(c_p->stop == E); I = handle_error(c_p, I, reg, &export->info.mfa); goto post_error_handling; + //| -no_next +} + +// +// Call a BIF tail-recursively, storing the result in x(0) and doing +// a return to the continuation poiner (c_p->cp). +// + +call_bif_only(Exp) { + ErtsBifFunc bf; + Eterm result; + ErlHeapFragment *live_hf_end; + Export *export = (Export*) $Exp; + + if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) { + /* + * If we have run out of reductions, do a context + * switch before calling the BIF. + */ + c_p->arity = GET_BIF_ARITY(export); + c_p->current = &export->info.mfa; + goto context_switch3; + } + + ERTS_MSACC_SET_BIF_STATE_CACHED_X(GET_BIF_MODULE(export), + GET_BIF_ADDRESS(export)); + + bf = GET_BIF_ADDRESS(export); + + PRE_BIF_SWAPOUT(c_p); + ERTS_DBG_CHK_REDS(c_p, FCALLS); + c_p->fcalls = FCALLS - 1; + if (FCALLS <= 0) { + save_calls(c_p, export); + } + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + live_hf_end = c_p->mbuf; + ERTS_CHK_MBUF_SZ(c_p); + result = (*bf)(c_p, reg, I); + ERTS_CHK_MBUF_SZ(c_p); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_HOLE_CHECK(c_p); + ERTS_REQ_PROC_MAIN_LOCK(c_p); + if (ERTS_IS_GC_DESIRED(c_p)) { + Uint arity = GET_BIF_ARITY(export); + result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result, + reg, arity); + E = c_p->stop; + } + PROCESS_MAIN_CHK_LOCKS(c_p); + HTOP = HEAP_TOP(c_p); + FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); + + /* + * We have to update the cache if we are enabled in order + * to make sure no bookkeeping is done after we disabled + * msacc. We don't always do this as it is quite expensive. + */ + if (ERTS_MSACC_IS_ENABLED_CACHED_X()) { + ERTS_MSACC_UPDATE_CACHE_X(); + } + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); + if (ERTS_LIKELY(is_value(result))) { + /* + * Success. Store the result and return to the caller. + */ + r(0) = result; + CHECK_TERM(r(0)); + $return(); + } else if (c_p->freason == TRAP) { + /* + * Dispatch to a trap. When the trap is done, a jump + * to the continuation pointer (c_p->cp) will be done. + */ + SET_I(c_p->i); + SWAPIN; + Dispatch(); + } + + /* + * Error handling. SWAPOUT is not needed because it was done above. + */ + ASSERT(c_p->stop == E); + I = handle_error(c_p, I, reg, &export->info.mfa); + goto post_error_handling; + //| -no_next } // @@ -313,7 +413,7 @@ send() { r(0) = result; CHECK_TERM(r(0)); } else if (c_p->freason == TRAP) { - SET_CP(c_p, I+1); + SET_CP(c_p, $NEXT_INSTRUCTION); SET_I(c_p->i); SWAPIN; Dispatch(); diff --git a/erts/emulator/beam/bs_instrs.tab b/erts/emulator/beam/bs_instrs.tab index 2dde70c2e1..652460a66d 100644 --- a/erts/emulator/beam/bs_instrs.tab +++ b/erts/emulator/beam/bs_instrs.tab @@ -21,12 +21,57 @@ %if ARCH_64 BS_SAFE_MUL(A, B, Fail, Dst) { - Uint64 res = ($A) * ($B); - if (res / $B != $A) { + Uint a = $A; + Uint b = $B; + Uint res; +#ifdef HAVE_OVERFLOW_CHECK_BUILTINS + if (__builtin_mul_overflow(a, b, &res)) { + $Fail; + } +#else + res = a * b; + if (res / b != a) { $Fail; } +#endif $Dst = res; } + +BS_GET_FIELD_SIZE(Bits, Unit, Fail, Dst) { + if (is_small($Bits)) { + Uint uint_size; + Sint signed_size = signed_val($Bits); + if (signed_size < 0) { + $Fail; + } + uint_size = (Uint) signed_size; + $BS_SAFE_MUL(uint_size, $Unit, $Fail, $Dst); + } else { + /* + * On a 64-bit architecture, the size of any binary + * that would fit in the memory fits in a small. + */ + $Fail; + } +} + +BS_GET_UNCHECKED_FIELD_SIZE(Bits, Unit, Fail, Dst) { + if (is_small($Bits)) { + Uint uint_size; + Sint signed_size = signed_val($Bits); + if (signed_size < 0) { + $Fail; + } + uint_size = (Uint) signed_size; + $Dst = uint_size * $Unit; + } else { + /* + * On a 64-bit architecture, the size of any binary + * that would fit in the memory fits in a small. + */ + $Fail; + } +} %else BS_SAFE_MUL(A, B, Fail, Dst) { Uint64 res = (Uint64)($A) * (Uint64)($B); @@ -35,7 +80,6 @@ BS_SAFE_MUL(A, B, Fail, Dst) { } $Dst = res; } -%endif BS_GET_FIELD_SIZE(Bits, Unit, Fail, Dst) { Sint signed_size; @@ -76,6 +120,7 @@ BS_GET_UNCHECKED_FIELD_SIZE(Bits, Unit, Fail, Dst) { } $Dst = uint_size * $Unit; } +%endif TEST_BIN_VHEAP(VNh, Nh, Live) { Uint need = $Nh; @@ -90,12 +135,22 @@ TEST_BIN_VHEAP(VNh, Nh, Live) { HEAP_SPACE_VERIFIED(need); } -i_bs_get_binary_all2(Fail, Ms, Live, Unit, Dst) { +i_bs_get_binary_all2 := i_bs_get_binary_all2.fetch.execute; + +i_bs_get_binary_all2.head() { + Eterm context; +} + +i_bs_get_binary_all2.fetch(Ctx) { + context = $Ctx; +} + +i_bs_get_binary_all2.execute(Fail, Live, Unit, Dst) { ErlBinMatchBuffer *_mb; Eterm _result; - $GC_TEST(0, ERL_SUB_BIN_SIZE, $Live); - _mb = ms_matchbuffer($Ms); + $GC_TEST_PRESERVE(ERL_SUB_BIN_SIZE, $Live, context); + _mb = ms_matchbuffer(context); if (((_mb->size - _mb->offset) % $Unit) == 0) { LIGHT_SWAPOUT; _result = erts_bs_get_binary_all_2(c_p, _mb); @@ -109,14 +164,23 @@ i_bs_get_binary_all2(Fail, Ms, Live, Unit, Dst) { $FAIL($Fail); } } +i_bs_get_binary2 := i_bs_get_binary2.fetch.execute; + +i_bs_get_binary2.head() { + Eterm context; +} + +i_bs_get_binary2.fetch(Ctx) { + context = $Ctx; +} -i_bs_get_binary2(Fail, Ms, Live, Sz, Flags, Dst) { +i_bs_get_binary2.execute(Fail, Live, Sz, Flags, Dst) { ErlBinMatchBuffer *_mb; Eterm _result; Uint _size; $BS_GET_FIELD_SIZE($Sz, (($Flags) >> 3), $FAIL($Fail), _size); - $GC_TEST(0, ERL_SUB_BIN_SIZE, $Live); - _mb = ms_matchbuffer($Ms); + $GC_TEST_PRESERVE(ERL_SUB_BIN_SIZE, $Live, context); + _mb = ms_matchbuffer(context); LIGHT_SWAPOUT; _result = erts_bs_get_binary_2(c_p, _size, $Flags, _mb); LIGHT_SWAPIN; @@ -129,11 +193,22 @@ i_bs_get_binary2(Fail, Ms, Live, Sz, Flags, Dst) { } } -i_bs_get_binary_imm2(Fail, Ms, Live, Sz, Flags, Dst) { +i_bs_get_binary_imm2 := i_bs_get_binary_imm2.fetch.execute; + +i_bs_get_binary_imm2.head() { + Eterm context; +} + +i_bs_get_binary_imm2.fetch(Ctx) { + context = $Ctx; +} + +i_bs_get_binary_imm2.execute(Fail, Live, Sz, Flags, Dst) { ErlBinMatchBuffer *_mb; Eterm _result; - $GC_TEST(0, heap_bin_size(ERL_ONHEAP_BIN_LIMIT), $Live); - _mb = ms_matchbuffer($Ms); + $GC_TEST_PRESERVE(heap_bin_size(ERL_ONHEAP_BIN_LIMIT), + $Live, context); + _mb = ms_matchbuffer(context); LIGHT_SWAPOUT; _result = erts_bs_get_binary_2(c_p, $Sz, $Flags, _mb); LIGHT_SWAPIN; @@ -145,8 +220,17 @@ i_bs_get_binary_imm2(Fail, Ms, Live, Sz, Flags, Dst) { $Dst = _result; } } +i_bs_get_float2 := i_bs_get_float2.fetch.execute; + +i_bs_get_float2.head() { + Eterm context; +} + +i_bs_get_float2.fetch(Ctx) { + context = $Ctx; +} -i_bs_get_float2(Fail, Ms, Live, Sz, Flags, Dst) { +i_bs_get_float2.execute(Fail, Live, Sz, Flags, Dst) { ErlBinMatchBuffer *_mb; Eterm _result; Sint _size; @@ -155,8 +239,8 @@ i_bs_get_float2(Fail, Ms, Live, Sz, Flags, Dst) { $FAIL($Fail); } _size *= (($Flags) >> 3); - $GC_TEST(0, FLOAT_SIZE_OBJECT, $Live); - _mb = ms_matchbuffer($Ms); + $GC_TEST_PRESERVE(FLOAT_SIZE_OBJECT, $Live, context); + _mb = ms_matchbuffer(context); LIGHT_SWAPOUT; _result = erts_bs_get_float_2(c_p, _size, ($Flags), _mb); LIGHT_SWAPIN; @@ -169,13 +253,24 @@ i_bs_get_float2(Fail, Ms, Live, Sz, Flags, Dst) { } } -i_bs_skip_bits2(Fail, Ms, Bits, Unit) { +i_bs_skip_bits2 := i_bs_skip_bits2.fetch.execute; + +i_bs_skip_bits2.head() { + Eterm context, bits; +} + +i_bs_skip_bits2.fetch(Ctx, Bits) { + context = $Ctx; + bits = $Bits; +} + +i_bs_skip_bits2.execute(Fail, Unit) { ErlBinMatchBuffer *_mb; size_t new_offset; Uint _size; - _mb = ms_matchbuffer($Ms); - $BS_GET_FIELD_SIZE($Bits, $Unit, $FAIL($Fail), _size); + _mb = ms_matchbuffer(context); + $BS_GET_FIELD_SIZE(bits, $Unit, $FAIL($Fail), _size); new_offset = _mb->offset + _size; if (new_offset <= _mb->size) { _mb->offset = new_offset; @@ -184,16 +279,6 @@ i_bs_skip_bits2(Fail, Ms, Bits, Unit) { } } -i_bs_skip_bits_all2(Fail, Ms, Unit) { - ErlBinMatchBuffer *_mb; - _mb = ms_matchbuffer($Ms); - if (((_mb->size - _mb->offset) % $Unit) == 0) { - _mb->offset = _mb->size; - } else { - $FAIL($Fail); - } -} - i_bs_skip_bits_imm2(Fail, Ms, Bits) { ErlBinMatchBuffer *_mb; size_t new_offset; @@ -207,15 +292,25 @@ i_bs_skip_bits_imm2(Fail, Ms, Bits) { } i_new_bs_put_binary(Fail, Sz, Flags, Src) { + Eterm sz = $Sz; Sint _size; - $BS_GET_UNCHECKED_FIELD_SIZE($Sz, (($Flags) >> 3), $BADARG($Fail), _size); + $BS_GET_UNCHECKED_FIELD_SIZE(sz, (($Flags) >> 3), $BADARG($Fail), _size); if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2(($Src), _size))) { $BADARG($Fail); } } +i_new_bs_put_binary_all := i_new_bs_put_binary_all.fetch.execute; + +i_new_bs_put_binary_all.head() { + Eterm src; +} + +i_new_bs_put_binary_all.fetch(Src) { + src = $Src; +} -i_new_bs_put_binary_all(Fail, Src, Unit) { - if (!erts_new_bs_put_binary_all(ERL_BITS_ARGS_2(($Src), ($Unit)))) { +i_new_bs_put_binary_all.execute(Fail, Unit) { + if (!erts_new_bs_put_binary_all(ERL_BITS_ARGS_2(src, ($Unit)))) { $BADARG($Fail); } } @@ -227,9 +322,11 @@ i_new_bs_put_binary_imm(Fail, Sz, Src) { } i_new_bs_put_float(Fail, Sz, Flags, Src) { + Eterm sz = $Sz; + Eterm flags = $Flags; Sint _size; - $BS_GET_UNCHECKED_FIELD_SIZE($Sz, (($Flags) >> 3), $BADARG($Fail), _size); - if (!erts_new_bs_put_float(c_p, ($Src), _size, ($Flags))) { + $BS_GET_UNCHECKED_FIELD_SIZE(sz, (flags >> 3), $BADARG($Fail), _size); + if (!erts_new_bs_put_float(c_p, ($Src), _size, flags)) { $BADARG($Fail); } } @@ -241,15 +338,27 @@ i_new_bs_put_float_imm(Fail, Sz, Flags, Src) { } i_new_bs_put_integer(Fail, Sz, Flags, Src) { - Sint _size; - $BS_GET_UNCHECKED_FIELD_SIZE($Sz, (($Flags) >> 3), $BADARG($Fail), _size); - if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(($Src), _size, ($Flags)))) { - $BADARG($Fail); - } + Eterm sz = $Sz; + Eterm flags = $Flags; + Sint _size; + $BS_GET_UNCHECKED_FIELD_SIZE(sz, (flags >> 3), $BADARG($Fail), _size); + if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(($Src), _size, flags))) { + $BADARG($Fail); + } +} + +i_new_bs_put_integer_imm := i_new_bs_put_integer_imm.fetch.execute; + +i_new_bs_put_integer_imm.head() { + Eterm src; } -i_new_bs_put_integer_imm(Fail, Sz, Flags, Src) { - if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(($Src), ($Sz), ($Flags)))) { +i_new_bs_put_integer_imm.fetch(Src) { + src = $Src; +} + +i_new_bs_put_integer_imm.execute(Fail, Sz, Flags) { + if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(src, ($Sz), ($Flags)))) { $BADARG($Fail); } } @@ -809,9 +918,19 @@ bs_test_unit8(Fail, Ctx) { } } -i_bs_get_integer_8(Ctx, Fail, Dst) { +i_bs_get_integer_8 := i_bs_get_integer_8.fetch.execute; + +i_bs_get_integer_8.head() { + Eterm context; +} + +i_bs_get_integer_8.fetch(Ctx) { + context = $Ctx; +} + +i_bs_get_integer_8.execute(Fail, Dst) { Eterm _result; - ErlBinMatchBuffer* _mb = ms_matchbuffer($Ctx); + ErlBinMatchBuffer* _mb = ms_matchbuffer(context); if (_mb->size - _mb->offset < 8) { $FAIL($Fail); @@ -825,9 +944,19 @@ i_bs_get_integer_8(Ctx, Fail, Dst) { $Dst = _result; } -i_bs_get_integer_16(Ctx, Fail, Dst) { +i_bs_get_integer_16 := i_bs_get_integer_16.fetch.execute; + +i_bs_get_integer_16.head() { + Eterm context; +} + +i_bs_get_integer_16.fetch(Ctx) { + context = $Ctx; +} + +i_bs_get_integer_16.execute(Fail, Dst) { Eterm _result; - ErlBinMatchBuffer* _mb = ms_matchbuffer($Ctx); + ErlBinMatchBuffer* _mb = ms_matchbuffer(context); if (_mb->size - _mb->offset < 16) { $FAIL($Fail); @@ -842,9 +971,19 @@ i_bs_get_integer_16(Ctx, Fail, Dst) { } %if ARCH_64 -i_bs_get_integer_32(Ctx, Fail, Dst) { +i_bs_get_integer_32 := i_bs_get_integer_32.fetch.execute; + +i_bs_get_integer_32.head() { + Eterm context; +} + +i_bs_get_integer_32.fetch(Ctx) { + context = $Ctx; +} + +i_bs_get_integer_32.execute(Fail, Dst) { Uint32 _integer; - ErlBinMatchBuffer* _mb = ms_matchbuffer($Ctx); + ErlBinMatchBuffer* _mb = ms_matchbuffer(context); if (_mb->size - _mb->offset < 32) { $FAIL($Fail); @@ -894,15 +1033,23 @@ bs_get_integer.execute(Fail, Flags, Dst) { $Dst = result; } -i_bs_get_integer(Fail, Live, FlagsAndUnit, Ms, Sz, Dst) { +i_bs_get_integer := i_bs_get_integer.fetch.execute; + +i_bs_get_integer.head() { + Eterm context; +} + +i_bs_get_integer.fetch(Ctx) { + context = $Ctx; +} + +i_bs_get_integer.execute(Fail, Live, FlagsAndUnit, Sz, Dst) { Uint flags; Uint size; - Eterm ms; ErlBinMatchBuffer* mb; Eterm result; flags = $FlagsAndUnit; - ms = $Ms; $BS_GET_FIELD_SIZE($Sz, (flags >> 3), $FAIL($Fail), size); if (size >= SMALL_BITS) { Uint wordsneeded; @@ -913,15 +1060,15 @@ i_bs_get_integer(Fail, Live, FlagsAndUnit, Ms, Sz, Dst) { * Remember to re-acquire the matchbuffer after gc. */ - mb = ms_matchbuffer(ms); + mb = ms_matchbuffer(context); if (mb->size - mb->offset < size) { $FAIL($Fail); } wordsneeded = 1+WSIZE(NBYTES((Uint) size)); - $GC_TEST_PRESERVE(wordsneeded, $Live, ms); + $GC_TEST_PRESERVE(wordsneeded, $Live, context); $REFRESH_GEN_DEST(); } - mb = ms_matchbuffer(ms); + mb = ms_matchbuffer(context); LIGHT_SWAPOUT; result = erts_bs_get_integer_2(c_p, size, flags, mb); LIGHT_SWAPIN; @@ -932,9 +1079,19 @@ i_bs_get_integer(Fail, Live, FlagsAndUnit, Ms, Sz, Dst) { $Dst = result; } -i_bs_get_utf8(Ctx, Fail, Dst) { +i_bs_get_utf8 := i_bs_get_utf8.fetch.execute; + +i_bs_get_utf8.head() { + Eterm context; +} + +i_bs_get_utf8.fetch(Ctx) { + context = $Ctx; +} + +i_bs_get_utf8.execute(Fail, Dst) { Eterm result; - ErlBinMatchBuffer* mb = ms_matchbuffer($Ctx); + ErlBinMatchBuffer* mb = ms_matchbuffer(context); if (mb->size - mb->offset < 8) { $FAIL($Fail); @@ -957,8 +1114,18 @@ i_bs_get_utf8(Ctx, Fail, Dst) { $Dst = result; } -i_bs_get_utf16(Ctx, Fail, Flags, Dst) { - ErlBinMatchBuffer* mb = ms_matchbuffer($Ctx); +i_bs_get_utf16 := i_bs_get_utf16.fetch.execute; + +i_bs_get_utf16.head() { + Eterm context; +} + +i_bs_get_utf16.fetch(Ctx) { + context = $Ctx; +} + +i_bs_get_utf16.execute(Fail, Flags, Dst) { + ErlBinMatchBuffer* mb = ms_matchbuffer(context); Eterm result = erts_bs_get_utf16(mb, $Flags); if (is_non_value(result)) { @@ -1055,13 +1222,20 @@ i_bs_restore2(Src, Slot) { _ms->mb.offset = _ms->save_offset[$Slot]; } -bs_get_tail(Src, Dst, Live) { - ErlBinMatchBuffer* mb; - Uint size, offs; - ErlSubBin* sb; +bs_get_tail := bs_get_tail.fetch.execute; + +bs_get_tail.head() { Eterm context; +} +bs_get_tail.fetch(Src) { context = $Src; +} + +bs_get_tail.execute(Dst, Live) { + ErlBinMatchBuffer* mb; + Uint size, offs; + ErlSubBin* sb; ASSERT(header_is_bin_matchstate(*boxed_val(context))); @@ -1090,11 +1264,20 @@ bs_get_tail(Src, Dst, Live) { %if ARCH_64 -i_bs_start_match3_gp(Src, Live, Fail, Dst, Pos) { - Eterm context, header; - Uint position, live; +i_bs_start_match3_gp := i_bs_start_match3_gp.fetch.execute; +i_bs_start_match3_gp.head() { + Eterm context; +} + +i_bs_start_match3_gp.fetch(Src) { context = $Src; +} + +i_bs_start_match3_gp.execute(Live, Fail, Dst, Pos) { + Eterm header; + Uint position, live; + live = $Live; if (!is_boxed(context)) { @@ -1139,11 +1322,20 @@ i_bs_start_match3_gp(Src, Live, Fail, Dst, Pos) { $Pos = make_small(position); } -i_bs_start_match3(Src, Live, Fail, Dst) { - Eterm context, header; - Uint live; +i_bs_start_match3 := i_bs_start_match3.fetch.execute; + +i_bs_start_match3.head() { + Eterm context; +} +i_bs_start_match3.fetch(Src) { context = $Src; +} + +i_bs_start_match3.execute(Live, Fail, Dst) { + Eterm header; + Uint live; + live = $Live; if (!is_boxed(context)) { @@ -1213,12 +1405,19 @@ i_bs_get_position(Ctx, Dst) { # match at a position beyond 16MB. # -bs_set_position(Ctx, Pos) { +bs_set_position := bs_set_position.fetch.execute; + +bs_set_position.head() { Eterm context, position; - ErlBinMatchState *ms; +} +bs_set_position.fetch(Ctx, Pos) { context = $Ctx; position = $Pos; +} + +bs_set_position.execute() { + ErlBinMatchState *ms; ASSERT(header_is_bin_matchstate(*boxed_val(context))); ms = (ErlBinMatchState*)boxed_val(context); @@ -1231,12 +1430,19 @@ bs_set_position(Ctx, Pos) { } } -bs_get_position(Ctx, Dst, Live) { - ErlBinMatchState *ms; +bs_get_position := bs_get_position.fetch.execute; + +bs_get_position.head() { Eterm context; - Uint position; +} +bs_get_position.fetch(Ctx) { context = $Ctx; +} + +bs_get_position.execute(Dst, Live) { + ErlBinMatchState *ms; + Uint position; ASSERT(header_is_bin_matchstate(*boxed_val(context))); ms = (ErlBinMatchState*)boxed_val(context); @@ -1261,11 +1467,20 @@ bs_get_position(Ctx, Dst, Live) { } } -i_bs_start_match3(Src, Live, Fail, Dst) { - Eterm context, header; - Uint live; +i_bs_start_match3 := i_bs_start_match3.fetch.execute; + +i_bs_start_match3.head() { + Eterm context; +} +i_bs_start_match3.fetch(Src) { context = $Src; +} + +i_bs_start_match3.execute(Live, Fail, Dst) { + Eterm header; + Uint live; + live = $Live; if (!is_boxed(context)) { diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 74708b2caa..8c51bdb630 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -4689,6 +4689,16 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_RET(am_ok); } } + else if (ERTS_IS_ATOM_STR("mbuf", BIF_ARG_1)) { + Uint sz = size_object(BIF_ARG_2); + ErlHeapFragment* frag = new_message_buffer(sz); + Eterm *hp = frag->mem; + Eterm copy = copy_struct(BIF_ARG_2, sz, &hp, &frag->off_heap); + frag->next = BIF_P->mbuf; + BIF_P->mbuf = frag; + BIF_P->mbuf_sz += sz; + BIF_RET(copy); + } } BIF_ERROR(BIF_P, BADARG); diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 4132a54934..0a50af4d1a 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -81,14 +81,6 @@ static BIF_RETTYPE db_bif_fail(Process* p, Uint freason, /* Get a key from any table structure and a tagged object */ #define TERM_GETKEY(tb, obj) db_getkey((tb)->common.keypos, (obj)) - -/* How safe are we from double-hits or missed objects -** when iterating without fixation? */ -enum DbIterSafety { - ITER_UNSAFE, /* Must fixate to be safe */ - ITER_SAFE_LOCKED, /* Safe while table is locked, not between trap calls */ - ITER_SAFE /* No need to fixate at all */ -}; # define ITERATION_SAFETY(Proc,Tab) \ ((IS_TREE_TABLE((Tab)->common.status) || IS_CATREE_TABLE((Tab)->common.status) \ || ONLY_WRITER(Proc,Tab)) ? ITER_SAFE \ @@ -195,9 +187,6 @@ static int fixed_tabs_find(DbFixation* first, DbFixation* fix) #define ERTS_RBT_WANT_DELETE #define ERTS_RBT_WANT_FOREACH #define ERTS_RBT_WANT_FOREACH_DESTROY -#ifdef DEBUG -# define ERTS_RBT_WANT_LOOKUP -#endif #define ERTS_RBT_UNDEF #include "erl_rbtree.h" @@ -2287,6 +2276,7 @@ static BIF_RETTYPE ets_select_delete_trap_1(BIF_ALIST_1) Eterm ret; Eterm *tptr; db_lock_kind_t kind = LCK_WRITE_REC; + enum DbIterSafety safety = ITER_SAFE; CHECK_TABLES(); ASSERT(is_tuple(a1)); @@ -2296,10 +2286,11 @@ static BIF_RETTYPE ets_select_delete_trap_1(BIF_ALIST_1) DB_TRAP_GET_TABLE(tb, tptr[1], DB_WRITE, kind, &ets_select_delete_continue_exp); - cret = tb->common.meth->db_select_delete_continue(p,tb,a1,&ret); + cret = tb->common.meth->db_select_delete_continue(p,tb,a1,&ret,&safety); - if(!DID_TRAP(p,ret) && ITERATION_SAFETY(p,tb) != ITER_SAFE) { - unfix_table_locked(p, tb, &kind); + if(!DID_TRAP(p,ret) && safety != ITER_SAFE) { + ASSERT(erts_refc_read(&tb->common.fix_count,1)); + unfix_table_locked(p, tb, &kind); } db_unlock(tb, kind); @@ -2337,7 +2328,8 @@ BIF_RETTYPE ets_internal_select_delete_2(BIF_ALIST_2) if (safety == ITER_UNSAFE) { local_fix_table(tb); } - cret = tb->common.meth->db_select_delete(BIF_P, tb, BIF_ARG_1, BIF_ARG_2, &ret); + cret = tb->common.meth->db_select_delete(BIF_P, tb, BIF_ARG_1, BIF_ARG_2, + &ret, safety); if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) { fix_table_locked(BIF_P,tb); @@ -2729,7 +2721,7 @@ ets_select3(Process* p, DbTable* tb, Eterm tid, Eterm ms, Sint chunk_size) cret = tb->common.meth->db_select_chunk(p, tb, tid, ms, chunk_size, 0 /* not reversed */, - &ret); + &ret, safety); if (DID_TRAP(p,ret) && safety != ITER_SAFE) { fix_table_locked(p, tb); } @@ -2756,7 +2748,8 @@ ets_select3(Process* p, DbTable* tb, Eterm tid, Eterm ms, Sint chunk_size) } -/* We get here instead of in the real BIF when trapping */ +/* Trap here from: ets_select_1/2/3 + */ static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1) { Process *p = BIF_P; @@ -2767,6 +2760,7 @@ static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1) Eterm ret; Eterm *tptr; db_lock_kind_t kind = LCK_READ; + enum DbIterSafety safety = ITER_SAFE; CHECK_TABLES(); @@ -2776,11 +2770,13 @@ static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1) DB_TRAP_GET_TABLE(tb, tptr[1], DB_READ, kind, &ets_select_continue_exp); - cret = tb->common.meth->db_select_continue(p, tb, a1, - &ret); + cret = tb->common.meth->db_select_continue(p, tb, a1, &ret, &safety); - if (!DID_TRAP(p,ret) && ITERATION_SAFETY(p,tb) != ITER_SAFE) { - unfix_table_locked(p, tb, &kind); + if (!DID_TRAP(p,ret)) { + if (safety != ITER_SAFE) { + ASSERT(erts_refc_read(&tb->common.fix_count,1)); + unfix_table_locked(p, tb, &kind); + } } db_unlock(tb, kind); @@ -2805,8 +2801,12 @@ static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1) BIF_RETTYPE ets_select_1(BIF_ALIST_1) { return ets_select1(BIF_P, BIF_ets_select_1, BIF_ARG_1); + /* TRAP: ets_select_trap_1 */ } +/* + * Common impl for select/1, select_reverse/1, match/1 and match_object/1 + */ static BIF_RETTYPE ets_select1(Process *p, int bif_ix, Eterm arg1) { BIF_RETTYPE result; @@ -2814,7 +2814,7 @@ static BIF_RETTYPE ets_select1(Process *p, int bif_ix, Eterm arg1) int cret; Eterm ret; Eterm *tptr; - enum DbIterSafety safety; + enum DbIterSafety safety, safety_copy; CHECK_TABLES(); @@ -2839,7 +2839,8 @@ static BIF_RETTYPE ets_select1(Process *p, int bif_ix, Eterm arg1) local_fix_table(tb); } - cret = tb->common.meth->db_select_continue(p,tb, arg1, &ret); + safety_copy = safety; + cret = tb->common.meth->db_select_continue(p,tb, arg1, &ret, &safety_copy); if (DID_TRAP(p,ret) && safety != ITER_SAFE) { fix_table_locked(p, tb); @@ -2871,6 +2872,7 @@ BIF_RETTYPE ets_select_2(BIF_ALIST_2) DbTable* tb; DB_BIF_GET_TABLE(tb, DB_READ, LCK_READ, BIF_ets_select_2); return ets_select2(BIF_P, tb, BIF_ARG_1, BIF_ARG_2); + /* TRAP: ets_select_trap_1 */ } static BIF_RETTYPE @@ -2888,7 +2890,7 @@ ets_select2(Process* p, DbTable* tb, Eterm tid, Eterm ms) local_fix_table(tb); } - cret = tb->common.meth->db_select(p, tb, tid, ms, 0, &ret); + cret = tb->common.meth->db_select(p, tb, tid, ms, 0, &ret, safety); if (DID_TRAP(p,ret) && safety != ITER_SAFE) { fix_table_locked(p, tb); @@ -2926,6 +2928,7 @@ static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1) Eterm ret; Eterm *tptr; db_lock_kind_t kind = LCK_READ; + enum DbIterSafety safety = ITER_SAFE; CHECK_TABLES(); @@ -2935,9 +2938,10 @@ static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1) DB_TRAP_GET_TABLE(tb, tptr[1], DB_READ, kind, &ets_select_count_continue_exp); - cret = tb->common.meth->db_select_count_continue(p, tb, a1, &ret); + cret = tb->common.meth->db_select_count_continue(p, tb, a1, &ret, &safety); - if (!DID_TRAP(p,ret) && ITERATION_SAFETY(p,tb) != ITER_SAFE) { + if (!DID_TRAP(p,ret) && safety != ITER_SAFE) { + ASSERT(erts_refc_read(&tb->common.fix_count,1)); unfix_table_locked(p, tb, &kind); } db_unlock(tb, kind); @@ -2975,7 +2979,8 @@ BIF_RETTYPE ets_select_count_2(BIF_ALIST_2) if (safety == ITER_UNSAFE) { local_fix_table(tb); } - cret = tb->common.meth->db_select_count(BIF_P,tb, BIF_ARG_1, BIF_ARG_2, &ret); + cret = tb->common.meth->db_select_count(BIF_P,tb, BIF_ARG_1, BIF_ARG_2, + &ret, safety); if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) { fix_table_locked(BIF_P, tb); @@ -3014,6 +3019,7 @@ static BIF_RETTYPE ets_select_replace_1(BIF_ALIST_1) Eterm ret; Eterm *tptr; db_lock_kind_t kind = LCK_WRITE_REC; + enum DbIterSafety safety = ITER_SAFE; CHECK_TABLES(); ASSERT(is_tuple(a1)); @@ -3023,9 +3029,10 @@ static BIF_RETTYPE ets_select_replace_1(BIF_ALIST_1) DB_TRAP_GET_TABLE(tb, tptr[1], DB_WRITE, kind, &ets_select_replace_continue_exp); - cret = tb->common.meth->db_select_replace_continue(p,tb,a1,&ret); + cret = tb->common.meth->db_select_replace_continue(p,tb,a1,&ret,&safety); - if(!DID_TRAP(p,ret) && ITERATION_SAFETY(p,tb) != ITER_SAFE) { + if(!DID_TRAP(p,ret) && safety != ITER_SAFE) { + ASSERT(erts_refc_read(&tb->common.fix_count,1)); unfix_table_locked(p, tb, &kind); } @@ -3068,7 +3075,8 @@ BIF_RETTYPE ets_select_replace_2(BIF_ALIST_2) if (safety == ITER_UNSAFE) { local_fix_table(tb); } - cret = tb->common.meth->db_select_replace(BIF_P, tb, BIF_ARG_1, BIF_ARG_2, &ret); + cret = tb->common.meth->db_select_replace(BIF_P, tb, BIF_ARG_1, BIF_ARG_2, + &ret, safety); if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) { fix_table_locked(BIF_P,tb); @@ -3120,7 +3128,7 @@ BIF_RETTYPE ets_select_reverse_3(BIF_ALIST_3) } cret = tb->common.meth->db_select_chunk(BIF_P,tb, BIF_ARG_1, BIF_ARG_2, chunk_size, - 1 /* reversed */, &ret); + 1 /* reversed */, &ret, safety); if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) { fix_table_locked(BIF_P, tb); } @@ -3165,7 +3173,7 @@ BIF_RETTYPE ets_select_reverse_2(BIF_ALIST_2) local_fix_table(tb); } cret = tb->common.meth->db_select(BIF_P,tb, BIF_ARG_1, BIF_ARG_2, - 1 /*reversed*/, &ret); + 1 /*reversed*/, &ret, safety); if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) { fix_table_locked(BIF_P, tb); diff --git a/erts/emulator/beam/erl_db_catree.c b/erts/emulator/beam/erl_db_catree.c index 75ac1c4a93..0402c6b7b4 100644 --- a/erts/emulator/beam/erl_db_catree.c +++ b/erts/emulator/beam/erl_db_catree.c @@ -116,24 +116,31 @@ static int db_erase_object_catree(DbTable *tbl, Eterm object,Eterm *ret); static int db_slot_catree(Process *p, DbTable *tbl, Eterm slot_term, Eterm *ret); static int db_select_catree(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, int reversed, Eterm *ret); + Eterm pattern, int reversed, Eterm *ret, + enum DbIterSafety); static int db_select_count_catree(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, Eterm *ret); + Eterm pattern, Eterm *ret, enum DbIterSafety); static int db_select_chunk_catree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Sint chunk_size, - int reversed, Eterm *ret); + int reversed, Eterm *ret, enum DbIterSafety); static int db_select_continue_catree(Process *p, DbTable *tbl, - Eterm continuation, Eterm *ret); + Eterm continuation, Eterm *ret, + enum DbIterSafety*); static int db_select_count_continue_catree(Process *p, DbTable *tbl, - Eterm continuation, Eterm *ret); + Eterm continuation, Eterm *ret, + enum DbIterSafety*); static int db_select_delete_catree(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, Eterm *ret); + Eterm pattern, Eterm *ret, + enum DbIterSafety); static int db_select_delete_continue_catree(Process *p, DbTable *tbl, - Eterm continuation, Eterm *ret); + Eterm continuation, Eterm *ret, + enum DbIterSafety*); static int db_select_replace_catree(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, Eterm *ret); + Eterm pattern, Eterm *ret, + enum DbIterSafety); static int db_select_replace_continue_catree(Process *p, DbTable *tbl, - Eterm continuation, Eterm *ret); + Eterm continuation, Eterm *ret, + enum DbIterSafety*); static int db_take_catree(Process *, DbTable *, Eterm, Eterm *); static void db_print_catree(fmtfn_t to, void *to_arg, int show, DbTable *tbl); @@ -1843,7 +1850,8 @@ static int db_slot_catree(Process *p, DbTable *tbl, static int db_select_continue_catree(Process *p, DbTable *tbl, Eterm continuation, - Eterm *ret) + Eterm *ret, + enum DbIterSafety* safety_p) { int result; CATreeRootIterator iter; @@ -1856,7 +1864,8 @@ static int db_select_continue_catree(Process *p, } static int db_select_catree(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, int reverse, Eterm *ret) + Eterm pattern, int reverse, Eterm *ret, + enum DbIterSafety safety) { int result; CATreeRootIterator iter; @@ -1871,7 +1880,8 @@ static int db_select_catree(Process *p, DbTable *tbl, Eterm tid, static int db_select_count_continue_catree(Process *p, DbTable *tbl, Eterm continuation, - Eterm *ret) + Eterm *ret, + enum DbIterSafety* safety_p) { int result; CATreeRootIterator iter; @@ -1885,7 +1895,8 @@ static int db_select_count_continue_catree(Process *p, } static int db_select_count_catree(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, Eterm *ret) + Eterm pattern, Eterm *ret, + enum DbIterSafety safety) { int result; CATreeRootIterator iter; @@ -1899,7 +1910,8 @@ static int db_select_count_catree(Process *p, DbTable *tbl, Eterm tid, static int db_select_chunk_catree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Sint chunk_size, - int reversed, Eterm *ret) + int reversed, Eterm *ret, + enum DbIterSafety safety) { int result; CATreeRootIterator iter; @@ -1915,7 +1927,8 @@ static int db_select_chunk_catree(Process *p, DbTable *tbl, Eterm tid, static int db_select_delete_continue_catree(Process *p, DbTable *tbl, Eterm continuation, - Eterm *ret) + Eterm *ret, + enum DbIterSafety* safety_p) { DbTreeStack stack; TreeDbTerm * stack_array[STACK_NEED]; @@ -1931,7 +1944,8 @@ static int db_select_delete_continue_catree(Process *p, } static int db_select_delete_catree(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, Eterm *ret) + Eterm pattern, Eterm *ret, + enum DbIterSafety safety) { DbTreeStack stack; TreeDbTerm * stack_array[STACK_NEED]; @@ -1948,7 +1962,8 @@ static int db_select_delete_catree(Process *p, DbTable *tbl, Eterm tid, } static int db_select_replace_catree(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, Eterm *ret) + Eterm pattern, Eterm *ret, + enum DbIterSafety safety_p) { int result; CATreeRootIterator iter; @@ -1961,7 +1976,8 @@ static int db_select_replace_catree(Process *p, DbTable *tbl, Eterm tid, } static int db_select_replace_continue_catree(Process *p, DbTable *tbl, - Eterm continuation, Eterm *ret) + Eterm continuation, Eterm *ret, + enum DbIterSafety* safety_p) { int result; CATreeRootIterator iter; diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index f05a3b51c9..f225730029 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -404,26 +404,31 @@ static int db_slot_hash(Process *p, DbTable *tbl, static int db_select_chunk_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Sint chunk_size, - int reverse, Eterm *ret); + int reverse, Eterm *ret, enum DbIterSafety); static int db_select_hash(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, int reverse, Eterm *ret); + Eterm pattern, int reverse, Eterm *ret, + enum DbIterSafety); static int db_select_continue_hash(Process *p, DbTable *tbl, - Eterm continuation, Eterm *ret); + Eterm continuation, Eterm *ret, + enum DbIterSafety*); static int db_select_count_hash(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, Eterm *ret); + Eterm pattern, Eterm *ret, enum DbIterSafety); static int db_select_count_continue_hash(Process *p, DbTable *tbl, - Eterm continuation, Eterm *ret); - + Eterm continuation, Eterm *ret, + enum DbIterSafety*); static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, Eterm *ret); + Eterm pattern, Eterm *ret, + enum DbIterSafety); static int db_select_delete_continue_hash(Process *p, DbTable *tbl, - Eterm continuation, Eterm *ret); + Eterm continuation, Eterm *ret, + enum DbIterSafety*); static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, Eterm *ret); + Eterm pattern, Eterm *ret, enum DbIterSafety); static int db_select_replace_continue_hash(Process *p, DbTable *tbl, - Eterm continuation, Eterm *ret); + Eterm continuation, Eterm *ret, + enum DbIterSafety*); static int db_take_hash(Process *, DbTable *, Eterm, Eterm *); static void db_print_hash(fmtfn_t to, @@ -535,7 +540,7 @@ DbTableMethod db_hash = db_select_chunk_hash, db_select_hash, db_select_delete_hash, - db_select_continue_hash, /* hmm continue_hash? */ + db_select_continue_hash, db_select_delete_continue_hash, db_select_count_hash, db_select_count_continue_hash, @@ -1154,8 +1159,9 @@ static int db_slot_hash(Process *p, DbTable *tbl, Eterm slot_term, Eterm *ret) * Match traversal callbacks */ -typedef struct match_callbacks_t_ match_callbacks_t; -struct match_callbacks_t_ + +typedef struct traverse_context_t_ traverse_context_t; +struct traverse_context_t_ { /* Called when no match is possible. * context_ptr: Pointer to context @@ -1163,7 +1169,7 @@ struct match_callbacks_t_ * * Both the direct return value and 'ret' are used as the traversal function return values. */ - int (*on_nothing_can_match)(match_callbacks_t* ctx, Eterm* ret); + int (*on_nothing_can_match)(traverse_context_t* ctx, Eterm* ret); /* Called for each match result. * context_ptr: Pointer to context @@ -1174,7 +1180,7 @@ struct match_callbacks_t_ * * Should return 1 for successful match, 0 otherwise. */ - int (*on_match_res)(match_callbacks_t* ctx, Sint slot_ix, + int (*on_match_res)(traverse_context_t* ctx, Sint slot_ix, HashDbTerm*** current_ptr_ptr, Eterm match_res); /* Called when either we've matched enough elements in this cycle or EOT was reached. @@ -1188,7 +1194,7 @@ struct match_callbacks_t_ * Both the direct return value and 'ret' are used as the traversal function return values. * If *mpp is set to NULL, it won't be deallocated (useful for trapping.) */ - int (*on_loop_ended)(match_callbacks_t* ctx, Sint slot_ix, Sint got, + int (*on_loop_ended)(traverse_context_t* ctx, Sint slot_ix, Sint got, Sint iterations_left, Binary** mpp, Eterm* ret); /* Called when it's time to trap @@ -1201,16 +1207,21 @@ struct match_callbacks_t_ * Both the direct return value and 'ret' are used as the traversal function return values. * If *mpp is set to NULL, it won't be deallocated (useful for trapping.) */ - int (*on_trap)(match_callbacks_t* ctx, Sint slot_ix, Sint got, Binary** mpp, + int (*on_trap)(traverse_context_t* ctx, Sint slot_ix, Sint got, Binary** mpp, Eterm* ret); + Process* p; + DbTableHash* tb; + Eterm tid; + Eterm* prev_continuation_tptr; + enum DbIterSafety safety; }; /* * Begin hash table match traversal */ -static int match_traverse(Process* p, DbTableHash* tb, +static int match_traverse(traverse_context_t* ctx, Eterm pattern, extra_match_validator_t extra_match_validator, /* Optional */ Sint chunk_size, /* If 0, no chunking */ @@ -1218,9 +1229,9 @@ static int match_traverse(Process* p, DbTableHash* tb, Eterm** hpp, /* Heap */ int lock_for_write, /* Set to 1 if we're going to delete or modify existing terms */ - match_callbacks_t* ctx, Eterm* ret) { + DbTableHash* tb = ctx->tb; Sint slot_ix; /* Slot index */ HashDbTerm** current_ptr; /* Refers to either the bucket pointer or * the 'next' pointer in the previous term @@ -1287,7 +1298,7 @@ static int match_traverse(Process* p, DbTableHash* tb, for(;;) { if (*current_ptr != NULL) { if (!is_pseudo_deleted(*current_ptr)) { - match_res = db_match_dbterm(&tb->common, p, mpi.mp, + match_res = db_match_dbterm(&tb->common, ctx->p, mpi.mp, &(*current_ptr)->dbterm, hpp, 2); saved_current = *current_ptr; if (ctx->on_match_res(ctx, slot_ix, ¤t_ptr, match_res)) { @@ -1323,11 +1334,7 @@ static int match_traverse(Process* p, DbTableHash* tb, unlock_hash_function(lck); break; } - if (iterations_left <= 0 || MBUF(p)) { - /* - * We have either reached our limit, or just created some heap fragments. - * Since many heap fragments will make the GC slower, trap and GC now. - */ + if (iterations_left <= 0) { unlock_hash_function(lck); ret_value = ctx->on_trap(ctx, slot_ix, got, &mpi.mp, ret); goto done; @@ -1356,7 +1363,7 @@ done: /* * Continue hash table match traversal */ -static int match_traverse_continue(Process* p, DbTableHash* tb, +static int match_traverse_continue(traverse_context_t* ctx, Sint chunk_size, /* If 0, no chunking */ Sint iterations_left, /* Nr. of iterations left */ Eterm** hpp, /* Heap */ @@ -1365,9 +1372,9 @@ static int match_traverse_continue(Process* p, DbTableHash* tb, Binary** mpp, /* Existing match program */ int lock_for_write, /* Set to 1 if we're going to delete or modify existing terms */ - match_callbacks_t* ctx, Eterm* ret) { + DbTableHash* tb = ctx->tb; HashDbTerm** current_ptr; /* Refers to either the bucket pointer or * the 'next' pointer in the previous term */ @@ -1410,7 +1417,7 @@ static int match_traverse_continue(Process* p, DbTableHash* tb, for(;;) { if (*current_ptr != NULL) { if (!is_pseudo_deleted(*current_ptr)) { - match_res = db_match_dbterm(&tb->common, p, *mpp, + match_res = db_match_dbterm(&tb->common, ctx->p, *mpp, &(*current_ptr)->dbterm, hpp, 2); saved_current = *current_ptr; if (ctx->on_match_res(ctx, slot_ix, ¤t_ptr, match_res)) { @@ -1433,11 +1440,7 @@ static int match_traverse_continue(Process* p, DbTableHash* tb, unlock_hash_function(lck); break; } - if (iterations_left <= 0 || MBUF(p)) { - /* - * We have either reached our limit, or just created some heap fragments. - * Since many heap fragments will make the GC slower, trap and GC now. - */ + if (iterations_left <= 0) { unlock_hash_function(lck); ret_value = ctx->on_trap(ctx, slot_ix, got, mpp, ret); goto done; @@ -1464,52 +1467,50 @@ done: */ static ERTS_INLINE int on_simple_trap(Export* trap_function, - Process* p, - DbTableHash* tb, - Eterm tid, - Eterm* prev_continuation_tptr, - Sint slot_ix, - Sint got, - Binary** mpp, - Eterm* ret) + traverse_context_t* ctx, + Sint slot_ix, + Sint got, + Binary** mpp, + Eterm* ret) { Eterm* hp; Eterm egot; Eterm mpb; Eterm continuation; - int is_first_trap = (prev_continuation_tptr == NULL); + int is_first_trap = (ctx->prev_continuation_tptr == NULL); size_t base_halloc_sz = (is_first_trap ? ERTS_MAGIC_REF_THING_SIZE : 0); - BUMP_ALL_REDS(p); + BUMP_ALL_REDS(ctx->p); if (IS_USMALL(0, got)) { - hp = HAllocX(p, base_halloc_sz + 5, ERTS_MAGIC_REF_THING_SIZE); + hp = HAllocX(ctx->p, base_halloc_sz + 6, ERTS_MAGIC_REF_THING_SIZE); egot = make_small(got); } else { - hp = HAllocX(p, base_halloc_sz + BIG_UINT_HEAP_SIZE + 5, + hp = HAllocX(ctx->p, base_halloc_sz + BIG_UINT_HEAP_SIZE + 6, ERTS_MAGIC_REF_THING_SIZE); egot = uint_to_big(got, hp); hp += BIG_UINT_HEAP_SIZE; } if (is_first_trap) { - if (is_atom(tid)) - tid = erts_db_make_tid(p, &tb->common); - mpb = erts_db_make_match_prog_ref(p, *mpp, &hp); + if (is_atom(ctx->tid)) + ctx->tid = erts_db_make_tid(ctx->p, &ctx->tb->common); + mpb = erts_db_make_match_prog_ref(ctx->p, *mpp, &hp); *mpp = NULL; /* otherwise the caller will destroy it */ } else { - ASSERT(!is_atom(tid)); - mpb = prev_continuation_tptr[3]; + ASSERT(!is_atom(ctx->tid)); + mpb = ctx->prev_continuation_tptr[3]; } - continuation = TUPLE4( + continuation = TUPLE5( hp, - tid, + ctx->tid, make_small(slot_ix), mpb, - egot); - ERTS_BIF_PREP_TRAP1(*ret, trap_function, p, continuation); + egot, + make_small(ctx->safety)); + ERTS_BIF_PREP_TRAP1(*ret, trap_function, ctx->p, continuation); return DB_ERROR_NONE; } @@ -1518,17 +1519,18 @@ static ERTS_INLINE int unpack_simple_continuation(Eterm continuation, Eterm* tid_ptr, Sint* slot_ix_p, Binary** mpp, - Sint* got_p) + Sint* got_p, + enum DbIterSafety* safety_p) { Eterm* tptr; ASSERT(is_tuple(continuation)); tptr = tuple_val(continuation); - if (arityval(*tptr) != 4) + if (*tptr != make_arityval(5)) return 1; - if (! is_small(tptr[2]) || !(is_big(tptr[4]) || is_small(tptr[4]))) { + if (!is_small(tptr[2]) || !(is_big(tptr[4]) || is_small(tptr[4])) + || !is_small(tptr[5])) return 1; - } *tptr_ptr = tptr; *tid_ptr = tptr[1]; @@ -1540,6 +1542,7 @@ static ERTS_INLINE int unpack_simple_continuation(Eterm continuation, else { *got_p = unsigned_val(tptr[4]); } + *safety_p = signed_val(tptr[5]); return 0; } @@ -1553,24 +1556,20 @@ static ERTS_INLINE int unpack_simple_continuation(Eterm continuation, #define MAX_SELECT_CHUNK_ITERATIONS 1000 typedef struct { - match_callbacks_t base; - Process* p; - DbTableHash* tb; - Eterm tid; + traverse_context_t base; Eterm* hp; Sint chunk_size; Eterm match_list; - Eterm* prev_continuation_tptr; } select_chunk_context_t; -static int select_chunk_on_nothing_can_match(match_callbacks_t* ctx_base, Eterm* ret) +static int select_chunk_on_nothing_can_match(traverse_context_t* ctx_base, Eterm* ret) { select_chunk_context_t* ctx = (select_chunk_context_t*) ctx_base; *ret = (ctx->chunk_size > 0 ? am_EOT : NIL); return DB_ERROR_NONE; } -static int select_chunk_on_match_res(match_callbacks_t* ctx_base, Sint slot_ix, +static int select_chunk_on_match_res(traverse_context_t* ctx_base, Sint slot_ix, HashDbTerm*** current_ptr_ptr, Eterm match_res) { @@ -1582,7 +1581,7 @@ static int select_chunk_on_match_res(match_callbacks_t* ctx_base, Sint slot_ix, return 0; } -static int select_chunk_on_loop_ended(match_callbacks_t* ctx_base, +static int select_chunk_on_loop_ended(traverse_context_t* ctx_base, Sint slot_ix, Sint got, Sint iterations_left, Binary** mpp, Eterm* ret) @@ -1598,7 +1597,7 @@ static int select_chunk_on_loop_ended(match_callbacks_t* ctx_base, } else { ASSERT(iterations_left < MAX_SELECT_CHUNK_ITERATIONS); - BUMP_REDS(ctx->p, MAX_SELECT_CHUNK_ITERATIONS - iterations_left); + BUMP_REDS(ctx->base.p, MAX_SELECT_CHUNK_ITERATIONS - iterations_left); if (ctx->chunk_size) { Eterm continuation; Eterm rest = NIL; @@ -1617,14 +1616,14 @@ static int select_chunk_on_loop_ended(match_callbacks_t* ctx_base, been in 'user space' */ } if (rest != NIL || slot_ix >= 0) { /* Need more calls */ - Eterm tid = ctx->tid; - ctx->hp = HAllocX(ctx->p, + Eterm tid = ctx->base.tid; + ctx->hp = HAllocX(ctx->base.p, 3 + 7 + ERTS_MAGIC_REF_THING_SIZE, ERTS_MAGIC_REF_THING_SIZE); - mpb = erts_db_make_match_prog_ref(ctx->p, *mpp, &ctx->hp); + mpb = erts_db_make_match_prog_ref(ctx->base.p, *mpp, &ctx->hp); if (is_atom(tid)) - tid = erts_db_make_tid(ctx->p, - &ctx->tb->common); + tid = erts_db_make_tid(ctx->base.p, + &ctx->base.tb->common); continuation = TUPLE6( ctx->hp, tid, @@ -1639,7 +1638,7 @@ static int select_chunk_on_loop_ended(match_callbacks_t* ctx_base, } else { /* All data is exhausted */ if (ctx->match_list != NIL) { /* No more data to search but still a result to return to the caller */ - ctx->hp = HAlloc(ctx->p, 3); + ctx->hp = HAlloc(ctx->base.p, 3); *ret = TUPLE2(ctx->hp, ctx->match_list, am_EOT); return DB_ERROR_NONE; } else { /* Reached the end of the ttable with no data to return */ @@ -1653,7 +1652,7 @@ static int select_chunk_on_loop_ended(match_callbacks_t* ctx_base, } } -static int select_chunk_on_trap(match_callbacks_t* ctx_base, +static int select_chunk_on_trap(traverse_context_t* ctx_base, Sint slot_ix, Sint got, Binary** mpp, Eterm* ret) { @@ -1662,74 +1661,77 @@ static int select_chunk_on_trap(match_callbacks_t* ctx_base, Eterm continuation; Eterm* hp; - BUMP_ALL_REDS(ctx->p); + BUMP_ALL_REDS(ctx->base.p); - if (ctx->prev_continuation_tptr == NULL) { - Eterm tid = ctx->tid; + if (ctx->base.prev_continuation_tptr == NULL) { + Eterm tid = ctx->base.tid; /* First time we're trapping */ - hp = HAllocX(ctx->p, 7 + ERTS_MAGIC_REF_THING_SIZE, + hp = HAllocX(ctx->base.p, 8 + ERTS_MAGIC_REF_THING_SIZE, ERTS_MAGIC_REF_THING_SIZE); if (is_atom(tid)) - tid = erts_db_make_tid(ctx->p, &ctx->tb->common); - mpb = erts_db_make_match_prog_ref(ctx->p, *mpp, &hp); - continuation = TUPLE6( + tid = erts_db_make_tid(ctx->base.p, &ctx->base.tb->common); + mpb = erts_db_make_match_prog_ref(ctx->base.p, *mpp, &hp); + continuation = TUPLE7( hp, tid, make_small(slot_ix), make_small(ctx->chunk_size), mpb, ctx->match_list, - make_small(got)); + make_small(got), + make_small(ctx->base.safety)); *mpp = NULL; /* otherwise the caller will destroy it */ } else { /* Not the first time we're trapping; reuse continuation terms */ - hp = HAlloc(ctx->p, 7); - continuation = TUPLE6( + hp = HAlloc(ctx->base.p, 8); + continuation = TUPLE7( hp, - ctx->prev_continuation_tptr[1], + ctx->base.prev_continuation_tptr[1], make_small(slot_ix), - ctx->prev_continuation_tptr[3], - ctx->prev_continuation_tptr[4], + ctx->base.prev_continuation_tptr[3], + ctx->base.prev_continuation_tptr[4], ctx->match_list, - make_small(got)); + make_small(got), + make_small(ctx->base.safety)); } - ERTS_BIF_PREP_TRAP1(*ret, &ets_select_continue_exp, ctx->p, + ERTS_BIF_PREP_TRAP1(*ret, &ets_select_continue_exp, ctx->base.p, continuation); return DB_ERROR_NONE; } static int db_select_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, - int reverse, Eterm *ret) + int reverse, Eterm *ret, enum DbIterSafety safety) { - return db_select_chunk_hash(p, tbl, tid, pattern, 0, reverse, ret); + return db_select_chunk_hash(p, tbl, tid, pattern, 0, reverse, ret, safety); } static int db_select_chunk_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Sint chunk_size, - int reverse, Eterm *ret) + int reverse, Eterm *ret, enum DbIterSafety safety) { select_chunk_context_t ctx; ctx.base.on_nothing_can_match = select_chunk_on_nothing_can_match; ctx.base.on_match_res = select_chunk_on_match_res; ctx.base.on_loop_ended = select_chunk_on_loop_ended; - ctx.base.on_trap = select_chunk_on_trap, - ctx.p = p; - ctx.tb = &tbl->hash; - ctx.tid = tid; + ctx.base.on_trap = select_chunk_on_trap; + ctx.base.p = p; + ctx.base.tb = &tbl->hash; + ctx.base.tid = tid; + ctx.base.prev_continuation_tptr = NULL; + ctx.base.safety = safety; ctx.hp = NULL; ctx.chunk_size = chunk_size; ctx.match_list = NIL; - ctx.prev_continuation_tptr = NULL; return match_traverse( - ctx.p, ctx.tb, + &ctx.base, pattern, NULL, ctx.chunk_size, MAX_SELECT_CHUNK_ITERATIONS, &ctx.hp, 0, - &ctx.base, ret); + ret); } /* @@ -1739,7 +1741,7 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, Eterm tid, */ static -int select_chunk_continue_on_loop_ended(match_callbacks_t* ctx_base, +int select_chunk_continue_on_loop_ended(traverse_context_t* ctx_base, Sint slot_ix, Sint got, Sint iterations_left, Binary** mpp, Eterm* ret) @@ -1750,14 +1752,14 @@ int select_chunk_continue_on_loop_ended(match_callbacks_t* ctx_base, Eterm* hp; ASSERT(iterations_left <= MAX_SELECT_CHUNK_ITERATIONS); - BUMP_REDS(ctx->p, MAX_SELECT_CHUNK_ITERATIONS - iterations_left); + BUMP_REDS(ctx->base.p, MAX_SELECT_CHUNK_ITERATIONS - iterations_left); if (ctx->chunk_size) { Sint rest_size = 0; if (got > ctx->chunk_size) { /* Cannot write destructively here, the list may have been in user space */ - hp = HAlloc(ctx->p, (got - ctx->chunk_size) * 2); + hp = HAlloc(ctx->base.p, (got - ctx->chunk_size) * 2); while (got-- > ctx->chunk_size) { rest = CONS(hp, CAR(list_val(ctx->match_list)), rest); hp += 2; @@ -1766,13 +1768,13 @@ int select_chunk_continue_on_loop_ended(match_callbacks_t* ctx_base, } } if (rest != NIL || slot_ix >= 0) { - hp = HAlloc(ctx->p, 3 + 7); + hp = HAlloc(ctx->base.p, 3 + 7); continuation = TUPLE6( hp, - ctx->prev_continuation_tptr[1], + ctx->base.prev_continuation_tptr[1], make_small(slot_ix), - ctx->prev_continuation_tptr[3], - ctx->prev_continuation_tptr[4], + ctx->base.prev_continuation_tptr[3], + ctx->base.prev_continuation_tptr[4], rest, make_small(rest_size)); hp += 7; @@ -1780,7 +1782,7 @@ int select_chunk_continue_on_loop_ended(match_callbacks_t* ctx_base, return DB_ERROR_NONE; } else { if (ctx->match_list != NIL) { - hp = HAlloc(ctx->p, 3); + hp = HAlloc(ctx->base.p, 3); *ret = TUPLE2(hp, ctx->match_list, am_EOT); return DB_ERROR_NONE; } else { @@ -1794,10 +1796,11 @@ int select_chunk_continue_on_loop_ended(match_callbacks_t* ctx_base, } /* - * This is called when select traps + * This is called when ets:select/1/2/3 traps + * and for ets:select/1 with user continuation term. */ static int db_select_continue_hash(Process* p, DbTable* tbl, Eterm continuation, - Eterm* ret) + Eterm* ret, enum DbIterSafety* safety_p) { select_chunk_context_t ctx; Eterm* tptr; @@ -1813,7 +1816,13 @@ static int db_select_continue_hash(Process* p, DbTable* tbl, Eterm continuation, ASSERT(is_tuple(continuation)); tptr = tuple_val(continuation); - if (arityval(*tptr) != 6) + /* + * 6-tuple is select/1 user continuation term + * 7-tuple is select trap continuation + */ + if (*tptr == make_arityval(7) && is_small(tptr[7])) + *safety_p = signed_val(tptr[7]); + else if (*tptr != make_arityval(6)) goto badparam; if (!is_small(tptr[2]) || !is_small(tptr[3]) || @@ -1837,18 +1846,19 @@ static int db_select_continue_hash(Process* p, DbTable* tbl, Eterm continuation, ctx.base.on_match_res = select_chunk_on_match_res; ctx.base.on_loop_ended = select_chunk_continue_on_loop_ended; ctx.base.on_trap = select_chunk_on_trap; - ctx.p = p; - ctx.tb = &tbl->hash; - ctx.tid = tid; + ctx.base.p = p; + ctx.base.tb = &tbl->hash; + ctx.base.tid = tid; + ctx.base.prev_continuation_tptr = tptr; + ctx.base.safety = *safety_p; ctx.hp = NULL; ctx.chunk_size = chunk_size; ctx.match_list = match_list; - ctx.prev_continuation_tptr = tptr; return match_traverse_continue( - ctx.p, ctx.tb, ctx.chunk_size, - iterations_left, &ctx.hp, slot_ix, got, &mp, 0, - &ctx.base, ret); + &ctx.base, ctx.chunk_size, + iterations_left, &ctx.hp, slot_ix, got, &mp, 0, + ret); badparam: *ret = NIL; @@ -1866,84 +1876,73 @@ badparam: #define MAX_SELECT_COUNT_ITERATIONS 1000 -typedef struct { - match_callbacks_t base; - Process* p; - DbTableHash* tb; - Eterm tid; - Eterm* prev_continuation_tptr; -} select_count_context_t; - -static int select_count_on_nothing_can_match(match_callbacks_t* ctx_base, +static int select_count_on_nothing_can_match(traverse_context_t* ctx_base, Eterm* ret) { *ret = make_small(0); return DB_ERROR_NONE; } -static int select_count_on_match_res(match_callbacks_t* ctx_base, Sint slot_ix, +static int select_count_on_match_res(traverse_context_t* ctx_base, Sint slot_ix, HashDbTerm*** current_ptr_ptr, Eterm match_res) { return (match_res == am_true); } -static int select_count_on_loop_ended(match_callbacks_t* ctx_base, +static int select_count_on_loop_ended(traverse_context_t* ctx, Sint slot_ix, Sint got, Sint iterations_left, Binary** mpp, Eterm* ret) { - select_count_context_t* ctx = (select_count_context_t*) ctx_base; ASSERT(iterations_left <= MAX_SELECT_COUNT_ITERATIONS); BUMP_REDS(ctx->p, MAX_SELECT_COUNT_ITERATIONS - iterations_left); *ret = erts_make_integer(got, ctx->p); return DB_ERROR_NONE; } -static int select_count_on_trap(match_callbacks_t* ctx_base, +static int select_count_on_trap(traverse_context_t* ctx, Sint slot_ix, Sint got, Binary** mpp, Eterm* ret) { - select_count_context_t* ctx = (select_count_context_t*) ctx_base; return on_simple_trap( - &ets_select_count_continue_exp, - ctx->p, - ctx->tb, - ctx->tid, - ctx->prev_continuation_tptr, + &ets_select_count_continue_exp, ctx, slot_ix, got, mpp, ret); } static int db_select_count_hash(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, Eterm *ret) + Eterm pattern, Eterm *ret, + enum DbIterSafety safety) { - select_count_context_t ctx; + traverse_context_t ctx; Sint iterations_left = MAX_SELECT_COUNT_ITERATIONS; Sint chunk_size = 0; - ctx.base.on_nothing_can_match = select_count_on_nothing_can_match; - ctx.base.on_match_res = select_count_on_match_res; - ctx.base.on_loop_ended = select_count_on_loop_ended; - ctx.base.on_trap = select_count_on_trap; + ctx.on_nothing_can_match = select_count_on_nothing_can_match; + ctx.on_match_res = select_count_on_match_res; + ctx.on_loop_ended = select_count_on_loop_ended; + ctx.on_trap = select_count_on_trap; ctx.p = p; ctx.tb = &tbl->hash; ctx.tid = tid; ctx.prev_continuation_tptr = NULL; + ctx.safety = safety; return match_traverse( - ctx.p, ctx.tb, + &ctx, pattern, NULL, chunk_size, iterations_left, NULL, 0, - &ctx.base, ret); + ret); } /* * This is called when select_count traps */ static int db_select_count_continue_hash(Process* p, DbTable* tbl, - Eterm continuation, Eterm* ret) + Eterm continuation, Eterm* ret, + enum DbIterSafety* safety_p) { - select_count_context_t ctx; + traverse_context_t ctx; Eterm* tptr; Eterm tid; Binary* mp; @@ -1952,24 +1951,26 @@ static int db_select_count_continue_hash(Process* p, DbTable* tbl, Sint chunk_size = 0; *ret = NIL; - if (unpack_simple_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) { + if (unpack_simple_continuation(continuation, &tptr, &tid, &slot_ix, &mp, + &got, safety_p)) { *ret = NIL; return DB_ERROR_BADPARAM; } - ctx.base.on_match_res = select_count_on_match_res; - ctx.base.on_loop_ended = select_count_on_loop_ended; - ctx.base.on_trap = select_count_on_trap; + ctx.on_match_res = select_count_on_match_res; + ctx.on_loop_ended = select_count_on_loop_ended; + ctx.on_trap = select_count_on_trap; ctx.p = p; ctx.tb = &tbl->hash; ctx.tid = tid; ctx.prev_continuation_tptr = tptr; + ctx.safety = *safety_p; return match_traverse_continue( - ctx.p, ctx.tb, chunk_size, + &ctx, chunk_size, MAX_SELECT_COUNT_ITERATIONS, NULL, slot_ix, got, &mp, 0, - &ctx.base, ret); + ret); } #undef MAX_SELECT_COUNT_ITERATIONS @@ -1984,24 +1985,20 @@ static int db_select_count_continue_hash(Process* p, DbTable* tbl, #define MAX_SELECT_DELETE_ITERATIONS 1000 typedef struct { - match_callbacks_t base; - Process* p; - DbTableHash* tb; - Eterm tid; - Eterm* prev_continuation_tptr; + traverse_context_t base; erts_aint_t fixated_by_me; Uint last_pseudo_delete; HashDbTerm* free_us; } select_delete_context_t; -static int select_delete_on_nothing_can_match(match_callbacks_t* ctx_base, +static int select_delete_on_nothing_can_match(traverse_context_t* ctx_base, Eterm* ret) { *ret = make_small(0); return DB_ERROR_NONE; } -static int select_delete_on_match_res(match_callbacks_t* ctx_base, Sint slot_ix, +static int select_delete_on_match_res(traverse_context_t* ctx_base, Sint slot_ix, HashDbTerm*** current_ptr_ptr, Eterm match_res) { @@ -2011,9 +2008,9 @@ static int select_delete_on_match_res(match_callbacks_t* ctx_base, Sint slot_ix, if (match_res != am_true) return 0; - if (NFIXED(ctx->tb) > ctx->fixated_by_me) { /* fixated by others? */ + if (NFIXED(ctx->base.tb) > ctx->fixated_by_me) { /* fixated by others? */ if (slot_ix != ctx->last_pseudo_delete) { - if (!add_fixed_deletion(ctx->tb, slot_ix, ctx->fixated_by_me)) + if (!add_fixed_deletion(ctx->base.tb, slot_ix, ctx->fixated_by_me)) goto do_erase; ctx->last_pseudo_delete = slot_ix; } @@ -2026,46 +2023,43 @@ static int select_delete_on_match_res(match_callbacks_t* ctx_base, Sint slot_ix, del->next = ctx->free_us; ctx->free_us = del; } - erts_atomic_dec_nob(&ctx->tb->common.nitems); + erts_atomic_dec_nob(&ctx->base.tb->common.nitems); return 1; } -static int select_delete_on_loop_ended(match_callbacks_t* ctx_base, +static int select_delete_on_loop_ended(traverse_context_t* ctx_base, Sint slot_ix, Sint got, Sint iterations_left, Binary** mpp, Eterm* ret) { select_delete_context_t* ctx = (select_delete_context_t*) ctx_base; - free_term_list(ctx->tb, ctx->free_us); + free_term_list(ctx->base.tb, ctx->free_us); ctx->free_us = NULL; ASSERT(iterations_left <= MAX_SELECT_DELETE_ITERATIONS); - BUMP_REDS(ctx->p, MAX_SELECT_DELETE_ITERATIONS - iterations_left); + BUMP_REDS(ctx->base.p, MAX_SELECT_DELETE_ITERATIONS - iterations_left); if (got) { - try_shrink(ctx->tb); + try_shrink(ctx->base.tb); } - *ret = erts_make_integer(got, ctx->p); + *ret = erts_make_integer(got, ctx->base.p); return DB_ERROR_NONE; } -static int select_delete_on_trap(match_callbacks_t* ctx_base, +static int select_delete_on_trap(traverse_context_t* ctx_base, Sint slot_ix, Sint got, Binary** mpp, Eterm* ret) { select_delete_context_t* ctx = (select_delete_context_t*) ctx_base; - free_term_list(ctx->tb, ctx->free_us); + free_term_list(ctx->base.tb, ctx->free_us); ctx->free_us = NULL; return on_simple_trap( - &ets_select_delete_continue_exp, - ctx->p, - ctx->tb, - ctx->tid, - ctx->prev_continuation_tptr, + &ets_select_delete_continue_exp, &ctx->base, slot_ix, got, mpp, ret); } static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, Eterm *ret) + Eterm pattern, Eterm *ret, + enum DbIterSafety safety) { select_delete_context_t ctx; Sint chunk_size = 0; @@ -2074,27 +2068,29 @@ static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid, ctx.base.on_match_res = select_delete_on_match_res; ctx.base.on_loop_ended = select_delete_on_loop_ended; ctx.base.on_trap = select_delete_on_trap; - ctx.p = p; - ctx.tb = &tbl->hash; - ctx.tid = tid; - ctx.prev_continuation_tptr = NULL; - ctx.fixated_by_me = ctx.tb->common.is_thread_safe ? 0 : 1; /* TODO: something nicer */ + ctx.base.p = p; + ctx.base.tb = &tbl->hash; + ctx.base.tid = tid; + ctx.base.prev_continuation_tptr = NULL; + ctx.base.safety = safety; + ctx.fixated_by_me = ctx.base.tb->common.is_thread_safe ? 0 : 1; ctx.last_pseudo_delete = (Uint) -1; ctx.free_us = NULL; return match_traverse( - ctx.p, ctx.tb, + &ctx.base, pattern, NULL, chunk_size, MAX_SELECT_DELETE_ITERATIONS, NULL, 1, - &ctx.base, ret); + ret); } /* * This is called when select_delete traps */ static int db_select_delete_continue_hash(Process* p, DbTable* tbl, - Eterm continuation, Eterm* ret) + Eterm continuation, Eterm* ret, + enum DbIterSafety* safety_p) { select_delete_context_t ctx; Eterm* tptr; @@ -2104,7 +2100,8 @@ static int db_select_delete_continue_hash(Process* p, DbTable* tbl, Sint slot_ix; Sint chunk_size = 0; - if (unpack_simple_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) { + if (unpack_simple_continuation(continuation, &tptr, &tid, &slot_ix, &mp, + &got, safety_p)) { *ret = NIL; return DB_ERROR_BADPARAM; } @@ -2112,19 +2109,20 @@ static int db_select_delete_continue_hash(Process* p, DbTable* tbl, ctx.base.on_match_res = select_delete_on_match_res; ctx.base.on_loop_ended = select_delete_on_loop_ended; ctx.base.on_trap = select_delete_on_trap; - ctx.p = p; - ctx.tb = &tbl->hash; - ctx.tid = tid; - ctx.prev_continuation_tptr = tptr; - ctx.fixated_by_me = ONLY_WRITER(p, ctx.tb) ? 0 : 1; /* TODO: something nicer */ + ctx.base.p = p; + ctx.base.tb = &tbl->hash; + ctx.base.tid = tid; + ctx.base.prev_continuation_tptr = tptr; + ctx.base.safety = *safety_p; + ctx.fixated_by_me = ONLY_WRITER(p, ctx.base.tb) ? 0 : 1; ctx.last_pseudo_delete = (Uint) -1; ctx.free_us = NULL; return match_traverse_continue( - ctx.p, ctx.tb, chunk_size, + &ctx.base, chunk_size, MAX_SELECT_DELETE_ITERATIONS, NULL, slot_ix, got, &mp, 1, - &ctx.base, ret); + ret); } #undef MAX_SELECT_DELETE_ITERATIONS @@ -2138,26 +2136,17 @@ static int db_select_delete_continue_hash(Process* p, DbTable* tbl, #define MAX_SELECT_REPLACE_ITERATIONS 1000 -typedef struct { - match_callbacks_t base; - Process* p; - DbTableHash* tb; - Eterm tid; - Eterm* prev_continuation_tptr; -} select_replace_context_t; - -static int select_replace_on_nothing_can_match(match_callbacks_t* ctx_base, +static int select_replace_on_nothing_can_match(traverse_context_t* ctx_base, Eterm* ret) { *ret = make_small(0); return DB_ERROR_NONE; } -static int select_replace_on_match_res(match_callbacks_t* ctx_base, Sint slot_ix, +static int select_replace_on_match_res(traverse_context_t* ctx, Sint slot_ix, HashDbTerm*** current_ptr_ptr, Eterm match_res) { - select_replace_context_t* ctx = (select_replace_context_t*) ctx_base; DbTableHash* tb = ctx->tb; HashDbTerm* new; HashDbTerm* next; @@ -2183,11 +2172,10 @@ static int select_replace_on_match_res(match_callbacks_t* ctx_base, Sint slot_ix return 0; } -static int select_replace_on_loop_ended(match_callbacks_t* ctx_base, Sint slot_ix, +static int select_replace_on_loop_ended(traverse_context_t* ctx, Sint slot_ix, Sint got, Sint iterations_left, Binary** mpp, Eterm* ret) { - select_replace_context_t* ctx = (select_replace_context_t*) ctx_base; ASSERT(iterations_left <= MAX_SELECT_REPLACE_ITERATIONS); /* the more objects we've replaced, the more reductions we've consumed */ BUMP_REDS(ctx->p, @@ -2197,23 +2185,20 @@ static int select_replace_on_loop_ended(match_callbacks_t* ctx_base, Sint slot_i return DB_ERROR_NONE; } -static int select_replace_on_trap(match_callbacks_t* ctx_base, +static int select_replace_on_trap(traverse_context_t* ctx, Sint slot_ix, Sint got, Binary** mpp, Eterm* ret) { - select_replace_context_t* ctx = (select_replace_context_t*) ctx_base; return on_simple_trap( - &ets_select_replace_continue_exp, - ctx->p, - ctx->tb, - ctx->tid, - ctx->prev_continuation_tptr, + &ets_select_replace_continue_exp, ctx, slot_ix, got, mpp, ret); } -static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret) +static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm tid, + Eterm pattern, Eterm *ret, + enum DbIterSafety safety) { - select_replace_context_t ctx; + traverse_context_t ctx; Sint chunk_size = 0; /* Bag implementation presented both semantic consistency and performance issues, @@ -2221,29 +2206,32 @@ static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pat */ ASSERT(!(tbl->hash.common.status & DB_BAG)); - ctx.base.on_nothing_can_match = select_replace_on_nothing_can_match; - ctx.base.on_match_res = select_replace_on_match_res; - ctx.base.on_loop_ended = select_replace_on_loop_ended; - ctx.base.on_trap = select_replace_on_trap; + ctx.on_nothing_can_match = select_replace_on_nothing_can_match; + ctx.on_match_res = select_replace_on_match_res; + ctx.on_loop_ended = select_replace_on_loop_ended; + ctx.on_trap = select_replace_on_trap; ctx.p = p; ctx.tb = &tbl->hash; ctx.tid = tid; ctx.prev_continuation_tptr = NULL; + ctx.safety = safety; return match_traverse( - ctx.p, ctx.tb, + &ctx, pattern, db_match_keeps_key, chunk_size, MAX_SELECT_REPLACE_ITERATIONS, NULL, 1, - &ctx.base, ret); + ret); } /* * This is called when select_replace traps */ -static int db_select_replace_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret) +static int db_select_replace_continue_hash(Process* p, DbTable* tbl, + Eterm continuation, Eterm* ret, + enum DbIterSafety* safety_p) { - select_replace_context_t ctx; + traverse_context_t ctx; Eterm* tptr; Eterm tid ; Binary* mp; @@ -2252,25 +2240,27 @@ static int db_select_replace_continue_hash(Process* p, DbTable* tbl, Eterm conti Sint chunk_size = 0; *ret = NIL; - if (unpack_simple_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) { + if (unpack_simple_continuation(continuation, &tptr, &tid, &slot_ix, &mp, + &got, safety_p)) { *ret = NIL; return DB_ERROR_BADPARAM; } /* Proceed */ - ctx.base.on_match_res = select_replace_on_match_res; - ctx.base.on_loop_ended = select_replace_on_loop_ended; - ctx.base.on_trap = select_replace_on_trap; + ctx.on_match_res = select_replace_on_match_res; + ctx.on_loop_ended = select_replace_on_loop_ended; + ctx.on_trap = select_replace_on_trap; ctx.p = p; ctx.tb = &tbl->hash; ctx.tid = tid; ctx.prev_continuation_tptr = tptr; + ctx.safety = *safety_p; return match_traverse_continue( - ctx.p, ctx.tb, chunk_size, + &ctx, chunk_size, MAX_SELECT_REPLACE_ITERATIONS, NULL, slot_ix, got, &mp, 1, - &ctx.base, ret); + ret); } diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 02a5934a6e..f9ba04f399 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -397,24 +397,31 @@ static int db_erase_object_tree(DbTable *tbl, Eterm object,Eterm *ret); static int db_slot_tree(Process *p, DbTable *tbl, Eterm slot_term, Eterm *ret); static int db_select_tree(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, int reversed, Eterm *ret); + Eterm pattern, int reversed, Eterm *ret, + enum DbIterSafety); static int db_select_count_tree(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, Eterm *ret); + Eterm pattern, Eterm *ret, enum DbIterSafety); static int db_select_chunk_tree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Sint chunk_size, - int reversed, Eterm *ret); + int reversed, Eterm *ret, enum DbIterSafety); static int db_select_continue_tree(Process *p, DbTable *tbl, - Eterm continuation, Eterm *ret); + Eterm continuation, Eterm *ret, + enum DbIterSafety*); static int db_select_count_continue_tree(Process *p, DbTable *tbl, - Eterm continuation, Eterm *ret); + Eterm continuation, Eterm *ret, + enum DbIterSafety*); static int db_select_delete_tree(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, Eterm *ret); + Eterm pattern, Eterm *ret, + enum DbIterSafety); static int db_select_delete_continue_tree(Process *p, DbTable *tbl, - Eterm continuation, Eterm *ret); + Eterm continuation, Eterm *ret, + enum DbIterSafety*); static int db_select_replace_tree(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, Eterm *ret); + Eterm pattern, Eterm *ret, + enum DbIterSafety); static int db_select_replace_continue_tree(Process *p, DbTable *tbl, - Eterm continuation, Eterm *ret); + Eterm continuation, Eterm *ret, + enum DbIterSafety*); static int db_take_tree(Process *, DbTable *, Eterm, Eterm *); static void db_print_tree(fmtfn_t to, void *to_arg, int show, DbTable *tbl); @@ -1160,7 +1167,8 @@ int db_select_continue_tree_common(Process *p, static int db_select_continue_tree(Process *p, DbTable *tbl, Eterm continuation, - Eterm *ret) + Eterm *ret, + enum DbIterSafety* safety_p) { DbTableTree *tb = &tbl->tree; return db_select_continue_tree_common(p, &tb->common, @@ -1297,7 +1305,8 @@ int db_select_tree_common(Process *p, DbTable *tb, } static int db_select_tree(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, int reverse, Eterm *ret) + Eterm pattern, int reverse, Eterm *ret, + enum DbIterSafety safety) { return db_select_tree_common(p, tbl, tid, pattern, reverse, ret, &tbl->tree, NULL); @@ -1408,7 +1417,8 @@ int db_select_count_continue_tree_common(Process *p, static int db_select_count_continue_tree(Process *p, DbTable *tbl, Eterm continuation, - Eterm *ret) + Eterm *ret, + enum DbIterSafety* safety_p) { DbTableTree *tb = &tbl->tree; return db_select_count_continue_tree_common(p, tbl, @@ -1527,7 +1537,8 @@ int db_select_count_tree_common(Process *p, DbTable *tb, } static int db_select_count_tree(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, Eterm *ret) + Eterm pattern, Eterm *ret, + enum DbIterSafety safety) { DbTableTree *tb = &tbl->tree; return db_select_count_tree_common(p, tbl, @@ -1704,7 +1715,7 @@ int db_select_chunk_tree_common(Process *p, DbTable *tb, static int db_select_chunk_tree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Sint chunk_size, int reverse, - Eterm *ret) + Eterm *ret, enum DbIterSafety safety) { DbTableTree *tb = &tbl->tree; return db_select_chunk_tree_common(p, tbl, @@ -1813,7 +1824,8 @@ int db_select_delete_continue_tree_common(Process *p, static int db_select_delete_continue_tree(Process *p, DbTable *tbl, Eterm continuation, - Eterm *ret) + Eterm *ret, + enum DbIterSafety* safety_p) { DbTableTree *tb = &tbl->tree; ASSERT(!erts_atomic_read_nob(&tb->is_stack_busy)); @@ -1942,7 +1954,8 @@ int db_select_delete_tree_common(Process *p, DbTable *tbl, } static int db_select_delete_tree(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, Eterm *ret) + Eterm pattern, Eterm *ret, + enum DbIterSafety safety) { DbTableTree *tb = &tbl->tree; return db_select_delete_tree_common(p, tbl, tid, pattern, ret, @@ -2052,7 +2065,8 @@ int db_select_replace_continue_tree_common(Process *p, static int db_select_replace_continue_tree(Process *p, DbTable *tbl, Eterm continuation, - Eterm *ret) + Eterm *ret, + enum DbIterSafety* safety_p) { return db_select_replace_continue_tree_common(p, tbl, continuation, ret, &tbl->tree, NULL); @@ -2177,7 +2191,8 @@ int db_select_replace_tree_common(Process *p, DbTable *tbl, } static int db_select_replace_tree(Process *p, DbTable *tbl, Eterm tid, - Eterm pattern, Eterm *ret) + Eterm pattern, Eterm *ret, + enum DbIterSafety safety) { return db_select_replace_tree_common(p, tbl, tid, pattern, ret, &tbl->tree, NULL); @@ -3743,13 +3758,6 @@ static int doit_select(DbTableCommon *tb, TreeDbTerm *this, if (is_value(ret)) { sc->accum = CONS(hp, ret, sc->accum); } - if (MBUF(sc->p)) { - /* - * Force a trap and GC if a heap fragment was created. Many heap fragments - * make the GC slow. - */ - sc->max = 0; - } if (--(sc->max) <= 0) { return 0; } @@ -3806,13 +3814,6 @@ static int doit_select_chunk(DbTableCommon *tb, TreeDbTerm *this, ++(sc->got); sc->accum = CONS(hp, ret, sc->accum); } - if (MBUF(sc->p)) { - /* - * Force a trap and GC if a heap fragment was created. Many heap fragments - * make the GC slow. - */ - sc->max = 0; - } if (--(sc->max) <= 0 || sc->got == sc->chunk_size) { return 0; } diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index e1af9210ea..e3d3c0e804 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -101,6 +101,14 @@ typedef struct { } u; } DbUpdateHandle; +/* How safe are we from double-hits or missed objects + * when iterating without fixation? + */ +enum DbIterSafety { + ITER_UNSAFE, /* Must fixate to be safe */ + ITER_SAFE_LOCKED, /* Safe while table is locked, not between trap calls */ + ITER_SAFE /* No need to fixate at all */ +}; typedef struct db_table_method { @@ -150,44 +158,53 @@ typedef struct db_table_method Eterm pattern, Sint chunk_size, int reverse, - Eterm* ret); + Eterm* ret, + enum DbIterSafety); int (*db_select)(Process* p, DbTable* tb, /* [in out] */ Eterm tid, Eterm pattern, int reverse, - Eterm* ret); + Eterm* ret, + enum DbIterSafety); int (*db_select_delete)(Process* p, DbTable* tb, /* [in out] */ Eterm tid, Eterm pattern, - Eterm* ret); + Eterm* ret, + enum DbIterSafety); int (*db_select_continue)(Process* p, DbTable* tb, /* [in out] */ Eterm continuation, - Eterm* ret); + Eterm* ret, + enum DbIterSafety*); int (*db_select_delete_continue)(Process* p, DbTable* tb, /* [in out] */ Eterm continuation, - Eterm* ret); + Eterm* ret, + enum DbIterSafety*); int (*db_select_count)(Process* p, DbTable* tb, /* [in out] */ Eterm tid, Eterm pattern, - Eterm* ret); + Eterm* ret, + enum DbIterSafety); int (*db_select_count_continue)(Process* p, DbTable* tb, /* [in out] */ Eterm continuation, - Eterm* ret); + Eterm* ret, + enum DbIterSafety*); int (*db_select_replace)(Process* p, DbTable* tb, /* [in out] */ Eterm tid, Eterm pattern, - Eterm* ret); + Eterm* ret, + enum DbIterSafety); int (*db_select_replace_continue)(Process* p, DbTable* tb, /* [in out] */ Eterm continuation, - Eterm* ret); + Eterm* ret, + enum DbIterSafety*); int (*db_take)(Process *, DbTable *, Eterm, Eterm *); SWord (*db_delete_all_objects)(Process* p, DbTable* db, SWord reds); diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 9d40754d2d..e350a20339 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -776,6 +776,13 @@ erts_send_message(Process* sender, #endif erts_queue_proc_message(sender, receiver, *receiver_locks, mp, message); + + if (msize > ERTS_MSG_COPY_WORDS_PER_REDUCTION) { + Uint reds = msize / ERTS_MSG_COPY_WORDS_PER_REDUCTION; + if (reds > CONTEXT_REDS) + reds = CONTEXT_REDS; + BUMP_REDS(sender, (int) reds); + } } diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 4c2674394e..e5f623a370 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -26,6 +26,12 @@ #include "erl_proc_sig_queue.h" #undef ERTS_PROC_SIG_QUEUE_TYPE_ONLY +#ifdef DEBUG +#define ERTS_MSG_COPY_WORDS_PER_REDUCTION 4 +#else +#define ERTS_MSG_COPY_WORDS_PER_REDUCTION 64 +#endif + struct proc_bin; struct external_thing_; diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 349d9bf13a..af1acbfc90 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -731,6 +731,23 @@ ErtsMessage* erts_create_message_from_nif_env(ErlNifEnv* msg_env) return mp; } +static ERTS_INLINE ERL_NIF_TERM make_copy(ErlNifEnv* dst_env, + ERL_NIF_TERM src_term, + Uint *cpy_szp) +{ + Uint sz; + Eterm* hp; + /* + * No preserved sharing allowed as long as literals are also preserved. + * Process independent environment can not be reached by purge. + */ + sz = size_object(src_term); + if (cpy_szp) + *cpy_szp += sz; + hp = alloc_heap(dst_env, sz); + return copy_struct(src_term, sz, &hp, &MSO(dst_env->proc)); +} + int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg) { @@ -743,6 +760,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, Eterm from; Eterm receiver = to_pid->pid; int scheduler; + Uint copy_sz = 0; execution_state(env, &c_p, &scheduler); @@ -806,14 +824,14 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, stoken = NIL; } #endif - token = enif_make_copy(msg_env, stoken); + token = make_copy(msg_env, stoken, ©_sz); #ifdef USE_VM_PROBES if (DT_UTAG_FLAGS(c_p) & DT_UTAG_SPREADING) { if (is_immed(DT_UTAG(c_p))) utag = DT_UTAG(c_p); else - utag = enif_make_copy(msg_env, DT_UTAG(c_p)); + utag = make_copy(msg_env, DT_UTAG(c_p), ©_sz); } if (DTRACE_ENABLED(message_send)) { if (have_seqtrace(stoken)) { @@ -835,6 +853,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, Uint sz; INITIALIZE_LITERAL_PURGE_AREA(litarea); sz = size_object_litopt(msg, &litarea); + copy_sz += sz; if (c_p && !env->tracee) { full_flush_env(env); mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); @@ -867,6 +886,12 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, trace_send(c_p, receiver, msg); full_cache_env(env); } + if (c_p && scheduler > 0 && copy_sz > ERTS_MSG_COPY_WORDS_PER_REDUCTION) { + Uint reds = copy_sz / ERTS_MSG_COPY_WORDS_PER_REDUCTION; + if (reds > CONTEXT_REDS) + reds = CONTEXT_REDS; + BUMP_REDS(c_p, (int) reds); + } } else { /* This clause is taken when the nif is called in the context @@ -935,6 +960,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, erts_queue_message(rp, rp_locks, mp, msg, from); done: + if (c_p == rp) rp_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp_locks & ~lc_locks) @@ -1047,18 +1073,9 @@ int enif_whereis_port(ErlNifEnv *env, ERL_NIF_TERM name, ErlNifPort *port) ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term) { - Uint sz; - Eterm* hp; - /* - * No preserved sharing allowed as long as literals are also preserved. - * Process independent environment cannot be reached by purge. - */ - sz = size_object(src_term); - hp = alloc_heap(dst_env, sz); - return copy_struct(src_term, sz, &hp, &MSO(dst_env->proc)); + return make_copy(dst_env, src_term, NULL); } - #ifdef DEBUG static int is_offheap(const ErlOffHeap* oh) { @@ -1158,6 +1175,47 @@ int enif_is_number(ErlNifEnv* env, ERL_NIF_TERM term) return is_number(term); } +ErlNifTermType enif_term_type(ErlNifEnv* env, ERL_NIF_TERM term) { + (void)env; + + switch (tag_val_def(term)) { + case ATOM_DEF: + return ERL_NIF_TERM_TYPE_ATOM; + case BINARY_DEF: + return ERL_NIF_TERM_TYPE_BITSTRING; + case FLOAT_DEF: + return ERL_NIF_TERM_TYPE_FLOAT; + case EXPORT_DEF: + case FUN_DEF: + return ERL_NIF_TERM_TYPE_FUN; + case BIG_DEF: + case SMALL_DEF: + return ERL_NIF_TERM_TYPE_INTEGER; + case LIST_DEF: + case NIL_DEF: + return ERL_NIF_TERM_TYPE_LIST; + case MAP_DEF: + return ERL_NIF_TERM_TYPE_MAP; + case EXTERNAL_PID_DEF: + case PID_DEF: + return ERL_NIF_TERM_TYPE_PID; + case EXTERNAL_PORT_DEF: + case PORT_DEF: + return ERL_NIF_TERM_TYPE_PORT; + case EXTERNAL_REF_DEF: + case REF_DEF: + return ERL_NIF_TERM_TYPE_REFERENCE; + case TUPLE_DEF: + return ERL_NIF_TERM_TYPE_TUPLE; + default: + /* tag_val_def() aborts on its own when passed complete garbage, but + * it's possible that the user has given us garbage that just happens + * to match something that tag_val_def() accepts but we don't, like + * binary match contexts. */ + ERTS_INTERNAL_ERROR("Invalid term passed to enif_term_type"); + } +} + static void aligned_binary_dtor(struct enif_tmp_obj_t* obj) { erts_free_aligned_binary_bytes_extra((byte*)obj, obj->allocator); diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 3fd1a8fd4c..a599511c78 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -55,6 +55,7 @@ ** 2.14: 21.0 add enif_ioq_peek_head, enif_(mutex|cond|rwlock|thread)_name ** enif_vfprintf, enif_vsnprintf, enif_make_map_from_arrays ** 2.15: 22.0 ERL_NIF_SELECT_CANCEL, enif_select_(read|write) +** enif_term_type */ #define ERL_NIF_MAJOR_VERSION 2 #define ERL_NIF_MINOR_VERSION 15 @@ -63,7 +64,7 @@ * with ticket syntax like "erts-@OTP-12345@", or a temporary placeholder * between two @ like "erts-@MyName@", if you don't know what a ticket is. */ -#define ERL_NIF_MIN_ERTS_VERSION "erts-@OTP-15095@ (OTP-22)" +#define ERL_NIF_MIN_ERTS_VERSION "erts-@OTP-15095 OTP-15640@ (OTP-22)" /* * The emulator will refuse to load a nif-lib with a major version @@ -282,6 +283,26 @@ typedef enum { ERL_NIF_IOQ_NORMAL = 1 } ErlNifIOQueueOpts; +typedef enum { + ERL_NIF_TERM_TYPE_ATOM = 1, + ERL_NIF_TERM_TYPE_BITSTRING = 2, + ERL_NIF_TERM_TYPE_FLOAT = 3, + ERL_NIF_TERM_TYPE_FUN = 4, + ERL_NIF_TERM_TYPE_INTEGER = 5, + ERL_NIF_TERM_TYPE_LIST = 6, + ERL_NIF_TERM_TYPE_MAP = 7, + ERL_NIF_TERM_TYPE_PID = 8, + ERL_NIF_TERM_TYPE_PORT = 9, + ERL_NIF_TERM_TYPE_REFERENCE = 10, + ERL_NIF_TERM_TYPE_TUPLE = 11, + + /* This is a dummy value intended to coax the compiler into warning about + * unhandled values in a switch even if all the above values have been + * handled. We can add new entries at any time so the user must always + * have a default case. */ + ERL_NIF_TERM_TYPE__MISSING_DEFAULT_CASE__READ_THE_MANUAL = -1 +} ErlNifTermType; + /* * Return values from enif_thread_type(). Negative values * reserved for specific types of non-scheduler threads. diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 8ab454c8dd..d57f6ec97c 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -215,6 +215,8 @@ ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_monitor_term,(ErlNifEnv* env, const ERL_NIF_API_FUNC_DECL(void,enif_set_pid_undefined,(ErlNifPid* pid)); ERL_NIF_API_FUNC_DECL(int,enif_is_pid_undefined,(const ErlNifPid* pid)); +ERL_NIF_API_FUNC_DECL(ErlNifTermType,enif_term_type,(ErlNifEnv* env, ERL_NIF_TERM term)); + /* ** ADD NEW ENTRIES HERE (before this comment) !!! */ @@ -401,6 +403,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_pid_undefined,(const ErlNifPid* pid)); # define enif_make_monitor_term ERL_NIF_API_FUNC_MACRO(enif_make_monitor_term) # define enif_set_pid_undefined ERL_NIF_API_FUNC_MACRO(enif_set_pid_undefined) # define enif_is_pid_undefined ERL_NIF_API_FUNC_MACRO(enif_is_pid_undefined) +# define enif_term_type ERL_NIF_API_FUNC_MACRO(enif_term_type) /* ** ADD NEW ENTRIES HERE (before this comment) diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab index fc88cab22f..1eb83b61f2 100644 --- a/erts/emulator/beam/instrs.tab +++ b/erts/emulator/beam/instrs.tab @@ -512,6 +512,15 @@ move_shift(Src, SD, D) { $SD = V; } +move_window2(S1, S2, D) { + Eterm xt0, xt1; + Eterm* y = &$D; + xt0 = $S1; + xt1 = $S2; + y[0] = xt0; + y[1] = xt1; +} + move_window3(S1, S2, S3, D) { Eterm xt0, xt1, xt2; Eterm* y = &$D; @@ -875,12 +884,16 @@ i_is_ne_exact_literal(Fail, Src, Literal) { } } -is_eq(Fail, X, Y) { - CMP_EQ_ACTION($X, $Y, $FAIL($Fail)); +is_eq(Fail, A, B) { + Eterm a = $A; + Eterm b = $B; + CMP_EQ_ACTION(a, b, $FAIL($Fail)); } -is_ne(Fail, X, Y) { - CMP_NE_ACTION($X, $Y, $FAIL($Fail)); +is_ne(Fail, A, B) { + Eterm a = $A; + Eterm b = $B; + CMP_NE_ACTION(a, b, $FAIL($Fail)); } is_lt(Fail, X, Y) { diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 3cfc685336..da5364183c 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -74,23 +74,19 @@ trace_jump W return +# To ensure that a "move Src x(0)" instruction can be combined with +# the following call instruction, we need to make sure that there is +# no line/1 instruction between the move and the call. # -# To ensure that a "move Src x(0)" instruction can be combined -# with the following call instruction, we need to make sure that -# there is no line/1 instruction between the move and the call. -# -# A tail-recursive call to an external function (non-BIF) will -# never be saved on the stack, so there is no reason to keep -# the line instruction. (The compiler did not remove the line -# instruction because it cannot tell the difference between -# BIFs and ordinary Erlang functions.) -# +# A tail-recursive call to an external function (BIF or non-BIF) will +# never be saved on the stack, so there is no reason to keep the line +# instruction. move S X0=x==0 | line Loc | call_ext Ar Func => \ line Loc | move S X0 | call_ext Ar Func -move S X0=x==0 | line Loc | call_ext_last Ar Func=u$is_not_bif D => \ +move S X0=x==0 | line Loc | call_ext_last Ar Func D => \ move S X0 | call_ext_last Ar Func D -move S X0=x==0 | line Loc | call_ext_only Ar Func=u$is_not_bif => \ +move S X0=x==0 | line Loc | call_ext_only Ar Func => \ move S X0 | call_ext_only Ar Func move S X0=x==0 | line Loc | call Ar Func => \ line Loc | move S X0 | call Ar Func @@ -102,9 +98,9 @@ line I allocate t t? allocate_heap t I t? -%cold +# This instruction when a BIF is called tail-recursively when +# ther is stack frame. deallocate Q -%hot init y allocate_zero t t? @@ -215,11 +211,7 @@ set_tuple_element s S P # Get tuple element -i_get_tuple_element xy P x - -%cold -i_get_tuple_element xy P y -%hot +i_get_tuple_element xy P xy i_get_tuple_element2 x P x i_get_tuple_element2_dst x P x x @@ -285,6 +277,9 @@ move_window/6 move X1=x Y1=y | move X2=x Y2=y | move X3=x Y3=y | succ(Y1,Y2) | succ(Y2,Y3) => \ move_window X1 X2 X3 Y1 Y3 +move X1=x Y1=y | move X2=x Y2=y | succ(Y1,Y2) => \ + move_window2 X1 X2 Y1 + move_window X1=x X2=x X3=x Y1=y Y3=y | move X4=x Y4=y | succ(Y3,Y4) => \ move_window X1 X2 X3 X4 Y1 Y4 @@ -294,6 +289,7 @@ move_window X1=x X2=x X3=x X4=x Y1=y Y4=y | move X5=x Y5=y | succ(Y4,Y5) => \ move_window X1=x X2=x X3=x Y1=y Y3=y => move_window3 X1 X2 X3 Y1 move_window X1=x X2=x X3=x X4=x Y1=y Y4=y => move_window4 X1 X2 X3 X4 Y1 +move_window2 x x y move_window3 x x x y move_window4 x x x x y move_window5 x x x x x y @@ -346,11 +342,6 @@ move X1=x X2=x | move X3=x X4=x | independent_moves(X1, X2, X3, X4) => \ move2_par X1 X2 X3 X4 move2_par x x x x -# move2_par x y x y - -move X1=x Y1=y | move X2=x Y2=y => move2_par X1 Y1 X2 Y2 -move2_par x y x y - # move2_par x x x y move X1=x X2=x | move X3=x Y1=y | independent_moves(X1, X2, X3, Y1) => \ @@ -393,20 +384,14 @@ move3 x x x x x x move C=aiq X=x==1 => move_x1 C move C=aiq X=x==2 => move_x2 C +move n D=y => init D + move_x1 c move_x2 c -# The compiler almost never generates a "move Literal y(Y)" instruction, -# so let's cheat if we encounter one. -move S=n D=y => init D -move S=c D=y => move S x | move x D - -move x x -move x y -move y x -move c x +move xy xy +move c xy move n x -move y y # The following move instructions using x(0) are frequently used. @@ -500,9 +485,13 @@ is_ge f? c x is_ge f? s s %hot -is_eq f? s s +is_eq Fail=f Const=c Reg=xy => is_eq Fail Reg Const +is_eq Fail=f C1=c C2=c => move C1 x | is_eq Fail x C2 +is_eq f? S s -is_ne f? s s +is_ne Fail=f Const=c Reg=xy => is_ne Fail Reg Const +is_ne Fail=f C1=c C2=c => move C1 x | is_ne Fail x C2 +is_ne f? S s # # Putting tuples. @@ -992,10 +981,9 @@ call_ext_only u==0 u$func:os:perf_counter/0 => \ call_ext u Bif=u$is_bif => call_bif Bif -call_ext_last u Bif=u$is_bif D => call_bif Bif | deallocate_return D +call_ext_last u Bif=u$is_bif D => deallocate D | call_bif_only Bif -call_ext_only Ar=u Bif=u$is_bif => \ - allocate u Ar | call_bif Bif | deallocate_return u +call_ext_only Ar=u Bif=u$is_bif => call_bif_only Bif # # Any remaining calls are calls to Erlang functions, not BIFs. @@ -1027,6 +1015,7 @@ i_perf_counter %hot call_bif e +call_bif_only e # # Calls to non-building and guard BIFs. @@ -1035,6 +1024,9 @@ call_bif e bif0 u$bif:erlang:self/0 Dst=d => self Dst bif0 u$bif:erlang:node/0 Dst=d => node Dst +bif1 Fail=f Bif=u$bif:erlang:hd/1 Src=x Dst=x => is_nonempty_list_get_hd Fail Src Dst +bif1 Fail=f Bif=u$bif:erlang:tl/1 Src=x Dst=x => is_nonempty_list_get_tl Fail Src Dst + bif1 Fail Bif=u$bif:erlang:get/1 Src=s Dst=d => gen_get(Src, Dst) bif2 Jump=j u$bif:erlang:element/2 S1=s S2=xy Dst=d => gen_element(Jump, S1, S2, Dst) @@ -1123,19 +1115,33 @@ is_function Fail=f c => jump Fail func_info M F A => i_func_info u M F A # ================================================================ -# New bit syntax matching (R11B). +# Bit syntax matching obsoleted in OTP 22. # ================================================================ -%warm +%cold bs_start_match2 Fail=f ica X Y D => jump Fail bs_start_match2 Fail Bin X Y D => i_bs_start_match2 Bin Fail X Y D i_bs_start_match2 xy f t t d +bs_save2 Y=y Index => move Y x | bs_save2 x Index bs_save2 Reg Index => gen_bs_save(Reg, Index) -i_bs_save2 xy t +i_bs_save2 x t +bs_restore2 Y=y Index => move Y x | bs_restore2 x Index bs_restore2 Reg Index => gen_bs_restore(Reg, Index) -i_bs_restore2 xy t +i_bs_restore2 x t + +bs_context_to_binary Y=y | line L | badmatch Y => \ + move Y x | bs_context_to_binary x | line L | badmatch x +bs_context_to_binary Y=y => move Y x | bs_context_to_binary x +bs_context_to_binary x +%warm + +# ================================================================ +# New bit syntax matching (R11B). +# ================================================================ + +%warm # Matching integers bs_match_string Fail Ms Bits Val => i_bs_match_string Ms Fail Bits Val @@ -1154,7 +1160,7 @@ i_bs_get_integer_imm Ms Bits Live Fail Flags Y=y => \ i_bs_get_integer_small_imm xy W f? t x i_bs_get_integer_imm xy W t f? t x -i_bs_get_integer f? t t xy s d +i_bs_get_integer xy f? t t s d i_bs_get_integer_8 xy f? d i_bs_get_integer_16 xy f? d @@ -1166,9 +1172,9 @@ i_bs_get_integer_32 xy f? d bs_get_binary2 Fail=f Ms=xy Live=u Sz=sq Unit=u Flags=u Dst=d => \ gen_get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst) -i_bs_get_binary_imm2 f? xy t W t d -i_bs_get_binary2 f xy t? s t d -i_bs_get_binary_all2 f? xy t t d +i_bs_get_binary_imm2 xy f? t W t d +i_bs_get_binary2 xy f t? s t d +i_bs_get_binary_all2 xy f? t t d i_bs_get_binary_all_reuse xy f? t # Fetching float from binaries. @@ -1177,7 +1183,7 @@ bs_get_float2 Fail=f Ms=xy Live=u Sz=s Unit=u Flags=u Dst=d => \ bs_get_float2 Fail=f Ms=x Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail -i_bs_get_float2 f? xy t s t d +i_bs_get_float2 xy f? t s t d # Miscellanous @@ -1185,8 +1191,7 @@ bs_skip_bits2 Fail=f Ms=xy Sz=sq Unit=u Flags=u => \ gen_skip_bits2(Fail, Ms, Sz, Unit, Flags) i_bs_skip_bits_imm2 f? xy W -i_bs_skip_bits2 f? xy xy t -i_bs_skip_bits_all2 f? xy t +i_bs_skip_bits2 xy xy f? t bs_test_tail2 Fail=f Ms=xy Bits=u==0 => bs_test_zero_tail2 Fail Ms bs_test_tail2 Fail=f Ms=xy Bits=u => bs_test_tail_imm2 Fail Ms Bits @@ -1197,16 +1202,6 @@ bs_test_unit F Ms Unit=u==8 => bs_test_unit8 F Ms bs_test_unit f? xy t bs_test_unit8 f? xy -# An y register operand for bs_context_to_binary is rare, -# but can happen because of inlining. - -bs_context_to_binary Y=y | line L | badmatch Y => \ - move Y x | bs_context_to_binary x | line L | badmatch x - -bs_context_to_binary Y=y => move Y x | bs_context_to_binary x - -bs_context_to_binary x - # Gets a bitstring from the tail of a context. bs_get_tail xy d t @@ -1332,31 +1327,35 @@ i_bs_private_append j? t s S x bs_put_integer Fail=j Sz=sq Unit=u Flags=u Src=s => \ gen_put_integer(Fail, Sz, Unit, Flags, Src) -i_new_bs_put_integer j? s t s -i_new_bs_put_integer_imm j? W t s +i_new_bs_put_integer j? S t s +i_new_bs_put_integer_imm xyc j? W t # # Utf8/utf16/utf32 support. (R12B-5) # -bs_utf8_size j Src=s Dst=d => i_bs_utf8_size Src Dst - -i_bs_utf8_size s x +bs_utf8_size j Src Dst=d => i_bs_utf8_size Src Dst +bs_utf16_size j Src Dst=d => i_bs_utf16_size Src Dst -bs_utf16_size j Src=s Dst=d => i_bs_utf16_size Src Dst +bs_put_utf8 Fail u Src => i_bs_put_utf8 Fail Src -i_bs_utf16_size s x - -bs_put_utf8 Fail u Src=s => i_bs_put_utf8 Fail Src +bs_put_utf32 Fail=j Flags=u Src=s => \ + i_bs_validate_unicode Fail Src | bs_put_integer Fail i=32 u=1 Flags Src -i_bs_put_utf8 j? s +i_bs_utf8_size S x +i_bs_utf16_size S x -bs_put_utf16 j? t s +i_bs_put_utf8 j? S +bs_put_utf16 j? t S -bs_put_utf32 Fail=j Flags=u Src=s => \ - i_bs_validate_unicode Fail Src | bs_put_integer Fail i=32 u=1 Flags Src +i_bs_validate_unicode j? S -i_bs_validate_unicode j? s +# Handle unoptimized code. +i_bs_utf8_size Src=c Dst => move Src x | i_bs_utf8_size x Dst +i_bs_utf16_size Src=c Dst => move Src x | i_bs_utf16_size x Dst +i_bs_put_utf8 Fail Src=c => move Src x | i_bs_put_utf8 Fail x +bs_put_utf16 Fail Flags Src=c => move Src x | bs_put_utf16 Fail Flags x +i_bs_validate_unicode Fail Src=c => move Src x | i_bs_validate_unicode Fail x # # Storing floats into binaries. @@ -1366,7 +1365,7 @@ bs_put_float Fail Sz=q Unit Flags Val => badarg Fail bs_put_float Fail=j Sz=s Unit=u Flags=u Src=s => \ gen_put_float(Fail, Sz, Unit, Flags, Src) -i_new_bs_put_float j? s t s +i_new_bs_put_float j? S t s i_new_bs_put_float_imm j? W t s # @@ -1376,9 +1375,18 @@ i_new_bs_put_float_imm j? W t s bs_put_binary Fail=j Sz=s Unit=u Flags=u Src=s => \ gen_put_binary(Fail, Sz, Unit, Flags, Src) -i_new_bs_put_binary j? s t s -i_new_bs_put_binary_imm j? W s -i_new_bs_put_binary_all j? s t +# In unoptimized code, the binary argument could be a literal. (In optimized code, +# there would be a bs_put_string instruction.) +i_new_bs_put_binary Fail Size Unit Lit=c => \ + move Lit x | i_new_bs_put_binary Fail Size Unit x +i_new_bs_put_binary_imm Fail Size Lit=c => \ + move Lit x | i_new_bs_put_binary_imm Fail Size x +i_new_bs_put_binary_all Lit=c Fail Unit => \ + move Lit x | i_new_bs_put_binary_all x Fail Unit + +i_new_bs_put_binary j? S t S +i_new_bs_put_binary_imm j? W S +i_new_bs_put_binary_all xy j? t # # Warning: The i_bs_put_string and i_new_bs_put_string instructions diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index a69da4d762..a6312293cc 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -111,6 +111,23 @@ #endif #endif +/* + * Test for clang's convenient __has_builtin feature checking macro. + */ +#ifndef __has_builtin + #define __has_builtin(x) 0 +#endif + +/* + * Define HAVE_OVERFLOW_CHECK_BUILTINS if the overflow checking arithmetic + * builtins are available. + */ +#if ERTS_AT_LEAST_GCC_VSN__(5, 1, 0) +# define HAVE_OVERFLOW_CHECK_BUILTINS 1 +#elif __has_builtin(__builtin_mul_overflow) +# define HAVE_OVERFLOW_CHECK_BUILTINS 1 +#endif + #include "erl_misc_utils.h" /* diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index 78411f324c..c93966d24f 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -574,7 +574,7 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) driver_select(port, e, mode | (on?ERL_DRV_USE:0), on) #define sock_select(d, flags, onoff) do { \ - ASSERT(!(d)->is_ignored); \ + ASSERT(!INET_IGNORED(d)); \ (d)->event_mask = (onoff) ? \ ((d)->event_mask | (flags)) : \ ((d)->event_mask & ~(flags)); \ @@ -898,6 +898,15 @@ static size_t my_strnlen(const char *s, size_t maxlen) #define INET_CMSG_RECVTCLASS (1 << 1) /* am_recvtclass, am_tclass */ #define INET_CMSG_RECVTTL (1 << 2) /* am_recvttl, am_ttl */ +/* Inet flags */ +#define INET_FLG_BUFFER_SET (1 << 0) /* am_buffer has been set by user */ +#define INET_FLG_IS_IGNORED (1 << 1) /* If a fd is ignored by the inet_drv. + This flag should be set to true when + the fd is used outside of inet_drv. */ +#define INET_FLG_IS_IGNORED_RD (1 << 2) +#define INET_FLG_IS_IGNORED_WR (1 << 3) +#define INET_FLG_IS_IGNORED_PASS (1 << 4) + /* ** End of interface constants. **--------------------------------------------------------------------------*/ @@ -943,10 +952,11 @@ static size_t my_strnlen(const char *s, size_t maxlen) #define INET_IFNAMSIZ 16 /* INET Ignore states */ -#define INET_IGNORE_NONE 0 -#define INET_IGNORE_READ (1 << 0) -#define INET_IGNORE_WRITE (1 << 1) -#define INET_IGNORE_PASSIVE (1 << 2) +#define INET_IGNORE_CLEAR(desc) ((desc)->flags & ~(INET_IGNORE_READ|INET_IGNORE_WRITE|INET_IGNORE_PASSIVE)) +#define INET_IGNORED(desc) ((desc)->flags & INET_FLG_IS_IGNORED) +#define INET_IGNORE_READ (INET_FLG_IS_IGNORED|INET_FLG_IS_IGNORED_RD) +#define INET_IGNORE_WRITE (INET_FLG_IS_IGNORED|INET_FLG_IS_IGNORED_WR) +#define INET_IGNORE_PASSIVE (INET_FLG_IS_IGNORED|INET_FLG_IS_IGNORED_PASS) /* Max length of Erlang Term Buffer (for outputting structured terms): */ #ifdef HAVE_SCTP @@ -1128,9 +1138,7 @@ typedef struct { double send_avg; /* average packet size sent */ subs_list empty_out_q_subs; /* Empty out queue subscribers */ - int is_ignored; /* if a fd is ignored by the inet_drv. - This flag should be set to true when - the fd is used outside of inet_drv. */ + int flags; #ifdef HAVE_SETNS char *netns; /* Socket network namespace name as full file path */ @@ -4551,7 +4559,7 @@ static void desc_close(inet_descriptor* desc) * We should close the fd here, but the other driver might still * be selecting on it. */ - if (!desc->is_ignored) + if (!INET_IGNORED(desc)) driver_select(desc->port,(ErlDrvEvent)(long)desc->event, ERL_DRV_USE, 0); else @@ -6312,6 +6320,7 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) (long)desc->port, desc->s, ival)); if (ival < INET_MIN_BUFFER) ival = INET_MIN_BUFFER; desc->bufsz = ival; + desc->flags |= INET_FLG_BUFFER_SET; continue; case INET_LOPT_ACTIVE: @@ -6507,6 +6516,16 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) case INET_OPT_RCVBUF: type = SO_RCVBUF; DEBUGF(("inet_set_opts(%ld): s=%d, SO_RCVBUF=%d\r\n", (long)desc->port, desc->s, ival)); + if (!(desc->flags & INET_FLG_BUFFER_SET)) { + /* make sure we have desc->bufsz >= SO_RCVBUF */ + if (ival > (1 << 16) && desc->stype == SOCK_DGRAM && !IS_SCTP(desc)) + /* For UDP we don't want to automatically + set the buffer size to be larger than + the theoretical max MTU */ + desc->bufsz = 1 << 16; + else if (ival > desc->bufsz) + desc->bufsz = ival; + } break; case INET_OPT_LINGER: type = SO_LINGER; if (len < 4) @@ -6752,23 +6771,17 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) } DEBUGF(("inet_set_opts(%ld): s=%d returned %d\r\n", (long)desc->port, desc->s, res)); - if (type == SO_RCVBUF) { - /* make sure we have desc->bufsz >= SO_RCVBUF */ - if (ival > (1 << 16) && desc->stype == SOCK_DGRAM && !IS_SCTP(desc)) - /* For UDP we don't want to automatically - set the buffer size to be larger than - the theoretical max MTU */ - desc->bufsz = 1 << 16; - else if (ival > desc->bufsz) - desc->bufsz = ival; - } } if ( ((desc->stype == SOCK_STREAM) && IS_CONNECTED(desc)) || ((desc->stype == SOCK_DGRAM) && IS_OPEN(desc))) { - if (desc->active != old_active) + if (desc->active != old_active) { + /* Need to cancel the read_packet timer if we go from active to passive. */ + if (desc->active == INET_PASSIVE && desc->stype == SOCK_DGRAM) + driver_cancel_timer(desc->port); sock_select(desc, (FD_READ|FD_CLOSE), (desc->active>0)); + } /* XXX: UDP sockets could also trigger immediate read here NIY */ if ((desc->stype==SOCK_STREAM) && desc->active) { @@ -6910,6 +6923,7 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len) if (desc->bufsz < INET_MIN_BUFFER) desc->bufsz = INET_MIN_BUFFER; + desc->flags |= INET_FLG_BUFFER_SET; res = 0; /* This does not affect the kernel buffer size */ continue; @@ -7031,6 +7045,7 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len) smaller than the kernel one: */ if (desc->bufsz <= arg.ival) desc->bufsz = arg.ival; + desc->flags |= INET_FLG_BUFFER_SET; break; } case INET_OPT_SNDBUF: @@ -7041,10 +7056,6 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len) arg_ptr = (char*) (&arg.ival); arg_sz = sizeof ( arg.ival); - /* Adjust the size of the user-level recv buffer, so it's not - smaller than the kernel one: */ - if (desc->bufsz <= arg.ival) - desc->bufsz = arg.ival; break; } case INET_OPT_REUSEADDR: @@ -9101,7 +9112,7 @@ static ErlDrvData inet_start(ErlDrvPort port, int size, int protocol) sys_memzero((char *)&desc->remote,sizeof(desc->remote)); - desc->is_ignored = 0; + desc->flags = 0; #ifdef HAVE_SETNS desc->netns = NULL; @@ -9503,19 +9514,19 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf, if (desc->stype != SOCK_STREAM) return ctl_error(EINVAL, rbuf, rsize); - if (*buf == 1 && !desc->is_ignored) { + if (*buf == 1 && !INET_IGNORED(desc)) { sock_select(desc, (FD_READ|FD_WRITE|FD_CLOSE|ERL_DRV_USE_NO_CALLBACK), 0); if (desc->active) - desc->is_ignored = INET_IGNORE_READ; + desc->flags |= INET_IGNORE_READ; else - desc->is_ignored = INET_IGNORE_PASSIVE; - } else if (*buf == 0 && desc->is_ignored) { + desc->flags |= INET_IGNORE_PASSIVE; + } else if (*buf == 0 && INET_IGNORED(desc)) { int flags = FD_CLOSE; - if (desc->is_ignored & INET_IGNORE_READ) + if (desc->flags & INET_IGNORE_READ) flags |= FD_READ; - if (desc->is_ignored & INET_IGNORE_WRITE) + if (desc->flags & INET_IGNORE_WRITE) flags |= FD_WRITE; - desc->is_ignored = INET_IGNORE_NONE; + desc->flags = INET_IGNORE_CLEAR(desc); if (flags != FD_CLOSE) sock_select(desc, flags, 1); } else @@ -10231,17 +10242,17 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd, if (enq_async(INETP(desc), tbuf, TCP_REQ_RECV) < 0) return ctl_error(EALREADY, rbuf, rsize); - if (INETP(desc)->is_ignored || tcp_recv(desc, n) == 0) { + if (INET_IGNORED(INETP(desc)) || tcp_recv(desc, n) == 0) { if (timeout == 0) async_error_am(INETP(desc), am_timeout); else { if (timeout != INET_INFINITY) add_multi_timer(desc, INETP(desc)->port, 0, timeout, &tcp_inet_recv_timeout); - if (!INETP(desc)->is_ignored) + if (!INET_IGNORED(INETP(desc))) sock_select(INETP(desc),(FD_READ|FD_CLOSE),1); else - INETP(desc)->is_ignored |= INET_IGNORE_READ; + INETP(desc)->flags |= INET_IGNORE_READ; } } return ctl_reply(INET_REP_OK, tbuf, 2, rbuf, rsize); @@ -11108,7 +11119,7 @@ static int tcp_inet_input(tcp_descriptor* desc, HANDLE event) #ifdef DEBUG long port = (long) desc->inet.port; /* Used after driver_exit() */ #endif - ASSERT(!INETP(desc)->is_ignored); + ASSERT(!INET_IGNORED(INETP(desc))); DEBUGF(("tcp_inet_input(%ld) {s=%d\r\n", port, desc->inet.s)); /* XXX fprintf(stderr,"tcp_inet_input(%ld) {s=%d}\r\n",(long) desc->inet.port, desc->inet.s); */ if (desc->inet.state == INET_STATE_ACCEPTING) { @@ -11443,8 +11454,8 @@ static int tcp_sendv(tcp_descriptor* desc, ErlIOVec* ev) DEBUGF(("tcp_sendv(%ld): s=%d, about to send "LLU","LLU" bytes\r\n", (long)desc->inet.port, desc->inet.s, (llu_t)h_len, (llu_t)len)); - if (INETP(desc)->is_ignored) { - INETP(desc)->is_ignored |= INET_IGNORE_WRITE; + if (INET_IGNORED(INETP(desc))) { + INETP(desc)->flags |= INET_IGNORE_WRITE; n = 0; } else if (desc->tcp_add_flags & TCP_ADDF_DELAY_SEND) { driver_enqv(ix, ev, 0); @@ -11479,7 +11490,7 @@ static int tcp_sendv(tcp_descriptor* desc, ErlIOVec* ev) DEBUGF(("tcp_sendv(%ld): s=%d, Send failed, queuing\r\n", (long)desc->inet.port, desc->inet.s)); driver_enqv(ix, ev, n); - if (!INETP(desc)->is_ignored) + if (!INET_IGNORED(INETP(desc))) sock_select(INETP(desc),(FD_WRITE|FD_CLOSE), 1); } return 0; @@ -11548,8 +11559,8 @@ static int tcp_send(tcp_descriptor* desc, char* ptr, ErlDrvSizeT len) DEBUGF(("tcp_send(%ld): s=%d, about to send "LLU","LLU" bytes\r\n", (long)desc->inet.port, desc->inet.s, (llu_t)h_len, (llu_t)len)); - if (INETP(desc)->is_ignored) { - INETP(desc)->is_ignored |= INET_IGNORE_WRITE; + if (INET_IGNORED(INETP(desc))) { + INETP(desc)->flags |= INET_IGNORE_WRITE; n = 0; } else if (desc->tcp_add_flags & TCP_ADDF_DELAY_SEND) { sock_send(desc->inet.s, buf, 0, 0); @@ -11582,7 +11593,7 @@ static int tcp_send(tcp_descriptor* desc, char* ptr, ErlDrvSizeT len) n -= h_len; driver_enq(ix, ptr+n, len-n); } - if (!INETP(desc)->is_ignored) + if (!INET_IGNORED(INETP(desc))) sock_select(INETP(desc),(FD_WRITE|FD_CLOSE), 1); } return 0; @@ -11863,7 +11874,7 @@ static int tcp_inet_output(tcp_descriptor* desc, HANDLE event) int ret = 0; ErlDrvPort ix = desc->inet.port; - ASSERT(!INETP(desc)->is_ignored); + ASSERT(!INET_IGNORED(INETP(desc))); DEBUGF(("tcp_inet_output(%ld) {s=%d\r\n", (long)desc->inet.port, desc->inet.s)); if (desc->inet.state == INET_STATE_CONNECTING) { @@ -12538,9 +12549,12 @@ static void packet_inet_timeout(ErlDrvData e) { udp_descriptor * udesc = (udp_descriptor*) e; inet_descriptor * desc = INETP(udesc); - if (!(desc->active)) + if (!(desc->active)) { sock_select(desc, FD_READ, 0); - async_error_am (desc, am_timeout); + async_error_am (desc, am_timeout); + } else { + (void)packet_inet_input(udesc, desc->s); + } } @@ -12896,6 +12910,15 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event) sock_select(desc, FD_READ, 1); } #endif + + /* We set a timer on the port to trigger now. + This emulates a "yield" operation as that is + what we want to do here. We do *NOT* do a deselect + as that is expensive, instead we check if the + socket it still active when the timeout triggers + and if it is not, then we just ignore the timeout */ + driver_set_timer(desc->port, 0); + return count; } diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index 4a0ad9c1d5..2309f844b9 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -65,7 +65,8 @@ nif_whereis/1, nif_whereis_parallel/1, nif_whereis_threaded/1, nif_whereis_proxy/1, nif_ioq/1, - pid/1 + pid/1, + nif_term_type/1 ]). -export([many_args_100/100]). @@ -105,7 +106,8 @@ all() -> nif_phash2, nif_whereis, nif_whereis_parallel, nif_whereis_threaded, nif_ioq, - pid]. + pid, + nif_term_type]. groups() -> [{G, [], api_repeaters()} || G <- api_groups()] @@ -3392,6 +3394,41 @@ pid(Config) -> {false, _} = get_local_pid_nif(undefined), ok. +nif_term_type(Config) -> + ensure_lib_loaded(Config), + + atom = term_type_nif(atom), + + bitstring = term_type_nif(<<1:1>>), + bitstring = term_type_nif(<<1:8>>), + + float = term_type_nif(0.0), + + 'fun' = term_type_nif(fun external:function/1), + 'fun' = term_type_nif(fun(A) -> A end), + 'fun' = term_type_nif(fun id/1), + + integer = term_type_nif(1 bsl 1024), %Bignum. + integer = term_type_nif(1), + + list = term_type_nif([list]), + list = term_type_nif([]), + + LargeMap = maps:from_list([{N, N} || N <- lists:seq(1, 256)]), + map = term_type_nif(LargeMap), + map = term_type_nif(#{ small => map }), + + pid = term_type_nif(self()), + + Port = open_port({spawn,echo_drv},[eof]), + port = term_type_nif(Port), + port_close(Port), + + reference = term_type_nif(make_ref()), + + tuple = term_type_nif({}), + + ok. id(I) -> I. @@ -3507,5 +3544,7 @@ set_pid_undefined_nif() -> ?nif_stub. is_pid_undefined_nif(_) -> ?nif_stub. compare_pids_nif(_, _) -> ?nif_stub. +term_type_nif(_) -> ?nif_stub. + nif_stub_error(Line) -> exit({nif_not_loaded,module,?MODULE,line,Line}). diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c index 0d5d900d31..1906384af4 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c +++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c @@ -3583,6 +3583,36 @@ static ERL_NIF_TERM compare_pids_nif(ErlNifEnv* env, int argc, const ERL_NIF_TER return enif_make_int(env, enif_compare_pids(&a, &b)); } +static ERL_NIF_TERM term_type_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + switch (enif_term_type(env, argv[0])) { + case ERL_NIF_TERM_TYPE_ATOM: + return enif_make_atom(env, "atom"); + case ERL_NIF_TERM_TYPE_BITSTRING: + return enif_make_atom(env, "bitstring"); + case ERL_NIF_TERM_TYPE_FLOAT: + return enif_make_atom(env, "float"); + case ERL_NIF_TERM_TYPE_FUN: + return enif_make_atom(env, "fun"); + case ERL_NIF_TERM_TYPE_INTEGER: + return enif_make_atom(env, "integer"); + case ERL_NIF_TERM_TYPE_LIST: + return enif_make_atom(env, "list"); + case ERL_NIF_TERM_TYPE_MAP: + return enif_make_atom(env, "map"); + case ERL_NIF_TERM_TYPE_PID: + return enif_make_atom(env, "pid"); + case ERL_NIF_TERM_TYPE_PORT: + return enif_make_atom(env, "port"); + case ERL_NIF_TERM_TYPE_REFERENCE: + return enif_make_atom(env, "reference"); + case ERL_NIF_TERM_TYPE_TUPLE: + return enif_make_atom(env, "tuple"); + default: + return enif_make_badarg(env); + } +} + static ErlNifFunc nif_funcs[] = { {"lib_version", 0, lib_version}, @@ -3690,7 +3720,8 @@ static ErlNifFunc nif_funcs[] = {"make_pid_nif", 1, make_pid_nif}, {"set_pid_undefined_nif", 0, set_pid_undefined_nif}, {"is_pid_undefined_nif", 1, is_pid_undefined_nif}, - {"compare_pids_nif", 2, compare_pids_nif} + {"compare_pids_nif", 2, compare_pids_nif}, + {"term_type_nif", 1, term_type_nif} }; ERL_NIF_INIT(nif_SUITE,nif_funcs,load,NULL,upgrade,unload) diff --git a/erts/emulator/test/trace_local_SUITE.erl b/erts/emulator/test/trace_local_SUITE.erl index 253d5fed23..ad802352b9 100644 --- a/erts/emulator/test/trace_local_SUITE.erl +++ b/erts/emulator/test/trace_local_SUITE.erl @@ -1181,7 +1181,9 @@ undef(X) -> ?MODULE:undef(X, X). % undef lists_reverse(A, B) -> - lists:reverse(A, B). + Res = lists:reverse(A, B), + _ = (catch abs(A)), + Res. diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops index f73e2362bf..1625b2cc65 100755 --- a/erts/emulator/utils/beam_makeops +++ b/erts/emulator/utils/beam_makeops @@ -1488,7 +1488,7 @@ sub code_gen { $var_decls .= "Eterm $tmp;\n"; $tmp_arg_num++; push(@f, $tmp); - $prefix .= "GetR($arg_offset, $tmp);\n"; + $prefix .= "GetSource(" . arg_offset($arg_offset) . ", $tmp);\n"; $need_block = 1; last SWITCH; }; |