diff options
160 files changed, 4947 insertions, 2950 deletions
diff --git a/.gitignore b/.gitignore index fee8cba0c7..3e7013ee6b 100644 --- a/.gitignore +++ b/.gitignore @@ -245,6 +245,7 @@ JAVADOC-GENERATED /lib/compiler/test/*_no_opt_SUITE.erl /lib/compiler/test/*_post_opt_SUITE.erl /lib/compiler/test/*_inline_SUITE.erl +/lib/compiler/test/*_r21_SUITE.erl # crypto /lib/crypto/test/crypto_SUITE_data/*.rsp diff --git a/Makefile.in b/Makefile.in index d880bfefa2..cc9a9375c9 100644 --- a/Makefile.in +++ b/Makefile.in @@ -982,7 +982,7 @@ primary_bootstrap_copy: # To remove modules left by the bootstrap building, but leave (restore) # the modules in kernel which are needed for an emulator build -KERNEL_PRELOAD = otp_ring0 init erl_prim_loader prim_inet prim_file zlib prim_zip erlang erts_code_purger +KERNEL_PRELOAD = erl_init init erl_prim_loader prim_inet prim_file zlib prim_zip erlang erts_code_purger KERNEL_PRELOAD_BEAMS=$(KERNEL_PRELOAD:%=$(BOOTSTRAP_TOP)/lib/kernel/ebin/%.beam) start_scripts: diff --git a/bootstrap/bin/no_dot_erlang.boot b/bootstrap/bin/no_dot_erlang.boot Binary files differindex 8cd235ddd2..7562bde377 100644 --- a/bootstrap/bin/no_dot_erlang.boot +++ b/bootstrap/bin/no_dot_erlang.boot diff --git a/bootstrap/bin/start.boot b/bootstrap/bin/start.boot Binary files differindex 8cd235ddd2..7562bde377 100644 --- a/bootstrap/bin/start.boot +++ b/bootstrap/bin/start.boot diff --git a/bootstrap/bin/start_clean.boot b/bootstrap/bin/start_clean.boot Binary files differindex 8cd235ddd2..7562bde377 100644 --- a/bootstrap/bin/start_clean.boot +++ b/bootstrap/bin/start_clean.boot diff --git a/bootstrap/lib/compiler/ebin/beam_a.beam b/bootstrap/lib/compiler/ebin/beam_a.beam Binary files differindex 2001f94830..d877b8962c 100644 --- a/bootstrap/lib/compiler/ebin/beam_a.beam +++ b/bootstrap/lib/compiler/ebin/beam_a.beam diff --git a/bootstrap/lib/compiler/ebin/beam_asm.beam b/bootstrap/lib/compiler/ebin/beam_asm.beam Binary files differindex 6a0c0498d2..4c7d37270e 100644 --- a/bootstrap/lib/compiler/ebin/beam_asm.beam +++ b/bootstrap/lib/compiler/ebin/beam_asm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_block.beam b/bootstrap/lib/compiler/ebin/beam_block.beam Binary files differindex 720bfcb277..e28b5daeb4 100644 --- a/bootstrap/lib/compiler/ebin/beam_block.beam +++ b/bootstrap/lib/compiler/ebin/beam_block.beam diff --git a/bootstrap/lib/compiler/ebin/beam_bs.beam b/bootstrap/lib/compiler/ebin/beam_bs.beam Binary files differindex d1e97c9e59..81b05a3ca7 100644 --- a/bootstrap/lib/compiler/ebin/beam_bs.beam +++ b/bootstrap/lib/compiler/ebin/beam_bs.beam diff --git a/bootstrap/lib/compiler/ebin/beam_bsm.beam b/bootstrap/lib/compiler/ebin/beam_bsm.beam Binary files differdeleted file mode 100644 index 77b69ee921..0000000000 --- a/bootstrap/lib/compiler/ebin/beam_bsm.beam +++ /dev/null diff --git a/bootstrap/lib/compiler/ebin/beam_clean.beam b/bootstrap/lib/compiler/ebin/beam_clean.beam Binary files differindex 3cc1461090..c6d3c34e1f 100644 --- a/bootstrap/lib/compiler/ebin/beam_clean.beam +++ b/bootstrap/lib/compiler/ebin/beam_clean.beam diff --git a/bootstrap/lib/compiler/ebin/beam_dict.beam b/bootstrap/lib/compiler/ebin/beam_dict.beam Binary files differindex 3a97f98bd5..d04130f560 100644 --- a/bootstrap/lib/compiler/ebin/beam_dict.beam +++ b/bootstrap/lib/compiler/ebin/beam_dict.beam diff --git a/bootstrap/lib/compiler/ebin/beam_disasm.beam b/bootstrap/lib/compiler/ebin/beam_disasm.beam Binary files differindex 3c491bd969..b0b444d886 100644 --- a/bootstrap/lib/compiler/ebin/beam_disasm.beam +++ b/bootstrap/lib/compiler/ebin/beam_disasm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_flatten.beam b/bootstrap/lib/compiler/ebin/beam_flatten.beam Binary files differindex eb60f9fe01..cc245af68a 100644 --- a/bootstrap/lib/compiler/ebin/beam_flatten.beam +++ b/bootstrap/lib/compiler/ebin/beam_flatten.beam diff --git a/bootstrap/lib/compiler/ebin/beam_jump.beam b/bootstrap/lib/compiler/ebin/beam_jump.beam Binary files differindex be124cd68b..bc6e006047 100644 --- a/bootstrap/lib/compiler/ebin/beam_jump.beam +++ b/bootstrap/lib/compiler/ebin/beam_jump.beam diff --git a/bootstrap/lib/compiler/ebin/beam_kernel_to_ssa.beam b/bootstrap/lib/compiler/ebin/beam_kernel_to_ssa.beam Binary files differindex 5e98c81977..4f5a916eb2 100644 --- a/bootstrap/lib/compiler/ebin/beam_kernel_to_ssa.beam +++ b/bootstrap/lib/compiler/ebin/beam_kernel_to_ssa.beam diff --git a/bootstrap/lib/compiler/ebin/beam_opcodes.beam b/bootstrap/lib/compiler/ebin/beam_opcodes.beam Binary files differindex 05c71c92b6..52a23f7f6c 100644 --- a/bootstrap/lib/compiler/ebin/beam_opcodes.beam +++ b/bootstrap/lib/compiler/ebin/beam_opcodes.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa.beam b/bootstrap/lib/compiler/ebin/beam_ssa.beam Binary files differindex 4e39b1d1c3..8b8ab2b69e 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_bsm.beam b/bootstrap/lib/compiler/ebin/beam_ssa_bsm.beam Binary files differnew file mode 100644 index 0000000000..d04b8767d2 --- /dev/null +++ b/bootstrap/lib/compiler/ebin/beam_ssa_bsm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_codegen.beam b/bootstrap/lib/compiler/ebin/beam_ssa_codegen.beam Binary files differindex e5efa5b734..4b4a56c18a 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_codegen.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_codegen.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_dead.beam b/bootstrap/lib/compiler/ebin/beam_ssa_dead.beam Binary files differindex a8f4a8c116..9abb8910d8 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_dead.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_dead.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_opt.beam b/bootstrap/lib/compiler/ebin/beam_ssa_opt.beam Binary files differindex add9be2ed3..5e31165ae6 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_opt.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_opt.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_pp.beam b/bootstrap/lib/compiler/ebin/beam_ssa_pp.beam Binary files differindex 85bcccd1cb..3e30ce48b2 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_pp.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_pp.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_pre_codegen.beam b/bootstrap/lib/compiler/ebin/beam_ssa_pre_codegen.beam Binary files differindex 23a8a9b6fb..34c29161e6 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_pre_codegen.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_pre_codegen.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_recv.beam b/bootstrap/lib/compiler/ebin/beam_ssa_recv.beam Binary files differindex b59d411105..28b5ede5fc 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_recv.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_recv.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_type.beam b/bootstrap/lib/compiler/ebin/beam_ssa_type.beam Binary files differindex 205093d254..a89c962f67 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_type.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_type.beam diff --git a/bootstrap/lib/compiler/ebin/beam_utils.beam b/bootstrap/lib/compiler/ebin/beam_utils.beam Binary files differindex eb3576393b..527e8ea2c5 100644 --- a/bootstrap/lib/compiler/ebin/beam_utils.beam +++ b/bootstrap/lib/compiler/ebin/beam_utils.beam diff --git a/bootstrap/lib/compiler/ebin/beam_validator.beam b/bootstrap/lib/compiler/ebin/beam_validator.beam Binary files differindex c813d609e6..c90e9a2ae7 100644 --- a/bootstrap/lib/compiler/ebin/beam_validator.beam +++ b/bootstrap/lib/compiler/ebin/beam_validator.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_inline.beam b/bootstrap/lib/compiler/ebin/cerl_inline.beam Binary files differindex 4d5f2510e5..7d6864335f 100644 --- a/bootstrap/lib/compiler/ebin/cerl_inline.beam +++ b/bootstrap/lib/compiler/ebin/cerl_inline.beam diff --git a/bootstrap/lib/compiler/ebin/compile.beam b/bootstrap/lib/compiler/ebin/compile.beam Binary files differindex bcf6d03176..ea3db59fea 100644 --- a/bootstrap/lib/compiler/ebin/compile.beam +++ b/bootstrap/lib/compiler/ebin/compile.beam diff --git a/bootstrap/lib/compiler/ebin/compiler.app b/bootstrap/lib/compiler/ebin/compiler.app index d83ca1a995..028ee62b42 100644 --- a/bootstrap/lib/compiler/ebin/compiler.app +++ b/bootstrap/lib/compiler/ebin/compiler.app @@ -19,13 +19,12 @@ {application, compiler, [{description, "ERTS CXC 138 10"}, - {vsn, "7.2.4"}, + {vsn, "7.2.5"}, {modules, [ beam_a, beam_asm, beam_block, beam_bs, - beam_bsm, beam_clean, beam_dict, beam_disasm, @@ -37,6 +36,7 @@ beam_opcodes, beam_peep, beam_ssa, + beam_ssa_bsm, beam_ssa_codegen, beam_ssa_dead, beam_ssa_lint, diff --git a/bootstrap/lib/compiler/ebin/core_pp.beam b/bootstrap/lib/compiler/ebin/core_pp.beam Binary files differindex ddef7dde88..e9d9399b4a 100644 --- a/bootstrap/lib/compiler/ebin/core_pp.beam +++ b/bootstrap/lib/compiler/ebin/core_pp.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_alias.beam b/bootstrap/lib/compiler/ebin/sys_core_alias.beam Binary files differindex 609cd5e71b..2829b49dce 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_alias.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_alias.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_bsm.beam b/bootstrap/lib/compiler/ebin/sys_core_bsm.beam Binary files differindex 706f9b58e8..38ce74115e 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_bsm.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_bsm.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_fold.beam b/bootstrap/lib/compiler/ebin/sys_core_fold.beam Binary files differindex 87d9e94d70..3a7a446886 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_fold.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_fold.beam diff --git a/bootstrap/lib/compiler/ebin/v3_core.beam b/bootstrap/lib/compiler/ebin/v3_core.beam Binary files differindex 6ecceba389..309669c70e 100644 --- a/bootstrap/lib/compiler/ebin/v3_core.beam +++ b/bootstrap/lib/compiler/ebin/v3_core.beam diff --git a/bootstrap/lib/compiler/ebin/v3_kernel.beam b/bootstrap/lib/compiler/ebin/v3_kernel.beam Binary files differindex b359037d7e..ce882eb80e 100644 --- a/bootstrap/lib/compiler/ebin/v3_kernel.beam +++ b/bootstrap/lib/compiler/ebin/v3_kernel.beam diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index a5826307f7..3820132d71 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -1252,7 +1252,7 @@ typedef struct { <p>Similar to <c>fprintf</c> but this format string also accepts <c>"%T"</c>, which formats Erlang terms of type <seealso marker="#ERL_NIF_TERM"><c>ERL_NIF_TERM</c></seealso>.</p> - <p>This function is primarily intenden for debugging purpose. It is not + <p>This function is primarily intended for debugging purpose. It is not recommended to print very large terms with <c>%T</c>. The function may change <c>errno</c>, even if successful.</p> </desc> @@ -3216,7 +3216,7 @@ if (retval & ERL_NIF_SELECT_STOP_CALLED) { <p>Similar to <c>snprintf</c> but this format string also accepts <c>"%T"</c>, which formats Erlang terms of type <seealso marker="#ERL_NIF_TERM"><c>ERL_NIF_TERM</c></seealso>.</p> - <p>This function is primarily intenden for debugging purpose. It is not + <p>This function is primarily intended for debugging purpose. It is not recommended to print very large terms with <c>%T</c>. The function may change <c>errno</c>, even if successful.</p> </desc> diff --git a/erts/doc/src/match_spec.xml b/erts/doc/src/match_spec.xml index 5cd6dc1750..48e502739a 100644 --- a/erts/doc/src/match_spec.xml +++ b/erts/doc/src/match_spec.xml @@ -113,6 +113,7 @@ <c><![CDATA[length]]></c> | <c><![CDATA[map_get]]></c> | <c><![CDATA[map_size]]></c> | <c><![CDATA[node]]></c> | <c><![CDATA[round]]></c> | <c><![CDATA[size]]></c> | + <c><![CDATA[bit_size]]></c> | <c><![CDATA[tl]]></c> | <c><![CDATA[trunc]]></c> | <c><![CDATA['+']]></c> | <c><![CDATA['-']]></c> | <c><![CDATA['*']]></c> | <c><![CDATA['div']]></c> | @@ -194,6 +195,7 @@ <c><![CDATA[length]]></c> | <c><![CDATA[map_get]]></c> | <c><![CDATA[map_size]]></c> | <c><![CDATA[node]]></c> | <c><![CDATA[round]]></c> | <c><![CDATA[size]]></c> | + <c><![CDATA[bit_size]]></c> | <c><![CDATA[tl]]></c> | <c><![CDATA[trunc]]></c> | <c><![CDATA['+']]></c> | <c><![CDATA['-']]></c> | <c><![CDATA['*']]></c> | <c><![CDATA['div']]></c> | @@ -269,8 +271,9 @@ other <c>false</c> to return <c><![CDATA[true]]></c>; otherwise <c><![CDATA['xor']]></c> returns false.</p> </item> - <tag><c>abs</c>, <c>element</c>, <c>hd</c>, <c>length</c>, <c>node</c>, - <c>round</c>, <c>size</c>, <c>tl</c>, <c>trunc</c>, <c>'+'</c>, + <tag><c>abs</c>, <c>element</c>, <c>hd</c>, <c>length</c>, + <c>map_get</c>, <c>map_size</c>, <c>node</c>, <c>round</c>, + <c>size</c>, <c>bit_size</c>, <c>tl</c>, <c>trunc</c>, <c>'+'</c>, <c>'-'</c>, <c>'*'</c>, <c>'div'</c>, <c>'rem'</c>, <c>'band'</c>, <c>'bor'</c>, <c>'bxor'</c>, <c>'bnot'</c>, <c>'bsl'</c>, <c>'bsr'</c>, <c>'>'</c>, <c>'>='</c>, <c>'<'</c>, <c>'=<'</c>, diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index eeb9577a33..f84c124a14 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -633,8 +633,8 @@ GENERATE += $(TTF_DIR)/driver_tab.c # This list must be consistent with PRE_LOADED_MODULES in # erts/preloaded/src/Makefile. -PRELOAD_BEAM = $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erts_code_purger.beam \ +PRELOAD_BEAM = $(ERL_TOP)/erts/preloaded/ebin/erts_code_purger.beam \ + $(ERL_TOP)/erts/preloaded/ebin/erl_init.beam \ $(ERL_TOP)/erts/preloaded/ebin/init.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_buffer.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_eval.beam \ diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 45b7540aeb..fb223d2a5d 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -208,7 +208,6 @@ atom dirty_nif_finalizer atom disable_trace atom disabled atom discard -atom display_items atom dist atom dist_cmd atom dist_ctrl_put_data @@ -237,6 +236,7 @@ atom eof atom eol atom Eq='=:=' atom Eqeq='==' +atom erl_init atom erl_tracer atom erlang atom erl_signal_server diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 50cbb37f3e..c28a5c57e6 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -2868,6 +2868,7 @@ load_code(LoaderState* stp) break; case op_bs_put_string_WW: case op_i_bs_match_string_xfWW: + case op_i_bs_match_string_yfWW: new_string_patch(stp, ci-1); break; diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 9ec863f259..3b45f968cf 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -4440,13 +4440,6 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(old_value); - } else if (BIF_ARG_1 == am_display_items) { - int oval = display_items; - if (!is_small(BIF_ARG_2) || (n = signed_val(BIF_ARG_2)) < 0) { - goto error; - } - display_items = n < 32 ? 32 : n; - BIF_RET(make_small(oval)); } else if (BIF_ARG_1 == am_debug_flags) { BIF_RET(am_true); } else if (BIF_ARG_1 == am_backtrace_depth) { diff --git a/erts/emulator/beam/bs_instrs.tab b/erts/emulator/beam/bs_instrs.tab index 61eb02a7a2..2dde70c2e1 100644 --- a/erts/emulator/beam/bs_instrs.tab +++ b/erts/emulator/beam/bs_instrs.tab @@ -102,6 +102,7 @@ i_bs_get_binary_all2(Fail, Ms, Live, Unit, Dst) { LIGHT_SWAPIN; HEAP_SPACE_VERIFIED(0); ASSERT(is_value(_result)); + $REFRESH_GEN_DEST(); $Dst = _result; } else { HEAP_SPACE_VERIFIED(0); @@ -123,6 +124,7 @@ i_bs_get_binary2(Fail, Ms, Live, Sz, Flags, Dst) { if (is_non_value(_result)) { $FAIL($Fail); } else { + $REFRESH_GEN_DEST(); $Dst = _result; } } @@ -139,6 +141,7 @@ i_bs_get_binary_imm2(Fail, Ms, Live, Sz, Flags, Dst) { if (is_non_value(_result)) { $FAIL($Fail); } else { + $REFRESH_GEN_DEST(); $Dst = _result; } } @@ -161,6 +164,7 @@ i_bs_get_float2(Fail, Ms, Live, Sz, Flags, Dst) { if (is_non_value(_result)) { $FAIL($Fail); } else { + $REFRESH_GEN_DEST(); $Dst = _result; } } @@ -724,26 +728,34 @@ bs_start_match.execute(Fail, Live, Slots, Dst) { $FAIL($Fail); } header = *boxed_val(context); - slots = $Slots; + + /* Reserve a slot for the start position. */ + slots = $Slots + 1; live = $Live; + if (header_is_bin_matchstate(header)) { ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(context); Uint actual_slots = HEADER_NUM_SLOTS(header); + + /* We're not compatible with contexts created by bs_start_match3. */ + ASSERT(actual_slots >= 1); + ms->save_offset[0] = ms->mb.offset; - if (actual_slots < slots) { - ErlBinMatchState* dst; + if (ERTS_UNLIKELY(actual_slots < slots)) { + ErlBinMatchState* expanded; Uint live = $Live; Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); - $GC_TEST_PRESERVE(wordsneeded, live, context); ms = (ErlBinMatchState *) boxed_val(context); - dst = (ErlBinMatchState *) HTOP; - *dst = *ms; + expanded = (ErlBinMatchState *) HTOP; + *expanded = *ms; *HTOP = HEADER_BIN_MATCHSTATE(slots); HTOP += wordsneeded; HEAP_SPACE_VERIFIED(0); - $Dst = make_matchstate(dst); + context = make_matchstate(expanded); + $REFRESH_GEN_DEST(); } + $Dst = context; } else if (is_binary_header(header)) { Eterm result; Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); @@ -758,6 +770,7 @@ bs_start_match.execute(Fail, Live, Slots, Dst) { if (is_non_value(result)) { $FAIL($Fail); } + $REFRESH_GEN_DEST(); $Dst = result; } else { $FAIL($Fail); @@ -906,6 +919,7 @@ i_bs_get_integer(Fail, Live, FlagsAndUnit, Ms, Sz, Dst) { } wordsneeded = 1+WSIZE(NBYTES((Uint) size)); $GC_TEST_PRESERVE(wordsneeded, $Live, ms); + $REFRESH_GEN_DEST(); } mb = ms_matchbuffer(ms); LIGHT_SWAPOUT; @@ -939,6 +953,7 @@ i_bs_get_utf8(Ctx, Fail, Dst) { if (is_non_value(result)) { $FAIL($Fail); } + $REFRESH_GEN_DEST(); $Dst = result; } @@ -949,6 +964,7 @@ i_bs_get_utf16(Ctx, Fail, Flags, Dst) { if (is_non_value(result)) { $FAIL($Fail); } + $REFRESH_GEN_DEST(); $Dst = result; } @@ -1029,10 +1045,289 @@ i_bs_match_string(Ctx, Fail, Bits, Ptr) { i_bs_save2(Src, Slot) { ErlBinMatchState* _ms = (ErlBinMatchState*) boxed_val((Eterm) $Src); + ASSERT(HEADER_NUM_SLOTS(_ms->thing_word) > $Slot); _ms->save_offset[$Slot] = _ms->mb.offset; } i_bs_restore2(Src, Slot) { ErlBinMatchState* _ms = (ErlBinMatchState*) boxed_val((Eterm) $Src); + ASSERT(HEADER_NUM_SLOTS(_ms->thing_word) > $Slot); _ms->mb.offset = _ms->save_offset[$Slot]; } + +bs_get_tail(Src, Dst, Live) { + ErlBinMatchBuffer* mb; + Uint size, offs; + ErlSubBin* sb; + Eterm context; + + context = $Src; + + ASSERT(header_is_bin_matchstate(*boxed_val(context))); + + $GC_TEST_PRESERVE(ERL_SUB_BIN_SIZE, $Live, context); + + mb = ms_matchbuffer(context); + + offs = mb->offset; + size = mb->size - offs; + + sb = (ErlSubBin *) HTOP; + HTOP += ERL_SUB_BIN_SIZE; + + sb->thing_word = HEADER_SUB_BIN; + sb->size = BYTE_OFFSET(size); + sb->bitsize = BIT_OFFSET(size); + sb->offs = BYTE_OFFSET(offs); + sb->bitoffs = BIT_OFFSET(offs); + sb->is_writable = 0; + sb->orig = mb->orig; + + $REFRESH_GEN_DEST(); + $Dst = make_binary(sb); +} + + +%if ARCH_64 + +i_bs_start_match3_gp(Src, Live, Fail, Dst, Pos) { + Eterm context, header; + Uint position, live; + + context = $Src; + live = $Live; + + if (!is_boxed(context)) { + $FAIL($Fail); + } + + header = *boxed_val(context); + + if (header_is_bin_matchstate(header)) { + ErlBinMatchBuffer *mb; + + ASSERT(HEADER_NUM_SLOTS(header) == 0); + + mb = ms_matchbuffer(context); + position = mb->offset; + + $Dst = context; + } else if (is_binary_header(header)) { + ErlBinMatchState *ms; + + $GC_TEST_PRESERVE(ERL_BIN_MATCHSTATE_SIZE(0), live, context); + HEAP_TOP(c_p) = HTOP; +#ifdef DEBUG + c_p->stop = E; /* Needed for checking in HeapOnlyAlloc(). */ +#endif + ms = erts_bs_start_match_3(c_p, context); + HTOP = HEAP_TOP(c_p); + HEAP_SPACE_VERIFIED(0); + + if (ms == NULL) { + $FAIL($Fail); + } + + $REFRESH_GEN_DEST(); + $Dst = make_matchstate(ms); + position = ms->mb.offset; + } else { + $FAIL($Fail); + } + + ASSERT(IS_USMALL(0, position)); + $Pos = make_small(position); +} + +i_bs_start_match3(Src, Live, Fail, Dst) { + Eterm context, header; + Uint live; + + context = $Src; + live = $Live; + + if (!is_boxed(context)) { + $FAIL($Fail); + } + + header = *boxed_val(context); + + if (header_is_bin_matchstate(header)) { + ASSERT(HEADER_NUM_SLOTS(header) == 0); + $Dst = context; + } else if (is_binary_header(header)) { + ErlBinMatchState *ms; + + $GC_TEST_PRESERVE(ERL_BIN_MATCHSTATE_SIZE(0), live, context); + HEAP_TOP(c_p) = HTOP; +#ifdef DEBUG + c_p->stop = E; /* Needed for checking in HeapOnlyAlloc(). */ +#endif + ms = erts_bs_start_match_3(c_p, context); + HTOP = HEAP_TOP(c_p); + HEAP_SPACE_VERIFIED(0); + + if (ms == NULL) { + $FAIL($Fail); + } + + $REFRESH_GEN_DEST(); + $Dst = make_matchstate(ms); + } else { + $FAIL($Fail); + } +} + +bs_set_position(Ctx, Pos) { + ErlBinMatchBuffer* mb; + Eterm context; + + context = $Ctx; + ASSERT(header_is_bin_matchstate(*boxed_val(context))); + + mb = ms_matchbuffer(context); + mb->offset = unsigned_val($Pos); +} + +i_bs_get_position(Ctx, Dst) { + ErlBinMatchBuffer* mb; + Eterm context; + + context = $Ctx; + ASSERT(header_is_bin_matchstate(*boxed_val(context))); + + mb = ms_matchbuffer(context); + $Dst = make_small(mb->offset); +} + +%else + +# +# Unlike their 64-bit counterparts, the 32-bit position instructions operate on +# an offset from the "base position" of the context because storing raw +# positions would lead to the creation of far too many bigints. +# +# When a match context is reused we check whether its position fits into an +# immediate, and create a new match context if it does not. This means we only +# have to allocate stuff roughly once every 16MB rather than every time we +# match at a position beyond 16MB. +# + +bs_set_position(Ctx, Pos) { + Eterm context, position; + ErlBinMatchState *ms; + + context = $Ctx; + position = $Pos; + + ASSERT(header_is_bin_matchstate(*boxed_val(context))); + ms = (ErlBinMatchState*)boxed_val(context); + + if (ERTS_LIKELY(is_small(position))) { + ms->mb.offset = ms->save_offset[0] + unsigned_val(position); + } else { + ASSERT(is_big(position)); + ms->mb.offset = ms->save_offset[0] + *BIG_V(big_val(position)); + } +} + +bs_get_position(Ctx, Dst, Live) { + ErlBinMatchState *ms; + Eterm context; + Uint position; + + context = $Ctx; + + ASSERT(header_is_bin_matchstate(*boxed_val(context))); + ms = (ErlBinMatchState*)boxed_val(context); + + position = ms->mb.offset - ms->save_offset[0]; + + if (ERTS_LIKELY(IS_USMALL(0, position))) { + $Dst = make_small(position); + } else { + Eterm *hp; + + $GC_TEST_PRESERVE(BIG_UINT_HEAP_SIZE, $Live, context); + + hp = HTOP; + HTOP += BIG_UINT_HEAP_SIZE; + + *hp = make_pos_bignum_header(1); + BIG_DIGIT(hp, 0) = position; + + $REFRESH_GEN_DEST(); + $Dst = make_big(hp); + } +} + +i_bs_start_match3(Src, Live, Fail, Dst) { + Eterm context, header; + Uint live; + + context = $Src; + live = $Live; + + if (!is_boxed(context)) { + $FAIL($Fail); + } + + header = *boxed_val(context); + + if (header_is_bin_matchstate(header)) { + ErlBinMatchState *current_ms; + Uint position; + + ASSERT(HEADER_NUM_SLOTS(header) == 1); + + current_ms = (ErlBinMatchState*)boxed_val(context); + position = current_ms->mb.offset - current_ms->save_offset[0]; + + if (ERTS_LIKELY(IS_USMALL(0, position))) { + $Dst = context; + } else { + ErlBinMatchState *new_ms; + + $GC_TEST_PRESERVE(ERL_BIN_MATCHSTATE_SIZE(1), live, context); + current_ms = (ErlBinMatchState*)boxed_val(context); + + new_ms = (ErlBinMatchState*)HTOP; + HTOP += ERL_BIN_MATCHSTATE_SIZE(1); + + new_ms->thing_word = HEADER_BIN_MATCHSTATE(1); + new_ms->save_offset[0] = current_ms->mb.offset; + new_ms->mb = current_ms->mb; + + $REFRESH_GEN_DEST(); + $Dst = make_matchstate(new_ms); + } + } else if (is_binary_header(header)) { + Eterm result; + + $GC_TEST_PRESERVE(ERL_BIN_MATCHSTATE_SIZE(1), live, context); + HEAP_TOP(c_p) = HTOP; + +#ifdef DEBUG + c_p->stop = E; /* Needed for checking in HeapOnlyAlloc(). */ +#endif + + /* We intentionally use erts_bs_start_match_2 so that we can use + * save_offset as a base for all saved positions on this context, + * allowing us to avoid bigints for much longer. */ + result = erts_bs_start_match_2(c_p, context, 1); + + HTOP = HEAP_TOP(c_p); + HEAP_SPACE_VERIFIED(0); + + if (is_non_value(result)) { + $FAIL($Fail); + } + + $REFRESH_GEN_DEST(); + $Dst = result; + } else { + $FAIL($Fail); + } +} + +%endif diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index ff919082c3..5b3f091ccc 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -471,6 +471,9 @@ static BMData *create_bmdata(MyAllocator *my, byte *x, Uint len, Binary **the_bin /* out */) { Uint datasize; + BMData *bmd; + Binary *mb; + byte *data; if(len > 1) { datasize = BM_SIZE_MULTI(len); @@ -478,9 +481,8 @@ static BMData *create_bmdata(MyAllocator *my, byte *x, Uint len, datasize = BM_SIZE_SINGLE(); } - BMData *bmd; - Binary *mb = erts_create_magic_binary(datasize,cleanup_my_data_bm); - byte *data = ERTS_MAGIC_BIN_DATA(mb); + mb = erts_create_magic_binary(datasize,cleanup_my_data_bm); + data = ERTS_MAGIC_BIN_DATA(mb); init_my_allocator(my, datasize, data); bmd = my_alloc(my, sizeof(BMData)); bmd->x = my_alloc(my,len); diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 3a16913473..e82c776e70 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -144,6 +144,42 @@ erts_bs_start_match_2(Process *p, Eterm Binary, Uint Max) return make_matchstate(ms); } +ErlBinMatchState *erts_bs_start_match_3(Process *p, Eterm Binary) +{ + Eterm Orig; + Uint offs; + Uint* hp; + Uint NeededSize; + ErlBinMatchState *ms; + Uint bitoffs; + Uint bitsize; + Uint total_bin_size; + ProcBin* pb; + + ASSERT(is_binary(Binary)); + total_bin_size = binary_size(Binary); + if ((total_bin_size >> (8*sizeof(Uint)-3)) != 0) { + return NULL; + } + + NeededSize = ERL_BIN_MATCHSTATE_SIZE(0); + hp = HeapOnlyAlloc(p, NeededSize); + ms = (ErlBinMatchState *) hp; + ERTS_GET_REAL_BIN(Binary, Orig, offs, bitoffs, bitsize); + pb = (ProcBin *) boxed_val(Orig); + if (pb->thing_word == HEADER_PROC_BIN && pb->flags != 0) { + erts_emasculate_writable_binary(pb); + } + + ms->thing_word = HEADER_BIN_MATCHSTATE(0); + (ms->mb).orig = Orig; + (ms->mb).base = binary_bytes(Orig); + (ms->mb).offset = 8 * offs + bitoffs; + (ms->mb).size = total_bin_size * 8 + (ms->mb).offset + bitsize; + + return ms; +} + #ifdef DEBUG # define CHECK_MATCH_BUFFER(MB) check_match_buffer(MB) diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h index 7beef5cfda..50d353e1fa 100644 --- a/erts/emulator/beam/erl_bits.h +++ b/erts/emulator/beam/erl_bits.h @@ -73,12 +73,16 @@ struct erl_bits_state { typedef struct erl_bin_match_struct{ Eterm thing_word; ErlBinMatchBuffer mb; /* Present match buffer */ - Eterm save_offset[1]; /* Saved offsets */ + Eterm save_offset[1]; /* Saved offsets, only valid for contexts + * created through bs_start_match2. */ } ErlBinMatchState; -#define ERL_BIN_MATCHSTATE_SIZE(_Max) ((sizeof(ErlBinMatchState) + (_Max)*sizeof(Eterm))/sizeof(Eterm)) -#define HEADER_BIN_MATCHSTATE(_Max) _make_header(ERL_BIN_MATCHSTATE_SIZE((_Max))-1, _TAG_HEADER_BIN_MATCHSTATE) -#define HEADER_NUM_SLOTS(hdr) (header_arity(hdr)-sizeof(ErlBinMatchState)/sizeof(Eterm)+1) +#define ERL_BIN_MATCHSTATE_SIZE(_Max) \ + ((offsetof(ErlBinMatchState, save_offset) + (_Max)*sizeof(Eterm))/sizeof(Eterm)) +#define HEADER_BIN_MATCHSTATE(_Max) \ + _make_header(ERL_BIN_MATCHSTATE_SIZE((_Max)) - 1, _TAG_HEADER_BIN_MATCHSTATE) +#define HEADER_NUM_SLOTS(hdr) \ + (header_arity(hdr) - (offsetof(ErlBinMatchState, save_offset) / sizeof(Eterm)) + 1) #define make_matchstate(_Ms) make_boxed((Eterm*)(_Ms)) #define ms_matchbuffer(_Ms) &(((ErlBinMatchState*) boxed_val(_Ms))->mb) @@ -144,6 +148,7 @@ void erts_bits_destroy_state(ERL_BITS_PROTO_0); */ Eterm erts_bs_start_match_2(Process *p, Eterm Bin, Uint Max); +ErlBinMatchState *erts_bs_start_match_3(Process *p, Eterm Bin); Eterm erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer* mb); Eterm erts_bs_get_binary_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer* mb); Eterm erts_bs_get_float_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer* mb); diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 1df972f4b6..3653c0bf7c 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -409,30 +409,17 @@ static void free_dbtable(void *vtb) { DbTable *tb = (DbTable *) vtb; -#ifdef HARDDEBUG - if (erts_atomic_read_nob(&tb->common.memory_size) != sizeof(DbTable)) { - erts_fprintf(stderr, "ets: free_dbtable memory remain=%ld fix=%x\n", - erts_atomic_read_nob(&tb->common.memory_size)-sizeof(DbTable), - tb->common.fixations); - } -#endif - if (erts_atomic_read_nob(&tb->common.memory_size) > sizeof(DbTable)) { - /* The CA tree implementation use delayed freeing and the DbTable needs to - be freed after all other memory blocks that are allocated by the table. */ - erts_schedule_thr_prgr_later_cleanup_op(free_dbtable, - (void *) tb, - &tb->release.data, - sizeof(DbTable)); - return; - } - erts_rwmtx_destroy(&tb->common.rwlock); - erts_mtx_destroy(&tb->common.fixlock); - ASSERT(is_immed(tb->common.heir_data)); - if (tb->common.btid) - erts_bin_release(tb->common.btid); + ASSERT(erts_atomic_read_nob(&tb->common.memory_size) == sizeof(DbTable)); + + erts_rwmtx_destroy(&tb->common.rwlock); + erts_mtx_destroy(&tb->common.fixlock); + ASSERT(is_immed(tb->common.heir_data)); + + if (tb->common.btid) + erts_bin_release(tb->common.btid); - erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable)); + erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable)); } static void schedule_free_dbtable(DbTable* tb) diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index 45d120ac0e..7a915ccea2 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -286,6 +286,12 @@ ERTS_GLB_INLINE void erts_db_free(ErtsAlcType_t type, void *ptr, Uint size); +ERTS_GLB_INLINE void erts_schedule_db_free(DbTableCommon* tab, + void (*free_func)(void *), + void *ptr, + ErtsThrPrgrLaterOp *lop, + Uint size); + ERTS_GLB_INLINE void erts_db_free_nt(ErtsAlcType_t type, void *ptr, Uint size); @@ -306,6 +312,26 @@ erts_db_free(ErtsAlcType_t type, DbTable *tab, void *ptr, Uint size) } ERTS_GLB_INLINE void +erts_schedule_db_free(DbTableCommon* tab, + void (*free_func)(void *), + void *ptr, + ErtsThrPrgrLaterOp *lop, + Uint size) +{ + ASSERT(ptr != 0); + ASSERT(((void *) tab) != ptr); + ASSERT(size == ERTS_ALC_DBG_BLK_SZ(ptr)); + + /* + * We update table memory stats here as table may already be gone + * when 'free_func' is finally called. + */ + ERTS_DB_ALC_MEM_UPDATE_((DbTable*)tab, size, 0); + + erts_schedule_thr_prgr_later_cleanup_op(free_func, ptr, lop, size); +} + +ERTS_GLB_INLINE void erts_db_free_nt(ErtsAlcType_t type, void *ptr, Uint size) { ASSERT(ptr != 0); diff --git a/erts/emulator/beam/erl_db_catree.c b/erts/emulator/beam/erl_db_catree.c index 37a299df35..a8e48bce1b 100644 --- a/erts/emulator/beam/erl_db_catree.c +++ b/erts/emulator/beam/erl_db_catree.c @@ -210,33 +210,32 @@ DbTableMethod db_catree = * Internal CA tree related helper functions and macros */ -#define GET_ROUTE_NODE_KEY(node) (node->baseOrRoute.route.key.tpl[0]) -#define GET_BASE_NODE_LOCK(node) (&(node->baseOrRoute.base.lock)) -#define GET_ROUTE_NODE_LOCK(node) (&(node->baseOrRoute.route.lock)) +#define GET_ROUTE_NODE_KEY(node) (node->u.route.key.term) +#define GET_BASE_NODE_LOCK(node) (&(node->u.base.lock)) +#define GET_ROUTE_NODE_LOCK(node) (&(node->u.route.lock)) /* Helpers for reading and writing shared atomic variables */ /* No memory barrier */ #define GET_ROOT(tb) ((DbTableCATreeNode*)erts_atomic_read_nob(&(tb->root))) -#define GET_LEFT(ca_tree_route_node) ((DbTableCATreeNode*)erts_atomic_read_nob(&(ca_tree_route_node->baseOrRoute.route.left))) -#define GET_RIGHT(ca_tree_route_node) ((DbTableCATreeNode*)erts_atomic_read_nob(&(ca_tree_route_node->baseOrRoute.route.right))) +#define GET_LEFT(ca_tree_route_node) ((DbTableCATreeNode*)erts_atomic_read_nob(&(ca_tree_route_node->u.route.left))) +#define GET_RIGHT(ca_tree_route_node) ((DbTableCATreeNode*)erts_atomic_read_nob(&(ca_tree_route_node->u.route.right))) #define SET_ROOT(tb, v) erts_atomic_set_nob(&((tb)->root), (erts_aint_t)(v)) -#define SET_LEFT(ca_tree_route_node, v) erts_atomic_set_nob(&(ca_tree_route_node->baseOrRoute.route.left), (erts_aint_t)(v)); -#define SET_RIGHT(ca_tree_route_node, v) erts_atomic_set_nob(&(ca_tree_route_node->baseOrRoute.route.right), (erts_aint_t)(v)); +#define SET_LEFT(ca_tree_route_node, v) erts_atomic_set_nob(&(ca_tree_route_node->u.route.left), (erts_aint_t)(v)); +#define SET_RIGHT(ca_tree_route_node, v) erts_atomic_set_nob(&(ca_tree_route_node->u.route.right), (erts_aint_t)(v)); /* Release or acquire barriers */ #define GET_ROOT_ACQB(tb) ((DbTableCATreeNode*)erts_atomic_read_acqb(&(tb->root))) -#define GET_LEFT_ACQB(ca_tree_route_node) ((DbTableCATreeNode*)erts_atomic_read_acqb(&(ca_tree_route_node->baseOrRoute.route.left))) -#define GET_RIGHT_ACQB(ca_tree_route_node) ((DbTableCATreeNode*)erts_atomic_read_acqb(&(ca_tree_route_node->baseOrRoute.route.right))) +#define GET_LEFT_ACQB(ca_tree_route_node) ((DbTableCATreeNode*)erts_atomic_read_acqb(&(ca_tree_route_node->u.route.left))) +#define GET_RIGHT_ACQB(ca_tree_route_node) ((DbTableCATreeNode*)erts_atomic_read_acqb(&(ca_tree_route_node->u.route.right))) #define SET_ROOT_RELB(tb, v) erts_atomic_set_relb(&((tb)->root), (erts_aint_t)(v)) -#define SET_LEFT_RELB(ca_tree_route_node, v) erts_atomic_set_relb(&(ca_tree_route_node->baseOrRoute.route.left), (erts_aint_t)(v)); -#define SET_RIGHT_RELB(ca_tree_route_node, v) erts_atomic_set_relb(&(ca_tree_route_node->baseOrRoute.route.right), (erts_aint_t)(v)); +#define SET_LEFT_RELB(ca_tree_route_node, v) erts_atomic_set_relb(&(ca_tree_route_node->u.route.left), (erts_aint_t)(v)); +#define SET_RIGHT_RELB(ca_tree_route_node, v) erts_atomic_set_relb(&(ca_tree_route_node->u.route.right), (erts_aint_t)(v)); /* Compares a key to the key in a route node */ -static ERTS_INLINE Sint cmp_key_route(DbTableCommon * tb, - Eterm key, +static ERTS_INLINE Sint cmp_key_route(Eterm key, DbTableCATreeNode *obj) { return CMP(key, GET_ROUTE_NODE_KEY(obj)); @@ -279,7 +278,7 @@ int less_than_two_elements(TreeDbTerm *root) * Inserts a TreeDbTerm into a tree. Returns the new root. */ static ERTS_INLINE -TreeDbTerm* insert_TreeDbTerm(DbTableCommon *common_table_data, +TreeDbTerm* insert_TreeDbTerm(DbTableCATree *tb, TreeDbTerm *insert_to_root, TreeDbTerm *value_to_insert) { /* Non recursive insertion in AVL tree, building our own stack */ @@ -295,7 +294,7 @@ TreeDbTerm* insert_TreeDbTerm(DbTableCommon *common_table_data, int dir; TreeDbTerm *p1, *p2, *p; - key = GETKEY(common_table_data, value_to_insert->dbterm.tpl); + key = GETKEY(tb, value_to_insert->dbterm.tpl); dstack[dpos++] = DIR_END; for (;;) @@ -305,7 +304,7 @@ TreeDbTerm* insert_TreeDbTerm(DbTableCommon *common_table_data, (*this)->balance = 0; (*this)->left = (*this)->right = NULL; break; - } else if ((c = cmp_key(common_table_data, key, *this)) < 0) { + } else if ((c = cmp_key(&tb->common, key, *this)) < 0) { /* go lefts */ dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; @@ -392,7 +391,7 @@ TreeDbTerm* insert_TreeDbTerm(DbTableCommon *common_table_data, * left_wb and the tree containing the rest of the keys in the write * back parameter right_wb. */ -static void split_tree(DbTableCommon *tb, +static void split_tree(DbTableCATree *tb, TreeDbTerm *root, TreeDbTerm **split_key_node_wb, TreeDbTerm **left_wb, @@ -413,9 +412,7 @@ static void split_tree(DbTableCommon *tb, split_node->left = NULL; right_root = split_node->right; split_node->right = NULL; - right_root = insert_TreeDbTerm(tb, - right_root, - split_node); + right_root = insert_TreeDbTerm(tb, right_root, split_node); *split_key_node_wb = split_node; *left_wb = left_root; *right_wb = right_root; @@ -700,15 +697,17 @@ void runlock_base_node(DbTableCATreeBaseNode *base_node) } static ERTS_INLINE -void lock_route_node(DbTableCATreeRouteNode *route_node) +void lock_route_node(DbTableCATreeNode *route_node) { - erts_mtx_lock(&route_node->lock); + ASSERT(!route_node->is_base_node); + erts_mtx_lock(&route_node->u.route.lock); } static ERTS_INLINE -void unlock_route_node(DbTableCATreeRouteNode *route_node) +void unlock_route_node(DbTableCATreeNode *route_node) { - erts_mtx_unlock(&route_node->lock); + ASSERT(!route_node->is_base_node); + erts_mtx_unlock(&route_node->u.route.lock); } @@ -722,7 +721,6 @@ void unlock_route_node(DbTableCATreeRouteNode *route_node) int retry; \ DbTableCATreeNode *current_node; \ DbTableCATreeNode *prev_node; \ - DbTableCommon* common_table_data = &tb->common; \ DbTableCATreeBaseNode *base_node; \ int current_level; \ (void)prev_node; \ @@ -734,13 +732,13 @@ void unlock_route_node(DbTableCATreeRouteNode *route_node) while ( ! current_node->is_base_node ) { \ current_level = current_level + 1; \ prev_node = current_node; \ - if (cmp_key_route(common_table_data,key,current_node) < 0) { \ + if (cmp_key_route(key,current_node) < 0) { \ current_node = GET_LEFT_ACQB(current_node); \ } else { \ current_node = GET_RIGHT_ACQB(current_node); \ } \ } \ - base_node = ¤t_node->baseOrRoute.base; \ + base_node = ¤t_node->u.base; \ LOCK(base_node); \ if ( ! base_node->is_valid ) { \ /* Retry */ \ @@ -753,7 +751,7 @@ void unlock_route_node(DbTableCATreeRouteNode *route_node) #define ERL_DB_CATREE_CREATE_DO_OPERATION_FUNCTION_ADAPT_AND_UNLOCK_PART \ if (base_node->lock_statistics > ERL_DB_CATREE_HIGH_CONTENTION_LIMIT \ && current_level < ERL_DB_CATREE_MAX_ROUTE_NODE_LAYER_HEIGHT) { \ - split_catree(&tb->common, prev_node, current_node); \ + split_catree(tb, prev_node, current_node); \ } else if (base_node->lock_statistics < ERL_DB_CATREE_LOW_CONTENTION_LIMIT) { \ join_catree(tb, prev_node, current_node); \ } else { \ @@ -784,141 +782,189 @@ void unlock_route_node(DbTableCATreeRouteNode *route_node) } +static ERTS_INLINE +void copy_route_key(DbRouteKey* dst, Eterm key, Uint key_size) +{ + dst->size = key_size; + if (key_size != 0) { + Eterm* hp = &dst->heap[0]; + ErlOffHeap tmp_offheap; + tmp_offheap.first = NULL; + dst->term = copy_struct(key, key_size, &hp, &tmp_offheap); + dst->oh = tmp_offheap.first; + } + else { + ASSERT(is_immed(key)); + dst->term = key; + dst->oh = NULL; + } +} -static DbTableCATreeNode *create_catree_base_node(DbTableCATree *tb) +static ERTS_INLINE +void destroy_route_key(DbRouteKey* key) { - DbTableCATreeNode *new_base_node_container = - erts_db_alloc(ERTS_ALC_T_DB_TABLE, - (DbTable *) tb, - sizeof(DbTableCATreeNode)); - DbTableCATreeBaseNode *new_base_node = - &new_base_node_container->baseOrRoute.base; + if (key->oh) { + ErlOffHeap oh; + oh.first = key->oh; + erts_cleanup_offheap(&oh); + } +} + + +#ifdef ERTS_ENABLE_LOCK_CHECK +# define sizeof_base_node(KEY_SZ) \ + (offsetof(DbTableCATreeNode, u.base.lc_key.heap) \ + + (KEY_SZ)*sizeof(Eterm)) +# define LC_ORDER(ORDER) ORDER +#else +# define sizeof_base_node(KEY_SZ) \ + offsetof(DbTableCATreeNode, u.base.end_of_struct__) +# define LC_ORDER(ORDER) NIL +#endif + +static DbTableCATreeNode *create_base_node(DbTableCATree *tb, + TreeDbTerm* root, + Eterm lc_key) +{ + DbTableCATreeNode *p; erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER; - new_base_node_container->is_base_node = 1; - new_base_node->root = NULL; +#ifdef ERTS_ENABLE_LOCK_CHECK + Eterm lc_key_size = size_object(lc_key); +#endif + p = erts_db_alloc(ERTS_ALC_T_DB_TABLE, (DbTable *) tb, + sizeof_base_node(lc_key_size)); + + p->is_base_node = 1; + p->u.base.root = root; if (tb->common.type & DB_FREQ_READ) rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ; if (erts_ets_rwmtx_spin_count >= 0) rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count; - erts_rwmtx_init_opt(&new_base_node->lock, &rwmtx_opt, - "erl_db_catree_base_node", tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB); - new_base_node->lock_statistics = 0; - new_base_node->is_valid = 1; - new_base_node->tab = (DbTable *) tb; - return new_base_node_container; + +#ifdef ERTS_ENABLE_LOCK_CHECK + copy_route_key(&p->u.base.lc_key, lc_key, lc_key_size); +#endif + erts_rwmtx_init_opt(&p->u.base.lock, &rwmtx_opt, + "erl_db_catree_base_node", + lc_key, + ERTS_LOCK_FLAGS_CATEGORY_DB); + p->u.base.lock_statistics = 0; + p->u.base.is_valid = 1; + return p; +} + +static ERTS_INLINE +DbTableCATreeNode *create_wlocked_base_node(DbTableCATree *tb, + TreeDbTerm* root, + Eterm lc_key) +{ + DbTableCATreeNode* p = create_base_node(tb, root, lc_key); + ethr_rwmutex_rwlock(&p->u.base.lock.rwmtx); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_trylock_flg(-1, &p->u.base.lock.lc, ERTS_LOCK_OPTIONS_RDWR); +#endif + return p; +} + + +static ERTS_INLINE Uint sizeof_route_node(Uint key_size) +{ + return (offsetof(DbTableCATreeNode, u.route.key.heap) + + key_size*sizeof(Eterm)); } static DbTableCATreeNode* -create_catree_route_node(DbTableCommon * common_table_data, - DbTableCATreeNode *left, - DbTableCATreeNode *right, - DbTerm * keyTerm) +create_route_node(DbTableCATree *tb, + DbTableCATreeNode *left, + DbTableCATreeNode *right, + DbTerm * keyTerm, + DbTableCATreeNode* lc_parent) { - Eterm* top; - Eterm key = GETKEY(common_table_data,keyTerm->tpl); + Eterm key = GETKEY(tb,keyTerm->tpl); int key_size = size_object(key); - Uint offset = offsetof(DbTableCATreeNode,baseOrRoute) + - offsetof(DbTableCATreeRouteNode,key); - size_t route_node_container_size = - offset + - sizeof(DbTerm) + - sizeof(Eterm)*key_size; - ErlOffHeap tmp_offheap; - byte* new_route_node_container_bytes = - erts_db_alloc(ERTS_ALC_T_DB_TABLE, - (DbTable *) common_table_data, - route_node_container_size); - DbTerm* newp = (DbTerm*) (new_route_node_container_bytes + offset); - DbTableCATreeNode *new_route_node_container = - (DbTableCATreeNode*)new_route_node_container_bytes; - DbTableCATreeRouteNode *new_route_node = - &new_route_node_container->baseOrRoute.route; - new_route_node->tab = (DbTable *)common_table_data; - if (key_size != 0) { - newp->size = key_size; - top = &newp->tpl[1]; - tmp_offheap.first = NULL; - newp->tpl[0] = copy_struct(key, key_size, &top, &tmp_offheap); - newp->first_oh = tmp_offheap.first; - } else { - newp->size = key_size; - newp->first_oh = NULL; - newp->tpl[0] = key; - } - new_route_node_container->is_base_node = 0; - new_route_node->is_valid = 1; - erts_atomic_init_nob(&(new_route_node->left), (erts_aint_t)left); - erts_atomic_init_nob(&(new_route_node->right), (erts_aint_t)right); - erts_mtx_init(&new_route_node->lock, "erl_db_catree_route_node", - NIL, ERTS_LOCK_FLAGS_CATEGORY_DB); - return new_route_node_container; + DbTableCATreeNode* p = erts_db_alloc(ERTS_ALC_T_DB_TABLE, + (DbTable *) tb, + sizeof_route_node(key_size)); + + copy_route_key(&p->u.route.key, key, key_size); + p->is_base_node = 0; + p->u.route.is_valid = 1; + erts_atomic_init_nob(&p->u.route.left, (erts_aint_t)left); + erts_atomic_init_nob(&p->u.route.right, (erts_aint_t)right); +#ifdef ERTS_ENABLE_LOCK_CHECK + /* Route node lock order is inverse tree depth (from leafs toward root) */ + p->u.route.lc_order = (lc_parent == NULL ? MAX_SMALL : + lc_parent->u.route.lc_order - 1); + /* + * This assert may eventually fail as we don't increase 'lc_order' in join + * operations when route nodes move up in the tree. + * Tough luck if you run a lock-checking VM for such a long time on 32-bit. + */ + ERTS_LC_ASSERT(p->u.route.lc_order >= 0); +#endif + erts_mtx_init(&p->u.route.lock, "erl_db_catree_route_node", + LC_ORDER(make_small(p->u.route.lc_order)), + ERTS_LOCK_FLAGS_CATEGORY_DB); + return p; } -static void free_catree_base_node(void* base_node_container_ptr) +static void do_free_base_node(void* vptr) { - DbTableCATreeNode *base_node_container = - (DbTableCATreeNode *)base_node_container_ptr; - DbTableCATreeBaseNode *base_node = - &base_node_container->baseOrRoute.base; - erts_rwmtx_destroy(&base_node->lock); - erts_db_free(ERTS_ALC_T_DB_TABLE, - base_node->tab, base_node_container, - sizeof(DbTableCATreeNode)); -} - -static void free_catree_routing_node(void *route_node_container_ptr) -{ - size_t route_node_container_size; - byte* route_node_container_bytes = route_node_container_ptr; - DbTableCATreeNode *route_node_container = - (DbTableCATreeNode *)route_node_container_bytes; - DbTableCATreeRouteNode *route_node = - &route_node_container->baseOrRoute.route; - int key_size = route_node->key.size; - Uint offset = offsetof(DbTableCATreeNode,baseOrRoute) + - offsetof(DbTableCATreeRouteNode,key); - ErlOffHeap tmp_oh; - DbTerm* db_term = (DbTerm*) (route_node_container_bytes + offset); - erts_mtx_destroy(&route_node->lock); - route_node_container_size = - offset + - sizeof(DbTerm) + - sizeof(Eterm)*key_size; - if (key_size != 0) { - tmp_oh.first = db_term->first_oh; - erts_cleanup_offheap(&tmp_oh); - } - erts_db_free(ERTS_ALC_T_DB_TABLE, - route_node->tab, - route_node_container, - route_node_container_size); + DbTableCATreeNode *p = (DbTableCATreeNode *)vptr; + ASSERT(p->is_base_node); + erts_rwmtx_destroy(&p->u.base.lock); +#ifdef ERTS_ENABLE_LOCK_CHECK + destroy_route_key(&p->u.base.lc_key); +#endif + erts_free(ERTS_ALC_T_DB_TABLE, p); +} + +static void free_catree_base_node(DbTableCATree* tb, DbTableCATreeNode* p) +{ + ASSERT(p->is_base_node); + ERTS_DB_ALC_MEM_UPDATE_(tb, sizeof_base_node(p->u.base.lc_key.size), 0); + do_free_base_node(p); +} + +static void do_free_route_node(void *vptr) +{ + DbTableCATreeNode *p = (DbTableCATreeNode *)vptr; + ASSERT(!p->is_base_node); + erts_mtx_destroy(&p->u.route.lock); + destroy_route_key(&p->u.route.key); + erts_free(ERTS_ALC_T_DB_TABLE, p); } +static void free_catree_route_node(DbTableCATree* tb, DbTableCATreeNode* p) +{ + ASSERT(!p->is_base_node); + ERTS_DB_ALC_MEM_UPDATE_(tb, sizeof_route_node(p->u.route.key.size), 0); + do_free_route_node(p); +} + + /* * Returns the parent routing node of the specified - * route_node_container if such a routing node exists or NULL if - * route_node_container is attached to the root + * route node 'child' if such a parent exists + * or NULL if 'child' is attached to the root. */ static ERTS_INLINE DbTableCATreeNode * parent_of(DbTableCATree *tb, - DbTableCATreeNode *route_node_container) + DbTableCATreeNode *child) { + Eterm key = GET_ROUTE_NODE_KEY(child); + DbTableCATreeNode *current = GET_ROOT_ACQB(tb); + DbTableCATreeNode *prev = NULL; - Eterm key = GET_ROUTE_NODE_KEY(route_node_container); - DbTableCATreeNode *current_node = GET_ROOT_ACQB(tb); - DbTableCATreeNode *prev_node = NULL; - if (current_node == route_node_container) { - return NULL; - } - while (current_node != route_node_container) { - prev_node = current_node; - if (cmp_key_route((DbTableCommon *)tb, key, current_node) < 0) { - current_node = GET_LEFT_ACQB(current_node); + while (current != child) { + prev = current; + if (cmp_key_route(key, current) < 0) { + current = GET_LEFT_ACQB(current); } else { - current_node = GET_RIGHT_ACQB(current_node); + current = GET_RIGHT_ACQB(current); } } - return prev_node; + return prev; } @@ -953,11 +999,7 @@ leftmost_route_node(DbTableCATreeNode *root) prev_node = node; node = GET_LEFT_ACQB(node); } - if (prev_node == NULL) { - return NULL; - } else { - return prev_node; - } + return prev_node; } static ERTS_INLINE DbTableCATreeNode* @@ -969,11 +1011,7 @@ rightmost_route_node(DbTableCATreeNode *root) prev_node = node; node = GET_RIGHT_ACQB(node); } - if (prev_node == NULL) { - return NULL; - } else { - return prev_node; - } + return prev_node; } static ERTS_INLINE DbTableCATreeNode* @@ -988,8 +1026,7 @@ leftmost_base_node_and_path(DbTableCATreeNode *root, CATreeNodeStack * stack) } static ERTS_INLINE DbTableCATreeNode* -get_next_base_node_and_path(DbTableCommon *common_table_data, - DbTableCATreeNode *base_node, +get_next_base_node_and_path(DbTableCATreeNode *base_node, CATreeNodeStack *stack) { if (EMPTY_NODE(stack)) { /* The parent of b is the root */ @@ -1001,11 +1038,11 @@ get_next_base_node_and_path(DbTableCommon *common_table_data, stack); } else { Eterm pkey = - TOP_NODE(stack)->baseOrRoute.route.key.tpl[0]; /* pKey = key of parent */ + TOP_NODE(stack)->u.route.key.term; /* pKey = key of parent */ POP_NODE(stack); while (!EMPTY_NODE(stack)) { - if (TOP_NODE(stack)->baseOrRoute.route.is_valid && - cmp_key_route(common_table_data, pkey, TOP_NODE(stack)) <= 0) { + if (TOP_NODE(stack)->u.route.is_valid && + cmp_key_route(pkey, TOP_NODE(stack)) <= 0) { return leftmost_base_node_and_path(GET_RIGHT_ACQB(TOP_NODE(stack)), stack); } else { POP_NODE(stack); @@ -1034,25 +1071,23 @@ lock_first_base_node(DbTable *tbl, CATreeNodeStack *search_stack_ptr, CATreeNodeStack *locked_base_nodes_stack_ptr) { - int retry; DbTableCATreeNode *current_node; DbTableCATreeBaseNode *base_node; DbTableCATree* tb = &tbl->catree; - do { - retry = 0; + while (1) { current_node = GET_ROOT_ACQB(tb); while ( ! current_node->is_base_node ) { PUSH_NODE(search_stack_ptr, current_node); current_node = GET_LEFT_ACQB(current_node); } - base_node = ¤t_node->baseOrRoute.base; + base_node = ¤t_node->u.base; rlock_base_node(base_node); - if ( ! base_node->is_valid ) { - /* Retry */ - runlock_base_node(base_node); - retry = 1; - } - } while(retry); + if (base_node->is_valid) + break; + /* Retry */ + runlock_base_node(base_node); + search_stack_ptr->pos = 0; + } push_node_dyn_array(tbl, locked_base_nodes_stack_ptr, current_node); return current_node; } @@ -1066,19 +1101,20 @@ find_and_lock_next_base_node_and_path(DbTable *tbl, DbTableCATreeNode *current_node; DbTableCATreeBaseNode *base_node; CATreeNodeStack * tmp_stack_ptr; - DbTableCommon* common_table_data; - retry_find_and_lock_next_base_node: - current_node = TOP_NODE(locked_base_nodes_stack_ptr); - common_table_data = &tbl->common; - clone_stack(*search_stack_ptr_ptr, *search_stack_copy_ptr_ptr); - current_node = - get_next_base_node_and_path(common_table_data, current_node, *search_stack_ptr_ptr); - if (current_node == NULL) { - return NULL; - } - base_node = ¤t_node->baseOrRoute.base; - rlock_base_node(base_node); - if ( ! base_node->is_valid ) { + + while (1) { + current_node = TOP_NODE(locked_base_nodes_stack_ptr); + clone_stack(*search_stack_ptr_ptr, *search_stack_copy_ptr_ptr); + current_node = + get_next_base_node_and_path(current_node, *search_stack_ptr_ptr); + if (current_node == NULL) { + return NULL; + } + base_node = ¤t_node->u.base; + rlock_base_node(base_node); + if (base_node->is_valid) + break; + /* Retry */ runlock_base_node(base_node); /* Revert to previous state */ @@ -1086,10 +1122,9 @@ find_and_lock_next_base_node_and_path(DbTable *tbl, tmp_stack_ptr = *search_stack_ptr_ptr; *search_stack_ptr_ptr = *search_stack_copy_ptr_ptr; *search_stack_copy_ptr_ptr = tmp_stack_ptr; - goto retry_find_and_lock_next_base_node; - } else { - push_node_dyn_array(tbl, locked_base_nodes_stack_ptr, current_node); } + + push_node_dyn_array(tbl, locked_base_nodes_stack_ptr, current_node); return base_node; } @@ -1102,7 +1137,7 @@ void unlock_and_release_locked_base_node_stack(DbTable *tbl, int i; for (i = 0; i < locked_base_nodes_stack_ptr->pos; i++) { current_node = locked_base_nodes_stack_ptr->array[i]; - base_node = ¤t_node->baseOrRoute.base; + base_node = ¤t_node->u.base; if (locked_base_nodes_stack_ptr->pos > 1) { base_node->lock_statistics = /* This is not atomic which is fine as */ base_node->lock_statistics + /* correctness does not depend on that. */ @@ -1166,19 +1201,18 @@ lock_base_node_with_key(DbTable *tbl, DbTableCATreeNode *current_node; DbTableCATreeBaseNode *base_node; DbTableCATree* tb = &tbl->catree; - DbTableCommon* common_table_data = &tbl->common; do { retry = 0; current_node = GET_ROOT_ACQB(tb); while ( ! current_node->is_base_node ) { PUSH_NODE(search_stack_ptr, current_node); - if( cmp_key_route(common_table_data,key,current_node) < 0 ) { + if( cmp_key_route(key,current_node) < 0 ) { current_node = GET_LEFT_ACQB(current_node); } else { current_node = GET_RIGHT_ACQB(current_node); } } - base_node = ¤t_node->baseOrRoute.base; + base_node = ¤t_node->u.base; rlock_base_node(base_node); if ( ! base_node->is_valid ) { /* Retry */ @@ -1196,100 +1230,90 @@ lock_base_node_with_key(DbTable *tbl, * node to join with. */ static DbTableCATreeNode* -erl_db_catree_force_join_right(DbTableCommon *common_table_data, - DbTableCATreeNode *parent_container, - DbTableCATreeNode *base_container, +erl_db_catree_force_join_right(DbTableCATree *tb, + DbTableCATreeNode *parent, + DbTableCATreeNode *thiz, DbTableCATreeNode **result_parent_wb) { - DbTableCATreeRouteNode *parent; - DbTableCATreeNode *gparent_container; - DbTableCATreeRouteNode *gparent; - DbTableCATreeBaseNode *base = &base_container->baseOrRoute.base; - DbTableCATree *tb = (DbTableCATree *)common_table_data; - DbTableCATreeNode *neighbor_base_container; - DbTableCATreeBaseNode *neighbor_base; - DbTableCATreeNode *new_neighbor_base; - DbTableCATreeNode *neighbor_base_parent; - int neighbour_not_valid; - if (parent_container == NULL) { + DbTableCATreeNode *gparent; + DbTableCATreeNode *neighbor; + DbTableCATreeNode *new_neighbor; + DbTableCATreeNode *neighbor_parent; + TreeDbTerm* new_root; + + if (parent == NULL) { return NULL; } - parent = &parent_container->baseOrRoute.route; - do { - neighbor_base_container = leftmost_base_node(GET_RIGHT_ACQB(parent_container)); - neighbor_base = &neighbor_base_container->baseOrRoute.base; - wlock_base_node_no_stats(neighbor_base); - neighbour_not_valid = !neighbor_base->is_valid; - if (neighbour_not_valid) { - wunlock_base_node(neighbor_base); - } - } while (neighbour_not_valid); + ASSERT(thiz == GET_LEFT(parent)); + while (1) { + neighbor = leftmost_base_node(GET_RIGHT_ACQB(parent)); + wlock_base_node_no_stats(&neighbor->u.base); + if (neighbor->u.base.is_valid) + break; + wunlock_base_node(&neighbor->u.base); + } lock_route_node(parent); - parent->is_valid = 0; - neighbor_base->is_valid = 0; - base->is_valid = 0; - gparent = NULL; - gparent_container = NULL; - do { - if (gparent != NULL) { - unlock_route_node(gparent); - } - gparent_container = parent_of(tb, parent_container); - if (gparent_container != NULL) { - gparent = &gparent_container->baseOrRoute.route; - lock_route_node(gparent); - } else { - gparent = NULL; - } - } while (gparent != NULL && !gparent->is_valid); + parent->u.route.is_valid = 0; + neighbor->u.base.is_valid = 0; + thiz->u.base.is_valid = 0; + while (1) { + gparent = parent_of(tb, parent); + if (gparent == NULL) + break; + lock_route_node(gparent); + if (gparent->u.route.is_valid) + break; + unlock_route_node(gparent); + } if (gparent == NULL) { - SET_ROOT_RELB(tb, GET_RIGHT(parent_container)); - } else if (GET_LEFT(gparent_container) == parent_container) { - SET_LEFT_RELB(gparent_container, GET_RIGHT(parent_container)); + SET_ROOT_RELB(tb, GET_RIGHT(parent)); + } else if (GET_LEFT(gparent) == parent) { + SET_LEFT_RELB(gparent, GET_RIGHT(parent)); } else { - SET_RIGHT_RELB(gparent_container, GET_RIGHT(parent_container)); + SET_RIGHT_RELB(gparent, GET_RIGHT(parent)); } unlock_route_node(parent); if (gparent != NULL) { unlock_route_node(gparent); } - new_neighbor_base = create_catree_base_node(tb); - new_neighbor_base->baseOrRoute.base.root = - join_trees(base->root, neighbor_base->root); - wlock_base_node_no_stats(&(new_neighbor_base->baseOrRoute.base)); - neighbor_base_parent = NULL; - if (GET_RIGHT(parent_container) == neighbor_base_container) { - neighbor_base_parent = gparent_container; + + new_root = join_trees(thiz->u.base.root, neighbor->u.base.root); + new_neighbor = create_wlocked_base_node(tb, new_root, + LC_ORDER(thiz->u.base.lc_key.term)); + + if (GET_RIGHT(parent) == neighbor) { + neighbor_parent = gparent; } else { - neighbor_base_parent = - leftmost_route_node(GET_RIGHT(parent_container)); + neighbor_parent = leftmost_route_node(GET_RIGHT(parent)); } - if(neighbor_base_parent == NULL) { - SET_ROOT_RELB(tb, new_neighbor_base); - } else if (GET_LEFT(neighbor_base_parent) == neighbor_base_container) { - SET_LEFT_RELB(neighbor_base_parent, new_neighbor_base); + if(neighbor_parent == NULL) { + SET_ROOT_RELB(tb, new_neighbor); + } else if (GET_LEFT(neighbor_parent) == neighbor) { + SET_LEFT_RELB(neighbor_parent, new_neighbor); } else { - SET_RIGHT_RELB(neighbor_base_parent, new_neighbor_base); + SET_RIGHT_RELB(neighbor_parent, new_neighbor); } - wunlock_base_node(base); - wunlock_base_node(neighbor_base); + wunlock_base_node(&thiz->u.base); + wunlock_base_node(&neighbor->u.base); /* Free the parent and base */ - erts_schedule_thr_prgr_later_op(free_catree_routing_node, - parent_container, - &parent->free_item); - erts_schedule_thr_prgr_later_op(free_catree_base_node, - base_container, - &base->free_item); - erts_schedule_thr_prgr_later_op(free_catree_base_node, - neighbor_base_container, - &neighbor_base->free_item); - - if (parent_container == neighbor_base_container) { - *result_parent_wb = gparent_container; - } else { - *result_parent_wb = neighbor_base_parent; - } - return new_neighbor_base; + erts_schedule_db_free(&tb->common, + do_free_route_node, + parent, + &parent->u.route.free_item, + sizeof_route_node(parent->u.route.key.size)); + erts_schedule_db_free(&tb->common, + do_free_base_node, + thiz, + &thiz->u.base.free_item, + sizeof_base_node(thiz->u.base.lc_key.size)); + erts_schedule_db_free(&tb->common, + do_free_base_node, + neighbor, + &neighbor->u.base.free_item, + sizeof_base_node(neighbor->u.base.lc_key.size)); + + *result_parent_wb = neighbor_parent; + return new_neighbor; } /* @@ -1298,243 +1322,232 @@ erl_db_catree_force_join_right(DbTableCommon *common_table_data, * locked state. */ static DbTableCATreeNode * -merge_to_one_locked_base_node(DbTableCommon * common_table_data) +merge_to_one_locked_base_node(DbTableCATree* tb) { - DbTableCATreeNode *parent_container; - DbTableCATreeNode *new_parent_container; - DbTableCATree *tb = (DbTableCATree *)common_table_data; - DbTableCATreeNode *base_container; - DbTableCATreeNode *new_base_container; + DbTableCATreeNode *parent; + DbTableCATreeNode *new_parent; + DbTableCATreeNode *base; + DbTableCATreeNode *new_base; int is_not_valid; /* Find first base node */ do { - parent_container = NULL; - base_container = GET_ROOT_ACQB(tb); - while ( ! base_container->is_base_node ) { - parent_container = base_container; - base_container = GET_LEFT_ACQB(base_container); + parent = NULL; + base = GET_ROOT_ACQB(tb); + while ( ! base->is_base_node ) { + parent = base; + base = GET_LEFT_ACQB(base); } - wlock_base_node_no_stats(&(base_container->baseOrRoute.base)); - is_not_valid = ! base_container->baseOrRoute.base.is_valid; + wlock_base_node_no_stats(&(base->u.base)); + is_not_valid = ! base->u.base.is_valid; if (is_not_valid) { - wunlock_base_node(&(base_container->baseOrRoute.base)); + wunlock_base_node(&(base->u.base)); } } while(is_not_valid); do { - new_base_container = erl_db_catree_force_join_right(common_table_data, - parent_container, - base_container, - &new_parent_container); - if (new_base_container != NULL) { - base_container = new_base_container; - parent_container = new_parent_container; + new_base = erl_db_catree_force_join_right(tb, + parent, + base, + &new_parent); + if (new_base != NULL) { + base = new_base; + parent = new_parent; } - } while(new_base_container != NULL); - return base_container; + } while(new_base != NULL); + return base; } static void join_catree(DbTableCATree *tb, - DbTableCATreeNode *parent_container, - DbTableCATreeNode *base_container) -{ - DbTableCATreeRouteNode *parent; - DbTableCATreeNode *gparent_container; - DbTableCATreeRouteNode *gparent; - DbTableCATreeBaseNode *base = &base_container->baseOrRoute.base; - DbTableCATreeNode *neighbor_base_container; - DbTableCATreeBaseNode *neighbor_base; - DbTableCATreeNode *new_neighbor_base; - DbTableCATreeNode *neighbor_base_parent; - if (parent_container == NULL) { - base->lock_statistics = 0; - wunlock_base_node(base); + DbTableCATreeNode *parent, + DbTableCATreeNode *thiz) +{ + DbTableCATreeNode *gparent; + DbTableCATreeNode *neighbor; + DbTableCATreeNode *new_neighbor; + DbTableCATreeNode *neighbor_parent; + + ASSERT(thiz->is_base_node); + if (parent == NULL) { + thiz->u.base.lock_statistics = 0; + wunlock_base_node(&thiz->u.base); return; } - parent = &parent_container->baseOrRoute.route; - if (GET_LEFT(parent_container) == base_container) { - neighbor_base_container = leftmost_base_node(GET_RIGHT_ACQB(parent_container)); - neighbor_base = &neighbor_base_container->baseOrRoute.base; - if (try_wlock_base_node(neighbor_base)) { + ASSERT(!parent->is_base_node); + if (GET_LEFT(parent) == thiz) { + neighbor = leftmost_base_node(GET_RIGHT_ACQB(parent)); + if (try_wlock_base_node(&neighbor->u.base)) { /* Failed to acquire lock */ - base->lock_statistics = 0; - wunlock_base_node(base); + thiz->u.base.lock_statistics = 0; + wunlock_base_node(&thiz->u.base); return; - } else if (!neighbor_base->is_valid) { - base->lock_statistics = 0; - wunlock_base_node(base); - wunlock_base_node(neighbor_base); + } else if (!neighbor->u.base.is_valid) { + thiz->u.base.lock_statistics = 0; + wunlock_base_node(&thiz->u.base); + wunlock_base_node(&neighbor->u.base); return; } else { lock_route_node(parent); - parent->is_valid = 0; - neighbor_base->is_valid = 0; - base->is_valid = 0; + parent->u.route.is_valid = 0; + neighbor->u.base.is_valid = 0; + thiz->u.base.is_valid = 0; gparent = NULL; - gparent_container = NULL; do { if (gparent != NULL) { unlock_route_node(gparent); } - gparent_container = parent_of(tb, parent_container); - if (gparent_container != NULL) { - gparent = &gparent_container->baseOrRoute.route; + gparent = parent_of(tb, parent); + if (gparent != NULL) lock_route_node(gparent); - } else { - gparent = NULL; - } - } while (gparent != NULL && !gparent->is_valid); + } while (gparent != NULL && !gparent->u.route.is_valid); + if (gparent == NULL) { - SET_ROOT_RELB(tb, GET_RIGHT(parent_container)); - } else if (GET_LEFT(gparent_container) == parent_container) { - SET_LEFT_RELB(gparent_container, GET_RIGHT(parent_container)); + SET_ROOT_RELB(tb, GET_RIGHT(parent)); + } else if (GET_LEFT(gparent) == parent) { + SET_LEFT_RELB(gparent, GET_RIGHT(parent)); } else { - SET_RIGHT_RELB(gparent_container, GET_RIGHT(parent_container)); + SET_RIGHT_RELB(gparent, GET_RIGHT(parent)); } unlock_route_node(parent); if (gparent != NULL) { unlock_route_node(gparent); } - new_neighbor_base = create_catree_base_node(tb); - new_neighbor_base->baseOrRoute.base.root = - join_trees(base->root, neighbor_base->root); - neighbor_base_parent = NULL; - if (GET_RIGHT(parent_container) == neighbor_base_container) { - neighbor_base_parent = gparent_container; + { + TreeDbTerm* new_root = join_trees(thiz->u.base.root, + neighbor->u.base.root); + new_neighbor = create_base_node(tb, new_root, + LC_ORDER(thiz->u.base.lc_key.term)); + } + if (GET_RIGHT(parent) == neighbor) { + neighbor_parent = gparent; } else { - neighbor_base_parent = - leftmost_route_node(GET_RIGHT(parent_container)); + neighbor_parent = leftmost_route_node(GET_RIGHT(parent)); } } } else { /* Symetric case */ - neighbor_base_container = rightmost_base_node(GET_LEFT_ACQB(parent_container)); - neighbor_base = &neighbor_base_container->baseOrRoute.base; - if (try_wlock_base_node(neighbor_base)) { + ASSERT(GET_RIGHT(parent) == thiz); + neighbor = rightmost_base_node(GET_LEFT_ACQB(parent)); + if (try_wlock_base_node(&neighbor->u.base)) { /* Failed to acquire lock */ - base->lock_statistics = 0; - wunlock_base_node(base); + thiz->u.base.lock_statistics = 0; + wunlock_base_node(&thiz->u.base); return; - } else if (!neighbor_base->is_valid) { - base->lock_statistics = 0; - wunlock_base_node(base); - wunlock_base_node(neighbor_base); + } else if (!neighbor->u.base.is_valid) { + thiz->u.base.lock_statistics = 0; + wunlock_base_node(&thiz->u.base); + wunlock_base_node(&neighbor->u.base); return; } else { lock_route_node(parent); - parent->is_valid = 0; - neighbor_base->is_valid = 0; - base->is_valid = 0; + parent->u.route.is_valid = 0; + neighbor->u.base.is_valid = 0; + thiz->u.base.is_valid = 0; gparent = NULL; - gparent_container = NULL; do { if (gparent != NULL) { unlock_route_node(gparent); } - gparent_container = parent_of(tb, parent_container); - if (gparent_container != NULL) { - gparent = &gparent_container->baseOrRoute.route; + gparent = parent_of(tb, parent); + if (gparent != NULL) { lock_route_node(gparent); } else { gparent = NULL; } - } while (gparent != NULL && !gparent->is_valid); + } while (gparent != NULL && !gparent->u.route.is_valid); if (gparent == NULL) { - SET_ROOT_RELB(tb, GET_LEFT(parent_container)); - } else if (GET_RIGHT(gparent_container) == parent_container) { - SET_RIGHT_RELB(gparent_container, GET_LEFT(parent_container)); + SET_ROOT_RELB(tb, GET_LEFT(parent)); + } else if (GET_RIGHT(gparent) == parent) { + SET_RIGHT_RELB(gparent, GET_LEFT(parent)); } else { - SET_LEFT_RELB(gparent_container, GET_LEFT(parent_container)); + SET_LEFT_RELB(gparent, GET_LEFT(parent)); } unlock_route_node(parent); if (gparent != NULL) { unlock_route_node(gparent); } - new_neighbor_base = create_catree_base_node(tb); - new_neighbor_base->baseOrRoute.base.root = - join_trees(neighbor_base->root, base->root); - neighbor_base_parent = NULL; - if (GET_LEFT(parent_container) == neighbor_base_container) { - neighbor_base_parent = gparent_container; + { + TreeDbTerm* new_root = join_trees(neighbor->u.base.root, + thiz->u.base.root); + new_neighbor = create_base_node(tb, new_root, + LC_ORDER(thiz->u.base.lc_key.term)); + } + if (GET_LEFT(parent) == neighbor) { + neighbor_parent = gparent; } else { - neighbor_base_parent = - rightmost_route_node(GET_LEFT(parent_container)); + neighbor_parent = + rightmost_route_node(GET_LEFT(parent)); } } } /* Link in new neighbor and free nodes that are no longer in the tree */ - if (neighbor_base_parent == NULL) { - SET_ROOT_RELB(tb, new_neighbor_base); - } else if (GET_LEFT(neighbor_base_parent) == neighbor_base_container) { - SET_LEFT_RELB(neighbor_base_parent, new_neighbor_base); + if (neighbor_parent == NULL) { + SET_ROOT_RELB(tb, new_neighbor); + } else if (GET_LEFT(neighbor_parent) == neighbor) { + SET_LEFT_RELB(neighbor_parent, new_neighbor); } else { - SET_RIGHT_RELB(neighbor_base_parent, new_neighbor_base); + SET_RIGHT_RELB(neighbor_parent, new_neighbor); } - wunlock_base_node(base); - wunlock_base_node(neighbor_base); + wunlock_base_node(&thiz->u.base); + wunlock_base_node(&neighbor->u.base); /* Free the parent and base */ - erts_schedule_thr_prgr_later_op(free_catree_routing_node, - parent_container, - &parent->free_item); - erts_schedule_thr_prgr_later_op(free_catree_base_node, - base_container, - &base->free_item); - erts_schedule_thr_prgr_later_op(free_catree_base_node, - neighbor_base_container, - &neighbor_base->free_item); -} - - -static void split_catree(DbTableCommon *tb, - DbTableCATreeNode *parent_container, - DbTableCATreeNode *base_container) { + erts_schedule_db_free(&tb->common, + do_free_route_node, + parent, + &parent->u.route.free_item, + sizeof_route_node(parent->u.route.key.size)); + erts_schedule_db_free(&tb->common, + do_free_base_node, + thiz, + &thiz->u.base.free_item, + sizeof_base_node(thiz->u.base.lc_key.size)); + erts_schedule_db_free(&tb->common, + do_free_base_node, + neighbor, + &neighbor->u.base.free_item, + sizeof_base_node(neighbor->u.base.lc_key.size)); +} + +static void split_catree(DbTableCATree *tb, + DbTableCATreeNode* ERTS_RESTRICT parent, + DbTableCATreeNode* ERTS_RESTRICT base) { TreeDbTerm *splitOutWriteBack; - TreeDbTerm *leftWriteBack; - TreeDbTerm *rightWriteBack; - DbTableCATreeNode *left_base_node; - DbTableCATreeNode *right_base_node; - DbTableCATreeNode *routing_node_container; - DbTableCATreeBaseNode *base = &base_container->baseOrRoute.base; - DbTableCATreeRouteNode *parent; - if (parent_container == NULL) { - parent = NULL; - } else { - parent = &parent_container->baseOrRoute.route; - } + DbTableCATreeNode* ERTS_RESTRICT new_left; + DbTableCATreeNode* ERTS_RESTRICT new_right; + DbTableCATreeNode* ERTS_RESTRICT new_route; - if (less_than_two_elements(base->root)) { - base->lock_statistics = 0; - wunlock_base_node(base); + if (less_than_two_elements(base->u.base.root)) { + base->u.base.lock_statistics = 0; + wunlock_base_node(&base->u.base); return; } else { - /* Split the tree */ - split_tree(tb, - base->root, - &splitOutWriteBack, - &leftWriteBack, - &rightWriteBack); - /* Create new base nodes */ - left_base_node = - create_catree_base_node((DbTableCATree*)tb); - right_base_node = - create_catree_base_node((DbTableCATree*)tb); - left_base_node->baseOrRoute.base.root = leftWriteBack; - right_base_node->baseOrRoute.base.root = rightWriteBack; - routing_node_container = create_catree_route_node(tb, - left_base_node, - right_base_node, - &splitOutWriteBack->dbterm); + TreeDbTerm *left_tree; + TreeDbTerm *right_tree; + + split_tree(tb, base->u.base.root, &splitOutWriteBack, + &left_tree, &right_tree); + + new_left = create_base_node(tb, left_tree, + LC_ORDER(GETKEY(tb, left_tree->dbterm.tpl))); + new_right = create_base_node(tb, right_tree, + LC_ORDER(GETKEY(tb, right_tree->dbterm.tpl))); + new_route = create_route_node(tb, + new_left, + new_right, + &splitOutWriteBack->dbterm, + parent); if (parent == NULL) { - SET_ROOT_RELB((DbTableCATree*)tb, routing_node_container); - } else if(GET_LEFT(parent_container) == base_container) { - SET_LEFT_RELB(parent_container, routing_node_container); + SET_ROOT_RELB(tb, new_route); + } else if(GET_LEFT(parent) == base) { + SET_LEFT_RELB(parent, new_route); } else { - SET_RIGHT_RELB(parent_container, routing_node_container); + SET_RIGHT_RELB(parent, new_route); } - base->is_valid = 0; - wunlock_base_node(base); - erts_schedule_thr_prgr_later_op(free_catree_base_node, - base_container, - &base->free_item); + base->u.base.is_valid = 0; + wunlock_base_node(&base->u.base); + erts_schedule_db_free(&tb->common, + do_free_base_node, + base, + &base->u.base.free_item, + sizeof_base_node(base->u.base.lc_key.size)); } } @@ -1546,7 +1559,7 @@ static void catree_add_base_node_to_free_list( DbTableCATree *tb, DbTableCATreeNode *base_node_container) { - base_node_container->baseOrRoute.base.next = + base_node_container->u.base.next = tb->base_nodes_to_free_list; tb->base_nodes_to_free_list = base_node_container; } @@ -1557,7 +1570,7 @@ static void catree_deque_base_node_from_free_list(DbTableCATree *tb) return; /* List empty */ } else { DbTableCATreeNode *first = tb->base_nodes_to_free_list; - tb->base_nodes_to_free_list = first->baseOrRoute.base.next; + tb->base_nodes_to_free_list = first->u.base.next; } } @@ -1596,7 +1609,7 @@ static SWord do_free_routing_nodes_catree_cont(DbTableCATree *tb, SWord num_left PUSH_NODE(&tb->free_stack_rnodes, root); root = p; } else { - free_catree_routing_node(root); + free_catree_route_node(tb, root); if (--num_left >= 0) { break; } else { @@ -1637,10 +1650,10 @@ static SWord do_free_base_node_cont(DbTableCATree *tb, SWord num_left) } } catree_deque_base_node_from_free_list(tb); - free_catree_base_node(base_node_container); + free_catree_base_node(tb, base_node_container); base_node_container = catree_first_base_node_from_free_list(tb); if (base_node_container != NULL) { - PUSH_NODE(&tb->free_stack_elems, base_node_container->baseOrRoute.base.root); + PUSH_NODE(&tb->free_stack_elems, base_node_container->u.base.root); } return num_left; } @@ -1662,7 +1675,7 @@ void db_initialize_catree(void) int db_create_catree(Process *p, DbTable *tbl) { DbTableCATree *tb = &tbl->catree; - DbTableCATreeNode *root = create_catree_base_node(tb); + DbTableCATreeNode *root = create_base_node(tb, NULL, NIL); tb->deletion = 0; tb->base_nodes_to_free_list = NULL; erts_atomic_init_relb(&(tb->root), (erts_aint_t)root); @@ -1675,7 +1688,7 @@ static int db_first_catree(Process *p, DbTable *tbl, Eterm *ret) int result; DECLARE_AND_INIT_BASE_NODE_SEARCH_STACKS; /* Find first base node */ - base_node = &(lock_first_base_node(tbl, &search_stack, &locked_base_nodes_stack)->baseOrRoute.base); + base_node = &(lock_first_base_node(tbl, &search_stack, &locked_base_nodes_stack)->u.base); /* Find next base node until non-empty base node is found */ while (base_node != NULL && base_node->root == NULL) { base_node = find_and_lock_next_base_node_and_path(tbl, &search_stack_ptr, &search_stack_copy_ptr, locked_base_nodes_stack_ptr); @@ -1716,10 +1729,9 @@ static int db_next_catree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) static int db_last_catree(Process *p, DbTable *tbl, Eterm *ret) { - DbTableCATree *tb = &tbl->catree; - DbTableCATreeNode *base = merge_to_one_locked_base_node(&tb->common); - int result = db_last_tree_common(p, tbl, base->baseOrRoute.base.root, ret, NULL); - wunlock_base_node(&(base->baseOrRoute.base)); + DbTableCATreeNode *base = merge_to_one_locked_base_node(&tbl->catree); + int result = db_last_tree_common(p, tbl, base->u.base.root, ret, NULL); + wunlock_base_node(&(base->u.base)); return result; } @@ -1728,13 +1740,12 @@ static int db_prev_catree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) { DbTreeStack stack; TreeDbTerm * stack_array[STACK_NEED]; - DbTableCATree *tb = &tbl->catree; int result; DbTableCATreeNode *base; init_tree_stack(&stack, stack_array, 0); - base = merge_to_one_locked_base_node(&tb->common); - result = db_prev_tree_common(p, tbl, base->baseOrRoute.base.root, key, ret, &stack); - wunlock_base_node(&(base->baseOrRoute.base)); + base = merge_to_one_locked_base_node(&tbl->catree); + result = db_prev_tree_common(p, tbl, base->u.base.root, key, ret, &stack); + wunlock_base_node(&(base->u.base)); return result; } @@ -1835,13 +1846,12 @@ static int db_erase_object_catree(DbTable *tbl, Eterm object, Eterm *ret) static int db_slot_catree(Process *p, DbTable *tbl, Eterm slot_term, Eterm *ret) { - DbTableCATree *tb = &tbl->catree; int result; DbTableCATreeNode *base; - base = merge_to_one_locked_base_node(&tb->common); - result = db_slot_tree_common(p, tbl, base->baseOrRoute.base.root, + base = merge_to_one_locked_base_node(&tbl->catree); + result = db_slot_tree_common(p, tbl, base->u.base.root, slot_term, ret, NULL); - wunlock_base_node(&(base->baseOrRoute.base)); + wunlock_base_node(&(base->u.base)); return result; } @@ -1850,13 +1860,12 @@ static int db_select_continue_catree(Process *p, Eterm continuation, Eterm *ret) { - DbTableCATree *tb = &tbl->catree; int result; DbTableCATreeNode *base; - base = merge_to_one_locked_base_node(&tb->common); - result = db_select_continue_tree_common(p, &tb->common, &base->baseOrRoute.base.root, + base = merge_to_one_locked_base_node(&tbl->catree); + result = db_select_continue_tree_common(p, &tbl->common, &base->u.base.root, continuation, ret, NULL); - wunlock_base_node(&(base->baseOrRoute.base)); + wunlock_base_node(&(base->u.base)); return result; } @@ -1864,14 +1873,13 @@ static int db_select_continue_catree(Process *p, static int db_select_catree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, int reverse, Eterm *ret) { - DbTableCATree *tb = &tbl->catree; int result; DbTableCATreeNode *base; - base = merge_to_one_locked_base_node(&tb->common); - result = db_select_tree_common(p, &tb->common, &base->baseOrRoute.base.root, + base = merge_to_one_locked_base_node(&tbl->catree); + result = db_select_tree_common(p, &tbl->common, &base->u.base.root, tid, pattern, reverse, ret, NULL); - wunlock_base_node(&(base->baseOrRoute.base)); + wunlock_base_node(&(base->u.base)); return result; } @@ -1880,27 +1888,25 @@ static int db_select_count_continue_catree(Process *p, Eterm continuation, Eterm *ret) { - DbTableCATree *tb = &tbl->catree; int result; DbTableCATreeNode *base; - base = merge_to_one_locked_base_node(&tb->common); - result = db_select_count_continue_tree_common(p, &tb->common, - &base->baseOrRoute.base.root, + base = merge_to_one_locked_base_node(&tbl->catree); + result = db_select_count_continue_tree_common(p, &tbl->common, + &base->u.base.root, continuation, ret, NULL); - wunlock_base_node(&(base->baseOrRoute.base)); + wunlock_base_node(&(base->u.base)); return result; } static int db_select_count_catree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret) { - DbTableCATree *tb = &tbl->catree; int result; DbTableCATreeNode *base; - base = merge_to_one_locked_base_node(&tb->common); - result = db_select_count_tree_common(p, &tb->common, &base->baseOrRoute.base.root, + base = merge_to_one_locked_base_node(&tbl->catree); + result = db_select_count_tree_common(p, &tbl->common, &base->u.base.root, tid, pattern, ret, NULL); - wunlock_base_node(&(base->baseOrRoute.base)); + wunlock_base_node(&(base->u.base)); return result; } @@ -1908,13 +1914,12 @@ static int db_select_chunk_catree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Sint chunk_size, int reversed, Eterm *ret) { - DbTableCATree *tb = &tbl->catree; int result; DbTableCATreeNode *base; - base = merge_to_one_locked_base_node(&tb->common); - result = db_select_chunk_tree_common(p, &tb->common, &base->baseOrRoute.base.root, + base = merge_to_one_locked_base_node(&tbl->catree); + result = db_select_chunk_tree_common(p, &tbl->common, &base->u.base.root, tid, pattern, chunk_size, reversed, ret, NULL); - wunlock_base_node(&(base->baseOrRoute.base)); + wunlock_base_node(&(base->u.base)); return result; } @@ -1923,60 +1928,56 @@ static int db_select_delete_continue_catree(Process *p, Eterm continuation, Eterm *ret) { - DbTableCATree *tb = &tbl->catree; DbTreeStack stack; TreeDbTerm * stack_array[STACK_NEED]; int result; DbTableCATreeNode *base; init_tree_stack(&stack, stack_array, 0); - base = merge_to_one_locked_base_node(&tb->common); - result = db_select_delete_continue_tree_common(p, tbl, &base->baseOrRoute.base.root, + base = merge_to_one_locked_base_node(&tbl->catree); + result = db_select_delete_continue_tree_common(p, tbl, &base->u.base.root, continuation, ret, &stack); - wunlock_base_node(&(base->baseOrRoute.base)); + wunlock_base_node(&(base->u.base)); return result; } static int db_select_delete_catree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret) { - DbTableCATree *tb = &tbl->catree; DbTreeStack stack; TreeDbTerm * stack_array[STACK_NEED]; int result; DbTableCATreeNode *base; init_tree_stack(&stack, stack_array, 0); - base = merge_to_one_locked_base_node(&tb->common); - result = db_select_delete_tree_common(p, tbl, &base->baseOrRoute.base.root, + base = merge_to_one_locked_base_node(&tbl->catree); + result = db_select_delete_tree_common(p, tbl, &base->u.base.root, tid, pattern, ret, &stack); - wunlock_base_node(&(base->baseOrRoute.base)); + wunlock_base_node(&(base->u.base)); return result; } static int db_select_replace_catree(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret) { - DbTableCATree *tb = &tbl->catree; int result; DbTableCATreeNode *base; - base = merge_to_one_locked_base_node(&tb->common); - result = db_select_replace_tree_common(p, &tb->common, - &base->baseOrRoute.base.root, + base = merge_to_one_locked_base_node(&tbl->catree); + result = db_select_replace_tree_common(p, &tbl->common, + &base->u.base.root, tid, pattern, ret, NULL); - wunlock_base_node(&(base->baseOrRoute.base)); + wunlock_base_node(&(base->u.base)); return result; } static int db_select_replace_continue_catree(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret) { - DbTableCATree *tb = &tbl->catree; int result; DbTableCATreeNode *base; - base = merge_to_one_locked_base_node(&tb->common); - result = db_select_replace_continue_tree_common(p, &tb->common, - &base->baseOrRoute.base.root, + base = merge_to_one_locked_base_node(&tbl->catree); + result = db_select_replace_continue_tree_common(p, &tbl->common, + &base->u.base.root, continuation, ret, NULL); - wunlock_base_node(&(base->baseOrRoute.base)); + wunlock_base_node(&(base->u.base)); return result; } @@ -2002,10 +2003,9 @@ static int db_take_catree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) static void db_print_catree(fmtfn_t to, void *to_arg, int show, DbTable *tbl) { - DbTableCATree *tb = &tbl->catree; - DbTableCATreeNode *base = merge_to_one_locked_base_node(&tb->common); - db_print_tree_common(to, to_arg, show, base->baseOrRoute.base.root, tbl); - wunlock_base_node(&(base->baseOrRoute.base)); + DbTableCATreeNode *base = merge_to_one_locked_base_node(&tbl->catree); + db_print_tree_common(to, to_arg, show, base->u.base.root, tbl); + wunlock_base_node(&(base->u.base)); } /* Release all memory occupied by a single table */ @@ -2045,7 +2045,7 @@ static SWord db_free_table_continue_catree(DbTable *tbl, SWord reds) } else { tb->is_routing_nodes_freed = 1; /* Ready with the routing nodes */ first_base_node = catree_first_base_node_from_free_list(tb); - PUSH_NODE(&tb->free_stack_elems, first_base_node->baseOrRoute.base.root); + PUSH_NODE(&tb->free_stack_elems, first_base_node->u.base.root); } } while (catree_first_base_node_from_free_list(tb) != NULL) { @@ -2081,10 +2081,9 @@ static void db_foreach_offheap_catree(DbTable *tbl, void (*func)(ErlOffHeap *, void *), void *arg) { - DbTableCATree *tb = &tbl->catree; - DbTableCATreeNode *base = merge_to_one_locked_base_node(&tb->common); - db_foreach_offheap_tree_common(base->baseOrRoute.base.root, func, arg); - wunlock_base_node(&(base->baseOrRoute.base)); + DbTableCATreeNode *base = merge_to_one_locked_base_node(&tbl->catree); + db_foreach_offheap_tree_common(base->u.base.root, func, arg); + wunlock_base_node(&(base->u.base)); } static int db_lookup_dbterm_catree(Process *p, DbTable *tbl, Eterm key, Eterm obj, @@ -2111,7 +2110,7 @@ static void db_finalize_dbterm_catree(int cret, DbUpdateHandle *handle) DbTableCATreeNode *prev_node = handle->lck; DbTableCATreeNode *current_node = handle->lck2; int current_level = handle->current_level; - DbTableCATreeBaseNode *base_node = ¤t_node->baseOrRoute.base; + DbTableCATreeBaseNode *base_node = ¤t_node->u.base; db_finalize_dbterm_tree_common(cret, handle, NULL); ERL_DB_CATREE_CREATE_DO_OPERATION_FUNCTION_ADAPT_AND_UNLOCK_PART; return; diff --git a/erts/emulator/beam/erl_db_catree.h b/erts/emulator/beam/erl_db_catree.h index 1f2c7da091..f9c0784289 100644 --- a/erts/emulator/beam/erl_db_catree.h +++ b/erts/emulator/beam/erl_db_catree.h @@ -34,23 +34,36 @@ struct DbTableCATreeNode; typedef struct { + Eterm term; + struct erl_off_heap_header* oh; + Uint size; + Eterm heap[1]; +} DbRouteKey; + +typedef struct { erts_rwmtx_t lock; /* The lock for this base node */ Sint lock_statistics; int is_valid; /* If this base node is still valid */ TreeDbTerm *root; /* The root of the sequential tree */ - DbTable * tab; /* Table ptr, used when freeing using thread progress */ ErtsThrPrgrLaterOp free_item; /* Used when freeing using thread progress */ struct DbTableCATreeNode * next; /* Used when gradually deleting */ + +#ifdef ERTS_ENABLE_LOCK_CHECK + DbRouteKey lc_key; +#endif + char end_of_struct__; } DbTableCATreeBaseNode; typedef struct { +#ifdef ERTS_ENABLE_LOCK_CHECK + Sint lc_order; +#endif ErtsThrPrgrLaterOp free_item; /* Used when freeing using thread progress */ - DbTable* tab; /* Table ptr, used when freeing using thread progress */ erts_mtx_t lock; /* Used when joining route nodes */ int is_valid; /* If this route node is still valid */ erts_atomic_t left; erts_atomic_t right; - DbTerm key; + DbRouteKey key; } DbTableCATreeRouteNode; typedef struct DbTableCATreeNode { @@ -58,7 +71,7 @@ typedef struct DbTableCATreeNode { union { DbTableCATreeRouteNode route; DbTableCATreeBaseNode base; - } baseOrRoute; + } u; } DbTableCATreeNode; typedef struct { diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 752d3ae3a8..61806876a7 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -2731,13 +2731,9 @@ static int free_seg(DbTableHash *tb, int free_records) * sure no lingering threads are still hanging in BUCKET macro * with an old segtab pointer. */ - Uint sz = SIZEOF_EXT_SEGTAB(est->nsegs); - ASSERT(sz == ERTS_ALC_DBG_BLK_SZ(est)); - ERTS_DB_ALC_MEM_UPDATE_(tb, sz, 0); - erts_schedule_thr_prgr_later_cleanup_op(dealloc_ext_segtab, - est, - &est->lop, - sz); + erts_schedule_db_free(&tb->common, dealloc_ext_segtab, + est, &est->lop, + SIZEOF_EXT_SEGTAB(est->nsegs)); } else erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable*)tb, est, diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index f643344477..c6f1a0fc6d 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -1995,22 +1995,14 @@ int db_select_replace_tree_common(Process *p, DbTableCommon *tb, TreeDbTerm **ro sc.mp = mpi.mp; - stack = get_static_stack(stack_container); if (!mpi.got_partial && mpi.some_limitation && CMP_EQ(mpi.least,mpi.most)) { - TreeDbTerm* term = *(mpi.save_term); doit_select_replace(tb,mpi.save_term,&sc,0 /* dummy */); - if (stack != NULL) { - if (TOP_NODE(stack) == term) - // throw away potentially invalid reference - REPLACE_TOP_NODE(stack, *(mpi.save_term)); - release_stack((DbTable*)tb,stack_container, stack); - } + reset_static_stack(stack_container); /* may refer replaced term */ RET_TO_BIF(erts_make_integer(sc.replaced,p),DB_ERROR_NONE); } - if (stack == NULL) - stack = get_any_stack((DbTable*)tb,stack_container); + stack = get_any_stack((DbTable*)tb,stack_container); if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, *root, stack, mpi.most)) != NULL) { diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 5da7b43b9e..9ba3ae226e 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -163,7 +163,6 @@ int erts_initialized = 0; * Configurable parameters. */ -Uint display_items; /* no of items to display in traces etc */ int H_MIN_SIZE; /* The minimum heap grain */ int BIN_VH_MIN_SIZE; /* The minimum binary virtual*/ int H_MAX_SIZE; /* The maximum heap size */ @@ -375,21 +374,19 @@ erl_init(int ncpu, } static Eterm -erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** argv) +erl_first_process_otp(char* mod_name, int argc, char** argv) { int i; - Eterm start_mod; Eterm args; Eterm res; Eterm* hp; Process parent; ErlSpawnOpts so; - Eterm env; - - start_mod = erts_atom_put((byte *) modname, sys_strlen(modname), ERTS_ATOM_ENC_LATIN1, 1); - if (erts_find_function(start_mod, am_start, 2, + Eterm boot_mod; + + if (erts_find_function(am_erl_init, am_start, 2, erts_active_code_ix()) == NULL) { - erts_exit(ERTS_ERROR_EXIT, "No function %s:start/2\n", modname); + erts_exit(ERTS_ERROR_EXIT, "No function erl_init:start/2\n"); } /* @@ -405,13 +402,13 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** args = CONS(hp, new_binary(&parent, (byte*)argv[i], len), args); hp += 2; } - env = new_binary(&parent, code, size); + boot_mod = erts_atom_put((byte *) mod_name, sys_strlen(mod_name), ERTS_ATOM_ENC_LATIN1, 1); args = CONS(hp, args, NIL); hp += 2; - args = CONS(hp, env, args); + args = CONS(hp, boot_mod, args); so.flags = erts_default_spo_flags|SPO_SYSTEM_PROC; - res = erl_create_process(&parent, start_mod, am_start, args, &so); + res = erl_create_process(&parent, am_erl_init, am_start, args, &so); erts_proc_unlock(&parent, ERTS_PROC_LOCK_MAIN); erts_cleanup_empty_process(&parent); return res; @@ -481,7 +478,6 @@ erts_preloaded(Process* p) /* static variables that must not change (use same values at restart) */ static char* program; static char* init = "init"; -static char* boot = "boot"; static int boot_argc; static char** boot_argv; @@ -535,9 +531,6 @@ void erts_usage(void) int this_rel = this_rel_num(); erts_fprintf(stderr, "Usage: %s [flags] [ -- [init_args] ]\n", progname(program)); erts_fprintf(stderr, "The flags are:\n\n"); - - /* erts_fprintf(stderr, "-# number set the number of items to be used in traces etc\n"); */ - erts_fprintf(stderr, "-a size suggested stack size in kilo words for threads\n"); erts_fprintf(stderr, " in the async-thread pool, valid range is [%d-%d]\n", ERTS_ASYNC_THREAD_MIN_STACK_SIZE, @@ -545,13 +538,9 @@ void erts_usage(void) erts_fprintf(stderr, "-A number set number of threads in async thread pool,\n"); erts_fprintf(stderr, " valid range is [0-%d]\n", ERTS_MAX_NO_OF_ASYNC_THREADS); - erts_fprintf(stderr, "-B[c|d|i] c to have Ctrl-c interrupt the Erlang shell,\n"); erts_fprintf(stderr, " d (or no extra option) to disable the break\n"); erts_fprintf(stderr, " handler, i to ignore break signals\n"); - - /* erts_fprintf(stderr, "-b func set the boot function (default boot)\n"); */ - erts_fprintf(stderr, "-c bool enable or disable time correction\n"); erts_fprintf(stderr, "-C mode set time warp mode; valid modes are:\n"); erts_fprintf(stderr, " no_time_warp|single_time_warp|multi_time_warp\n"); @@ -570,7 +559,6 @@ void erts_usage(void) erts_pd_initial_size); erts_fprintf(stderr, "-hmqd val set default message queue data flag for processes,\n"); erts_fprintf(stderr, " valid values are: off_heap | on_heap\n"); - erts_fprintf(stderr, "-IOp number set number of pollsets to be used to poll for I/O,\n"); erts_fprintf(stderr, " This value has to be equal or smaller than the\n"); erts_fprintf(stderr, " number of poll threads. If the current platform\n"); @@ -581,9 +569,7 @@ void erts_usage(void) erts_fprintf(stderr, " number of poll threads."); erts_fprintf(stderr, "-IOPt number set number of threads to be used to poll for I/O\n"); erts_fprintf(stderr, " as a percentage of the number of schedulers."); - - /* erts_fprintf(stderr, "-i module set the boot module (default init)\n"); */ - + erts_fprintf(stderr, "-i module set the boot module (default init)\n"); erts_fprintf(stderr, "-n[s|a|d] Control behavior of signals to ports\n"); erts_fprintf(stderr, " Note that this flag is deprecated!\n"); erts_fprintf(stderr, "-M<X> <Y> memory allocator switches,\n"); @@ -598,7 +584,6 @@ void erts_usage(void) erts_fprintf(stderr, "-R number set compatibility release number,\n"); erts_fprintf(stderr, " valid range [%d-%d]\n", this_rel-2, this_rel); - erts_fprintf(stderr, "-r force ets memory block to be moved on realloc\n"); erts_fprintf(stderr, "-rg amount set reader groups limit\n"); erts_fprintf(stderr, "-sbt type set scheduler bind type, valid types are:\n"); @@ -669,9 +654,7 @@ void erts_usage(void) erts_fprintf(stderr, "-T number set modified timing level, valid range is [0-%d]\n", ERTS_MODIFIED_TIMING_LEVELS-1); erts_fprintf(stderr, "-V print Erlang version\n"); - erts_fprintf(stderr, "-v turn on chatty mode (GCs will be reported etc)\n"); - erts_fprintf(stderr, "-W<i|w|e> set error logger warnings mapping,\n"); erts_fprintf(stderr, " see error_logger documentation for details\n"); erts_fprintf(stderr, "-zdbbl size set the distribution buffer busy limit in kilobytes\n"); @@ -762,7 +745,6 @@ early_init(int *argc, char **argv) /* erts_sched_compact_load = 1; erts_printf_eterm_func = erts_printf_term; - display_items = 200; erts_backtrace_depth = DEFAULT_BACKTRACE_SIZE; erts_async_max_threads = ERTS_DEFAULT_NO_ASYNC_THREADS; erts_async_thread_suggested_stack_size = ERTS_ASYNC_THREAD_MIN_STACK_SIZE; @@ -1269,25 +1251,9 @@ erl_start(int argc, char **argv) /* * NOTE: -M flags are handled (and removed from argv) by - * erts_alloc_init(). - * - * The -d, -m, -S, -t, and -T flags was removed in - * Erlang 5.3/OTP R9C. - * - * -S, and -T has been reused in Erlang 5.5/OTP R11B. - * - * -d has been reused in a patch R12B-4. + * erts_alloc_init(). */ - case '#' : - arg = get_arg(argv[i]+2, argv[i+1], &i); - if ((display_items = atoi(arg)) == 0) { - erts_fprintf(stderr, "bad display items%s\n", arg); - erts_usage(); - } - VERBOSE(DEBUG_SYSTEM, - ("using display items %d\n",display_items)); - break; case 'p': if (!sys_strncmp(argv[i],"-pc",3)) { int printable_chars = ERL_PRINTABLE_CHARACTERS_LATIN1; @@ -1566,11 +1532,6 @@ erl_start(int argc, char **argv) init = get_arg(argv[i]+2, argv[i+1], &i); break; - case 'b': - /* define name of initial function */ - boot = get_arg(argv[i]+2, argv[i+1], &i); - break; - case 'B': if (argv[i][2] == 'i') /* +Bi */ ignore_break = 1; @@ -2256,9 +2217,8 @@ erl_start(int argc, char **argv) erts_initialized = 1; - erts_init_process_id = erl_first_process_otp("otp_ring0", NULL, 0, - boot_argc, boot_argv); - ASSERT(erts_init_process_id != ERTS_INVALID_PID); + erts_init_process_id = erl_first_process_otp(init, boot_argc, boot_argv); + ASSERT(erts_init_process_id != ERTS_INVALID_PID); { /* diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 3b10ef8c25..987e370341 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -42,6 +42,7 @@ #include "erl_term.h" #include "erl_threads.h" #include "erl_atom_table.h" +#include "erl_utils.h" typedef struct { char *name; @@ -91,8 +92,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "db_tab", "address" }, { "db_tab_fix", "address" }, { "db_hash_slot", "address" }, - { "erl_db_catree_base_node", "dynamic" }, - { "erl_db_catree_route_node", "dynamic" }, + { "erl_db_catree_base_node", "term" }, + { "erl_db_catree_route_node", "index" }, { "resource_monitors", "address" }, { "driver_list", NULL }, { "proc_msgq", "pid" }, @@ -709,26 +710,14 @@ erts_lc_get_lock_order_id(char *name) return (Sint16) -1; } -int -erts_lc_is_check_order(char *name) +static int +lc_is_term_order(Sint16 id) { - int i; - if (!name || name[0] == '\0') - erts_fprintf(stderr, "Missing lock name\n"); - - for (i = 0; i < ERTS_LOCK_ORDER_SIZE; i++) { - if (sys_strcmp(erts_lock_order[i].name, name) == 0) { - if (erts_lock_order[i].internal_order != NULL && - sys_strcmp(erts_lock_order[i].internal_order, "dynamic") == 0) { - return 0; - }else{ - return 1; - } - } - } - return 1; + return erts_lock_order[id].internal_order != NULL + && sys_strcmp(erts_lock_order[id].internal_order, "term") == 0; } + static int compare_locked_by_id(lc_locked_lock_t *locked_lock, erts_lc_lock_t *comparand) { if(locked_lock->id < comparand->id) { @@ -740,18 +729,23 @@ static int compare_locked_by_id(lc_locked_lock_t *locked_lock, erts_lc_lock_t *c return 0; } -static int compare_locked_by_id_extra(lc_locked_lock_t *locked_lock, erts_lc_lock_t *comparand) +static int compare_locked_by_id_extra(lc_locked_lock_t *ll, erts_lc_lock_t *comparand) { - int order = compare_locked_by_id(locked_lock, comparand); + int order = compare_locked_by_id(ll, comparand); if(order) { return order; - } else if(locked_lock->extra < comparand->extra) { + } + if (ll->flags & ERTS_LOCK_FLAGS_PROPERTY_TERM_ORDER) { + ASSERT(!is_header(ll->extra) && !is_header(comparand->extra)); + return CMP(ll->extra, comparand->extra); + } + + if(ll->extra < comparand->extra) { return -1; - } else if(locked_lock->extra > comparand->extra) { + } else if(ll->extra > comparand->extra) { return 1; } - return 0; } @@ -990,7 +984,8 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, erts_lock_options_t options) return 0; } else { - lc_locked_lock_t *tl_lck; + lc_locked_lock_t *ll; + int order; ASSERT(thr->locked.last); @@ -999,26 +994,27 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, erts_lock_options_t options) type_order_violation("trylocking ", thr, lck); #endif - if (thr->locked.last->id < lck->id - || (thr->locked.last->id == lck->id - && thr->locked.last->extra < lck->extra)) - return 0; + ll = thr->locked.last; + order = compare_locked_by_id_extra(ll, lck); + + if (order < 0) + return 0; /* - * Lock order violation + * TryLock order violation */ - - if (lck->check_order) { - /* Check that we are not trying to lock this lock twice */ - for (tl_lck = thr->locked.last; tl_lck; tl_lck = tl_lck->prev) { - if (tl_lck->id < lck->id - || (tl_lck->id == lck->id && tl_lck->extra <= lck->extra)) { - if (tl_lck->id == lck->id && tl_lck->extra == lck->extra) - lock_twice("Trylocking", thr, lck, options); - break; - } + /* Check that we are not trying to lock this lock twice */ + while (1) { + if (order <= 0) { + if (order == 0) + lock_twice("Trylocking", thr, lck, options); + break; } + ll = ll->prev; + if (!ll) + break; + order = compare_locked_by_id_extra(ll, lck); } #ifndef ERTS_LC_ALLWAYS_FORCE_BUSY_TRYLOCK_ON_LOCK_ORDER_VIOLATION @@ -1038,6 +1034,11 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, erts_lock_options_t options) #endif } +/* + * locked = 0 trylock failed + * locked > 0 trylock succeeded + * locked < 0 prelocking of newly created lock (no lock order check) + */ void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, erts_lock_options_t options, char *file, unsigned int line) { @@ -1066,9 +1067,9 @@ void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, erts_lock_options_t #endif for (tl_lck = thr->locked.last; tl_lck; tl_lck = tl_lck->prev) { - if (tl_lck->id < lck->id - || (tl_lck->id == lck->id && tl_lck->extra <= lck->extra)) { - if (tl_lck->id == lck->id && tl_lck->extra == lck->extra && lck->check_order) + int order = compare_locked_by_id_extra(tl_lck, lck); + if (order <= 0) { + if (order == 0 && locked >= 0) lock_twice("Trylocking", thr, lck, options); if (locked) { ll->next = tl_lck->next; @@ -1111,10 +1112,10 @@ void erts_lc_require_lock_flg(erts_lc_lock_t *lck, erts_lock_options_t options, for (l_lck2 = thr->required.last; l_lck2; l_lck2 = l_lck2->prev) { - if (l_lck2->id < lck->id - || (l_lck2->id == lck->id && l_lck2->extra < lck->extra)) + int order = compare_locked_by_id_extra(l_lck2, lck); + if (order < 0) break; - else if (l_lck2->id == lck->id && l_lck2->extra == lck->extra) + if (order == 0) require_twice(thr, lck); } if (!l_lck2) { @@ -1172,6 +1173,7 @@ void erts_lc_lock_flg_x(erts_lc_lock_t *lck, erts_lock_options_t options, { lc_thread_t *thr; lc_locked_lock_t *new_ll; + int order; if (lck->inited != ERTS_LC_INITITALIZED) uninitialized_lock(); @@ -1187,11 +1189,10 @@ void erts_lc_lock_flg_x(erts_lc_lock_t *lck, erts_lock_options_t options, thr->locked.last = thr->locked.first = new_ll; ASSERT(0 < lck->id && lck->id < ERTS_LOCK_ORDER_SIZE); thr->matrix.m[lck->id][0] = 1; + return; } - else if (( ! lck->check_order && thr->locked.last->id == lck->id) || - (thr->locked.last->id < lck->id - || (thr->locked.last->id == lck->id - && thr->locked.last->extra < lck->extra))) { + order = compare_locked_by_id_extra(thr->locked.last, lck); + if (order < 0) { lc_locked_lock_t* ll; if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, thr->locked.last->flags)) { type_order_violation("locking ", thr, lck); @@ -1321,7 +1322,6 @@ void erts_lc_init_lock(erts_lc_lock_t *lck, char *name, erts_lock_flags_t flags) { lck->id = erts_lc_get_lock_order_id(name); - lck->check_order = erts_lc_is_check_order(name); lck->extra = (UWord) &lck->extra; ASSERT(is_not_immed(lck->extra)); lck->flags = flags; @@ -1333,10 +1333,14 @@ void erts_lc_init_lock_x(erts_lc_lock_t *lck, char *name, erts_lock_flags_t flags, Eterm extra) { lck->id = erts_lc_get_lock_order_id(name); - lck->check_order = erts_lc_is_check_order(name); lck->extra = extra; - ASSERT(is_immed(lck->extra)); lck->flags = flags; + if (lc_is_term_order(lck->id)) { + lck->flags |= ERTS_LOCK_FLAGS_PROPERTY_TERM_ORDER; + ASSERT(!is_header(lck->extra)); + } + else + ASSERT(is_immed(lck->extra)); lck->taken_options = 0; lck->inited = ERTS_LC_INITITALIZED; } diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h index 472e88ad65..b32f27d9f9 100644 --- a/erts/emulator/beam/erl_lock_check.h +++ b/erts/emulator/beam/erl_lock_check.h @@ -46,7 +46,6 @@ typedef struct { int inited; Sint16 id; - int check_order; erts_lock_flags_t flags; erts_lock_options_t taken_options; UWord extra; @@ -54,12 +53,11 @@ typedef struct { #define ERTS_LC_INITITALIZED 0x7f7f7f7f -#define ERTS_LC_LOCK_INIT(ID, X, F) {ERTS_LC_INITITALIZED, (ID), 1, (F), 0, (X)} +#define ERTS_LC_LOCK_INIT(ID, X, F) {ERTS_LC_INITITALIZED, (ID), (F), 0, (X)} void erts_lc_init(void); void erts_lc_late_init(void); Sint16 erts_lc_get_lock_order_id(char *name); -int erts_lc_is_check_order(char *name); void erts_lc_check(erts_lc_lock_t *have, int have_len, erts_lc_lock_t *have_not, int have_not_len); void erts_lc_check_exact(erts_lc_lock_t *have, int have_len); @@ -106,7 +104,7 @@ Eterm erts_lc_dump_graph(void); #define erts_lc_lock(lck) erts_lc_lock_x(lck,__FILE__,__LINE__) #define erts_lc_trylock(res,lck) erts_lc_trylock_x(res,lck,__FILE__,__LINE__) -#define erts_lc_lock_flg(lck) erts_lc_lock_flg_x(lck,__FILE__,__LINE__) -#define erts_lc_trylock_flg(res,lck) erts_lc_trylock_flg_x(res,lck,__FILE__,__LINE__) +#define erts_lc_lock_flg(lck,flg) erts_lc_lock_flg_x(lck,flg,__FILE__,__LINE__) +#define erts_lc_trylock_flg(res,lck,flg) erts_lc_trylock_flg_x(res,lck,flg,__FILE__,__LINE__) #endif /* #ifndef ERTS_LOCK_CHECK_H__ */ diff --git a/erts/emulator/beam/erl_lock_flags.h b/erts/emulator/beam/erl_lock_flags.h index d711f69456..2db133b598 100644 --- a/erts/emulator/beam/erl_lock_flags.h +++ b/erts/emulator/beam/erl_lock_flags.h @@ -28,15 +28,17 @@ /* Property/category are bitfields to simplify their use in masks. */ #define ERTS_LOCK_FLAGS_MASK_CATEGORY (0xFFC0) -#define ERTS_LOCK_FLAGS_MASK_PROPERTY (0x0030) +#define ERTS_LOCK_FLAGS_MASK_PROPERTY (0x0038) /* Type is a plain number. */ -#define ERTS_LOCK_FLAGS_MASK_TYPE (0x000F) +#define ERTS_LOCK_FLAGS_MASK_TYPE (0x0007) #define ERTS_LOCK_FLAGS_TYPE_SPINLOCK (1) #define ERTS_LOCK_FLAGS_TYPE_MUTEX (2) #define ERTS_LOCK_FLAGS_TYPE_PROCLOCK (3) +/* Lock checker use real term order instead of raw word compare */ +#define ERTS_LOCK_FLAGS_PROPERTY_TERM_ORDER (1 << 3) /* "Static" guarantees that the lock will never be destroyed once created. */ #define ERTS_LOCK_FLAGS_PROPERTY_STATIC (1 << 4) #define ERTS_LOCK_FLAGS_PROPERTY_READ_WRITE (1 << 5) diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 21ae205237..187d2635ea 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -292,7 +292,6 @@ union erl_off_heap_ptr { /* controls warning mapping in error_logger */ extern Eterm node_cookie; -extern Uint display_items; /* no of items to display in traces etc */ extern int erts_backtrace_depth; extern erts_atomic32_t erts_max_gen_gcs; diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index d859c4bb24..349034e8ac 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -1089,21 +1089,21 @@ func_info M F A => i_func_info u M F A %warm bs_start_match2 Fail=f ica X Y D => jump Fail bs_start_match2 Fail Bin X Y D => i_bs_start_match2 Bin Fail X Y D -i_bs_start_match2 xy f t t x +i_bs_start_match2 xy f t t d bs_save2 Reg Index => gen_bs_save(Reg, Index) -i_bs_save2 x t +i_bs_save2 xy t bs_restore2 Reg Index => gen_bs_restore(Reg, Index) -i_bs_restore2 x t +i_bs_restore2 xy t # Matching integers bs_match_string Fail Ms Bits Val => i_bs_match_string Ms Fail Bits Val -i_bs_match_string x f W W +i_bs_match_string xy f W W # Fetching integers from binaries. -bs_get_integer2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \ +bs_get_integer2 Fail=f Ms=xy Live=u Sz=sq Unit=u Flags=u Dst=d => \ gen_get_integer2(Fail, Ms, Live, Sz, Unit, Flags, Dst) i_bs_get_integer_small_imm Ms Bits Fail Flags Y=y => \ @@ -1112,50 +1112,50 @@ i_bs_get_integer_small_imm Ms Bits Fail Flags Y=y => \ i_bs_get_integer_imm Ms Bits Live Fail Flags Y=y => \ i_bs_get_integer_imm Ms Bits Live Fail Flags x | move x Y -i_bs_get_integer_small_imm x W f? t x -i_bs_get_integer_imm x W t f? t x -i_bs_get_integer f? t t x s xy -i_bs_get_integer_8 x f? xy -i_bs_get_integer_16 x f? xy +i_bs_get_integer_small_imm xy W f? t x +i_bs_get_integer_imm xy W t f? t x +i_bs_get_integer f? t t xy s d +i_bs_get_integer_8 xy f? d +i_bs_get_integer_16 xy f? d %if ARCH_64 -i_bs_get_integer_32 x f? xy +i_bs_get_integer_32 xy f? d %endif # Fetching binaries from binaries. -bs_get_binary2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \ +bs_get_binary2 Fail=f Ms=xy Live=u Sz=sq Unit=u Flags=u Dst=d => \ gen_get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst) -i_bs_get_binary_imm2 f? x t W t xy -i_bs_get_binary2 f x t? s t xy -i_bs_get_binary_all2 f? x t t xy -i_bs_get_binary_all_reuse x f? t +i_bs_get_binary_imm2 f? xy t W t d +i_bs_get_binary2 f xy t? s t d +i_bs_get_binary_all2 f? xy t t d +i_bs_get_binary_all_reuse xy f? t # Fetching float from binaries. -bs_get_float2 Fail=f Ms=x Live=u Sz=s Unit=u Flags=u Dst=d => \ +bs_get_float2 Fail=f Ms=xy Live=u Sz=s Unit=u Flags=u Dst=d => \ gen_get_float2(Fail, Ms, Live, Sz, Unit, Flags, Dst) bs_get_float2 Fail=f Ms=x Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail -i_bs_get_float2 f? x t s t xy +i_bs_get_float2 f? xy t s t d # Miscellanous -bs_skip_bits2 Fail=f Ms=x Sz=sq Unit=u Flags=u => \ +bs_skip_bits2 Fail=f Ms=xy Sz=sq Unit=u Flags=u => \ gen_skip_bits2(Fail, Ms, Sz, Unit, Flags) -i_bs_skip_bits_imm2 f? x W -i_bs_skip_bits2 f? x xy t -i_bs_skip_bits_all2 f? x t +i_bs_skip_bits_imm2 f? xy W +i_bs_skip_bits2 f? xy xy t +i_bs_skip_bits_all2 f? xy t -bs_test_tail2 Fail=f Ms=x Bits=u==0 => bs_test_zero_tail2 Fail Ms -bs_test_tail2 Fail=f Ms=x Bits=u => bs_test_tail_imm2 Fail Ms Bits -bs_test_zero_tail2 f? x -bs_test_tail_imm2 f? x W +bs_test_tail2 Fail=f Ms=xy Bits=u==0 => bs_test_zero_tail2 Fail Ms +bs_test_tail2 Fail=f Ms=xy Bits=u => bs_test_tail_imm2 Fail Ms Bits +bs_test_zero_tail2 f? xy +bs_test_tail_imm2 f? xy W bs_test_unit F Ms Unit=u==8 => bs_test_unit8 F Ms -bs_test_unit f? x t -bs_test_unit8 f? x +bs_test_unit f? xy t +bs_test_unit8 f? xy # An y register operand for bs_context_to_binary is rare, # but can happen because of inlining. @@ -1167,23 +1167,55 @@ bs_context_to_binary Y=y => move Y x | bs_context_to_binary x bs_context_to_binary x +# Gets a bitstring from the tail of a context. +bs_get_tail xy d t + +# New bs_start_match variant for contexts with external position storage. +# +# bs_get/set_position is used to save positions into registers instead of +# "slots" in the context itself, which lets us continue matching even after +# we've passed it off to another function. + +%if ARCH_64 +bs_start_match3 Fail Bin Live Ctx | bs_get_position Ctx Pos=x Ignored => \ + i_bs_start_match3_gp Bin Live Fail Ctx Pos +i_bs_start_match3_gp xy t f d x +%endif + +bs_start_match3 Fail=f ica Live Dst => jump Fail +bs_start_match3 Fail Bin Live Dst => i_bs_start_match3 Bin Live Fail Dst + +i_bs_start_match3 xy t f d + +# Match context position instructions. 64-bit assumes that all positions can +# fit into an unsigned small. + +%if ARCH_64 + bs_get_position Src Dst Live => i_bs_get_position Src Dst + i_bs_get_position xy xy + bs_set_position xy xy +%else + bs_get_position xy d t? + bs_set_position xy xy +%endif + # # Utf8/utf16/utf32 support. (R12B-5) # -bs_get_utf8 Fail=f Ms=x u u Dst=d => i_bs_get_utf8 Ms Fail Dst -i_bs_get_utf8 x f? xy +bs_get_utf8 Fail=f Ms=xy u u Dst=d => i_bs_get_utf8 Ms Fail Dst +i_bs_get_utf8 xy f? d -bs_skip_utf8 Fail=f Ms=x u u => i_bs_get_utf8 Ms Fail x +bs_skip_utf8 Fail=f Ms=xy u u => i_bs_get_utf8 Ms Fail x -bs_get_utf16 Fail=f Ms=x u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst -bs_skip_utf16 Fail=f Ms=x u Flags=u => i_bs_get_utf16 Ms Fail Flags x +bs_get_utf16 Fail=f Ms=xy u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst +bs_skip_utf16 Fail=f Ms=xy u Flags=u => i_bs_get_utf16 Ms Fail Flags x -i_bs_get_utf16 x f? t xy +i_bs_get_utf16 xy f? t d -bs_get_utf32 Fail=f Ms=x Live=u Flags=u Dst=d => \ +bs_get_utf32 Fail=f Ms=xy Live=u Flags=u Dst=d => \ bs_get_integer2 Fail Ms Live i=32 u=1 Flags Dst | \ i_bs_validate_unicode_retract Fail Dst Ms -bs_skip_utf32 Fail=f Ms=x Live=u Flags=u => \ +bs_skip_utf32 Fail=f Ms=xy Live=u Flags=u => \ bs_get_integer2 Fail Ms Live i=32 u=1 Flags x | \ i_bs_validate_unicode_retract Fail x Ms diff --git a/erts/emulator/nifs/common/prim_file_nif.c b/erts/emulator/nifs/common/prim_file_nif.c index a05d50b333..9b98ac3f9a 100644 --- a/erts/emulator/nifs/common/prim_file_nif.c +++ b/erts/emulator/nifs/common/prim_file_nif.c @@ -442,7 +442,8 @@ static ERL_NIF_TERM open_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[] ERL_NIF_TERM result; efile_path_t path; - if(argc != 2 || !enif_is_list(env, argv[1])) { + ASSERT(argc == 2); + if(!enif_is_list(env, argv[1])) { return enif_make_badarg(env); } @@ -469,9 +470,7 @@ static ERL_NIF_TERM open_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[] static ERL_NIF_TERM close_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { enum efile_state_t previous_state; - if(argc != 0) { - return enif_make_badarg(env); - } + ASSERT(argc == 0); previous_state = erts_atomic32_cmpxchg_acqb(&d->state, EFILE_STATE_CLOSED, EFILE_STATE_BUSY); @@ -495,7 +494,8 @@ static ERL_NIF_TERM read_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, con SysIOVec read_vec[1]; ErlNifBinary result; - if(argc != 1 || !enif_is_number(env, argv[0])) { + ASSERT(argc == 1); + if(!enif_is_number(env, argv[0])) { return enif_make_badarg(env); } @@ -532,7 +532,8 @@ static ERL_NIF_TERM write_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, co Sint64 bytes_written; ERL_NIF_TERM tail; - if(argc != 1 || !enif_inspect_iovec(env, 64, argv[0], &tail, &input)) { + ASSERT(argc == 1); + if(!enif_inspect_iovec(env, 64, argv[0], &tail, &input)) { return enif_make_badarg(env); } @@ -555,8 +556,8 @@ static ERL_NIF_TERM pread_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, co SysIOVec read_vec[1]; ErlNifBinary result; - if(argc != 2 || !enif_is_number(env, argv[0]) - || !enif_is_number(env, argv[1])) { + ASSERT(argc == 2); + if(!enif_is_number(env, argv[0]) || !enif_is_number(env, argv[1])) { return enif_make_badarg(env); } @@ -594,8 +595,9 @@ static ERL_NIF_TERM pwrite_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, c Sint64 bytes_written, offset; ERL_NIF_TERM tail; - if(argc != 2 || !enif_is_number(env, argv[0]) - || !enif_inspect_iovec(env, 64, argv[1], &tail, &input)) { + ASSERT(argc == 2); + if(!enif_is_number(env, argv[0]) + || !enif_inspect_iovec(env, 64, argv[1], &tail, &input)) { return enif_make_badarg(env); } @@ -622,7 +624,8 @@ static ERL_NIF_TERM seek_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, con Sint64 new_position, offset; enum efile_seek_t seek; - if(argc != 2 || !enif_get_int64(env, argv[1], &offset)) { + ASSERT(argc == 2); + if(!enif_get_int64(env, argv[1], &offset)) { return enif_make_badarg(env); } @@ -646,7 +649,8 @@ static ERL_NIF_TERM seek_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, con static ERL_NIF_TERM sync_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { int data_only; - if(argc != 1 || !enif_get_int(env, argv[0], &data_only)) { + ASSERT(argc == 1); + if(!enif_get_int(env, argv[0], &data_only)) { return enif_make_badarg(env); } @@ -658,9 +662,7 @@ static ERL_NIF_TERM sync_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, con } static ERL_NIF_TERM truncate_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - if(argc != 0) { - return enif_make_badarg(env); - } + ASSERT(argc == 0); if(!efile_truncate(d)) { return posix_error_to_tuple(env, d->posix_errno); @@ -672,8 +674,8 @@ static ERL_NIF_TERM truncate_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, static ERL_NIF_TERM allocate_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { Sint64 offset, length; - if(argc != 2 || !enif_is_number(env, argv[0]) - || !enif_is_number(env, argv[1])) { + ASSERT(argc == 2); + if(!enif_is_number(env, argv[0]) || !enif_is_number(env, argv[1])) { return enif_make_badarg(env); } @@ -694,8 +696,8 @@ static ERL_NIF_TERM advise_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, c enum efile_advise_t advise; Sint64 offset, length; - if(argc != 3 || !enif_is_number(env, argv[0]) - || !enif_is_number(env, argv[1])) { + ASSERT(argc == 3); + if(!enif_is_number(env, argv[0]) || !enif_is_number(env, argv[1])) { return enif_make_badarg(env); } @@ -758,8 +760,8 @@ static ERL_NIF_TERM ipread_s32bu_p32bu_nif_impl(efile_data_t *d, ErlNifEnv *env, ErlNifBinary payload; - if(argc != 2 || !enif_is_number(env, argv[0]) - || !enif_is_number(env, argv[1])) { + ASSERT(argc == 2); + if(!enif_is_number(env, argv[0]) || !enif_is_number(env, argv[1])) { return enif_make_badarg(env); } @@ -825,9 +827,7 @@ static ERL_NIF_TERM ipread_s32bu_p32bu_nif_impl(efile_data_t *d, ErlNifEnv *env, } static ERL_NIF_TERM get_handle_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - if(argc != 0) { - return enif_make_badarg(env); - } + ASSERT(argc == 0); return efile_get_handle(env, d); } @@ -839,7 +839,8 @@ static ERL_NIF_TERM read_info_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM a efile_path_t path; int follow_links; - if(argc != 2 || !enif_get_int(env, argv[1], &follow_links)) { + ASSERT(argc == 2); + if(!enif_get_int(env, argv[1], &follow_links)) { return enif_make_badarg(env); } @@ -874,7 +875,8 @@ static ERL_NIF_TERM set_permissions_nif(ErlNifEnv *env, int argc, const ERL_NIF_ efile_path_t path; Uint32 permissions; - if(argc != 2 || !enif_get_uint(env, argv[1], &permissions)) { + ASSERT(argc == 2); + if(!enif_get_uint(env, argv[1], &permissions)) { return enif_make_badarg(env); } @@ -893,8 +895,8 @@ static ERL_NIF_TERM set_owner_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM a efile_path_t path; Sint32 uid, gid; - if(argc != 3 || !enif_get_int(env, argv[1], &uid) - || !enif_get_int(env, argv[2], &gid)) { + ASSERT(argc == 3); + if(!enif_get_int(env, argv[1], &uid) || !enif_get_int(env, argv[2], &gid)) { return enif_make_badarg(env); } @@ -913,9 +915,10 @@ static ERL_NIF_TERM set_time_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar Sint64 accessed, modified, created; efile_path_t path; - if(argc != 4 || !enif_get_int64(env, argv[1], &accessed) - || !enif_get_int64(env, argv[2], &modified) - || !enif_get_int64(env, argv[3], &created)) { + ASSERT(argc == 4); + if(!enif_get_int64(env, argv[1], &accessed) + || !enif_get_int64(env, argv[2], &modified) + || !enif_get_int64(env, argv[3], &created)) { return enif_make_badarg(env); } @@ -934,9 +937,7 @@ static ERL_NIF_TERM read_link_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM a efile_path_t path; ERL_NIF_TERM result; - if(argc != 1) { - return enif_make_badarg(env); - } + ASSERT(argc == 1); if((posix_errno = efile_marshal_path(env, argv[0], &path))) { return posix_error_to_tuple(env, posix_errno); @@ -953,9 +954,7 @@ static ERL_NIF_TERM list_dir_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar efile_path_t path; ERL_NIF_TERM result; - if(argc != 1) { - return enif_make_badarg(env); - } + ASSERT(argc == 1); if((posix_errno = efile_marshal_path(env, argv[0], &path))) { return posix_error_to_tuple(env, posix_errno); @@ -971,9 +970,7 @@ static ERL_NIF_TERM rename_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv efile_path_t existing_path, new_path; - if(argc != 2) { - return enif_make_badarg(env); - } + ASSERT(argc == 2); if((posix_errno = efile_marshal_path(env, argv[0], &existing_path))) { return posix_error_to_tuple(env, posix_errno); @@ -991,9 +988,7 @@ static ERL_NIF_TERM make_hard_link_nif(ErlNifEnv *env, int argc, const ERL_NIF_T efile_path_t existing_path, new_path; - if(argc != 2) { - return enif_make_badarg(env); - } + ASSERT(argc == 2); if((posix_errno = efile_marshal_path(env, argv[0], &existing_path))) { return posix_error_to_tuple(env, posix_errno); @@ -1011,9 +1006,7 @@ static ERL_NIF_TERM make_soft_link_nif(ErlNifEnv *env, int argc, const ERL_NIF_T efile_path_t existing_path, new_path; - if(argc != 2) { - return enif_make_badarg(env); - } + ASSERT(argc == 2); if((posix_errno = efile_marshal_path(env, argv[0], &existing_path))) { return posix_error_to_tuple(env, posix_errno); @@ -1031,9 +1024,7 @@ static ERL_NIF_TERM make_dir_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar efile_path_t path; - if(argc != 1) { - return enif_make_badarg(env); - } + ASSERT(argc == 1); if((posix_errno = efile_marshal_path(env, argv[0], &path))) { return posix_error_to_tuple(env, posix_errno); @@ -1049,9 +1040,7 @@ static ERL_NIF_TERM del_file_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar efile_path_t path; - if(argc != 1) { - return enif_make_badarg(env); - } + ASSERT(argc == 1); if((posix_errno = efile_marshal_path(env, argv[0], &path))) { return posix_error_to_tuple(env, posix_errno); @@ -1067,9 +1056,7 @@ static ERL_NIF_TERM del_dir_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM arg efile_path_t path; - if(argc != 1) { - return enif_make_badarg(env); - } + ASSERT(argc == 1); if((posix_errno = efile_marshal_path(env, argv[0], &path))) { return posix_error_to_tuple(env, posix_errno); @@ -1086,7 +1073,8 @@ static ERL_NIF_TERM get_device_cwd_nif(ErlNifEnv *env, int argc, const ERL_NIF_T ERL_NIF_TERM result; int device_index; - if(argc != 1 || !enif_get_int(env, argv[0], &device_index)) { + ASSERT(argc == 1); + if(!enif_get_int(env, argv[0], &device_index)) { return enif_make_badarg(env); } @@ -1101,9 +1089,7 @@ static ERL_NIF_TERM get_cwd_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM arg posix_errno_t posix_errno; ERL_NIF_TERM result; - if(argc != 0) { - return enif_make_badarg(env); - } + ASSERT(argc == 0); if((posix_errno = efile_get_cwd(env, &result))) { return posix_error_to_tuple(env, posix_errno); @@ -1117,9 +1103,7 @@ static ERL_NIF_TERM set_cwd_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM arg efile_path_t path; - if(argc != 1) { - return enif_make_badarg(env); - } + ASSERT(argc == 1); if((posix_errno = efile_marshal_path(env, argv[0], &path))) { return posix_error_to_tuple(env, posix_errno); @@ -1195,9 +1179,7 @@ static ERL_NIF_TERM read_file_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM a ErlNifBinary result; - if(argc != 1) { - return enif_make_badarg(env); - } + ASSERT(argc == 1); if((posix_errno = efile_marshal_path(env, argv[0], &path))) { return posix_error_to_tuple(env, posix_errno); @@ -1223,9 +1205,7 @@ static ERL_NIF_TERM altname_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM arg efile_path_t path; ERL_NIF_TERM result; - if(argc != 1) { - return enif_make_badarg(env); - } + ASSERT(argc == 1); if((posix_errno = efile_marshal_path(env, argv[0], &path))) { return posix_error_to_tuple(env, posix_errno); diff --git a/erts/emulator/test/smoke_test_SUITE.erl b/erts/emulator/test/smoke_test_SUITE.erl index 26c610e3a8..5b46342127 100644 --- a/erts/emulator/test/smoke_test_SUITE.erl +++ b/erts/emulator/test/smoke_test_SUITE.erl @@ -56,7 +56,7 @@ end_per_testcase(_Case, Config) when is_list(Config) -> %%% boot_combo(Config) when is_list(Config) -> - ZFlags = os:getenv("ERL_ZFLAGS"), + ZFlags = os:getenv("ERL_ZFLAGS", ""), NOOP = fun () -> ok end, A42 = fun () -> case erlang:system_info(threads) of @@ -87,10 +87,7 @@ boot_combo(Config) when is_list(Config) -> %% A lot more combos could be implemented... ok after - os:putenv("ERL_ZFLAGS", case ZFlags of - false -> ""; - _ -> ZFlags - end) + os:putenv("ERL_ZFLAGS", ZFlags) end. native_atomics(Config) when is_list(Config) -> diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index 0cb01fd4ef..ec4a4ead23 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -819,10 +819,8 @@ int main(int argc, char **argv) case '+': switch (argv[i][1]) { - case '#': case 'a': case 'A': - case 'b': case 'C': case 'e': case 'i': diff --git a/erts/preloaded/ebin/erl_init.beam b/erts/preloaded/ebin/erl_init.beam Binary files differnew file mode 100644 index 0000000000..93f23377b5 --- /dev/null +++ b/erts/preloaded/ebin/erl_init.beam diff --git a/erts/preloaded/ebin/erl_prim_loader.beam b/erts/preloaded/ebin/erl_prim_loader.beam Binary files differindex 0f5f5036f0..42629bbe8e 100644 --- a/erts/preloaded/ebin/erl_prim_loader.beam +++ b/erts/preloaded/ebin/erl_prim_loader.beam diff --git a/erts/preloaded/ebin/erl_tracer.beam b/erts/preloaded/ebin/erl_tracer.beam Binary files differindex 6017112dac..a74e87d19c 100644 --- a/erts/preloaded/ebin/erl_tracer.beam +++ b/erts/preloaded/ebin/erl_tracer.beam diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam Binary files differindex 7a4167b0e9..54557d9d18 100644 --- a/erts/preloaded/ebin/erlang.beam +++ b/erts/preloaded/ebin/erlang.beam diff --git a/erts/preloaded/ebin/erts_code_purger.beam b/erts/preloaded/ebin/erts_code_purger.beam Binary files differindex c899b69a2c..3cbfd94a43 100644 --- a/erts/preloaded/ebin/erts_code_purger.beam +++ b/erts/preloaded/ebin/erts_code_purger.beam diff --git a/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam b/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam Binary files differindex 9490a56758..43604e0480 100644 --- a/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam +++ b/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam diff --git a/erts/preloaded/ebin/erts_internal.beam b/erts/preloaded/ebin/erts_internal.beam Binary files differindex 15c59de80a..960a4451d3 100644 --- a/erts/preloaded/ebin/erts_internal.beam +++ b/erts/preloaded/ebin/erts_internal.beam diff --git a/erts/preloaded/ebin/erts_literal_area_collector.beam b/erts/preloaded/ebin/erts_literal_area_collector.beam Binary files differindex e650a6b5af..1f7fbef85a 100644 --- a/erts/preloaded/ebin/erts_literal_area_collector.beam +++ b/erts/preloaded/ebin/erts_literal_area_collector.beam diff --git a/erts/preloaded/ebin/init.beam b/erts/preloaded/ebin/init.beam Binary files differindex 858a9dc63e..b6107f8974 100644 --- a/erts/preloaded/ebin/init.beam +++ b/erts/preloaded/ebin/init.beam diff --git a/erts/preloaded/ebin/prim_buffer.beam b/erts/preloaded/ebin/prim_buffer.beam Binary files differindex 4ad1380d0b..70675f47ee 100644 --- a/erts/preloaded/ebin/prim_buffer.beam +++ b/erts/preloaded/ebin/prim_buffer.beam diff --git a/erts/preloaded/ebin/prim_eval.beam b/erts/preloaded/ebin/prim_eval.beam Binary files differindex 2ae18846bf..5aa5a7f384 100644 --- a/erts/preloaded/ebin/prim_eval.beam +++ b/erts/preloaded/ebin/prim_eval.beam diff --git a/erts/preloaded/ebin/prim_file.beam b/erts/preloaded/ebin/prim_file.beam Binary files differindex df611f2bb0..6a9528d1e2 100644 --- a/erts/preloaded/ebin/prim_file.beam +++ b/erts/preloaded/ebin/prim_file.beam diff --git a/erts/preloaded/ebin/prim_inet.beam b/erts/preloaded/ebin/prim_inet.beam Binary files differindex 4a345f8152..b36be923a3 100644 --- a/erts/preloaded/ebin/prim_inet.beam +++ b/erts/preloaded/ebin/prim_inet.beam diff --git a/erts/preloaded/ebin/prim_zip.beam b/erts/preloaded/ebin/prim_zip.beam Binary files differindex 4923cadbdc..8483d11aed 100644 --- a/erts/preloaded/ebin/prim_zip.beam +++ b/erts/preloaded/ebin/prim_zip.beam diff --git a/erts/preloaded/ebin/zlib.beam b/erts/preloaded/ebin/zlib.beam Binary files differindex 07e7e97814..f3737a524e 100644 --- a/erts/preloaded/ebin/zlib.beam +++ b/erts/preloaded/ebin/zlib.beam diff --git a/erts/preloaded/src/Makefile b/erts/preloaded/src/Makefile index 4333f6643a..3c8d75b205 100644 --- a/erts/preloaded/src/Makefile +++ b/erts/preloaded/src/Makefile @@ -41,7 +41,7 @@ PRE_LOADED_ERL_MODULES = \ prim_inet \ zlib \ prim_zip \ - otp_ring0 \ + erl_init \ erts_code_purger \ erlang \ erts_internal \ diff --git a/erts/preloaded/src/otp_ring0.erl b/erts/preloaded/src/erl_init.erl index 62a60fffe2..d681f05398 100644 --- a/erts/preloaded/src/otp_ring0.erl +++ b/erts/preloaded/src/erl_init.erl @@ -17,15 +17,26 @@ %% %% %CopyrightEnd% %% --module(otp_ring0). +-module(erl_init). -%% Purpose : Start up of erlang system. +%% Initial process of an Erlang system. -export([start/2]). --spec start(_, term()) -> term(). -start(_Env, Argv) -> - run(init, boot, Argv). +%% This gets the module name given by the +i option (default 'init') +%% and the list of command line arguments + +-spec start(Mod, BootArgs) -> no_return() when + Mod :: module(), + BootArgs :: [binary()]. +start(Mod, BootArgs) -> + %% Load the static nifs + zlib:on_load(), + erl_tracer:on_load(), + prim_buffer:on_load(), + prim_file:on_load(), + %% Proceed to the specified boot module + run(Mod, boot, BootArgs). run(M, F, A) -> case erlang:function_exported(M, F, 1) of diff --git a/erts/preloaded/src/erts.app.src b/erts/preloaded/src/erts.app.src index 8c34c99a98..dc97b86d8c 100644 --- a/erts/preloaded/src/erts.app.src +++ b/erts/preloaded/src/erts.app.src @@ -26,7 +26,7 @@ erl_prim_loader, erts_internal, init, - otp_ring0, + erl_init, erts_code_purger, prim_buffer, prim_eval, diff --git a/erts/preloaded/src/init.erl b/erts/preloaded/src/init.erl index 253fcf7a1f..277e60c180 100644 --- a/erts/preloaded/src/init.erl +++ b/erts/preloaded/src/init.erl @@ -200,12 +200,6 @@ boot(BootArgs) -> register(init, self()), process_flag(trap_exit, true), - %% Load the static nifs - zlib:on_load(), - erl_tracer:on_load(), - prim_buffer:on_load(), - prim_file:on_load(), - {Start0,Flags,Args} = parse_boot_args(BootArgs), %% We don't get to profile parsing of BootArgs case b2a(get_flag(profile_boot, Flags, false)) of diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 778ea2e9e2..bfa7b25862 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -390,11 +390,7 @@ testcases(TestDir, Suite) -> end. make_and_load(Dir, Suite) -> - EnvInclude = - case os:getenv("CT_INCLUDE_PATH") of - false -> []; - CtInclPath -> string:lexemes(CtInclPath, [$:,$ ,$,]) - end, + EnvInclude = string:lexemes(os:getenv("CT_INCLUDE_PATH", ""), [$:,$ ,$,]), StartInclude = case init:get_argument(include) of {ok,[Dirs]} -> Dirs; diff --git a/lib/common_test/test/ct_config_SUITE.erl b/lib/common_test/test/ct_config_SUITE.erl index 5ffc735d6a..ec5278b96d 100644 --- a/lib/common_test/test/ct_config_SUITE.erl +++ b/lib/common_test/test/ct_config_SUITE.erl @@ -211,18 +211,12 @@ reformat_events(Events, EH) -> %%% Test related to 'localtime' will often fail if the test host is %%% time warping, so let's just skip the 'dynamic' tests then. skip_dynamic() -> - case os:getenv("TS_EXTRA_PLATFORM_LABEL") of - TSExtraPlatformLabel when is_list(TSExtraPlatformLabel) -> - case string:find(TSExtraPlatformLabel,"TimeWarpingOS") of - nomatch -> false; - _ -> true - end; - _ -> - false + case string:find(os:getenv("TS_EXTRA_PLATFORM_LABEL", ""), "TimeWarpingOS") of + nomatch -> false; + _ -> true end. - %%%----------------------------------------------------------------- %%% TEST EVENTS %%%----------------------------------------------------------------- diff --git a/lib/common_test/test_server/ts_install.erl b/lib/common_test/test_server/ts_install.erl index 048e5493d2..09f3da860a 100644 --- a/lib/common_test/test_server/ts_install.erl +++ b/lib/common_test/test_server/ts_install.erl @@ -112,12 +112,6 @@ get_vars([], name, [], Result) -> get_vars(_, _, _, _) -> {error, fatal_bad_conf_vars}. -config_flags() -> - case os:getenv("CONFIG_FLAGS") of - false -> []; - CF -> string:lexemes(CF, " \t\n") - end. - unix_autoconf(XConf) -> Configure = filename:absname("configure"), Flags = proplists:get_value(crossflags,XConf,[]), @@ -128,7 +122,7 @@ unix_autoconf(XConf) -> erlang:system_info(threads) /= false], Debug = [" --enable-debug-mode" || string:find(erlang:system_info(system_version),"debug") =/= nomatch], - MXX_Build = [Y || Y <- config_flags(), + MXX_Build = [Y || Y <- string:lexemes(os:getenv("CONFIG_FLAGS", ""), " \t\n"), Y == "--enable-m64-build" orelse Y == "--enable-m32-build"], Args = Host ++ Build ++ Threads ++ Debug ++ " " ++ MXX_Build, @@ -164,7 +158,7 @@ assign_vars(FlagsStr) -> assign_all_vars([$$ | Rest], FlagSoFar) -> {VarName,Rest1} = get_var_name(Rest, []), - assign_all_vars(Rest1, FlagSoFar ++ assign_var(VarName)); + assign_all_vars(Rest1, FlagSoFar ++ os:getenv(VarName, "")); assign_all_vars([Char | Rest], FlagSoFar) -> assign_all_vars(Rest, FlagSoFar ++ [Char]); assign_all_vars([], Flag) -> @@ -177,12 +171,6 @@ get_var_name([Ch | Rest] = Str, VarR) -> end; get_var_name([], VarR) -> {lists:reverse(VarR),[]}. - -assign_var(VarName) -> - case os:getenv(VarName) of - false -> ""; - Val -> Val - end. valid_char(Ch) when Ch >= $a, Ch =< $z -> true; valid_char(Ch) when Ch >= $A, Ch =< $Z -> true; @@ -280,7 +268,7 @@ add_vars(Vars0, Opts0) -> {Opts, [{longnames, LongNames}, {platform_id, PlatformId}, {platform_filename, PlatformFilename}, - {rsh_name, get_rsh_name()}, + {rsh_name, os:getenv("ERL_RSH", "rsh")}, {platform_label, PlatformLabel}, {ts_net_dir, Mounted}, {erl_flags, []}, @@ -301,16 +289,10 @@ get_testcase_callback() -> end end. -get_rsh_name() -> - case os:getenv("ERL_RSH") of - false -> "rsh"; - Str -> Str - end. - platform_id(Vars) -> {Id,_,_,_} = platform(Vars), Id. - + platform(Vars) -> Hostname = hostname(), diff --git a/lib/compiler/doc/src/compile.xml b/lib/compiler/doc/src/compile.xml index 1a71c83521..45e442f5c2 100644 --- a/lib/compiler/doc/src/compile.xml +++ b/lib/compiler/doc/src/compile.xml @@ -203,7 +203,8 @@ <tag><c>deterministic</c></tag> <item> <p>Omit the <c>options</c> and <c>source</c> tuples in - the list returned by <c>Module:module_info(compile)</c>. + the list returned by <c>Module:module_info(compile)</c>, and + reduce the paths in stack traces to the module name alone. This option will make it easier to achieve reproducible builds. </p> </item> @@ -347,8 +348,8 @@ module.beam: module.erl \ <tag><c>{source,FileName}</c></tag> <item> - <p>Sets the value of the source, as returned by - <c>module_info(compile)</c>.</p> + <p>Overrides the source file name as presented in + <c>module_info(compile)</c> and stack traces.</p> </item> <tag><c>{outdir,Dir}</c></tag> @@ -415,6 +416,17 @@ module.beam: module.erl \ is not documented, and can change between releases.</p> </item> + <tag><c>no_spawn_compiler_process</c></tag> + <item> + <p>By default, all code is compiled in a separate process + which is terminated at the end of compilation. However, + some tools, like Dialyzer or compilers for other BEAM languages, + may already manage their own worker processes and spawning + an extra process may slow the compilation down. + In such scenarios, you can pass this option to stop the + compiler from spawning an additional process.</p> + </item> + <tag><c>no_strict_record_tests</c></tag> <item> <p>This option is not recommended.</p> diff --git a/lib/compiler/src/Makefile b/lib/compiler/src/Makefile index 522523726b..d475e5a19a 100644 --- a/lib/compiler/src/Makefile +++ b/lib/compiler/src/Makefile @@ -50,7 +50,6 @@ MODULES = \ beam_asm \ beam_block \ beam_bs \ - beam_bsm \ beam_clean \ beam_dict \ beam_disasm \ @@ -61,8 +60,10 @@ MODULES = \ beam_opcodes \ beam_peep \ beam_ssa \ + beam_ssa_bsm \ beam_ssa_codegen \ beam_ssa_dead \ + beam_ssa_funs \ beam_ssa_lint \ beam_ssa_opt \ beam_ssa_pp \ diff --git a/lib/compiler/src/beam_a.erl b/lib/compiler/src/beam_a.erl index 0abc845310..dd2537a699 100644 --- a/lib/compiler/src/beam_a.erl +++ b/lib/compiler/src/beam_a.erl @@ -59,6 +59,9 @@ rename_instrs([{test,is_eq_exact,_,[Dst,Src]}=Test, rename_instrs([{test,is_eq_exact,_,[Same,Same]}|Is]) -> %% Same literal or same register. Will always succeed. rename_instrs(Is); +rename_instrs([{loop_rec,{f,Fail},{x,0}},{loop_rec_end,_},{label,Fail}|Is]) -> + %% This instruction sequence does nothing. + rename_instrs(Is); rename_instrs([{apply_last,A,N}|Is]) -> [{apply,A},{deallocate,N},return|rename_instrs(Is)]; rename_instrs([{call_last,A,F,N}|Is]) -> diff --git a/lib/compiler/src/beam_block.erl b/lib/compiler/src/beam_block.erl index d28c0fd9e4..9d8d5b2b0c 100644 --- a/lib/compiler/src/beam_block.erl +++ b/lib/compiler/src/beam_block.erl @@ -49,9 +49,6 @@ function({function,Name,Arity,CLabel,Is0}) -> blockify(Is) -> blockify(Is, []). -blockify([{loop_rec,{f,Fail},{x,0}},{loop_rec_end,_Lbl},{label,Fail}|Is], Acc) -> - %% Useless instruction sequence. - blockify(Is, Acc); blockify([I|Is0]=IsAll, Acc) -> case collect(I) of error -> blockify(Is0, [I|Acc]); diff --git a/lib/compiler/src/beam_bsm.erl b/lib/compiler/src/beam_bsm.erl deleted file mode 100644 index abc6e96c85..0000000000 --- a/lib/compiler/src/beam_bsm.erl +++ /dev/null @@ -1,719 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2007-2018. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% - --module(beam_bsm). --export([module/2,format_error/1]). - --import(lists, [member/2,foldl/3,reverse/1,sort/1,all/2]). - -%%% -%%% We optimize bit syntax matching where the tail end of a binary is -%%% matched out and immediately passed on to a bs_start_match2 instruction, -%%% such as in this code sequence: -%%% -%%% func_info ... -%%% L1 test bs_start_match2 {f,...} {x,0} Live SavePositions {x,0} -%%% . . . -%%% test bs_get_binary2 {f,...} {x,0} all 1 Flags {x,0} -%%% . . . -%%% call_only 2 L1 -%%% -%%% The sequence can be optimized simply by removing the bs_get_binary2 -%%% instruction. Another example: -%%% -%%% func_info ... -%%% L1 test bs_start_match2 {f,...} {x,0} Live SavePositions {x,0} -%%% . . . -%%% test bs_get_binary2 {f,...} {x,0} all 8 Flags {x,1} -%%% . . . -%%% move {x,1} {x,0} -%%% call_only 2 L1 -%%% -%%% In this case, the bs_get_binary2 instruction must be replaced by -%%% -%%% test bs_unit {x,1} 8 -%%% -%%% to ensure that the match fail if the length of the binary in bits -%%% is not evenly divisible by 8. -%%% -%%% Note that the bs_start_match2 instruction doesn't need to be in the same -%%% function as the caller. It can be in the beginning of any function, or -%%% follow the bs_get_binary2 instruction in the same function. The important -%%% thing is that the match context register is not copied or built into -%%% data structures or passed to BIFs. -%%% - --type label() :: beam_asm:label(). --type func_info() :: {beam_asm:reg(),boolean()}. - --record(btb, - {f :: gb_trees:tree(label(), func_info()), - index :: beam_utils:code_index(), %{Label,Code} index (for liveness). - ok_br=gb_sets:empty() :: gb_sets:set(label()), %Labels that are OK. - must_not_save=false :: boolean(), %Must not save position when - % optimizing (reaches - % bs_context_to_binary). - must_save=false :: boolean() %Must save position when optimizing. - }). - - --spec module(beam_utils:module_code(), [compile:option()]) -> - {'ok',beam_utils:module_code()}. - -module({Mod,Exp,Attr,Fs0,Lc}, Opts) -> - FIndex = btb_index(Fs0), - Fs = [function(F, FIndex) || F <- Fs0], - Code = {Mod,Exp,Attr,Fs,Lc}, - case proplists:get_bool(bin_opt_info, Opts) of - true -> - {ok,Code,collect_warnings(Fs)}; - false -> - {ok,Code} - end. - --spec format_error('bin_opt' | {'no_bin_opt', term()}) -> nonempty_string(). - -format_error(bin_opt) -> - "OPTIMIZED: creation of sub binary delayed"; -format_error({no_bin_opt,Reason}) -> - lists:flatten(["NOT OPTIMIZED: "|format_error_1(Reason)]). - -%%% -%%% Local functions. -%%% - -function({function,Name,Arity,Entry,Is}, FIndex) -> - try - Index = beam_utils:index_labels(Is), - D = #btb{f=FIndex,index=Index}, - {function,Name,Arity,Entry,btb_opt_1(Is, D, [])} - catch - Class:Error:Stack -> - io:fwrite("Function: ~w/~w\n", [Name,Arity]), - erlang:raise(Class, Error, Stack) - end. - -btb_opt_1([{test,bs_get_binary2,F,_,[Reg,{atom,all},U,Fs],Reg}=I0|Is], D, Acc0) -> - case btb_reaches_match(Is, [Reg], D) of - {error,Reason} -> - Comment = btb_comment_no_opt(Reason, Fs), - btb_opt_1(Is, D, [Comment,I0|Acc0]); - {ok,MustSave} -> - Comment = btb_comment_opt(Fs), - Acc1 = btb_gen_save(MustSave, Reg, [Comment|Acc0]), - Acc = case U of - 1 -> Acc1; - _ -> [{test,bs_test_unit,F,[Reg,U]}|Acc1] - end, - btb_opt_1(Is, D, Acc) - end; -btb_opt_1([{test,bs_get_binary2,F,_,[Ctx,{atom,all},U,Fs],Dst}=I0|Is0], D, Acc0) -> - case btb_reaches_match(Is0, [Ctx,Dst], D) of - {error,Reason} -> - Comment = btb_comment_no_opt(Reason, Fs), - btb_opt_1(Is0, D, [Comment,I0|Acc0]); - {ok,MustSave} when U =:= 1 -> - Comment = btb_comment_opt(Fs), - Acc = btb_gen_save(MustSave, Ctx, [Comment|Acc0]), - Is = prepend_move(Ctx, Dst, Is0), - btb_opt_1(Is, D, Acc); - {ok,MustSave} -> - Comment = btb_comment_opt(Fs), - Acc1 = btb_gen_save(MustSave, Ctx, [Comment|Acc0]), - Acc = [{test,bs_test_unit,F,[Ctx,U]}|Acc1], - Is = prepend_move(Ctx, Dst, Is0), - btb_opt_1(Is, D, Acc) - end; -btb_opt_1([I|Is], D, Acc) -> - %%io:format("~p\n", [I]), - btb_opt_1(Is, D, [I|Acc]); -btb_opt_1([], _, Acc) -> - reverse(Acc). - -btb_gen_save(true, Reg, Acc) -> - [{bs_save2,Reg,{atom,start}}|Acc]; -btb_gen_save(false, _, Acc) -> Acc. - -prepend_move(Ctx, Dst, [{block,Bl0}|Is]) -> - Bl = [{set,[Dst],[Ctx],move}|Bl0], - [{block,Bl}|Is]; -prepend_move(Ctx, Dst, Is) -> - [{move,Ctx,Dst}|Is]. - -%% btb_reaches_match([Instruction], [Register], D) -> -%% {ok,MustSave}|{error,Reason} -%% -%% The list of Registers should be a list of registers referencing a -%% match context. The Register may contain one element if the -%% bs_get_binary2 instruction looks like -%% -%% test bs_get_binary2 {f,...} Ctx all _ _ Ctx -%% -%% or two elements if the instruction looks like -%% -%% test bs_get_binary2 {f,...} Ctx all _ _ Dst -%% -%% This function determines whether the bs_get_binary2 instruction -%% can be omitted (retaining the match context instead of creating -%% a sub binary). -%% -%% The rule is that the match context ultimately must end up at a -%% bs_start_match2 instruction and nowhere else. That it, it must not -%% be passed to BIFs, or copied or put into data structures. There -%% must only be one copy alive when the match context reaches the -%% bs_start_match2 instruction. -%% -%% At a branch, we must follow all branches and make sure that the above -%% rule is followed (or that the branch kills the match context). -%% -%% The MustSave return value will be true if control may end up at -%% bs_context_to_binary instruction. Since that instruction uses the -%% saved start position, we must use "bs_save2 Ctx start" to -%% update the saved start position. An additional complication is that -%% "bs_save2 Ctx start" must not be used if Dst and Ctx are -%% different registers and both registers may be passed to -%% a bs_context_to_binary instruction. -%% - -btb_reaches_match(Is, RegList, D) -> - try - Regs = btb_regs_from_list(RegList), - #btb{must_not_save=MustNotSave,must_save=MustSave} = - btb_reaches_match_1(Is, Regs, D), - case MustNotSave andalso MustSave of - true -> btb_error(must_and_must_not_save); - false -> {ok,MustSave} - end - catch - throw:{error,_}=Error -> Error - end. - -btb_reaches_match_1(Is, Regs, D) -> - case btb_are_registers_empty(Regs) of - false -> - btb_reaches_match_2(Is, Regs, D); - true -> - %% The context was killed, which is OK. - D - end. - -btb_reaches_match_2([{block,Bl}|Is], Regs0, D) -> - Regs = btb_reaches_match_block(Bl, Regs0), - btb_reaches_match_1(Is, Regs, D); -btb_reaches_match_2([{call,Arity,{f,Lbl}}|Is], Regs0, D) -> - case is_tail_call(Is) of - true -> - Regs1 = btb_kill_not_live(Arity, Regs0), - Regs = btb_kill_yregs(Regs1), - btb_tail_call(Lbl, Regs, D); - false -> - btb_call(Arity, Lbl, Regs0, Is, D) - end; -btb_reaches_match_2([{apply,Arity}|Is], Regs, D) -> - btb_call(Arity+2, apply, Regs, Is, D); -btb_reaches_match_2([{call_fun,Live}=I|Is], Regs, D) -> - btb_ensure_not_used([{x,Live}], I, Regs), - btb_call(Live, I, Regs, Is, D); -btb_reaches_match_2([{make_fun2,_,_,_,Live}|Is], Regs, D) -> - btb_call(Live, make_fun2, Regs, Is, D); -btb_reaches_match_2([{call_ext,Arity,Func}=I|Is], Regs0, D) -> - %% Allow us scanning beyond the call in case the match - %% context is saved on the stack. - case beam_jump:is_exit_instruction(I) of - false -> - btb_call(Arity, Func, Regs0, Is, D); - true -> - Regs = btb_kill_not_live(Arity, Regs0), - btb_tail_call(Func, Regs, D) - end; -btb_reaches_match_2([{kill,Y}|Is], Regs, D) -> - btb_reaches_match_1(Is, btb_kill([Y], Regs), D); -btb_reaches_match_2([{deallocate,_}|Is], Regs0, D) -> - Regs = btb_kill_yregs(Regs0), - btb_reaches_match_1(Is, Regs, D); -btb_reaches_match_2([return=I|_], Regs0, D) -> - btb_ensure_not_used([{x,0}], I, Regs0), - D; -btb_reaches_match_2([{gc_bif,_,{f,F},Live,Ss,Dst}=I|Is], Regs0, D0) -> - btb_ensure_not_used(Ss, I, Regs0), - Regs1 = btb_kill_not_live(Live, Regs0), - Regs = btb_kill([Dst], Regs1), - D = btb_follow_branch(F, Regs, D0), - btb_reaches_match_1(Is, Regs, D); -btb_reaches_match_2([{bif,_,{f,F},Ss,Dst}=I|Is], Regs0, D0) -> - btb_ensure_not_used(Ss, I, Regs0), - Regs = btb_kill([Dst], Regs0), - D = btb_follow_branch(F, Regs, D0), - btb_reaches_match_1(Is, Regs, D); -btb_reaches_match_2([{get_map_elements,{f,F},Src,{list,Ls}}=I|Is], Regs0, D0) -> - {Ss,Ds} = beam_utils:split_even(Ls), - btb_ensure_not_used([Src|Ss], I, Regs0), - Regs = btb_kill(Ds, Regs0), - D = btb_follow_branch(F, Regs, D0), - btb_reaches_match_1(Is, Regs, D); -btb_reaches_match_2([{test,bs_start_match2,{f,F},Live,[Ctx,_],Ctx}=I|Is], - Regs0, D0) -> - CtxRegs = btb_context_regs(Regs0), - case member(Ctx, CtxRegs) of - false -> - %% This bs_start_match2 instruction does not use "our" - %% match state. Therefore we can continue the search - %% for another bs_start_match2 instruction. - D = btb_follow_branch(F, Regs0, D0), - Regs = btb_kill_not_live(Live, Regs0), - btb_reaches_match_2(Is, Regs, D); - true -> - %% OK. This instruction will use "our" match state, - %% but we must make sure that all other copies of the - %% match state are killed in the code that follows - %% the instruction. (We know that the fail branch cannot - %% be taken in this case.) - OtherCtxRegs = CtxRegs -- [Ctx], - case btb_are_all_unused(OtherCtxRegs, Is, D0) of - false -> btb_error({OtherCtxRegs,not_all_unused_after,I}); - true -> D0 - end - end; -btb_reaches_match_2([{test,bs_start_match2,{f,F},Live,[Bin,_],Ctx}|Is], - Regs0, D0) -> - CtxRegs = btb_context_regs(Regs0), - case member(Bin, CtxRegs) orelse member(Ctx, CtxRegs) of - false -> - %% This bs_start_match2 does not reference any copy of the - %% match state. Therefore it can safely be passed on the - %% way to another (perhaps more suitable) bs_start_match2 - %% instruction. - D = btb_follow_branch(F, Regs0, D0), - Regs = btb_kill_not_live(Live, Regs0), - btb_reaches_match_2(Is, Regs, D); - true -> - %% This variant of the bs_start_match2 instruction does - %% not accept a match state as source. - btb_error(unsuitable_bs_start_match) - end; -btb_reaches_match_2([{test,_,{f,F},Ss}=I|Is], Regs, D0) -> - btb_ensure_not_used(Ss, I, Regs), - D1 = btb_follow_branch(F, Regs, D0), - D = case Is of - [{bs_context_to_binary,_}|_] -> - %% bs_context_to_binary following a test instruction - %% probably needs the current position to be saved as - %% the new start position, but we can't be sure. - %% Therefore, conservatively disable the optimization - %% (instead of forcing a saving of the position). - D1#btb{must_save=true,must_not_save=true}; - _ -> - D1 - end, - btb_reaches_match_1(Is, Regs, D); -btb_reaches_match_2([{test,_,{f,F},_,Ss,_}=I|Is], Regs, D0) -> - btb_ensure_not_used(Ss, I, Regs), - D = btb_follow_branch(F, Regs, D0), - btb_reaches_match_1(Is, Regs, D); -btb_reaches_match_2([{select,_,Src,{f,F},Conds}=I|Is], Regs, D0) -> - btb_ensure_not_used([Src], I, Regs), - D1 = btb_follow_branch(F, Regs, D0), - D = btb_follow_branches(Conds, Regs, D1), - btb_reaches_match_1(Is, Regs, D); -btb_reaches_match_2([{jump,{f,Lbl}}|_], Regs, #btb{index=Li}=D) -> - Is = fetch_code_at(Lbl, Li), - btb_reaches_match_2(Is, Regs, D); -btb_reaches_match_2([{label,_}|Is], Regs, D) -> - btb_reaches_match_2(Is, Regs, D); -btb_reaches_match_2([{bs_init,{f,0},_,_,Ss,Dst}=I|Is], Regs, D) -> - btb_ensure_not_used(Ss, I, Regs), - btb_reaches_match_1(Is, btb_kill([Dst], Regs), D); -btb_reaches_match_2([{bs_put,{f,0},_,Ss}=I|Is], Regs, D) -> - btb_ensure_not_used(Ss, I, Regs), - btb_reaches_match_1(Is, Regs, D); -btb_reaches_match_2([{bs_restore2,Src,_}=I|Is], Regs0, D) -> - case btb_contains_context(Src, Regs0) of - false -> - btb_reaches_match_1(Is, Regs0, D); - true -> - %% Check that all other copies of the context registers - %% are unused by the following instructions. - Regs = btb_kill([Src], Regs0), - CtxRegs = btb_context_regs(Regs), - case btb_are_all_unused(CtxRegs, Is, D) of - false -> btb_error({CtxRegs,not_all_unused_after,I}); - true -> D#btb{must_not_save=true} - end - end; -btb_reaches_match_2([{bs_context_to_binary,Src}=I|Is], Regs0, D) -> - case btb_contains_context(Src, Regs0) of - false -> - btb_reaches_match_1(Is, Regs0, D); - true -> - %% Check that all other copies of the context registers - %% are unused by the following instructions. - Regs = btb_kill([Src], Regs0), - CtxRegs = btb_context_regs(Regs), - case btb_are_all_unused(CtxRegs, Is, D) of - false -> btb_error({CtxRegs,not_all_unused_after,I}); - true -> D#btb{must_not_save=true} - end - end; -btb_reaches_match_2([{badmatch,Src}=I|_], Regs, D) -> - btb_ensure_not_used([Src], I, Regs), - D; -btb_reaches_match_2([{case_end,Src}=I|_], Regs, D) -> - btb_ensure_not_used([Src], I, Regs), - D; -btb_reaches_match_2([if_end|_], _Regs, D) -> - D; -btb_reaches_match_2([{func_info,_,_,Arity}=I|_], Regs0, D) -> - Regs = btb_kill_yregs(btb_kill_not_live(Arity, Regs0)), - case btb_context_regs(Regs) of - [] -> D; - _ -> {binary_used_in,I} - end; -btb_reaches_match_2([{line,_}|Is], Regs, D) -> - btb_reaches_match_1(Is, Regs, D); -btb_reaches_match_2([I|_], Regs, _) -> - btb_error({btb_context_regs(Regs),I,not_handled}). - -is_tail_call([{deallocate,_}|_]) -> true; -is_tail_call([return|_]) -> true; -is_tail_call(_) -> false. - -btb_call(Arity, Lbl, Regs0, Is, D0) -> - Regs = btb_kill_not_live(Arity, Regs0), - case btb_are_x_registers_empty(Regs) of - false -> - %% There is a match context in one of the x registers. - %% First handle the call as if it were a tail call. - D = btb_tail_call(Lbl, Regs, D0), - - %% No problem so far (the called function can handle a - %% match context). Now we must make sure that we don't - %% have any copies of the match context tucked away in an - %% y register. - RegList = btb_context_regs(Regs), - case [R || {y,_}=R <- RegList] of - [] -> - D; - [_|_] -> - btb_error({multiple_uses,RegList}) - end; - true -> - %% No match context in any x register. It could have been - %% saved to an y register, so continue to scan the code following - %% the call. - btb_reaches_match_1(Is, Regs, D0) - end. - -btb_tail_call(Lbl, Regs, #btb{f=Ftree,must_save=MustSave0}=D) -> - %% Ignore any y registers here. - case [R || {x,_}=R <- btb_context_regs(Regs)] of - [] -> - D; - [{x,_}=Reg] -> - case gb_trees:lookup(Lbl, Ftree) of - {value,{Reg,MustSave}} -> - D#btb{must_save=MustSave0 or MustSave}; - _ when is_integer(Lbl) -> - btb_error({{label,Lbl},no_suitable_bs_start_match}); - _ -> - btb_error({binary_used_in,Lbl}) - end; - [_|_] when not is_integer(Lbl) -> - btb_error({binary_used_in,Lbl}); - [_|_]=RegList -> - btb_error({multiple_uses,RegList}) - end. - -%% btb_follow_branches([Cond], Regs, D) -> D' -%% Recursively follow all the branches. - -btb_follow_branches([{f,Lbl}|T], Regs, D0) -> - D = btb_follow_branch(Lbl, Regs, D0), - btb_follow_branches(T, Regs, D); -btb_follow_branches([_|T], Regs, D) -> - btb_follow_branches(T, Regs, D); -btb_follow_branches([], _, D) -> D. - -%% btb_follow_branch(Lbl, Regs, D) -> D' -%% Recursively follow the branch. - -btb_follow_branch(0, _Regs, D) -> D; -btb_follow_branch(Lbl, Regs, #btb{ok_br=Br0,index=Li}=D) -> - Key = {Lbl,Regs}, - case gb_sets:is_member(Key, Br0) of - true -> - %% We have already followed this branch and it was OK. - D; - false -> - %% New branch. Try it. - Is = fetch_code_at(Lbl, Li), - #btb{ok_br=Br,must_not_save=MustNotSave,must_save=MustSave} = - btb_reaches_match_1(Is, Regs, D), - - %% Since we got back, this branch is OK. - D#btb{ok_br=gb_sets:insert(Key, Br),must_not_save=MustNotSave, - must_save=MustSave} - end. - -btb_reaches_match_block([{set,Ds,Ss,{alloc,Live,_}}=I|Is], Regs0) -> - %% An allocation instruction or a GC bif. We'll kill all registers - %% if any copy of the context is used as the source to the BIF. - btb_ensure_not_used(Ss, I, Regs0), - Regs1 = btb_kill_not_live(Live, Regs0), - Regs = btb_kill(Ds, Regs1), - btb_reaches_match_block(Is, Regs); -btb_reaches_match_block([{set,[Dst]=Ds,[Src],move}|Is], Regs0) -> - Regs1 = btb_kill(Ds, Regs0), - Regs = case btb_contains_context(Src, Regs1) of - false -> Regs1; - true -> btb_set_context(Dst, Regs1) - end, - btb_reaches_match_block(Is, Regs); -btb_reaches_match_block([{set,Ds,Ss,_}=I|Is], Regs0) -> - btb_ensure_not_used(Ss, I, Regs0), - Regs = btb_kill(Ds, Regs0), - btb_reaches_match_block(Is, Regs); -btb_reaches_match_block([], Regs) -> - Regs. - -%% btb_are_all_killed([Register], [Instruction], D) -> true|false -%% Test whether all of the register are unused in the instruction stream. - -btb_are_all_unused(RegList, Is, #btb{index=Li}) -> - all(fun(R) -> - beam_utils:is_not_used(R, Is, Li) - end, RegList). - -%% btp_regs_from_list([Register]) -> RegisterSet. -%% Create a register set from a list of registers. - -btb_regs_from_list(L) -> - foldl(fun(R, Regs) -> - btb_set_context(R, Regs) - end, {0,0}, L). - -%% btb_set_context(Register, RegisterSet) -> RegisterSet' -%% Update RegisterSet to indicate that Register contains the matching context. - -btb_set_context({x,N}, {Xregs,Yregs}) -> - {Xregs bor (1 bsl N),Yregs}; -btb_set_context({y,N}, {Xregs,Yregs}) -> - {Xregs,Yregs bor (1 bsl N)}. - -%% btb_ensure_not_used([Register], Instruction, RegisterSet) -> ok -%% If any register in RegisterSet (the register(s) known to contain -%% the match context) is used in the list of registers, generate an error. - -btb_ensure_not_used(Rs, I, Regs) -> - case lists:any(fun(R) -> btb_contains_context(R, Regs) end, Rs) of - true -> btb_error({binary_used_in,I}); - false -> ok - end. - -%% btb_kill([Register], RegisterSet) -> RegisterSet' -%% Kill all registers mentioned in the list of registers. - -btb_kill([{x,N}|Rs], {Xregs,Yregs}) -> - btb_kill(Rs, {Xregs band (bnot (1 bsl N)),Yregs}); -btb_kill([{y,N}|Rs], {Xregs,Yregs}) -> - btb_kill(Rs, {Xregs,Yregs band (bnot (1 bsl N))}); -btb_kill([{fr,_}|Rs], Regs) -> - btb_kill(Rs, Regs); -btb_kill([], Regs) -> Regs. - -%% btb_kill_not_live(Live, RegisterSet) -> RegisterSet' -%% Kill all registers indicated not live by Live. - -btb_kill_not_live(Live, {Xregs,Yregs}) -> - {Xregs band ((1 bsl Live)-1),Yregs}. - -%% btb_kill(Regs0) -> Regs -%% Kill all y registers. - -btb_kill_yregs({Xregs,_}) -> {Xregs,0}. - -%% btb_are_registers_empty(RegisterSet) -> true|false -%% Test whether the register set is empty. - -btb_are_registers_empty({0,0}) -> true; -btb_are_registers_empty({_,_}) -> false. - -%% btb_are_x_registers_empty(Regs) -> true|false -%% Test whether the x registers are empty. - -btb_are_x_registers_empty({0,_}) -> true; -btb_are_x_registers_empty({_,_}) -> false. - -%% btb_contains_context(Register, RegisterSet) -> true|false -%% Test whether Register contains the context. - -btb_contains_context({x,N}, {Regs,_}) -> Regs band (1 bsl N) =/= 0; -btb_contains_context({y,N}, {_,Regs}) -> Regs band (1 bsl N) =/= 0; -btb_contains_context(_, _) -> false. - -%% btb_context_regs(RegisterSet) -> [Register] -%% Convert the register set to an explicit list of registers. -btb_context_regs({Xregs,Yregs}) -> - btb_context_regs_1(Xregs, 0, x, btb_context_regs_1(Yregs, 0, y, [])). - -btb_context_regs_1(0, _, _, Acc) -> - Acc; -btb_context_regs_1(Regs, N, Tag, Acc) when (Regs band 1) =:= 1 -> - btb_context_regs_1(Regs bsr 1, N+1, Tag, [{Tag,N}|Acc]); -btb_context_regs_1(Regs, N, Tag, Acc) -> - btb_context_regs_1(Regs bsr 1, N+1, Tag, Acc). - -%% btb_index([Function]) -> GbTree({EntryLabel,{Register,MustSave}}) -%% Build an index of functions that accept a match context instead of -%% a binary. MustSave is true if the function may pass the match -%% context to the bs_context_to_binary instruction (in which case -%% the current position in the binary must have saved into the -%% start position using "bs_save_2 Ctx start"). - -btb_index(Fs) -> - btb_index_1(Fs, []). - -btb_index_1([{function,_,_,Entry,Is0}|Fs], Acc0) -> - Is = drop_to_label(Is0, Entry), - Acc = btb_index_2(Is, Entry, false, Acc0), - btb_index_1(Fs, Acc); -btb_index_1([], Acc) -> gb_trees:from_orddict(sort(Acc)). - -btb_index_2([{test,bs_start_match2,{f,_},_,[Reg,_],Reg}|_], - Entry, MustSave, Acc) -> - [{Entry,{Reg,MustSave}}|Acc]; -btb_index_2(Is0, Entry, _, Acc) -> - try btb_index_find_start_match(Is0) of - Is -> btb_index_2(Is, Entry, true, Acc) - catch - throw:none -> Acc - end. - -drop_to_label([{label,L}|Is], L) -> Is; -drop_to_label([_|Is], L) -> drop_to_label(Is, L). - -btb_index_find_start_match([{test,_,{f,F},_},{bs_context_to_binary,_}|Is]) -> - btb_index_find_label(Is, F); -btb_index_find_start_match(_) -> - throw(none). - -btb_index_find_label([{label,L}|Is], L) -> Is; -btb_index_find_label([_|Is], L) -> btb_index_find_label(Is, L). - -btb_error(Error) -> - throw({error,Error}). - -fetch_code_at(Lbl, Li) -> - case beam_utils:code_at(Lbl, Li) of - Is when is_list(Is) -> Is - end. - -%%% -%%% Compilation information warnings. -%%% - -btb_comment_opt({field_flags,[{anno,A}|_]}) -> - {'%',{bin_opt,A}}; -btb_comment_opt(_) -> - {'%',{bin_opt,[]}}. - -btb_comment_no_opt(Reason, {field_flags,[{anno,A}|_]}) -> - {'%',{no_bin_opt,Reason,A}}; -btb_comment_no_opt(Reason, _) -> - {'%',{no_bin_opt,Reason,[]}}. - -collect_warnings(Fs) -> - D = warning_index_functions(Fs), - foldl(fun(F, A) -> collect_warnings_fun(F, D, A) end, [], Fs). - -collect_warnings_fun({function,_,_,_,Is}, D, A) -> - collect_warnings_instr(Is, D, A). - -collect_warnings_instr([{'%',{bin_opt,Where}}|Is], D, Acc0) -> - Acc = add_warning(bin_opt, Where, Acc0), - collect_warnings_instr(Is, D, Acc); -collect_warnings_instr([{'%',{no_bin_opt,Reason0,Where}}|Is], D, Acc0) -> - Reason = warning_translate_label(Reason0, D), - Acc = add_warning({no_bin_opt,Reason}, Where, Acc0), - collect_warnings_instr(Is, D, Acc); -collect_warnings_instr([_|Is], D, Acc) -> - collect_warnings_instr(Is, D, Acc); -collect_warnings_instr([], _, Acc) -> Acc. - -add_warning(Term, Anno, Ws) -> - Line = get_line(Anno), - File = get_file(Anno), - [{File,[{Line,?MODULE,Term}]}|Ws]. - -warning_translate_label(Term, D) when is_tuple(Term) -> - case element(1, Term) of - {label,F} -> - FA = gb_trees:get(F, D), - setelement(1, Term, FA); - _ -> Term - end; -warning_translate_label(Term, _) -> Term. - -get_line([Line|_]) when is_integer(Line) -> Line; -get_line([_|T]) -> get_line(T); -get_line([]) -> none. - -get_file([{file,File}|_]) -> File; -get_file([_|T]) -> get_file(T); -get_file([]) -> "no_file". % should not happen - -warning_index_functions(Fs) -> - D = [{Entry,{F,A}} || {function,F,A,Entry,_} <- Fs], - gb_trees:from_orddict(sort(D)). - -format_error_1({binary_used_in,{extfunc,M,F,A}}) -> - [io_lib:format("sub binary used by ~p:~p/~p", [M,F,A])| - case {M,F,A} of - {erlang,split_binary,2} -> - "; SUGGEST using binary matching instead of split_binary/2"; - _ -> - "" - end]; -format_error_1({binary_used_in,_}) -> - "sub binary is used or returned"; -format_error_1({multiple_uses,_}) -> - "sub binary is matched or used in more than one place"; -format_error_1(unsuitable_bs_start_match) -> - "the binary matching instruction that follows in the same function " - "have problems that prevent delayed sub binary optimization " - "(probably indicated by INFO warnings)"; -format_error_1({{F,A},no_suitable_bs_start_match}) -> - io_lib:format("called function ~p/~p does not begin with a suitable " - "binary matching instruction", [F,A]); -format_error_1(must_and_must_not_save) -> - "different control paths use different positions in the binary"; -format_error_1({_,I,not_handled}) -> - case I of - {'catch',_,_} -> - "the compiler currently does not attempt the delayed sub binary " - "optimization when catch is used"; - {'try',_,_} -> - "the compiler currently does not attempt the delayed sub binary " - "optimization when try/catch is used"; - _ -> - io_lib:format("compiler limitation: instruction ~p prevents " - "delayed sub binary optimization", [I]) - end; -format_error_1(Term) -> - io_lib:format("~w", [Term]). diff --git a/lib/compiler/src/beam_clean.erl b/lib/compiler/src/beam_clean.erl index f5f0ac2218..7299654476 100644 --- a/lib/compiler/src/beam_clean.erl +++ b/lib/compiler/src/beam_clean.erl @@ -23,17 +23,15 @@ -export([module/2]). -export([clean_labels/1]). --import(lists, [foldl/3]). -spec module(beam_utils:module_code(), [compile:option()]) -> {'ok',beam_utils:module_code()}. module({Mod,Exp,Attr,Fs0,_}, Opts) -> Order = [Lbl || {function,_,_,Lbl,_} <- Fs0], - All = foldl(fun({function,_,_,Lbl,_}=Func,D) -> dict:store(Lbl, Func, D) end, - dict:new(), Fs0), + All = maps:from_list([{Lbl,Func} || {function,_,_,Lbl,_}=Func <- Fs0]), WorkList = rootset(Fs0, Exp, Attr), - Used = find_all_used(WorkList, All, sets:from_list(WorkList)), + Used = find_all_used(WorkList, All, cerl_sets:from_list(WorkList)), Fs1 = remove_unused(Order, Used, All), {Fs2,Lc} = clean_labels(Fs1), Fs = maybe_remove_lines(Fs2, Opts), @@ -55,16 +53,16 @@ rootset(Fs, Root0, Attr) -> %% Remove the unused functions. remove_unused([F|Fs], Used, All) -> - case sets:is_element(F, Used) of + case cerl_sets:is_element(F, Used) of false -> remove_unused(Fs, Used, All); - true -> [dict:fetch(F, All)|remove_unused(Fs, Used, All)] + true -> [map_get(F, All)|remove_unused(Fs, Used, All)] end; remove_unused([], _, _) -> []. - + %% Find all used functions. find_all_used([F|Fs0], All, Used0) -> - {function,_,_,_,Code} = dict:fetch(F, All), + {function,_,_,_,Code} = map_get(F, All), {Fs,Used} = update_work_list(Code, {Fs0,Used0}), find_all_used(Fs, All, Used); find_all_used([], _All, Used) -> Used. @@ -78,9 +76,9 @@ update_work_list([_|Is], Sets) -> update_work_list([], Sets) -> Sets. add_to_work_list(F, {Fs,Used}=Sets) -> - case sets:is_element(F, Used) of + case cerl_sets:is_element(F, Used) of true -> Sets; - false -> {[F|Fs],sets:add_element(F, Used)} + false -> {[F|Fs],cerl_sets:add_element(F, Used)} end. diff --git a/lib/compiler/src/beam_disasm.erl b/lib/compiler/src/beam_disasm.erl index d0be39f520..7d048716e4 100644 --- a/lib/compiler/src/beam_disasm.erl +++ b/lib/compiler/src/beam_disasm.erl @@ -1105,6 +1105,16 @@ resolve_inst({get_hd,[Src,Dst]},_,_,_) -> resolve_inst({get_tl,[Src,Dst]},_,_,_) -> {get_tl,Src,Dst}; +%% OTP 22 +resolve_inst({bs_start_match3,[Fail,Bin,Live,Dst]},_,_,_) -> + {bs_start_match3,Fail,Bin,Live,Dst}; +resolve_inst({bs_get_tail,[Src,Dst,Live]},_,_,_) -> + {bs_get_tail,Src,Dst,Live}; +resolve_inst({bs_get_position,[Src,Dst,Live]},_,_,_) -> + {bs_get_position,Src,Dst,Live}; +resolve_inst({bs_set_position,[Src,Dst]},_,_,_) -> + {bs_set_position,Src,Dst}; + %% %% OTP 22. %% diff --git a/lib/compiler/src/beam_flatten.erl b/lib/compiler/src/beam_flatten.erl index 973d16a1bc..3e6bc1b1ed 100644 --- a/lib/compiler/src/beam_flatten.erl +++ b/lib/compiler/src/beam_flatten.erl @@ -32,8 +32,7 @@ module({Mod,Exp,Attr,Fs,Lc}, _Opt) -> {ok,{Mod,Exp,Attr,[function(F) || F <- Fs],Lc}}. function({function,Name,Arity,CLabel,Is0}) -> - Is1 = block(Is0), - Is = opt(Is1), + Is = block(Is0), {function,Name,Arity,CLabel,Is}. block(Is) -> @@ -43,21 +42,12 @@ block([{block,Is0}|Is1], Acc) -> block(Is1, norm_block(Is0, Acc)); block([I|Is], Acc) -> block(Is, [I|Acc]); block([], Acc) -> reverse(Acc). -norm_block([{set,[],[],{alloc,R,{_,nostack,_,_}=Alloc}}|Is], Acc0) -> - case insert_alloc_in_bs_init(Acc0, Alloc) of - impossible -> - norm_block(Is, reverse(norm_allocate(Alloc, R), Acc0)); - Acc -> - norm_block(Is, Acc) - end; norm_block([{set,[],[],{alloc,R,Alloc}}|Is], Acc0) -> norm_block(Is, reverse(norm_allocate(Alloc, R), Acc0)); -norm_block([{set,[D1],[S],get_hd},{set,[D2],[S],get_tl}|Is], Acc) -> - I = {get_list,S,D1,D2}, - norm_block(Is, [I|Acc]); -norm_block([I|Is], Acc) -> norm_block(Is, [norm(I)|Acc]); +norm_block([I|Is], Acc) -> + norm_block(Is, [norm(I)|Acc]); norm_block([], Acc) -> Acc. - + norm({set,[D],As,{bif,N,F}}) -> {bif,N,F,As,D}; norm({set,[D],As,{alloc,R,{gc_bif,N,F}}}) -> {gc_bif,N,F,R,As,D}; norm({set,[D],[],init}) -> {init,D}; @@ -91,57 +81,3 @@ norm_allocate({nozero,Ns,0,Inits}, Regs) -> [{allocate,Ns,Regs}|Inits]; norm_allocate({nozero,Ns,Nh,Inits}, Regs) -> [{allocate_heap,Ns,Nh,Regs}|Inits]. - -%% insert_alloc_in_bs_init(ReverseInstructionStream, AllocationInfo) -> -%% impossible | ReverseInstructionStream' -%% A bs_init/6 instruction should not be followed by a test heap instruction. -%% Given the AllocationInfo from a test heap instruction, merge the -%% allocation amounts into the previous bs_init/6 instruction (if any). -%% -insert_alloc_in_bs_init([{bs_put,_,_,_}=I|Is], Alloc) -> - %% The instruction sequence ends with an bs_put/4 instruction. - %% We'll need to search backwards for the bs_init/6 instruction. - insert_alloc_1(Is, Alloc, [I]); -insert_alloc_in_bs_init(_, _) -> impossible. - -insert_alloc_1([{bs_init=Op,Fail,Info0,Live,Ss,Dst}|Is], - {_,nostack,Ws2,[]}, Acc) when is_integer(Live) -> - %% The number of extra heap words is always in the second position - %% in the Info tuple. - Ws1 = element(2, Info0), - Al = beam_utils:combine_heap_needs(Ws1, Ws2), - Info = setelement(2, Info0, Al), - I = {Op,Fail,Info,Live,Ss,Dst}, - reverse(Acc, [I|Is]); -insert_alloc_1([{bs_put,_,_,_}=I|Is], Alloc, Acc) -> - insert_alloc_1(Is, Alloc, [I|Acc]). - -%% opt(Is0) -> Is -%% Simple peep-hole optimization to move a {move,Any,{x,0}} past -%% any kill up to the next call instruction. (To give the loader -%% an opportunity to combine the 'move' and the 'call' instructions.) -%% -opt(Is) -> - opt_1(Is, []). - -opt_1([{move,_,{x,0}}=I|Is0], Acc0) -> - case move_past_kill(Is0, I, Acc0) of - impossible -> opt_1(Is0, [I|Acc0]); - {Is,Acc} -> opt_1(Is, Acc) - end; -opt_1([I|Is], Acc) -> - opt_1(Is, [I|Acc]); -opt_1([], Acc) -> reverse(Acc). - -move_past_kill([{kill,Src}|_], {move,Src,_}, _) -> - impossible; -move_past_kill([{kill,_}=I|Is], Move, Acc) -> - move_past_kill(Is, Move, [I|Acc]); -move_past_kill([{trim,N,_}=I|Is], {move,Src,Dst}=Move, Acc) -> - case Src of - {y,Y} when Y < N-> impossible; - {y,Y} -> {Is,[{move,{y,Y-N},Dst},I|Acc]}; - _ -> {Is,[Move,I|Acc]} - end; -move_past_kill(Is, Move, Acc) -> - {Is,[Move|Acc]}. diff --git a/lib/compiler/src/beam_jump.erl b/lib/compiler/src/beam_jump.erl index 6d0a8db2dc..fbff4cfd79 100644 --- a/lib/compiler/src/beam_jump.erl +++ b/lib/compiler/src/beam_jump.erl @@ -367,8 +367,6 @@ extract_seq([{line,_}=Line|Is], Acc) -> extract_seq(Is, [Line|Acc]); extract_seq([{block,_}=Bl|Is], Acc) -> extract_seq_1(Is, [Bl|Acc]); -extract_seq([{bs_context_to_binary,_}=I|Is], Acc) -> - extract_seq_1(Is, [I|Acc]); extract_seq([{label,_}|_]=Is, Acc) -> extract_seq_1(Is, Acc); extract_seq(_, _) -> no. diff --git a/lib/compiler/src/beam_kernel_to_ssa.erl b/lib/compiler/src/beam_kernel_to_ssa.erl index c55a57ab32..d6e675ae72 100644 --- a/lib/compiler/src/beam_kernel_to_ssa.erl +++ b/lib/compiler/src/beam_kernel_to_ssa.erl @@ -276,12 +276,11 @@ select_nil(#k_val_clause{val=#k_nil{},body=B}, V, Tf, Vf, St0) -> {Is ++ Bis,St}. select_binary(#k_val_clause{val=#k_binary{segs=#k_var{name=Ctx0}},body=B}, - #k_var{anno=Anno0}=Src, Tf, Vf, St0) -> - Anno = #{reuse_for_context=>member(reuse_for_context, Anno0)}, + #k_var{}=Src, Tf, Vf, St0) -> {Ctx,St1} = new_ssa_var(Ctx0, St0), {Bis0,St2} = match_cg(B, Vf, St1), {TestIs,St} = make_cond_branch(succeeded, [Ctx], Tf, St2), - Bis1 = [#b_set{anno=Anno,op=bs_start_match,dst=Ctx, + Bis1 = [#b_set{op=bs_start_match,dst=Ctx, args=[ssa_arg(Src, St)]}] ++ TestIs ++ Bis0, Bis = finish_bs_matching(Bis1), {Bis,St}. @@ -708,10 +707,6 @@ bif_cg(#k_bif{op=#k_remote{mod=#k_atom{val=erlang},name=#k_atom{val=Name}}, %% internal_cg(Bif, [Arg], [Ret], Le, State) -> %% {[Ainstr],State}. -internal_cg(bs_context_to_binary, [Src0], [], _Le, St) -> - Src = ssa_arg(Src0, St), - Set = #b_set{op=context_to_binary,args=[Src]}, - {[Set],St}; internal_cg(dsetelement, [Index0,Tuple0,New0], _Rs, _Le, St) -> [New,Tuple,#b_literal{val=Index1}] = ssa_args([New0,Tuple0,Index0], St), Index = #b_literal{val=Index1-1}, diff --git a/lib/compiler/src/beam_ssa.erl b/lib/compiler/src/beam_ssa.erl index 9d10d4aec3..c5e23d2ae0 100644 --- a/lib/compiler/src/beam_ssa.erl +++ b/lib/compiler/src/beam_ssa.erl @@ -20,12 +20,15 @@ %% Purpose: Type definitions and utilities for the SSA format. -module(beam_ssa). --export([add_anno/3,get_anno/2, - clobbers_xregs/1,def/2,def_used/2,dominators/1, +-export([add_anno/3,get_anno/2,get_anno/3, + clobbers_xregs/1,def/2,def_used/2, + definitions/1, + dominators/1, flatmapfold_instrs_rpo/4, fold_po/3,fold_po/4,fold_rpo/3,fold_rpo/4, fold_instrs_rpo/4, linearize/1, + mapfold_blocks_rpo/4, mapfold_instrs_rpo/4, normalize/1, no_side_effect/1, @@ -35,14 +38,17 @@ split_blocks/3, successors/1,successors/2, trim_unreachable/1, - update_phi_labels/4,used/1]). + update_phi_labels/4,used/1, + uses/1,uses/2]). -export_type([b_module/0,b_function/0,b_blk/0,b_set/0, b_ret/0,b_br/0,b_switch/0,terminator/0, b_var/0,b_literal/0,b_remote/0,b_local/0, value/0,argument/0,label/0, var_name/0,var_base/0,literal_value/0, - op/0,anno/0,block_map/0]). + op/0,anno/0,block_map/0,dominator_map/0, + rename_map/0,rename_proplist/0,usage_map/0, + definition_map/0]). -include("beam_ssa.hrl"). @@ -56,6 +62,9 @@ -type b_switch() :: #b_switch{}. -type terminator() :: b_br() | b_ret() | b_switch(). +-type construct() :: b_module() | b_function() | b_blk() | b_set() | + terminator(). + -type b_var() :: #b_var{}. -type b_literal() :: #b_literal{}. -type b_remote() :: #b_remote{}. @@ -76,6 +85,11 @@ -type anno() :: #{atom() := any()}. -type block_map() :: #{label():=b_blk()}. +-type dominator_map() :: #{label():=ordsets:ordset(label())}. +-type usage_map() :: #{b_var():=[{label(),b_set() | terminator()}]}. +-type definition_map() :: #{b_var():=b_set()}. +-type rename_map() :: #{b_var():=value()}. +-type rename_proplist() :: [{b_var(),value()}]. %% Note: By default, dialyzer will collapse this type to atom(). %% To avoid the collapsing, change the value of SET_LIMIT to 50 in the @@ -84,7 +98,7 @@ -type prim_op() :: 'bs_add' | 'bs_extract' | 'bs_init' | 'bs_init_writable' | 'bs_match' | 'bs_put' | 'bs_start_match' | 'bs_test_tail' | 'bs_utf16_size' | 'bs_utf8_size' | 'build_stacktrace' | - 'call' | 'catch_end' | 'context_to_binary' | + 'call' | 'catch_end' | 'extract' | 'get_hd' | 'get_map_element' | 'get_tl' | 'get_tuple_element' | 'has_map_field' | @@ -103,14 +117,14 @@ %% Primops only used internally during code generation. -type cg_prim_op() :: 'bs_get' | 'bs_match_string' | 'bs_restore' | 'bs_skip' | -'copy' | 'put_tuple_arity' | 'put_tuple_element'. + 'copy' | 'put_tuple_arity' | 'put_tuple_element'. -import(lists, [foldl/3,keyfind/3,mapfoldl/3,member/2,reverse/1]). -spec add_anno(Key, Value, Construct) -> Construct when Key :: atom(), Value :: any(), - Construct :: b_function() | b_blk() | b_set() | terminator(). + Construct :: construct(). add_anno(Key, Val, #b_function{anno=Anno}=Bl) -> Bl#b_function{anno=Anno#{Key=>Val}}; @@ -125,11 +139,17 @@ add_anno(Key, Val, #b_ret{anno=Anno}=Bl) -> add_anno(Key, Val, #b_switch{anno=Anno}=Bl) -> Bl#b_switch{anno=Anno#{Key=>Val}}. --spec get_anno(atom(), b_blk()|b_set()|terminator()) -> any(). +-spec get_anno(atom(), construct()) -> any(). get_anno(Key, Construct) -> maps:get(Key, get_anno(Construct)). +-spec get_anno(atom(), construct(),any()) -> any(). + +get_anno(Key, Construct, Default) -> + maps:get(Key, get_anno(Construct), Default). + +get_anno(#b_function{anno=Anno}) -> Anno; get_anno(#b_blk{anno=Anno}) -> Anno; get_anno(#b_set{anno=Anno}) -> Anno; get_anno(#b_br{anno=Anno}) -> Anno; @@ -169,6 +189,7 @@ no_side_effect(#b_set{op=Op}) -> bs_match -> true; bs_start_match -> true; bs_test_tail -> true; + bs_get_tail -> true; bs_put -> true; extract -> true; get_hd -> true; @@ -177,6 +198,7 @@ no_side_effect(#b_set{op=Op}) -> has_map_field -> true; is_nonempty_list -> true; is_tagged_tuple -> true; + make_fun -> true; put_map -> true; put_list -> true; put_tuple -> true; @@ -306,7 +328,7 @@ def_used(Ls, Blocks) -> -spec dominators(Blocks) -> Result when Blocks :: block_map(), - Result :: #{label():=ordsets:ordset(label())}. + Result :: dominator_map(). dominators(Blocks) -> Preds = predecessors(Blocks), @@ -327,6 +349,26 @@ fold_instrs_rpo(Fun, From, Acc0, Blocks) -> Top = rpo(From, Blocks), fold_instrs_rpo_1(Top, Fun, Blocks, Acc0). +%% Like mapfold_instrs_rpo but at the block level to support lookahead and +%% scope-dependent transformations. +-spec mapfold_blocks_rpo(Fun, From, Acc, Blocks) -> Result when + Fun :: fun((label(), b_blk(), any()) -> {b_blk(), any()}), + From :: [label()], + Acc :: any(), + Blocks :: block_map(), + Result :: {block_map(), any()}. +mapfold_blocks_rpo(Fun, From, Acc, Blocks) -> + Successors = rpo(From, Blocks), + foldl(fun(Lbl, A) -> + mapfold_blocks_rpo_1(Fun, Lbl, A) + end, {Blocks, Acc}, Successors). + +mapfold_blocks_rpo_1(Fun, Lbl, {Blocks0, Acc0}) -> + Block0 = maps:get(Lbl, Blocks0), + {Block, Acc} = Fun(Lbl, Block0, Acc0), + Blocks = maps:put(Lbl, Block, Blocks0), + {Blocks, Acc}. + -spec mapfold_instrs_rpo(Fun, From, Acc0, Blocks0) -> {Blocks,Acc} when Fun :: fun((b_blk()|terminator(), any()) -> any()), From :: [label()], @@ -442,7 +484,7 @@ rpo(From, Blocks) -> Ls. -spec rename_vars(Rename, [label()], block_map()) -> block_map() when - Rename :: [{var_name(),value()}] | #{var_name():=value()}. + Rename :: rename_map() | rename_proplist(). rename_vars(Rename, From, Blocks) when is_list(Rename) -> rename_vars(maps:from_list(Rename), From, Blocks); @@ -535,6 +577,34 @@ used(#b_switch{arg=#b_var{}=V}) -> [V]; used(_) -> []. +-spec definitions(Blocks :: block_map()) -> definition_map(). +definitions(Blocks) -> + fold_instrs_rpo(fun(#b_set{ dst = Var }=I, Acc) -> + maps:put(Var, I, Acc); + (_Terminator, Acc) -> + Acc + end, [0], #{}, Blocks). + +-spec uses(Blocks :: block_map()) -> usage_map(). +uses(Blocks) -> + uses([0], Blocks). + +-spec uses(From, Blocks) -> usage_map() when + From :: [label()], + Blocks :: block_map(). +uses(From, Blocks) -> + fold_rpo(fun fold_uses_block/3, From, #{}, Blocks). + +fold_uses_block(Lbl, #b_blk{is=Is,last=Last}, UseMap0) -> + F = fun(I, UseMap) -> + foldl(fun(Var, Acc) -> + Uses0 = maps:get(Var, Acc, []), + Uses = [{Lbl, I} | Uses0], + maps:put(Var, Uses, Acc) + end, UseMap, used(I)) + end, + F(Last, foldl(F, UseMap0, Is)). + %%% %%% Internal functions. %%% @@ -728,8 +798,8 @@ split_blocks_1([L|Ls], P, Blocks0, Count0) -> BefBlk = Blk#b_blk{is=Bef,last=Br}, NewBlk = Blk#b_blk{is=Aft}, Blocks1 = Blocks0#{L:=BefBlk,NewLbl=>NewBlk}, - Successors = beam_ssa:successors(NewBlk), - Blocks = beam_ssa:update_phi_labels(Successors, L, NewLbl, Blocks1), + Successors = successors(NewBlk), + Blocks = update_phi_labels(Successors, L, NewLbl, Blocks1), split_blocks_1([NewLbl|Ls], P, Blocks, Count); no -> split_blocks_1(Ls, P, Blocks0, Count0) diff --git a/lib/compiler/src/beam_ssa_bsm.erl b/lib/compiler/src/beam_ssa_bsm.erl new file mode 100644 index 0000000000..9631bf3334 --- /dev/null +++ b/lib/compiler/src/beam_ssa_bsm.erl @@ -0,0 +1,1043 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%%% +%%% This pass optimizes bit syntax matching, and is centered around the concept +%%% of "match context reuse" which is best explained through example. To put it +%%% shortly we attempt to turn this: +%%% +%%% <<0,B/bits>> = A, +%%% <<1,C/bits>> = B, +%%% <<D,_/bits>> = C, +%%% D. +%%% +%%% ... Into this: +%%% +%%% <<0,1,D,_/bits>>=A, +%%% D. +%%% +%%% Which is much faster as it avoids the creation of intermediate terms. This +%%% is especially noticeable in loops where such garbage is generated on each +%%% iteration. +%%% +%%% The optimization itself is very simple and can be applied whenever there's +%%% matching on the tail end of a binary; instead of creating a new binary and +%%% starting a new match context on it, we reuse the match context used to +%%% extract the tail and avoid the creation of both objects. +%%% +%%% The catch is that a match context isn't a proper type and nothing outside +%%% of bit syntax match operations can handle them. We therefore need to make +%%% sure that they never "leak" into other instructions, and most of the pass +%%% revolves around getting around that limitation. +%%% +%%% Unlike most other passes we look at the whole module so we can combine +%%% matches across function boundaries, greatly increasing the performance of +%%% complex matches and loops. +%%% + +-module(beam_ssa_bsm). + +-export([module/2, format_error/1]). + +-include("beam_ssa.hrl"). + +-import(lists, [member/2, reverse/1, splitwith/2, map/2, foldl/3, mapfoldl/3, + nth/2, max/1, unzip/1]). + +-spec format_error(term()) -> nonempty_string(). + +format_error(OptInfo) -> + format_opt_info(OptInfo). + +-spec module(Module, Options) -> Result when + Module :: beam_ssa:b_module(), + Options :: [compile:option()], + Result :: {ok, beam_ssa:b_module(), list()}. + +-define(PASS(N), {N,fun N/1}). + +module(#b_module{body=Fs0}=Module, Opts) -> + ModInfo = analyze_module(Module), + + %% combine_matches is repeated after accept_context_args as the control + %% flow changes can enable further optimizations, as in the example below: + %% + %% a(<<0,X/binary>>) -> a(X); + %% a(A) when bit_size(A) =:= 52 -> bar; + %% a(<<1,X/binary>>) -> X. %% Match context will be reused here when + %% %% when repeated. + + {Fs, _} = compile:run_sub_passes( + [?PASS(combine_matches), + ?PASS(accept_context_args), + ?PASS(combine_matches), + ?PASS(allow_context_passthrough), + ?PASS(skip_outgoing_tail_extraction), + ?PASS(annotate_context_parameters)], + {Fs0, ModInfo}), + + Ws = case proplists:get_bool(bin_opt_info, Opts) of + true -> collect_opt_info(Fs); + false -> [] + end, + + {ok, Module#b_module{body=Fs}, Ws}. + +-type module_info() :: #{ func_id() => func_info() }. + +-type func_id() :: {Name :: atom(), Arity :: non_neg_integer()}. + +-type func_info() :: #{ has_bsm_ops => boolean(), + parameters => [#b_var{}], + parameter_info => #{ #b_var{} => param_info() } }. + +-type param_info() :: suitable_for_reuse | + {Problem :: atom(), Where :: term()}. + +-spec analyze_module(#b_module{}) -> module_info(). + +analyze_module(#b_module{body=Fs}) -> + foldl(fun(#b_function{args=Parameters}=F, I) -> + FuncInfo = #{ has_bsm_ops => has_bsm_ops(F), + parameters => Parameters, + parameter_info => #{} }, + FuncId = get_fa(F), + I#{ FuncId => FuncInfo } + end, #{}, Fs). + +has_bsm_ops(#b_function{bs=Blocks}) -> + hbo_blocks(maps:to_list(Blocks)). + +hbo_blocks([{_,#b_blk{is=Is}} | Blocks]) -> + case hbo_is(Is) of + false -> hbo_blocks(Blocks); + true -> true + end; +hbo_blocks([]) -> + false. + +hbo_is([#b_set{op=bs_start_match} | _]) -> true; +hbo_is([_I | Is]) -> hbo_is(Is); +hbo_is([]) -> false. + +%% Checks whether it's legal to make a call with the given argument as a match +%% context, returning the param_info() of the relevant parameter. +-spec check_context_call(#b_set{}, Arg, CtxChain, ModInfo) -> param_info() when + Arg :: #b_var{}, + CtxChain :: [#b_var{}], + ModInfo :: module_info(). +check_context_call(#b_set{args=Args}, Arg, CtxChain, ModInfo) -> + Aliases = [Arg | CtxChain], + ccc_1(Args, Arg, Aliases, ModInfo). + +ccc_1([#b_local{}=Call | Args], Ctx, Aliases, ModInfo) -> + %% Matching operations assume that their context isn't aliased (as in + %% pointer aliasing), so we must reject calls whose arguments contain more + %% than one reference to the context. + %% + %% TODO: Try to fall back to passing binaries in these cases. Partial reuse + %% is better than nothing. + UseCount = foldl(fun(Arg, C) -> + case member(Arg, Aliases) of + true -> C + 1; + false -> C + end + end, 0, Args), + if + UseCount =:= 1 -> + #b_local{name=#b_literal{val=Name},arity=Arity} = Call, + Callee = {Name, Arity}, + + ParamInfo = funcinfo_get(Callee, parameter_info, ModInfo), + Parameters = funcinfo_get(Callee, parameters, ModInfo), + Parameter = nth(1 + arg_index(Ctx, Args), Parameters), + + case maps:find(Parameter, ParamInfo) of + {ok, suitable_for_reuse} -> + suitable_for_reuse; + {ok, Other} -> + {unsuitable_call, {Call, Other}}; + error -> + {no_match_on_entry, Call} + end; + UseCount > 1 -> + {multiple_uses_in_call, Call} + end; +ccc_1([#b_remote{}=Call | _Args], _Ctx, _CtxChain, _ModInfo) -> + {remote_call, Call}; +ccc_1([Fun | _Args], _Ctx, _CtxChain, _ModInfo) -> + %% TODO: It may be possible to support this in the future for locally + %% defined funs, including ones with free variables. + {fun_call, Fun}. + +%% Returns the index of Var in Args. +arg_index(Var, Args) -> arg_index_1(Var, Args, 0). + +arg_index_1(Var, [Var | _Args], Index) -> Index; +arg_index_1(Var, [_Arg | Args], Index) -> arg_index_1(Var, Args, Index + 1). + +is_tail_binary(#b_set{op=bs_match,args=[#b_literal{val=binary} | Rest]}) -> + member(#b_literal{val=all}, Rest); +is_tail_binary(#b_set{op=bs_get_tail}) -> + true; +is_tail_binary(_) -> + false. + +is_tail_binary(#b_var{}=Var, Defs) -> + case find_match_definition(Var, Defs) of + {ok, Def} -> is_tail_binary(Def); + _ -> false + end; +is_tail_binary(_Literal, _Defs) -> + false. + +assert_match_context(#b_var{}=Var, Defs) -> + case maps:find(Var, Defs) of + {ok, #b_set{op=bs_match,args=[_,#b_var{}=Ctx|_]}} -> + assert_match_context(Ctx, Defs); + {ok, #b_set{op=bs_start_match}} -> + ok + end. + +find_match_definition(#b_var{}=Var, Defs) -> + case maps:find(Var, Defs) of + {ok, #b_set{op=bs_extract,args=[Ctx]}} -> maps:find(Ctx, Defs); + {ok, #b_set{op=bs_get_tail}=Def} -> {ok, Def}; + _ -> error + end. + +%% Returns a list of all contexts that were used to extract Var. +context_chain_of(#b_var{}=Var, Defs) -> + case maps:find(Var, Defs) of + {ok, #b_set{op=bs_match,args=[_,#b_var{}=Ctx|_]}} -> + [Ctx | context_chain_of(Ctx, Defs)]; + {ok, #b_set{op=bs_get_tail,args=[Ctx]}} -> + [Ctx | context_chain_of(Ctx, Defs)]; + {ok, #b_set{op=bs_extract,args=[Ctx]}} -> + [Ctx | context_chain_of(Ctx, Defs)]; + _ -> + [] + end. + +%% Grabs the match context used to produce the given variable. +match_context_of(#b_var{}=Var, Defs) -> + Ctx = match_context_of_1(Var, Defs), + assert_match_context(Ctx, Defs), + Ctx. + +match_context_of_1(Var, Defs) -> + case maps:get(Var, Defs) of + #b_set{op=bs_extract,args=[#b_var{}=Ctx0]} -> + #b_set{op=bs_match, + args=[_,#b_var{}=Ctx|_]} = maps:get(Ctx0, Defs), + Ctx; + #b_set{op=bs_get_tail,args=[#b_var{}=Ctx]} -> + Ctx + end. + +funcinfo_get(#b_function{}=F, Attribute, ModInfo) -> + funcinfo_get(get_fa(F), Attribute, ModInfo); +funcinfo_get({_,_}=Key, Attribute, ModInfo) -> + FuncInfo = maps:get(Key, ModInfo), + maps:get(Attribute, FuncInfo). + +funcinfo_set(#b_function{}=F, Attribute, Value, ModInfo) -> + funcinfo_set(get_fa(F), Attribute, Value, ModInfo); +funcinfo_set(Key, Attribute, Value, ModInfo) -> + FuncInfo = maps:put(Attribute, Value, maps:get(Key, ModInfo, #{})), + maps:put(Key, FuncInfo, ModInfo). + +get_fa(#b_function{ anno = Anno }) -> + {_,Name,Arity} = maps:get(func_info, Anno), + {Name,Arity}. + +%% Replaces matched-out binaries with aliases that are lazily converted to +%% binary form when used, allowing us to keep the "match path" free of binary +%% creation. + +-spec alias_matched_binaries(Blocks, Counter, AliasMap) -> Result when + Blocks :: beam_ssa:block_map(), + Counter :: non_neg_integer(), + AliasMap :: match_alias_map(), + Result :: {Blocks, Counter}. + +-type match_alias_map() :: + #{ Binary :: #b_var{} => + { %% Replace all uses of Binary with an alias after this + %% label. + AliasAfter :: beam_ssa:label(), + %% The match context whose tail is equal to Binary. + Context :: #b_var{} } }. + +%% Keeps track of the promotions we need to insert. They're partially keyed by +%% location because they may not be valid on all execution paths and we may +%% need to add redundant promotions in some cases. +-type promotion_map() :: + #{ { PromoteAt :: beam_ssa:label(), + Variable :: #b_var{} } => + Instruction :: #b_set{} }. + +-record(amb, { dominators :: beam_ssa:dominator_map(), + match_aliases :: match_alias_map(), + cnt :: non_neg_integer(), + promotions = #{} :: promotion_map() }). + +alias_matched_binaries(Blocks0, Counter, AliasMap) when AliasMap =/= #{} -> + State0 = #amb{ dominators = beam_ssa:dominators(Blocks0), + match_aliases = AliasMap, + cnt = Counter }, + {Blocks, State} = beam_ssa:mapfold_blocks_rpo(fun amb_1/3, [0], State0, + Blocks0), + {amb_insert_promotions(Blocks, State), State#amb.cnt}; +alias_matched_binaries(Blocks, Counter, _AliasMap) -> + {Blocks, Counter}. + +amb_1(Lbl, #b_blk{is=Is0,last=Last0}=Block, State0) -> + {Is, State1} = mapfoldl(fun(I, State) -> + amb_assign_set(I, Lbl, State) + end, State0, Is0), + {Last, State} = amb_assign_last(Last0, Lbl, State1), + {Block#b_blk{is=Is,last=Last}, State}. + +amb_assign_set(#b_set{op=phi,args=Args0}=I, _Lbl, State0) -> + %% Phi node aliases are relative to their source block, not their + %% containing block. + {Args, State} = + mapfoldl(fun({Arg0, Lbl}, Acc) -> + {Arg, State} = amb_get_alias(Arg0, Lbl, Acc), + {{Arg, Lbl}, State} + end, State0, Args0), + {I#b_set{args=Args}, State}; +amb_assign_set(#b_set{args=Args0}=I, Lbl, State0) -> + {Args, State} = mapfoldl(fun(Arg0, Acc) -> + amb_get_alias(Arg0, Lbl, Acc) + end, State0, Args0), + {I#b_set{args=Args}, State}. + +amb_assign_last(#b_ret{arg=Arg0}=T, Lbl, State0) -> + {Arg, State} = amb_get_alias(Arg0, Lbl, State0), + {T#b_ret{arg=Arg}, State}; +amb_assign_last(#b_switch{arg=Arg0}=T, Lbl, State0) -> + {Arg, State} = amb_get_alias(Arg0, Lbl, State0), + {T#b_switch{arg=Arg}, State}; +amb_assign_last(#b_br{bool=Arg0}=T, Lbl, State0) -> + {Arg, State} = amb_get_alias(Arg0, Lbl, State0), + {T#b_br{bool=Arg}, State}. + +amb_get_alias(#b_var{}=Arg, Lbl, State) -> + case maps:find(Arg, State#amb.match_aliases) of + {ok, {AliasAfter, Context}} -> + %% Our context may not have been created yet, so we skip assigning + %% an alias unless the given block is among our dominators. + Dominators = maps:get(Lbl, State#amb.dominators), + case ordsets:is_element(AliasAfter, Dominators) of + true -> amb_create_alias(Arg, Context, Lbl, State); + false -> {Arg, State} + end; + error -> + {Arg, State} + end; +amb_get_alias(#b_remote{mod=Mod0,name=Name0}=Arg0, Lbl, State0) -> + {Mod, State1} = amb_get_alias(Mod0, Lbl, State0), + {Name, State} = amb_get_alias(Name0, Lbl, State1), + Arg = Arg0#b_remote{mod=Mod,name=Name}, + {Arg, State}; +amb_get_alias(Arg, _Lbl, State) -> + {Arg, State}. + +amb_create_alias(#b_var{}=Arg0, Context, Lbl, State0) -> + Dominators = maps:get(Lbl, State0#amb.dominators), + Promotions0 = State0#amb.promotions, + + PrevPromotions = + [maps:get({Dom, Arg0}, Promotions0) + || Dom <- Dominators, is_map_key({Dom, Arg0}, Promotions0)], + + case PrevPromotions of + [_|_] -> + %% We've already created an alias prior to this block, so we'll + %% grab the most recent one to minimize stack use. + + #b_set{dst=Alias} = max(PrevPromotions), + {Alias, State0}; + [] -> + %% If we haven't created an alias we need to do so now. The + %% promotion will be inserted later by amb_insert_promotions/2. + + Counter = State0#amb.cnt, + Alias = #b_var{name={'@ssa_bsm_alias', Counter}}, + Promotion = #b_set{op=bs_get_tail,dst=Alias,args=[Context]}, + + Promotions = maps:put({Lbl, Arg0}, Promotion, Promotions0), + State = State0#amb{ promotions=Promotions, cnt=Counter+1 }, + + {Alias, State} + end. + +amb_insert_promotions(Blocks0, State) -> + F = fun({Lbl, #b_var{}}, Promotion, Blocks) -> + Block = maps:get(Lbl, Blocks), + + Alias = Promotion#b_set.dst, + {Before, After} = splitwith( + fun(#b_set{args=Args}) -> + not is_var_in_args(Alias, Args) + end, Block#b_blk.is), + Is = Before ++ [Promotion | After], + + maps:put(Lbl, Block#b_blk{is=Is}, Blocks) + end, + maps:fold(F, Blocks0, State#amb.promotions). + +is_var_in_args(Var, [Var | _]) -> true; +is_var_in_args(Var, [#b_remote{name=Var} | _]) -> true; +is_var_in_args(Var, [#b_remote{mod=Var} | _]) -> true; +is_var_in_args(Var, [_ | Args]) -> is_var_in_args(Var, Args); +is_var_in_args(_Var, []) -> false. + +%%% +%%% Subpasses +%%% + +%% Removes superflous chained bs_start_match instructions in the same +%% function. When matching on an extracted tail binary, or on a binary we've +%% already matched on, we reuse the original match context. +%% +%% This pass runs first since it makes subsequent optimizations more effective +%% by removing spots where promotion would be required. + +-type prior_match_map() :: + #{ Binary :: #b_var{} => + [{ %% The context and success label of a previous + %% bs_start_match made on this binary. + ValidAfter :: beam_ssa:label(), + Context :: #b_var{} }] }. + +-record(cm, { definitions :: beam_ssa:definition_map(), + dominators :: beam_ssa:dominator_map(), + blocks :: beam_ssa:block_map(), + match_aliases = #{} :: match_alias_map(), + prior_matches = #{} :: prior_match_map(), + renames = #{} :: beam_ssa:rename_map() }). + +combine_matches({Fs0, ModInfo}) -> + Fs = map(fun(F) -> combine_matches(F, ModInfo) end, Fs0), + {Fs, ModInfo}. + +combine_matches(#b_function{bs=Blocks0,cnt=Counter0}=F, ModInfo) -> + case funcinfo_get(F, has_bsm_ops, ModInfo) of + true -> + {Blocks1, State} = + beam_ssa:mapfold_blocks_rpo( + fun(Lbl, #b_blk{is=Is0}=Block0, State0) -> + {Is, State} = cm_1(Is0, [], Lbl, State0), + {Block0#b_blk{is=Is}, State} + end, [0], + #cm{ definitions = beam_ssa:definitions(Blocks0), + dominators = beam_ssa:dominators(Blocks0), + blocks = Blocks0 }, + Blocks0), + + Blocks2 = beam_ssa:rename_vars(State#cm.renames, [0], Blocks1), + + {Blocks, Counter} = alias_matched_binaries(Blocks2, Counter0, + State#cm.match_aliases), + + F#b_function{ bs=Blocks, cnt=Counter }; + false -> + F + end. + +cm_1([#b_set{ op=bs_start_match, + dst=Ctx, + args=[Src] }, + #b_set{ op=succeeded, + dst=Bool, + args=[Ctx] }]=MatchSeq, Acc0, Lbl, State0) -> + Acc = reverse(Acc0), + case is_tail_binary(Src, State0#cm.definitions) of + true -> cm_combine_tail(Src, Ctx, Bool, Acc, State0); + false -> cm_handle_priors(Src, Ctx, Bool, Acc, MatchSeq, Lbl, State0) + end; +cm_1([I | Is], Acc, Lbl, State) -> + cm_1(Is, [I | Acc], Lbl, State); +cm_1([], Acc, _Lbl, State) -> + {reverse(Acc), State}. + +%% If we're dominated by at least one match on the same source, we can reuse +%% the context created by that match. +cm_handle_priors(Src, DstCtx, Bool, Acc, MatchSeq, Lbl, State0) -> + PriorCtxs = case maps:find(Src, State0#cm.prior_matches) of + {ok, Priors} -> + %% We've seen other match contexts on this source, but + %% we can only consider the ones whose success path + %% dominate us. + Dominators = maps:get(Lbl, State0#cm.dominators, []), + [Ctx || {ValidAfter, Ctx} <- Priors, + ordsets:is_element(ValidAfter, Dominators)]; + error -> + [] + end, + case PriorCtxs of + [Ctx|_] -> + Renames0 = State0#cm.renames, + Renames = Renames0#{ Bool => #b_literal{val=true}, DstCtx => Ctx }, + {Acc, State0#cm{ renames = Renames }}; + [] -> + %% Since we lack a prior match, we need to register this one in + %% case we dominate another. + State = cm_register_prior(Src, DstCtx, Lbl, State0), + {Acc ++ MatchSeq, State} + end. + +cm_register_prior(Src, DstCtx, Lbl, State) -> + Block = maps:get(Lbl, State#cm.blocks), + #b_br{succ=ValidAfter} = Block#b_blk.last, + + Priors0 = maps:get(Src, State#cm.prior_matches, []), + Priors = [{ValidAfter, DstCtx} | Priors0], + + PriorMatches = maps:put(Src, Priors, State#cm.prior_matches), + State#cm{ prior_matches = PriorMatches }. + +cm_combine_tail(Src, DstCtx, Bool, Acc, State0) -> + SrcCtx = match_context_of(Src, State0#cm.definitions), + + %% We replace the source with a context alias as it normally won't be used + %% on the happy path after being matched, and the added cost of conversion + %% is negligible if it is. + Aliases = maps:put(Src, {0, SrcCtx}, State0#cm.match_aliases), + + Renames0 = State0#cm.renames, + Renames = Renames0#{ Bool => #b_literal{val=true}, DstCtx => SrcCtx }, + + State = State0#cm{ match_aliases = Aliases, renames = Renames }, + + {Acc, State}. + +%% Lets functions accept match contexts as arguments. The parameter must be +%% unused before the bs_start_match instruction, and it must be matched in the +%% first block. + +-record(aca, { unused_parameters :: ordsets:ordset(#b_var{}), + counter :: non_neg_integer(), + parameter_info = #{} :: #{ #b_var{} => param_info() }, + match_aliases = #{} :: match_alias_map() }). + +accept_context_args({Fs, ModInfo}) -> + mapfoldl(fun accept_context_args/2, ModInfo, Fs). + +accept_context_args(#b_function{bs=Blocks0}=F, ModInfo0) -> + case funcinfo_get(F, has_bsm_ops, ModInfo0) of + true -> + Parameters = ordsets:from_list(funcinfo_get(F, parameters, ModInfo0)), + State0 = #aca{ unused_parameters = Parameters, + counter = F#b_function.cnt }, + + {Blocks1, State} = aca_1(Blocks0, State0), + {Blocks, Counter} = alias_matched_binaries(Blocks1, + State#aca.counter, + State#aca.match_aliases), + + ModInfo = funcinfo_set(F, parameter_info, State#aca.parameter_info, + ModInfo0), + + {F#b_function{bs=Blocks,cnt=Counter}, ModInfo}; + false -> + {F, ModInfo0} + end. + +aca_1(Blocks, State) -> + %% We only handle block 0 as we don't yet support starting a match after a + %% test. This is generally good enough as the sys_core_bsm pass makes the + %% match instruction come first if possible, and it's rare for a function + %% to binary-match several parameters at once. + EntryBlock = maps:get(0, Blocks), + aca_enable_reuse(EntryBlock#b_blk.is, EntryBlock, Blocks, [], State). + +aca_enable_reuse([#b_set{op=bs_start_match,args=[Src]}=I0 | Rest], + EntryBlock, Blocks0, Acc, State0) -> + case aca_is_reuse_safe(Src, State0) of + true -> + {I, Last, Blocks1, State} = + aca_reuse_context(I0, EntryBlock, Blocks0, State0), + + Is = reverse([I|Acc]) ++ Rest, + Blocks = maps:put(0, EntryBlock#b_blk{is=Is,last=Last}, Blocks1), + + {Blocks, State}; + false -> + {Blocks0, State0} + end; +aca_enable_reuse([I | Is], EntryBlock, Blocks, Acc, State0) -> + UnusedParams0 = State0#aca.unused_parameters, + case ordsets:intersection(UnusedParams0, beam_ssa:used(I)) of + [] -> + aca_enable_reuse(Is, EntryBlock, Blocks, [I | Acc], State0); + PrematureUses -> + UnusedParams = ordsets:subtract(UnusedParams0, PrematureUses), + + %% Mark the offending parameters as unsuitable for context reuse. + ParamInfo = foldl(fun(A, Ps) -> + maps:put(A, {used_before_match, I}, Ps) + end, State0#aca.parameter_info, PrematureUses), + + State = State0#aca{ unused_parameters = UnusedParams, + parameter_info = ParamInfo }, + aca_enable_reuse(Is, EntryBlock, Blocks, [I | Acc], State) + end; +aca_enable_reuse([], _EntryBlock, Blocks, _Acc, State) -> + {Blocks, State}. + +aca_is_reuse_safe(Src, State) -> + %% Context reuse is unsafe unless all uses are dominated by the start_match + %% instruction. Since we only process block 0 it's enough to check if + %% they're unused so far. + ordsets:is_element(Src, State#aca.unused_parameters). + +aca_reuse_context(#b_set{dst=Dst, args=[Src]}=I0, Block, Blocks0, State0) -> + %% When matching fails on a reused context it needs to be converted back + %% to a binary. We only need to do this on the success path since it can't + %% be a context on the type failure path, but it's very common for these + %% to converge which requires special handling. + {State1, Last, Blocks} = + aca_handle_convergence(Src, State0, Block#b_blk.last, Blocks0), + + Aliases = maps:put(Src, {Last#b_br.succ, Dst}, State1#aca.match_aliases), + ParamInfo = maps:put(Src, suitable_for_reuse, State1#aca.parameter_info), + + State = State1#aca{ match_aliases = Aliases, + parameter_info = ParamInfo }, + + I = beam_ssa:add_anno(accepts_match_contexts, true, I0), + + {I, Last, Blocks, State}. + +aca_handle_convergence(Src, State0, Last0, Blocks0) -> + #b_br{fail=Fail0,succ=Succ0} = Last0, + + SuccPath = beam_ssa:rpo([Succ0], Blocks0), + FailPath = beam_ssa:rpo([Fail0], Blocks0), + + %% The promotion logic in alias_matched_binaries breaks down if the source + %% is used after the fail/success paths converge, as we have no way to tell + %% whether the source is a match context or something else past that point. + %% + %% We could handle this through clever insertion of phi nodes but it's + %% far simpler to copy either branch in its entirety. It doesn't matter + %% which one as long as they become disjoint. + ConvergedPaths = ordsets:intersection( + ordsets:from_list(SuccPath), + ordsets:from_list(FailPath)), + + case maps:is_key(Src, beam_ssa:uses(ConvergedPaths, Blocks0)) of + true -> + case shortest(SuccPath, FailPath) of + left -> + {Succ, Blocks, Counter} = + aca_copy_successors(Succ0, Blocks0, State0#aca.counter), + State = State0#aca{ counter = Counter }, + {State, Last0#b_br{succ=Succ}, Blocks}; + right -> + {Fail, Blocks, Counter} = + aca_copy_successors(Fail0, Blocks0, State0#aca.counter), + State = State0#aca{ counter = Counter }, + {State, Last0#b_br{fail=Fail}, Blocks} + end; + false -> + {State0, Last0, Blocks0} + end. + +shortest([_|As], [_|Bs]) -> shortest(As, Bs); +shortest([], _) -> left; +shortest(_, []) -> right. + +%% Copies all successor blocks of Lbl, returning the label to the entry block +%% of this copy. Since the copied blocks aren't referenced anywhere else, they +%% are all guaranteed to be dominated by Lbl. +aca_copy_successors(Lbl0, Blocks0, Counter0) -> + %% Building the block rename map up front greatly simplifies phi node + %% handling. + Path = beam_ssa:rpo([Lbl0], Blocks0), + {BRs, Counter1} = aca_cs_build_brs(Path, Counter0, #{}), + {Blocks, Counter} = aca_cs_1(Path, Blocks0, Counter1, #{}, BRs, #{}), + Lbl = maps:get(Lbl0, BRs), + {Lbl, Blocks, Counter}. + +aca_cs_build_brs([Lbl | Path], Counter0, Acc) -> + aca_cs_build_brs(Path, Counter0 + 1, maps:put(Lbl, Counter0, Acc)); +aca_cs_build_brs([], Counter, Acc) -> + {Acc, Counter}. + +aca_cs_1([Lbl0 | Path], Blocks, Counter0, VRs0, BRs, Acc0) -> + Block0 = maps:get(Lbl0, Blocks), + Lbl = maps:get(Lbl0, BRs), + {VRs, Block, Counter} = aca_cs_block(Block0, Counter0, VRs0, BRs), + Acc = maps:put(Lbl, Block, Acc0), + aca_cs_1(Path, Blocks, Counter, VRs, BRs, Acc); +aca_cs_1([], Blocks, Counter, _VRs, _BRs, Acc) -> + {maps:merge(Blocks, Acc), Counter}. + +aca_cs_block(#b_blk{is=Is0,last=Last0}=Block0, Counter0, VRs0, BRs) -> + {VRs, Is, Counter} = aca_cs_is(Is0, Counter0, VRs0, BRs, []), + Last = aca_cs_last(Last0, VRs, BRs), + Block = Block0#b_blk{is=Is,last=Last}, + {VRs, Block, Counter}. + +aca_cs_is([#b_set{op=Op, + dst=Dst0, + args=Args0}=I0 | Is], + Counter0, VRs0, BRs, Acc) -> + Args = case Op of + phi -> aca_cs_args_phi(Args0, VRs0, BRs); + _ -> aca_cs_args(Args0, VRs0) + end, + Counter = Counter0 + 1, + Dst = #b_var{name={'@ssa_bsm_aca',Counter}}, + I = I0#b_set{dst=Dst,args=Args}, + VRs = maps:put(Dst0, Dst, VRs0), + aca_cs_is(Is, Counter, VRs, BRs, [I | Acc]); +aca_cs_is([], Counter, VRs, _BRs, Acc) -> + {VRs, reverse(Acc), Counter}. + +aca_cs_last(#b_switch{arg=Arg0,list=Switch0,fail=Fail0}=Sw, VRs, BRs) -> + Switch = [{Literal, maps:get(Lbl, BRs)} || {Literal, Lbl} <- Switch0], + Sw#b_switch{arg=aca_cs_arg(Arg0, VRs), + fail=maps:get(Fail0, BRs), + list=Switch}; +aca_cs_last(#b_br{bool=Arg0,succ=Succ0,fail=Fail0}=Br, VRs, BRs) -> + Br#b_br{bool=aca_cs_arg(Arg0, VRs), + succ=maps:get(Succ0, BRs), + fail=maps:get(Fail0, BRs)}; +aca_cs_last(#b_ret{arg=Arg0}=Ret, VRs, _BRs) -> + Ret#b_ret{arg=aca_cs_arg(Arg0, VRs)}. + +aca_cs_args_phi([{Arg, Lbl} | Args], VRs, BRs) -> + case BRs of + #{ Lbl := New } -> + [{aca_cs_arg(Arg, VRs), New} | aca_cs_args_phi(Args, VRs, BRs)]; + #{} -> + aca_cs_args_phi(Args, VRs, BRs) + end; +aca_cs_args_phi([], _VRs, _BRs) -> + []. + +aca_cs_args([Arg | Args], VRs) -> + [aca_cs_arg(Arg, VRs) | aca_cs_args(Args, VRs)]; +aca_cs_args([], _VRs) -> + []. + +aca_cs_arg(#b_remote{mod=Mod0,name=Name0}=Rem, VRs) -> + Mod = aca_cs_arg(Mod0, VRs), + Name = aca_cs_arg(Name0, VRs), + Rem#b_remote{mod=Mod,name=Name}; +aca_cs_arg(Arg, VRs) -> + case VRs of + #{ Arg := New } -> New; + #{} -> Arg + end. + +%% Allows contexts to pass through "wrapper functions" where the context is +%% passed directly to a function that accepts match contexts (including other +%% wrappers). +%% +%% This does not alter the function in any way, it only changes parameter info +%% so that skip_outgoing_tail_extraction is aware that it's safe to pass +%% contexts to us. + +allow_context_passthrough({Fs, ModInfo0}) -> + ModInfo = + acp_forward_params([{F, beam_ssa:uses(F#b_function.bs)} || F <- Fs], + ModInfo0), + {Fs, ModInfo}. + +acp_forward_params(FsUses, ModInfo0) -> + F = fun({#b_function{args=Parameters}=Func, UseMap}, ModInfo) -> + ParamInfo = + foldl(fun(Param, ParamInfo) -> + Uses = maps:get(Param, UseMap, []), + acp_1(Param, Uses, ModInfo, ParamInfo) + end, + funcinfo_get(Func, parameter_info, ModInfo), + Parameters), + funcinfo_set(Func, parameter_info, ParamInfo, ModInfo) + end, + %% Allowing context passthrough on one function may make it possible to + %% enable it on another, so it needs to be repeated for maximum effect. + case foldl(F, ModInfo0, FsUses) of + ModInfo0 -> ModInfo0; + Changed -> acp_forward_params(FsUses, Changed) + end. + +%% We have no way to know if an argument is a context, so it's only safe to +%% forward them if they're passed exactly once in the first block. Any other +%% uses are unsafe, including function_clause errors. +acp_1(Param, [{0, #b_set{op=call}=I}], ModInfo, ParamInfo) -> + %% We don't need to provide a context chain as our callers make sure that + %% multiple arguments never reference the same context. + case check_context_call(I, Param, [], ModInfo) of + {no_match_on_entry, _} -> ParamInfo; + Other -> maps:put(Param, Other, ParamInfo) + end; +acp_1(_Param, _Uses, _ModInfo, ParamInfo) -> + ParamInfo. + +%% This is conceptually similar to combine_matches but operates across +%% functions. Whenever a tail binary is passed to a parameter that accepts +%% match contexts we'll pass the context instead, improving performance by +%% avoiding the creation of a new match context in the callee. +%% +%% We also create an alias to delay extraction until it's needed as an actual +%% binary, which is often rare on the happy path. The cost of being wrong is +%% negligible (`bs_test_unit + bs_get_tail` vs `bs_get_binary`) so we're +%% applying it unconditionally to keep things simple. + +-record(sote, { definitions :: beam_ssa:definition_map(), + mod_info :: module_info(), + match_aliases = #{} :: match_alias_map() }). + +skip_outgoing_tail_extraction({Fs0, ModInfo}) -> + Fs = map(fun(F) -> skip_outgoing_tail_extraction(F, ModInfo) end, Fs0), + {Fs, ModInfo}. + +skip_outgoing_tail_extraction(#b_function{bs=Blocks0}=F, ModInfo) -> + case funcinfo_get(F, has_bsm_ops, ModInfo) of + true -> + State0 = #sote{ definitions = beam_ssa:definitions(Blocks0), + mod_info = ModInfo }, + + {Blocks1, State} = beam_ssa:mapfold_instrs_rpo( + fun sote_rewrite_calls/2, [0], State0, Blocks0), + + {Blocks, Counter} = alias_matched_binaries(Blocks1, + F#b_function.cnt, + State#sote.match_aliases), + + F#b_function{bs=Blocks,cnt=Counter}; + false -> + F + end. + +sote_rewrite_calls(#b_set{op=call,args=Args}=Call, State) -> + sote_rewrite_call(Call, Args, [], State); +sote_rewrite_calls(I, State) -> + {I, State}. + +sote_rewrite_call(Call, [], ArgsOut, State) -> + {Call#b_set{args=reverse(ArgsOut)}, State}; +sote_rewrite_call(Call0, [Arg | ArgsIn], ArgsOut, State0) -> + case is_tail_binary(Arg, State0#sote.definitions) of + true -> + CtxChain = context_chain_of(Arg, State0#sote.definitions), + case check_context_call(Call0, Arg, CtxChain, State0#sote.mod_info) of + suitable_for_reuse -> + Ctx = match_context_of(Arg, State0#sote.definitions), + + MatchAliases0 = State0#sote.match_aliases, + MatchAliases = maps:put(Arg, {0, Ctx}, MatchAliases0), + State = State0#sote{ match_aliases = MatchAliases }, + + Call = beam_ssa:add_anno(bsm_info, context_reused, Call0), + sote_rewrite_call(Call, ArgsIn, [Ctx | ArgsOut], State); + Other -> + Call = beam_ssa:add_anno(bsm_info, Other, Call0), + sote_rewrite_call(Call, ArgsIn, [Arg | ArgsOut], State0) + end; + false -> + sote_rewrite_call(Call0, ArgsIn, [Arg | ArgsOut], State0) + end. + +%% Adds parameter_type_info annotations to help the validator determine whether +%% our optimizations were safe. + +annotate_context_parameters({Fs, ModInfo}) -> + mapfoldl(fun annotate_context_parameters/2, ModInfo, Fs). + +annotate_context_parameters(F, ModInfo) -> + ParamInfo = funcinfo_get(F, parameter_info, ModInfo), + TypeAnno0 = beam_ssa:get_anno(parameter_type_info, F, #{}), + TypeAnno = maps:fold(fun(K, _V, Acc) when is_map_key(K, Acc) -> + %% Assertion. + error(conflicting_parameter_types); + (K, suitable_for_reuse, Acc) -> + Acc#{ K => match_context }; + (_K, _V, Acc) -> + Acc + end, TypeAnno0, ParamInfo), + {beam_ssa:add_anno(parameter_type_info, TypeAnno, F), ModInfo}. + +%%% +%%% +bin_opt_info +%%% + +collect_opt_info(Fs) -> + foldl(fun(#b_function{bs=Blocks}=F, Acc0) -> + UseMap = beam_ssa:uses(Blocks), + Where = beam_ssa:get_anno(location, F, []), + beam_ssa:fold_instrs_rpo( + fun(I, Acc) -> + collect_opt_info_1(I, Where, UseMap, Acc) + end, [0], Acc0, Blocks) + end, [], Fs). + +collect_opt_info_1(#b_set{op=Op,anno=Anno,dst=Dst}=I, Where, UseMap, Acc0) -> + case is_tail_binary(I) of + true when Op =:= bs_match -> + %% The uses include when the context is passed raw, so we discard + %% everything but the bs_extract instruction to limit warnings to + %% unoptimized uses. + Uses0 = maps:get(Dst, UseMap, []), + case [E || {_, #b_set{op=bs_extract}=E} <- Uses0] of + [Use] -> add_unopt_binary_info(Use, false, Where, UseMap, Acc0); + [] -> Acc0 + end; + true -> + %% Add a warning for each use. Note that we don't do anything + %% special if unused as a later pass will remove this instruction + %% anyway. + Uses = maps:get(Dst, UseMap, []), + foldl(fun({_Lbl, Use}, Acc) -> + add_unopt_binary_info(Use, false, Where, UseMap, Acc) + end, Acc0, Uses); + false -> + add_opt_info(Anno, Where, Acc0) + end; +collect_opt_info_1(#b_ret{anno=Anno}, Where, _UseMap, Acc) -> + add_opt_info(Anno, Where, Acc); +collect_opt_info_1(_I, _Where, _Uses, Acc) -> + Acc. + +add_opt_info(Anno, Where, Acc) -> + case maps:find(bsm_info, Anno) of + {ok, Term} -> [make_warning(Term, Anno, Where) | Acc]; + error -> Acc + end. + +%% When an alias is promoted we need to figure out where it goes to ignore +%% warnings for compiler-generated things, and provide more useful warnings in +%% general. +%% +%% We track whether the binary has been used to build another term because it +%% can be helpful when there's no line information. + +add_unopt_binary_info(#b_set{op=Follow,dst=Dst}, _Nested, Where, UseMap, Acc0) + when Follow =:= put_tuple; + Follow =:= put_list; + Follow =:= put_map -> + %% Term-building instructions. + {_, Uses} = unzip(maps:get(Dst, UseMap, [])), + foldl(fun(Use, Acc) -> + add_unopt_binary_info(Use, true, Where, UseMap, Acc) + end, Acc0, Uses); +add_unopt_binary_info(#b_set{op=Follow,dst=Dst}, Nested, Where, UseMap, Acc0) + when Follow =:= bs_extract; + Follow =:= phi -> + %% Non-building instructions that need to be followed. + {_, Uses} = unzip(maps:get(Dst, UseMap, [])), + foldl(fun(Use, Acc) -> + add_unopt_binary_info(Use, Nested, Where, UseMap, Acc) + end, Acc0, Uses); +add_unopt_binary_info(#b_set{op=call, + args=[#b_remote{mod=#b_literal{val=erlang}, + name=#b_literal{val=error}} | + _Ignored]}, + _Nested, _Where, _UseMap, Acc) -> + %% There's no nice way to tell compiler-generated exceptions apart from + %% user ones so we ignore them all. I doubt anyone cares. + Acc; +add_unopt_binary_info(#b_switch{anno=Anno}=I, Nested, Where, _UseMap, Acc) -> + [make_promotion_warning(I, Nested, Anno, Where) | Acc]; +add_unopt_binary_info(#b_set{anno=Anno}=I, Nested, Where, _UseMap, Acc) -> + [make_promotion_warning(I, Nested, Anno, Where) | Acc]; +add_unopt_binary_info(#b_ret{anno=Anno}=I, Nested, Where, _UseMap, Acc) -> + [make_promotion_warning(I, Nested, Anno, Where) | Acc]; +add_unopt_binary_info(#b_br{anno=Anno}=I, Nested, Where, _UseMap, Acc) -> + [make_promotion_warning(I, Nested, Anno, Where) | Acc]. + +make_promotion_warning(I, Nested, Anno, Where) -> + make_warning({binary_created, I, Nested}, Anno, Where). + +make_warning(Term, Anno, Where) -> + {File, Line} = maps:get(location, Anno, Where), + {File,[{Line,?MODULE,Term}]}. + +format_opt_info(context_reused) -> + "OPTIMIZED: match context reused"; +format_opt_info({binary_created, _, _}=Promotion) -> + io_lib:format("BINARY CREATED: ~s", [format_opt_info_1(Promotion)]); +format_opt_info(Other) -> + io_lib:format("NOT OPTIMIZED: ~s", [format_opt_info_1(Other)]). + +format_opt_info_1({binary_created, #b_set{op=call,args=[Call|_]}, false}) -> + io_lib:format("binary is used in call to ~s which doesn't support " + "context reuse", [format_call(Call)]); +format_opt_info_1({binary_created, #b_set{op=call,args=[Call|_]}, true}) -> + io_lib:format("binary is used in term passed to ~s", + [format_call(Call)]); +format_opt_info_1({binary_created, #b_set{op={bif, BIF},args=Args}, false}) -> + io_lib:format("binary is used in ~p/~p which doesn't support context " + "reuse", [BIF, length(Args)]); +format_opt_info_1({binary_created, #b_set{op={bif, BIF},args=Args}, true}) -> + io_lib:format("binary is used in term passed to ~p/~p", + [BIF, length(Args)]); +format_opt_info_1({binary_created, #b_set{op=Op}, false}) -> + io_lib:format("binary is used in '~p' which doesn't support context " + "reuse", [Op]); +format_opt_info_1({binary_created, #b_set{op=Op}, true}) -> + io_lib:format("binary is used in term passed to '~p'", [Op]); +format_opt_info_1({binary_created, #b_ret{}, false}) -> + io_lib:format("binary is returned from the function", []); +format_opt_info_1({binary_created, #b_ret{}, true}) -> + io_lib:format("binary is used in a term that is returned from the " + "function", []); +format_opt_info_1({unsuitable_call, {Call, Inner}}) -> + io_lib:format("binary used in call to ~s, where ~s", + [format_call(Call), format_opt_info_1(Inner)]); +format_opt_info_1({remote_call, Call}) -> + io_lib:format("binary is used in remote call to ~s", [format_call(Call)]); +format_opt_info_1({fun_call, Call}) -> + io_lib:format("binary is used in fun call (~s)", + [format_call(Call)]); +format_opt_info_1({multiple_uses_in_call, Call}) -> + io_lib:format("binary is passed as multiple arguments to ~s", + [format_call(Call)]); +format_opt_info_1({no_match_on_entry, Call}) -> + io_lib:format("binary is used in call to ~s which does not begin with a " + "suitable binary match", [format_call(Call)]); +format_opt_info_1({used_before_match, #b_set{op=call,args=[Call|_]}}) -> + io_lib:format("binary is used in call to ~s before being matched", + [format_call(Call)]); +format_opt_info_1({used_before_match, #b_set{op={bif, BIF},args=Args}}) -> + io_lib:format("binary is used in ~p/~p before being matched", + [BIF, length(Args)]); +format_opt_info_1({used_before_match, #b_set{op=phi}}) -> + io_lib:format("binary is returned from an expression before being " + "matched", []); +format_opt_info_1({used_before_match, #b_set{op=Op}}) -> + io_lib:format("binary is used in '~p' before being matched",[Op]); +format_opt_info_1(Term) -> + io_lib:format("~w", [Term]). + +format_call(#b_local{name=#b_literal{val=F},arity=A}) -> + io_lib:format("~p/~p", [F, A]); +format_call(#b_remote{mod=#b_literal{val=M},name=#b_literal{val=F},arity=A}) -> + io_lib:format("~p:~p/~p", [M, F, A]); +format_call(Fun) -> + io_lib:format("~p", [Fun]). diff --git a/lib/compiler/src/beam_ssa_codegen.erl b/lib/compiler/src/beam_ssa_codegen.erl index 5085c950b5..3c14062d0b 100644 --- a/lib/compiler/src/beam_ssa_codegen.erl +++ b/lib/compiler/src/beam_ssa_codegen.erl @@ -108,7 +108,8 @@ module(#b_module{name=Mod,exports=Es,attributes=Attrs,body=Fs}, _Opts) -> -type ssa_register() :: xreg() | yreg() | {'fr',reg_num()} | {'z',reg_num()}. functions(Forms, AtomMod) -> - mapfoldl(fun (F, St) -> function(F, AtomMod, St) end, #cg{lcount=1}, Forms). + mapfoldl(fun (F, St) -> function(F, AtomMod, St) end, + #cg{lcount=1}, Forms). function(#b_function{anno=Anno,bs=Blocks}, AtomMod, St0) -> #{func_info:={_,Name,Arity}} = Anno, @@ -125,8 +126,9 @@ function(#b_function{anno=Anno,bs=Blocks}, AtomMod, St0) -> ultimate_fail=Ult}, {Body,St} = cg_fun(Blocks, St5), Asm = [{label,Fi},line(Anno), - {func_info,AtomMod,{atom,Name},Arity}] ++ Body ++ - [{label,Ult},if_end], + {func_info,AtomMod,{atom,Name},Arity}] ++ + add_parameter_annos(Body, Anno) ++ + [{label,Ult},if_end], Func = {function,Name,Arity,Entry,Asm}, {Func,St} catch @@ -150,6 +152,17 @@ assert_badarg_block(Blocks) -> ok end. +add_parameter_annos([{label, _}=Entry | Body], Anno) -> + ParamInfo = maps:get(parameter_type_info, Anno, #{}), + Annos = maps:fold( + fun(K, V, Acc) when is_map_key(K, ParamInfo) -> + TypeInfo = maps:get(K, ParamInfo), + [{'%', {type_info, V, TypeInfo}} | Acc]; + (_K, _V, Acc) -> + Acc + end, [], maps:get(registers, Anno)), + [Entry | Annos] ++ Body. + cg_fun(Blocks, St0) -> Linear0 = linearize(Blocks), St = collect_catch_labels(Linear0, St0), @@ -218,7 +231,7 @@ need_heap_never(_) -> false. need_heap_blks([{L,#cg_blk{is=Is0}=Blk0}|Bs], H0, Acc) -> {Is1,H1} = need_heap_is(reverse(Is0), H0, []), - {Ns,H} = need_heap_terminator(Bs, H1), + {Ns,H} = need_heap_terminator(Bs, L, H1), Is = Ns ++ Is1, Blk = Blk0#cg_blk{is=Is}, need_heap_blks(Bs, H, [{L,Blk}|Acc]); @@ -228,6 +241,13 @@ need_heap_blks([], H, Acc) -> need_heap_is([#cg_alloc{words=Words}=Alloc0|Is], N, Acc) -> Alloc = Alloc0#cg_alloc{words=add_heap_words(N, Words)}, need_heap_is(Is, #need{}, [Alloc|Acc]); +need_heap_is([#cg_set{anno=Anno,op=bs_init}=I0|Is], N, Acc) -> + Alloc = case need_heap_need(N) of + [#cg_alloc{words=Need}] -> alloc(Need); + [] -> 0 + end, + I = I0#cg_set{anno=Anno#{alloc=>Alloc}}, + need_heap_is(Is, #need{}, [I|Acc]); need_heap_is([#cg_set{op=Op,args=Args}=I|Is], N, Acc) -> case classify_heap_need(Op, Args) of {put,Words} -> @@ -243,11 +263,31 @@ need_heap_is([#cg_set{op=Op,args=Args}=I|Is], N, Acc) -> need_heap_is([], N, Acc) -> {Acc,N}. -need_heap_terminator([{_,#cg_blk{last=#cg_br{succ=Same,fail=Same}}}|_], N) -> +need_heap_terminator([{_,#cg_blk{last=#cg_br{succ=L,fail=L}}}|_], L, N) -> + %% Fallthrough. {[],N}; -need_heap_terminator([{_,#cg_blk{}}|_], N) -> +need_heap_terminator([{_,#cg_blk{is=Is,last=#cg_br{succ=L}}}|_], L, N) -> + case need_heap_need(N) of + [] -> + {[],#need{}}; + [_|_]=Alloc -> + %% If the preceding instructions are a binary construction, + %% hoist the allocation and incorporate into the bs_init + %% instruction. + case reverse(Is) of + [#cg_set{op=succeeded},#cg_set{op=bs_init}|_] -> + {[],N}; + [#cg_set{op=bs_put}|_] -> + {[],N}; + _ -> + %% Not binary construction. Must emit an allocation + %% instruction in this block. + {Alloc,#need{}} + end + end; +need_heap_terminator([{_,#cg_blk{}}|_], _, N) -> {need_heap_need(N),#need{}}; -need_heap_terminator([], H) -> +need_heap_terminator([], _, H) -> {need_heap_need(H),#need{}}. need_heap_need(#need{h=0,f=0}) -> []; @@ -315,12 +355,15 @@ classify_heap_need(Name, _Args) -> classify_heap_need(bs_add) -> gc; classify_heap_need(bs_get) -> gc; +classify_heap_need(bs_get_tail) -> gc; classify_heap_need(bs_init) -> gc; classify_heap_need(bs_init_writable) -> gc; classify_heap_need(bs_match_string) -> gc; classify_heap_need(bs_put) -> neutral; classify_heap_need(bs_restore) -> neutral; classify_heap_need(bs_save) -> neutral; +classify_heap_need(bs_get_position) -> gc; +classify_heap_need(bs_set_position) -> neutral; classify_heap_need(bs_skip) -> gc; classify_heap_need(bs_start_match) -> neutral; classify_heap_need(bs_test_tail) -> neutral; @@ -329,7 +372,6 @@ classify_heap_need(bs_utf8_size) -> neutral; classify_heap_need(build_stacktrace) -> gc; classify_heap_need(call) -> gc; classify_heap_need(catch_end) -> gc; -classify_heap_need(context_to_binary) -> gc; classify_heap_need(copy) -> neutral; classify_heap_need(extract) -> gc; classify_heap_need(get_hd) -> neutral; @@ -695,6 +737,8 @@ need_live_anno(Op) -> {bif,_} -> true; bs_get -> true; bs_init -> true; + bs_get_position -> true; + bs_get_tail -> true; bs_start_match -> true; bs_skip -> true; call -> true; @@ -794,6 +838,8 @@ def_successors([], _, DefMap) -> DefMap. need_y_init(#cg_set{anno=#{clobbers:=Clobbers}}) -> Clobbers; need_y_init(#cg_set{op=bs_get}) -> true; +need_y_init(#cg_set{op=bs_get_position}) -> true; +need_y_init(#cg_set{op=bs_get_tail}) -> true; need_y_init(#cg_set{op=bs_init}) -> true; need_y_init(#cg_set{op=bs_skip,args=[#b_literal{val=Type}|_]}) -> case Type of @@ -1022,12 +1068,13 @@ cg_block([#cg_set{op=bs_init,dst=Dst0,args=Args0,anno=Anno}=I, #cg_set{op=succeeded,dst=Bool}], {Bool,Fail0}, St) -> Fail = bif_fail(Fail0), Line = line(Anno), + Alloc = map_get(alloc, Anno), [#b_literal{val=Kind}|Args1] = Args0, case Kind of new -> [Dst,Size,{integer,Unit}] = beam_args([Dst0|Args1], St), Live = get_live(I), - {[Line|cg_bs_init(Dst, Size, Unit, Live, Fail)],St}; + {[Line|cg_bs_init(Dst, Size, Alloc, Unit, Live, Fail)],St}; private_append -> [Dst,Src,Bits,{integer,Unit}] = beam_args([Dst0|Args1], St), Flags = {field_flags,[]}, @@ -1037,17 +1084,23 @@ cg_block([#cg_set{op=bs_init,dst=Dst0,args=Args0,anno=Anno}=I, [Dst,Src,Bits,{integer,Unit}] = beam_args([Dst0|Args1], St), Flags = {field_flags,[]}, Live = get_live(I), - Is = [Line,{bs_append,Fail,Bits,0,Live,Unit,Src,Flags,Dst}], + Is = [Line,{bs_append,Fail,Bits,Alloc,Live,Unit,Src,Flags,Dst}], {Is,St} end; cg_block([#cg_set{anno=Anno,op=bs_start_match,dst=Ctx0,args=[Bin0]}=I, #cg_set{op=succeeded,dst=Bool}], {Bool,Fail}, St) -> - #{num_slots:=Slots} = Anno, [Dst,Bin1] = beam_args([Ctx0,Bin0], St), {Bin,Pre} = force_reg(Bin1, Dst), Live = get_live(I), - Is = Pre ++ [{test,bs_start_match2,Fail,Live,[Bin,Slots],Dst}], - {Is,St}; + %% num_slots is only set when using the old instructions. + case maps:find(num_slots, Anno) of + {ok, Slots} -> + Is = Pre ++ [{test,bs_start_match2,Fail,Live,[Bin,Slots],Dst}], + {Is,St}; + error -> + Is = Pre ++ [{test,bs_start_match3,Fail,Live,[Bin],Dst}], + {Is,St} + end; cg_block([#cg_set{op=bs_get}=Set, #cg_set{op=succeeded,dst=Bool}], {Bool,Fail}, St) -> {cg_bs_get(Fail, Set, St),St}; @@ -1179,6 +1232,12 @@ cg_copy_1([#cg_set{dst=Dst0,args=Args}|T], St) -> end; cg_copy_1([], _St) -> []. +-define(IS_LITERAL(Val), (Val =:= nil orelse + element(1, Val) =:= integer orelse + element(1, Val) =:= float orelse + element(1, Val) =:= atom orelse + element(1, Val) =:= literal)). + bif_to_test('and', [V1,V2], Fail) -> [{test,is_eq_exact,Fail,[V1,{atom,true}]}, {test,is_eq_exact,Fail,[V2,{atom,true}]}]; @@ -1192,15 +1251,99 @@ bif_to_test('or', [V1,V2], {f,Lbl}=Fail) when Lbl =/= 0 -> bif_to_test('not', [Var], Fail) -> [{test,is_eq_exact,Fail,[Var,{atom,false}]}]; bif_to_test(Name, Args, Fail) -> - [beam_utils:bif_to_test(Name, Args, Fail)]. + [bif_to_test_1(Name, Args, Fail)]. + +bif_to_test_1(is_atom, [_]=Ops, Fail) -> + {test,is_atom,Fail,Ops}; +bif_to_test_1(is_boolean, [_]=Ops, Fail) -> + {test,is_boolean,Fail,Ops}; +bif_to_test_1(is_binary, [_]=Ops, Fail) -> + {test,is_binary,Fail,Ops}; +bif_to_test_1(is_bitstring,[_]=Ops, Fail) -> + {test,is_bitstr,Fail,Ops}; +bif_to_test_1(is_float, [_]=Ops, Fail) -> + {test,is_float,Fail,Ops}; +bif_to_test_1(is_function, [_]=Ops, Fail) -> + {test,is_function,Fail,Ops}; +bif_to_test_1(is_function, [_,_]=Ops, Fail) -> + {test,is_function2,Fail,Ops}; +bif_to_test_1(is_integer, [_]=Ops, Fail) -> + {test,is_integer,Fail,Ops}; +bif_to_test_1(is_list, [_]=Ops, Fail) -> + {test,is_list,Fail,Ops}; +bif_to_test_1(is_map, [_]=Ops, Fail) -> + {test,is_map,Fail,Ops}; +bif_to_test_1(is_number, [_]=Ops, Fail) -> + {test,is_number,Fail,Ops}; +bif_to_test_1(is_pid, [_]=Ops, Fail) -> + {test,is_pid,Fail,Ops}; +bif_to_test_1(is_port, [_]=Ops, Fail) -> + {test,is_port,Fail,Ops}; +bif_to_test_1(is_reference, [_]=Ops, Fail) -> + {test,is_reference,Fail,Ops}; +bif_to_test_1(is_tuple, [_]=Ops, Fail) -> + {test,is_tuple,Fail,Ops}; +bif_to_test_1('=<', [A,B], Fail) -> + {test,is_ge,Fail,[B,A]}; +bif_to_test_1('>', [A,B], Fail) -> + {test,is_lt,Fail,[B,A]}; +bif_to_test_1('<', [_,_]=Ops, Fail) -> + {test,is_lt,Fail,Ops}; +bif_to_test_1('>=', [_,_]=Ops, Fail) -> + {test,is_ge,Fail,Ops}; +bif_to_test_1('==', [C,A], Fail) when ?IS_LITERAL(C) -> + {test,is_eq,Fail,[A,C]}; +bif_to_test_1('==', [_,_]=Ops, Fail) -> + {test,is_eq,Fail,Ops}; +bif_to_test_1('/=', [C,A], Fail) when ?IS_LITERAL(C) -> + {test,is_ne,Fail,[A,C]}; +bif_to_test_1('/=', [_,_]=Ops, Fail) -> + {test,is_ne,Fail,Ops}; +bif_to_test_1('=:=', [C,A], Fail) when ?IS_LITERAL(C) -> + {test,is_eq_exact,Fail,[A,C]}; +bif_to_test_1('=:=', [_,_]=Ops, Fail) -> + {test,is_eq_exact,Fail,Ops}; +bif_to_test_1('=/=', [C,A], Fail) when ?IS_LITERAL(C) -> + {test,is_ne_exact,Fail,[A,C]}; +bif_to_test_1('=/=', [_,_]=Ops, Fail) -> + {test,is_ne_exact,Fail,Ops}. opt_call_moves(Is0, Arity) -> {Moves0,Is} = splitwith(fun({move,_,_}) -> true; + ({kill,_}) -> true; (_) -> false end, Is0), Moves = opt_call_moves_1(Moves0, Arity), Moves ++ Is. +opt_call_moves_1([{move,Src,{x,_}=Tmp}=M1|[{kill,_}|_]=Is], Arity) -> + %% There could be a {move,Tmp,{x,0}} instruction after the + %% kill/1 instructions (moved to there by opt_move_to_x0/1). + case splitwith(fun({kill,_}) -> true; + (_) -> false + end, Is) of + {Kills,[{move,{x,_}=Tmp,{x,0}}=M2]} -> + %% The two move/2 instructions (M1 and M2) can be combined + %% to one. The question is, though, is it safe to place + %% them after the kill/1 instructions? + case is_killed(Src, Kills, Arity) of + true -> + %% Src (a Y register) is killed by one of the + %% kill/1 instructions. Thus M1 and M2 + %% must be placed before the kill/1 instructions + %% (essentially undoing what opt_move_to_x0/1 + %% did, which turned out to be a pessimization + %% in this case). + opt_call_moves_1([M1,M2|Kills], Arity); + false -> + %% Src is not killed by any of the kill/1 + %% instructions. Thus it is safe to place + %% M1 and M2 after the kill/1 instructions. + opt_call_moves_1(Kills++[M1,M2], Arity) + end; + {_,_} -> + [M1|Is] + end; opt_call_moves_1([{move,Src,{x,_}=Tmp}=M1,{move,Tmp,Dst}=M2|Is], Arity) -> case is_killed(Tmp, Is, Arity) of true -> @@ -1214,6 +1357,10 @@ opt_call_moves_1([M|Ms], Arity) -> [M|opt_call_moves_1(Ms, Arity)]; opt_call_moves_1([], _Arity) -> []. +is_killed(Y, [{kill,Y}|_], _) -> + true; +is_killed(R, [{kill,_}|Is], Arity) -> + is_killed(R, Is, Arity); is_killed(R, [{move,R,_}|_], _) -> false; is_killed(R, [{move,_,R}|_], _) -> @@ -1221,7 +1368,9 @@ is_killed(R, [{move,_,R}|_], _) -> is_killed(R, [{move,_,_}|Is], Arity) -> is_killed(R, Is, Arity); is_killed({x,X}, [], Arity) -> - X >= Arity. + X >= Arity; +is_killed({y,_}, [], _) -> + false. cg_alloc(#cg_alloc{stack=none,words=#need{h=0,f=0}}, _St) -> []; @@ -1344,6 +1493,12 @@ build_apply(Arity, none, Dst) -> cg_instr(put_map, [{atom,assoc},SrcMap|Ss], Dst, Set) -> Live = get_live(Set), [{put_map_assoc,{f,0},SrcMap,Dst,Live,{list,Ss}}]; +cg_instr(bs_get_tail, [Src], Dst, Set) -> + Live = get_live(Set), + [{bs_get_tail,Src,Dst,Live}]; +cg_instr(bs_get_position, [Ctx], Dst, Set) -> + Live = get_live(Set), + [{bs_get_position,Ctx,Dst,Live}]; cg_instr(Op, Args, Dst, _Set) -> cg_instr(Op, Args, Dst). @@ -1359,10 +1514,10 @@ cg_instr(bs_restore, [Ctx,Slot], _Dst) -> cg_instr(bs_save, [Ctx,Slot], _Dst) -> {integer,N} = Slot, [{bs_save2,Ctx,N}]; +cg_instr(bs_set_position, [Ctx,Pos], _Dst) -> + [{bs_set_position,Ctx,Pos}]; cg_instr(build_stacktrace, Args, Dst) -> setup_args(Args) ++ [build_stacktrace|copy({x,0}, Dst)]; -cg_instr(context_to_binary, [Src], _Dst) -> - [{bs_context_to_binary,Src}]; cg_instr(set_tuple_element=Op, [New,Tuple,{integer,Index}], _Dst) -> [{Op,New,Tuple,Index}]; cg_instr({float,clearerror}, [], _Dst) -> @@ -1496,13 +1651,13 @@ cg_bs_put(Fail, [{atom,Type},{literal,Flags}|Args]) -> [{Op,Fail,{field_flags,Flags},Src}] end. -cg_bs_init(Dst, Size0, Unit, Live, Fail) -> +cg_bs_init(Dst, Size0, Alloc, Unit, Live, Fail) -> Op = case Unit of 1 -> bs_init_bits; 8 -> bs_init2 end, Size = cg_bs_init_size(Size0), - [{Op,Fail,Size,0,Live,{field_flags,[]},Dst}]. + [{Op,Fail,Size,Alloc,Live,{field_flags,[]},Dst}]. cg_bs_init_size({x,_}=R) -> R; cg_bs_init_size({y,_}=R) -> R; @@ -1621,12 +1776,41 @@ phi_copies([#b_set{dst=Dst,args=PhiArgs}|Sets], L) -> [#cg_set{op=copy,dst=Dst,args=CopyArgs}|phi_copies(Sets, L)]; phi_copies([], _) -> []. +%% opt_move_to_x0([Instruction]) -> [Instruction]. +%% Simple peep-hole optimization to move a {move,Any,{x,0}} past +%% any kill up to the next call instruction. (To give the loader +%% an opportunity to combine the 'move' and the 'call' instructions.) + +opt_move_to_x0(Moves) -> + opt_move_to_x0(Moves, []). + +opt_move_to_x0([{move,_,{x,0}}=I|Is0], Acc0) -> + case move_past_kill(Is0, I, Acc0) of + impossible -> opt_move_to_x0(Is0, [I|Acc0]); + {Is,Acc} -> opt_move_to_x0(Is, Acc) + end; +opt_move_to_x0([I|Is], Acc) -> + opt_move_to_x0(Is, [I|Acc]); +opt_move_to_x0([], Acc) -> reverse(Acc). + +move_past_kill([{kill,Src}|_], {move,Src,_}, _) -> + impossible; +move_past_kill([{kill,_}=I|Is], Move, Acc) -> + move_past_kill(Is, Move, [I|Acc]); +move_past_kill(Is, Move, Acc) -> + {Is,[Move|Acc]}. + %% setup_args(Args, Anno, Context) -> [Instruction]. %% setup_args(Args) -> [Instruction]. %% Set up X registers for a call. setup_args(Args, Anno, none, St) -> - setup_args(Args) ++ kill_yregs(Anno, St); + case {setup_args(Args),kill_yregs(Anno, St)} of + {Moves,[]} -> + Moves; + {Moves,Kills} -> + opt_move_to_x0(Moves ++ Kills) + end; setup_args(Args, _, _, _) -> setup_args(Args). diff --git a/lib/compiler/src/beam_ssa_funs.erl b/lib/compiler/src/beam_ssa_funs.erl new file mode 100644 index 0000000000..38df50fd74 --- /dev/null +++ b/lib/compiler/src/beam_ssa_funs.erl @@ -0,0 +1,149 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%%% +%%% If a fun is defined locally and only used for calls, it can be replaced +%%% with direct calls to the relevant function. This greatly speeds up "named +%%% functions" (which rely on make_fun to recreate themselves) and macros that +%%% wrap their body in a fun. +%%% + +-module(beam_ssa_funs). + +-export([module/2]). + +-include("beam_ssa.hrl"). + +-import(lists, [foldl/3]). + +-spec module(Module, Options) -> Result when + Module :: beam_ssa:b_module(), + Options :: [compile:option()], + Result :: {ok, beam_ssa:b_module()}. + +module(#b_module{body=Fs0}=Module, _Opts) -> + Trampolines = foldl(fun find_trampolines/2, #{}, Fs0), + Fs = [lfo(F, Trampolines) || F <- Fs0], + {ok, Module#b_module{body=Fs}}. + +%% If a function does absolutely nothing beyond calling another function with +%% the same arguments in the same order, we can shave off a call by short- +%% circuiting it. +find_trampolines(#b_function{args=Args,bs=Blocks}=F, Trampolines) -> + case maps:get(0, Blocks) of + #b_blk{is=[#b_set{op=call, + args=[#b_local{}=Actual | Args], + dst=Dst}], + last=#b_ret{arg=Dst}} -> + {_, Name, Arity} = beam_ssa:get_anno(func_info, F), + Trampoline = #b_local{name=#b_literal{val=Name},arity=Arity}, + maps:put(Trampoline, Actual, Trampolines); + _ -> + Trampolines + end. + +lfo(#b_function{bs=Blocks0}=F, Trampolines) -> + Linear0 = beam_ssa:linearize(Blocks0), + Linear = lfo_optimize(Linear0, lfo_analyze(Linear0, #{}), Trampolines), + F#b_function{bs=maps:from_list(Linear)}. + +%% Gather a map of the locally defined funs that are only used for calls. +lfo_analyze([{_L,#b_blk{is=Is,last=Last}}|Bs], LFuns0) -> + LFuns = lfo_analyze_last(Last, lfo_analyze_is(Is, LFuns0)), + lfo_analyze(Bs, LFuns); +lfo_analyze([], LFuns) -> + LFuns. + +lfo_analyze_is([#b_set{op=make_fun, + dst=Dst, + args=[#b_local{} | FreeVars]}=Def | Is], + LFuns0) -> + LFuns = maps:put(Dst, Def, maps:without(FreeVars, LFuns0)), + lfo_analyze_is(Is, LFuns); +lfo_analyze_is([#b_set{op=call, + args=[Fun | CallArgs]} | Is], + LFuns) when is_map_key(Fun, LFuns) -> + #b_set{args=[#b_local{arity=Arity} | FreeVars]} = maps:get(Fun, LFuns), + case length(CallArgs) + length(FreeVars) of + Arity -> + lfo_analyze_is(Is, maps:without(CallArgs, LFuns)); + _ -> + %% This will `badarity` at runtime, and it's easier to disable the + %% optimization than to simulate it. + lfo_analyze_is(Is, maps:without([Fun | CallArgs], LFuns)) + end; +lfo_analyze_is([#b_set{args=Args} | Is], LFuns) when map_size(LFuns) =/= 0 -> + %% We disqualify funs that are used outside calls because this forces them + %% to be created anyway, and the slight performance gain from direct calls + %% is not enough to offset the potential increase in stack frame size (the + %% free variables need to be kept alive until the call). + %% + %% This is also a kludge to make HiPE work, as the latter will generate + %% code with the assumption that the functions referenced in a make_fun + %% will only be used by funs, which will not be the case if we mix it with + %% direct calls. See cerl_cconv.erl for details. + %% + %% Future optimizations like delaying fun creation until use may require us + %% to copy affected functions so that HiPE gets its own to play with (until + %% HiPE is fixed anyway). + lfo_analyze_is(Is, maps:without(Args, LFuns)); +lfo_analyze_is([_ | Is], LFuns) -> + lfo_analyze_is(Is, LFuns); +lfo_analyze_is([], LFuns) -> + LFuns. + +lfo_analyze_last(#b_switch{arg=Arg}, LFuns) -> + maps:remove(Arg, LFuns); +lfo_analyze_last(#b_ret{arg=Arg}, LFuns) -> + maps:remove(Arg, LFuns); +lfo_analyze_last(_, LFuns) -> + LFuns. + +%% Replace all calls of suitable funs with a direct call to their +%% implementation. Liveness optimization will get rid of the make_fun +%% instruction. +lfo_optimize(Linear, LFuns, _Trampolines) when map_size(LFuns) =:= 0 -> + Linear; +lfo_optimize(Linear, LFuns, Trampolines) -> + lfo_optimize_1(Linear, LFuns, Trampolines). + +lfo_optimize_1([{L,#b_blk{is=Is0}=Blk}|Bs], LFuns, Trampolines) -> + Is = lfo_optimize_is(Is0, LFuns, Trampolines), + [{L,Blk#b_blk{is=Is}} | lfo_optimize_1(Bs, LFuns, Trampolines)]; +lfo_optimize_1([], _LFuns, _Trampolines) -> + []. + +lfo_optimize_is([#b_set{op=call, + args=[Fun | CallArgs]}=Call0 | Is], + LFuns, Trampolines) when is_map_key(Fun, LFuns) -> + #b_set{args=[Local | FreeVars]} = maps:get(Fun, LFuns), + Args = [lfo_short_circuit(Local, Trampolines) | CallArgs ++ FreeVars], + Call = beam_ssa:add_anno(local_fun_opt, Fun, Call0#b_set{args=Args}), + [Call | lfo_optimize_is(Is, LFuns, Trampolines)]; +lfo_optimize_is([I | Is], LFuns, Trampolines) -> + [I | lfo_optimize_is(Is, LFuns, Trampolines)]; +lfo_optimize_is([], _LFuns, _Trampolines) -> + []. + +lfo_short_circuit(Call, Trampolines) -> + case maps:find(Call, Trampolines) of + {ok, Other} -> lfo_short_circuit(Other, Trampolines); + error -> Call + end. diff --git a/lib/compiler/src/beam_ssa_opt.erl b/lib/compiler/src/beam_ssa_opt.erl index 89cfbd7a84..ac2d943fef 100644 --- a/lib/compiler/src/beam_ssa_opt.erl +++ b/lib/compiler/src/beam_ssa_opt.erl @@ -66,13 +66,15 @@ passes(Opts0) -> ?PASS(ssa_opt_live), %Second time. ?PASS(ssa_opt_bsm), + ?PASS(ssa_opt_bsm_units), ?PASS(ssa_opt_bsm_shortcut), ?PASS(ssa_opt_misc), ?PASS(ssa_opt_tuple_size), ?PASS(ssa_opt_sw), ?PASS(ssa_opt_blockify), ?PASS(ssa_opt_sink), - ?PASS(ssa_opt_merge_blocks)], + ?PASS(ssa_opt_merge_blocks), + ?PASS(ssa_opt_trim_unreachable)], Negations = [{list_to_atom("no_"++atom_to_list(N)),N} || {N,_} <- Ps], Opts = proplists:substitute_negations(Negations, Opts0), @@ -112,6 +114,9 @@ ssa_opt_type(#st{ssa=Linear,args=Args}=St) -> ssa_opt_blockify(#st{ssa=Linear}=St) -> St#st{ssa=maps:from_list(Linear)}. +ssa_opt_trim_unreachable(#st{ssa=Blocks}=St) -> + St#st{ssa=beam_ssa:trim_unreachable(Blocks)}. + %%% %%% Split blocks before certain instructions to enable more optimizations. %%% @@ -1002,6 +1007,110 @@ bsm_shortcut([{L,#b_blk{is=Is,last=Last0}=Blk}|Bs], PosMap) -> bsm_shortcut([], _PosMap) -> []. %%% +%%% Eliminate redundant bs_test_unit2 instructions. +%%% + +ssa_opt_bsm_units(#st{ssa=Linear}=St) -> + St#st{ssa=bsm_units(Linear, #{})}. + +bsm_units([{L,#b_blk{last=#b_br{succ=Succ,fail=Fail}}=Block0} | Bs], UnitMaps0) -> + UnitsIn = maps:get(L, UnitMaps0, #{}), + {Block, UnitsOut} = bsm_units_skip(Block0, UnitsIn), + UnitMaps1 = bsm_units_join(Succ, UnitsOut, UnitMaps0), + UnitMaps = bsm_units_join(Fail, UnitsIn, UnitMaps1), + [{L, Block} | bsm_units(Bs, UnitMaps)]; +bsm_units([{L,#b_blk{last=#b_switch{fail=Fail,list=Switch}}=Block} | Bs], UnitMaps0) -> + UnitsIn = maps:get(L, UnitMaps0, #{}), + Labels = [Fail | [Lbl || {_Arg, Lbl} <- Switch]], + UnitMaps = foldl(fun(Lbl, UnitMaps) -> + bsm_units_join(Lbl, UnitsIn, UnitMaps) + end, UnitMaps0, Labels), + [{L, Block} | bsm_units(Bs, UnitMaps)]; +bsm_units([{L, Block} | Bs], UnitMaps) -> + [{L, Block} | bsm_units(Bs, UnitMaps)]; +bsm_units([], _UnitMaps) -> + []. + +bsm_units_skip(Block, Units) -> + bsm_units_skip_1(Block#b_blk.is, Block, Units). + +bsm_units_skip_1([#b_set{op=bs_start_match,dst=New}|_], Block, Units) -> + %% We bail early since there can't be more than one match per block. + {Block, Units#{ New => 1 }}; +bsm_units_skip_1([#b_set{op=bs_match, + dst=New, + args=[#b_literal{val=skip}, + Ctx, + #b_literal{val=binary}, + _Flags, + #b_literal{val=all}, + #b_literal{val=OpUnit}]}=Skip | Test], + Block0, Units) -> + [#b_set{op=succeeded,dst=Bool,args=[New]}] = Test, %Assertion. + #b_br{bool=Bool} = Last0 = Block0#b_blk.last, %Assertion. + CtxUnit = maps:get(Ctx, Units), + if + CtxUnit rem OpUnit =:= 0 -> + Is = takewhile(fun(I) -> I =/= Skip end, Block0#b_blk.is), + Last = Last0#b_br{bool=#b_literal{val=true}}, + Block = Block0#b_blk{is=Is,last=Last}, + {Block, Units#{ New => CtxUnit }}; + CtxUnit rem OpUnit =/= 0 -> + {Block0, Units#{ New => OpUnit, Ctx => OpUnit }} + end; +bsm_units_skip_1([#b_set{op=bs_match,dst=New,args=Args}|_], Block, Units) -> + [_,Ctx|_] = Args, + CtxUnit = maps:get(Ctx, Units), + OpUnit = bsm_op_unit(Args), + {Block, Units#{ New => gcd(OpUnit, CtxUnit) }}; +bsm_units_skip_1([_I | Is], Block, Units) -> + bsm_units_skip_1(Is, Block, Units); +bsm_units_skip_1([], Block, Units) -> + {Block, Units}. + +bsm_op_unit([_,_,_,Size,#b_literal{val=U}]) -> + case Size of + #b_literal{val=Sz} when is_integer(Sz) -> Sz*U; + _ -> U + end; +bsm_op_unit([#b_literal{val=string},_,#b_literal{val=String}]) -> + bit_size(String); +bsm_op_unit([#b_literal{val=utf8}|_]) -> + 8; +bsm_op_unit([#b_literal{val=utf16}|_]) -> + 16; +bsm_op_unit([#b_literal{val=utf32}|_]) -> + 32; +bsm_op_unit(_) -> + 1. + +%% Several paths can lead to the same match instruction and the inferred units +%% may differ between them, so we can only keep the information that is common +%% to all paths. +bsm_units_join(Lbl, MapA, UnitMaps0) when is_map_key(Lbl, UnitMaps0) -> + MapB = maps:get(Lbl, UnitMaps0), + Merged = if + map_size(MapB) =< map_size(MapA) -> + bsm_units_join_1(maps:keys(MapB), MapA, MapB); + map_size(MapB) > map_size(MapA) -> + bsm_units_join_1(maps:keys(MapA), MapB, MapA) + end, + maps:put(Lbl, Merged, UnitMaps0); +bsm_units_join(Lbl, MapA, UnitMaps0) when MapA =/= #{} -> + maps:put(Lbl, MapA, UnitMaps0); +bsm_units_join(_Lbl, _MapA, UnitMaps0) -> + UnitMaps0. + +bsm_units_join_1([Key | Keys], Left, Right) when is_map_key(Key, Left) -> + UnitA = maps:get(Key, Left), + UnitB = maps:get(Key, Right), + bsm_units_join_1(Keys, Left, maps:put(Key, gcd(UnitA, UnitB), Right)); +bsm_units_join_1([Key | Keys], Left, Right) -> + bsm_units_join_1(Keys, Left, maps:remove(Key, Right)); +bsm_units_join_1([], _MapA, Right) -> + Right. + +%%% %%% Miscellanous optimizations in execution order. %%% @@ -1618,6 +1727,12 @@ insert_def_is([], _V, Def) -> %%% Common utilities. %%% +gcd(A, B) -> + case A rem B of + 0 -> B; + X -> gcd(B, X) + end. + rel2fam(S0) -> S1 = sofs:relation(S0), S = sofs:rel2fam(S1), diff --git a/lib/compiler/src/beam_ssa_pre_codegen.erl b/lib/compiler/src/beam_ssa_pre_codegen.erl index ca3b792ed6..9175931375 100644 --- a/lib/compiler/src/beam_ssa_pre_codegen.erl +++ b/lib/compiler/src/beam_ssa_pre_codegen.erl @@ -78,15 +78,14 @@ {'ok',beam_ssa:b_module()}. module(#b_module{body=Fs0}=Module, Opts) -> - FixTuples = proplists:get_bool(no_put_tuple2, Opts), - ExtraAnnos = proplists:get_bool(dprecg, Opts), - Ps = passes(FixTuples, ExtraAnnos), - Fs = functions(Fs0, Ps), + UseBSM3 = not proplists:get_bool(no_bsm3, Opts), + Ps = passes(Opts), + Fs = functions(Fs0, Ps, UseBSM3), {ok,Module#b_module{body=Fs}}. -functions([F|Fs], Ps) -> - [function(F, Ps)|functions(Fs, Ps)]; -functions([], _Ps) -> []. +functions([F|Fs], Ps, UseBSM3) -> + [function(F, Ps, UseBSM3)|functions(Fs, Ps, UseBSM3)]; +functions([], _Ps, _UseBSM3) -> []. -type b_var() :: beam_ssa:b_var(). -type var_name() :: beam_ssa:var_name(). @@ -104,16 +103,18 @@ functions([], _Ps) -> []. -record(st, {ssa :: beam_ssa:block_map(), args :: [b_var()], cnt :: beam_ssa:label(), + use_bsm3 :: boolean(), frames=[] :: [beam_ssa:label()], intervals=[] :: [{b_var(),[range()]}], - aliases=[] :: [{b_var(),b_var()}], res=[] :: [{b_var(),reservation()}] | #{b_var():=reservation()}, regs=#{} :: #{b_var():=ssa_register()}, extra_annos=[] :: [{atom(),term()}] }). -define(PASS(N), {N,fun N/1}). -passes(FixTuples, ExtraAnnos) -> +passes(Opts) -> + AddPrecgAnnos = proplists:get_bool(dprecg, Opts), + FixTuples = proplists:get_bool(no_put_tuple2, Opts), Ps = [?PASS(assert_no_critical_edges), %% Preliminaries. @@ -130,6 +131,10 @@ passes(FixTuples, ExtraAnnos) -> ?PASS(find_yregs), ?PASS(reserve_yregs), + %% Handle legacy binary match instruction that don't + %% accept a Y register as destination. + ?PASS(legacy_bs), + %% Improve reuse of Y registers to potentially %% reduce the size of the stack frame. ?PASS(copy_retval), @@ -138,27 +143,25 @@ passes(FixTuples, ExtraAnnos) -> %% Calculate live intervals. ?PASS(number_instructions), ?PASS(live_intervals), - ?PASS(remove_unsuitable_aliases), ?PASS(reserve_regs), - ?PASS(merge_intervals), %% If needed for a .precg file, save the live intervals %% so they can be included in an annotation. - case ExtraAnnos of + case AddPrecgAnnos of false -> ignore; true -> ?PASS(save_live_intervals) end, %% Allocate registers. ?PASS(linear_scan), - ?PASS(fix_aliased_regs), ?PASS(frame_size), ?PASS(turn_yregs)], [P || P <- Ps, P =/= ignore]. -function(#b_function{anno=Anno,args=Args,bs=Blocks0,cnt=Count0}=F0, Ps) -> +function(#b_function{anno=Anno,args=Args,bs=Blocks0,cnt=Count0}=F0, + Ps, UseBSM3) -> try - St0 = #st{ssa=Blocks0,args=Args,cnt=Count0}, + St0 = #st{ssa=Blocks0,args=Args,use_bsm3=UseBSM3,cnt=Count0}, St = compile:run_sub_passes(Ps, St0), #st{ssa=Blocks,cnt=Count,regs=Regs,extra_annos=ExtraAnnos} = St, F1 = add_extra_annos(F0, ExtraAnnos), @@ -174,14 +177,6 @@ function(#b_function{anno=Anno,args=Args,bs=Blocks0,cnt=Count0}=F0, Ps) -> save_live_intervals(#st{intervals=Intervals}=St) -> St#st{extra_annos=[{live_intervals,Intervals}]}. -fix_aliased_regs(#st{aliases=Aliases,regs=Regs}=St) -> - St#st{regs=fix_aliased_regs(Aliases, Regs)}. - -fix_aliased_regs([{Alias,V}|Aliases], Regs) -> - #{V:=Reg} = Regs, - fix_aliased_regs(Aliases, Regs#{Alias=>Reg}); -fix_aliased_regs([], Regs) -> Regs. - %% Add extra annotations when a .precg listing file is being produced. add_extra_annos(F, Annos) -> foldl(fun({Name,Value}, Acc) -> @@ -214,7 +209,7 @@ assert_no_ces(_, _, Blocks) -> Blocks. %% * Combine bs_match and bs_extract instructions to bs_get %% instructions. -fix_bs(#st{ssa=Blocks,cnt=Count0}=St) -> +fix_bs(#st{ssa=Blocks,cnt=Count0,use_bsm3=UseBSM3}=St) -> F = fun(#b_set{op=bs_start_match,dst=Dst}, A) -> %% Mark the root of the match context list. [{Dst,{context,Dst}}|A]; @@ -232,8 +227,13 @@ fix_bs(#st{ssa=Blocks,cnt=Count0}=St) -> CtxChain = maps:from_list(M), Linear0 = beam_ssa:linearize(Blocks), - %% Insert bs_save / bs_restore instructions where needed. - {Linear1,Count} = bs_save_restore(Linear0, CtxChain, Count0), + %% Insert position instructions where needed. + {Linear1,Count} = case UseBSM3 of + true -> + bs_pos_bsm3(Linear0, CtxChain, Count0); + false -> + bs_pos_bsm2(Linear0, CtxChain, Count0) + end, %% Rename instructions. Linear = bs_instrs(Linear1, CtxChain, []), @@ -241,10 +241,54 @@ fix_bs(#st{ssa=Blocks,cnt=Count0}=St) -> St#st{ssa=maps:from_list(Linear),cnt=Count} end. +%% Insert bs_get_position and bs_set_position instructions as needed. +bs_pos_bsm3(Linear0, CtxChain, Count0) -> + Rs0 = bs_restores(Linear0, CtxChain, #{}, #{}), + Rs = maps:values(Rs0), + S0 = sofs:relation(Rs, [{context,save_point}]), + S1 = sofs:relation_to_family(S0), + S = sofs:to_external(S1), -%% Insert bs_save and bs_restore instructions as needed. + {SavePoints,Count1} = make_bs_pos_dict(S, Count0, []), + {Gets,Count2} = make_bs_setpos_map(Rs, SavePoints, Count1, []), + {Sets,Count} = make_bs_getpos_map(maps:to_list(Rs0), SavePoints, Count2, []), + + %% Now insert all saves and restores. + {bs_insert_bsm3(Linear0, Gets, Sets, SavePoints),Count}. -bs_save_restore(Linear0, CtxChain, Count0) -> +make_bs_setpos_map([{Ctx,Save}=Ps|T], SavePoints, Count, Acc) -> + SavePoint = get_savepoint(Ps, SavePoints), + I = #b_set{op=bs_get_position,dst=SavePoint,args=[Ctx]}, + make_bs_setpos_map(T, SavePoints, Count+1, [{Save,I}|Acc]); +make_bs_setpos_map([], _, Count, Acc) -> + {maps:from_list(Acc),Count}. + +make_bs_getpos_map([{Bef,{Ctx,_}=Ps}|T], SavePoints, Count, Acc) -> + Ignored = #b_var{name={'@ssa_ignored',Count}}, + Args = [Ctx, get_savepoint(Ps, SavePoints)], + I = #b_set{op=bs_set_position,dst=Ignored,args=Args}, + make_bs_getpos_map(T, SavePoints, Count+1, [{Bef,I}|Acc]); +make_bs_getpos_map([], _, Count, Acc) -> + {maps:from_list(Acc),Count}. + +get_savepoint({_,_}=Ps, SavePoints) -> + Name = {'@ssa_bs_position', maps:get(Ps, SavePoints)}, + #b_var{name=Name}. + +make_bs_pos_dict([{Ctx,Pts}|T], Count0, Acc0) -> + {Acc, Count} = make_bs_pos_dict_1(Pts, Ctx, Count0, Acc0), + make_bs_pos_dict(T, Count, Acc); +make_bs_pos_dict([], Count, Acc) -> + {maps:from_list(Acc), Count}. + +make_bs_pos_dict_1([H|T], Ctx, I, Acc) -> + make_bs_pos_dict_1(T, Ctx, I+1, [{{Ctx,H},I}|Acc]); +make_bs_pos_dict_1([], Ctx, I, Acc) -> + {[{Ctx,I}|Acc], I}. + +%% As bs_position but without OTP-22 instructions. This is only used when +%% cross-compiling to older versions. +bs_pos_bsm2(Linear0, CtxChain, Count0) -> Rs0 = bs_restores(Linear0, CtxChain, #{}, #{}), Rs = maps:values(Rs0), S0 = sofs:relation(Rs, [{context,save_point}]), @@ -255,7 +299,7 @@ bs_save_restore(Linear0, CtxChain, Count0) -> {Restores,Count} = make_restore_map(maps:to_list(Rs0), Slots, Count1, []), %% Now insert all saves and restores. - {bs_insert(Linear0, Saves, Restores, Slots),Count}. + {bs_insert_bsm2(Linear0, Saves, Restores, Slots),Count}. make_save_map([{Ctx,Save}=Ps|T], Slots, Count, Acc) -> Ignored = #b_var{name={'@ssa_ignored',Count}}, @@ -308,8 +352,7 @@ bs_restores([], _, _, Rs) -> Rs. bs_update_successors(#b_br{succ=Succ,fail=Fail}, SPos, FPos, D) -> join_positions([{Succ,SPos},{Fail,FPos}], D); -bs_update_successors(#b_switch{fail=Fail,list=List}, SPos, FPos, D) -> - SPos = FPos, %Assertion. +bs_update_successors(#b_switch{fail=Fail,list=List}, SPos, _FPos, D) -> Update = [{L,SPos} || {_,L} <- List] ++ [{Fail,SPos}], join_positions(Update, D); bs_update_successors(#b_ret{}, _, _, D) -> D. @@ -388,10 +431,19 @@ bs_restores_is([#b_set{op=bs_extract,args=[FromPos|_]}|Is], Start = bs_subst_ctx(FromPos, CtxChain), #{Start:=FromPos} = PosMap, %Assertion. bs_restores_is(Is, CtxChain, PosMap, Rs); +bs_restores_is([#b_set{op=call,dst=Dst,args=Args}|Is], + CtxChain, PosMap0, Rs0) -> + {Rs,PosMap1} = bs_restore_args(Args, PosMap0, CtxChain, Dst, Rs0), + PosMap = bs_invalidate_pos(Args, PosMap1, CtxChain), + bs_restores_is(Is, CtxChain, PosMap, Rs); +bs_restores_is([#b_set{op=landingpad}|Is], CtxChain, PosMap0, Rs) -> + %% We can land here from any point, so all positions are invalid. + PosMap = maps:map(fun(_Start,_Pos) -> unknown end, PosMap0), + bs_restores_is(Is, CtxChain, PosMap, Rs); bs_restores_is([#b_set{op=Op,dst=Dst,args=Args}|Is], CtxChain, PosMap0, Rs0) when Op =:= bs_test_tail; - Op =:= call -> + Op =:= bs_get_tail -> {Rs,PosMap} = bs_restore_args(Args, PosMap0, CtxChain, Dst, Rs0), bs_restores_is(Is, CtxChain, PosMap, Rs); bs_restores_is([_|Is], CtxChain, PosMap, Rs) -> @@ -399,7 +451,6 @@ bs_restores_is([_|Is], CtxChain, PosMap, Rs) -> bs_restores_is([], _CtxChain, PosMap, Rs) -> {PosMap,Rs}. - bs_match_type(#b_set{args=[#b_literal{val=skip},_Ctx, #b_literal{val=binary},_Flags, #b_literal{val=all},#b_literal{val=U}]}) -> @@ -410,6 +461,23 @@ bs_match_type(#b_set{args=[#b_literal{val=skip},_Ctx, bs_match_type(_) -> plain. +%% Call instructions leave the match position in an undefined state, +%% requiring us to invalidate each affected argument. +bs_invalidate_pos([#b_var{}=Arg|Args], PosMap0, CtxChain) -> + Start = bs_subst_ctx(Arg, CtxChain), + case PosMap0 of + #{Start:=_} -> + PosMap = PosMap0#{Start:=unknown}, + bs_invalidate_pos(Args, PosMap, CtxChain); + #{} -> + %% Not a match context. + bs_invalidate_pos(Args, PosMap0, CtxChain) + end; +bs_invalidate_pos([_|Args], PosMap, CtxChain) -> + bs_invalidate_pos(Args, PosMap, CtxChain); +bs_invalidate_pos([], PosMap, _CtxChain) -> + PosMap. + bs_restore_args([#b_var{}=Arg|Args], PosMap0, CtxChain, Dst, Rs0) -> Start = bs_subst_ctx(Arg, CtxChain), case PosMap0 of @@ -432,33 +500,45 @@ bs_restore_args([], PosMap, _CtxChain, _Dst, Rs) -> %% Insert all bs_save and bs_restore instructions. -bs_insert([{L,#b_blk{is=Is0}=Blk}|Bs0], Saves, Restores, Slots) -> - Is = bs_insert_is_1(Is0, Restores, Slots), +bs_insert_bsm3(Blocks, Saves, Restores, SavePoints) -> + bs_insert_1(Blocks, Saves, Restores, SavePoints, fun(I) -> I end). + +bs_insert_bsm2(Blocks, Saves, Restores, SavePoints) -> + %% The old instructions require bs_start_match to be annotated with the + %% number of position slots it needs. + bs_insert_1(Blocks, Saves, Restores, SavePoints, + fun(#b_set{op=bs_start_match,dst=Dst}=I0) -> + NumSlots = case SavePoints of + #{Dst:=NumSlots0} -> NumSlots0; + #{} -> 0 + end, + beam_ssa:add_anno(num_slots, NumSlots, I0); + (I) -> + I + end). + +bs_insert_1([{L,#b_blk{is=Is0}=Blk}|Bs0], Saves, Restores, Slots, XFrm) -> + Is = bs_insert_is_1(Is0, Restores, Slots, XFrm), Bs = bs_insert_saves(Is, Bs0, Saves), - [{L,Blk#b_blk{is=Is}}|bs_insert(Bs, Saves, Restores, Slots)]; -bs_insert([], _, _, _) -> []. + [{L,Blk#b_blk{is=Is}}|bs_insert_1(Bs, Saves, Restores, Slots, XFrm)]; +bs_insert_1([], _, _, _, _) -> []. -bs_insert_is_1([#b_set{op=Op,dst=Dst}=I0|Is], Restores, Slots) -> +bs_insert_is_1([#b_set{op=Op,dst=Dst}=I0|Is], Restores, SavePoints, XFrm) -> + I = XFrm(I0), if Op =:= bs_test_tail; + Op =:= bs_get_tail; Op =:= bs_match; Op =:= call -> Rs = case Restores of #{Dst:=R} -> [R]; #{} -> [] end, - Rs ++ [I0|bs_insert_is_1(Is, Restores, Slots)]; - Op =:= bs_start_match -> - NumSlots = case Slots of - #{Dst:=NumSlots0} -> NumSlots0; - #{} -> 0 - end, - I = beam_ssa:add_anno(num_slots, NumSlots, I0), - [I|bs_insert_is_1(Is, Restores, Slots)]; + Rs ++ [I|bs_insert_is_1(Is, Restores, SavePoints, XFrm)]; true -> - [I0|bs_insert_is_1(Is, Restores, Slots)] + [I|bs_insert_is_1(Is, Restores, SavePoints, XFrm)] end; -bs_insert_is_1([], _, _) -> []. +bs_insert_is_1([], _, _, _) -> []. bs_insert_saves([#b_set{dst=Dst}|Is], Bs, Saves) -> case Saves of @@ -504,6 +584,8 @@ bs_instrs_is([#b_set{op=Op,args=Args0}=I0|Is], CtxChain, Acc) -> I1#b_set{op=bs_skip,args=[Type,Ctx|As]}; {bs_match,[#b_literal{val=string},Ctx|As]} -> I1#b_set{op=bs_match_string,args=[Ctx|As]}; + {bs_get_tail,[Ctx|As]} -> + I1#b_set{op=bs_get_tail,args=[Ctx|As]}; {_,_} -> I1 end, @@ -534,6 +616,59 @@ bs_subst_ctx(#b_var{}=Var, CtxChain) -> bs_subst_ctx(Other, _CtxChain) -> Other. +%% legacy_bs(St0) -> St. +%% Binary matching instructions in OTP 21 and earlier don't support +%% a Y register as destination. If St#st.use_bsm3 is false, +%% we will need to rewrite those instructions so that the result +%% is first put in an X register and then moved to a Y register +%% if the operation succeeded. + +legacy_bs(#st{use_bsm3=false,ssa=Blocks0,cnt=Count0,res=Res}=St) -> + IsYreg = maps:from_list([{V,true} || {V,{y,_}} <- Res]), + Linear0 = beam_ssa:linearize(Blocks0), + {Linear,Count} = legacy_bs(Linear0, IsYreg, Count0, #{}, []), + Blocks = maps:from_list(Linear), + St#st{ssa=Blocks,cnt=Count}; +legacy_bs(#st{use_bsm3=true}=St) -> St. + +legacy_bs([{L,Blk}|Bs], IsYreg, Count0, Copies0, Acc) -> + #b_blk{is=Is0,last=Last} = Blk, + Is1 = case Copies0 of + #{L:=Copy} -> [Copy|Is0]; + #{} -> Is0 + end, + {Is,Count,Copies} = legacy_bs_is(Is1, Last, IsYreg, Count0, Copies0, []), + legacy_bs(Bs, IsYreg, Count, Copies, [{L,Blk#b_blk{is=Is}}|Acc]); +legacy_bs([], _IsYreg, Count, _Copies, Acc) -> + {Acc,Count}. + +legacy_bs_is([#b_set{op=Op,dst=Dst}=I0, + #b_set{op=succeeded,dst=SuccDst,args=[Dst]}=SuccI0], + Last, IsYreg, Count0, Copies0, Acc) -> + NeedsFix = is_map_key(Dst, IsYreg) andalso + case Op of + bs_get -> true; + bs_init -> true; + _ -> false + end, + case NeedsFix of + true -> + TempDst = #b_var{name={'@bs_temp_dst',Count0}}, + Count = Count0 + 1, + I = I0#b_set{dst=TempDst}, + SuccI = SuccI0#b_set{args=[TempDst]}, + Copy = #b_set{op=copy,dst=Dst,args=[TempDst]}, + #b_br{bool=SuccDst,succ=SuccL} = Last, + Copies = Copies0#{SuccL=>Copy}, + legacy_bs_is([], Last, IsYreg, Count, Copies, [SuccI,I|Acc]); + false -> + legacy_bs_is([], Last, IsYreg, Count0, Copies0, [SuccI0,I0|Acc]) + end; +legacy_bs_is([I|Is], Last, IsYreg, Count, Copies, Acc) -> + legacy_bs_is(Is, Last, IsYreg, Count, Copies, [I|Acc]); +legacy_bs_is([], _Last, _IsYreg, Count, Copies, Acc) -> + {reverse(Acc),Count,Copies}. + %% sanitize(St0) -> St. %% Remove constructs that can cause problems later: %% @@ -574,21 +709,24 @@ sanitize([], Count, Blocks0, Values) -> false -> remove_unreachable(Ls, Blocks, Reachable, []) end,Count}. -sanitize_is([#b_set{op=get_map_element, - args=[#b_literal{}=Map,Key]}=I0|Is], - Count0, Values, _Changed, Acc) -> - {MapVar,Count} = new_var('@ssa_map', Count0), - I = I0#b_set{args=[MapVar,Key]}, - Copy = #b_set{op=copy,dst=MapVar,args=[Map]}, - sanitize_is(Is, Count, Values, true, [I,Copy|Acc]); +sanitize_is([#b_set{op=get_map_element,args=Args0}=I0|Is], + Count0, Values, Changed, Acc) -> + case sanitize_args(Args0, Values) of + [#b_literal{}=Map,Key] -> + %% Bind the literal map to a variable. + {MapVar,Count} = new_var('@ssa_map', Count0), + I = I0#b_set{args=[MapVar,Key]}, + Copy = #b_set{op=copy,dst=MapVar,args=[Map]}, + sanitize_is(Is, Count, Values, true, [I,Copy|Acc]); + [_,_]=Args0 -> + sanitize_is(Is, Count0, Values, Changed, [I0|Acc]); + [_,_]=Args -> + I = I0#b_set{args=Args}, + sanitize_is(Is, Count0, Values, Changed, [I|Acc]) + end; sanitize_is([#b_set{op=Op,dst=Dst,args=Args0}=I0|Is0], - Count, Values, Changed, Acc) -> - Args = map(fun(Var) -> - case Values of - #{Var:=New} -> New; - #{} -> Var - end - end, Args0), + Count, Values, Changed0, Acc) -> + Args = sanitize_args(Args0, Values), case sanitize_instr(Op, Args, I0) of {value,Value0} -> Value = #b_literal{val=Value0}, @@ -596,7 +734,9 @@ sanitize_is([#b_set{op=Op,dst=Dst,args=Args0}=I0|Is0], {ok,I} -> sanitize_is(Is0, Count, Values, true, [I|Acc]); ok -> - sanitize_is(Is0, Count, Values, Changed, [I0|Acc]) + I = I0#b_set{args=Args}, + Changed = Changed0 orelse Args =/= Args0, + sanitize_is(Is0, Count, Values, Changed, [I|Acc]) end; sanitize_is([], Count, Values, Changed, Acc) -> case Changed of @@ -606,6 +746,14 @@ sanitize_is([], Count, Values, Changed, Acc) -> no_change end. +sanitize_args(Args, Values) -> + map(fun(Var) -> + case Values of + #{Var:=New} -> New; + #{} -> Var + end + end, Args). + sanitize_instr({bif,Bif}, [#b_literal{val=Lit}], _I) -> case erl_bifs:is_pure(erlang, Bif, 1) of false -> @@ -895,11 +1043,11 @@ need_frame_1([#b_set{op=call,args=[Func|_]}|Is], Context) -> #b_remote{} -> %% This is an apply(), which always needs a frame. true; - #b_var{} -> - %% A fun call always needs a frame. - true; + #b_local{} -> + Context =:= body orelse Is =/= []; _ -> - Context =:= body orelse Is =/= [] + %% A fun call always needs a frame. + true end; need_frame_1([I|Is], Context) -> beam_ssa:clobbers_xregs(I) orelse need_frame_1(Is, Context); @@ -1520,10 +1668,10 @@ live_intervals(#st{args=Args,ssa=Blocks}=St) -> Vars0 = [{V,{0,1}} || #b_var{}=V <- Args], F = fun(L, _, A) -> live_interval_blk(L, Blocks, A) end, LiveMap0 = #{}, - Acc0 = {[],[],LiveMap0}, - {Vars,Aliases,_} = beam_ssa:fold_po(F, Acc0, Blocks), + Acc0 = {[],LiveMap0}, + {Vars,_} = beam_ssa:fold_po(F, Acc0, Blocks), Intervals = merge_ranges(rel2fam(Vars0++Vars)), - St#st{intervals=Intervals,aliases=Aliases}. + St#st{intervals=Intervals}. merge_ranges([{V,Rs}|T]) -> [{V,merge_ranges_1(Rs)}|merge_ranges(T)]; @@ -1535,7 +1683,7 @@ merge_ranges_1([R|Rs]) -> [R|merge_ranges_1(Rs)]; merge_ranges_1([]) -> []. -live_interval_blk(L, Blocks, {Vars0,Aliases0,LiveMap0}) -> +live_interval_blk(L, Blocks, {Vars0,LiveMap0}) -> Live0 = [], Successors = beam_ssa:successors(L, Blocks), Live1 = update_successors(Successors, L, Blocks, LiveMap0, Live0), @@ -1547,8 +1695,7 @@ live_interval_blk(L, Blocks, {Vars0,Aliases0,LiveMap0}) -> %% Determine used and defined variables in this block. FirstNumber = first_number(Is, Last), - {UseDef0,Aliases} = live_interval_blk_1([Last|reverse(Is)], - FirstNumber, Aliases0, Use), + UseDef0 = live_interval_blk_1([Last|reverse(Is)], FirstNumber, Use), UseDef = rel2fam(UseDef0), %% Update what is live at the beginning of this block and @@ -1561,7 +1708,7 @@ live_interval_blk(L, Blocks, {Vars0,Aliases0,LiveMap0}) -> %% Construct the ranges for this block. Vars = make_block_ranges(UseDef, FirstNumber, Vars0), - {Vars,Aliases,LiveMap}. + {Vars,LiveMap}. make_block_ranges([{V,[{def,Def}]}|Vs], First, Acc) -> make_block_ranges(Vs, First, [{V,{Def,Def}}|Acc]); @@ -1573,25 +1720,17 @@ make_block_ranges([{V,[{use,_}|_]=Uses}|Vs], First, Acc) -> make_block_ranges(Vs, First, [{V,{First,Last}}|Acc]); make_block_ranges([], _, Acc) -> Acc. -live_interval_blk_1([#b_set{op=phi,dst=Dst}|Is], - FirstNumber, Aliases, Acc0) -> +live_interval_blk_1([#b_set{op=phi,dst=Dst}|Is], FirstNumber, Acc0) -> Acc = [{Dst,{def,FirstNumber}}|Acc0], - live_interval_blk_1(Is, FirstNumber, Aliases, Acc); -live_interval_blk_1([#b_set{op=bs_start_match}=I|Is], FirstNumber, - Aliases0, Acc0) -> + live_interval_blk_1(Is, FirstNumber, Acc); +live_interval_blk_1([#b_set{op=bs_start_match}=I|Is], + FirstNumber, Acc0) -> N = beam_ssa:get_anno(n, I), #b_set{dst=Dst} = I, Acc1 = [{Dst,{def,N}}|Acc0], - Aliases = case beam_ssa:get_anno(reuse_for_context, I) of - true -> - #b_set{args=[Src]} = I, - [{Dst,Src}|Aliases0]; - false -> - Aliases0 - end, Acc = [{V,{use,N}} || V <- beam_ssa:used(I)] ++ Acc1, - live_interval_blk_1(Is, FirstNumber, Aliases, Acc); -live_interval_blk_1([I|Is], FirstNumber, Aliases, Acc0) -> + live_interval_blk_1(Is, FirstNumber, Acc); +live_interval_blk_1([I|Is], FirstNumber, Acc0) -> N = beam_ssa:get_anno(n, I), Acc1 = case I of #b_set{dst=Dst} -> @@ -1601,9 +1740,9 @@ live_interval_blk_1([I|Is], FirstNumber, Aliases, Acc0) -> end, Used = beam_ssa:used(I), Acc = [{V,{use,N}} || V <- Used] ++ Acc1, - live_interval_blk_1(Is, FirstNumber, Aliases, Acc); -live_interval_blk_1([], _FirstNumber, Aliases, Acc) -> - {Acc,Aliases}. + live_interval_blk_1(Is, FirstNumber, Acc); +live_interval_blk_1([], _FirstNumber, Acc) -> + Acc. %% first_number([#b_set{}]) -> InstructionNumber. %% Return the number for the first instruction for the block. @@ -1865,10 +2004,10 @@ reserve_zreg([#b_set{op={bif,tuple_size},dst=Dst}], reserve_zreg_1(Dst, ShortLived, A); reserve_zreg([#b_set{op=Op,dst=Dst}|Is], Last, ShortLived, A0) -> IsZReg = case Op of - context_to_binary -> true; bs_match_string -> true; - bs_restore -> true; bs_save -> true; + bs_restore -> true; + bs_set_position -> true; {float,clearerror} -> true; kill_try_tag -> true; landingpad -> true; @@ -2040,82 +2179,6 @@ res_xregs_prune(Xs, Used, Res) -> maps:filter(fun(_, {x,X}) -> X < NumSafe end, Xs). %%% -%%% Remove unsuitable aliases. -%%% -%%% If a binary is matched more than once, we must not put the -%%% the match context in the same register as the binary to -%%% avoid the following situation: -%%% -%%% {test,bs_start_match2,{f,3},1,[{x,0},0],{x,0}}. -%%% . -%%% . -%%% . -%%% {test,bs_start_match2,{f,6},1,[{x,0},0],{x,1}}. %% ILLEGAL! -%%% -%%% The second instruction is illegal because a match context source -%%% is only allowed if source and destination registers are identical. -%%% - -remove_unsuitable_aliases(#st{aliases=[_|_]=Aliases0,ssa=Blocks}=St) -> - R = rem_unsuitable(maps:values(Blocks)), - Unsuitable0 = [V || {V,[_,_|_]} <- rel2fam(R)], - Unsuitable = gb_sets:from_list(Unsuitable0), - Aliases =[P || {_,V}=P <- Aliases0, - not gb_sets:is_member(V, Unsuitable)], - St#st{aliases=Aliases}; -remove_unsuitable_aliases(#st{aliases=[]}=St) -> St. - -rem_unsuitable([#b_blk{is=Is}|Bs]) -> - Vs = [{Var,Dst} || - #b_set{op=bs_start_match,dst=Dst, - args=[#b_var{}=Var]} <- Is], - Vs ++ rem_unsuitable(Bs); -rem_unsuitable([]) -> []. - -%%% -%%% Merge intervals. -%%% - -merge_intervals(#st{aliases=Aliases0,intervals=Intervals0, - res=Reserved}=St) -> - Aliases1 = [A || A <- Aliases0, - is_suitable_alias(A, Reserved)], - case Aliases1 of - [] -> - St#st{aliases=Aliases1}; - [_|_] -> - Intervals1 = maps:from_list(Intervals0), - {Intervals,Aliases} = - merge_intervals_1(Aliases1, Intervals1, []), - St#st{aliases=Aliases,intervals=Intervals} - end. - -merge_intervals_1([{Alias,V}|Vs], Intervals0, Acc) -> - #{Alias:=Int1,V:=Int2} = Intervals0, - Int3 = lists:merge(Int1, Int2), - Int = merge_intervals_2(Int3), - Intervals1 = maps:remove(Alias, Intervals0), - Intervals = Intervals1#{V:=Int}, - merge_intervals_1(Vs, Intervals, [{Alias,V}|Acc]); -merge_intervals_1([], Intervals, Acc) -> - {maps:to_list(Intervals),Acc}. - -merge_intervals_2([{A1,B1},{A2,B2}|Is]) when A2 =< B1 -> - merge_intervals_2([{min(A1, A2),max(B1, B2)}|Is]); -merge_intervals_2([{_A1,B1}=R|[{A2,_B2}|_]=Is]) when B1 < A2 -> - [R|merge_intervals_2(Is)]; -merge_intervals_2([_]=Is) -> Is. - -is_suitable_alias({V1,V2}, Reserved) -> - #{V1:=Res1,V2:=Res2} = Reserved, - case {Res1,Res2} of - {x,x} -> true; - {x,{x,_}} -> true; - {{x,_},x} -> true; - {_,_} -> false - end. - -%%% %%% Register allocation using linear scan. %%% diff --git a/lib/compiler/src/beam_ssa_type.erl b/lib/compiler/src/beam_ssa_type.erl index 5ccc9c371b..18e6e73a46 100644 --- a/lib/compiler/src/beam_ssa_type.erl +++ b/lib/compiler/src/beam_ssa_type.erl @@ -486,6 +486,8 @@ type(bs_extract, [Ctx], Ts, _Ds) -> Type; type(bs_match, Args, _Ts, _Ds) -> #t_bs_match{type=bs_match_type(Args)}; +type(bs_get_tail, _Args, _Ts, _Ds) -> + {binary, 1}; type(call, [#b_remote{mod=#b_literal{val=Mod}, name=#b_literal{val=Name}}|Args], Ts, _Ds) -> case {Mod,Name,Args} of diff --git a/lib/compiler/src/beam_utils.erl b/lib/compiler/src/beam_utils.erl index 686d314c2d..5156a04f6b 100644 --- a/lib/compiler/src/beam_utils.erl +++ b/lib/compiler/src/beam_utils.erl @@ -23,14 +23,12 @@ -module(beam_utils). -export([is_killed/3,is_killed_at/3,is_not_used/3, empty_label_index/0,index_label/3,index_labels/1,replace_labels/4, - code_at/2,bif_to_test/3,is_pure_test/1, - combine_heap_needs/2, - split_even/1 - ]). + code_at/2,is_pure_test/1, + split_even/1]). -export_type([code_index/0,module_code/0,instruction/0]). --import(lists, [flatmap/2,map/2,member/2,sort/1,reverse/1]). +-import(lists, [map/2,member/2,sort/1,reverse/1]). -define(is_const(Val), (Val =:= nil orelse element(1, Val) =:= integer orelse @@ -38,7 +36,7 @@ element(1, Val) =:= atom orelse element(1, Val) =:= literal)). -%% instruction() describes all instructions that are used during optimzation +%% instruction() describes all instructions that are used during optimization %% (from beam_a to beam_z). -type instruction() :: atom() | tuple(). @@ -158,44 +156,6 @@ code_at(L, Ll) -> replace_labels(Is, Acc, D, Fb) -> replace_labels_1(Is, Acc, D, Fb). -%% bif_to_test(Bif, [Op], Fail) -> {test,Test,Fail,[Op]} -%% Convert a BIF to a test. Fail if not possible. - --spec bif_to_test(atom(), list(), fail()) -> test(). - -bif_to_test(is_atom, [_]=Ops, Fail) -> {test,is_atom,Fail,Ops}; -bif_to_test(is_boolean, [_]=Ops, Fail) -> {test,is_boolean,Fail,Ops}; -bif_to_test(is_binary, [_]=Ops, Fail) -> {test,is_binary,Fail,Ops}; -bif_to_test(is_bitstring,[_]=Ops, Fail) -> {test,is_bitstr,Fail,Ops}; -bif_to_test(is_float, [_]=Ops, Fail) -> {test,is_float,Fail,Ops}; -bif_to_test(is_function, [_]=Ops, Fail) -> {test,is_function,Fail,Ops}; -bif_to_test(is_function, [_,_]=Ops, Fail) -> {test,is_function2,Fail,Ops}; -bif_to_test(is_integer, [_]=Ops, Fail) -> {test,is_integer,Fail,Ops}; -bif_to_test(is_list, [_]=Ops, Fail) -> {test,is_list,Fail,Ops}; -bif_to_test(is_map, [_]=Ops, Fail) -> {test,is_map,Fail,Ops}; -bif_to_test(is_number, [_]=Ops, Fail) -> {test,is_number,Fail,Ops}; -bif_to_test(is_pid, [_]=Ops, Fail) -> {test,is_pid,Fail,Ops}; -bif_to_test(is_port, [_]=Ops, Fail) -> {test,is_port,Fail,Ops}; -bif_to_test(is_reference, [_]=Ops, Fail) -> {test,is_reference,Fail,Ops}; -bif_to_test(is_tuple, [_]=Ops, Fail) -> {test,is_tuple,Fail,Ops}; -bif_to_test('=<', [A,B], Fail) -> {test,is_ge,Fail,[B,A]}; -bif_to_test('>', [A,B], Fail) -> {test,is_lt,Fail,[B,A]}; -bif_to_test('<', [_,_]=Ops, Fail) -> {test,is_lt,Fail,Ops}; -bif_to_test('>=', [_,_]=Ops, Fail) -> {test,is_ge,Fail,Ops}; -bif_to_test('==', [C,A], Fail) when ?is_const(C) -> - {test,is_eq,Fail,[A,C]}; -bif_to_test('==', [_,_]=Ops, Fail) -> {test,is_eq,Fail,Ops}; -bif_to_test('/=', [C,A], Fail) when ?is_const(C) -> - {test,is_ne,Fail,[A,C]}; -bif_to_test('/=', [_,_]=Ops, Fail) -> {test,is_ne,Fail,Ops}; -bif_to_test('=:=', [C,A], Fail) when ?is_const(C) -> - {test,is_eq_exact,Fail,[A,C]}; -bif_to_test('=:=', [_,_]=Ops, Fail) -> {test,is_eq_exact,Fail,Ops}; -bif_to_test('=/=', [C,A], Fail) when ?is_const(C) -> - {test,is_ne_exact,Fail,[A,C]}; -bif_to_test('=/=', [_,_]=Ops, Fail) -> {test,is_ne_exact,Fail,Ops}. - - %% is_pure_test({test,Op,Fail,Ops}) -> true|false. %% Return 'true' if the test instruction does not modify any %% registers and/or bit syntax matching state. @@ -218,19 +178,6 @@ is_pure_test({test,is_function2,_,[_,_]}) -> true; is_pure_test({test,Op,_,Ops}) -> erl_internal:new_type_test(Op, length(Ops)). -%% combine_heap_needs(HeapNeed1, HeapNeed2) -> HeapNeed -%% Combine the heap need for two allocation instructions. - --type heap_need_tag() :: 'floats' | 'words'. --type heap_need() :: non_neg_integer() | - {'alloc',[{heap_need_tag(),non_neg_integer()}]}. --spec combine_heap_needs(heap_need(), heap_need()) -> heap_need(). - -combine_heap_needs(H1, H2) when is_integer(H1), is_integer(H2) -> - H1 + H2; -combine_heap_needs(H1, H2) -> - {alloc,combine_alloc_lists([H1,H2])}. - %% split_even/1 %% [1,2,3,4,5,6] -> {[1,3,5],[2,4,6]} @@ -458,11 +405,6 @@ check_liveness(R, [{get_tuple_element,S,_,D}|Is], St) -> D -> {killed,St}; _ -> check_liveness(R, Is, St) end; -check_liveness(R, [{bs_context_to_binary,S}|Is], St) -> - case R of - S -> {used,St}; - _ -> check_liveness(R, Is, St) - end; check_liveness(R, [{loop_rec,{f,_},{x,0}}|_], St) -> case R of {x,_} -> @@ -734,19 +676,6 @@ label(Old, D, Fb) -> _ -> Fb(Old) end. -%% Help function for combine_heap_needs. - -combine_alloc_lists(Al0) -> - Al1 = flatmap(fun(Words) when is_integer(Words) -> - [{words,Words}]; - ({alloc,List}) -> - List - end, Al0), - Al2 = sofs:relation(Al1), - Al3 = sofs:relation_to_family(Al2), - Al4 = sofs:to_external(Al3), - [{Tag,lists:sum(L)} || {Tag,L} <- Al4]. - %% live_opt/4. split_even([], Ss, Ds) -> diff --git a/lib/compiler/src/beam_validator.erl b/lib/compiler/src/beam_validator.erl index e76b097500..7d908df3bf 100644 --- a/lib/compiler/src/beam_validator.erl +++ b/lib/compiler/src/beam_validator.erl @@ -90,33 +90,30 @@ format_error(Error) -> %% format as used in the compiler and in .S files. validate(Module, Fs) -> - Ft = index_bs_start_match(Fs, []), + Ft = index_parameter_types(Fs, []), validate_0(Module, Fs, Ft). -index_bs_start_match([{function,_,_,Entry,Code0}|Fs], Acc0) -> +index_parameter_types([{function,_,_,Entry,Code0}|Fs], Acc0) -> Code = dropwhile(fun({label,L}) when L =:= Entry -> false; (_) -> true end, Code0), case Code of [{label,Entry}|Is] -> - Acc = index_bs_start_match_1(Is, Entry, Acc0), - index_bs_start_match(Fs, Acc); + Acc = index_parameter_types_1(Is, Entry, Acc0), + index_parameter_types(Fs, Acc); _ -> %% Something serious is wrong. Ignore it for now. %% It will be detected and diagnosed later. - index_bs_start_match(Fs, Acc0) + index_parameter_types(Fs, Acc0) end; -index_bs_start_match([], Acc) -> +index_parameter_types([], Acc) -> gb_trees:from_orddict(lists:sort(Acc)). -index_bs_start_match_1([{test,bs_start_match2,_,_,_,_}=I|_], Entry, Acc) -> - [{Entry,[I]}|Acc]; -index_bs_start_match_1([{test,_,{f,F},_},{bs_context_to_binary,_}|Is0], Entry, Acc) -> - [{label,F}|Is] = dropwhile(fun({label,L}) when L =:= F -> false; - (_) -> true - end, Is0), - index_bs_start_match_1(Is, Entry, Acc); -index_bs_start_match_1(_, _, Acc) -> Acc. +index_parameter_types_1([{'%', {type_info, Reg, Type}} | Is], Entry, Acc) -> + Key = {Entry, Reg}, + index_parameter_types_1(Is, Entry, [{Key, Type} | Acc]); +index_parameter_types_1(_, _, Acc) -> + Acc. validate_0(_Module, [], _) -> []; validate_0(Module, [{function,Name,Ar,Entry,Code}|Fs], Ft) -> @@ -289,17 +286,21 @@ valfun_1({try_case_end,Src}, Vst) -> assert_term(Src, Vst), kill_state(Vst); %% Instructions that cannot cause exceptions -valfun_1({bs_context_to_binary,Ctx}, #vst{current=#st{x=Xs}}=Vst) -> - case Ctx of - {Tag,X} when Tag =:= x; Tag =:= y -> - Type = case gb_trees:lookup(X, Xs) of - {value,#ms{}} -> term; - _ -> get_term_type(Ctx, Vst) - end, - set_type_reg(Type, Ctx, Vst); - _ -> - error({bad_source,Ctx}) - end; +valfun_1({bs_get_tail,Ctx,Dst,Live}, Vst0) -> + verify_live(Live, Vst0), + verify_y_init(Vst0), + Vst = prune_x_regs(Live, Vst0), + #vst{current=#st{x=Xs,y=Ys}} = Vst, + {Reg, Tree} = case Ctx of + {x,X} -> {X, Xs}; + {y,Y} -> {Y, Ys}; + _ -> error({bad_source,Ctx}) + end, + Type = case gb_trees:lookup(Reg, Tree) of + {value,#ms{}} -> propagate_fragility(term, [Ctx], Vst); + _ -> error({bad_context,Reg}) + end, + set_type_reg(Type, Dst, Vst); valfun_1(bs_init_writable=I, Vst) -> call(I, 1, Vst); valfun_1(build_stacktrace=I, Vst) -> @@ -385,6 +386,25 @@ valfun_1(remove_message, Vst) -> %% The message term is no longer fragile. It can be used %% without restrictions. remove_fragility(Vst); +valfun_1({'%', {type_info, Reg, Info0}}, Vst0) -> + %% Explicit type information inserted by optimization passes to indicate + %% that Reg has a certain type, so that we can accept cross-function type + %% optimizations. + %% + %% At the moment we only allow this when narrowing from 'term' which is + %% what to expect with function parameters, but in theory any narrowing + %% conversion should be legal. + case get_move_term_type(Reg, Vst0) of + term -> + Type0 = case Info0 of + match_context -> #ms{}; + _ -> Info0 + end, + Type = propagate_fragility(Type0, [Reg], Vst0), + set_type_reg(Type, Reg, Vst0); + _ -> + error(bad_type_info) + end; valfun_1({'%',_}, Vst) -> Vst; valfun_1({line,_}, Vst) -> @@ -459,16 +479,20 @@ valfun_1({try_case,Reg}, #vst{current=#st{ct=[Fail|Fails]}}=Vst0) -> error({bad_type,Type}) end; valfun_1({get_list,Src,D1,D2}, Vst0) -> + assert_not_literal(Src), assert_type(cons, Src, Vst0), Vst = set_type_reg(term, Src, D1, Vst0), set_type_reg(term, Src, D2, Vst); valfun_1({get_hd,Src,Dst}, Vst) -> + assert_not_literal(Src), assert_type(cons, Src, Vst), set_type_reg(term, Src, Dst, Vst); valfun_1({get_tl,Src,Dst}, Vst) -> + assert_not_literal(Src), assert_type(cons, Src, Vst), set_type_reg(term, Src, Dst, Vst); valfun_1({get_tuple_element,Src,I,Dst}, Vst) -> + assert_not_literal(Src), assert_type({tuple_element,I+1}, Src, Vst), set_type_reg(term, Src, Dst, Vst); valfun_1({jump,{f,Lbl}}, Vst) -> @@ -676,32 +700,44 @@ valfun_4({select_tuple_arity,Tuple,{f,Fail},{list,Choices}}, Vst) -> branch_state(Fail, Vst))); %% New bit syntax matching instructions. -valfun_4({test,bs_start_match2,{f,Fail},Live,[Ctx,NeedSlots],Ctx}, Vst0) -> - %% If source and destination registers are the same, match state - %% is OK as input. - CtxType = get_move_term_type(Ctx, Vst0), +valfun_4({test,bs_start_match3,{f,Fail},Live,[Src],Dst}, Vst0) -> + %% Match states are always okay as input. + SrcType = get_move_term_type(Src, Vst0), + DstType = propagate_fragility(bsm_match_state(), [Src], Vst0), verify_live(Live, Vst0), verify_y_init(Vst0), Vst1 = prune_x_regs(Live, Vst0), - BranchVst = case CtxType of - #ms{} -> - %% The failure branch will never be taken when Ctx - %% is a match context. Therefore, the type for Ctx - %% at the failure label must not be match_context - %% (or we could reject legal code). - set_type_reg(term, Ctx, Vst1); - _ -> - Vst1 - end, + BranchVst = case SrcType of + #ms{} -> + %% The failure branch will never be taken when Src is a + %% match context. Therefore, the type for Src at the + %% failure label must not be match_context (or we could + %% reject legal code). + set_type_reg(term, Src, Vst1); + _ -> + Vst1 + end, Vst = branch_state(Fail, BranchVst), - set_type_reg(bsm_match_state(NeedSlots), Ctx, Vst); + set_type_reg(DstType, Dst, Vst); valfun_4({test,bs_start_match2,{f,Fail},Live,[Src,Slots],Dst}, Vst0) -> - assert_term(Src, Vst0), + %% Match states are always okay as input. + SrcType = get_move_term_type(Src, Vst0), + DstType = propagate_fragility(bsm_match_state(Slots), [Src], Vst0), verify_live(Live, Vst0), verify_y_init(Vst0), Vst1 = prune_x_regs(Live, Vst0), - Vst = branch_state(Fail, Vst1), - set_type_reg(bsm_match_state(Slots), Src, Dst, Vst); + BranchVst = case SrcType of + #ms{} -> + %% The failure branch will never be taken when Src is a + %% match context. Therefore, the type for Src at the + %% failure label must not be match_context (or we could + %% reject legal code). + set_type_reg(term, Src, Vst1); + _ -> + Vst1 + end, + Vst = branch_state(Fail, BranchVst), + set_type_reg(DstType, Dst, Vst); valfun_4({test,bs_match_string,{f,Fail},[Ctx,_,_]}, Vst) -> bsm_validate_context(Ctx, Vst), branch_state(Fail, Vst); @@ -738,6 +774,16 @@ valfun_4({bs_save2,Ctx,SavePoint}, Vst) -> bsm_save(Ctx, SavePoint, Vst); valfun_4({bs_restore2,Ctx,SavePoint}, Vst) -> bsm_restore(Ctx, SavePoint, Vst); +valfun_4({bs_get_position, Ctx, Dst, Live}, Vst0) -> + bsm_validate_context(Ctx, Vst0), + verify_live(Live, Vst0), + verify_y_init(Vst0), + Vst = prune_x_regs(Live, Vst0), + set_type_reg(bs_position, Dst, Vst); +valfun_4({bs_set_position, Ctx, Pos}, Vst) -> + bsm_validate_context(Ctx, Vst), + assert_type(bs_position, Pos, Vst), + Vst; %% Other test instructions. valfun_4({test,is_float,{f,Lbl},[Float]}, Vst) -> @@ -875,6 +921,7 @@ valfun_4(_, _) -> error(unknown_instruction). verify_get_map(Fail, Src, List, Vst0) -> + assert_not_literal(Src), %OTP 22. assert_type(map, Src, Vst0), Vst1 = foldl(fun(D, Vsti) -> case is_reg_defined(D,Vsti) of @@ -999,26 +1046,12 @@ verify_call_args_1(N, Vst) -> verify_call_args_1(X, Vst). verify_local_call(Lbl, Live, Vst) -> - case all_ms_in_x_regs(Live, Vst) of - [{R,Ctx}] -> - %% Verify that there is a suitable bs_start_match2 instruction. - verify_call_match_context(Lbl, R, Vst), - - %% Since the callee has consumed the match context, - %% there must be no additional copies in Y registers. - #ms{id=Id} = Ctx, - case ms_in_y_regs(Id, Vst) of - [] -> - ok; - [_|_]=Ys -> - error({multiple_match_contexts,[R|Ys]}) - end; - [_,_|_]=Xs0 -> - Xs = [R || {R,_} <- Xs0], - error({multiple_match_contexts,Xs}); - [] -> - ok - end. + F = fun({R, _Ctx}) -> + verify_call_match_context(Lbl, R, Vst) + end, + MsRegs = all_ms_in_x_regs(Live, Vst), + verify_no_ms_aliases(MsRegs), + foreach(F, MsRegs). all_ms_in_x_regs(0, _Vst) -> []; @@ -1026,24 +1059,26 @@ all_ms_in_x_regs(Live0, Vst) -> Live = Live0 - 1, R = {x,Live}, case get_move_term_type(R, Vst) of - #ms{}=M -> - [{R,M}|all_ms_in_x_regs(Live, Vst)]; - _ -> - all_ms_in_x_regs(Live, Vst) + #ms{}=M -> [{R,M} | all_ms_in_x_regs(Live, Vst)]; + _ -> all_ms_in_x_regs(Live, Vst) end. -ms_in_y_regs(Id, #vst{current=#st{y=Ys0}}) -> - Ys = gb_trees:to_list(Ys0), - [{y,Y} || {Y,#ms{id=OtherId}} <- Ys, OtherId =:= Id]. +%% Verifies that the same match context isn't present twice. +verify_no_ms_aliases(MsRegs) -> + CtxIds = [Id || {_, #ms{id=Id}} <- MsRegs], + UniqueCtxIds = ordsets:from_list(CtxIds), + if + length(UniqueCtxIds) < length(CtxIds) -> + error({multiple_match_contexts, MsRegs}); + length(UniqueCtxIds) =:= length(CtxIds) -> + ok + end. +%% Verifies that the target label accepts match contexts in the given register. verify_call_match_context(Lbl, Ctx, #vst{ft=Ft}) -> - case gb_trees:lookup(Lbl, Ft) of - none -> - error(no_bs_start_match2); - {value,[{test,bs_start_match2,_,_,[Ctx,_],Ctx}|_]} -> - ok; - {value,[{test,bs_start_match2,_,_,_,_}=I|_]} -> - error({unsuitable_bs_start_match2,I}) + case gb_trees:lookup({Lbl, Ctx}, Ft) of + {value, match_context} -> ok; + none -> error(no_bs_start_match2) end. allocate(Zero, Stk, Heap, Live, #vst{current=#st{numy=none}}=Vst0) -> @@ -1212,6 +1247,8 @@ assert_unique_map_keys([_,_|_]=Ls) -> %%% New binary matching instructions. %%% +bsm_match_state() -> + #ms{}. bsm_match_state(Slots) -> #ms{slots=Slots}. @@ -1225,6 +1262,12 @@ bsm_get_context({x,X}=Reg, #vst{current=#st{x=Xs}}=_Vst) when is_integer(X) -> {value,{fragile,#ms{}=Ctx}} -> Ctx; _ -> error({no_bsm_context,Reg}) end; +bsm_get_context({y,Y}=Reg, #vst{current=#st{y=Ys}}=_Vst) when is_integer(Y) -> + case gb_trees:lookup(Y, Ys) of + {value,#ms{}=Ctx} -> Ctx; + {value,{fragile,#ms{}=Ctx}} -> Ctx; + _ -> error({no_bsm_context,Reg}) + end; bsm_get_context(Reg, _) -> error({bad_source,Reg}). bsm_save(Reg, {atom,start}, Vst) -> @@ -1255,6 +1298,7 @@ bsm_restore(Reg, SavePoint, Vst) -> _ -> error({illegal_restore,SavePoint,range}) end. + select_val_branches(Src, Choices, Vst) -> Infer = infer_types(Src, Vst), select_val_branches_1(Choices, Infer, Vst). @@ -1427,6 +1471,10 @@ assert_term(Src, Vst) -> get_term_type(Src, Vst), ok. +assert_not_literal({x,_}) -> ok; +assert_not_literal({y,_}) -> ok; +assert_not_literal(Literal) -> error({literal_not_allowed,Literal}). + %% The possible types. %% %% First non-term types: diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index 74ad864163..27e6e8fe00 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -210,8 +210,11 @@ do_compile(Input, Opts0) -> {error,Reason} end end, - %% Dialyzer has already spawned workers. - case lists:member(dialyzer, Opts) of + %% Some tools, like Dialyzer, has already spawned workers + %% and spawning extra workers actually slow the compilation + %% down instead of speeding it up, so we provide a mechanism + %% to bypass the compiler process. + case lists:member(no_spawn_compiler_process, Opts) of true -> IntFun(); false -> @@ -248,6 +251,9 @@ expand_opt(report, Os) -> [report_errors,report_warnings|Os]; expand_opt(return, Os) -> [return_errors,return_warnings|Os]; +expand_opt(no_bsm3, Os) -> + %% The new bsm pass requires bsm3 instructions. + [no_bsm3,no_bsm_opt|Os]; expand_opt(r16, Os) -> expand_opt_before_21(Os); expand_opt(r17, Os) -> @@ -259,13 +265,14 @@ expand_opt(r19, Os) -> expand_opt(r20, Os) -> expand_opt_before_21(Os); expand_opt(r21, Os) -> - [no_put_tuple2|Os]; + [no_put_tuple2 | expand_opt(no_bsm3, Os)]; expand_opt({debug_info_key,_}=O, Os) -> [encrypt_debug_info,O|Os]; expand_opt(O, Os) -> [O|Os]. expand_opt_before_21(Os) -> - [no_put_tuple2,no_get_hd_tl,no_ssa_opt_record,no_utf8_atoms|Os]. + [no_put_tuple2, no_get_hd_tl, no_ssa_opt_record, + no_utf8_atoms | expand_opt(no_bsm3, Os)]. %% format_error(ErrorDescriptor) -> string() @@ -816,6 +823,12 @@ kernel_passes() -> {pass,beam_kernel_to_ssa}, {iff,dssa,{listing,"ssa"}}, {iff,ssalint,{pass,beam_ssa_lint}}, + {unless,no_bsm_opt,{pass,beam_ssa_bsm}}, + {iff,dssabsm,{listing,"ssabsm"}}, + {iff,ssalint,{pass,beam_ssa_lint}}, + {unless,no_fun_opt,{pass,beam_ssa_funs}}, + {iff,dssafuns,{listing,"ssafuns"}}, + {iff,ssalint,{pass,beam_ssa_lint}}, {unless,no_ssa_opt,{pass,beam_ssa_opt}}, {iff,dssaopt,{listing,"ssaopt"}}, {iff,ssalint,{pass,beam_ssa_lint}}, @@ -847,8 +860,6 @@ asm_passes() -> {iff,dpeep,{listing,"peep"}}, {pass,beam_clean}, {iff,dclean,{listing,"clean"}}, - {unless,no_bsm_opt,{pass,beam_bsm}}, - {iff,dbsm,{listing,"bsm"}}, {unless,no_stack_trimming,{pass,beam_trim}}, {iff,dtrim,{listing,"trim"}}, {pass,beam_flatten}]}, @@ -1001,11 +1012,17 @@ parse_module(_Code, St0) -> end. do_parse_module(DefEncoding, #compile{ifile=File,options=Opts,dir=Dir}=St) -> + SourceName0 = proplists:get_value(source, Opts, File), + SourceName = case member(deterministic, Opts) of + true -> filename:basename(SourceName0); + false -> SourceName0 + end, R = epp:parse_file(File, - [{includes,[".",Dir|inc_paths(Opts)]}, - {macros,pre_defs(Opts)}, - {default_encoding,DefEncoding}, - extra]), + [{includes,[".",Dir|inc_paths(Opts)]}, + {source_name, SourceName}, + {macros,pre_defs(Opts)}, + {default_encoding,DefEncoding}, + extra]), case R of {ok,Forms,Extra} -> Encoding = proplists:get_value(encoding, Extra), @@ -2030,7 +2047,6 @@ pre_load() -> beam_asm, beam_block, beam_bs, - beam_bsm, beam_clean, beam_dict, beam_except, @@ -2040,8 +2056,10 @@ pre_load() -> beam_opcodes, beam_peep, beam_ssa, + beam_ssa_bsm, beam_ssa_codegen, beam_ssa_dead, + beam_ssa_funs, beam_ssa_opt, beam_ssa_pre_codegen, beam_ssa_recv, diff --git a/lib/compiler/src/compiler.app.src b/lib/compiler/src/compiler.app.src index bb281376ef..86259270bd 100644 --- a/lib/compiler/src/compiler.app.src +++ b/lib/compiler/src/compiler.app.src @@ -25,7 +25,6 @@ beam_asm, beam_block, beam_bs, - beam_bsm, beam_clean, beam_dict, beam_disasm, @@ -37,8 +36,10 @@ beam_opcodes, beam_peep, beam_ssa, + beam_ssa_bsm, beam_ssa_codegen, beam_ssa_dead, + beam_ssa_funs, beam_ssa_lint, beam_ssa_opt, beam_ssa_pp, diff --git a/lib/compiler/src/genop.tab b/lib/compiler/src/genop.tab index 8e34e3e291..86590fad87 100755 --- a/lib/compiler/src/genop.tab +++ b/lib/compiler/src/genop.tab @@ -574,7 +574,25 @@ BEAM_FORMAT_NUMBER=0 ## put it into the register Tail. 163: get_tl/2 +# OTP 22 + ## @spec put_tuple2 Destination Elements ## @doc Build a tuple with the elements in the list Elements and put it ## put into register Destination. 164: put_tuple2/2 + +## @spec bs_get_tail Ctx Dst Live +## @doc Sets Dst to the tail of Ctx at the current position +165: bs_get_tail/3 + +## @spec bs_start_match3 Fail Bin Live Dst +## @doc Starts a binary match sequence +166: bs_start_match3/4 + +## @spec bs_get_position Ctx Dst Live +## @doc Sets Dst to the current position of Ctx +167: bs_get_position/3 + +## @spec bs_set_positon Ctx Pos +## @doc Sets the current position of Ctx to Pos +168: bs_set_position/2 diff --git a/lib/compiler/src/sys_core_bsm.erl b/lib/compiler/src/sys_core_bsm.erl index d7b26c3a56..685e807e65 100644 --- a/lib/compiler/src/sys_core_bsm.erl +++ b/lib/compiler/src/sys_core_bsm.erl @@ -24,161 +24,52 @@ -export([module/2,format_error/1]). -include("core_parse.hrl"). --import(lists, [member/2,reverse/1,usort/1]). -spec module(cerl:c_module(), [compile:option()]) -> {'ok', cerl:c_module()}. -module(#c_module{defs=Ds0}=Mod, Opts) -> - {Ds,Ws0} = function(Ds0, [], []), - case member(bin_opt_info, Opts) of - false -> - {ok,Mod#c_module{defs=Ds}}; - true -> - Ws1 = [make_warning(Where, What) || {Where,What} <- Ws0], - Ws = usort(Ws1), - {ok,Mod#c_module{defs=Ds},Ws} - end. +module(#c_module{defs=Ds}=Mod, _Opts) -> + {ok,Mod#c_module{defs=function(Ds)}}. -function([{#c_var{name={F,Arity}}=Name,B0}|Fs], FsAcc, Ws0) -> - try cerl_trees:mapfold(fun bsm_an/2, Ws0, B0) of - {B,Ws} -> - function(Fs, [{Name,B}|FsAcc], Ws) +function([{#c_var{name={F,Arity}}=Name,B0}|Fs]) -> + try cerl_trees:map(fun bsm_reorder/1, B0) of + B -> [{Name,B} | function(Fs)] catch Class:Error:Stack -> - io:fwrite("Function: ~w/~w\n", [F,Arity]), - erlang:raise(Class, Error, Stack) + io:fwrite("Function: ~w/~w\n", [F,Arity]), + erlang:raise(Class, Error, Stack) end; -function([], Fs, Ws) -> - {reverse(Fs),Ws}. +function([]) -> + []. -type error() :: atom(). -spec format_error(error()) -> nonempty_string(). -format_error(bin_opt_alias) -> - "INFO: the '=' operator will prevent delayed sub binary optimization"; -format_error(bin_partition) -> - "INFO: matching non-variables after a previous clause matching a variable " - "will prevent delayed sub binary optimization"; -format_error(bin_var_used) -> - "INFO: using a matched out sub binary will prevent " - "delayed sub binary optimization"; -format_error(orig_bin_var_used_in_guard) -> - "INFO: using the original binary variable in a guard will prevent " - "delayed sub binary optimization"; -format_error(bin_var_used_in_guard) -> - "INFO: using a matched out sub binary in a guard will prevent " - "delayed sub binary optimization". - +format_error(_) -> error(badarg). -%%% -%%% Annotate bit syntax matching to faciliate optimization in further passes. -%%% +%%% Reorder bit syntax matching to faciliate optimization in further passes. -bsm_an(Core0, Ws0) -> - case bsm_an(Core0) of - {ok,Core} -> - {Core,Ws0}; - {ok,Core,W} -> - {Core,[W|Ws0]} - end. +bsm_reorder(#c_case{arg=#c_var{}=V}=Case) -> + bsm_reorder_1([V], Case); +bsm_reorder(#c_case{arg=#c_values{es=Es}}=Case) -> + bsm_reorder_1(Es, Case); +bsm_reorder(Core) -> + Core. -bsm_an(#c_case{arg=#c_var{}=V}=Case) -> - bsm_an_1([V], Case); -bsm_an(#c_case{arg=#c_values{es=Es}}=Case) -> - bsm_an_1(Es, Case); -bsm_an(Other) -> - {ok,Other}. - -bsm_an_1(Vs0, #c_case{clauses=Cs0}=Case) -> +bsm_reorder_1(Vs0, #c_case{clauses=Cs0}=Case) -> case bsm_leftmost(Cs0) of - none -> - {ok,Case}; - 1 -> - bsm_an_2(Vs0, Cs0, Case); - Pos -> - Vs = move_from_col(Pos, Vs0), - Cs = [C#c_clause{pats=move_from_col(Pos, Ps)} || - #c_clause{pats=Ps}=C <- Cs0], - bsm_an_2(Vs, Cs, Case) - end. - -bsm_an_2(Vs, Cs, Case) -> - try - bsm_ensure_no_partition(Cs), - {ok,bsm_do_an(Vs, Cs, Case)} - catch - throw:{problem,Where,What} -> - {ok,Case,{Where,What}} + Pos when Pos > 0, Pos =/= none -> + Vs = core_lib:make_values(move_from_col(Pos, Vs0)), + Cs = [C#c_clause{pats=move_from_col(Pos, Ps)} + || #c_clause{pats=Ps}=C <- Cs0], + Case#c_case{arg=Vs,clauses=Cs}; + _ -> + Case end. move_from_col(Pos, L) -> {First,[Col|Rest]} = lists:split(Pos - 1, L), [Col|First] ++ Rest. -bsm_do_an([#c_var{name=Vname}=V0|Vs0], Cs0, Case) -> - Cs = bsm_do_an_var(Vname, Cs0), - V = bsm_annotate_for_reuse(V0), - Vs = core_lib:make_values([V|Vs0]), - Case#c_case{arg=Vs,clauses=Cs}; -bsm_do_an(_Vs, _Cs, Case) -> Case. - -bsm_do_an_var(V, [#c_clause{pats=[P|_],guard=G,body=B0}=C0|Cs]) -> - case P of - #c_var{name=VarName} -> - case core_lib:is_var_used(V, G) of - true -> bsm_problem(C0, orig_bin_var_used_in_guard); - false -> ok - end, - case core_lib:is_var_used(VarName, G) of - true -> bsm_problem(C0, bin_var_used_in_guard); - false -> ok - end, - B1 = bsm_maybe_ctx_to_binary(VarName, B0), - B = bsm_maybe_ctx_to_binary(V, B1), - C = C0#c_clause{body=B}, - [C|bsm_do_an_var(V, Cs)]; - #c_alias{} -> - case bsm_could_match_binary(P) of - false -> - [C0|bsm_do_an_var(V, Cs)]; - true -> - bsm_problem(C0, bin_opt_alias) - end; - _ -> - case bsm_could_match_binary(P) andalso bsm_is_var_used(V, G, B0) of - false -> - [C0|bsm_do_an_var(V, Cs)]; - true -> - bsm_problem(C0, bin_var_used) - end - end; -bsm_do_an_var(_, []) -> []. - -bsm_annotate_for_reuse(#c_var{anno=Anno}=Var) -> - Var#c_var{anno=[reuse_for_context|Anno]}. - -bsm_is_var_used(V, G, B) -> - core_lib:is_var_used(V, G) orelse core_lib:is_var_used(V, B). - -bsm_maybe_ctx_to_binary(V, B) -> - case core_lib:is_var_used(V, B) andalso not previous_ctx_to_binary(V, B) of - false -> - B; - true -> - #c_seq{arg=#c_primop{name=#c_literal{val=bs_context_to_binary}, - args=[#c_var{name=V}]}, - body=B} - end. - -previous_ctx_to_binary(V, Core) -> - case Core of - #c_seq{arg=#c_primop{name=#c_literal{val=bs_context_to_binary}, - args=[#c_var{name=V}]}} -> - true; - _ -> - false - end. - %% bsm_leftmost(Cs) -> none | ArgumentNumber %% Find the leftmost argument that matches a nonempty binary. %% Return either 'none' or the argument number (1-N). @@ -200,94 +91,3 @@ bsm_leftmost_2([_|Ps], Cs, N, Pos) -> bsm_leftmost_2(Ps, Cs, N+1, Pos); bsm_leftmost_2([], Cs, _, Pos) -> bsm_leftmost_1(Cs, Pos). - -%% bsm_ensure_no_partition(Cs) -> ok (exception if problem) -%% There must only be a single bs_start_match2 instruction if we -%% are to reuse the binary variable for the match context. -%% -%% To make sure that there is only a single bs_start_match2 -%% instruction, we will check for partitions such as: -%% -%% foo(<<...>>) -> ... -%% foo(<Variable>) when ... -> ... -%% foo(<Non-variable pattern>) -> -%% -%% If there is such partition, we reject the optimization. - -bsm_ensure_no_partition(Cs) -> - bsm_ensure_no_partition_1(Cs, before). - -%% Loop through each clause. -bsm_ensure_no_partition_1([#c_clause{pats=Ps,guard=G}|Cs], State0) -> - State = bsm_ensure_no_partition_2(Ps, G, State0), - case State of - 'after' -> - bsm_ensure_no_partition_after(Cs); - _ -> - ok - end, - bsm_ensure_no_partition_1(Cs, State); -bsm_ensure_no_partition_1([], _) -> ok. - -bsm_ensure_no_partition_2([#c_binary{}|_], _, _State) -> - within; -bsm_ensure_no_partition_2([#c_alias{}=Alias|_], N, State) -> - %% Retrieve the real pattern that the alias refers to and check that. - P = bsm_real_pattern(Alias), - bsm_ensure_no_partition_2([P], N, State); -bsm_ensure_no_partition_2([_|_], _, before=State) -> - %% No binary matching yet - therefore no partition. - State; -bsm_ensure_no_partition_2([P|_], _, State) -> - case bsm_could_match_binary(P) of - false -> - State; - true -> - %% The pattern P *may* match a binary, so we must update the state. - %% (P must be a variable.) - 'after' - end. - -bsm_ensure_no_partition_after([#c_clause{pats=Ps}=C|Cs]) -> - case Ps of - [#c_var{}|_] -> - bsm_ensure_no_partition_after(Cs); - _ -> - bsm_problem(C, bin_partition) - end; -bsm_ensure_no_partition_after([]) -> ok. - -bsm_could_match_binary(#c_alias{pat=P}) -> bsm_could_match_binary(P); -bsm_could_match_binary(#c_cons{}) -> false; -bsm_could_match_binary(#c_tuple{}) -> false; -bsm_could_match_binary(#c_literal{val=Lit}) -> is_bitstring(Lit); -bsm_could_match_binary(_) -> true. - -bsm_real_pattern(#c_alias{pat=P}) -> bsm_real_pattern(P); -bsm_real_pattern(P) -> P. - -bsm_problem(Where, What) -> - throw({problem,Where,What}). - -make_warning(Core, Term) -> - case should_suppress_warning(Core) of - true -> - ok; - false -> - Anno = cerl:get_ann(Core), - Line = get_line(Anno), - File = get_file(Anno), - {File,[{Line,?MODULE,Term}]} - end. - -should_suppress_warning(Core) -> - Ann = cerl:get_ann(Core), - member(compiler_generated, Ann). - -get_line([Line|_]) when is_integer(Line) -> Line; -get_line([_|T]) -> get_line(T); -get_line([]) -> none. - -get_file([{file,File}|_]) -> File; -get_file([_|T]) -> get_file(T); -get_file([]) -> "no_file". % should not happen diff --git a/lib/compiler/src/v3_kernel.erl b/lib/compiler/src/v3_kernel.erl index fe8e252e5a..f7ca66b1da 100644 --- a/lib/compiler/src/v3_kernel.erl +++ b/lib/compiler/src/v3_kernel.erl @@ -82,8 +82,7 @@ -export([module/2,format_error/1]). -import(lists, [map/2,foldl/3,foldr/3,mapfoldl/3,splitwith/2,member/2, - keymember/3,keyfind/3,partition/2,droplast/1,last/1,sort/1, - reverse/1]). + keyfind/3,partition/2,droplast/1,last/1,sort/1,reverse/1]). -import(ordsets, [add_element/2,del_element/2,union/2,union/1,subtract/2]). -import(cerl, [c_tuple/1]). @@ -1416,7 +1415,6 @@ is_remote_bif(_, _, _) -> false. %% called for effect only. bif_vals(dsetelement, 3) -> 0; -bif_vals(bs_context_to_binary, 1) -> 0; bif_vals(_, _) -> 1. bif_vals(_, _, _) -> 1. @@ -2337,8 +2335,7 @@ uexpr(#k_bif{anno=A,op=Op,args=As}=Bif, {break,Rs}, St0) -> {Brs,St1} = bif_returns(Op, Rs, St0), {Bif#k_bif{anno=#k{us=Used,ns=lit_list_vars(Brs),a=A},ret=Brs}, Used,St1}; -uexpr(#k_match{anno=A,vars=Vs0,body=B0}, Br, St0) -> - Vs = handle_reuse_annos(Vs0, St0), +uexpr(#k_match{anno=A,vars=Vs,body=B0}, Br, St0) -> Rs = break_rets(Br), {B1,Bu,St1} = umatch(B0, Br, St0), case is_in_guard(St1) of @@ -2441,33 +2438,6 @@ make_fdef(Anno, Name, Arity, Vs, Body) -> vars=Vs,body=Body,ret=[]}, #k_fdef{anno=Anno,func=Name,arity=Arity,vars=Vs,body=Match}. - -%% handle_reuse_annos([#k_var{}], State) -> State. -%% In general, it is only safe to reuse a variable for a match context -%% if the original value of the variable will no longer be needed. -%% -%% If a variable has been bound in an outer letrec and is therefore -%% free in the current function, the variable may still be used. -%% We don't bother to check whether the variable is actually used, -%% but simply clears the 'reuse_for_context' annotation for any variable -%% that is free. -handle_reuse_annos(Vs, St) -> - [handle_reuse_anno(V, St) || V <- Vs]. - -handle_reuse_anno(#k_var{anno=A}=V, St) -> - case member(reuse_for_context, A) of - false -> V; - true -> handle_reuse_anno_1(V, St) - end. - -handle_reuse_anno_1(#k_var{anno=Anno,name=Vname}=V, #kern{ff={F,A}}=St) -> - FreeVs = get_free(F, A, St), - case keymember(Vname, #k_var.name, FreeVs) of - true -> V#k_var{anno=Anno--[reuse_for_context]}; - false -> V - end; -handle_reuse_anno_1(V, _St) -> V. - %% get_free(Name, Arity, State) -> [Free]. %% store_free(Name, Arity, [Free], State) -> State. @@ -2511,8 +2481,7 @@ umatch(#k_alt{anno=A,first=F0,then=T0}, Br, St0) -> Used = union(Fu, Tu), {#k_alt{anno=#k{us=Used,ns=[],a=A},first=F1,then=T1}, Used,St2}; -umatch(#k_select{anno=A,var=V0,types=Ts0}, Br, St0) -> - V = handle_reuse_anno(V0, St0), +umatch(#k_select{anno=A,var=V,types=Ts0}, Br, St0) -> {Ts1,Tus,St1} = umatch_list(Ts0, Br, St0), Used = case member(no_usage, get_kanno(V)) of true -> Tus; diff --git a/lib/compiler/test/Makefile b/lib/compiler/test/Makefile index 2a5004aa4c..40428b7f2d 100644 --- a/lib/compiler/test/Makefile +++ b/lib/compiler/test/Makefile @@ -97,6 +97,10 @@ INLINE= \ receive \ record +R21= \ + bs_construct \ + bs_match + CORE_MODULES = \ lfe_andor_SUITE \ lfe_guard_SUITE @@ -107,6 +111,8 @@ POST_OPT_MODULES= $(NO_OPT:%=%_post_opt_SUITE) POST_OPT_ERL_FILES= $(POST_OPT_MODULES:%=%.erl) INLINE_MODULES= $(INLINE:%=%_inline_SUITE) INLINE_ERL_FILES= $(INLINE_MODULES:%=%.erl) +R21_MODULES= $(R21:%=%_r21_SUITE) +R21_ERL_FILES= $(R21_MODULES:%=%.erl) ERL_FILES= $(MODULES:%=%.erl) CORE_FILES= $(CORE_MODULES:%=%.core) @@ -135,7 +141,8 @@ EBIN = . # Targets # ---------------------------------------------------- -make_emakefile: $(NO_OPT_ERL_FILES) $(POST_OPT_ERL_FILES) $(INLINE_ERL_FILES) +make_emakefile: $(NO_OPT_ERL_FILES) $(POST_OPT_ERL_FILES) \ + $(INLINE_ERL_FILES) $(R21_ERL_FILES) $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) -o$(EBIN) $(MODULES) \ > $(EMAKEFILE) $(ERL_TOP)/make/make_emakefile +no_copt +no_postopt \ @@ -145,6 +152,8 @@ make_emakefile: $(NO_OPT_ERL_FILES) $(POST_OPT_ERL_FILES) $(INLINE_ERL_FILES) -o$(EBIN) $(POST_OPT_MODULES) >> $(EMAKEFILE) $(ERL_TOP)/make/make_emakefile +inline $(ERL_COMPILE_FLAGS) \ -o$(EBIN) $(INLINE_MODULES) >> $(EMAKEFILE) + $(ERL_TOP)/make/make_emakefile +r21 $(ERL_COMPILE_FLAGS) \ + -o$(EBIN) $(R21_MODULES) >> $(EMAKEFILE) $(ERL_TOP)/make/make_emakefile +from_core $(ERL_COMPILE_FLAGS) \ -o$(EBIN) $(CORE_MODULES) >> $(EMAKEFILE) @@ -171,6 +180,9 @@ docs: %_inline_SUITE.erl: %_SUITE.erl sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@ +%_r21_SUITE.erl: %_SUITE.erl + sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@ + # ---------------------------------------------------- # Release Target # ---------------------------------------------------- @@ -183,7 +195,7 @@ release_tests_spec: make_emakefile $(INSTALL_DATA) compiler.spec compiler.cover \ $(EMAKEFILE) $(ERL_FILES) "$(RELSYSDIR)" $(INSTALL_DATA) $(NO_OPT_ERL_FILES) $(POST_OPT_ERL_FILES) \ - $(INLINE_ERL_FILES) "$(RELSYSDIR)" + $(INLINE_ERL_FILES) $(R21_ERL_FILES) "$(RELSYSDIR)" $(INSTALL_DATA) $(CORE_FILES) "$(RELSYSDIR)" for file in $(ERL_DUMMY_FILES); do \ module=`basename $$file .erl`; \ diff --git a/lib/compiler/test/beam_validator_SUITE_data/bad_bin_match.S b/lib/compiler/test/beam_validator_SUITE_data/bad_bin_match.S index a60ca1e89a..c7610971f1 100644 --- a/lib/compiler/test/beam_validator_SUITE_data/bad_bin_match.S +++ b/lib/compiler/test/beam_validator_SUITE_data/bad_bin_match.S @@ -11,5 +11,5 @@ {label,1}. {func_info,{atom,t},{atom,t},1}. {label,2}. - {test,bs_start_match2,{f,1},1,[{x,0},0],{x,0}}. + {test,bs_start_match3,{f,1},1,[{x,0}],{x,0}}. return. diff --git a/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.S b/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.S index cca052a9c4..5b974119c6 100644 --- a/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.S +++ b/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.S @@ -172,7 +172,7 @@ {allocate_zero,1,0}. {label,28}. {loop_rec,{f,30},{x,0}}. - {test,bs_start_match2,{f,29},1,[{x,0},0],{x,0}}. + {test,bs_start_match3,{f,29},1,[{x,0}],{x,0}}. {test,bs_get_integer2, {f,29}, 1, @@ -219,7 +219,7 @@ {allocate_zero,1,0}. {label,33}. {loop_rec,{f,35},{x,0}}. - {test,bs_start_match2,{f,34},1,[{x,0},0],{x,0}}. + {test,bs_start_match3,{f,34},1,[{x,0}],{x,0}}. {test,bs_get_integer2, {f,34}, 1, @@ -262,7 +262,7 @@ {allocate_zero,1,0}. {label,38}. {loop_rec,{f,40},{x,0}}. - {test,bs_start_match2,{f,39},1,[{x,0},0],{x,1}}. + {test,bs_start_match3,{f,39},1,[{x,0}],{x,1}}. {test,bs_get_integer2, {f,39}, 2, diff --git a/lib/compiler/test/bs_match_SUITE.erl b/lib/compiler/test/bs_match_SUITE.erl index b4277f0705..8e105c6244 100644 --- a/lib/compiler/test/bs_match_SUITE.erl +++ b/lib/compiler/test/bs_match_SUITE.erl @@ -40,7 +40,10 @@ map_and_binary/1,unsafe_branch_caching/1, bad_literals/1,good_literals/1,constant_propagation/1, parse_xml/1,get_payload/1,escape/1,num_slots_different/1, - beam_bsm/1,guard/1,is_ascii/1,non_opt_eq/1,erl_689/1]). + beam_bsm/1,guard/1,is_ascii/1,non_opt_eq/1, + expression_before_match/1,erl_689/1,restore_on_call/1, + restore_after_catch/1,matches_on_parameter/1,big_positions/1, + matching_meets_apply/1]). -export([coverage_id/1,coverage_external_ignore/2]). @@ -56,7 +59,7 @@ all() -> [{group,p}]. groups() -> - [{p,[parallel], + [{p,[], [size_shadow,int_float,otp_5269,null_fields,wiger, bin_tail,save_restore, partitioned_bs_match,function_clause,unit, @@ -72,7 +75,10 @@ groups() -> map_and_binary,unsafe_branch_caching, bad_literals,good_literals,constant_propagation,parse_xml, get_payload,escape,num_slots_different, - beam_bsm,guard,is_ascii,non_opt_eq,erl_689]}]. + beam_bsm,guard,is_ascii,non_opt_eq, + expression_before_match,erl_689,restore_on_call, + matches_on_parameter,big_positions, + matching_meets_apply]}]. init_per_suite(Config) -> @@ -1734,34 +1740,175 @@ non_opt_eq([], <<>>) -> %% ERL-689 -erl_689(Config) -> +erl_689(_Config) -> {{0, 0, 0}, <<>>} = do_erl_689_1(<<0>>, ?MODULE), {{2018, 8, 7}, <<>>} = do_erl_689_1(<<4,2018:16/little,8,7>>, ?MODULE), {{0, 0, 0}, <<>>} = do_erl_689_2(?MODULE, <<0>>), {{2018, 8, 7}, <<>>} = do_erl_689_2(?MODULE, <<4,2018:16/little,8,7>>), ok. -do_erl_689_1(<<Length, Data/binary>>, _) -> +do_erl_689_1(Arg1, Arg2) -> + Res = do_erl_689_1a(Arg1, Arg2), + Res = do_erl_689_1b(Arg1, Arg2). + +do_erl_689_2(Arg1, Arg2) -> + Res = do_erl_689_2a(Arg1, Arg2), + Res = do_erl_689_2b(Arg1, Arg2). + +do_erl_689_1a(<<Length, Data/binary>>, _) -> + case {Data, Length} of + {_, 0} -> + %% bs_context_to_binary would incorrectly set Data to the original + %% binary (before matching in the function head). + {{0, 0, 0}, Data}; + {<<Y:16/little, M, D, Rest/binary>>, 4} -> + {{Y, M, D}, Rest} + end. + +do_erl_689_1b(<<Length, Data/binary>>, _) -> case {Data, Length} of {_, 0} -> %% bs_context_to_binary would incorrectly set Data to the original %% binary (before matching in the function head). + id(0), {{0, 0, 0}, Data}; {<<Y:16/little, M, D, Rest/binary>>, 4} -> + id(1), + {{Y, M, D}, Rest} + end. + +do_erl_689_2a(_, <<Length, Data/binary>>) -> + case {Length, Data} of + {0, _} -> + %% bs_context_to_binary would incorrectly set Data to the original + %% binary (before matching in the function head). + {{0, 0, 0}, Data}; + {4, <<Y:16/little, M, D, Rest/binary>>} -> {{Y, M, D}, Rest} end. -do_erl_689_2(_, <<Length, Data/binary>>) -> +do_erl_689_2b(_, <<Length, Data/binary>>) -> case {Length, Data} of {0, _} -> %% bs_context_to_binary would incorrectly set Data to the original %% binary (before matching in the function head). + id(0), {{0, 0, 0}, Data}; {4, <<Y:16/little, M, D, Rest/binary>>} -> + id(1), {{Y, M, D}, Rest} end. check(F, R) -> R = F(). +%% Make sure that an expression that comes between function start and a match +%% expression passes validation. +expression_before_match(Config) when is_list(Config) -> + <<_,R/binary>> = id(<<0,1,2,3>>), + {1, <<2,3>>} = expression_before_match_1(R), + ok. + +expression_before_match_1(R) -> + A = id(1), + case R of + <<1,Bar/binary>> -> {A, Bar}; + <<>> -> {A, baz} + end. + +%% Make sure that context positions are updated on calls. +restore_on_call(Config) when is_list(Config) -> + ok = restore_on_call_1(<<0, 1, 2>>). + +restore_on_call_1(<<0, Rest/binary>>) -> + <<2>> = restore_on_call_2(Rest), + <<2>> = restore_on_call_2(Rest), %% {badmatch, <<>>} on missing restore. + ok. + +restore_on_call_2(<<1, Rest/binary>>) -> Rest; +restore_on_call_2(Other) -> Other. + +%% 'catch' must invalidate positions. +restore_after_catch(Config) when is_list(Config) -> + <<0, 1>> = restore_after_catch_1(<<0, 1>>), + ok. + +restore_after_catch_1(<<A/binary>>) -> + try throw_after_byte(A) of + _ -> impossible + catch + throw:_Any -> + %% Will equal <<1>> if the bug is present. + A + end. + +throw_after_byte(<<_,_/binary>>) -> + throw(away). + +matches_on_parameter(Config) when is_list(Config) -> + %% This improves coverage for matching on "naked" parameters. + {<<"urka">>, <<"a">>} = matches_on_parameter_1(<<"gurka">>), + ok = (catch matches_on_parameter_2(<<"10001110101">>, 0)). + +matches_on_parameter_1(Bin) -> + <<"g", A/binary>> = Bin, + <<_,_,"rk", B/binary>> = Bin, + {A, B}. + +matches_on_parameter_2(Bin, Offset) -> + <<_:Offset, Bit:1, Rest/bits>> = Bin, + case bit_size(Rest) of + 0 -> throw(ok); + _ -> [Bit | matches_on_parameter_2(Bin, Offset + 1)] + end. + +big_positions(Config) when is_list(Config) -> + %% This provides coverage for when match context positions no longer fit + %% into an immediate on 32-bit platforms. + + A = <<0:((1 bsl 27) - 8), $A, 1:1, "gurka", $A>>, + B = <<0:((1 bsl 27) - 8), $B, "hello", $B>>, + + {a,$A} = bp_start_match(A), + {b,$B} = bp_start_match(B), + {a,$A} = bp_getpos(A), + {b,$B} = bp_getpos(B), + + ok. + +%% After the first iteration the context's position will no longer fit into an +%% immediate. To improve performance the bs_start_match3 instruction will +%% return a new context with an updated base position so that we won't have to +%% resort to using bigints. +bp_start_match(<<_:(1 bsl 27),T/bits>>) -> bp_start_match(T); +bp_start_match(<<1:1,"gurka",A>>) -> {a,A}; +bp_start_match(<<"hello",B>>) -> {b,B}. + +%% This is a corner case where the above didn't work perfectly; if the position +%% was _just_ small enough to fit into an immediate when bs_start_match3 was +%% hit, but too large at bs_get_position, then it must be saved as a bigint. +bp_getpos(<<_:((1 bsl 27) - 8),T/bits>>) -> bp_getpos(T); +bp_getpos(<<A,1:1,"gurka",A>>) -> {a,A}; +bp_getpos(<<B,"hello",B>>) -> {b,B}. + +matching_meets_apply(_Config) -> + <<"abc">> = do_matching_meets_apply(<<"/abc">>, []), + 42 = do_matching_meets_apply(<<"">>, {erlang,-42}), + 100 = do_matching_meets_apply(no_binary, {erlang,-100}), + ok. + +do_matching_meets_apply(<<$/, Rest/binary>>, _Handler) -> + id(Rest); +do_matching_meets_apply(<<_/binary>>=Name, never_matches_a) -> + %% Used to crash the compiler because variables in a remote + %% were not handled properly by beam_ssa_bsm. + Name:foo(gurka); +do_matching_meets_apply(<<_/binary>>=Name, never_matches_b) -> + %% Another case of the above. + foo:Name(gurka); +do_matching_meets_apply(_Bin, {Handler, State}) -> + %% Another case of the above. + Handler:abs(State). + + id(I) -> I. diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl index 38bc928f85..5656743c76 100644 --- a/lib/compiler/test/compile_SUITE.erl +++ b/lib/compiler/test/compile_SUITE.erl @@ -36,7 +36,7 @@ core_roundtrip/1, asm/1, optimized_guards/1, sys_pre_attributes/1, dialyzer/1, warnings/1, pre_load_check/1, env_compiler_options/1, - bc_options/1, deterministic_include/1 + bc_options/1, deterministic_include/1, deterministic_paths/1 ]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -53,7 +53,7 @@ all() -> cover, env, core_pp, core_roundtrip, asm, optimized_guards, sys_pre_attributes, dialyzer, warnings, pre_load_check, env_compiler_options, custom_debug_info, bc_options, - custom_compile_info, deterministic_include]. + custom_compile_info, deterministic_include, deterministic_paths]. groups() -> []. @@ -1481,7 +1481,10 @@ bc_options(Config) -> 132 = highest_opcode(DataDir, small, [no_put_tuple2,no_get_hd_tl,no_ssa_opt_record, - no_ssa_opt_float,no_line_info]), + no_ssa_opt_float,no_line_info,no_bsm3]), + + 153 = highest_opcode(DataDir, small, [r20]), + 153 = highest_opcode(DataDir, small, [r21]), 136 = highest_opcode(DataDir, big, [no_put_tuple2,no_get_hd_tl, no_ssa_opt_record,no_line_info]), @@ -1530,6 +1533,30 @@ deterministic_include(Config) when is_list(Config) -> ok. +deterministic_paths(Config) when is_list(Config) -> + DataDir = proplists:get_value(data_dir, Config), + + %% Files without +deterministic should differ if they were compiled from a + %% different directory. + true = deterministic_paths_1(DataDir, "simple", []), + + %% ... but files with +deterministic shouldn't. + false = deterministic_paths_1(DataDir, "simple", [deterministic]), + + ok. + +deterministic_paths_1(DataDir, Name, Opts) -> + Simple = filename:join(DataDir, "simple"), + {ok, Cwd} = file:get_cwd(), + try + {ok,_,A} = compile:file(Simple, [binary | Opts]), + ok = file:set_cwd(DataDir), + {ok,_,B} = compile:file(Name, [binary | Opts]), + A =/= B + after + file:set_cwd(Cwd) + end. + %%% %%% Utilities. %%% diff --git a/lib/compiler/test/fun_SUITE.erl b/lib/compiler/test/fun_SUITE.erl index e00885fcd6..1df0a05275 100644 --- a/lib/compiler/test/fun_SUITE.erl +++ b/lib/compiler/test/fun_SUITE.erl @@ -249,6 +249,13 @@ badfun(_Config) -> expect_badfun(X, catch X(put(?FUNCTION_NAME, of_course))), of_course = erase(?FUNCTION_NAME), + %% A literal as a Fun used to crash the code generator. This only happened + %% when type optimization had reduced `Fun` to a literal, hence the match. + Literal = fun(literal = Fun) -> + Fun() + end, + expect_badfun(literal, catch Literal(literal)), + ok. expect_badfun(Term, Exit) -> diff --git a/lib/compiler/test/misc_SUITE.erl b/lib/compiler/test/misc_SUITE.erl index 4c2d0116c9..c9acda2b6d 100644 --- a/lib/compiler/test/misc_SUITE.erl +++ b/lib/compiler/test/misc_SUITE.erl @@ -254,17 +254,6 @@ silly_coverage(Config) when is_list(Config) -> 2}, expect_error(fun() -> beam_peep:module(PeepInput, []) end), - %% beam_bsm. This is tricky. Our function must be sane enough to not crash - %% btb_index/1, but must crash the main optimization pass. - BsmInput = {?MODULE,[{foo,0}],[], - [{function,foo,0,2, - [{label,1}, - {func_info,{atom,?MODULE},{atom,foo},0}, - {label,2}, - {test,bs_get_binary2,{f,99},0,[{x,0},{atom,all},1,[]],{x,0}}, - {block,[a|b]}]}],0}, - expect_error(fun() -> beam_bsm:module(BsmInput, []) end), - BeamZInput = {?MODULE,[{foo,0}],[], [{function,foo,0,2, [{label,1}, diff --git a/lib/compiler/test/test_lib.erl b/lib/compiler/test/test_lib.erl index c6baa611ec..4502f5b68a 100644 --- a/lib/compiler/test/test_lib.erl +++ b/lib/compiler/test/test_lib.erl @@ -78,6 +78,9 @@ opt_opts(Mod) -> (no_stack_trimming) -> true; (debug_info) -> true; (inline) -> true; + (no_put_tuple2) -> true; + (no_bsm3) -> true; + (no_bsm_opt) -> true; (_) -> false end, Opts). @@ -89,8 +92,9 @@ get_data_dir(Config) -> Data0 = proplists:get_value(data_dir, Config), Opts = [{return,list}], Data1 = re:replace(Data0, "_no_opt_SUITE", "_SUITE", Opts), - Data = re:replace(Data1, "_post_opt_SUITE", "_SUITE", Opts), - re:replace(Data, "_inline_SUITE", "_SUITE", Opts). + Data2 = re:replace(Data1, "_post_opt_SUITE", "_SUITE", Opts), + Data = re:replace(Data2, "_inline_SUITE", "_SUITE", Opts), + re:replace(Data, "_r21_SUITE", "_SUITE", Opts). is_cloned_mod(Mod) -> is_cloned_mod_1(atom_to_list(Mod)). @@ -100,6 +104,7 @@ is_cloned_mod(Mod) -> is_cloned_mod_1("no_opt_SUITE") -> true; is_cloned_mod_1("post_opt_SUITE") -> true; is_cloned_mod_1("inline_SUITE") -> true; +is_cloned_mod_1("21_SUITE") -> true; is_cloned_mod_1([_|T]) -> is_cloned_mod_1(T); is_cloned_mod_1([]) -> false. diff --git a/lib/compiler/test/warnings_SUITE.erl b/lib/compiler/test/warnings_SUITE.erl index 33d55996ad..1f39348998 100644 --- a/lib/compiler/test/warnings_SUITE.erl +++ b/lib/compiler/test/warnings_SUITE.erl @@ -522,25 +522,43 @@ bin_opt_info(Config) when is_list(Config) -> <<>> -> ok end. + %% We use a tail in a BIF instruction, remote call, function + %% return, and an optimizable tail call for better coverage. + t2(<<A,B,T/bytes>>) -> + if + A > B -> t2(T); + A =< B -> T + end; + t2(<<_,T/bytes>>) when byte_size(T) < 4 -> + foo; t2(<<_,T/bytes>>) -> - split_binary(T, 4). + split_binary(T, 4). ">>, - Ts1 = [{bsm1, - Code, - [bin_opt_info], - {warnings, - [{4,sys_core_bsm,orig_bin_var_used_in_guard}, - {5,beam_bsm,{no_bin_opt,{{t1,1},no_suitable_bs_start_match}}}, - {9,beam_bsm,{no_bin_opt, - {binary_used_in,{extfunc,erlang,split_binary,2}}}} ]}}], - [] = run(Config, Ts1), + + Ws = (catch run_test(Config, Code, [bin_opt_info])), + + %% This is an inexact match since the pass reports exact instructions as + %% part of the warnings, which may include annotations that vary from run + %% to run. + {warnings, + [{5,beam_ssa_bsm,{unsuitable_call, + {{b_local,{b_literal,t1},1}, + {used_before_match, + {b_set,_,_,{bif,byte_size},[_]}}}}}, + {5,beam_ssa_bsm,{binary_created,_,_}}, + {11,beam_ssa_bsm,{binary_created,_,_}}, %% A =< B -> T + {13,beam_ssa_bsm,context_reused}, %% A > B -> t2(T); + {16,beam_ssa_bsm,{binary_created,_,_}}, %% when byte_size(T) < 4 -> + {19,beam_ssa_bsm,{remote_call, + {b_remote, + {b_literal,erlang}, + {b_literal,split_binary},2}}}, + {19,beam_ssa_bsm,{binary_created,_,_}} %% split_binary(T, 4) + ]} = Ws, %% For coverage: don't give the bin_opt_info option. - Ts2 = [{bsm2, - Code, - [], - []}], - [] = run(Config, Ts2), + [] = (catch run_test(Config, Code, [])), + ok. bin_construction(Config) when is_list(Config) -> @@ -746,7 +764,7 @@ maps_bin_opt_info(Config) when is_list(Config) -> M. ">>, [bin_opt_info], - {warnings,[{2,beam_bsm,bin_opt}]}}], + {warnings,[{3,beam_ssa_bsm,context_reused}]}}], [] = run(Config, Ts), ok. @@ -969,7 +987,6 @@ run(Config, Tests) -> end, lists:foldl(F, [], Tests). - %% Compiles a test module and returns the list of errors and warnings. run_test(Conf, Test0, Warnings) -> diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl index abd89034f3..ebe4040c34 100644 --- a/lib/dialyzer/src/dialyzer_utils.erl +++ b/lib/dialyzer/src/dialyzer_utils.erl @@ -643,7 +643,7 @@ sets_filter([Mod|Mods], ExpTypes) -> src_compiler_opts() -> [no_copt, to_core, binary, return_errors, no_inline, strict_record_tests, strict_record_updates, - dialyzer]. + dialyzer, no_spawn_compiler_process]. -spec format_errors([{module(), string()}]) -> [string()]. diff --git a/lib/hipe/doc/src/hipe_app.xml b/lib/hipe/doc/src/hipe_app.xml index 99759a2f2c..480290cd9e 100644 --- a/lib/hipe/doc/src/hipe_app.xml +++ b/lib/hipe/doc/src/hipe_app.xml @@ -62,6 +62,13 @@ and the runtime system that have limited or no support for HiPE compiled modules. </p> <taglist> + <tag>Binary matching</tag> + <item><p>The HiPE compiler will crash on modules containing binary + matching unless they have been compiled with the <c>+no_bsm3</c> flag. + Note that this will disable all related optimizations done by the BEAM + compiler.</p> + </item> + <tag>Stack traces</tag> <item><p>Stack traces returned from <seealso marker="erts:erlang#get_stacktrace/0"> <c>erlang:get_stacktrace/0</c></seealso> or as part of <c>'EXIT'</c> terms diff --git a/lib/hipe/test/Makefile b/lib/hipe/test/Makefile index efeb0887ab..3650cda663 100644 --- a/lib/hipe/test/Makefile +++ b/lib/hipe/test/Makefile @@ -13,7 +13,6 @@ MODULES= \ # .erl files for these modules are automatically generated GEN_MODULES= \ basic_SUITE \ - bs_SUITE \ maps_SUITE \ sanity_SUITE diff --git a/lib/hipe/test/basic_SUITE_data/basic_inline_function.erl b/lib/hipe/test/basic_SUITE_data/basic_inline_function.erl deleted file mode 100644 index 4c08064670..0000000000 --- a/lib/hipe/test/basic_SUITE_data/basic_inline_function.erl +++ /dev/null @@ -1,73 +0,0 @@ -%%% -*- erlang-indent-level: 2 -*- -%%%------------------------------------------------------------------- -%%% Author: Kostis Sagonas -%%% -%%% Contains tests that depend on the compiler inliner being turned on. -%%%------------------------------------------------------------------- --module(basic_inline_function). - --export([test/0]). - --compile({inline, [{to_objects, 3}]}). - -test() -> - ok = test_inline_match(), - ok. - -%%-------------------------------------------------------------------- - -test_inline_match() -> - bad_object = test1(a, {binary, foo, set}, c), - bad_object = test2(a, {binary, foo, set}, c), - bad_object = test3(a, {binary, foo, set}, c), - ok. - -%% Inlined -test1(KeysObjs, C, Ts) -> - case catch to_objects(KeysObjs, C, Ts) of - {'EXIT', _} -> - bad_object; - ok -> - ok - end. - -%% "Inlined" by hand -test2(KeysObjs, C, _Ts) -> - case catch (case C of - {binary, _, set} -> - <<_ObjSz0:32, _T/binary>> = KeysObjs; - _ -> ok - end) of - {'EXIT', _} -> - bad_object; - ok -> - ok - end. - -%% Not inlined -test3(KeysObjs, C, Ts) -> - case catch fto_objects(KeysObjs, C, Ts) of - {'EXIT', _} -> - bad_object; - ok -> - ok - end. - -%% Inlined. -to_objects(Bin, {binary, _, set}, _Ts) -> - <<_ObjSz0:32, _T/binary>> = Bin, - ok; -to_objects(<<_ObjSz0:32, _T/binary>> ,_, _) -> - ok; -to_objects(_Bin, _, _Ts) -> - ok. - -%% Not Inlined. -fto_objects(Bin, {binary, _, set}, _Ts) -> - <<_ObjSz0:32, _T/binary>> = Bin, - ok; -fto_objects(<<_ObjSz0:32, _T/binary>> ,_,_) -> - ok; -fto_objects(_Bin, _, _Ts) -> - ok. - diff --git a/lib/inets/src/http_client/httpc_request.erl b/lib/inets/src/http_client/httpc_request.erl index 9b81bd7a80..0f20d93bc1 100644 --- a/lib/inets/src/http_client/httpc_request.erl +++ b/lib/inets/src/http_client/httpc_request.erl @@ -213,15 +213,18 @@ update_body(Headers, Body) -> update_headers(Headers, ContentType, Body, []) -> case Body of [] -> - Headers#http_request_h{'content-length' = "0"}; + Headers1 = Headers#http_request_h{'content-length' = "0"}, + handle_content_type(Headers1, ContentType); <<>> -> - Headers#http_request_h{'content-length' = "0"}; + Headers1 = Headers#http_request_h{'content-length' = "0"}, + handle_content_type(Headers1, ContentType); {Fun, _Acc} when is_function(Fun, 1) -> %% A client MUST NOT generate a 100-continue expectation in a request %% that does not include a message body. This implies that either the %% Content-Length or the Transfer-Encoding header MUST be present. %% DO NOT send content-type when Body is empty. - Headers#http_request_h{'content-type' = ContentType}; + Headers1 = Headers#http_request_h{'content-type' = ContentType}, + handle_transfer_encoding(Headers1); _ -> Headers#http_request_h{ 'content-length' = body_length(Body), @@ -230,12 +233,26 @@ update_headers(Headers, ContentType, Body, []) -> update_headers(_, _, _, HeadersAsIs) -> HeadersAsIs. +handle_transfer_encoding(Headers = #http_request_h{'transfer-encoding' = undefined}) -> + Headers; +handle_transfer_encoding(Headers) -> + %% RFC7230 3.3.2 + %% A sender MUST NOT send a 'Content-Length' header field in any message + %% that contains a 'Transfer-Encoding' header field. + Headers#http_request_h{'content-length' = undefined}. + body_length(Body) when is_binary(Body) -> integer_to_list(size(Body)); body_length(Body) when is_list(Body) -> integer_to_list(length(Body)). +%% Set 'Content-Type' when it is explicitly set. +handle_content_type(Headers, "") -> + Headers; +handle_content_type(Headers, ContentType) -> + Headers#http_request_h{'content-type' = ContentType}. + method(Method) -> http_util:to_upper(atom_to_list(Method)). diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 3d375222b5..8357e02014 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -156,6 +156,7 @@ only_simulated() -> multipart_chunks, get_space, delete_no_body, + post_with_content_type, stream_fun_server_close ]. @@ -170,7 +171,8 @@ misc() -> server_does_not_exist, timeout_memory_leak, wait_for_whole_response, - post_204_chunked + post_204_chunked, + chunkify_fun ]. sim_mixed() -> @@ -1408,7 +1410,8 @@ post_204_chunked(_Config) -> {ok, ListenSocket} = gen_tcp:listen(0, [{active,once}, binary]), {ok,{_,Port}} = inet:sockname(ListenSocket), - spawn(fun () -> custom_server(Msg, Chunk, ListenSocket) end), + spawn(fun () -> custom_server(Msg, Chunk, ListenSocket, + fun post_204_receive/0) end), {ok,Host} = inet:gethostname(), End = "/cgi-bin/erl/httpd_example:post_204", @@ -1418,16 +1421,26 @@ post_204_chunked(_Config) -> %% Second request times out in the faulty case. {ok, _} = httpc:request(post, {URL, [], "text/html", []}, [], []). -custom_server(Msg, Chunk, ListenSocket) -> +post_204_receive() -> + receive + {tcp, _, Msg} -> + ct:log("Message received: ~p", [Msg]) + after + 1000 -> + ct:fail("Timeout: did not recive packet") + end. + +%% Custom server is used to test special cases when using chunked encoding +custom_server(Msg, Chunk, ListenSocket, ReceiveFun) -> {ok, Accept} = gen_tcp:accept(ListenSocket), - receive_packet(), + ReceiveFun(), send_response(Msg, Chunk, Accept), - custom_server_loop(Msg, Chunk, Accept). + custom_server_loop(Msg, Chunk, Accept, ReceiveFun). -custom_server_loop(Msg, Chunk, Accept) -> - receive_packet(), +custom_server_loop(Msg, Chunk, Accept, ReceiveFun) -> + ReceiveFun(), send_response(Msg, Chunk, Accept), - custom_server_loop(Msg, Chunk, Accept). + custom_server_loop(Msg, Chunk, Accept, ReceiveFun). send_response(Msg, Chunk, Socket) -> inet:setopts(Socket, [{active, once}]), @@ -1435,15 +1448,54 @@ send_response(Msg, Chunk, Socket) -> timer:sleep(250), gen_tcp:send(Socket, Chunk). -receive_packet() -> +%%-------------------------------------------------------------------- +chunkify_fun() -> + [{doc,"Test that a chunked encoded request does not include the 'Content-Length header'"}]. +chunkify_fun(_Config) -> + Msg = "HTTP/1.1 204 No Content\r\n" ++ + "Date: Thu, 23 Aug 2018 13:36:29 GMT\r\n" ++ + "Content-Type: text/html\r\n" ++ + "Server: inets/6.5.2.3\r\n" ++ + "Cache-Control: no-cache\r\n" ++ + "Pragma: no-cache\r\n" ++ + "Expires: Fri, 24 Aug 2018 07:49:35 GMT\r\n" ++ + "Transfer-Encoding: chunked\r\n" ++ + "\r\n", + Chunk = "0\r\n\r\n", + + {ok, ListenSocket} = gen_tcp:listen(0, [{active,once}, binary]), + {ok,{_,Port}} = inet:sockname(ListenSocket), + spawn(fun () -> custom_server(Msg, Chunk, ListenSocket, + fun chunkify_receive/0) end), + + {ok,Host} = inet:gethostname(), + End = "/cgi-bin/erl/httpd_example", + URL = ?URL_START ++ Host ++ ":" ++ integer_to_list(Port) ++ End, + Fun = fun(_) -> {ok,<<1>>,eof_body} end, + Acc = start, + + {ok, {{_,204,_}, _, _}} = + httpc:request(put, {URL, [], "text/html", {chunkify, Fun, Acc}}, [], []). + +chunkify_receive() -> + Error = "HTTP/1.1 500 Internal Server Error\r\n" ++ + "Content-Length: 0\r\n\r\n", receive - {tcp, _, Msg} -> - ct:log("Message received: ~p", [Msg]) + {tcp, Port, Msg} -> + case binary:match(Msg, <<"content-length">>) of + nomatch -> + ct:log("Message received: ~s", [binary_to_list(Msg)]); + {_, _} -> + ct:log("Message received (negative): ~s", [binary_to_list(Msg)]), + %% Signal a testcase failure when the received HTTP request + %% contains a 'Content-Length' header. + gen_tcp:send(Port, Error), + ct:fail("Content-Length present in received headers.") + end after 1000 -> ct:fail("Timeout: did not recive packet") end. - %%-------------------------------------------------------------------- stream_fun_server_close() -> [{doc, "Test that an error msg is received when using a receiver fun as stream target"}]. @@ -1550,6 +1602,15 @@ delete_no_body(Config) when is_list(Config) -> httpc:request(delete, {URL, [], "text/plain", "TEST"}, [], []). %%-------------------------------------------------------------------- +post_with_content_type(doc) -> + ["Test that a POST request with explicit 'Content-Type' does not drop the 'Content-Type' header - Solves ERL-736"]; +post_with_content_type(Config) when is_list(Config) -> + URL = url(group_name(Config), "/delete_no_body.html", Config), + %% Simulated server replies 500 if 'Content-Type' header is present + {ok, {{_,500,_}, _, _}} = + httpc:request(post, {URL, [], "application/x-www-form-urlencoded", ""}, [], []). + +%%-------------------------------------------------------------------- request_options() -> [{doc, "Test http get request with socket options against local server (IPv6)"}]. request_options(Config) when is_list(Config) -> diff --git a/lib/kernel/src/kernel.appup.src b/lib/kernel/src/kernel.appup.src index 305a1c788c..0c0435e051 100644 --- a/lib/kernel/src/kernel.appup.src +++ b/lib/kernel/src/kernel.appup.src @@ -20,9 +20,11 @@ %% Up from - max one major revision back [{<<"5\\.3(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-20.0 {<<"5\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-20.1+ - {<<"6\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-21 + {<<"6\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-21.0 + {<<"6\\.1(\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-21.1 %% Down to - max one major revision back [{<<"5\\.3(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-20.0 {<<"5\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-20.1+ - {<<"6\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-21 + {<<"6\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-21.0 + {<<"6\\.1(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-21.1 }. diff --git a/lib/kernel/test/init_SUITE.erl b/lib/kernel/test/init_SUITE.erl index 6a006cdc01..7828cc4716 100644 --- a/lib/kernel/test/init_SUITE.erl +++ b/lib/kernel/test/init_SUITE.erl @@ -451,7 +451,7 @@ find_system_procs([], SysProcs) -> SysProcs#sys_procs.dirty_sig_handler_max]; find_system_procs([P|Ps], SysProcs) -> case process_info(P, [initial_call, priority]) of - [{initial_call,{otp_ring0,start,2}},_] -> + [{initial_call,{erl_init,start,2}},_] -> undefined = SysProcs#sys_procs.init, find_system_procs(Ps, SysProcs#sys_procs{init = P}); [{initial_call,{erts_code_purger,start,0}},_] -> diff --git a/lib/observer/src/cdv_wx.erl b/lib/observer/src/cdv_wx.erl index 78a897111c..e546997879 100644 --- a/lib/observer/src/cdv_wx.erl +++ b/lib/observer/src/cdv_wx.erl @@ -448,10 +448,7 @@ maybe_warn_filename(FileName) -> true -> continue; false -> - DumpName = case os:getenv("ERL_CRASH_DUMP") of - false -> filename:absname("erl_crash.dump"); - Name -> filename:absname(Name) - end, + DumpName = filename:absname(os:getenv("ERL_CRASH_DUMP", "erl_crash.dump")), case filename:absname(FileName) of DumpName -> Warning = diff --git a/lib/observer/test/crashdump_viewer_SUITE_data/old_format.dump b/lib/observer/test/crashdump_viewer_SUITE_data/old_format.dump index 2737757bfa..0b30fdf2f1 100644 --- a/lib/observer/test/crashdump_viewer_SUITE_data/old_format.dump +++ b/lib/observer/test/crashdump_viewer_SUITE_data/old_format.dump @@ -10,7 +10,7 @@ Compiled on Wed Dec 4 11:11:21 2002 Process Information -------------------------------------------------- <0.0.0> Waiting. Registered as: init -Spawned as: otp_ring0:start/2 +Spawned as: erl_init:start/2 Message buffer data: 4 words Link list: [<0.5.0>,<0.4.0>,<0.2.0>] Reductions 3125 stack+heap 610 old_heap_sz=233 @@ -68,7 +68,7 @@ y(3) [gen_event,<0.1.0>,<0.1.0>,{local,error_logger},[],[],[]] Spawned as: proc_lib:init_p/5 Message buffer data: 4 words Link list: [<0.7.0>,<0.0.0>] -Dictionary: [{'$initial_call',{gen,init_it,[gen_server,<0.1.0>,<0.1.0>,{local,application_controller},application_controller,{application,kernel,[{description,"ERTS CXC 138 10"},{vsn,"2.6.3.15"},{id,[]},{modules,[application,application_controller,application_master,application_starter,auth,code,code_aux,code_server,code_server_int,dist_util,erl_boot_server,erl_distribution,erl_open_port,erl_prim_loader,erl_reply,erlang,error_handler,error_logger,file,global,global_group,global_search,group,heart,inet6_tcp,inet6_tcp_dist,inet6_udp,inet_config,inet_hosts,inet_gethost_native,inet_tcp_dist,otp_pre_init,init,kernel,kernel_config,net,net_adm,net_kernel,os,ram_file,rpc,user,user_drv,user_sup,disk_log,disk_log_1,disk_log_server,disk_log_sup,dist_ac,erl_atom_cache,erl_ddll,erl_epmd,erl_external,erts_debug,fixtable_server,gen_tcp,gen_udp,prim_inet,inet,inet_db,inet_dns,inet_parse,inet_res,inet_tcp,inet_udp,pg2,seq_trace,socks5,socks5_auth,socks5_tcp,socks5_udp,wrap_log_reader,otp_ring0]},{registered,[application_controller,erl_reply,auth,boot_server,code_server,disk_log_server,disk_log_sup,erl_prim_loader,error_logger,file_server,fixtable_server,global_group,global_name_server,heart,init,kernel_config,kernel_sup,net_kernel,net_sup,rex,user,os_server,ddll_server,erl_epmd,inet_db,pg2]},{applications,[]},{included_applications,[]},{env,[{error_logger,tty}]},{start_phases,undefined},{maxT,infinity},{maxP,infinity},{mod,{kernel,[]}}]},[]]}},{'$ancestors',[<0.1.0>]}] +Dictionary: [{'$initial_call',{gen,init_it,[gen_server,<0.1.0>,<0.1.0>,{local,application_controller},application_controller,{application,kernel,[{description,"ERTS CXC 138 10"},{vsn,"2.6.3.15"},{id,[]},{modules,[application,application_controller,application_master,application_starter,auth,code,code_aux,code_server,code_server_int,dist_util,erl_boot_server,erl_distribution,erl_open_port,erl_prim_loader,erl_reply,erlang,error_handler,error_logger,file,global,global_group,global_search,group,heart,inet6_tcp,inet6_tcp_dist,inet6_udp,inet_config,inet_hosts,inet_gethost_native,inet_tcp_dist,otp_pre_init,init,kernel,kernel_config,net,net_adm,net_kernel,os,ram_file,rpc,user,user_drv,user_sup,disk_log,disk_log_1,disk_log_server,disk_log_sup,dist_ac,erl_atom_cache,erl_ddll,erl_epmd,erl_external,erts_debug,fixtable_server,gen_tcp,gen_udp,prim_inet,inet,inet_db,inet_dns,inet_parse,inet_res,inet_tcp,inet_udp,pg2,seq_trace,socks5,socks5_auth,socks5_tcp,socks5_udp,wrap_log_reader,erl_init]},{registered,[application_controller,erl_reply,auth,boot_server,code_server,disk_log_server,disk_log_sup,erl_prim_loader,error_logger,file_server,fixtable_server,global_group,global_name_server,heart,init,kernel_config,kernel_sup,net_kernel,net_sup,rex,user,os_server,ddll_server,erl_epmd,inet_db,pg2]},{applications,[]},{included_applications,[]},{env,[{error_logger,tty}]},{start_phases,undefined},{maxT,infinity},{maxP,infinity},{mod,{kernel,[]}}]},[]]}},{'$ancestors',[<0.1.0>]}] Reductions 527 stack+heap 1597 old_heap_sz=0 Heap unused=833 OldHeap unused=0 Stack dump: @@ -88,13 +88,13 @@ y(5) <0.1.0> y(0) Catch 0x33480C (proc_lib:init_p/5 + 164) y(1) gen y(2) init_it -y(3) [gen_server,<0.1.0>,<0.1.0>,{local,application_controller},application_controller,{application,kernel,[{description,"ERTS CXC 138 10"},{vsn,"2.6.3.15"},{id,[]},{modules,[application,application_controller,application_master,application_starter,auth,code,code_aux,code_server,code_server_int,dist_util,erl_boot_server,erl_distribution,erl_open_port,erl_prim_loader,erl_reply,erlang,error_handler,error_logger,file,global,global_group,global_search,group,heart,inet6_tcp,inet6_tcp_dist,inet6_udp,inet_config,inet_hosts,inet_gethost_native,inet_tcp_dist,otp_pre_init,init,kernel,kernel_config,net,net_adm,net_kernel,os,ram_file,rpc,user,user_drv,user_sup,disk_log,disk_log_1,disk_log_server,disk_log_sup,dist_ac,erl_atom_cache,erl_ddll,erl_epmd,erl_external,erts_debug,fixtable_server,gen_tcp,gen_udp,prim_inet,inet,inet_db,inet_dns,inet_parse,inet_res,inet_tcp,inet_udp,pg2,seq_trace,socks5,socks5_auth,socks5_tcp,socks5_udp,wrap_log_reader,otp_ring0]},{registered,[application_controller,erl_reply,auth,boot_server,code_server,disk_log_server,disk_log_sup,erl_prim_loader,error_logger,file_server,fixtable_server,global_group,global_name_server,heart,init,kernel_config,kernel_sup,net_kernel,net_sup,rex,user,os_server,ddll_server,erl_epmd,inet_db,pg2]},{applications,[]},{included_applications,[]},{env,[{error_logger,tty}]},{start_phases,undefined},{maxT,infinity},{maxP,infinity},{mod,{kernel,[]}}]},[]] +y(3) [gen_server,<0.1.0>,<0.1.0>,{local,application_controller},application_controller,{application,kernel,[{description,"ERTS CXC 138 10"},{vsn,"2.6.3.15"},{id,[]},{modules,[application,application_controller,application_master,application_starter,auth,code,code_aux,code_server,code_server_int,dist_util,erl_boot_server,erl_distribution,erl_open_port,erl_prim_loader,erl_reply,erlang,error_handler,error_logger,file,global,global_group,global_search,group,heart,inet6_tcp,inet6_tcp_dist,inet6_udp,inet_config,inet_hosts,inet_gethost_native,inet_tcp_dist,otp_pre_init,init,kernel,kernel_config,net,net_adm,net_kernel,os,ram_file,rpc,user,user_drv,user_sup,disk_log,disk_log_1,disk_log_server,disk_log_sup,dist_ac,erl_atom_cache,erl_ddll,erl_epmd,erl_external,erts_debug,fixtable_server,gen_tcp,gen_udp,prim_inet,inet,inet_db,inet_dns,inet_parse,inet_res,inet_tcp,inet_udp,pg2,seq_trace,socks5,socks5_auth,socks5_tcp,socks5_udp,wrap_log_reader,erl_init]},{registered,[application_controller,erl_reply,auth,boot_server,code_server,disk_log_server,disk_log_sup,erl_prim_loader,error_logger,file_server,fixtable_server,global_group,global_name_server,heart,init,kernel_config,kernel_sup,net_kernel,net_sup,rex,user,os_server,ddll_server,erl_epmd,inet_db,pg2]},{applications,[]},{included_applications,[]},{env,[{error_logger,tty}]},{start_phases,undefined},{maxT,infinity},{maxP,infinity},{mod,{kernel,[]}}]},[]] -------------------------------------------------- <0.7.0> Waiting. Spawned as: proc_lib:init_p/5 Message buffer data: 0 words Link list: [<0.8.0>,<0.5.0>] -Dictionary: [{'$initial_call',{application_master,init,[<0.5.0>,<0.6.0>,{appl_data,kernel,[application_controller,erl_reply,auth,boot_server,code_server,disk_log_server,disk_log_sup,erl_prim_loader,error_logger,file_server,fixtable_server,global_group,global_name_server,heart,init,kernel_config,kernel_sup,net_kernel,net_sup,rex,user,os_server,ddll_server,erl_epmd,inet_db,pg2],undefined,{kernel,[]},[application,application_controller,application_master,application_starter,auth,code,code_aux,code_server,code_server_int,dist_util,erl_boot_server,erl_distribution,erl_open_port,erl_prim_loader,erl_reply,erlang,error_handler,error_logger,file,global,global_group,global_search,group,heart,inet6_tcp,inet6_tcp_dist,inet6_udp,inet_config,inet_hosts,inet_gethost_native,inet_tcp_dist,otp_pre_init,init,kernel,kernel_config,net,net_adm,net_kernel,os,ram_file,rpc,user,user_drv,user_sup,disk_log,disk_log_1,disk_log_server,disk_log_sup,dist_ac,erl_atom_cache,erl_ddll,erl_epmd,erl_external,erts_debug,fixtable_server,gen_tcp,gen_udp,prim_inet,inet,inet_db,inet_dns,inet_parse,inet_res,inet_tcp,inet_udp,pg2,seq_trace,socks5,socks5_auth,socks5_tcp,socks5_udp,wrap_log_reader,otp_ring0],[],infinity,infinity},normal]}},{'$ancestors',[<0.6.0>]}] +Dictionary: [{'$initial_call',{application_master,init,[<0.5.0>,<0.6.0>,{appl_data,kernel,[application_controller,erl_reply,auth,boot_server,code_server,disk_log_server,disk_log_sup,erl_prim_loader,error_logger,file_server,fixtable_server,global_group,global_name_server,heart,init,kernel_config,kernel_sup,net_kernel,net_sup,rex,user,os_server,ddll_server,erl_epmd,inet_db,pg2],undefined,{kernel,[]},[application,application_controller,application_master,application_starter,auth,code,code_aux,code_server,code_server_int,dist_util,erl_boot_server,erl_distribution,erl_open_port,erl_prim_loader,erl_reply,erlang,error_handler,error_logger,file,global,global_group,global_search,group,heart,inet6_tcp,inet6_tcp_dist,inet6_udp,inet_config,inet_hosts,inet_gethost_native,inet_tcp_dist,otp_pre_init,init,kernel,kernel_config,net,net_adm,net_kernel,os,ram_file,rpc,user,user_drv,user_sup,disk_log,disk_log_1,disk_log_server,disk_log_sup,dist_ac,erl_atom_cache,erl_ddll,erl_epmd,erl_external,erts_debug,fixtable_server,gen_tcp,gen_udp,prim_inet,inet,inet_db,inet_dns,inet_parse,inet_res,inet_tcp,inet_udp,pg2,seq_trace,socks5,socks5_auth,socks5_tcp,socks5_udp,wrap_log_reader,erl_init],[],infinity,infinity},normal]}},{'$ancestors',[<0.6.0>]}] Reductions 45 stack+heap 377 old_heap_sz=377 Heap unused=306 OldHeap unused=377 Stack dump: @@ -103,14 +103,14 @@ cp = 0x33480c (proc_lib:init_p/5 + 164) arity = 0 347174 Return addr 0x33480C (proc_lib:init_p/5 + 164) -y(0) {state,<0.8.0>,{appl_data,kernel,[application_controller,erl_reply,auth,boot_server,code_server,disk_log_server,disk_log_sup,erl_prim_loader,error_logger,file_server,fixtable_server,global_group,global_name_server,heart,init,kernel_config,kernel_sup,net_kernel,net_sup,rex,user,os_server,ddll_server,erl_epmd,inet_db,pg2],undefined,{kernel,[]},[application,application_controller,application_master,application_starter,auth,code,code_aux,code_server,code_server_int,dist_util,erl_boot_server,erl_distribution,erl_open_port,erl_prim_loader,erl_reply,erlang,error_handler,error_logger,file,global,global_group,global_search,group,heart,inet6_tcp,inet6_tcp_dist,inet6_udp,inet_config,inet_hosts,inet_gethost_native,inet_tcp_dist,otp_pre_init,init,kernel,kernel_config,net,net_adm,net_kernel,os,ram_file,rpc,user,user_drv,user_sup,disk_log,disk_log_1,disk_log_server,disk_log_sup,dist_ac,erl_atom_cache,erl_ddll,erl_epmd,erl_external,erts_debug,fixtable_server,gen_tcp,gen_udp,prim_inet,inet,inet_db,inet_dns,inet_parse,inet_res,inet_tcp,inet_udp,pg2,seq_trace,socks5,socks5_auth,socks5_tcp,socks5_udp,wrap_log_reader,otp_ring0],[],infinity,infinity},[],0,<0.0.0>} +y(0) {state,<0.8.0>,{appl_data,kernel,[application_controller,erl_reply,auth,boot_server,code_server,disk_log_server,disk_log_sup,erl_prim_loader,error_logger,file_server,fixtable_server,global_group,global_name_server,heart,init,kernel_config,kernel_sup,net_kernel,net_sup,rex,user,os_server,ddll_server,erl_epmd,inet_db,pg2],undefined,{kernel,[]},[application,application_controller,application_master,application_starter,auth,code,code_aux,code_server,code_server_int,dist_util,erl_boot_server,erl_distribution,erl_open_port,erl_prim_loader,erl_reply,erlang,error_handler,error_logger,file,global,global_group,global_search,group,heart,inet6_tcp,inet6_tcp_dist,inet6_udp,inet_config,inet_hosts,inet_gethost_native,inet_tcp_dist,otp_pre_init,init,kernel,kernel_config,net,net_adm,net_kernel,os,ram_file,rpc,user,user_drv,user_sup,disk_log,disk_log_1,disk_log_server,disk_log_sup,dist_ac,erl_atom_cache,erl_ddll,erl_epmd,erl_external,erts_debug,fixtable_server,gen_tcp,gen_udp,prim_inet,inet,inet_db,inet_dns,inet_parse,inet_res,inet_tcp,inet_udp,pg2,seq_trace,socks5,socks5_auth,socks5_tcp,socks5_udp,wrap_log_reader,erl_init],[],infinity,infinity},[],0,<0.0.0>} y(1) <0.5.0> 347180 Return addr 0xFCEC8 (<terminate process normally>) y(0) Catch 0x33480C (proc_lib:init_p/5 + 164) y(1) application_master y(2) init -y(3) [<0.5.0>,<0.6.0>,{appl_data,kernel,[application_controller,erl_reply,auth,boot_server,code_server,disk_log_server,disk_log_sup,erl_prim_loader,error_logger,file_server,fixtable_server,global_group,global_name_server,heart,init,kernel_config,kernel_sup,net_kernel,net_sup,rex,user,os_server,ddll_server,erl_epmd,inet_db,pg2],undefined,{kernel,[]},[application,application_controller,application_master,application_starter,auth,code,code_aux,code_server,code_server_int,dist_util,erl_boot_server,erl_distribution,erl_open_port,erl_prim_loader,erl_reply,erlang,error_handler,error_logger,file,global,global_group,global_search,group,heart,inet6_tcp,inet6_tcp_dist,inet6_udp,inet_config,inet_hosts,inet_gethost_native,inet_tcp_dist,otp_pre_init,init,kernel,kernel_config,net,net_adm,net_kernel,os,ram_file,rpc,user,user_drv,user_sup,disk_log,disk_log_1,disk_log_server,disk_log_sup,dist_ac,erl_atom_cache,erl_ddll,erl_epmd,erl_external,erts_debug,fixtable_server,gen_tcp,gen_udp,prim_inet,inet,inet_db,inet_dns,inet_parse,inet_res,inet_tcp,inet_udp,pg2,seq_trace,socks5,socks5_auth,socks5_tcp,socks5_udp,wrap_log_reader,otp_ring0],[],infinity,infinity},normal] +y(3) [<0.5.0>,<0.6.0>,{appl_data,kernel,[application_controller,erl_reply,auth,boot_server,code_server,disk_log_server,disk_log_sup,erl_prim_loader,error_logger,file_server,fixtable_server,global_group,global_name_server,heart,init,kernel_config,kernel_sup,net_kernel,net_sup,rex,user,os_server,ddll_server,erl_epmd,inet_db,pg2],undefined,{kernel,[]},[application,application_controller,application_master,application_starter,auth,code,code_aux,code_server,code_server_int,dist_util,erl_boot_server,erl_distribution,erl_open_port,erl_prim_loader,erl_reply,erlang,error_handler,error_logger,file,global,global_group,global_search,group,heart,inet6_tcp,inet6_tcp_dist,inet6_udp,inet_config,inet_hosts,inet_gethost_native,inet_tcp_dist,otp_pre_init,init,kernel,kernel_config,net,net_adm,net_kernel,os,ram_file,rpc,user,user_drv,user_sup,disk_log,disk_log_1,disk_log_server,disk_log_sup,dist_ac,erl_atom_cache,erl_ddll,erl_epmd,erl_external,erts_debug,fixtable_server,gen_tcp,gen_udp,prim_inet,inet,inet_db,inet_dns,inet_parse,inet_res,inet_tcp,inet_udp,pg2,seq_trace,socks5,socks5_auth,socks5_tcp,socks5_udp,wrap_log_reader,erl_init],[],infinity,infinity},normal] -------------------------------------------------- <0.8.0> Waiting. Spawned as: application_master:start_it/4 @@ -652,7 +652,7 @@ Not alive Loaded Modules Information -------------------------------------------------- -otp_ring0 448 +erl_init 448 init 28000 prim_inet 34800 erl_prim_loader 14187 @@ -1202,7 +1202,7 @@ udp_error inet_async inet_reply empty_out_q -otp_ring0 +erl_init boot init run diff --git a/lib/reltool/doc/src/reltool_examples.xml b/lib/reltool/doc/src/reltool_examples.xml index 2a103119e6..f9a6fcf342 100644 --- a/lib/reltool/doc/src/reltool_examples.xml +++ b/lib/reltool/doc/src/reltool_examples.xml @@ -160,7 +160,7 @@ Eshell V9.0 (abort with ^G) {mod,erts_internal,[]}, {mod,erts_literal_area_collector,[]}, {mod,init,[]}, - {mod,otp_ring0,...}, + {mod,erl_init,...}, {mod,...}, {...}|...]}]}, {app,compiler, @@ -331,7 +331,7 @@ Eshell V10.0 (abort with ^G) {ok,{script,{"NAME","VSN"}, [{preLoaded,[erl_prim_loader,erl_tracer,erlang, erts_code_purger,erts_dirty_process_signal_handler, - erts_internal,erts_literal_area_collector,init,otp_ring0, + erts_internal,erts_literal_area_collector,init,erl_init, prim_eval,prim_file,prim_inet,prim_zip,zlib]}, {progress,preloaded}, {path,["$ROOT/lib/kernel-5.2/ebin", diff --git a/lib/runtime_tools/test/system_information_SUITE_data/information_test_report.dat b/lib/runtime_tools/test/system_information_SUITE_data/information_test_report.dat index a0e3806981..a9399a5d53 100644 --- a/lib/runtime_tools/test/system_information_SUITE_data/information_test_report.dat +++ b/lib/runtime_tools/test/system_information_SUITE_data/information_test_report.dat @@ -6829,7 +6829,7 @@ {native,false}, {compiler,"4.9.1"}, {md5,"55bb9fddcdf820938be2efee15eccd82"}]}, - {otp_ring0, + {erl_init, [{loaded,true}, {native,false}, {compiler,"4.9.1"}, diff --git a/lib/sasl/src/sasl.appup.src b/lib/sasl/src/sasl.appup.src index 83ee328af2..d37c5b3d95 100644 --- a/lib/sasl/src/sasl.appup.src +++ b/lib/sasl/src/sasl.appup.src @@ -19,10 +19,10 @@ {"%VSN%", %% Up from - max one major revision back [{<<"3\\.0\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-20.0 - {<<"3\\.1(\\.[0-2]+)*">>,[restart_new_emulator]}, % OTP-20.1+ - {<<"3\\.1(\\.[3-9]+)*">>,[restart_new_emulator]}], % OTP-21 + {<<"3\\.1(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-20.1+ + {<<"3\\.2(\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-21.* %% Down to - max one major revision back [{<<"3\\.0\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-20.* - {<<"3\\.1(\\.[0-2]+)*">>,[restart_new_emulator]}, % OTP-20.1+ - {<<"3\\.1(\\.[3-9]+)*">>,[restart_new_emulator]}] % OTP-21 + {<<"3\\.1(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-20.1+ + {<<"3\\.2(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-21.* }. diff --git a/lib/sasl/src/systools_make.erl b/lib/sasl/src/systools_make.erl index 6916107623..a0331fafd1 100644 --- a/lib/sasl/src/systools_make.erl +++ b/lib/sasl/src/systools_make.erl @@ -1562,10 +1562,10 @@ mandatory_modules() -> preloaded() -> %% Sorted - [erl_prim_loader,erl_tracer,erlang, + [erl_init,erl_prim_loader,erl_tracer,erlang, erts_code_purger,erts_dirty_process_signal_handler, erts_internal,erts_literal_area_collector, - init,otp_ring0,prim_buffer,prim_eval,prim_file, + init,prim_buffer,prim_eval,prim_file, prim_inet,prim_zip,zlib]. %%______________________________________________________________________ diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl index b05e4b7f24..8ed4505256 100644 --- a/lib/ssl/src/dtls_connection.erl +++ b/lib/ssl/src/dtls_connection.erl @@ -350,8 +350,8 @@ reinit_handshake_data(#state{protocol_buffers = Buffers} = State) -> dtls_handshake_later_fragments = [] }}. -select_sni_extension(#client_hello{extensions = HelloExtensions}) -> - HelloExtensions#hello_extensions.sni; +select_sni_extension(#client_hello{extensions = #{sni := SNI}}) -> + SNI; select_sni_extension(_) -> undefined. @@ -551,12 +551,12 @@ hello(internal, #client_hello{extensions = Extensions} = Hello, #state{ssl_optio start_or_recv_from = From} = State) -> {next_state, user_hello, State#state{start_or_recv_from = undefined, hello = Hello}, - [{reply, From, {ok, ssl_connection:map_extensions(Extensions)}}]}; + [{reply, From, {ok, Extensions}}]}; hello(internal, #server_hello{extensions = Extensions} = Hello, #state{ssl_options = #ssl_options{handshake = hello}, start_or_recv_from = From} = State) -> {next_state, user_hello, State#state{start_or_recv_from = undefined, hello = Hello}, - [{reply, From, {ok, ssl_connection:map_extensions(Extensions)}}]}; + [{reply, From, {ok, Extensions}}]}; hello(internal, #client_hello{cookie = Cookie} = Hello, #state{role = server, transport_cb = Transport, socket = Socket, diff --git a/lib/ssl/src/dtls_handshake.erl b/lib/ssl/src/dtls_handshake.erl index 3f70eaec8a..c8daa11433 100644 --- a/lib/ssl/src/dtls_handshake.erl +++ b/lib/ssl/src/dtls_handshake.erl @@ -169,10 +169,7 @@ handle_client_hello(Version, cipher_suites = CipherSuites, compression_methods = Compressions, random = Random, - extensions = - #hello_extensions{elliptic_curves = Curves, - signature_algs = ClientHashSigns} - = HelloExt}, + extensions = HelloExt}, #ssl_options{versions = Versions, signature_algs = SupportedHashSigns, eccs = SupportedECCs, @@ -181,6 +178,8 @@ handle_client_hello(Version, Renegotiation) -> case dtls_record:is_acceptable_version(Version, Versions) of true -> + Curves = maps:get(elliptic_curves, HelloExt, undefined), + ClientHashSigns = maps:get(signature_algs, HelloExt, undefined), TLSVersion = dtls_v1:corresponding_tls_version(Version), AvailableHashSigns = ssl_handshake:available_signature_algs( ClientHashSigns, SupportedHashSigns, Cert,TLSVersion), @@ -335,7 +334,7 @@ decode_handshake(Version, <<?BYTE(Type), Bin/binary>>) -> decode_handshake(_, ?HELLO_REQUEST, <<>>) -> #hello_request{}; -decode_handshake(_Version, ?CLIENT_HELLO, <<?UINT24(_), ?UINT16(_), +decode_handshake(Version, ?CLIENT_HELLO, <<?UINT24(_), ?UINT16(_), ?UINT24(_), ?UINT24(_), ?BYTE(Major), ?BYTE(Minor), Random:32/binary, ?BYTE(SID_length), Session_ID:SID_length/binary, @@ -343,8 +342,9 @@ decode_handshake(_Version, ?CLIENT_HELLO, <<?UINT24(_), ?UINT16(_), ?UINT16(Cs_length), CipherSuites:Cs_length/binary, ?BYTE(Cm_length), Comp_methods:Cm_length/binary, Extensions/binary>>) -> - - DecodedExtensions = ssl_handshake:decode_hello_extensions({client, Extensions}), + TLSVersion = dtls_v1:corresponding_tls_version(Version), + Exts = ssl_handshake:decode_vector(Extensions), + DecodedExtensions = ssl_handshake:decode_hello_extensions(Exts, TLSVersion, client), #client_hello{ client_version = {Major,Minor}, diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index ef9aac34bf..18d94adc18 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -614,6 +614,25 @@ getopts(#sslsocket{}, OptionTags) -> %% %% Description: Sets options %%-------------------------------------------------------------------- +setopts(#sslsocket{pid = [Pid, Sender]}, Options0) when is_pid(Pid), is_list(Options0) -> + try proplists:expand([{binary, [{mode, binary}]}, + {list, [{mode, list}]}], Options0) of + Options -> + case proplists:get_value(packet, Options, undefined) of + undefined -> + ssl_connection:set_opts(Pid, Options); + PacketOpt -> + case tls_sender:setopts(Sender, [{packet, PacketOpt}]) of + ok -> + ssl_connection:set_opts(Pid, Options); + Error -> + Error + end + end + catch + _:_ -> + {error, {options, {not_a_proplist, Options0}}} + end; setopts(#sslsocket{pid = [Pid|_]}, Options0) when is_pid(Pid), is_list(Options0) -> try proplists:expand([{binary, [{mode, binary}]}, {list, [{mode, list}]}], Options0) of diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index 66e96f8da5..3eb3b0a980 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -60,7 +60,7 @@ %% Help functions for tls|dtls_connection.erl -export([handle_session/7, ssl_config/3, - prepare_connection/2, hibernate_after/3, map_extensions/1]). + prepare_connection/2, hibernate_after/3]). %% General gen_statem state functions with extra callback argument %% to determine if it is an SSL/TLS or DTLS gen_statem machine @@ -335,21 +335,12 @@ prf(ConnectionPid, Secret, Label, Seed, WantedLength) -> %%==================================================================== %% Alert and close handling %%==================================================================== -handle_own_alert(Alert, Version, StateName, +handle_own_alert(Alert, _, StateName, #state{role = Role, - transport_cb = Transport, - socket = Socket, protocol_cb = Connection, - connection_states = ConnectionStates, ssl_options = SslOpts} = State) -> try %% Try to tell the other side - {BinMsg, _} = - Connection:encode_alert(Alert, Version, ConnectionStates), - Connection:send(Transport, Socket, BinMsg), - Report = #{direction => outbound, - protocol => 'tls_record', - message => BinMsg}, - ssl_logger:debug(SslOpts#ssl_options.log_level, Report, #{domain => [otp,ssl,tls_record]}) + send_alert(Alert, StateName, State) catch _:_ -> %% Can crash if we are in a uninitialized state ignore end, @@ -1170,24 +1161,20 @@ handle_call({close, {Pid, Timeout}}, From, StateName, State0, Connection) when i %% we must recive the close alert from the peer before releasing the %% transport socket. {next_state, downgrade, State#state{terminated = true}, [{timeout, Timeout, downgrade}]}; -handle_call({close, _} = Close, From, StateName, State, Connection) -> +handle_call({close, _} = Close, From, StateName, State, _Connection) -> %% Run terminate before returning so that the reuseaddr %% inet-option works properly - Result = Connection:terminate(Close, StateName, State#state{terminated = true}), + Result = terminate(Close, StateName, State), stop_and_reply( {shutdown, normal}, - {reply, From, Result}, State); -handle_call({shutdown, How0}, From, _, + {reply, From, Result}, State#state{terminated = true}); +handle_call({shutdown, How0}, From, StateName, #state{transport_cb = Transport, - negotiated_version = Version, - connection_states = ConnectionStates, - socket = Socket} = State, Connection) -> + socket = Socket} = State, _) -> case How0 of How when How == write; How == both -> - Alert = ?ALERT_REC(?WARNING, ?CLOSE_NOTIFY), - {BinMsg, _} = - Connection:encode_alert(Alert, Version, ConnectionStates), - Connection:send(Transport, Socket, BinMsg); + send_alert(?ALERT_REC(?WARNING, ?CLOSE_NOTIFY), + StateName, State); _ -> ok end, @@ -1353,14 +1340,20 @@ terminate({shutdown, own_alert}, _StateName, #state{ _ -> Connection:close({timeout, ?DEFAULT_TIMEOUT}, Socket, Transport, undefined, undefined) end; +terminate(downgrade = Reason, connection, #state{protocol_cb = Connection, + transport_cb = Transport, socket = Socket + } = State) -> + handle_trusted_certs_db(State), + Connection:close(Reason, Socket, Transport, undefined, undefined); terminate(Reason, connection, #state{protocol_cb = Connection, - connection_states = ConnectionStates, - ssl_options = #ssl_options{padding_check = Check}, - transport_cb = Transport, socket = Socket - } = State) -> + connection_states = ConnectionStates, + ssl_options = #ssl_options{padding_check = Check}, + transport_cb = Transport, socket = Socket + } = State) -> handle_trusted_certs_db(State), Alert = terminate_alert(Reason), - ok = Connection:send_alert_in_connection(Alert, State), + %% Send the termination ALERT if possible + catch (ok = Connection:send_alert_in_connection(Alert, State)), Connection:close(Reason, Socket, Transport, ConnectionStates, Check); terminate(Reason, _StateName, #state{transport_cb = Transport, protocol_cb = Connection, socket = Socket @@ -1397,6 +1390,11 @@ format_status(terminate, [_, StateName, State]) -> %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- +send_alert(Alert, connection, #state{protocol_cb = Connection} = State) -> + Connection:send_alert_in_connection(Alert, State); +send_alert(Alert, _, #state{protocol_cb = Connection} = State) -> + Connection:send_alert(Alert, State). + connection_info(#state{sni_hostname = SNIHostname, session = #session{session_id = SessionId, cipher_suite = CipherSuite, ecc = ECCCurve}, @@ -1427,7 +1425,7 @@ security_info(#state{connection_states = ConnectionStates}) -> ssl_record:current_connection_state(ConnectionStates, read), [{client_random, ClientRand}, {server_random, ServerRand}, {master_secret, MasterSecret}]. -do_server_hello(Type, #hello_extensions{next_protocol_negotiation = NextProtocols} = +do_server_hello(Type, #{next_protocol_negotiation := NextProtocols} = ServerHelloExt, #state{negotiated_version = Version, session = #session{session_id = SessId}, @@ -2351,22 +2349,6 @@ hibernate_after(connection = StateName, hibernate_after(StateName, State, Actions) -> {next_state, StateName, State, Actions}. -map_extensions(#hello_extensions{renegotiation_info = RenegotiationInfo, - signature_algs = SigAlg, - alpn = Alpn, - next_protocol_negotiation = Next, - srp = SRP, - ec_point_formats = ECPointFmt, - elliptic_curves = ECCCurves, - sni = SNI}) -> - #{renegotiation_info => ssl_handshake:extension_value(RenegotiationInfo), - signature_algs => ssl_handshake:extension_value(SigAlg), - alpn => ssl_handshake:extension_value(Alpn), - srp => ssl_handshake:extension_value(SRP), - next_protocol => ssl_handshake:extension_value(Next), - ec_point_formats => ssl_handshake:extension_value(ECPointFmt), - elliptic_curves => ssl_handshake:extension_value(ECCCurves), - sni => ssl_handshake:extension_value(SNI)}. terminate_alert(normal) -> ?ALERT_REC(?WARNING, ?CLOSE_NOTIFY); diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index ced3c2675e..ba0b670091 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -57,10 +57,10 @@ ]). %% Encode --export([encode_handshake/2, encode_hello_extensions/1, +-export([encode_handshake/2, encode_hello_extensions/1, encode_extensions/1, encode_extensions/2, encode_client_protocol_negotiation/2, encode_protocols_advertised_on_server/1]). %% Decode --export([decode_handshake/3, decode_hello_extensions/1, +-export([decode_handshake/3, decode_vector/1, decode_hello_extensions/3, decode_extensions/1, decode_server_key/3, decode_client_key/3, decode_suites/2 ]). @@ -93,7 +93,7 @@ hello_request() -> %%-------------------------------------------------------------------- -spec server_hello(#session{}, ssl_record:ssl_version(), ssl_record:connection_states(), - #hello_extensions{}) -> #server_hello{}. + Extension::map()) -> #server_hello{}. %% %% Description: Creates a server hello message. %%-------------------------------------------------------------------- @@ -532,7 +532,7 @@ encode_handshake(#server_hello{server_version = {Major, Minor}, session_id = Session_ID, cipher_suite = CipherSuite, compression_method = Comp_method, - extensions = #hello_extensions{} = Extensions}, _Version) -> + extensions = Extensions}, _Version) -> SID_length = byte_size(Session_ID), ExtensionsBin = encode_hello_extensions(Extensions), {?SERVER_HELLO, <<?BYTE(Major), ?BYTE(Minor), Random:32/binary, @@ -582,93 +582,94 @@ encode_handshake(#certificate_verify{signature = BinSig, hashsign_algorithm = Ha encode_handshake(#finished{verify_data = VerifyData}, _Version) -> {?FINISHED, VerifyData}. -encode_hello_extensions(#hello_extensions{} = Extensions) -> - encode_hello_extensions(hello_extensions_list(Extensions), <<>>). -encode_hello_extensions([], <<>>) -> +encode_hello_extensions(Extensions) -> + encode_extensions(hello_extensions_list(Extensions), <<>>). + +encode_extensions(Exts) -> + encode_extensions(Exts, <<>>). + +encode_extensions([], <<>>) -> <<>>; -encode_hello_extensions([], Acc) -> +encode_extensions([], Acc) -> Size = byte_size(Acc), <<?UINT16(Size), Acc/binary>>; - -encode_hello_extensions([#alpn{extension_data = ExtensionData} | Rest], Acc) -> - Len = byte_size(ExtensionData), +encode_extensions([#alpn{extension_data = ExtensionData} | Rest], Acc) -> + Len = byte_size(ExtensionData), ExtLen = Len + 2, - encode_hello_extensions(Rest, <<?UINT16(?ALPN_EXT), ?UINT16(ExtLen), ?UINT16(Len), - ExtensionData/binary, Acc/binary>>); -encode_hello_extensions([#next_protocol_negotiation{extension_data = ExtensionData} | Rest], Acc) -> + encode_extensions(Rest, <<?UINT16(?ALPN_EXT), ?UINT16(ExtLen), ?UINT16(Len), + ExtensionData/binary, Acc/binary>>); +encode_extensions([#next_protocol_negotiation{extension_data = ExtensionData} | Rest], Acc) -> Len = byte_size(ExtensionData), - encode_hello_extensions(Rest, <<?UINT16(?NEXTPROTONEG_EXT), ?UINT16(Len), + encode_extensions(Rest, <<?UINT16(?NEXTPROTONEG_EXT), ?UINT16(Len), ExtensionData/binary, Acc/binary>>); -encode_hello_extensions([#renegotiation_info{renegotiated_connection = undefined} | Rest], Acc) -> - encode_hello_extensions(Rest, Acc); -encode_hello_extensions([#renegotiation_info{renegotiated_connection = ?byte(0) = Info} | Rest], Acc) -> +encode_extensions([#renegotiation_info{renegotiated_connection = undefined} | Rest], Acc) -> + encode_extensions(Rest, Acc); +encode_extensions([#renegotiation_info{renegotiated_connection = ?byte(0) = Info} | Rest], Acc) -> Len = byte_size(Info), - encode_hello_extensions(Rest, <<?UINT16(?RENEGOTIATION_EXT), ?UINT16(Len), Info/binary, Acc/binary>>); + encode_extensions(Rest, <<?UINT16(?RENEGOTIATION_EXT), ?UINT16(Len), Info/binary, Acc/binary>>); -encode_hello_extensions([#renegotiation_info{renegotiated_connection = Info} | Rest], Acc) -> +encode_extensions([#renegotiation_info{renegotiated_connection = Info} | Rest], Acc) -> InfoLen = byte_size(Info), Len = InfoLen +1, - encode_hello_extensions(Rest, <<?UINT16(?RENEGOTIATION_EXT), ?UINT16(Len), ?BYTE(InfoLen), + encode_extensions(Rest, <<?UINT16(?RENEGOTIATION_EXT), ?UINT16(Len), ?BYTE(InfoLen), Info/binary, Acc/binary>>); -encode_hello_extensions([#elliptic_curves{elliptic_curve_list = EllipticCurves} | Rest], Acc) -> +encode_extensions([#elliptic_curves{elliptic_curve_list = EllipticCurves} | Rest], Acc) -> EllipticCurveList = << <<(tls_v1:oid_to_enum(X)):16>> || X <- EllipticCurves>>, ListLen = byte_size(EllipticCurveList), Len = ListLen + 2, - encode_hello_extensions(Rest, <<?UINT16(?ELLIPTIC_CURVES_EXT), + encode_extensions(Rest, <<?UINT16(?ELLIPTIC_CURVES_EXT), ?UINT16(Len), ?UINT16(ListLen), EllipticCurveList/binary, Acc/binary>>); -encode_hello_extensions([#ec_point_formats{ec_point_format_list = ECPointFormats} | Rest], Acc) -> +encode_extensions([#ec_point_formats{ec_point_format_list = ECPointFormats} | Rest], Acc) -> ECPointFormatList = list_to_binary(ECPointFormats), ListLen = byte_size(ECPointFormatList), Len = ListLen + 1, - encode_hello_extensions(Rest, <<?UINT16(?EC_POINT_FORMATS_EXT), + encode_extensions(Rest, <<?UINT16(?EC_POINT_FORMATS_EXT), ?UINT16(Len), ?BYTE(ListLen), ECPointFormatList/binary, Acc/binary>>); -encode_hello_extensions([#srp{username = UserName} | Rest], Acc) -> +encode_extensions([#srp{username = UserName} | Rest], Acc) -> SRPLen = byte_size(UserName), Len = SRPLen + 2, - encode_hello_extensions(Rest, <<?UINT16(?SRP_EXT), ?UINT16(Len), ?BYTE(SRPLen), + encode_extensions(Rest, <<?UINT16(?SRP_EXT), ?UINT16(Len), ?BYTE(SRPLen), UserName/binary, Acc/binary>>); -encode_hello_extensions([#hash_sign_algos{hash_sign_algos = HashSignAlgos} | Rest], Acc) -> +encode_extensions([#hash_sign_algos{hash_sign_algos = HashSignAlgos} | Rest], Acc) -> SignAlgoList = << <<(ssl_cipher:hash_algorithm(Hash)):8, (ssl_cipher:sign_algorithm(Sign)):8>> || {Hash, Sign} <- HashSignAlgos >>, ListLen = byte_size(SignAlgoList), Len = ListLen + 2, - encode_hello_extensions(Rest, <<?UINT16(?SIGNATURE_ALGORITHMS_EXT), + encode_extensions(Rest, <<?UINT16(?SIGNATURE_ALGORITHMS_EXT), ?UINT16(Len), ?UINT16(ListLen), SignAlgoList/binary, Acc/binary>>); -encode_hello_extensions([#signature_scheme_list{ +encode_extensions([#signature_scheme_list{ signature_scheme_list = SignatureSchemes} | Rest], Acc) -> SignSchemeList = << <<(ssl_cipher:signature_scheme(SignatureScheme)):16 >> || SignatureScheme <- SignatureSchemes >>, ListLen = byte_size(SignSchemeList), Len = ListLen + 2, - encode_hello_extensions(Rest, <<?UINT16(?SIGNATURE_ALGORITHMS_CERT_EXT), + encode_extensions(Rest, <<?UINT16(?SIGNATURE_ALGORITHMS_CERT_EXT), ?UINT16(Len), ?UINT16(ListLen), SignSchemeList/binary, Acc/binary>>); -encode_hello_extensions([#sni{hostname = Hostname} | Rest], Acc) -> +encode_extensions([#sni{hostname = Hostname} | Rest], Acc) -> HostLen = length(Hostname), HostnameBin = list_to_binary(Hostname), % Hostname type (1 byte) + Hostname length (2 bytes) + Hostname (HostLen bytes) ServerNameLength = 1 + 2 + HostLen, % ServerNameListSize (2 bytes) + ServerNameLength ExtLength = 2 + ServerNameLength, - encode_hello_extensions(Rest, <<?UINT16(?SNI_EXT), ?UINT16(ExtLength), - ?UINT16(ServerNameLength), - ?BYTE(?SNI_NAMETYPE_HOST_NAME), - ?UINT16(HostLen), HostnameBin/binary, - Acc/binary>>); -encode_hello_extensions([#client_hello_versions{versions = Versions0} | Rest], Acc) -> + encode_extensions(Rest, <<?UINT16(?SNI_EXT), ?UINT16(ExtLength), + ?UINT16(ServerNameLength), + ?BYTE(?SNI_NAMETYPE_HOST_NAME), + ?UINT16(HostLen), HostnameBin/binary, + Acc/binary>>); +encode_extensions([#client_hello_versions{versions = Versions0} | Rest], Acc) -> Versions = encode_versions(Versions0), VerLen = byte_size(Versions), Len = VerLen + 2, - encode_hello_extensions(Rest, <<?UINT16(?SUPPORTED_VERSIONS_EXT), + encode_extensions(Rest, <<?UINT16(?SUPPORTED_VERSIONS_EXT), ?UINT16(Len), ?UINT16(VerLen), Versions/binary, Acc/binary>>); -encode_hello_extensions([#server_hello_selected_version{selected_version = Version0} | Rest], Acc) -> - Version = encode_versions(Version0), +encode_extensions([#server_hello_selected_version{selected_version = Version0} | Rest], Acc) -> + Version = encode_versions([Version0]), Len = byte_size(Version), %% 2 - encode_hello_extensions(Rest, <<?UINT16(?SUPPORTED_VERSIONS_EXT), + encode_extensions(Rest, <<?UINT16(?SUPPORTED_VERSIONS_EXT), ?UINT16(Len), Version/binary, Acc/binary>>). - - encode_client_protocol_negotiation(undefined, _) -> undefined; encode_client_protocol_negotiation(_, false) -> @@ -693,7 +694,7 @@ decode_handshake(_, ?NEXT_PROTOCOL, <<?BYTE(SelectedProtocolLength), ?BYTE(PaddingLength), _Padding:PaddingLength/binary>>) -> #next_protocol{selected_protocol = SelectedProtocol}; -decode_handshake(_Version, ?SERVER_HELLO, <<?BYTE(Major), ?BYTE(Minor), Random:32/binary, +decode_handshake(Version, ?SERVER_HELLO, <<?BYTE(Major), ?BYTE(Minor), Random:32/binary, ?BYTE(SID_length), Session_ID:SID_length/binary, Cipher_suite:2/binary, ?BYTE(Comp_method)>>) -> #server_hello{ @@ -702,14 +703,14 @@ decode_handshake(_Version, ?SERVER_HELLO, <<?BYTE(Major), ?BYTE(Minor), Random:3 session_id = Session_ID, cipher_suite = Cipher_suite, compression_method = Comp_method, - extensions = #hello_extensions{}}; + extensions = empty_hello_extensions(Version, server)}; -decode_handshake(_Version, ?SERVER_HELLO, <<?BYTE(Major), ?BYTE(Minor), Random:32/binary, +decode_handshake(Version, ?SERVER_HELLO, <<?BYTE(Major), ?BYTE(Minor), Random:32/binary, ?BYTE(SID_length), Session_ID:SID_length/binary, Cipher_suite:2/binary, ?BYTE(Comp_method), ?UINT16(ExtLen), Extensions:ExtLen/binary>>) -> - HelloExtensions = decode_hello_extensions(Extensions), + HelloExtensions = decode_hello_extensions(Extensions, Version, server), #server_hello{ server_version = {Major,Minor}, @@ -752,17 +753,34 @@ decode_handshake(_Version, ?FINISHED, VerifyData) -> decode_handshake(_, Message, _) -> throw(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, {unknown_or_malformed_handshake, Message})). + %%-------------------------------------------------------------------- --spec decode_hello_extensions({client, binary()} | binary()) -> #hello_extensions{}. +-spec decode_vector(binary()) -> binary(). +%% +%% Description: Remove length tag from TLS Vector type. Needed +%% for client hello when extensions in older versions may be empty. +%% +%%-------------------------------------------------------------------- +decode_vector(<<>>) -> + <<>>; +decode_vector(<<?UINT16(Len), Vector:Len/binary>>) -> + Vector. + +%%-------------------------------------------------------------------- +-spec decode_hello_extensions(binary(), ssl_record:ssl_version(), client | server) -> map(). +%% +%% Description: Decodes TLS hello extensions +%%-------------------------------------------------------------------- +decode_hello_extensions(Extensions, Version, Role) -> + decode_extensions(Extensions, empty_hello_extensions(Version, Role)). + +%%-------------------------------------------------------------------- +-spec decode_extensions(binary()) -> map(). %% %% Description: Decodes TLS hello extensions %%-------------------------------------------------------------------- -decode_hello_extensions({client, <<>>}) -> - #hello_extensions{}; -decode_hello_extensions({client, <<?UINT16(ExtLen), Extensions:ExtLen/binary>>}) -> - decode_hello_extensions(Extensions); -decode_hello_extensions(Extensions) -> - dec_hello_extensions(Extensions, #hello_extensions{}). +decode_extensions(Extensions) -> + decode_extensions(Extensions, empty_extensions()). %%-------------------------------------------------------------------- -spec decode_server_key(binary(), ssl_cipher_format:key_algo(), ssl_record:ssl_version()) -> @@ -979,57 +997,52 @@ client_hello_extensions(Version, CipherSuites, end, SRP = srp_user(SslOpts), - HelloExtensions = - #hello_extensions{ - renegotiation_info = renegotiation_info(tls_record, client, - ConnectionStates, Renegotiation), - srp = SRP, - signature_algs = available_signature_algs(SupportedHashSigns, Version), - ec_point_formats = EcPointFormats, - elliptic_curves = EllipticCurves, - alpn = encode_alpn(SslOpts#ssl_options.alpn_advertised_protocols, Renegotiation), - next_protocol_negotiation = - encode_client_protocol_negotiation(SslOpts#ssl_options.next_protocol_selector, - Renegotiation), - sni = sni(SslOpts#ssl_options.server_name_indication)}, + HelloExtensions = #{renegotiation_info => renegotiation_info(tls_record, client, + ConnectionStates, Renegotiation), + srp => SRP, + signature_algs => available_signature_algs(SupportedHashSigns, Version), + ec_point_formats => EcPointFormats, + elliptic_curves => EllipticCurves, + alpn => encode_alpn(SslOpts#ssl_options.alpn_advertised_protocols, Renegotiation), + next_protocol_negotiation => + encode_client_protocol_negotiation(SslOpts#ssl_options.next_protocol_selector, + Renegotiation), + sni => sni(SslOpts#ssl_options.server_name_indication) + }, %% Add "supported_versions" extension if TLS 1.3 case Version of {3,4} -> - HelloExtensions#hello_extensions{ - client_hello_versions = #client_hello_versions{ - versions = Versions}, - signature_algs_cert = #signature_scheme_list{ - signature_scheme_list = SignatureSchemes}}; + HelloExtensions#{client_hello_versions => + #client_hello_versions{versions = Versions}, + signature_algs_cert => + #signature_scheme_list{signature_scheme_list = SignatureSchemes}}; _Else -> HelloExtensions end. handle_client_hello_extensions(RecordCB, Random, ClientCipherSuites, - #hello_extensions{renegotiation_info = Info, - srp = SRP, - ec_point_formats = ECCFormat, - alpn = ALPN, - next_protocol_negotiation = NextProtocolNegotiation}, Version, + Exts, Version, #ssl_options{secure_renegotiate = SecureRenegotation, alpn_preferred_protocols = ALPNPreferredProtocols} = Opts, #session{cipher_suite = NegotiatedCipherSuite, compression_method = Compression} = Session0, ConnectionStates0, Renegotiation) -> - Session = handle_srp_extension(SRP, Session0), - ConnectionStates = handle_renegotiation_extension(server, RecordCB, Version, Info, + Session = handle_srp_extension(maps:get(srp, Exts, undefined), Session0), + ConnectionStates = handle_renegotiation_extension(server, RecordCB, Version, maps:get(renegotiation_info, Exts, undefined), Random, NegotiatedCipherSuite, ClientCipherSuites, Compression, ConnectionStates0, Renegotiation, SecureRenegotation), - ServerHelloExtensions = #hello_extensions{ - renegotiation_info = renegotiation_info(RecordCB, server, - ConnectionStates, Renegotiation), - ec_point_formats = server_ecc_extension(Version, ECCFormat) - }, - + Empty = empty_hello_extensions(Version, client), + ServerHelloExtensions = Empty#{renegotiation_info => renegotiation_info(RecordCB, server, + ConnectionStates, Renegotiation), + ec_point_formats => server_ecc_extension(Version, maps:get(ec_point_formats, Exts, undefined)) + }, + %% If we receive an ALPN extension and have ALPN configured for this connection, %% we handle it. Otherwise we check for the NPN extension. + ALPN = maps:get(alpn, Exts, undefined), if ALPN =/= undefined, ALPNPreferredProtocols =/= undefined -> case handle_alpn_extension(ALPNPreferredProtocols, decode_alpn(ALPN)) of @@ -1037,35 +1050,36 @@ handle_client_hello_extensions(RecordCB, Random, ClientCipherSuites, Alert; Protocol -> {Session, ConnectionStates, Protocol, - ServerHelloExtensions#hello_extensions{alpn=encode_alpn([Protocol], Renegotiation)}} + ServerHelloExtensions#{alpn => encode_alpn([Protocol], Renegotiation)}} end; true -> + NextProtocolNegotiation = maps:get(next_protocol_negotiation, Exts, undefined), ProtocolsToAdvertise = handle_next_protocol_extension(NextProtocolNegotiation, Renegotiation, Opts), {Session, ConnectionStates, undefined, - ServerHelloExtensions#hello_extensions{next_protocol_negotiation= - encode_protocols_advertised_on_server(ProtocolsToAdvertise)}} + ServerHelloExtensions#{next_protocol_negotiation => + encode_protocols_advertised_on_server(ProtocolsToAdvertise)}} end. handle_server_hello_extensions(RecordCB, Random, CipherSuite, Compression, - #hello_extensions{renegotiation_info = Info, - alpn = ALPN, - next_protocol_negotiation = NextProtocolNegotiation}, Version, + Exts, Version, #ssl_options{secure_renegotiate = SecureRenegotation, next_protocol_selector = NextProtoSelector}, ConnectionStates0, Renegotiation) -> - ConnectionStates = handle_renegotiation_extension(client, RecordCB, Version, Info, Random, + ConnectionStates = handle_renegotiation_extension(client, RecordCB, Version, maps:get(renegotiation_info, Exts, undefined), Random, CipherSuite, undefined, Compression, ConnectionStates0, Renegotiation, SecureRenegotation), %% If we receive an ALPN extension then this is the protocol selected, %% otherwise handle the NPN extension. + ALPN = maps:get(alpn, Exts, undefined), case decode_alpn(ALPN) of %% ServerHello contains exactly one protocol: the one selected. %% We also ignore the ALPN extension during renegotiation (see encode_alpn/2). [Protocol] when not Renegotiation -> {ConnectionStates, alpn, Protocol}; undefined -> + NextProtocolNegotiation = maps:get(next_protocol_negotiation, Exts, undefined), case handle_next_protocol(NextProtocolNegotiation, NextProtoSelector, Renegotiation) of #alert{} = Alert -> Alert; @@ -1823,21 +1837,8 @@ encode_versions([{M,N}|T], Acc) -> encode_versions(T, <<?BYTE(M),?BYTE(N),Acc/binary>>). -hello_extensions_list(#hello_extensions{renegotiation_info = RenegotiationInfo, - srp = SRP, - signature_algs = HashSigns, - signature_algs_cert = SignatureSchemes, - ec_point_formats = EcPointFormats, - elliptic_curves = EllipticCurves, - alpn = ALPN, - next_protocol_negotiation = NextProtocolNegotiation, - sni = Sni, - client_hello_versions = Versions, - server_hello_selected_version = Version}) -> - [Ext || Ext <- [RenegotiationInfo, SRP, HashSigns, SignatureSchemes, - EcPointFormats, EllipticCurves, ALPN, - NextProtocolNegotiation, Sni, - Versions, Version], Ext =/= undefined]. +hello_extensions_list(HelloExtensions) -> + [Ext || {_, Ext} <- maps:to_list(HelloExtensions), Ext =/= undefined]. %%-------------Decode handshakes--------------------------------- dec_server_key(<<?UINT16(PLen), P:PLen/binary, @@ -1977,16 +1978,16 @@ dec_server_key_signature(Params, <<?UINT16(Len), Signature:Len/binary>>, _) -> dec_server_key_signature(_, _, _) -> throw(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, failed_to_decrypt_server_key_sign)). -dec_hello_extensions(<<>>, Acc) -> +decode_extensions(<<>>, Acc) -> Acc; -dec_hello_extensions(<<?UINT16(?ALPN_EXT), ?UINT16(ExtLen), ?UINT16(Len), ExtensionData:Len/binary, Rest/binary>>, Acc) +decode_extensions(<<?UINT16(?ALPN_EXT), ?UINT16(ExtLen), ?UINT16(Len), ExtensionData:Len/binary, Rest/binary>>, Acc) when Len + 2 =:= ExtLen -> ALPN = #alpn{extension_data = ExtensionData}, - dec_hello_extensions(Rest, Acc#hello_extensions{alpn = ALPN}); -dec_hello_extensions(<<?UINT16(?NEXTPROTONEG_EXT), ?UINT16(Len), ExtensionData:Len/binary, Rest/binary>>, Acc) -> + decode_extensions(Rest, Acc#{alpn => ALPN}); +decode_extensions(<<?UINT16(?NEXTPROTONEG_EXT), ?UINT16(Len), ExtensionData:Len/binary, Rest/binary>>, Acc) -> NextP = #next_protocol_negotiation{extension_data = ExtensionData}, - dec_hello_extensions(Rest, Acc#hello_extensions{next_protocol_negotiation = NextP}); -dec_hello_extensions(<<?UINT16(?RENEGOTIATION_EXT), ?UINT16(Len), Info:Len/binary, Rest/binary>>, Acc) -> + decode_extensions(Rest, Acc#{next_protocol_negotiation => NextP}); +decode_extensions(<<?UINT16(?RENEGOTIATION_EXT), ?UINT16(Len), Info:Len/binary, Rest/binary>>, Acc) -> RenegotiateInfo = case Len of 1 -> % Initial handshake Info; % should be <<0>> will be matched in handle_renegotiation_info @@ -1995,34 +1996,34 @@ dec_hello_extensions(<<?UINT16(?RENEGOTIATION_EXT), ?UINT16(Len), Info:Len/binar <<?BYTE(VerifyLen), VerifyInfo/binary>> = Info, VerifyInfo end, - dec_hello_extensions(Rest, Acc#hello_extensions{renegotiation_info = - #renegotiation_info{renegotiated_connection = - RenegotiateInfo}}); + decode_extensions(Rest, Acc#{renegotiation_info => + #renegotiation_info{renegotiated_connection = + RenegotiateInfo}}); -dec_hello_extensions(<<?UINT16(?SRP_EXT), ?UINT16(Len), ?BYTE(SRPLen), SRP:SRPLen/binary, Rest/binary>>, Acc) +decode_extensions(<<?UINT16(?SRP_EXT), ?UINT16(Len), ?BYTE(SRPLen), SRP:SRPLen/binary, Rest/binary>>, Acc) when Len == SRPLen + 2 -> - dec_hello_extensions(Rest, Acc#hello_extensions{srp = #srp{username = SRP}}); + decode_extensions(Rest, Acc#{srp => #srp{username = SRP}}); -dec_hello_extensions(<<?UINT16(?SIGNATURE_ALGORITHMS_EXT), ?UINT16(Len), +decode_extensions(<<?UINT16(?SIGNATURE_ALGORITHMS_EXT), ?UINT16(Len), ExtData:Len/binary, Rest/binary>>, Acc) -> SignAlgoListLen = Len - 2, <<?UINT16(SignAlgoListLen), SignAlgoList/binary>> = ExtData, HashSignAlgos = [{ssl_cipher:hash_algorithm(Hash), ssl_cipher:sign_algorithm(Sign)} || <<?BYTE(Hash), ?BYTE(Sign)>> <= SignAlgoList], - dec_hello_extensions(Rest, Acc#hello_extensions{signature_algs = - #hash_sign_algos{hash_sign_algos = HashSignAlgos}}); + decode_extensions(Rest, Acc#{signature_algs => + #hash_sign_algos{hash_sign_algos = HashSignAlgos}}); -dec_hello_extensions(<<?UINT16(?SIGNATURE_ALGORITHMS_CERT_EXT), ?UINT16(Len), +decode_extensions(<<?UINT16(?SIGNATURE_ALGORITHMS_CERT_EXT), ?UINT16(Len), ExtData:Len/binary, Rest/binary>>, Acc) -> SignSchemeListLen = Len - 2, <<?UINT16(SignSchemeListLen), SignSchemeList/binary>> = ExtData, SignSchemes = [ssl_cipher:signature_scheme(SignScheme) || <<?UINT16(SignScheme)>> <= SignSchemeList], - dec_hello_extensions(Rest, Acc#hello_extensions{signature_algs_cert = - #signature_scheme_list{ - signature_scheme_list = SignSchemes}}); + decode_extensions(Rest, Acc#{signature_algs_cert => + #signature_scheme_list{ + signature_scheme_list = SignSchemes}}); -dec_hello_extensions(<<?UINT16(?ELLIPTIC_CURVES_EXT), ?UINT16(Len), +decode_extensions(<<?UINT16(?ELLIPTIC_CURVES_EXT), ?UINT16(Len), ExtData:Len/binary, Rest/binary>>, Acc) -> <<?UINT16(_), EllipticCurveList/binary>> = ExtData, %% Ignore unknown curves @@ -2035,44 +2036,42 @@ dec_hello_extensions(<<?UINT16(?ELLIPTIC_CURVES_EXT), ?UINT16(Len), end end, EllipticCurves = lists:filtermap(Pick, [ECC || <<ECC:16>> <= EllipticCurveList]), - dec_hello_extensions(Rest, Acc#hello_extensions{elliptic_curves = - #elliptic_curves{elliptic_curve_list = - EllipticCurves}}); -dec_hello_extensions(<<?UINT16(?EC_POINT_FORMATS_EXT), ?UINT16(Len), + decode_extensions(Rest, Acc#{elliptic_curves => + #elliptic_curves{elliptic_curve_list = + EllipticCurves}}); +decode_extensions(<<?UINT16(?EC_POINT_FORMATS_EXT), ?UINT16(Len), ExtData:Len/binary, Rest/binary>>, Acc) -> <<?BYTE(_), ECPointFormatList/binary>> = ExtData, ECPointFormats = binary_to_list(ECPointFormatList), - dec_hello_extensions(Rest, Acc#hello_extensions{ec_point_formats = - #ec_point_formats{ec_point_format_list = - ECPointFormats}}); + decode_extensions(Rest, Acc#{ec_point_formats => + #ec_point_formats{ec_point_format_list = + ECPointFormats}}); -dec_hello_extensions(<<?UINT16(?SNI_EXT), ?UINT16(Len), Rest/binary>>, Acc) when Len == 0 -> - dec_hello_extensions(Rest, Acc#hello_extensions{sni = #sni{hostname = ""}}); %% Server may send an empy SNI +decode_extensions(<<?UINT16(?SNI_EXT), ?UINT16(Len), Rest/binary>>, Acc) when Len == 0 -> + decode_extensions(Rest, Acc#{sni => #sni{hostname = ""}}); %% Server may send an empy SNI -dec_hello_extensions(<<?UINT16(?SNI_EXT), ?UINT16(Len), +decode_extensions(<<?UINT16(?SNI_EXT), ?UINT16(Len), ExtData:Len/binary, Rest/binary>>, Acc) -> <<?UINT16(_), NameList/binary>> = ExtData, - dec_hello_extensions(Rest, Acc#hello_extensions{sni = dec_sni(NameList)}); + decode_extensions(Rest, Acc#{sni => dec_sni(NameList)}); -dec_hello_extensions(<<?UINT16(?SUPPORTED_VERSIONS_EXT), ?UINT16(Len), +decode_extensions(<<?UINT16(?SUPPORTED_VERSIONS_EXT), ?UINT16(Len), ExtData:Len/binary, Rest/binary>>, Acc) when Len > 2 -> <<?UINT16(_),Versions/binary>> = ExtData, - dec_hello_extensions(Rest, Acc#hello_extensions{ - client_hello_versions = + decode_extensions(Rest, Acc#{client_hello_versions => #client_hello_versions{versions = decode_versions(Versions)}}); -dec_hello_extensions(<<?UINT16(?SUPPORTED_VERSIONS_EXT), ?UINT16(Len), +decode_extensions(<<?UINT16(?SUPPORTED_VERSIONS_EXT), ?UINT16(Len), ?UINT16(Version), Rest/binary>>, Acc) when Len =:= 2, Version =:= 16#0304 -> - dec_hello_extensions(Rest, Acc#hello_extensions{ - server_hello_selected_version = - #server_hello_selected_version{selected_version = [{3,4}]}}); + decode_extensions(Rest, Acc#{server_hello_selected_version => + #server_hello_selected_version{selected_version = {3,4}}}); %% Ignore data following the ClientHello (i.e., %% extensions) if not understood. -dec_hello_extensions(<<?UINT16(_), ?UINT16(Len), _Unknown:Len/binary, Rest/binary>>, Acc) -> - dec_hello_extensions(Rest, Acc); +decode_extensions(<<?UINT16(_), ?UINT16(Len), _Unknown:Len/binary, Rest/binary>>, Acc) -> + decode_extensions(Rest, Acc); %% This theoretically should not happen if the protocol is followed, but if it does it is ignored. -dec_hello_extensions(_, Acc) -> +decode_extensions(_, Acc) -> Acc. dec_hashsign(<<?BYTE(HashAlgo), ?BYTE(SignAlgo)>>) -> @@ -2576,4 +2575,39 @@ cert_curve(Cert, ECCCurve0, CipherSuite) -> {ECCCurve0, CipherSuite} end. - +empty_hello_extensions({3, 4}, server) -> + #{server_hello_selected_version => undefined, + key_share => undefined, + pre_shared_key => undefined, + sni => undefined + }; +empty_hello_extensions({3, 4}, client) -> + #{client_hello_versions => undefined, + signature_algs => undefined, + signature_algs_cert => undefined, + sni => undefined, + alpn => undefined, + key_share => undefined, + pre_shared_key => undefined + }; +empty_hello_extensions({3, 3}, client) -> + Ext = empty_hello_extensions({3,2}, client), + Ext#{client_hello_versions => undefined, + signature_algs => undefined, + signature_algs_cert => undefined}; +empty_hello_extensions(_, client) -> + #{renegotiation_info => undefined, + alpn => undefined, + next_protocol_negotiation => undefined, + srp => undefined, + ec_point_formats => undefined, + elliptic_curves => undefined, + sni => undefined}; +empty_hello_extensions(_, server) -> + #{renegotiation_info => undefined, + alpn => undefined, + next_protocol_negotiation => undefined, + ec_point_formats => undefined, + sni => undefined}. +empty_extensions() -> + #{}. diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index 94a4fa511d..8ded2cbff7 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -57,7 +57,9 @@ empty_connection_state/2]). %% Alert and close handling --export([send_alert/2, send_alert_in_connection/2, encode_alert/3, close/5, protocol_name/0]). +-export([send_alert/2, send_alert_in_connection/2, + send_sync_alert/2, + encode_alert/3, close/5, protocol_name/0]). %% Data handling -export([encode_data/3, passive_receive/2, next_record_if_active/1, @@ -341,8 +343,8 @@ reinit_handshake_data(State) -> tls_handshake_history = ssl_handshake:init_handshake_history() }. -select_sni_extension(#client_hello{extensions = HelloExtensions}) -> - HelloExtensions#hello_extensions.sni; +select_sni_extension(#client_hello{extensions = #{sni := SNI}}) -> + SNI; select_sni_extension(_) -> undefined. @@ -364,21 +366,38 @@ encode_alert(#alert{} = Alert, Version, ConnectionStates) -> send_alert(Alert, #state{negotiated_version = Version, socket = Socket, - protocol_cb = Connection, transport_cb = Transport, connection_states = ConnectionStates0, ssl_options = SslOpts} = StateData0) -> - {BinMsg, ConnectionStates} = - Connection:encode_alert(Alert, Version, ConnectionStates0), - Connection:send(Transport, Socket, BinMsg), + {BinMsg, ConnectionStates} = encode_alert(Alert, Version, ConnectionStates0), + send(Transport, Socket, BinMsg), Report = #{direction => outbound, protocol => 'tls_record', message => BinMsg}, ssl_logger:debug(SslOpts#ssl_options.log_level, Report, #{domain => [otp,ssl,tls_record]}), StateData0#state{connection_states = ConnectionStates}. -send_alert_in_connection(Alert, #state{protocol_specific = #{sender := Sender}}) -> +%% If an ALERT sent in the connection state, should cause the TLS +%% connection to end, we need to synchronize with the tls_sender +%% process so that the ALERT if possible (that is the tls_sender process is +%% not blocked) is sent before the connection process terminates and +%% thereby closes the transport socket. +send_alert_in_connection(#alert{level = ?FATAL} = Alert, State) -> + send_sync_alert(Alert, State); +send_alert_in_connection(#alert{description = ?CLOSE_NOTIFY} = Alert, State) -> + send_sync_alert(Alert, State); +send_alert_in_connection(Alert, + #state{protocol_specific = #{sender := Sender}}) -> tls_sender:send_alert(Sender, Alert). +send_sync_alert(Alert, #state{protocol_specific = #{sender := Sender}}= State) -> + tls_sender:send_and_ack_alert(Sender, Alert), + receive + {Sender, ack_alert} -> + ok + after ?DEFAULT_TIMEOUT -> + %% Sender is blocked terminate anyway + throw({stop, {shutdown, own_alert}, State}) + end. %% User closes or recursive call! close({close, Timeout}, Socket, Transport = gen_tcp, _,_) -> @@ -517,13 +536,13 @@ hello(internal, #client_hello{extensions = Extensions} = Hello, start_or_recv_from = From} = State) -> {next_state, user_hello, State#state{start_or_recv_from = undefined, hello = Hello}, - [{reply, From, {ok, ssl_connection:map_extensions(Extensions)}}]}; + [{reply, From, {ok, Extensions}}]}; hello(internal, #server_hello{extensions = Extensions} = Hello, #state{ssl_options = #ssl_options{handshake = hello}, start_or_recv_from = From} = State) -> {next_state, user_hello, State#state{start_or_recv_from = undefined, hello = Hello}, - [{reply, From, {ok, ssl_connection:map_extensions(Extensions)}}]}; + [{reply, From, {ok, Extensions}}]}; hello(internal, #client_hello{client_version = ClientVersion} = Hello, #state{connection_states = ConnectionStates0, port = Port, session = #session{own_certificate = Cert} = Session0, @@ -536,7 +555,9 @@ hello(internal, #client_hello{client_version = ClientVersion} = Hello, case tls_handshake:hello(Hello, SslOpts, {Port, Session0, Cache, CacheCb, ConnectionStates0, Cert, KeyExAlg}, Renegotiation) of #alert{} = Alert -> - ssl_connection:handle_own_alert(Alert, ClientVersion, hello, State); + ssl_connection:handle_own_alert(Alert, ClientVersion, hello, + State#state{negotiated_version + = ClientVersion}); {Version, {Type, Session}, ConnectionStates, Protocol0, ServerHelloExt, HashSign} -> Protocol = case Protocol0 of @@ -559,7 +580,8 @@ hello(internal, #server_hello{} = Hello, ssl_options = SslOptions} = State) -> case tls_handshake:hello(Hello, SslOptions, ConnectionStates0, Renegotiation) of #alert{} = Alert -> - ssl_connection:handle_own_alert(Alert, ReqVersion, hello, State); + ssl_connection:handle_own_alert(Alert, ReqVersion, hello, + State#state{negotiated_version = ReqVersion}); {Version, NewId, ConnectionStates, ProtoExt, Protocol} -> ssl_connection:handle_session(Hello, Version, NewId, ConnectionStates, ProtoExt, Protocol, State) @@ -667,8 +689,8 @@ callback_mode() -> state_functions. terminate(Reason, StateName, State) -> - ensure_sender_terminate(Reason, State), - catch ssl_connection:terminate(Reason, StateName, State). + catch ssl_connection:terminate(Reason, StateName, State), + ensure_sender_terminate(Reason, State). format_status(Type, Data) -> ssl_connection:format_status(Type, Data). @@ -827,8 +849,8 @@ handle_info({CloseTag, Socket}, StateName, %% and then receive the final message. next_event(StateName, no_record, State) end; -handle_info({'EXIT', Pid, Reason}, _, - #state{protocol_specific = Pid} = State) -> +handle_info({'EXIT', Sender, Reason}, _, + #state{protocol_specific = #{sender := Sender}} = State) -> {stop, {shutdown, sender_died, Reason}, State}; handle_info(Msg, StateName, State) -> ssl_connection:StateName(info, Msg, State, ?MODULE). diff --git a/lib/ssl/src/tls_handshake.erl b/lib/ssl/src/tls_handshake.erl index 82ed2e8d14..050b4be870 100644 --- a/lib/ssl/src/tls_handshake.erl +++ b/lib/ssl/src/tls_handshake.erl @@ -100,7 +100,7 @@ client_hello(Host, Port, ConnectionStates, ssl_record:connection_states(), alpn | npn, binary() | undefined}| {tls_record:tls_version(), {resumed | new, #session{}}, ssl_record:connection_states(), binary() | undefined, - #hello_extensions{}, {ssl_cipher_format:hash(), ssl_cipher_format:sign_algo()} | + HelloExt::map(), {ssl_cipher_format:hash(), ssl_cipher_format:sign_algo()} | undefined} | #alert{}. %% %% Description: Handles a received hello message @@ -145,10 +145,9 @@ hello(#server_hello{server_version = {Major, Minor}, %% - If "supported_version" is present (ServerHello): %% - Abort handshake with an "illegal_parameter" alert hello(#server_hello{server_version = Version, - extensions = #hello_extensions{ - server_hello_selected_version = - #server_hello_selected_version{selected_version = Version} - }}, + extensions = #{server_hello_selected_version := + #server_hello_selected_version{selected_version = Version}} + }, #ssl_options{versions = SupportedVersions}, _ConnectionStates0, _Renegotiation) -> case tls_record:is_higher({3,4}, Version) of @@ -196,10 +195,9 @@ hello(#server_hello{server_version = Version, random = Random, %% e.g. Server 1.0,1.2 Client 1.1 -> ServerHello 1.0 hello(#client_hello{client_version = _ClientVersion, cipher_suites = CipherSuites, - extensions = #hello_extensions{ - client_hello_versions = - #client_hello_versions{versions = ClientVersions} - }} = Hello, + extensions = #{client_hello_versions := + #client_hello_versions{versions = ClientVersions} + }} = Hello, #ssl_options{versions = Versions} = SslOpts, Info, Renegotiation) -> try @@ -267,10 +265,7 @@ handle_client_hello(Version, cipher_suites = CipherSuites, compression_methods = Compressions, random = Random, - extensions = - #hello_extensions{elliptic_curves = Curves, - signature_algs = ClientHashSigns} - = HelloExt}, + extensions = HelloExt}, #ssl_options{versions = Versions, signature_algs = SupportedHashSigns, eccs = SupportedECCs, @@ -279,6 +274,8 @@ handle_client_hello(Version, Renegotiation) -> case tls_record:is_acceptable_version(Version, Versions) of true -> + Curves = maps:get(elliptic_curves, HelloExt, undefined), + ClientHashSigns = maps:get(signature_algs, HelloExt, undefined), AvailableHashSigns = ssl_handshake:available_signature_algs( ClientHashSigns, SupportedHashSigns, Cert, Version), ECCCurve = ssl_handshake:select_curve(Curves, SupportedECCs, ECCOrder), @@ -407,13 +404,14 @@ get_tls_handshake_aux(_Version, Data, _, Acc) -> decode_handshake({3, N}, ?HELLO_REQUEST, <<>>) when N < 4 -> #hello_request{}; -decode_handshake(_Version, ?CLIENT_HELLO, +decode_handshake(Version, ?CLIENT_HELLO, <<?BYTE(Major), ?BYTE(Minor), Random:32/binary, ?BYTE(SID_length), Session_ID:SID_length/binary, ?UINT16(Cs_length), CipherSuites:Cs_length/binary, ?BYTE(Cm_length), Comp_methods:Cm_length/binary, Extensions/binary>>) -> - DecodedExtensions = ssl_handshake:decode_hello_extensions({client, Extensions}), + Exts = ssl_handshake:decode_vector(Extensions), + DecodedExtensions = ssl_handshake:decode_hello_extensions(Exts, Version, client), #client_hello{ client_version = {Major,Minor}, random = Random, @@ -426,5 +424,3 @@ decode_handshake({3, 4}, Tag, Msg) -> tls_handshake_1_3:decode_handshake(Tag, Msg); decode_handshake(Version, Tag, Msg) -> ssl_handshake:decode_handshake(Version, Tag, Msg). - - diff --git a/lib/ssl/src/tls_handshake_1_3.erl b/lib/ssl/src/tls_handshake_1_3.erl index 2957e3a5b4..199054b43b 100644 --- a/lib/ssl/src/tls_handshake_1_3.erl +++ b/lib/ssl/src/tls_handshake_1_3.erl @@ -89,7 +89,7 @@ decode_handshake(?CERTIFICATE, <<?BYTE(CSize), Context:CSize/binary, certificate_request_context = Context, entries = CertList }; -decode_handshake(?ENCRYPTED_EXTENSIONS, EncExts) -> +decode_handshake(?ENCRYPTED_EXTENSIONS, <<?UINT16(Size), EncExts:Size/binary>>) -> #encrypted_extensions{ extensions = decode_extensions(EncExts) }; @@ -127,23 +127,27 @@ encode_cert_entries([], Acc) -> iolist_to_binary(lists:reverse(Acc)); encode_cert_entries([#certificate_entry{data = Data, extensions = Exts} | Rest], Acc) -> + DSize = byte_size(Data), BinExts = encode_extensions(Exts), - Size = byte_size(Data), + ExtSize = byte_size(BinExts), encode_cert_entries(Rest, - [<<?UINT24(Size), Data/binary, BinExts/binary>> | Acc]). + [<<?UINT24(DSize), Data/binary, ?UINT16(ExtSize), BinExts/binary>> | Acc]). decode_cert_entries(Entries) -> decode_cert_entries(Entries, []). decode_cert_entries(<<>>, Acc) -> lists:reverse(Acc); -decode_cert_entries(<<?UINT24(DSize), Data:DSize/binary, ?UINT24(Esize), BinExts:Esize/binary, +decode_cert_entries(<<?UINT24(DSize), Data:DSize/binary, ?UINT16(Esize), BinExts:Esize/binary, Rest/binary>>, Acc) -> Exts = decode_extensions(BinExts), decode_cert_entries(Rest, [#certificate_entry{data = Data, extensions = Exts} | Acc]). encode_extensions(Exts)-> - ssl_handshake:encode_hello_extensions(Exts). + ssl_handshake:encode_extensions(extensions_list(Exts)). decode_extensions(Exts) -> - ssl_handshake:decode_hello_extensions(Exts). + ssl_handshake:decode_extensions(Exts). + +extensions_list(HelloExtensions) -> + [Ext || {_, Ext} <- maps:to_list(HelloExtensions)]. diff --git a/lib/ssl/src/tls_sender.erl b/lib/ssl/src/tls_sender.erl index db67d7ddff..ad6e462279 100644 --- a/lib/ssl/src/tls_sender.erl +++ b/lib/ssl/src/tls_sender.erl @@ -28,7 +28,8 @@ -include("ssl_api.hrl"). %% API --export([start/0, start/1, initialize/2, send_data/2, send_alert/2, renegotiate/1, +-export([start/0, start/1, initialize/2, send_data/2, send_alert/2, + send_and_ack_alert/2, setopts/2, renegotiate/1, update_connection_state/3, dist_tls_socket/1, dist_handshake_complete/3]). %% gen_statem callbacks @@ -80,7 +81,7 @@ initialize(Pid, InitMsg) -> gen_statem:call(Pid, {self(), InitMsg}). %%-------------------------------------------------------------------- --spec send_data(pid(), iodata()) -> ok. +-spec send_data(pid(), iodata()) -> ok | {error, term()}. %% Description: Send application data %%-------------------------------------------------------------------- send_data(Pid, AppData) -> @@ -89,13 +90,28 @@ send_data(Pid, AppData) -> %%-------------------------------------------------------------------- -spec send_alert(pid(), #alert{}) -> _. -%% Description: TLS connection process wants to end an Alert +%% Description: TLS connection process wants to send an Alert %% in the connection state. %%-------------------------------------------------------------------- send_alert(Pid, Alert) -> gen_statem:cast(Pid, Alert). %%-------------------------------------------------------------------- +-spec send_and_ack_alert(pid(), #alert{}) -> _. +%% Description: TLS connection process wants to send an Alert +%% in the connection state and recive an ack. +%%-------------------------------------------------------------------- +send_and_ack_alert(Pid, Alert) -> + gen_statem:cast(Pid, {ack_alert, Alert}). + +%%-------------------------------------------------------------------- +-spec setopts(pid(), [{packet, integer() | atom()}]) -> ok | {error, term()}. +%% Description: Send application data +%%-------------------------------------------------------------------- +setopts(Pid, Opts) -> + call(Pid, {set_opts, Opts}). + +%%-------------------------------------------------------------------- -spec renegotiate(pid()) -> {ok, WriteState::map()} | {error, closed}. %% Description: So TLS connection process can synchronize the %% encryption state to be used when handshaking. @@ -192,6 +208,8 @@ connection({call, From}, {application_data, AppData}, Data -> send_application_data(Data, From, ?FUNCTION_NAME, StateData) end; +connection({call, From}, {set_opts, _} = Call, StateData) -> + handle_call(From, Call, ?FUNCTION_NAME, StateData); connection({call, From}, dist_get_tls_socket, #data{protocol_cb = Connection, transport_cb = Transport, @@ -207,6 +225,10 @@ connection({call, From}, {dist_handshake_complete, _Node, DHandle}, #data{connec process_flag(priority, normal), Events = dist_data_events(DHandle, []), {next_state, ?FUNCTION_NAME, StateData#data{dist_handle = DHandle}, [{reply, From, ok} | Events]}; +connection(cast, {ack_alert, #alert{} = Alert}, #data{connection_pid = Pid} =StateData0) -> + StateData = send_tls_alert(Alert, StateData0), + Pid ! {self(), ack_alert}, + {next_state, ?FUNCTION_NAME, StateData}; connection(cast, #alert{} = Alert, StateData0) -> StateData = send_tls_alert(Alert, StateData0), {next_state, ?FUNCTION_NAME, StateData}; @@ -241,6 +263,8 @@ connection(info, Msg, StateData) -> StateData :: term()) -> gen_statem:event_handler_result(atom()). %%-------------------------------------------------------------------- +handshake({call, From}, {set_opts, _} = Call, StateData) -> + handle_call(From, Call, ?FUNCTION_NAME, StateData); handshake({call, _}, _, _) -> {keep_state_and_data, [postpone]}; handshake(cast, {new_write, WritesState, Version}, @@ -285,6 +309,9 @@ code_change(_OldVsn, State, Data, _Extra) -> %%%=================================================================== %%% Internal functions %%%=================================================================== +handle_call(From, {set_opts, Opts}, StateName, #data{socket_options = SockOpts} = StateData) -> + {next_state, StateName, StateData#data{socket_options = set_opts(SockOpts, Opts)}, [{reply, From, ok}]}. + handle_info({'DOWN', Monitor, _, _, Reason}, _, #data{connection_monitor = Monitor, dist_handle = Handle} = StateData) when Handle =/= undefined-> @@ -303,6 +330,11 @@ send_tls_alert(Alert, #data{negotiated_version = Version, {BinMsg, ConnectionStates} = Connection:encode_alert(Alert, Version, ConnectionStates0), Connection:send(Transport, Socket, BinMsg), + %% TODO: fix ssl_options for this process + %% Report = #{direction => outbound, + %% protocol => 'tls_record', + %% message => BinMsg}, + %% ssl_logger:debug(SslOpts#ssl_options.log_level, Report, #{domain => [otp,ssl,tls_record]}), StateData0#data{connection_states = ConnectionStates}. send_application_data(Data, From, StateName, @@ -351,6 +383,10 @@ encode_size_packet(Bin, Size, Max) -> false -> <<Len:Size, Bin/binary>> end. + +set_opts(SocketOptions, [{packet, N}]) -> + SocketOptions#socket_options{packet = N}. + time_to_renegotiate(_Data, #{current_write := #{sequence_number := Num}}, RenegotiateAt) -> diff --git a/lib/ssl/test/property_test/ssl_eqc_handshake.erl b/lib/ssl/test/property_test/ssl_eqc_handshake.erl index 88046f7386..99c6554f15 100644 --- a/lib/ssl/test/property_test/ssl_eqc_handshake.erl +++ b/lib/ssl/test/property_test/ssl_eqc_handshake.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2015. All Rights Reserved. +%% Copyright Ericsson AB 2018-2018. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -51,6 +51,7 @@ -endif. -endif. +-include_lib("kernel/include/inet.hrl"). -include_lib("ssl/src/tls_handshake_1_3.hrl"). -include_lib("ssl/src/tls_handshake.hrl"). -include_lib("ssl/src/ssl_handshake.hrl"). @@ -83,79 +84,261 @@ prop_tls_hs_encode_decode() -> end ). +%%-------------------------------------------------------------------- +%% Message Generators -------------------------------------------------- +%%-------------------------------------------------------------------- + tls_version() -> - oneof([?'TLS_v1.2', ?'TLS_v1.1', ?'TLS_v1', ?'SSL_v3']). + oneof([?'TLS_v1.3', ?'TLS_v1.2', ?'TLS_v1.1', ?'TLS_v1', ?'SSL_v3']). tls_msg(?'TLS_v1.3'= Version) -> oneof([client_hello(Version), - %%server_hello(Version) + server_hello(Version), %%new_session_ticket() #end_of_early_data{}, - %%encrypted_extensions() - %%certificate_1_3(), - %%certificate_request() + encrypted_extensions(), + certificate_1_3(), + %%certificate_request_1_3, %%certificate_verify() - %%finished() + finished(), key_update() - %%message_hash() ]); tls_msg(Version) -> oneof([#hello_request{}, client_hello(Version), - %%server_hello(Version) - %%certificate(), + server_hello(Version), + certificate(), %%server_key_exchange() - %%certificate_request() - #server_hello_done{} + certificate_request(Version), + #server_hello_done{}, %%certificate_verify() %%client_key_exchange() - %%finished() + finished() ]). client_hello(?'TLS_v1.3' = Version) -> #client_hello{session_id = session_id(), client_version = ?'TLS_v1.2', - cipher_suites = ssl_cipher:suites(Version), + cipher_suites = cipher_suites(Version), compression_methods = compressions(Version), random = client_random(Version), - extensions = client_extensions(Version) + extensions = client_hello_extensions(Version) }; client_hello(Version) -> #client_hello{session_id = session_id(), client_version = Version, - cipher_suites = ssl_cipher:suites(Version), + cipher_suites = cipher_suites(Version), compression_methods = compressions(Version), random = client_random(Version), - extensions = client_extensions(Version) + extensions = client_hello_extensions(Version) }. + +server_hello(?'TLS_v1.3' = Version) -> + #server_hello{server_version = ?'TLS_v1.2', + session_id = session_id(), + random = server_random(Version), + cipher_suite = cipher_suite(Version), + compression_method = compression(Version), + extensions = server_hello_extensions(Version) + }; +server_hello(Version) -> + #server_hello{server_version = Version, + session_id = session_id(), + random = server_random(Version), + cipher_suite = cipher_suite(Version), + compression_method = compression(Version), + extensions = server_hello_extensions(Version) + }. + +encrypted_extensions() -> + ?LET(Exts, extensions(?'TLS_v1.3'), + #encrypted_extensions{extensions = Exts}). + +certificate() -> + #certificate{ + asn1_certificates = certificate_chain() + }. + +certificate_1_3() -> + ?LET(Certs, certificate_chain(), + #certificate_1_3{ + certificate_request_context = certificate_request_context(), + entries = certificate_entries(Certs, []) + }). + +key_update() -> + #key_update{request_update = request_update()}. + +finished() -> + ?LET(Size, digest_size(), + #finished{verify_data = crypto:strong_rand_bytes(Size)}). + +%%-------------------------------------------------------------------- +%% Messge Data Generators ------------------------------------------- +%%-------------------------------------------------------------------- + + +cipher_suite(Version) -> + oneof(cipher_suites(Version)). + +cipher_suites(Version) -> + ssl_cipher:suites(Version). + session_id() -> crypto:strong_rand_bytes(?NUM_OF_SESSION_ID_BYTES). +compression(Version) -> + oneof(compressions(Version)). + compressions(_) -> ssl_record:compressions(). + client_random(_) -> crypto:strong_rand_bytes(32). - -client_extensions(?'TLS_v1.3' = Version) -> - #hello_extensions{ - client_hello_versions = - #client_hello_versions{ - versions = supported_versions(Version) - }, - signature_algs_cert = - #signature_scheme_list{ - signature_scheme_list = signature_scheme_list() - } - }; -client_extensions(Version) -> - #hello_extensions{ - client_hello_versions = - #client_hello_versions{ - versions = supported_versions(Version) - } - }. + +server_random(_) -> + crypto:strong_rand_bytes(32). + + +client_hello_extensions(?'TLS_v1.3' = Version) -> + ?LET({Versions, Ext}, {supported_versions(Version), c_hello_extensions(Version)}, + maps:merge(Ext, #{client_hello_versions => client_hello_versions(Versions)}) + ); +client_hello_extensions(?'TLS_v1.2' = Version) -> + ?LET({Versions, Exts}, {supported_versions(Version), c_hello_extensions(Version)}, + maps:merge(Exts, #{client_hello_versions => client_hello_versions(Versions)}) + ); +client_hello_extensions(Version) -> + ?LET(Exts, + c_hello_extensions(Version), + maps:merge(empty_hello_extensions(Version, client), Exts)). + +server_hello_extensions(?'TLS_v1.3' = Version) -> + ?LET(Exts, + s_hello_extensions(Version), + maps:merge(Exts, #{server_hello_selected_version => server_hello_selected_version(Version)})); +server_hello_extensions(Version) -> + ?LET(Exts, + s_hello_extensions(Version), + Exts). + +c_hello_extensions(?'TLS_v1.3'= Version) -> + ?LET({KeyShare, PreShare}, {key_share_client_hello(), + pre_shared_keyextension()}, + maps:merge(empty_hello_extensions(Version, client), + #{key_share => KeyShare, + pre_shared_key => PreShare + }) + ); +c_hello_extensions(Version) -> + ?LET(Exts, extensions(Version), + maps:merge(empty_hello_extensions(Version, client), + Exts)). + +s_hello_extensions(?'TLS_v1.3'= Version) -> + ?LET({KeyShare, PreShare}, {key_share_server_hello(), + pre_shared_keyextension()}, + maps:merge(empty_hello_extensions(Version, server), + #{key_share => KeyShare, + pre_shared_key => PreShare + }) + ); +s_hello_extensions(Version) -> + ?LET(Exts, extensions(Version), + maps:merge(empty_hello_extensions(Version, server), + Exts)). + +key_share_client_hello() -> + oneof([undefined]). + %%oneof([#key_share_client_hello{}, undefined]). + +key_share_server_hello() -> + oneof([undefined]). + %%oneof([#key_share_server_hello{}, undefined]). + +pre_shared_keyextension() -> + oneof([undefined]). + %%oneof([#pre_shared_keyextension{},undefined]). + +extensions(?'TLS_v1.3') -> + ?LET({Ext_1_3, Exts}, {extensions_1_3(), extensions(?'TLS_v1.2')}, maps:merge(Ext_1_3, Exts)); +extensions(?'SSL_v3') -> + #{}; +extensions(Version) -> + ?LET({SNI, ECPoitF, ECCurves, ALPN, NextP, SRP}, + {oneof([sni(), undefined]), + oneof([ec_poit_formats(), undefined]), + oneof([elliptic_curves(Version), undefined]), + oneof([alpn(), undefined]), + oneof([next_protocol_negotiation(), undefined]), + oneof([srp(), undefined])}, + maps:filter(fun(_, undefined) -> + false; + (_,_) -> + true + end, + #{sni => SNI, + ec_point_formats => ECPoitF, + elliptic_curves => ECCurves, + alpn => ALPN, + next_protocol_negotiation => NextP, + srp => SRP})). + +extensions_1_3() -> + %% ?LET(Entry, key_share_entry(), + %% maps:filter(fun(_, undefined) -> + %% false; + %% (_,_) -> + %% true + %% end, #{key_share_entry => Entry})). + ?LET({HashSign, SigAlgCert}, {oneof([hash_sign_algos(?'TLS_v1.2')]), oneof([signature_scheme_list()])}, + #{signature_algs => HashSign, + signature_algs_cert => SigAlgCert}). + +empty_hello_extensions({3, 4}, server) -> + #{server_hello_selected_version => undefined, + key_share => undefined, + pre_shared_key => undefined, + sni => undefined + }; +empty_hello_extensions({3, 4}, client) -> + #{client_hello_versions => undefined, + signature_algs => undefined, + signature_algs_cert => undefined, + sni => undefined, + alpn => undefined, + key_share => undefined, + pre_shared_key => undefined + }; +empty_hello_extensions({3, 3}, client) -> + Ext = empty_hello_extensions({3,2}, client), + Ext#{client_hello_versions => undefined, + signature_algs => undefined, + signature_algs_cert => undefined}; +empty_hello_extensions(_, client) -> + #{renegotiation_info => undefined, + alpn => undefined, + next_protocol_negotiation => undefined, + srp => undefined, + ec_point_formats => undefined, + elliptic_curves => undefined, + sni => undefined}; +empty_hello_extensions(_, server) -> + #{renegotiation_info => undefined, + alpn => undefined, + next_protocol_negotiation => undefined, + ec_point_formats => undefined, + sni => undefined}. + +signature_algs_cert() -> + ?LET(Algs, signature_scheme_list(), + Algs). signature_scheme_list() -> + ?LET(List, sig_scheme_list(), + #signature_scheme_list{signature_scheme_list = List}). + +sig_scheme_list() -> oneof([[rsa_pkcs1_sha256], [rsa_pkcs1_sha256, ecdsa_sha1], [rsa_pkcs1_sha256, @@ -185,8 +368,167 @@ supported_versions(_) -> [{3,3},{3,2},{3,1},{3,0}] ]). -key_update() -> - #key_update{request_update = request_update()}. - request_update() -> oneof([?UPDATE_NOT_REQUESTED, ?UPDATE_REQUESTED]). + +certificate_chain()-> + Conf = cert_conf(), + ?LET(Chain, + choose_certificate_chain(Conf), + Chain). + +choose_certificate_chain(#{server_config := ServerConf, + client_config := ClientConf}) -> + oneof([certificate_chain(ServerConf), certificate_chain(ClientConf)]). + +certificate_request_context() -> + <<>>. +certificate_entries([], Acc) -> + lists:reverse(Acc); +certificate_entries([Cert | Rest], Acc) -> + certificate_entries(Rest, [certificate_entry(Cert) | Acc]). + +certificate_entry(Cert) -> + #certificate_entry{data = Cert, + extensions = certificate_entry_extensions() + }. +certificate_entry_extensions() -> + #{}. + +certificate_chain(Conf) -> + CAs = proplists:get_value(cacerts, Conf), + Cert = proplists:get_value(cert, Conf), + %% Middle argument are of correct type but will not be used + {ok, _, Chain} = ssl_certificate:certificate_chain(Cert, ets:new(foo, []), make_ref(), CAs), + Chain. + +cert_conf()-> + Hostname = net_adm:localhost(), + {ok, #hostent{h_addr_list = [_IP |_]}} = inet:gethostbyname(net_adm:localhost()), + public_key:pkix_test_data(#{server_chain => + #{root => [{key, ssl_test_lib:hardcode_rsa_key(1)}], + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}]], + peer => [{extensions, [#'Extension'{extnID = + ?'id-ce-subjectAltName', + extnValue = [{dNSName, Hostname}], + critical = false}]}, + {key, ssl_test_lib:hardcode_rsa_key(3)} + ]}, + client_chain => + #{root => [{key, ssl_test_lib:hardcode_rsa_key(4)}], + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(5)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(6)}]}}). + +certificate_request(Version) -> + #certificate_request{certificate_types = certificate_types(Version), + hashsign_algorithms = hashsign_algorithms(Version), + certificate_authorities = certificate_authorities()}. + +certificate_types(?'TLS_v1.3') -> + iolist_to_binary([<<?BYTE(?ECDSA_SIGN)>>, <<?BYTE(?RSA_SIGN)>>]); +certificate_types(?'TLS_v1.2') -> + iolist_to_binary([<<?BYTE(?ECDSA_SIGN)>>, <<?BYTE(?RSA_SIGN)>>, <<?BYTE(?DSS_SIGN)>>]); +certificate_types(_) -> + iolist_to_binary([<<?BYTE(?ECDSA_SIGN)>>, <<?BYTE(?RSA_SIGN)>>, <<?BYTE(?DSS_SIGN)>>]). + +hashsign_algorithms({_, N} = Version) when N >= 3 -> + #hash_sign_algos{hash_sign_algos = hash_alg_list(Version)}; +hashsign_algorithms(_) -> + undefined. + +hash_alg_list(Version) -> + ?LET(NumOf, choose(0,15), + ?LET(List, [hash_alg(Version) || _ <- lists:seq(1,NumOf)], + lists:usort(List) + )). + +hash_alg(Version) -> + ?LET(Alg, sign_algorithm(Version), + {hash_algorithm(Version, Alg), Alg} + ). + +hash_algorithm(?'TLS_v1.3', _) -> + oneof([sha, sha224, sha256, sha384, sha512]); +hash_algorithm(?'TLS_v1.2', rsa) -> + oneof([sha, sha224, sha256, sha384, sha512]); +hash_algorithm(_, rsa) -> + oneof([md5, sha, sha224, sha256, sha384, sha512]); +hash_algorithm(_, ecdsa) -> + oneof([sha, sha224, sha256, sha384, sha512]); +hash_algorithm(_, dsa) -> + sha. + +sign_algorithm(?'TLS_v1.3') -> + oneof([rsa, ecdsa]); +sign_algorithm(_) -> + oneof([rsa, dsa, ecdsa]). + +certificate_authorities() -> + #{server_config := ServerConf} = cert_conf(), + Authorities = proplists:get_value(cacerts, ServerConf), + Enc = fun(#'OTPCertificate'{tbsCertificate=TBSCert}) -> + OTPSubj = TBSCert#'OTPTBSCertificate'.subject, + DNEncodedBin = public_key:pkix_encode('Name', OTPSubj, otp), + DNEncodedLen = byte_size(DNEncodedBin), + <<?UINT16(DNEncodedLen), DNEncodedBin/binary>> + end, + list_to_binary([Enc(public_key:pkix_decode_cert(DERCert, otp)) || DERCert <- Authorities]). + +digest_size()-> + oneof([160,224,256,384,512]). + +key_share_entry() -> + undefined. + %%#key_share_entry{}. + +client_hello_versions(Versions) -> + #client_hello_versions{versions = Versions}. + +server_hello_selected_version(Version) -> + #server_hello_selected_version{selected_version = Version}. + +sni() -> + #sni{hostname = net_adm:localhost()}. + +ec_poit_formats() -> + #ec_point_formats{ec_point_format_list = ec_point_format_list()}. + +ec_point_format_list() -> + [?ECPOINT_UNCOMPRESSED]. + +elliptic_curves({_, Minor}) -> + Curves = tls_v1:ecc_curves(Minor), + #elliptic_curves{elliptic_curve_list = Curves}. + +hash_sign_algos(Version) -> + #hash_sign_algos{hash_sign_algos = hash_alg_list(Version)}. + +alpn() -> + ?LET(ExtD, alpn_protocols(), #alpn{extension_data = ExtD}). + +alpn_protocols() -> + oneof([<<"spdy/2">>, <<"spdy/3">>, <<"http/2">>, <<"http/1.0">>, <<"http/1.1">>]). + +next_protocol_negotiation() -> + %% Predecessor to APLN + ?LET(ExtD, alpn_protocols(), #next_protocol_negotiation{extension_data = ExtD}). + +srp() -> + ?LET(Name, gen_name(), #srp{username = list_to_binary(Name)}). + +renegotiation_info() -> + #renegotiation_info{renegotiated_connection = 0}. + +gen_name() -> + ?LET(Size, choose(0,10), gen_string(Size)). + +gen_char() -> + choose($a,$z). + +gen_string(N) -> + gen_string(N, []). + +gen_string(0, Acc) -> + Acc; +gen_string(N, Acc) -> + ?LET(Char, gen_char(), gen_string(N-1, [Char | Acc])). diff --git a/lib/ssl/test/ssl_alpn_handshake_SUITE.erl b/lib/ssl/test/ssl_alpn_handshake_SUITE.erl index 27062d4801..04c4b257d9 100644 --- a/lib/ssl/test/ssl_alpn_handshake_SUITE.erl +++ b/lib/ssl/test/ssl_alpn_handshake_SUITE.erl @@ -155,7 +155,7 @@ empty_client(Config) when is_list(Config) -> run_failing_handshake(Config, [{alpn_advertised_protocols, []}], [{alpn_preferred_protocols, [<<"spdy/2">>, <<"spdy/3">>, <<"http/2">>]}], - {connect_failed,{tls_alert,"no application protocol"}}). + {error,{tls_alert,"no application protocol"}}). %-------------------------------------------------------------------------------- @@ -163,7 +163,7 @@ empty_server(Config) when is_list(Config) -> run_failing_handshake(Config, [{alpn_advertised_protocols, [<<"http/1.0">>, <<"http/1.1">>]}], [{alpn_preferred_protocols, []}], - {connect_failed,{tls_alert,"no application protocol"}}). + {error,{tls_alert,"no application protocol"}}). %-------------------------------------------------------------------------------- @@ -171,7 +171,7 @@ empty_client_empty_server(Config) when is_list(Config) -> run_failing_handshake(Config, [{alpn_advertised_protocols, []}], [{alpn_preferred_protocols, []}], - {connect_failed,{tls_alert,"no application protocol"}}). + {error,{tls_alert,"no application protocol"}}). %-------------------------------------------------------------------------------- @@ -179,7 +179,7 @@ no_matching_protocol(Config) when is_list(Config) -> run_failing_handshake(Config, [{alpn_advertised_protocols, [<<"http/1.0">>, <<"http/1.1">>]}], [{alpn_preferred_protocols, [<<"spdy/2">>, <<"spdy/3">>, <<"http/2">>]}], - {connect_failed,{tls_alert,"no application protocol"}}). + {error,{tls_alert,"no application protocol"}}). %-------------------------------------------------------------------------------- @@ -342,18 +342,19 @@ run_failing_handshake(Config, ClientExtraOpts, ServerExtraOpts, ExpectedResult) ServerOpts = ServerExtraOpts ++ ssl_test_lib:ssl_options(server_rsa_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), - Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {?MODULE, placeholder, []}}, {options, ServerOpts}]), Port = ssl_test_lib:inet_port(Server), - ExpectedResult - = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, - {host, Hostname}, - {from, self()}, - {mfa, {?MODULE, placeholder, []}}, - {options, ClientOpts}]). + Client = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, placeholder, []}}, + {options, ClientOpts}]), + ssl_test_lib:check_result(Server, ExpectedResult, + Client, ExpectedResult). run_handshake(Config, ClientExtraOpts, ServerExtraOpts, ExpectedProtocol) -> Data = "hello world", diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index 1970c16f1d..a0cc9f5c08 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -1183,16 +1183,16 @@ fallback(Config) when is_list(Config) -> Port = ssl_test_lib:inet_port(Server), - Client = - ssl_test_lib:start_client_error([{node, ClientNode}, - {port, Port}, {host, Hostname}, - {from, self()}, {options, - [{fallback, true}, - {versions, ['tlsv1']} - | ClientOpts]}]), + Client = + ssl_test_lib:start_client_error([{node, ClientNode}, + {port, Port}, {host, Hostname}, + {from, self()}, {options, + [{fallback, true}, + {versions, ['tlsv1']} + | ClientOpts]}]), - ssl_test_lib:check_result(Server, {error,{tls_alert,"inappropriate fallback"}}, - Client, {error,{tls_alert,"inappropriate fallback"}}). + ssl_test_lib:check_result(Server, {error,{tls_alert,"inappropriate fallback"}}, + Client, {error,{tls_alert,"inappropriate fallback"}}). %%-------------------------------------------------------------------- cipher_format() -> @@ -2645,14 +2645,14 @@ default_reject_anonymous(Config) when is_list(Config) -> {options, ServerOpts}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port}, - {host, Hostname}, - {from, self()}, - {options, - [{ciphers,[CipherSuite]} | - ClientOpts]}]), + {host, Hostname}, + {from, self()}, + {options, + [{ciphers,[CipherSuite]} | + ClientOpts]}]), ssl_test_lib:check_result(Server, {error, {tls_alert, "insufficient security"}}, - Client, {error, {tls_alert, "insufficient security"}}). + Client, {error, {tls_alert, "insufficient security"}}). %%-------------------------------------------------------------------- ciphers_ecdsa_signed_certs() -> @@ -3605,14 +3605,14 @@ no_common_signature_algs(Config) when is_list(Config) -> | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port}, - {host, Hostname}, - {from, self()}, - {options, [{signature_algs, [{sha384, rsa}]} - | ClientOpts]}]), + {host, Hostname}, + {from, self()}, + {options, [{signature_algs, [{sha384, rsa}]} + | ClientOpts]}]), ssl_test_lib:check_result(Server, {error, {tls_alert, "insufficient security"}}, - Client, {error, {tls_alert, "insufficient security"}}). - + Client, {error, {tls_alert, "insufficient security"}}). + %%-------------------------------------------------------------------- tls_dont_crash_on_handshake_garbage() -> diff --git a/lib/ssl/test/ssl_certificate_verify_SUITE.erl b/lib/ssl/test/ssl_certificate_verify_SUITE.erl index f677bf8a6e..bddcc2514d 100644 --- a/lib/ssl/test/ssl_certificate_verify_SUITE.erl +++ b/lib/ssl/test/ssl_certificate_verify_SUITE.erl @@ -620,8 +620,8 @@ cert_expired(Config) when is_list(Config) -> {from, self()}, {options, [{verify, verify_peer}, {active, Active} | ClientOpts]}]), - tcp_delivery_workaround(Server, {error, {tls_alert, "certificate expired"}}, - Client, {error, {tls_alert, "certificate expired"}}). + ssl_test_lib:check_result(Server, {error, {tls_alert, "certificate expired"}}, + Client, {error, {tls_alert, "certificate expired"}}). two_digits_str(N) when N < 10 -> lists:flatten(io_lib:format("0~p", [N])); @@ -729,8 +729,8 @@ critical_extension_verify_server(Config) when is_list(Config) -> %% This certificate has a critical extension that we don't %% understand. Therefore, verification should fail. - tcp_delivery_workaround(Server, {error, {tls_alert, "unsupported certificate"}}, - Client, {error, {tls_alert, "unsupported certificate"}}), + ssl_test_lib:check_result(Server, {error, {tls_alert, "unsupported certificate"}}, + Client, {error, {tls_alert, "unsupported certificate"}}), ssl_test_lib:close(Server). %%-------------------------------------------------------------------- @@ -909,8 +909,8 @@ invalid_signature_server(Config) when is_list(Config) -> {from, self()}, {options, [{verify, verify_peer} | ClientOpts]}]), - tcp_delivery_workaround(Server, {error, {tls_alert, "unknown ca"}}, - Client, {error, {tls_alert, "unknown ca"}}). + ssl_test_lib:check_result(Server, {error, {tls_alert, "unknown ca"}}, + Client, {error, {tls_alert, "unknown ca"}}). %%-------------------------------------------------------------------- @@ -946,8 +946,8 @@ invalid_signature_client(Config) when is_list(Config) -> {from, self()}, {options, NewClientOpts}]), - tcp_delivery_workaround(Server, {error, {tls_alert, "unknown ca"}}, - Client, {error, {tls_alert, "unknown ca"}}). + ssl_test_lib:check_result(Server, {error, {tls_alert, "unknown ca"}}, + Client, {error, {tls_alert, "unknown ca"}}). %%-------------------------------------------------------------------- @@ -1236,41 +1236,3 @@ incomplete_chain(Config) when is_list(Config) -> %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- -tcp_delivery_workaround(Server, ServerMsg, Client, ClientMsg) -> - receive - {Server, ServerMsg} -> - client_msg(Client, ClientMsg); - {Client, ClientMsg} -> - server_msg(Server, ServerMsg); - {Client, {error,closed}} -> - server_msg(Server, ServerMsg); - {Server, {error,closed}} -> - client_msg(Client, ClientMsg) - end. - -client_msg(Client, ClientMsg) -> - receive - {Client, ClientMsg} -> - ok; - {Client, {error,closed}} -> - ct:log("client got close"), - ok; - {Client, {error, Reason}} -> - ct:log("client got econnaborted: ~p", [Reason]), - ok; - Unexpected -> - ct:fail(Unexpected) - end. -server_msg(Server, ServerMsg) -> - receive - {Server, ServerMsg} -> - ok; - {Server, {error,closed}} -> - ct:log("server got close"), - ok; - {Server, {error, Reason}} -> - ct:log("server got econnaborted: ~p", [Reason]), - ok; - Unexpected -> - ct:fail(Unexpected) - end. diff --git a/lib/ssl/test/ssl_handshake_SUITE.erl b/lib/ssl/test/ssl_handshake_SUITE.erl index b8b9989d30..ef1f6be286 100644 --- a/lib/ssl/test/ssl_handshake_SUITE.erl +++ b/lib/ssl/test/ssl_handshake_SUITE.erl @@ -104,15 +104,13 @@ decode_hello_handshake(_Config) -> #ssl_options{}), {Hello, _Data} = hd(Records), - #renegotiation_info{renegotiated_connection = <<0>>} - = (Hello#server_hello.extensions)#hello_extensions.renegotiation_info. - + Extensions = Hello#server_hello.extensions, + #{renegotiation_info := #renegotiation_info{renegotiated_connection = <<0>>}} = Extensions. decode_single_hello_extension_correctly(_Config) -> Renegotiation = <<?UINT16(?RENEGOTIATION_EXT), ?UINT16(1), 0>>, - Extensions = ssl_handshake:decode_hello_extensions(Renegotiation), - #renegotiation_info{renegotiated_connection = <<0>>} - = Extensions#hello_extensions.renegotiation_info. + Extensions = ssl_handshake:decode_extensions(Renegotiation), + #{renegotiation_info := #renegotiation_info{renegotiated_connection = <<0>>}} = Extensions. decode_supported_elliptic_curves_hello_extension_correctly(_Config) -> % List of supported and unsupported curves (RFC4492:S5.1.1) @@ -123,37 +121,34 @@ decode_supported_elliptic_curves_hello_extension_correctly(_Config) -> Len = ListLen + 2, Extension = <<?UINT16(?ELLIPTIC_CURVES_EXT), ?UINT16(Len), ?UINT16(ListLen), EllipticCurveList/binary>>, % after decoding we should see only valid curves - #hello_extensions{elliptic_curves = DecodedCurves} = ssl_handshake:decode_hello_extensions(Extension), - #elliptic_curves{elliptic_curve_list = [?sect233k1, ?sect193r2]} = DecodedCurves. + Extensions = ssl_handshake:decode_hello_extensions(Extension, {3,2}, client), + #{elliptic_curves := #elliptic_curves{elliptic_curve_list = [?sect233k1, ?sect193r2]}} = Extensions. decode_unknown_hello_extension_correctly(_Config) -> FourByteUnknown = <<16#CA,16#FE, ?UINT16(4), 3, 0, 1, 2>>, Renegotiation = <<?UINT16(?RENEGOTIATION_EXT), ?UINT16(1), 0>>, - Extensions = ssl_handshake:decode_hello_extensions(<<FourByteUnknown/binary, Renegotiation/binary>>), - #renegotiation_info{renegotiated_connection = <<0>>} - = Extensions#hello_extensions.renegotiation_info. + Extensions = ssl_handshake:decode_hello_extensions(<<FourByteUnknown/binary, Renegotiation/binary>>, {3,2}, client), + #{renegotiation_info := #renegotiation_info{renegotiated_connection = <<0>>}} = Extensions. + encode_single_hello_sni_extension_correctly(_Config) -> - Exts = #hello_extensions{sni = #sni{hostname = "test.com"}}, SNI = <<16#00, 16#00, 16#00, 16#0d, 16#00, 16#0b, 16#00, 16#00, 16#08, $t, $e, $s, $t, $., $c, $o, $m>>, ExtSize = byte_size(SNI), HelloExt = <<ExtSize:16/unsigned-big-integer, SNI/binary>>, - Encoded = ssl_handshake:encode_hello_extensions(Exts), + Encoded = ssl_handshake:encode_extensions([#sni{hostname = "test.com"}]), HelloExt = Encoded. decode_single_hello_sni_extension_correctly(_Config) -> - Exts = #hello_extensions{sni = #sni{hostname = "test.com"}}, SNI = <<16#00, 16#00, 16#00, 16#0d, 16#00, 16#0b, 16#00, 16#00, 16#08, $t, $e, $s, $t, $., $c, $o, $m>>, - Decoded = ssl_handshake:decode_hello_extensions(SNI), - Exts = Decoded. + Decoded = ssl_handshake:decode_hello_extensions(SNI, {3,3}, client), + #{sni := #sni{hostname = "test.com"}} = Decoded. decode_empty_server_sni_correctly(_Config) -> - Exts = #hello_extensions{sni = #sni{hostname = ""}}, SNI = <<?UINT16(?SNI_EXT),?UINT16(0)>>, - Decoded = ssl_handshake:decode_hello_extensions(SNI), - Exts = Decoded. + Decoded = ssl_handshake:decode_hello_extensions(SNI, {3,3}, server), + #{sni := #sni{hostname = ""}} = Decoded. select_proper_tls_1_2_rsa_default_hashsign(_Config) -> diff --git a/lib/ssl/test/ssl_npn_hello_SUITE.erl b/lib/ssl/test/ssl_npn_hello_SUITE.erl index 35af666e9e..46734ba180 100644 --- a/lib/ssl/test/ssl_npn_hello_SUITE.erl +++ b/lib/ssl/test/ssl_npn_hello_SUITE.erl @@ -71,44 +71,46 @@ encode_and_decode_client_hello_test(Config) -> Version = ssl_test_lib:protocol_version(Config), {[{DecodedHandshakeMessage, _Raw}], _} = tls_handshake:get_tls_handshake(Version, list_to_binary(HandShakeData), <<>>, #ssl_options{}), - NextProtocolNegotiation = (DecodedHandshakeMessage#client_hello.extensions)#hello_extensions.next_protocol_negotiation, - NextProtocolNegotiation = undefined. + Extensions = DecodedHandshakeMessage#client_hello.extensions, + #{next_protocol_negotiation := undefined} = Extensions. %%-------------------------------------------------------------------- encode_and_decode_npn_client_hello_test(Config) -> HandShakeData = create_client_handshake(#next_protocol_negotiation{extension_data = <<>>}), Version = ssl_test_lib:protocol_version(Config), {[{DecodedHandshakeMessage, _Raw}], _} = tls_handshake:get_tls_handshake(Version, list_to_binary(HandShakeData), <<>>, #ssl_options{}), - NextProtocolNegotiation = (DecodedHandshakeMessage#client_hello.extensions)#hello_extensions.next_protocol_negotiation, - NextProtocolNegotiation = #next_protocol_negotiation{extension_data = <<>>}. + Extensions = DecodedHandshakeMessage#client_hello.extensions, + #{next_protocol_negotiation := #next_protocol_negotiation{extension_data = <<>>}} = Extensions. %%-------------------------------------------------------------------- encode_and_decode_server_hello_test(Config) -> HandShakeData = create_server_handshake(undefined), Version = ssl_test_lib:protocol_version(Config), {[{DecodedHandshakeMessage, _Raw}], _} = tls_handshake:get_tls_handshake(Version, list_to_binary(HandShakeData), <<>>, #ssl_options{}), - NextProtocolNegotiation = (DecodedHandshakeMessage#server_hello.extensions)#hello_extensions.next_protocol_negotiation, - NextProtocolNegotiation = undefined. + Extensions = DecodedHandshakeMessage#server_hello.extensions, + #{next_protocol_negotiation := undefined} = Extensions. + %%-------------------------------------------------------------------- encode_and_decode_npn_server_hello_test(Config) -> HandShakeData = create_server_handshake(#next_protocol_negotiation{extension_data = <<6, "spdy/2">>}), Version = ssl_test_lib:protocol_version(Config), {[{DecodedHandshakeMessage, _Raw}], _} = tls_handshake:get_tls_handshake(Version, list_to_binary(HandShakeData), <<>>, #ssl_options{}), - NextProtocolNegotiation = (DecodedHandshakeMessage#server_hello.extensions)#hello_extensions.next_protocol_negotiation, - ct:log("~p ~n", [NextProtocolNegotiation]), - NextProtocolNegotiation = #next_protocol_negotiation{extension_data = <<6, "spdy/2">>}. + Extensions = DecodedHandshakeMessage#server_hello.extensions, + ct:log("~p ~n", [Extensions]), + #{next_protocol_negotiation := #next_protocol_negotiation{extension_data = <<6, "spdy/2">>}} = Extensions. %%-------------------------------------------------------------------- create_server_hello_with_no_advertised_protocols_test(_Config) -> - Hello = ssl_handshake:server_hello(<<>>, {3, 0}, create_connection_states(), #hello_extensions{}), - undefined = (Hello#server_hello.extensions)#hello_extensions.next_protocol_negotiation. + Hello = ssl_handshake:server_hello(<<>>, {3, 0}, create_connection_states(), #{}), + Extensions = Hello#server_hello.extensions, + #{} = Extensions. %%-------------------------------------------------------------------- create_server_hello_with_advertised_protocols_test(_Config) -> Hello = ssl_handshake:server_hello(<<>>, {3, 0}, create_connection_states(), - #hello_extensions{next_protocol_negotiation = [<<"spdy/1">>, <<"http/1.0">>, <<"http/1.1">>]}), - [<<"spdy/1">>, <<"http/1.0">>, <<"http/1.1">>] = - (Hello#server_hello.extensions)#hello_extensions.next_protocol_negotiation. + #{next_protocol_negotiation => [<<"spdy/1">>, <<"http/1.0">>, <<"http/1.1">>]}), + Extensions = Hello#server_hello.extensions, + #{next_protocol_negotiation := [<<"spdy/1">>, <<"http/1.0">>, <<"http/1.1">>]} = Extensions. %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- @@ -120,9 +122,8 @@ create_client_handshake(Npn) -> session_id = <<>>, cipher_suites = [?TLS_DHE_DSS_WITH_DES_CBC_SHA], compression_methods = "", - extensions = #hello_extensions{ - next_protocol_negotiation = Npn, - renegotiation_info = #renegotiation_info{}} + extensions = #{next_protocol_negotiation => Npn, + renegotiation_info => #renegotiation_info{}} }, Vsn). create_server_handshake(Npn) -> @@ -133,9 +134,8 @@ create_server_handshake(Npn) -> session_id = <<>>, cipher_suite = ?TLS_DHE_DSS_WITH_DES_CBC_SHA, compression_method = 1, - extensions = #hello_extensions{ - next_protocol_negotiation = Npn, - renegotiation_info = #renegotiation_info{}} + extensions = #{next_protocol_negotiation => Npn, + renegotiation_info => #renegotiation_info{}} }, Vsn). create_connection_states() -> @@ -146,5 +146,5 @@ create_connection_states() -> } }, current_read => #{secure_renegotiation => false - } + } }. diff --git a/lib/ssl/test/ssl_packet_SUITE.erl b/lib/ssl/test/ssl_packet_SUITE.erl index 3261244ace..ebf8ddbfac 100644 --- a/lib/ssl/test/ssl_packet_SUITE.erl +++ b/lib/ssl/test/ssl_packet_SUITE.erl @@ -141,6 +141,7 @@ socket_active_packet_tests() -> packet_4_active_some_big, packet_wait_active, packet_size_active, + packet_switch, %% inet header option should be deprecated! header_decode_one_byte_active, header_decode_two_bytes_active, @@ -702,6 +703,34 @@ packet_size_passive(Config) when is_list(Config) -> ssl_test_lib:close(Server), ssl_test_lib:close(Client). + +%%-------------------------------------------------------------------- +packet_switch() -> + [{doc,"Test packet option {packet, 2} followd by {packet, 4}"}]. + +packet_switch(Config) when is_list(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ClientNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, send_switch_packet ,["Hello World", 4]}}, + {options, [{nodelay, true},{packet, 2} | ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + Client = ssl_test_lib:start_client([{node, ServerNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, recv_switch_packet, ["Hello World", 4]}}, + {options, [{nodelay, true}, {packet, 2} | + ClientOpts]}]), + + ssl_test_lib:check_result(Client, ok), + + ssl_test_lib:close(Server), + ssl_test_lib:close(Client). + + %%-------------------------------------------------------------------- packet_cdr_decode() -> [{doc,"Test setting the packet option {packet, cdr}, {mode, binary}"}]. @@ -2286,3 +2315,26 @@ client_reject_packet_opt(Config, PacketOpt) -> ClientOpts]}]), ssl_test_lib:check_result(Client, {error, {options, {not_supported, PacketOpt}}}). + + +send_switch_packet(SslSocket, Data, NextPacket) -> + ssl:send(SslSocket, Data), + receive + {ssl, SslSocket, "Hello World"} -> + ssl:setopts(SslSocket, [{packet, NextPacket}]), + ssl:send(SslSocket, Data), + receive + {ssl, SslSocket, "Hello World"} -> + ok + end + end. +recv_switch_packet(SslSocket, Data, NextPacket) -> + receive + {ssl, SslSocket, "Hello World"} -> + ssl:send(SslSocket, Data), + ssl:setopts(SslSocket, [{packet, NextPacket}]), + receive + {ssl, SslSocket, "Hello World"} -> + ssl:send(SslSocket, Data) + end + end. diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index f3235f5614..39a5bcaad6 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -1003,7 +1003,6 @@ ecc_test_error(COpts, SOpts, CECCOpts, SECCOpts, Config) -> Error = {error, {tls_alert, "insufficient security"}}, check_result(Server, Error, Client, Error). - start_client(openssl, Port, ClientOpts, Config) -> Cert = proplists:get_value(certfile, ClientOpts), Key = proplists:get_value(keyfile, ClientOpts), @@ -2061,3 +2060,40 @@ hardcode_dsa_key(3) -> y = 48598545580251057979126570873881530215432219542526130654707948736559463436274835406081281466091739849794036308281564299754438126857606949027748889019480936572605967021944405048011118039171039273602705998112739400664375208228641666852589396502386172780433510070337359132965412405544709871654840859752776060358, x = 1457508827177594730669011716588605181448418352823}. +tcp_delivery_workaround(Server, ServerMsg, Client, ClientMsg) -> + receive + {Server, ServerMsg} -> + client_msg(Client, ClientMsg); + {Client, ClientMsg} -> + server_msg(Server, ServerMsg); + {Client, {error,closed}} -> + server_msg(Server, ServerMsg); + {Server, {error,closed}} -> + client_msg(Client, ClientMsg) + end. +client_msg(Client, ClientMsg) -> + receive + {Client, ClientMsg} -> + ok; + {Client, {error,closed}} -> + ct:log("client got close"), + ok; + {Client, {error, Reason}} -> + ct:log("client got econnaborted: ~p", [Reason]), + ok; + Unexpected -> + ct:fail(Unexpected) + end. +server_msg(Server, ServerMsg) -> + receive + {Server, ServerMsg} -> + ok; + {Server, {error,closed}} -> + ct:log("server got close"), + ok; + {Server, {error, Reason}} -> + ct:log("server got econnaborted: ~p", [Reason]), + ok; + Unexpected -> + ct:fail(Unexpected) + end. diff --git a/lib/stdlib/doc/src/epp.xml b/lib/stdlib/doc/src/epp.xml index 1dc0161398..d803d259aa 100644 --- a/lib/stdlib/doc/src/epp.xml +++ b/lib/stdlib/doc/src/epp.xml @@ -124,6 +124,10 @@ <fsummary>Open a file for preprocessing.</fsummary> <desc> <p>Opens a file for preprocessing.</p> + <p>If you want to change the file name of the implicit -file() + attributes inserted during preprocessing, you can do with + <c>{source_name, <anno>SourceName</anno>}</c>. If unset it will + default to the name of the opened file.</p> <p>If <c>extra</c> is specified in <c><anno>Options</anno></c>, the return value is <c>{ok, <anno>Epp</anno>, <anno>Extra</anno>}</c> instead @@ -169,6 +173,10 @@ <p>Preprocesses and parses an Erlang source file. Notice that tuple <c>{eof, <anno>Line</anno>}</c> returned at the end of the file is included as a "form".</p> + <p>If you want to change the file name of the implicit -file() + attributes inserted during preprocessing, you can do with + <c>{source_name, <anno>SourceName</anno>}</c>. If unset it will + default to the name of the opened file.</p> <p>If <c>extra</c> is specified in <c><anno>Options</anno></c>, the return value is <c>{ok, [<anno>Form</anno>], <anno>Extra</anno>}</c> instead diff --git a/lib/stdlib/doc/src/rand.xml b/lib/stdlib/doc/src/rand.xml index 25eec216ef..8e657698c6 100644 --- a/lib/stdlib/doc/src/rand.xml +++ b/lib/stdlib/doc/src/rand.xml @@ -38,34 +38,50 @@ <p> This module provides a pseudo random number generator. The module contains a number of algorithms. - The uniform distribution algorithms use the + The uniform distribution algorithms are based on the <url href="http://xorshift.di.unimi.it"> - xoroshiro116+ and xorshift1024* algorithms by Sebastiano Vigna. + Xoroshiro and Xorshift algorithms </url> + by Sebastiano Vigna. The normal distribution algorithm uses the <url href="http://www.jstatsoft.org/v05/i08"> Ziggurat Method by Marsaglia and Tsang </url> on top of the uniform distribution algorithm. </p> - <p>For some algorithms, jump functions are provided for generating - non-overlapping sequences for parallel computations. - The jump functions perform calculations - equivalent to perform a large number of repeated calls - for calculating new states. </p> + <p> + For most algorithms, jump functions are provided for generating + non-overlapping sequences for parallel computations. + The jump functions perform calculations + equivalent to perform a large number of repeated calls + for calculating new states. + </p> <p>The following algorithms are provided:</p> <taglist> - <tag><c>exrop</c></tag> + <tag><c>exsss</c></tag> <item> - <p>Xoroshiro116+, 58 bits precision and period of 2^116-1</p> + <p>Xorshift116**, 58 bits precision and period of 2^116-1</p> <p>Jump function: equivalent to 2^64 calls</p> - </item> - <tag><c>exs1024s</c></tag> - <item> - <p>Xorshift1024*, 64 bits precision and a period of 2^1024-1</p> - <p>Jump function: equivalent to 2^512 calls</p> + <p> + This is the Xorshift116 generator combined with the StarStar scrambler + from the 2018 paper by David Blackman and Sebastiano Vigna: + <url href="http://vigna.di.unimi.it/ftp/papers/ScrambledLinear.pdf"> + Scrambled Linear Pseudorandom Number Generators + </url> + </p> + <p> + The generator does not need 58-bit rotates so it is faster + than the Xoroshiro116 generator, and when combined with + the StarStar scrambler it does not have any weak low bits + like <c>exrop</c> (Xoroshiro116+). + </p> + <p> + Alas, this combination is about 10% slower than <c>exrop</c>, + but is despite that the default algorithm thanks to its + statistical qualities. + </p> </item> <tag><c>exro928ss</c></tag> <item> @@ -77,8 +93,8 @@ <url href="http://vigna.di.unimi.it/ftp/papers/ScrambledLinear.pdf"> Scrambled Linear Pseudorandom Number Generators </url> - that on a 64 bit Erlang system executes only about 30% slower than - the default <c>exrop</c> algorithm but with much longer period + that on a 64 bit Erlang system executes only about 40% slower than + the default <c>exsss</c> algorithm but with much longer period and better statistical properties, and on the flip side a larger state. </p> @@ -87,6 +103,16 @@ the 58 bit adaption. </p> </item> + <tag><c>exrop</c></tag> + <item> + <p>Xoroshiro116+, 58 bits precision and period of 2^116-1</p> + <p>Jump function: equivalent to 2^64 calls</p> + </item> + <tag><c>exs1024s</c></tag> + <item> + <p>Xorshift1024*, 64 bits precision and a period of 2^1024-1</p> + <p>Jump function: equivalent to 2^512 calls</p> + </item> <tag><c>exsp</c></tag> <item> <p>Xorshift116+, 58 bits precision and period of 2^116-1</p> @@ -103,7 +129,7 @@ </taglist> <p> - The default algorithm is <c>exrop</c> (Xoroshiro116+). + The default algorithm is <c>exsss</c> (Xorshift116**). If a specific algorithm is required, ensure to always use <seealso marker="#seed-1"> <c>seed/1</c></seealso> to initialize the state. @@ -174,19 +200,19 @@ R1 = rand:uniform(),</pre> <p>Use a specified algorithm:</p> <pre> -_ = rand:seed(exs1024s), +_ = rand:seed(exs928ss), R2 = rand:uniform(),</pre> <p>Use a specified algorithm with a constant seed:</p> <pre> -_ = rand:seed(exs1024s, {123, 123534, 345345}), +_ = rand:seed(exs928ss, {123, 123534, 345345}), R3 = rand:uniform(),</pre> <p>Use the functional API with a non-constant seed:</p> <pre> -S0 = rand:seed_s(exrop), +S0 = rand:seed_s(exsss), {R4, S1} = rand:uniform_s(S0),</pre> <p>Textbook basic form Box-Muller standard normal deviate</p> @@ -215,8 +241,9 @@ SND0 = math:sqrt(-2 * math:log(R5)) * math:cos(math:pi() * R6)</pre> </note> <p> - For all these generators except <c>exro928ss</c> the lowest bit(s) - has got a slightly less random behaviour than all other bits. + For all these generators except <c>exro928ss</c> and <c>exsss</c> + the lowest bit(s) has got a slightly less + random behaviour than all other bits. 1 bit for <c>exrop</c> (and <c>exsp</c>), and 3 bits for <c>exs1024s</c>. See for example the explanation in the @@ -231,7 +258,7 @@ up to (and included) 16TB, with the exception of binary rank tests, which fail due to the lowest bit being an LFSR; all other bits pass all tests. We suggest to use a sign test to extract a random Boolean value.</pre> <p> - If this is a problem; to generate a boolean + If this is a problem; to generate a boolean with these algorithms use something like this: </p> <pre>(rand:uniform(16) > 8)</pre> @@ -299,19 +326,19 @@ tests. We suggest to use a sign test to extract a random Boolean value.</pre> </desc> </datatype> <datatype> - <name name="exrop_state"/> + <name name="exsplus_state"/> <desc><p>Algorithm specific internal state</p></desc> </datatype> <datatype> - <name name="exs1024_state"/> + <name name="exro928_state"/> <desc><p>Algorithm specific internal state</p></desc> </datatype> <datatype> - <name name="exro928_state"/> + <name name="exrop_state"/> <desc><p>Algorithm specific internal state</p></desc> </datatype> <datatype> - <name name="exsplus_state"/> + <name name="exs1024_state"/> <desc><p>Algorithm specific internal state</p></desc> </datatype> <datatype> diff --git a/lib/stdlib/src/epp.erl b/lib/stdlib/src/epp.erl index cc34d4bdd3..181a524db6 100644 --- a/lib/stdlib/src/epp.erl +++ b/lib/stdlib/src/epp.erl @@ -117,6 +117,7 @@ open(Name, File, StartLocation, Path, Pdm) -> {'ok', Epp} | {'ok', Epp, Extra} | {'error', ErrorDescriptor} when Options :: [{'default_encoding', DefEncoding :: source_encoding()} | {'includes', IncludePath :: [DirectoryName :: file:name()]} | + {'source_name', SourceName :: file:name()} | {'macros', PredefMacros :: macros()} | {'name',FileName :: file:name()} | 'extra'], @@ -248,6 +249,7 @@ parse_file(Ifile, Path, Predefs) -> {'ok', [Form]} | {'ok', [Form], Extra} | {error, OpenError} when FileName :: file:name(), Options :: [{'includes', IncludePath :: [DirectoryName :: file:name()]} | + {'source_name', SourceName :: file:name()} | {'macros', PredefMacros :: macros()} | {'default_encoding', DefEncoding :: source_encoding()} | 'extra'], @@ -540,9 +542,10 @@ server(Pid, Name, Options, #epp{pre_opened=PreOpened}=St) -> init_server(Pid, Name, Options, St) end. -init_server(Pid, Name, Options, St0) -> +init_server(Pid, FileName, Options, St0) -> + SourceName = proplists:get_value(source_name, Options, FileName), Pdm = proplists:get_value(macros, Options, []), - Ms0 = predef_macros(Name), + Ms0 = predef_macros(FileName), case user_predef(Pdm, Ms0) of {ok,Ms1} -> #epp{file = File, location = AtLocation} = St0, @@ -552,14 +555,14 @@ init_server(Pid, Name, Options, St0) -> epp_reply(Pid, {ok,self(),Encoding}), %% ensure directory of current source file is %% first in path - Path = [filename:dirname(Name) | + Path = [filename:dirname(FileName) | proplists:get_value(includes, Options, [])], - St = St0#epp{delta=0, name=Name, name2=Name, + St = St0#epp{delta=0, name=SourceName, name2=SourceName, path=Path, macs=Ms1, default_encoding=DefEncoding}, From = wait_request(St), Anno = erl_anno:new(AtLocation), - enter_file_reply(From, file_name(Name), Anno, + enter_file_reply(From, file_name(SourceName), Anno, AtLocation, code), wait_req_scan(St); {error,E} -> diff --git a/lib/stdlib/src/ms_transform.erl b/lib/stdlib/src/ms_transform.erl index 3845e35e9b..6d243e1bec 100644 --- a/lib/stdlib/src/ms_transform.erl +++ b/lib/stdlib/src/ms_transform.erl @@ -946,6 +946,7 @@ real_guard_function(node,0) -> true; real_guard_function(node,1) -> true; real_guard_function(round,1) -> true; real_guard_function(size,1) -> true; +real_guard_function(bit_size,1) -> true; real_guard_function(map_size,1) -> true; real_guard_function(map_get,2) -> true; real_guard_function(tl,1) -> true; diff --git a/lib/stdlib/src/rand.erl b/lib/stdlib/src/rand.erl index 9854c778a1..3a9a1e007b 100644 --- a/lib/stdlib/src/rand.erl +++ b/lib/stdlib/src/rand.erl @@ -43,13 +43,13 @@ %% Debug -export([make_float/3, float2str/1, bc64/1]). --compile({inline, [exs64_next/1, exsplus_next/1, +-compile({inline, [exs64_next/1, exsplus_next/1, exsss_next/1, exs1024_next/1, exs1024_calc/2, exro928_next_state/4, exrop_next/1, exrop_next_s/2, get_52/1, normal_kiwi/1]}). --define(DEFAULT_ALG_HANDLER, exrop). +-define(DEFAULT_ALG_HANDLER, exsss). -define(SEED_DICT, rand_seed). %% ===================================================================== @@ -86,7 +86,7 @@ %% This depends on the algorithm handler function -type alg_state() :: - exrop_state() | exs1024_state() | exro928_state() | exsplus_state() | + exsplus_state() | exro928_state() | exrop_state() | exs1024_state() | exs64_state() | term(). %% This is the algorithm handling definition within this module, @@ -131,7 +131,7 @@ %% Algorithm state -type state() :: {alg_handler(), alg_state()}. -type builtin_alg() :: - exrop | exs1024s | exro928ss | exsp | exs64 | exsplus | exs1024. + exsss | exro928ss | exrop | exs1024s | exsp | exs64 | exsplus | exs1024. -type alg() :: builtin_alg() | atom(). -type export_state() :: {alg(), alg_state()}. -type seed() :: [integer()] | integer() | {integer(), integer(), integer()}. @@ -139,7 +139,7 @@ [builtin_alg/0, alg/0, alg_handler/0, alg_state/0, state/0, export_state/0, seed/0]). -export_type( - [exrop_state/0, exs1024_state/0, exro928_state/0, exsplus_state/0, + [exsplus_state/0, exro928_state/0, exrop_state/0, exs1024_state/0, exs64_state/0]). %% ===================================================================== @@ -618,6 +618,11 @@ mk_alg(exsp) -> uniform=>fun exsp_uniform/1, uniform_n=>fun exsp_uniform/2, jump=>fun exsplus_jump/1}, fun exsplus_seed/1}; +mk_alg(exsss) -> + {#{type=>exsss, bits=>58, next=>fun exsss_next/1, + uniform=>fun exsss_uniform/1, uniform_n=>fun exsss_uniform/2, + jump=>fun exsplus_jump/1}, + fun exsss_seed/1}; mk_alg(exs1024) -> {#{type=>exs1024, max=>?MASK(64), next=>fun exs1024_next/1, jump=>fun exs1024_jump/1}, @@ -675,6 +680,36 @@ exs64_next(R) -> %% 58 bits fits into an immediate on 64bits erlang and is thus much faster. %% Modification of the original Xorshift128+ algorithm to 116 %% by Sebastiano Vigna, a lot of thanks for his help and work. +%% +%% Reference C code for Xorshift116+ and Xorshift116** +%% +%% #include <stdint.h> +%% +%% #define MASK(b, v) (((UINT64_C(1) << (b)) - 1) & (v)) +%% #define BSL(b, v, n) (MASK((b)-(n), (v)) << (n)) +%% #define ROTL(b, v, n) (BSL((b), (v), (n)) | ((v) >> ((b)-(n)))) +%% +%% uint64_t s[2]; +%% +%% uint64_t next(void) { +%% uint64_t s1 = s[0]; +%% const uint64_t s0 = s[1]; +%% +%% s1 ^= BSL(58, s1, 24); // a +%% s1 ^= s0 ^ (s1 >> 11) ^ (s0 >> 41); // b, c +%% s[0] = s0; +%% s[1] = s1; +%% +%% const uint64_t result_plus = MASK(58, s0 + s1); +%% uint64_t result_starstar = s0; +%% result_starstar = MASK(58, result_starstar * 5); +%% result_starstar = ROTL(58, result_starstar, 7); +%% result_starstar = MASK(58, result_starstar * 9); +%% +%% return result_plus; +%% return result_starstar; +%% } +%% %% ===================================================================== -opaque exsplus_state() :: nonempty_improper_list(uint58(), uint58()). @@ -697,16 +732,62 @@ exsplus_seed({A1, A2, A3}) -> tl(R1)]), R2. +-dialyzer({no_improper_lists, exsss_seed/1}). + +exsss_seed(L) when is_list(L) -> + [S0,S1] = seed58_nz(2, L), + [S0|S1]; +exsss_seed(X) when is_integer(X) -> + [S0,S1] = seed58(2, ?MASK(64, X)), + [S0|S1]; +%% +%% Seed from traditional integer triple - mix into splitmix +exsss_seed({A1, A2, A3}) -> + {_, X0} = seed58(?MASK(64, A1)), + {S0, X1} = seed58(?MASK(64, A2) bxor X0), + {S1, _} = seed58(?MASK(64, A3) bxor X1), + [S0|S1]. + +%% Advance Xorshift116 state one step +-define( + exs_next(S0, S1, S1_b), + begin + S1_b = S1 bxor ?BSL(58, S1, 24), + S1_b bxor S0 bxor (S1_b bsr 11) bxor (S0 bsr 41) + end). + +-define( + scramble_starstar(S, V_a, V_b), + begin + %% The multiply by add shifted trick avoids creating bignums + %% which improves performance significantly + %% + V_a = ?MASK(58, S + ?BSL(58, S, 2)), % V_a = S * 5 + V_b = ?ROTL(58, V_a, 7), + ?MASK(58, V_b + ?BSL(58, V_b, 3)) % V_b * 9 + end). + -dialyzer({no_improper_lists, exsplus_next/1}). -%% Advance xorshift116+ state for one step and generate 58bit unsigned integer +%% Advance state and generate 58bit unsigned integer -spec exsplus_next(exsplus_state()) -> {uint58(), exsplus_state()}. exsplus_next([S1|S0]) -> %% Note: members s0 and s1 are swapped here - S11 = S1 bxor ?BSL(58, S1, 24), - S12 = S11 bxor S0 bxor (S11 bsr 11) bxor (S0 bsr 41), - {?MASK(58, S0 + S12), [S0|S12]}. + NewS1 = ?exs_next(S0, S1, S1_1), + {?MASK(58, S0 + NewS1), [S0|NewS1]}. +%% %% Note: members s0 and s1 are swapped here +%% S11 = S1 bxor ?BSL(58, S1, 24), +%% S12 = S11 bxor S0 bxor (S11 bsr 11) bxor (S0 bsr 41), +%% {?MASK(58, S0 + S12), [S0|S12]}. + +-dialyzer({no_improper_lists, exsss_next/1}). +-spec exsss_next(exsplus_state()) -> {uint58(), exsplus_state()}. +exsss_next([S1|S0]) -> + %% Note: members s0 and s1 are swapped here + NewS1 = ?exs_next(S0, S1, S1_1), + {?scramble_starstar(S0, V_0, V_1), [S0|NewS1]}. +%% {?MASK(58, S0 + NewS1), [S0|NewS1]}. exsp_uniform({Alg, R0}) -> {I, R1} = exsplus_next(R0), @@ -714,18 +795,48 @@ exsp_uniform({Alg, R0}) -> %% randomness quality than the others {(I bsr (58-53)) * ?TWO_POW_MINUS53, {Alg, R1}}. +exsss_uniform({Alg, R0}) -> + {I, R1} = exsss_next(R0), + {(I bsr (58-53)) * ?TWO_POW_MINUS53, {Alg, R1}}. + exsp_uniform(Range, {Alg, R}) -> {V, R1} = exsplus_next(R), MaxMinusRange = ?BIT(58) - Range, ?uniform_range(Range, Alg, R1, V, MaxMinusRange, I). +exsss_uniform(Range, {Alg, R}) -> + {V, R1} = exsss_next(R), + MaxMinusRange = ?BIT(58) - Range, + ?uniform_range(Range, Alg, R1, V, MaxMinusRange, I). + -%% This is the jump function for the exsplus generator, equivalent +%% This is the jump function for the exs* generators, +%% i.e the Xorshift116 generators, equivalent %% to 2^64 calls to next/1; it can be used to generate 2^52 %% non-overlapping subsequences for parallel computations. %% Note: the jump function takes 116 times of the execution time of %% next/1. - +%% +%% #include <stdint.h> +%% +%% void jump(void) { +%% static const uint64_t JUMP[] = { 0x02f8ea6bc32c797, +%% 0x345d2a0f85f788c }; +%% int i, b; +%% uint64_t s0 = 0; +%% uint64_t s1 = 0; +%% for(i = 0; i < sizeof JUMP / sizeof *JUMP; i++) +%% for(b = 0; b < 58; b++) { +%% if (JUMP[i] & 1ULL << b) { +%% s0 ^= s[0]; +%% s1 ^= s[1]; +%% } +%% next(); +%% } +%% s[0] = s0; +%% s[1] = s1; +%% } +%% %% -define(JUMPCONST, 16#000d174a83e17de2302f8ea6bc32c797). %% split into 58-bit chunks %% and two iterative executions @@ -973,13 +1084,14 @@ exro928ss_next({[S15,S0|Ss], Rs}) -> %% {S, R, T} = {5, 7, 9} %% const uint64_t result_starstar = rotl(s0 * S, R) * T; %% - %% The multiply by add shifted trick avoids creating bignums - %% which improves performance significantly - %% - V0 = ?MASK(58, S0 + ?BSL(58, S0, 2)), % V0 = S0 * 5 - V1 = ?ROTL(58, V0, 7), - V = ?MASK(58, V1 + ?BSL(58, V1, 3)), % V = V1 * 9 - {V, SR}; + {?scramble_starstar(S0, V_0, V_1), SR}; +%% %% The multiply by add shifted trick avoids creating bignums +%% %% which improves performance significantly +%% %% +%% V0 = ?MASK(58, S0 + ?BSL(58, S0, 2)), % V0 = S0 * 5 +%% V1 = ?ROTL(58, V0, 7), +%% V = ?MASK(58, V1 + ?BSL(58, V1, 3)), % V = V1 * 9 +%% {V, SR}; exro928ss_next({[S15], Rs}) -> exro928ss_next({[S15|lists:reverse(Rs)], []}). diff --git a/lib/stdlib/src/stdlib.appup.src b/lib/stdlib/src/stdlib.appup.src index 8d1cc09a8b..8c0b186288 100644 --- a/lib/stdlib/src/stdlib.appup.src +++ b/lib/stdlib/src/stdlib.appup.src @@ -19,8 +19,10 @@ {"%VSN%", %% Up from - max one major revision back [{<<"3\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-20.* - {<<"3\\.5(\\.[0-9]+)*">>,[restart_new_emulator]}],% OTP-21.* + {<<"3\\.5(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-21.0 + {<<"3\\.6(\\.[0-9]+)*">>,[restart_new_emulator]}],% OTP-21.1 %% Down to - max one major revision back [{<<"3\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-20.* - {<<"3\\.5(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-20.* + {<<"3\\.5(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-21.0 + {<<"3\\.6(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-21.1 }. diff --git a/lib/stdlib/test/epp_SUITE.erl b/lib/stdlib/test/epp_SUITE.erl index 10e1b75e0f..a90beed4f3 100644 --- a/lib/stdlib/test/epp_SUITE.erl +++ b/lib/stdlib/test/epp_SUITE.erl @@ -29,7 +29,7 @@ otp_8562/1, otp_8665/1, otp_8911/1, otp_10302/1, otp_10820/1, otp_11728/1, encoding/1, extends/1, function_macro/1, test_error/1, test_warning/1, otp_14285/1, - test_if/1]). + test_if/1,source_name/1]). -export([epp_parse_erl_form/2]). @@ -70,7 +70,7 @@ all() -> overload_mac, otp_8388, otp_8470, otp_8562, otp_8665, otp_8911, otp_10302, otp_10820, otp_11728, encoding, extends, function_macro, test_error, test_warning, - otp_14285, test_if]. + otp_14285, test_if, source_name]. groups() -> [{upcase_mac, [], [upcase_mac_1, upcase_mac_2]}, @@ -1702,6 +1702,18 @@ function_macro(Config) -> ok. +source_name(Config) when is_list(Config) -> + DataDir = proplists:get_value(data_dir, Config), + File = filename:join(DataDir, "source_name.erl"), + + source_name_1(File, "/test/gurka.erl"), + source_name_1(File, "gaffel.erl"), + + ok. + +source_name_1(File, Expected) -> + Res = epp:parse_file(File, [{source_name, Expected}]), + {ok, [{attribute,_,file,{Expected,_}} | _Forms]} = Res. check(Config, Tests) -> eval_tests(Config, fun check_test/2, Tests). diff --git a/lib/stdlib/test/epp_SUITE_data/source_name.erl b/lib/stdlib/test/epp_SUITE_data/source_name.erl new file mode 100644 index 0000000000..71ad2dddb9 --- /dev/null +++ b/lib/stdlib/test/epp_SUITE_data/source_name.erl @@ -0,0 +1,27 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(source_name). +-export([ok/0]). + +%% Changing source name should not affect headers +-include("bar.hrl"). + +ok() -> ok. diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 8940f0b58c..2c0692855f 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -41,7 +41,7 @@ -export([t_delete_object/1, t_init_table/1, t_whitebox/1, select_bound_chunk/1, t_delete_all_objects/1, t_insert_list/1, t_test_ms/1, - t_select_delete/1,t_select_replace/1,t_ets_dets/1]). + t_select_delete/1,t_select_replace/1,t_select_replace_next_bug/1,t_ets_dets/1]). -export([ordered/1, ordered_match/1, interface_equality/1, fixtable_next/1, fixtable_insert/1, rename/1, rename_unnamed/1, evil_rename/1, @@ -126,6 +126,7 @@ all() -> select_bound_chunk, t_init_table, t_whitebox, t_delete_all_objects, t_insert_list, t_test_ms, t_select_delete, t_select_replace, + t_select_replace_next_bug, t_ets_dets, memory, t_select_reverse, t_bucket_disappears, t_named_select, select_fail, t_insert_new, t_repair_continuation, @@ -1471,6 +1472,25 @@ t_select_replace(Config) when is_list(Config) -> verify_etsmem(EtsMem). +%% OTP-15346: Bug caused select_replace of bound key to corrupt static stack +%% used by ets:next and ets:prev. +t_select_replace_next_bug(Config) when is_list(Config) -> + T = ets:new(k, [ordered_set]), + [ets:insert(T, {I, value}) || I <- lists:seq(1,10)], + 1 = ets:first(T), + + %% Make sure select_replace does not leave pointer + %% to deallocated {2,value} in static stack. + MS = [{{2,value}, [], [{{2,"new_value"}}]}], + 1 = ets:select_replace(T, MS), + + %% This would crash or give wrong result at least on DEBUG emulator + %% where deallocated memory is overwritten. + 2 = ets:next(T, 1), + + ets:delete(T). + + %% Test that partly bound keys gives faster matches. partly_bound(Config) when is_list(Config) -> case os:type() of @@ -3457,6 +3477,13 @@ delete_large_tab_do(Opts,Data) -> delete_large_tab_1(Name, Flags, Data, Fix) -> + case is_redundant_opts_combo(Flags) of + true -> skip; + false -> + delete_large_tab_2(Name, Flags, Data, Fix) + end. + +delete_large_tab_2(Name, Flags, Data, Fix) -> Tab = ets_new(Name, Flags), ets:insert(Tab, Data), @@ -3528,6 +3555,13 @@ delete_large_named_table_do(Opts,Data) -> delete_large_named_table_1(foo_hash, [named_table | Opts], Data, true). delete_large_named_table_1(Name, Flags, Data, Fix) -> + case is_redundant_opts_combo(Flags) of + true -> skip; + false -> + delete_large_named_table_2(Name, Flags, Data, Fix) + end. + +delete_large_named_table_2(Name, Flags, Data, Fix) -> Tab = ets_new(Name, Flags), ets:insert(Tab, Data), @@ -3576,6 +3610,13 @@ evil_delete_do(Opts,Data) -> [TabA,TabB,TabC,TabD]). evil_delete_not_owner(Name, Flags, Data, Fix) -> + case is_redundant_opts_combo(Flags) of + true -> skip; + false -> + evil_delete_not_owner_1(Name, Flags, Data, Fix) + end. + +evil_delete_not_owner_1(Name, Flags, Data, Fix) -> io:format("Not owner: ~p, fix = ~p", [Name,Fix]), Tab = ets_new(Name, [public|Flags]), ets:insert(Tab, Data), @@ -3601,6 +3642,13 @@ evil_delete_not_owner(Name, Flags, Data, Fix) -> Tab. evil_delete_owner(Name, Flags, Data, Fix) -> + case is_redundant_opts_combo(Flags) of + true -> skip; + false -> + evil_delete_owner_1(Name, Flags, Data, Fix) + end. + +evil_delete_owner_1(Name, Flags, Data, Fix) -> Fun = fun() -> Tab = ets_new(Name, [public|Flags]), ets:insert(Tab, Data), @@ -5299,7 +5347,7 @@ otp_7665_act(Tab,Min,Max,DelNr) -> %% Whitebox testing of meta name table hashing. meta_wb(Config) when is_list(Config) -> EtsMem = etsmem(), - repeat_for_opts_all_table_types(fun meta_wb_do/1), + repeat_for_opts_all_non_stim_table_types(fun meta_wb_do/1), verify_etsmem(EtsMem). @@ -5586,25 +5634,28 @@ meta_newdel_named(Config) when is_list(Config) -> %% Concurrent insert's on same table. smp_insert(Config) when is_list(Config) -> - repeat_for_all_set_table_types( - fun(Opts) -> - ets_new(smp_insert,[named_table,public,{write_concurrency,true}|Opts]), - InitF = fun(_) -> ok end, - ExecF = fun(_) -> true = ets:insert(smp_insert,{rand:uniform(10000)}) - end, - FiniF = fun(_) -> ok end, - run_smp_workers(InitF,ExecF,FiniF,100000), - verify_table_load(smp_insert), - ets:delete(smp_insert) - end). + repeat_for_opts(fun smp_insert_do/1, + [[set,ordered_set,stim_cat_ord_set]]). + +smp_insert_do(Opts) -> + ets_new(smp_insert,[named_table,public,{write_concurrency,true}|Opts]), + InitF = fun(_) -> ok end, + ExecF = fun(_) -> true = ets:insert(smp_insert,{rand:uniform(10000)}) + end, + FiniF = fun(_) -> ok end, + run_smp_workers(InitF,ExecF,FiniF,100000), + verify_table_load(smp_insert), + ets:delete(smp_insert). %% Concurrent deletes on same fixated table. smp_fixed_delete(Config) when is_list(Config) -> - only_if_smp(fun()->smp_fixed_delete_do() end). + only_if_smp(fun()-> + repeat_for_opts(fun smp_fixed_delete_do/1, + [[set,ordered_set,stim_cat_ord_set]]) + end). -smp_fixed_delete_do() -> - repeat_for_opts_all_set_table_types( - fun(Opts) -> +smp_fixed_delete_do(Opts) -> + begin T = ets_new(foo,[public,{write_concurrency,true}|Opts]), %%Mem = ets:info(T,memory), NumOfObjs = 100000, @@ -5632,7 +5683,7 @@ smp_fixed_delete_do() -> %%Mem = ets:info(T,memory), %%verify_table_load(T), ets:delete(T) - end). + end. %% ERL-720 %% Provoke race between ets:delete and table unfix (by select_count) @@ -5887,8 +5938,11 @@ otp_8732(Config) when is_list(Config) -> %% Run concurrent select_delete (and inserts) on same table. smp_select_delete(Config) when is_list(Config) -> - repeat_for_opts_all_set_table_types( - fun(Opts) -> + repeat_for_opts(fun smp_select_delete_do/1, + [[set,ordered_set,stim_cat_ord_set], read_concurrency, compressed]). + +smp_select_delete_do(Opts) -> + begin % indentation T = ets_new(smp_select_delete,[named_table,public,{write_concurrency,true}|Opts]), Mod = 17, Zeros = erlang:make_tuple(Mod,0), @@ -5942,7 +5996,7 @@ smp_select_delete(Config) when is_list(Config) -> 0 = ets:info(T,size), false = ets:info(T,fixed), ets:delete(T) - end), + end, % indentation ok. smp_select_replace(Config) when is_list(Config) -> @@ -6253,6 +6307,19 @@ do_work(WorksDoneSoFar, Table, ProbHelpTab, Range, Operations) -> do_work(WorksDoneSoFar + 1, Table, ProbHelpTab, Range, Operations) end. +prefill_table(T, KeyRange, Num) -> + Seed = rand:uniform(KeyRange), + %%io:format("prefill_table: Seed = ~p\n", [Seed]), + RState = unique_rand_start(KeyRange, Seed), + prefill_table_loop(T, RState, Num), + Num = ets:info(T, size). + +prefill_table_loop(_, _, 0) -> + ok; +prefill_table_loop(T, RS0, N) -> + {Key, RS1} = unique_rand_next(RS0), + ets:insert(T, {Key}), + prefill_table_loop(T, RS1, N-1). throughput_benchmark() -> throughput_benchmark(false, not_set, not_set). @@ -6337,14 +6404,6 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> false -> Calculate([Count*2,Count|Rest]) end end, - PrefillTable = fun Prefill(T, KeyRange) -> - Size = ets:info(T, size), - case Size > KeyRange / 2 of - true -> ok; - false -> ets:insert(T, {rand:uniform(KeyRange)}), - Prefill(T, KeyRange) - end - end, CalculateOpsProbHelpTab = fun Calculate([{_, OpName}], _) -> [{1.0, OpName}]; @@ -6386,7 +6445,7 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> Range, Duration, RecoverTime) -> ProbHelpTab = CalculateOpsProbHelpTab(Scenario, 0), Table = ets:new(t, TableConfig), - PrefillTable(Table, Range), + prefill_table(Table, Range, Range div 2), SafeFixTableIfRequired(Table, Scenario, true), ParentPid = self(), ChildPids = @@ -6426,7 +6485,7 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> end, KeyRanges = % Sizes of the key ranges case TestMode of - true -> [100000]; + true -> [50000]; false -> [1000000] end, Duration = @@ -7225,16 +7284,22 @@ repeat_for_opts(F, OptGenList) when is_function(F, 1) -> repeat_for_opts(F, [], Acc) -> lists:foldl(fun(Opts, RV_Acc) -> OptList = lists:filter(fun(E) -> E =/= void end, Opts), - io:format("Calling with options ~p\n",[OptList]), - RV = F(OptList), - case RV_Acc of - {comment,_} -> RV_Acc; - _ -> case RV of - {comment,_} -> RV; - _ -> [RV | RV_Acc] - end - end - end, [], Acc); + case is_redundant_opts_combo(OptList) of + true -> + %%io:format("Ignoring redundant options ~p\n",[OptList]), + ok; + false -> + io:format("Calling with options ~p\n",[OptList]), + RV = F(OptList), + case RV_Acc of + {comment,_} -> RV_Acc; + _ -> case RV of + {comment,_} -> RV; + _ -> [RV | RV_Acc] + end + end + end + end, [], Acc); repeat_for_opts(F, [OptList | Tail], []) when is_list(OptList) -> repeat_for_opts(F, Tail, [[Opt] || Opt <- OptList]); repeat_for_opts(F, [OptList | Tail], AccList) when is_list(OptList) -> @@ -7242,15 +7307,22 @@ repeat_for_opts(F, [OptList | Tail], AccList) when is_list(OptList) -> repeat_for_opts(F, [Atom | Tail], AccList) when is_atom(Atom) -> repeat_for_opts(F, [repeat_for_opts_atom2list(Atom) | Tail ], AccList). -repeat_for_opts_atom2list(set_types) -> [void,set,ordered_set,stim_cat_ord_set,cat_ord_set]; +repeat_for_opts_atom2list(set_types) -> [set,ordered_set,stim_cat_ord_set,cat_ord_set]; repeat_for_opts_atom2list(ord_set_types) -> [ordered_set,stim_cat_ord_set,cat_ord_set]; -repeat_for_opts_atom2list(all_types) -> [void,set,ordered_set,stim_cat_ord_set,cat_ord_set,bag,duplicate_bag]; -repeat_for_opts_atom2list(all_non_stim_types) -> [void,set,ordered_set,cat_ord_set,bag,duplicate_bag]; -repeat_for_opts_atom2list(all_non_stim_set_types) -> [void,set,ordered_set,cat_ord_set]; +repeat_for_opts_atom2list(all_types) -> [set,ordered_set,stim_cat_ord_set,cat_ord_set,bag,duplicate_bag]; +repeat_for_opts_atom2list(all_non_stim_types) -> [set,ordered_set,cat_ord_set,bag,duplicate_bag]; +repeat_for_opts_atom2list(all_non_stim_set_types) -> [set,ordered_set,cat_ord_set]; repeat_for_opts_atom2list(write_concurrency) -> [{write_concurrency,false},{write_concurrency,true}]; repeat_for_opts_atom2list(read_concurrency) -> [{read_concurrency,false},{read_concurrency,true}]; repeat_for_opts_atom2list(compressed) -> [compressed,void]. +is_redundant_opts_combo(Opts) -> + (lists:member(stim_cat_ord_set, Opts) orelse + lists:member(cat_ord_set, Opts)) + andalso + (lists:member({write_concurrency, false}, Opts) orelse + lists:member(private, Opts) orelse + lists:member(protected, Opts)). ets_new(Name, Opts) -> ReplaceStimOrdSetHelper = @@ -7361,3 +7433,49 @@ syrup_factor() -> valgrind -> 20; _ -> 1 end. + + +%% +%% This is a pseudo random number generator for UNIQUE integers. +%% All integers between 1 and Max will be generated before it repeat itself. +%% It's a variant of this one using quadratic residues by Jeff Preshing: +%% http://preshing.com/20121224/how-to-generate-a-sequence-of-unique-random-integers/ +%% +unique_rand_start(Max, Seed) -> + L = lists:dropwhile(fun(P) -> P < Max end, + primes_3mod4()), + [P | _] = case L of + [] -> + error("Random range too large"); + _ -> + L + end, + 3 = P rem 4, + {0, {Max, P, Seed}}. + +unique_rand_next({N, {Max, P, Seed}=Const}) -> + case dquad(P, N, Seed) + 1 of + RND when RND > Max -> % Too large, skip + unique_rand_next({N+1, Const}); + RND -> + {RND, {N+1, Const}} + end. + +%% A one-to-one relation between all integers 0 =< X < Prime +%% if Prime rem 4 == 3. +quad(Prime, X) -> + Rem = X*X rem Prime, + case 2*X < Prime of + true -> + Rem; + false -> + Prime - Rem + end. + +dquad(Prime, X, Seed) -> + quad(Prime, (quad(Prime, X) + Seed) rem Prime). + +%% Primes where P rem 4 == 3. +primes_3mod4() -> + [103, 211, 503, 1019, 2003, 5003, 10007, 20011, 50023, + 100003, 200003, 500083, 1000003, 2000003]. diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl index 4cb1c0b13d..7685c17967 100644 --- a/lib/stdlib/test/rand_SUITE.erl +++ b/lib/stdlib/test/rand_SUITE.erl @@ -79,7 +79,7 @@ test() -> end, Tests). algs() -> - [exrop, exsp, exs1024s, exs64, exsplus, exs1024, exro928ss]. + [exsss, exrop, exsp, exs1024s, exs64, exsplus, exs1024, exro928ss]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -276,7 +276,7 @@ gen(Algo) -> if Algo =:= exs64 -> %% Printed with orig 'C' code and this seed rand:seed_s(exs64, [12345678]); - Algo =:= exsplus; Algo =:= exsp; Algo =:= exrop -> + Algo =:= exsplus; Algo =:= exsp; Algo =:= exrop; Algo =:= exsss -> %% Printed with orig 'C' code and this seed rand:seed_s(Algo, [12345678,12345678]); Algo =:= exs1024; Algo =:= exs1024s; Algo =:= exro928ss -> @@ -438,7 +438,7 @@ stats_standard_normal_box_muller(Config) when is_list(Config) -> ([S|Z]) -> {Z, [S]} end, - State = [rand:seed(exrop)], + State = [rand:seed(exsss)], stats_standard_normal(NormalS, State, 3) catch error:_ -> {skip, "math:erfc/1 not supported"} @@ -463,7 +463,7 @@ stats_standard_normal_box_muller_2(Config) when is_list(Config) -> ([S|Z]) -> {Z, [S]} end, - State = [rand:seed(exrop)], + State = [rand:seed(exsss)], stats_standard_normal(NormalS, State, 3) catch error:_ -> {skip, "math:erfc/1 not supported"} @@ -471,10 +471,11 @@ stats_standard_normal_box_muller_2(Config) when is_list(Config) -> stats_standard_normal(Config) when is_list(Config) -> + Retries = 7, try math:erfc(1.0) of _ -> stats_standard_normal( - fun rand:normal_s/1, rand:seed_s(exrop), 3) + fun rand:normal_s/1, rand:seed_s(exsss), Retries) catch error:_ -> {skip, "math:erfc/1 not supported"} end. @@ -1070,7 +1071,7 @@ do_measure(_Config) -> end, State) end, - exrop, TMarkNormalFloat), + exsss, TMarkNormalFloat), ok. -define(LOOP_MEASURE, (?LOOP div 5)). @@ -1169,7 +1170,7 @@ gen_jump_1(Algo) -> catch error:not_implemented -> [error_not_implemented] end; - _ when Algo =:= exsplus; Algo =:= exsp; Algo =:= exrop -> + _ when Algo =:= exsplus; Algo =:= exsp; Algo =:= exrop; Algo =:= exsss -> %% Printed with orig 'C' code and this seed gen_jump_2( rand:seed_s(Algo, [12345678,12345678])); @@ -1223,7 +1224,7 @@ gen_jump_p1(Algo) -> catch error:not_implemented -> [error_not_implemented] end; - _ when Algo =:= exsplus; Algo =:= exsp; Algo =:= exrop -> + _ when Algo =:= exsplus; Algo =:= exsp; Algo =:= exrop; Algo =:= exsss -> %% Printed with orig 'C' code and this seed gen_jump_p2( rand:seed(Algo, [12345678,12345678])); @@ -1376,6 +1377,34 @@ reference_val(exsplus) -> 16#3dd493b8012970f,16#be13bed1e00e5c,16#ceef033b74ae10,16#3da38c6a50abe03, 16#15cbd1a421c7a8c,16#22794e3ec6ef3b1,16#26154d26e7ea99f,16#3a66681359a6ab6]; +reference_val(exsss) -> + [16#108e8d5b01,16#33b72092117209a,16#224d4d2961a2d0a,16#2c4c81aac3da48d, + 16#2f4bc39bfc36f3a,16#41826d4c4d243a,16#19871b8bb4e23ee,16#3e2112cdf9384b1, + 16#69801943bf91ab,16#2de1a603c31ec45,16#a90ca1991b831e,16#51ca29571a69a7, + 16#93ce3e511906cf,16#93ebc5768aef75,16#2412f284b902ae7,16#1ac10e758410c52, + 16#3f32494560368f6,16#39a5e82dcf0de95,16#3f4b14d59cc6a21,16#3174668db0b36ae, + 16#1449812fb8bd54e,16#eaca1f8ece51e1,16#2564b2545fd23c1,16#3cf3a2d2217e0d7, + 16#226f4164ba1d054,16#10dac9ae207ceef,16#17f2c4b2d40fcb9,16#1c1b282d386fdcb, + 16#a264f450ba2912,16#2a0a1dd67e52666,16#2be84eb835cb1e1,16#2a1cd9aa16ccc37, + 16#7dd5e8c2b3f490,16#254a3db4976c05b,16#2a0a67971ec1e63,16#13a0cbf7c0eed8a, + 16#3192d7edc0a20bc,16#2705ad756292e84,16#3ec429a18119c81,16#25944b38baa975b, + 16#291dcc43e3256f4,16#30d10b759237db,16#c1522a652058a,16#8ef1e9378381e6, + 16#1f442f33c2439f4,16#186087710a73818,16#12887f94b2b8387,16#3e42e8b1f3c9b4b, + 16#e462859d55f9d8,16#2356ae85be908de,16#15e96a927b3bc52,16#35c6dc52511ce46, + 16#7bc0624ce66e01,16#33ab7d95b738322,16#26f01effc182aa0,16#1b66ae7eaafea88, + 16#278f3dc14943b90,16#22178bc8d8faf28,16#396c37d53c11985,16#5e0d79d0b10f18, + 16#1be3de3b5675ec,16#d4db298f1f4b50,16#2da6cb99bb5c7b1,16#130b2dc17d03be8, + 16#f1847e7e059e9f,16#2da6591788326e7,16#222e4a18c24211c,16#949213ca49baab, + 16#b5129fec56f6a2,16#30f25f1e926f43e,16#1ddd8d04445fb4d,16#15995b542514150, + 16#1595fe879296296,16#e2f237a488453b,16#23e5cd2d6047890,16#3a5dc88fc954666, + 16#89bca9969b103,16#5e6893cd35dc63,16#1fed534feeeef5a,16#26f40e2147ee558, + 16#30c131a00625837,16#2618a7e617422e9,16#23630b297e45e7,16#1143b17502f3219, + 16#15607dac41168da,16#2886bdc314b3fb8,16#465d1cc1536546,16#30b09123e3a02e4, + 16#245a375f810be52,16#6a1b0792376a03,16#221425f59f2470f,16#867ce16dfac81c, + 16#9c62d95fae9b58,16#380381db1394426,16#34908dedc01c324,16#1f0ff517089b561, + 16#1571366dd873d32,16#3ee353dc56e192,16#15a1dee8d889b11,16#41036ad76d9888 + ]; + reference_val(exsp) -> reference_val(exsplus); reference_val(exs1024s) -> @@ -1516,6 +1545,33 @@ reference_jump_val(exsplus) -> 12504080415362731, 45083100453836317, 270968267812126657, 93505647407734103, 252852934678537969, 258758309277167202, 74250882143432077, 141629095984552833]; +reference_jump_val(exsss) -> + [16#304ae783d40db2b,16#1dfb196b3a5600a,16#2a24116effc6a0d,16#1f138d68c56725, + 16#9360a445e2f989,16#32ed8080390e242,16#294ca85a270cff6,16#1418e6296a88bf, + 16#114fae3dc578ba7,16#479c42c760eb72,16#334a40655df22d6,16#e7a85dd4d37d72, + 16#181db16c8925c77,16#1b8a5a8afd16cbd,16#329107bf9777a39,16#2fc915c08535e42, + 16#16696d142c6078,16#2e2a2601c919448,16#2246150d1000568,16#26109007cb3dd44, + 16#3761360723e3175,16#169abd352db74de,16#1c97d520983684f,16#12455f0adee8c66, + 16#46719cff00622d,16#1fc92792ed4e437,16#18e2edae21affb5,16#3a67fa9e3e7d46e, + 16#1313fdc2728aa74,16#1c1a2b577581db8,16#db49357ea196b1,16#10e219a21d93fc7, + 16#3c43abede083666,16#3eef5055a58bbf9,16#1975056f95d90e3,16#3916c133ab16d87, + 16#2bc0bea891c26f1,16#391e4b369fc6b36,16#183f83155a359f6,16#1d9f137e9d2e488, + 16#ef084de5f4cd3c,16#36a9cf7e29e55d3,16#19eca704e0409a7,16#1bdb99902896c69, + 16#21777e2ad128203,16#5d0369ec0563e4,16#36db40b863bd74a,16#33feb71b7515159, + 16#208d923ce26f257,16#3841b32891c082d,16#2748f224c2ba226,16#2fcd93b2daf79bb, + 16#2c8e6cacad58ec4,16#39850131a1a85f,16#134648d6eea624d,16#2e102e197d5725c, + 16#12ac280fa744758,16#1c18266c7442d16,16#22b5f91b15fe17e,16#316740ca870f7c8, + 16#720ed4836c426,16#1aac0f738d04f8c,16#34fcd2a647b462c,16#3d430ac755114a3, + 16#3692e3670fdf2a,16#265279ab0fc0a15,16#10bd883dee80945,16#10e7843413175e4, + 16#b291deba08cee2,16#3915a8234caf11,16#34b911b96707dbd,16#ae63fcda15fde6, + 16#b13b9091e82e41,16#29de1b6d70dc04f,16#23fbcbc409617e8,16#1389a0738061066, + 16#360f39af790f5d1,16#f436da2a7d12f5,16#2d06ba8da21e08,16#3601a6492b887d, + 16#2b2590b8c6cc186,16#f8d613b6904464,16#e5456786e46b78,16#201b8b1f96ed80c, + 16#1b75b86d9b843f2,16#2e8bfaa7243a630,16#125ff068a78c3b4,16#3875a28c48bd26e, + 16#f09a06941fc9d7,16#107c4de8ca77744,16#357c34144bb9ed6,16#3ccc55d3ebb3378, + 16#28db7cea7d3fdee,16#3197fd0b49f6370,16#11af6fedb708ea6,16#2bde0382e37469e, + 16#10666171abddb3f,16#1a8876c1f4e78a8,16#169c0efd4422043,16#1501c49abf0440f]; + reference_jump_val(exs1024) -> [2655961906500790629, 17003395417078685063, 10466831598958356428, 7603399148503548021, 1650550950190587188, 12294992315080723704, 15743995773860389219, 5492181000145247327, diff --git a/lib/xmerl/src/xmerl_sax_parser_base.erlsrc b/lib/xmerl/src/xmerl_sax_parser_base.erlsrc index 1dca9608cb..ef753c7148 100644 --- a/lib/xmerl/src/xmerl_sax_parser_base.erlsrc +++ b/lib/xmerl/src/xmerl_sax_parser_base.erlsrc @@ -3679,7 +3679,7 @@ create_tempfile(Template) -> false -> case os:getenv("TEMP") of false -> - throw({error, "Variabel TMP or TEMP doesn't exist"}); + throw({error, "Variable TMP or TEMP doesn't exist"}); P2 -> P2 end; diff --git a/system/doc/efficiency_guide/binaryhandling.xml b/system/doc/efficiency_guide/binaryhandling.xml index b500329ef9..d92da17390 100644 --- a/system/doc/efficiency_guide/binaryhandling.xml +++ b/system/doc/efficiency_guide/binaryhandling.xml @@ -384,8 +384,8 @@ export ERL_COMPILER_OPTIONS=bin_opt_info]]></code> <p>The warnings look as follows:</p> <code type="erl"><![CDATA[ -./efficiency_guide.erl:60: Warning: NOT OPTIMIZED: sub binary is used or returned -./efficiency_guide.erl:62: Warning: OPTIMIZED: creation of sub binary delayed]]></code> +./efficiency_guide.erl:60: Warning: NOT OPTIMIZED: binary is returned from the function +./efficiency_guide.erl:62: Warning: OPTIMIZED: match context reused]]></code> <p>To make it clearer exactly what code the warnings refer to, the warnings in the following examples are inserted as comments @@ -393,10 +393,10 @@ export ERL_COMPILER_OPTIONS=bin_opt_info]]></code> <code type="erl"><![CDATA[ after_zero(<<0,T/binary>>) -> - %% NOT OPTIMIZED: sub binary is used or returned + %% BINARY CREATED: binary is returned from the function T; after_zero(<<_,T/binary>>) -> - %% OPTIMIZED: creation of sub binary delayed + %% OPTIMIZED: match context reused after_zero(T); after_zero(<<>>) -> <<>>.]]></code> |