diff options
Diffstat (limited to 'lib')
227 files changed, 9864 insertions, 4139 deletions
diff --git a/lib/Makefile b/lib/Makefile index 6605c6145c..ea2827bef0 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -58,10 +58,10 @@ else SUB_DIRECTORIES = hipe parsetools asn1/src else ifdef TERTIARY_BOOTSTRAP - SUB_DIRECTORIES = snmp sasl jinterface syntax_tools wx + SUB_DIRECTORIES = snmp sasl erl_interface jinterface syntax_tools wx else ifdef DOC_BOOTSTRAP - SUB_DIRECTORIES = xmerl edoc erl_docgen + SUB_DIRECTORIES = xmerl edoc erl_docgen public_key else # Not bootstrap build SUB_DIRECTORIES = $(ERTS_APPLICATIONS) \ $(ERLANG_APPLICATIONS) \ diff --git a/lib/common_test/doc/src/basics_chapter.xml b/lib/common_test/doc/src/basics_chapter.xml index 95599ca1f1..899a52fa31 100644 --- a/lib/common_test/doc/src/basics_chapter.xml +++ b/lib/common_test/doc/src/basics_chapter.xml @@ -125,7 +125,7 @@ The test case is the smallest unit that the <c>Common Test</c> test server deals with. </p> <p> - Subsets of test cases, called test case groups, can also be defined. A test case + Sets of test cases, called test case groups, can also be defined. A test case group can have execution properties associated with it. Execution properties specify if the test cases in the group are to be executed in random order, in parallel, or in sequence, and if the execution of the group diff --git a/lib/common_test/doc/src/notes.xml b/lib/common_test/doc/src/notes.xml index 2f53f1c29e..c8e0722a0f 100644 --- a/lib/common_test/doc/src/notes.xml +++ b/lib/common_test/doc/src/notes.xml @@ -33,6 +33,66 @@ <file>notes.xml</file> </header> +<section><title>Common_Test 1.17</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + A bug caused <c>ct:encrypt_config_file/3</c> and + <c>ct:decrypt_config_file/3</c> to fail with + <c>badmatch</c> if input parameter <c>KeyOrFile</c> was + <c>{key,string()}</c>. This is now corrected.</p> + <p> + Own Id: OTP-15540</p> + </item> + <item> + <p> + The status of a test case which failed with timetrap + timeout in <c>end_per_testcase</c> could not be modified + by returning <c>{fail,Reason}</c> from a + <c>post_end_per_testcase</c> hook function. This is now + corrected.</p> + <p> + Own Id: OTP-15584 Aux Id: ERIERL-282 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + A new variant of the <c>newline</c> option to + <c>ct_telnet:cmd/3</c> and <c>ct_telnet:send/3</c> is + added, which allows to specify a string to append as + newline indicator on a command. By default, the value is + "\n", but in some cases it is required to be "\r\n", + which this option allows.</p> + <p> + A faulty regular expression given as parameter to + <c>ct_telnet:expect/2,3</c> would earlier crash and look + like an internal error in common_test. A better error + indication is now given, but the test case will still + fail.</p> + <p> + Own Id: OTP-15229 Aux Id: ERIERL-203 </p> + </item> + <item> + <p> + Since the yang RFC allows more than one top element of + config data in an <c>edit-config</c> element, + <c>ct_netconfc:edit_config/3,4,5</c> can now take a list + of XML elements.</p> + <p> + Own Id: OTP-15298</p> + </item> + </list> + </section> + +</section> + <section><title>Common_Test 1.16.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/common_test/test_server/ts_erl_config.erl b/lib/common_test/test_server/ts_erl_config.erl index c2852131d6..f3972bea4e 100644 --- a/lib/common_test/test_server/ts_erl_config.erl +++ b/lib/common_test/test_server/ts_erl_config.erl @@ -197,28 +197,24 @@ system_include(Root, Vars) -> " -I" ++ quote(filename:nativename(filename:join([Root, "erts", "emulator", SysDir]))). erl_interface(Vars,OsType) -> - {Incl, TargetIncl, {LibPath, MkIncl}} = + {Incl, {LibPath, MkIncl}} = case lib_dir(Vars, erl_interface) of {error, bad_name} -> throw({cannot_find_app, erl_interface}); Dir -> - BaseIncl = filename:join(Dir, "include"), - case erl_root(Vars) of - {installed, _Root} -> - {BaseIncl, - [], - {filename:join(Dir, "lib"), - filename:join([Dir, "src", "eidefs.mk"])}}; - {srctree, _Root, Target} -> - Obj = case is_debug_build() of - true -> "obj.debug"; - false -> "obj" - end, - {BaseIncl, - filename:join(BaseIncl, Target), - {filename:join([Dir, Obj, Target]), - filename:join([Dir, "src", Target, "eidefs.mk"])}} - end + {filename:join(Dir, "include"), + case erl_root(Vars) of + {installed, _Root} -> + {filename:join(Dir, "lib"), + filename:join([Dir, "src", "eidefs.mk"])}; + {srctree, _Root, Target} -> + Obj = case is_debug_build() of + true -> "obj.debug"; + false -> "obj" + end, + {filename:join([Dir, Obj, Target]), + filename:join([Dir, "src", Target, "eidefs.mk"])} + end} end, Lib = link_library("erl_interface",OsType), Lib1 = link_library("ei",OsType), @@ -264,10 +260,6 @@ erl_interface(Vars,OsType) -> {erl_interface_eilib_drv, quote(filename:join(LibPath, Lib1Drv))}, {erl_interface_threadlib, ThreadLib}, {erl_interface_include, quote(filename:nativename(Incl))}, - {erl_interface_target_include, case TargetIncl of - [] -> []; - _ -> "-I" ++ quote(filename:nativename(TargetIncl)) - end}, {erl_interface_mk_include, quote(filename:nativename(MkIncl))} | Vars]. diff --git a/lib/common_test/vsn.mk b/lib/common_test/vsn.mk index fd5d4a57aa..23eb8d9656 100644 --- a/lib/common_test/vsn.mk +++ b/lib/common_test/vsn.mk @@ -1 +1 @@ -COMMON_TEST_VSN = 1.16.1 +COMMON_TEST_VSN = 1.17 diff --git a/lib/compiler/doc/src/notes.xml b/lib/compiler/doc/src/notes.xml index 02e6203137..d45dfef8f3 100644 --- a/lib/compiler/doc/src/notes.xml +++ b/lib/compiler/doc/src/notes.xml @@ -32,6 +32,38 @@ <p>This document describes the changes made to the Compiler application.</p> +<section><title>Compiler 7.3.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>An expression such as <c>(A / B) band 16#ff</c> would + crash the compiler.</p> + <p> + Own Id: OTP-15518 Aux Id: ERL-829 </p> + </item> + <item> + <p>There could be an incorrect warning when the + <c>tuple_calls</c> option was given. The generated code + would be correct. Here is an example of code that would + trigger the warning:</p> + <p><c>(list_to_atom("prefix_" ++ + atom_to_list(suffix))):doit(X)</c>.</p> + <p> + Own Id: OTP-15552 Aux Id: ERL-838 </p> + </item> + <item> + <p>Optimize (again) Dialyzer's handling of + left-associative use of <c>andalso</c> and <c>orelse</c> + in guards.</p> + <p> + Own Id: OTP-15577 Aux Id: ERL-851, PR-2141, PR-1944 </p> + </item> + </list> + </section> + +</section> + <section><title>Compiler 7.3.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/compiler/scripts/smoke b/lib/compiler/scripts/smoke index 2429f104c0..ae31c923b8 100755 --- a/lib/compiler/scripts/smoke +++ b/lib/compiler/scripts/smoke @@ -54,6 +54,7 @@ setup_mix() -> ElixirBin = filename:join([SmokeDir,"elixir","bin"]), PATH = ElixirBin ++ ":" ++ os:getenv("PATH"), os:putenv("PATH", PATH), + mix("local.hex --force"), mix("local.rebar --force"), ok. diff --git a/lib/compiler/scripts/smoke-mix.exs b/lib/compiler/scripts/smoke-mix.exs index 82ae3370fe..ba0815e465 100644 --- a/lib/compiler/scripts/smoke-mix.exs +++ b/lib/compiler/scripts/smoke-mix.exs @@ -25,6 +25,14 @@ defmodule Smoke.MixProject do [ {:bear, "~> 0.8.7"}, {:cloudi_core, "~> 1.7"}, + {:cloudi_service_monitoring, "~> 1.7"}, + {:cloudi_service_tcp, "~> 1.7"}, + {:cloudi_service_queue, "~> 1.7"}, + {:cloudi_service_udp, "~> 1.7"}, + {:cloudi_service_map_reduce, "~> 1.7"}, + {:cloudi_service_api_requests, "~> 1.7"}, + {:cloudi_service_router, "~> 1.7"}, + {:cloudi_service_request_rate, "~> 1.7"}, {:concuerror, "~> 0.20.0"}, {:cowboy, "~> 2.6.1"}, {:ecto, "~> 3.0.6"}, diff --git a/lib/compiler/src/beam_jump.erl b/lib/compiler/src/beam_jump.erl index 6f50bfdb9c..74f80ca70e 100644 --- a/lib/compiler/src/beam_jump.erl +++ b/lib/compiler/src/beam_jump.erl @@ -179,8 +179,9 @@ function({function,Name,Arity,CLabel,Asm0}, Lc0) -> eliminate_moves(Is) -> eliminate_moves(Is, #{}, []). -eliminate_moves([{select,select_val,Reg,_,List}=I|Is], D0, Acc) -> - D = update_value_dict(List, Reg, D0), +eliminate_moves([{select,select_val,Reg,{f,Fail},List}=I|Is], D0, Acc) -> + D1 = add_unsafe_label(Fail, D0), + D = update_value_dict(List, Reg, D1), eliminate_moves(Is, D, [I|Acc]); eliminate_moves([{test,is_eq_exact,_,[Reg,Val]}=I, {block,BlkIs0}|Is], D0, Acc) -> @@ -229,6 +230,9 @@ update_value_dict([Lit,{f,Lbl}|T], Reg, D0) -> update_value_dict(T, Reg, D); update_value_dict([], _, D) -> D. +add_unsafe_label(L, D) -> + D#{L=>unsafe}. + update_unsafe_labels(I, D) -> Ls = instr_labels(I), update_unsafe_labels_1(Ls, D). diff --git a/lib/compiler/src/beam_kernel_to_ssa.erl b/lib/compiler/src/beam_kernel_to_ssa.erl index 410bafe0bb..df95749fb3 100644 --- a/lib/compiler/src/beam_kernel_to_ssa.erl +++ b/lib/compiler/src/beam_kernel_to_ssa.erl @@ -327,7 +327,7 @@ select_bin_seg(#k_val_clause{val=#k_bin_seg{size=Size,unit=U,type=T, {Mis,St1} = select_extract_bin(Next, Size, U, T, Fs, Fail, Ctx, LineAnno, St0), {Extracted,St2} = new_ssa_var(Seg#k_var.name, St1), - {Bis,St} = bin_match_cg(Size, B, Fail, St2), + {Bis,St} = match_cg(B, Fail, St2), BsGet = #b_set{op=bs_extract,dst=Extracted,args=[ssa_arg(Next, St)]}, Is = Mis ++ [BsGet] ++ Bis, {Is,St}; @@ -362,14 +362,6 @@ select_bin_seg(#k_val_clause{val=#k_bin_int{size=Sz,unit=U,flags=Fs, end, {Is,St}. -bin_match_cg(#k_atom{val=all}, B0, Fail, St) -> - #k_select{types=Types} = B0, - [#k_type_clause{type=k_bin_end,values=Values}] = Types, - [#k_val_clause{val=#k_bin_end{},body=B}] = Values, - match_cg(B, Fail, St); -bin_match_cg(_, B, Fail, St) -> - match_cg(B, Fail, St). - get_context(#k_var{}=Var, St) -> ssa_arg(Var, St). diff --git a/lib/compiler/src/beam_ssa_dead.erl b/lib/compiler/src/beam_ssa_dead.erl index 2cca9ebadf..bb43a550ae 100644 --- a/lib/compiler/src/beam_ssa_dead.erl +++ b/lib/compiler/src/beam_ssa_dead.erl @@ -27,7 +27,8 @@ -export([opt/1]). -include("beam_ssa.hrl"). --import(lists, [append/1,last/1,member/2,takewhile/2,reverse/1]). +-import(lists, [append/1,keymember/3,last/1,member/2, + takewhile/2,reverse/1]). -type used_vars() :: #{beam_ssa:label():=ordsets:ordset(beam_ssa:var_name())}. @@ -58,7 +59,7 @@ opt(Linear) -> Blocks0 = maps:from_list(Linear), St0 = #st{bs=Blocks0,us=Used,skippable=Skippable}, St = shortcut_opt(St0), - #st{bs=Blocks} = combine_eqs(St), + #st{bs=Blocks} = combine_eqs(St#st{us=#{}}), beam_ssa:linearize(Blocks). %%% @@ -87,13 +88,22 @@ shortcut_opt(#st{bs=Blocks}=St) -> %% opportunities for optimizations compared to post order. (Based on %% running scripts/diffable with both PO and RPO and looking at %% the diff.) + %% + %% Unfortunately, processing the blocks in reverse post order + %% potentially makes the time complexity quadratic or even cubic if + %% the ordset of unset variables grows large, instead of + %% linear for post order processing. We try to still get reasonable + %% compilation times by optimizations that will keep the constant + %% factor as low as possible, and we try to avoid the cubic time + %% complexity by trying to keep the set of unset variables as small + %% as possible. + Ls = beam_ssa:rpo(Blocks), - shortcut_opt(Ls, #{from=>0}, St). + shortcut_opt(Ls, #{}, St). -shortcut_opt([L|Ls], Bs0, #st{bs=Blocks0}=St) -> +shortcut_opt([L|Ls], Bs, #st{bs=Blocks0}=St) -> #b_blk{is=Is,last=Last0} = Blk0 = get_block(L, St), - Bs = Bs0#{from:=L}, - case shortcut_terminator(Last0, Is, Bs, St) of + case shortcut_terminator(Last0, Is, L, Bs, St) of Last0 -> %% No change. No need to update the block. shortcut_opt(Ls, Bs, St); @@ -107,17 +117,17 @@ shortcut_opt([L|Ls], Bs0, #st{bs=Blocks0}=St) -> shortcut_opt([], _, St) -> St. shortcut_terminator(#b_br{bool=#b_literal{val=true},succ=Succ0}, - _Is, Bs, St0) -> + _Is, From, Bs, St0) -> St = St0#st{rel_op=none}, - shortcut(Succ0, Bs, St); + shortcut(Succ0, From, Bs, St); shortcut_terminator(#b_br{bool=#b_var{}=Bool,succ=Succ0,fail=Fail0}=Br, - Is, Bs, St0) -> + Is, From, Bs, St0) -> St = St0#st{target=one_way}, RelOp = get_rel_op(Bool, Is), SuccBs = bind_var(Bool, #b_literal{val=true}, Bs), - BrSucc = shortcut(Succ0, SuccBs, St#st{rel_op=RelOp}), + BrSucc = shortcut(Succ0, From, SuccBs, St#st{rel_op=RelOp}), FailBs = bind_var(Bool, #b_literal{val=false}, Bs), - BrFail = shortcut(Fail0, FailBs, St#st{rel_op=invert_op(RelOp)}), + BrFail = shortcut(Fail0, From, FailBs, St#st{rel_op=invert_op(RelOp)}), case {BrSucc,BrFail} of {#b_br{bool=#b_literal{val=true},succ=Succ}, #b_br{bool=#b_literal{val=true},succ=Fail}} @@ -128,25 +138,25 @@ shortcut_terminator(#b_br{bool=#b_var{}=Bool,succ=Succ0,fail=Fail0}=Br, %% No change. Br end; -shortcut_terminator(#b_switch{arg=Bool,list=List0}=Sw, _Is, Bs, St) -> - List = shortcut_switch(List0, Bool, Bs, St), +shortcut_terminator(#b_switch{arg=Bool,list=List0}=Sw, _Is, From, Bs, St) -> + List = shortcut_switch(List0, Bool, From, Bs, St), beam_ssa:normalize(Sw#b_switch{list=List}); -shortcut_terminator(Last, _Is, _Bs, _St) -> +shortcut_terminator(Last, _Is, _Bs, _From, _St) -> Last. -shortcut_switch([{Lit,L0}|T], Bool, Bs, St0) -> +shortcut_switch([{Lit,L0}|T], Bool, From, Bs, St0) -> RelOp = {'=:=',Bool,Lit}, St = St0#st{rel_op=RelOp}, #b_br{bool=#b_literal{val=true},succ=L} = - shortcut(L0, bind_var(Bool, Lit, Bs), St#st{target=one_way}), - [{Lit,L}|shortcut_switch(T, Bool, Bs, St0)]; -shortcut_switch([], _, _, _) -> []. + shortcut(L0, From, bind_var(Bool, Lit, Bs), St#st{target=one_way}), + [{Lit,L}|shortcut_switch(T, Bool, From, Bs, St0)]; +shortcut_switch([], _, _, _, _) -> []. -shortcut(L, Bs, St) -> - shortcut_1(L, Bs, ordsets:new(), St). +shortcut(L, From, Bs, St) -> + shortcut_1(L, From, Bs, ordsets:new(), St). -shortcut_1(L, Bs0, UnsetVars0, St) -> - case shortcut_2(L, Bs0, UnsetVars0, St) of +shortcut_1(L, From, Bs0, UnsetVars0, St) -> + case shortcut_2(L, From, Bs0, UnsetVars0, St) of none -> %% No more shortcuts found. Package up the previous %% label in an unconditional branch. @@ -156,13 +166,13 @@ shortcut_1(L, Bs0, UnsetVars0, St) -> Br; {#b_br{bool=#b_literal{val=true},succ=Succ},Bs,UnsetVars} -> %% This is a safe `br`, but try to find a better one. - shortcut_1(Succ, Bs#{from:=L}, UnsetVars, St) + shortcut_1(Succ, L, Bs, UnsetVars, St) end. %% Try to shortcut this block, branching to a successor. -shortcut_2(L, Bs0, UnsetVars0, St) -> +shortcut_2(L, From, Bs0, UnsetVars0, St) -> #b_blk{is=Is,last=Last} = get_block(L, St), - case eval_is(Is, Bs0, St) of + case eval_is(Is, From, Bs0, St) of none -> %% It is not safe to avoid this block because it %% has instructions with potential side effects. @@ -181,139 +191,147 @@ shortcut_2(L, Bs0, UnsetVars0, St) -> %% We have a potentially suitable br. %% Now update the set of variables that will never %% be set if this block will be skipped. - SetInThisBlock = [V || #b_set{dst=V} <- Is], - UnsetVars = update_unset_vars(L, Br, SetInThisBlock, - UnsetVars0, St), - - %% Continue checking whether this br is suitable. - shortcut_3(Br, Bs#{from:=L}, UnsetVars, St) + case update_unset_vars(L, Is, Br, UnsetVars0, St) of + unsafe -> + %% It is unsafe to use this br, + %% because it refers to a variable defined + %% in this block. + shortcut_unsafe_br(Br, L, Bs, UnsetVars0, St); + UnsetVars -> + %% Continue checking whether this br is + %% suitable. + shortcut_test_br(Br, L, Bs, UnsetVars, St) + end end end. -shortcut_3(Br, Bs, UnsetVars, #st{target=Target}=St) -> +shortcut_test_br(Br, From, Bs, UnsetVars, St) -> case is_br_safe(UnsetVars, Br, St) of false -> - %% Branching using this `br` is unsafe, either because it - %% is an unconditional branch to a phi node, or because - %% one or more of the variables that are not set will be - %% used. Try to follow branches of this `br`, to find a - %% safe `br`. - case Br of - #b_br{bool=#b_literal{val=true},succ=L} -> - case Target of - L -> - %% We have reached the forced target, and it - %% is unsafe. Give up. - none; - _ -> - %% Try following this branch to see whether it - %% leads to a safe `br`. - shortcut_2(L, Bs, UnsetVars, St) - end; - #b_br{bool=#b_var{},succ=Succ,fail=Fail} -> - case {Succ,Fail} of - {L,Target} -> - %% The failure label is the forced target. - %% Try following the success label to see - %% whether it also ultimately ends up at the - %% forced target. - shortcut_2(L, Bs, UnsetVars, St); - {Target,L} -> - %% The success label is the forced target. - %% Try following the failure label to see - %% whether it also ultimately ends up at the - %% forced target. - shortcut_2(L, Bs, UnsetVars, St); - {_,_} -> - case Target of - any -> - %% This two-way branch is unsafe. Try reducing - %% it to a one-way branch. - shortcut_two_way(Br, Bs, UnsetVars, St); - one_way -> - %% This two-way branch is unsafe. Try reducing - %% it to a one-way branch. - shortcut_two_way(Br, Bs, UnsetVars, St); - _ when is_integer(Target) -> - %% This two-way branch is unsafe, and - %% there already is a forced target. - %% Give up. - none - end - end - end; + shortcut_unsafe_br(Br, From, Bs, UnsetVars, St); true -> - %% This `br` instruction is safe. It does not - %% branch to a phi node, and all variables that - %% will be used are guaranteed to be defined. - case Br of - #b_br{bool=#b_literal{val=true},succ=L} -> - %% This is a one-way branch. + shortcut_safe_br(Br, From, Bs, UnsetVars, St) + end. + +shortcut_unsafe_br(Br, From, Bs, UnsetVars, #st{target=Target}=St) -> + %% Branching using this `br` is unsafe, either because it + %% is an unconditional branch to a phi node, or because + %% one or more of the variables that are not set will be + %% used. Try to follow branches of this `br`, to find a + %% safe `br`. + case Br of + #b_br{bool=#b_literal{val=true},succ=L} -> + case Target of + L -> + %% We have reached the forced target, and it + %% is unsafe. Give up. + none; + _ -> + %% Try following this branch to see whether it + %% leads to a safe `br`. + shortcut_2(L, From, Bs, UnsetVars, St) + end; + #b_br{bool=#b_var{},succ=Succ,fail=Fail} -> + case {Succ,Fail} of + {L,Target} -> + %% The failure label is the forced target. + %% Try following the success label to see + %% whether it also ultimately ends up at the + %% forced target. + shortcut_2(L, From, Bs, UnsetVars, St); + {Target,L} -> + %% The success label is the forced target. + %% Try following the failure label to see + %% whether it also ultimately ends up at the + %% forced target. + shortcut_2(L, From, Bs, UnsetVars, St); + {_,_} -> case Target of any -> - %% No forced target. Success! - {Br,Bs,UnsetVars}; + %% This two-way branch is unsafe. Try + %% reducing it to a one-way branch. + shortcut_two_way(Br, From, Bs, UnsetVars, St); one_way -> - %% The target must be a one-way branch, which this - %% `br` is. Success! - {Br,Bs,UnsetVars}; - L when is_integer(Target) -> - %% The forced target is L. Success! - {Br,Bs,UnsetVars}; + %% This two-way branch is unsafe. Try + %% reducing it to a one-way branch. + shortcut_two_way(Br, From, Bs, UnsetVars, St); _ when is_integer(Target) -> - %% Wrong forced target. Try following this branch - %% to see if it ultimately ends up at the forced - %% target. - shortcut_2(L, Bs, UnsetVars, St) - end; - #b_br{bool=#b_var{}} -> - %% This is a two-way branch. - if - Target =:= any; Target =:= one_way -> - %% No specific forced target. Try to reduce the - %% two-way branch to an one-way branch. - case shortcut_two_way(Br, Bs, UnsetVars, St) of - none when Target =:= any -> - %% This `br` can't be reduced to a one-way - %% branch. Return the `br` as-is. - {Br,Bs,UnsetVars}; - none when Target =:= one_way -> - %% This `br` can't be reduced to a one-way - %% branch. The caller wants a one-way branch. - %% Give up. - none; - {_,_,_}=Res -> - %% This `br` was successfully reduced to a - %% one-way branch. - Res - end; - is_integer(Target) -> - %% There is a forced target, which can't - %% be reached because this `br` is a two-way - %% branch. Give up. + %% This two-way branch is unsafe, and + %% there already is a forced target. + %% Give up. none end end end. -update_unset_vars(L, Br, SetInThisBlock, UnsetVars, #st{skippable=Skippable}) -> +shortcut_safe_br(Br, From, Bs, UnsetVars, #st{target=Target}=St) -> + %% This `br` instruction is safe. It does not branch to a phi + %% node, and all variables that will be used are guaranteed to be + %% defined. + case Br of + #b_br{bool=#b_literal{val=true},succ=L} -> + %% This is a one-way branch. + case Target of + any -> + %% No forced target. Success! + {Br,Bs,UnsetVars}; + one_way -> + %% The target must be a one-way branch, which this + %% `br` is. Success! + {Br,Bs,UnsetVars}; + L when is_integer(Target) -> + %% The forced target is L. Success! + {Br,Bs,UnsetVars}; + _ when is_integer(Target) -> + %% Wrong forced target. Try following this branch + %% to see if it ultimately ends up at the forced + %% target. + shortcut_2(L, From, Bs, UnsetVars, St) + end; + #b_br{bool=#b_var{}} -> + %% This is a two-way branch. + if + Target =:= any; Target =:= one_way -> + %% No specific forced target. Try to reduce the + %% two-way branch to an one-way branch. + case shortcut_two_way(Br, From, Bs, UnsetVars, St) of + none when Target =:= any -> + %% This `br` can't be reduced to a one-way + %% branch. Return the `br` as-is. + {Br,Bs,UnsetVars}; + none when Target =:= one_way -> + %% This `br` can't be reduced to a one-way + %% branch. The caller wants a one-way + %% branch. Give up. + none; + {_,_,_}=Res -> + %% This `br` was successfully reduced to a + %% one-way branch. + Res + end; + is_integer(Target) -> + %% There is a forced target, which can't + %% be reached because this `br` is a two-way + %% branch. Give up. + none + end + end. + +update_unset_vars(L, Is, Br, UnsetVars, #st{skippable=Skippable}) -> case is_map_key(L, Skippable) of true -> %% None of the variables used in this block are used in - %% the successors. We can speed up compilation by avoiding - %% adding variables to the UnsetVars if the presence of - %% those variable would not change the outcome of the - %% tests in is_br_safe/2. + %% the successors. Thus, there is no need to add the + %% variables to the set of unset variables. case Br of - #b_br{bool=Bool} -> - case member(Bool, SetInThisBlock) of + #b_br{bool=#b_var{}=Bool} -> + case keymember(Bool, #b_set.dst, Is) of true -> %% Bool is a variable defined in this - %% block. It will change the outcome of - %% the `not member(V, UnsetVars)` check in - %% is_br_safe/2. The other variables - %% defined in this block will not. - ordsets:add_element(Bool, UnsetVars); + %% block. Using the br instruction from + %% this block (and skipping the body of + %% the block) is unsafe. + unsafe; false -> %% Bool is either a variable not defined %% in this block or a literal. Adding it @@ -321,18 +339,24 @@ update_unset_vars(L, Br, SetInThisBlock, UnsetVars, #st{skippable=Skippable}) -> %% the outcome of the tests in %% is_br_safe/2. UnsetVars - end + end; + #b_br{} -> + UnsetVars end; false -> + %% Some variables defined in this block are used by + %% successors. We must update the set of unset variables. + SetInThisBlock = [V || #b_set{dst=V} <- Is], ordsets:union(UnsetVars, ordsets:from_list(SetInThisBlock)) end. -shortcut_two_way(#b_br{succ=Succ,fail=Fail}, Bs0, UnsetVars0, St) -> - case shortcut_2(Succ, Bs0, UnsetVars0, St#st{target=Fail}) of +shortcut_two_way(#b_br{succ=Succ,fail=Fail}, From, Bs0, UnsetVars0, St0) -> + case shortcut_2(Succ, From, Bs0, UnsetVars0, St0#st{target=Fail}) of {#b_br{bool=#b_literal{},succ=Fail},_,_}=Res -> Res; none -> - case shortcut_2(Fail, Bs0, UnsetVars0, St#st{target=Succ}) of + St = St0#st{target=Succ}, + case shortcut_2(Fail, From, Bs0, UnsetVars0, St) of {#b_br{bool=#b_literal{},succ=Succ},_,_}=Res -> Res; none -> @@ -374,40 +398,42 @@ is_forbidden(L, St) -> %% Return the updated bindings, or 'none' if there is %% any instruction with potential side effects. -eval_is([#b_set{op=phi,dst=Dst,args=Args}|Is], Bs0, St) -> - From = map_get(from, Bs0), - [Val] = [Val || {Val,Pred} <- Args, Pred =:= From], +eval_is([#b_set{op=phi,dst=Dst,args=Args}|Is], From, Bs0, St) -> + Val = get_phi_arg(Args, From), Bs = bind_var(Dst, Val, Bs0), - eval_is(Is, Bs, St); -eval_is([#b_set{op={bif,_},dst=Dst}=I0|Is], Bs, St) -> + eval_is(Is, From, Bs, St); +eval_is([#b_set{op={bif,_},dst=Dst}=I0|Is], From, Bs, St) -> I = sub(I0, Bs), case eval_bif(I, St) of #b_literal{}=Val -> - eval_is(Is, bind_var(Dst, Val, Bs), St); + eval_is(Is, From, bind_var(Dst, Val, Bs), St); none -> - eval_is(Is, Bs, St) + eval_is(Is, From, Bs, St) end; -eval_is([#b_set{op=Op,dst=Dst}=I|Is], Bs, St) +eval_is([#b_set{op=Op,dst=Dst}=I|Is], From, Bs, St) when Op =:= is_tagged_tuple; Op =:= is_nonempty_list -> #b_set{args=Args} = sub(I, Bs), case eval_rel_op(Op, Args, St) of #b_literal{}=Val -> - eval_is(Is, bind_var(Dst, Val, Bs), St); + eval_is(Is, From, bind_var(Dst, Val, Bs), St); none -> - eval_is(Is, Bs, St) + eval_is(Is, From, Bs, St) end; -eval_is([#b_set{}=I|Is], Bs, St) -> +eval_is([#b_set{}=I|Is], From, Bs, St) -> case beam_ssa:no_side_effect(I) of true -> %% This instruction has no side effects. It can %% safely be omitted. - eval_is(Is, Bs, St); + eval_is(Is, From, Bs, St); false -> %% This instruction may have some side effect. %% It is not safe to avoid this instruction. none end; -eval_is([], Bs, _St) -> Bs. +eval_is([], _From, Bs, _St) -> Bs. + +get_phi_arg([{Val,From}|_], From) -> Val; +get_phi_arg([_|As], From) -> get_phi_arg(As, From). eval_terminator(#b_br{bool=#b_var{}=Bool}=Br, Bs, _St) -> Val = get_value(Bool, Bs), @@ -477,20 +503,31 @@ eval_bif(#b_set{op={bif,Bif},args=Args}, St) -> false -> none; true -> - case [Lit || #b_literal{val=Lit} <- Args] of - LitArgs when length(LitArgs) =:= Arity -> + case get_lit_args(Args) of + none -> + %% Not literal arguments. Try to evaluate + %% it based on a previous relational operator. + eval_rel_op({bif,Bif}, Args, St); + LitArgs -> try apply(erlang, Bif, LitArgs) of Val -> #b_literal{val=Val} catch error:_ -> none - end; - _ -> - %% Not literal arguments. Try to evaluate - %% it based on a previous relational operator. - eval_rel_op({bif,Bif}, Args, St) + end end end. +get_lit_args([#b_literal{val=Lit1}]) -> + [Lit1]; +get_lit_args([#b_literal{val=Lit1}, + #b_literal{val=Lit2}]) -> + [Lit1,Lit2]; +get_lit_args([#b_literal{val=Lit1}, + #b_literal{val=Lit2}, + #b_literal{val=Lit3}]) -> + [Lit1,Lit2,Lit3]; +get_lit_args(_) -> none. + %%% %%% Handling of relational operators. %%% @@ -1026,11 +1063,12 @@ used_vars_is([], Used) -> sub(#b_set{args=Args}=I, Sub) -> I#b_set{args=[sub_arg(A, Sub) || A <- Args]}. -sub_arg(Old, Sub) -> +sub_arg(#b_var{}=Old, Sub) -> case Sub of #{Old:=New} -> New; #{} -> Old - end. + end; +sub_arg(Old, _Sub) -> Old. rel2fam(S0) -> S1 = sofs:relation(S0), diff --git a/lib/compiler/src/beam_ssa_opt.erl b/lib/compiler/src/beam_ssa_opt.erl index 6e548dd529..90c0d3cf16 100644 --- a/lib/compiler/src/beam_ssa_opt.erl +++ b/lib/compiler/src/beam_ssa_opt.erl @@ -175,6 +175,7 @@ epilogue_passes(Opts) -> ?PASS(ssa_opt_blockify), ?PASS(ssa_opt_sink), ?PASS(ssa_opt_merge_blocks), + ?PASS(ssa_opt_get_tuple_element), ?PASS(ssa_opt_trim_unreachable)], passes_1(Ps, Opts). @@ -682,6 +683,14 @@ record_opt_is([#b_set{op={bif,is_tuple},dst=Bool,args=[Tuple]}=Set], no -> [Set] end; +record_opt_is([I|Is]=Is0, #b_br{bool=Bool}=Last, Blocks) -> + case is_tagged_tuple_1(Is0, Last, Blocks) of + {yes,_Fail,Tuple,Arity,Tag} -> + Args = [Tuple,Arity,Tag], + [I#b_set{op=is_tagged_tuple,dst=Bool,args=Args}]; + no -> + [I|record_opt_is(Is, Last, Blocks)] + end; record_opt_is([I|Is], Last, Blocks) -> [I|record_opt_is(Is, Last, Blocks)]; record_opt_is([], _Last, _Blocks) -> []. @@ -689,29 +698,30 @@ record_opt_is([], _Last, _Blocks) -> []. is_tagged_tuple(#b_var{}=Tuple, Bool, #b_br{bool=Bool,succ=Succ,fail=Fail}, Blocks) -> - SuccBlk = map_get(Succ, Blocks), - is_tagged_tuple_1(SuccBlk, Tuple, Fail, Blocks); + #b_blk{is=Is,last=Last} = map_get(Succ, Blocks), + case is_tagged_tuple_1(Is, Last, Blocks) of + {yes,Fail,Tuple,Arity,Tag} -> + {yes,Arity,Tag}; + _ -> + no + end; is_tagged_tuple(_, _, _, _) -> no. -is_tagged_tuple_1(#b_blk{is=Is,last=Last}, Tuple, Fail, Blocks) -> - case Is of - [#b_set{op={bif,tuple_size},dst=ArityVar, - args=[#b_var{}=Tuple]}, - #b_set{op={bif,'=:='}, - dst=Bool, - args=[ArityVar, #b_literal{val=ArityVal}=Arity]}] - when is_integer(ArityVal) -> - case Last of - #b_br{bool=Bool,succ=Succ,fail=Fail} -> - SuccBlk = map_get(Succ, Blocks), - case is_tagged_tuple_2(SuccBlk, Tuple, Fail) of - no -> - no; - {yes,Tag} -> - {yes,Arity,Tag} - end; - _ -> - no +is_tagged_tuple_1(Is, Last, Blocks) -> + case {Is,Last} of + {[#b_set{op={bif,tuple_size},dst=ArityVar, + args=[#b_var{}=Tuple]}, + #b_set{op={bif,'=:='}, + dst=Bool, + args=[ArityVar, #b_literal{val=ArityVal}=Arity]}], + #b_br{bool=Bool,succ=Succ,fail=Fail}} + when is_integer(ArityVal) -> + SuccBlk = map_get(Succ, Blocks), + case is_tagged_tuple_2(SuccBlk, Tuple, Fail) of + no -> + no; + {yes,Tag} -> + {yes,Fail,Tuple,Arity,Tag} end; _ -> no @@ -901,6 +911,11 @@ ssa_opt_float({#st{ssa=Linear0,cnt=Count0}=St, FuncDb}) -> {Linear,Count} = float_opt(Linear0, Count0, Fs), {St#st{ssa=Linear,cnt=Count}, FuncDb}. +float_blk_is_in_guard(#b_blk{last=#b_br{fail=F}}, #fs{non_guards=NonGuards}) -> + not gb_sets:is_member(F, NonGuards); +float_blk_is_in_guard(#b_blk{}, #fs{}) -> + false. + float_non_guards([{L,#b_blk{is=Is}}|Bs]) -> case Is of [#b_set{op=landingpad}|_] -> @@ -910,21 +925,18 @@ float_non_guards([{L,#b_blk{is=Is}}|Bs]) -> end; float_non_guards([]) -> [?BADARG_BLOCK]. -float_opt([{L,#b_blk{last=#b_br{fail=F}}=Blk}|Bs0], - Count0, #fs{non_guards=NonGuards}=Fs) -> - case gb_sets:is_member(F, NonGuards) of +float_opt([{L,Blk}|Bs0], Count0, Fs) -> + case float_blk_is_in_guard(Blk, Fs) of true -> - %% This block is not inside a guard. - %% We can do the optimization. - float_opt_1(L, Blk, Bs0, Count0, Fs); - false -> %% This block is inside a guard. Don't do %% any floating point optimizations. {Bs,Count} = float_opt(Bs0, Count0, Fs), - {[{L,Blk}|Bs],Count} + {[{L,Blk}|Bs],Count}; + false -> + %% This block is not inside a guard. + %% We can do the optimization. + float_opt_1(L, Blk, Bs0, Count0, Fs) end; -float_opt([{L,Blk}|Bs], Count, Fs) -> - float_opt_1(L, Blk, Bs, Count, Fs); float_opt([], Count, _Fs) -> {[],Count}. @@ -1000,10 +1012,14 @@ float_conv([{L,#b_blk{is=Is0}=Blk0}|Bs0], Fail, Count0) -> float_maybe_flush(Blk0, #fs{s=cleared,fail=Fail,bs=Blocks}=Fs0, Count0) -> #b_blk{last=#b_br{bool=#b_var{},succ=Succ}=Br} = Blk0, - #b_blk{is=Is} = map_get(Succ, Blocks), + + %% If the success block starts with a floating point operation, we can + %% defer flushing to that block as long as it isn't a guard. + #b_blk{is=Is} = SuccBlk = map_get(Succ, Blocks), + SuccIsGuard = float_blk_is_in_guard(SuccBlk, Fs0), + case Is of - [#b_set{anno=#{float_op:=_}}|_] -> - %% The next operation is also a floating point operation. + [#b_set{anno=#{float_op:=_}}|_] when not SuccIsGuard -> %% No flush needed. {[],Blk0,Fs0,Count0}; _ -> @@ -2174,6 +2190,46 @@ insert_def_is([#b_set{op=Op}=I|Is]=Is0, V, Def) -> insert_def_is([], _V, Def) -> [Def]. +%%% +%%% Order consecutive get_tuple_element instructions in ascending +%%% position order. This will give the loader more opportunities +%%% for combining get_tuple_element instructions. +%%% + +ssa_opt_get_tuple_element({#st{ssa=Blocks0}=St, FuncDb}) -> + Blocks = opt_get_tuple_element(maps:to_list(Blocks0), Blocks0), + {St#st{ssa=Blocks}, FuncDb}. + +opt_get_tuple_element([{L,#b_blk{is=Is0}=Blk0}|Bs], Blocks) -> + case opt_get_tuple_element_is(Is0, false, []) of + {yes,Is} -> + Blk = Blk0#b_blk{is=Is}, + opt_get_tuple_element(Bs, Blocks#{L:=Blk}); + no -> + opt_get_tuple_element(Bs, Blocks) + end; +opt_get_tuple_element([], Blocks) -> Blocks. + +opt_get_tuple_element_is([#b_set{op=get_tuple_element, + args=[#b_var{}=Src,_]}=I0|Is0], + _AnyChange, Acc) -> + {GetIs0,Is} = collect_get_tuple_element(Is0, Src, [I0]), + GetIs1 = sort([{Pos,I} || #b_set{args=[_,Pos]}=I <- GetIs0]), + GetIs = [I || {_,I} <- GetIs1], + opt_get_tuple_element_is(Is, true, reverse(GetIs, Acc)); +opt_get_tuple_element_is([I|Is], AnyChange, Acc) -> + opt_get_tuple_element_is(Is, AnyChange, [I|Acc]); +opt_get_tuple_element_is([], AnyChange, Acc) -> + case AnyChange of + true -> {yes,reverse(Acc)}; + false -> no + end. + +collect_get_tuple_element([#b_set{op=get_tuple_element, + args=[Src,_]}=I|Is], Src, Acc) -> + collect_get_tuple_element(Is, Src, [I|Acc]); +collect_get_tuple_element(Is, _Src, Acc) -> + {Acc,Is}. %%% %%% Common utilities. diff --git a/lib/compiler/src/beam_ssa_type.erl b/lib/compiler/src/beam_ssa_type.erl index aa4720d222..c01ea4af91 100644 --- a/lib/compiler/src/beam_ssa_type.erl +++ b/lib/compiler/src/beam_ssa_type.erl @@ -170,7 +170,8 @@ opt_finish_1([], [], ParamInfo) -> validator_anno(#t_tuple{size=Size,exact=Exact,elements=Elements0}) -> Elements = maps:fold(fun(Index, Type, Acc) -> - Acc#{ Index => validator_anno(Type) } + Key = beam_validator:type_anno(integer, Index), + Acc#{ Key => validator_anno(Type) } end, #{}, Elements0), beam_validator:type_anno(tuple, Size, Exact, Elements); validator_anno(#t_integer{elements={Same,Same}}) -> diff --git a/lib/compiler/src/beam_validator.erl b/lib/compiler/src/beam_validator.erl index 5175be3ad5..4fba3fa1c6 100644 --- a/lib/compiler/src/beam_validator.erl +++ b/lib/compiler/src/beam_validator.erl @@ -28,8 +28,7 @@ -export([module/2, format_error/1]). -export([type_anno/1, type_anno/2, type_anno/4]). --import(lists, [any/2,dropwhile/2,foldl/3,map/2,member/2,reverse/1, - seq/2,sort/1,zip/2]). +-import(lists, [dropwhile/2,foldl/3,member/2,reverse/1,sort/1,zip/2]). %% To be called by the compiler. @@ -51,7 +50,7 @@ module({Mod,Exp,Attr,Fs,Lc}=Code, _Opts) -spec type_anno(term()) -> term(). type_anno(atom) -> {atom,[]}; type_anno(bool) -> bool; -type_anno({binary,_}) -> term; +type_anno({binary,_}) -> binary; type_anno(cons) -> cons; type_anno(float) -> {float,[]}; type_anno(integer) -> {integer,[]}; @@ -62,9 +61,9 @@ type_anno(number) -> number; type_anno(nil) -> nil. -spec type_anno(term(), term()) -> term(). -type_anno(atom, Value) -> {atom, Value}; -type_anno(float, Value) -> {float, Value}; -type_anno(integer, Value) -> {integer, Value}. +type_anno(atom, Value) when is_atom(Value) -> {atom, Value}; +type_anno(float, Value) when is_float(Value) -> {float, Value}; +type_anno(integer, Value) when is_integer(Value) -> {integer, Value}. -spec type_anno(term(), term(), term(), term()) -> term(). type_anno(tuple, Size, Exact, Elements) when is_integer(Size), Size >= 0, @@ -137,43 +136,100 @@ validate_0(Module, [{function,Name,Ar,Entry,Code}|Fs], Ft) -> erlang:raise(Class, Error, Stack) end. +-record(value_ref, {id :: index()}). +-record(value, {op :: term(), args :: [argument()], type :: type()}). + +-type argument() :: #value_ref{} | literal(). + -type index() :: non_neg_integer(). --type reg_tab() :: gb_trees:tree(index(), 'none' | {'value', _}). - --record(st, %Emulation state - {x :: reg_tab(), %x register info. - y :: reg_tab(), %y register info. - f=init_fregs(), % - numy=none, %Number of y registers. - h=0, %Available heap size. - hf=0, %Available heap size for floats. - fls=undefined, %Floating point state. - ct=[], %List of hot catch/try labels - setelem=false, %Previous instruction was setelement/3. - puts_left=none, %put/1 instructions left. - defs=#{}, %Defining expression for each register. - aliases=#{} - }). + +-type literal() :: {atom, [] | atom()} | + {float, [] | float()} | + {integer, [] | integer()} | + {literal, term()} | + nil. + +-type tuple_sz() :: [non_neg_integer()] | %% Inexact + non_neg_integer(). %% Exact. + +%% Match context type. +-record(ms, + {id=make_ref() :: reference(), %Unique ID. + valid=0 :: non_neg_integer(), %Valid slots + slots=0 :: non_neg_integer() %Number of slots + }). + +-type type() :: binary | + cons | + list | + map | + nil | + #ms{} | + ms_position | + none | + number | + term | + tuple_in_progress | + {tuple, tuple_sz(), #{ literal() => type() }} | + literal(). + +-type tag() :: initialized | + uninitialized | + {catchtag, [label()]} | + {trytag, [label()]}. + +-type x_regs() :: #{ {x, index()} => #value_ref{} }. +-type y_regs() :: #{ {y, index()} => tag() | #value_ref{} }. + +%% Emulation state +-record(st, + {%% All known values. + vs=#{} :: #{ #value_ref{} => #value{} }, + %% Register states. + xs=#{} :: x_regs(), + ys=#{} :: y_regs(), + f=init_fregs(), + %% A set of all registers containing "fragile" terms. That is, terms + %% that don't exist on our process heap and would be destroyed by a + %% GC. + fragile=cerl_sets:new() :: cerl_sets:set(), + %% Number of Y registers. + %% + %% Note that this may be 0 if there's a frame without saved values, + %% such as on a body-recursive call. + numy=none :: none | undecided | index(), + %% Available heap size. + h=0, + %Available heap size for floats. + hf=0, + %% Floating point state. + fls=undefined, + %% List of hot catch/try labels + ct=[], + %% Previous instruction was setelement/3. + setelem=false, + %% put/1 instructions left. + puts_left=none + }). -type label() :: integer(). -type label_set() :: gb_sets:set(label()). -type branched_tab() :: gb_trees:tree(label(), #st{}). -type ft_tab() :: gb_trees:tree(). --record(vst, %Validator state - {current=none :: #st{} | 'none', %Current state - branched=gb_trees:empty() :: branched_tab(), %States at jumps - labels=gb_sets:empty() :: label_set(), %All defined labels - ft=gb_trees:empty() :: ft_tab() %Some other functions - % in the module (those that start with bs_start_match2). - }). - -%% Match context type. --record(ms, - {id=make_ref() :: reference(), %Unique ID. - valid=0 :: non_neg_integer(), %Valid slots - slots=0 :: non_neg_integer() %Number of slots - }). +%% Validator state +-record(vst, + {%% Current state + current=none :: #st{} | 'none', + %% States at labels + branched=gb_trees:empty() :: branched_tab(), + %% All defined labels + labels=gb_sets:empty() :: label_set(), + %% Argument information of other functions in the module + ft=gb_trees:empty() :: ft_tab(), + %% Counter for #value_ref{} creation + ref_ctr=0 :: index() + }). index_parameter_types([{function,_,_,Entry,Code0}|Fs], Acc0) -> Code = dropwhile(fun({label,L}) when L =:= Entry -> false; @@ -238,7 +294,7 @@ validate_fun_info_branches_1(X, {Mod,Name,Arity}=MFA, Vst) -> #vst{current=#st{numy=Size}} -> error({unexpected_stack_frame,Size}) end, - get_term_type({x,X}, Vst) + assert_term({x,X}, Vst) catch Error -> I = {func_info,{atom,Mod},{atom,Name},Arity}, Offset = 2, @@ -260,24 +316,21 @@ labels_1(Is, R) -> {reverse(R),Is}. init_vst(Arity, Ls1, Ls2, Ft) -> - Xs = init_regs(Arity, term), - Ys = init_regs(0, initialized), - St = #st{x=Xs,y=Ys}, - Branches = gb_trees_from_list([{L,St} || L <- Ls1]), + Vst0 = init_function_args(Arity - 1, #vst{current=#st{}}), + Branches = gb_trees_from_list([{L,Vst0#vst.current} || L <- Ls1]), Labels = gb_sets:from_list(Ls1++Ls2), - #vst{branched=Branches, - current=St, - labels=Labels, - ft=Ft}. + Vst0#vst{branched=Branches, + labels=Labels, + ft=Ft}. + +init_function_args(-1, Vst) -> + Vst; +init_function_args(X, Vst) -> + init_function_args(X - 1, create_term(term, argument, [], {x,X}, Vst)). kill_heap_allocation(St) -> St#st{h=0,hf=0}. -init_regs(0, _) -> - gb_trees:empty(); -init_regs(N, Type) -> - gb_trees_from_list([{R,Type} || R <- seq(0, N-1)]). - valfun([], MFA, _Offset, #vst{branched=Targets0,labels=Labels0}=Vst) -> Targets = gb_trees:keys(Targets0), Labels = gb_sets:to_list(Labels0), @@ -298,20 +351,25 @@ valfun([I|Is], MFA, Offset, Vst0) -> %% Instructions that are allowed in dead code or when failing, %% that is while the state is undecided in some way. -valfun_1({label,Lbl}, #vst{current=St0,branched=B,labels=Lbls}=Vst) -> - St = merge_states(Lbl, St0, B), - Vst#vst{current=St,branched=gb_trees:enter(Lbl, St, B), - labels=gb_sets:add(Lbl, Lbls)}; +valfun_1({label,Lbl}, #vst{current=St0, + ref_ctr=Counter0, + branched=B, + labels=Lbls}=Vst) -> + {St, Counter} = merge_states(Lbl, St0, B, Counter0), + Vst#vst{current=St, + ref_ctr=Counter, + branched=gb_trees:enter(Lbl, St, B), + labels=gb_sets:add(Lbl, Lbls)}; valfun_1(_I, #vst{current=none}=Vst) -> %% Ignore instructions after erlang:error/1,2, which %% the original R10B compiler thought would return. Vst; valfun_1({badmatch,Src}, Vst) -> - assert_not_fragile(Src, Vst), + assert_durable_term(Src, Vst), verify_y_init(Vst), kill_state(Vst); valfun_1({case_end,Src}, Vst) -> - assert_not_fragile(Src, Vst), + assert_durable_term(Src, Vst), verify_y_init(Vst), kill_state(Vst); valfun_1(if_end, Vst) -> @@ -319,7 +377,7 @@ valfun_1(if_end, Vst) -> kill_state(Vst); valfun_1({try_case_end,Src}, Vst) -> verify_y_init(Vst), - assert_not_fragile(Src, Vst), + assert_durable_term(Src, Vst), kill_state(Vst); %% Instructions that cannot cause exceptions valfun_1({bs_get_tail,Ctx,Dst,Live}, Vst0) -> @@ -363,17 +421,17 @@ valfun_1({bif,Op,{f,_},Ss,Dst}=I, Vst) -> end; %% Put instructions. valfun_1({put_list,A,B,Dst}, Vst0) -> - assert_not_fragile(A, Vst0), - assert_not_fragile(B, Vst0), + assert_term(A, Vst0), + assert_term(B, Vst0), Vst = eat_heap(2, Vst0), create_term(cons, put_list, [A, B], Dst, Vst); valfun_1({put_tuple2,Dst,{list,Elements}}, Vst0) -> - _ = [assert_not_fragile(El, Vst0) || El <- Elements], + _ = [assert_term(El, Vst0) || El <- Elements], Size = length(Elements), Vst = eat_heap(Size+1, Vst0), {Es,_} = foldl(fun(Val, {Es0, Index}) -> Type = get_term_type(Val, Vst0), - Es = set_element_type(Index, Type, Es0), + Es = set_element_type({integer,Index}, Type, Es0), {Es, Index + 1} end, {#{}, 1}, Elements), Type = {tuple,Size,Es}, @@ -385,18 +443,19 @@ valfun_1({put_tuple,Sz,Dst}, Vst0) when is_integer(Sz) -> St = St0#st{puts_left={Sz,{Dst,Sz,#{}}}}, Vst#vst{current=St}; valfun_1({put,Src}, Vst0) -> - assert_not_fragile(Src, Vst0), + assert_term(Src, Vst0), Vst = eat_heap(1, Vst0), #vst{current=St0} = Vst, case St0 of #st{puts_left=none} -> error(not_building_a_tuple); - #st{puts_left={1,{Dst,Sz,Es}}} -> + #st{puts_left={1,{Dst,Sz,Es0}}} -> + Es = Es0#{ {integer,Sz} => get_term_type(Src, Vst0) }, St = St0#st{puts_left=none}, create_term({tuple,Sz,Es}, put_tuple, [], Dst, Vst#vst{current=St}); #st{puts_left={PutsLeft,{Dst,Sz,Es0}}} when is_integer(PutsLeft) -> Index = Sz - PutsLeft + 1, - Es = Es0#{ Index => get_term_type(Src, Vst0) }, + Es = Es0#{ {integer,Index} => get_term_type(Src, Vst0) }, St = St0#st{puts_left={PutsLeft-1,{Dst,Sz,Es}}}, Vst#vst{current=St} end; @@ -417,6 +476,14 @@ valfun_1({'%', {type_info, Reg, Type}}, Vst) -> %% that Reg has a certain type, so that we can accept cross-function type %% optimizations. update_type(fun meet/2, Type, Reg, Vst); +valfun_1({'%', {remove_fragility, Reg}}, Vst) -> + %% This is a hack to make prim_eval:'receive'/2 work. + %% + %% Normally it's illegal to pass fragile terms as a function argument as we + %% have no way of knowing what the callee will do with it, but we know that + %% prim_eval:'receive'/2 won't leak the term, nor cause a GC since it's + %% disabled while matching messages. + remove_fragility(Reg, Vst); valfun_1({'%',_}, Vst) -> Vst; valfun_1({line,_}, Vst) -> @@ -438,13 +505,13 @@ valfun_1(_I, #vst{current=#st{ct=undecided}}) -> %% %% Allocate and deallocate, et.al valfun_1({allocate,Stk,Live}, Vst) -> - allocate(false, Stk, 0, Live, Vst); + allocate(uninitialized, Stk, 0, Live, Vst); valfun_1({allocate_heap,Stk,Heap,Live}, Vst) -> - allocate(false, Stk, Heap, Live, Vst); + allocate(uninitialized, Stk, Heap, Live, Vst); valfun_1({allocate_zero,Stk,Live}, Vst) -> - allocate(true, Stk, 0, Live, Vst); + allocate(initialized, Stk, 0, Live, Vst); valfun_1({allocate_heap_zero,Stk,Heap,Live}, Vst) -> - allocate(true, Stk, Heap, Live, Vst); + allocate(initialized, Stk, Heap, Live, Vst); valfun_1({deallocate,StkSize}, #vst{current=#st{numy=StkSize}}=Vst) -> verify_no_ct(Vst), deallocate(Vst); @@ -508,46 +575,66 @@ valfun_1({get_tl,Src,Dst}, Vst) -> valfun_1({get_tuple_element,Src,N,Dst}, Vst) -> assert_not_literal(Src), assert_type({tuple_element,N+1}, Src, Vst), - Type = get_element_type(N+1, Src, Vst), - extract_term(Type, get_tuple_element, [Src], Dst, Vst); + Index = {integer,N+1}, + Type = get_element_type(Index, Src, Vst), + extract_term(Type, {bif,element}, [Index, Src], Dst, Vst); valfun_1({jump,{f,Lbl}}, Vst) -> - kill_state(branch_state(Lbl, Vst)); + branch(Lbl, Vst, + fun(SuccVst) -> + %% The next instruction is never executed. + kill_state(SuccVst) + end); valfun_1(I, Vst) -> valfun_2(I, Vst). init_try_catch_branch(Tag, Dst, Fail, Vst0) -> Vst1 = create_tag({Tag,[Fail]}, 'try_catch', [], Dst, Vst0), #vst{current=#st{ct=Fails}=St0} = Vst1, - CurrentSt = St0#st{ct=[[Fail]|Fails]}, - - %% Set the initial state at the try/catch label. - %% Assume that Y registers contain terms or try/catch - %% tags. - Yregs0 = map(fun({Y,uninitialized}) -> {Y,term}; - ({Y,initialized}) -> {Y,term}; - (E) -> E - end, gb_trees:to_list(CurrentSt#st.y)), - Yregs = gb_trees:from_orddict(Yregs0), - BranchSt = CurrentSt#st{y=Yregs}, - - Vst = branch_state(Fail, Vst1#vst{current=BranchSt}), - Vst#vst{current=CurrentSt}. - -%% Update branched state if necessary and try next set of instructions. -valfun_2(I, #vst{current=#st{ct=[]}}=Vst) -> - valfun_3(I, Vst); + St = St0#st{ct=[[Fail]|Fails]}, + Vst = Vst0#vst{current=St}, + + branch(Fail, Vst, + fun(CatchVst) -> + #vst{current=#st{ys=Ys}} = CatchVst, + maps:fold(fun init_catch_handler_1/3, CatchVst, Ys) + end, + fun(SuccVst) -> + %% All potentially-throwing instructions after this + %% one will implicitly branch to the fail label; + %% see valfun_2/2 + SuccVst + end). + +%% Set the initial state at the try/catch label. Assume that Y registers +%% contain terms or try/catch tags. +init_catch_handler_1(Reg, initialized, Vst) -> + create_term(term, 'catch_handler', [], Reg, Vst); +init_catch_handler_1(Reg, uninitialized, Vst) -> + create_term(term, 'catch_handler', [], Reg, Vst); +init_catch_handler_1(_, _, Vst) -> + Vst. + valfun_2(I, #vst{current=#st{ct=[[Fail]|_]}}=Vst) when is_integer(Fail) -> - %% Update branched state. + %% We have an active try/catch tag and we can jump there from this + %% instruction, so we need to update the branched state of the try/catch + %% handler. valfun_3(I, branch_state(Fail, Vst)); +valfun_2(I, #vst{current=#st{ct=[]}}=Vst) -> + valfun_3(I, Vst); valfun_2(_, _) -> error(ambiguous_catch_try_state). %% Handle the remaining floating point instructions here. %% Floating point. -valfun_3({fconv,Src,{fr,_}=Dst}, Vst0) -> - assert_term(Src, Vst0), - Vst = update_type(fun meet/2, number, Src, Vst0), - set_freg(Dst, Vst); +valfun_3({fconv,Src,{fr,_}=Dst}, Vst) -> + assert_term(Src, Vst), + + %% An exception is raised on error, hence branching to 0. + branch(0, Vst, + fun(SuccVst0) -> + SuccVst = update_type(fun meet/2, number, Src, SuccVst0), + set_freg(Dst, SuccVst) + end); valfun_3({bif,fadd,_,[_,_]=Ss,Dst}, Vst) -> float_op(Ss, Dst, Vst); valfun_3({bif,fdiv,_,[_,_]=Ss,Dst}, Vst) -> @@ -608,68 +695,87 @@ valfun_4({call_ext_last,_,_,_}, #vst{current=#st{numy=NumY}}) -> valfun_4({make_fun2,_,_,_,Live}, Vst) -> call(make_fun, Live, Vst); %% Other BIFs -valfun_4({bif,element,{f,Fail},[Pos,Tuple],Dst}, Vst0) -> - PosType = get_durable_term_type(Pos, Vst0), - ElementType = case PosType of - {integer,I} -> get_element_type(I, Tuple, Vst0); - _ -> term - end, - InferredType = {tuple,[get_tuple_size(PosType)],#{}}, - Vst1 = branch_state(Fail, Vst0), - Vst = update_type(fun meet/2, InferredType, Tuple, Vst1), - extract_term(ElementType, {bif,element}, [Tuple], Dst, Vst); +valfun_4({bif,element,{f,Fail},[Pos,Src],Dst}, Vst) -> + branch(Fail, Vst, + fun(SuccVst0) -> + PosType = get_term_type(Pos, SuccVst0), + TupleType = {tuple,[get_tuple_size(PosType)],#{}}, + + SuccVst1 = update_type(fun meet/2, TupleType, + Src, SuccVst0), + SuccVst = update_type(fun meet/2, {integer,[]}, + Pos, SuccVst1), + + ElementType = get_element_type(PosType, Src, SuccVst), + extract_term(ElementType, {bif,element}, [Pos,Src], + Dst, SuccVst) + end); valfun_4({bif,raise,{f,0},Src,_Dst}, Vst) -> validate_src(Src, Vst), kill_state(Vst); valfun_4(raw_raise=I, Vst) -> call(I, 3, Vst); -valfun_4({bif,Op,{f,Fail},[Cons]=Ss,Dst}, Vst0) - when Op =:= hd; Op =:= tl -> - validate_src(Ss, Vst0), - Vst = type_test(Fail, cons, Cons, Vst0), - Type = bif_return_type(Op, Ss, Vst), - extract_term(Type, {bif,Op}, Ss, Dst, Vst); -valfun_4({bif,Op,{f,Fail},Ss,Dst}, Vst0) -> - validate_src(Ss, Vst0), - Vst1 = branch_state(Fail, Vst0), - - %% Infer argument types. Note that we can't type_test in the general case - %% as the BIF could fail for reasons other than bad arguments. - ArgTypes = bif_arg_types(Op, Ss), - Vst = foldl(fun({Arg, T}, Vsti) -> - update_type(fun meet/2, T, Arg, Vsti) - end, Vst1, zip(Ss, ArgTypes)), - - Type = bif_return_type(Op, Ss, Vst), - extract_term(Type, {bif,Op}, Ss, Dst, Vst); +valfun_4({bif,Op,{f,Fail},[Src]=Ss,Dst}, Vst) when Op =:= hd; Op =:= tl -> + assert_term(Src, Vst), + branch(Fail, Vst, + fun(FailVst) -> + update_type(fun subtract/2, cons, Src, FailVst) + end, + fun(SuccVst0) -> + SuccVst = update_type(fun meet/2, cons, Src, SuccVst0), + extract_term(term, {bif,Op}, Ss, Dst, SuccVst) + end); +valfun_4({bif,Op,{f,Fail},Ss,Dst}, Vst) -> + validate_src(Ss, Vst), + branch(Fail, Vst, + fun(SuccVst0) -> + %% Infer argument types. Note that we can't subtract + %% types as the BIF could fail for reasons other than + %% bad argument types. + ArgTypes = bif_arg_types(Op, Ss), + SuccVst = foldl(fun({Arg, T}, V) -> + update_type(fun meet/2, T, Arg, V) + end, SuccVst0, zip(Ss, ArgTypes)), + Type = bif_return_type(Op, Ss, SuccVst), + extract_term(Type, {bif,Op}, Ss, Dst, SuccVst) + end); valfun_4({gc_bif,Op,{f,Fail},Live,Ss,Dst}, #vst{current=St0}=Vst0) -> validate_src(Ss, Vst0), verify_live(Live, Vst0), verify_y_init(Vst0), + + %% Heap allocations and X registers are killed regardless of whether we + %% fail or not, as we may fail after GC. St = kill_heap_allocation(St0), - Vst1 = Vst0#vst{current=St}, - Vst2 = branch_state(Fail, Vst1), + Vst = prune_x_regs(Live, Vst0#vst{current=St}), - ArgTypes = bif_arg_types(Op, Ss), - Vst3 = foldl(fun({Arg, T}, Vsti) -> - update_type(fun meet/2, T, Arg, Vsti) - end, Vst2, zip(Ss, ArgTypes)), + branch(Fail, Vst, + fun(SuccVst0) -> + ArgTypes = bif_arg_types(Op, Ss), + SuccVst = foldl(fun({Arg, T}, V) -> + update_type(fun meet/2, T, Arg, V) + end, SuccVst0, zip(Ss, ArgTypes)), - Type = bif_return_type(Op, Ss, Vst3), - Vst = prune_x_regs(Live, Vst3), - extract_term(Type, {gc_bif,Op}, Ss, Dst, Vst, Vst0); + Type = bif_return_type(Op, Ss, SuccVst), + + %% We're passing Vst0 as the original because the + %% registers were pruned before the branch. + extract_term(Type, {gc_bif,Op}, Ss, Dst, SuccVst, Vst0) + end); valfun_4(return, #vst{current=#st{numy=none}}=Vst) -> - assert_not_fragile({x,0}, Vst), + assert_durable_term({x,0}, Vst), kill_state(Vst); valfun_4(return, #vst{current=#st{numy=NumY}}) -> error({stack_frame,NumY}); -valfun_4({loop_rec,{f,Fail},Dst}, Vst0) -> - Vst = branch_state(Fail, Vst0), - %% This term may not be part of the root set until - %% remove_message/0 is executed. If control transfers - %% to the loop_rec_end/1 instruction, no part of - %% this term must be stored in a Y register. - create_term({fragile,term}, loop_rec, [], Dst, Vst); +valfun_4({loop_rec,{f,Fail},Dst}, Vst) -> + %% This term may not be part of the root set until remove_message/0 is + %% executed. If control transfers to the loop_rec_end/1 instruction, no + %% part of this term must be stored in a Y register. + branch(Fail, Vst, + fun(SuccVst0) -> + {Ref, SuccVst} = new_value(term, loop_rec, [], SuccVst0), + mark_fragile(Dst, set_reg_vref(Ref, Dst, SuccVst)) + end); valfun_4({wait,_}, Vst) -> verify_y_init(Vst), kill_state(Vst); @@ -680,30 +786,30 @@ valfun_4({wait_timeout,_,Src}, Vst) -> valfun_4({loop_rec_end,_}, Vst) -> verify_y_init(Vst), kill_state(Vst); -valfun_4(timeout, #vst{current=St}=Vst) -> - Vst#vst{current=St#st{x=init_regs(0, term)}}; +valfun_4(timeout, Vst) -> + prune_x_regs(0, Vst); valfun_4(send, Vst) -> call(send, 2, Vst); valfun_4({set_tuple_element,Src,Tuple,N}, Vst) -> I = N + 1, - assert_not_fragile(Src, Vst), + assert_term(Src, Vst), assert_type({tuple_element,I}, Tuple, Vst), %% Manually update the tuple type; we can't rely on the ordinary update %% helpers as we must support overwriting (rather than just widening or %% narrowing) known elements, and we can't use extract_term either since %% the source tuple may be aliased. {tuple, Sz, Es0} = get_term_type(Tuple, Vst), - Es = set_element_type(I, get_term_type(Src, Vst), Es0), + Es = set_element_type({integer,I}, get_term_type(Src, Vst), Es0), override_type({tuple, Sz, Es}, Tuple, Vst); %% Match instructions. -valfun_4({select_val,Src,{f,Fail},{list,Choices}}, Vst0) -> - assert_term(Src, Vst0), +valfun_4({select_val,Src,{f,Fail},{list,Choices}}, Vst) -> + assert_term(Src, Vst), assert_choices(Choices), - select_val_branches(Fail, Src, Choices, Vst0); + validate_select_val(Fail, Choices, Src, Vst); valfun_4({select_tuple_arity,Tuple,{f,Fail},{list,Choices}}, Vst) -> assert_type(tuple, Tuple, Vst), assert_arities(Choices), - select_arity_branches(Fail, Choices, Tuple, Vst); + validate_select_tuple_arity(Fail, Choices, Tuple, Vst); %% New bit syntax matching instructions. valfun_4({test,bs_start_match3,{f,Fail},Live,[Src],Dst}, Vst) -> @@ -712,17 +818,17 @@ valfun_4({test,bs_start_match2,{f,Fail},Live,[Src,Slots],Dst}, Vst) -> validate_bs_start_match(Fail, Live, bsm_match_state(Slots), Src, Dst, Vst); valfun_4({test,bs_match_string,{f,Fail},[Ctx,_,_]}, Vst) -> bsm_validate_context(Ctx, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, fun(V) -> V end); valfun_4({test,bs_skip_bits2,{f,Fail},[Ctx,Src,_,_]}, Vst) -> bsm_validate_context(Ctx, Vst), assert_term(Src, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, fun(V) -> V end); valfun_4({test,bs_test_tail2,{f,Fail},[Ctx,_]}, Vst) -> bsm_validate_context(Ctx, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, fun(V) -> V end); valfun_4({test,bs_test_unit,{f,Fail},[Ctx,_]}, Vst) -> bsm_validate_context(Ctx, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, fun(V) -> V end); valfun_4({test,bs_skip_utf8,{f,Fail},[Ctx,Live,_]}, Vst) -> validate_bs_skip_utf(Fail, Ctx, Live, Vst); valfun_4({test,bs_skip_utf16,{f,Fail},[Ctx,Live,_]}, Vst) -> @@ -750,15 +856,23 @@ valfun_4({bs_get_position, Ctx, Dst, Live}, Vst0) -> verify_live(Live, Vst0), verify_y_init(Vst0), Vst = prune_x_regs(Live, Vst0), - create_term(bs_position, bs_get_position, [Ctx], Dst, Vst); + create_term(ms_position, bs_get_position, [Ctx], Dst, Vst, Vst0); valfun_4({bs_set_position, Ctx, Pos}, Vst) -> bsm_validate_context(Ctx, Vst), - assert_type(bs_position, Pos, Vst), + assert_type(ms_position, Pos, Vst), Vst; %% Other test instructions. +valfun_4({test,has_map_fields,{f,Lbl},Src,{list,List}}, Vst) -> + assert_type(map, Src, Vst), + assert_unique_map_keys(List), + branch(Lbl, Vst, fun(V) -> V end); valfun_4({test,is_atom,{f,Lbl},[Src]}, Vst) -> type_test(Lbl, {atom,[]}, Src, Vst); +valfun_4({test,is_binary,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, binary, Src, Vst); +valfun_4({test,is_bitstr,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, binary, Src, Vst); valfun_4({test,is_boolean,{f,Lbl},[Src]}, Vst) -> type_test(Lbl, bool, Src, Vst); valfun_4({test,is_float,{f,Lbl},[Src]}, Vst) -> @@ -769,63 +883,72 @@ valfun_4({test,is_integer,{f,Lbl},[Src]}, Vst) -> type_test(Lbl, {integer,[]}, Src, Vst); valfun_4({test,is_nonempty_list,{f,Lbl},[Src]}, Vst) -> type_test(Lbl, cons, Src, Vst); +valfun_4({test,is_number,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, number, Src, Vst); valfun_4({test,is_list,{f,Lbl},[Src]}, Vst) -> type_test(Lbl, list, Src, Vst); -valfun_4({test,is_nil,{f,Lbl},[Src]}, Vst) -> - type_test(Lbl, nil, Src, Vst); valfun_4({test,is_map,{f,Lbl},[Src]}, Vst) -> - case Src of - {Tag,_} when Tag =:= x; Tag =:= y -> - type_test(Lbl, map, Src, Vst); - {literal,Map} when is_map(Map) -> - Vst; - _ -> - assert_term(Src, Vst), - kill_state(Vst) - end; -valfun_4({test,test_arity,{f,Lbl},[Tuple,Sz]}, Vst0) when is_integer(Sz) -> - assert_type(tuple, Tuple, Vst0), - Vst = branch_state(Lbl, Vst0), - update_type(fun meet/2, {tuple,Sz,#{}}, Tuple, Vst); -valfun_4({test,is_tagged_tuple,{f,Lbl},[Src,Sz,Atom]}, Vst0) -> - assert_term(Src, Vst0), - Vst = branch_state(Lbl, Vst0), - update_type(fun meet/2, {tuple,Sz,#{ 1 => Atom }}, Src, Vst); -valfun_4({test,has_map_fields,{f,Lbl},Src,{list,List}}, Vst) -> - assert_type(map, Src, Vst), - assert_unique_map_keys(List), - branch_state(Lbl, Vst); + type_test(Lbl, map, Src, Vst); +valfun_4({test,is_nil,{f,Lbl},[Src]}, Vst) -> + %% is_nil is an exact check against the 'nil' value, and should not be + %% treated as a simple type test. + assert_term(Src, Vst), + branch(Lbl, Vst, + fun(FailVst) -> + update_ne_types(Src, nil, FailVst) + end, + fun(SuccVst) -> + update_eq_types(Src, nil, SuccVst) + end); +valfun_4({test,test_arity,{f,Lbl},[Tuple,Sz]}, Vst) when is_integer(Sz) -> + assert_type(tuple, Tuple, Vst), + Type = {tuple, Sz, #{}}, + type_test(Lbl, Type, Tuple, Vst); +valfun_4({test,is_tagged_tuple,{f,Lbl},[Src,Sz,Atom]}, Vst) -> + assert_term(Src, Vst), + Type = {tuple, Sz, #{ {integer,1} => Atom }}, + type_test(Lbl, Type, Src, Vst); valfun_4({test,is_eq_exact,{f,Lbl},[Src,Val]=Ss}, Vst) -> validate_src(Ss, Vst), - complex_test(Lbl, - fun(FailVst) -> - update_ne_types(Src, Val, FailVst) - end, - fun(SuccVst) -> - update_eq_types(Src, Val, SuccVst) - end, Vst); + branch(Lbl, Vst, + fun(FailVst) -> + update_ne_types(Src, Val, FailVst) + end, + fun(SuccVst) -> + update_eq_types(Src, Val, SuccVst) + end); valfun_4({test,is_ne_exact,{f,Lbl},[Src,Val]=Ss}, Vst) -> validate_src(Ss, Vst), - complex_test(Lbl, - fun(FailVst) -> - update_eq_types(Src, Val, FailVst) - end, - fun(SuccVst) -> - update_ne_types(Src, Val, SuccVst) - end, Vst); + branch(Lbl, Vst, + fun(FailVst) -> + update_eq_types(Src, Val, FailVst) + end, + fun(SuccVst) -> + update_ne_types(Src, Val, SuccVst) + end); valfun_4({test,_Op,{f,Lbl},Src}, Vst) -> + %% is_pid, is_reference, et cetera. validate_src(Src, Vst), - branch_state(Lbl, Vst); + branch(Lbl, Vst, fun(V) -> V end); valfun_4({bs_add,{f,Fail},[A,B,_],Dst}, Vst) -> - assert_not_fragile(A, Vst), - assert_not_fragile(B, Vst), - create_term({integer,[]}, bs_add, [A, B], Dst, branch_state(Fail, Vst)); + assert_term(A, Vst), + assert_term(B, Vst), + branch(Fail, Vst, + fun(SuccVst) -> + create_term({integer,[]}, bs_add, [A, B], Dst, SuccVst) + end); valfun_4({bs_utf8_size,{f,Fail},A,Dst}, Vst) -> assert_term(A, Vst), - create_term({integer,[]}, bs_utf8_size, [A], Dst, branch_state(Fail, Vst)); + branch(Fail, Vst, + fun(SuccVst) -> + create_term({integer,[]}, bs_utf8_size, [A], Dst, SuccVst) + end); valfun_4({bs_utf16_size,{f,Fail},A,Dst}, Vst) -> assert_term(A, Vst), - create_term({integer,[]}, bs_utf16_size, [A], Dst, branch_state(Fail, Vst)); + branch(Fail, Vst, + fun(SuccVst) -> + create_term({integer,[]}, bs_utf16_size, [A], Dst, SuccVst) + end); valfun_4({bs_init2,{f,Fail},Sz,Heap,Live,_,Dst}, Vst0) -> verify_live(Live, Vst0), verify_y_init(Vst0), @@ -833,12 +956,14 @@ valfun_4({bs_init2,{f,Fail},Sz,Heap,Live,_,Dst}, Vst0) -> is_integer(Sz) -> ok; true -> - assert_not_fragile(Sz, Vst0) + assert_term(Sz, Vst0) end, - Vst1 = heap_alloc(Heap, Vst0), - Vst2 = branch_state(Fail, Vst1), - Vst = prune_x_regs(Live, Vst2), - create_term(binary, bs_init2, [], Dst, Vst); + Vst = heap_alloc(Heap, Vst0), + branch(Fail, Vst, + fun(SuccVst0) -> + SuccVst = prune_x_regs(Live, SuccVst0), + create_term(binary, bs_init2, [], Dst, SuccVst, SuccVst0) + end); valfun_4({bs_init_bits,{f,Fail},Sz,Heap,Live,_,Dst}, Vst0) -> verify_live(Live, Vst0), verify_y_init(Vst0), @@ -848,47 +973,71 @@ valfun_4({bs_init_bits,{f,Fail},Sz,Heap,Live,_,Dst}, Vst0) -> true -> assert_term(Sz, Vst0) end, - Vst1 = heap_alloc(Heap, Vst0), - Vst2 = branch_state(Fail, Vst1), - Vst = prune_x_regs(Live, Vst2), - create_term(binary, bs_init_bits, [], Dst, Vst); + Vst = heap_alloc(Heap, Vst0), + branch(Fail, Vst, + fun(SuccVst0) -> + SuccVst = prune_x_regs(Live, SuccVst0), + create_term(binary, bs_init_bits, [], Dst, SuccVst) + end); valfun_4({bs_append,{f,Fail},Bits,Heap,Live,_Unit,Bin,_Flags,Dst}, Vst0) -> verify_live(Live, Vst0), verify_y_init(Vst0), - assert_not_fragile(Bits, Vst0), - assert_not_fragile(Bin, Vst0), - Vst1 = heap_alloc(Heap, Vst0), - Vst2 = branch_state(Fail, Vst1), - Vst = prune_x_regs(Live, Vst2), - create_term(binary, bs_append, [Bin], Dst, Vst); -valfun_4({bs_private_append,{f,Fail},Bits,_Unit,Bin,_Flags,Dst}, Vst0) -> - assert_not_fragile(Bits, Vst0), - assert_not_fragile(Bin, Vst0), - Vst = branch_state(Fail, Vst0), - create_term(binary, bs_private_append, [Bin], Dst, Vst); + assert_term(Bits, Vst0), + assert_term(Bin, Vst0), + Vst = heap_alloc(Heap, Vst0), + branch(Fail, Vst, + fun(SuccVst0) -> + SuccVst = prune_x_regs(Live, SuccVst0), + create_term(binary, bs_append, [Bin], Dst, SuccVst, SuccVst0) + end); +valfun_4({bs_private_append,{f,Fail},Bits,_Unit,Bin,_Flags,Dst}, Vst) -> + assert_term(Bits, Vst), + assert_term(Bin, Vst), + branch(Fail, Vst, + fun(SuccVst) -> + create_term(binary, bs_private_append, [Bin], Dst, SuccVst) + end); valfun_4({bs_put_string,Sz,_}, Vst) when is_integer(Sz) -> Vst; valfun_4({bs_put_binary,{f,Fail},Sz,_,_,Src}, Vst) -> - assert_not_fragile(Sz, Vst), - assert_not_fragile(Src, Vst), - branch_state(Fail, Vst); + assert_term(Sz, Vst), + assert_term(Src, Vst), + branch(Fail, Vst, + fun(SuccVst) -> + update_type(fun meet/2, binary, Src, SuccVst) + end); valfun_4({bs_put_float,{f,Fail},Sz,_,_,Src}, Vst) -> - assert_not_fragile(Sz, Vst), - assert_not_fragile(Src, Vst), - branch_state(Fail, Vst); + assert_term(Sz, Vst), + assert_term(Src, Vst), + branch(Fail, Vst, + fun(SuccVst) -> + update_type(fun meet/2, {float,[]}, Src, SuccVst) + end); valfun_4({bs_put_integer,{f,Fail},Sz,_,_,Src}, Vst) -> - assert_not_fragile(Sz, Vst), - assert_not_fragile(Src, Vst), - branch_state(Fail, Vst); + assert_term(Sz, Vst), + assert_term(Src, Vst), + branch(Fail, Vst, + fun(SuccVst) -> + update_type(fun meet/2, {integer,[]}, Src, SuccVst) + end); valfun_4({bs_put_utf8,{f,Fail},_,Src}, Vst) -> - assert_not_fragile(Src, Vst), - branch_state(Fail, Vst); + assert_term(Src, Vst), + branch(Fail, Vst, + fun(SuccVst) -> + update_type(fun meet/2, {integer,[]}, Src, SuccVst) + end); valfun_4({bs_put_utf16,{f,Fail},_,Src}, Vst) -> - assert_not_fragile(Src, Vst), - branch_state(Fail, Vst); + assert_term(Src, Vst), + branch(Fail, Vst, + fun(SuccVst) -> + update_type(fun meet/2, {integer,[]}, Src, SuccVst) + end); valfun_4({bs_put_utf32,{f,Fail},_,Src}, Vst) -> - assert_not_fragile(Src, Vst), - branch_state(Fail, Vst); + assert_term(Src, Vst), + branch(Fail, Vst, + fun(SuccVst) -> + update_type(fun meet/2, {integer,[]}, Src, SuccVst) + end); %% Map instructions. valfun_4({put_map_assoc=Op,{f,Fail},Src,Dst,Live,{list,List}}, Vst) -> verify_put_map(Op, Fail, Src, Dst, Live, List, Vst); @@ -903,15 +1052,15 @@ verify_get_map(Fail, Src, List, Vst0) -> assert_not_literal(Src), %OTP 22. assert_type(map, Src, Vst0), - complex_test(Fail, - fun(FailVst) -> - clobber_map_vals(List, Src, FailVst) - end, - fun(SuccVst) -> - Keys = extract_map_keys(List), - assert_unique_map_keys(Keys), - extract_map_vals(List, Src, SuccVst, SuccVst) - end, Vst0). + branch(Fail, Vst0, + fun(FailVst) -> + clobber_map_vals(List, Src, FailVst) + end, + fun(SuccVst) -> + Keys = extract_map_keys(List), + assert_unique_map_keys(Keys), + extract_map_vals(List, Src, SuccVst, SuccVst) + end). %% get_map_elements may leave its destinations in an inconsistent state when %% the fail label is taken. Consider the following: @@ -945,13 +1094,16 @@ verify_put_map(Op, Fail, Src, Dst, Live, List, Vst0) -> assert_type(map, Src, Vst0), verify_live(Live, Vst0), verify_y_init(Vst0), - [assert_not_fragile(Term, Vst0) || Term <- List], - Vst1 = heap_alloc(0, Vst0), - Vst2 = branch_state(Fail, Vst1), - Vst = prune_x_regs(Live, Vst2), - Keys = extract_map_keys(List), - assert_unique_map_keys(Keys), - create_term(map, Op, [Src], Dst, Vst). + _ = [assert_term(Term, Vst0) || Term <- List], + Vst = heap_alloc(0, Vst0), + + branch(Fail, Vst, + fun(SuccVst0) -> + SuccVst = prune_x_regs(Live, SuccVst0), + Keys = extract_map_keys(List), + assert_unique_map_keys(Keys), + create_term(map, Op, [Src], Dst, SuccVst, SuccVst0) + end). %% %% Common code for validating bs_start_match* instructions. @@ -962,40 +1114,62 @@ validate_bs_start_match(Fail, Live, Type, Src, Dst, Vst) -> verify_y_init(Vst), %% #ms{} can represent either a match context or a term, so we have to mark - %% the source as a term if it fails, and retain the incoming type if it - %% succeeds (match context or not). - %% - %% The override_type hack is only needed until we get proper union types. - complex_test(Fail, - fun(FailVst) -> - override_type(term, Src, FailVst) - end, - fun(SuccVst0) -> - SuccVst = prune_x_regs(Live, SuccVst0), - extract_term(Type, bs_start_match, [Src], Dst, - SuccVst, Vst) - end, Vst). + %% the source as a term if it fails with a match context as an input. This + %% hack is only needed until we get proper union types. + branch(Fail, Vst, + fun(FailVst) -> + case get_movable_term_type(Src, FailVst) of + #ms{} -> override_type(term, Src, FailVst); + _ -> FailVst + end + end, + fun(SuccVst0) -> + SuccVst1 = update_type(fun meet/2, binary, + Src, SuccVst0), + SuccVst = prune_x_regs(Live, SuccVst1), + extract_term(Type, bs_start_match, [Src], Dst, + SuccVst, SuccVst0) + end). %% %% Common code for validating bs_get* instructions. %% -validate_bs_get(Op, Fail, Ctx, Live, Type, Dst, Vst0) -> - bsm_validate_context(Ctx, Vst0), - verify_live(Live, Vst0), - verify_y_init(Vst0), - Vst1 = prune_x_regs(Live, Vst0), - Vst = branch_state(Fail, Vst1), - extract_term(Type, Op, [Ctx], Dst, Vst). +validate_bs_get(Op, Fail, Ctx, Live, Type, Dst, Vst) -> + bsm_validate_context(Ctx, Vst), + verify_live(Live, Vst), + verify_y_init(Vst), + + branch(Fail, Vst, + fun(SuccVst0) -> + SuccVst = prune_x_regs(Live, SuccVst0), + extract_term(Type, Op, [Ctx], Dst, SuccVst, SuccVst0) + end). %% %% Common code for validating bs_skip_utf* instructions. %% -validate_bs_skip_utf(Fail, Ctx, Live, Vst0) -> - bsm_validate_context(Ctx, Vst0), - verify_y_init(Vst0), - verify_live(Live, Vst0), - Vst = prune_x_regs(Live, Vst0), - branch_state(Fail, Vst). +validate_bs_skip_utf(Fail, Ctx, Live, Vst) -> + bsm_validate_context(Ctx, Vst), + verify_y_init(Vst), + verify_live(Live, Vst), + + branch(Fail, Vst, + fun(SuccVst) -> + prune_x_regs(Live, SuccVst) + end). + +%% +%% Common code for is_$type instructions. +%% +type_test(Fail, Type, Reg, Vst) -> + assert_term(Reg, Vst), + branch(Fail, Vst, + fun(FailVst) -> + update_type(fun subtract/2, Type, Reg, FailVst) + end, + fun(SuccVst) -> + update_type(fun meet/2, Type, Reg, SuccVst) + end). %% %% Special state handling for setelement/3 and set_tuple_element/3 instructions. @@ -1032,7 +1206,7 @@ call(Name, Live, #vst{current=St0}=Vst0) -> case call_return_type(Name, Vst0) of Type when Type =/= exception -> %% Type is never 'exception' because it has been handled earlier. - St = St0#st{f=init_fregs(),aliases=#{}}, + St = St0#st{f=init_fregs()}, Vst = prune_x_regs(0, Vst0#vst{current=St}), create_term(Type, call, [], {x,0}, Vst) end. @@ -1059,14 +1233,15 @@ verify_call_args(_, Live, _) -> verify_remote_args_1(-1, _) -> ok; verify_remote_args_1(X, Vst) -> - assert_not_fragile({x, X}, Vst), + assert_durable_term({x, X}, Vst), verify_remote_args_1(X - 1, Vst). verify_local_args(-1, _Lbl, _CtxIds, _Vst) -> ok; verify_local_args(X, Lbl, CtxIds, Vst) -> Reg = {x, X}, - case get_raw_type(Reg, Vst) of + assert_not_fragile(Reg, Vst), + case get_movable_term_type(Reg, Vst) of #ms{id=Id}=Type -> case CtxIds of #{ Id := Other } -> @@ -1075,8 +1250,6 @@ verify_local_args(X, Lbl, CtxIds, Vst) -> verify_arg_type(Lbl, Reg, Type, Vst), verify_local_args(X - 1, Lbl, CtxIds#{ Id => Reg }, Vst) end; - {fragile,_} -> - error({fragile_message_reference, Reg}); Type -> verify_arg_type(Lbl, Reg, Type, Vst), verify_local_args(X - 1, Lbl, CtxIds, Vst) @@ -1092,56 +1265,89 @@ verify_arg_type(Lbl, Reg, #ms{}, #vst{ft=Ft}) -> end; verify_arg_type(Lbl, Reg, GivenType, #vst{ft=Ft}) -> case gb_trees:lookup({Lbl, Reg}, Ft) of - {value, bool} when GivenType =:= {atom, true}; - GivenType =:= {atom, false}; - GivenType =:= {atom, []} -> - %% We don't yet support upgrading true/false to bool, so we - %% assume unknown atoms can be bools when validating calls. - ok; {value, #ms{}} -> %% Functions that accept match contexts also accept all other %% terms. This will change once we support union types. ok; {value, RequiredType} -> - case meet(GivenType, RequiredType) of - none -> error({bad_arg_type, Reg, GivenType, RequiredType}); - _ -> ok + case vat_1(GivenType, RequiredType) of + true -> ok; + false -> error({bad_arg_type, Reg, GivenType, RequiredType}) end; none -> ok end. -allocate(Zero, Stk, Heap, Live, #vst{current=#st{numy=none}}=Vst0) -> +%% Checks whether the Given argument is compatible with the Required one. This +%% is essentially a relaxed version of 'meet(Given, Req) =:= Given', where we +%% accept that the Given value has the right type but not necessarily the exact +%% same value; if {atom,gurka} is required, we'll consider {atom,[]} valid. +%% +%% This will catch all problems that could crash the emulator, like passing a +%% 1-tuple when the callee expects a 3-tuple, but some value errors might slip +%% through. +vat_1(Same, Same) -> true; +vat_1({atom,A}, {atom,B}) -> A =:= B orelse is_list(A) orelse is_list(B); +vat_1({atom,A}, bool) -> is_boolean(A) orelse is_list(A); +vat_1(bool, {atom,B}) -> is_boolean(B) orelse is_list(B); +vat_1(cons, list) -> true; +vat_1({float,A}, {float,B}) -> A =:= B orelse is_list(A) orelse is_list(B); +vat_1({float,_}, number) -> true; +vat_1({integer,A}, {integer,B}) -> A =:= B orelse is_list(A) orelse is_list(B); +vat_1({integer,_}, number) -> true; +vat_1(_, {literal,_}) -> false; +vat_1({literal,_}=Lit, Required) -> vat_1(get_literal_type(Lit), Required); +vat_1(nil, list) -> true; +vat_1({tuple,SzA,EsA}, {tuple,SzB,EsB}) -> + if + is_list(SzB) -> + tuple_sz(SzA) >= tuple_sz(SzB) andalso vat_elements(EsA, EsB); + SzA =:= SzB -> + vat_elements(EsA, EsB); + SzA =/= SzB -> + false + end; +vat_1(_, _) -> false. + +vat_elements(EsA, EsB) -> + maps:fold(fun(Key, Req, Acc) -> + case EsA of + #{ Key := Given } -> Acc andalso vat_1(Given, Req); + #{} -> false + end + end, true, EsB). + +allocate(Tag, Stk, Heap, Live, #vst{current=#st{numy=none}=St}=Vst0) -> verify_live(Live, Vst0), - Vst = #vst{current=St} = prune_x_regs(Live, Vst0), - Ys = init_regs(Stk, case Zero of - true -> initialized; - false -> uninitialized - end), - heap_alloc(Heap, Vst#vst{current=St#st{y=Ys,numy=Stk}}); + Vst1 = Vst0#vst{current=St#st{numy=Stk}}, + Vst2 = prune_x_regs(Live, Vst1), + Vst = init_stack(Tag, Stk - 1, Vst2), + heap_alloc(Heap, Vst); allocate(_, _, _, _, #vst{current=#st{numy=Numy}}) -> error({existing_stack_frame,{size,Numy}}). deallocate(#vst{current=St}=Vst) -> - Vst#vst{current=St#st{y=init_regs(0, initialized),numy=none}}. - -trim_stack(From, To, Top, #st{y=Ys0}=St) when From =:= Top -> - Ys = foldl(fun(Y, Acc) -> - gb_trees:delete(Y, Acc) - end, Ys0, seq(To, From - 1)), - %% Note that all aliases and defs are wiped. This is perhaps a bit too - %% conservative, but preserving them won't be easy until type management - %% is refactored. - St#st{aliases=#{},defs=#{},numy=To,y=Ys}; + Vst#vst{current=St#st{ys=#{},numy=none}}. + +init_stack(_Tag, -1, Vst) -> + Vst; +init_stack(Tag, Y, Vst) -> + init_stack(Tag, Y - 1, create_tag(Tag, allocate, [], {y,Y}, Vst)). + +trim_stack(From, To, Top, #st{ys=Ys0}=St) when From =:= Top -> + Ys = maps:filter(fun({y,Y}, _) -> Y < To end, Ys0), + St#st{numy=To,ys=Ys}; trim_stack(From, To, Top, St0) -> - #st{y=Ys0} = St0, + Src = {y, From}, + Dst = {y, To}, - Ys = case gb_trees:lookup(From, Ys0) of - none -> error({invalid_shift,{y,From},{y,To}}); - {value,Type} -> gb_trees:enter(To, Type, Ys0) + #st{ys=Ys0} = St0, + Ys = case Ys0 of + #{ Src := Ref } -> Ys0#{ Dst => Ref }; + #{} -> error({invalid_shift,Src,Dst}) end, + St = St0#st{ys=Ys}, - St = St0#st{y=Ys}, trim_stack(From + 1, To + 1, Top, St). test_heap(Heap, Live, Vst0) -> @@ -1168,24 +1374,17 @@ heap_alloc_2([{floats,Floats}|T], St0) -> heap_alloc_2(T, St); heap_alloc_2([], St) -> St. -prune_x_regs(Live, #vst{current=St0}=Vst) - when is_integer(Live) -> - #st{x=Xs0,defs=Defs0,aliases=Aliases0} = St0, - Xs1 = gb_trees:to_list(Xs0), - Xs = [P || {R,_}=P <- Xs1, R < Live], - Defs = maps:filter(fun({x,X}, _) -> X < Live; - ({y,_}, _) -> true - end, Defs0), - Aliases = maps:filter(fun({x,X1}, {x,X2}) -> - X1 < Live andalso X2 < Live; - ({x,X}, _) -> - X < Live; - (_, {x,X}) -> - X < Live; - (_, _) -> - true - end, Aliases0), - St = St0#st{x=gb_trees:from_orddict(Xs),defs=Defs,aliases=Aliases}, +prune_x_regs(Live, #vst{current=St0}=Vst) when is_integer(Live) -> + #st{fragile=Fragile0,xs=Xs0} = St0, + Fragile = cerl_sets:filter(fun({x,X}) -> + X < Live; + ({y,_}) -> + true + end, Fragile0), + Xs = maps:filter(fun({x,X}, _) -> + X < Live + end, Xs0), + St = St0#st{fragile=Fragile,xs=Xs}, Vst#vst{current=St}. %% All choices in a select_val list must be integers, floats, or atoms. @@ -1231,7 +1430,7 @@ assert_arities(_) -> error(bad_tuple_arity_list). %%% float_op(Ss, Dst, Vst0) -> - [assert_freg_set(S, Vst0) || S <- Ss], + _ = [assert_freg_set(S, Vst0) || S <- Ss], assert_fls(cleared, Vst0), Vst = set_fls(cleared, Vst0), set_freg(Dst, Vst). @@ -1249,8 +1448,7 @@ get_fls(#vst{current=#st{fls=Fls}}) when is_atom(Fls) -> Fls. init_fregs() -> 0. -set_freg({fr,Fr}=Freg, #vst{current=#st{f=Fregs0}=St}=Vst) - when is_integer(Fr), 0 =< Fr -> +set_freg({fr,Fr}=Freg, #vst{current=#st{f=Fregs0}=St}=Vst) -> check_limit(Freg), Bit = 1 bsl Fr, if @@ -1308,19 +1506,13 @@ bsm_validate_context(Reg, Vst) -> _ = bsm_get_context(Reg, Vst), ok. -bsm_get_context({x,X}=Reg, #vst{current=#st{x=Xs}}=_Vst) when is_integer(X) -> - case gb_trees:lookup(X, Xs) of - {value,#ms{}=Ctx} -> Ctx; - {value,{fragile,#ms{}=Ctx}} -> Ctx; - _ -> error({no_bsm_context,Reg}) - end; -bsm_get_context({y,Y}=Reg, #vst{current=#st{y=Ys}}=_Vst) when is_integer(Y) -> - case gb_trees:lookup(Y, Ys) of - {value,#ms{}=Ctx} -> Ctx; - {value,{fragile,#ms{}=Ctx}} -> Ctx; - _ -> error({no_bsm_context,Reg}) +bsm_get_context({Kind,_}=Reg, Vst) when Kind =:= x; Kind =:= y-> + case get_movable_term_type(Reg, Vst) of + #ms{}=Ctx -> Ctx; + _ -> error({no_bsm_context,Reg}) end; -bsm_get_context(Reg, _) -> error({bad_source,Reg}). +bsm_get_context(Reg, _) -> + error({bad_source,Reg}). bsm_save(Reg, {atom,start}, Vst) -> %% Save point refering to where the match started. @@ -1350,81 +1542,98 @@ bsm_restore(Reg, SavePoint, Vst) -> _ -> error({illegal_restore,SavePoint,range}) end. -select_val_branches(Fail, Src, Choices, Vst0) -> - Vst = svb_1(Choices, Src, Vst0), - kill_state(branch_state(Fail, Vst)). - -svb_1([Val,{f,L}|T], Src, Vst0) -> - Vst = complex_test(L, - fun(BranchVst) -> - update_eq_types(Val, Src, BranchVst) - end, - fun(FailVst) -> - update_ne_types(Val, Src, FailVst) - end, Vst0), - svb_1(T, Src, Vst); -svb_1([], _, Vst) -> - Vst. - -select_arity_branches(Fail, List, Tuple, Vst0) -> - Type = get_durable_term_type(Tuple, Vst0), - Vst = sab_1(List, Tuple, Type, Vst0), - kill_state(branch_state(Fail, Vst)). - -sab_1([Sz,{f,L}|T], Tuple, {tuple,[_],Es}=Type0, Vst0) -> - #vst{current=St0} = Vst0, - Vst1 = update_type(fun meet/2, {tuple,Sz,Es}, Tuple, Vst0), - Vst2 = branch_state(L, Vst1), - Vst = Vst2#vst{current=St0}, - - sab_1(T, Tuple, Type0, Vst); -sab_1([Sz,{f,L}|T], Tuple, {tuple,Sz,_Es}=Type, Vst0) -> - %% The type is already correct. (This test is redundant.) - Vst = branch_state(L, Vst0), - sab_1(T, Tuple, Type, Vst); -sab_1([_,{f,_}|T], Tuple, Type, Vst) -> - %% We already have an established different exact size for the tuple. - %% This label can't possibly be reached. - sab_1(T, Tuple, Type, Vst); -sab_1([], _, _, #vst{}=Vst) -> - Vst. +validate_select_val(_Fail, _Choices, _Src, #vst{current=none}=Vst) -> + %% We've already branched on all of Src's possible values, so we know we + %% can't reach the fail label or any of the remaining choices. + Vst; +validate_select_val(Fail, [Val,{f,L}|T], Src, Vst0) -> + Vst = branch(L, Vst0, + fun(BranchVst) -> + update_eq_types(Src, Val, BranchVst) + end, + fun(FailVst) -> + update_ne_types(Src, Val, FailVst) + end), + validate_select_val(Fail, T, Src, Vst); +validate_select_val(Fail, [], _, Vst) -> + branch(Fail, Vst, + fun(SuccVst) -> + %% The next instruction is never executed. + kill_state(SuccVst) + end). + +validate_select_tuple_arity(_Fail, _Choices, _Src, #vst{current=none}=Vst) -> + %% We've already branched on all of Src's possible values, so we know we + %% can't reach the fail label or any of the remaining choices. + Vst; +validate_select_tuple_arity(Fail, [Arity,{f,L}|T], Tuple, Vst0) -> + Type = {tuple, Arity, #{}}, + Vst = branch(L, Vst0, + fun(BranchVst) -> + update_type(fun meet/2, Type, Tuple, BranchVst) + end, + fun(FailVst) -> + update_type(fun subtract/2, Type, Tuple, FailVst) + end), + validate_select_tuple_arity(Fail, T, Tuple, Vst); +validate_select_tuple_arity(Fail, [], _, #vst{}=Vst) -> + branch(Fail, Vst, + fun(SuccVst) -> + %% The next instruction is never executed. + kill_state(SuccVst) + end). + +infer_types({Kind,_}=Reg, Vst) when Kind =:= x; Kind =:= y -> + infer_types(get_reg_vref(Reg, Vst), Vst); +infer_types(#value_ref{}=Ref, #vst{current=#st{vs=Vs}}) -> + case Vs of + #{ Ref := Entry } -> infer_types_1(Entry); + #{} -> fun(_, S) -> S end + end; +infer_types(_, #vst{}) -> + fun(_, S) -> S end. -infer_types(Src, Vst) -> - case get_def(Src, Vst) of - {{bif,tuple_size}, [Tuple]} -> - fun({integer,Arity}, S) -> - update_type(fun meet/2, {tuple,Arity,#{}}, Tuple, S); - (_, S) -> S - end; - {{bif,'=:='},[ArityReg,{integer,_}=Val]} when ArityReg =/= Src -> - fun({atom,true}, S) -> - Infer = infer_types(ArityReg, S), - Infer(Val, S); - (_, S) -> S - end; - {{bif,is_atom},[Src]} -> - infer_type_test_bif({atom,[]}, Src); - {{bif,is_boolean},[Src]} -> - infer_type_test_bif(bool, Src); - {{bif,is_binary},[Src]} -> - infer_type_test_bif(binary, Src); - {{bif,is_bitstring},[Src]} -> - infer_type_test_bif(binary, Src); - {{bif,is_float},[Src]} -> - infer_type_test_bif(float, Src); - {{bif,is_integer},[Src]} -> - infer_type_test_bif({integer,{}}, Src); - {{bif,is_list},[Src]} -> - infer_type_test_bif(list, Src); - {{bif,is_map},[Src]} -> - infer_type_test_bif(map, Src); - {{bif,is_number},[Src]} -> - infer_type_test_bif(number, Src); - {{bif,is_tuple},[Src]} -> - infer_type_test_bif({tuple,[0],#{}}, Src); - _ -> - fun(_, S) -> S end - end. +infer_types_1(#value{op={bif,'=:='},args=[LHS,RHS]}) -> + fun({atom,true}, S) -> + %% Either side might contain something worth inferring, so we need + %% to check them both. + Infer_L = infer_types(RHS, S), + Infer_R = infer_types(LHS, S), + Infer_R(RHS, Infer_L(LHS, S)); + (_, S) -> S + end; +infer_types_1(#value{op={bif,element},args=[{integer,Index}=Key,Tuple]}) -> + fun(Val, S) -> + Type = get_term_type(Val, S), + update_type(fun meet/2,{tuple,[Index],#{ Key => Type }}, Tuple, S) + end; +infer_types_1(#value{op={bif,is_atom},args=[Src]}) -> + infer_type_test_bif({atom,[]}, Src); +infer_types_1(#value{op={bif,is_boolean},args=[Src]}) -> + infer_type_test_bif(bool, Src); +infer_types_1(#value{op={bif,is_binary},args=[Src]}) -> + infer_type_test_bif(binary, Src); +infer_types_1(#value{op={bif,is_bitstring},args=[Src]}) -> + infer_type_test_bif(binary, Src); +infer_types_1(#value{op={bif,is_float},args=[Src]}) -> + infer_type_test_bif(float, Src); +infer_types_1(#value{op={bif,is_integer},args=[Src]}) -> + infer_type_test_bif({integer,{}}, Src); +infer_types_1(#value{op={bif,is_list},args=[Src]}) -> + infer_type_test_bif(list, Src); +infer_types_1(#value{op={bif,is_map},args=[Src]}) -> + infer_type_test_bif(map, Src); +infer_types_1(#value{op={bif,is_number},args=[Src]}) -> + infer_type_test_bif(number, Src); +infer_types_1(#value{op={bif,is_tuple},args=[Src]}) -> + infer_type_test_bif({tuple,[0],#{}}, Src); +infer_types_1(#value{op={bif,tuple_size}, args=[Tuple]}) -> + fun({integer,Arity}, S) -> + update_type(fun meet/2, {tuple,Arity,#{}}, Tuple, S); + (_, S) -> S + end; +infer_types_1(_) -> + fun(_, S) -> S end. infer_type_test_bif(Type, Src) -> fun({atom,true}, S) -> @@ -1445,69 +1654,76 @@ assign({y,_}=Src, {y,_}=Dst, Vst) -> initialized -> create_tag(initialized, init, [], Dst, Vst); _ -> assign_1(Src, Dst, Vst) end; -assign({Kind,_}=Reg, Dst, Vst) when Kind =:= x; Kind =:= y -> - assign_1(Reg, Dst, Vst); +assign({Kind,_}=Src, Dst, Vst) when Kind =:= x; Kind =:= y -> + assign_1(Src, Dst, Vst); assign(Literal, Dst, Vst) -> - Type = get_term_type(Literal, Vst), + Type = get_literal_type(Literal), create_term(Type, move, [Literal], Dst, Vst). -%% Creates a special tag value that isn't a regular term, such as -%% 'initialized' or 'catchtag' -create_tag(Type, Op, Ss, {y,_}=Dst, Vst) -> - set_type_reg_expr(Type, {Op, Ss}, Dst, Vst); -create_tag(_Type, _Op, _Ss, Dst, _Vst) -> +%% Creates a special tag value that isn't a regular term, such as 'initialized' +%% or 'catchtag' +create_tag(Tag, _Op, _Ss, {y,_}=Dst, #vst{current=#st{ys=Ys0}=St0}=Vst) -> + case maps:get(Dst, Ys0, uninitialized) of + {catchtag,_}=Prev -> + error(Prev); + {trytag,_}=Prev -> + error(Prev); + _ -> + check_try_catch_tags(Tag, Dst, Vst), + Ys = Ys0#{ Dst => Tag }, + St = St0#st{ys=Ys}, + remove_fragility(Dst, Vst#vst{current=St}) + end; +create_tag(_Tag, _Op, _Ss, Dst, _Vst) -> error({invalid_tag_register, Dst}). %% Wipes a special tag, leaving the register initialized but empty. -kill_tag({y,Y}=Reg, #vst{current=#st{y=Ys0}=St0}=Vst) -> +kill_tag({y,_}=Reg, #vst{current=#st{ys=Ys0}=St0}=Vst) -> _ = get_tag_type(Reg, Vst), %Assertion. - Ys = gb_trees:update(Y, initialized, Ys0), - Vst#vst{current=St0#st{y=Ys}}. + Ys = Ys0#{ Reg => initialized }, + Vst#vst{current=St0#st{ys=Ys}}. %% Creates a completely new term with the given type. -create_term(Type, Op, Ss, Dst, Vst) -> - set_type_reg_expr(Type, {Op, Ss}, Dst, Vst). +create_term(Type, Op, Ss0, Dst, Vst0) -> + create_term(Type, Op, Ss0, Dst, Vst0, Vst0). -%% Extracts a term from Ss, propagating fragility. -extract_term(Type, Op, Ss, Dst, Vst) -> - extract_term(Type, Op, Ss, Dst, Vst, Vst). - -%% As extract_term/4, but uses the incoming Vst for fragility in case x-regs -%% have been pruned and the sources can no longer be found. -extract_term(Type0, Op, Ss, Dst, Vst, OrigVst) -> - Type = propagate_fragility(Type0, Ss, OrigVst), - set_type_reg_expr(Type, {Op, Ss}, Dst, Vst). - -%% Helper functions for tests that alter state on both the success and fail -%% branches, keeping the states from tainting each other. -complex_test(Fail, FailFun, SuccFun, Vst0) -> - #vst{current=St0} = Vst0, - Vst1 = FailFun(Vst0), - Vst2 = branch_state(Fail, Vst1), - Vst = Vst2#vst{current=St0}, - SuccFun(Vst). +%% As create_term/4, but uses the incoming Vst for argument resolution in +%% case x-regs have been pruned and the sources can no longer be found. +create_term(Type, Op, Ss0, Dst, Vst0, OrigVst) -> + {Ref, Vst1} = new_value(Type, Op, resolve_args(Ss0, OrigVst), Vst0), + Vst = remove_fragility(Dst, Vst1), + set_reg_vref(Ref, Dst, Vst). -%% Helper function for simple "is_type" tests. -type_test(Fail, Type, Reg, Vst) -> - assert_term(Reg, Vst), - complex_test(Fail, - fun(FailVst) -> - update_type(fun subtract/2, Type, Reg, FailVst) - end, - fun(SuccVst) -> - update_type(fun meet/2, Type, Reg, SuccVst) - end, Vst). +%% Extracts a term from Ss, propagating fragility. +extract_term(Type, Op, Ss0, Dst, Vst0) -> + extract_term(Type, Op, Ss0, Dst, Vst0, Vst0). + +%% As extract_term/4, but uses the incoming Vst for argument resolution in +%% case x-regs have been pruned and the sources can no longer be found. +extract_term(Type, Op, Ss0, Dst, Vst0, OrigVst) -> + {Ref, Vst1} = new_value(Type, Op, resolve_args(Ss0, OrigVst), Vst0), + Vst = propagate_fragility(Dst, Ss0, Vst1), + set_reg_vref(Ref, Dst, Vst). + +%% Translates instruction arguments into the argument() type, decoupling them +%% from their registers, allowing us to infer their types after they've been +%% clobbered or moved. +resolve_args([{Kind,_}=Src | Args], Vst) when Kind =:= x; Kind =:= y -> + [get_reg_vref(Src, Vst) | resolve_args(Args, Vst)]; +resolve_args([Lit | Args], Vst) -> + assert_literal(Lit), + [Lit | resolve_args(Args, Vst)]; +resolve_args([], _Vst) -> + []. %% Overrides the type of Reg. This is ugly but a necessity for certain %% destructive operations. override_type(Type, Reg, Vst) -> - %% Once the new type format is in, this should be expressed as: - %% update_type(fun(_, T) -> T end, Type, Reg, Vst). - set_aliased_type(Type, Reg, Vst). + update_type(fun(_, T) -> T end, Type, Reg, Vst). %% This is used when linear code finds out more and more information about a %% type, so that the type gets more specialized. -update_type(Merge, Type0, Reg, Vst) -> +update_type(Merge, With, #value_ref{}=Ref, Vst) -> %% If the old type can't be merged with the new one, the type information %% is inconsistent and we know that some instructions will never be %% executed at run-time. For example: @@ -1516,23 +1732,57 @@ update_type(Merge, Type0, Reg, Vst) -> %% {test,is_tuple,Fail,[Reg]}. %% {test,test_arity,Fail,[Reg,5]}. %% - %% Note that the test_arity instruction can never be reached, so we use the - %% new type instead of 'none'. - Type = case Merge(get_durable_term_type(Reg, Vst), Type0) of - none -> Type0; - T -> T - end, - set_aliased_type(Type, Reg, Vst). + %% Note that the test_arity instruction can never be reached, so we need to + %% kill the state to avoid raising an error when we encounter it. + %% + %% Simply returning `kill_state(Vst)` is unsafe however as we might be in + %% the middle of an instruction, and altering the rest of the validator + %% (eg. prune_x_regs/2) to no-op on dead states is prone to error. + %% + %% We therefore throw a 'type_conflict' error instead, which causes + %% validation to fail unless we're in a context where such errors can be + %% handled, such as in a branch handler. + Current = get_raw_type(Ref, Vst), + case Merge(Current, With) of + none -> throw({type_conflict, Current, With}); + Type -> set_type(Type, Ref, Vst) + end; +update_type(Merge, With, {Kind,_}=Reg, Vst) when Kind =:= x; Kind =:= y -> + update_type(Merge, With, get_reg_vref(Reg, Vst), Vst); +update_type(Merge, With, Literal, Vst) -> + assert_literal(Literal), + %% Literals always retain their type, but we still need to bail on type + %% conflicts. + case Merge(Literal, With) of + none -> throw({type_conflict, Literal, With}); + _Type -> Vst + end. update_ne_types(LHS, RHS, Vst) -> - update_type(fun subtract/2, get_durable_term_type(RHS, Vst), LHS, Vst). + %% While updating types on equality is fairly straightforward, inequality + %% is a bit trickier since all we know is that the *value* of LHS differs + %% from RHS, so we can't blindly subtract their types. + %% + %% Consider `number =/= {integer,[]}`; all we know is that LHS isn't equal + %% to some *specific integer* of unknown value, and if we were to subtract + %% {integer,[]} we would erroneously infer that the new type is {float,[]}. + %% + %% Therefore, we only subtract when we know that RHS has a specific value. + RType = get_term_type(RHS, Vst), + case is_literal(RType) of + true -> update_type(fun subtract/2, RType, LHS, Vst); + false -> Vst + end. update_eq_types(LHS, RHS, Vst0) -> - Infer = infer_types(LHS, Vst0), - Vst1 = Infer(RHS, Vst0), + %% Either side might contain something worth inferring, so we need + %% to check them both. + Infer_L = infer_types(RHS, Vst0), + Infer_R = infer_types(LHS, Vst0), + Vst1 = Infer_R(RHS, Infer_L(LHS, Vst0)), - T1 = get_durable_term_type(LHS, Vst1), - T2 = get_durable_term_type(RHS, Vst1), + T1 = get_term_type(LHS, Vst1), + T2 = get_term_type(RHS, Vst1), Vst = update_type(fun meet/2, T2, LHS, Vst1), update_type(fun meet/2, T1, RHS, Vst). @@ -1540,102 +1790,69 @@ update_eq_types(LHS, RHS, Vst0) -> %% Helper functions for the above. assign_1(Src, Dst, Vst0) -> - Type = get_move_term_type(Src, Vst0), - Def = get_def(Src, Vst0), - - Vst = set_type_reg_expr(Type, Def, Dst, Vst0), - - #vst{current=St0} = Vst, - #st{aliases=Aliases0} = St0, - - Aliases = Aliases0#{Src=>Dst,Dst=>Src}, - - St = St0#st{aliases=Aliases}, - Vst#vst{current=St}. - -set_aliased_type(Type, Reg, #vst{current=#st{aliases=Aliases}}=Vst0) -> - Vst1 = set_type(Type, Reg, Vst0), - case Aliases of - #{Reg:=OtherReg} -> - Vst = set_type_reg(Type, OtherReg, Vst1), - #vst{current=St} = Vst, - Vst#vst{current=St#st{aliases=Aliases}}; + assert_movable(Src, Vst0), + Vst = propagate_fragility(Dst, [Src], Vst0), + set_reg_vref(get_reg_vref(Src, Vst), Dst, Vst). + +set_reg_vref(Ref, {x,_}=Dst, Vst) -> + check_limit(Dst), + #vst{current=#st{xs=Xs0}=St0} = Vst, + St = St0#st{xs=Xs0#{ Dst => Ref }}, + Vst#vst{current=St}; +set_reg_vref(Ref, {y,_}=Dst, #vst{current=#st{ys=Ys0}=St0} = Vst) -> + check_limit(Dst), + case Ys0 of + #{ Dst := {catchtag,_}=Tag } -> + error(Tag); + #{ Dst := {trytag,_}=Tag } -> + error(Tag); + #{ Dst := _ } -> + St = St0#st{ys=Ys0#{ Dst => Ref }}, + Vst#vst{current=St}; #{} -> - Vst1 + %% Storing into a non-existent Y register means that we haven't set + %% up a (sufficiently large) stack. + error({invalid_store, Dst}) end. -kill_aliases(Reg, #st{aliases=Aliases0}=St) -> - case Aliases0 of - #{Reg:=OtherReg} -> - Aliases = maps:without([Reg,OtherReg], Aliases0), - St#st{aliases=Aliases}; +get_reg_vref({x,_}=Src, #vst{current=#st{xs=Xs}}) -> + check_limit(Src), + case Xs of + #{ Src := #value_ref{}=Ref } -> + Ref; #{} -> - St + error({uninitialized_reg, Src}) + end; +get_reg_vref({y,_}=Src, #vst{current=#st{ys=Ys}}) -> + check_limit(Src), + case Ys of + #{ Src := #value_ref{}=Ref } -> + Ref; + #{ Src := initialized } -> + error({unassigned, Src}); + #{ Src := Tag } when Tag =/= uninitialized -> + error(Tag); + #{} -> + error({uninitialized_reg, Src}) end. -set_type(Type, {x,_}=Reg, Vst) -> - set_type_reg(Type, Reg, Reg, Vst); -set_type(Type, {y,_}=Reg, Vst) -> - set_type_reg(Type, Reg, Reg, Vst); -set_type(_, _, #vst{}=Vst) -> Vst. - -set_type_reg(Type, Src, Dst, Vst) -> - case get_raw_type(Src, Vst) of - uninitialized -> - error({uninitialized_reg, Src}); - {fragile,_} -> - set_type_reg(make_fragile(Type), Dst, Vst); - _ -> - set_type_reg(Type, Dst, Vst) +set_type(Type, #value_ref{}=Ref, #vst{current=#st{vs=Vs0}=St}=Vst) -> + case Vs0 of + #{ Ref := #value{}=Entry } -> + Vs = Vs0#{ Ref => Entry#value{type=Type} }, + Vst#vst{current=St#st{vs=Vs}}; + #{} -> + %% Dead references may happen during type inference and are not an + %% error in and of themselves. If a problem were to arise from this + %% it'll explode elsewhere. + Vst end. -set_type_reg(Type, Reg, Vst) -> - set_type_reg_expr(Type, none, Reg, Vst). - -set_type_reg_expr(Type, Expr, {x,_}=Reg, Vst) -> - set_type_x(Type, Expr, Reg, Vst); -set_type_reg_expr(Type, Expr, Reg, Vst) -> - set_type_y(Type, Expr, Reg, Vst). - -set_type_x(Type, Expr, {x,X}=Reg, #vst{current=#st{x=Xs0,defs=Defs0}=St0}=Vst) - when is_integer(X), 0 =< X -> - check_limit(Reg), - Xs = case gb_trees:lookup(X, Xs0) of - none -> - gb_trees:insert(X, Type, Xs0); - {value,{fragile,_}} -> - gb_trees:update(X, make_fragile(Type), Xs0); - {value,_} -> - gb_trees:update(X, Type, Xs0) - end, - Defs = Defs0#{Reg=>Expr}, - St = kill_aliases(Reg, St0), - Vst#vst{current=St#st{x=Xs,defs=Defs}}; -set_type_x(Type, _Expr, Reg, #vst{}) -> - error({invalid_store,Reg,Type}). - -set_type_y(Type, Expr, {y,Y}=Reg, #vst{current=#st{y=Ys0,defs=Defs0}=St0}=Vst) - when is_integer(Y), 0 =< Y -> - check_limit(Reg), - Ys = case gb_trees:lookup(Y, Ys0) of - none -> - error({invalid_store,Reg,Type}); - {value,{catchtag,_}=Tag} -> - error(Tag); - {value,{trytag,_}=Tag} -> - error(Tag); - {value,_} -> - gb_trees:update(Y, Type, Ys0) - end, - check_try_catch_tags(Type, Reg, Vst), - Defs = Defs0#{Reg=>Expr}, - St = kill_aliases(Reg, St0), - Vst#vst{current=St#st{y=Ys,defs=Defs}}; -set_type_y(Type, _Expr, Reg, #vst{}) -> - error({invalid_store,Reg,Type}). - -make_fragile({fragile,_}=Type) -> Type; -make_fragile(Type) -> {fragile,Type}. +new_value(Type, Op, Ss, #vst{current=#st{vs=Vs0}=St,ref_ctr=Counter}=Vst) -> + Ref = #value_ref{id=Counter}, + Vs = Vs0#{ Ref => #value{op=Op,args=Ss,type=Type} }, + + {Ref, Vst#vst{current=St#st{vs=Vs},ref_ctr=Counter+1}}. kill_catch_tag(Reg, #vst{current=#st{ct=[Fail|Fails]}=St}=Vst0) -> Vst = Vst0#vst{current=St#st{ct=Fails,fls=undefined}}, @@ -1657,36 +1874,36 @@ check_try_catch_tags(Type, {y,N}=Reg, Vst) -> ok end. -is_reg_defined({x,_}=Reg, Vst) -> is_type_defined_x(Reg, Vst); -is_reg_defined({y,_}=Reg, Vst) -> is_type_defined_y(Reg, Vst); +is_reg_defined({x,_}=Reg, #vst{current=#st{xs=Xs}}) -> is_map_key(Reg, Xs); +is_reg_defined({y,_}=Reg, #vst{current=#st{ys=Ys}}) -> is_map_key(Reg, Ys); is_reg_defined(V, #vst{}) -> error({not_a_register, V}). -is_type_defined_x({x,X}, #vst{current=#st{x=Xs}}) -> - gb_trees:is_defined(X,Xs). - -is_type_defined_y({y,Y}, #vst{current=#st{y=Ys}}) -> - gb_trees:is_defined(Y,Ys). - assert_term(Src, Vst) -> - get_term_type(Src, Vst), + _ = get_term_type(Src, Vst), ok. -assert_not_fragile(Src, Vst) -> - case get_term_type(Src, Vst) of - {fragile, _} -> error({fragile_message_reference, Src}); - _ -> ok +assert_movable(Src, Vst) -> + _ = get_movable_term_type(Src, Vst), + ok. + +assert_literal(Src) -> + case is_literal(Src) of + true -> ok; + false -> error({literal_required,Src}) end. -assert_literal(nil) -> ok; -assert_literal({atom,A}) when is_atom(A) -> ok; -assert_literal({float,F}) when is_float(F) -> ok; -assert_literal({integer,I}) when is_integer(I) -> ok; -assert_literal({literal,_L}) -> ok; -assert_literal(T) -> error({literal_required,T}). +assert_not_literal(Src) -> + case is_literal(Src) of + true -> error({literal_not_allowed,Src}); + false -> ok + end. -assert_not_literal({x,_}) -> ok; -assert_not_literal({y,_}) -> ok; -assert_not_literal(Literal) -> error({literal_not_allowed,Literal}). +is_literal(nil) -> true; +is_literal({atom,A}) when is_atom(A) -> true; +is_literal({float,F}) when is_float(F) -> true; +is_literal({integer,I}) when is_integer(I) -> true; +is_literal({literal,_L}) -> true; +is_literal(_) -> false. %% The possible types. %% @@ -1753,16 +1970,106 @@ assert_not_literal(Literal) -> error({literal_not_allowed,Literal}). %% %% none A conflict in types. There will be an exception at runtime. %% -%% FRAGILITY -%% --------- -%% -%% The loop_rec/2 instruction may return a reference to a term that is -%% not part of the root set. That term or any part of it must not be -%% included in a garbage collection. Therefore, the term (or any part -%% of it) must not be stored in an Y register. -%% -%% Such terms are wrapped in a {fragile,Type} tuple, where Type is one -%% of the types described above. + +%% join(Type1, Type2) -> Type +%% Return the most specific type possible. +join(Same, Same) -> + Same; +join(none, Other) -> + Other; +join(Other, none) -> + Other; +join({literal,_}=T1, T2) -> + join_literal(T1, T2); +join(T1, {literal,_}=T2) -> + join_literal(T2, T1); +join({tuple,Size,EsA}, {tuple,Size,EsB}) -> + Es = join_tuple_elements(tuple_sz(Size), EsA, EsB), + {tuple, Size, Es}; +join({tuple,A,EsA}, {tuple,B,EsB}) -> + Size = min(tuple_sz(A), tuple_sz(B)), + Es = join_tuple_elements(Size, EsA, EsB), + {tuple, [Size], Es}; +join({Type,A}, {Type,B}) + when Type =:= atom; Type =:= integer; Type =:= float -> + if A =:= B -> {Type,A}; + true -> {Type,[]} + end; +join({Type,_}, number) + when Type =:= integer; Type =:= float -> + number; +join(number, {Type,_}) + when Type =:= integer; Type =:= float -> + number; +join({integer,_}, {float,_}) -> + number; +join({float,_}, {integer,_}) -> + number; +join(bool, {atom,A}) -> + join_bool(A); +join({atom,A}, bool) -> + join_bool(A); +join({atom,A}, {atom,B}) when is_boolean(A), is_boolean(B) -> + bool; +join({atom,_}, {atom,_}) -> + {atom,[]}; +join(#ms{id=Id1,valid=B1,slots=Slots1}, + #ms{id=Id2,valid=B2,slots=Slots2}) -> + Id = if + Id1 =:= Id2 -> Id1; + true -> make_ref() + end, + #ms{id=Id,valid=B1 band B2,slots=min(Slots1, Slots2)}; +join(T1, T2) when T1 =/= T2 -> + %% We've exhaused all other options, so the type must either be a list or + %% a 'term'. + join_list(T1, T2). + +join_tuple_elements(Limit, EsA, EsB) -> + Es0 = join_elements(EsA, EsB), + maps:filter(fun({integer,Index}, _Type) -> Index =< Limit end, Es0). + +join_elements(Es1, Es2) -> + Keys = if + map_size(Es1) =< map_size(Es2) -> maps:keys(Es1); + map_size(Es1) > map_size(Es2) -> maps:keys(Es2) + end, + join_elements_1(Keys, Es1, Es2, #{}). + +join_elements_1([Key | Keys], Es1, Es2, Acc0) -> + Type = case {Es1, Es2} of + {#{ Key := Same }, #{ Key := Same }} -> Same; + {#{ Key := Type1 }, #{ Key := Type2 }} -> join(Type1, Type2); + {#{}, #{}} -> term + end, + Acc = set_element_type(Key, Type, Acc0), + join_elements_1(Keys, Es1, Es2, Acc); +join_elements_1([], _Es1, _Es2, Acc) -> + Acc. + +%% Joins types of literals; note that the left argument must either be a +%% literal or exactly equal to the second argument. +join_literal(Same, Same) -> + Same; +join_literal({literal,_}=Lit, T) -> + join_literal(T, get_literal_type(Lit)); +join_literal(T1, T2) -> + %% We're done extracting the types, try merging them again. + join(T1, T2). + +join_list(nil, cons) -> list; +join_list(nil, list) -> list; +join_list(cons, list) -> list; +join_list(T, nil) -> join_list(nil, T); +join_list(T, cons) -> join_list(cons, T); +join_list(_, _) -> + %% Not a list, so it must be a term. + term. + +join_bool([]) -> {atom,[]}; +join_bool(true) -> bool; +join_bool(false) -> bool; +join_bool(_) -> {atom,[]}. %% meet(Type1, Type2) -> Type %% Return the meet of two types. The meet is a more specific type. @@ -1770,14 +2077,23 @@ assert_not_literal(Literal) -> error({literal_not_allowed,Literal}). meet(Same, Same) -> Same; -meet({literal,_}=T1, T2) -> - meet_literal(T1, T2); -meet(T1, {literal,_}=T2) -> - meet_literal(T2, T1); meet(term, Other) -> Other; meet(Other, term) -> Other; +meet(#ms{}, binary) -> + #ms{}; +meet(binary, #ms{}) -> + #ms{}; +meet({literal,_}, {literal,_}) -> + none; +meet(T1, {literal,_}=T2) -> + meet(T2, T1); +meet({literal,_}=T1, T2) -> + case meet(get_literal_type(T1), T2) of + none -> none; + _ -> T1 + end; meet(T1, T2) -> case {erlang:min(T1, T2),erlang:max(T1, T2)} of {{atom,_}=A,{atom,[]}} -> A; @@ -1810,13 +2126,6 @@ meet(T1, T2) -> {_,_} -> none end. -%% Meets types of literals. -meet_literal({literal,_}=Lit, T) -> - meet_literal(T, get_literal_type(Lit)); -meet_literal(T1, T2) -> - %% We're done extracting the types, try merging them again. - meet(T1, T2). - meet_elements(Es1, Es2) -> Keys = maps:keys(Es1) ++ maps:keys(Es2), meet_elements_1(Keys, Es1, Es2, #{}). @@ -1838,7 +2147,7 @@ meet_elements_1([], _Es1, _Es2, Acc) -> %% No tuple elements may have an index above the known size. assert_tuple_elements(Limit, Es) -> - true = maps:fold(fun(Index, _T, true) -> + true = maps:fold(fun({integer,Index}, _T, true) -> Index =< Limit end, true, Es). %Assertion. @@ -1846,6 +2155,7 @@ assert_tuple_elements(Limit, Es) -> %% Subtract Type2 from Type2. Example: %% subtract(list, nil) -> cons +subtract(Same, Same) -> none; subtract(list, nil) -> cons; subtract(list, cons) -> nil; subtract(number, {integer,[]}) -> {float,[]}; @@ -1855,7 +2165,7 @@ subtract(bool, {atom,true}) -> {atom, false}; subtract(Type, _) -> Type. assert_type(WantedType, Term, Vst) -> - Type = get_durable_term_type(Term, Vst), + Type = get_term_type(Term, Vst), assert_type(WantedType, Type). assert_type(Correct, Correct) -> ok; @@ -1876,13 +2186,12 @@ assert_type(Needed, Actual) -> error({bad_type,{needed,Needed},{actual,Actual}}). get_element_type(Key, Src, Vst) -> - get_element_type_1(Key, get_durable_term_type(Src, Vst)). + get_element_type_1(Key, get_term_type(Src, Vst)). -get_element_type_1(Index, {tuple,Sz,Es}) -> +get_element_type_1({integer,_}=Key, {tuple,_Sz,Es}) -> case Es of - #{ Index := Type } -> Type; - #{} when Index =< Sz -> term; - #{} -> none + #{ Key := Type } -> Type; + #{} -> term end; get_element_type_1(_Index, _Type) -> term. @@ -1899,7 +2208,7 @@ get_tuple_size({integer,Sz}) -> Sz; get_tuple_size(_) -> 0. validate_src(Ss, Vst) when is_list(Ss) -> - [assert_term(S, Vst) || S <- Ss], + _ = [assert_term(S, Vst) || S <- Ss], ok. %% get_term_type(Src, ValidatorState) -> Type @@ -1907,33 +2216,23 @@ validate_src(Ss, Vst) when is_list(Ss) -> %% a standard Erlang type (no catch/try tags or match contexts). get_term_type(Src, Vst) -> - case get_move_term_type(Src, Vst) of + case get_movable_term_type(Src, Vst) of #ms{} -> error({match_context,Src}); Type -> Type end. -%% get_durable_term_type(Src, ValidatorState) -> Type -%% Get the type of the source Src. The returned type Type will be -%% a standard Erlang type (no catch/try tags or match contexts). -%% Fragility will be stripped. - -get_durable_term_type(Src, Vst) -> - case get_term_type(Src, Vst) of - {fragile,Type} -> Type; - Type -> Type - end. - -%% get_move_term_type(Src, ValidatorState) -> Type +%% get_movable_term_type(Src, ValidatorState) -> Type %% Get the type of the source Src. The returned type Type will be %% a standard Erlang type (no catch/try tags). Match contexts are OK. -get_move_term_type(Src, Vst) -> +get_movable_term_type(Src, Vst) -> case get_raw_type(Src, Vst) of initialized -> error({unassigned,Src}); uninitialized -> error({uninitialized_reg,Src}); {catchtag,_} -> error({catchtag,Src}); {trytag,_} -> error({trytag,Src}); tuple_in_progress -> error({tuple_in_progress,Src}); + {literal,_}=Lit -> get_literal_type(Lit); Type -> Type end. @@ -1952,26 +2251,29 @@ get_tag_type(Src, _) -> error({invalid_tag_register,Src}). %% get_raw_type(Src, ValidatorState) -> Type -%% Return the type of a register without doing any validity checks. -get_raw_type({x,X}, #vst{current=#st{x=Xs}}) when is_integer(X) -> - case gb_trees:lookup(X, Xs) of - {value,Type} -> Type; - none -> uninitialized +%% Return the type of a register without doing any validity checks or +%% conversions. +get_raw_type({x,X}=Src, #vst{current=#st{xs=Xs}}=Vst) when is_integer(X) -> + check_limit(Src), + case Xs of + #{ Src := #value_ref{}=Ref } -> get_raw_type(Ref, Vst); + #{} -> uninitialized + end; +get_raw_type({y,Y}=Src, #vst{current=#st{ys=Ys}}=Vst) when is_integer(Y) -> + check_limit(Src), + case Ys of + #{ Src := #value_ref{}=Ref } -> get_raw_type(Ref, Vst); + #{ Src := Tag } -> Tag; + #{} -> uninitialized end; -get_raw_type({y,Y}, #vst{current=#st{y=Ys}}) when is_integer(Y) -> - case gb_trees:lookup(Y, Ys) of - {value,Type} -> Type; - none -> uninitialized +get_raw_type(#value_ref{}=Ref, #vst{current=#st{vs=Vs}}) -> + case Vs of + #{ Ref := #value{type=Type} } -> Type; + #{} -> none end; get_raw_type(Src, #vst{}) -> get_literal_type(Src). -get_def(Src, #vst{current=#st{defs=Defs}}) -> - case Defs of - #{Src:=Def} -> Def; - #{} -> none - end. - get_literal_type(nil=T) -> T; get_literal_type({atom,A}=T) when is_atom(A) -> T; get_literal_type({float,F}=T) when is_float(F) -> T; @@ -1979,59 +2281,201 @@ get_literal_type({integer,I}=T) when is_integer(I) -> T; get_literal_type({literal,[_|_]}) -> cons; get_literal_type({literal,Bitstring}) when is_bitstring(Bitstring) -> binary; get_literal_type({literal,Map}) when is_map(Map) -> map; -get_literal_type({literal,Tuple}) when is_tuple(Tuple) -> value_to_type(Tuple); +get_literal_type({literal,Tuple}) when is_tuple(Tuple) -> glt_1(Tuple); get_literal_type({literal,_}) -> term; get_literal_type(T) -> error({not_literal,T}). -value_to_type([]) -> nil; -value_to_type(A) when is_atom(A) -> {atom, A}; -value_to_type(F) when is_float(F) -> {float, F}; -value_to_type(I) when is_integer(I) -> {integer, I}; -value_to_type(T) when is_tuple(T) -> - {Es,_} = foldl(fun(Val, {Es0, Index}) -> - Type = value_to_type(Val), - Es = set_element_type(Index, Type, Es0), - {Es, Index + 1} - end, {#{}, 1}, tuple_to_list(T)), - {tuple, tuple_size(T), Es}; -value_to_type(L) -> {literal, L}. +glt_1([]) -> nil; +glt_1(A) when is_atom(A) -> {atom, A}; +glt_1(F) when is_float(F) -> {float, F}; +glt_1(I) when is_integer(I) -> {integer, I}; +glt_1(T) when is_tuple(T) -> + {Es,_} = foldl(fun(Val, {Es0, Index}) -> + Type = glt_1(Val), + Es = set_element_type({integer,Index}, Type, Es0), + {Es, Index + 1} + end, {#{}, 1}, tuple_to_list(T)), + {tuple, tuple_size(T), Es}; +glt_1(L) -> + {literal, L}. +%%% +%%% Branch tracking +%%% + +%% Forks the execution flow, with the provided funs returning the new state of +%% their respective branch; the "fail" fun returns the state where the branch +%% is taken, and the "success" fun returns the state where it's not. +%% +%% If either path is known not to be taken at runtime (eg. due to a type +%% conflict), it will simply be discarded. +-spec branch(Lbl :: label(), + Original :: #vst{}, + FailFun :: BranchFun, + SuccFun :: BranchFun) -> #vst{} when + BranchFun :: fun((#vst{}) -> #vst{}). +branch(Lbl, Vst0, FailFun, SuccFun) -> + #vst{current=St0} = Vst0, + try FailFun(Vst0) of + Vst1 -> + Vst2 = branch_state(Lbl, Vst1), + Vst = Vst2#vst{current=St0}, + try SuccFun(Vst) of + V -> V + catch + {type_conflict, _, _} -> + %% The instruction is guaranteed to fail; kill the state. + kill_state(Vst) + end + catch + {type_conflict, _, _} -> + %% This instruction is guaranteed not to fail, so we run the + %% success branch *without* catching type conflicts to avoid hiding + %% errors in the validator itself; one of the branches must + %% succeed. + SuccFun(Vst0) + end. + +%% A shorthand version of branch/4 for when the state is only altered on +%% success. +branch(Fail, Vst, SuccFun) -> + branch(Fail, Vst, fun(V) -> V end, SuccFun). + +%% Directly branches off the state. This is an "internal" operation that should +%% be used sparingly. branch_state(0, #vst{}=Vst) -> - %% If the instruction fails, the stack may be scanned - %% looking for a catch tag. Therefore the Y registers - %% must be initialized at this point. + %% If the instruction fails, the stack may be scanned looking for a catch + %% tag. Therefore the Y registers must be initialized at this point. verify_y_init(Vst), Vst; -branch_state(L, #vst{current=St,branched=B}=Vst) -> - Vst#vst{ - branched=case gb_trees:is_defined(L, B) of - false -> - gb_trees:insert(L, St, B); - true -> - MergedSt = merge_states(L, St, B), - gb_trees:update(L, MergedSt, B) - end}. - -%% merge_states/3 is used when there are more than one way to arrive -%% at this point, and the type states for the different paths has -%% to be merged. The type states are downgraded to the least common -%% subset for the subsequent code. - -merge_states(L, St, Branched) when L =/= 0 -> +branch_state(L, #vst{current=St,branched=B,ref_ctr=Counter0}=Vst) -> + case gb_trees:is_defined(L, B) of + true -> + {MergedSt, Counter} = merge_states(L, St, B, Counter0), + Branched = gb_trees:update(L, MergedSt, B), + Vst#vst{branched=Branched,ref_ctr=Counter}; + false -> + Vst#vst{branched=gb_trees:insert(L, St, B)} + end. + +%% merge_states/3 is used when there's more than one way to arrive at a +%% certain point, requiring the states to be merged down to the least +%% common subset for the subsequent code. + +merge_states(L, St, Branched, Counter) when L =/= 0 -> case gb_trees:lookup(L, Branched) of - none -> St; - {value,OtherSt} when St =:= none -> OtherSt; - {value,OtherSt} -> merge_states_1(St, OtherSt) + none -> + {St, Counter}; + {value,OtherSt} when St =:= none -> + {OtherSt, Counter}; + {value,OtherSt} -> + merge_states_1(St, OtherSt, Counter) end. -merge_states_1(#st{x=Xs0,y=Ys0,numy=NumY0,h=H0,ct=Ct0,aliases=Aliases0}, - #st{x=Xs1,y=Ys1,numy=NumY1,h=H1,ct=Ct1,aliases=Aliases1}) -> - NumY = merge_stk(NumY0, NumY1), - Xs = merge_regs(Xs0, Xs1), - Ys = merge_y_regs(Ys0, Ys1), - Ct = merge_ct(Ct0, Ct1), - Aliases = merge_aliases(Aliases0, Aliases1), - #st{x=Xs,y=Ys,numy=NumY,h=min(H0, H1),ct=Ct,aliases=Aliases}. +merge_states_1(#st{xs=XsA,ys=YsA,vs=VsA,fragile=FragA,numy=NumYA,h=HA,ct=CtA}, + #st{xs=XsB,ys=YsB,vs=VsB,fragile=FragB,numy=NumYB,h=HB,ct=CtB}, + Counter0) -> + %% When merging registers we drop all registers that aren't defined in both + %% states, and resolve conflicts by creating new values (similar to phi + %% nodes in SSA). + %% + %% While doing this we build a "merge map" detailing which values need to + %% be kept and which new values need to be created to resolve conflicts. + %% This map is then used to create a new value database where the types of + %% all values have been joined. + {Xs, Merge0, Counter1} = merge_regs(XsA, XsB, #{}, Counter0), + {Ys, Merge, Counter} = merge_regs(YsA, YsB, Merge0, Counter1), + Vs = merge_values(Merge, VsA, VsB), + + Fragile = merge_fragility(FragA, FragB), + NumY = merge_stk(NumYA, NumYB), + Ct = merge_ct(CtA, CtB), + + St = #st{xs=Xs,ys=Ys,vs=Vs,fragile=Fragile,numy=NumY,h=min(HA, HB),ct=Ct}, + {St, Counter}. + +%% Merges the contents of two register maps, returning the updated "merge map" +%% and the new registers. +merge_regs(RsA, RsB, Merge, Counter) -> + Keys = if + map_size(RsA) =< map_size(RsB) -> maps:keys(RsA); + map_size(RsA) > map_size(RsB) -> maps:keys(RsB) + end, + merge_regs_1(Keys, RsA, RsB, #{}, Merge, Counter). + +merge_regs_1([Reg | Keys], RsA, RsB, Regs, Merge0, Counter0) -> + case {RsA, RsB} of + {#{ Reg := #value_ref{}=RefA }, #{ Reg := #value_ref{}=RefB }} -> + {Ref, Merge, Counter} = merge_vrefs(RefA, RefB, Merge0, Counter0), + merge_regs_1(Keys, RsA, RsB, Regs#{ Reg => Ref }, Merge, Counter); + {#{ Reg := TagA }, #{ Reg := TagB }} -> + %% Tags describe the state of the register rather than the value it + %% contains, so if a register contains a tag in one state we have + %% to merge it as a tag regardless of whether the other state says + %% it's a value. + {y, _} = Reg, %Assertion. + merge_regs_1(Keys, RsA, RsB, Regs#{ Reg => merge_tags(TagA,TagB) }, + Merge0, Counter0); + {#{}, #{}} -> + merge_regs_1(Keys, RsA, RsB, Regs, Merge0, Counter0) + end; +merge_regs_1([], _, _, Regs, Merge, Counter) -> + {Regs, Merge, Counter}. + +merge_tags(Same, Same) -> + Same; +merge_tags(uninitialized, _) -> + uninitialized; +merge_tags(_, uninitialized) -> + uninitialized; +merge_tags({catchtag,T0}, {catchtag,T1}) -> + {catchtag, ordsets:from_list(T0 ++ T1)}; +merge_tags({trytag,T0}, {trytag,T1}) -> + {trytag, ordsets:from_list(T0 ++ T1)}; +merge_tags(_A, _B) -> + %% All other combinations leave the register initialized. Errors arising + %% from this will be caught later on. + initialized. + +merge_vrefs(Ref, Ref, Merge, Counter) -> + %% We have two (potentially) different versions of the same value, so we + %% should join their types into the same value. + {Ref, Merge#{ Ref => Ref }, Counter}; +merge_vrefs(RefA, RefB, Merge, Counter) -> + %% We have two different values, so we need to create a new value from + %% their joined type if we haven't already done so. + Key = {RefA, RefB}, + case Merge of + #{ Key := Ref } -> + {Ref, Merge, Counter}; + #{} -> + Ref = #value_ref{id=Counter}, + {Ref, Merge#{ Key => Ref }, Counter + 1} + end. + +merge_values(Merge, VsA, VsB) -> + maps:fold(fun(Spec, New, Acc) -> + merge_values_1(Spec, New, VsA, VsB, Acc) + end, #{}, Merge). + +merge_values_1(Same, Same, VsA, VsB, Acc) -> + %% We're merging different versions of the same value, so it's safe to + %% reuse old entries if the type's unchanged. + #value{type=TypeA}=EntryA = map_get(Same, VsA), + #value{type=TypeB}=EntryB = map_get(Same, VsB), + Entry = case join(TypeA, TypeB) of + TypeA -> EntryA; + TypeB -> EntryB; + JoinedType -> EntryA#value{type=JoinedType} + end, + Acc#{ Same => Entry }; +merge_values_1({RefA, RefB}, New, VsA, VsB, Acc) -> + #value{type=TypeA} = map_get(RefA, VsA), + #value{type=TypeB} = map_get(RefB, VsB), + Acc#{ New => #value{op=join,args=[],type=join(TypeA, TypeB)} }. + +merge_fragility(FragileA, FragileB) -> + cerl_sets:union(FragileA, FragileB). merge_stk(S, S) -> S; merge_stk(_, _) -> undecided. @@ -2044,174 +2488,17 @@ merge_ct_1([C0|Ct0], [C1|Ct1]) -> merge_ct_1([], []) -> []; merge_ct_1(_, _) -> undecided. -merge_regs(Rs0, Rs1) -> - Rs = merge_regs_1(gb_trees:to_list(Rs0), gb_trees:to_list(Rs1)), - gb_trees_from_list(Rs). - -merge_regs_1([Same|Rs1], [Same|Rs2]) -> - [Same|merge_regs_1(Rs1, Rs2)]; -merge_regs_1([{R1,_}|Rs1], [{R2,_}|_]=Rs2) when R1 < R2 -> - merge_regs_1(Rs1, Rs2); -merge_regs_1([{R1,_}|_]=Rs1, [{R2,_}|Rs2]) when R1 > R2 -> - merge_regs_1(Rs1, Rs2); -merge_regs_1([{R,Type1}|Rs1], [{R,Type2}|Rs2]) -> - [{R,join(Type1, Type2)}|merge_regs_1(Rs1, Rs2)]; -merge_regs_1([], []) -> []; -merge_regs_1([], [_|_]) -> []; -merge_regs_1([_|_], []) -> []. - -merge_y_regs(Rs0, Rs1) -> - case {gb_trees:size(Rs0),gb_trees:size(Rs1)} of - {Sz0,Sz1} when Sz0 < Sz1 -> - merge_y_regs_1(Sz0-1, Rs1, Rs0); - {_,Sz1} -> - merge_y_regs_1(Sz1-1, Rs0, Rs1) - end. - -merge_y_regs_1(Y, S, Regs0) when Y >= 0 -> - Type0 = gb_trees:get(Y, Regs0), - case gb_trees:get(Y, S) of - Type0 -> - merge_y_regs_1(Y-1, S, Regs0); - Type1 -> - Type = join(Type0, Type1), - Regs = gb_trees:update(Y, Type, Regs0), - merge_y_regs_1(Y-1, S, Regs) - end; -merge_y_regs_1(_, _, Regs) -> Regs. - -%% join(Type1, Type2) -> Type -%% Return the most specific type possible. -%% Note: Type1 must NOT be the same as Type2. -join({literal,_}=T1, T2) -> - join_literal(T1, T2); -join(T1, {literal,_}=T2) -> - join_literal(T2, T1); -join({fragile,Same}=Type, Same) -> - Type; -join({fragile,T1}, T2) -> - make_fragile(join(T1, T2)); -join(Same, {fragile,Same}=Type) -> - Type; -join(T1, {fragile,T2}) -> - make_fragile(join(T1, T2)); -join(uninitialized=I, _) -> I; -join(_, uninitialized=I) -> I; -join(initialized=I, _) -> I; -join(_, initialized=I) -> I; -join({catchtag,T0},{catchtag,T1}) -> - {catchtag,ordsets:from_list(T0++T1)}; -join({trytag,T0},{trytag,T1}) -> - {trytag,ordsets:from_list(T0++T1)}; -join({tuple,Size,EsA}, {tuple,Size,EsB}) -> - Es = join_tuple_elements(tuple_sz(Size), EsA, EsB), - {tuple, Size, Es}; -join({tuple,A,EsA}, {tuple,B,EsB}) -> - Size = min(tuple_sz(A), tuple_sz(B)), - Es = join_tuple_elements(Size, EsA, EsB), - {tuple, [Size], Es}; -join({Type,A}, {Type,B}) - when Type =:= atom; Type =:= integer; Type =:= float -> - if A =:= B -> {Type,A}; - true -> {Type,[]} - end; -join({Type,_}, number) - when Type =:= integer; Type =:= float -> - number; -join(number, {Type,_}) - when Type =:= integer; Type =:= float -> - number; -join(bool, {atom,A}) -> - join_bool(A); -join({atom,A}, bool) -> - join_bool(A); -join({atom,_}, {atom,_}) -> - {atom,[]}; -join(#ms{id=Id1,valid=B1,slots=Slots1}, - #ms{id=Id2,valid=B2,slots=Slots2}) -> - Id = if - Id1 =:= Id2 -> Id1; - true -> make_ref() - end, - #ms{id=Id,valid=B1 band B2,slots=min(Slots1, Slots2)}; -join(T1, T2) when T1 =/= T2 -> - %% We've exhaused all other options, so the type must either be a list or - %% a 'term'. - join_list(T1, T2). - -join_tuple_elements(Limit, EsA, EsB) -> - Es0 = join_elements(EsA, EsB), - maps:filter(fun(Index, _Type) -> Index =< Limit end, Es0). - -join_elements(Es1, Es2) -> - Keys = if - map_size(Es1) =< map_size(Es2) -> maps:keys(Es1); - map_size(Es1) > map_size(Es2) -> maps:keys(Es2) - end, - join_elements_1(Keys, Es1, Es2, #{}). - -join_elements_1([Key | Keys], Es1, Es2, Acc0) -> - Type = case {Es1, Es2} of - {#{ Key := Same }, #{ Key := Same }} -> Same; - {#{ Key := Type1 }, #{ Key := Type2 }} -> join(Type1, Type2); - {#{}, #{}} -> term - end, - Acc = set_element_type(Key, Type, Acc0), - join_elements_1(Keys, Es1, Es2, Acc); -join_elements_1([], _Es1, _Es2, Acc) -> - Acc. - -%% Joins types of literals; note that the left argument must either be a -%% literal or exactly equal to the second argument. -join_literal(Same, Same) -> - Same; -join_literal({literal,_}=Lit, T) -> - join_literal(T, get_literal_type(Lit)); -join_literal(T1, T2) -> - %% We're done extracting the types, try merging them again. - join(T1, T2). - -join_list(nil, cons) -> list; -join_list(nil, list) -> list; -join_list(cons, list) -> list; -join_list(T, nil) -> join_list(nil, T); -join_list(T, cons) -> join_list(cons, T); -join_list(_, _) -> - %% Not a list, so it must be a term. - term. - -join_bool([]) -> {atom,[]}; -join_bool(true) -> bool; -join_bool(false) -> bool; -join_bool(_) -> {atom,[]}. - tuple_sz([Sz]) -> Sz; tuple_sz(Sz) -> Sz. -merge_aliases(Al0, Al1) when map_size(Al0) =< map_size(Al1) -> - maps:filter(fun(K, V) -> - case Al1 of - #{K:=V} -> true; - #{} -> false - end - end, Al0); -merge_aliases(Al0, Al1) -> - merge_aliases(Al1, Al0). - -verify_y_init(#vst{current=#st{numy=NumY,y=Ys}}=Vst) - when is_integer(NumY), NumY > 0 -> - {HighestY, _} = gb_trees:largest(Ys), +verify_y_init(#vst{current=#st{numy=NumY,ys=Ys}}=Vst) when is_integer(NumY) -> + HighestY = maps:fold(fun({y,Y}, _, Acc) -> max(Y, Acc) end, -1, Ys), true = NumY > HighestY, %Assertion. verify_y_init_1(NumY - 1, Vst), ok; -verify_y_init(#vst{current=#st{numy=undecided,y=Ys}}=Vst) -> - case gb_trees:is_empty(Ys) of - true -> - ok; - false -> - {HighestY, _} = gb_trees:largest(Ys), - verify_y_init_1(HighestY, Vst) - end; +verify_y_init(#vst{current=#st{numy=undecided,ys=Ys}}=Vst) -> + HighestY = maps:fold(fun({y,Y}, _, Acc) -> max(Y, Acc) end, -1, Ys), + verify_y_init_1(HighestY, Vst); verify_y_init(#vst{}) -> ok. @@ -2219,15 +2506,10 @@ verify_y_init_1(-1, _Vst) -> ok; verify_y_init_1(Y, Vst) -> Reg = {y, Y}, + assert_not_fragile(Reg, Vst), case get_raw_type(Reg, Vst) of - uninitialized -> - error({uninitialized_reg,Reg}); - {fragile, _} -> - %% Unsafe. This term may be outside any heap belonging to the - %% process and would be corrupted by a GC. - error({fragile_message_reference,Reg}); - _ -> - verify_y_init_1(Y - 1, Vst) + uninitialized -> error({uninitialized_reg,Reg}); + _ -> verify_y_init_1(Y - 1, Vst) end. verify_live(0, _Vst) -> @@ -2287,27 +2569,68 @@ eat_heap_float(#vst{current=#st{hf=HeapFloats0}=St}=Vst) -> Vst#vst{current=St#st{hf=HeapFloats}} end. -remove_fragility(#vst{current=#st{x=Xs0,y=Ys0}=St0}=Vst) -> - F = fun(_, {fragile,Type}) -> Type; - (_, Type) -> Type - end, - Xs = gb_trees:map(F, Xs0), - Ys = gb_trees:map(F, Ys0), - St = St0#st{x=Xs,y=Ys}, +%%% FRAGILITY +%%% +%%% The loop_rec/2 instruction may return a reference to a term that is not +%%% part of the root set. That term or any part of it must not be included in a +%%% garbage collection. Therefore, the term (or any part of it) must not be +%%% passed to another function, placed in another term, or live in a Y register +%%% over an instruction that may GC. +%%% +%%% Fragility is marked on a per-register (rather than per-value) basis. + +%% Marks Reg as fragile. +mark_fragile(Reg, Vst) -> + #vst{current=#st{fragile=Fragile0}=St0} = Vst, + Fragile = cerl_sets:add_element(Reg, Fragile0), + St = St0#st{fragile=Fragile}, Vst#vst{current=St}. -propagate_fragility(Type, Ss, Vst) -> - F = fun(S) -> - case get_raw_type(S, Vst) of - {fragile,_} -> true; - _ -> false - end - end, - case any(F, Ss) of - true -> make_fragile(Type); - false -> Type +propagate_fragility(Reg, Args, #vst{current=St0}=Vst) -> + #st{fragile=Fragile0} = St0, + + Sources = cerl_sets:from_list(Args), + Fragile = case cerl_sets:is_disjoint(Sources, Fragile0) of + true -> cerl_sets:del_element(Reg, Fragile0); + false -> cerl_sets:add_element(Reg, Fragile0) + end, + + St = St0#st{fragile=Fragile}, + Vst#vst{current=St}. + +%% Marks Reg as durable, must be used when assigning a newly created value to +%% a register. +remove_fragility(Reg, Vst) -> + #vst{current=#st{fragile=Fragile0}=St0} = Vst, + case cerl_sets:is_element(Reg, Fragile0) of + true -> + Fragile = cerl_sets:del_element(Reg, Fragile0), + St = St0#st{fragile=Fragile}, + Vst#vst{current=St}; + false -> + Vst end. +%% Marks all registers as durable. +remove_fragility(#vst{current=St0}=Vst) -> + St = St0#st{fragile=cerl_sets:new()}, + Vst#vst{current=St}. + +assert_durable_term(Src, Vst) -> + assert_term(Src, Vst), + assert_not_fragile(Src, Vst). + +assert_not_fragile({Kind,_}=Src, Vst) when Kind =:= x; Kind =:= y -> + check_limit(Src), + #vst{current=#st{fragile=Fragile}} = Vst, + case cerl_sets:is_element(Src, Fragile) of + true -> error({fragile_message_reference, Src}); + false -> ok + end; +assert_not_fragile(Lit, #vst{}) -> + assert_literal(Lit), + ok. + %%% %%% Return/argument types of BIFs %%% @@ -2319,7 +2642,7 @@ bif_return_type('+', Src, Vst) -> bif_return_type('*', Src, Vst) -> arith_return_type(Src, Vst); bif_return_type(abs, [Num], Vst) -> - case get_durable_term_type(Num, Vst) of + case get_term_type(Num, Vst) of {float,_}=T -> T; {integer,_}=T -> T; _ -> number @@ -2327,8 +2650,10 @@ bif_return_type(abs, [Num], Vst) -> bif_return_type(float, _, _) -> {float,[]}; bif_return_type('/', _, _) -> {float,[]}; %% Binary operations -bif_return_type('byte_size', _, _) -> {integer,[]}; -bif_return_type('bit_size', _, _) -> {integer,[]}; +bif_return_type('binary_part', [_,_], _) -> binary; +bif_return_type('binary_part', [_,_,_], _) -> binary; +bif_return_type('bit_size', [_], _) -> {integer,[]}; +bif_return_type('byte_size', [_], _) -> {integer,[]}; %% Integer operations. bif_return_type(ceil, [_], _) -> {integer,[]}; bif_return_type('div', [_,_], _) -> {integer,[]}; @@ -2362,6 +2687,7 @@ bif_return_type(is_boolean, [_], _) -> bool; bif_return_type(is_binary, [_], _) -> bool; bif_return_type(is_float, [_], _) -> bool; bif_return_type(is_function, [_], _) -> bool; +bif_return_type(is_function, [_,_], _) -> bool; bif_return_type(is_integer, [_], _) -> bool; bif_return_type(is_list, [_], _) -> bool; bif_return_type(is_map, [_], _) -> bool; @@ -2372,6 +2698,7 @@ bif_return_type(is_reference, [_], _) -> bool; bif_return_type(is_tuple, [_], _) -> bool; %% Misc. bif_return_type(tuple_size, [_], _) -> {integer,[]}; +bif_return_type(map_size, [_], _) -> {integer,[]}; bif_return_type(node, [], _) -> {atom,[]}; bif_return_type(node, [_], _) -> {atom,[]}; bif_return_type(hd, [_], _) -> term; @@ -2393,14 +2720,24 @@ bif_arg_types('and', [_,_]) -> [bool, bool]; bif_arg_types('or', [_,_]) -> [bool, bool]; bif_arg_types('xor', [_,_]) -> [bool, bool]; %% Binary -bif_arg_types('byte_size', [_]) -> [binary]; +bif_arg_types('binary_part', [_,_]) -> + PosLen = {tuple, 2, #{ {integer,1} => {integer,[]}, + {integer,2} => {integer,[]} }}, + [binary, PosLen]; +bif_arg_types('binary_part', [_,_,_]) -> + [binary, {integer,[]}, {integer,[]}]; bif_arg_types('bit_size', [_]) -> [binary]; +bif_arg_types('byte_size', [_]) -> [binary]; %% Numerical bif_arg_types('-', [_]) -> [number]; +bif_arg_types('-', [_,_]) -> [number,number]; bif_arg_types('+', [_]) -> [number]; +bif_arg_types('+', [_,_]) -> [number,number]; bif_arg_types('*', [_,_]) -> [number, number]; bif_arg_types('/', [_,_]) -> [number, number]; +bif_arg_types(abs, [_]) -> [number]; bif_arg_types(ceil, [_]) -> [number]; +bif_arg_types(float, [_]) -> [number]; bif_arg_types(floor, [_]) -> [number]; bif_arg_types(trunc, [_]) -> [number]; bif_arg_types(round, [_]) -> [number]; @@ -2446,14 +2783,14 @@ is_bif_safe(_, _) -> false. arith_return_type([A], Vst) -> %% Unary '+' or '-'. - case get_durable_term_type(A, Vst) of + case get_term_type(A, Vst) of {integer,_} -> {integer,[]}; {float,_} -> {float,[]}; _ -> number end; arith_return_type([A,B], Vst) -> - TypeA = get_durable_term_type(A, Vst), - TypeB = get_durable_term_type(B, Vst), + TypeA = get_term_type(A, Vst), + TypeB = get_term_type(B, Vst), case {TypeA, TypeB} of {{integer,_},{integer,_}} -> {integer,[]}; {{float,_},_} -> {float,[]}; @@ -2482,7 +2819,7 @@ call_return_type_1(erlang, setelement, 3, Vst) -> case meet({tuple,[I],#{}}, TupleType) of {tuple, Sz, Es0} -> ValueType = get_term_type({x,2}, Vst), - Es = set_element_type(I, ValueType, Es0), + Es = set_element_type({integer,I}, ValueType, Es0), {tuple, Sz, Es}; none -> TupleType @@ -2542,19 +2879,33 @@ math_mod_return_type(fmod, 2) -> {float,[]}; math_mod_return_type(pi, 0) -> {float,[]}; math_mod_return_type(F, A) when is_atom(F), is_integer(A), A >= 0 -> term. +lists_mod_return_type(all, 2, _Vst) -> + bool; +lists_mod_return_type(any, 2, _Vst) -> + bool; +lists_mod_return_type(keymember, 3, _Vst) -> + bool; +lists_mod_return_type(member, 2, _Vst) -> + bool; +lists_mod_return_type(prefix, 2, _Vst) -> + bool; +lists_mod_return_type(suffix, 2, _Vst) -> + bool; lists_mod_return_type(dropwhile, 2, _Vst) -> list; lists_mod_return_type(duplicate, 2, _Vst) -> list; lists_mod_return_type(filter, 2, _Vst) -> list; +lists_mod_return_type(flatten, 1, _Vst) -> + list; lists_mod_return_type(flatten, 2, _Vst) -> list; lists_mod_return_type(map, 2, Vst) -> same_length_type({x,1}, Vst); lists_mod_return_type(MF, 3, Vst) when MF =:= mapfoldl; MF =:= mapfoldr -> ListType = same_length_type({x,2}, Vst), - {tuple,2,#{1=>ListType}}; + {tuple,2,#{ {integer,1} => ListType} }; lists_mod_return_type(partition, 2, _Vst) -> two_tuple(list, list); lists_mod_return_type(reverse, 1, Vst) -> @@ -2590,7 +2941,8 @@ lists_mod_return_type(_, _, _) -> term. two_tuple(Type1, Type2) -> - {tuple,2,#{1=>Type1,2=>Type2}}. + {tuple,2,#{ {integer,1} => Type1, + {integer,2} => Type2 }}. same_length_type(Reg, Vst) -> case get_term_type(Reg, Vst) of @@ -2600,15 +2952,25 @@ same_length_type(Reg, Vst) -> _ -> list end. -check_limit({x,X}) when is_integer(X), X < 1023 -> - %% Note: x(1023) is reserved for use by the BEAM loader. - ok; -check_limit({y,Y}) when is_integer(Y), Y < 1024 -> - ok; -check_limit({fr,Fr}) when is_integer(Fr), Fr < 1024 -> - ok; -check_limit(_) -> - error(limit). +check_limit({x,X}=Src) when is_integer(X) -> + if + %% Note: x(1023) is reserved for use by the BEAM loader. + 0 =< X, X < 1023 -> ok; + 1023 =< X -> error(limit); + X < 0 -> error({bad_register, Src}) + end; +check_limit({y,Y}=Src) when is_integer(Y) -> + if + 0 =< Y, Y < 1024 -> ok; + 1024 =< Y -> error(limit); + Y < 0 -> error({bad_register, Src}) + end; +check_limit({fr,Fr}=Src) when is_integer(Fr) -> + if + 0 =< Fr, Fr < 1023 -> ok; + 1023 =< Fr -> error(limit); + Fr < 0 -> error({bad_register, Src}) + end. min(A, B) when is_integer(A), is_integer(B), A < B -> A; min(A, B) when is_integer(A), is_integer(B) -> B. diff --git a/lib/compiler/src/cerl_sets.erl b/lib/compiler/src/cerl_sets.erl index 0361186713..f489baf238 100644 --- a/lib/compiler/src/cerl_sets.erl +++ b/lib/compiler/src/cerl_sets.erl @@ -204,4 +204,4 @@ fold(F, Init, D) -> Set2 :: set(Element). filter(F, D) -> - maps:from_list(lists:filter(fun({K,_}) -> F(K) end, maps:to_list(D))). + maps:filter(fun(K,_) -> F(K) end, D). diff --git a/lib/compiler/src/v3_kernel.erl b/lib/compiler/src/v3_kernel.erl index 86351bc0c5..e2b8787224 100644 --- a/lib/compiler/src/v3_kernel.erl +++ b/lib/compiler/src/v3_kernel.erl @@ -1590,19 +1590,12 @@ match_var([U|Us], Cs0, Def, St) -> %% constructor/constant as first argument. Group the constructors %% according to type, the order is really irrelevant but tries to be %% smart. - -match_con(Us, Cs0, Def, St) -> - %% Expand literals at the top level. - Cs = [expand_pat_lit_clause(C) || C <- Cs0], - match_con_1(Us, Cs, Def, St). - -match_con_1([U|_Us] = L, Cs, Def, St0) -> +match_con([U|_Us] = L, Cs, Def, St0) -> %% Extract clauses for different constructors (types). %%ok = io:format("match_con ~p~n", [Cs]), - Ttcs0 = select_types([k_binary], Cs) ++ select_bin_con(Cs) ++ - select_types([k_cons,k_tuple,k_map,k_atom,k_float, - k_int,k_nil], Cs), - Ttcs = opt_single_valued(Ttcs0), + Ttcs0 = select_types(Cs, [], [], [], [], [], [], [], [], []), + Ttcs1 = [{T, Types} || {T, [_ | _] = Types} <- Ttcs0], + Ttcs = opt_single_valued(Ttcs1), %%ok = io:format("ttcs = ~p~n", [Ttcs]), {Scs,St1} = mapfoldl(fun ({T,Tcs}, St) -> @@ -1613,8 +1606,41 @@ match_con_1([U|_Us] = L, Cs, Def, St0) -> St0, Ttcs), {build_alt_1st_no_fail(build_select(U, Scs), Def),St1}. -select_types(Types, Cs) -> - [{T,Tcs} || T <- Types, begin Tcs = select(T, Cs), Tcs =/= [] end]. +select_types([NoExpC | Cs], Bin, BinCon, Cons, Tuple, Map, Atom, Float, Int, Nil) -> + C = expand_pat_lit_clause(NoExpC), + case clause_con(C) of + k_binary -> + select_types(Cs, [C |Bin], BinCon, Cons, Tuple, Map, Atom, Float, Int, Nil); + k_bin_seg -> + select_types(Cs, Bin, [C | BinCon], Cons, Tuple, Map, Atom, Float, Int, Nil); + k_bin_end -> + select_types(Cs, Bin, [C | BinCon], Cons, Tuple, Map, Atom, Float, Int, Nil); + k_cons -> + select_types(Cs, Bin, BinCon, [C | Cons], Tuple, Map, Atom, Float, Int, Nil); + k_tuple -> + select_types(Cs, Bin, BinCon, Cons, [C | Tuple], Map, Atom, Float, Int, Nil); + k_map -> + select_types(Cs, Bin, BinCon, Cons, Tuple, [C | Map], Atom, Float, Int, Nil); + k_atom -> + select_types(Cs, Bin, BinCon, Cons, Tuple, Map, [C | Atom], Float, Int, Nil); + k_float -> + select_types(Cs, Bin, BinCon, Cons, Tuple, Map, Atom, [C | Float], Int, Nil); + k_int -> + select_types(Cs, Bin, BinCon, Cons, Tuple, Map, Atom, Float, [C | Int], Nil); + k_nil -> + select_types(Cs, Bin, BinCon, Cons, Tuple, Map, Atom, Float, Int, [C | Nil]) + end; +select_types([], Bin, BinCon, Cons, Tuple, Map, Atom, Float, Int, Nil) -> + [{k_binary, reverse(Bin)}] ++ handle_bin_con(reverse(BinCon)) ++ + [ + {k_cons, reverse(Cons)}, + {k_tuple, reverse(Tuple)}, + {k_map, reverse(Map)}, + {k_atom, reverse(Atom)}, + {k_float, reverse(Float)}, + {k_int, reverse(Int)}, + {k_nil, reverse(Nil)} + ]. expand_pat_lit_clause(#iclause{pats=[#ialias{pat=#k_literal{anno=A,val=Val}}=Alias|Ps]}=C) -> P = expand_pat_lit(Val, A), @@ -1743,20 +1769,12 @@ combine_bin_segs(#k_bin_end{}) -> combine_bin_segs(_) -> throw(not_possible). -%% select_bin_con([Clause]) -> [{Type,[Clause]}]. -%% Extract clauses for the k_bin_seg constructor. As k_bin_seg +%% handle_bin_con([Clause]) -> [{Type,[Clause]}]. +%% Handle clauses for the k_bin_seg constructor. As k_bin_seg %% matching can overlap, the k_bin_seg constructors cannot be %% reordered, only grouped. -select_bin_con(Cs0) -> - Cs1 = lists:filter(fun (C) -> - Con = clause_con(C), - (Con =:= k_bin_seg) or (Con =:= k_bin_end) - end, Cs0), - select_bin_con_1(Cs1). - - -select_bin_con_1(Cs) -> +handle_bin_con(Cs) -> try %% The usual way to match literals is to first extract the %% value to a register, and then compare the register to the @@ -1795,14 +1813,14 @@ select_bin_con_1(Cs) -> end catch throw:not_possible -> - select_bin_con_2(Cs) + handle_bin_con_not_possible(Cs) end. -select_bin_con_2([C1|Cs]) -> +handle_bin_con_not_possible([C1|Cs]) -> Con = clause_con(C1), {More,Rest} = splitwith(fun (C) -> clause_con(C) =:= Con end, Cs), - [{Con,[C1|More]}|select_bin_con_2(Rest)]; -select_bin_con_2([]) -> []. + [{Con,[C1|More]}|handle_bin_con_not_possible(Rest)]; +handle_bin_con_not_possible([]) -> []. %% select_bin_int([Clause]) -> {k_bin_int,[Clause]} %% If the first pattern in each clause selects the same integer, @@ -1902,10 +1920,6 @@ select_utf8(Val0) -> throw(not_possible) end. -%% select(Con, [Clause]) -> [Clause]. - -select(T, Cs) -> [ C || C <- Cs, clause_con(C) =:= T ]. - %% match_value([Var], Con, [Clause], Default, State) -> {SelectExpr,State}. %% At this point all the clauses have the same constructor, we must %% now separate them according to value. @@ -2040,6 +2054,10 @@ get_match(#k_cons{}, St0) -> get_match(#k_binary{}, St0) -> {[V]=Mes,St1} = new_vars(1, St0), {#k_binary{segs=V},Mes,St1}; +get_match(#k_bin_seg{size=#k_atom{val=all},next={k_bin_end,[]}}=Seg, St0) -> + {[S,N0],St1} = new_vars(2, St0), + N = set_kanno(N0, [no_usage]), + {Seg#k_bin_seg{seg=S,next=N},[S],St1}; get_match(#k_bin_seg{}=Seg, St0) -> {[S,N0],St1} = new_vars(2, St0), N = set_kanno(N0, [no_usage]), @@ -2067,6 +2085,9 @@ new_clauses(Cs0, U, St) -> #k_cons{hd=H,tl=T} -> [H,T|As]; #k_tuple{es=Es} -> Es ++ As; #k_binary{segs=E} -> [E|As]; + #k_bin_seg{size=#k_atom{val=all}, + seg=S,next={k_bin_end,[]}} -> + [S|As]; #k_bin_seg{seg=S,next=N} -> [S,N|As]; #k_bin_int{next=N} -> diff --git a/lib/compiler/test/beam_except_SUITE.erl b/lib/compiler/test/beam_except_SUITE.erl index 9380fe06c8..8e3b373d29 100644 --- a/lib/compiler/test/beam_except_SUITE.erl +++ b/lib/compiler/test/beam_except_SUITE.erl @@ -84,9 +84,16 @@ coverage(_) -> {'EXIT',{function_clause, [{?MODULE,fc,[y],[File,{line,2}]}|_]}} = (catch fc(y)), - {'EXIT',{function_clause, - [{?MODULE,fc,[[a,b,c]],[File,{line,6}]}|_]}} = - (catch fc([a,b,c])), + case ?MODULE of + beam_except_no_opt_SUITE -> + %% There will be a different stack fram in + %% unoptimized code. + ok; + _ -> + {'EXIT',{function_clause, + [{?MODULE,fc,[[a,b,c]],[File,{line,6}]}|_]}} = + (catch fc([a,b,c])) + end, {'EXIT',{undef,[{erlang,error,[a,b,c],_}|_]}} = (catch erlang:error(a, b, c)), diff --git a/lib/compiler/test/beam_jump_SUITE.erl b/lib/compiler/test/beam_jump_SUITE.erl index 759d884dc4..a456f31d79 100644 --- a/lib/compiler/test/beam_jump_SUITE.erl +++ b/lib/compiler/test/beam_jump_SUITE.erl @@ -79,12 +79,13 @@ checks(Wanted) -> {catch case river() of sheet -> begin +Wanted, if "da" -> Wanted end end end, catch case river() of sheet -> begin + Wanted, if "da" -> Wanted end end end}. unsafe_move_elimination(_Config) -> - {{left,right,false},false} = unsafe_move_elimination(left, right, false), - {{false,right,false},false} = unsafe_move_elimination(false, right, true), - {{true,right,right},right} = unsafe_move_elimination(true, right, true), + {{left,right,false},false} = unsafe_move_elimination_1(left, right, false), + {{false,right,false},false} = unsafe_move_elimination_1(false, right, true), + {{true,right,right},right} = unsafe_move_elimination_1(true, right, true), + [ok = unsafe_move_elimination_2(I) || I <- lists:seq(0,16)], ok. -unsafe_move_elimination(Left, Right, Simple0) -> +unsafe_move_elimination_1(Left, Right, Simple0) -> id(1), %% The move at label 29 would be removed by beam_jump, which is unsafe because @@ -115,6 +116,44 @@ unsafe_move_elimination(Left, Right, Simple0) -> end, {id({Left,Right,Simple}),Simple}. +unsafe_move_elimination_2(Int) -> + %% The type optimization pass would recognize that TagInt can only be + %% [0 .. 7], so the first 'case' would select_val over [0 .. 6] and swap + %% out the fail label with the block for 7. + %% + %% A later optimization would merge this block with 'expects_h' in the + %% second case, as the latter is only reachable from the former. + %% + %% ... but this broke down when the move elimination optimization didn't + %% take the fail label of the first select_val into account. This caused it + %% to believe that the only way to reach 'expects_h' was through the second + %% case when 'Tag' =:= 'h', which made it remove the move instruction + %% added in the first case, passing garbage to expects_h/2. + TagInt = Int band 2#111, + Tag = case TagInt of + 0 -> a; + 1 -> b; + 2 -> c; + 3 -> d; + 4 -> e; + 5 -> f; + 6 -> g; + 7 -> h + end, + case Tag of + g -> expects_g(TagInt, Tag); + h -> expects_h(TagInt, Tag); + _ -> Tag = id(Tag), ok + end. + +expects_g(6, Atom) -> + Atom = id(g), + ok. + +expects_h(7, Atom) -> + Atom = id(h), + ok. + -record(message2, {id, p1}). -record(message3, {id, p1, p2}). diff --git a/lib/compiler/test/beam_validator_SUITE.erl b/lib/compiler/test/beam_validator_SUITE.erl index 2660bf222c..de5a3c2873 100644 --- a/lib/compiler/test/beam_validator_SUITE.erl +++ b/lib/compiler/test/beam_validator_SUITE.erl @@ -34,7 +34,8 @@ undef_label/1,illegal_instruction/1,failing_gc_guard_bif/1, map_field_lists/1,cover_bin_opt/1, val_dsetel/1,bad_tuples/1,bad_try_catch_nesting/1, - receive_stacked/1,aliased_types/1]). + receive_stacked/1,aliased_types/1,type_conflict/1, + infer_on_eq/1]). -include_lib("common_test/include/ct.hrl"). @@ -63,7 +64,8 @@ groups() -> undef_label,illegal_instruction,failing_gc_guard_bif, map_field_lists,cover_bin_opt,val_dsetel, bad_tuples,bad_try_catch_nesting, - receive_stacked,aliased_types]}]. + receive_stacked,aliased_types,type_conflict, + infer_on_eq]}]. init_per_suite(Config) -> test_lib:recompile(?MODULE), @@ -107,13 +109,12 @@ xrange(Config) when is_list(Config) -> Errors = do_val(xrange, Config), [{{t,sum_1,2}, {{bif,'+',{f,0},[{x,-1},{x,1}],{x,0}},4, - {uninitialized_reg,{x,-1}}}}, + {bad_register,{x,-1}}}}, {{t,sum_2,2}, - {{bif,'+',{f,0},[{x,0},{x,1023}],{x,0}},4, - {uninitialized_reg,{x,1023}}}}, + {{bif,'+',{f,0},[{x,0},{x,1023}],{x,0}},4,limit}}, {{t,sum_3,2}, {{bif,'+',{f,0},[{x,0},{x,1}],{x,-1}},4, - {invalid_store,{x,-1},number}}}, + {bad_register,{x,-1}}}}, {{t,sum_4,2}, {{bif,'+',{f,0},[{x,0},{x,1}],{x,1023}},4,limit}}] = Errors, ok. @@ -122,15 +123,15 @@ yrange(Config) when is_list(Config) -> Errors = do_val(yrange, Config), [{{t,sum_1,2}, {{move,{x,1},{y,-1}},5, - {invalid_store,{y,-1},term}}}, + {bad_register,{y,-1}}}}, {{t,sum_2,2}, {{bif,'+',{f,0},[{x,0},{y,1024}],{x,0}},7, - {uninitialized_reg,{y,1024}}}}, + limit}}, {{t,sum_3,2}, {{move,{x,1},{y,1024}},5,limit}}, {{t,sum_4,2}, {{move,{x,1},{y,-1}},5, - {invalid_store,{y,-1},term}}}] = Errors, + {bad_register,{y,-1}}}}] = Errors, ok. stack(Config) when is_list(Config) -> @@ -157,8 +158,8 @@ call_last(Config) when is_list(Config) -> merge_undefined(Config) when is_list(Config) -> Errors = do_val(merge_undefined, Config), [{{t,handle_call,2}, - {{call_ext,1,{extfunc,erlang,exit,1}}, - 10, + {{call_ext,2,{extfunc,debug,filter,2}}, + 22, {uninitialized_reg,{y,_}}}}] = Errors, ok. @@ -178,7 +179,7 @@ unsafe_catch(Config) when is_list(Config) -> Errors = do_val(unsafe_catch, Config), [{{t,small,2}, {{bs_put_integer,{f,0},{integer,16},1, - {field_flags,[unsigned,big]},{y,0}}, + {field_flags,[unsigned,big]},{y,0}}, 20, {unassigned,{y,0}}}}] = Errors, ok. @@ -223,7 +224,7 @@ bad_catch_try(Config) when is_list(Config) -> {{try_case,{y,1}},12,{invalid_tag,{y,1},term}}}, {{bad_catch_try,bad_6,1}, {{move,{integer,1},{y,1}},7, - {invalid_store,{y,1},{integer,1}}}}] = Errors, + {invalid_store,{y,1}}}}] = Errors, ok. cons_guard(Config) when is_list(Config) -> @@ -247,7 +248,7 @@ freg_range(Config) when is_list(Config) -> {{t,sum_3,2}, {{bif,fadd,{f,0},[{fr,0},{fr,1}],{fr,-1}}, 7, - {bad_target,{fr,-1}}}}, + {bad_register,{fr,-1}}}}, {{t,sum_4,2}, {{bif,fadd,{f,0},[{fr,0},{fr,1}],{fr,1024}}, 7, @@ -631,6 +632,53 @@ aliased_types_3(Bug) -> hd(List) end. + +%% ERL-867; validation proceeded after a type conflict, causing incorrect types +%% to be joined. + +-record(r, { e1 = e1, e2 = e2 }). + +type_conflict(Config) when is_list(Config) -> + {e1, e2} = type_conflict_1(#r{}), + ok. + +type_conflict_1(C) -> + Src = id(C#r.e2), + TRes = try id(Src) of + R -> R + catch + %% C:R can never match, yet it assumed that the type of 'C' was + %% an atom from here on. + C:R -> R + end, + {C#r.e1, TRes}. + +%% ERL-886; validation failed to infer types on both sides of '=:=' + +infer_on_eq(Config) when is_list(Config) -> + {ok, gurka} = infer_on_eq_1(id({gurka})), + {ok, gaffel} = infer_on_eq_2(id({gaffel})), + {ok, elefant} = infer_on_eq_3(id({elefant})), + {ok, myra} = infer_on_eq_4(id({myra})), + ok. + +infer_on_eq_1(T) -> + 1 = erlang:tuple_size(T), + {ok, erlang:element(1, T)}. + +infer_on_eq_2(T) -> + Size = erlang:tuple_size(T), + Size = 1, + {ok, erlang:element(1, T)}. + +infer_on_eq_3(T) -> + true = 1 =:= erlang:tuple_size(T), + {ok, erlang:element(1, T)}. + +infer_on_eq_4(T) -> + true = erlang:tuple_size(T) =:= 1, + {ok, erlang:element(1, T)}. + %%%------------------------------------------------------------------------- transform_remove(Remove, Module) -> diff --git a/lib/compiler/test/beam_validator_SUITE_data/merge_undefined.S b/lib/compiler/test/beam_validator_SUITE_data/merge_undefined.S index 481d55045d..aa344807e4 100644 --- a/lib/compiler/test/beam_validator_SUITE_data/merge_undefined.S +++ b/lib/compiler/test/beam_validator_SUITE_data/merge_undefined.S @@ -15,8 +15,9 @@ {select_val,{x,0},{f,1},{list,[{atom,gurka},{f,3},{atom,delete},{f,4}]}}. {label,3}. {allocate_heap,2,6,2}. - %% The Y registers are not initialized here. {test,is_eq_exact,{f,5},[{x,0},{atom,ok}]}. + %% This is unreachable since {x,0} is known not to be 'ok'. We should not + %% fail with "uninitialized y registers" on erlang:exit/1 {move,{atom,nisse},{x,0}}. {call_ext,1,{extfunc,erlang,exit,1}}. {label,4}. @@ -29,6 +30,7 @@ {call_ext,2,{extfunc,io,format,2}}. {test,is_ne_exact,{f,6},[{x,0},{atom,ok}]}. {label,5}. + %% The Y registers are not initialized here. {move,{atom,logReader},{x,1}}. {move,{atom,console},{x,0}}. {call_ext,2,{extfunc,debug,filter,2}}. diff --git a/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.S b/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.S index 5b974119c6..a878204d16 100644 --- a/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.S +++ b/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.S @@ -240,7 +240,7 @@ {y,0}}. {'%',{no_bin_opt,{binary_used_in,{test,is_binary,{f,34},[{y,0}]}}, [63,{file,"receive_stacked.erl"}]}}. - {test,is_binary,{f,34},[{y,0}]}. + {test,is_eq_exact,{f,34},[{y,0},{literal,<<0,1,2,3>>}]}. remove_message. {move,{integer,42},{x,0}}. {line,[{location,"receive_stacked.erl",64}]}. @@ -283,7 +283,7 @@ {y,0}}. {'%',{no_bin_opt,{[{x,1},{y,0}],{loop_rec_end,{f,38}},not_handled}, [70,{file,"receive_stacked.erl"}]}}. - {test,is_binary,{f,39},[{x,0}]}. + {test,is_eq_exact,{f,39},[{x,0},{literal,<<0,1,2,3>>}]}. remove_message. {move,{integer,42},{x,0}}. {line,[{location,"receive_stacked.erl",71}]}. diff --git a/lib/compiler/test/float_SUITE.erl b/lib/compiler/test/float_SUITE.erl index 831e8279aa..0fa8070dc8 100644 --- a/lib/compiler/test/float_SUITE.erl +++ b/lib/compiler/test/float_SUITE.erl @@ -21,15 +21,16 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, pending/1,bif_calls/1,math_functions/1,mixed_float_and_int/1, - subtract_number_type/1]). + subtract_number_type/1,float_followed_by_guard/1]). -include_lib("common_test/include/ct.hrl"). suite() -> [{ct_hooks,[ts_install_cth]}]. -all() -> +all() -> [pending, bif_calls, math_functions, - mixed_float_and_int, subtract_number_type]. + mixed_float_and_int, subtract_number_type, + float_followed_by_guard]. groups() -> []. @@ -187,5 +188,21 @@ fact(0, P) -> P; fact(1, P) -> P; fact(N, P) -> fact(N-1, P*N). +float_followed_by_guard(Config) when is_list(Config) -> + true = ffbg_1(5, 1), + false = ffbg_1(1, 5), + ok. + +ffbg_1(A, B0) -> + %% This is a non-guard block followed by a *guard block* that starts with a + %% floating point operation, and the compiler erroneously assumed that it + %% was safe to skip fcheckerror because the next block started with a float + %% op. + B = id(B0) / 1.0, + if + A - B > 0.0 -> true; + A - B =< 0.0 -> false + end. + id(I) -> I. diff --git a/lib/compiler/test/receive_SUITE.erl b/lib/compiler/test/receive_SUITE.erl index 12108445f0..0038eb1a4b 100644 --- a/lib/compiler/test/receive_SUITE.erl +++ b/lib/compiler/test/receive_SUITE.erl @@ -25,7 +25,8 @@ init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2, export/1,recv/1,coverage/1,otp_7980/1,ref_opt/1, - wait/1,recv_in_try/1,double_recv/1,receive_var_zero/1]). + wait/1,recv_in_try/1,double_recv/1,receive_var_zero/1, + match_built_terms/1]). -include_lib("common_test/include/ct.hrl"). @@ -45,7 +46,8 @@ all() -> groups() -> [{p,test_lib:parallel(), [recv,coverage,otp_7980,ref_opt,export,wait, - recv_in_try,double_recv,receive_var_zero]}]. + recv_in_try,double_recv,receive_var_zero, + match_built_terms]}]. init_per_suite(Config) -> @@ -400,5 +402,29 @@ receive_var_zero(Config) when is_list(Config) -> zero() -> 0. +%% ERL-862; the validator would explode when a term was constructed in a +%% receive guard. + +-define(MATCH_BUILT_TERM(Ref, Expr), + (fun() -> + Ref = make_ref(), + A = id($a), + B = id($b), + Built = id(Expr), + self() ! {Ref, A, B}, + receive + {Ref, A, B} when Expr =:= Built -> + ok + after 5000 -> + ct:fail("Failed to match message with term built in " + "receive guard.") + end + end)()). + +match_built_terms(Config) when is_list(Config) -> + ?MATCH_BUILT_TERM(Ref, [A, B]), + ?MATCH_BUILT_TERM(Ref, {A, B}), + ?MATCH_BUILT_TERM(Ref, <<A, B>>), + ?MATCH_BUILT_TERM(Ref, #{ 1 => A, 2 => B}). id(I) -> I. diff --git a/lib/compiler/vsn.mk b/lib/compiler/vsn.mk index efedb414ad..a523627384 100644 --- a/lib/compiler/vsn.mk +++ b/lib/compiler/vsn.mk @@ -1 +1 @@ -COMPILER_VSN = 7.3.1 +COMPILER_VSN = 7.3.2 diff --git a/lib/crypto/c_src/Makefile.in b/lib/crypto/c_src/Makefile.in index e1e7f71538..b6a65d7488 100644 --- a/lib/crypto/c_src/Makefile.in +++ b/lib/crypto/c_src/Makefile.in @@ -77,9 +77,7 @@ CRYPTO_OBJS = $(OBJDIR)/crypto$(TYPEMARKER).o \ $(OBJDIR)/algorithms$(TYPEMARKER).o \ $(OBJDIR)/api_ng$(TYPEMARKER).o \ $(OBJDIR)/atoms$(TYPEMARKER).o \ - $(OBJDIR)/block$(TYPEMARKER).o \ $(OBJDIR)/bn$(TYPEMARKER).o \ - $(OBJDIR)/chacha20$(TYPEMARKER).o \ $(OBJDIR)/cipher$(TYPEMARKER).o \ $(OBJDIR)/cmac$(TYPEMARKER).o \ $(OBJDIR)/dh$(TYPEMARKER).o \ @@ -98,7 +96,6 @@ CRYPTO_OBJS = $(OBJDIR)/crypto$(TYPEMARKER).o \ $(OBJDIR)/pkey$(TYPEMARKER).o \ $(OBJDIR)/poly1305$(TYPEMARKER).o \ $(OBJDIR)/rand$(TYPEMARKER).o \ - $(OBJDIR)/rc4$(TYPEMARKER).o \ $(OBJDIR)/rsa$(TYPEMARKER).o \ $(OBJDIR)/srp$(TYPEMARKER).o CALLBACK_OBJS = $(OBJDIR)/crypto_callback$(TYPEMARKER).o diff --git a/lib/crypto/c_src/aes.c b/lib/crypto/c_src/aes.c index ee2bb70fb7..4b01e629f9 100644 --- a/lib/crypto/c_src/aes.c +++ b/lib/crypto/c_src/aes.c @@ -166,156 +166,7 @@ ERL_NIF_TERM aes_ige_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv } -#ifdef HAVE_EVP_AES_CTR -ERL_NIF_TERM aes_ctr_stream_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Key, IVec) */ - ErlNifBinary key_bin, ivec_bin; - struct evp_cipher_ctx *ctx = NULL; - const EVP_CIPHER *cipher; - ERL_NIF_TERM ret; - - ASSERT(argc == 2); - - if (!enif_inspect_iolist_as_binary(env, argv[0], &key_bin)) - goto bad_arg; - if (!enif_inspect_binary(env, argv[1], &ivec_bin)) - goto bad_arg; - if (ivec_bin.size != 16) - goto bad_arg; - - switch (key_bin.size) - { - case 16: - cipher = EVP_aes_128_ctr(); - break; - case 24: - cipher = EVP_aes_192_ctr(); - break; - case 32: - cipher = EVP_aes_256_ctr(); - break; - default: - goto bad_arg; - } - - if ((ctx = enif_alloc_resource(evp_cipher_ctx_rtype, sizeof(struct evp_cipher_ctx))) == NULL) - goto err; - if ((ctx->ctx = EVP_CIPHER_CTX_new()) == NULL) - goto err; - - if (EVP_CipherInit_ex(ctx->ctx, cipher, NULL, - key_bin.data, ivec_bin.data, 1) != 1) - goto err; - - if (EVP_CIPHER_CTX_set_padding(ctx->ctx, 0) != 1) - goto err; - - ret = enif_make_resource(env, ctx); - goto done; - - bad_arg: - return enif_make_badarg(env); - - err: - ret = enif_make_badarg(env); - - done: - if (ctx) - enif_release_resource(ctx); - return ret; -} - -ERL_NIF_TERM aes_ctr_stream_encrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Context, Data) */ - struct evp_cipher_ctx *ctx = NULL, *new_ctx = NULL; - ErlNifBinary data_bin; - ERL_NIF_TERM ret, cipher_term; - unsigned char *out; - int outl = 0; - - ASSERT(argc == 2); - - if (!enif_get_resource(env, argv[0], evp_cipher_ctx_rtype, (void**)&ctx)) - goto bad_arg; - if (!enif_inspect_iolist_as_binary(env, argv[1], &data_bin)) - goto bad_arg; - if (data_bin.size > INT_MAX) - goto bad_arg; - - if ((new_ctx = enif_alloc_resource(evp_cipher_ctx_rtype, sizeof(struct evp_cipher_ctx))) == NULL) - goto err; - if ((new_ctx->ctx = EVP_CIPHER_CTX_new()) == NULL) - goto err; - - if (EVP_CIPHER_CTX_copy(new_ctx->ctx, ctx->ctx) != 1) - goto err; - - if ((out = enif_make_new_binary(env, data_bin.size, &cipher_term)) == NULL) - goto err; - - if (EVP_CipherUpdate(new_ctx->ctx, out, &outl, data_bin.data, (int)data_bin.size) != 1) - goto err; - ASSERT(outl >= 0 && (size_t)outl == data_bin.size); - - ret = enif_make_tuple2(env, enif_make_resource(env, new_ctx), cipher_term); - CONSUME_REDS(env,data_bin); - goto done; - - bad_arg: - return enif_make_badarg(env); - - err: - ret = enif_make_badarg(env); - - done: - if (new_ctx) - enif_release_resource(new_ctx); - return ret; -} - -#else /* if not HAVE_EVP_AES_CTR */ - -ERL_NIF_TERM aes_ctr_stream_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Key, IVec) */ - ASSERT(argc == 2); - - return aes_ctr_stream_init_compat(env, argv[0], argv[1]); -} - - -ERL_NIF_TERM aes_ctr_stream_init_compat(ErlNifEnv* env, const ERL_NIF_TERM key_term, const ERL_NIF_TERM iv_term) -{ - ErlNifBinary key_bin, ivec_bin; - ERL_NIF_TERM ecount_bin; - unsigned char *outp; - - if (!enif_inspect_iolist_as_binary(env, key_term, &key_bin)) - goto bad_arg; - if (key_bin.size != 16 && key_bin.size != 24 && key_bin.size != 32) - goto bad_arg; - if (!enif_inspect_binary(env, iv_term, &ivec_bin)) - goto bad_arg; - if (ivec_bin.size != 16) - goto bad_arg; - if ((outp = enif_make_new_binary(env, AES_BLOCK_SIZE, &ecount_bin)) == NULL) - goto err; - memset(outp, 0, AES_BLOCK_SIZE); - - return enif_make_tuple4(env, key_term, iv_term, ecount_bin, enif_make_int(env, 0)); - - bad_arg: - err: - return enif_make_badarg(env); -} - -ERL_NIF_TERM aes_ctr_stream_encrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - ASSERT(argc == 2); - - return aes_ctr_stream_encrypt_compat(env, argv[0], argv[1]); -} - - +#if !defined(HAVE_EVP_AES_CTR) ERL_NIF_TERM aes_ctr_stream_encrypt_compat(ErlNifEnv* env, const ERL_NIF_TERM state_arg, const ERL_NIF_TERM data_arg) {/* ({Key, IVec, ECount, Num}, Data) */ ErlNifBinary key_bin, ivec_bin, text_bin, ecount_bin; diff --git a/lib/crypto/c_src/aes.h b/lib/crypto/c_src/aes.h index 527d041410..c0b2b91f8d 100644 --- a/lib/crypto/c_src/aes.h +++ b/lib/crypto/c_src/aes.h @@ -27,10 +27,7 @@ ERL_NIF_TERM aes_cfb_8_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[] ERL_NIF_TERM aes_cfb_128_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM aes_ige_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -ERL_NIF_TERM aes_ctr_stream_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -ERL_NIF_TERM aes_ctr_stream_encrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); #if !defined(HAVE_EVP_AES_CTR) -ERL_NIF_TERM aes_ctr_stream_init_compat(ErlNifEnv* env, const ERL_NIF_TERM key_term, const ERL_NIF_TERM iv_term); ERL_NIF_TERM aes_ctr_stream_encrypt_compat(ErlNifEnv* env, const ERL_NIF_TERM state_arg, const ERL_NIF_TERM data_arg); #endif diff --git a/lib/crypto/c_src/api_ng.c b/lib/crypto/c_src/api_ng.c index c4114d1626..6a833a0984 100644 --- a/lib/crypto/c_src/api_ng.c +++ b/lib/crypto/c_src/api_ng.c @@ -25,199 +25,532 @@ /* * A unified set of functions for encryption/decryption. * - * EXPERIMENTAL!! - * */ ERL_NIF_TERM ng_crypto_update(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM ng_crypto_one_shot(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -/* Try better error messages in new functions */ -#define ERROR_Term(Env, ReasonTerm) enif_make_tuple2((Env), atom_error, (ReasonTerm)) -#define ERROR_Str(Env, ReasonString) ERROR_Term((Env), enif_make_string((Env),(ReasonString),(ERL_NIF_LATIN1))) +/* All nif functions return a valid value or throws an exception */ +#define EXCP(Env, Class, Str) enif_raise_exception((Env), \ + enif_make_tuple2((Env), (Class), \ + enif_make_string((Env),(Str),(ERL_NIF_LATIN1)) )) -/* Initializes state for (de)encryption - */ -ERL_NIF_TERM ng_crypto_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Cipher, Key, IVec, Encrypt) % if no IV for the Cipher, set IVec = <<>> - */ - ErlNifBinary key_bin, ivec_bin; - unsigned char *iv = NULL; - struct evp_cipher_ctx *ctx; - const struct cipher_type_t *cipherp; - const EVP_CIPHER *cipher; - ERL_NIF_TERM enc_flg_arg, ret; - int enc; - unsigned iv_len; - - enc_flg_arg = argv[argc-1]; - if (enc_flg_arg == atom_true) - enc = 1; - else if (enc_flg_arg == atom_false) - enc = 0; - else if (enc_flg_arg == atom_undefined) +#define EXCP_NOTSUP(Env, Str) EXCP((Env), atom_notsup, (Str)) +#define EXCP_BADARG(Env, Str) EXCP((Env), atom_badarg, (Str)) +#define EXCP_ERROR(Env, Str) EXCP((Env), atom_error, (Str)) + + +#ifdef HAVE_ECB_IVEC_BUG + /* <= 0.9.8l returns faulty ivec length */ +# define GET_IV_LEN(Ciph) ((Ciph)->flags & ECB_BUG_0_9_8L) ? 0 : EVP_CIPHER_iv_length((Ciph)->cipher.p) +#else +# define GET_IV_LEN(Ciph) EVP_CIPHER_iv_length((Ciph)->cipher.p) +#endif + +/*************************************************************************/ +/* Get the arguments for the initialization of the EVP_CIPHER_CTX. Check */ +/* them and initialize that context. */ +/*************************************************************************/ +static int get_init_args(ErlNifEnv* env, + struct evp_cipher_ctx *ctx_res, + const ERL_NIF_TERM cipher_arg, + const ERL_NIF_TERM key_arg, + const ERL_NIF_TERM ivec_arg, + const ERL_NIF_TERM encflg_arg, + const struct cipher_type_t **cipherp, + ERL_NIF_TERM *return_term) +{ + int ivec_len; + ErlNifBinary key_bin; + ErlNifBinary ivec_bin; + int encflg; + + ctx_res->ctx = NULL; /* For testing if *ctx should be freed after errors */ + + /* Fetch the flag telling if we are going to encrypt (=true) or decrypt (=false) */ + if (encflg_arg == atom_true) + encflg = 1; + else if (encflg_arg == atom_false) + encflg = 0; + else if (encflg_arg == atom_undefined) /* For compat funcs in crypto.erl */ - enc = -1; + encflg = -1; else - return ERROR_Str(env, "Bad enc flag"); + { + *return_term = EXCP_BADARG(env, "Bad enc flag"); + goto err; + } - if (!enif_inspect_binary(env, argv[1], &key_bin)) - return ERROR_Str(env, "Bad key"); + /* Fetch the key */ + if (!enif_inspect_iolist_as_binary(env, key_arg, &key_bin)) + { + *return_term = EXCP_BADARG(env, "Bad key"); + goto err; + } - if (!(cipherp = get_cipher_type(argv[0], key_bin.size))) - return ERROR_Str(env, "Unknown cipher or bad key size"); + /* Fetch cipher type */ + if (!enif_is_atom(env, cipher_arg)) + { + *return_term = EXCP_BADARG(env, "Cipher id is not an atom"); + goto err; + } - if (FORBIDDEN_IN_FIPS(cipherp)) - return enif_raise_exception(env, atom_notsup); + if (!(*cipherp = get_cipher_type(cipher_arg, key_bin.size))) + { + if (!get_cipher_type_no_key(cipher_arg)) + *return_term = EXCP_BADARG(env, "Unknown cipher"); + else + *return_term = EXCP_BADARG(env, "Bad key size"); + goto err; + } - if (enc == -1) - return atom_undefined; + if (FORBIDDEN_IN_FIPS(*cipherp)) + { + *return_term = EXCP_NOTSUP(env, "Forbidden in FIPS"); + goto err; + } - if (!(cipher = cipherp->cipher.p)) { + /* Get ivec_len for this cipher (if we found one) */ #if !defined(HAVE_EVP_AES_CTR) - if (cipherp->flags & AES_CTR_COMPAT) - return aes_ctr_stream_init_compat(env, argv[1], argv[2]); - else + /* This code is for historic OpenSSL where EVP_aes_*_ctr is not defined.... */ + if ((*cipherp)->cipher.p) { + /* Not aes_ctr compatibility code since EVP_* + was defined and assigned to (*cipherp)->cipher.p */ + ivec_len = GET_IV_LEN(*cipherp); + } else { + /* No EVP_* was found */ + if ((*cipherp)->flags & AES_CTR_COMPAT) + /* Use aes_ctr compatibility code later */ + ivec_len = 16; + else { + /* Unsupported crypto */ + *return_term = EXCP_NOTSUP(env, "Cipher not supported in this libcrypto version"); + goto err; + } + } +#else + /* Normal code */ + if (!((*cipherp)->cipher.p)) { + *return_term = EXCP_NOTSUP(env, "Cipher not supported in this libcrypto version"); + goto err; + } + ivec_len = GET_IV_LEN(*cipherp); #endif - return enif_raise_exception(env, atom_notsup); + + /* (*cipherp)->cipher.p != NULL and ivec_len has a value */ + + /* Fetch IV */ + if (ivec_len && (ivec_arg != atom_undefined)) { + if (!enif_inspect_iolist_as_binary(env, ivec_arg, &ivec_bin)) + { + *return_term = EXCP_BADARG(env, "Bad iv type"); + goto err; + } + + if (ivec_len != ivec_bin.size) + { + *return_term = EXCP_BADARG(env, "Bad iv size"); + goto err; + } } -#ifdef HAVE_ECB_IVEC_BUG - if (cipherp->flags & ECB_BUG_0_9_8L) - iv_len = 0; /* <= 0.9.8l returns faulty ivec length */ - else + ctx_res->iv_len = ivec_len; + +#if !defined(HAVE_EVP_AES_CTR) + if (!((*cipherp)->cipher.p) + && ((*cipherp)->flags & AES_CTR_COMPAT) + ) { + /* Must use aes_ctr compatibility code */ + ERL_NIF_TERM ecount_bin; + unsigned char *outp; + if ((outp = enif_make_new_binary(env, AES_BLOCK_SIZE, &ecount_bin)) == NULL) { + *return_term = EXCP_ERROR(env, "Can't allocate ecount_bin"); + goto err; + } + memset(outp, 0, AES_BLOCK_SIZE); + + ctx_res->env = enif_alloc_env(); + if (!ctx_res->env) { + *return_term = EXCP_ERROR(env, "Can't allocate env"); + goto err; + } + ctx_res->state = + enif_make_copy(ctx_res->env, + enif_make_tuple4(env, key_arg, ivec_arg, ecount_bin, enif_make_int(env, 0))); + goto success; + } else { + /* Flag for subsequent calls that no aes_ctr compatibility code should be called */ + ctx_res->state = atom_undefined; + ctx_res->env = NULL; + } #endif - iv_len = EVP_CIPHER_iv_length(cipher); - if (iv_len) { - if (!enif_inspect_binary(env, argv[2], &ivec_bin)) - return ERROR_Str(env, "Bad iv type"); + /* Initialize the EVP_CIPHER_CTX */ - if (iv_len != ivec_bin.size) - return ERROR_Str(env, "Bad iv size"); + ctx_res->ctx = EVP_CIPHER_CTX_new(); + if (! ctx_res->ctx) + { + *return_term = EXCP_ERROR(env, "Can't allocate context"); + goto err; + } - iv = ivec_bin.data; - } + if (!EVP_CipherInit_ex(ctx_res->ctx, (*cipherp)->cipher.p, NULL, NULL, NULL, encflg)) + { + *return_term = EXCP_ERROR(env, "Can't initialize context, step 1"); + goto err; + } - if ((ctx = enif_alloc_resource(evp_cipher_ctx_rtype, sizeof(struct evp_cipher_ctx))) == NULL) - return ERROR_Str(env, "Can't allocate resource"); + if (!EVP_CIPHER_CTX_set_key_length(ctx_res->ctx, (int)key_bin.size)) + { + *return_term = EXCP_ERROR(env, "Can't initialize context, key_length"); + goto err; + } - ctx->ctx = EVP_CIPHER_CTX_new(); - if (! ctx->ctx) - return ERROR_Str(env, "Can't allocate context"); - if (!EVP_CipherInit_ex(ctx->ctx, cipher, NULL, NULL, NULL, enc)) { - enif_release_resource(ctx); - return ERROR_Str(env, "Can't initialize context, step 1"); + if (EVP_CIPHER_type((*cipherp)->cipher.p) == NID_rc2_cbc) { + if (key_bin.size > INT_MAX / 8) { + *return_term = EXCP_BADARG(env, "To large rc2_cbc key"); + goto err; + } + if (!EVP_CIPHER_CTX_ctrl(ctx_res->ctx, EVP_CTRL_SET_RC2_KEY_BITS, (int)key_bin.size * 8, NULL)) { + *return_term = EXCP_ERROR(env, "ctrl rc2_cbc key"); + goto err; + } } - if (!EVP_CIPHER_CTX_set_key_length(ctx->ctx, (int)key_bin.size)) { - enif_release_resource(ctx); - return ERROR_Str(env, "Can't initialize context, key_length"); - } + if (ivec_arg == atom_undefined || ivec_len == 0) + { + if (!EVP_CipherInit_ex(ctx_res->ctx, NULL, NULL, key_bin.data, NULL, -1)) { + *return_term = EXCP_ERROR(env, "Can't initialize key"); + goto err; + } + } + else + if (!EVP_CipherInit_ex(ctx_res->ctx, NULL, NULL, key_bin.data, ivec_bin.data, -1)) + { + *return_term = EXCP_ERROR(env, "Can't initialize key or iv"); + goto err; + } - if (EVP_CIPHER_type(cipher) == NID_rc2_cbc) { - if (key_bin.size > INT_MAX / 8) { - enif_release_resource(ctx); - return ERROR_Str(env, "To large rc2_cbc key"); + EVP_CIPHER_CTX_set_padding(ctx_res->ctx, 0); + + *return_term = atom_ok; + +#if !defined(HAVE_EVP_AES_CTR) + success: +#endif + return 1; + + err: + if (ctx_res->ctx) EVP_CIPHER_CTX_free(ctx_res->ctx); + return 0; +} + +/*************************************************************************/ +/* Get the arguments for the EVP_CipherUpdate function, and call it. */ +/*************************************************************************/ + +static int get_update_args(ErlNifEnv* env, + struct evp_cipher_ctx *ctx_res, + const ERL_NIF_TERM indata_arg, + ERL_NIF_TERM *return_term) +{ + ErlNifBinary in_data_bin, out_data_bin; + int out_len, block_size; + + if (!enif_inspect_binary(env, indata_arg, &in_data_bin) ) + { + *return_term = EXCP_BADARG(env, "Bad 2:nd arg"); + goto err; } - if (!EVP_CIPHER_CTX_ctrl(ctx->ctx, EVP_CTRL_SET_RC2_KEY_BITS, (int)key_bin.size * 8, NULL)) { - enif_release_resource(ctx); - return ERROR_Str(env, "ctrl rc2_cbc key"); + + ASSERT(in_data_bin.size <= INT_MAX); + +#if !defined(HAVE_EVP_AES_CTR) + if (ctx_res->state != atom_undefined) { + ERL_NIF_TERM state0, newstate_and_outdata; + const ERL_NIF_TERM *tuple_argv; + int tuple_argc; + + state0 = enif_make_copy(env, ctx_res->state); + + if (enif_get_tuple(env, state0, &tuple_argc, &tuple_argv) && (tuple_argc == 4)) { + /* A compatibility state term */ + /* encrypt and decrypt is performed by calling the same function */ + newstate_and_outdata = aes_ctr_stream_encrypt_compat(env, state0, indata_arg); + + if (enif_get_tuple(env, newstate_and_outdata, &tuple_argc, &tuple_argv) && (tuple_argc == 2)) { + /* newstate_and_outdata = {NewState, OutData} */ + ctx_res->state = enif_make_copy(ctx_res->env, tuple_argv[0]); + /* Return the OutData (from the newstate_and_outdata tuple) only: */ + *return_term = tuple_argv[1]; + } } + } else +#endif + { + block_size = EVP_CIPHER_CTX_block_size(ctx_res->ctx); + + if (!enif_alloc_binary((size_t)in_data_bin.size+block_size, &out_data_bin)) + { + *return_term = EXCP_ERROR(env, "Can't allocate outdata"); + goto err; + } + + if (!EVP_CipherUpdate(ctx_res->ctx, out_data_bin.data, &out_len, in_data_bin.data, in_data_bin.size)) + { + *return_term = EXCP_ERROR(env, "Can't update"); + goto err; + } + + if (!enif_realloc_binary(&out_data_bin, (size_t)out_len)) + { + *return_term = EXCP_ERROR(env, "Can't reallocate"); + goto err; + } + + CONSUME_REDS(env, in_data_bin); + /* return the result text as a binary: */ + *return_term = enif_make_binary(env, &out_data_bin); } - if (!EVP_CipherInit_ex(ctx->ctx, NULL, NULL, key_bin.data, iv, enc)) { - enif_release_resource(ctx); - return ERROR_Str(env, "Can't initialize key and/or iv"); - } + /* success: */ + return 1; - EVP_CIPHER_CTX_set_padding(ctx->ctx, 0); + err: + return 0; +} - ret = enif_make_resource(env, ctx); - enif_release_resource(ctx); +/*************************************************************************/ +/* Initialize the state for (de/en)cryption */ +/*************************************************************************/ + +ERL_NIF_TERM ng_crypto_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Cipher, Key, IVec, Encrypt) % if no IV for the Cipher, set IVec = <<>> + */ + struct evp_cipher_ctx *ctx_res = NULL; + const struct cipher_type_t *cipherp; + ERL_NIF_TERM ret; + int encflg; + + if (enif_is_atom(env, argv[0])) { + if ((ctx_res = enif_alloc_resource(evp_cipher_ctx_rtype, sizeof(struct evp_cipher_ctx))) == NULL) + return EXCP_ERROR(env, "Can't allocate resource"); + + if (!get_init_args(env, ctx_res, argv[0], argv[1], argv[2], argv[argc-1], + &cipherp, &ret)) + /* Error msg in &ret */ + goto ret; + + ret = enif_make_resource(env, ctx_res); + if(ctx_res) enif_release_resource(ctx_res); + + } else if (enif_get_resource(env, argv[0], evp_cipher_ctx_rtype, (void**)&ctx_res)) { + /* Fetch the flag telling if we are going to encrypt (=true) or decrypt (=false) */ + if (argv[3] == atom_true) + encflg = 1; + else if (argv[3] == atom_false) + encflg = 0; + else { + ret = EXCP_BADARG(env, "Bad enc flag"); + goto ret; + } + if (ctx_res->ctx) { + /* It is *not* a ctx_res for the compatibility handling of non-EVP aes_ctr */ + if (!EVP_CipherInit_ex(ctx_res->ctx, NULL, NULL, NULL, NULL, encflg)) { + ret = EXCP_ERROR(env, "Can't initialize encflag"); + goto ret; + } + } + ret = argv[0]; + } else { + ret = EXCP_BADARG(env, "Bad 1:st arg"); + goto ret; + } + + ret: return ret; } -ERL_NIF_TERM ng_crypto_update(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Context, Data) - (Context, Data, IV) */ - struct evp_cipher_ctx *ctx; - ErlNifBinary in_data_bin, ivec_bin, out_data_bin; - int out_len, block_size; -#if !defined(HAVE_EVP_AES_CTR) - const ERL_NIF_TERM *state_term; - int state_arity; +/*************************************************************************/ +/* Encrypt/decrypt */ +/*************************************************************************/ - if (enif_get_tuple(env, argv[0], &state_arity, &state_term) && (state_arity == 4)) { - return aes_ctr_stream_encrypt_compat(env, argv[0], argv[1]); - } +#if !defined(HAVE_EVP_CIPHER_CTX_COPY) +/* + The EVP_CIPHER_CTX_copy is not available in older cryptolibs although + the function is needed. + Instead of implement it in-place, we have a copy here as a compatibility + function +*/ + +int EVP_CIPHER_CTX_copy(EVP_CIPHER_CTX *out, const EVP_CIPHER_CTX *in); + +int EVP_CIPHER_CTX_copy(EVP_CIPHER_CTX *out, const EVP_CIPHER_CTX *in) +{ + if ((in == NULL) || (in->cipher == NULL)) + { + return 0; + } +#ifdef HAS_ENGINE_SUPPORT + /* Make sure it's safe to copy a cipher context using an ENGINE */ + if (in->engine && !ENGINE_init(in->engine)) + return 0; #endif - if (!enif_get_resource(env, argv[0], evp_cipher_ctx_rtype, (void**)&ctx)) - return ERROR_Str(env, "Bad 1:st arg"); - - if (!enif_inspect_binary(env, argv[1], &in_data_bin) ) - return ERROR_Str(env, "Bad 2:nd arg"); + EVP_CIPHER_CTX_cleanup(out); + memcpy(out,in,sizeof *out); - /* arg[1] was checked by the caller */ - ASSERT(in_data_bin.size =< INT_MAX); + if (in->cipher_data && in->cipher->ctx_size) + { + out->cipher_data=OPENSSL_malloc(in->cipher->ctx_size); + if (!out->cipher_data) + return 0; + memcpy(out->cipher_data,in->cipher_data,in->cipher->ctx_size); + } - block_size = EVP_CIPHER_CTX_block_size(ctx->ctx); - if (in_data_bin.size % (size_t)block_size != 0) - return ERROR_Str(env, "Data not a multiple of block size"); +#if defined(EVP_CIPH_CUSTOM_COPY) && defined(EVP_CTRL_COPY) + if (in->cipher->flags & EVP_CIPH_CUSTOM_COPY) + return in->cipher->ctrl((EVP_CIPHER_CTX *)in, EVP_CTRL_COPY, 0, out); +#endif + return 1; +} +/****** End of compatibility function ******/ +#endif - if (argc==3) { - if (!enif_inspect_iolist_as_binary(env, argv[2], &ivec_bin)) - return ERROR_Str(env, "Not binary IV"); - - if (ivec_bin.size > INT_MAX) - return ERROR_Str(env, "Too big IV"); - - if (!EVP_CipherInit_ex(ctx->ctx, NULL, NULL, NULL, ivec_bin.data, -1)) - return ERROR_Str(env, "Can't set IV"); - } - if (!enif_alloc_binary((size_t)in_data_bin.size+block_size, &out_data_bin)) - return ERROR_Str(env, "Can't allocate outdata"); +ERL_NIF_TERM ng_crypto_update(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Context, Data [, IV]) */ + struct evp_cipher_ctx *ctx_res; + ERL_NIF_TERM ret; + + if (!enif_get_resource(env, argv[0], evp_cipher_ctx_rtype, (void**)&ctx_res)) + return EXCP_BADARG(env, "Bad 1:st arg"); + + if (argc == 3) { + struct evp_cipher_ctx ctx_res_copy; + ErlNifBinary ivec_bin; - if (!EVP_CipherUpdate(ctx->ctx, out_data_bin.data, &out_len, in_data_bin.data, in_data_bin.size)) - return ERROR_Str(env, "Can't update"); + memcpy(&ctx_res_copy, ctx_res, sizeof ctx_res_copy); +#if !defined(HAVE_EVP_AES_CTR) + if (ctx_res_copy.state == atom_undefined) + /* Not going to use aes_ctr compat functions */ +#endif + { + ctx_res_copy.ctx = EVP_CIPHER_CTX_new(); - if (!enif_realloc_binary(&out_data_bin, (size_t)out_len)) - return ERROR_Str(env, "Can't reallocate"); + if (!EVP_CIPHER_CTX_copy(ctx_res_copy.ctx, ctx_res->ctx)) { + ret = EXCP_ERROR(env, "Can't copy ctx_res"); + goto err; + } + } - CONSUME_REDS(env, in_data_bin); - return enif_make_binary(env, &out_data_bin); + ctx_res = &ctx_res_copy; + + if (!enif_inspect_iolist_as_binary(env, argv[2], &ivec_bin)) + { + ret = EXCP_BADARG(env, "Bad iv type"); + goto err; + } + + if (ctx_res_copy.iv_len != ivec_bin.size) + { + ret = EXCP_BADARG(env, "Bad iv size"); + goto err; + } + +#if !defined(HAVE_EVP_AES_CTR) + if ((ctx_res_copy.state != atom_undefined) ) { + /* replace the iv in state with argv[2] */ + ERL_NIF_TERM state0; + const ERL_NIF_TERM *tuple_argv; + int tuple_argc; + state0 = enif_make_copy(env, ctx_res_copy.state); + if (enif_get_tuple(env, state0, &tuple_argc, &tuple_argv) && (tuple_argc == 4)) { + /* A compatibility state term */ + ctx_res_copy.state = enif_make_tuple4(env, tuple_argv[0], argv[2], tuple_argv[2], tuple_argv[3]); + } + } else +#endif + if (!EVP_CipherInit_ex(ctx_res_copy.ctx, NULL, NULL, NULL, ivec_bin.data, -1)) + { + ret = EXCP_ERROR(env, "Can't set iv"); + goto err; + } + + get_update_args(env, &ctx_res_copy, argv[1], &ret); + } else + get_update_args(env, ctx_res, argv[1], &ret); + + err: + return ret; /* Both success and error */ } ERL_NIF_TERM ng_crypto_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Context, Data) - (Context, Data, IV) */ - int i; +{/* (Context, Data [, IV]) */ ErlNifBinary data_bin; - ERL_NIF_TERM new_argv[3]; - ASSERT(argc =< 3); + ASSERT(argc <= 3); - if (!enif_inspect_iolist_as_binary(env, argv[1], &data_bin)) - return ERROR_Str(env, "iodata expected as data"); + if (!enif_inspect_binary(env, argv[1], &data_bin)) + return EXCP_BADARG(env, "expected binary as data"); if (data_bin.size > INT_MAX) - return ERROR_Str(env, "to long data"); - - for (i=0; i<argc; i++) new_argv[i] = argv[i]; - new_argv[1] = enif_make_binary(env, &data_bin); + return EXCP_BADARG(env, "to long data"); /* Run long jobs on a dirty scheduler to not block the current emulator thread */ if (data_bin.size > MAX_BYTES_TO_NIF) { return enif_schedule_nif(env, "ng_crypto_update", ERL_NIF_DIRTY_JOB_CPU_BOUND, - ng_crypto_update, argc, new_argv); + ng_crypto_update, argc, argv); } - return ng_crypto_update(env, argc, new_argv); + return ng_crypto_update(env, argc, argv); +} + +/*************************************************************************/ +/* One shot */ +/*************************************************************************/ + +ERL_NIF_TERM ng_crypto_one_shot(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Cipher, Key, IVec, Data, Encrypt) */ + struct evp_cipher_ctx ctx_res; + const struct cipher_type_t *cipherp; + ERL_NIF_TERM ret; + + if (!get_init_args(env, &ctx_res, argv[0], argv[1], argv[2], argv[4], &cipherp, &ret)) + goto ret; + + get_update_args(env, &ctx_res, argv[3], &ret); + + ret: + if (ctx_res.ctx) + EVP_CIPHER_CTX_free(ctx_res.ctx); + return ret; } +ERL_NIF_TERM ng_crypto_one_shot_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Cipher, Key, IVec, Data, Encrypt) % if no IV for the Cipher, set IVec = <<>> + */ + ErlNifBinary data_bin; + + ASSERT(argc == 5); + + if (!enif_inspect_binary(env, argv[3], &data_bin)) + return EXCP_BADARG(env, "expected binary as data"); + + if (data_bin.size > INT_MAX) + return EXCP_BADARG(env, "to long data"); + + /* Run long jobs on a dirty scheduler to not block the current emulator thread */ + if (data_bin.size > MAX_BYTES_TO_NIF) { + return enif_schedule_nif(env, "ng_crypto_one_shot", + ERL_NIF_DIRTY_JOB_CPU_BOUND, + ng_crypto_one_shot, argc, argv); + } + + return ng_crypto_one_shot(env, argc, argv); +} diff --git a/lib/crypto/c_src/api_ng.h b/lib/crypto/c_src/api_ng.h index a3b40fe7fc..5c7d9af3c5 100644 --- a/lib/crypto/c_src/api_ng.h +++ b/lib/crypto/c_src/api_ng.h @@ -25,5 +25,6 @@ ERL_NIF_TERM ng_crypto_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM ng_crypto_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM ng_crypto_one_shot_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); #endif /* E_AES_H__ */ diff --git a/lib/crypto/c_src/atoms.c b/lib/crypto/c_src/atoms.c index 2e417da7f4..114e3c1985 100644 --- a/lib/crypto/c_src/atoms.c +++ b/lib/crypto/c_src/atoms.c @@ -33,6 +33,7 @@ ERL_NIF_TERM atom_undefined; ERL_NIF_TERM atom_ok; ERL_NIF_TERM atom_none; ERL_NIF_TERM atom_notsup; +ERL_NIF_TERM atom_badarg; ERL_NIF_TERM atom_digest; #ifdef FIPS_SUPPORT ERL_NIF_TERM atom_enabled; @@ -41,6 +42,18 @@ ERL_NIF_TERM atom_not_enabled; ERL_NIF_TERM atom_not_supported; #endif +ERL_NIF_TERM atom_type; +ERL_NIF_TERM atom_size; +ERL_NIF_TERM atom_block_size; +ERL_NIF_TERM atom_key_length; +ERL_NIF_TERM atom_iv_length; +ERL_NIF_TERM atom_mode; +ERL_NIF_TERM atom_ecb_mode; +ERL_NIF_TERM atom_cbc_mode; +ERL_NIF_TERM atom_cfb_mode; +ERL_NIF_TERM atom_ofb_mode; +ERL_NIF_TERM atom_stream_cipher; + #if defined(HAVE_EC) ERL_NIF_TERM atom_prime_field; ERL_NIF_TERM atom_characteristic_two_field; @@ -138,8 +151,21 @@ int init_atoms(ErlNifEnv *env, const ERL_NIF_TERM fips_mode, const ERL_NIF_TERM atom_ok = enif_make_atom(env,"ok"); atom_none = enif_make_atom(env,"none"); atom_notsup = enif_make_atom(env,"notsup"); + atom_badarg = enif_make_atom(env,"badarg"); atom_digest = enif_make_atom(env,"digest"); + atom_type = enif_make_atom(env,"type"); + atom_size = enif_make_atom(env,"size"); + atom_block_size = enif_make_atom(env,"block_size"); + atom_key_length = enif_make_atom(env,"key_length"); + atom_iv_length = enif_make_atom(env,"iv_length"); + atom_mode = enif_make_atom(env,"mode"); + atom_ecb_mode = enif_make_atom(env,"ecb_mode"); + atom_cbc_mode = enif_make_atom(env,"cbc_mode"); + atom_cfb_mode = enif_make_atom(env,"cfb_mode"); + atom_ofb_mode = enif_make_atom(env,"ofb_mode"); + atom_stream_cipher = enif_make_atom(env,"stream_cipher"); + #if defined(HAVE_EC) atom_prime_field = enif_make_atom(env,"prime_field"); atom_characteristic_two_field = enif_make_atom(env,"characteristic_two_field"); diff --git a/lib/crypto/c_src/atoms.h b/lib/crypto/c_src/atoms.h index f15523d865..fc46d838aa 100644 --- a/lib/crypto/c_src/atoms.h +++ b/lib/crypto/c_src/atoms.h @@ -37,6 +37,7 @@ extern ERL_NIF_TERM atom_undefined; extern ERL_NIF_TERM atom_ok; extern ERL_NIF_TERM atom_none; extern ERL_NIF_TERM atom_notsup; +extern ERL_NIF_TERM atom_badarg; extern ERL_NIF_TERM atom_digest; #ifdef FIPS_SUPPORT extern ERL_NIF_TERM atom_enabled; @@ -45,6 +46,18 @@ extern ERL_NIF_TERM atom_not_enabled; extern ERL_NIF_TERM atom_not_supported; #endif +extern ERL_NIF_TERM atom_type; +extern ERL_NIF_TERM atom_size; +extern ERL_NIF_TERM atom_block_size; +extern ERL_NIF_TERM atom_key_length; +extern ERL_NIF_TERM atom_iv_length; +extern ERL_NIF_TERM atom_mode; +extern ERL_NIF_TERM atom_ecb_mode; +extern ERL_NIF_TERM atom_cbc_mode; +extern ERL_NIF_TERM atom_cfb_mode; +extern ERL_NIF_TERM atom_ofb_mode; +extern ERL_NIF_TERM atom_stream_cipher; + #if defined(HAVE_EC) extern ERL_NIF_TERM atom_prime_field; extern ERL_NIF_TERM atom_characteristic_two_field; diff --git a/lib/crypto/c_src/block.c b/lib/crypto/c_src/block.c deleted file mode 100644 index 0a4fd72623..0000000000 --- a/lib/crypto/c_src/block.c +++ /dev/null @@ -1,149 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2010-2018. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -#include "block.h" -#include "aes.h" -#include "cipher.h" - -ERL_NIF_TERM block_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Type, Key, Ivec, Text, IsEncrypt) or (Type, Key, Text, IsEncrypt) */ - const struct cipher_type_t *cipherp; - const EVP_CIPHER *cipher; - ErlNifBinary key, ivec, text; - EVP_CIPHER_CTX *ctx = NULL; - ERL_NIF_TERM ret; - unsigned char *out; - int ivec_size, out_size = 0; - int cipher_len; - - ASSERT(argc == 4 || argc == 5); - - if (!enif_inspect_iolist_as_binary(env, argv[1], &key)) - goto bad_arg; - if (key.size > INT_MAX) - goto bad_arg; - if ((cipherp = get_cipher_type(argv[0], key.size)) == NULL) - goto bad_arg; - if (cipherp->flags & (NON_EVP_CIPHER | AEAD_CIPHER)) - goto bad_arg; - if (!enif_inspect_iolist_as_binary(env, argv[argc - 2], &text)) - goto bad_arg; - if (text.size > INT_MAX) - goto bad_arg; - - if (FORBIDDEN_IN_FIPS(cipherp)) - return enif_raise_exception(env, atom_notsup); - if ((cipher = cipherp->cipher.p) == NULL) - return enif_raise_exception(env, atom_notsup); - - if (cipherp->flags & AES_CFBx) { - if (argv[0] == atom_aes_cfb8 - && (key.size == 24 || key.size == 32)) { - /* Why do EVP_CIPHER_CTX_set_key_length() fail on these key sizes? - * Fall back on low level API - */ - return aes_cfb_8_crypt(env, argc-1, argv+1); - } - else if (argv[0] == atom_aes_cfb128 - && (key.size == 24 || key.size == 32)) { - /* Why do EVP_CIPHER_CTX_set_key_length() fail on these key sizes? - * Fall back on low level API - */ - return aes_cfb_128_crypt_nif(env, argc-1, argv+1); - } - } - - ivec_size = EVP_CIPHER_iv_length(cipher); - -#ifdef HAVE_ECB_IVEC_BUG - if (cipherp->flags & ECB_BUG_0_9_8L) - ivec_size = 0; /* 0.9.8l returns faulty ivec_size */ -#endif - - if (ivec_size < 0) - goto bad_arg; - - if ((cipher_len = EVP_CIPHER_block_size(cipher)) < 0) - goto bad_arg; - if (text.size % (size_t)cipher_len != 0) - goto bad_arg; - - if (ivec_size == 0) { - if (argc != 4) - goto bad_arg; - } else { - if (argc != 5) - goto bad_arg; - if (!enif_inspect_iolist_as_binary(env, argv[2], &ivec)) - goto bad_arg; - if (ivec.size != (size_t)ivec_size) - goto bad_arg; - } - - if ((out = enif_make_new_binary(env, text.size, &ret)) == NULL) - goto err; - if ((ctx = EVP_CIPHER_CTX_new()) == NULL) - goto err; - - if (!EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL, - (argv[argc - 1] == atom_true))) - goto err; - if (!EVP_CIPHER_CTX_set_key_length(ctx, (int)key.size)) - goto err; - - if (EVP_CIPHER_type(cipher) == NID_rc2_cbc) { - if (key.size > INT_MAX / 8) - goto err; - if (!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_SET_RC2_KEY_BITS, (int)key.size * 8, NULL)) - goto err; - } - - if (!EVP_CipherInit_ex(ctx, NULL, NULL, key.data, - ivec_size ? ivec.data : NULL, -1)) - goto err; - if (!EVP_CIPHER_CTX_set_padding(ctx, 0)) - goto err; - - /* OpenSSL 0.9.8h asserts text.size > 0 */ - if (text.size > 0) { - if (!EVP_CipherUpdate(ctx, out, &out_size, text.data, (int)text.size)) - goto err; - if (ASSERT(out_size == text.size), 0) - goto err; - if (!EVP_CipherFinal_ex(ctx, out + out_size, &out_size)) - goto err; - } - - ASSERT(out_size == 0); - CONSUME_REDS(env, text); - goto done; - - bad_arg: - ret = enif_make_badarg(env); - goto done; - - err: - ret = enif_raise_exception(env, atom_notsup); - - done: - if (ctx) - EVP_CIPHER_CTX_free(ctx); - return ret; -} diff --git a/lib/crypto/c_src/block.h b/lib/crypto/c_src/block.h deleted file mode 100644 index cc5e78ce12..0000000000 --- a/lib/crypto/c_src/block.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2010-2018. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -#ifndef E_BLOCK_H__ -#define E_BLOCK_H__ 1 - -#include "common.h" - -ERL_NIF_TERM block_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); - -#endif /* E_BLOCK_H__ */ diff --git a/lib/crypto/c_src/chacha20.c b/lib/crypto/c_src/chacha20.c deleted file mode 100644 index cfcc395dca..0000000000 --- a/lib/crypto/c_src/chacha20.c +++ /dev/null @@ -1,124 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2010-2018. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -#include "chacha20.h" -#include "cipher.h" - -ERL_NIF_TERM chacha20_stream_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Key, IV) */ -#if defined(HAVE_CHACHA20) - ErlNifBinary key_bin, ivec_bin; - struct evp_cipher_ctx *ctx = NULL; - const EVP_CIPHER *cipher; - ERL_NIF_TERM ret; - - ASSERT(argc == 2); - - if (!enif_inspect_iolist_as_binary(env, argv[0], &key_bin)) - goto bad_arg; - if (key_bin.size != 32) - goto bad_arg; - if (!enif_inspect_binary(env, argv[1], &ivec_bin)) - goto bad_arg; - if (ivec_bin.size != 16) - goto bad_arg; - - cipher = EVP_chacha20(); - - if ((ctx = enif_alloc_resource(evp_cipher_ctx_rtype, sizeof(struct evp_cipher_ctx))) == NULL) - goto err; - if ((ctx->ctx = EVP_CIPHER_CTX_new()) == NULL) - goto err; - - if (EVP_CipherInit_ex(ctx->ctx, cipher, NULL, - key_bin.data, ivec_bin.data, 1) != 1) - goto err; - if (EVP_CIPHER_CTX_set_padding(ctx->ctx, 0) != 1) - goto err; - - ret = enif_make_resource(env, ctx); - goto done; - - bad_arg: - return enif_make_badarg(env); - - err: - ret = enif_make_badarg(env); - - done: - if (ctx) - enif_release_resource(ctx); - return ret; - -#else - return enif_raise_exception(env, atom_notsup); -#endif -} - -ERL_NIF_TERM chacha20_stream_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (State, Data) */ -#if defined(HAVE_CHACHA20) - struct evp_cipher_ctx *ctx = NULL, *new_ctx = NULL; - ErlNifBinary data_bin; - ERL_NIF_TERM ret, cipher_term; - unsigned char *out; - int outl = 0; - - ASSERT(argc == 2); - - if (!enif_get_resource(env, argv[0], evp_cipher_ctx_rtype, (void**)&ctx)) - goto bad_arg; - if (!enif_inspect_iolist_as_binary(env, argv[1], &data_bin)) - goto bad_arg; - if (data_bin.size > INT_MAX) - goto bad_arg; - - if ((new_ctx = enif_alloc_resource(evp_cipher_ctx_rtype, sizeof(struct evp_cipher_ctx))) == NULL) - goto err; - if ((new_ctx->ctx = EVP_CIPHER_CTX_new()) == NULL) - goto err; - - if (EVP_CIPHER_CTX_copy(new_ctx->ctx, ctx->ctx) != 1) - goto err; - if ((out = enif_make_new_binary(env, data_bin.size, &cipher_term)) == NULL) - goto err; - if (EVP_CipherUpdate(new_ctx->ctx, out, &outl, data_bin.data, (int)data_bin.size) != 1) - goto err; - ASSERT(outl >= 0 && (size_t)outl == data_bin.size); - - ret = enif_make_tuple2(env, enif_make_resource(env, new_ctx), cipher_term); - CONSUME_REDS(env, data_bin); - goto done; - - bad_arg: - return enif_make_badarg(env); - - err: - ret = enif_make_badarg(env); - - done: - if (new_ctx) - enif_release_resource(new_ctx); - return ret; - -#else - return enif_raise_exception(env, atom_notsup); -#endif -} diff --git a/lib/crypto/c_src/chacha20.h b/lib/crypto/c_src/chacha20.h deleted file mode 100644 index 7e2ccae2bb..0000000000 --- a/lib/crypto/c_src/chacha20.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2010-2018. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -#ifndef E_CHACHA20_H__ -#define E_CHACHA20_H__ 1 - -#include "common.h" - -ERL_NIF_TERM chacha20_stream_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -ERL_NIF_TERM chacha20_stream_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); - -#endif /* E_CHACHA20_H__ */ diff --git a/lib/crypto/c_src/cipher.c b/lib/crypto/c_src/cipher.c index f8e44b228a..5c57898c50 100644 --- a/lib/crypto/c_src/cipher.c +++ b/lib/crypto/c_src/cipher.c @@ -67,14 +67,26 @@ static struct cipher_type_t cipher_types[] = {{"aes_cfb8"}, {&EVP_aes_192_cfb8}, 24, NO_FIPS_CIPHER | AES_CFBx}, {{"aes_cfb8"}, {&EVP_aes_256_cfb8}, 32, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_128_cfb8"}, {&EVP_aes_128_cfb8}, 16, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_192_cfb8"}, {&EVP_aes_192_cfb8}, 24, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_256_cfb8"}, {&EVP_aes_256_cfb8}, 32, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_cfb128"}, {&EVP_aes_128_cfb128}, 16, NO_FIPS_CIPHER | AES_CFBx}, {{"aes_cfb128"}, {&EVP_aes_192_cfb128}, 24, NO_FIPS_CIPHER | AES_CFBx}, {{"aes_cfb128"}, {&EVP_aes_256_cfb128}, 32, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_128_cfb128"}, {&EVP_aes_128_cfb128}, 16, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_192_cfb128"}, {&EVP_aes_192_cfb128}, 24, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_256_cfb128"}, {&EVP_aes_256_cfb128}, 32, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_ecb"}, {&EVP_aes_128_ecb}, 16, ECB_BUG_0_9_8L}, {{"aes_ecb"}, {&EVP_aes_192_ecb}, 24, ECB_BUG_0_9_8L}, {{"aes_ecb"}, {&EVP_aes_256_ecb}, 32, ECB_BUG_0_9_8L}, + {{"aes_128_ecb"}, {&EVP_aes_128_ecb}, 16, ECB_BUG_0_9_8L}, + {{"aes_192_ecb"}, {&EVP_aes_192_ecb}, 24, ECB_BUG_0_9_8L}, + {{"aes_256_ecb"}, {&EVP_aes_256_ecb}, 32, ECB_BUG_0_9_8L}, + #if defined(HAVE_EVP_AES_CTR) {{"aes_128_ctr"}, {&EVP_aes_128_ctr}, 16, 0}, {{"aes_192_ctr"}, {&EVP_aes_192_ctr}, 24, 0}, @@ -86,7 +98,9 @@ static struct cipher_type_t cipher_types[] = {{"aes_128_ctr"}, {NULL}, 16, AES_CTR_COMPAT}, {{"aes_192_ctr"}, {NULL}, 24, AES_CTR_COMPAT}, {{"aes_256_ctr"}, {NULL}, 32, AES_CTR_COMPAT}, - {{"aes_ctr"}, {NULL}, 0, AES_CTR_COMPAT}, + {{"aes_ctr"}, {NULL}, 16, AES_CTR_COMPAT}, + {{"aes_ctr"}, {NULL}, 24, AES_CTR_COMPAT}, + {{"aes_ctr"}, {NULL}, 32, AES_CTR_COMPAT}, #endif #if defined(HAVE_CHACHA20) @@ -150,6 +164,11 @@ static void evp_cipher_ctx_dtor(ErlNifEnv* env, struct evp_cipher_ctx* ctx) { if (ctx->ctx) EVP_CIPHER_CTX_free(ctx->ctx); + +#if !defined(HAVE_EVP_AES_CTR) + if (ctx->env) + enif_free_env(ctx->env); +#endif } int init_cipher_ctx(ErlNifEnv *env) { @@ -207,6 +226,87 @@ int cmp_cipher_types(const void *keyp, const void *elemp) { } +ERL_NIF_TERM cipher_info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Type) */ + const struct cipher_type_t *cipherp; + const EVP_CIPHER *cipher; + ERL_NIF_TERM ret, ret_mode; + unsigned type; + unsigned long mode; + + if ((cipherp = get_cipher_type_no_key(argv[0])) == NULL) + return enif_make_badarg(env); + + if (FORBIDDEN_IN_FIPS(cipherp)) + return enif_raise_exception(env, atom_notsup); + if ((cipher = cipherp->cipher.p) == NULL) + return enif_raise_exception(env, atom_notsup); + + ret = enif_make_new_map(env); + + type = EVP_CIPHER_type(cipher); + enif_make_map_put(env, ret, atom_type, + type == NID_undef ? atom_undefined : enif_make_int(env, type), + &ret); + + enif_make_map_put(env, ret, atom_key_length, + enif_make_int(env, EVP_CIPHER_key_length(cipher)), &ret); + enif_make_map_put(env, ret, atom_iv_length, + enif_make_int(env, EVP_CIPHER_iv_length(cipher)), &ret); + enif_make_map_put(env, ret, atom_block_size, + enif_make_int(env, EVP_CIPHER_block_size(cipher)), &ret); + + mode = EVP_CIPHER_mode(cipher); + switch (mode) { + case EVP_CIPH_ECB_MODE: + ret_mode = atom_ecb_mode; + break; + + case EVP_CIPH_CBC_MODE: + ret_mode = atom_cbc_mode; + break; + + case EVP_CIPH_CFB_MODE: + ret_mode = atom_cfb_mode; + break; + + case EVP_CIPH_OFB_MODE: + ret_mode = atom_ofb_mode; + break; + + case EVP_CIPH_STREAM_CIPHER: + ret_mode = atom_stream_cipher; + break; + + default: + ret_mode = atom_undefined; + break; + } + + enif_make_map_put(env, ret, atom_mode, ret_mode, &ret); + + return ret; +} + +const struct cipher_type_t* get_cipher_type_no_key(ERL_NIF_TERM type) +{ + struct cipher_type_t key; + + key.type.atom = type; + + return bsearch(&key, cipher_types, num_cipher_types, sizeof(cipher_types[0]), cmp_cipher_types_no_key); +} + +int cmp_cipher_types_no_key(const void *keyp, const void *elemp) { + const struct cipher_type_t *key = keyp; + const struct cipher_type_t *elem = elemp; + + if (key->type.atom < elem->type.atom) return -1; + else if (key->type.atom > elem->type.atom) return 1; + else /* key->type.atom == elem->type.atom */ return 0; +} + + ERL_NIF_TERM cipher_types_as_list(ErlNifEnv* env) { struct cipher_type_t* p; diff --git a/lib/crypto/c_src/cipher.h b/lib/crypto/c_src/cipher.h index 6b43afea99..b94873940f 100644 --- a/lib/crypto/c_src/cipher.h +++ b/lib/crypto/c_src/cipher.h @@ -59,14 +59,23 @@ struct cipher_type_t { extern ErlNifResourceType* evp_cipher_ctx_rtype; struct evp_cipher_ctx { EVP_CIPHER_CTX* ctx; + int iv_len; +#if !defined(HAVE_EVP_AES_CTR) + ErlNifEnv* env; + ERL_NIF_TERM state; +#endif }; +ERL_NIF_TERM cipher_info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + int init_cipher_ctx(ErlNifEnv *env); void init_cipher_types(ErlNifEnv* env); +const struct cipher_type_t* get_cipher_type_no_key(ERL_NIF_TERM type); const struct cipher_type_t* get_cipher_type(ERL_NIF_TERM type, size_t key_len); int cmp_cipher_types(const void *keyp, const void *elemp); +int cmp_cipher_types_no_key(const void *keyp, const void *elemp); ERL_NIF_TERM cipher_types_as_list(ErlNifEnv* env); diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c index 06439c34b2..4aed06a489 100644 --- a/lib/crypto/c_src/crypto.c +++ b/lib/crypto/c_src/crypto.c @@ -29,9 +29,7 @@ #include "aes.h" #include "algorithms.h" #include "api_ng.h" -#include "block.h" #include "bn.h" -#include "chacha20.h" #include "cipher.h" #include "cmac.h" #include "dh.h" @@ -50,7 +48,6 @@ #include "pkey.h" #include "poly1305.h" #include "rand.h" -#include "rc4.h" #include "rsa.h" #include "srp.h" @@ -67,6 +64,7 @@ static ErlNifFunc nif_funcs[] = { {"info_fips", 0, info_fips, 0}, {"enable_fips_mode", 1, enable_fips_mode, 0}, {"algorithms", 0, algorithms, 0}, + {"hash_info", 1, hash_info_nif, 0}, {"hash_nif", 2, hash_nif, 0}, {"hash_init_nif", 1, hash_init_nif, 0}, {"hash_update_nif", 2, hash_update_nif, 0}, @@ -78,22 +76,17 @@ static ErlNifFunc nif_funcs[] = { {"hmac_final_nif", 1, hmac_final_nif, 0}, {"hmac_final_nif", 2, hmac_final_nif, 0}, {"cmac_nif", 3, cmac_nif, 0}, - {"block_crypt_nif", 5, block_crypt_nif, 0}, - {"block_crypt_nif", 4, block_crypt_nif, 0}, + {"cipher_info_nif", 1, cipher_info_nif, 0}, {"aes_ige_crypt_nif", 4, aes_ige_crypt_nif, 0}, - {"aes_ctr_stream_init", 2, aes_ctr_stream_init, 0}, - {"aes_ctr_stream_encrypt", 2, aes_ctr_stream_encrypt, 0}, - {"aes_ctr_stream_decrypt", 2, aes_ctr_stream_encrypt, 0}, {"ng_crypto_init_nif", 4, ng_crypto_init_nif, 0}, {"ng_crypto_update_nif", 2, ng_crypto_update_nif, 0}, {"ng_crypto_update_nif", 3, ng_crypto_update_nif, 0}, + {"ng_crypto_one_shot_nif", 5, ng_crypto_one_shot_nif, 0}, {"strong_rand_bytes_nif", 1, strong_rand_bytes_nif, 0}, {"strong_rand_range_nif", 1, strong_rand_range_nif, 0}, {"rand_uniform_nif", 2, rand_uniform_nif, 0}, {"mod_exp_nif", 4, mod_exp_nif, 0}, {"do_exor", 2, do_exor, 0}, - {"rc4_set_key", 1, rc4_set_key, 0}, - {"rc4_encrypt_with_state", 2, rc4_encrypt_with_state, 0}, {"pkey_sign_nif", 5, pkey_sign_nif, 0}, {"pkey_verify_nif", 6, pkey_verify_nif, 0}, {"pkey_crypt_nif", 6, pkey_crypt_nif, 0}, @@ -115,10 +108,6 @@ static ErlNifFunc nif_funcs[] = { {"aead_encrypt", 6, aead_encrypt, 0}, {"aead_decrypt", 6, aead_decrypt, 0}, - {"chacha20_stream_init", 2, chacha20_stream_init, 0}, - {"chacha20_stream_encrypt", 2, chacha20_stream_crypt, 0}, - {"chacha20_stream_decrypt", 2, chacha20_stream_crypt, 0}, - {"poly1305_nif", 2, poly1305_nif, 0}, {"engine_by_id_nif", 1, engine_by_id_nif, 0}, diff --git a/lib/crypto/c_src/hash.c b/lib/crypto/c_src/hash.c index 457e9d071a..0a9f64acef 100644 --- a/lib/crypto/c_src/hash.c +++ b/lib/crypto/c_src/hash.c @@ -61,6 +61,32 @@ int init_hash_ctx(ErlNifEnv* env) { #endif } +ERL_NIF_TERM hash_info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Type) */ + struct digest_type_t *digp = NULL; + const EVP_MD *md; + ERL_NIF_TERM ret; + + ASSERT(argc == 1); + + if ((digp = get_digest_type(argv[0])) == NULL) + return enif_make_badarg(env); + + if ((md = digp->md.p) == NULL) + return atom_notsup; + + ret = enif_make_new_map(env); + + enif_make_map_put(env, ret, atom_type, + enif_make_int(env, EVP_MD_type(md)), &ret); + enif_make_map_put(env, ret, atom_size, + enif_make_int(env, EVP_MD_size(md)), &ret); + enif_make_map_put(env, ret, atom_block_size, + enif_make_int(env, EVP_MD_block_size(md)), &ret); + + return ret; +} + ERL_NIF_TERM hash_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {/* (Type, Data) */ struct digest_type_t *digp = NULL; diff --git a/lib/crypto/c_src/hash.h b/lib/crypto/c_src/hash.h index 8bae07f39a..92a25cedb7 100644 --- a/lib/crypto/c_src/hash.h +++ b/lib/crypto/c_src/hash.h @@ -25,6 +25,7 @@ int init_hash_ctx(ErlNifEnv *env); +ERL_NIF_TERM hash_info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM hash_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM hash_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM hash_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); diff --git a/lib/crypto/c_src/openssl_config.h b/lib/crypto/c_src/openssl_config.h index 45144a0c25..46868cb987 100644 --- a/lib/crypto/c_src/openssl_config.h +++ b/lib/crypto/c_src/openssl_config.h @@ -109,6 +109,7 @@ #ifndef HAS_LIBRESSL # if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) # define HAS_EVP_PKEY_CTX +# define HAVE_EVP_CIPHER_CTX_COPY # endif #endif @@ -203,12 +204,17 @@ #if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,1,0) # ifndef HAS_LIBRESSL -# define HAVE_CHACHA20 # define HAVE_CHACHA20_POLY1305 # define HAVE_RSA_OAEP_MD # endif #endif +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION(1,1,0,'d') +# ifndef HAS_LIBRESSL +# define HAVE_CHACHA20 +# endif +#endif + // OPENSSL_VERSION_NUMBER >= 1.1.1-pre8 #if OPENSSL_VERSION_NUMBER >= (PACKED_OPENSSL_VERSION_PLAIN(1,1,1)-7) # ifndef HAS_LIBRESSL diff --git a/lib/crypto/c_src/pkey.c b/lib/crypto/c_src/pkey.c index 567e8df08a..638bb588fa 100644 --- a/lib/crypto/c_src/pkey.c +++ b/lib/crypto/c_src/pkey.c @@ -70,10 +70,13 @@ static int get_pkey_digest_type(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_ if (type == atom_none && algorithm == atom_rsa) return PKEY_OK; + if (algorithm == atom_eddsa) { #ifdef HAVE_EDDSA - if (algorithm == atom_eddsa) - return PKEY_OK; + if (!FIPS_mode()) return PKEY_OK; +#else + return PKEY_NOTSUP; #endif + } if ((digp = get_digest_type(type)) == NULL) return PKEY_BADARG; if (digp->md.p == NULL) @@ -312,11 +315,16 @@ static int get_pkey_private_key(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_ return PKEY_NOTSUP; #endif } else if (algorithm == atom_eddsa) { -#if defined(HAVE_EDDSA) - if (!get_eddsa_key(env, 0, key, &result)) - goto err; +#ifdef HAVE_EDDSA + if (!FIPS_mode()) + { + if (!get_eddsa_key(env, 0, key, &result)) + goto err; + else + goto done; // Not nice.... + } #else - return PKEY_NOTSUP; + return PKEY_NOTSUP; #endif } else if (algorithm == atom_dss) { if ((dsa = DSA_new()) == NULL) @@ -432,10 +440,11 @@ static int get_pkey_public_key(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_T return PKEY_NOTSUP; #endif } else if (algorithm == atom_eddsa) { -#if defined(HAVE_EDDSA) - if (!get_eddsa_key(env, 1, key, &result)) - goto err; - +#ifdef HAVE_EDDSA + if (!FIPS_mode()) { + if (!get_eddsa_key(env, 1, key, &result)) + goto err; + } #else return PKEY_NOTSUP; #endif @@ -516,7 +525,6 @@ ERL_NIF_TERM pkey_sign_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) /*char buf[1024]; enif_get_atom(env,argv[0],buf,1024,ERL_NIF_LATIN1); printf("algo=%s ",buf); enif_get_atom(env,argv[1],buf,1024,ERL_NIF_LATIN1); printf("hash=%s ",buf); -printf("\r\n"); */ #ifndef HAS_ENGINE_SUPPORT @@ -583,22 +591,24 @@ printf("\r\n"); if (argv[0] == atom_eddsa) { #ifdef HAVE_EDDSA - if ((mdctx = EVP_MD_CTX_new()) == NULL) - goto err; + if (!FIPS_mode()) { + if ((mdctx = EVP_MD_CTX_new()) == NULL) + goto err; - if (EVP_DigestSignInit(mdctx, NULL, NULL, NULL, pkey) != 1) - goto err; - if (EVP_DigestSign(mdctx, NULL, &siglen, tbs, tbslen) != 1) - goto err; - if (!enif_alloc_binary(siglen, &sig_bin)) - goto err; - sig_bin_alloc = 1; + if (EVP_DigestSignInit(mdctx, NULL, NULL, NULL, pkey) != 1) + goto err; + if (EVP_DigestSign(mdctx, NULL, &siglen, tbs, tbslen) != 1) + goto err; + if (!enif_alloc_binary(siglen, &sig_bin)) + goto err; + sig_bin_alloc = 1; - if (EVP_DigestSign(mdctx, sig_bin.data, &siglen, tbs, tbslen) != 1) - goto bad_key; -#else - goto bad_arg; + if (EVP_DigestSign(mdctx, sig_bin.data, &siglen, tbs, tbslen) != 1) + goto bad_key; + } + else #endif + goto notsup; } else { if (EVP_PKEY_sign(ctx, NULL, &siglen, tbs, tbslen) != 1) goto err; @@ -709,6 +719,11 @@ printf("\r\n"); if (pkey) EVP_PKEY_free(pkey); +#ifdef HAVE_EDDSA + if (mdctx) + EVP_MD_CTX_free(mdctx); +#endif + return ret; } @@ -805,16 +820,18 @@ ERL_NIF_TERM pkey_verify_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[] if (argv[0] == atom_eddsa) { #ifdef HAVE_EDDSA - if ((mdctx = EVP_MD_CTX_new()) == NULL) - goto err; - - if (EVP_DigestVerifyInit(mdctx, NULL, NULL, NULL, pkey) != 1) - goto err; + if (!FIPS_mode()) { + if ((mdctx = EVP_MD_CTX_new()) == NULL) + goto err; - result = EVP_DigestVerify(mdctx, sig_bin.data, sig_bin.size, tbs, tbslen); -#else - goto bad_arg; + if (EVP_DigestVerifyInit(mdctx, NULL, NULL, NULL, pkey) != 1) + goto err; + + result = EVP_DigestVerify(mdctx, sig_bin.data, sig_bin.size, tbs, tbslen); + } + else #endif + goto notsup; } else { if (md != NULL) { ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, EVP_MD_size(md)); diff --git a/lib/crypto/c_src/poly1305.c b/lib/crypto/c_src/poly1305.c index db3433dce3..76579c0a29 100644 --- a/lib/crypto/c_src/poly1305.c +++ b/lib/crypto/c_src/poly1305.c @@ -85,6 +85,6 @@ ERL_NIF_TERM poly1305_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) return ret; #else - return atom_notsup; + return enif_raise_exception(env, atom_notsup); #endif } diff --git a/lib/crypto/c_src/rc4.c b/lib/crypto/c_src/rc4.c deleted file mode 100644 index e423661097..0000000000 --- a/lib/crypto/c_src/rc4.c +++ /dev/null @@ -1,92 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2010-2018. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -#include "rc4.h" - -ERL_NIF_TERM rc4_set_key(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Key) */ -#ifndef OPENSSL_NO_RC4 - ErlNifBinary key; - ERL_NIF_TERM ret; - RC4_KEY *rc4_key; - - CHECK_NO_FIPS_MODE(); - - ASSERT(argc == 1); - - if (!enif_inspect_iolist_as_binary(env, argv[0], &key)) - goto bad_arg; - if (key.size > INT_MAX) - goto bad_arg; - - if ((rc4_key = (RC4_KEY*)enif_make_new_binary(env, sizeof(RC4_KEY), &ret)) == NULL) - goto err; - - RC4_set_key(rc4_key, (int)key.size, key.data); - return ret; - - bad_arg: - err: - return enif_make_badarg(env); - -#else - return enif_raise_exception(env, atom_notsup); -#endif -} - -ERL_NIF_TERM rc4_encrypt_with_state(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (State, Data) */ -#ifndef OPENSSL_NO_RC4 - ErlNifBinary state, data; - RC4_KEY* rc4_key; - ERL_NIF_TERM new_state, new_data; - unsigned char *outp; - - CHECK_NO_FIPS_MODE(); - - ASSERT(argc == 2); - - if (!enif_inspect_iolist_as_binary(env, argv[0], &state)) - goto bad_arg; - if (state.size != sizeof(RC4_KEY)) - goto bad_arg; - if (!enif_inspect_iolist_as_binary(env, argv[1], &data)) - goto bad_arg; - - if ((rc4_key = (RC4_KEY*)enif_make_new_binary(env, sizeof(RC4_KEY), &new_state)) == NULL) - goto err; - if ((outp = enif_make_new_binary(env, data.size, &new_data)) == NULL) - goto err; - - memcpy(rc4_key, state.data, sizeof(RC4_KEY)); - RC4(rc4_key, data.size, data.data, outp); - - CONSUME_REDS(env, data); - return enif_make_tuple2(env, new_state, new_data); - - bad_arg: - err: - return enif_make_badarg(env); - -#else - return enif_raise_exception(env, atom_notsup); -#endif -} - diff --git a/lib/crypto/c_src/rc4.h b/lib/crypto/c_src/rc4.h deleted file mode 100644 index 28bf674253..0000000000 --- a/lib/crypto/c_src/rc4.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2010-2018. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -#ifndef E_RC4_H__ -#define E_RC4_H__ 1 - -#include "common.h" - -ERL_NIF_TERM rc4_set_key(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -ERL_NIF_TERM rc4_encrypt_with_state(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); - -#endif /* E_RC4_H__ */ diff --git a/lib/crypto/doc/src/notes.xml b/lib/crypto/doc/src/notes.xml index 0a3f68ade2..c0b302734e 100644 --- a/lib/crypto/doc/src/notes.xml +++ b/lib/crypto/doc/src/notes.xml @@ -31,6 +31,38 @@ </header> <p>This document describes the changes made to the Crypto application.</p> +<section><title>Crypto 4.4.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixes a bug that caused <c>crypto:sign</c> and + <c>crypto:verify</c> to return the error message + <c>badarg</c> instead of <c>notsup</c> in one case. That + case was when signing or verifying with eddsa keys (that + is, ed15519 or ed448), but only when FIPS was supported + and enabled.</p> + <p> + Own Id: OTP-15634</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Added a crypto benchmark test suite.</p> + <p> + Own Id: OTP-15447</p> + </item> + </list> + </section> + +</section> + <section><title>Crypto 4.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index fe8390c5b8..5cf34f8069 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -24,6 +24,7 @@ -export([start/0, stop/0, info_lib/0, info_fips/0, supports/0, enable_fips_mode/1, version/0, bytes_to_integer/1]). +-export([cipher_info/1, hash_info/1]). -export([hash/2, hash_init/1, hash_update/2, hash_final/1]). -export([sign/4, sign/5, verify/5, verify/6]). -export([generate_key/2, generate_key/3, compute_key/4]). @@ -39,24 +40,27 @@ -export([rand_plugin_uniform/2]). -export([rand_cache_plugin_next/1]). -export([rand_uniform/2]). --export([block_encrypt/3, block_decrypt/3, block_encrypt/4, block_decrypt/4]). -export([next_iv/2, next_iv/3]). --export([stream_init/2, stream_init/3, stream_encrypt/2, stream_decrypt/2]). -export([public_encrypt/4, private_decrypt/4]). -export([private_encrypt/4, public_decrypt/4]). -export([privkey_to_pubkey/2]). -export([ec_curve/1, ec_curves/0]). -export([rand_seed/1]). -%% Experiment --export([crypto_init/4, - crypto_update/2, crypto_update/3, - %% Emulates old api: - crypto_stream_init/2, crypto_stream_init/3, - crypto_stream_encrypt/2, - crypto_stream_decrypt/2, - crypto_block_encrypt/3, crypto_block_encrypt/4, - crypto_block_decrypt/3, crypto_block_decrypt/4 +%% Old interface. Now implemented with the New interface +-export([stream_init/2, stream_init/3, + stream_encrypt/2, + stream_decrypt/2, + block_encrypt/3, block_encrypt/4, + block_decrypt/3, block_decrypt/4 + ]). + +%% New interface +-export([crypto_init/4, crypto_init/3, + crypto_update/2, + crypto_one_shot/5, + crypto_init_dyn_iv/3, + crypto_update_dyn_iv/3 ]). @@ -403,6 +407,11 @@ enable_fips_mode(_) -> ?nif_stub. -define(HASH_HASH_ALGORITHM, sha1() | sha2() | sha3() | blake2() | ripemd160 | compatibility_only_hash() ). +-spec hash_info(Type) -> map() when Type :: ?HASH_HASH_ALGORITHM. + +hash_info(Type) -> + notsup_to_error(hash_info_nif(Type)). + -spec hash(Type, Data) -> Digest when Type :: ?HASH_HASH_ALGORITHM, Data :: iodata(), Digest :: binary(). @@ -527,12 +536,24 @@ poly1305(Key, Data) -> %%%================================================================ %%% -%%% Encrypt/decrypt +%%% Encrypt/decrypt, The "Old API" %%% %%%================================================================ -%%%---- Block ciphers +-define(COMPAT(CALL), + try CALL + catch + error:{E,_Reason} when E==notsup ; E==badarg -> + error(E) + end). + +-spec cipher_info(Type) -> map() when Type :: block_cipher_with_iv() + | aead_cipher() + | block_cipher_without_iv(). +cipher_info(Type) -> + cipher_info_nif(Type). +%%%---- Block ciphers %%%---------------------------------------------------------------- -spec block_encrypt(Type::block_cipher_with_iv(), Key::key()|des3_key(), Ivec::binary(), PlainText::iodata()) -> binary(); (Type::aead_cipher(), Key::iodata(), Ivec::binary(), {AAD::binary(), PlainText::iodata()}) -> @@ -544,11 +565,6 @@ poly1305(Key, Data) -> block_encrypt(Type, Key, Ivec, Data) -> do_block_encrypt(alias(Type), Key, Ivec, Data). -do_block_encrypt(Type, Key0, Ivec, Data) when Type =:= des_ede3_cbc; - Type =:= des_ede3_cfb -> - Key = check_des3_key(Key0), - block_crypt_nif(Type, Key, Ivec, Data, true); - do_block_encrypt(Type, Key, Ivec, PlainText) when Type =:= aes_ige256 -> notsup_to_error(aes_ige_crypt_nif(Key, Ivec, PlainText, true)); @@ -565,14 +581,13 @@ do_block_encrypt(Type, Key, Ivec, Data) when Type =:= aes_gcm; end; do_block_encrypt(Type, Key, Ivec, PlainText) -> - block_crypt_nif(Type, Key, Ivec, PlainText, true). - + ?COMPAT(crypto_one_shot(Type, Key, Ivec, PlainText, true)). -spec block_encrypt(Type::block_cipher_without_iv(), Key::key(), PlainText::iodata()) -> binary(). block_encrypt(Type, Key, PlainText) -> - block_crypt_nif(alias(Type), Key, PlainText, true). + ?COMPAT(crypto_one_shot(Type, Key, <<>>, PlainText, true)). %%%---------------------------------------------------------------- %%%---------------------------------------------------------------- @@ -583,11 +598,6 @@ block_encrypt(Type, Key, PlainText) -> block_decrypt(Type, Key, Ivec, Data) -> do_block_decrypt(alias(Type), Key, Ivec, Data). -do_block_decrypt(Type, Key0, Ivec, Data) when Type =:= des_ede3_cbc; - Type =:= des_ede3_cfb -> - Key = check_des3_key(Key0), - block_crypt_nif(Type, Key, Ivec, Data, false); - do_block_decrypt(aes_ige256, Key, Ivec, Data) -> notsup_to_error(aes_ige_crypt_nif(Key, Ivec, Data, false)); @@ -597,14 +607,80 @@ do_block_decrypt(Type, Key, Ivec, {AAD, Data, Tag}) when Type =:= aes_gcm; aead_decrypt(Type, Key, Ivec, AAD, Data, Tag); do_block_decrypt(Type, Key, Ivec, Data) -> - block_crypt_nif(Type, Key, Ivec, Data, false). - + ?COMPAT(crypto_one_shot(Type, Key, Ivec, Data, false)). -spec block_decrypt(Type::block_cipher_without_iv(), Key::key(), Data::iodata()) -> binary(). block_decrypt(Type, Key, Data) -> - block_crypt_nif(alias(Type), Key, Data, false). + ?COMPAT(crypto_one_shot(Type, Key, <<>>, Data, false)). + +%%%-------- Stream ciphers API + +-opaque stream_state() :: {stream_cipher(), + crypto_state() | {crypto_state(),flg_undefined} + }. + +-type stream_cipher() :: stream_cipher_iv() | stream_cipher_no_iv() . +-type stream_cipher_no_iv() :: rc4 . +-type stream_cipher_iv() :: aes_ctr + | aes_128_ctr + | aes_192_ctr + | aes_256_ctr + | chacha20 . + +%%%---- stream_init +-spec stream_init(Type, Key, IVec) -> State | no_return() + when Type :: stream_cipher_iv(), + Key :: iodata(), + IVec ::binary(), + State :: stream_state() . +stream_init(Type, Key, IVec) when is_binary(IVec) -> + Ref = ?COMPAT(ng_crypto_init_nif(alias(Type), + iolist_to_binary(Key), iolist_to_binary(IVec), + undefined) + ), + {Type, {Ref,flg_undefined}}. + + +-spec stream_init(Type, Key) -> State | no_return() + when Type :: stream_cipher_no_iv(), + Key :: iodata(), + State :: stream_state() . +stream_init(rc4 = Type, Key) -> + Ref = ?COMPAT(ng_crypto_init_nif(alias(Type), + iolist_to_binary(Key), <<>>, + undefined) + ), + {Type, {Ref,flg_undefined}}. + +%%%---- stream_encrypt +-spec stream_encrypt(State, PlainText) -> {NewState, CipherText} | no_return() + when State :: stream_state(), + PlainText :: iodata(), + NewState :: stream_state(), + CipherText :: iodata() . +stream_encrypt(State, Data) -> + crypto_stream_emulate(State, Data, true). + +%%%---- stream_decrypt +-spec stream_decrypt(State, CipherText) -> {NewState, PlainText} | no_return() + when State :: stream_state(), + CipherText :: iodata(), + NewState :: stream_state(), + PlainText :: iodata() . +stream_decrypt(State, Data) -> + crypto_stream_emulate(State, Data, false). + +%%%-------- helpers +crypto_stream_emulate({Cipher,{Ref0,flg_undefined}}, Data, EncryptFlag) when is_reference(Ref0) -> + ?COMPAT(begin + Ref = ng_crypto_init_nif(Ref0, <<>>, <<>>, EncryptFlag), + {{Cipher,Ref}, crypto_update(Ref, Data)} + end); + +crypto_stream_emulate({Cipher,Ref}, Data, _) when is_reference(Ref) -> + ?COMPAT({{Cipher,Ref}, crypto_update(Ref, Data)}). %%%---------------------------------------------------------------- -spec next_iv(Type:: cbc_cipher(), Data) -> NextIVec when % Type :: cbc_cipher(), %des_cbc | des3_cbc | aes_cbc | aes_ige, @@ -633,59 +709,155 @@ next_iv(des_cfb, Data, IVec) -> next_iv(Type, Data, _Ivec) -> next_iv(Type, Data). -%%%---- Stream ciphers +%%%================================================================ +%%% +%%% Encrypt/decrypt, The "New API" +%%% +%%%================================================================ --opaque stream_state() :: {stream_cipher(), reference()}. +-opaque crypto_state() :: reference() . --type stream_cipher() :: stream_cipher_iv() | stream_cipher_no_iv() . --type stream_cipher_no_iv() :: rc4 . --type stream_cipher_iv() :: aes_ctr - | aes_128_ctr - | aes_192_ctr - | aes_256_ctr - | chacha20 . --spec stream_init(Type, Key, IVec) -> State when Type :: stream_cipher_iv(), - Key :: iodata(), - IVec :: binary(), - State :: stream_state() . -stream_init(aes_ctr, Key, Ivec) -> - {aes_ctr, aes_ctr_stream_init(Key, Ivec)}; -stream_init(aes_128_ctr, Key, Ivec) -> - {aes_ctr, aes_ctr_stream_init(Key, Ivec)}; -stream_init(aes_192_ctr, Key, Ivec) -> - {aes_ctr, aes_ctr_stream_init(Key, Ivec)}; -stream_init(aes_256_ctr, Key, Ivec) -> - {aes_ctr, aes_ctr_stream_init(Key, Ivec)}; -stream_init(chacha20, Key, Ivec) -> - {chacha20, chacha20_stream_init(Key,Ivec)}. - --spec stream_init(Type, Key) -> State when Type :: stream_cipher_no_iv(), - Key :: iodata(), - State :: stream_state() . -stream_init(rc4, Key) -> - {rc4, notsup_to_error(rc4_set_key(Key))}. - --spec stream_encrypt(State, PlainText) -> {NewState, CipherText} - when State :: stream_state(), - PlainText :: iodata(), - NewState :: stream_state(), - CipherText :: iodata() . -stream_encrypt(State, Data0) -> - Data = iolist_to_binary(Data0), - MaxByts = max_bytes(), - stream_crypt(fun do_stream_encrypt/2, State, Data, erlang:byte_size(Data), MaxByts, []). +%%%---------------------------------------------------------------- +%%% +%%% Create and initialize a new state for encryption or decryption +%%% --spec stream_decrypt(State, CipherText) -> {NewState, PlainText} - when State :: stream_state(), - CipherText :: iodata(), - NewState :: stream_state(), - PlainText :: iodata() . -stream_decrypt(State, Data0) -> - Data = iolist_to_binary(Data0), - MaxByts = max_bytes(), - stream_crypt(fun do_stream_decrypt/2, State, Data, erlang:byte_size(Data), MaxByts, []). +-spec crypto_init(Cipher, Key, EncryptFlag) -> State | ng_crypto_error() + when Cipher :: block_cipher_without_iv() + | stream_cipher_no_iv(), + Key :: iodata(), + EncryptFlag :: boolean(), + State :: crypto_state() . +crypto_init(Cipher, Key, EncryptFlag) -> + %% The IV is supposed to be supplied by calling crypto_update/3 + ng_crypto_init_nif(alias(Cipher), iolist_to_binary(Key), <<>>, EncryptFlag). + + +-spec crypto_init(Cipher, Key, IV, EncryptFlag) -> State | ng_crypto_error() + when Cipher :: stream_cipher_iv() + | block_cipher_with_iv(), + Key :: iodata(), + IV :: iodata(), + EncryptFlag :: boolean(), + State :: crypto_state() . +crypto_init(Cipher, Key, IV, EncryptFlag) -> + ng_crypto_init_nif(alias(Cipher), iolist_to_binary(Key), iolist_to_binary(IV), EncryptFlag). + + + +%%%---------------------------------------------------------------- +-spec crypto_init_dyn_iv(Cipher, Key, EncryptFlag) -> State | ng_crypto_error() + when Cipher :: stream_cipher_iv() + | block_cipher_with_iv(), + Key :: iodata(), + EncryptFlag :: boolean(), + State :: crypto_state() . +crypto_init_dyn_iv(Cipher, Key, EncryptFlag) -> + %% The IV is supposed to be supplied by calling crypto_update/3 + ng_crypto_init_nif(alias(Cipher), iolist_to_binary(Key), undefined, EncryptFlag). + +%%%---------------------------------------------------------------- +%%% +%%% Encrypt/decrypt a sequence of bytes. The sum of the sizes +%%% of all blocks must be an integer multiple of the crypto's +%%% blocksize. +%%% + +-spec crypto_update(State, Data) -> Result | ng_crypto_error() + when State :: crypto_state(), + Data :: iodata(), + Result :: binary() . +crypto_update(State, Data0) -> + case iolist_to_binary(Data0) of + <<>> -> + <<>>; % Known to fail on OpenSSL 0.9.8h + Data -> + ng_crypto_update_nif(State, Data) + end. + + +%%%---------------------------------------------------------------- +-spec crypto_update_dyn_iv(State, Data, IV) -> Result | ng_crypto_error() + when State :: crypto_state(), + Data :: iodata(), + IV :: iodata(), + Result :: binary() . +crypto_update_dyn_iv(State, Data0, IV) -> + %% When State is from State = crypto_init(Cipher, Key, undefined, EncryptFlag) + case iolist_to_binary(Data0) of + <<>> -> + <<>>; % Known to fail on OpenSSL 0.9.8h + Data -> + ng_crypto_update_nif(State, Data, iolist_to_binary(IV)) + end. + +%%%---------------------------------------------------------------- +%%% +%%% Encrypt/decrypt one set bytes. +%%% The size must be an integer multiple of the crypto's blocksize. +%%% + +-spec crypto_one_shot(Cipher, Key, IV, Data, EncryptFlag) -> Result | ng_crypto_error() + when Cipher :: stream_cipher() + | block_cipher_with_iv() + | block_cipher_without_iv(), + Key :: iodata(), + IV :: iodata() | undefined, + Data :: iodata(), + EncryptFlag :: boolean(), + Result :: binary() . +crypto_one_shot(Cipher, Key, undefined, Data, EncryptFlag) -> + crypto_one_shot(Cipher, Key, <<>>, Data, EncryptFlag); + +crypto_one_shot(Cipher, Key, IV, Data0, EncryptFlag) -> + case iolist_to_binary(Data0) of + <<>> -> + <<>>; % Known to fail on OpenSSL 0.9.8h + Data -> + ng_crypto_one_shot_nif(alias(Cipher), + iolist_to_binary(Key), iolist_to_binary(IV), Data, + EncryptFlag) + end. + +%%%---------------------------------------------------------------- +%%% NIFs + +-type ng_crypto_error() :: no_return() . + +-spec ng_crypto_init_nif(atom(), binary(), binary()|undefined, boolean()|undefined ) -> crypto_state() | ng_crypto_error() + ; (crypto_state(), <<>>, <<>>, boolean()) -> crypto_state() | ng_crypto_error(). +ng_crypto_init_nif(_Cipher, _Key, _IVec, _EncryptFlg) -> ?nif_stub. + + +-spec ng_crypto_update_nif(crypto_state(), binary()) -> binary() | ng_crypto_error() . +ng_crypto_update_nif(_State, _Data) -> ?nif_stub. + +-spec ng_crypto_update_nif(crypto_state(), binary(), binary()) -> binary() | ng_crypto_error() . +ng_crypto_update_nif(_State, _Data, _IV) -> ?nif_stub. + + +-spec ng_crypto_one_shot_nif(atom(), binary(), binary(), binary(), boolean() ) -> binary() | ng_crypto_error(). +ng_crypto_one_shot_nif(_Cipher, _Key, _IVec, _Data, _EncryptFlg) -> ?nif_stub. + +%%%---------------------------------------------------------------- +%%% Cipher aliases +%%% +prepend_cipher_aliases(L) -> + [des3_cbc, des_ede3, des_ede3_cbf, des3_cbf, des3_cfb, aes_cbc128, aes_cbc256 | L]. +%%%---- des_ede3_cbc +alias(des3_cbc) -> des_ede3_cbc; +alias(des_ede3) -> des_ede3_cbc; +%%%---- des_ede3_cfb +alias(des_ede3_cbf) -> des_ede3_cfb; +alias(des3_cbf) -> des_ede3_cfb; +alias(des3_cfb) -> des_ede3_cfb; +%%%---- aes_*_cbc +alias(aes_cbc128) -> aes_128_cbc; +alias(aes_cbc256) -> aes_256_cbc; + +alias(Alg) -> Alg. %%%================================================================ %%% @@ -1726,6 +1898,7 @@ hash_update(State0, Data, _, MaxBytes) -> State = notsup_to_error(hash_update_nif(State0, Increment)), hash_update(State, Rest, erlang:byte_size(Rest), MaxBytes). +hash_info_nif(_Hash) -> ?nif_stub. hash_nif(_Hash, _Data) -> ?nif_stub. hash_init_nif(_Hash) -> ?nif_stub. hash_update_nif(_State, _Data) -> ?nif_stub. @@ -1770,18 +1943,7 @@ poly1305_nif(_Key, _Data) -> ?nif_stub. %% CIPHERS -------------------------------------------------------------------- -block_crypt_nif(_Type, _Key, _Ivec, _Text, _IsEncrypt) -> ?nif_stub. -block_crypt_nif(_Type, _Key, _Text, _IsEncrypt) -> ?nif_stub. - -check_des3_key(Key) -> - case lists:map(fun erlang:iolist_to_binary/1, Key) of - ValidKey = [B1, B2, B3] when byte_size(B1) =:= 8, - byte_size(B2) =:= 8, - byte_size(B3) =:= 8 -> - ValidKey; - _ -> - error(badarg) - end. +cipher_info_nif(_Type) -> ?nif_stub. %% %% AES - in Galois/Counter Mode (GCM) @@ -1799,59 +1961,7 @@ aead_decrypt(_Type, _Key, _Ivec, _AAD, _In, _Tag) -> ?nif_stub. aes_ige_crypt_nif(_Key, _IVec, _Data, _IsEncrypt) -> ?nif_stub. - -%% Stream ciphers -------------------------------------------------------------------- - -stream_crypt(Fun, State, Data, Size, MaxByts, []) when Size =< MaxByts -> - Fun(State, Data); -stream_crypt(Fun, State0, Data, Size, MaxByts, Acc) when Size =< MaxByts -> - {State, Cipher} = Fun(State0, Data), - {State, list_to_binary(lists:reverse([Cipher | Acc]))}; -stream_crypt(Fun, State0, Data, _, MaxByts, Acc) -> - <<Increment:MaxByts/binary, Rest/binary>> = Data, - {State, CipherText} = Fun(State0, Increment), - stream_crypt(Fun, State, Rest, erlang:byte_size(Rest), MaxByts, [CipherText | Acc]). - -do_stream_encrypt({aes_ctr, State0}, Data) -> - {State, Cipher} = aes_ctr_stream_encrypt(State0, Data), - {{aes_ctr, State}, Cipher}; -do_stream_encrypt({rc4, State0}, Data) -> - {State, Cipher} = rc4_encrypt_with_state(State0, Data), - {{rc4, State}, Cipher}; -do_stream_encrypt({chacha20, State0}, Data) -> - {State, Cipher} = chacha20_stream_encrypt(State0, Data), - {{chacha20, State}, Cipher}. - -do_stream_decrypt({aes_ctr, State0}, Data) -> - {State, Text} = aes_ctr_stream_decrypt(State0, Data), - {{aes_ctr, State}, Text}; -do_stream_decrypt({rc4, State0}, Data) -> - {State, Text} = rc4_encrypt_with_state(State0, Data), - {{rc4, State}, Text}; -do_stream_decrypt({chacha20, State0}, Data) -> - {State, Cipher} = chacha20_stream_decrypt(State0, Data), - {{chacha20, State}, Cipher}. - - -%% -%% AES - in counter mode (CTR) with state maintained for multi-call streaming -%% -aes_ctr_stream_init(_Key, _IVec) -> ?nif_stub. -aes_ctr_stream_encrypt(_State, _Data) -> ?nif_stub. -aes_ctr_stream_decrypt(_State, _Cipher) -> ?nif_stub. - -%% -%% RC4 - symmetric stream cipher -%% -rc4_set_key(_Key) -> ?nif_stub. -rc4_encrypt_with_state(_State, _Data) -> ?nif_stub. - -%% -%% CHACHA20 - stream cipher -%% -chacha20_stream_init(_Key, _IVec) -> ?nif_stub. -chacha20_stream_encrypt(_State, _Data) -> ?nif_stub. -chacha20_stream_decrypt(_State, _Data) -> ?nif_stub. +%%%================================================================ %% Secure remote password ------------------------------------------------------------------- @@ -2217,176 +2327,3 @@ check_otp_test_engine(LibDir) -> end. -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%%% -%%% Experimental NG -%%% - -%%% -> {ok,State::ref()} | {error,Reason} - --opaque crypto_state() :: reference() | {any(),any(),any(),any()}. - - -%%%---------------------------------------------------------------- -%%% -%%% Create and initialize a new state for encryption or decryption -%%% - --spec crypto_init(Cipher, Key, IV, EncryptFlag) -> {ok,State} | {error,term()} | undefined - when Cipher :: stream_cipher() - | block_cipher_with_iv() - | block_cipher_without_iv() , - Key :: iodata(), - IV :: binary(), - EncryptFlag :: boolean() | undefined, - State :: crypto_state() . - -crypto_init(Cipher, Key, IV, EncryptFlag) when is_atom(Cipher), - is_binary(Key), - is_binary(IV), - is_atom(EncryptFlag) -> - case ng_crypto_init_nif(alias(Cipher), Key, IV, EncryptFlag) of - {error,Error} -> - {error,Error}; - undefined -> % For compatibility function crypto_stream_init/3 - undefined; - Ref when is_reference(Ref) -> - {ok,Ref}; - State when is_tuple(State), - size(State)==4 -> - {ok,State} % compatibility with old cryptolibs < 1.0.1 - end. - - -%%%---------------------------------------------------------------- -%%% -%%% Encrypt/decrypt a sequence of bytes. The sum of the sizes -%%% of all blocks must be an integer multiple of the crypto's -%%% blocksize. -%%% - --spec crypto_update(State, Data) -> {ok,Result} | {error,term()} - when State :: crypto_state(), - Data :: iodata(), - Result :: binary() | {crypto_state(),binary()}. -crypto_update(State, Data) -> - mk_ret(ng_crypto_update_nif(State, Data)). - -%%%---------------------------------------------------------------- -%%% -%%% Encrypt/decrypt a sequence of bytes but change the IV first. -%%% Not applicable for all modes. -%%% - --spec crypto_update(State, Data, IV) -> {ok,Result} | {error,term()} - when State :: crypto_state(), - Data :: iodata(), - IV :: binary(), - Result :: binary() | {crypto_state(),binary()}. -crypto_update(State, Data, IV) -> - mk_ret(ng_crypto_update_nif(State, Data, IV)). - -%%%---------------------------------------------------------------- -%%% Helpers -mk_ret(R) -> mk_ret(R, []). - -mk_ret({error,Error}, _) -> - {error,Error}; -mk_ret(Bin, Acc) when is_binary(Bin) -> - {ok, iolist_to_binary(lists:reverse([Bin|Acc]))}; -mk_ret({State1,Bin}, Acc) when is_tuple(State1), - size(State1) == 4, - is_binary(Bin) -> - %% compatibility with old cryptolibs < 1.0.1 - {ok, {State1, iolist_to_binary(lists:reverse([Bin|Acc]))}}. - -%%%---------------------------------------------------------------- -%%% NIFs -ng_crypto_init_nif(_Cipher, _Key, _IVec, _EncryptFlg) -> ?nif_stub. -ng_crypto_update_nif(_State, _Data) -> ?nif_stub. -ng_crypto_update_nif(_State, _Data, _IV) -> ?nif_stub. - -%%%================================================================ -%%% Compatibility functions to be called by "old" api functions. - -%%%-------------------------------- -%%%---- block encrypt/decrypt -crypto_block_encrypt(Cipher, Key, Data) -> crypto_block_encrypt(Cipher, Key, <<>>, Data). -crypto_block_decrypt(Cipher, Key, Data) -> crypto_block_decrypt(Cipher, Key, <<>>, Data). - -crypto_block_encrypt(Cipher, Key, Ivec, Data) -> crypto_block(Cipher, Key, Ivec, Data, true). -crypto_block_decrypt(Cipher, Key, Ivec, Data) -> crypto_block(Cipher, Key, Ivec, Data, false). - -%% AEAD: use old funcs - -%%%---- helper -crypto_block(Cipher, Key, IV, Data, EncryptFlag) -> - case crypto_init(Cipher, iolist_to_binary(Key), iolist_to_binary(IV), EncryptFlag) of - {ok, Ref} -> - case crypto_update(Ref, Data) of - {ok, {_,Bin}} when is_binary(Bin) -> Bin; - {ok, Bin} when is_binary(Bin) -> Bin; - {error,_} -> error(badarg) - end; - - {error,_} -> error(badarg) - end. - -%%%-------------------------------- -%%%---- stream init, encrypt/decrypt - -crypto_stream_init(Cipher, Key) -> - crypto_stream_init(Cipher, Key, <<>>). - -crypto_stream_init(Cipher, Key0, IV0) -> - Key = iolist_to_binary(Key0), - IV = iolist_to_binary(IV0), - %% First check the argumensts: - case crypto_init(Cipher, Key, IV, undefined) of - undefined -> - {Cipher, {Key, IV}}; - {error,_} -> - {error,badarg} - end. - -crypto_stream_encrypt(State, PlainText) -> - crypto_stream_emulate(State, PlainText, true). - -crypto_stream_decrypt(State, CryptoText) -> - crypto_stream_emulate(State, CryptoText, false). - - -%%%---- helper -crypto_stream_emulate({Cipher,{Key,IV}}, Data, EncryptFlag) -> - case crypto_init(Cipher, Key, IV, EncryptFlag) of - {ok,State} -> - crypto_stream_emulate({Cipher,State}, Data, EncryptFlag); - {error,_} -> - error(badarg) - end; -crypto_stream_emulate({Cipher,State}, Data, _) -> - case crypto_update(State, Data) of - {ok, {State1,Bin}} when is_binary(Bin) -> {{Cipher,State1},Bin}; - {ok,Bin} when is_binary(Bin) -> {{Cipher,State},Bin}; - {error,_} -> error(badarg) - end. - - -%%%================================================================ - -prepend_cipher_aliases(L) -> - [des3_cbc, des_ede3, des_ede3_cbf, des3_cbf, des3_cfb, aes_cbc128, aes_cbc256 | L]. - - -%%%---- des_ede3_cbc -alias(des3_cbc) -> des_ede3_cbc; -alias(des_ede3) -> des_ede3_cbc; -%%%---- des_ede3_cfb -alias(des_ede3_cbf) -> des_ede3_cfb; -alias(des3_cbf) -> des_ede3_cfb; -alias(des3_cfb) -> des_ede3_cfb; -%%%---- aes_*_cbc -alias(aes_cbc128) -> aes_128_cbc; -alias(aes_cbc256) -> aes_256_cbc; - -alias(Alg) -> Alg. diff --git a/lib/crypto/test/crypto_SUITE.erl b/lib/crypto/test/crypto_SUITE.erl index ab6d88deb2..7dbbde68e9 100644 --- a/lib/crypto/test/crypto_SUITE.erl +++ b/lib/crypto/test/crypto_SUITE.erl @@ -9,7 +9,7 @@ %% %% http://www.apache.org/licenses/LICENSE-2.0 %% -%% Unless required by applicable law or agreed to in writing, software + %% distributed under the License is distributed on an "AS IS" BASIS, %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and @@ -40,57 +40,73 @@ all() -> rand_uniform, rand_threads, rand_plugin, - rand_plugin_s + rand_plugin_s, + cipher_info, + hash_info ]. groups() -> - [{non_fips, [], [{group, md4}, + [{non_fips, [], [ + {group, blake2b}, + {group, blake2s}, + {group, dss}, + {group, ecdsa}, + {group, ed25519}, + {group, ed448}, + {group, rsa}, + + {group, md4}, {group, md5}, {group, ripemd160}, - {group, sha}, {group, sha224}, {group, sha256}, {group, sha384}, - {group, sha512}, {group, sha3_224}, {group, sha3_256}, {group, sha3_384}, {group, sha3_512}, - {group, blake2b}, - {group, blake2s}, - {group, rsa}, - {group, dss}, - {group, ecdsa}, - {group, ed25519}, - {group, ed448}, + {group, sha512}, + {group, sha}, + {group, dh}, {group, ecdh}, {group, srp}, - {group, des_cbc}, - {group, des_cfb}, - {group, des3_cbc}, - {group, des3_cbf}, - {group, des3_cfb}, - {group, des_ede3}, - {group, blowfish_cbc}, - {group, blowfish_ecb}, - {group, blowfish_cfb64}, - {group, blowfish_ofb64}, - {group, aes_cbc128}, - {group, aes_cfb8}, - {group, aes_cfb128}, - {group, aes_cbc256}, - {group, aes_ige256}, - {group, rc2_cbc}, - {group, rc4}, - {group, aes_ctr}, + + {group, aes_cbc}, {group, aes_ccm}, {group, aes_gcm}, {group, chacha20_poly1305}, {group, chacha20}, + {group, des3_cfb}, + {group, aes_cbc128}, + {group, aes_cbc256}, + {group, aes_cfb128}, + {group, aes_cfb8}, + {group, aes_ctr}, + {group, aes_ige256}, + {group, blowfish_cbc}, + {group, blowfish_cfb64}, + {group, blowfish_ecb}, + {group, blowfish_ofb64}, + {group, des3_cbc}, + {group, des3_cbf}, + {group, des_cbc}, + {group, des_cfb}, + {group, des_ede3}, {group, poly1305}, - {group, aes_cbc}]}, - {fips, [], [{group, no_md4}, + {group, rc2_cbc}, + {group, rc4} + ]}, + {fips, [], [ + {group, no_blake2b}, + {group, no_blake2s}, + {group, dss}, + {group, ecdsa}, + {group, no_ed25519}, + {group, no_ed448}, + {group, rsa}, + + {group, no_md4}, {group, no_md5}, {group, no_ripemd160}, {group, sha}, @@ -98,37 +114,36 @@ groups() -> {group, sha256}, {group, sha384}, {group, sha512}, - {group, rsa}, - {group, dss}, - {group, ecdsa}, - {group, no_ed25519}, - {group, no_ed448}, + {group, dh}, {group, ecdh}, {group, no_srp}, - {group, no_des_cbc}, - {group, no_des_cfb}, - {group, des3_cbc}, - {group, des3_cbf}, + + {group, aes_cbc}, + {group, aes_ccm}, + {group, aes_gcm}, + {group, no_chacha20_poly1305}, + {group, no_chacha20}, {group, des3_cfb}, - {group, des_ede3}, - {group, no_blowfish_cbc}, - {group, no_blowfish_ecb}, - {group, no_blowfish_cfb64}, - {group, no_blowfish_ofb64}, {group, aes_cbc128}, - {group, no_aes_cfb8}, - {group, no_aes_cfb128}, {group, aes_cbc256}, + {group, no_aes_cfb128}, + {group, no_aes_cfb8}, + {group, aes_ctr}, {group, no_aes_ige256}, + {group, no_blowfish_cbc}, + {group, no_blowfish_cfb64}, + {group, no_blowfish_ecb}, + {group, no_blowfish_ofb64}, + {group, des3_cbc}, + {group, des3_cbf}, + {group, no_des_cbc}, + {group, no_des_cfb}, + {group, des_ede3}, + {group, no_poly1305}, {group, no_rc2_cbc}, - {group, no_rc4}, - {group, aes_ctr}, - {group, aes_ccm}, - {group, aes_gcm}, - {group, no_chacha20_poly1305}, - {group, no_chacha20}, - {group, aes_cbc}]}, + {group, no_rc4} + ]}, {md4, [], [hash]}, {md5, [], [hash, hmac]}, {ripemd160, [], [hash]}, @@ -143,6 +158,8 @@ groups() -> {sha3_512, [], [hash, hmac]}, {blake2b, [], [hash, hmac]}, {blake2s, [], [hash, hmac]}, + {no_blake2b, [], [no_hash, no_hmac]}, + {no_blake2s, [], [no_hash, no_hmac]}, {rsa, [], [sign_verify, public_encrypt, private_encrypt, @@ -164,31 +181,32 @@ groups() -> compute_bug]}, {ecdh, [], [use_all_elliptic_curves, compute, generate]}, {srp, [], [generate_compute]}, - {des_cbc, [], [block]}, - {des_cfb, [], [block]}, - {des3_cbc,[], [block]}, - {des_ede3,[], [block]}, - {des3_cbf,[], [block]}, - {des3_cfb,[], [block]}, - {rc2_cbc,[], [block]}, - {aes_cbc128,[], [block, cmac]}, - {aes_cfb8,[], [block]}, - {aes_cfb128,[], [block]}, - {aes_cbc256,[], [block, cmac]}, - {aes_ecb,[], [block]}, + {des_cbc, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {des_cfb, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {des3_cbc,[], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {des_ede3,[], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {des3_cbf,[], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {des3_cfb,[], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {rc2_cbc,[], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_cbc128,[], [block, api_ng, api_ng_one_shot, api_ng_tls, cmac]}, + {aes_cfb8,[], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_cfb128,[], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_cbc256,[], [block, api_ng, api_ng_one_shot, api_ng_tls, cmac]}, + {aes_ecb,[], [block, api_ng, api_ng_one_shot, api_ng_tls]}, {aes_ige256,[], [block]}, - {blowfish_cbc, [], [block]}, - {blowfish_ecb, [], [block]}, - {blowfish_cfb64, [], [block]}, - {blowfish_ofb64,[], [block]}, - {rc4, [], [stream]}, - {aes_ctr, [], [stream]}, + {blowfish_cbc, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {blowfish_ecb, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {blowfish_cfb64, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {blowfish_ofb64,[], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {rc4, [], [stream, api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_ctr, [], [stream, api_ng, api_ng_one_shot, api_ng_tls]}, {aes_ccm, [], [aead]}, {aes_gcm, [], [aead]}, {chacha20_poly1305, [], [aead]}, - {chacha20, [], [stream]}, + {chacha20, [], [stream, api_ng, api_ng_one_shot, api_ng_tls]}, {poly1305, [], [poly1305]}, - {aes_cbc, [], [block]}, + {no_poly1305, [], [no_poly1305]}, + {aes_cbc, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, {no_aes_cfb8,[], [no_support, no_block]}, {no_aes_cfb128,[], [no_support, no_block]}, {no_md4, [], [no_support, no_hash]}, @@ -410,11 +428,19 @@ poly1305(Config) -> end, proplists:get_value(poly1305, Config)). %%-------------------------------------------------------------------- +no_poly1305() -> + [{doc, "Test disabled poly1305 function"}]. +no_poly1305(Config) -> + Type = ?config(type, Config), + Key = <<133,214,190,120,87,85,109,51,127,68,82,254,66,213,6,168,1, + 3,128,138,251,13,178,253,74,191,246,175,65,73,245,27>>, + Txt = <<"Cryptographic Forum Research Group">>, + notsup(fun crypto:poly1305/2, [Key,Txt]). + +%%-------------------------------------------------------------------- block() -> [{doc, "Test block ciphers"}]. block(Config) when is_list(Config) -> - Fips = proplists:get_bool(fips, Config), - Type = ?config(type, Config), Blocks = lazy_eval(proplists:get_value(block, Config)), lists:foreach(fun block_cipher/1, Blocks), lists:foreach(fun block_cipher/1, block_iolistify(Blocks)), @@ -437,6 +463,156 @@ no_block(Config) when is_list(Config) -> notsup(fun crypto:block_encrypt/N, Args), notsup(fun crypto:block_decrypt/N, Args). %%-------------------------------------------------------------------- +api_ng() -> + [{doc, "Test new api"}]. + +api_ng(Config) when is_list(Config) -> + Blocks = lazy_eval(proplists:get_value(block, Config, [])), + Streams = lazy_eval(proplists:get_value(stream, Config, [])), + lists:foreach(fun api_ng_cipher_increment/1, Blocks++Streams). + + +api_ng_cipher_increment({Type, Key, PlainTexts}=_X) -> + ct:log("~p",[_X]), + api_ng_cipher_increment({Type, Key, <<>>, PlainTexts}); + +api_ng_cipher_increment({Type, Key, IV, PlainTexts}=_X) -> + ct:log("~p",[_X]), + api_ng_cipher_increment({Type, Key, IV, PlainTexts, undefined}); + +api_ng_cipher_increment({Type, Key, IV, PlainText0, ExpectedEncText}=_X) -> + ct:log("~p",[_X]), + PlainTexts = iolistify(PlainText0), + RefEnc = crypto:crypto_init(Type, Key, IV, true), + RefDec = crypto:crypto_init(Type, Key, IV, false), + EncTexts = api_ng_cipher_increment_loop(RefEnc, PlainTexts), + Enc = iolist_to_binary(EncTexts), + case ExpectedEncText of + undefined -> + ok; + Enc -> + ok; + _ -> + ct:log("encode~nIn: ~p~nExpected: ~p~nEnc: ~p~n", [{Type,Key,IV,PlainTexts}, ExpectedEncText, Enc]), + ct:fail("api_ng_cipher_increment (encode)",[]) + end, + Plain = iolist_to_binary(PlainTexts), + case iolist_to_binary(api_ng_cipher_increment_loop(RefDec, EncTexts)) of + Plain -> + ok; + OtherPT -> + ct:log("decode~nIn: ~p~nExpected: ~p~nDec: ~p~n", [{Type,Key,IV,EncTexts}, Plain, OtherPT]), + ct:fail("api_ng_cipher_increment (encode)",[]) + end. + + +api_ng_cipher_increment_loop(Ref, InTexts) -> + lists:map(fun(Txt) -> + try crypto:crypto_update(Ref, Txt) + of + Bin when is_binary(Bin) -> + Bin + catch + error:Error -> + ct:pal("Txt = ~p",[Txt]), + ct:fail("~p",[Error]) + end + end, InTexts). + +%%-------------------------------------------------------------------- +api_ng_one_shot() -> + [{doc, "Test new api"}]. + +api_ng_one_shot(Config) when is_list(Config) -> + Blocks = lazy_eval(proplists:get_value(block, Config, [])), + Streams = lazy_eval(proplists:get_value(stream, Config, [])), + lists:foreach(fun do_api_ng_one_shot/1, Blocks++Streams). + +do_api_ng_one_shot({Type, Key, PlainTexts}=_X) -> + ct:log("~p",[_X]), + do_api_ng_one_shot({Type, Key, <<>>, PlainTexts}); + +do_api_ng_one_shot({Type, Key, IV, PlainTexts}=_X) -> + ct:log("~p",[_X]), + do_api_ng_one_shot({Type, Key, IV, PlainTexts, undefined}); + +do_api_ng_one_shot({Type, Key, IV, PlainText0, ExpectedEncText}=_X) -> + ct:log("~p",[_X]), + PlainText = iolist_to_binary(PlainText0), + EncTxt = crypto:crypto_one_shot(Type, Key, IV, PlainText, true), + case ExpectedEncText of + undefined -> + ok; + EncTxt -> + ok; + _ -> + ct:log("encode~nIn: ~p~nExpected: ~p~nEnc: ~p~n", [{Type,Key,IV,PlainText}, ExpectedEncText, EncTxt]), + ct:fail("api_ng_one_shot (encode)",[]) + end, + case crypto:crypto_one_shot(Type, Key, IV, EncTxt, false) of + PlainText -> + ok; + OtherPT -> + ct:log("decode~nIn: ~p~nExpected: ~p~nDec: ~p~n", [{Type,Key,IV,EncTxt}, PlainText, OtherPT]), + ct:fail("api_ng_one_shot (decode)",[]) + end. + +%%-------------------------------------------------------------------- +api_ng_tls() -> + [{doc, "Test special tls api"}]. + +api_ng_tls(Config) when is_list(Config) -> + Blocks = lazy_eval(proplists:get_value(block, Config, [])), + Streams = lazy_eval(proplists:get_value(stream, Config, [])), + lists:foreach(fun do_api_ng_tls/1, Blocks++Streams). + + +do_api_ng_tls({Type, Key, PlainTexts}=_X) -> + ct:log("~p",[_X]), + do_api_ng_tls({Type, Key, <<>>, PlainTexts}); + +do_api_ng_tls({Type, Key, IV, PlainTexts}=_X) -> + ct:log("~p",[_X]), + do_api_ng_tls({Type, Key, IV, PlainTexts, undefined}); + +do_api_ng_tls({Type, Key, IV, PlainText0, ExpectedEncText}=_X) -> + ct:log("~p",[_X]), + PlainText = iolist_to_binary(PlainText0), + Renc = crypto:crypto_init_dyn_iv(Type, Key, true), + Rdec = crypto:crypto_init_dyn_iv(Type, Key, false), + EncTxt = crypto:crypto_update_dyn_iv(Renc, PlainText, IV), + case ExpectedEncText of + undefined -> + ok; + EncTxt -> + %% Now check that the state is NOT updated: + case crypto:crypto_update_dyn_iv(Renc, PlainText, IV) of + EncTxt -> + ok; + EncTxt2 -> + ct:log("2nd encode~nIn: ~p~nExpected: ~p~nEnc: ~p~n", [{Type,Key,IV,PlainText}, EncTxt, EncTxt2]), + ct:fail("api_ng_tls (second encode)",[]) + end; + OtherEnc -> + ct:log("1st encode~nIn: ~p~nExpected: ~p~nEnc: ~p~n", [{Type,Key,IV,PlainText}, ExpectedEncText, OtherEnc]), + ct:fail("api_ng_tls (encode)",[]) + end, + case crypto:crypto_update_dyn_iv(Rdec, EncTxt, IV) of + PlainText -> + %% Now check that the state is NOT updated: + case crypto:crypto_update_dyn_iv(Rdec, EncTxt, IV) of + PlainText -> + ok; + PlainText2 -> + ct:log("2nd decode~nIn: ~p~nExpected: ~p~nDec: ~p~n", [{Type,Key,IV,EncTxt}, PlainText, PlainText2]), + ct:fail("api_ng_tls (second decode)",[]) + end; + OtherPT -> + ct:log("1st decode~nIn: ~p~nExpected: ~p~nDec: ~p~n", [{Type,Key,IV,EncTxt}, PlainText, OtherPT]), + ct:fail("api_ng_tlst (decode)",[]) + end. + +%%-------------------------------------------------------------------- no_aead() -> [{doc, "Test disabled aead ciphers"}]. no_aead(Config) when is_list(Config) -> @@ -666,6 +842,23 @@ rand_plugin_s(Config) when is_list(Config) -> rand_plugin_aux(explicit_state). %%-------------------------------------------------------------------- +cipher_info() -> + [{doc, "crypto cipher_info testing"}]. +cipher_info(Config) when is_list(Config) -> + #{type := _,key_length := _,iv_length := _, + block_size := _,mode := _} = crypto:cipher_info(aes_128_cbc), + {'EXIT',_} = (catch crypto:cipher_info(not_a_cipher)), + ok. + +%%-------------------------------------------------------------------- +hash_info() -> + [{doc, "crypto hash_info testing"}]. +hash_info(Config) when is_list(Config) -> + #{type := _,size := _,block_size := _} = crypto:hash_info(sha256), + {'EXIT',_} = (catch crypto:hash_info(not_a_hash)), + ok. + +%%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- hash(_, [], []) -> @@ -755,6 +948,7 @@ cmac_check({Type, Key, Text, Size, CMac}) -> ct:fail({{crypto, cmac, [Type, Key, Text, Size]}, {expected, ExpCMac}, {got, Other}}) end. + block_cipher({Type, Key, PlainText}) -> Plain = iolist_to_binary(PlainText), CipherText = crypto:block_encrypt(Type, Key, PlainText), @@ -832,46 +1026,51 @@ block_cipher_increment(Type, Key, IV0, IV, [PlainText | PlainTexts], Plain, Ciph stream_cipher({Type, Key, PlainText}) -> Plain = iolist_to_binary(PlainText), - State = crypto:stream_init(Type, Key), - {_, CipherText} = crypto:stream_encrypt(State, PlainText), - case crypto:stream_decrypt(State, CipherText) of + StateE = crypto:stream_init(Type, Key), + StateD = crypto:stream_init(Type, Key), + {_, CipherText} = crypto:stream_encrypt(StateE, PlainText), + case crypto:stream_decrypt(StateD, CipherText) of {_, Plain} -> ok; Other -> - ct:fail({{crypto, stream_decrypt, [State, CipherText]}, {expected, PlainText}, {got, Other}}) + ct:fail({{crypto, stream_decrypt, [StateD, CipherText]}, {expected, PlainText}, {got, Other}}) end; stream_cipher({Type, Key, IV, PlainText}) -> Plain = iolist_to_binary(PlainText), - State = crypto:stream_init(Type, Key, IV), - {_, CipherText} = crypto:stream_encrypt(State, PlainText), - case crypto:stream_decrypt(State, CipherText) of + StateE = crypto:stream_init(Type, Key, IV), + StateD = crypto:stream_init(Type, Key, IV), + {_, CipherText} = crypto:stream_encrypt(StateE, PlainText), + case crypto:stream_decrypt(StateD, CipherText) of {_, Plain} -> ok; Other -> - ct:fail({{crypto, stream_decrypt, [State, CipherText]}, {expected, PlainText}, {got, Other}}) + ct:fail({{crypto, stream_decrypt, [StateD, CipherText]}, {expected, PlainText}, {got, Other}}) end; stream_cipher({Type, Key, IV, PlainText, CipherText}) -> Plain = iolist_to_binary(PlainText), - State = crypto:stream_init(Type, Key, IV), - case crypto:stream_encrypt(State, PlainText) of + StateE = crypto:stream_init(Type, Key, IV), + StateD = crypto:stream_init(Type, Key, IV), + case crypto:stream_encrypt(StateE, PlainText) of {_, CipherText} -> ok; {_, Other0} -> - ct:fail({{crypto, stream_encrypt, [State, Type, Key, IV, Plain]}, {expected, CipherText}, {got, Other0}}) + ct:fail({{crypto, stream_encrypt, [StateE, Type, Key, IV, Plain]}, {expected, CipherText}, {got, Other0}}) end, - case crypto:stream_decrypt(State, CipherText) of + case crypto:stream_decrypt(StateD, CipherText) of {_, Plain} -> ok; Other1 -> - ct:fail({{crypto, stream_decrypt, [State, CipherText]}, {expected, PlainText}, {got, Other1}}) + ct:fail({{crypto, stream_decrypt, [StateD, CipherText]}, {expected, PlainText}, {got, Other1}}) end. stream_cipher_incment({Type, Key, PlainTexts}) -> - State = crypto:stream_init(Type, Key), - stream_cipher_incment_loop(State, State, PlainTexts, [], iolist_to_binary(PlainTexts)); + StateE = crypto:stream_init(Type, Key), + StateD = crypto:stream_init(Type, Key), + stream_cipher_incment_loop(StateE, StateD, PlainTexts, [], iolist_to_binary(PlainTexts)); stream_cipher_incment({Type, Key, IV, PlainTexts}) -> - State = crypto:stream_init(Type, Key, IV), - stream_cipher_incment_loop(State, State, PlainTexts, [], iolist_to_binary(PlainTexts)); + StateE = crypto:stream_init(Type, Key, IV), + StateD = crypto:stream_init(Type, Key, IV), + stream_cipher_incment_loop(StateE, StateD, PlainTexts, [], iolist_to_binary(PlainTexts)); stream_cipher_incment({Type, Key, IV, PlainTexts, _CipherText}) -> stream_cipher_incment({Type, Key, IV, PlainTexts}). diff --git a/lib/crypto/vsn.mk b/lib/crypto/vsn.mk index 6a91244715..deba17fb66 100644 --- a/lib/crypto/vsn.mk +++ b/lib/crypto/vsn.mk @@ -1 +1 @@ -CRYPTO_VSN = 4.4 +CRYPTO_VSN = 4.4.1 diff --git a/lib/debugger/doc/src/Makefile b/lib/debugger/doc/src/Makefile index 56d6085e9c..49b5a4be57 100644 --- a/lib/debugger/doc/src/Makefile +++ b/lib/debugger/doc/src/Makefile @@ -114,8 +114,7 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_docs_spec: docs $(INSTALL_DIR) "$(RELSYSDIR)/doc/pdf" $(INSTALL_DATA) $(TOP_PDF_FILE) "$(RELSYSDIR)/doc/pdf" - $(INSTALL_DIR) "$(RELSYSDIR)/doc/html" - ($(CP) -rf $(HTMLDIR) "$(RELSYSDIR)/doc") + $(INSTALL_DIR_DATA) $(HTMLDIR) "$(RELSYSDIR)/doc/html" $(INSTALL_DATA) $(INFO_FILE) "$(RELSYSDIR)" $(INSTALL_DIR) "$(RELEASE_PATH)/man/man3" $(INSTALL_DATA) $(MAN3DIR)/* "$(RELEASE_PATH)/man/man3" diff --git a/lib/dialyzer/doc/src/notes.xml b/lib/dialyzer/doc/src/notes.xml index 3cf776e566..bc422c43a0 100644 --- a/lib/dialyzer/doc/src/notes.xml +++ b/lib/dialyzer/doc/src/notes.xml @@ -32,6 +32,36 @@ <p>This document describes the changes made to the Dialyzer application.</p> +<section><title>Dialyzer 3.3.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Fix a bug that caused Dialyzer to crash when + analyzing a contract with a module name differing from + the analyzed module's name. The bug was introduced in + Erlang/OTP 18. </p> + <p> + Own Id: OTP-15562 Aux Id: ERL-845 </p> + </item> + <item> + <p> Fix a bug in the handling of the <c>Key</c> argument + of <c>lists:{keysearch, keyfind, keymember}</c>. </p> + <p> + Own Id: OTP-15570</p> + </item> + <item> + <p>Optimize (again) Dialyzer's handling of + left-associative use of <c>andalso</c> and <c>orelse</c> + in guards.</p> + <p> + Own Id: OTP-15577 Aux Id: ERL-851, PR-2141, PR-1944 </p> + </item> + </list> + </section> + +</section> + <section><title>Dialyzer 3.3.1</title> <section><title>Improvements and New Features</title> diff --git a/lib/dialyzer/vsn.mk b/lib/dialyzer/vsn.mk index 98ab533a58..7221993963 100644 --- a/lib/dialyzer/vsn.mk +++ b/lib/dialyzer/vsn.mk @@ -1 +1 @@ -DIALYZER_VSN = 3.3.1 +DIALYZER_VSN = 3.3.2 diff --git a/lib/diameter/doc/src/diameter.xml b/lib/diameter/doc/src/diameter.xml index 0a0194af2d..85522c99b2 100644 --- a/lib/diameter/doc/src/diameter.xml +++ b/lib/diameter/doc/src/diameter.xml @@ -1,7 +1,9 @@ <?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE erlref SYSTEM "erlref.dtd" [ - <!ENTITY spawn_opt + <!ENTITY spawn_opt2 '<seealso marker="erts:erlang#spawn_opt-2">erlang:spawn_opt/2</seealso>'> + <!ENTITY spawn_opt5 + '<seealso marker="erts:erlang#spawn_opt-5">erlang:spawn_opt/5</seealso>'> <!ENTITY nodes '<seealso marker="erts:erlang#nodes-0">erlang:nodes/0</seealso>'> <!ENTITY make_ref @@ -21,7 +23,7 @@ <copyright> <year>2011</year> -<year>2017</year> +<year>2019</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -1384,12 +1386,22 @@ the same peer.</p> </item> <tag> -<marker id="spawn_opt"/><c>{spawn_opt, [term()]}</c></tag> +<marker id="spawn_opt"/><c>{spawn_opt, [term()] | {M,F,A}}</c></tag> <item> <p> -Options passed to &spawn_opt; when spawning a process for an -incoming Diameter request. -Options <c>monitor</c> and <c>link</c> are ignored.</p> +An options list passed to &spawn_opt2; to spawn a handler process for an +incoming Diameter request on the local node, or an MFA that returns +the pid of a handler process.</p> + +<p> +Options <c>monitor</c> and <c>link</c> are ignored in the list-valued +case. +An MFA is applied with an additional term prepended to its argument +list, and should return either the pid of the handler process that +invokes <c>diameter_traffic:request/1</c> on the term in order to +process the request, or the atom <c>discard</c>. +The handler process need not be local, but diameter must be started on +the remote node.</p> <p> Defaults to the empty list.</p> diff --git a/lib/diameter/doc/src/diameter_transport.xml b/lib/diameter/doc/src/diameter_transport.xml index 67fd54bc56..0a8ef321c6 100644 --- a/lib/diameter/doc/src/diameter_transport.xml +++ b/lib/diameter/doc/src/diameter_transport.xml @@ -14,7 +14,8 @@ <erlref> <header> <copyright> -<year>2011</year><year>2016</year> +<year>2011</year> +<year>2019</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -174,10 +175,13 @@ its parent.</p> <taglist> -<tag><c>{diameter, {send, &message;}}</c></tag> +<tag><c>{diameter, {send, &message; | false}}</c></tag> <item> <p> -An outbound Diameter message.</p> +An outbound Diameter message. +The atom <c>false</c> can only be received when request +acknowledgements have been requests: see the <c>ack</c> message +below.</p> </item> <tag><c>{diameter, {close, Pid}}</c></tag> @@ -246,6 +250,27 @@ A <c>LocalAddr</c> list has the same semantics as one returned from &start;.</p> </item> +<tag><c>{diameter, ack}</c></tag> +<item> +<p> +Request acknowledgements of unanswered requests. +A transport process should send this once before passing incoming +Diameter messages into diameter. +As a result, every Diameter request passed into diameter with a +<c>recv</c> message (below) will be answered with a +<c>send</c> message (above), either a &message; for the transport +process to send or the atom <c>false</c> if the request has been +discarded or otherwise not answered.</p> + +<p> +This is to allow a transport process to keep count of the number +of incoming request messages that have not yet been answered or +discarded, to allow it to regulate the amount of incoming traffic. +Both diameter_tcp and diameter_sctp request acknowledgements when a +<c>message_cb</c> is configured, turning send/recv message into +callbacks that can be used to regulate traffic.</p> +</item> + <tag><c>{diameter, {recv, &message;}}</c></tag> <item> <p> diff --git a/lib/diameter/doc/src/notes.xml b/lib/diameter/doc/src/notes.xml index cc92bd99f0..5777225ae7 100644 --- a/lib/diameter/doc/src/notes.xml +++ b/lib/diameter/doc/src/notes.xml @@ -43,6 +43,41 @@ first.</p> <!-- ===================================================================== --> +<section><title>diameter 2.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix failure of incoming answer message with faulty + Experimental-Result-Code. Failure to decode the AVP + resulted in an uncaught exception, with no no + handle_answer/error callback as a consequence.</p> + <p> + Own Id: OTP-15569 Aux Id: ERIERL-302 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Add spawn_opt MFA configuration to allow a callback to + spawn a handler process for an incoming Diameter request + on an an arbitrary node. Module diameter_dist provides a + route_session/2 that can be used to distribute requests + based on Session-Id, although this module is currently + only documented in the module itself and may change.</p> + <p> + Own Id: OTP-15398</p> + </item> + </list> + </section> + +</section> + <section><title>diameter 2.1.6</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/diameter/src/base/diameter.erl b/lib/diameter/src/base/diameter.erl index b90b794611..7f172e1fa1 100644 --- a/lib/diameter/src/base/diameter.erl +++ b/lib/diameter/src/base/diameter.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% Copyright Ericsson AB 2010-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -365,7 +365,7 @@ call(SvcName, App, Message) -> | {connect_timer, 'Unsigned32'()} | {watchdog_timer, 'Unsigned32'() | {module(), atom(), list()}} | {watchdog_config, [{okay|suspect, non_neg_integer()}]} - | {spawn_opt, list()}. + | {spawn_opt, list() | mfa()}. %% Options passed to start_service/2 diff --git a/lib/diameter/src/base/diameter_callback.erl b/lib/diameter/src/base/diameter_callback.erl index d04a416bef..3bcf550cd8 100644 --- a/lib/diameter/src/base/diameter_callback.erl +++ b/lib/diameter/src/base/diameter_callback.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% Copyright Ericsson AB 2010-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -70,7 +70,7 @@ -module(diameter_callback). -%% Default callbacks when no aleternate is specified. +%% Default callbacks when no alternate is specified. -export([peer_up/3, peer_down/3, pick_peer/4, diff --git a/lib/diameter/src/base/diameter_dist.erl b/lib/diameter/src/base/diameter_dist.erl new file mode 100644 index 0000000000..5c29ea95a4 --- /dev/null +++ b/lib/diameter/src/base/diameter_dist.erl @@ -0,0 +1,525 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(diameter_dist). + +-behaviour(gen_server). + +%% +%% Implements callbacks that can be configured as a spawn_opt +%% transport configuration, to be able to distribute incoming Diameter +%% requests to handler processes (local or remote) in various ways. +%% + +%% spawn_opt callbacks +-export([spawn_local/2, + spawn_local/1, + route_session/2, + route_session/1]). + +%% signal availability for handling incoming requests to route_sesssion/2 +-export([attach/1, + detach/1]). + +%% consistent hashing +-export([hash/3, %% for use as default MFA in route_session/2 options map + hash/2]). %% arbitrary key/values + +-include_lib("diameter/include/diameter.hrl"). + +%% server start +-export([start_link/0]). + +%% gen_server callbacks +-export([init/1, + handle_info/2, + handle_cast/2, + handle_call/3, + code_change/3, + terminate/2]). + +-type request() :: tuple(). %% callback argument from diameter_traffic + +-define(SERVER, ?MODULE). %% server monitoring node connections + +%% Maps a node name binary to the corresponding atom. Used by +%% route_session/2 to map the optional value of a Session-Id to +%% node(). +-define(NODE_TABLE, diameter_dist_node). + +%% Maps a diameter:service_name() to a node() that has called attach/1 +%% to declare its willingness to handle incoming requests for the +%% service. Use by route_session/2 in case the optional value mapping +%% has failed. +-define(SERVICE_TABLE, diameter_dist_service). + +-define(B(A), atom_to_binary(A, utf8)). +-define(ORCOND(List), list_to_tuple(['orelse', false | List])). +-define(HASH(T), erlang:phash2(T, 16#100000000)). + +%% spawn_local/2 +%% +%% Callback that is equivalent to an options list. That is, the +%% following are equivalent when passed as options to +%% diameter:add_transport/2. +%% +%% {spawn_opt, Opts} +%% {spawn_opt, {diameter_dist, spawn_local, [Opts]}} + +-spec spawn_local(ReqT :: request(), Opts :: list()) + -> pid(). + +spawn_local(ReqT, Opts) -> + spawn_opt(diameter_traffic, request, [ReqT], Opts). + +%% spawn_local/1 + +spawn_local(ReqT) -> + spawn_local(ReqT, []). + +%% route_session/2 +%% +%% Callback that maps the Session-Id of an incoming request to a +%% handler node. +%% +%% With an options list, maps an id whose optional value is the name +%% of a connected node to the same node, to handle the case that the +%% session id has been returned from diameter:session_id/1; otherwise +%% to a node that has called diameter_dist:attach/1 using the +%% consistent hashing provided by hash/3, or to the local node() if a +%% session id could not be extracted or there are no attached nodes. A +%% handler process is spawned on the selected node using +%% erlang:spawn_opt/4. +%% +%% Different behaviour can be configured by supplying an options map +%% of the following form: +%% +%% #{search => non_neg_integer(), +%% id => [binary()], +%% default => discard | local | mfa(), +%% dispatch => list() | mfa()} +%% +%% The search member limits the number of AVPs that are examined in +%% the message (from the front), to avoid searching entire message in +%% case it's known that peers follow RFC 6733's recommendation that +%% Session-Id be placed at the head of a message. The default is to +%% search the entire message. +%% +%% The id member restricts the optional value mapping to session ids +%% whose DiamterIdentity is one of those specified. Set this to the +%% list of Diameter identities advertised by the service in question +%% (typically one) to ensure that only locally generated session ids +%% are mapped; or to the empty list to disable the mapping. +%% +%% The default member determines where to handle a message whose +%% Session-Id isn't found or whose optional value isn't mapped to the +%% name of a connected node. The atom local says the local node, an +%% MFA is invoked on Session-Id | false, the name of the diameter +%% service, and the message binary, and should return either a node() +%% or false to discard the message. Defaults to {diameter_dist, hash, []}. +%% +%% The dispatch member determines how the pid() of the request handler +%% process is retrieved. An MFA is applied to a previously selected +%% node(), and the module, function, and arguments list to apply in +%% the handler process to handle the request, the MFA being supplied +%% by diameter, and returns pid() | discard. A list is equivalent to +%% {erlang, spawn_opt, []}. Defaults to []. +%% +%% This can be used with search = 0 to route on something other than +%% Session-Id, but this is probably no simpler than just implementing +%% an own spawn_opt callback. (Except with the default dispatch possibly.) +%% +%% Note that if the peer is also implemented with OTP diameter and +%% generating session ids with diameter:session_id/1 then +%% route_session/2 can map an optional value to a local node that +%% happens to have the same name as one of the peer's nodes. This +%% could lead to an uneven distribution; for example, if the peer +%% nodes are a subset of the local nodes. In practice, it's typically +%% known if it's peers or the local node originating sessions; if the +%% former then setting id = [] disables the optional value mapping, if +%% the latter then setting default = local disables the hashing. +-spec route_session(ReqT :: request(), Opts) + -> discard + | pid() + when Opts :: pos_integer() %% aka #{search => N} + | list() %% aka #{dispatch => Opts} + | #{search => non_neg_integer(), %% limit number of examined AVPs + id => [binary()], %% restrict optional value map on DiamIdent + default => local %% handle locally + | discard + | mfa(), %% return node() | false + dispatch => list() %% spawn options + | mfa()}. %% (Node, M, F, A) -> pid() | discard + +route_session(ReqT, Opts) -> + {_, Bin} = Info = diameter_traffic:request_info(ReqT), + Sid = session_id(avps(Bin), search(Opts)), + Node = default(node_of_session_id(Sid, Opts), Sid, Opts, Info), + dispatch(Node, ReqT, dispatch(Opts)). + +%% avps/1 + +avps(<<_:20/binary, Bin/binary>>) -> + Bin; + +avps(_) -> + false. + +%% dispatch/3 + +dispatch(false, _, _) -> + discard; + +dispatch(Node, ReqT, {M,F,A}) -> + apply(M, F, [Node, diameter_traffic, request, [ReqT] | A]); + +dispatch(Node, ReqT, Opts) -> + spawn_opt(Node, diameter_traffic, request, [ReqT], Opts). + +%% route_session/1 + +route_session(ReqT) -> + route_session(ReqT, []). + +%% node_of_session_id/2 +%% +%% Return the node name encoded as optional value in a Session-Id, +%% assuming the id has been created with diameter:session_id/0. Lookup +%% the node name to ensure we don't convert arbitrary binaries to +%% atom. + +node_of_session_id([Id, _, _, Bin], #{id := Ids}) -> + lists:member(Id, Ids) andalso nodemap(Bin); + +node_of_session_id([_, _, _, Bin], _) -> + nodemap(Bin); + +node_of_session_id(_, _) -> + false. + +%% nodemap/1 + +nodemap(Bin) -> + try + ets:lookup_element(?NODE_TABLE, Bin, 2) + catch + error: badarg -> false + end. + +%% session_id/2 + +session_id(_, 0) -> %% give up + false; + +%% Session-Id = Command Code 263, V-bit = 0. +session_id(<<263:32, 0:1, _:7, Len:24, _/binary>> = Bin, _) -> + case Bin of + <<Avp:Len/binary, _/binary>> -> + <<_:8/binary, Sid/binary>> = Avp, + split(Sid); + _ -> + false + end; + +%% Jump to the next AVP. This is potentially costly for a message with +%% many AVPs and no Session-Id, which an attacker is prone to send. +%% 8.8 or RFC 6733 says that Session-Id SHOULD (but not MUST) appear +%% immediately following the Diameter Header, so there is no +%% guarantee. +session_id(<<_:40, Len:24, _/binary>> = Bin, N) -> + Pad = (4 - (Len rem 4)) rem 4, + case Bin of + <<_:Len/binary, _:Pad/binary, Rest/binary>> -> + session_id(Rest, if N == infinity -> N; true -> N-1 end); + _ -> + false + end; + +session_id(_, _) -> + false. + +%% split/1 +%% +%% Split a Session-Id at no more than three semicolons: the optional +%% value (if any) follows the third. binary:split/2 does better than +%% matching character by character, especially when the pattern is +%% compiled. + +split(Bin) -> + split(3, Bin, pattern()). + +%% split/3 + +split(0, Bin, _) -> + [Bin]; + +split(N, Bin, Pattern) -> + [H|T] = binary:split(Bin, Pattern), + [H | case T of + [] -> + T; + [Rest] -> + split(N-1, Rest, Pattern) + end]. + +%% pattern/0 +%% +%% Since this is being called in a watchdog process, compile the +%% pattern once and maintain it in the process dictionary. + +pattern() -> + case get(?MODULE) of + undefined -> + CP = binary:compile_pattern(<<$;>>), %% tuple + put(?MODULE, CP), + CP; + CP -> + CP + end. + +%% dispatch/1 + +dispatch(#{} = Opts) -> + maps:get(dispatch, Opts, []); + +dispatch(Opts) + when is_list(Opts) -> + Opts; + +dispatch(_) -> + []. + +%% search/1 +%% +%% Bound number of AVPs examined when looking for Session-Id. + +search(#{search := N}) + when is_integer(N), 0 =< N -> + N; + +search(N) + when is_integer(N), 0 =< N -> + N; + +search(_) -> + infinity. + +%% default/3 +%% +%% Choose a node when Session-Id lookup has failed. + +default(false, _, #{default := discard}, _) -> + false; + +default(false, _, #{default := local}, _) -> + node(); + +default(false, Sid, #{default := {M,F,A}}, Info) -> + {ServiceName, Bin} = Info, + apply(M, F, [Sid, ServiceName, Bin | A]); %% node() | false + +default(false, Sid, _, Info) -> %% aka {?MODULE, hash, []} + {ServiceName, Bin} = Info, + hash(Sid, ServiceName, Bin); + +default(Node, _, _, _) -> + Node. + +%% =========================================================================== + +%% hash/3 +%% +%% Consistent hashing of Session-Id to an attached node, or the local +%% node if Session-Id = false or no attached nodes. + +hash(Sid, ServiceName, _) -> + case false /= Sid andalso attached(ServiceName) of + [_|_] = Nodes -> + hash(Sid, Nodes); + _ -> + node() + end. + +%% hash/2 +%% +%% Consistent hashing on arbitrary key/values. Returns false if the +%% list is empty. + +%% No key or no values. +hash(_, []) -> + false; + +%% Not much choice. +hash(_, [Value]) -> + Value; + +%% Hash on a circle and choose the closest predecessor. +hash(Key, Values) -> + Hash = ?HASH(Key), + tl(lists:foldl(fun(V,A) -> + choose(Hash, [?HASH({Key, V}) | V], A) + end, + false, %% < list() + Values)). + +%% choose/3 + +choose(Hash, [Hash1 | _] = T, [Hash2 | _]) + when Hash1 =< Hash, Hash < Hash2 -> + T; + +choose(Hash, [Hash1 | _], [Hash2 | _] = T) + when Hash2 =< Hash, Hash < Hash1 -> + T; + +choose(_, T1, T2) -> + max(T1, T2). + +%% =========================================================================== + +%% attach/1 +%% +%% Register the local node as a handler of incoming requests for the +%% specified services when using the route_session/2 spawn_opt +%% callback. + +attach(ServiceNames) -> + abcast({attach, node(), ServiceNames}). + +%% detach/1 +%% +%% Deregister the local node as a handler of incoming requests. + +detach(ServiceNames) -> + abcast({detach, node(), ServiceNames}). + +%% abcast/1 + +abcast(T) -> + gen_server:abcast([node() | nodes()], ?SERVER, T), + ok. + +%% attached/1 + +attached(ServiceName) -> + try + ets:lookup_element(?SERVICE_TABLE, ServiceName, 2) + catch + error: badarg -> [] + end. + +%% cast/2 + +cast(Node, T) -> + gen_server:cast({?SERVER, Node}, T). + +%% attach/2 + +attach(Node, S) -> + case sets:to_list(S) of + [] -> + ok; + Services -> + cast(Node, {attach, node(), Services}) + end. + +%% =========================================================================== + +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, _Args = [], _Opts = []). + +%% init/1 +%% +%% Maintain [node() | nodes()] in a table that maps from binary-valued +%% names, so we can lookup the corresponding atoms rather than convert +%% binaries that aren't necessarily node names. + +init([]) -> + ets:new(?NODE_TABLE, [set, named_table]), + ets:new(?SERVICE_TABLE, [bag, named_table]), + ok = net_kernel:monitor_nodes(true, [{node_type, all}, nodedown_reason]), + ets:insert(?NODE_TABLE, [{?B(N), N} || N <- [node() | nodes()]]), + abcast({attach, node()}), + {ok, sets:new()}. + +%% handle_call/3 + +handle_call(_, _From, S) -> + {reply, nok, S}. + +%% handle_cast/2 + +%% Remote node is asking which services the local node wants to handle. +handle_cast({attach, Node}, S) + when Node /= node() -> + attach(Node, S), + {noreply, S}; + +%% Node wants to handle incoming requests ... +handle_cast({attach, Node, ServiceNames}, S) -> + ets:insert(?SERVICE_TABLE, [{N, Node} || N <- ServiceNames]), + {noreply, case node() of + Node -> + sets:union(S, sets:from_list(ServiceNames)); + _ -> + S + end}; + +%% ... or not. +handle_cast({detach, Node, ServiceNames}, S) -> + ets:select_delete(?SERVICE_TABLE, [{{'$1', Node}, + [?ORCOND([{'==', '$1', {const, N}} + || N <- ServiceNames])], + [true]}]), + {noreply, case node() of + Node -> + sets:subtract(S, sets:from_list(ServiceNames)); + _ -> + S + end}; + +handle_cast(_, S) -> + {noreply, S}. + +%% handle_info/2 + +handle_info({nodeup, Node, _}, S) -> + ets:insert(?NODE_TABLE, {?B(Node), Node}), + cast(Node, {attach, node()}), %% ask which services remote node handles + attach(Node, S), %% say which service local node handles + {noreply, S}; + +handle_info({nodedown, Node, _}, S) -> + ets:delete(?NODE_TABLE, ?B(Node)), + ets:select_delete(?SERVICE_TABLE, [{{'_', Node}, [], [true]}]), + {noreply, S}; + +handle_info(_, S) -> + {noreply, S}. + +%% terminate/2 + +terminate(_, _) -> + ok. + +%% code_change/3 + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. diff --git a/lib/diameter/src/base/diameter_misc_sup.erl b/lib/diameter/src/base/diameter_misc_sup.erl index 343688be23..fec5a41b5c 100644 --- a/lib/diameter/src/base/diameter_misc_sup.erl +++ b/lib/diameter/src/base/diameter_misc_sup.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -35,6 +35,7 @@ diameter_stats, %% statistics counter management diameter_reg, %% service/property publishing diameter_peer, %% remote peer manager + diameter_dist, %% request distribution diameter_config]). %% configuration/restart %% start_link/0 diff --git a/lib/diameter/src/base/diameter_traffic.erl b/lib/diameter/src/base/diameter_traffic.erl index 2d3e4a2ac9..8423e30269 100644 --- a/lib/diameter/src/base/diameter_traffic.erl +++ b/lib/diameter/src/base/diameter_traffic.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2017. All Rights Reserved. +%% Copyright Ericsson AB 2013-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -42,8 +42,12 @@ peer_up/1, peer_down/1]). +%% towards diameter_dist +-export([request_info/1]). + %% internal -export([send/1, %% send from remote node + request/1, %% process request in handler process init/1]). %% monitor process start -include_lib("diameter/include/diameter.hrl"). @@ -232,7 +236,7 @@ incr_rc(Dir, Pkt, TPid, MsgDict, AppDict, Dict0) -> -spec receive_message(pid(), Route, #diameter_packet{}, module(), RecvData) -> pid() %% request handler | boolean() %% answer, known request or not - | discard %% request discarded by MFA + | discard %% request discarded when Route :: {Handler, RequestRef, TPid} | Ack, RecvData :: {[SpawnOpt], #recvdata{}}, @@ -252,7 +256,8 @@ receive_message(TPid, Route, Pkt, Dict0, RecvData) -> recv(true, Ack, TPid, Pkt, Dict0, T) when is_boolean(Ack) -> {Opts, RecvData} = T, - spawn_request(Ack, TPid, Pkt, Dict0, RecvData, Opts); + AppT = find_app(TPid, Pkt, RecvData), + ack(Ack, TPid, spawn_request(AppT, Opts, Ack, TPid, Pkt, Dict0, RecvData)); %% ... answer to known request ... recv(false, {Pid, Ref, TPid}, _, Pkt, Dict0, _) -> @@ -274,58 +279,73 @@ recv(false, false, TPid, Pkt, _, _) -> incr(TPid, {{unknown, 0}, recv, discarded}), false. -%% spawn_request/6 +%% spawn_request/7 + +spawn_request(false, _, _, _, _, _, _) -> %% no transport + discard; -%% An MFA should return a pid() or the atom 'discard'. The latter -%% results in an acknowledgment back to the transport process when -%% appropriate, to ensure that send/recv callbacks can count -%% outstanding requests. Acknowledgement is implicit if the +%% An MFA should return the pid() of a process in which the argument +%% fun in applied, or the atom 'discard' if the fun is not applied. +%% The latter results in an acknowledgment back to the transport +%% process when appropriate, to ensure that send/recv callbacks can +%% count outstanding requests. Acknowledgement is implicit if the %% handler process dies (in a handle_request callback for example). -spawn_request(Ack, TPid, Pkt, Dict0, RecvData, {M,F,A}) -> - ReqF = fun() -> - ack(Ack, TPid, recv_request(Ack, TPid, Pkt, Dict0, RecvData)) - end, - ack(Ack, TPid, apply(M, F, [ReqF | A])); +spawn_request(AppT, {M,F,A}, Ack, TPid, Pkt, Dict0, RecvData) -> + %% Term to pass to request/1 in an appropriate process. Module + %% diameter_dist implements callbacks. + ReqT = {Pkt, AppT, Ack, TPid, Dict0, RecvData}, + apply(M, F, [ReqT | A]); %% A spawned process acks implicitly when it dies, so there's no need %% to handle 'discard'. -spawn_request(Ack, TPid, Pkt, Dict0, RecvData, Opts) -> +spawn_request(AppT, Opts, Ack, TPid, Pkt, Dict0, RecvData) -> spawn_opt(fun() -> - recv_request(Ack, TPid, Pkt, Dict0, RecvData) + recv_request(Ack, TPid, Pkt, Dict0, RecvData, AppT) end, Opts). +%% request_info/1 +%% +%% Limited request information for diameter_dist. + +request_info({Pkt, _AppT, _Ack, _TPid, _Dict0, RecvData} = _ReqT) -> + {RecvData#recvdata.service_name, Pkt#diameter_packet.bin}. + +%% request/1 +%% +%% Called from a handler process chosen by a transport spawn_opt MFA +%% to process an incoming request. + +request({Pkt, AppT, Ack, TPid, Dict0, RecvData} = _ReqT) -> + ack(Ack, TPid, recv_request(Ack, TPid, Pkt, Dict0, RecvData, AppT)). + %% ack/3 ack(Ack, TPid, RC) -> - RC == discard andalso Ack andalso (TPid ! {send, false}), + RC == discard + andalso Ack + andalso (TPid ! {send, false}), RC. %% --------------------------------------------------------------------------- -%% recv_request/5 +%% recv_request/6 %% --------------------------------------------------------------------------- -spec recv_request(Ack :: boolean(), TPid :: pid(), #diameter_packet{}, Dict0 :: module(), - #recvdata{}) + #recvdata{}, + AppT :: {#diameter_app{}, #diameter_caps{}} + | #diameter_caps{}) %% no suitable app -> ok %% answer was sent - | discard %% or not - | false. %% no transport - -recv_request(Ack, - TPid, - #diameter_packet{header = #diameter_header{application_id = Id}} - = Pkt, - Dict0, - #recvdata{peerT = PeerT, - apps = Apps, - counters = Count} - = RecvData) -> + | discard. %% or not + +recv_request(Ack, TPid, Pkt, Dict0, RecvData, AppT) -> Ack andalso (TPid ! {handler, self()}), - case diameter_service:find_incoming_app(PeerT, TPid, Id, Apps) of + case AppT of {#diameter_app{id = Aid, dictionary = AppDict} = App, Caps} -> + Count = RecvData#recvdata.counters, Count andalso incr(recv, Pkt, TPid, AppDict), DecPkt = decode(Aid, AppDict, RecvData, Pkt), Count andalso incr_error(recv, DecPkt, TPid, AppDict), @@ -349,11 +369,20 @@ recv_request(Ack, Dict0, RecvData, DecPkt, - [[]]); - false = No -> %% transport has gone down - No + [[]]) end. +%% find_app/3 +%% +%% Lookup the application of a received Diameter request on the node +%% on which it's received. + +find_app(TPid, + #diameter_packet{header = #diameter_header{application_id = Id}}, + #recvdata{peerT = PeerT, + apps = Apps}) -> + diameter_service:find_incoming_app(PeerT, TPid, Id, Apps). + %% decode/4 decode(Id, Dict, #recvdata{codec = Opts}, Pkt) -> diff --git a/lib/diameter/src/diameter.appup.src b/lib/diameter/src/diameter.appup.src index 4e6b983bac..52263633fb 100644 --- a/lib/diameter/src/diameter.appup.src +++ b/lib/diameter/src/diameter.appup.src @@ -60,7 +60,8 @@ {"2.1.3", [{restart_application, diameter}]}, %% 20.2 {"2.1.4", [{restart_application, diameter}]}, %% 20.3 {"2.1.4.1", [{restart_application, diameter}]}, %% 20.3.8.19 - {"2.1.5", [{update, diameter_peer_fsm}]} %% 21.0 + {"2.1.5", [{restart_application, diameter}]}, %% 21.0 + {"2.1.6", [{restart_application, diameter}]} %% 21.1 ], [ {"0.9", [{restart_application, diameter}]}, @@ -102,6 +103,7 @@ {"2.1.3", [{restart_application, diameter}]}, {"2.1.4", [{restart_application, diameter}]}, {"2.1.4.1", [{restart_application, diameter}]}, - {"2.1.5", [{update, diameter_peer_fsm}]} + {"2.1.5", [{restart_application, diameter}]}, + {"2.1.6", [{restart_application, diameter}]} ] }. diff --git a/lib/diameter/src/modules.mk b/lib/diameter/src/modules.mk index bb86de016a..d16292bb88 100644 --- a/lib/diameter/src/modules.mk +++ b/lib/diameter/src/modules.mk @@ -1,7 +1,7 @@ # %CopyrightBegin% # -# Copyright Ericsson AB 2010-2017. All Rights Reserved. +# Copyright Ericsson AB 2010-2019. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -40,6 +40,7 @@ RT_MODULES = \ base/diameter_config \ base/diameter_config_sup \ base/diameter_codec \ + base/diameter_dist \ base/diameter_gen \ base/diameter_lib \ base/diameter_misc_sup \ diff --git a/lib/diameter/test/diameter_dist_SUITE.erl b/lib/diameter/test/diameter_dist_SUITE.erl new file mode 100644 index 0000000000..b2e4c35b9a --- /dev/null +++ b/lib/diameter/test/diameter_dist_SUITE.erl @@ -0,0 +1,332 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +%% Tests of traffic between two Diameter nodes, the server being +%% spread across three Erlang nodes. +%% + +-module(diameter_dist_SUITE). + +-export([suite/0, + all/0]). + +%% testcases +-export([enslave/1, enslave/0, + ping/1, + start/1, + connect/1, + send/1, + stop/1, stop/0]). + +%% diameter callbacks +-export([peer_up/3, + peer_down/3, + pick_peer/4, + prepare_request/3, + prepare_retransmit/3, + handle_answer/4, + handle_error/4, + handle_request/3]). + +-export([call/1]). + +-include("diameter.hrl"). +-include("diameter_gen_base_rfc6733.hrl"). + +%% =========================================================================== + +-define(util, diameter_util). + +-define(CLIENT, 'CLIENT'). +-define(SERVER, 'SERVER'). +-define(REALM, "erlang.org"). +-define(DICT, diameter_gen_base_rfc6733). +-define(ADDR, {127,0,0,1}). + +%% Config for diameter:start_service/2. +-define(SERVICE(Host), + [{'Origin-Host', Host ++ [$.|?REALM]}, + {'Origin-Realm', ?REALM}, + {'Host-IP-Address', [?ADDR]}, + {'Vendor-Id', 12345}, + {'Product-Name', "OTP/diameter"}, + {'Auth-Application-Id', [?DICT:id()]}, + {'Origin-State-Id', origin()}, + {spawn_opt, {diameter_dist, route_session, [#{id => []}]}}, + {sequence, fun sequence/0}, + {string_decode, false}, + {application, [{dictionary, ?DICT}, + {module, ?MODULE}, + {request_errors, callback}, + {answer_errors, callback}]}]). + +-define(SUCCESS, 2001). +-define(BUSY, 3004). +-define(LOGOUT, ?'DIAMETER_BASE_TERMINATION-CAUSE_LOGOUT'). +-define(MOVED, ?'DIAMETER_BASE_TERMINATION-CAUSE_USER_MOVED'). +-define(TIMEOUT, ?'DIAMETER_BASE_TERMINATION-CAUSE_SESSION_TIMEOUT'). + +-define(L, atom_to_list). +-define(A, list_to_atom). + +%% The order here is significant and causes the server to listen +%% before the clients connect. The server listens on the first node, +%% and distributes requests to the other two. +-define(NODES, [{server0, ?SERVER}, + {server1, ?SERVER}, + {server2, ?SERVER}, + {client, ?CLIENT}]). + +%% Options to ct_slave:start/2. +-define(TIMEOUTS, [{T, 15000} || T <- [boot_timeout, + init_timeout, + start_timeout]]). + +%% =========================================================================== + +suite() -> + [{timetrap, {seconds, 60}}]. + +all() -> + [enslave, + ping, + start, + connect, + send, + stop]. + +%% =========================================================================== +%% start/stop testcases + +%% enslave/1 +%% +%% Start four slave nodes, three to implement a Diameter server, +%% one to implement a client. + +enslave() -> + [{timetrap, {seconds, 30*length(?NODES)}}]. + +enslave(Config) -> + Here = filename:dirname(code:which(?MODULE)), + Ebin = filename:join([Here, "..", "ebin"]), + Dirs = [Here, Ebin], + Nodes = [{N,S} || {M,S} <- ?NODES, N <- [slave(M, Dirs)]], + ?util:write_priv(Config, nodes, [{N,S} || {{N,ok},S} <- Nodes]), + [] = [{T,S} || {{_,E} = T, S} <- Nodes, E /= ok]. + +slave(Name, Dirs) -> + add_pathsa(Dirs, ct_slave:start(Name, ?TIMEOUTS)). + +add_pathsa(Dirs, {ok, Node}) -> + {Node, rpc:call(Node, code, add_pathsa, [Dirs])}; +add_pathsa(_, No) -> + {No, error}. + +%% ping/1 +%% +%% Ensure the server nodes are connected so that diameter_dist can attach. + +ping({S, Nodes}) -> + ?SERVER = S, + [N || {N,_} <- Nodes, + node() /= N, + pang <- [net_adm:ping(N)]]; + +ping(Config) -> + Nodes = lists:droplast(?util:read_priv(Config, nodes)), + [] = [{N,RC} || {N,S} <- Nodes, + RC <- [rpc:call(N, ?MODULE, ping, [{S,Nodes}])], + RC /= []]. + +%% start/1 +%% +%% Start diameter services. + +start(SvcName) + when is_atom(SvcName) -> + ok = diameter:start(), + ok = diameter:start_service(SvcName, ?SERVICE((?L(SvcName)))); + +start(Config) -> + Nodes = ?util:read_priv(Config, nodes), + [] = [{N,RC} || {N,S} <- Nodes, + RC <- [rpc:call(N, ?MODULE, start, [S])], + RC /= ok]. + +sequence() -> + sequence(sname()). + +sequence(client) -> + {0,32}; +sequence(Server) -> + "server" ++ N = ?L(Server), + {list_to_integer(N), 30}. + +origin() -> + origin(sname()). + +origin(client) -> + 99; +origin(Server) -> + "server" ++ N = ?L(Server), + list_to_integer(N). + +%% connect/1 +%% +%% Establish one connection from the client, terminated on the first +%% server node, the others handling requests. + +connect({?SERVER, Config, [{Node, _} | _]}) -> + if Node == node() -> %% server0 + ?util:write_priv(Config, lref, {Node, ?util:listen(?SERVER, tcp)}); + true -> + diameter_dist:attach([?SERVER]) + end, + ok; + +connect({?CLIENT, Config, _}) -> + ?util:connect(?CLIENT, tcp, ?util:read_priv(Config, lref)), + ok; + +connect(Config) -> + Nodes = ?util:read_priv(Config, nodes), + [] = [{N,RC} || {N,S} <- Nodes, + RC <- [rpc:call(N, ?MODULE, connect, [{S, Config, Nodes}])], + RC /= ok]. + +%% stop/1 +%% +%% Stop the slave nodes. + +stop() -> + [{timetrap, {seconds, 30*length(?NODES)}}]. + +stop(_Config) -> + [] = [{N,E} || {N,_} <- ?NODES, + {error, _, _} = E <- [ct_slave:stop(N)]]. + +%% =========================================================================== +%% traffic testcases + +%% send/1 +%% +%% Send 100 requests and ensure the node name sent as User-Name isn't +%% the node terminating transport. + +send(Config) -> + send(Config, 100, dict:new()). + +%% send/2 + +send(Config, 0, Dict) -> + [{Server0, _} | _] = ?util:read_priv(Config, nodes) , + Node = atom_to_binary(Server0, utf8), + {false, _} = {dict:is_key(Node, Dict), dict:to_list(Dict)}; + +send(Config, N, Dict) -> + #diameter_base_STA{'Result-Code' = ?SUCCESS, + 'User-Name' = [ServerNode]} + = send(Config, str(?LOGOUT)), + true = is_binary(ServerNode), + send(Config, N-1, dict:update_counter(ServerNode, 1, Dict)). + +%% =========================================================================== + +str(Cause) -> + #diameter_base_STR{'Destination-Realm' = ?REALM, + 'Auth-Application-Id' = ?DICT:id(), + 'Termination-Cause' = Cause}. + +%% send/2 + +send(Config, Req) -> + {Node, _} = lists:last(?util:read_priv(Config, nodes)), + rpc:call(Node, ?MODULE, call, [Req]). + +%% call/1 + +call(Req) -> + diameter:call(?CLIENT, ?DICT, Req, []). + +%% sname/0 + +sname() -> + ?A(hd(string:tokens(?L(node()), "@"))). + +%% =========================================================================== +%% diameter callbacks + +%% peer_up/3 + +peer_up(_SvcName, _Peer, State) -> + State. + +%% peer_down/3 + +peer_down(_SvcName, _Peer, State) -> + State. + +%% pick_peer/4 + +pick_peer([Peer], [], ?CLIENT, _State) -> + {ok, Peer}. + +%% prepare_request/3 + +prepare_request(Pkt, ?CLIENT, {_Ref, Caps}) -> + #diameter_packet{msg = Req} + = Pkt, + #diameter_caps{origin_host = {OH, _}, + origin_realm = {OR, _}} + = Caps, + {send, Req#diameter_base_STR{'Origin-Host' = OH, + 'Origin-Realm' = OR, + 'Session-Id' = diameter:session_id(OH)}}. + +%% prepare_retransmit/3 + +prepare_retransmit(_, ?CLIENT, _) -> + discard. + +%% handle_answer/5 + +handle_answer(Pkt, _Req, ?CLIENT, _Peer) -> + #diameter_packet{msg = Rec, errors = []} = Pkt, + Rec. + +%% handle_error/5 + +handle_error(Reason, _Req, ?CLIENT, _Peer) -> + {error, Reason}. + +%% handle_request/3 + +handle_request(Pkt, ?SERVER, {_, Caps}) -> + #diameter_packet{msg = #diameter_base_STR{'Session-Id' = SId}} + = Pkt, + #diameter_caps{origin_host = {OH, _}, + origin_realm = {OR, _}} + = Caps, + {reply, #diameter_base_STA{'Result-Code' = ?SUCCESS, + 'Session-Id' = SId, + 'Origin-Host' = OH, + 'Origin-Realm' = OR, + 'User-Name' = [atom_to_binary(node(), utf8)]}}. diff --git a/lib/diameter/test/diameter_distribution_SUITE.erl b/lib/diameter/test/diameter_distribution_SUITE.erl index 5146f68ff1..5fe02284ae 100644 --- a/lib/diameter/test/diameter_distribution_SUITE.erl +++ b/lib/diameter/test/diameter_distribution_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2016. All Rights Reserved. +%% Copyright Ericsson AB 2013-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -76,6 +76,7 @@ {share_peers, peers()}, {use_shared_peers, peers()}, {restrict_connections, false}, + {spawn_opt, {diameter_dist, spawn_local, []}}, {sequence, fun sequence/0}, {application, [{dictionary, ?DICT}, {module, ?MODULE}, @@ -125,7 +126,7 @@ all() -> %% enslave/1 %% %% Start four slave nodes, one to implement a Diameter server, -%% two three to implement a client. +%% three to implement a client. enslave() -> [{timetrap, {seconds, 30*length(?NODES)}}]. @@ -331,6 +332,8 @@ prepare_request(Pkt, ?CLIENT, {_Ref, Caps}, {_, client0}) -> 'Origin-Realm' = OR, 'Session-Id' = diameter:session_id(OH)}}. +%% prepare_retransmit/4 + prepare_retransmit(Pkt, ?CLIENT, _, {_, client0}) -> #diameter_packet{msg = #diameter_base_STR{'Termination-Cause' = ?MOVED}} = Pkt, %% assert diff --git a/lib/diameter/test/diameter_pool_SUITE.erl b/lib/diameter/test/diameter_pool_SUITE.erl index 97c16940ff..a36a4fa17a 100644 --- a/lib/diameter/test/diameter_pool_SUITE.erl +++ b/lib/diameter/test/diameter_pool_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2015-2017. All Rights Reserved. +%% Copyright Ericsson AB 2015-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -51,6 +51,7 @@ {'Auth-Application-Id', [0]}, %% common {'Acct-Application-Id', [3]}, %% accounting {restrict_connections, false}, + {spawn_opt, {diameter_dist, route_session, []}}, {application, [{alias, common}, {dictionary, diameter_gen_base_rfc6733}, {module, diameter_callback}]}, diff --git a/lib/diameter/test/diameter_traffic_SUITE.erl b/lib/diameter/test/diameter_traffic_SUITE.erl index 434aef01dd..47b00c25a2 100644 --- a/lib/diameter/test/diameter_traffic_SUITE.erl +++ b/lib/diameter/test/diameter_traffic_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2018. All Rights Reserved. +%% Copyright Ericsson AB 2010-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -539,8 +539,7 @@ add_transports(Config) -> ++ [{unordered, unordered()} || T == sctp], [{capabilities_cb, fun capx/2}, {pool_size, 8} - | server_apps()] - ++ [{spawn_opt, {erlang, spawn, []}} || CS]), + | server_apps()]), Cs = [?util:connect(CN, [T, {sender, CS} | client_opts(T)], LRef, diff --git a/lib/diameter/test/modules.mk b/lib/diameter/test/modules.mk index 0c73adca12..90b0a25d5f 100644 --- a/lib/diameter/test/modules.mk +++ b/lib/diameter/test/modules.mk @@ -1,7 +1,7 @@ # %CopyrightBegin% # -# Copyright Ericsson AB 2010-2017. All Rights Reserved. +# Copyright Ericsson AB 2010-2019. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -31,6 +31,7 @@ MODULES = \ diameter_codec_test \ diameter_config_SUITE \ diameter_compiler_SUITE \ + diameter_dist_SUITE \ diameter_distribution_SUITE \ diameter_dpr_SUITE \ diameter_event_SUITE \ diff --git a/lib/diameter/vsn.mk b/lib/diameter/vsn.mk index 8c75c9e55e..a900e8f28e 100644 --- a/lib/diameter/vsn.mk +++ b/lib/diameter/vsn.mk @@ -17,5 +17,5 @@ # %CopyrightEnd% APPLICATION = diameter -DIAMETER_VSN = 2.1.6 +DIAMETER_VSN = 2.2 APP_VSN = $(APPLICATION)-$(DIAMETER_VSN)$(PRE_VSN) diff --git a/lib/edoc/doc/src/notes.xml b/lib/edoc/doc/src/notes.xml index e818887eb8..145856bcaa 100644 --- a/lib/edoc/doc/src/notes.xml +++ b/lib/edoc/doc/src/notes.xml @@ -32,6 +32,22 @@ <p>This document describes the changes made to the EDoc application.</p> +<section><title>Edoc 0.10</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> Print a helpful message explaining that adding + <c>{preprocess, true}</c> can help if reading a source + file fails. </p> + <p> + Own Id: OTP-15605 Aux Id: ERL-841 </p> + </item> + </list> + </section> + +</section> + <section><title>Edoc 0.9.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/edoc/vsn.mk b/lib/edoc/vsn.mk index 0b3636f030..b6e1422623 100644 --- a/lib/edoc/vsn.mk +++ b/lib/edoc/vsn.mk @@ -1 +1 @@ -EDOC_VSN = 0.9.4 +EDOC_VSN = 0.10 diff --git a/lib/erl_docgen/doc/src/notes.xml b/lib/erl_docgen/doc/src/notes.xml index 97c842a324..54f0a36b27 100644 --- a/lib/erl_docgen/doc/src/notes.xml +++ b/lib/erl_docgen/doc/src/notes.xml @@ -31,7 +31,31 @@ </header> <p>This document describes the changes made to the <em>erl_docgen</em> application.</p> - <section><title>Erl_Docgen 0.8.1</title> + <section><title>Erl_Docgen 0.9</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + The HTML reference documentation shows the OTP version + where modules and functions were first introduced. + Versions older than R13B04 are not shown.</p> + <p> + Own Id: OTP-15460</p> + </item> + <item> + <p> + Make html documentation of C functions with many + arguments more readable.</p> + <p> + Own Id: OTP-15637 Aux Id: PR-2160 </p> + </item> + </list> + </section> + +</section> + +<section><title>Erl_Docgen 0.8.1</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/erl_docgen/priv/css/otp_doc.css b/lib/erl_docgen/priv/css/otp_doc.css index 89b278215c..17d9f8dd56 100644 --- a/lib/erl_docgen/priv/css/otp_doc.css +++ b/lib/erl_docgen/priv/css/otp_doc.css @@ -74,7 +74,7 @@ a:visited { color: #1b6ec2; text-decoration: none } /* Invisible table for function specs, * just to get since-version out in right margin */ -.func-table, .func-tr, .func-td, .func-since-td { +.func-table, .func-tr, .func-td, .cfunc-td, .func-since-td { width: 200%; border: 0; padding: 0; @@ -86,7 +86,13 @@ a:visited { color: #1b6ec2; text-decoration: none } } .func-td { - width: 50% + width: 50%; +} + +.cfunc-td { + width: 50%; + padding-left: 100px; + text-indent: -100px; } .func-since-td { diff --git a/lib/erl_docgen/priv/xsl/db_html.xsl b/lib/erl_docgen/priv/xsl/db_html.xsl index c5150d447c..c9be926e1e 100644 --- a/lib/erl_docgen/priv/xsl/db_html.xsl +++ b/lib/erl_docgen/priv/xsl/db_html.xsl @@ -2119,17 +2119,10 @@ <xsl:when test="ancestor::cref"> <table class="func-table"> <tr class="func-tr"> - <td class="func-td"> + <td class="cfunc-td"> <span class="bold_code bc-7"> <xsl:call-template name="title_link"> <xsl:with-param name="link" select="substring-before(nametext, '(')"/> - <xsl:with-param name="title"> - <xsl:value-of select="ret"/> - <xsl:call-template name="maybe-space-after-ret"> - <xsl:with-param name="s" select="ret"/> - </xsl:call-template> - <xsl:value-of select="nametext"/> - </xsl:with-param> </xsl:call-template> </span> </td> @@ -2199,18 +2192,6 @@ </xsl:template> - <xsl:template name="maybe-space-after-ret"> - <xsl:param name="s"/> - <xsl:variable name="last_char" - select="substring($s, string-length($s), 1)"/> - <xsl:choose> - <xsl:when test="$last_char != '*'"> - <xsl:text> </xsl:text> - </xsl:when> - </xsl:choose> - </xsl:template> - - <!-- Type --> <xsl:template match="type"> <xsl:param name="partnum"/> @@ -2264,7 +2245,7 @@ </xsl:template> <xsl:template name="title_link"> - <xsl:param name="title"/> + <xsl:param name="title" select="'APPLY'"/> <xsl:param name="link" select="erl:to-link(title)"/> <xsl:param name="ghlink" select="ancestor-or-self::*[@ghlink][position() = 1]/@ghlink"/> <xsl:variable name="id" select="concat(concat($link,'-'), generate-id(.))"/> @@ -2274,7 +2255,16 @@ <xsl:with-param name="id" select="$id"/> <xsl:with-param name="ghlink" select="$ghlink"/> </xsl:call-template> - <a class="title_link" name="{$link}" href="#{$link}"><xsl:value-of select="$title"/></a> + <a class="title_link" name="{$link}" href="#{$link}"> + <xsl:choose> + <xsl:when test="$title = 'APPLY'"> + <xsl:apply-templates/> <!-- like <ret> and <nametext> --> + </xsl:when> + <xsl:otherwise> + <xsl:value-of select="$title"/> + </xsl:otherwise> + </xsl:choose> + </a> </span> </xsl:template> @@ -2704,4 +2694,46 @@ <xsl:value-of select="normalize-space(.)"/> </xsl:template> + <xsl:template match="ret"> + <xsl:value-of select="."/> + <xsl:variable name="last_char" select="substring(., string-length(.), 1)"/> + <xsl:if test="$last_char != '*'"> + <xsl:text> </xsl:text> + </xsl:if> + </xsl:template> + + <xsl:template match="nametext"> + <xsl:value-of select="substring-before(.,'(')"/> + <xsl:text>(</xsl:text> + <xsl:variable name="arglist" select="substring-after(.,'(')"/> + <xsl:choose> + <xsl:when test="$arglist = ')' or $arglist = 'void)'"> + <xsl:value-of select="$arglist"/> + </xsl:when> + <xsl:otherwise> + <br/> + <xsl:call-template name="cfunc-arglist"> + <xsl:with-param name="text" select="$arglist"/> + </xsl:call-template> + </xsl:otherwise> + </xsl:choose> + </xsl:template> + + <!-- Format C function argument list with <br> after comma --> + <xsl:template name="cfunc-arglist"> + <xsl:param name="text"/> + <xsl:variable name="line" select="normalize-space($text)"/> + <xsl:choose> + <xsl:when test="contains($line,',')"> + <xsl:value-of select="substring-before($line,',')"/>,<br/> + <xsl:call-template name="cfunc-arglist"> + <xsl:with-param name="text" select="substring-after($line,',')"/> + </xsl:call-template> + </xsl:when> + <xsl:otherwise> + <xsl:value-of select="$line"/> + </xsl:otherwise> + </xsl:choose> + </xsl:template> + </xsl:stylesheet> diff --git a/lib/erl_docgen/vsn.mk b/lib/erl_docgen/vsn.mk index 3b2f6db6a1..6321f229dd 100644 --- a/lib/erl_docgen/vsn.mk +++ b/lib/erl_docgen/vsn.mk @@ -1 +1 @@ -ERL_DOCGEN_VSN = 0.8.1 +ERL_DOCGEN_VSN = 0.9 diff --git a/lib/erl_interface/configure.in b/lib/erl_interface/configure.in index 53c2f7069c..f0e9b2eb3f 100644 --- a/lib/erl_interface/configure.in +++ b/lib/erl_interface/configure.in @@ -61,7 +61,7 @@ fi TARGET=$host AC_SUBST(TARGET) -AC_CONFIG_HEADER([src/$host/config.h:config.h.in include/$host/ei_config.h:include/ei_config.h.in]) +AC_CONFIG_HEADER([src/$host/config.h:config.h.in]) dnl ---------------------------------------------------------------------- dnl Optional features @@ -184,14 +184,6 @@ AC_TRY_COMPILE([#include <sys/types.h> AC_DEFINE(HAVE_SOCKLEN_T, [], [Define if you have the `socklen_t' type])], [AC_MSG_RESULT(no)]) -AC_MSG_CHECKING([for deprecated attribute]) -AC_TRY_COMPILE([], -[void my_function(void) __attribute__((deprecated));], -[AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_DEPRECATED_ATTRIBUTE, [], [Define to 1 if you have the `deprecated' attribute])], -[AC_MSG_RESULT(no)]) - - # Checks for library functions. AC_FUNC_ALLOCA dnl AC_FUNC_FORK diff --git a/lib/erl_interface/doc/src/ei.xml b/lib/erl_interface/doc/src/ei.xml index 2bdb390644..f081ca926a 100644 --- a/lib/erl_interface/doc/src/ei.xml +++ b/lib/erl_interface/doc/src/ei.xml @@ -736,7 +736,7 @@ ei_encode_tuple_header(buf, &i, 0);</pre> </func> <func> - <name since="OTP @OTP-15442@"><ret>int</ret><nametext>ei_init(void)</nametext></name> + <name since="OTP 21.3"><ret>int</ret><nametext>ei_init(void)</nametext></name> <fsummary>Initialize the ei library.</fsummary> <desc> <p>Initialize the <c>ei</c> library. This function should be called once diff --git a/lib/erl_interface/doc/src/ei_connect.xml b/lib/erl_interface/doc/src/ei_connect.xml index df40973270..795f1249b3 100644 --- a/lib/erl_interface/doc/src/ei_connect.xml +++ b/lib/erl_interface/doc/src/ei_connect.xml @@ -412,7 +412,7 @@ typedef struct { </func> <func> - <name since="OTP @OTP-15442@"><ret>int</ret><nametext>ei_close_connection(int fd)</nametext></name> + <name since="OTP 21.3"><ret>int</ret><nametext>ei_close_connection(int fd)</nametext></name> <fsummary>Close a connection.</fsummary> <desc> <p>Closes a previously opened connection or listen socket.</p> @@ -472,9 +472,9 @@ fd = ei_xconnect(&ec, &addr, ALIVE); <func> <name since=""><ret>int</ret><nametext>ei_connect_init(ei_cnode* ec, const char* this_node_name, const char *cookie, short creation)</nametext></name> - <name since="OTP @OTP-15442@"><ret>int</ret><nametext>ei_connect_init_ussi(ei_cnode* ec, const char* this_node_name, const char *cookie, short creation, ei_socket_callbacks *cbs, int cbs_sz, void *setup_context)</nametext></name> + <name since="OTP 21.3"><ret>int</ret><nametext>ei_connect_init_ussi(ei_cnode* ec, const char* this_node_name, const char *cookie, short creation, ei_socket_callbacks *cbs, int cbs_sz, void *setup_context)</nametext></name> <name since=""><ret>int</ret><nametext>ei_connect_xinit(ei_cnode* ec, const char *thishostname, const char *thisalivename, const char *thisnodename, Erl_IpAddr thisipaddr, const char *cookie, short creation)</nametext></name> - <name since="OTP @OTP-15442@"><ret>int</ret><nametext>ei_connect_xinit_ussi(ei_cnode* ec, const char *thishostname, const char *thisalivename, const char *thisnodename, Erl_IpAddr thisipaddr, const char *cookie, short creation, ei_socket_callbacks *cbs, int cbs_sz, void *setup_context)</nametext></name> + <name since="OTP 21.3"><ret>int</ret><nametext>ei_connect_xinit_ussi(ei_cnode* ec, const char *thishostname, const char *thisalivename, const char *thisnodename, Erl_IpAddr thisipaddr, const char *cookie, short creation, ei_socket_callbacks *cbs, int cbs_sz, void *setup_context)</nametext></name> <fsummary>Initialize for a connection.</fsummary> <desc> <p>Initializes the <c>ec</c> structure, to @@ -595,8 +595,8 @@ if (ei_connect_init(&ec, "madonna", "cookie...", n++) < 0) { </func> <func> - <name since="OTP @OTP-15442@"><ret>int</ret><nametext>ei_listen(ei_cnode *ec, int *port, int backlog)</nametext></name> - <name since="OTP @OTP-15442@"><ret>int</ret><nametext>ei_xlisten(ei_cnode *ec, Erl_IpAddr adr, int *port, int backlog)</nametext></name> + <name since="OTP 21.3"><ret>int</ret><nametext>ei_listen(ei_cnode *ec, int *port, int backlog)</nametext></name> + <name since="OTP 21.3"><ret>int</ret><nametext>ei_xlisten(ei_cnode *ec, Erl_IpAddr adr, int *port, int backlog)</nametext></name> <fsummary>Create a listen socket.</fsummary> <desc> <p>Used by a server process to setup a listen socket which diff --git a/lib/erl_interface/doc/src/notes.xml b/lib/erl_interface/doc/src/notes.xml index 07ddd82718..32d28b853b 100644 --- a/lib/erl_interface/doc/src/notes.xml +++ b/lib/erl_interface/doc/src/notes.xml @@ -31,6 +31,43 @@ </header> <p>This document describes the changes made to the Erl_interface application.</p> +<section><title>Erl_Interface 3.11.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed two bugs in the <c>erl_call</c> program. A missing + initialization (introduced in <c>erl_interface-3.11</c>) + which either caused a crash or failure to connect to or + start a node, and an incorrectly calculated timeout which + could cause failure to start an erlang node. These bugs + only caused failures on some platforms.</p> + <p> + Own Id: OTP-15676 Aux Id: OTP-15442, ERL-881 </p> + </item> + </list> + </section> + +</section> + +<section><title>Erl_Interface 3.11</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Support for plugin of a <seealso + marker="ei_connect#ussi">user supplied socket + implementation</seealso> has been added.</p> + <p> + Own Id: OTP-15442 Aux Id: ERIERL-258 </p> + </item> + </list> + </section> + +</section> + <section><title>Erl_Interface 3.10.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/erl_interface/include/ei.h b/lib/erl_interface/include/ei.h index 10a6e5bf50..aa2a49098f 100644 --- a/lib/erl_interface/include/ei.h +++ b/lib/erl_interface/include/ei.h @@ -47,21 +47,26 @@ typedef LONG_PTR ssize_t; /* Sigh... */ # include <netdb.h> #endif -#ifndef EI_INCLUDED_CONFIG_H__ -#include "ei_config.h" +#ifdef __has_attribute +#if __has_attribute(deprecated) +#define EI_HAVE_DEPRECATED_ATTR__ 1 +#else +#undef EI_HAVE_DEPRECATED_ATTR__ +#endif #endif -#if defined(HAVE_DEPRECATED_ATTRIBUTE) && !defined(EI_NO_DEPR_WARN) -#define EI_HAVE_DEPRECATED_ATTR__ 1 +#ifdef EI_NO_DEPR_WARN +#undef EI_HAVE_DEPRECATED_ATTR__ +#endif + +#ifdef EI_HAVE_DEPRECATED_ATTR__ #define EI_DEPRECATED_ATTR_NAME deprecated #define EI_DEPRECATED_ATTR __attribute__((EI_DEPRECATED_ATTR_NAME)) #else -#undef EI_HAVE_DEPRECATED_ATTR__ #define EI_DEPRECATED_ATTR_NAME #define EI_DEPRECATED_ATTR #endif - /* -------------------------------------------------------------------- */ /* Defines part of API */ /* -------------------------------------------------------------------- */ diff --git a/lib/erl_interface/include/ei_config.h.in b/lib/erl_interface/include/ei_config.h.in deleted file mode 100644 index dfcf676bed..0000000000 --- a/lib/erl_interface/include/ei_config.h.in +++ /dev/null @@ -1,3 +0,0 @@ - -/* Define to 1 if you have the `deprecated' attribute */ -#undef HAVE_DEPRECATED_ATTRIBUTE diff --git a/lib/erl_interface/src/Makefile b/lib/erl_interface/src/Makefile index 800557fbfe..00c49f1622 100644 --- a/lib/erl_interface/src/Makefile +++ b/lib/erl_interface/src/Makefile @@ -27,7 +27,9 @@ include $(ERL_TOP)/make/output.mk include $(ERL_TOP)/make/target.mk debug opt shared purify quantify purecov gcov: +ifndef TERTIARY_BOOTSTRAP $(make_verbose)$(MAKE) -f $(TARGET)/Makefile TYPE=$@ +endif clean depend docs release release_docs tests release_tests check xmllint: $(make_verbose)$(MAKE) -f $(TARGET)/Makefile $@ diff --git a/lib/erl_interface/src/Makefile.in b/lib/erl_interface/src/Makefile.in index 1b19a85f1e..b2600f0fab 100644 --- a/lib/erl_interface/src/Makefile.in +++ b/lib/erl_interface/src/Makefile.in @@ -118,7 +118,7 @@ else AR_FLAGS=rcv endif -INCFLAGS = -I. -I../include -I../include/$(TARGET) -Iconnect -Iencode -Idecode -Imisc -Iepmd \ +INCFLAGS = -I. -I../include -Iconnect -Iencode -Idecode -Imisc -Iepmd \ -Iregistry -I$(TARGET) ifeq ($(USING_VC),yes) @@ -322,8 +322,7 @@ HEADERS = \ ../include/ei.h \ ../include/ei_connect.h \ ../include/eicode.h \ - ../include/erl_interface.h \ - ../include/$(TARGET)/ei_config.h + ../include/erl_interface.h EISOURCES = \ $(CONNECTSRC) \ diff --git a/lib/erl_interface/src/misc/eidef.h b/lib/erl_interface/src/misc/eidef.h index 083814c6e9..f38824d826 100644 --- a/lib/erl_interface/src/misc/eidef.h +++ b/lib/erl_interface/src/misc/eidef.h @@ -25,7 +25,6 @@ /* Common definitions used in ei user interface */ -#define EI_INCLUDED_CONFIG_H__ #include "config.h" /* Central include of config.h */ /* vxWorks.h needs to be before stddef.h */ diff --git a/lib/erl_interface/src/prog/ei_fake_prog.c b/lib/erl_interface/src/prog/ei_fake_prog.c index c7a16dc7c4..158464b385 100644 --- a/lib/erl_interface/src/prog/ei_fake_prog.c +++ b/lib/erl_interface/src/prog/ei_fake_prog.c @@ -98,11 +98,18 @@ int main(void) EI_ULONGLONG ulonglongx = 0; #endif erlang_char_encoding enc; + ei_socket_callbacks cbs; intx = erl_errno; + ei_init(); + + ei_close_connection(intx); + ei_connect_init(&xec, charp, charp, creation); + ei_connect_init_ussi(&xec, charp, charp, creation, &cbs, sizeof(cbs), NULL); ei_connect_xinit (&xec, charp, charp, charp, thisipaddr, charp, creation); + ei_connect_xinit_ussi(&xec, charp, charp, charp, thisipaddr, charp, creation, &cbs, sizeof(cbs), NULL); ei_connect(&xec, charp); ei_xconnect (&xec, thisipaddr, charp); @@ -121,6 +128,8 @@ int main(void) ei_publish(&xec, intx); ei_accept(&xec, intx, &conp); ei_unpublish(&xec); + ei_listen(&xec, intp, intx); + ei_xlisten(&xec, thisipaddr, intp, intx); ei_thisnodename(&xec); ei_thishostname(&xec); @@ -187,7 +196,7 @@ int main(void) ei_decode_char(charp, intp, charp); ei_decode_string(charp, intp, charp); ei_decode_atom(charp, intp, charp); - ei_decode_atom_as(charp, intp, charp, MAXATOMLEN_UTF8, ERLANG_WHATEVER, &enc, &enc); + ei_decode_atom_as(charp, intp, charp, MAXATOMLEN_UTF8, ERLANG_UTF8, &enc, &enc); ei_decode_binary(charp, intp, (void *)0, longp); ei_decode_fun(charp, intp, &efun); free_fun(&efun); diff --git a/lib/erl_interface/src/prog/erl_call.c b/lib/erl_interface/src/prog/erl_call.c index 52ad6885e8..ab91157035 100644 --- a/lib/erl_interface/src/prog/erl_call.c +++ b/lib/erl_interface/src/prog/erl_call.c @@ -88,10 +88,6 @@ #include "ei_resolve.h" #include "erl_start.h" /* FIXME remove dependency */ -#ifdef __WIN32__ -static void initWinSock(void); -#endif - /* * Some nice global variables * (I don't think "nice" is the right word actually... -gordon) @@ -157,6 +153,8 @@ int erl_call(int argc, char **argv) char* progname = argv[0]; ei_cnode ec; + ei_init(); + /* Get the command line options */ while (i < argc) { if (argv[i][0] != '-') { @@ -317,14 +315,6 @@ int erl_call(int argc, char **argv) struct in_addr h_ipadr; char* ct; -#ifdef __WIN32__ - /* - * FIXME Extremly ugly, but needed to get ei_gethostbyname() below - * to work. - */ - initWinSock(); -#endif - /* gethostname requires len to be max(hostname) + 1 */ if (gethostname(h_hostname, EI_MAXHOSTNAMELEN+1) < 0) { fprintf(stderr,"erl_call: failed to get host name: %d\n", errno); @@ -857,46 +847,6 @@ static void usage(const char *progname) { exit(0); } - -/*************************************************************************** - * - * OS specific functions - * - ***************************************************************************/ - -#ifdef __WIN32__ -/* - * FIXME This should not be here. This is a quick fix to make erl_call - * work at all on Windows NT. - */ -static void initWinSock(void) -{ - WORD wVersionRequested; - WSADATA wsaData; - int err; - static int initialized; - - wVersionRequested = MAKEWORD(1, 1); - if (!initialized) { - initialized = 1; - err = WSAStartup(wVersionRequested, &wsaData); - - if (err != 0) { - fprintf(stderr,"erl_call: " - "Can't initialize windows sockets: %d\n", err); - } - - if ( LOBYTE( wsaData.wVersion ) != 1 || - HIBYTE( wsaData.wVersion ) != 1 ) { - fprintf(stderr,"erl_call: This version of " - "windows sockets not supported\n"); - WSACleanup(); - } - } -} -#endif - - /*************************************************************************** * * Utility functions diff --git a/lib/erl_interface/src/prog/erl_start.c b/lib/erl_interface/src/prog/erl_start.c index ba495ac818..b7aa451946 100644 --- a/lib/erl_interface/src/prog/erl_start.c +++ b/lib/erl_interface/src/prog/erl_start.c @@ -657,7 +657,7 @@ static int wait_for_erlang(int sockd, int magic, struct timeval *timeout) gettimeofday(&now,NULL); to.tv_sec = stop_time.tv_sec - now.tv_sec; to.tv_usec = stop_time.tv_usec - now.tv_usec; - while ((to.tv_usec <= 0) && (to.tv_sec >= 0)) { + while ((to.tv_usec < 0) && (to.tv_sec > 0)) { to.tv_usec += 1000000; to.tv_sec--; } diff --git a/lib/erl_interface/test/Makefile b/lib/erl_interface/test/Makefile index 94f4b422d6..f8f2ef0156 100644 --- a/lib/erl_interface/test/Makefile +++ b/lib/erl_interface/test/Makefile @@ -33,6 +33,7 @@ MODULES= \ ei_format_SUITE \ ei_print_SUITE \ ei_tmo_SUITE \ + erl_call_SUITE \ erl_connect_SUITE \ erl_global_SUITE \ erl_eterm_SUITE \ diff --git a/lib/erl_interface/test/all_SUITE_data/Makefile.src b/lib/erl_interface/test/all_SUITE_data/Makefile.src index 57e522fd3e..4f27b097c8 100644 --- a/lib/erl_interface/test/all_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/all_SUITE_data/Makefile.src @@ -21,8 +21,8 @@ include @erl_interface_mk_include@ CC0 = @CC@ CC = .@DS@gccifier@exe@ -CC"$(CC0)" -CFLAGS0 = @CFLAGS@ -I@erl_interface_include@ @erl_interface_target_include@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ +CFLAGS0 = @CFLAGS@ -I@erl_interface_include@ +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ EI_COMMON_OBJS = runner@obj@ ei_runner@obj@ ALL_OBJS = gccifier@exe@ $(EI_COMMON_OBJS) diff --git a/lib/erl_interface/test/ei_accept_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_accept_SUITE_data/Makefile.src index 76dc84221f..10ef437f8b 100644 --- a/lib/erl_interface/test/ei_accept_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_accept_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_ACCEPT_OBJS = ei_accept_test@obj@ ei_accept_test_decl@obj@ EIACCNODE_OBJS = eiaccnode@obj@ diff --git a/lib/erl_interface/test/ei_connect_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_connect_SUITE_data/Makefile.src index d1694e607d..c2d8261dd8 100644 --- a/lib/erl_interface/test/ei_connect_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_connect_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBERL) $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_CONNECT_OBJS = ei_connect_test@obj@ ei_connect_test_decl@obj@ EINODE_OBJS = einode@obj@ diff --git a/lib/erl_interface/test/ei_decode_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_decode_SUITE_data/Makefile.src index 76591d893c..e678914a40 100644 --- a/lib/erl_interface/test/ei_decode_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_decode_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_DECODE_OBJS = ei_decode_test@obj@ ei_decode_test_decl@obj@ all: ei_decode_test@exe@ diff --git a/lib/erl_interface/test/ei_decode_encode_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_decode_encode_SUITE_data/Makefile.src index 3f5fa4f295..853fe9ddeb 100644 --- a/lib/erl_interface/test/ei_decode_encode_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_decode_encode_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_DECODE_ENCODE_OBJS = ei_decode_encode_test@obj@ ei_decode_encode_test_decl@obj@ all: ei_decode_encode_test@exe@ diff --git a/lib/erl_interface/test/ei_encode_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_encode_SUITE_data/Makefile.src index 489382d85e..3b2cab7af4 100644 --- a/lib/erl_interface/test/ei_encode_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_encode_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_ENCODE_OBJS = ei_encode_test@obj@ ei_encode_test_decl@obj@ all: ei_encode_test@exe@ diff --git a/lib/erl_interface/test/ei_format_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_format_SUITE_data/Makefile.src index 9e5a271db6..b89dcae45a 100644 --- a/lib/erl_interface/test/ei_format_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_format_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_FORMAT_OBJS = ei_format_test@obj@ ei_format_test_decl@obj@ all: ei_format_test@exe@ diff --git a/lib/erl_interface/test/ei_print_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_print_SUITE_data/Makefile.src index 354011f1a5..150c11b99c 100644 --- a/lib/erl_interface/test/ei_print_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_print_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_PRINT_OBJS = ei_print_test@obj@ ei_print_test_decl@obj@ all: ei_print_test@exe@ diff --git a/lib/erl_interface/test/ei_tmo_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_tmo_SUITE_data/Makefile.src index 76a9c6a606..b4ee361939 100644 --- a/lib/erl_interface/test/ei_tmo_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_tmo_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_TMO_OBJS = ei_tmo_test@obj@ ei_tmo_test_decl@obj@ all: ei_tmo_test@exe@ diff --git a/lib/erl_interface/test/erl_call_SUITE.erl b/lib/erl_interface/test/erl_call_SUITE.erl new file mode 100644 index 0000000000..9e2b2e4251 --- /dev/null +++ b/lib/erl_interface/test/erl_call_SUITE.erl @@ -0,0 +1,96 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +-module(erl_call_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([all/0, smoke/1]). + +all() -> + [smoke]. + +smoke(Config) when is_list(Config) -> + ErlCall = find_erl_call(), + NameSwitch = case net_kernel:longnames() of + true -> + "-name"; + false -> + "-sname" + end, + Name = atom_to_list(?MODULE) + ++ "-" + ++ integer_to_list(erlang:system_time(microsecond)), + + ArgsList = ["-s", "-a", "erlang node", NameSwitch, Name], + io:format("erl_call: \"~ts\"\n~nargs list: ~p~n", [ErlCall, ArgsList]), + CmdRes = get_smoke_port_res(open_port({spawn_executable, ErlCall}, + [{args, ArgsList}, eof]), []), + io:format("CmdRes: ~p~n", [CmdRes]), + + [_, Hostname] = string:lexemes(atom_to_list(node()), "@"), + NodeName = list_to_atom(Name ++ "@" ++ Hostname), + io:format("NodeName: ~p~n~n", [NodeName]), + + pong = net_adm:ping(NodeName), + rpc:cast(NodeName, erlang, halt, []), + NodeName = list_to_atom(string:trim(CmdRes, both, "'")), + ok. + +% +% Utility functions... +% + +find_erl_call() -> + ErlCallName = case os:type() of + {win32, _} -> "erl_call.exe"; + _ -> "erl_call" + end, + LibDir = code:lib_dir(erl_interface), + InstalledErlCall = filename:join([LibDir, "bin", ErlCallName]), + TargetDir = erlang:system_info(system_architecture), + TargetErlCall = filename:join([LibDir, "bin", TargetDir, ErlCallName]), + + try + lists:foreach(fun (F) -> + io:format("Checking: \"~ts\"~n", [F]), + case file:read_file_info(F) of + {ok, _} -> + throw(F); + _ -> + ok + end + end, + [InstalledErlCall, TargetErlCall]), + exit({missing, erl_call}) + catch + throw:ErlCall -> + ErlCall + end. + +get_smoke_port_res(Port, Acc) when is_port(Port) -> + receive + {Port, {data, Data}} -> + get_smoke_port_res(Port, [Acc|Data]); + {Port, eof} -> + lists:flatten(Acc) + end. + diff --git a/lib/erl_interface/test/erl_connect_SUITE_data/Makefile.src b/lib/erl_interface/test/erl_connect_SUITE_data/Makefile.src index 19b076794a..ff4c382c97 100644 --- a/lib/erl_interface/test/erl_connect_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/erl_connect_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/runner@obj@ \ $(LIBERL) $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data OBJS = erl_connect_test@obj@ erl_connect_test_decl@obj@ all: erl_connect_test@exe@ diff --git a/lib/erl_interface/test/erl_eterm_SUITE_data/Makefile.src b/lib/erl_interface/test/erl_eterm_SUITE_data/Makefile.src index cd4de56d7c..4b1ddf77b6 100644 --- a/lib/erl_interface/test/erl_eterm_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/erl_eterm_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/runner@obj@ \ $(LIBERL) $(LIBEI) @erl_interface_sock_libs@ @LIBS@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data ETERM_OBJS = eterm_test@obj@ eterm_test_decl@obj@ CNODE_OBJS = cnode@obj@ PRINT_OBJS = print_term@obj@ diff --git a/lib/erl_interface/test/erl_ext_SUITE_data/Makefile.src b/lib/erl_interface/test/erl_ext_SUITE_data/Makefile.src index 50b60637cd..fe8caebbd6 100644 --- a/lib/erl_interface/test/erl_ext_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/erl_ext_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/runner@obj@ \ $(LIBERL) $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EXT_OBJS = ext_test@obj@ ext_test_decl@obj@ all: ext_test@exe@ diff --git a/lib/erl_interface/test/erl_format_SUITE_data/Makefile.src b/lib/erl_interface/test/erl_format_SUITE_data/Makefile.src index 7d51cd6007..2ba59ab651 100644 --- a/lib/erl_interface/test/erl_format_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/erl_format_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/runner@obj@ \ $(LIBERL) $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data FORMAT_OBJS = format_test@obj@ format_test_decl@obj@ all: format_test@exe@ diff --git a/lib/erl_interface/test/erl_global_SUITE_data/Makefile.src b/lib/erl_interface/test/erl_global_SUITE_data/Makefile.src index 9f2a8619ac..1c1530d1b6 100644 --- a/lib/erl_interface/test/erl_global_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/erl_global_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/runner@obj@ \ $(LIBERL) $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data OBJS = erl_global_test@obj@ erl_global_test_decl@obj@ all: erl_global_test@exe@ diff --git a/lib/erl_interface/test/erl_match_SUITE_data/Makefile.src b/lib/erl_interface/test/erl_match_SUITE_data/Makefile.src index 56ed1df203..156214a269 100644 --- a/lib/erl_interface/test/erl_match_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/erl_match_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/runner@obj@ \ $(LIBERL) $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data MATCH_OBJS = match_test@obj@ match_test_decl@obj@ all: match_test@exe@ diff --git a/lib/erl_interface/test/port_call_SUITE_data/Makefile.src b/lib/erl_interface/test/port_call_SUITE_data/Makefile.src index e09e0fe175..0f97ce9f70 100644 --- a/lib/erl_interface/test/port_call_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/port_call_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBERL = @erl_interface_lib_drv@ LIBEI = @erl_interface_eilib_drv@ SHLIB_EXTRA_LDLIBS = $(LIBERL) $(LIBEI) @erl_interface_threadlib@ -SHLIB_EXTRA_CFLAGS = -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +SHLIB_EXTRA_CFLAGS = -I@erl_interface_include@ -I../all_SUITE_data all: port_call_drv@dll@ diff --git a/lib/erl_interface/vsn.mk b/lib/erl_interface/vsn.mk index 06ef907d6c..dae6052d55 100644 --- a/lib/erl_interface/vsn.mk +++ b/lib/erl_interface/vsn.mk @@ -1,2 +1,2 @@ -EI_VSN = 3.10.4 +EI_VSN = 3.11.1 ERL_INTERFACE_VSN = $(EI_VSN) diff --git a/lib/ftp/doc/src/notes.xml b/lib/ftp/doc/src/notes.xml index 01c1f88cf1..61da079900 100644 --- a/lib/ftp/doc/src/notes.xml +++ b/lib/ftp/doc/src/notes.xml @@ -33,7 +33,23 @@ <file>notes.xml</file> </header> - <section><title>Ftp 1.0.1</title> + <section><title>Ftp 1.0.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed timing related bug that could make ftp functions + behave badly.</p> + <p> + Own Id: OTP-15659 Aux Id: ERIERL-316 </p> + </item> + </list> + </section> + +</section> + +<section><title>Ftp 1.0.1</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/ftp/src/ftp.erl b/lib/ftp/src/ftp.erl index 40f6b53fa3..18cd8c7524 100644 --- a/lib/ftp/src/ftp.erl +++ b/lib/ftp/src/ftp.erl @@ -1065,9 +1065,9 @@ handle_call({_, {open, ip_comm, Opts, {CtrlOpts, DataPassOpts, DataActOpts}}}, F end end; -handle_call({_, {open, tls_upgrade, TLSOptions}}, From, State) -> - _ = send_ctrl_message(State, mk_cmd("AUTH TLS", [])), - activate_ctrl_connection(State), +handle_call({_, {open, tls_upgrade, TLSOptions}}, From, State0) -> + _ = send_ctrl_message(State0, mk_cmd("AUTH TLS", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From, caller = open, tls_options = TLSOptions}}; handle_call({_, {user, User, Password}}, From, @@ -1081,17 +1081,17 @@ handle_call({_, {user, User, Password, Acc}}, From, handle_call({_, {account, Acc}}, From, State)-> handle_user_account(Acc, State#state{client = From}); -handle_call({_, pwd}, From, #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("PWD", [])), - activate_ctrl_connection(State), +handle_call({_, pwd}, From, #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("PWD", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From, caller = pwd}}; handle_call({_, lpwd}, From, #state{ldir = LDir} = State) -> {reply, {ok, LDir}, State#state{client = From}}; -handle_call({_, {cd, Dir}}, From, #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("CWD ~s", [Dir])), - activate_ctrl_connection(State), +handle_call({_, {cd, Dir}}, From, #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("CWD ~s", [Dir])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From, caller = cd}}; handle_call({_,{lcd, Dir}}, _From, #state{ldir = LDir0} = State) -> @@ -1108,41 +1108,41 @@ handle_call({_, {dir, Len, Dir}}, {_Pid, _} = From, setup_data_connection(State#state{caller = {dir, Dir, Len}, client = From}); handle_call({_, {rename, CurrFile, NewFile}}, From, - #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("RNFR ~s", [CurrFile])), - activate_ctrl_connection(State), + #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("RNFR ~s", [CurrFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {rename, NewFile}, client = From}}; handle_call({_, {delete, File}}, {_Pid, _} = From, - #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("DELE ~s", [File])), - activate_ctrl_connection(State), + #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("DELE ~s", [File])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From}}; -handle_call({_, {mkdir, Dir}}, From, #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("MKD ~s", [Dir])), - activate_ctrl_connection(State), +handle_call({_, {mkdir, Dir}}, From, #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("MKD ~s", [Dir])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From}}; -handle_call({_,{rmdir, Dir}}, From, #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("RMD ~s", [Dir])), - activate_ctrl_connection(State), +handle_call({_,{rmdir, Dir}}, From, #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("RMD ~s", [Dir])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From}}; -handle_call({_,{type, Type}}, From, #state{chunk = false} = State) -> +handle_call({_,{type, Type}}, From, #state{chunk = false} = State0) -> case Type of ascii -> - _ = send_ctrl_message(State, mk_cmd("TYPE A", [])), - activate_ctrl_connection(State), + _ = send_ctrl_message(State0, mk_cmd("TYPE A", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = type, type = ascii, client = From}}; binary -> - _ = send_ctrl_message(State, mk_cmd("TYPE I", [])), - activate_ctrl_connection(State), + _ = send_ctrl_message(State0, mk_cmd("TYPE I", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = type, type = binary, client = From}}; _ -> - {reply, {error, etype}, State} + {reply, {error, etype}, State0} end; handle_call({_,{recv, RemoteFile, LocalFile}}, From, @@ -1181,8 +1181,8 @@ handle_call({_, recv_chunk}, _From, #state{chunk = true, } = State0) -> %% The ftp:recv_chunk call was the last event we waited for, finnish and clean up ?DBG("recv_chunk_closing ftp:recv_chunk, last event",[]), - activate_ctrl_connection(State0), - {reply, ok, State0#state{caller = undefined, + State = activate_ctrl_connection(State0), + {reply, ok, State#state{caller = undefined, chunk = false, client = undefined}}; @@ -1238,18 +1238,18 @@ handle_call({_, {transfer_chunk, Bin}}, _, #state{chunk = true} = State) -> handle_call({_, {transfer_chunk, _}}, _, #state{chunk = false} = State) -> {reply, {error, echunk}, State}; -handle_call({_, chunk_end}, From, #state{chunk = true} = State) -> - close_data_connection(State), - activate_ctrl_connection(State), +handle_call({_, chunk_end}, From, #state{chunk = true} = State0) -> + close_data_connection(State0), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From, dsock = undefined, caller = end_chunk_transfer, chunk = false}}; handle_call({_, chunk_end}, _, #state{chunk = false} = State) -> {reply, {error, echunk}, State}; -handle_call({_, {quote, Cmd}}, From, #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd(Cmd, [])), - activate_ctrl_connection(State), +handle_call({_, {quote, Cmd}}, From, #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd(Cmd, [])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From, caller = quote}}; handle_call({_, _Req}, _From, #state{csock = CSock} = State) @@ -1325,38 +1325,38 @@ handle_info({Trpt, Socket, Data}, #state{dsock = {Trpt,Socket}} = State0) when T Data/binary>>}}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, - caller = {recv_file, Fd}} = State) + caller = {recv_file, Fd}} = State0) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> file_close(Fd), - progress_report({transfer_size, 0}, State), - activate_ctrl_connection(State), + progress_report({transfer_size, 0}, State0), + State = activate_ctrl_connection(State0), ?DBG("Data channel close",[]), {noreply, State#state{dsock = undefined, data = <<>>}}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, client = Client, - caller = recv_chunk} = State) + caller = recv_chunk} = State0) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> ?DBG("Data channel close recv_chunk",[]), - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{dsock = undefined, caller = #recv_chunk_closing{dconn_closed = true, client_called_us = Client =/= undefined} }}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, caller = recv_bin, - data = Data} = State) + data = Data} = State0) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> ?DBG("Data channel close",[]), - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{dsock = undefined, data = <<>>, caller = {recv_bin, Data}}}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, data = Data, - caller = {handle_dir_result, Dir}} - = State) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> + caller = {handle_dir_result, Dir}} + = State0) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> ?DBG("Data channel close",[]), - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{dsock = undefined, caller = {handle_dir_result, Dir, Data}, % data = <<?CR,?LF>>}}; @@ -1377,7 +1377,7 @@ handle_info({Transport, Socket, Data}, #state{csock = {Transport, Socket}, client = From, ctrl_data = {CtrlData, AccLines, LineStatus}} - = State) -> + = State0) -> ?DBG('--ctrl ~p ----> ~s~p~n',[Socket,<<CtrlData/binary, Data/binary>>,State]), case ftp_response:parse_lines(<<CtrlData/binary, Data/binary>>, AccLines, LineStatus) of @@ -1387,21 +1387,21 @@ handle_info({Transport, Socket, Data}, #state{csock = {Transport, Socket}, case Caller of quote -> gen_server:reply(From, string:tokens(Lines, [?CR, ?LF])), - {noreply, State#state{client = undefined, - caller = undefined, - latest_ctrl_response = Lines, - ctrl_data = {NextMsgData, [], - start}}}; + {noreply, State0#state{client = undefined, + caller = undefined, + latest_ctrl_response = Lines, + ctrl_data = {NextMsgData, [], + start}}}; _ -> ?DBG(' ...handle_ctrl_result(~p,...) ctrl_data=~p~n',[CtrlResult,{NextMsgData, [], start}]), handle_ctrl_result(CtrlResult, - State#state{latest_ctrl_response = Lines, - ctrl_data = - {NextMsgData, [], start}}) + State0#state{latest_ctrl_response = Lines, + ctrl_data = + {NextMsgData, [], start}}) end; {continue, NewCtrlData} -> ?DBG(' ...Continue... ctrl_data=~p~n',[NewCtrlData]), - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{ctrl_data = NewCtrlData}} end; @@ -1567,19 +1567,19 @@ start_link(Opts, GenServerOptions) -> %%% Help functions to handle_call and/or handle_ctrl_result %%-------------------------------------------------------------------------- %% User handling -handle_user(User, Password, Acc, State) -> - _ = send_ctrl_message(State, mk_cmd("USER ~s", [User])), - activate_ctrl_connection(State), +handle_user(User, Password, Acc, State0) -> + _ = send_ctrl_message(State0, mk_cmd("USER ~s", [User])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {handle_user, Password, Acc}}}. -handle_user_passwd(Password, Acc, State) -> - _ = send_ctrl_message(State, mk_cmd("PASS ~s", [Password])), - activate_ctrl_connection(State), +handle_user_passwd(Password, Acc, State0) -> + _ = send_ctrl_message(State0, mk_cmd("PASS ~s", [Password])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {handle_user_passwd, Acc}}}. -handle_user_account(Acc, State) -> - _ = send_ctrl_message(State, mk_cmd("ACCT ~s", [Acc])), - activate_ctrl_connection(State), +handle_user_account(Acc, State0) -> + _ = send_ctrl_message(State0, mk_cmd("ACCT ~s", [Acc])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = handle_user_account}}. @@ -1594,9 +1594,9 @@ handle_ctrl_result({tls_upgrade, _}, #state{csock = {tcp, Socket}, ?DBG('<--ctrl ssl:connect(~p, ~p)~n~p~n',[Socket,TLSOptions,State0]), case ssl:connect(Socket, TLSOptions, Timeout) of {ok, TLSSocket} -> - State = State0#state{csock = {ssl,TLSSocket}}, - _ = send_ctrl_message(State, mk_cmd("PBSZ 0", [])), - activate_ctrl_connection(State), + State1 = State0#state{csock = {ssl,TLSSocket}}, + _ = send_ctrl_message(State1, mk_cmd("PBSZ 0", [])), + State = activate_ctrl_connection(State1), {noreply, State#state{tls_upgrading_data_connection = {true, pbsz}} }; {error, _} = Error -> gen_server:reply(From, {Error, self()}), @@ -1605,9 +1605,9 @@ handle_ctrl_result({tls_upgrade, _}, #state{csock = {tcp, Socket}, tls_upgrading_data_connection = false}} end; -handle_ctrl_result({pos_compl, _}, #state{tls_upgrading_data_connection = {true, pbsz}} = State) -> - _ = send_ctrl_message(State, mk_cmd("PROT P", [])), - activate_ctrl_connection(State), +handle_ctrl_result({pos_compl, _}, #state{tls_upgrading_data_connection = {true, pbsz}} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("PROT P", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{tls_upgrading_data_connection = {true, prot}}}; handle_ctrl_result({pos_compl, _}, #state{tls_upgrading_data_connection = {true, prot}, @@ -1801,10 +1801,10 @@ handle_ctrl_result({pos_compl, _}, #state{caller = {handle_dir_result, Dir, handle_ctrl_result({pos_compl, Lines}, #state{caller = {handle_dir_data, Dir, DirData}} = - State) -> + State0) -> OldDir = pwd_result(Lines), - _ = send_ctrl_message(State, mk_cmd("CWD ~s", [Dir])), - activate_ctrl_connection(State), + _ = send_ctrl_message(State0, mk_cmd("CWD ~s", [Dir])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {handle_dir_data_second_phase, OldDir, DirData}}}; handle_ctrl_result({Status, _}, @@ -1818,9 +1818,9 @@ handle_ctrl_result(S={_Status, _}, handle_ctrl_result({pos_compl, _}, #state{caller = {handle_dir_data_second_phase, OldDir, - DirData}} = State) -> - _ = send_ctrl_message(State, mk_cmd("CWD ~s", [OldDir])), - activate_ctrl_connection(State), + DirData}} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("CWD ~s", [OldDir])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {handle_dir_data_third_phase, DirData}}}; handle_ctrl_result({Status, _}, #state{caller = {handle_dir_data_second_phase, _, _}} @@ -1840,9 +1840,9 @@ handle_ctrl_result(Status={epath, _}, #state{caller = {dir,_}} = State) -> %%-------------------------------------------------------------------------- %% File renaming handle_ctrl_result({pos_interm, _}, #state{caller = {rename, NewFile}} - = State) -> - _ = send_ctrl_message(State, mk_cmd("RNTO ~s", [NewFile])), - activate_ctrl_connection(State), + = State0) -> + _ = send_ctrl_message(State0, mk_cmd("RNTO ~s", [NewFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = rename_second_phase}}; handle_ctrl_result({Status, _}, @@ -1916,10 +1916,10 @@ handle_ctrl_result({pos_compl, _}, #state{client = From, %% The pos_compl was the last event we waited for, finnish and clean up ?DBG("recv_chunk_closing pos_compl, last event",[]), gen_server:reply(From, ok), - activate_ctrl_connection(State0), - {noreply, State0#state{caller = undefined, - chunk = false, - client = undefined}}; + State = activate_ctrl_connection(State0), + {noreply, State#state{caller = undefined, + chunk = false, + client = undefined}}; handle_ctrl_result({pos_compl, _}, #state{caller = #recv_chunk_closing{}=R} = State0) -> @@ -2013,71 +2013,71 @@ ctrl_result_response(_, #state{client = From} = State, ErrorMsg) -> {noreply, State#state{client = undefined, caller = undefined}}. %%-------------------------------------------------------------------------- -handle_caller(#state{caller = {dir, Dir, Len}} = State) -> +handle_caller(#state{caller = {dir, Dir, Len}} = State0) -> Cmd = case Len of short -> "NLST"; long -> "LIST" end, _ = case Dir of "" -> - send_ctrl_message(State, mk_cmd(Cmd, "")); + send_ctrl_message(State0, mk_cmd(Cmd, "")); _ -> - send_ctrl_message(State, mk_cmd(Cmd ++ " ~s", [Dir])) + send_ctrl_message(State0, mk_cmd(Cmd ++ " ~s", [Dir])) end, - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {dir, Dir}}}; -handle_caller(#state{caller = {recv_bin, RemoteFile}} = State) -> - _ = send_ctrl_message(State, mk_cmd("RETR ~s", [RemoteFile])), - activate_ctrl_connection(State), +handle_caller(#state{caller = {recv_bin, RemoteFile}} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("RETR ~s", [RemoteFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = recv_bin}}; handle_caller(#state{caller = {start_chunk_transfer, Cmd, RemoteFile}} = - State) -> - _ = send_ctrl_message(State, mk_cmd("~s ~s", [Cmd, RemoteFile])), - activate_ctrl_connection(State), + State0) -> + _ = send_ctrl_message(State0, mk_cmd("~s ~s", [Cmd, RemoteFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = start_chunk_transfer}}; -handle_caller(#state{caller = {recv_file, RemoteFile, Fd}} = State) -> - _ = send_ctrl_message(State, mk_cmd("RETR ~s", [RemoteFile])), - activate_ctrl_connection(State), +handle_caller(#state{caller = {recv_file, RemoteFile, Fd}} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("RETR ~s", [RemoteFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {recv_file, Fd}}}; handle_caller(#state{caller = {transfer_file, {Cmd, LocalFile, RemoteFile}}, - ldir = LocalDir, client = From} = State) -> + ldir = LocalDir, client = From} = State0) -> case file_open(filename:absname(LocalFile, LocalDir), read) of {ok, Fd} -> - _ = send_ctrl_message(State, mk_cmd("~s ~s", [Cmd, RemoteFile])), - activate_ctrl_connection(State), + _ = send_ctrl_message(State0, mk_cmd("~s ~s", [Cmd, RemoteFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {transfer_file, Fd}}}; {error, _} -> gen_server:reply(From, {error, epath}), - {noreply, State#state{client = undefined, caller = undefined, - dsock = undefined}} + {noreply, State0#state{client = undefined, caller = undefined, + dsock = undefined}} end; handle_caller(#state{caller = {transfer_data, {Cmd, Bin, RemoteFile}}} = - State) -> - _ = send_ctrl_message(State, mk_cmd("~s ~s", [Cmd, RemoteFile])), - activate_ctrl_connection(State), + State0) -> + _ = send_ctrl_message(State0, mk_cmd("~s ~s", [Cmd, RemoteFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {transfer_data, Bin}}}. %% ----------- FTP SERVER COMMUNICATION ------------------------- %% Connect to FTP server at Host (default is TCP port 21) %% in order to establish a control connection. -setup_ctrl_connection(Host, Port, Timeout, #state{sockopts_ctrl = SockOpts} = State) -> +setup_ctrl_connection(Host, Port, Timeout, #state{sockopts_ctrl = SockOpts} = State0) -> MsTime = erlang:monotonic_time(), - case connect(Host, Port, SockOpts, Timeout, State) of + case connect(Host, Port, SockOpts, Timeout, State0) of {ok, IpFam, CSock} -> - NewState = State#state{csock = {tcp, CSock}, ipfamily = IpFam}, - activate_ctrl_connection(NewState), + State1 = State0#state{csock = {tcp, CSock}, ipfamily = IpFam}, + State = activate_ctrl_connection(State1), case Timeout - millisec_passed(MsTime) of Timeout2 when (Timeout2 >= 0) -> - {ok, NewState#state{caller = open}, Timeout2}; + {ok, State#state{caller = open}, Timeout2}; _ -> %% Oups: Simulate timeout - {ok, NewState#state{caller = open}, 0} + {ok, State#state{caller = open}, 0} end; Error -> Error @@ -2087,7 +2087,7 @@ setup_data_connection(#state{mode = active, caller = Caller, csock = CSock, sockopts_data_active = SockOpts, - ftp_extension = FtpExt} = State) -> + ftp_extension = FtpExt} = State0) -> case (catch sockname(CSock)) of {ok, {{_, _, _, _, _, _, _, _} = IP0, _}} -> IP = proplists:get_value(ip, SockOpts, IP0), @@ -2098,8 +2098,8 @@ setup_data_connection(#state{mode = active, {ok, {_, Port}} = sockname({tcp,LSock}), IpAddress = inet_parse:ntoa(IP), Cmd = mk_cmd("EPRT |2|~s|~p|", [IpAddress, Port]), - _ = send_ctrl_message(State, Cmd), - activate_ctrl_connection(State), + _ = send_ctrl_message(State0, Cmd), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {setup_data_connection, {LSock, Caller}}}}; {ok, {{_,_,_,_} = IP0, _}} -> @@ -2112,37 +2112,37 @@ setup_data_connection(#state{mode = active, false -> {IP1, IP2, IP3, IP4} = IP, {Port1, Port2} = {Port div 256, Port rem 256}, - send_ctrl_message(State, + send_ctrl_message(State0, mk_cmd("PORT ~w,~w,~w,~w,~w,~w", [IP1, IP2, IP3, IP4, Port1, Port2])); true -> IpAddress = inet_parse:ntoa(IP), Cmd = mk_cmd("EPRT |1|~s|~p|", [IpAddress, Port]), - send_ctrl_message(State, Cmd) + send_ctrl_message(State0, Cmd) end, - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {setup_data_connection, {LSock, Caller}}}} end; setup_data_connection(#state{mode = passive, ipfamily = inet6, - caller = Caller} = State) -> - _ = send_ctrl_message(State, mk_cmd("EPSV", [])), - activate_ctrl_connection(State), + caller = Caller} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("EPSV", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {setup_data_connection, Caller}}}; setup_data_connection(#state{mode = passive, ipfamily = inet, caller = Caller, - ftp_extension = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("PASV", [])), - activate_ctrl_connection(State), + ftp_extension = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("PASV", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {setup_data_connection, Caller}}}; setup_data_connection(#state{mode = passive, ipfamily = inet, caller = Caller, - ftp_extension = true} = State) -> - _ = send_ctrl_message(State, mk_cmd("EPSV", [])), - activate_ctrl_connection(State), + ftp_extension = true} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("EPSV", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {setup_data_connection, Caller}}}. connect(Host, Port, SockOpts, Timeout, #state{ipfamily = inet = IpFam}) -> @@ -2248,14 +2248,15 @@ send_message({tcp, Socket}, Message) -> send_message({ssl, Socket}, Message) -> ssl:send(Socket, Message). -activate_ctrl_connection(#state{csock = CSock, ctrl_data = {<<>>, _, _}}) -> - activate_connection(CSock); -activate_ctrl_connection(#state{csock = CSock}) -> +activate_ctrl_connection(#state{csock = CSock, ctrl_data = {<<>>, _, _}} = State) -> + activate_connection(CSock), + State; +activate_ctrl_connection(#state{csock = CSock} = State0) -> activate_connection(CSock), %% We have already received at least part of the next control message, %% that has been saved in ctrl_data, process this first. - self() ! {socket_type(CSock), unwrap_socket(CSock), <<>>}, - ok. + {noreply, State} = handle_info({socket_type(CSock), unwrap_socket(CSock), <<>>}, State0), + State. activate_data_connection(#state{dsock = DSock} = State) -> activate_connection(DSock), @@ -2290,22 +2291,22 @@ close_connection({ssl, Socket}) -> ignore_return_value( ssl:close(Socket) ). %% ------------ FILE HANDLING ---------------------------------------- send_file(#state{tls_upgrading_data_connection = {true, CTRL, _}} = State, Fd) -> {noreply, State#state{tls_upgrading_data_connection = {true, CTRL, ?MODULE, send_file, Fd}}}; -send_file(State, Fd) -> +send_file(State0, Fd) -> case file_read(Fd) of {ok, N, Bin} when N > 0 -> - send_data_message(State, Bin), - progress_report({binary, Bin}, State), - send_file(State, Fd); + send_data_message(State0, Bin), + progress_report({binary, Bin}, State0), + send_file(State0, Fd); {ok, _, _} -> file_close(Fd), - close_data_connection(State), - progress_report({transfer_size, 0}, State), - activate_ctrl_connection(State), + close_data_connection(State0), + progress_report({transfer_size, 0}, State0), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = transfer_file_second_phase, dsock = undefined}}; {error, Reason} -> - gen_server:reply(State#state.client, {error, Reason}), - {stop, normal, State#state{client = undefined}} + gen_server:reply(State0#state.client, {error, Reason}), + {stop, normal, State0#state{client = undefined}} end. file_open(File, Option) -> @@ -2347,10 +2348,10 @@ cast(GenServer, Msg) -> send_bin(#state{tls_upgrading_data_connection = {true, CTRL, _}} = State, Bin) -> State#state{tls_upgrading_data_connection = {true, CTRL, ?MODULE, send_bin, Bin}}; -send_bin(State, Bin) -> - send_data_message(State, Bin), - close_data_connection(State), - activate_ctrl_connection(State), +send_bin(State0, Bin) -> + send_data_message(State0, Bin), + close_data_connection(State0), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = transfer_data_second_phase, dsock = undefined}}. diff --git a/lib/ftp/test/ftp_SUITE.erl b/lib/ftp/test/ftp_SUITE.erl index 7c87d5cbdb..0b070ee8cb 100644 --- a/lib/ftp/test/ftp_SUITE.erl +++ b/lib/ftp/test/ftp_SUITE.erl @@ -96,6 +96,7 @@ ftp_tests()-> recv_chunk, recv_chunk_twice, recv_chunk_three_times, + recv_chunk_delay, type, quote, error_elogin, @@ -669,9 +670,9 @@ recv_chunk(Config0) -> Contents = list_to_binary( lists:duplicate(1000, lists:seq(0,255)) ), Config = set_state([reset, {mkfile,File,Contents}], Config0), Pid = proplists:get_value(ftp, Config), - {{error, "ftp:recv_chunk_start/2 not called"},_} = recv_chunk(Pid, <<>>), + {error, "ftp:recv_chunk_start/2 not called"} = do_recv_chunk(Pid), ok = ftp:recv_chunk_start(Pid, id2ftp(File,Config)), - {ok, ReceivedContents, _Ncunks} = recv_chunk(Pid, <<>>), + {ok, ReceivedContents} = do_recv_chunk(Pid), find_diff(ReceivedContents, Contents). recv_chunk_twice() -> @@ -683,11 +684,11 @@ recv_chunk_twice(Config0) -> Contents2 = crypto:strong_rand_bytes(1200), Config = set_state([reset, {mkfile,File1,Contents1}, {mkfile,File2,Contents2}], Config0), Pid = proplists:get_value(ftp, Config), - {{error, "ftp:recv_chunk_start/2 not called"},_} = recv_chunk(Pid, <<>>), + {error, "ftp:recv_chunk_start/2 not called"} = do_recv_chunk(Pid), ok = ftp:recv_chunk_start(Pid, id2ftp(File1,Config)), - {ok, ReceivedContents1, _Ncunks1} = recv_chunk(Pid, <<>>), + {ok, ReceivedContents1} = do_recv_chunk(Pid), ok = ftp:recv_chunk_start(Pid, id2ftp(File2,Config)), - {ok, ReceivedContents2, _Ncunks2} = recv_chunk(Pid, <<>>), + {ok, ReceivedContents2} = do_recv_chunk(Pid), find_diff(ReceivedContents1, Contents1), find_diff(ReceivedContents2, Contents2). @@ -704,46 +705,56 @@ recv_chunk_three_times(Config0) -> Config = set_state([reset, {mkfile,File1,Contents1}, {mkfile,File2,Contents2}, {mkfile,File3,Contents3}], Config0), Pid = proplists:get_value(ftp, Config), - {{error, "ftp:recv_chunk_start/2 not called"},_} = recv_chunk(Pid, <<>>), + {error, "ftp:recv_chunk_start/2 not called"} = do_recv_chunk(Pid), + ok = ftp:recv_chunk_start(Pid, id2ftp(File3,Config)), + {ok, ReceivedContents3} = do_recv_chunk(Pid), + ok = ftp:recv_chunk_start(Pid, id2ftp(File1,Config)), - {ok, ReceivedContents1, Nchunks1} = recv_chunk(Pid, <<>>), + {ok, ReceivedContents1} = do_recv_chunk(Pid), ok = ftp:recv_chunk_start(Pid, id2ftp(File2,Config)), - {ok, ReceivedContents2, _Nchunks2} = recv_chunk(Pid, <<>>), - - ok = ftp:recv_chunk_start(Pid, id2ftp(File3,Config)), - {ok, ReceivedContents3, _Nchunks3} = recv_chunk(Pid, <<>>, 10000, 0, Nchunks1), + {ok, ReceivedContents2} = do_recv_chunk(Pid), find_diff(ReceivedContents1, Contents1), find_diff(ReceivedContents2, Contents2), find_diff(ReceivedContents3, Contents3). - +do_recv_chunk(Pid) -> + recv_chunk(Pid, <<>>). recv_chunk(Pid, Acc) -> - recv_chunk(Pid, Acc, 0, 0, undefined). - - - -%% ExpectNchunks :: integer() | undefined -recv_chunk(Pid, Acc, DelayMilliSec, N, ExpectNchunks) when N+1 < ExpectNchunks -> - %% for all I in integer(), I < undefined - recv_chunk1(Pid, Acc, DelayMilliSec, N, ExpectNchunks); - -recv_chunk(Pid, Acc, DelayMilliSec, N, ExpectNchunks) -> - %% N >= ExpectNchunks-1 - timer:sleep(DelayMilliSec), - recv_chunk1(Pid, Acc, DelayMilliSec, N, ExpectNchunks). + case ftp:recv_chunk(Pid) of + ok -> + {ok, Acc}; + {ok, Bin} -> + recv_chunk(Pid, <<Acc/binary, Bin/binary>>); + Error -> + Error + end. +recv_chunk_delay(Config0) when is_list(Config0) -> + File1 = "big_file1.txt", + Contents = list_to_binary(lists:duplicate(1000, lists:seq(0,255))), + Config = set_state([reset, {mkfile,File1,Contents}], Config0), + Pid = proplists:get_value(ftp, Config), + ok = ftp:recv_chunk_start(Pid, id2ftp(File1,Config)), + {ok, ReceivedContents} = delay_recv_chunk(Pid), + find_diff(ReceivedContents, Contents). -recv_chunk1(Pid, Acc, DelayMilliSec, N, ExpectNchunks) -> - ct:log("Call ftp:recv_chunk",[]), +delay_recv_chunk(Pid) -> + delay_recv_chunk(Pid, <<>>). +delay_recv_chunk(Pid, Acc) -> + ct:pal("Recived size ~p", [byte_size(Acc)]), case ftp:recv_chunk(Pid) of - ok -> {ok, Acc, N}; - {ok, Bin} -> recv_chunk(Pid, <<Acc/binary, Bin/binary>>, DelayMilliSec, N+1, ExpectNchunks); - Error -> {Error, N} - end. + ok -> + {ok, Acc}; + {ok, Bin} -> + ct:sleep(100), + delay_recv_chunk(Pid, <<Acc/binary, Bin/binary>>); + Error -> + Error + end. %%------------------------------------------------------------------------- type() -> diff --git a/lib/ftp/vsn.mk b/lib/ftp/vsn.mk index d5d6c45b28..9f14658099 100644 --- a/lib/ftp/vsn.mk +++ b/lib/ftp/vsn.mk @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = ftp -FTP_VSN = 1.0.1 +FTP_VSN = 1.0.2 PRE_VSN = APP_VSN = "$(APPLICATION)-$(FTP_VSN)$(PRE_VSN)" diff --git a/lib/hipe/doc/src/notes.xml b/lib/hipe/doc/src/notes.xml index e9cdf42018..9a803cb9df 100644 --- a/lib/hipe/doc/src/notes.xml +++ b/lib/hipe/doc/src/notes.xml @@ -31,6 +31,21 @@ </header> <p>This document describes the changes made to HiPE.</p> +<section><title>Hipe 3.18.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Fix a bug in the handling of the <c>Key</c> argument + of <c>lists:{keysearch, keyfind, keymember}</c>. </p> + <p> + Own Id: OTP-15570</p> + </item> + </list> + </section> + +</section> + <section><title>Hipe 3.18.2</title> <section><title>Improvements and New Features</title> diff --git a/lib/hipe/vsn.mk b/lib/hipe/vsn.mk index 12d621bf01..39565d721f 100644 --- a/lib/hipe/vsn.mk +++ b/lib/hipe/vsn.mk @@ -1 +1 @@ -HIPE_VSN = 3.18.2 +HIPE_VSN = 3.18.3 diff --git a/lib/inets/doc/src/notes.xml b/lib/inets/doc/src/notes.xml index 31dae6317e..91dd9cd6ed 100644 --- a/lib/inets/doc/src/notes.xml +++ b/lib/inets/doc/src/notes.xml @@ -33,7 +33,43 @@ <file>notes.xml</file> </header> - <section><title>Inets 7.0.5</title> + <section><title>Inets 7.0.6</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix the internal handling of the option + erl_script_timeout in httpd. When httpd was started with + explicit erl_script_timeout, the value of the option was + converted to milliseconds before storage. Subsequent + calls to httpd:info/1 returned the input value multiplied + by 1000.</p> + <p> + This change fixes the handing of erl_script_timeout by + storing the timeout in seconds and converting to + milliseconds before usage.</p> + <p> + Own Id: OTP-15669 Aux Id: ERIERL-321 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Enhance documentation</p> + <p> + Own Id: OTP-15508 Aux Id: ERL-816 </p> + </item> + </list> + </section> + +</section> + +<section><title>Inets 7.0.5</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/inets/src/http_server/mod_esi.erl b/lib/inets/src/http_server/mod_esi.erl index 443b7ee564..f495f12f03 100644 --- a/lib/inets/src/http_server/mod_esi.erl +++ b/lib/inets/src/http_server/mod_esi.erl @@ -33,7 +33,7 @@ -include("httpd_internal.hrl"). -define(VMODULE,"ESI"). --define(DEFAULT_ERL_TIMEOUT,15000). +-define(DEFAULT_ERL_TIMEOUT,15). %%%========================================================================= @@ -174,7 +174,7 @@ store({erl_script_alias, Value}, _) -> {error, {wrong_type, {erl_script_alias, Value}}}; store({erl_script_timeout, TimeoutSec}, _) when is_integer(TimeoutSec) andalso (TimeoutSec >= 0) -> - {ok, {erl_script_timeout, TimeoutSec * 1000}}; + {ok, {erl_script_timeout, TimeoutSec}}; store({erl_script_timeout, Value}, _) -> {error, {wrong_type, {erl_script_timeout, Value}}}; store({erl_script_nocache, Value} = Conf, _) @@ -500,7 +500,7 @@ kill_esi_delivery_process(Pid) -> erl_script_timeout(Db) -> - httpd_util:lookup(Db, erl_script_timeout, ?DEFAULT_ERL_TIMEOUT). + httpd_util:lookup(Db, erl_script_timeout, ?DEFAULT_ERL_TIMEOUT * 1000). script_elements(FuncAndInput, Input) -> case input_type(FuncAndInput) of diff --git a/lib/inets/test/httpd_SUITE.erl b/lib/inets/test/httpd_SUITE.erl index 5b6740fba3..fcb9ad7905 100644 --- a/lib/inets/test/httpd_SUITE.erl +++ b/lib/inets/test/httpd_SUITE.erl @@ -78,7 +78,8 @@ all() -> {group, http_rel_path_script_alias}, {group, http_not_sup}, {group, https_not_sup}, - mime_types_format + mime_types_format, + erl_script_timeout_option ]. groups() -> @@ -1777,6 +1778,18 @@ mime_types_format(Config) when is_list(Config) -> {"hqx","application/mac-binhex40"}]} = httpd_conf:load_mime_types(MimeTypes). +erl_script_timeout_option(Config) when is_list(Config) -> + inets:start(), + {ok, Pid} = inets:start(httpd, [{erl_script_timeout, 215}, + {server_name, "test"}, + {port,0}, + {server_root, "."}, + {document_root, "."}]), + Info = httpd:info(Pid), + 215 = proplists:get_value(erl_script_timeout, Info), + inets:stop(). + + %%-------------------------------------------------------------------- %% Internal functions ----------------------------------- %%-------------------------------------------------------------------- diff --git a/lib/inets/vsn.mk b/lib/inets/vsn.mk index 921161dce1..b7ddf39ebd 100644 --- a/lib/inets/vsn.mk +++ b/lib/inets/vsn.mk @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = inets -INETS_VSN = 7.0.5 +INETS_VSN = 7.0.6 PRE_VSN = APP_VSN = "$(APPLICATION)-$(INETS_VSN)$(PRE_VSN)" diff --git a/lib/jinterface/doc/src/Makefile b/lib/jinterface/doc/src/Makefile index d80bb38ec1..f5cba9d074 100644 --- a/lib/jinterface/doc/src/Makefile +++ b/lib/jinterface/doc/src/Makefile @@ -137,6 +137,7 @@ clean clean_docs: jdoc:$(JAVA_SRC_FILES) (cd ../../java_src;$(JAVADOC) -sourcepath . -d $(JAVADOC_DEST) \ -windowtitle $(JAVADOC_TITLE) $(JAVADOC_PKGS)) + touch jdoc man: @@ -152,15 +153,8 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_docs_spec: docs $(INSTALL_DIR) "$(RELSYSDIR)/doc/pdf" $(INSTALL_DATA) $(TOP_PDF_FILE) "$(RELSYSDIR)/doc/pdf" - $(INSTALL_DIR) "$(RELSYSDIR)/doc/html" - $(INSTALL_DIR) "$(RELSYSDIR)/doc/html/java/$(JAVA_PKG_PATH)" $(INSTALL_DATA) $(INFO_FILE) "$(RELSYSDIR)" - ($(CP) -rf ../html "$(RELSYSDIR)/doc") - -# $(INSTALL_DATA) $(GIF_FILES) $(EXTRA_FILES) $(HTML_FILES) \ -# "$(RELSYSDIR)/doc/html" -# $(INSTALL_DATA) $(JAVA_EXTRA_FILES) "$(RELSYSDIR)/doc/html/java" -# $(INSTALL_DATA) $(TOP_HTML_FILES) "$(RELSYSDIR)/doc" + $(INSTALL_DIR_DATA) $(HTMLDIR) "$(RELSYSDIR)/doc/html" release_spec: diff --git a/lib/kernel/doc/src/application.xml b/lib/kernel/doc/src/application.xml index 4e32c1a3a5..83a83ebad2 100644 --- a/lib/kernel/doc/src/application.xml +++ b/lib/kernel/doc/src/application.xml @@ -238,6 +238,41 @@ Nodes = [cp1@cave, {cp2@cave, cp3@cave}]</code> </desc> </func> <func> + <name name="set_env" arity="1" since="OTP 21.3"/> + <name name="set_env" arity="2" since="OTP 21.3"/> + <fsummary>Sets the configuration parameters of multiple applications.</fsummary> + <desc> + <p>Sets the configuration <c><anno>Config</anno></c> for multiple + applications. It is equivalent to calling <c>set_env/4</c> on + each application individially, except it is more efficient. + The given <c><anno>Config</anno></c> is validated before the + configuration is set.</p> + <p><c>set_env/2</c> uses the standard <c>gen_server</c> time-out + value (5000 ms). Option <c>timeout</c> can be specified + if another time-out value is useful, for example, in situations + where the application controller is heavily loaded.</p> + <p>Option <c>persistent</c> can be set to <c>true</c> + to guarantee that parameters set with <c>set_env/2</c> + are not overridden by those defined in the application resource + file on load. This means that persistent values will stick after the application + is loaded and also on application reload.</p> + <p>If an application is given more than once or if an application + has the same key given more than once, the behaviour is undefined + and a warning message will be logged. In future releases, an error + will be raised.</p> + <p><c>set_env/1</c> is equivalent to <c>set_env(Config, [])</c>.</p> + <warning> + <p>Use this function only if you know what you are doing, + that is, on your own applications. It is very + application-dependent and + configuration parameter-dependent when and how often + the value is read by the application. Careless use + of this function can put the application in a + weird, inconsistent, and malfunctioning state.</p> + </warning> + </desc> + </func> + <func> <name name="permit" arity="2" since=""/> <fsummary>Change the permission for an application to run at a node.</fsummary> <desc> diff --git a/lib/kernel/doc/src/kernel_app.xml b/lib/kernel/doc/src/kernel_app.xml index 15dbdb47dc..7f9609d5c1 100644 --- a/lib/kernel/doc/src/kernel_app.xml +++ b/lib/kernel/doc/src/kernel_app.xml @@ -369,6 +369,14 @@ MaxT = TickTime + TickTime / 4</code> performed. This option ensures that <c>global</c> is synchronized.</p> </item> + <tag><c>start_distribution = true | false</c></tag> + <item> + <p>Starts all distribution services, such as <c>rpc</c>, + <c>global</c>, and <c>net_kernel</c> if the parameter is + <c>true</c>. This parameter is to be set to <c>false</c> + for systems who want to disable all distribution functionality.</p> + <p>Defaults to <c>true</c>.</p> + </item> <tag><c>start_dist_ac = true | false</c></tag> <item> <p>Starts the <c>dist_ac</c> server if the parameter is @@ -510,11 +518,13 @@ MaxT = TickTime + TickTime / 4</code> parameters for Logger are not set.</p> <taglist> <tag><c>error_logger</c></tag> - <item>Replaced by setting the type of the default - <seealso marker="logger_std_h#type"><c>logger_std_h</c></seealso> - to the same value. Example: + <item>Replaced by setting the <seealso + marker="logger_std_h#type"><c>type</c></seealso>, and possibly + <seealso marker="logger_std_h#file"><c>file</c></seealso> and + <seealso marker="logger_std_h#modes"><c>modes</c></seealso> + parameters of the default <c>logger_std_h</c> handler. Example: <code type="none"> -erl -kernel logger '[{handler,default,logger_std_h,#{config=>#{type=>{file,"/tmp/erlang.log"}}}}]' +erl -kernel logger '[{handler,default,logger_std_h,#{config=>#{file=>"/tmp/erlang.log"}}}]' </code> </item> <tag><c>error_logger_format_depth</c></tag> diff --git a/lib/kernel/doc/src/logger.xml b/lib/kernel/doc/src/logger.xml index df2d081d76..2a060ce1f9 100644 --- a/lib/kernel/doc/src/logger.xml +++ b/lib/kernel/doc/src/logger.xml @@ -66,7 +66,7 @@ logger:error("error happened because: ~p", [Reason]). % Without macro [{kernel, [{logger, [{handler, default, logger_std_h, - #{config => #{type => {file, "path/to/file.log"}}}}]}]}]. + #{config => #{file => "path/to/file.log"}}}]}]}]. </code> <p> For more information about: @@ -190,7 +190,7 @@ logger:error("error happened because: ~p", [Reason]). % Without macro <list> <item><c>pid => self()</c></item> <item><c>gl => group_leader()</c></item> - <item><c>time => erlang:system_time(microsecond)</c></item> + <item><c>time => logger:timestamp()</c></item> </list> <p>When a log macro is used, Logger also inserts location information:</p> @@ -288,8 +288,8 @@ logger:error("error happened because: ~p", [Reason]). % Without macro <name name="timestamp"/> <desc> <p>A timestamp produced - with <seealso marker="erts:erlang#system_time-1"> - <c>erlang:system_time(microsecond)</c></seealso>.</p> + with <seealso marker="#timestamp-0"> + <c>logger:timestamp()</c></seealso>.</p> </desc> </datatype> </datatypes> @@ -1117,6 +1117,24 @@ logger:set_proxy_config(maps:merge(Old, Config)). a key-value list before formatting as such.</p> </desc> </func> + + <func> + <name name="timestamp" arity="0" since="OTP 21.3"/> + <fsummary>Return a timestamp to insert in meta data for a log + event.</fsummary> + <desc> + <p>Return a timestamp that can be inserted as the <c>time</c> + field in the meta data for a log event. It is produced with + <seealso marker="kernel:os#system_time-1"> + <c>os:system_time(microsecond)</c></seealso>.</p> + <p>Notice that Logger automatically inserts a timestamp in the + meta data unless it already exists. This function is + exported for the rare case when the timestamp must be taken + at a different point in time than when the log event is + issued.</p> + </desc> + </func> + </funcs> <section> @@ -1333,5 +1351,3 @@ logger:set_proxy_config(maps:merge(Old, Config)). </p> </section> </erlref> - - diff --git a/lib/kernel/doc/src/logger_chapter.xml b/lib/kernel/doc/src/logger_chapter.xml index 03b9edcf8f..8458ffa042 100644 --- a/lib/kernel/doc/src/logger_chapter.xml +++ b/lib/kernel/doc/src/logger_chapter.xml @@ -801,7 +801,7 @@ logger:debug(#{got => connection_request, id => Id, state => State}, [{kernel, [{logger, [{handler, default, logger_std_h, % {handler, HandlerId, Module, - #{config => #{type => {file,"log/erlang.log"}}}} % Config} + #{config => #{file => "log/erlang.log"}}} % Config} ]}]}]. </code> <p>Modify the default handler to print each log event as a @@ -831,10 +831,10 @@ logger:debug(#{got => connection_request, id => Id, state => State}, [{logger, [{handler, default, logger_std_h, #{level => error, - config => #{type => {file, "log/erlang.log"}}}}, + config => #{file => "log/erlang.log"}}}, {handler, info, logger_std_h, #{level => debug, - config => #{type => {file, "log/debug.log"}}}} + config => #{file => "log/debug.log"}}} ]}]}]. </code> </section> @@ -1004,10 +1004,10 @@ ok</pre> <p>Then, add a new handler which prints to file. You can use the handler module <seealso marker="logger_std_h"><c>logger_std_h</c></seealso>, - and specify type <c>{file,File}</c>.:</p> + and configure it to log to file:</p> <pre> -4> <input>Config = #{config => #{type => {file,"./info.log"}}, level => info}.</input> -#{config => #{type => {file,"./info.log"}},level => info} +4> <input>Config = #{config => #{file => "./info.log"}, level => info}.</input> +#{config => #{file => "./info.log"},level => info} 5> <input>logger:add_handler(myhandler, logger_std_h, Config).</input> ok</pre> <p>Since <c>filter_default</c> defaults to <c>log</c>, this @@ -1246,7 +1246,7 @@ do_log(Fd, LogEvent, #{formatter := {FModule, FConfig}}) -> <p>A configuration example:</p> <code type="none"> logger:add_handler(my_standard_h, logger_std_h, - #{config => #{type => {file,"./system_info.log"}, + #{config => #{file => "./system_info.log", sync_mode_qlen => 100, drop_mode_qlen => 1000, flush_qlen => 2000}}). diff --git a/lib/kernel/doc/src/logger_std_h.xml b/lib/kernel/doc/src/logger_std_h.xml index fcd180abd6..5ed1a2f210 100644 --- a/lib/kernel/doc/src/logger_std_h.xml +++ b/lib/kernel/doc/src/logger_std_h.xml @@ -55,30 +55,105 @@ is stored in a sub map with the key <c>config</c>, and can contain the following parameters:</p> <taglist> - <tag><marker id="type"/><c>type</c></tag> + <tag><marker id="type"/><c>type = standard_io | standard_error | file</c></tag> <item> - <p>This has the value <c>standard_io</c>, <c>standard_error</c>, - <c>{file,LogFileName}</c>, or <c>{file,LogFileName,LogFileOpts}</c>.</p> - <p>If <c>LogFileOpts</c> is specified, it replaces the default - list of options used when opening the log file. The default - list is <c>[raw,append,delayed_write]</c>. One reason to do - so can be to change <c>append</c> to, for - example, <c>write</c>, ensuring that the old log is - truncated when a node is restarted. See the reference manual - for <seealso marker="file#open-2"><c>file:open/2</c></seealso> - for more information about file options.</p> + <p>Specifies the log destination.</p> + <p>The value is set when the handler is added, and it can not + be changed in runtime.</p> + <p>Defaults to <c>standard_io</c>, unless + parameter <seealso marker="#file"><c>file</c></seealso> is + given, in which case it defaults to <c>file</c>.</p> + </item> + <tag><marker id="file"/><c>file = </c><seealso marker="file#type-filename"><c>file:filename()</c></seealso></tag> + <item> + <p>This specifies the name of the log file when the handler is + of type <c>file</c>.</p> + <p>The value is set when the handler is added, and it can not + be changed in runtime.</p> + <p>Defaults to the same name as the handler identity, in the + current directory.</p> + </item> + <tag><marker id="modes"/><c>modes = [</c><seealso marker="file#type-mode"><c>file:mode()</c></seealso><c>]</c></tag> + <item> + <p>This specifies the file modes to use when opening the log + file, + see <seealso marker="file#open-2"><c>file:open/2</c></seealso>. + If <c>modes</c> are not specified, the default list used + is <c>[raw,append,delayed_write]</c>. If <c>modes</c> are + specified, the list replaces the default modes list with the + following adjustments:</p> + <list> + <item> + If <c>raw</c> is not found in the list, it is added. + </item> + <item> + If none of <c>write</c>, <c>append</c> or <c>exclusive</c> is + found in the list, <c>append</c> is added.</item> + <item>If none of <c>delayed_write</c> + or <c>{delayed_write,Size,Delay}</c> is found in the + list, <c>delayed_write</c> is added.</item> + </list> <p>Log files are always UTF-8 encoded. The encoding can not be - changed by setting the option <c>{encoding,Encoding}</c> - in <c>LogFileOpts</c>.</p> - <p>Notice that the standard handler does not have support for - circular logging. Use the disk_log handler, - <seealso marker="logger_disk_log_h"><c>logger_disk_log_h</c></seealso>, - for this.</p> + changed by setting the mode <c>{encoding,Encoding}</c>.</p> <p>The value is set when the handler is added, and it can not be changed in runtime.</p> - <p>Defaults to <c>standard_io</c>.</p> + <p>Defaults to <c>[raw,append,delayed_write]</c>.</p> + </item> + <tag><marker id="max_no_bytes"/><c>max_no_bytes = pos_integer() | infinity</c></tag> + <item> + <p>This parameter specifies if the log file should be rotated + or not. The value <c>infinity</c> means the log file will + grow indefinitely, while an integer value specifies at which + file size (bytes) the file is rotated.</p> + <p>Defaults to <c>infinity</c>.</p> + </item> + <tag><marker id="max_no_files"/><c>max_no_files = non_neg_integer()</c></tag> + <item> + <p>This parameter specifies the number of rotated log file + archives to keep. This has meaning only + if <seealso marker="#max_no_bytes"><c>max_no_bytes</c></seealso> + is set to an integer value.</p> + <p>The log archives are + named <c>FileName.0</c>, <c>FileName.1</c>, + ... <c>FileName.N</c>, where <c>FileName</c> is the name of + the current log file. <c>FileName.0</c> is the newest of the + archives. The maximum value for <c>N</c> is the value + of <c>max_no_files</c> minus 1.</p> + <p>Notice that setting this value to <c>0</c> does not turn of + rotation. It only specifies that no archives are kept.</p> + <p>Defaults to <c>0</c>.</p> + </item> + <tag><marker id="compress_on_rotate"/><c>compress_on_rotate = boolean()</c></tag> + <item> + <p>This parameter specifies if the rotated log file archives + shall be compressed or not. If set to <c>true</c>, all + archives are compressed with <c>gzip</c>, and renamed + to <c>FileName.N.gz</c></p> + <p><c>compress_on_rotate</c> has no meaning if <seealso + marker="#max_no_bytes"><c>max_no_bytes</c></seealso> has the + value <c>infinity</c>.</p> + <p>Defaults to <c>false</c>.</p> + </item> + <tag><marker id="file_check"/><c>file_check = non_neg_integer()</c></tag> + <item> + <p>When <c>logger_std_h</c> logs to a file, it reads the file + information of the log file prior to each write + operation. This is to make sure the file still exists and + has the same inode as when it was opened. This implies some + performance loss, but ensures that no log events are lost in + the case when the file has been removed or renamed by an + external actor.</p> + <p>In order to allow minimizing the performance loss, the + <c>file_check</c> parameter can be set to a positive integer + value, <c>N</c>. The handler will then skip reading the file + information prior to writing, as long as no more + than <c>N</c> milliseconds have passed since it was last + read.</p> + <p>Notice that the risk of loosing log events grows when + the <c>file_check</c> value grows.</p> + <p>Defaults to 0.</p> </item> - <tag><c>filesync_repeat_interval</c></tag> + <tag><c>filesync_repeat_interval = pos_integer() | no_repeat</c></tag> <item> <p>This value, in milliseconds, specifies how often the handler does a file sync operation to write buffered data to disk. The handler attempts @@ -97,12 +172,13 @@ standard handler and the disk_log handler, and are documented in the <seealso marker="logger_chapter#overload_protection"><c>User's Guide</c> </seealso>.</p> - <p>Notice that if changing the configuration of the handler in runtime, - the <c>type</c> parameter must not be modified.</p> + <p>Notice that if changing the configuration of the handler in + runtime, the <c>type</c>, <c>file</c>, or <c>modes</c> parameters + must not be modified.</p> <p>Example of adding a standard handler:</p> <code type="none"> logger:add_handler(my_standard_h, logger_std_h, - #{config => #{type => {file,"./system_info.log"}, + #{config => #{file => "./system_info.log", filesync_repeat_interval => 1000}}). </code> <p>To set the default handler, that starts initially with @@ -110,7 +186,7 @@ logger:add_handler(my_standard_h, logger_std_h, change the Kernel default logger configuration. Example:</p> <code type="none"> erl -kernel logger '[{handler,default,logger_std_h, - #{config => #{type => {file,"./log.log"}}}}]' + #{config => #{file => "./log.log"}}}]' </code> <p>An example of how to replace the standard handler with a disk_log handler at startup is found in the diff --git a/lib/kernel/doc/src/notes.xml b/lib/kernel/doc/src/notes.xml index 021ecfa40d..0c187eb19f 100644 --- a/lib/kernel/doc/src/notes.xml +++ b/lib/kernel/doc/src/notes.xml @@ -31,6 +31,204 @@ </header> <p>This document describes the changes made to the Kernel application.</p> +<section><title>Kernel 6.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + If for example the <c>/etc/hosts</c> did not come into + existence until after the kernel application had started, + its content was never read. This bug has now been + corrected.</p> + <p> + Own Id: OTP-14702 Aux Id: PR-2066 </p> + </item> + <item> + <p> + Fix bug where doing <c>seq_trace:reset_trace()</c> while + another process was doing a garbage collection could + cause the run-time system to segfault.</p> + <p> + Own Id: OTP-15490</p> + </item> + <item> + <p> + Fix <c>erl_epmd:port_please</c> spec to include + <c>atom()</c> and <c>string()</c>.</p> + <p> + Own Id: OTP-15557 Aux Id: PR-2117 </p> + </item> + <item> + <p> + The Logger handler logger_std_h now keeps track of the + inode of its log file in order to re-open the file if the + inode changes. This may happen, for instance, if the log + file is opened and saved by an editor.</p> + <p> + Own Id: OTP-15578 Aux Id: ERL-850 </p> + </item> + <item> + <p> + When user specific file modes are given to the logger + handler <c>logger_std_h</c>, they were earlier accepted + without any control. This is now changes, so Logger will + adjust the file modes as follows:</p> + <p> + - If <c>raw</c> is not found in the list, it is + added.<br/> - If none of <c>write</c>, <c>append</c> or + <c>exclusive</c> are found in the list, <c>append</c> is + added.<br/> - If none of <c>delayed_write</c> or + <c>{delayed_write,Size,Delay}</c> are found in the list, + <c>delayed_write</c> is added.</p> + <p> + Own Id: OTP-15602</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + The standard logger handler, <c>logger_std_h</c>, now has + a new internal feature for log rotation. The rotation + scheme is as follows:</p> + <p> + The log file to which the handler currently writes always + has the same name, i.e. the name which is configured for + the handler. The archived files have the same name, but + with extension ".N", where N is an integer. The newest + archive has extension ".0", and the oldest archive has + the highest number. </p> + <p> + The size at which the log file is rotated, and the number + of archive files that are kept, is specified with the + handler specific configuration parameters + <c>max_no_bytes</c> and <c>max_no_files</c> respectively. </p> + <p> + Archives can be compressed, in which case they get a + ".gz" file extension after the integer. Compression is + specified with the handler specific configuration + parameter <c>compress_on_rotate</c>.</p> + <p> + Own Id: OTP-15479</p> + </item> + <item> + <p> + The new functions <c>logger:i/0</c> and <c>logger:i/1</c> + are added. These provide the same information as + <c>logger:get_config/0</c> and other + <c>logger:get_*_config</c> functions, but the information + is more logically sorted and more readable.</p> + <p> + Own Id: OTP-15600</p> + </item> + <item> + <p> + Logger is now protected against overload due to massive + amounts of log events from the emulator or from remote + nodes.</p> + <p> + Own Id: OTP-15601</p> + </item> + <item> + <p> + Logger now uses os:system_time/1 instead of + erlang:system_time/1 to generate log event timestamps.</p> + <p> + Own Id: OTP-15625</p> + </item> + <item> + <p> + Add functions <c>application:set_env/1,2</c> and + <c>application:set_env/2</c>. These take a list of + application configuration parameters, and the behaviour + is equivalent to calling <c>application:set_env/4</c> + individually for each application/key combination, except + it is more efficient.</p> + <p> + <c>set_env/1,2</c> warns about duplicated applications or + keys. The warning is also emitted during boot, if + applications or keys are duplicated within one + configuration file, e.g. sys.config.</p> + <p> + Own Id: OTP-15642 Aux Id: PR-2164 </p> + </item> + <item> + <p> + Handler specific configuration parameters for the + standard handler <c>logger_std_h</c> are changed to be + more intuitive and more similar to the disk_log handler.</p> + <p> + Earlier there was only one parameter, <c>type</c>, which + could have the values <c>standard_io</c>, + <c>standard_error</c>, <c>{file,FileName}</c> or + <c>{file,FileName,Modes}</c>.</p> + <p> + This is now changed, so the following parameters are + allowed:</p> + <p> + <c>type = standard_io | standard_error | file</c><br/> + <c>file = file:filename()</c><br/> <c>modes = + [file:mode()]</c></p> + <p> + All parameters are optional. <c>type</c> defaults to + <c>standard_io</c>, unless a file name is given, in which + case it defaults to <c>file</c>. If <c>type</c> is set to + <c>file</c>, the file name defaults to the same as the + handler id.</p> + <p> + The potential incompatibility is that + <c>logger:get_config/0</c> and + <c>logger:get_handler_config/1</c> now returns the new + parameters, even if the configuration was set with the + old variant, e.g. <c>#{type=>{file,FileName}}</c>.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-15662</p> + </item> + <item> + <p> + The new configuration parameter <c>file_check</c> is + added to the Logger handler <c>logger_std_h</c>. This + parameter specifies how long (in milliseconds) the + handler may wait before checking if the log file still + exists and the inode is the same as when it was opened.</p> + <p> + The default value is 0, which means that this check is + done prior to each write operation. Setting a higher + number may improve performance, but adds the risk of + loosing log events.</p> + <p> + Own Id: OTP-15663</p> + </item> + </list> + </section> + +</section> + +<section><title>Kernel 6.2.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Setting the <c>recbuf</c> size of an inet socket the + <c>buffer</c> is also automatically increased. Fix a bug + where the auto adjustment of inet buffer size would be + triggered even if an explicit inet buffer size had + already been set.</p> + <p> + Own Id: OTP-15651 Aux Id: ERIERL-304 </p> + </item> + </list> + </section> + +</section> + <section><title>Kernel 6.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/kernel/src/application.erl b/lib/kernel/src/application.erl index bc6be2f8f5..5c2e981e4b 100644 --- a/lib/kernel/src/application.erl +++ b/lib/kernel/src/application.erl @@ -25,7 +25,7 @@ which_applications/0, which_applications/1, loaded_applications/0, permit/2]). -export([ensure_started/1, ensure_started/2]). --export([set_env/3, set_env/4, unset_env/2, unset_env/3]). +-export([set_env/1, set_env/2, set_env/3, set_env/4, unset_env/2, unset_env/3]). -export([get_env/1, get_env/2, get_env/3, get_all_env/0, get_all_env/1]). -export([get_key/1, get_key/2, get_all_key/0, get_all_key/1]). -export([get_application/0, get_application/1, info/0]). @@ -279,6 +279,26 @@ loaded_applications() -> info() -> application_controller:info(). +-spec set_env(Config) -> 'ok' when + Config :: [{Application, Env}], + Application :: atom(), + Env :: [{Par :: atom(), Val :: term()}]. + +set_env(Config) when is_list(Config) -> + set_env(Config, []). + +-spec set_env(Config, Opts) -> 'ok' when + Config :: [{Application, Env}], + Application :: atom(), + Env :: [{Par :: atom(), Val :: term()}], + Opts :: [{timeout, timeout()} | {persistent, boolean()}]. + +set_env(Config, Opts) when is_list(Config), is_list(Opts) -> + case application_controller:set_env(Config, Opts) of + ok -> ok; + {error, Msg} -> erlang:error({badarg, Msg}, [Config, Opts]) + end. + -spec set_env(Application, Par, Val) -> 'ok' when Application :: atom(), Par :: atom(), diff --git a/lib/kernel/src/application_controller.erl b/lib/kernel/src/application_controller.erl index a074d2e74b..7715dca7c6 100644 --- a/lib/kernel/src/application_controller.erl +++ b/lib/kernel/src/application_controller.erl @@ -26,7 +26,7 @@ control_application/1, change_application_data/2, prep_config_change/0, config_change/1, which_applications/0, which_applications/1, - loaded_applications/0, info/0, + loaded_applications/0, info/0, set_env/2, get_pid_env/2, get_env/2, get_pid_all_env/1, get_all_env/1, get_pid_key/2, get_key/2, get_pid_all_key/1, get_all_key/1, get_master/1, get_application/1, get_application_module/1, @@ -345,9 +345,6 @@ get_all_env(AppName) -> map(fun([Key, Val]) -> {Key, Val} end, ets:match(ac_tab, {{env, AppName, '$1'}, '$2'})). - - - get_pid_key(Master, Key) -> case ets:match(ac_tab, {{application_master, '$1'}, Master}) of [[AppName]] -> get_key(AppName, Key); @@ -461,6 +458,15 @@ permit_application(ApplName, Flag) -> {permit_application, ApplName, Flag}, infinity). +set_env(Config, Opts) -> + case check_conf_data(Config) of + ok -> + Timeout = proplists:get_value(timeout, Opts, 5000), + gen_server:call(?AC, {set_env, Config, Opts}, Timeout); + + {error, _} = Error -> + Error + end. set_env(AppName, Key, Val) -> gen_server:call(?AC, {set_env, AppName, Key, Val, []}). @@ -528,19 +534,15 @@ check_conf_data([]) -> check_conf_data(ConfData) when is_list(ConfData) -> [Application | ConfDataRem] = ConfData, case Application of - {kernel, List} when is_list(List) -> - case check_para_kernel(List) of - ok -> - check_conf_data(ConfDataRem); - Error1 -> - Error1 - end; {AppName, List} when is_atom(AppName), is_list(List) -> - case check_para(List, atom_to_list(AppName)) of - ok -> - check_conf_data(ConfDataRem); - Error2 -> - Error2 + case lists:keymember(AppName, 1, ConfDataRem) of + true -> + {error, "duplicate application config: " ++ atom_to_list(AppName)}; + false -> + case check_para(List, AppName) of + ok -> check_conf_data(ConfDataRem); + Error -> Error + end end; {AppName, List} when is_list(List) -> ErrMsg = "application: " @@ -553,36 +555,39 @@ check_conf_data(ConfData) when is_list(ConfData) -> ++ "; parameters must be a list", {error, ErrMsg}; Else -> - ErrMsg = "invalid application name: " ++ - lists:flatten(io_lib:format(" ~tp",[Else])), + ErrMsg = "invalid application config: " + ++ lists:flatten(io_lib:format("~tp",[Else])), {error, ErrMsg} end; check_conf_data(_ConfData) -> - {error, 'configuration must be a list ended by <dot><whitespace>'}. - + {error, "configuration must be a list ended by <dot><whitespace>"}. -%% Special check of distributed parameter for kernel -check_para_kernel([]) -> + +check_para([], _AppName) -> ok; -check_para_kernel([{distributed, Apps} | ParaList]) when is_list(Apps) -> - case check_distributed(Apps) of - {error, _ErrorMsg} = Error -> - Error; - _ -> - check_para_kernel(ParaList) +check_para([{Para, Val} | ParaList], AppName) when is_atom(Para) -> + case lists:keymember(Para, 1, ParaList) of + true -> + ErrMsg = "application: " ++ atom_to_list(AppName) + ++ "; duplicate parameter: " ++ atom_to_list(Para), + {error, ErrMsg}; + false -> + case check_para_value(Para, Val, AppName) of + ok -> check_para(ParaList, AppName); + {error, _} = Error -> Error + end end; -check_para_kernel([{distributed, _Apps} | _ParaList]) -> - {error, "application: kernel; erroneous parameter: distributed"}; -check_para_kernel([{Para, _Val} | ParaList]) when is_atom(Para) -> - check_para_kernel(ParaList); -check_para_kernel([{Para, _Val} | _ParaList]) -> - {error, "application: kernel; invalid parameter: " ++ +check_para([{Para, _Val} | _ParaList], AppName) -> + {error, "application: " ++ atom_to_list(AppName) ++ "; invalid parameter name: " ++ lists:flatten(io_lib:format("~tp",[Para]))}; -check_para_kernel(Else) -> - {error, "application: kernel; invalid parameter list: " ++ +check_para([Else | _ParaList], AppName) -> + {error, "application: " ++ atom_to_list(AppName) ++ "; invalid parameter: " ++ lists:flatten(io_lib:format("~tp",[Else]))}. - +check_para_value(distributed, Apps, kernel) -> check_distributed(Apps); +check_para_value(_Para, _Val, _AppName) -> ok. + +%% Special check of distributed parameter for kernel check_distributed([]) -> ok; check_distributed([{App, List} | Apps]) when is_atom(App), is_list(List) -> @@ -595,18 +600,6 @@ check_distributed(_Else) -> {error, "application: kernel; erroneous parameter: distributed"}. -check_para([], _AppName) -> - ok; -check_para([{Para, _Val} | ParaList], AppName) when is_atom(Para) -> - check_para(ParaList, AppName); -check_para([{Para, _Val} | _ParaList], AppName) -> - {error, "application: " ++ AppName ++ "; invalid parameter: " ++ - lists:flatten(io_lib:format("~tp",[Para]))}; -check_para([Else | _ParaList], AppName) -> - {error, "application: " ++ AppName ++ "; invalid parameter: " ++ - lists:flatten(io_lib:format("~tp",[Else]))}. - - -type calls() :: 'info' | 'prep_config_change' | 'which_applications' | {'config_change' | 'control_application' | 'load_application' | 'start_type' | 'stop_application' | @@ -863,6 +856,16 @@ handle_call(which_applications, _From, S) -> end, S#state.running), {reply, Reply, S}; +handle_call({set_env, Config, Opts}, _From, S) -> + _ = [add_env(AppName, Env) || {AppName, Env} <- Config], + + case proplists:get_value(persistent, Opts, false) of + true -> + {reply, ok, S#state{conf_data = merge_env(S#state.conf_data, Config)}}; + false -> + {reply, ok, S} + end; + handle_call({set_env, AppName, Key, Val, Opts}, _From, S) -> ets:insert(ac_tab, {{env, AppName, Key}, Val}), case proplists:get_value(persistent, Opts, false) of diff --git a/lib/kernel/src/code_server.erl b/lib/kernel/src/code_server.erl index 1b4a67ecb7..68e1205301 100644 --- a/lib/kernel/src/code_server.erl +++ b/lib/kernel/src/code_server.erl @@ -1434,19 +1434,25 @@ all_loaded(Db) -> -spec error_msg(io:format(), [term()]) -> 'ok'. error_msg(Format, Args) -> + %% This is equal to calling logger:error/3 which we don't want to + %% do from code_server. We don't want to call logger:timestamp() + %% either. logger ! {log,error,Format,Args, #{pid=>self(), gl=>group_leader(), - time=>erlang:system_time(microsecond), + time=>os:system_time(microsecond), error_logger=>#{tag=>error}}}, ok. -spec info_msg(io:format(), [term()]) -> 'ok'. info_msg(Format, Args) -> + %% This is equal to calling logger:info/3 which we don't want to + %% do from code_server. We don't want to call logger:timestamp() + %% either. logger ! {log,info,Format,Args, #{pid=>self(), gl=>group_leader(), - time=>erlang:system_time(microsecond), + time=>os:system_time(microsecond), error_logger=>#{tag=>info_msg}}}, ok. diff --git a/lib/kernel/src/global.erl b/lib/kernel/src/global.erl index 5bb56f2de6..3875074d74 100644 --- a/lib/kernel/src/global.erl +++ b/lib/kernel/src/global.erl @@ -50,10 +50,9 @@ %% This is for backward compatibility only; the functionality is broken. -define(WARN_DUPLICATED_NAME, global_multi_name_action). -%% Undocumented Kernel variable. Set this to 0 (zero) to get the old -%% behaviour. +%% Undocumented Kernel variable. -define(N_CONNECT_RETRIES, global_connect_retries). --define(DEFAULT_N_CONNECT_RETRIES, 5). +-define(DEFAULT_N_CONNECT_RETRIES, 0). %%% In certain places in the server, calling io:format hangs everything, %%% so we'd better use erlang:display/1. diff --git a/lib/kernel/src/inet_db.erl b/lib/kernel/src/inet_db.erl index 6cbb6ac2da..3f5a2ea5ee 100644 --- a/lib/kernel/src/inet_db.erl +++ b/lib/kernel/src/inet_db.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2016. All Rights Reserved. +%% Copyright Ericsson AB 1997-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1223,7 +1223,10 @@ handle_set_file(Option, Fname, TagTm, TagInfo, ParseFun, From, {ok, B, _} -> B; _ -> <<>> end; - _ -> <<>> + _ -> + ets:insert(Db, {TagInfo, undefined}), + TimeZero = - (?RES_FILE_UPDATE_TM + 1), % Early enough + ets:insert(Db, {TagTm, TimeZero}) end, handle_set_file(ParseFun, Bin, From, State); false -> {reply,error,State} diff --git a/lib/kernel/src/kernel.app.src b/lib/kernel/src/kernel.app.src index 4b48f6cd1d..8fe6bdd1ca 100644 --- a/lib/kernel/src/kernel.app.src +++ b/lib/kernel/src/kernel.app.src @@ -147,6 +147,6 @@ {logger_sasl_compatible, false} ]}, {mod, {kernel, []}}, - {runtime_dependencies, ["erts-10.1", "stdlib-3.5", "sasl-3.0"]} + {runtime_dependencies, ["erts-10.2.5", "stdlib-3.5", "sasl-3.0"]} ] }. diff --git a/lib/kernel/src/kernel.appup.src b/lib/kernel/src/kernel.appup.src index ccf0a82ced..8fa3f5c588 100644 --- a/lib/kernel/src/kernel.appup.src +++ b/lib/kernel/src/kernel.appup.src @@ -40,7 +40,10 @@ {<<"^6\\.0\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^6\\.1$">>,[restart_new_emulator]}, {<<"^6\\.1\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], + {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^6\\.2$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], [{<<"^5\\.3$">>,[restart_new_emulator]}, {<<"^5\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^5\\.3\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -54,4 +57,7 @@ {<<"^6\\.0\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^6\\.1$">>,[restart_new_emulator]}, {<<"^6\\.1\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. + {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^6\\.2$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. diff --git a/lib/kernel/src/kernel.erl b/lib/kernel/src/kernel.erl index c68d04e279..111d103df2 100644 --- a/lib/kernel/src/kernel.erl +++ b/lib/kernel/src/kernel.erl @@ -145,26 +145,11 @@ init([]) -> {ok, {SupFlags, [Code, File, StdError, User, LoggerSup, Config, RefC, SafeSup]}}; _ -> - Rpc = #{id => rex, - start => {rpc, start_link, []}, - restart => permanent, - shutdown => 2000, - type => worker, - modules => [rpc]}, - - Global = #{id => global_name_server, - start => {global, start_link, []}, - restart => permanent, - shutdown => 2000, - type => worker, - modules => [global]}, - - GlGroup = #{id => global_group, - start => {global_group,start_link,[]}, - restart => permanent, - shutdown => 2000, - type => worker, - modules => [global_group]}, + DistChildren = + case application:get_env(kernel, start_distribution) of + {ok, false} -> []; + _ -> start_distribution() + end, InetDb = #{id => inet_db, start => {inet_db, start_link, []}, @@ -173,13 +158,6 @@ init([]) -> type => worker, modules => [inet_db]}, - NetSup = #{id => net_sup, - start => {erl_distribution, start_link, []}, - restart => permanent, - shutdown => infinity, - type => supervisor, - modules => [erl_distribution]}, - SigSrv = #{id => erl_signal_server, start => {gen_event, start_link, [{local, erl_signal_server}]}, restart => permanent, @@ -187,14 +165,11 @@ init([]) -> type => worker, modules => dynamic}, - DistAC = start_dist_ac(), - Timer = start_timer(), {ok, {SupFlags, - [Code, Rpc, Global, InetDb | DistAC] ++ - [NetSup, GlGroup, File, SigSrv, - StdError, User, Config, RefC, SafeSup, LoggerSup] ++ Timer}} + [Code, InetDb | DistChildren] ++ + [File, SigSrv, StdError, User, Config, RefC, SafeSup, LoggerSup] ++ Timer}} end; init(safe) -> SupFlags = #{strategy => one_for_one, @@ -213,6 +188,39 @@ init(safe) -> {ok, {SupFlags, Boot ++ DiskLog ++ Pg2}}. +start_distribution() -> + Rpc = #{id => rex, + start => {rpc, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [rpc]}, + + Global = #{id => global_name_server, + start => {global, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [global]}, + + DistAC = start_dist_ac(), + + NetSup = #{id => net_sup, + start => {erl_distribution, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [erl_distribution]}, + + GlGroup = #{id => global_group, + start => {global_group,start_link,[]}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [global_group]}, + + [Rpc, Global | DistAC] ++ [NetSup, GlGroup]. + start_dist_ac() -> Spec = [#{id => dist_ac, start => {dist_ac,start_link,[]}, diff --git a/lib/kernel/src/logger.erl b/lib/kernel/src/logger.erl index 7d36640f52..38bd2f481c 100644 --- a/lib/kernel/src/logger.erl +++ b/lib/kernel/src/logger.erl @@ -61,6 +61,7 @@ -export([set_process_metadata/1, update_process_metadata/1, unset_process_metadata/0, get_process_metadata/0]). -export([i/0, i/1]). +-export([timestamp/0]). %% Basic report formatting -export([format_report/1, format_otp_report/1]). @@ -154,7 +155,8 @@ filter_return/0, config_handler/0, formatter_config/0, - olp_config/0]). + olp_config/0, + timestamp/0]). %%%----------------------------------------------------------------- %%% API @@ -354,6 +356,10 @@ internal_log(Level,Term) when is_atom(Level) -> erlang:display_string("Logger - "++ atom_to_list(Level) ++ ": "), erlang:display(Term). +-spec timestamp() -> timestamp(). +timestamp() -> + os:system_time(microsecond). + %%%----------------------------------------------------------------- %%% Configuration -spec add_primary_filter(FilterId,Filter) -> ok | {error,term()} when @@ -1129,7 +1135,7 @@ proc_meta() -> default(pid) -> self(); default(gl) -> group_leader(); -default(time) -> erlang:system_time(microsecond). +default(time) -> timestamp(). %% Remove everything upto and including this module from the stacktrace filter_stacktrace(Module,[{Module,_,_,_}|_]) -> diff --git a/lib/kernel/src/logger_formatter.erl b/lib/kernel/src/logger_formatter.erl index ded89bac9f..8696adbd72 100644 --- a/lib/kernel/src/logger_formatter.erl +++ b/lib/kernel/src/logger_formatter.erl @@ -64,7 +64,7 @@ format(#{level:=Level,msg:=Msg0,meta:=Meta},Config0) Config; Size0 -> Size = - case Size0 - string:length([B,A]) of + case Size0 - io_lib:chars_length([B,A]) of S when S>=0 -> S; _ -> 0 end, @@ -75,7 +75,11 @@ format(#{level:=Level,msg:=Msg0,meta:=Meta},Config0) true -> %% Trim leading and trailing whitespaces, and replace %% newlines with ", " - re:replace(string:trim(MsgStr0),",?\r?\n\s*",", ", + T = lists:reverse( + trim( + lists:reverse( + trim(MsgStr0,false)),true)), + re:replace(T,",?\r?\n\s*",", ", [{return,list},global,unicode]); _false -> MsgStr0 @@ -83,7 +87,26 @@ format(#{level:=Level,msg:=Msg0,meta:=Meta},Config0) true -> "" end, - truncate([B,MsgStr,A],maps:get(max_size,Config)). + truncate(B,MsgStr,A,maps:get(max_size,Config)). + +trim([H|T],Rev) when H==$\s; H==$\r; H==$\n -> + trim(T,Rev); +trim([H|T],false) when is_list(H) -> + case trim(H,false) of + [] -> + trim(T,false); + TrimmedH -> + [TrimmedH|T] + end; +trim([H|T],true) when is_list(H) -> + case trim(lists:reverse(H),true) of + [] -> + trim(T,true); + TrimmedH -> + [lists:reverse(TrimmedH)|T] + end; +trim(String,_) -> + String. do_format(Level,Data,[level|Format],Config) -> [to_string(level,Level,Config)|do_format(Level,Data,Format,Config)]; @@ -239,21 +262,47 @@ chardata_to_list(Chardata) -> throw(Error) end. -truncate(String,unlimited) -> - String; -truncate(String,Size) -> - Length = string:length(String), +truncate(B,Msg,A,unlimited) -> + [B,Msg,A]; +truncate(B,Msg,A,Size) -> + String = [B,Msg,A], + Length = io_lib:chars_length(String), if Length>Size -> - case lists:reverse(lists:flatten(String)) of - [$\n|_] -> - string:slice(String,0,Size-4)++"...\n"; + {Last,FlatString} = + case A of + [] -> + case Msg of + [] -> + {get_last(B),lists:flatten(B)}; + _ -> + {get_last(Msg),lists:flatten([B,Msg])} + end; + _ -> + {get_last(A),lists:flatten(String)} + end, + case Last of + $\n-> + lists:sublist(FlatString,1,Size-4)++"...\n"; _ -> - string:slice(String,0,Size-3)++"..." + lists:sublist(FlatString,1,Size-3)++"..." end; true -> String end. +get_last(L) -> + get_first(lists:reverse(L)). + +get_first([]) -> + error; +get_first([C|_]) when is_integer(C) -> + C; +get_first([L|Rest]) when is_list(L) -> + case get_last(L) of + error -> get_first(Rest); + First -> First + end. + %% SysTime is the system time in microseconds format_time(SysTime,#{time_offset:=Offset,time_designator:=Des}) when is_integer(SysTime) -> diff --git a/lib/kernel/src/logger_h_common.erl b/lib/kernel/src/logger_h_common.erl index e69f6de38d..16946ff97c 100644 --- a/lib/kernel/src/logger_h_common.erl +++ b/lib/kernel/src/logger_h_common.erl @@ -142,8 +142,9 @@ changing_config(SetOrUpdate, maps:with(?OLP_KEYS,NewHConfig0)), case logger_olp:set_opts(Olp,NewOlpOpts) of ok -> - maybe_set_repeated_filesync(Olp,OldCommonConfig, - NewCommonConfig), + logger_olp:cast(Olp, {config_changed, + NewCommonConfig, + NewHandlerConfig}), ReadOnly = maps:with(?READ_ONLY_KEYS,OldHConfig), NewHConfig = maps:merge( @@ -281,11 +282,24 @@ handle_cast(repeated_filesync, State#{handler_state => HS, last_op => sync} end, {noreply,set_repeated_filesync(State1)}; - -handle_cast({set_repeated_filesync,FSyncInt},State) -> - State1 = State#{filesync_repeat_interval=>FSyncInt}, - State2 = set_repeated_filesync(cancel_repeated_filesync(State1)), - {noreply, State2}. +handle_cast({config_changed, CommonConfig, HConfig}, + State = #{id := Name, + module := Module, + handler_state := HandlerState, + filesync_repeat_interval := OldFSyncInt}) -> + State1 = + case maps:get(filesync_repeat_interval,CommonConfig) of + OldFSyncInt -> + State; + FSyncInt -> + set_repeated_filesync( + cancel_repeated_filesync( + State#{filesync_repeat_interval=>FSyncInt})) + end, + HS = try Module:config_changed(Name, HConfig, HandlerState) + catch error:undef -> HandlerState + end, + {noreply, State1#{handler_state => HS}}. handle_info(Info, #{id := Name, module := Module, handler_state := HandlerState} = State) -> @@ -351,7 +365,7 @@ log_handler_info(Name, Format, Args, #{module:=Module, {ok,Conf} -> Conf; _ -> #{formatter=>{?DEFAULT_FORMATTER,?DEFAULT_FORMAT_CONFIG}} end, - Meta = #{time=>erlang:system_time(microsecond)}, + Meta = #{time=>logger:timestamp()}, Bin = log_to_binary(#{level => notice, msg => {Format,Args}, meta => Meta}, Config), @@ -447,10 +461,3 @@ cancel_repeated_filesync(State) -> end. error_notify(Term) -> ?internal_log(error, Term). - -maybe_set_repeated_filesync(_Olp, - #{filesync_repeat_interval:=FSyncInt}, - #{filesync_repeat_interval:=FSyncInt}) -> - ok; -maybe_set_repeated_filesync(Olp,_,#{filesync_repeat_interval:=FSyncInt}) -> - logger_olp:cast(Olp,{set_repeated_filesync,FSyncInt}). diff --git a/lib/kernel/src/logger_olp.erl b/lib/kernel/src/logger_olp.erl index 009280a9c9..8365383fe2 100644 --- a/lib/kernel/src/logger_olp.erl +++ b/lib/kernel/src/logger_olp.erl @@ -515,10 +515,11 @@ check_load(State = #{id:=_Name, mode_ref := ModeRef, mode := Mode, end, State1 = ?update_other(drops,DROPS,_NewDrops,State), State2 = ?update_max_qlen(QLen,State1), - State3 = maybe_notify_mode_change(Mode1,State2), + State3 = ?update_max_mem(Mem,State2), + State4 = maybe_notify_mode_change(Mode1,State3), {Mode1, QLen, Mem, ?update_other(flushes,FLUSHES,_NewFlushes, - State3#{last_qlen => QLen})}. + State4#{last_qlen => QLen})}. limit_burst(#{burst_limit_enable := false}=State) -> {true,State}; diff --git a/lib/kernel/src/logger_olp.hrl b/lib/kernel/src/logger_olp.hrl index 9b4f5ebf27..d68b5c048d 100644 --- a/lib/kernel/src/logger_olp.hrl +++ b/lib/kernel/src/logger_olp.hrl @@ -114,12 +114,16 @@ flushes => 0, flushed => 0, drops => 0, burst_drops => 0, casts => 0, calls => 0, writes => 0, max_qlen => 0, max_time => 0, - freq => {TIME,0,0}} end). + max_mem => 0, freq => {TIME,0,0}} end). -define(update_max_qlen(QLEN, STATE), begin #{max_qlen := QLEN0} = STATE, STATE#{max_qlen => ?max(QLEN0,QLEN)} end). + -define(update_max_mem(MEM, STATE), + begin #{max_mem := MEM0} = STATE, + STATE#{max_mem => ?max(MEM0,MEM)} end). + -define(update_calls_or_casts(CALL_OR_CAST, INC, STATE), case CALL_OR_CAST of cast -> @@ -154,6 +158,7 @@ -else. % DEFAULT! -define(merge_with_stats(STATE), STATE). -define(update_max_qlen(_QLEN, STATE), STATE). + -define(update_max_mem(_MEM, STATE), STATE). -define(update_calls_or_casts(_CALL_OR_CAST, _INC, STATE), STATE). -define(update_max_time(_TIME, STATE), STATE). -define(update_other(_OTHER, _VAR, _INCVAL, STATE), STATE). diff --git a/lib/kernel/src/logger_simple_h.erl b/lib/kernel/src/logger_simple_h.erl index fe181722f3..a0d51dba25 100644 --- a/lib/kernel/src/logger_simple_h.erl +++ b/lib/kernel/src/logger_simple_h.erl @@ -69,7 +69,7 @@ log(#{msg:=_,meta:=#{time:=_}}=Log,_Config) -> do_log( #{level=>error, msg=>{report,{error,simple_handler_process_dead}}, - meta=>#{time=>erlang:system_time(microsecond)}}), + meta=>#{time=>logger:timestamp()}}), do_log(Log); _ -> ?MODULE ! {log,Log} @@ -129,7 +129,7 @@ drop_msg(0) -> drop_msg(N) -> [#{level=>info, msg=>{"Simple handler buffer full, dropped ~w messages",[N]}, - meta=>#{time=>erlang:system_time(microsecond)}}]. + meta=>#{time=>logger:timestamp()}}]. %%%----------------------------------------------------------------- %%% Internal diff --git a/lib/kernel/src/logger_std_h.erl b/lib/kernel/src/logger_std_h.erl index 65f5b3876e..c8f1acfca4 100644 --- a/lib/kernel/src/logger_std_h.erl +++ b/lib/kernel/src/logger_std_h.erl @@ -29,7 +29,7 @@ -export([filesync/1]). %% logger_h_common callbacks --export([init/2, check_config/4, reset_state/2, +-export([init/2, check_config/4, config_changed/3, reset_state/2, filesync/3, write/4, handle_info/3, terminate/3]). %% logger callbacks @@ -105,85 +105,169 @@ filter_config(Config) -> %%%=================================================================== %%% logger_h_common callbacks %%%=================================================================== -init(Name, #{type := Type}) -> - case open_log_file(Name, Type) of +init(Name, Config) -> + MyConfig = maps:with([type,file,modes,file_check,max_no_bytes, + max_no_files,compress_on_rotate],Config), + case file_ctrl_start(Name, MyConfig) of {ok,FileCtrlPid} -> - {ok,#{type=>Type,file_ctrl_pid=>FileCtrlPid}}; + {ok,MyConfig#{file_ctrl_pid=>FileCtrlPid}}; Error -> Error end. -check_config(_Name,set,undefined,NewHConfig) -> - check_config(maps:merge(get_default_config(),NewHConfig)); -check_config(_Name,SetOrUpdate,OldHConfig,NewHConfig0) -> - WriteOnce = maps:with([type],OldHConfig), +check_config(Name,set,undefined,NewHConfig) -> + check_h_config(merge_default_config(Name,normalize_config(NewHConfig))); +check_config(Name,SetOrUpdate,OldHConfig,NewHConfig0) -> + WriteOnce = maps:with([type,file,modes],OldHConfig), Default = case SetOrUpdate of set -> %% Do not reset write-once fields to defaults - maps:merge(get_default_config(),WriteOnce); + merge_default_config(Name,WriteOnce); update -> OldHConfig end, - NewHConfig = maps:merge(Default, NewHConfig0), + NewHConfig = maps:merge(Default, normalize_config(NewHConfig0)), %% Fail if write-once fields are changed - case maps:with([type],NewHConfig) of + case maps:with([type,file,modes],NewHConfig) of WriteOnce -> - check_config(NewHConfig); + check_h_config(NewHConfig); Other -> {error,{illegal_config_change,?MODULE,WriteOnce,Other}} end. -check_config(#{type:=Type}=HConfig) -> - case check_h_config(maps:to_list(HConfig)) of - ok when is_atom(Type) -> - {ok,HConfig#{filesync_repeat_interval=>no_repeat}}; +check_h_config(HConfig) -> + case check_h_config(maps:get(type,HConfig),maps:to_list(HConfig)) of ok -> - {ok,HConfig}; + {ok,fix_file_opts(HConfig)}; {error,{Key,Value}} -> {error,{invalid_config,?MODULE,#{Key=>Value}}} end. -check_h_config([{type,Type} | Config]) when Type == standard_io; - Type == standard_error -> - check_h_config(Config); -check_h_config([{type,{file,File}} | Config]) when is_list(File) -> - check_h_config(Config); -check_h_config([{type,{file,File,Modes}} | Config]) when is_list(File), - is_list(Modes) -> - check_h_config(Config); -check_h_config([Other | _]) -> +check_h_config(Type,[{type,Type} | Config]) when Type =:= standard_io; + Type =:= standard_error; + Type =:= file -> + check_h_config(Type,Config); +check_h_config(file,[{file,File} | Config]) when is_list(File) -> + check_h_config(file,Config); +check_h_config(file,[{modes,Modes} | Config]) when is_list(Modes) -> + check_h_config(file,Config); +check_h_config(file,[{max_no_bytes,Size} | Config]) + when (is_integer(Size) andalso Size>0) orelse Size=:=infinity -> + check_h_config(file,Config); +check_h_config(file,[{max_no_files,Num} | Config]) when is_integer(Num), Num>=0 -> + check_h_config(file,Config); +check_h_config(file,[{compress_on_rotate,Bool} | Config]) when is_boolean(Bool) -> + check_h_config(file,Config); +check_h_config(file,[{file_check,FileCheck} | Config]) + when is_integer(FileCheck), FileCheck>=0 -> + check_h_config(file,Config); +check_h_config(_Type,[Other | _]) -> {error,Other}; -check_h_config([]) -> +check_h_config(_Type,[]) -> ok. -get_default_config() -> - #{type => standard_io}. +normalize_config(#{type:={file,File}}=HConfig) -> + HConfig#{type=>file,file=>File}; +normalize_config(#{type:={file,File,Modes}}=HConfig) -> + HConfig#{type=>file,file=>File,modes=>Modes}; +normalize_config(HConfig) -> + HConfig. + +merge_default_config(Name,#{type:=Type}=HConfig) -> + merge_default_config(Name,Type,HConfig); +merge_default_config(Name,#{file:=_}=HConfig) -> + merge_default_config(Name,file,HConfig); +merge_default_config(Name,HConfig) -> + merge_default_config(Name,standard_io,HConfig). + +merge_default_config(Name,Type,HConfig) -> + maps:merge(get_default_config(Name,Type),HConfig). + +get_default_config(Name,file) -> + #{type => file, + file => atom_to_list(Name), + modes => [raw,append], + file_check => 0, + max_no_bytes => infinity, + max_no_files => 0, + compress_on_rotate => false}; +get_default_config(_Name,Type) -> + #{type => Type}. + +fix_file_opts(#{modes:=Modes}=HConfig) -> + HConfig#{modes=>fix_modes(Modes)}; +fix_file_opts(HConfig) -> + HConfig#{filesync_repeat_interval=>no_repeat}. + +fix_modes(Modes) -> + %% Ensure write|append|exclusive + Modes1 = + case [M || M <- Modes, + lists:member(M,[write,append,exclusive])] of + [] -> [append|Modes]; + _ -> Modes + end, + %% Ensure raw + Modes2 = + case lists:member(raw,Modes) of + false -> [raw|Modes1]; + true -> Modes1 + end, + %% Ensure delayed_write + case lists:partition(fun(delayed_write) -> true; + ({delayed_write,_,_}) -> true; + (_) -> false + end, Modes2) of + {[],_} -> + [delayed_write|Modes2]; + _ -> + Modes2 + end. -filesync(_Name, _Mode, #{type := Type}=State) when is_atom(Type) -> - {ok,State}; -filesync(_Name, async, #{file_ctrl_pid := FileCtrlPid} = State) -> - ok = file_ctrl_filesync_async(FileCtrlPid), - {ok,State}; -filesync(_Name, sync, #{file_ctrl_pid := FileCtrlPid} = State) -> - Result = file_ctrl_filesync_sync(FileCtrlPid), +config_changed(_Name, + #{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}, + #{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}=State) -> + State; +config_changed(_Name, + #{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}, + #{file_ctrl_pid := FileCtrlPid} = State) -> + FileCtrlPid ! {update_config,#{file_check=>FileCheck, + max_no_bytes=>Size, + max_no_files=>Count, + compress_on_rotate=>Compress}}, + State#{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}; +config_changed(_Name,_NewHConfig,State) -> + State. + +filesync(_Name, SyncAsync, #{file_ctrl_pid := FileCtrlPid} = State) -> + Result = file_ctrl_filesync(SyncAsync, FileCtrlPid), {Result,State}. -write(_Name, async, Bin, #{file_ctrl_pid:=FileCtrlPid} = State) -> - ok = file_write_async(FileCtrlPid, Bin), - {ok,State}; -write(_Name, sync, Bin, #{file_ctrl_pid:=FileCtrlPid} = State) -> - Result = file_write_sync(FileCtrlPid, Bin), +write(_Name, SyncAsync, Bin, #{file_ctrl_pid:=FileCtrlPid} = State) -> + Result = file_write(SyncAsync, FileCtrlPid, Bin), {Result,State}. reset_state(_Name, State) -> State. -handle_info(_Name, {'EXIT',Pid,Why}, #{type := FileInfo, file_ctrl_pid := Pid}) -> +handle_info(_Name, {'EXIT',Pid,Why}, #{file_ctrl_pid := Pid}=State) -> %% file_ctrl_pid died, file error, terminate handler - exit({error,{write_failed,FileInfo,Why}}); + exit({error,{write_failed,maps:with([type,file,modes],State),Why}}); handle_info(_, _, State) -> State. @@ -211,27 +295,33 @@ terminate(_Name, _Reason, #{file_ctrl_pid:=FWPid}) -> %%%----------------------------------------------------------------- %%% -open_log_file(HandlerName, FileInfo) -> - case file_ctrl_start(HandlerName, FileInfo) of - OK = {ok,_FileCtrlPid} -> OK; - Error -> Error - end. - -do_open_log_file({file,FileName}) -> - do_open_log_file({file,FileName,[raw,append,delayed_write]}); - -do_open_log_file({file,FileName,[]}) -> - do_open_log_file({file,FileName,[raw,append,delayed_write]}); - -do_open_log_file({file,FileName,Modes}) -> +open_log_file(HandlerName,#{type:=file, + file:=FileName, + modes:=Modes, + file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}) -> try case filelib:ensure_dir(FileName) of ok -> case file:open(FileName, Modes) of {ok, Fd} -> {ok,#file_info{inode=INode}} = - file:read_file_info(FileName), - {ok, {Fd, INode}}; + file:read_file_info(FileName,[raw]), + UpdateModes = [append | Modes--[write,append,exclusive]], + State0 = #{handler_name=>HandlerName, + file_name=>FileName, + modes=>UpdateModes, + file_check=>FileCheck, + fd=>Fd, + inode=>INode, + last_check=>timestamp(), + synced=>false, + write_res=>ok, + sync_res=>ok}, + State = update_rotation({Size,Count,Compress},State0), + {ok,State}; Error -> Error end; @@ -242,21 +332,23 @@ do_open_log_file({file,FileName,Modes}) -> _:Reason -> {error,Reason} end. -close_log_file(Std) when Std == standard_io; Std == standard_error -> - ok; -close_log_file({Fd,_}) -> +close_log_file(#{fd:=Fd}) -> _ = file:datasync(Fd), - _ = file:close(Fd). + _ = file:close(Fd), + ok; +close_log_file(_) -> + ok. + %%%----------------------------------------------------------------- %%% File control process -file_ctrl_start(HandlerName, FileInfo) -> +file_ctrl_start(HandlerName, HConfig) -> Starter = self(), FileCtrlPid = spawn_link(fun() -> - file_ctrl_init(HandlerName, FileInfo, Starter) + file_ctrl_init(HandlerName, HConfig, Starter) end), receive {FileCtrlPid,ok} -> @@ -271,18 +363,16 @@ file_ctrl_start(HandlerName, FileInfo) -> file_ctrl_stop(Pid) -> Pid ! stop. -file_write_async(Pid, Bin) -> +file_write(async, Pid, Bin) -> Pid ! {log,Bin}, - ok. - -file_write_sync(Pid, Bin) -> + ok; +file_write(sync, Pid, Bin) -> file_ctrl_call(Pid, {log,Bin}). -file_ctrl_filesync_async(Pid) -> +file_ctrl_filesync(async, Pid) -> Pid ! filesync, - ok. - -file_ctrl_filesync_sync(Pid) -> + ok; +file_ctrl_filesync(sync, Pid) -> file_ctrl_call(Pid, filesync). file_ctrl_call(Pid, Msg) -> @@ -299,98 +389,255 @@ file_ctrl_call(Pid, Msg) -> {error,{no_response,Pid}} end. -file_ctrl_init(HandlerName, FileInfo, Starter) when is_tuple(FileInfo) -> +file_ctrl_init(HandlerName, + #{type:=file, + file:=FileName} = HConfig, + Starter) -> process_flag(message_queue_data, off_heap), - FileName = element(2, FileInfo), - case do_open_log_file(FileInfo) of - {ok,File} -> + case open_log_file(HandlerName,HConfig) of + {ok,State} -> Starter ! {self(),ok}, - file_ctrl_loop(File, FileName, false, ok, ok, HandlerName); + file_ctrl_loop(State); {error,Reason} -> Starter ! {self(),{error,{open_failed,FileName,Reason}}} end; -file_ctrl_init(HandlerName, StdDev, Starter) -> +file_ctrl_init(HandlerName, #{type:=StdDev}, Starter) -> Starter ! {self(),ok}, - file_ctrl_loop(StdDev, StdDev, false, ok, ok, HandlerName). + file_ctrl_loop(#{handler_name=>HandlerName,dev=>StdDev}). -file_ctrl_loop(File, DevName, Synced, - PrevWriteResult, PrevSyncResult, HandlerName) -> +file_ctrl_loop(State) -> receive %% asynchronous event {log,Bin} -> - File1 = ensure(File, DevName), - Result = write_to_dev(File1, Bin, DevName, - PrevWriteResult, HandlerName), - file_ctrl_loop(File1, DevName, false, - Result, PrevSyncResult, HandlerName); + State1 = write_to_dev(Bin,State), + file_ctrl_loop(State1); %% synchronous event {{log,Bin},{From,MRef}} -> - File1 = ensure(File, DevName), - Result = write_to_dev(File1, Bin, DevName, - PrevWriteResult, HandlerName), + State1 = ensure_file(State), + State2 = write_to_dev(Bin,State1), From ! {MRef,ok}, - file_ctrl_loop(File1, DevName, false, - Result, PrevSyncResult, HandlerName); + file_ctrl_loop(State2); filesync -> - File1 = ensure(File, DevName), - Result = sync_dev(File1, DevName, Synced, - PrevSyncResult, HandlerName), - file_ctrl_loop(File1, DevName, true, - PrevWriteResult, Result, HandlerName); + State1 = sync_dev(State), + file_ctrl_loop(State1); {filesync,{From,MRef}} -> - File1 = ensure(File, DevName), - Result = sync_dev(File1, DevName, Synced, - PrevSyncResult, HandlerName), + State1 = ensure_file(State), + State2 = sync_dev(State1), From ! {MRef,ok}, - file_ctrl_loop(File1, DevName, true, - PrevWriteResult, Result, HandlerName); + file_ctrl_loop(State2); + + {update_config,#{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}} -> + State1 = update_rotation({Size,Count,Compress},State), + file_ctrl_loop(State1#{file_check=>FileCheck}); stop -> - _ = close_log_file(File), + close_log_file(State), stopped end. +maybe_ensure_file(#{file_check:=0}=State) -> + ensure_file(State); +maybe_ensure_file(#{last_check:=T0,file_check:=CheckInt}=State) + when is_integer(CheckInt) -> + T = timestamp(), + if T-T0 > CheckInt -> ensure_file(State); + true -> State + end; +maybe_ensure_file(State) -> + State. + %% In order to play well with tools like logrotate, we need to be able %% to re-create the file if it has disappeared (e.g. if rotated by %% logrotate) -ensure(Fd,DevName) when is_atom(DevName) -> - Fd; -ensure({Fd,INode},FileName) -> - case file:read_file_info(FileName) of - {ok,#file_info{inode=INode}} -> - {Fd,INode}; +ensure_file(#{fd:=Fd0,inode:=INode0,file_name:=FileName,modes:=Modes}=State) -> + case file:read_file_info(FileName,[raw]) of + {ok,#file_info{inode=INode0}} -> + State#{last_check=>timestamp()}; _ -> - _ = file:close(Fd), - _ = file:close(Fd), % delayed_write cause close not to close - case do_open_log_file({file,FileName}) of - {ok,File} -> - File; + close_log_file(Fd0), + case file:open(FileName,Modes) of + {ok,Fd} -> + {ok,#file_info{inode=INode}} = + file:read_file_info(FileName,[raw]), + State#{fd=>Fd,inode=>INode, + last_check=>timestamp(), + synced=>true,sync_res=>ok}; Error -> exit({could_not_reopen_file,Error}) end - end. + end; +ensure_file(State) -> + State. -write_to_dev(DevName, Bin, _DevName, _PrevWriteResult, _HandlerName) - when is_atom(DevName) -> - io:put_chars(DevName, Bin); -write_to_dev({Fd,_}, Bin, FileName, PrevWriteResult, HandlerName) -> +write_to_dev(Bin,#{dev:=DevName}=State) -> + io:put_chars(DevName, Bin), + State; +write_to_dev(Bin, State) -> + State1 = #{fd:=Fd} = maybe_ensure_file(State), Result = ?file_write(Fd, Bin), - maybe_notify_error(write,Result,PrevWriteResult,FileName,HandlerName). + State2 = maybe_rotate_file(Bin,State1), + maybe_notify_error(write,Result,State2), + State2#{synced=>false,write_res=>Result}. -sync_dev(_, _FileName, true, PrevSyncResult, _HandlerName) -> - PrevSyncResult; -sync_dev({Fd,_}, FileName, false, PrevSyncResult, HandlerName) -> +sync_dev(#{synced:=false}=State) -> + State1 = #{fd:=Fd} = maybe_ensure_file(State), Result = ?file_datasync(Fd), - maybe_notify_error(filesync,Result,PrevSyncResult,FileName,HandlerName). + maybe_notify_error(filesync,Result,State1), + State1#{synced=>true,sync_res=>Result}; +sync_dev(State) -> + State. -maybe_notify_error(_Op, ok, _PrevResult, _FileName, _HandlerName) -> +update_rotation({infinity,_,_},State) -> + maybe_remove_archives(0,State), + maps:remove(rotation,State); +update_rotation({Size,Count,Compress},#{file_name:=FileName} = State) -> + maybe_remove_archives(Count,State), + {ok,#file_info{size=CurrSize}} = file:read_file_info(FileName,[raw]), + State1 = State#{rotation=>#{size=>Size, + count=>Count, + compress=>Compress, + curr_size=>CurrSize}}, + maybe_update_compress(0,State1), + maybe_rotate_file(0,State1). + +maybe_remove_archives(Count,#{file_name:=FileName}=State) -> + Archive = rot_file_name(FileName,Count,false), + CompressedArchive = rot_file_name(FileName,Count,true), + case {file:read_file_info(Archive,[raw]), + file:read_file_info(CompressedArchive,[raw])} of + {{error,enoent},{error,enoent}} -> + ok; + _ -> + _ = file:delete(Archive), + _ = file:delete(CompressedArchive), + maybe_remove_archives(Count+1,State) + end. + +maybe_update_compress(Count,#{rotation:=#{count:=Count}}) -> + ok; +maybe_update_compress(N,#{file_name:=FileName, + rotation:=#{compress:=Compress}}=State) -> + Archive = rot_file_name(FileName,N,not Compress), + case file:read_file_info(Archive,[raw]) of + {ok,_} when Compress -> + compress_file(Archive); + {ok,_} -> + decompress_file(Archive); + _ -> + ok + end, + maybe_update_compress(N+1,State). + +maybe_rotate_file(Bin,#{rotation:=_}=State) when is_binary(Bin) -> + maybe_rotate_file(byte_size(Bin),State); +maybe_rotate_file(AddSize,#{rotation:=#{size:=RotSize, + curr_size:=CurrSize}=Rotation}=State) -> + NewSize = CurrSize + AddSize, + if NewSize>RotSize -> + rotate_file(State#{rotation=>Rotation#{curr_size=>NewSize}}); + true -> + State#{rotation=>Rotation#{curr_size=>NewSize}} + end; +maybe_rotate_file(_Bin,State) -> + State. + +rotate_file(#{fd:=Fd0,file_name:=FileName,modes:=Modes,rotation:=Rotation}=State) -> + State1 = sync_dev(State), + _ = file:close(Fd0), + _ = file:close(Fd0), + rotate_files(FileName,maps:get(count,Rotation),maps:get(compress,Rotation)), + case file:open(FileName,Modes) of + {ok,Fd} -> + {ok,#file_info{inode=INode}} = file:read_file_info(FileName,[raw]), + State1#{fd=>Fd,inode=>INode,rotation=>Rotation#{curr_size=>0}}; + Error -> + exit({could_not_reopen_file,Error}) + end. + +rotate_files(FileName,0,_Compress) -> + _ = file:delete(FileName), + ok; +rotate_files(FileName,1,Compress) -> + FileName0 = FileName++".0", + _ = file:rename(FileName,FileName0), + if Compress -> compress_file(FileName0); + true -> ok + end, ok; -maybe_notify_error(_Op, PrevResult, PrevResult, _FileName, _HandlerName) -> +rotate_files(FileName,Count,Compress) -> + _ = file:rename(rot_file_name(FileName,Count-2,Compress), + rot_file_name(FileName,Count-1,Compress)), + rotate_files(FileName,Count-1,Compress). + +rot_file_name(FileName,Count,false) -> + FileName ++ "." ++ integer_to_list(Count); +rot_file_name(FileName,Count,true) -> + rot_file_name(FileName,Count,false) ++ ".gz". + +compress_file(FileName) -> + {ok,In} = file:open(FileName,[read,binary]), + {ok,Out} = file:open(FileName++".gz",[write]), + Z = zlib:open(), + zlib:deflateInit(Z, default, deflated, 31, 8, default), + compress_data(Z,In,Out), + zlib:deflateEnd(Z), + zlib:close(Z), + _ = file:close(In), + _ = file:close(Out), + _ = file:delete(FileName), + ok. + +compress_data(Z,In,Out) -> + case file:read(In,100000) of + {ok,Data} -> + Compressed = zlib:deflate(Z, Data), + _ = file:write(Out,Compressed), + compress_data(Z,In,Out); + eof -> + Compressed = zlib:deflate(Z, <<>>, finish), + _ = file:write(Out,Compressed), + ok + end. + +decompress_file(FileName) -> + {ok,In} = file:open(FileName,[read,binary]), + {ok,Out} = file:open(filename:rootname(FileName,".gz"),[write]), + Z = zlib:open(), + zlib:inflateInit(Z, 31), + decompress_data(Z,In,Out), + zlib:inflateEnd(Z), + zlib:close(Z), + _ = file:close(In), + _ = file:close(Out), + _ = file:delete(FileName), + ok. + +decompress_data(Z,In,Out) -> + case file:read(In,1000) of + {ok,Data} -> + Decompressed = zlib:inflate(Z, Data), + _ = file:write(Out,Decompressed), + decompress_data(Z,In,Out); + eof -> + ok + end. + +maybe_notify_error(_Op, ok, _State) -> + ok; +maybe_notify_error(Op, Result, #{write_res:=WR,sync_res:=SR}) + when (Op==write andalso Result==WR) orelse + (Op==filesync andalso Result==SR) -> %% don't report same error twice - PrevResult; -maybe_notify_error(Op, Error, _PrevResult, FileName, HandlerName) -> + ok; +maybe_notify_error(Op, Error, #{handler_name:=HandlerName,file_name:=FileName}) -> logger_h_common:error_notify({HandlerName,Op,FileName,Error}), - Error. + ok. + +timestamp() -> + erlang:monotonic_time(millisecond). diff --git a/lib/kernel/test/application_SUITE.erl b/lib/kernel/test/application_SUITE.erl index 5c35b82207..1ab554db7c 100644 --- a/lib/kernel/test/application_SUITE.erl +++ b/lib/kernel/test/application_SUITE.erl @@ -31,6 +31,7 @@ otp_3002/1, otp_3184/1, otp_4066/1, otp_4227/1, otp_5363/1, otp_5606/1, start_phases/1, get_key/1, get_env/1, + set_env/1, set_env_persistent/1, set_env_errors/1, permit_false_start_local/1, permit_false_start_dist/1, script_start/1, nodedown_start/1, init2973/0, loop2973/0, loop5606/1]). @@ -55,6 +56,7 @@ all() -> load_use_cache, ensure_started, {group, reported_bugs}, start_phases, script_start, nodedown_start, permit_false_start_local, permit_false_start_dist, get_key, get_env, ensure_all_started, + set_env, set_env_persistent, set_env_errors, {group, distr_changed}, config_change, shutdown_func, shutdown_timeout, shutdown_deadlock, config_relative_paths, persistent_env]. @@ -1944,6 +1946,94 @@ get_appls([_ | T], Res) -> get_appls([], Res) -> Res. +%% Test set_env/1. +set_env(Conf) when is_list(Conf) -> + ok = application:set_env([{appinc, [{own2, persist}, {not_in_app, persist}]}, + {unknown_app, [{key, persist}]}]), + + %% own_env1 and own2 are set in appinc + undefined = application:get_env(appinc, own_env1), + {ok, persist} = application:get_env(appinc, own2), + {ok, persist} = application:get_env(appinc, not_in_app), + {ok, persist} = application:get_env(unknown_app, key), + + ok = application:load(appinc()), + {ok, value1} = application:get_env(appinc, own_env1), + {ok, val2} = application:get_env(appinc, own2), + {ok, persist} = application:get_env(appinc, not_in_app), + {ok, persist} = application:get_env(unknown_app, key), + + %% On reload, values are lost + ok = application:unload(appinc), + ok = application:load(appinc()), + {ok, value1} = application:get_env(appinc, own_env1), + {ok, val2} = application:get_env(appinc, own2), + undefined = application:get_env(appinc, not_in_app), + + %% Clean up + ok = application:unload(appinc). + +%% Test set_env/2 with persistent true. +set_env_persistent(Conf) when is_list(Conf) -> + Opts = [{persistent, true}], + ok = application:set_env([{appinc, [{own2, persist}, {not_in_app, persist}]}, + {unknown_app, [{key, persist}]}], Opts), + + %% own_env1 and own2 are set in appinc + undefined = application:get_env(appinc, own_env1), + {ok, persist} = application:get_env(appinc, own2), + {ok, persist} = application:get_env(appinc, not_in_app), + {ok, persist} = application:get_env(unknown_app, key), + + ok = application:load(appinc()), + {ok, value1} = application:get_env(appinc, own_env1), + {ok, persist} = application:get_env(appinc, own2), + {ok, persist} = application:get_env(appinc, not_in_app), + {ok, persist} = application:get_env(unknown_app, key), + + %% On reload, values are not lost + ok = application:unload(appinc), + ok = application:load(appinc()), + {ok, value1} = application:get_env(appinc, own_env1), + {ok, persist} = application:get_env(appinc, own2), + {ok, persist} = application:get_env(appinc, not_in_app), + + %% Clean up + ok = application:unload(appinc). + +set_env_errors(Conf) when is_list(Conf) -> + "application: 1; application name must be an atom" = + badarg_msg(fun() -> application:set_env([{1, []}]) end), + + "application: foo; parameters must be a list" = + badarg_msg(fun() -> application:set_env([{foo, bar}]) end), + + "invalid application config: foo_bar" = + badarg_msg(fun() -> application:set_env([foo_bar]) end), + + "application: foo; invalid parameter name: 1" = + badarg_msg(fun() -> application:set_env([{foo, [{1, 2}]}]) end), + + "application: foo; invalid parameter: config" = + badarg_msg(fun() -> application:set_env([{foo, [config]}]) end), + + "application: kernel; erroneous parameter: distributed" = + badarg_msg(fun() -> application:set_env([{kernel, [{distributed, config}]}]) end), + + "duplicate application config: foo" = + badarg_msg(fun() -> application:set_env([{foo, []}, {foo, []}]) end), + + "application: foo; duplicate parameter: bar" = + badarg_msg(fun() -> application:set_env([{foo, [{bar, baz}, {bar, bat}]}]) end), + + ok. + +badarg_msg(Fun) -> + try Fun() of + _ -> ct:fail(try_succeeded) + catch + error:{badarg, Msg} -> Msg + end. %% Test set_env/4 and unset_env/3 with persistent true. persistent_env(Conf) when is_list(Conf) -> diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl index 64e0b9d8dd..99fecbe970 100644 --- a/lib/kernel/test/code_SUITE.erl +++ b/lib/kernel/test/code_SUITE.erl @@ -140,6 +140,11 @@ end_per_testcase(on_load_embedded, Config) -> LinkName = proplists:get_value(link_name, Config), _ = del_link(LinkName), end_per_testcase(Config); +end_per_testcase(upgrade, Config) -> + %% Make sure tracing is turned off even if the test times out. + erlang:trace_pattern({error_handler,undefined_function,3}, false, [global]), + erlang:trace(self(), false, [call]), + end_per_testcase(Config); end_per_testcase(_Func, Config) -> end_per_testcase(Config). @@ -1556,6 +1561,11 @@ on_load_update_code_1(3, Mod) -> %% Test -on_load while trace feature 'on_load' is enabled (OTP-14612) on_load_trace_on_load(Config) -> + %% 'on_load' enables tracing for all newly loaded modules, so we make a dry + %% run to ensure that ancillary modules like 'merl' won't be loaded during + %% the actual test. + on_load_update(Config), + Papa = self(), Tracer = spawn_link(fun F() -> receive M -> Papa ! M end, F() end), {tracer,[]} = erlang:trace_info(self(),tracer), diff --git a/lib/kernel/test/gen_sctp_SUITE.erl b/lib/kernel/test/gen_sctp_SUITE.erl index a0ae792ba9..e4c489bd10 100644 --- a/lib/kernel/test/gen_sctp_SUITE.erl +++ b/lib/kernel/test/gen_sctp_SUITE.erl @@ -1459,11 +1459,11 @@ do_open_and_connect(ServerAddresses, AddressToConnectTo) -> do_open_and_connect(ServerAddresses, AddressToConnectTo, Fun). %% do_open_and_connect(ServerAddresses, AddressToConnectTo, Fun) -> - ServerFamily = get_family_by_addrs(ServerAddresses), + {ServerFamily, ServerOpts} = get_family_by_addrs(ServerAddresses), io:format("Serving ~p addresses: ~p~n", [ServerFamily, ServerAddresses]), S1 = ok(gen_sctp:open(0, [{ip,Addr} || Addr <- ServerAddresses] ++ - [ServerFamily])), + [ServerFamily|ServerOpts])), ok = gen_sctp:listen(S1, true), P1 = ok(inet:port(S1)), ClientFamily = get_family_by_addr(AddressToConnectTo), @@ -1493,9 +1493,9 @@ do_open_and_connect(ServerAddresses, AddressToConnectTo, Fun) -> %% If at least one of the addresses is an ipv6 address, return inet6, else inet. get_family_by_addrs(Addresses) -> case lists:usort([get_family_by_addr(Addr) || Addr <- Addresses]) of - [inet, inet6] -> inet6; - [inet] -> inet; - [inet6] -> inet6 + [inet, inet6] -> {inet6, [{ipv6_v6only, false}]}; + [inet] -> {inet, []}; + [inet6] -> {inet6, []} end. get_family_by_addr(Addr) when tuple_size(Addr) =:= 4 -> inet; diff --git a/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c b/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c index b91dca61d4..96938f9071 100644 --- a/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c +++ b/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c @@ -30,6 +30,7 @@ #define sock_close(s) closesocket(s) #else #include <sys/socket.h> +#include <unistd.h> #define sock_close(s) close(s) #endif diff --git a/lib/kernel/test/gen_tcp_misc_SUITE.erl b/lib/kernel/test/gen_tcp_misc_SUITE.erl index 52edfaee29..edf30448c4 100644 --- a/lib/kernel/test/gen_tcp_misc_SUITE.erl +++ b/lib/kernel/test/gen_tcp_misc_SUITE.erl @@ -2086,8 +2086,39 @@ test_pktoptions(Family, Spec, CheckConnect, OSType, OSVer) -> %%% {ok,<<"hi">>} = gen_tcp:recv(S1, 2, Timeout), %% %% Verify returned remote options - {ok,[{pktoptions,OptsVals1}]} = inet:getopts(S1, [pktoptions]), - {ok,[{pktoptions,OptsVals2}]} = inet:getopts(S2, [pktoptions]), + VerifyRemOpts = + fun(S, Role) -> + case inet:getopts(S, [pktoptions]) of + {ok, [{pktoptions, PktOpts1}]} -> + PktOpts1; + {ok, UnexpOK1} -> + io:format("Unexpected OK (~w): " + "~n ~p" + "~n", [Role, UnexpOK1]), + exit({unexpected_getopts_ok, + Role, + Spec, + TrueRecvOpts, + OptsVals, + OptsValsDefault, + UnexpOK1}); + {error, UnexpERR1} -> + io:format("Unexpected ERROR (~w): " + "~n ~p" + "~n", [Role, UnexpERR1]), + exit({unexpected_getopts_failure, + Role, + Spec, + TrueRecvOpts, + OptsVals, + OptsValsDefault, + UnexpERR1}) + end + end, + OptsVals1 = VerifyRemOpts(S1, dest), + OptsVals2 = VerifyRemOpts(S2, orig), + %% {ok,[{pktoptions,OptsVals1}]} = inet:getopts(S1, [pktoptions]), + %% {ok,[{pktoptions,OptsVals2}]} = inet:getopts(S2, [pktoptions]), (Result1 = sets_eq(OptsVals1, OptsVals)) orelse io:format( "Accept differs: ~p neq ~p~n", [OptsVals1,OptsVals]), @@ -3430,7 +3461,7 @@ wait(Mref) -> %% OTP-15536 %% Test that send error works correctly for delay_send -delay_send_error(Config) -> +delay_send_error(_Config) -> {ok, LS} = gen_tcp:listen(0, [{reuseaddr, true}, {packet, 1}, {active, false}]), {ok,{{0,0,0,0},PortNum}}=inet:sockname(LS), P = spawn_link( diff --git a/lib/kernel/test/inet_SUITE.erl b/lib/kernel/test/inet_SUITE.erl index 8b33f4a679..44ec7e7076 100644 --- a/lib/kernel/test/inet_SUITE.erl +++ b/lib/kernel/test/inet_SUITE.erl @@ -21,6 +21,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("kernel/include/inet.hrl"). +-include_lib("kernel/src/inet_res.hrl"). -include_lib("kernel/src/inet_dns.hrl"). -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, @@ -34,7 +35,7 @@ ipv4_to_ipv6/0, ipv4_to_ipv6/1, host_and_addr/0, host_and_addr/1, t_gethostnative/1, - gethostnative_parallell/1, cname_loop/1, + gethostnative_parallell/1, cname_loop/1, missing_hosts_reload/1, gethostnative_soft_restart/0, gethostnative_soft_restart/1, gethostnative_debug_level/0, gethostnative_debug_level/1, lookup_bad_search_option/1, @@ -56,7 +57,7 @@ all() -> [t_gethostbyaddr, t_gethostbyname, t_getaddr, t_gethostbyaddr_v6, t_gethostbyname_v6, t_getaddr_v6, ipv4_to_ipv6, host_and_addr, {group, parse}, - t_gethostnative, gethostnative_parallell, cname_loop, + t_gethostnative, gethostnative_parallell, cname_loop, missing_hosts_reload, gethostnative_debug_level, gethostnative_soft_restart, lookup_bad_search_option, getif, getif_ifr_name_overflow, getservbyname_overflow, @@ -840,6 +841,32 @@ cname_loop(Config) when is_list(Config) -> ok. +%% Test that hosts file gets reloaded correctly in case when it +% was missing during initial startup +missing_hosts_reload(Config) when is_list(Config) -> + RootDir = proplists:get_value(priv_dir,Config), + HostsFile = filename:join(RootDir, atom_to_list(?MODULE) ++ ".hosts"), + InetRc = filename:join(RootDir, "inetrc"), + ok = file:write_file(InetRc, "{hosts_file, \"" ++ HostsFile ++ "\"}.\n"), + {error, enoent} = file:read_file_info(HostsFile), + % start a node + Pa = filename:dirname(code:which(?MODULE)), + {ok, TestNode} = test_server:start_node(?MODULE, slave, + [{args, "-pa " ++ Pa ++ " -kernel inetrc '\"" ++ InetRc ++ "\"'"}]), + % ensure it has our RC + Rc = rpc:call(TestNode, inet_db, get_rc, []), + {hosts_file, HostsFile} = lists:keyfind(hosts_file, 1, Rc), + % ensure it does not resolve + {error, nxdomain} = rpc:call(TestNode, inet_hosts, gethostbyname, ["somehost"]), + % write hosts file + ok = file:write_file(HostsFile, "1.2.3.4 somehost"), + % wait for cached timestamp to expire + timer:sleep(?RES_FILE_UPDATE_TM * 1000 + 100), + % ensure it DOES resolve + {ok,{hostent,"somehost",[],inet,4,[{1,2,3,4}]}} = + rpc:call(TestNode, inet_hosts, gethostbyname, ["somehost"]), + % cleanup + true = test_server:stop_node(TestNode). %% These must be run in the whole suite since they need %% the host list and require inet_gethost_native to be started. diff --git a/lib/kernel/test/kernel.spec b/lib/kernel/test/kernel.spec index 62afc9f97b..eaa17f3a59 100644 --- a/lib/kernel/test/kernel.spec +++ b/lib/kernel/test/kernel.spec @@ -2,3 +2,4 @@ {config, "../test_server/ts.unix.config"}. {suites,"../kernel_test", all}. +{skip_suites,"../kernel_test",[logger_stress_SUITE],"Benchmarks only"}. diff --git a/lib/kernel/test/kernel_config_SUITE.erl b/lib/kernel/test/kernel_config_SUITE.erl index 9207025a2c..57c44c2498 100644 --- a/lib/kernel/test/kernel_config_SUITE.erl +++ b/lib/kernel/test/kernel_config_SUITE.erl @@ -21,7 +21,8 @@ -include_lib("common_test/include/ct.hrl"). --export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2, sync/1]). +-export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2, + start_distribution_false/1, sync/1]). -export([init_per_suite/1, end_per_suite/1]). @@ -30,7 +31,7 @@ suite() -> {timetrap,{minutes,2}}]. all() -> - [sync]. + [sync, start_distribution_false]. groups() -> []. @@ -59,12 +60,9 @@ from(H, [H | T]) -> T; from(H, [_ | T]) -> from(H, T); from(_, []) -> []. -%%----------------------------------------------------------------- -%% Test suite for sync_nodes. This is quite tricky. -%% +%% Test sync_nodes. This is quite tricky. %% Should be started in a CC view with: %% erl -sname XXX where XX not in [cp1, cp2] -%%----------------------------------------------------------------- sync(Conf) when is_list(Conf) -> %% Write a config file Dir = proplists:get_value(priv_dir,Conf), @@ -106,9 +104,18 @@ wait_for_node(Node) -> _Other -> wait_for_node(Node) end. - stop_node(Node) -> M = list_to_atom(lists:concat([Node, [$@], from($@,atom_to_list(node()))])), rpc:cast(M, erlang, halt, []). + +start_distribution_false(Config) when is_list(Config) -> + %% When distribution is disabled, -sname/-name has no effect + Str = os:cmd(ct:get_progname() + ++ " -kernel start_distribution false" + ++ " -sname no_distribution" + ++ " -eval \"erlang:display(node())\"" + ++ " -noshell -s erlang halt"), + "'nonode@nohost'" ++ _ = Str, + ok. diff --git a/lib/kernel/test/logger_SUITE.erl b/lib/kernel/test/logger_SUITE.erl index 2dad651f9c..035e5d8974 100644 --- a/lib/kernel/test/logger_SUITE.erl +++ b/lib/kernel/test/logger_SUITE.erl @@ -899,14 +899,14 @@ process_metadata(_Config) -> undefined = logger:get_process_metadata(), {error,badarg} = ?TRY(logger:set_process_metadata(bad)), ok = logger:add_handler(h1,?MODULE,#{level=>notice,filter_default=>log}), - Time = erlang:system_time(microsecond), + Time = logger:timestamp(), ProcMeta = #{time=>Time,line=>0,custom=>proc}, ok = logger:set_process_metadata(ProcMeta), S1 = ?str, ?LOG_NOTICE(S1,#{custom=>macro}), check_logged(notice,S1,#{time=>Time,line=>0,custom=>macro}), - Time2 = erlang:system_time(microsecond), + Time2 = logger:timestamp(), S2 = ?str, ?LOG_NOTICE(S2,#{time=>Time2,line=>1,custom=>macro}), check_logged(notice,S2,#{time=>Time2,line=>1,custom=>macro}), @@ -1048,8 +1048,11 @@ kernel_config(Config) -> ok = rpc:call(Node,logger,internal_init_logger,[]), ok = rpc:call(Node,logger,add_handlers,[kernel]), #{primary:=#{filter_default:=log,filters:=[]}, - handlers:=[#{id:=default,filters:=DF,config:=#{type:={file,F}}}], + handlers:=[#{id:=default,filters:=DF, + config:=#{type:=file,file:=F,modes:=Modes}}], module_levels:=[]} = rpc:call(Node,logger,get_config,[]), + [append,delayed_write,raw] = lists:sort(Modes), + %% Same, but using 'logger' parameter instead of 'error_logger' ok = rpc:call(Node,logger,remove_handler,[default]),% so it can be added again @@ -1060,26 +1063,27 @@ kernel_config(Config) -> ok = rpc:call(Node,logger,internal_init_logger,[]), ok = rpc:call(Node,logger,add_handlers,[kernel]), #{primary:=#{filter_default:=log,filters:=[]}, - handlers:=[#{id:=default,filters:=DF,config:=#{type:={file,F}}}], + handlers:=[#{id:=default,filters:=DF, + config:=#{type:=file,file:=F,modes:=Modes}}], module_levels:=[]} = rpc:call(Node,logger,get_config,[]), %% Same, but with type={file,File,Modes} ok = rpc:call(Node,logger,remove_handler,[default]),% so it can be added again ok = rpc:call(Node,application,unset_env,[kernel,error_logger]), - M = [raw,write,delayed_write], + M = [raw,write], ok = rpc:call(Node,application,set_env,[kernel,logger, [{handler,default,logger_std_h, #{config=>#{type=>{file,F,M}}}}]]), ok = rpc:call(Node,logger,internal_init_logger,[]), ok = rpc:call(Node,logger,add_handlers,[kernel]), #{primary:=#{filter_default:=log,filters:=[]}, - handlers:=[#{id:=default,filters:=DF,config:=#{type:={file,F,M}}}], + handlers:=[#{id:=default,filters:=DF, + config:=#{type:=file,file:=F,modes:=[delayed_write|M]}}], module_levels:=[]} = rpc:call(Node,logger,get_config,[]), %% Same, but with disk_log handler ok = rpc:call(Node,logger,remove_handler,[default]),% so it can be added again ok = rpc:call(Node,application,unset_env,[kernel,error_logger]), - M = [raw,write,delayed_write], ok = rpc:call(Node,application,set_env,[kernel,logger, [{handler,default,logger_disk_log_h, #{config=>#{file=>F}}}]]), diff --git a/lib/kernel/test/logger_formatter_SUITE.erl b/lib/kernel/test/logger_formatter_SUITE.erl index 8c13f0f908..83e3e6c40a 100644 --- a/lib/kernel/test/logger_formatter_SUITE.erl +++ b/lib/kernel/test/logger_formatter_SUITE.erl @@ -867,7 +867,7 @@ my_try(Fun) -> try Fun() catch C:R:S -> {C,R,hd(S)} end. timestamp() -> - erlang:system_time(microsecond). + logger:timestamp(). %% necessary? add_time(#{time:=_}=Meta) -> diff --git a/lib/kernel/test/logger_std_h_SUITE.erl b/lib/kernel/test/logger_std_h_SUITE.erl index b2c2c8ba67..0c5516f82b 100644 --- a/lib/kernel/test/logger_std_h_SUITE.erl +++ b/lib/kernel/test/logger_std_h_SUITE.erl @@ -112,6 +112,8 @@ all() -> add_remove_instance_standard_error, add_remove_instance_file1, add_remove_instance_file2, + add_remove_instance_file3, + add_remove_instance_file4, default_formatter, filter_config, errors, @@ -142,7 +144,12 @@ all() -> restart_after, handler_requests_under_load, recreate_deleted_log, - reopen_changed_log + reopen_changed_log, + rotate_size, + rotate_size_compressed, + rotate_size_reopen, + rotation_opts, + rotation_opts_restart_handler ]. add_remove_instance_tty(_Config) -> @@ -179,10 +186,27 @@ add_remove_instance_file2(Config) -> add_remove_instance_file2(cleanup,_Config) -> logger_std_h_remove(). -add_remove_instance_file(Log, Type) -> +add_remove_instance_file3(_Config) -> + Log = atom_to_list(?MODULE), + StdHConfig = #{type=>file}, + add_remove_instance_file(Log, StdHConfig). +add_remove_instance_file3(cleanup,_Config) -> + logger_std_h_remove(). + +add_remove_instance_file4(Config) -> + Dir = ?config(priv_dir,Config), + Log = filename:join(Dir,"stdlog4.txt"), + StdHConfig = #{file=>Log,modes=>[]}, + add_remove_instance_file(Log, StdHConfig). +add_remove_instance_file4(cleanup,_Config) -> + logger_std_h_remove(). + +add_remove_instance_file(Log, Type) when not is_map(Type) -> + add_remove_instance_file(Log,#{type=>Type}); +add_remove_instance_file(Log, StdHConfig) when is_map(StdHConfig) -> ok = logger:add_handler(?MODULE, logger_std_h, - #{config => #{type => Type}, + #{config => StdHConfig, filter_default=>stop, filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), formatter=>{?MODULE,self()}}), @@ -257,9 +281,10 @@ errors(Config) -> end, {error, - {handler_not_added,{open_failed,Log,_}}} = + {handler_not_added, + {invalid_config,logger_std_h,#{modes:=bad_file_opt}}}} = logger:add_handler(myh3,logger_std_h, - #{config=>#{type=>{file,Log,[bad_file_opt]}}}), + #{config=>#{type=>{file,Log,bad_file_opt}}}), ok = logger:notice(?msg). @@ -607,24 +632,51 @@ reconfig(cleanup, _Config) -> file_opts(Config) -> Dir = ?config(priv_dir,Config), Log = filename:join(Dir, lists:concat([?FUNCTION_NAME,".log"])), - BadFileOpts = [raw], - BadType = {file,Log,BadFileOpts}, - {error,{handler_not_added,{open_failed,Log,enoent}}} = - logger:add_handler(?MODULE, logger_std_h, - #{config => #{type => BadType}}), + MissingOpts = [raw], + Type1 = {file,Log,MissingOpts}, + ok = logger:add_handler(?MODULE, logger_std_h, + #{config => #{type => Type1}}), + {ok,#{config:=#{type:=file,file:=Log,modes:=Modes1}}} = + logger:get_handler_config(?MODULE), + [append,delayed_write,raw] = lists:sort(Modes1), + ok = logger:remove_handler(?MODULE), OkFileOpts = [raw,append], OkType = {file,Log,OkFileOpts}, ok = logger:add_handler(?MODULE, logger_std_h, - #{config => #{type => OkType}, + #{config => #{type => OkType}, % old format filter_default=>log, filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), formatter=>{?MODULE,self()}}), - #{cb_state := #{handler_state := #{type := OkType}}} = + ModOpts = [delayed_write|OkFileOpts], + #{cb_state := #{handler_state := #{type:=file, + file:=Log, + modes:=ModOpts}}} = logger_olp:info(h_proc_name()), - {ok,#{config := #{type := OkType}}} = logger:get_handler_config(?MODULE), + {ok,#{config := #{type:=file, + file:=Log, + modes:=ModOpts}}} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + + ok = logger:add_handler(?MODULE, + logger_std_h, + #{config => #{type => file, + file => Log, + modes => OkFileOpts}, % new format + filter_default=>log, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter=>{?MODULE,self()}}), + + #{cb_state := #{handler_state := #{type:=file, + file:=Log, + modes:=ModOpts}}} = + logger_olp:info(h_proc_name()), + {ok,#{config := #{type:=file, + file:=Log, + modes:=ModOpts}}} = + logger:get_handler_config(?MODULE), logger:notice(M1=?msg,?domain), ?check(M1), B1 = ?bin(M1), @@ -640,13 +692,14 @@ sync(Config) -> Type = {file,Log}, ok = logger:add_handler(?MODULE, logger_std_h, - #{config => #{type => Type}, + #{config => #{type => Type, + file_check => 10000}, filter_default=>log, filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), formatter=>{?MODULE,nl}}), %% check repeated filesync happens - start_tracer([{logger_std_h, write_to_dev, 5}, + start_tracer([{logger_std_h, write_to_dev, 2}, {file, datasync, 1}], [{logger_std_h, write_to_dev, <<"first\n">>}, {file,datasync}]), @@ -656,7 +709,7 @@ sync(Config) -> check_tracer(filesync_rep_int()*2), %% check that explicit filesync is only done once - start_tracer([{logger_std_h, write_to_dev, 5}, + start_tracer([{logger_std_h, write_to_dev, 2}, {file, datasync, 1}], [{logger_std_h, write_to_dev, <<"second\n">>}, {file,datasync}, @@ -675,7 +728,7 @@ sync(Config) -> #{filesync_repeat_interval => no_repeat}), no_repeat = maps:get(filesync_repeat_interval, maps:get(cb_state, logger_olp:info(h_proc_name()))), - start_tracer([{logger_std_h, write_to_dev, 5}, + start_tracer([{logger_std_h, write_to_dev, 2}, {file, datasync, 1}], [{logger_std_h, write_to_dev, <<"third\n">>}, {file,datasync}, @@ -1285,6 +1338,331 @@ reopen_changed_log(Config) -> reopen_changed_log(cleanup, _Config) -> ok = stop_handler(?MODULE). +rotate_size(Config) -> + {Log,_HConfig,_StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + ok = logger:update_handler_config(?MODULE,#{config=>#{max_no_bytes=>1000, + max_no_files=>2}}), + + Str = lists:duplicate(19,$a), + [logger:notice(Str,?domain) || _ <- lists:seq(1,50)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=1000}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + + logger:notice(Str,?domain), + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".0"), + {error,enoent} = file:read_file_info(Log++".1"), + + [logger:notice(Str,?domain) || _ <- lists:seq(1,51)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".0"), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".1"), + {error,enoent} = file:read_file_info(Log++".2"), + + [logger:notice(Str,?domain) || _ <- lists:seq(1,50)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=1000}} = file:read_file_info(Log), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".0"), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".1"), + {error,enoent} = file:read_file_info(Log++".2"), + + logger:notice("bbbb",?domain), + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=1005}} = file:read_file_info(Log++".0"), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".1"), + {error,enoent} = file:read_file_info(Log++".2"), + + ok. +rotate_size(cleanup,_Config) -> + ok = stop_handler(?MODULE). + +rotate_size_compressed(Config) -> + {Log,_HConfig,_StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + ok = logger:update_handler_config(?MODULE, + #{config=>#{max_no_bytes=>1000, + max_no_files=>2, + compress_on_rotate=>true}}), + Str = lists:duplicate(19,$a), + [logger:notice(Str,?domain) || _ <- lists:seq(1,50)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=1000}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + {error,enoent} = file:read_file_info(Log++".0.gz"), + + logger:notice(Str,?domain), + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".0.gz"), + {error,enoent} = file:read_file_info(Log++".1"), + {error,enoent} = file:read_file_info(Log++".1.gz"), + + [logger:notice(Str,?domain) || _ <- lists:seq(1,51)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".0.gz"), + {error,enoent} = file:read_file_info(Log++".1"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".1.gz"), + {error,enoent} = file:read_file_info(Log++".2"), + {error,enoent} = file:read_file_info(Log++".2.gz"), + + [logger:notice(Str,?domain) || _ <- lists:seq(1,50)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=1000}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".0.gz"), + {error,enoent} = file:read_file_info(Log++".1"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".1.gz"), + {error,enoent} = file:read_file_info(Log++".2"), + {error,enoent} = file:read_file_info(Log++".2.gz"), + + logger:notice("bbbb",?domain), + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + {ok,#file_info{size=38}} = file:read_file_info(Log++".0.gz"), + {error,enoent} = file:read_file_info(Log++".1"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".1.gz"), + {error,enoent} = file:read_file_info(Log++".2"), + {error,enoent} = file:read_file_info(Log++".2.gz"), + + ok. +rotate_size_compressed(cleanup,_Config) -> + ok = stop_handler(?MODULE). + +rotate_size_reopen(Config) -> + {Log,_HConfig,_StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + ok = logger:update_handler_config(?MODULE,#{config=>#{max_no_bytes=>1000, + max_no_files=>2}}), + + Str = lists:duplicate(19,$a), + [logger:notice(Str,?domain) || _ <- lists:seq(1,40)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=800}} = file:read_file_info(Log), + + {ok,HConfig} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + ok = logger:add_handler(?MODULE,maps:get(module,HConfig),HConfig), + {ok,#file_info{size=800}} = file:read_file_info(Log), + + [logger:notice(Str,?domain) || _ <- lists:seq(1,40)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=580}} = file:read_file_info(Log), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".0"), + ok. +rotate_size_reopen(cleanup,_Config) -> + ok = stop_handler(?MODULE). + +rotation_opts(Config) -> + {Log,_HConfig,StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + #{max_no_bytes:=infinity, + max_no_files:=0, + compress_on_rotate:=false} = StdHConfig, + + %% Test bad rotation config + {error,{invalid_config,_,_}} = + logger:update_handler_config(?MODULE,config,#{max_no_bytes=>0}), + {error,{invalid_config,_,_}} = + logger:update_handler_config(?MODULE,config,#{max_no_files=>infinity}), + {error,{invalid_config,_,_}} = + logger:update_handler_config(?MODULE,config, + #{compress_on_rotate=>undefined}), + + + %% Test good rotation config - start with no rotation + Str = lists:duplicate(19,$a), + [logger:notice(Str,?domain) || _ <- lists:seq(1,10)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=200}} = file:read_file_info(Log), + [] = filelib:wildcard(Log++".*"), + + %% Turn on rotation, check that existing file is rotated since its + %% size exceeds max_no_bytes + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_bytes=>100, + max_no_files=>2}), + timer:sleep(100), % give some time to execute config_changed + {ok,#file_info{size=0}} = file:read_file_info(Log), + Log0 = Log++".0", + {ok,#file_info{size=200}} = file:read_file_info(Log0), + [Log0] = filelib:wildcard(Log++".*"), + + %% Fill all logs + [logger:notice(Str,?domain) || _ <- lists:seq(1,13)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=20}} = file:read_file_info(Log), + {ok,#file_info{size=120}} = file:read_file_info(Log0), + {ok,#file_info{size=120}} = file:read_file_info(Log++".1"), + [_,_] = filelib:wildcard(Log++".*"), + + %% Extend size and count and check that nothing changes with existing files + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_bytes=>200, + max_no_files=>3}), + timer:sleep(100), % give some time to execute config_changed + {ok,#file_info{size=20}} = file:read_file_info(Log), + {ok,#file_info{size=120}} = file:read_file_info(Log0), + {ok,#file_info{size=120}} = file:read_file_info(Log++".1"), + [_,_] = filelib:wildcard(Log++".*"), + + %% Add more log events and see that extended size and count works + [logger:notice(Str,?domain) || _ <- lists:seq(1,10)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=220}} = file:read_file_info(Log0), + {ok,#file_info{size=120}} = file:read_file_info(Log++".1"), + {ok,#file_info{size=120}} = file:read_file_info(Log++".2"), + [_,_,_] = filelib:wildcard(Log++".*"), + + %% Reduce count and check that archive files that exceed the new + %% count are moved + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_files=>1}), + timer:sleep(100), % give some time to execute config_changed + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=220}} = file:read_file_info(Log0), + [Log0] = filelib:wildcard(Log++".*"), + + %% Extend size and count again, and turn on compression. Check + %% that archives are compressed + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_bytes=>100, + max_no_files=>2, + compress_on_rotate=>true}), + timer:sleep(100), % give some time to execute config_changed + {ok,#file_info{size=0}} = file:read_file_info(Log), + Log0gz = Log0++".gz", + {ok,#file_info{size=29}} = file:read_file_info(Log0gz), + [Log0gz] = filelib:wildcard(Log++".*"), + + %% Fill all logs + [logger:notice(Str,?domain) || _ <- lists:seq(1,13)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=20}} = file:read_file_info(Log), + {ok,#file_info{size=29}} = file:read_file_info(Log0gz), + {ok,#file_info{size=29}} = file:read_file_info(Log++".1.gz"), + [_,_] = filelib:wildcard(Log++".*"), + + %% Reduce count and turn off compression. Check that archives that + %% exceeds the new count are removed, and the rest are + %% uncompressed. + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_files=>1, + compress_on_rotate=>false}), + timer:sleep(100), % give some time to execute config_changed + {ok,#file_info{size=20}} = file:read_file_info(Log), + {ok,#file_info{size=120}} = file:read_file_info(Log0), + [Log0] = filelib:wildcard(Log++".*"), + + %% Check that config and handler state agree on the current rotation settings + {ok,#{config:=#{max_no_bytes:=100, + max_no_files:=1, + compress_on_rotate:=false}}} = + logger:get_handler_config(?MODULE), + #{cb_state:=#{handler_state:=#{max_no_bytes:=100, + max_no_files:=1, + compress_on_rotate:=false}}} = + logger_olp:info(h_proc_name()), + ok. +rotation_opts(cleanup,_Config) -> + ok = stop_handler(?MODULE). + +rotation_opts_restart_handler(Config) -> + {Log,_HConfig,_StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_bytes=>100, + max_no_files=>2}), + + %% Fill all logs + Str = lists:duplicate(19,$a), + [logger:notice(Str,?domain) || _ <- lists:seq(1,15)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=60}} = file:read_file_info(Log), + {ok,#file_info{size=120}} = file:read_file_info(Log++".0"), + {ok,#file_info{size=120}} = file:read_file_info(Log++".1"), + [_,_] = filelib:wildcard(Log++".*"), + + %% Stop/start handler and turn off rotation. Check that archives are removed. + {ok,#{config:=StdHConfig1}=HConfig1} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + ok = logger:add_handler( + ?MODULE,logger_std_h, + HConfig1#{config=>StdHConfig1#{max_no_bytes=>infinity}}), + timer:sleep(100), + {ok,#file_info{size=60}} = file:read_file_info(Log), + [] = filelib:wildcard(Log++".*"), + + %% Add some log events and check that file is no longer rotated. + [logger:notice(Str,?domain) || _ <- lists:seq(1,10)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=260}} = file:read_file_info(Log), + [] = filelib:wildcard(Log++".*"), + + %% Stop/start handler and trun on rotation. Check that file is rotated. + {ok,#{config:=StdHConfig2}=HConfig2} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + ok = logger:add_handler( + ?MODULE,logger_std_h, + HConfig2#{config=>StdHConfig2#{max_no_bytes=>100, + max_no_files=>2}}), + timer:sleep(100), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=260}} = file:read_file_info(Log++".0"), + [_] = filelib:wildcard(Log++".*"), + + %% Fill all logs + [logger:notice(Str,?domain) || _ <- lists:seq(1,10)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=80}} = file:read_file_info(Log), + {ok,#file_info{size=120}} = file:read_file_info(Log++".0"), + {ok,#file_info{size=260}} = file:read_file_info(Log++".1"), + + %% Stop/start handler, reduce count and turn on compression. Check + %% that excess archives are removed, and the rest compressed. + {ok,#{config:=StdHConfig3}=HConfig3} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + ok = logger:add_handler( + ?MODULE,logger_std_h, + HConfig3#{config=>StdHConfig3#{max_no_bytes=>75, + max_no_files=>1, + compress_on_rotate=>true}}), + timer:sleep(100), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=29}} = file:read_file_info(Log++".0.gz"), + [_] = filelib:wildcard(Log++".*"), + + %% Stop/start handler and turn off compression. Check that achives + %% are decompressed. + {ok,#{config:=StdHConfig4}=HConfig4} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + ok = logger:add_handler( + ?MODULE,logger_std_h, + HConfig4#{config=>StdHConfig4#{compress_on_rotate=>false}}), + timer:sleep(100), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=80}} = file:read_file_info(Log++".0"), + [_] = filelib:wildcard(Log++".*"), + + ok. +rotation_opts_restart_handler(cleanup,_Config) -> + ok = stop_handler(?MODULE). + %%%----------------------------------------------------------------- %%% send_requests(TO, Reqs = [{Mod,Func,Args,Res}|Rs]) -> @@ -1305,8 +1683,8 @@ start_handler(Name, TTY, _Config) when TTY == standard_io; ok = logger:add_handler(Name, logger_std_h, #{config => #{type => TTY}, - filter_default=>log, - filters=>?DEFAULT_HANDLER_FILTERS([Name]), + filter_default=>stop, + filters=>filter_only_this_domain(Name), formatter=>{?MODULE,op}}), {ok,HConfig = #{config := StdHConfig}} = logger:get_handler_config(Name), {HConfig,StdHConfig}; @@ -1320,12 +1698,17 @@ start_handler(Name, FuncName, Config) -> ok = logger:add_handler(Name, logger_std_h, #{config => #{type => Type}, - filter_default=>log, - filters=>?DEFAULT_HANDLER_FILTERS([Name]), + filter_default=>stop, + filters=>filter_only_this_domain(Name), formatter=>{?MODULE,op}}), {ok,HConfig = #{config := StdHConfig}} = logger:get_handler_config(Name), {Log,HConfig,StdHConfig}. + +filter_only_this_domain(Name) -> + [{remote_gl,{fun logger_filters:remote_gl/2,stop}}, + {domain,{fun logger_filters:domain/2,{log,super,[Name]}}}]. + stop_handler(Name) -> R = logger:remove_handler(Name), ct:pal("Handler ~p stopped! Result: ~p", [Name,R]), @@ -1658,7 +2041,7 @@ tpl([]) -> tracer({trace,_,call,{logger_h_common,handle_cast,[Op|_]}}, {Pid,[{Mod,Func,Op}|Expected]}) -> maybe_tracer_done(Pid,Expected,{Mod,Func,Op}); -tracer({trace,_,call,{Mod=logger_std_h,Func=write_to_dev,[_,Data,_,_,_]}}, +tracer({trace,_,call,{Mod=logger_std_h,Func=write_to_dev,[Data,_]}}, {Pid,[{Mod,Func,Data}|Expected]}) -> maybe_tracer_done(Pid,Expected,{Mod,Func,Data}); tracer({trace,_,call,{Mod,Func,_}}, {Pid,[{Mod,Func}|Expected]}) -> @@ -1742,4 +2125,3 @@ filesync_rep_int() -> file_delete(Log) -> file:delete(Log). - diff --git a/lib/kernel/test/logger_stress_SUITE.erl b/lib/kernel/test/logger_stress_SUITE.erl index 4072e8c86a..1a278fb1b2 100644 --- a/lib/kernel/test/logger_stress_SUITE.erl +++ b/lib/kernel/test/logger_stress_SUITE.erl @@ -67,6 +67,10 @@ all() -> reject_events, std_handler, disk_log_handler, + std_handler_time, + std_handler_time_big, + disk_log_handler_time, + disk_log_handler_time_big, emulator_events, remote_events, remote_to_disk_log, @@ -123,6 +127,20 @@ std_handler(cleanup,_Config) -> _ = file:delete("default.log"), ok. +%% Disable overload protection and just print a lot - measure time. +%% The IOPS reported is the number of log events written per millisecond. +std_handler_time(Config) -> + measure_handler_time(logger_std_h,#{type=>{file,"default.log"}},Config). +std_handler_time(cleanup,_Config) -> + _ = file:delete("default.log"), + ok. + +std_handler_time_big(Config) -> + measure_handler_time_big(logger_std_h,#{type=>{file,"default.log"}},Config). +std_handler_time_big(cleanup,_Config) -> + _ = file:delete("default.log"), + ok. + %% Cascading failure that produce gen_server and proc_lib reports - %% how many of the produced log events are actually written to a log %% with logger_disk_log_h wrap file handler. @@ -140,6 +158,20 @@ disk_log_handler(cleanup,_Config) -> [_ = file:delete(F) || F <- Files], ok. +%% Disable overload protection and just print a lot - measure time. +%% The IOPS reported is the number of log events written per millisecond. +disk_log_handler_time(Config) -> + measure_handler_time(logger_disk_log_h,#{type=>halt},Config). +disk_log_handler_time(cleanup,_Config) -> + _ = file:delete("default"), + ok. + +disk_log_handler_time_big(Config) -> + measure_handler_time_big(logger_disk_log_h,#{type=>halt},Config). +disk_log_handler_time_big(cleanup,_Config) -> + _ = file:delete("default"), + ok. + %% Cascading failure that produce log events from the emulator - how %% many of the produced log events pass through the proxy. emulator_events(Config) -> @@ -221,6 +253,68 @@ remote_emulator_to_disk_log(cleanup,_Config) -> %%%----------------------------------------------------------------- %%% Internal functions +measure_handler_time(Module,HCfg,Config) -> + measure_handler_time(Module,100000,small_fa(),millisecond,HCfg,#{},Config). + +measure_handler_time_big(Module,HCfg,Config) -> + FCfg = #{chars_limit=>4096, max_size=>1024}, + measure_handler_time(Module,100,big_fa(),second,HCfg,FCfg,Config). + +measure_handler_time(Module,N,FA,Unit,HCfg,FCfg,Config) -> + {ok,_,Node} = + logger_test_lib:setup( + Config, + [{logger, + [{handler,default,Module, + #{formatter => {logger_formatter, + maps:merge(#{legacy_header=>false, + single_line=>true},FCfg)}, + config=>maps:merge(#{sync_mode_qlen => N+1, + drop_mode_qlen => N+1, + flush_qlen => N+1, + burst_limit_enable => false, + filesync_repeat_interval => no_repeat}, + HCfg)}}]}]), + %% HPid = rpc:call(Node,erlang,whereis,[?name_to_reg_name(Module,default)]), + %% {links,[_,FCPid]} = rpc:call(Node,erlang,process_info,[HPid,links]), + T0 = erlang:monotonic_time(millisecond), + ok = rpc:call(Node,?MODULE,nlogs_wait,[N,FA,Module]), + %% ok = rpc:call(Node,fprof,apply,[fun ?MODULE:nlogs_wait/2,[N div 10,FA,Module],[{procs,[HPid,FCPid]}]]), + T1 = erlang:monotonic_time(millisecond), + T = T1-T0, + M = case Unit of + millisecond -> 1; + second -> 1000 + end, + IOPS = M*N/T, + ct:pal("N: ~p~nT: ~p~nIOPS: ~.2f events pr ~w",[N,T,IOPS,Unit]), + %% Stats = rpc:call(Node,logger_olp,info,[?name_to_reg_name(Module,default)]), + %% ct:pal("Stats: ~p",[Stats]), + ct_event:notify(#event{name = benchmark_data, + data = [{value,IOPS}]}), + {comment,io_lib:format("~.2f events written pr ~w",[IOPS,Unit])}. + +nlogs_wait(N,{F,A},Module) -> + group_leader(whereis(user),self()), + [?LOG_NOTICE(F,A) || _ <- lists:seq(1,N)], + wait(Module). + +wait(Module) -> + case Module:filesync(default) of + {error,handler_busy} -> + wait(Module); + ok -> + ok + end. + +small_fa() -> + Str = "\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "[\\]^_`abcdefghijklmnopqr", + {"~ts",[Str]}. + +big_fa() -> + {"~p",[lists:duplicate(1048576,"a")]}. + nlogs(N) -> group_leader(whereis(user),self()), Str = "\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ" diff --git a/lib/kernel/vsn.mk b/lib/kernel/vsn.mk index 4b43c6ae9d..7bebe1ba70 100644 --- a/lib/kernel/vsn.mk +++ b/lib/kernel/vsn.mk @@ -1 +1 @@ -KERNEL_VSN = 6.2 +KERNEL_VSN = 6.3 diff --git a/lib/mnesia/doc/src/notes.xml b/lib/mnesia/doc/src/notes.xml index 8fc3610bb6..01d1666b8d 100644 --- a/lib/mnesia/doc/src/notes.xml +++ b/lib/mnesia/doc/src/notes.xml @@ -39,7 +39,22 @@ thus constitutes one section in this document. The title of each section is the version number of Mnesia.</p> - <section><title>Mnesia 4.15.5</title> + <section><title>Mnesia 4.15.6</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Avoid overload warnings caused by a race condition.</p> + <p> + Own Id: OTP-15619 Aux Id: ERIERL-310 </p> + </item> + </list> + </section> + +</section> + +<section><title>Mnesia 4.15.5</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/mnesia/src/mnesia.erl b/lib/mnesia/src/mnesia.erl index 77afb8250c..02bc884e36 100644 --- a/lib/mnesia/src/mnesia.erl +++ b/lib/mnesia/src/mnesia.erl @@ -160,7 +160,7 @@ {'sync_transaction', Retries::non_neg_integer()}. -type table() :: atom(). -type storage_type() :: 'ram_copies' | 'disc_copies' | 'disc_only_copies'. --type index_attr() :: atom() | non_neg_integer(). +-type index_attr() :: atom() | non_neg_integer() | {atom()}. -type write_locks() :: 'write' | 'sticky_write'. -type read_locks() :: 'read'. -type lock_kind() :: write_locks() | read_locks(). @@ -1277,6 +1277,14 @@ match_object(Tid, Ts, Tab, Pat, LockKind) match_object(_Tid, _Ts, Tab, Pat, _LockKind) -> abort({bad_type, Tab, Pat}). +add_written_index(Store, Pos, Tab, Key, Objs) when is_integer(Pos) -> + Pat = setelement(Pos, val({Tab, wild_pattern}), Key), + add_written_match(Store, Pat, Tab, Objs); +add_written_index(Store, Pos, Tab, Key, Objs) when is_tuple(Pos) -> + IxF = mnesia_index:index_vals_f(val({Tab, storage_type}), Tab, Pos), + Ops = find_ops(Store, Tab, '_'), + add_ix_match(Ops, Objs, IxF, Key, val({Tab, setorbag})). + add_written_match(S, Pat, Tab, Objs) -> Ops = find_ops(S, Tab, Pat), FixedRes = add_match(Ops, Objs, val({Tab, setorbag})), @@ -1303,6 +1311,46 @@ add_match([{_Oid, Val, write}|R], Objs, bag) -> add_match([{Oid, Val, write}|R], Objs, set) -> add_match(R, [Val | deloid(Oid,Objs)],set). +add_ix_match([], Objs, _IxF, _Key, _Type) -> + Objs; +add_ix_match(Written, Objs, IxF, Key, ordered_set) -> + %% Must use keysort which is stable + add_ordered_match(lists:keysort(1, ix_filter_ops(IxF, Key, Written)), Objs, []); +add_ix_match([{Oid, _, delete}|R], Objs, IxF, Key, Type) -> + add_ix_match(R, deloid(Oid, Objs), IxF, Key, Type); +add_ix_match([{_Oid, Val, delete_object}|R], Objs, IxF, Key, Type) -> + case ix_match(Val, IxF, Key) of + true -> + add_ix_match(R, lists:delete(Val, Objs), IxF, Key, Type); + false -> + add_ix_match(R, Objs, IxF, Key, Type) + end; +add_ix_match([{_Oid, Val, write}|R], Objs, IxF, Key, bag) -> + case ix_match(Val, IxF, Key) of + true -> + add_ix_match(R, [Val | lists:delete(Val, Objs)], IxF, Key, bag); + false -> + add_ix_match(R, Objs, IxF, Key, bag) + end; +add_ix_match([{Oid, Val, write}|R], Objs, IxF, Key, set) -> + case ix_match(Val, IxF, Key) of + true -> + add_ix_match(R, [Val | deloid(Oid,Objs)],IxF,Key,set); + false -> + add_ix_match(R, Objs, IxF, Key, set) + end. + +ix_match(Val, IxF, Key) -> + lists:member(Key, IxF(Val)). + +ix_filter_ops(IxF, Key, Ops) -> + lists:filter( + fun({_Oid, Obj, write}) -> + ix_match(Obj, IxF, Key); + (_) -> + true + end, Ops). + %% For ordered_set only !! add_ordered_match(Written = [{{_, Key}, _, _}|_], [Obj|Objs], Acc) when Key > element(2, Obj) -> @@ -1641,6 +1689,16 @@ index_match_object(Tid, Ts, Tab, Pat, Attr, LockKind) dirty_index_match_object(Tab, Pat, Attr); % Should be optimized? tid -> case mnesia_schema:attr_tab_to_pos(Tab, Attr) of + {_} -> + case LockKind of + read -> + Store = Ts#tidstore.store, + mnesia_locker:rlock_table(Tid, Store, Tab), + Objs = dirty_match_object(Tab, Pat), + add_written_match(Store, Pat, Tab, Objs); + _ -> + abort({bad_type, Tab, LockKind}) + end; Pos when Pos =< tuple_size(Pat) -> case LockKind of read -> @@ -1688,8 +1746,8 @@ index_read(Tid, Ts, Tab, Key, Attr, LockKind) false -> Store = Ts#tidstore.store, Objs = mnesia_index:read(Tid, Store, Tab, Key, Pos), - Pat = setelement(Pos, val({Tab, wild_pattern}), Key), - add_written_match(Store, Pat, Tab, Objs); + add_written_index( + Ts#tidstore.store, Pos, Tab, Key, Objs); true -> abort({bad_type, Tab, Attr, Key}) end; @@ -1825,7 +1883,7 @@ remote_dirty_match_object(Tab, Pat) -> false -> mnesia_lib:db_match_object(Tab, Pat); true -> - PosList = val({Tab, index}), + PosList = regular_indexes(Tab), remote_dirty_match_object(Tab, Pat, PosList) end. @@ -1857,7 +1915,7 @@ remote_dirty_select(Tab, Spec) -> false -> mnesia_lib:db_select(Tab, Spec); true -> - PosList = val({Tab, index}), + PosList = regular_indexes(Tab), remote_dirty_select(Tab, Spec, PosList) end; _ -> @@ -1924,6 +1982,8 @@ dirty_index_match_object(Pat, _Attr) -> dirty_index_match_object(Tab, Pat, Attr) when is_atom(Tab), Tab /= schema, is_tuple(Pat), tuple_size(Pat) > 2 -> case mnesia_schema:attr_tab_to_pos(Tab, Attr) of + {_} -> + dirty_match_object(Tab, Pat); Pos when Pos =< tuple_size(Pat) -> case has_var(element(2, Pat)) of false -> @@ -3254,3 +3314,7 @@ put_activity_id(Activity) -> mnesia_tm:put_activity_id(Activity). put_activity_id(Activity,Fun) -> mnesia_tm:put_activity_id(Activity,Fun). + +regular_indexes(Tab) -> + PosList = val({Tab, index}), + [P || P <- PosList, is_integer(P)]. diff --git a/lib/mnesia/src/mnesia_index.erl b/lib/mnesia/src/mnesia_index.erl index 098265d5fc..6f1c21e3b9 100644 --- a/lib/mnesia/src/mnesia_index.erl +++ b/lib/mnesia/src/mnesia_index.erl @@ -1,8 +1,8 @@ %% %% %CopyrightBegin% -%% +%% %% Copyright Ericsson AB 1996-2018. All Rights Reserved. -%% +%% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. %% You may obtain a copy of the License at @@ -14,7 +14,7 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%% +%% %% %CopyrightEnd% %% @@ -37,7 +37,7 @@ db_match_erase/2, get_index_table/2, get_index_table/3, - + tab2filename/2, init_index/2, init_indecies/3, @@ -45,6 +45,7 @@ del_transient/3, del_index_table/3, + index_vals_f/3, index_info/2, ext_index_instances/1]). @@ -60,9 +61,14 @@ read(Tid, Store, Tab, IxKey, Pos) -> ResList = mnesia_locker:ixrlock(Tid, Store, Tab, IxKey, Pos), %% Remove all tuples which don't include Ixkey, happens when Tab is a bag case val({Tab, setorbag}) of - bag -> + bag when is_integer(Pos) -> mnesia_lib:key_search_all(IxKey, Pos, ResList); - _ -> + bag when is_tuple(Pos) -> + TabStorage = val({Tab, storage_type}), + ValsF = index_vals_f(TabStorage, Tab, Pos), + [Obj || Obj <- ResList, + lists:member(IxKey, ValsF(Obj))]; + _ -> ResList end. @@ -136,7 +142,7 @@ del_object_index2([], _, _Storage, _Tab, _K, _Obj) -> ok; del_object_index2([{{Pos, Type}, Ixt} | Tail], SoB, Storage, Tab, K, Obj) -> ValsF = index_vals_f(Storage, Tab, Pos), case SoB of - bag -> + bag -> del_object_bag(Type, ValsF, Tab, K, Obj, Ixt); _ -> %% If set remove the tuple in index table del_ixes(Type, Ixt, ValsF, Obj, K) @@ -197,7 +203,7 @@ merge([], _, _, Ack) -> realkeys(Tab, Pos, IxKey) -> Index = get_index_table(Tab, Pos), db_get(Index, IxKey). % a list on the form [{IxKey, RealKey1} , .... - + dirty_select(Tab, Spec, Pos) when is_integer(Pos) -> %% Assume that we are on the node where the replica is %% Returns the records without applying the match spec @@ -233,7 +239,7 @@ dirty_read2(Tab, IxKey, Pos) -> end, Acc, mnesia_lib:db_get(Storage, Tab, K)) end, [], Keys)). -pick_index([{{{Pfx,_},IxType}, Ixt}|_], _Tab, {_} = Pfx) -> +pick_index([{{{Pfx,_,_},IxType}, Ixt}|_], _Tab, {_} = Pfx) -> {IxType, Ixt}; pick_index([{{Pos,IxType}, Ixt}|_], _Tab, Pos) -> {IxType, Ixt}; @@ -242,7 +248,7 @@ pick_index([_|T], Tab, Pos) -> pick_index([], Tab, Pos) -> mnesia:abort({no_exist, Tab, {index, Pos}}). - + %%%%%%% Creation, Init and deletion routines for index tables %% We can have several indexes on the same table @@ -387,12 +393,12 @@ init_ext_index(Tab, Storage, Alias, Mod, [{Pos,Type} | Tail]) -> create_fun(Cont, Tab, Pos) -> IxF = index_vals_f(disc_only_copies, Tab, Pos), fun(read) -> - Data = + Data = case Cont of {start, KeysPerChunk} -> mnesia_lib:db_init_chunk( disc_only_copies, Tab, KeysPerChunk); - '$end_of_table' -> + '$end_of_table' -> '$end_of_table'; _Else -> mnesia_lib:db_chunk(disc_only_copies, Cont) @@ -462,7 +468,7 @@ add_index_info(Tab, SetOrBag, IxElem) -> %% Check later if mnesia_tm is sensitive about the order mnesia_lib:set({Tab, index_info}, IndexInfo), mnesia_lib:set({Tab, index}, index_positions(IndexInfo)), - mnesia_lib:set({Tab, commit_work}, + mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit([IndexInfo | Commit])); {value, Old} -> %% We could check for consistency here @@ -470,7 +476,7 @@ add_index_info(Tab, SetOrBag, IxElem) -> mnesia_lib:set({Tab, index_info}, Index), mnesia_lib:set({Tab, index}, index_positions(Index)), NewC = lists:keyreplace(index, 1, Commit, Index), - mnesia_lib:set({Tab, commit_work}, + mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)) end. @@ -488,19 +494,19 @@ del_index_info(Tab, Pos) -> element(1,P)=/=Pos end, Old#index.pos_list) of - [] -> + [] -> IndexInfo = index_info(Old#index.setorbag,[]), mnesia_lib:set({Tab, index_info}, IndexInfo), mnesia_lib:set({Tab, index}, index_positions(IndexInfo)), NewC = lists:keydelete(index, 1, Commit), - mnesia_lib:set({Tab, commit_work}, + mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)); New -> Index = Old#index{pos_list = New}, mnesia_lib:set({Tab, index_info}, Index), mnesia_lib:set({Tab, index}, index_positions(Index)), NewC = lists:keyreplace(index, 1, Commit, Index), - mnesia_lib:set({Tab, commit_work}, + mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)) end end. @@ -537,7 +543,7 @@ db_match_erase({{ext,_,_} = Ext, Ixt}, Pat) -> mnesia_lib:db_match_erase(Ext, Ixt, Pat); db_match_erase({dets, Ixt}, Pat) -> ok = dets:match_delete(Ixt, Pat). - + db_select({ram, Ixt}, Pat) -> ets:select(Ixt, Pat); db_select({{ext,_,_} = Ext, Ixt}, Pat) -> @@ -545,7 +551,7 @@ db_select({{ext,_,_} = Ext, Ixt}, Pat) -> db_select({dets, Ixt}, Pat) -> dets:select(Ixt, Pat). - + get_index_table(Tab, Pos) -> get_index_table(Tab, val({Tab, storage_type}), Pos). diff --git a/lib/mnesia/test/Makefile b/lib/mnesia/test/Makefile index 5b61b1af65..b43bc82801 100644 --- a/lib/mnesia/test/Makefile +++ b/lib/mnesia/test/Makefile @@ -53,7 +53,8 @@ MODULES= \ mnesia_measure_test \ mnesia_cost \ mnesia_dbn_meters \ - ext_test + ext_test \ + mnesia_index_plugin_test DocExamplesDir := ../doc/src/ diff --git a/lib/mnesia/test/mnesia_SUITE.erl b/lib/mnesia/test/mnesia_SUITE.erl index 24c1def6da..b41bf22efa 100644 --- a/lib/mnesia/test/mnesia_SUITE.erl +++ b/lib/mnesia/test/mnesia_SUITE.erl @@ -69,12 +69,13 @@ groups() -> %% covered. [{light, [], [{group, install}, {group, nice}, {group, evil}, - {group, mnesia_frag_test, light}, {group, qlc}, + {group, mnesia_frag_test, light}, {group, qlc}, {group, index_plugins}, {group, registry}, {group, config}, {group, examples}]}, {install, [], [{mnesia_install_test, all}]}, {nice, [], [{mnesia_nice_coverage_test, all}]}, {evil, [], [{mnesia_evil_coverage_test, all}]}, {qlc, [], [{mnesia_qlc_test, all}]}, + {index_plugins, [], [{mnesia_index_plugin_test, all}]}, {registry, [], [{mnesia_registry_test, all}]}, {config, [], [{mnesia_config_test, all}]}, {examples, [], [{mnesia_examples_test, all}]}, diff --git a/lib/mnesia/test/mnesia_index_plugin_test.erl b/lib/mnesia/test/mnesia_index_plugin_test.erl new file mode 100644 index 0000000000..44fe047c50 --- /dev/null +++ b/lib/mnesia/test/mnesia_index_plugin_test.erl @@ -0,0 +1,261 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +-module(mnesia_index_plugin_test). +-author('[email protected]'). + +-export([init_per_testcase/2, end_per_testcase/2, + init_per_group/2, end_per_group/2, + init_per_suite/1, end_per_suite/1, + all/0, groups/0]). + +-export([ + add_rm_plugin/1, + tab_with_plugin_index/1, + tab_with_multiple_plugin_indexes/1, + ix_match_w_plugin/1, + ix_match_w_plugin_ordered/1, + ix_match_w_plugin_bag/1 + ]). + +-export([ix_prefixes/3, % test plugin + ix_prefixes2/3]). % test plugin 2 + +-include("mnesia_test_lib.hrl"). + +init_per_suite(Conf) -> + Conf. + +end_per_suite(Conf) -> + Conf. + +init_per_testcase(Func, Conf) -> + mnesia_test_lib:init_per_testcase(Func, Conf). + +end_per_testcase(Func, Conf) -> + mnesia_test_lib:end_per_testcase(Func, Conf). + +all() -> + [add_rm_plugin, + tab_with_plugin_index, + tab_with_multiple_plugin_indexes, + ix_match_w_plugin, + ix_match_w_plugin_ordered, + ix_match_w_plugin_bag]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + +add_rm_plugin(suite) -> []; +add_rm_plugin(Config) when is_list(Config) -> + [N1, N2] = Nodes = ?acquire_nodes(2, Config), + ok = add_plugin(), + ok = rpc_check_plugin(N1), + ok = rpc_check_plugin(N2), + ok = add_plugin2(), + ok = del_plugin(), + ok = del_plugin2(), + ok = add_plugin(), + ok = add_plugin2(), + ok = del_plugin(), + ok = del_plugin2(), + ?verify_mnesia(Nodes, []). + +-define(PLUGIN1, {{pfx},?MODULE,ix_prefixes}). +-define(PLUGIN2, {{pfx2},?MODULE,ix_prefixes2}). + +add_plugin() -> + {atomic, ok} = mnesia_schema:add_index_plugin({pfx}, ?MODULE, ix_prefixes), + [?PLUGIN1] = mnesia_schema:index_plugins(), + ok. + +add_plugin2() -> + {atomic, ok} = mnesia_schema:add_index_plugin({pfx2}, ?MODULE, ix_prefixes2), + [?PLUGIN1, ?PLUGIN2] = lists:sort(mnesia_schema:index_plugins()), + ok. + +del_plugin() -> + {atomic, ok} = mnesia_schema:delete_index_plugin({pfx}), + [?PLUGIN2] = mnesia_schema:index_plugins(), + ok. + +del_plugin2() -> + {atomic, ok} = mnesia_schema:delete_index_plugin({pfx2}), + [] = mnesia_schema:index_plugins(), + ok. + +rpc_check_plugin(N) -> + [?PLUGIN1] = + rpc:call(N, mnesia_schema, index_plugins, []), + ok. + +tab_with_plugin_index(suite) -> []; +tab_with_plugin_index(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + {atomic, ok} = mnesia:create_table(t, [{attributes, [k,v1,v2]}, + {index, [{{pfx}, ordered}, + {v1, ordered}, + v2]}]), + [ok,ok,ok,ok] = + [mnesia:dirty_write({t, K, V1, V2}) + || {K,V1,V2} <- [{1,a,"123"}, + {2,b,"12345"}, + {3,c,"6789"}, + {4,d,nil}]], + [{t,1,a,"123"},{t,2,b,"12345"}] = + mnesia:dirty_index_read(t,<<"123">>,{pfx}), + [{t,3,c,"6789"}] = + mnesia:dirty_index_read(t,"6789",v2), + [{t,1,a,"123"}] = + mnesia:dirty_match_object({t,'_',a,"123"}), + [{t,1,a,"123"}] = + mnesia:dirty_select(t, [{ {t,'_',a,"123"}, [], ['$_']}]), + mnesia:dirty_delete(t,2), + [{t,1,a,"123"}] = + mnesia:dirty_index_read(t,<<"123">>,{pfx}), + ?verify_mnesia(Nodes, []). + +tab_with_multiple_plugin_indexes(suite) -> []; +tab_with_multiple_plugin_indexes(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + ok = add_plugin2(), + {atomic, ok} = + mnesia:create_table(u, [{attributes, [k,v1,v2]}, + {index, [{{pfx}, ordered}, + {{pfx2}, ordered}]}]), + [ok,ok,ok,ok] = + [mnesia:dirty_write({u, K, V1, V2}) + || {K,V1,V2} <- [{1,a,"123"}, + {2,b,"12345"}, + {3,c,"6789"}, + {4,d,nil}]], + [{u,1,a,"123"},{u,2,b,"12345"}] = + mnesia:dirty_index_read(u,<<"123">>,{pfx}), + [{u,1,a,"123"},{u,2,b,"12345"}] = + mnesia:dirty_index_read(u,<<"321">>,{pfx2}), + ?verify_mnesia(Nodes, []). + +ix_match_w_plugin(suite) -> []; +ix_match_w_plugin(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + {atomic, ok} = mnesia:create_table(im1, [{attributes, [k, v1, v2]}, + {index, [{{pfx}, ordered}, + {v1, ordered}]}]), + fill_and_test_index_match(im1, set), + ?verify_mnesia(Nodes, []). + + +ix_match_w_plugin_ordered(suite) -> []; +ix_match_w_plugin_ordered(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + {atomic, ok} = mnesia:create_table(im2, [{attributes, [k, v1, v2]}, + {type, ordered_set}, + {index, [{{pfx}, ordered}, + {v1, ordered}]}]), + fill_and_test_index_match(im2, ordered_set), + ?verify_mnesia(Nodes, []). + +ix_match_w_plugin_bag(suite) -> []; +ix_match_w_plugin_bag(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + {atomic, ok} = mnesia:create_table(im3, [{attributes, [k, v1, v2]}, + {type, bag}, + {index, [{{pfx}, ordered}, + {v1, ordered}]}]), + fill_and_test_index_match(im3, bag), + ?verify_mnesia(Nodes, []). + +fill_and_test_index_match(Tab, Type) -> + [ok,ok,ok,ok,ok,ok,ok,ok,ok] = + [mnesia:dirty_write({Tab, K, V1, V2}) + || {K,V1,V2} <- [{1,a,"123"}, + {2,b,"12345"}, + {3,c,"123"}, + {4,d,nil}, + {5,e,nil}, + {6,f,nil}, + {7,g,nil}, %% overwritten if not bag + {7,g,"234"}, + {8,h,"123"}]], + mnesia:activity( + transaction, + fun() -> + ok = mnesia:write({Tab, 1, aa, "1234"}), %% replaces if not bag + ok = mnesia:delete({Tab, 2}), + ok = mnesia:delete({Tab, 4}), + ok = mnesia:write({Tab, 6, ff, nil}), + ok = mnesia:write({Tab, 7, gg, "123"}), + ok = mnesia:write({Tab, 100, x, nil}), + ok = mnesia:delete_object({Tab,3,c,"123"}), + ok = mnesia:delete_object({Tab,5,e,nil}), + Res = mnesia:index_read(Tab, <<"123">>, {pfx}), + SetRes = [{Tab,1,aa,"1234"}, {Tab,7,gg,"123"}, {Tab,8,h,"123"}], + case Type of + set -> + SetRes = lists:sort(Res); + ordered_set -> + SetRes = Res; + bag -> + [{Tab,1,a,"123"}, {Tab,1,aa,"1234"}, + {Tab,7,gg,"123"}, {Tab,8,h,"123"}] = lists:sort(Res) + end + end). + +%% ============================================================ +%% +ix_prefixes(_Tab, _Pos, Obj) -> + lists:foldl( + fun(V, Acc) when is_list(V) -> + try Pfxs = prefixes(list_to_binary(V)), + Pfxs ++ Acc + catch + error:_ -> + Acc + end; + (V, Acc) when is_binary(V) -> + Pfxs = prefixes(V), + Pfxs ++ Acc; + (_, Acc) -> + Acc + end, [], tl(tuple_to_list(Obj))). + +ix_prefixes2(Tab, Pos, Obj) -> + [rev(P) || P <- ix_prefixes(Tab, Pos, Obj)]. + +rev(B) when is_binary(B) -> + list_to_binary(lists:reverse(binary_to_list(B))). + +prefixes(<<P:3/binary, _/binary>>) -> + [P]; +prefixes(_) -> + []. diff --git a/lib/mnesia/test/mt.erl b/lib/mnesia/test/mt.erl index 5a981bf539..037d6adb38 100644 --- a/lib/mnesia/test/mt.erl +++ b/lib/mnesia/test/mt.erl @@ -67,6 +67,7 @@ alias(recovery) -> mnesia_recovery_test; alias(registry) -> mnesia_registry_test; alias(suite) -> mnesia_SUITE; alias(trans) -> mnesia_trans_access_test; +alias(ixp) -> mnesia_index_plugin_test; alias(Other) -> Other. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/mnesia/vsn.mk b/lib/mnesia/vsn.mk index 1cfb35c774..781a4830a0 100644 --- a/lib/mnesia/vsn.mk +++ b/lib/mnesia/vsn.mk @@ -1 +1 @@ -MNESIA_VSN = 4.15.5 +MNESIA_VSN = 4.15.6 diff --git a/lib/observer/doc/src/notes.xml b/lib/observer/doc/src/notes.xml index 22035af982..2d914f8c61 100644 --- a/lib/observer/doc/src/notes.xml +++ b/lib/observer/doc/src/notes.xml @@ -32,6 +32,45 @@ <p>This document describes the changes made to the Observer application.</p> +<section><title>Observer 2.9</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Since Logger was introduced in OTP-21.0, menu choice + <em>Log > Toggle Log View</em> in observer would cause + a crash unless an <c>error_logger</c> event handler was + explicitly installed. This is now corrected.</p> + <p> + Own Id: OTP-15553 Aux Id: ERL-848 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Since persistent_term was introduced, observer would + sometimes crash when expanding a term from a process + state. This is now corrected.</p> + <p> + Own Id: OTP-15493 Aux Id: ERL-810 </p> + </item> + <item> + <p> + Add <c>OBSERVER_SCALE</c> environment variable for HiDPI + support.</p> + <p> + Own Id: OTP-15586 Aux Id: PR-2105 </p> + </item> + </list> + </section> + +</section> + <section><title>Observer 2.8.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/observer/vsn.mk b/lib/observer/vsn.mk index 5ce0aca589..0e9c8b302c 100644 --- a/lib/observer/vsn.mk +++ b/lib/observer/vsn.mk @@ -1 +1 @@ -OBSERVER_VSN = 2.8.2 +OBSERVER_VSN = 2.9 diff --git a/lib/odbc/doc/src/notes.xml b/lib/odbc/doc/src/notes.xml index dba7663bb9..696fcaa479 100644 --- a/lib/odbc/doc/src/notes.xml +++ b/lib/odbc/doc/src/notes.xml @@ -32,7 +32,22 @@ <p>This document describes the changes made to the odbc application. </p> - <section><title>ODBC 2.12.2</title> + <section><title>ODBC 2.12.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Enhance error handling to avoid stack corruption</p> + <p> + Own Id: OTP-15667 Aux Id: ERL-808, PR-2065 </p> + </item> + </list> + </section> + +</section> + +<section><title>ODBC 2.12.2</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/odbc/vsn.mk b/lib/odbc/vsn.mk index bb21016fad..ff023e666b 100644 --- a/lib/odbc/vsn.mk +++ b/lib/odbc/vsn.mk @@ -1 +1 @@ -ODBC_VSN = 2.12.2 +ODBC_VSN = 2.12.3 diff --git a/lib/public_key/asn1/OTP-PKIX.asn1 b/lib/public_key/asn1/OTP-PKIX.asn1 index 9bcd99fba3..ff3250b383 100644 --- a/lib/public_key/asn1/OTP-PKIX.asn1 +++ b/lib/public_key/asn1/OTP-PKIX.asn1 @@ -233,9 +233,13 @@ countryName ATTRIBUTE-TYPE-AND-VALUE-CLASS ::= { -- regarding how to handle and sometimes accept incorrect certificates -- we define and use the type below instead of X520countryName + -- We accept utf8String encoding of the US-ASCII + -- country name code and the mix up with other country code systems + -- that uses three characters instead of two. + OTP-X520countryname ::= CHOICE { - printableString PrintableString (SIZE (2)), - utf8String UTF8String (SIZE (2)) + printableString PrintableString (SIZE (2..3)), + utf8String UTF8String (SIZE (2..3)) } serialNumber ATTRIBUTE-TYPE-AND-VALUE-CLASS ::= { diff --git a/lib/public_key/doc/src/notes.xml b/lib/public_key/doc/src/notes.xml index b3e6023c41..f6bc0dc797 100644 --- a/lib/public_key/doc/src/notes.xml +++ b/lib/public_key/doc/src/notes.xml @@ -35,6 +35,21 @@ <file>notes.xml</file> </header> +<section><title>Public_Key 1.6.5</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Add export of dialyzer type</p> + <p> + Own Id: OTP-15624</p> + </item> + </list> + </section> + +</section> + <section><title>Public_Key 1.6.4</title> <section><title>Improvements and New Features</title> diff --git a/lib/public_key/doc/src/public_key.xml b/lib/public_key/doc/src/public_key.xml index 9fcedf6ef9..fb81ea68a4 100644 --- a/lib/public_key/doc/src/public_key.xml +++ b/lib/public_key/doc/src/public_key.xml @@ -423,7 +423,7 @@ <p>Available options:</p> <taglist> - <tag>{verify_fun, fun()}</tag> + <tag>{verify_fun, {fun(), InitialUserState::term()}</tag> <item> <p>The fun must be defined as:</p> diff --git a/lib/public_key/src/pubkey_pbe.erl b/lib/public_key/src/pubkey_pbe.erl index 806f7c5b0f..e6bcedd1b1 100644 --- a/lib/public_key/src/pubkey_pbe.erl +++ b/lib/public_key/src/pubkey_pbe.erl @@ -42,15 +42,14 @@ encode(Data, Password, "DES-CBC" = Cipher, KeyDevParams) -> {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), crypto:block_encrypt(des_cbc, Key, IV, pbe_pad(Data, KeyDevParams)); - encode(Data, Password, "DES-EDE3-CBC" = Cipher, KeyDevParams) -> {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), <<Key1:8/binary, Key2:8/binary, Key3:8/binary>> = Key, crypto:block_encrypt(des3_cbc, [Key1, Key2, Key3], IV, pbe_pad(Data)); - encode(Data, Password, "RC2-CBC" = Cipher, KeyDevParams) -> {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), crypto:block_encrypt(rc2_cbc, Key, IV, pbe_pad(Data, KeyDevParams)). + %%-------------------------------------------------------------------- -spec decode(binary(), string(), string(), term()) -> binary(). %% @@ -59,21 +58,20 @@ encode(Data, Password, "RC2-CBC" = Cipher, KeyDevParams) -> decode(Data, Password,"DES-CBC"= Cipher, KeyDevParams) -> {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), crypto:block_decrypt(des_cbc, Key, IV, Data); - decode(Data, Password,"DES-EDE3-CBC" = Cipher, KeyDevParams) -> {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), <<Key1:8/binary, Key2:8/binary, Key3:8/binary>> = Key, crypto:block_decrypt(des3_cbc, [Key1, Key2, Key3], IV, Data); - decode(Data, Password,"RC2-CBC"= Cipher, KeyDevParams) -> {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), crypto:block_decrypt(rc2_cbc, Key, IV, Data); +decode(Data, Password,"AES-128-CBC"= Cipher, KeyDevParams) -> + {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), + crypto:block_decrypt(aes_cbc128, Key, IV, Data); +decode(Data, Password,"AES-256-CBC"= Cipher, KeyDevParams) -> + {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), + crypto:block_decrypt(aes_cbc256, Key, IV, Data). -decode(Data, Password,"AES-128-CBC"= Cipher, IV) -> - %% PKCS5_SALT_LEN is 8 bytes - <<Salt:8/binary,_/binary>> = IV, - {Key, _} = password_to_key_and_iv(Password, Cipher, Salt), - crypto:block_decrypt(aes_cbc128, Key, IV, Data). %%-------------------------------------------------------------------- -spec pbdkdf1(string(), iodata(), integer(), atom()) -> binary(). @@ -131,13 +129,15 @@ password_to_key_and_iv(Password, _Cipher, {#'PBEParameter'{salt = Salt, <<Key:8/binary, IV:8/binary, _/binary>> = pbdkdf1(Password, Salt, Count, Hash), {Key, IV}; -password_to_key_and_iv(Password, Cipher, Salt) -> - KeyLen = derived_key_length(Cipher, undefined), +password_to_key_and_iv(Password, Cipher, KeyDevParams) -> + %% PKCS5_SALT_LEN is 8 bytes + <<Salt:8/binary,_/binary>> = KeyDevParams, + KeyLen = derived_key_length(Cipher, undefined), <<Key:KeyLen/binary, _/binary>> = pem_encrypt(<<>>, Password, Salt, ceiling(KeyLen div 16), <<>>, md5), %% Old PEM encryption does not use standard encryption method - %% pbdkdf1 and uses then salt as IV - {Key, Salt}. + %% pbdkdf1 + {Key, KeyDevParams}. pem_encrypt(_, _, _, 0, Acc, _) -> Acc; pem_encrypt(Prev, Password, Salt, Count, Acc, Hash) -> @@ -267,7 +267,9 @@ derived_key_length(Cipher,_) when (Cipher == ?'des-EDE3-CBC') or (Cipher == "DES-EDE3-CBC") -> 24; derived_key_length(Cipher,_) when (Cipher == "AES-128-CBC") -> - 16. + 16; +derived_key_length(Cipher,_) when (Cipher == "AES-256-CBC") -> + 32. cipher(#'PBES2-params_encryptionScheme'{algorithm = ?'desCBC'}) -> "DES-CBC"; diff --git a/lib/public_key/src/pubkey_pem.erl b/lib/public_key/src/pubkey_pem.erl index d7e5bc3ad8..0fd1453f7c 100644 --- a/lib/public_key/src/pubkey_pem.erl +++ b/lib/public_key/src/pubkey_pem.erl @@ -101,10 +101,10 @@ encode_pem_entry({'PrivateKeyInfo', Der, EncParams}) -> EncDer = encode_encrypted_private_keyinfo(Der, EncParams), StartStr = pem_start('EncryptedPrivateKeyInfo'), [StartStr, "\n", b64encode_and_split(EncDer), "\n", pem_end(StartStr) ,"\n\n"]; -encode_pem_entry({Type, Der, {Cipher, Salt}}) -> +encode_pem_entry({Type, Decrypted, {Cipher, Salt}}) -> StartStr = pem_start(Type), [StartStr,"\n", pem_decrypt(),"\n", pem_decrypt_info(Cipher, Salt),"\n\n", - b64encode_and_split(Der), "\n", pem_end(StartStr) ,"\n\n"]. + b64encode_and_split(Decrypted), "\n", pem_end(StartStr) ,"\n\n"]. decode_pem_entries([], Entries) -> lists:reverse(Entries); diff --git a/lib/public_key/test/pbe_SUITE.erl b/lib/public_key/test/pbe_SUITE.erl index 523c9e2515..1136267411 100644 --- a/lib/public_key/test/pbe_SUITE.erl +++ b/lib/public_key/test/pbe_SUITE.erl @@ -37,7 +37,7 @@ all() -> [ pbdkdf1, pbdkdf2, - old_enc, + old_pbe, pbes1, pbes2]. @@ -197,23 +197,11 @@ pbdkdf2(Config) when is_list(Config) -> = pubkey_pbe:pbdkdf2("pass\0word", "sa\0lt", 4096, 16, fun crypto:hmac/4, sha, 20). -old_enc() -> - [{doc,"Tests encode/decode RSA key encrypted with different ciphers using old PEM encryption scheme"}]. -old_enc(Config) when is_list(Config) -> - Datadir = proplists:get_value(data_dir, Config), - %% key generated with ssh-keygen -N hello_aes -f old_aes_128_cbc_enc_key.pem - {ok, PemAesCbc} = file:read_file(filename:join(Datadir, "old_aes_128_cbc_enc_key.pem")), - - PemAesCbcEntry = public_key:pem_decode(PemAesCbc), - ct:print("Pem entry: ~p" , [PemAesCbcEntry]), - [{'RSAPrivateKey', _, {"AES-128-CBC",_}} = PubAesCbcEntry] = PemAesCbcEntry, - #'RSAPrivateKey'{} = public_key:pem_entry_decode(PubAesCbcEntry, "hello_aes"). - pbes1() -> [{doc,"Tests encode/decode EncryptedPrivateKeyInfo encrypted with different ciphers using PBES1"}]. pbes1(Config) when is_list(Config) -> decode_encode_key_file("pbes1_des_cbc_md5_enc_key.pem", "password", "DES-CBC", Config). - + pbes2() -> [{doc,"Tests encode/decode EncryptedPrivateKeyInfo encrypted with different ciphers using PBES2"}]. pbes2(Config) when is_list(Config) -> @@ -225,13 +213,33 @@ pbes2(Config) when is_list(Config) -> false -> ok end. +old_pbe() -> + [{doc,"Tests encode/decode with old format used before PBE"}]. +old_pbe(Config) when is_list(Config) -> + Datadir = proplists:get_value(data_dir, Config), + % key generated with ssh-keygen -N hello_aes -f old_aes_128_cbc.pem + {ok, PemAes128Cbc} = file:read_file(filename:join(Datadir, "old_aes_128_cbc.pem")), + + PemAes128CbcEntries = public_key:pem_decode(PemAes128Cbc), + ct:print("Pem entry: ~p" , [PemAes128CbcEntries]), + [{'RSAPrivateKey', _, {"AES-128-CBC",_}} = Aes128CbcEntry] = PemAes128CbcEntries, + #'RSAPrivateKey'{} = Key = public_key:pem_entry_decode(Aes128CbcEntry, "hello_aes"), + + %% Converted with openssl rsa -in old_aes_128_cbc.pem -out old_aes_256_cbc.pem -aes256 + {ok, PemAes256Cbc} = file:read_file(filename:join(Datadir, "old_aes_256_cbc.pem")), + + PemAes256CbcEntries = public_key:pem_decode(PemAes256Cbc), + ct:print("Pem entry: ~p" , [PemAes256CbcEntries]), + [{'RSAPrivateKey', _, {"AES-256-CBC",_}} = Aes256CbcEntry] = PemAes256CbcEntries, + Key = public_key:pem_entry_decode(Aes256CbcEntry, "hello_aes"). + decode_encode_key_file(File, Password, Cipher, Config) -> Datadir = proplists:get_value(data_dir, Config), {ok, PemKey} = file:read_file(filename:join(Datadir, File)), PemEntry = public_key:pem_decode(PemKey), - ct:print("Pem entry: ~p" , [PemEntry]), + ct:pal("Pem entry: ~p" , [PemEntry]), [{Asn1Type, _, {Cipher,_} = CipherInfo} = PubEntry] = PemEntry, #'RSAPrivateKey'{} = KeyInfo = public_key:pem_entry_decode(PubEntry, Password), PemKey1 = public_key:pem_encode([public_key:pem_entry_encode(Asn1Type, KeyInfo, {CipherInfo, Password})]), diff --git a/lib/public_key/test/pbe_SUITE_data/old_aes_128_cbc_enc_key.pem b/lib/public_key/test/pbe_SUITE_data/old_aes_128_cbc.pem index 34c7543f30..34c7543f30 100644 --- a/lib/public_key/test/pbe_SUITE_data/old_aes_128_cbc_enc_key.pem +++ b/lib/public_key/test/pbe_SUITE_data/old_aes_128_cbc.pem diff --git a/lib/public_key/test/pbe_SUITE_data/old_aes_256_cbc.pem b/lib/public_key/test/pbe_SUITE_data/old_aes_256_cbc.pem new file mode 100644 index 0000000000..e6aec2869d --- /dev/null +++ b/lib/public_key/test/pbe_SUITE_data/old_aes_256_cbc.pem @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-256-CBC,ABDA22398E511E9465983E1A50706044 + +XhIcPOb6pTWL++pgeeTH5rsx0tackhllVqyXyOfbYMBJnVFRhQ/V/1MDg3Jt4wD1 +Nerhcv5srHeiwmf+vwXwDFOzvFzLVM1jFMUJe/2XloYFX4TBiLZAF/zekQA3uPY6 +DKJuBuO5vVSZ0VlxGpu3jphIAwxbssfZkZmryCP3b1/oX5Y3Em/wEWW3RduaeWFu +Z2nTsPH3yNmHkuqFF3cq0aZs+VxtjcuYo0gbkN5hNVgOoOVxIBzw7DsgBAkdXvr3 +LRCMGg7Y2pVthA963s0xkN37XtZEiYbydoLnzHlW/Kx5QSvED8/Fn94Lcdy5NsQN +7nYWWgYZRH39Wnsi0BrTZv399U5rBe7DnpStKWPn2Sa8Bdu3CX2oLajM1cZjAp1X +y6vuasK7SoZ8rWpcsHQpV1HyNBTl/uRU5nrYc/KGD8l2x7tdNidGx2Hey/6O4H5v +rxL1TB4PlzDYwCsReuyLbEjyuZQ10j0SK/SFzikHuvD2IEpM/oUaPdVFqcW8/sjw +VhcsupAf4EXfsi2dJBylmDjI2W7h9XwBDLzQN+69DtkBmvimE5CguTITf8MAHQQ6 +Q1vYF2ij7W675tj9ulksRPaMxSsI03luai/Jrieh0mPqGEenGEEC5QU0XPOvsfyw +GMYaBUbdYrMpbHM3wFPxM2wRlXf4gX5BhZKRGZX7OaEs54pfglyQtTPuZmD0VcAp +EWHq70G9mbuBlhbMX1rKAomuDmIvgLeLRUpAFf59Qr8KoJSLD1S8TJB3uEPk6i39 +4GnylbpmqS4gv/OIc6WTeOeUZTAD3A77HBwSlELPk3/s1d/MLyfciYClOBEuZ6NB +FXEKCGCEC786zJA678gLEaa2XPNkEM+2gjzNFqtYMIn5ehAq/HRRsFvW+wkTbee4 +z+qe5HbVKAQ3EOfbidvYrDaGd7HvHVG8zosl+O61iIFs04lLEMDFXBIdvIgEncOK +Rq3yXdpBKMg89aoZLniaPobSvuvdjNOMzW6EKlb5FKZduCiR68MEZ+rLHYHTwE3W +Z5+TCbrbV2F6WQpq3zqnB14wGu8igEb5Veq+N2vMkx4iTMTUyCty1SwIjj4NidM1 +dJM7Ighdal6tQ6hIwbDfpIPsY4eGH/UrdVZ0SkxuDR2s76cZ8nFX3lJ/BNwTZLKo +IqAC4NjUOv3ID+0Q6Lz+sxLCi5pLYUf0E+s4pgi1BYAOu+BF3GwxyqnqVoq/Fs5D +LXxuY0946YM+WcrYzke4mq3MPx6QQYj04H5KJ2mzxtnbZJrfLF23PVRVhvgKSjyV +I3/zgJ16fV2H/fb26oCpTNbb11pQvhorkLwdvpwtM+go7dJGebAi1762Nbj/CqnW +fbBPxPRvNPZn6pEodJ/L/APhvGv1K7eC9THj66H7Kmeoq8Lz74idhywP9I3QS0ZO +15ORbTDjuiRYNJPxxu79A3/tWMUlprJ9ljhI/0DXRB0M3UGic52D/32Q64I7eewy +qRNS/3C3ejDShIRBDFTdDkM3s/42LySXJjmjU9bpZY4POQ3kOaJb3EzSvbzTyXzu +3FiHvDQY+b8XwbxtE/kTMaAPQZ7TtWOao7SRi7J94MvCQ5/tbakFP2suM8psnigC +-----END RSA PRIVATE KEY----- diff --git a/lib/public_key/vsn.mk b/lib/public_key/vsn.mk index 5e2643f0ea..11c06fb158 100644 --- a/lib/public_key/vsn.mk +++ b/lib/public_key/vsn.mk @@ -1 +1 @@ -PUBLIC_KEY_VSN = 1.6.4 +PUBLIC_KEY_VSN = 1.6.5 diff --git a/lib/runtime_tools/doc/src/notes.xml b/lib/runtime_tools/doc/src/notes.xml index 810bb8207c..58a2a66c4b 100644 --- a/lib/runtime_tools/doc/src/notes.xml +++ b/lib/runtime_tools/doc/src/notes.xml @@ -32,6 +32,21 @@ <p>This document describes the changes made to the Runtime_Tools application.</p> +<section><title>Runtime_Tools 1.13.2</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Update of systemtap trace example scripts.</p> + <p> + Own Id: OTP-15670</p> + </item> + </list> + </section> + +</section> + <section><title>Runtime_Tools 1.13.1</title> <section><title>Improvements and New Features</title> diff --git a/lib/runtime_tools/vsn.mk b/lib/runtime_tools/vsn.mk index aa3d702997..fa2f338ec2 100644 --- a/lib/runtime_tools/vsn.mk +++ b/lib/runtime_tools/vsn.mk @@ -1 +1 @@ -RUNTIME_TOOLS_VSN = 1.13.1 +RUNTIME_TOOLS_VSN = 1.13.2 diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index 0bc4baf5eb..17f14bdea2 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -30,6 +30,23 @@ <file>notes.xml</file> </header> +<section><title>Ssh 4.7.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + SSH sftp daemon now accepts an SSH_FXP_STAT message + encoded according to the wrong sftp version. Some clients + sends such messages.</p> + <p> + Own Id: OTP-15498 Aux Id: ERL-822, PR-2077 </p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.7.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssh/src/ssh_transport.erl b/lib/ssh/src/ssh_transport.erl index 9ff20454cd..1f4e281a30 100644 --- a/lib/ssh/src/ssh_transport.erl +++ b/lib/ssh/src/ssh_transport.erl @@ -162,14 +162,14 @@ supported_algorithms(cipher) -> select_crypto_supported( [ {'[email protected]', [{ciphers,chacha20}, {macs,poly1305}]}, - {'[email protected]', [{ciphers,{aes_gcm,256}}]}, - {'aes256-ctr', [{ciphers,{aes_ctr,256}}]}, - {'aes192-ctr', [{ciphers,{aes_ctr,192}}]}, - {'[email protected]', [{ciphers,{aes_gcm,128}}]}, - {'aes128-ctr', [{ciphers,{aes_ctr,128}}]}, - {'AEAD_AES_256_GCM', [{ciphers,{aes_gcm,256}}]}, - {'AEAD_AES_128_GCM', [{ciphers,{aes_gcm,128}}]}, - {'aes128-cbc', [{ciphers,aes_cbc128}]}, + {'[email protected]', [{ciphers,aes_256_gcm}]}, + {'aes256-ctr', [{ciphers,aes_256_ctr}]}, + {'aes192-ctr', [{ciphers,aes_192_ctr}]}, + {'[email protected]', [{ciphers,aes_128_gcm}]}, + {'aes128-ctr', [{ciphers,aes_128_ctr}]}, + {'AEAD_AES_256_GCM', [{ciphers,aes_256_gcm}]}, + {'AEAD_AES_128_GCM', [{ciphers,aes_128_gcm}]}, + {'aes128-cbc', [{ciphers,aes_128_cbc}]}, {'3des-cbc', [{ciphers,des3_cbc}]} ] )); @@ -179,8 +179,8 @@ supported_algorithms(mac) -> [{'hmac-sha2-256', [{macs,hmac}, {hashs,sha256}]}, {'hmac-sha2-512', [{macs,hmac}, {hashs,sha512}]}, {'hmac-sha1', [{macs,hmac}, {hashs,sha}]}, - {'AEAD_AES_128_GCM', [{ciphers,{aes_gcm,128}}]}, - {'AEAD_AES_256_GCM', [{ciphers,{aes_gcm,256}}]} + {'AEAD_AES_128_GCM', [{ciphers,aes_128_gcm}]}, + {'AEAD_AES_256_GCM', [{ciphers,aes_256_gcm}]} ] )); supported_algorithms(compression) -> @@ -1256,11 +1256,6 @@ get_length(aead, EncryptedBuffer, Ssh) -> end. -pkt_type('AEAD_AES_128_GCM') -> aead; -pkt_type('AEAD_AES_256_GCM') -> aead; -pkt_type('[email protected]') -> aead; -pkt_type(_) -> common. - payload(<<PacketLen:32, PaddingLen:8, PayloadAndPadding/binary>>) -> PayloadLen = PacketLen - PaddingLen - 1, <<Payload:PayloadLen/binary, _/binary>> = PayloadAndPadding, @@ -1323,162 +1318,115 @@ verify(PlainText, HashAlg, Sig, Key, _) -> %%% Unit: bytes --record(cipher_data, { - key_bytes, - iv_bytes, - block_bytes - }). +-record(cipher, { + impl, + key_bytes, + iv_bytes, + block_bytes, + pkt_type = common + }). %%% Start of a more parameterized crypto handling. cipher('AEAD_AES_128_GCM') -> - #cipher_data{key_bytes = 16, - iv_bytes = 12, - block_bytes = 16}; + #cipher{key_bytes = 16, + iv_bytes = 12, + block_bytes = 16, + pkt_type = aead}; cipher('AEAD_AES_256_GCM') -> - #cipher_data{key_bytes = 32, - iv_bytes = 12, - block_bytes = 16}; + #cipher{key_bytes = 32, + iv_bytes = 12, + block_bytes = 16, + pkt_type = aead}; cipher('3des-cbc') -> - #cipher_data{key_bytes = 24, - iv_bytes = 8, - block_bytes = 8}; + #cipher{impl = des3_cbc, + key_bytes = 24, + iv_bytes = 8, + block_bytes = 8}; cipher('aes128-cbc') -> - #cipher_data{key_bytes = 16, - iv_bytes = 16, - block_bytes = 16}; + #cipher{impl = aes_cbc, + key_bytes = 16, + iv_bytes = 16, + block_bytes = 16}; cipher('aes128-ctr') -> - #cipher_data{key_bytes = 16, - iv_bytes = 16, - block_bytes = 16}; + #cipher{impl = aes_128_ctr, + key_bytes = 16, + iv_bytes = 16, + block_bytes = 16}; cipher('aes192-ctr') -> - #cipher_data{key_bytes = 24, - iv_bytes = 16, - block_bytes = 16}; + #cipher{impl = aes_192_ctr, + key_bytes = 24, + iv_bytes = 16, + block_bytes = 16}; cipher('aes256-ctr') -> - #cipher_data{key_bytes = 32, - iv_bytes = 16, - block_bytes = 16}; + #cipher{impl = aes_256_ctr, + key_bytes = 32, + iv_bytes = 16, + block_bytes = 16}; cipher('[email protected]') -> % FIXME: Verify!! - #cipher_data{key_bytes = 32, - iv_bytes = 12, - block_bytes = 8}. - + #cipher{key_bytes = 32, + iv_bytes = 12, + block_bytes = 8, + pkt_type = aead}; + +cipher(_) -> + #cipher{}. + + +pkt_type(SshCipher) -> (cipher(SshCipher))#cipher.pkt_type. + +decrypt_magic(server) -> {"A", "C"}; +decrypt_magic(client) -> {"B", "D"}. + +encrypt_magic(client) -> decrypt_magic(server); +encrypt_magic(server) -> decrypt_magic(client). + encrypt_init(#ssh{encrypt = none} = Ssh) -> {ok, Ssh}; -encrypt_init(#ssh{encrypt = '[email protected]', role = client} = Ssh) -> + +encrypt_init(#ssh{encrypt = '[email protected]', role = Role} = Ssh) -> %% [email protected] uses two independent crypto streams, one (chacha20) %% for the length used in stream mode, and the other (chacha20-poly1305) as AEAD for %% the payload and to MAC the length||payload. %% See draft-josefsson-ssh-chacha20-poly1305-openssh-00 - <<K2:32/binary,K1:32/binary>> = hash(Ssh, "C", 512), + {_, KeyMagic} = encrypt_magic(Role), + <<K2:32/binary,K1:32/binary>> = hash(Ssh, KeyMagic, 8*64), {ok, Ssh#ssh{encrypt_keys = {K1,K2} % encrypt_block_size = 16, %default = 8. What to set it to? 64 (openssl chacha.h) % ctx and iv is setup for each packet }}; -encrypt_init(#ssh{encrypt = '[email protected]', role = server} = Ssh) -> - <<K2:32/binary,K1:32/binary>> = hash(Ssh, "D", 512), - {ok, Ssh#ssh{encrypt_keys = {K1,K2} - % encrypt_block_size = 16, %default = 8. What to set it to? - }}; -encrypt_init(#ssh{encrypt = 'AEAD_AES_128_GCM', role = client} = Ssh) -> - IV = hash(Ssh, "A", 12*8), - <<K:16/binary>> = hash(Ssh, "C", 128), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = 'AEAD_AES_128_GCM', role = server} = Ssh) -> - IV = hash(Ssh, "B", 12*8), - <<K:16/binary>> = hash(Ssh, "D", 128), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = 'AEAD_AES_256_GCM', role = client} = Ssh) -> - IV = hash(Ssh, "A", 12*8), - <<K:32/binary>> = hash(Ssh, "C", 256), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = 'AEAD_AES_256_GCM', role = server} = Ssh) -> - IV = hash(Ssh, "B", 12*8), - <<K:32/binary>> = hash(Ssh, "D", 256), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = '3des-cbc', role = client} = Ssh) -> - IV = hash(Ssh, "A", 64), - <<K1:8/binary, K2:8/binary, K3:8/binary>> = hash(Ssh, "C", 192), - {ok, Ssh#ssh{encrypt_keys = {K1,K2,K3}, - encrypt_block_size = 8, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = '3des-cbc', role = server} = Ssh) -> - IV = hash(Ssh, "B", 64), - <<K1:8/binary, K2:8/binary, K3:8/binary>> = hash(Ssh, "D", 192), - {ok, Ssh#ssh{encrypt_keys = {K1,K2,K3}, - encrypt_block_size = 8, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = 'aes128-cbc', role = client} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:16/binary>> = hash(Ssh, "C", 128), + +encrypt_init(#ssh{encrypt = SshCipher, role = Role} = Ssh) when SshCipher == 'AEAD_AES_128_GCM'; + SshCipher == 'AEAD_AES_256_GCM' -> + {IvMagic, KeyMagic} = encrypt_magic(Role), + #cipher{key_bytes = KeyBytes, + iv_bytes = IvBytes, + block_bytes = BlockBytes} = cipher(SshCipher), + IV = hash(Ssh, IvMagic, 8*IvBytes), + K = hash(Ssh, KeyMagic, 8*KeyBytes), {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, + encrypt_block_size = BlockBytes, encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = 'aes128-cbc', role = server} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:16/binary>> = hash(Ssh, "D", 128), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = 'aes128-ctr', role = client} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:16/binary>> = hash(Ssh, "C", 128), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = State}}; -encrypt_init(#ssh{encrypt = 'aes192-ctr', role = client} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:24/binary>> = hash(Ssh, "C", 192), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = State}}; -encrypt_init(#ssh{encrypt = 'aes256-ctr', role = client} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:32/binary>> = hash(Ssh, "C", 256), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = State}}; -encrypt_init(#ssh{encrypt = 'aes128-ctr', role = server} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:16/binary>> = hash(Ssh, "D", 128), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = State}}; -encrypt_init(#ssh{encrypt = 'aes192-ctr', role = server} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:24/binary>> = hash(Ssh, "D", 192), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = State}}; -encrypt_init(#ssh{encrypt = 'aes256-ctr', role = server} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:32/binary>> = hash(Ssh, "D", 256), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = State}}. + +encrypt_init(#ssh{encrypt = SshCipher, role = Role} = Ssh) -> + {IvMagic, KeyMagic} = encrypt_magic(Role), + #cipher{impl = CryptoCipher, + key_bytes = KeyBytes, + iv_bytes = IvBytes, + block_bytes = BlockBytes} = cipher(SshCipher), + IV = hash(Ssh, IvMagic, 8*IvBytes), + K = hash(Ssh, KeyMagic, 8*KeyBytes), + Ctx0 = crypto:crypto_init(CryptoCipher, K, IV, true), + {ok, Ssh#ssh{encrypt_block_size = BlockBytes, + encrypt_ctx = Ctx0}}. encrypt_final(Ssh) -> {ok, Ssh#ssh{encrypt = none, @@ -1487,249 +1435,126 @@ encrypt_final(Ssh) -> encrypt_ctx = undefined }}. + encrypt(#ssh{encrypt = none} = Ssh, Data) -> {Ssh, Data}; + encrypt(#ssh{encrypt = '[email protected]', encrypt_keys = {K1,K2}, send_sequence = Seq} = Ssh, <<LenData:4/binary, PayloadData/binary>>) -> %% Encrypt length IV1 = <<0:8/unit:8, Seq:8/unit:8>>, - {_,EncLen} = crypto:stream_encrypt(crypto:stream_init(chacha20, K1, IV1), - LenData), + EncLen = crypto:crypto_one_shot(chacha20, K1, IV1, LenData, true), %% Encrypt payload IV2 = <<1:8/little-unit:8, Seq:8/unit:8>>, - {_,EncPayloadData} = crypto:stream_encrypt(crypto:stream_init(chacha20, K2, IV2), - PayloadData), - + EncPayloadData = crypto:crypto_one_shot(chacha20, K2, IV2, PayloadData, true), %% MAC tag - {_,PolyKey} = crypto:stream_encrypt(crypto:stream_init(chacha20, K2, <<0:8/unit:8,Seq:8/unit:8>>), - <<0:32/unit:8>>), + PolyKey = crypto:crypto_one_shot(chacha20, K2, <<0:8/unit:8,Seq:8/unit:8>>, <<0:32/unit:8>>, true), EncBytes = <<EncLen/binary,EncPayloadData/binary>>, Ctag = crypto:poly1305(PolyKey, EncBytes), %% Result {Ssh, {EncBytes,Ctag}}; -encrypt(#ssh{encrypt = 'AEAD_AES_128_GCM', - encrypt_keys = K, + +encrypt(#ssh{encrypt = SshCipher, + encrypt_keys = K, encrypt_ctx = IV0} = Ssh, - <<LenData:4/binary, PayloadData/binary>>) -> - {Ctext,Ctag} = crypto:block_encrypt(aes_gcm, K, IV0, {LenData,PayloadData}), - IV = next_gcm_iv(IV0), - {Ssh#ssh{encrypt_ctx = IV}, {<<LenData/binary,Ctext/binary>>,Ctag}}; -encrypt(#ssh{encrypt = 'AEAD_AES_256_GCM', - encrypt_keys = K, - encrypt_ctx = IV0} = Ssh, - <<LenData:4/binary, PayloadData/binary>>) -> + <<LenData:4/binary, PayloadData/binary>>) when SshCipher == 'AEAD_AES_128_GCM' ; + SshCipher == 'AEAD_AES_256_GCM' -> {Ctext,Ctag} = crypto:block_encrypt(aes_gcm, K, IV0, {LenData,PayloadData}), IV = next_gcm_iv(IV0), {Ssh#ssh{encrypt_ctx = IV}, {<<LenData/binary,Ctext/binary>>,Ctag}}; -encrypt(#ssh{encrypt = '3des-cbc', - encrypt_keys = {K1,K2,K3}, - encrypt_ctx = IV0} = Ssh, Data) -> - Enc = crypto:block_encrypt(des3_cbc, [K1,K2,K3], IV0, Data), - IV = crypto:next_iv(des3_cbc, Enc), - {Ssh#ssh{encrypt_ctx = IV}, Enc}; -encrypt(#ssh{encrypt = 'aes128-cbc', - encrypt_keys = K, - encrypt_ctx = IV0} = Ssh, Data) -> - Enc = crypto:block_encrypt(aes_cbc128, K,IV0,Data), - IV = crypto:next_iv(aes_cbc, Enc), - {Ssh#ssh{encrypt_ctx = IV}, Enc}; -encrypt(#ssh{encrypt = 'aes128-ctr', - encrypt_ctx = State0} = Ssh, Data) -> - {State, Enc} = crypto:stream_encrypt(State0,Data), - {Ssh#ssh{encrypt_ctx = State}, Enc}; -encrypt(#ssh{encrypt = 'aes192-ctr', - encrypt_ctx = State0} = Ssh, Data) -> - {State, Enc} = crypto:stream_encrypt(State0,Data), - {Ssh#ssh{encrypt_ctx = State}, Enc}; -encrypt(#ssh{encrypt = 'aes256-ctr', - encrypt_ctx = State0} = Ssh, Data) -> - {State, Enc} = crypto:stream_encrypt(State0,Data), - {Ssh#ssh{encrypt_ctx = State}, Enc}. - +encrypt(#ssh{encrypt_ctx = Ctx0} = Ssh, Data) -> + Enc = crypto:crypto_update(Ctx0, Data), + {Ssh, Enc}. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Decryption %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% decrypt_init(#ssh{decrypt = none} = Ssh) -> {ok, Ssh}; -decrypt_init(#ssh{decrypt = '[email protected]', role = client} = Ssh) -> - <<K2:32/binary,K1:32/binary>> = hash(Ssh, "D", 512), - {ok, Ssh#ssh{decrypt_keys = {K1,K2} - }}; -decrypt_init(#ssh{decrypt = '[email protected]', role = server} = Ssh) -> - <<K2:32/binary,K1:32/binary>> = hash(Ssh, "C", 512), + +decrypt_init(#ssh{decrypt = '[email protected]', role = Role} = Ssh) -> + {_, KeyMagic} = decrypt_magic(Role), + <<K2:32/binary,K1:32/binary>> = hash(Ssh, KeyMagic, 8*64), {ok, Ssh#ssh{decrypt_keys = {K1,K2} }}; -decrypt_init(#ssh{decrypt = 'AEAD_AES_128_GCM', role = client} = Ssh) -> - IV = hash(Ssh, "B", 12*8), - <<K:16/binary>> = hash(Ssh, "D", 128), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = IV}}; -decrypt_init(#ssh{decrypt = 'AEAD_AES_128_GCM', role = server} = Ssh) -> - IV = hash(Ssh, "A", 12*8), - <<K:16/binary>> = hash(Ssh, "C", 128), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = IV}}; -decrypt_init(#ssh{decrypt = 'AEAD_AES_256_GCM', role = client} = Ssh) -> - IV = hash(Ssh, "B", 12*8), - <<K:32/binary>> = hash(Ssh, "D", 256), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = IV}}; -decrypt_init(#ssh{decrypt = 'AEAD_AES_256_GCM', role = server} = Ssh) -> - IV = hash(Ssh, "A", 12*8), - <<K:32/binary>> = hash(Ssh, "C", 256), + +decrypt_init(#ssh{decrypt = SshCipher, role = Role} = Ssh) when SshCipher == 'AEAD_AES_128_GCM'; + SshCipher == 'AEAD_AES_256_GCM' -> + {IvMagic, KeyMagic} = decrypt_magic(Role), + #cipher{key_bytes = KeyBytes, + iv_bytes = IvBytes, + block_bytes = BlockBytes} = cipher(SshCipher), + IV = hash(Ssh, IvMagic, 8*IvBytes), + K = hash(Ssh, KeyMagic, 8*KeyBytes), {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, + decrypt_block_size = BlockBytes, decrypt_ctx = IV}}; -decrypt_init(#ssh{decrypt = '3des-cbc', role = client} = Ssh) -> - {IV, KD} = {hash(Ssh, "B", 64), - hash(Ssh, "D", 192)}, - <<K1:8/binary, K2:8/binary, K3:8/binary>> = KD, - {ok, Ssh#ssh{decrypt_keys = {K1,K2,K3}, decrypt_ctx = IV, - decrypt_block_size = 8}}; -decrypt_init(#ssh{decrypt = '3des-cbc', role = server} = Ssh) -> - {IV, KD} = {hash(Ssh, "A", 64), - hash(Ssh, "C", 192)}, - <<K1:8/binary, K2:8/binary, K3:8/binary>> = KD, - {ok, Ssh#ssh{decrypt_keys = {K1, K2, K3}, decrypt_ctx = IV, - decrypt_block_size = 8}}; -decrypt_init(#ssh{decrypt = 'aes128-cbc', role = client} = Ssh) -> - {IV, KD} = {hash(Ssh, "B", 128), - hash(Ssh, "D", 128)}, - <<K:16/binary>> = KD, - {ok, Ssh#ssh{decrypt_keys = K, decrypt_ctx = IV, - decrypt_block_size = 16}}; -decrypt_init(#ssh{decrypt = 'aes128-cbc', role = server} = Ssh) -> - {IV, KD} = {hash(Ssh, "A", 128), - hash(Ssh, "C", 128)}, - <<K:16/binary>> = KD, - {ok, Ssh#ssh{decrypt_keys = K, decrypt_ctx = IV, - decrypt_block_size = 16}}; -decrypt_init(#ssh{decrypt = 'aes128-ctr', role = client} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:16/binary>> = hash(Ssh, "D", 128), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = State}}; -decrypt_init(#ssh{decrypt = 'aes192-ctr', role = client} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:24/binary>> = hash(Ssh, "D", 192), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = State}}; -decrypt_init(#ssh{decrypt = 'aes256-ctr', role = client} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:32/binary>> = hash(Ssh, "D", 256), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = State}}; -decrypt_init(#ssh{decrypt = 'aes128-ctr', role = server} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:16/binary>> = hash(Ssh, "C", 128), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = State}}; -decrypt_init(#ssh{decrypt = 'aes192-ctr', role = server} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:24/binary>> = hash(Ssh, "C", 192), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = State}}; -decrypt_init(#ssh{decrypt = 'aes256-ctr', role = server} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:32/binary>> = hash(Ssh, "C", 256), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = State}}. - +decrypt_init(#ssh{decrypt = SshCipher, role = Role} = Ssh) -> + {IvMagic, KeyMagic} = decrypt_magic(Role), + #cipher{impl = CryptoCipher, + key_bytes = KeyBytes, + iv_bytes = IvBytes, + block_bytes = BlockBytes} = cipher(SshCipher), + IV = hash(Ssh, IvMagic, 8*IvBytes), + K = hash(Ssh, KeyMagic, 8*KeyBytes), + Ctx0 = crypto:crypto_init(CryptoCipher, K, IV, false), + {ok, Ssh#ssh{decrypt_block_size = BlockBytes, + decrypt_ctx = Ctx0}}. + decrypt_final(Ssh) -> {ok, Ssh#ssh {decrypt = none, decrypt_keys = undefined, decrypt_ctx = undefined, decrypt_block_size = 8}}. + decrypt(Ssh, <<>>) -> {Ssh, <<>>}; + decrypt(#ssh{decrypt = '[email protected]', decrypt_keys = {K1,_K2}, recv_sequence = Seq} = Ssh, {length,EncryptedLen}) -> - {_State,PacketLenBin} = - crypto:stream_decrypt(crypto:stream_init(chacha20, K1, <<0:8/unit:8, Seq:8/unit:8>>), - EncryptedLen), + PacketLenBin = crypto:crypto_one_shot(chacha20, K1, <<0:8/unit:8, Seq:8/unit:8>>, EncryptedLen, false), {Ssh, PacketLenBin}; + decrypt(#ssh{decrypt = '[email protected]', decrypt_keys = {_K1,K2}, recv_sequence = Seq} = Ssh, {AAD,Ctext,Ctag}) -> %% The length is already decoded and used to divide the input %% Check the mac (important that it is timing-safe): - {_,PolyKey} = - crypto:stream_encrypt(crypto:stream_init(chacha20, K2, <<0:8/unit:8,Seq:8/unit:8>>), - <<0:32/unit:8>>), + PolyKey = crypto:crypto_one_shot(chacha20, K2, <<0:8/unit:8,Seq:8/unit:8>>, <<0:32/unit:8>>, false), case equal_const_time(Ctag, crypto:poly1305(PolyKey, <<AAD/binary,Ctext/binary>>)) of true -> %% MAC is ok, decode IV2 = <<1:8/little-unit:8, Seq:8/unit:8>>, - {_,PlainText} = - crypto:stream_decrypt(crypto:stream_init(chacha20,K2,IV2), Ctext), + PlainText = crypto:crypto_one_shot(chacha20, K2, IV2, Ctext, false), {Ssh, PlainText}; false -> {Ssh,error} end; + decrypt(#ssh{decrypt = none} = Ssh, Data) -> {Ssh, Data}; -decrypt(#ssh{decrypt = 'AEAD_AES_128_GCM', - decrypt_keys = K, - decrypt_ctx = IV0} = Ssh, Data = {_AAD,_Ctext,_Ctag}) -> - Dec = crypto:block_decrypt(aes_gcm, K, IV0, Data), % Dec = PlainText | error - IV = next_gcm_iv(IV0), - {Ssh#ssh{decrypt_ctx = IV}, Dec}; -decrypt(#ssh{decrypt = 'AEAD_AES_256_GCM', + +decrypt(#ssh{decrypt = SshCipher, decrypt_keys = K, - decrypt_ctx = IV0} = Ssh, Data = {_AAD,_Ctext,_Ctag}) -> + decrypt_ctx = IV0} = Ssh, Data = {_AAD,_Ctext,_Ctag}) when SshCipher == 'AEAD_AES_128_GCM' ; + SshCipher == 'AEAD_AES_256_GCM' -> Dec = crypto:block_decrypt(aes_gcm, K, IV0, Data), % Dec = PlainText | error IV = next_gcm_iv(IV0), {Ssh#ssh{decrypt_ctx = IV}, Dec}; -decrypt(#ssh{decrypt = '3des-cbc', decrypt_keys = Keys, - decrypt_ctx = IV0} = Ssh, Data) -> - {K1, K2, K3} = Keys, - Dec = crypto:block_decrypt(des3_cbc, [K1,K2,K3], IV0, Data), - IV = crypto:next_iv(des3_cbc, Data), - {Ssh#ssh{decrypt_ctx = IV}, Dec}; -decrypt(#ssh{decrypt = 'aes128-cbc', decrypt_keys = Key, - decrypt_ctx = IV0} = Ssh, Data) -> - Dec = crypto:block_decrypt(aes_cbc128, Key,IV0,Data), - IV = crypto:next_iv(aes_cbc, Data), - {Ssh#ssh{decrypt_ctx = IV}, Dec}; -decrypt(#ssh{decrypt = 'aes128-ctr', - decrypt_ctx = State0} = Ssh, Data) -> - {State, Enc} = crypto:stream_decrypt(State0,Data), - {Ssh#ssh{decrypt_ctx = State}, Enc}; -decrypt(#ssh{decrypt = 'aes192-ctr', - decrypt_ctx = State0} = Ssh, Data) -> - {State, Enc} = crypto:stream_decrypt(State0,Data), - {Ssh#ssh{decrypt_ctx = State}, Enc}; -decrypt(#ssh{decrypt = 'aes256-ctr', - decrypt_ctx = State0} = Ssh, Data) -> - {State, Enc} = crypto:stream_decrypt(State0,Data), - {Ssh#ssh{decrypt_ctx = State}, Enc}. +decrypt(#ssh{decrypt_ctx = Ctx0} = Ssh, Data) -> + Dec = crypto:crypto_update(Ctx0, Data), + {Ssh, Dec}. next_gcm_iv(<<Fixed:32, InvCtr:64>>) -> <<Fixed:32, (InvCtr+1):64>>. - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Compression %% @@ -2058,9 +1883,9 @@ compute_key(Algorithm, OthersPublic, MyPrivate, Args) -> dh_bits(#alg{encrypt = Encrypt, send_mac = SendMac}) -> C = cipher(Encrypt), - 8 * lists:max([C#cipher_data.key_bytes, - C#cipher_data.block_bytes, - C#cipher_data.iv_bytes, + 8 * lists:max([C#cipher.key_bytes, + C#cipher.block_bytes, + C#cipher.iv_bytes, mac_key_bytes(SendMac) ]). @@ -2091,40 +1916,13 @@ select_crypto_supported(L) -> crypto_supported(Conditions, Supported) -> lists:all( fun({Tag,CryptoName}) when is_atom(CryptoName) -> - crypto_name_supported(Tag,CryptoName,Supported); - ({Tag,{Name,Len}}) when is_integer(Len) -> - crypto_name_supported(Tag,Name,Supported) andalso - len_supported(Name,Len) + crypto_name_supported(Tag,CryptoName,Supported) end, Conditions). crypto_name_supported(Tag, CryptoName, Supported) -> - Vs = case proplists:get_value(Tag,Supported,[]) of - [] when Tag == curves -> crypto:ec_curves(); - L -> L - end, + Vs = proplists:get_value(Tag,Supported,[]), lists:member(CryptoName, Vs). -len_supported(Name, Len) -> - try - case Name of - aes_ctr -> - {_, <<_/binary>>} = - %% Test encryption - crypto:stream_encrypt(crypto:stream_init(Name, <<0:Len>>, <<0:128>>), <<"">>); - aes_gcm -> - {<<_/binary>>, <<_/binary>>} = - crypto:block_encrypt(Name, - _Key = <<0:Len>>, - _IV = <<0:12/unsigned-unit:8>>, - {<<"AAD">>,"PT"}) - end - of - _ -> true - catch - _:_ -> false - end. - - same(Algs) -> [{client2server,Algs}, {server2client,Algs}]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/ssh/test/ssh_protocol_SUITE.erl b/lib/ssh/test/ssh_protocol_SUITE.erl index 3e3e151781..b12ddfeef6 100644 --- a/lib/ssh/test/ssh_protocol_SUITE.erl +++ b/lib/ssh/test/ssh_protocol_SUITE.erl @@ -38,7 +38,11 @@ -define(EXTRA_KEX, 'diffie-hellman-group1-sha1'). -define(CIPHERS, ['aes256-ctr','aes192-ctr','aes128-ctr','aes128-cbc','3des-cbc']). --define(DEFAULT_CIPHERS, [{client2server,?CIPHERS}, {server2client,?CIPHERS}]). +-define(DEFAULT_CIPHERS, (fun() -> Ciphs = filter_supported(cipher, ?CIPHERS), + [{client2server,Ciphs}, {server2client,Ciphs}] + end)() + ). + -define(v(Key, Config), proplists:get_value(Key, Config)). -define(v(Key, Config, Default), proplists:get_value(Key, Config, Default)). diff --git a/lib/ssh/vsn.mk b/lib/ssh/vsn.mk index 2890d7fe5b..0f9eee887c 100644 --- a/lib/ssh/vsn.mk +++ b/lib/ssh/vsn.mk @@ -1,4 +1,4 @@ #-*-makefile-*- ; force emacs to enter makefile-mode -SSH_VSN = 4.7.3 +SSH_VSN = 4.7.4 APP_VSN = "ssh-$(SSH_VSN)" diff --git a/lib/ssl/doc/src/notes.xml b/lib/ssl/doc/src/notes.xml index 82eb8ff700..a511cb4db3 100644 --- a/lib/ssl/doc/src/notes.xml +++ b/lib/ssl/doc/src/notes.xml @@ -27,6 +27,117 @@ </header> <p>This document describes the changes made to the SSL application.</p> +<section><title>SSL 9.2.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The timeout for a passive receive was sometimes not + cancelled and later caused a server crash. This bug has + now been corrected.</p> + <p> + Own Id: OTP-14701 Aux Id: ERL-883, ERL-884 </p> + </item> + <item> + <p> + Add tag for passive message (active N) in cb_info to + retain transport transparency.</p> + <p> + Own Id: OTP-15679 Aux Id: ERL-861 </p> + </item> + </list> + </section> + +</section> + +<section><title>SSL 9.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix bug that an incorrect return value for gen_statem + could be created when alert was a result of handling + renegotiation info extension</p> + <p> + Own Id: OTP-15502</p> + </item> + <item> + <p> + Correct check for 3des_ede_cbc, could cause ssl to claim + to support 3des_ede_cbc when cryptolib does not.</p> + <p> + Own Id: OTP-15539</p> + </item> + <item> + <p> + Improved DTLS error handling, avoids unexpected + connection failure in rare cases.</p> + <p> + Own Id: OTP-15561</p> + </item> + <item> + <p> + Corrected active once emulation bug that could cause the + ssl_closed meassage to not be sent. Bug introduced by + OTP-15449</p> + <p> + Own Id: OTP-15666 Aux Id: ERIERL-316, </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Add client option {reuse_session, SessionID::binary()} + that can be used together with new option value + {reuse_sessions, save}. This makes it possible to reuse a + session from a specific connection establishment.</p> + <p> + Own Id: OTP-15369</p> + </item> + <item> + <p> + The Reason part of of the error return from the functions + connect and handshake has a better and documented format. + This will sometimes differ from previous returned + reasons, however those where only documented as term() + and should for that reason not be relied on.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-15423</p> + </item> + <item> + <p> + Refactor of state handling to improve TLS application + data throughput and reduce CPU overhead</p> + <p> + Own Id: OTP-15445</p> + </item> + <item> + <p> + The SSL code has been optimized in many small ways to + reduce CPU load for encryption/decryption, especially for + Erlang's distribution protocol over TLS.</p> + <p> + Own Id: OTP-15529</p> + </item> + <item> + <p> + Add support for active N</p> + <p> + Own Id: OTP-15665 Aux Id: ERL-811, PR-2072 </p> + </item> + </list> + </section> + +</section> + <section><title>SSL 9.1.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index a2dd1f29b7..37bf9033a1 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -89,25 +89,33 @@ <datatype> <name name="active_msgs"/> <desc> - <p>When an TLS/DTLS socket is in active mode (the default), data from the + <p>When a TLS/DTLS socket is in active mode (the default), data from the socket is delivered to the owner of the socket in the form of messages as described above.</p> + <p>The <c>ssl_passive</c> message is sent only when the socket is in + <c>{active, N}</c> mode and the counter dropped to 0. It indicates + that the socket has transitioned to passive (<c>{active, false}</c>) mode.</p> </desc> </datatype> <datatype> <name name="transport_option"/> <desc> - <p>Defaults to <c>{gen_tcp, tcp, tcp_closed, tcp_error}</c> - for TLS and <c>{gen_udp, udp, udp_closed, udp_error}</c> for - DTLS. Can be used to customize the transport layer. The tag - values should be the values used by the underlying transport - in its active mode messages. For TLS the callback module must implement a - reliable transport protocol, behave as <c>gen_tcp</c>, and have functions - corresponding to <c>inet:setopts/2</c>, <c>inet:getopts/2</c>, - <c>inet:peername/1</c>, <c>inet:sockname/1</c>, and <c>inet:port/1</c>. - The callback <c>gen_tcp</c> is treated specially and calls <c>inet</c> - directly. For DTLS this feature must be considered exprimental. + <p>Defaults to <c>{gen_tcp, tcp, tcp_closed, tcp_error, + tcp_passive}</c> for TLS (for backward compatibility a four + tuple will be converted to a five tuple with the last element + "second_element"_passive) and <c>{gen_udp, udp, udp_closed, + udp_error}</c> for DTLS (might also be changed to five tuple in + the future). Can be used to customize the transport layer. The + tag values should be the values used by the underlying + transport in its active mode messages. For TLS the callback + module must implement a reliable transport protocol, behave as + <c>gen_tcp</c>, and have functions corresponding to + <c>inet:setopts/2</c>, <c>inet:getopts/2</c>, + <c>inet:peername/1</c>, <c>inet:sockname/1</c>, and + <c>inet:port/1</c>. The callback <c>gen_tcp</c> is treated + specially and calls <c>inet</c> directly. For DTLS this + feature must be considered exprimental. </p> </desc> </datatype> @@ -1204,8 +1212,8 @@ fun(srp, Username :: string(), UserState :: term()) -> marker="#handshake_cancel-1"><c>handshake_cancel/1</c></seealso>. </p> - <p> If the option <c>active</c> is set to <c>once</c> or <c>true</c> the - process owning the sslsocket will receive messages of type + <p> If the option <c>active</c> is set to <c>once</c>, <c>true</c> or an integer value, + the process owning the sslsocket will receive messages of type <seealso marker="#type-active_msgs"> active_msgs() </seealso> </p> </desc> @@ -1252,8 +1260,8 @@ fun(srp, Username :: string(), UserState :: term()) -> marker="#handshake_cancel-1"><c>handshake_cancel/1</c></seealso>. </p> - <p> If the option <c>active</c> is set to <c>once</c> or <c>true</c> the - process owning the sslsocket will receive messages of type + <p> If the option <c>active</c> is set to <c>once</c>, <c>true</c> or an integer value, + the process owning the sslsocket will receive messages of type <seealso marker="#type-active_msgs"> active_msgs() </seealso> </p> </desc> @@ -1414,8 +1422,8 @@ fun(srp, Username :: string(), UserState :: term()) -> <p>Performs the SSL/TLS/DTLS server-side handshake.</p> <p>Returns a new TLS/DTLS socket if the handshake is successful.</p> - <p> If the option <c>active</c> is set to <c>once</c> or <c>true</c> the - process owning the sslsocket will receive messages of type + <p> If the option <c>active</c> is set to <c>once</c>, <c>true</c> or an integer value, + the process owning the sslsocket will receive messages of type <seealso marker="#type-active_msgs"> active_msgs() </seealso> </p> </desc> @@ -1459,8 +1467,8 @@ fun(srp, Username :: string(), UserState :: term()) -> marker="#handshake_cancel-1"><c>handshake_cancel/1</c></seealso>. </p> - <p> If the option <c>active</c> is set to <c>once</c> or <c>true</c> the - process owning the sslsocket will receive messages of type + <p> If the option <c>active</c> is set to <c>once</c>, <c>true</c> or an integer value, + the process owning the sslsocket will receive messages of type <seealso marker="#type-active_msgs"> active_msgs() </seealso> </p> @@ -1655,17 +1663,6 @@ fun(srp, Username :: string(), UserState :: term()) -> </func> <func> - <name since="OTP 22.0">set_log_level(Level) -> ok | {error, Reason}</name> - <fsummary>Sets log level for the SSL application.</fsummary> - <type> - <v>Level = atom()</v> - </type> - <desc> - <p>Sets log level for the SSL application.</p> - </desc> - </func> - - <func> <name since="OTP R14B">shutdown(SslSocket, How) -> ok | {error, Reason}</name> <fsummary>Immediately closes a socket.</fsummary> <type> diff --git a/lib/ssl/src/dtls_packet_demux.erl b/lib/ssl/src/dtls_packet_demux.erl index afcd4af000..e0423b07b4 100644 --- a/lib/ssl/src/dtls_packet_demux.erl +++ b/lib/ssl/src/dtls_packet_demux.erl @@ -298,6 +298,9 @@ do_set_emulated_opts([], Opts) -> Opts; do_set_emulated_opts([{mode, Value} | Rest], Opts) -> do_set_emulated_opts(Rest, Opts#socket_options{mode = Value}); +do_set_emulated_opts([{active, N0} | Rest], Opts=#socket_options{active = Active}) when is_integer(N0) -> + N = tls_socket:update_active_n(N0, Active), + do_set_emulated_opts(Rest, Opts#socket_options{active = N}); do_set_emulated_opts([{active, Value} | Rest], Opts) -> do_set_emulated_opts(Rest, Opts#socket_options{active = Value}). diff --git a/lib/ssl/src/dtls_socket.erl b/lib/ssl/src/dtls_socket.erl index 2001afd02f..4d07372e31 100644 --- a/lib/ssl/src/dtls_socket.erl +++ b/lib/ssl/src/dtls_socket.erl @@ -38,7 +38,9 @@ listen(Port, #config{transport_info = TransportInfo, case dtls_listener_sup:start_child([Port, TransportInfo, emulated_socket_options(EmOpts, #socket_options{}), Options ++ internal_inet_values(), SslOpts]) of {ok, Pid} -> - {ok, #sslsocket{pid = {dtls, Config#config{dtls_handler = {Pid, Port}}}}}; + Socket = #sslsocket{pid = {dtls, Config#config{dtls_handler = {Pid, Port}}}}, + check_active_n(EmOpts, Socket), + {ok, Socket}; Err = {error, _} -> Err end. @@ -81,8 +83,9 @@ socket(Pids, Transport, Socket, ConnectionCb) -> #sslsocket{pid = Pids, %% "The name "fd" is keept for backwards compatibility fd = {Transport, Socket, ConnectionCb}}. -setopts(_, #sslsocket{pid = {dtls, #config{dtls_handler = {ListenPid, _}}}}, Options) -> - SplitOpts = tls_socket:split_options(Options), +setopts(_, Socket = #sslsocket{pid = {dtls, #config{dtls_handler = {ListenPid, _}}}}, Options) -> + SplitOpts = {_, EmOpts} = tls_socket:split_options(Options), + check_active_n(EmOpts, Socket), dtls_packet_demux:set_sock_opts(ListenPid, SplitOpts); %%% Following clauses will not be called for emulated options, they are handled in the connection process setopts(gen_udp, Socket, Options) -> @@ -90,6 +93,32 @@ setopts(gen_udp, Socket, Options) -> setopts(Transport, Socket, Options) -> Transport:setopts(Socket, Options). +check_active_n(EmulatedOpts, Socket = #sslsocket{pid = {dtls, #config{dtls_handler = {ListenPid, _}}}}) -> + %% We check the resulting options to send an ssl_passive message if necessary. + case proplists:lookup(active, EmulatedOpts) of + %% The provided value is out of bound. + {_, N} when is_integer(N), N < -32768 -> + throw(einval); + {_, N} when is_integer(N), N > 32767 -> + throw(einval); + {_, N} when is_integer(N) -> + {ok, #socket_options{active = Active}, _} = dtls_packet_demux:get_all_opts(ListenPid), + case Active of + Atom when is_atom(Atom), N =< 0 -> + self() ! {ssl_passive, Socket}; + %% The result of the addition is out of bound. + %% We do not need to check < -32768 because Active can't be below 1. + A when is_integer(A), A + N > 32767 -> + throw(einval); + A when is_integer(A), A + N =< 0 -> + self() ! {ssl_passive, Socket}; + _ -> + ok + end; + _ -> + ok + end. + getopts(_, #sslsocket{pid = {dtls, #config{dtls_handler = {ListenPid, _}}}}, Options) -> SplitOpts = tls_socket:split_options(Options), dtls_packet_demux:get_sock_opts(ListenPid, SplitOpts); @@ -161,9 +190,18 @@ emulated_socket_options(InetValues, #socket_options{ mode = proplists:get_value(mode, InetValues, Mode), packet = proplists:get_value(packet, InetValues, Packet), packet_size = proplists:get_value(packet_size, InetValues, PacketSize), - active = proplists:get_value(active, InetValues, Active) + active = emulated_active_option(InetValues, Active) }. +emulated_active_option([], Active) -> + Active; +emulated_active_option([{active, Active} | _], _) when Active =< 0 -> + false; +emulated_active_option([{active, Active} | _], _) -> + Active; +emulated_active_option([_|Tail], Active) -> + emulated_active_option(Tail, Active). + emulated_options([{mode, Value} = Opt |Opts], Inet, Emulated) -> validate_inet_option(mode, Value), emulated_options(Opts, Inet, [Opt | proplists:delete(mode, Emulated)]); @@ -185,6 +223,9 @@ validate_inet_option(mode, Value) when Value =/= list, Value =/= binary -> throw({error, {options, {mode,Value}}}); validate_inet_option(active, Value) + when Value >= -32768, Value =< 32767 -> + ok; +validate_inet_option(active, Value) when Value =/= true, Value =/= false, Value =/= once -> throw({error, {options, {active,Value}}}); validate_inet_option(_, _) -> diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index 3516bd6d49..bfa349c8d8 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -55,8 +55,7 @@ format_error/1, renegotiate/1, prf/5, negotiated_protocol/1, connection_information/1, connection_information/2]). %% Misc --export([handle_options/2, tls_version/1, new_ssl_options/3, suite_to_str/1, - set_log_level/1]). +-export([handle_options/2, tls_version/1, new_ssl_options/3, suite_to_str/1]). -deprecated({ssl_accept, 1, eventually}). -deprecated({ssl_accept, 2, eventually}). @@ -94,9 +93,11 @@ -type tls_client_option() :: client_option() | common_option() | socket_option() | transport_option(). -type tls_server_option() :: server_option() | common_option() | socket_option() | transport_option(). -type active_msgs() :: {ssl, sslsocket(), Data::binary() | list()} | {ssl_closed, sslsocket()} | - {ssl_error, sslsocket(), Reason::term()}. + {ssl_error, sslsocket(), Reason::term()} | {ssl_passive, sslsocket()}. -type transport_option() :: {cb_info, {CallbackModule::atom(), DataTag::atom(), - ClosedTag::atom(), ErrTag::atom()}}. + ClosedTag::atom(), ErrTag::atom()}} | + {cb_info, {CallbackModule::atom(), DataTag::atom(), + ClosedTag::atom(), ErrTag::atom(), PassiveTag::atom()}}. -type host() :: hostname() | ip_address(). -type hostname() :: string(). -type ip_address() :: inet:ip_address(). @@ -422,9 +423,9 @@ connect(Socket, SslOptions) when is_port(Socket) -> timeout() | list()) -> {ok, #sslsocket{}} | {error, reason()}. connect(Socket, SslOptions0, Timeout) when is_port(Socket), - (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> - {Transport,_,_,_} = proplists:get_value(cb_info, SslOptions0, - {gen_tcp, tcp, tcp_closed, tcp_error}), + (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> + CbInfo = handle_option(cb_info, SslOptions0, default_cb_info(tls)), + Transport = element(1, CbInfo), EmulatedOptions = tls_socket:emulated_options(), {ok, SocketValues} = tls_socket:getopts(Transport, Socket, EmulatedOptions), try handle_options(SslOptions0 ++ SocketValues, client) of @@ -572,8 +573,8 @@ handshake(#sslsocket{pid = [Pid|_], fd = {_, _, _}} = Socket, SslOpts, Timeout) end; handshake(Socket, SslOptions, Timeout) when is_port(Socket), (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> - {Transport,_,_,_} = - proplists:get_value(cb_info, SslOptions, {gen_tcp, tcp, tcp_closed, tcp_error}), + CbInfo = handle_option(cb_info, SslOptions, default_cb_info(tls)), + Transport = element(1, CbInfo), EmulatedOptions = tls_socket:emulated_options(), {ok, SocketValues} = tls_socket:getopts(Transport, Socket, EmulatedOptions), ConnetionCb = connection_cb(SslOptions), @@ -625,7 +626,7 @@ close(#sslsocket{pid = [Pid|_]}) when is_pid(Pid) -> ssl_connection:close(Pid, {close, ?DEFAULT_TIMEOUT}); close(#sslsocket{pid = {dtls, #config{dtls_handler = {Pid, _}}}}) -> dtls_packet_demux:close(Pid); -close(#sslsocket{pid = {ListenSocket, #config{transport_info={Transport,_, _, _}}}}) -> +close(#sslsocket{pid = {ListenSocket, #config{transport_info={Transport,_,_,_,_}}}}) -> Transport:close(ListenSocket). %%-------------------------------------------------------------------- @@ -641,7 +642,7 @@ close(#sslsocket{pid = [TLSPid|_]}, close(#sslsocket{pid = [TLSPid|_]}, Timeout) when is_pid(TLSPid), (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> ssl_connection:close(TLSPid, {close, Timeout}); -close(#sslsocket{pid = {ListenSocket, #config{transport_info={Transport,_, _, _}}}}, _) -> +close(#sslsocket{pid = {ListenSocket, #config{transport_info={Transport,_,_,_,_}}}}, _) -> Transport:close(ListenSocket). %%-------------------------------------------------------------------- @@ -657,7 +658,8 @@ send(#sslsocket{pid = {_, #config{transport_info={_, udp, _, _}}}}, _) -> {error,enotconn}; %% Emulate connection behaviour send(#sslsocket{pid = {dtls,_}}, _) -> {error,enotconn}; %% Emulate connection behaviour -send(#sslsocket{pid = {ListenSocket, #config{transport_info={Transport, _, _, _}}}}, Data) -> +send(#sslsocket{pid = {ListenSocket, #config{transport_info = Info}}}, Data) -> + Transport = element(1, Info), Transport:send(ListenSocket, Data). %% {error,enotconn} %%-------------------------------------------------------------------- @@ -675,7 +677,8 @@ recv(#sslsocket{pid = [Pid|_]}, Length, Timeout) when is_pid(Pid), recv(#sslsocket{pid = {dtls,_}}, _, _) -> {error,enotconn}; recv(#sslsocket{pid = {Listen, - #config{transport_info = {Transport, _, _, _}}}}, _,_) when is_port(Listen)-> + #config{transport_info = Info}}},_,_) when is_port(Listen)-> + Transport = element(1, Info), Transport:recv(Listen, 0). %% {error,enotconn} %%-------------------------------------------------------------------- @@ -690,7 +693,7 @@ controlling_process(#sslsocket{pid = {dtls, _}}, NewOwner) when is_pid(NewOwner) -> ok; %% Meaningless but let it be allowed to conform with TLS controlling_process(#sslsocket{pid = {Listen, - #config{transport_info = {Transport, _, _, _}}}}, + #config{transport_info = {Transport,_,_,_,_}}}}, NewOwner) when is_port(Listen), is_pid(NewOwner) -> %% Meaningless but let it be allowed to conform with normal sockets @@ -733,13 +736,13 @@ connection_information(#sslsocket{pid = [Pid|_]}, Items) when is_pid(Pid) -> %% %% Description: same as inet:peername/1. %%-------------------------------------------------------------------- -peername(#sslsocket{pid = [Pid|_], fd = {Transport, Socket, _}}) when is_pid(Pid)-> +peername(#sslsocket{pid = [Pid|_], fd = {Transport, Socket,_}}) when is_pid(Pid)-> dtls_socket:peername(Transport, Socket); -peername(#sslsocket{pid = [Pid|_], fd = {Transport, Socket, _, _}}) when is_pid(Pid)-> +peername(#sslsocket{pid = [Pid|_], fd = {Transport, Socket,_,_}}) when is_pid(Pid)-> tls_socket:peername(Transport, Socket); -peername(#sslsocket{pid = {dtls, #config{dtls_handler = {_Pid, _}}}}) -> +peername(#sslsocket{pid = {dtls, #config{dtls_handler = {_Pid,_}}}}) -> dtls_socket:peername(dtls, undefined); -peername(#sslsocket{pid = {ListenSocket, #config{transport_info = {Transport,_,_,_}}}}) -> +peername(#sslsocket{pid = {ListenSocket, #config{transport_info = {Transport,_,_,_,_}}}}) -> tls_socket:peername(Transport, ListenSocket); %% Will return {error, enotconn} peername(#sslsocket{pid = {dtls,_}}) -> {error,enotconn}. @@ -931,7 +934,7 @@ getopts(#sslsocket{pid = {dtls, #config{transport_info = {Transport,_,_,_}}}} = _:Error -> {error, {options, {socket_options, OptionTags, Error}}} end; -getopts(#sslsocket{pid = {_, #config{transport_info = {Transport,_,_,_}}}} = ListenSocket, +getopts(#sslsocket{pid = {_, #config{transport_info = {Transport,_,_,_,_}}}} = ListenSocket, OptionTags) when is_list(OptionTags) -> try tls_socket:getopts(Transport, ListenSocket, OptionTags) of {ok, _} = Result -> @@ -988,7 +991,7 @@ setopts(#sslsocket{pid = {dtls, #config{transport_info = {Transport,_,_,_}}}} = _:Error -> {error, {options, {socket_options, Options, Error}}} end; -setopts(#sslsocket{pid = {_, #config{transport_info = {Transport,_,_,_}}}} = ListenSocket, Options) when is_list(Options) -> +setopts(#sslsocket{pid = {_, #config{transport_info = {Transport,_,_,_,_}}}} = ListenSocket, Options) when is_list(Options) -> try tls_socket:setopts(Transport, ListenSocket, Options) of ok -> ok; @@ -1032,8 +1035,9 @@ getstat(#sslsocket{pid = [Pid|_], fd = {Transport, Socket, _, _}}, Options) when %% %% Description: Same as gen_tcp:shutdown/2 %%-------------------------------------------------------------------- -shutdown(#sslsocket{pid = {Listen, #config{transport_info = {Transport,_, _, _}}}}, +shutdown(#sslsocket{pid = {Listen, #config{transport_info = Info}}}, How) when is_port(Listen) -> + Transport = element(1, Info), Transport:shutdown(Listen, How); shutdown(#sslsocket{pid = {dtls,_}},_) -> {error, enotconn}; @@ -1045,13 +1049,13 @@ shutdown(#sslsocket{pid = [Pid|_]}, How) when is_pid(Pid) -> %% %% Description: Same as inet:sockname/1 %%-------------------------------------------------------------------- -sockname(#sslsocket{pid = {Listen, #config{transport_info = {Transport, _, _, _}}}}) when is_port(Listen) -> +sockname(#sslsocket{pid = {Listen, #config{transport_info = {Transport,_,_,_,_}}}}) when is_port(Listen) -> tls_socket:sockname(Transport, Listen); sockname(#sslsocket{pid = {dtls, #config{dtls_handler = {Pid, _}}}}) -> dtls_packet_demux:sockname(Pid); -sockname(#sslsocket{pid = [Pid|_], fd = {Transport, Socket, _}}) when is_pid(Pid) -> +sockname(#sslsocket{pid = [Pid|_], fd = {Transport, Socket,_}}) when is_pid(Pid) -> dtls_socket:sockname(Transport, Socket); -sockname(#sslsocket{pid = [Pid| _], fd = {Transport, Socket, _, _}}) when is_pid(Pid) -> +sockname(#sslsocket{pid = [Pid| _], fd = {Transport, Socket,_,_}}) when is_pid(Pid) -> tls_socket:sockname(Transport, Socket). %%--------------------------------------------------------------- @@ -1167,32 +1171,6 @@ suite_to_str(Cipher) -> ssl_cipher_format:suite_to_str(Cipher). -%%-------------------------------------------------------------------- --spec set_log_level(atom()) -> ok | {error, term()}. -%% -%% Description: Set log level for the SSL application -%%-------------------------------------------------------------------- -set_log_level(Level) -> - case application:get_all_key(ssl) of - {ok, PropList} -> - Modules = proplists:get_value(modules, PropList), - set_module_level(Modules, Level); - undefined -> - {error, ssl_not_started} - end. - -set_module_level(Modules, Level) -> - Fun = fun (Module) -> - ok = logger:set_module_level(Module, Level) - end, - try lists:map(Fun, Modules) of - _ -> - ok - catch - error:{badmatch, Error} -> - Error - end. - %%%-------------------------------------------------------------- %%% Internal functions %%%-------------------------------------------------------------------- @@ -1211,7 +1189,7 @@ supported_suites(all, Version) -> supported_suites(anonymous, Version) -> ssl_cipher:anonymous_suites(Version). -do_listen(Port, #config{transport_info = {Transport, _, _, _}} = Config, tls_connection) -> +do_listen(Port, #config{transport_info = {Transport, _, _, _,_}} = Config, tls_connection) -> tls_socket:listen(Transport, Port, Config); do_listen(Port, Config, dtls_connection) -> @@ -1381,7 +1359,7 @@ handle_options(Opts0, Role, Host) -> log_level = handle_option(log_level, Opts, LogLevel) }, - CbInfo = proplists:get_value(cb_info, Opts, default_cb_info(Protocol)), + CbInfo = handle_option(cb_info, Opts, default_cb_info(Protocol)), SslOptions = [protocol, versions, verify, verify_fun, partial_chain, fail_if_no_peer_cert, verify_client_once, depth, cert, certfile, key, keyfile, @@ -1425,6 +1403,10 @@ handle_option(sni_fun, Opts, Default) -> _ -> throw({error, {conflict_options, [sni_fun, sni_hosts]}}) end; +handle_option(cb_info, Opts, Default) -> + CbInfo = proplists:get_value(cb_info, Opts, Default), + true = validate_option(cb_info, CbInfo), + handle_cb_info(CbInfo, Default); handle_option(OptionName, Opts, Default) -> validate_option(OptionName, proplists:get_value(OptionName, Opts, Default)). @@ -1659,9 +1641,29 @@ validate_option(handshake, full = Value) -> Value; validate_option(customize_hostname_check, Value) when is_list(Value) -> Value; +validate_option(cb_info, {V1, V2, V3, V4}) when is_atom(V1), + is_atom(V2), + is_atom(V3), + is_atom(V4) + -> + true; +validate_option(cb_info, {V1, V2, V3, V4, V5}) when is_atom(V1), + is_atom(V2), + is_atom(V3), + is_atom(V4), + is_atom(V5) + -> + true; +validate_option(cb_info, _) -> + false; validate_option(Opt, Value) -> throw({error, {options, {Opt, Value}}}). +handle_cb_info({V1, V2, V3, V4}, {_,_,_,_,_}) -> + {V1,V2,V3,V4, list_to_atom(atom_to_list(V2) ++ "passive")}; +handle_cb_info(CbInfo, _) -> + CbInfo. + handle_hashsigns_option(Value, Version) when is_list(Value) andalso Version >= {3, 4} -> case tls_v1:signature_schemes(Version, Value) of @@ -2132,7 +2134,7 @@ default_option_role(_,_,_) -> default_cb_info(tls) -> - {gen_tcp, tcp, tcp_closed, tcp_error}; + {gen_tcp, tcp, tcp_closed, tcp_error, tcp_passive}; default_cb_info(dtls) -> {gen_udp, udp, udp_closed, udp_error}. diff --git a/lib/ssl/src/ssl_alert.erl b/lib/ssl/src/ssl_alert.erl index e17476f33b..06b1b005a5 100644 --- a/lib/ssl/src/ssl_alert.erl +++ b/lib/ssl/src/ssl_alert.erl @@ -161,6 +161,8 @@ description_txt(?INSUFFICIENT_SECURITY) -> "Insufficient Security"; description_txt(?INTERNAL_ERROR) -> "Internal Error"; +description_txt(?INAPPROPRIATE_FALLBACK) -> + "Inappropriate Fallback"; description_txt(?USER_CANCELED) -> "User Canceled"; description_txt(?NO_RENEGOTIATION) -> @@ -179,8 +181,6 @@ description_txt(?BAD_CERTIFICATE_HASH_VALUE) -> "Bad Certificate Hash Value"; description_txt(?UNKNOWN_PSK_IDENTITY) -> "Unknown Psk Identity"; -description_txt(?INAPPROPRIATE_FALLBACK) -> - "Inappropriate Fallback"; description_txt(?CERTIFICATE_REQUIRED) -> "Certificate required"; description_txt(?NO_APPLICATION_PROTOCOL) -> @@ -232,10 +232,14 @@ description_atom(?INSUFFICIENT_SECURITY) -> insufficient_security; description_atom(?INTERNAL_ERROR) -> internal_error; +description_atom(?INAPPROPRIATE_FALLBACK) -> + inappropriate_fallback; description_atom(?USER_CANCELED) -> user_canceled; description_atom(?NO_RENEGOTIATION) -> no_renegotiation; +description_atom(?MISSING_EXTENSION) -> + missing_extension; description_atom(?UNSUPPORTED_EXTENSION) -> unsupported_extension; description_atom(?CERTIFICATE_UNOBTAINABLE) -> @@ -248,9 +252,9 @@ description_atom(?BAD_CERTIFICATE_HASH_VALUE) -> bad_certificate_hash_value; description_atom(?UNKNOWN_PSK_IDENTITY) -> unknown_psk_identity; -description_atom(?INAPPROPRIATE_FALLBACK) -> - inappropriate_fallback; +description_atom(?CERTIFICATE_REQUIRED) -> + certificate_required; description_atom(?NO_APPLICATION_PROTOCOL) -> no_application_protocol; description_atom(_) -> - 'unsupported/unkonwn_alert'. + 'unsupported/unknown_alert'. diff --git a/lib/ssl/src/ssl_app.erl b/lib/ssl/src/ssl_app.erl index 2a5047c75c..9e6d676bef 100644 --- a/lib/ssl/src/ssl_app.erl +++ b/lib/ssl/src/ssl_app.erl @@ -44,11 +44,11 @@ start_logger() -> formatter => {ssl_logger, #{}}}, Filter = {fun logger_filters:domain/2,{log,sub,[otp,ssl]}}, logger:add_handler(ssl_handler, logger_std_h, Config), - logger:add_handler_filter(ssl_handler, filter_non_ssl, Filter). + logger:add_handler_filter(ssl_handler, filter_non_ssl, Filter), + logger:set_application_level(ssl, debug). %% %% Description: Stop SSL logger stop_logger() -> + logger:unset_application_level(ssl), logger:remove_handler(ssl_handler). - - diff --git a/lib/ssl/src/ssl_cipher.erl b/lib/ssl/src/ssl_cipher.erl index 6e751f9ceb..fe8736d2df 100644 --- a/lib/ssl/src/ssl_cipher.erl +++ b/lib/ssl/src/ssl_cipher.erl @@ -45,7 +45,7 @@ random_bytes/1, calc_mac_hash/4, calc_mac_hash/6, is_stream_ciphersuite/1, signature_scheme/1, scheme_to_components/1, hash_size/1, effective_key_bits/1, - key_material/1]). + key_material/1, signature_algorithm_to_scheme/1]). %% RFC 8446 TLS 1.3 -export([generate_client_shares/1, generate_server_share/1, add_zero_padding/2]). @@ -900,6 +900,18 @@ scheme_to_components(rsa_pss_pss_sha512) -> {sha512, rsa_pss_pss, undefined}; scheme_to_components(rsa_pkcs1_sha1) -> {sha1, rsa_pkcs1, undefined}; scheme_to_components(ecdsa_sha1) -> {sha1, ecdsa, undefined}. + +%% TODO: Add support for EC and RSA-SSA signatures +signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?sha1WithRSAEncryption}) -> + rsa_pkcs1_sha1; +signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?sha256WithRSAEncryption}) -> + rsa_pkcs1_sha256; +signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?sha384WithRSAEncryption}) -> + rsa_pkcs1_sha384; +signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?sha512WithRSAEncryption}) -> + rsa_pkcs1_sha512. + + %% RFC 5246: 6.2.3.2. CBC Block Cipher %% %% Implementation note: Canvel et al. [CBCTIME] have demonstrated a diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index f194610d72..1e97fe046b 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -385,7 +385,8 @@ handle_alert(#alert{level = ?FATAL} = Alert, StateName, log_alert(SslOpts#ssl_options.log_level, Role, Connection:protocol_name(), StateName, Alert#alert{role = opposite_role(Role)}), Pids = Connection:pids(State), - alert_user(Pids, Transport, Tracker, Socket, StateName, Opts, Pid, From, Alert, Role, Connection), + alert_user(Pids, Transport, Tracker, Socket, StateName, Opts, Pid, From, Alert, + opposite_role(Role), Connection), {stop, {shutdown, normal}, State}; handle_alert(#alert{level = ?WARNING, description = ?CLOSE_NOTIFY} = Alert, @@ -1383,10 +1384,22 @@ handle_call({get_opts, OptTags}, From, _, {keep_state_and_data, [{reply, From, OptsReply}]}; handle_call({set_opts, Opts0}, From, StateName, #state{static_env = #static_env{socket = Socket, - transport_cb = Transport}, + transport_cb = Transport, + tracker = Tracker}, + connection_env = + #connection_env{user_application = {_Mon, Pid}}, socket_options = Opts1 } = State0, Connection) -> {Reply, Opts} = set_socket_opts(Connection, Transport, Socket, Opts0, Opts1, []), + case {proplists:lookup(active, Opts0), Opts} of + {{_, N}, #socket_options{active=false}} when is_integer(N) -> + send_user( + Pid, + format_passive( + Connection:pids(State0), Transport, Socket, Tracker, Connection)); + _ -> + ok + end, State = State0#state{socket_options = Opts}, handle_active_option(Opts#socket_options.active, StateName, From, Reply, State); @@ -2516,6 +2529,30 @@ set_socket_opts(ConnectionCb, Transport, Socket, [{active, Active}| Opts], SockO Active == false -> set_socket_opts(ConnectionCb, Transport, Socket, Opts, SockOpts#socket_options{active = Active}, Other); +set_socket_opts(ConnectionCb, Transport, Socket, [{active, Active1} = Opt| Opts], + SockOpts=#socket_options{active = Active0}, Other) + when Active1 >= -32768, Active1 =< 32767 -> + Active = if + is_integer(Active0), Active0 + Active1 < -32768 -> + error; + is_integer(Active0), Active0 + Active1 =< 0 -> + false; + is_integer(Active0), Active0 + Active1 > 32767 -> + error; + Active1 =< 0 -> + false; + is_integer(Active0) -> + Active0 + Active1; + true -> + Active1 + end, + case Active of + error -> + {{error, {options, {socket_options, Opt}} }, SockOpts}; + _ -> + set_socket_opts(ConnectionCb, Transport, Socket, Opts, + SockOpts#socket_options{active = Active}, Other) + end; set_socket_opts(_,_, _, [{active, _} = Opt| _], SockOpts, _) -> {{error, {options, {socket_options, Opt}} }, SockOpts}; set_socket_opts(ConnectionCb, Transport, Socket, [Opt | Opts], SockOpts, Other) -> @@ -2700,6 +2737,11 @@ ssl_options_list([Key | Keys], [Value | Values], Acc) -> handle_active_option(false, connection = StateName, To, Reply, State) -> hibernate_after(StateName, State, [{reply, To, Reply}]); +handle_active_option(_, connection = StateName, To, _Reply, #state{connection_env = #connection_env{terminated = true}, + user_data_buffer = {_,0,_}} = State) -> + handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY, all_data_deliverd), StateName, + State#state{start_or_recv_from = To}), + {stop,{shutdown, peer_close}, State}; handle_active_option(_, connection = StateName0, To, Reply, #state{static_env = #static_env{protocol_cb = Connection}, user_data_buffer = {_,0,_}} = State0) -> case Connection:next_event(StateName0, no_record, State0) of @@ -2795,6 +2837,14 @@ deliver_app_data( case Active of once -> SO#socket_options{active=false}; + 1 -> + send_user( + Pid, + format_passive( + CPids, Transport, Socket, Tracker, Connection)), + SO#socket_options{active=false}; + N when is_integer(N) -> + SO#socket_options{active=N - 1}; _ -> SO end. @@ -2831,6 +2881,9 @@ do_format_reply(list, Packet, _, Data) do_format_reply(list, _,_, Data) -> binary_to_list(Data). +format_passive(CPids, Transport, Socket, Tracker, Connection) -> + {ssl_passive, Connection:socket(CPids, Transport, Socket, Tracker)}. + header(0, <<>>) -> <<>>; header(_, <<>>) -> diff --git a/lib/ssl/src/ssl_connection.hrl b/lib/ssl/src/ssl_connection.hrl index 201164949a..ff7207a8ce 100644 --- a/lib/ssl/src/ssl_connection.hrl +++ b/lib/ssl/src/ssl_connection.hrl @@ -40,6 +40,7 @@ data_tag :: atom(), % ex tcp. close_tag :: atom(), % ex tcp_closed error_tag :: atom(), % ex tcp_error + passive_tag :: atom(), % ex tcp_passive host :: string() | inet:ip_address(), port :: integer(), socket :: port() | tuple(), %% TODO: dtls socket diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index 6b1e3b6e07..6c95a7edf8 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -39,7 +39,7 @@ -type oid() :: tuple(). -type public_key_params() :: #'Dss-Parms'{} | {namedCurve, oid()} | #'ECParameters'{} | term(). -type public_key_info() :: {oid(), #'RSAPublicKey'{} | integer() | #'ECPoint'{}, public_key_params()}. --type ssl_handshake_history() :: {[binary()], [binary()]}. +-type ssl_handshake_history() :: {iodata(), iodata()}. -type ssl_handshake() :: #server_hello{} | #server_hello_done{} | #certificate{} | #certificate_request{} | #client_key_exchange{} | #finished{} | #certificate_verify{} | @@ -76,10 +76,13 @@ handle_client_hello_extensions/9, %% Returns server hello extensions handle_server_hello_extensions/9, select_curve/2, select_curve/3, select_hashsign/4, select_hashsign/5, - select_hashsign_algs/3, empty_extensions/2, add_server_share/2 + select_hashsign_algs/3, empty_extensions/2, add_server_share/3 ]). --export([get_cert_params/1]). +-export([get_cert_params/1, + server_name/3, + validation_fun_and_state/9, + handle_path_validation_error/7]). %%==================================================================== %% Create handshake messages @@ -1150,12 +1153,18 @@ maybe_add_key_share(HelloExtensions, KeyShare) -> HelloExtensions#{key_share => #key_share_client_hello{ client_shares = ClientShares}}. -add_server_share(Extensions, KeyShare) -> +add_server_share(server_hello, Extensions, KeyShare) -> #key_share_server_hello{server_share = ServerShare0} = KeyShare, %% Keep only public keys ServerShare = kse_remove_private_key(ServerShare0), Extensions#{key_share => #key_share_server_hello{ - server_share = ServerShare}}. + server_share = ServerShare}}; +add_server_share(hello_retry_request, Extensions, + #key_share_server_hello{ + server_share = #key_share_entry{group = Group}}) -> + Extensions#{key_share => #key_share_hello_retry_request{ + selected_group = Group}}. + kse_remove_private_key(#key_share_entry{ group = Group, diff --git a/lib/ssl/src/ssl_handshake.hrl b/lib/ssl/src/ssl_handshake.hrl index d4233bea9b..b248edcaa9 100644 --- a/lib/ssl/src/ssl_handshake.hrl +++ b/lib/ssl/src/ssl_handshake.hrl @@ -47,7 +47,9 @@ srp_username, is_resumable, time_stamp, - ecc + ecc, %% TLS 1.3 Group + sign_alg, %% TLS 1.3 Signature Algorithm + dh_public_value %% TLS 1.3 DH Public Value from peer }). -define(NUM_OF_SESSION_ID_BYTES, 32). % TSL 1.1 & SSL 3 diff --git a/lib/ssl/src/ssl_logger.erl b/lib/ssl/src/ssl_logger.erl index b82b3937a1..f497315235 100644 --- a/lib/ssl/src/ssl_logger.erl +++ b/lib/ssl/src/ssl_logger.erl @@ -181,6 +181,11 @@ parse_handshake(Direction, #hello_request{} = HelloRequest) -> [header_prefix(Direction)]), Message = io_lib:format("~p", [?rec_info(hello_request, HelloRequest)]), {Header, Message}; +parse_handshake(Direction, #certificate_request_1_3{} = CertificateRequest) -> + Header = io_lib:format("~s Handshake, CertificateRequest", + [header_prefix(Direction)]), + Message = io_lib:format("~p", [?rec_info(certificate_request_1_3, CertificateRequest)]), + {Header, Message}; parse_handshake(Direction, #certificate_1_3{} = Certificate) -> Header = io_lib:format("~s Handshake, Certificate", [header_prefix(Direction)]), diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index 8eb9e56375..fde73cdef1 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -98,7 +98,7 @@ %% Setup %%==================================================================== start_fsm(Role, Host, Port, Socket, {#ssl_options{erl_dist = false},_, Tracker} = Opts, - User, {CbModule, _,_, _} = CbInfo, + User, {CbModule, _,_, _, _} = CbInfo, Timeout) -> try {ok, Sender} = tls_sender:start(), @@ -112,7 +112,7 @@ start_fsm(Role, Host, Port, Socket, {#ssl_options{erl_dist = false},_, Tracker} end; start_fsm(Role, Host, Port, Socket, {#ssl_options{erl_dist = true},_, Tracker} = Opts, - User, {CbModule, _,_, _} = CbInfo, + User, {CbModule, _,_, _, _} = CbInfo, Timeout) -> try {ok, Sender} = tls_sender:start([{spawn_opt, ?DIST_CNTRL_SPAWN_OPTS}]), @@ -251,13 +251,28 @@ next_event(StateName, Record, State, Actions) -> %%% TLS record protocol level application data messages - -handle_protocol_record(#ssl_tls{type = ?APPLICATION_DATA, fragment = Data}, StateName0, State0) -> +handle_protocol_record(#ssl_tls{type = ?APPLICATION_DATA, fragment = Data}, StateName, + #state{start_or_recv_from = From, + socket_options = #socket_options{active = false}} = State0) when From =/= undefined -> + case ssl_connection:read_application_data(Data, State0) of + {stop, _, _} = Stop-> + Stop; + {Record, #state{start_or_recv_from = Caller} = State1} -> + TimerAction = case Caller of + undefined -> %% Passive recv complete cancel timer + [{{timeout, recv}, infinity, timeout}]; + _ -> + [] + end, + {next_state, StateName, State, Actions} = next_event(StateName, Record, State1, TimerAction), + ssl_connection:hibernate_after(StateName, State, Actions) + end; +handle_protocol_record(#ssl_tls{type = ?APPLICATION_DATA, fragment = Data}, StateName, State0) -> case ssl_connection:read_application_data(Data, State0) of {stop, _, _} = Stop-> Stop; {Record, State1} -> - case next_event(StateName0, Record, State1) of + case next_event(StateName, Record, State1) of {next_state, StateName, State, Actions} -> ssl_connection:hibernate_after(StateName, State, Actions); {stop, _, _} = Stop -> @@ -308,9 +323,7 @@ handle_protocol_record(#ssl_tls{type = ?ALERT, fragment = EncAlerts}, StateName, handle_alerts(Alerts, {next_state, StateName, State}); [] -> ssl_connection:handle_own_alert(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, empty_alert), - Version, StateName, State); - #alert{} = Alert -> - ssl_connection:handle_own_alert(Alert, Version, StateName, State) + Version, StateName, State) catch _:_ -> ssl_connection:handle_own_alert(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, alert_decode_error), @@ -941,7 +954,7 @@ code_change(_OldVsn, StateName, State, _) -> %%% Internal functions %%-------------------------------------------------------------------- initial_state(Role, Sender, Host, Port, Socket, {SSLOptions, SocketOptions, Tracker}, User, - {CbModule, DataTag, CloseTag, ErrorTag}) -> + {CbModule, DataTag, CloseTag, ErrorTag, PassiveTag}) -> #ssl_options{beast_mitigation = BeastMitigation, erl_dist = IsErlDist} = SSLOptions, ConnectionStates = tls_record:init_connection_states(Role, BeastMitigation), @@ -965,6 +978,7 @@ initial_state(Role, Sender, Host, Port, Socket, {SSLOptions, SocketOptions, Trac data_tag = DataTag, close_tag = CloseTag, error_tag = ErrorTag, + passive_tag = PassiveTag, host = Host, port = Port, socket = Socket, @@ -1061,8 +1075,9 @@ handle_info({Protocol, _, Data}, StateName, ssl_connection:handle_normal_shutdown(Alert, StateName, State0), {stop, {shutdown, own_alert}, State0} end; -handle_info({tcp_passive, Socket}, StateName, - #state{static_env = #static_env{socket = Socket}, +handle_info({PassiveTag, Socket}, StateName, + #state{static_env = #static_env{socket = Socket, + passive_tag = PassiveTag}, protocol_specific = PS } = State) -> next_event(StateName, no_record, @@ -1135,6 +1150,7 @@ encode_handshake(Handshake, Version, ConnectionStates0, Hist0) -> encode_change_cipher(#change_cipher_spec{}, Version, ConnectionStates) -> tls_record:encode_change_cipher_spec(Version, ConnectionStates). +-spec decode_alerts(binary()) -> list(). decode_alerts(Bin) -> ssl_alert:decode(Bin). diff --git a/lib/ssl/src/tls_connection_1_3.erl b/lib/ssl/src/tls_connection_1_3.erl index de786d0875..701a5860c2 100644 --- a/lib/ssl/src/tls_connection_1_3.erl +++ b/lib/ssl/src/tls_connection_1_3.erl @@ -110,51 +110,75 @@ %% gen_statem helper functions -export([start/4, negotiated/4, + wait_cert/4, + wait_cv/4, wait_finished/4 ]). -start(internal, - #client_hello{} = Hello, - #state{connection_states = _ConnectionStates0, - ssl_options = #ssl_options{ciphers = _ServerCiphers, - signature_algs = _ServerSignAlgs, - signature_algs_cert = _SignatureSchemes, %% TODO: Check?? - supported_groups = _ServerGroups0, - versions = _Versions} = SslOpts, - session = #session{own_certificate = Cert}} = State0, - _Module) -> - Env = #{cert => Cert}, - case tls_handshake_1_3:handle_client_hello(Hello, SslOpts, Env) of +start(internal, #change_cipher_spec{}, State0, _Module) -> + {Record, State} = tls_connection:next_record(State0), + tls_connection:next_event(?FUNCTION_NAME, Record, State); +start(internal, #client_hello{} = Hello, State0, _Module) -> + case tls_handshake_1_3:do_start(Hello, State0) of #alert{} = Alert -> ssl_connection:handle_own_alert(Alert, {3,4}, start, State0); - M -> - %% update connection_states with cipher - State = update_state(State0, M), - {next_state, negotiated, State, [{next_event, internal, M}]} - - end. + {State, start} -> + {next_state, start, State, []}; + {State, negotiated} -> + {next_state, negotiated, State, [{next_event, internal, start_handshake}]} + end; +start(Type, Msg, State, Connection) -> + ssl_connection:handle_common_event(Type, Msg, ?FUNCTION_NAME, State, Connection). -negotiated(internal, Map, State0, _Module) -> - case tls_handshake_1_3:do_negotiated(Map, State0) of +negotiated(internal, #change_cipher_spec{}, State0, _Module) -> + {Record, State} = tls_connection:next_record(State0), + tls_connection:next_event(?FUNCTION_NAME, Record, State); +negotiated(internal, Message, State0, _Module) -> + case tls_handshake_1_3:do_negotiated(Message, State0) of #alert{} = Alert -> ssl_connection:handle_own_alert(Alert, {3,4}, negotiated, State0); - State -> - {next_state, wait_finished, State, []} - + {State, NextState} -> + {next_state, NextState, State, []} end. -wait_finished(internal, - #change_cipher_spec{} = ChangeCipherSpec, State0, _Module) -> - case tls_handshake_1_3:do_wait_finished(ChangeCipherSpec, State0) of - #alert{} = Alert -> - ssl_connection:handle_own_alert(Alert, {3,4}, wait_finished, State0); - State1 -> +wait_cert(internal, #change_cipher_spec{}, State0, _Module) -> + {Record, State} = tls_connection:next_record(State0), + tls_connection:next_event(?FUNCTION_NAME, Record, State); +wait_cert(internal, + #certificate_1_3{} = Certificate, State0, _Module) -> + case tls_handshake_1_3:do_wait_cert(Certificate, State0) of + {#alert{} = Alert, State} -> + ssl_connection:handle_own_alert(Alert, {3,4}, wait_cert, State); + {State1, NextState} -> + {Record, State} = tls_connection:next_record(State1), + tls_connection:next_event(NextState, Record, State) + end; +wait_cert(Type, Msg, State, Connection) -> + ssl_connection:handle_common_event(Type, Msg, ?FUNCTION_NAME, State, Connection). + + +wait_cv(internal, #change_cipher_spec{}, State0, _Module) -> + {Record, State} = tls_connection:next_record(State0), + tls_connection:next_event(?FUNCTION_NAME, Record, State); +wait_cv(internal, + #certificate_verify_1_3{} = CertificateVerify, State0, _Module) -> + case tls_handshake_1_3:do_wait_cv(CertificateVerify, State0) of + {#alert{} = Alert, State} -> + ssl_connection:handle_own_alert(Alert, {3,4}, wait_cv, State); + {State1, NextState} -> {Record, State} = tls_connection:next_record(State1), - tls_connection:next_event(?FUNCTION_NAME, Record, State) + tls_connection:next_event(NextState, Record, State) end; +wait_cv(Type, Msg, State, Connection) -> + ssl_connection:handle_common_event(Type, Msg, ?FUNCTION_NAME, State, Connection). + + +wait_finished(internal, #change_cipher_spec{}, State0, _Module) -> + {Record, State} = tls_connection:next_record(State0), + tls_connection:next_event(?FUNCTION_NAME, Record, State); wait_finished(internal, #finished{} = Finished, State0, Module) -> case tls_handshake_1_3:do_wait_finished(Finished, State0) of @@ -166,24 +190,3 @@ wait_finished(internal, end; wait_finished(Type, Msg, State, Connection) -> ssl_connection:handle_common_event(Type, Msg, ?FUNCTION_NAME, State, Connection). - - -update_state(#state{connection_states = ConnectionStates0, - connection_env = CEnv, - session = Session} = State, - #{cipher := Cipher, - key_share := KeyShare, - session_id := SessionId}) -> - #{security_parameters := SecParamsR0} = PendingRead = - maps:get(pending_read, ConnectionStates0), - #{security_parameters := SecParamsW0} = PendingWrite = - maps:get(pending_write, ConnectionStates0), - SecParamsR = ssl_cipher:security_parameters_1_3(SecParamsR0, Cipher), - SecParamsW = ssl_cipher:security_parameters_1_3(SecParamsW0, Cipher), - ConnectionStates = - ConnectionStates0#{pending_read => PendingRead#{security_parameters => SecParamsR}, - pending_write => PendingWrite#{security_parameters => SecParamsW}}, - State#state{connection_states = ConnectionStates, - key_share = KeyShare, - session = Session#session{session_id = SessionId}, - connection_env = CEnv#connection_env{negotiated_version = {3,4}}}. diff --git a/lib/ssl/src/tls_handshake_1_3.erl b/lib/ssl/src/tls_handshake_1_3.erl index 6a6de4b988..0efedf3400 100644 --- a/lib/ssl/src/tls_handshake_1_3.erl +++ b/lib/ssl/src/tls_handshake_1_3.erl @@ -36,38 +36,51 @@ %% Encode -export([encode_handshake/1, decode_handshake/2]). -%% Handshake --export([handle_client_hello/3]). - %% Create handshake messages -export([certificate/5, certificate_verify/4, encrypted_extensions/0, server_hello/4]). --export([do_negotiated/2, +-export([do_start/2, + do_negotiated/2, + do_wait_cert/2, + do_wait_cv/2, do_wait_finished/2]). %%==================================================================== %% Create handshake messages %%==================================================================== -server_hello(SessionId, KeyShare, ConnectionStates, _Map) -> +server_hello(MsgType, SessionId, KeyShare, ConnectionStates) -> #{security_parameters := SecParams} = ssl_record:pending_connection_state(ConnectionStates, read), - Extensions = server_hello_extensions(KeyShare), + Extensions = server_hello_extensions(MsgType, KeyShare), #server_hello{server_version = {3,3}, %% legacy_version cipher_suite = SecParams#security_parameters.cipher_suite, compression_method = 0, %% legacy attribute - random = SecParams#security_parameters.server_random, + random = server_hello_random(MsgType, SecParams), session_id = SessionId, extensions = Extensions }. -server_hello_extensions(KeyShare) -> +server_hello_extensions(MsgType, KeyShare) -> SupportedVersions = #server_hello_selected_version{selected_version = {3,4}}, Extensions = #{server_hello_selected_version => SupportedVersions}, - ssl_handshake:add_server_share(Extensions, KeyShare). + ssl_handshake:add_server_share(MsgType, Extensions, KeyShare). + +server_hello_random(server_hello, #security_parameters{server_random = Random}) -> + Random; +%% For reasons of backward compatibility with middleboxes (see +%% Appendix D.4), the HelloRetryRequest message uses the same structure +%% as the ServerHello, but with Random set to the special value of the +%% SHA-256 of "HelloRetryRequest": +%% +%% CF 21 AD 74 E5 9A 61 11 BE 1D 8C 02 1E 65 B8 91 +%% C2 A2 11 16 7A BB 8C 5E 07 9E 09 E2 C8 A8 33 9C +server_hello_random(hello_retry_request, _) -> + crypto:hash(sha256, "HelloRetryRequest"). + %% TODO: implement support for encrypted_extensions encrypted_extensions() -> @@ -75,6 +88,37 @@ encrypted_extensions() -> extensions = #{} }. + +certificate_request(SignAlgs0, SignAlgsCert0) -> + %% Input arguments contain TLS 1.2 algorithms due to backward compatibility + %% reasons. These {Hash, Algo} tuples must be filtered before creating the + %% the extensions. + SignAlgs = filter_tls13_algs(SignAlgs0), + SignAlgsCert = filter_tls13_algs(SignAlgsCert0), + Extensions0 = add_signature_algorithms(#{}, SignAlgs), + Extensions = add_signature_algorithms_cert(Extensions0, SignAlgsCert), + #certificate_request_1_3{ + certificate_request_context = <<>>, + extensions = Extensions}. + + +add_signature_algorithms(Extensions, SignAlgs) -> + Extensions#{signature_algorithms => + #signature_algorithms{signature_scheme_list = SignAlgs}}. + + +add_signature_algorithms_cert(Extensions, undefined) -> + Extensions; +add_signature_algorithms_cert(Extensions, SignAlgsCert) -> + Extensions#{signature_algorithms_cert => + #signature_algorithms{signature_scheme_list = SignAlgsCert}}. + + +filter_tls13_algs(undefined) -> undefined; +filter_tls13_algs(Algo) -> + lists:filter(fun is_atom/1, Algo). + + %% TODO: use maybe monad for error handling! %% enum { %% X509(0), @@ -132,7 +176,7 @@ certificate_verify(PrivateKey, SignatureScheme, %% Digital signatures use the hash function defined by the selected signature %% scheme. - case digitally_sign(THash, <<"TLS 1.3, server CertificateVerify">>, + case sign(THash, <<"TLS 1.3, server CertificateVerify">>, HashAlgo, PrivateKey) of {ok, Signature} -> {ok, #certificate_verify_1_3{ @@ -340,7 +384,7 @@ certificate_entry(DER) -> %% 79 %% 00 %% 0101010101010101010101010101010101010101010101010101010101010101 -digitally_sign(THash, Context, HashAlgo, PrivateKey) -> +sign(THash, Context, HashAlgo, PrivateKey) -> Content = build_content(Context, THash), %% The length of the Salt MUST be equal to the length of the output @@ -357,24 +401,41 @@ digitally_sign(THash, Context, HashAlgo, PrivateKey) -> end. +verify(THash, Context, HashAlgo, Signature, PublicKey) -> + Content = build_content(Context, THash), + + %% The length of the Salt MUST be equal to the length of the output + %% of the digest algorithm: rsa_pss_saltlen = -1 + try public_key:verify(Content, HashAlgo, Signature, PublicKey, + [{rsa_padding, rsa_pkcs1_pss_padding}, + {rsa_pss_saltlen, -1}, + {rsa_mgf1_md, HashAlgo}]) of + Result -> + {ok, Result} + catch + error:badarg -> + {error, badarg} + end. + + build_content(Context, THash) -> Prefix = binary:copy(<<32>>, 64), <<Prefix/binary,Context/binary,?BYTE(0),THash/binary>>. + %%==================================================================== %% Handle handshake messages %%==================================================================== -handle_client_hello(#client_hello{cipher_suites = ClientCiphers, - session_id = SessionId, - extensions = Extensions} = _Hello, - #ssl_options{ciphers = ServerCiphers, - signature_algs = ServerSignAlgs, - signature_algs_cert = _SignatureSchemes, %% TODO: Check?? - supported_groups = ServerGroups0} = _SslOpts, - Env) -> - Cert = maps:get(cert, Env, undefined), +do_start(#client_hello{cipher_suites = ClientCiphers, + session_id = SessionId, + extensions = Extensions} = _Hello, + #state{connection_states = _ConnectionStates0, + ssl_options = #ssl_options{ciphers = ServerCiphers, + signature_algs = ServerSignAlgs, + supported_groups = ServerGroups0}, + session = #session{own_certificate = Cert}} = State0) -> ClientGroups0 = maps:get(elliptic_curves, Extensions, undefined), ClientGroups = get_supported_groups(ClientGroups0), @@ -398,11 +459,9 @@ handle_client_hello(#client_hello{cipher_suites = ClientCiphers, %% and a signature algorithm/certificate pair to authenticate itself to %% the client. Cipher = Maybe(select_cipher_suite(ClientCiphers, ServerCiphers)), - Group = Maybe(select_server_group(ServerGroups, ClientGroups)), + Groups = Maybe(select_common_groups(ServerGroups, ClientGroups)), Maybe(validate_key_share(ClientGroups, ClientShares)), - ClientPubKey = Maybe(get_client_public_key(Group, ClientShares)), - {PublicKeyAlgo, SignAlgo, SignHash} = get_certificate_params(Cert), %% Check if client supports signature algorithm of server certificate @@ -411,14 +470,25 @@ handle_client_hello(#client_hello{cipher_suites = ClientCiphers, %% Select signature algorithm (used in CertificateVerify message). SelectedSignAlg = Maybe(select_sign_algo(PublicKeyAlgo, ClientSignAlgs, ServerSignAlgs)), + %% Select client public key. If no public key found in ClientShares or + %% ClientShares is empty, trigger HelloRetryRequest as we were able + %% to find an acceptable set of parameters but the ClientHello does not + %% contain sufficient information. + {Group, ClientPubKey} = get_client_public_key(Groups, ClientShares), + %% Generate server_share KeyShare = ssl_cipher:generate_server_share(Group), - _Ret = #{cipher => Cipher, - group => Group, - sign_alg => SelectedSignAlg, - client_share => ClientPubKey, - key_share => KeyShare, - session_id => SessionId} + + State1 = update_start_state(State0, Cipher, KeyShare, SessionId, + Group, SelectedSignAlg, ClientPubKey), + + %% 4.1.4. Hello Retry Request + %% + %% The server will send this message in response to a ClientHello + %% message if it is able to find an acceptable set of parameters but the + %% ClientHello does not contain sufficient information to proceed with + %% the handshake. + Maybe(send_hello_retry_request(State1, ClientPubKey, KeyShare, SessionId)) %% TODO: %% - session handling @@ -430,41 +500,38 @@ handle_client_hello(#client_hello{cipher_suites = ClientCiphers, ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_groups); {Ref, illegal_parameter} -> ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER); - {Ref, {hello_retry_request, _Group0}} -> - %% TODO - ?ALERT_REC(?FATAL, ?INTERNAL_ERROR, "hello_retry_request not implemented"); {Ref, no_suitable_cipher} -> ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_cipher); {Ref, {insufficient_security, no_suitable_signature_algorithm}} -> - ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_signature_algorithm); + ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, "No suitable signature algorithm"); {Ref, {insufficient_security, no_suitable_public_key}} -> ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_public_key) end. -do_negotiated(#{client_share := ClientKey, - group := SelectedGroup, - sign_alg := SignatureScheme - } = Map, - #state{connection_states = ConnectionStates0, - session = #session{session_id = SessionId, - own_certificate = OwnCert}, - ssl_options = #ssl_options{} = _SslOpts, - key_share = KeyShare, - handshake_env = #handshake_env{tls_handshake_history = _HHistory0}, - connection_env = #connection_env{private_key = CertPrivateKey}, - static_env = #static_env{ - cert_db = CertDbHandle, - cert_db_ref = CertDbRef, - socket = _Socket, - transport_cb = _Transport} - } = State0) -> +do_negotiated(start_handshake, + #state{connection_states = ConnectionStates0, + session = #session{session_id = SessionId, + own_certificate = OwnCert, + ecc = SelectedGroup, + sign_alg = SignatureScheme, + dh_public_value = ClientKey}, + ssl_options = #ssl_options{} = SslOpts, + key_share = KeyShare, + handshake_env = #handshake_env{tls_handshake_history = _HHistory0}, + connection_env = #connection_env{private_key = CertPrivateKey}, + static_env = #static_env{ + cert_db = CertDbHandle, + cert_db_ref = CertDbRef, + socket = _Socket, + transport_cb = _Transport} + } = State0) -> {Ref,Maybe} = maybe(), try %% Create server_hello %% Extensions: supported_versions, key_share, (pre_shared_key) - ServerHello = server_hello(SessionId, KeyShare, ConnectionStates0, Map), + ServerHello = server_hello(server_hello, SessionId, KeyShare, ConnectionStates0), {State1, _} = tls_connection:send_handshake(ServerHello, State0), @@ -479,59 +546,71 @@ do_negotiated(#{client_share := ClientKey, %% Encode EncryptedExtensions State4 = tls_connection:queue_handshake(EncryptedExtensions, State3), + %% Create and send CertificateRequest ({verify, verify_peer}) + {State5, NextState} = maybe_send_certificate_request(State4, SslOpts), + %% Create Certificate Certificate = certificate(OwnCert, CertDbHandle, CertDbRef, <<>>, server), %% Encode Certificate - State5 = tls_connection:queue_handshake(Certificate, State4), + State6 = tls_connection:queue_handshake(Certificate, State5), %% Create CertificateVerify CertificateVerify = Maybe(certificate_verify(CertPrivateKey, SignatureScheme, - State5, server)), + State6, server)), %% Encode CertificateVerify - State6 = tls_connection:queue_handshake(CertificateVerify, State5), + State7 = tls_connection:queue_handshake(CertificateVerify, State6), %% Create Finished - Finished = finished(State6), + Finished = finished(State7), %% Encode Finished - State7 = tls_connection:queue_handshake(Finished, State6), + State8 = tls_connection:queue_handshake(Finished, State7), %% Send first flight - {State8, _} = tls_connection:send_handshake_flight(State7), + {State9, _} = tls_connection:send_handshake_flight(State8), - State8 + {State9, NextState} catch - {Ref, {state_not_implemented, State}} -> - %% TODO - ?ALERT_REC(?FATAL, ?INTERNAL_ERROR, {state_not_implemented, State}) + {Ref, badarg} -> + ?ALERT_REC(?FATAL, ?INTERNAL_ERROR, {digitally_sign, badarg}) end. -do_wait_finished(#change_cipher_spec{}, - #state{connection_states = _ConnectionStates0, - session = #session{session_id = _SessionId, - own_certificate = _OwnCert}, - ssl_options = #ssl_options{} = _SslOpts, - key_share = _KeyShare, - handshake_env = #handshake_env{tls_handshake_history = _HHistory0}, - static_env = #static_env{ - cert_db = _CertDbHandle, - cert_db_ref = _CertDbRef, - socket = _Socket, - transport_cb = _Transport} - } = State0) -> - %% {Ref,Maybe} = maybe(), - +do_wait_cert(#certificate_1_3{} = Certificate, State0) -> + {Ref,Maybe} = maybe(), try + Maybe(process_client_certificate(Certificate, State0)) + catch + {Ref, {certificate_required, State}} -> + {?ALERT_REC(?FATAL, ?CERTIFICATE_REQUIRED, certificate_required), State}; + {Ref, {{certificate_unknown, Reason}, State}} -> + {?ALERT_REC(?FATAL, ?CERTIFICATE_UNKNOWN, Reason), State}; + {Ref, {{internal_error, Reason}, State}} -> + {?ALERT_REC(?FATAL, ?INTERNAL_ERROR, Reason), State}; + {Ref, {{handshake_failure, Reason}, State}} -> + {?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, Reason), State}; + {#alert{} = Alert, State} -> + {Alert, State} + end. - State0 +do_wait_cv(#certificate_verify_1_3{} = CertificateVerify, State0) -> + {Ref,Maybe} = maybe(), + try + Maybe(verify_signature_algorithm(State0, CertificateVerify)), + Maybe(verify_certificate_verify(State0, CertificateVerify)) catch - {_Ref, {state_not_implemented, State}} -> - ?ALERT_REC(?FATAL, ?INTERNAL_ERROR, {state_not_implemented, State}) - end; + {Ref, {{bad_certificate, Reason}, State}} -> + {?ALERT_REC(?FATAL, ?BAD_CERTIFICATE, {bad_certificate, Reason}), State}; + {Ref, {badarg, State}} -> + {?ALERT_REC(?FATAL, ?INTERNAL_ERROR, {verify, badarg}), State}; + {Ref, {{handshake_failure, Reason}, State}} -> + {?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, {handshake_failure, Reason}), State} + end. + + do_wait_finished(#finished{verify_data = VerifyData}, #state{connection_states = _ConnectionStates0, session = #session{session_id = _SessionId, @@ -559,16 +638,19 @@ do_wait_finished(#finished{verify_data = VerifyData}, catch {Ref, decrypt_error} -> - ?ALERT_REC(?FATAL, ?DECRYPT_ERROR, decrypt_error); - {_, {state_not_implemented, State}} -> - %% TODO - ?ALERT_REC(?FATAL, ?INTERNAL_ERROR, {state_not_implemented, State}) + ?ALERT_REC(?FATAL, ?DECRYPT_ERROR, decrypt_error) end. %% TODO: Remove this function! -%% not_implemented(State) -> -%% {error, {state_not_implemented, State}}. +%% not_implemented(State, Reason) -> +%% {error, {not_implemented, State, Reason}}. +%% +%% not_implemented(update_secrets, State0, Reason) -> +%% State1 = calculate_traffic_secrets(State0), +%% State = ssl_record:step_encryption_state(State1), +%% {error, {not_implemented, State, Reason}}. + %% Recipients of Finished messages MUST verify that the contents are @@ -597,6 +679,188 @@ compare_verify_data(_, _) -> {error, decrypt_error}. +send_hello_retry_request(#state{connection_states = ConnectionStates0} = State0, + no_suitable_key, KeyShare, SessionId) -> + ServerHello = server_hello(hello_retry_request, SessionId, KeyShare, ConnectionStates0), + {State1, _} = tls_connection:send_handshake(ServerHello, State0), + + %% TODO: Fix handshake history! + State2 = replace_ch1_with_message_hash(State1), + + {ok, {State2, start}}; +send_hello_retry_request(State0, _, _, _) -> + %% Suitable key found. + {ok, {State0, negotiated}}. + + +maybe_send_certificate_request(State, #ssl_options{verify = verify_none}) -> + {State, wait_finished}; +maybe_send_certificate_request(State, #ssl_options{ + verify = verify_peer, + signature_algs = SignAlgs, + signature_algs_cert = SignAlgsCert}) -> + CertificateRequest = certificate_request(SignAlgs, SignAlgsCert), + {tls_connection:queue_handshake(CertificateRequest, State), wait_cert}. + + +process_client_certificate(#certificate_1_3{ + certificate_request_context = <<>>, + certificate_list = []}, + #state{ssl_options = + #ssl_options{ + fail_if_no_peer_cert = false}} = State) -> + {ok, {State, wait_finished}}; +process_client_certificate(#certificate_1_3{ + certificate_request_context = <<>>, + certificate_list = []}, + #state{ssl_options = + #ssl_options{ + fail_if_no_peer_cert = true}} = State0) -> + + %% At this point the client believes that the connection is up and starts using + %% its traffic secrets. In order to be able send an proper Alert to the client + %% the server should also change its connection state and use the traffic + %% secrets. + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {error, {certificate_required, State}}; +process_client_certificate(#certificate_1_3{certificate_list = Certs0}, + #state{ssl_options = + #ssl_options{signature_algs = SignAlgs, + signature_algs_cert = SignAlgsCert} = SslOptions, + static_env = + #static_env{ + role = Role, + host = Host, + cert_db = CertDbHandle, + cert_db_ref = CertDbRef, + crl_db = CRLDbHandle}} = State0) -> + %% TODO: handle extensions! + + %% Remove extensions from list of certificates! + Certs = convert_certificate_chain(Certs0), + case is_supported_signature_algorithm(Certs, SignAlgs, SignAlgsCert) of + true -> + case validate_certificate_chain(Certs, CertDbHandle, CertDbRef, + SslOptions, CRLDbHandle, Role, Host) of + {ok, {PeerCert, PublicKeyInfo}} -> + State = store_peer_cert(State0, PeerCert, PublicKeyInfo), + {ok, {State, wait_cv}}; + {error, Reason} -> + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {error, {Reason, State}}; + #alert{} = Alert -> + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {Alert, State} + end; + false -> + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {error, {{handshake_failure, + "Client certificate uses unsupported signature algorithm"}, State}} + end. + + +%% TODO: check whole chain! +is_supported_signature_algorithm(Certs, SignAlgs, undefined) -> + is_supported_signature_algorithm(Certs, SignAlgs); +is_supported_signature_algorithm(Certs, _, SignAlgsCert) -> + is_supported_signature_algorithm(Certs, SignAlgsCert). +%% +is_supported_signature_algorithm([BinCert|_], SignAlgs0) -> + #'OTPCertificate'{signatureAlgorithm = SignAlg} = + public_key:pkix_decode_cert(BinCert, otp), + SignAlgs = filter_tls13_algs(SignAlgs0), + Scheme = ssl_cipher:signature_algorithm_to_scheme(SignAlg), + lists:member(Scheme, SignAlgs). + + +validate_certificate_chain(Certs, CertDbHandle, CertDbRef, SslOptions, CRLDbHandle, Role, Host) -> + ServerName = ssl_handshake:server_name(SslOptions#ssl_options.server_name_indication, Host, Role), + [PeerCert | ChainCerts ] = Certs, + try + {TrustedCert, CertPath} = + ssl_certificate:trusted_cert_and_path(Certs, CertDbHandle, CertDbRef, + SslOptions#ssl_options.partial_chain), + ValidationFunAndState = + ssl_handshake:validation_fun_and_state(SslOptions#ssl_options.verify_fun, Role, + CertDbHandle, CertDbRef, ServerName, + SslOptions#ssl_options.customize_hostname_check, + SslOptions#ssl_options.crl_check, CRLDbHandle, CertPath), + Options = [{max_path_length, SslOptions#ssl_options.depth}, + {verify_fun, ValidationFunAndState}], + %% TODO: Validate if Certificate is using a supported signature algorithm + %% (signature_algs_cert)! + case public_key:pkix_path_validation(TrustedCert, CertPath, Options) of + {ok, {PublicKeyInfo,_}} -> + {ok, {PeerCert, PublicKeyInfo}}; + {error, Reason} -> + ssl_handshake:handle_path_validation_error(Reason, PeerCert, ChainCerts, + SslOptions, Options, + CertDbHandle, CertDbRef) + end + catch + error:{badmatch,{asn1, Asn1Reason}} -> + %% ASN-1 decode of certificate somehow failed + {error, {certificate_unknown, {failed_to_decode_certificate, Asn1Reason}}}; + error:OtherReason -> + {error, {internal_error, {unexpected_error, OtherReason}}} + end. + + +store_peer_cert(#state{session = Session, + handshake_env = HsEnv} = State, PeerCert, PublicKeyInfo) -> + State#state{session = Session#session{peer_certificate = PeerCert}, + handshake_env = HsEnv#handshake_env{public_key_info = PublicKeyInfo}}. + + +convert_certificate_chain(Certs) -> + Fun = fun(#certificate_entry{data = Data}) -> + {true, Data}; + (_) -> + false + end, + lists:filtermap(Fun, Certs). + + +%% 4.4.1. The Transcript Hash +%% +%% As an exception to this general rule, when the server responds to a +%% ClientHello with a HelloRetryRequest, the value of ClientHello1 is +%% replaced with a special synthetic handshake message of handshake type +%% "message_hash" containing Hash(ClientHello1). I.e., +%% +%% Transcript-Hash(ClientHello1, HelloRetryRequest, ... Mn) = +%% Hash(message_hash || /* Handshake type */ +%% 00 00 Hash.length || /* Handshake message length (bytes) */ +%% Hash(ClientHello1) || /* Hash of ClientHello1 */ +%% HelloRetryRequest || ... || Mn) +%% +%% NOTE: Hash.length is used in practice (openssl) and not message length! +%% It is most probably a fault in the RFC. +replace_ch1_with_message_hash(#state{connection_states = ConnectionStates, + handshake_env = + #handshake_env{ + tls_handshake_history = + {[HRR,CH1|HHistory], LM}} = HSEnv} = State0) -> + #{security_parameters := SecParamsR} = + ssl_record:pending_connection_state(ConnectionStates, read), + #security_parameters{prf_algorithm = HKDFAlgo} = SecParamsR, + MessageHash = message_hash(CH1, HKDFAlgo), + State0#state{handshake_env = + HSEnv#handshake_env{ + tls_handshake_history = + {[HRR,MessageHash|HHistory], LM}}}. + + +message_hash(ClientHello1, HKDFAlgo) -> + [?MESSAGE_HASH, + 0,0,ssl_cipher:hash_size(HKDFAlgo), + crypto:hash(HKDFAlgo, ClientHello1)]. + + calculate_handshake_secrets(ClientKey, SelectedGroup, KeyShare, #state{connection_states = ConnectionStates, handshake_env = @@ -648,10 +912,8 @@ calculate_traffic_secrets(#state{connection_states = ConnectionStates, MasterSecret = tls_v1:key_schedule(master_secret, HKDFAlgo, HandshakeSecret), - {Messages0, _} = HHistory, - - %% Drop Client Finish - [_|Messages] = Messages0, + %% Get the correct list messages for the handshake context. + Messages = get_handshake_context(HHistory), %% Calculate [sender]_application_traffic_secret_0 ClientAppTrafficSecret0 = @@ -680,6 +942,11 @@ get_private_key(#key_share_entry{ {_, PrivateKey}}) -> PrivateKey. +%% TODO: implement EC keys +get_public_key({?'rsaEncryption', PublicKey, _}) -> + PublicKey. + + %% X25519, X448 calculate_shared_secret(OthersKey, MyKey, Group) when is_binary(OthersKey) andalso is_binary(MyKey) andalso @@ -721,6 +988,29 @@ update_connection_state(ConnectionState = #{security_parameters := SecurityParam cipher_state => cipher_init(Key, IV, FinishedKey)}. +update_start_state(#state{connection_states = ConnectionStates0, + connection_env = CEnv, + session = Session} = State, + Cipher, KeyShare, SessionId, + Group, SelectedSignAlg, ClientPubKey) -> + #{security_parameters := SecParamsR0} = PendingRead = + maps:get(pending_read, ConnectionStates0), + #{security_parameters := SecParamsW0} = PendingWrite = + maps:get(pending_write, ConnectionStates0), + SecParamsR = ssl_cipher:security_parameters_1_3(SecParamsR0, Cipher), + SecParamsW = ssl_cipher:security_parameters_1_3(SecParamsW0, Cipher), + ConnectionStates = + ConnectionStates0#{pending_read => PendingRead#{security_parameters => SecParamsR}, + pending_write => PendingWrite#{security_parameters => SecParamsW}}, + State#state{connection_states = ConnectionStates, + key_share = KeyShare, + session = Session#session{session_id = SessionId, + ecc = Group, + sign_alg = SelectedSignAlg, + dh_public_value = ClientPubKey, + cipher_suite = Cipher}, + connection_env = CEnv#connection_env{negotiated_version = {3,4}}}. + cipher_init(Key, IV, FinishedKey) -> #cipher_state{key = Key, @@ -729,18 +1019,139 @@ cipher_init(Key, IV, FinishedKey) -> tag_len = 16}. +%% Get handshake context for verification of CertificateVerify. +%% +%% Verify CertificateVerify: +%% ClientHello (client) (1) +%% ServerHello (server) (2) +%% EncryptedExtensions (server) (8) +%% CertificateRequest (server) (13) +%% Certificate (server) (11) +%% CertificateVerify (server) (15) +%% Finished (server) (20) +%% Certificate (client) (11) +%% CertificateVerify (client) (15) - Drop! Not included in calculations! +get_handshake_context_cv({[<<15,_/binary>>|Messages], _}) -> + Messages. + + +%% Get handshake context for traffic key calculation. +%% +%% Client is authenticated with certificate: +%% ClientHello (client) (1) +%% ServerHello (server) (2) +%% EncryptedExtensions (server) (8) +%% CertificateRequest (server) (13) +%% Certificate (server) (11) +%% CertificateVerify (server) (15) +%% Finished (server) (20) +%% Certificate (client) (11) - Drop! Not included in calculations! +%% CertificateVerify (client) (15) - Drop! Not included in calculations! +%% Finished (client) (20) - Drop! Not included in calculations! +%% +%% Client is authenticated but sends empty certificate: +%% ClientHello (client) (1) +%% ServerHello (server) (2) +%% EncryptedExtensions (server) (8) +%% CertificateRequest (server) (13) +%% Certificate (server) (11) +%% CertificateVerify (server) (15) +%% Finished (server) (20) +%% Certificate (client) (11) - Drop! Not included in calculations! +%% Finished (client) (20) - Drop! Not included in calculations! +%% +%% Client is not authenticated: +%% ClientHello (client) (1) +%% ServerHello (server) (2) +%% EncryptedExtensions (server) (8) +%% Certificate (server) (11) +%% CertificateVerify (server) (15) +%% Finished (server) (20) +%% Finished (client) (20) - Drop! Not included in calculations! +%% +%% Drop all client messages from the front of the iolist using the property that +%% incoming messages are binaries. +get_handshake_context({Messages, _}) -> + get_handshake_context(Messages); +get_handshake_context([H|T]) when is_binary(H) -> + get_handshake_context(T); +get_handshake_context(L) -> + L. + + +%% If sent by a client, the signature algorithm used in the signature +%% MUST be one of those present in the supported_signature_algorithms +%% field of the "signature_algorithms" extension in the +%% CertificateRequest message. +verify_signature_algorithm(#state{ssl_options = + #ssl_options{ + signature_algs = ServerSignAlgs}} = State0, + #certificate_verify_1_3{algorithm = ClientSignAlg}) -> + case lists:member(ClientSignAlg, ServerSignAlgs) of + true -> + ok; + false -> + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {error, {{handshake_failure, + "CertificateVerify uses unsupported signature algorithm"}, State}} + end. + + +verify_certificate_verify(#state{connection_states = ConnectionStates, + handshake_env = + #handshake_env{ + public_key_info = PublicKeyInfo, + tls_handshake_history = HHistory}} = State0, + #certificate_verify_1_3{ + algorithm = SignatureScheme, + signature = Signature}) -> + #{security_parameters := SecParamsR} = + ssl_record:pending_connection_state(ConnectionStates, write), + #security_parameters{prf_algorithm = HKDFAlgo} = SecParamsR, + + {HashAlgo, _, _} = + ssl_cipher:scheme_to_components(SignatureScheme), + + Messages = get_handshake_context_cv(HHistory), + + Context = lists:reverse(Messages), + + %% Transcript-Hash uses the HKDF hash function defined by the cipher suite. + THash = tls_v1:transcript_hash(Context, HKDFAlgo), + + PublicKey = get_public_key(PublicKeyInfo), + + %% Digital signatures use the hash function defined by the selected signature + %% scheme. + case verify(THash, <<"TLS 1.3, client CertificateVerify">>, + HashAlgo, Signature, PublicKey) of + {ok, true} -> + {ok, {State0, wait_finished}}; + {ok, false} -> + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {error, {{handshake_failure, "Failed to verify CertificateVerify"}, State}}; + {error, badarg} -> + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {error, {badarg, State}} + end. + + %% If there is no overlap between the received %% "supported_groups" and the groups supported by the server, then the %% server MUST abort the handshake with a "handshake_failure" or an %% "insufficient_security" alert. -select_server_group(_, []) -> +select_common_groups(_, []) -> {error, {insufficient_security, no_suitable_groups}}; -select_server_group(ServerGroups, [C|ClientGroups]) -> - case lists:member(C, ServerGroups) of - true -> - {ok, C}; - false -> - select_server_group(ServerGroups, ClientGroups) +select_common_groups(ServerGroups, ClientGroups) -> + Fun = fun(E) -> lists:member(E, ClientGroups) end, + case lists:filter(Fun, ServerGroups) of + [] -> + {error, {insufficient_security, no_suitable_groups}}; + L -> + {ok, L} end. @@ -771,20 +1182,36 @@ validate_key_share([_|ClientGroups], [_|_] = ClientShares) -> validate_key_share(ClientGroups, ClientShares). -get_client_public_key(Group, ClientShares) -> +get_client_public_key([Group|_] = Groups, ClientShares) -> + get_client_public_key(Groups, ClientShares, Group). +%% +get_client_public_key(_, [], PreferredGroup) -> + {PreferredGroup, no_suitable_key}; +get_client_public_key([], _, PreferredGroup) -> + {PreferredGroup, no_suitable_key}; +get_client_public_key([Group|Groups], ClientShares, PreferredGroup) -> case lists:keysearch(Group, 2, ClientShares) of {value, {_, _, ClientPublicKey}} -> - {ok, ClientPublicKey}; + {Group, ClientPublicKey}; false -> - %% 4.1.4. Hello Retry Request - %% - %% The server will send this message in response to a ClientHello - %% message if it is able to find an acceptable set of parameters but the - %% ClientHello does not contain sufficient information to proceed with - %% the handshake. - {error, {hello_retry_request, Group}} + get_client_public_key(Groups, ClientShares, PreferredGroup) end. + +%% get_client_public_key(Group, ClientShares) -> +%% case lists:keysearch(Group, 2, ClientShares) of +%% {value, {_, _, ClientPublicKey}} -> +%% ClientPublicKey; +%% false -> +%% %% 4.1.4. Hello Retry Request +%% %% +%% %% The server will send this message in response to a ClientHello +%% %% message if it is able to find an acceptable set of parameters but the +%% %% ClientHello does not contain sufficient information to proceed with +%% %% the handshake. +%% no_suitable_key +%% end. + select_cipher_suite([], _) -> {error, no_suitable_cipher}; select_cipher_suite([Cipher|ClientCiphers], ServerCiphers) -> diff --git a/lib/ssl/src/tls_record_1_3.erl b/lib/ssl/src/tls_record_1_3.erl index 05acc08392..97331e1510 100644 --- a/lib/ssl/src/tls_record_1_3.erl +++ b/lib/ssl/src/tls_record_1_3.erl @@ -124,6 +124,20 @@ decode_cipher_text(#ssl_tls{type = ?OPAQUE_TYPE, {decode_inner_plaintext(PlainFragment), ConnectionStates} end; + +%% RFC8446 - TLS 1.3 (OpenSSL compatibility) +%% Handle unencrypted Alerts from openssl s_client when server's +%% connection states are already stepped into traffic encryption. +%% (E.g. openssl s_client receives a CertificateRequest with +%% a signature_algorithms_cert extension that does not contain +%% the signature algorithm of the client's certificate.) +decode_cipher_text(#ssl_tls{type = ?ALERT, + version = ?LEGACY_VERSION, + fragment = <<2,47>>}, + ConnectionStates0) -> + {#ssl_tls{type = ?ALERT, + version = {3,4}, %% Internally use real version + fragment = <<2,47>>}, ConnectionStates0}; %% RFC8446 - TLS 1.3 %% D.4. Middlebox Compatibility Mode %% - If not offering early data, the client sends a dummy @@ -139,7 +153,6 @@ decode_cipher_text(#ssl_tls{type = ?CHANGE_CIPHER_SPEC, {#ssl_tls{type = ?CHANGE_CIPHER_SPEC, version = {3,4}, %% Internally use real version fragment = <<1>>}, ConnectionStates0}; - decode_cipher_text(#ssl_tls{type = Type, version = ?LEGACY_VERSION, fragment = CipherFragment}, diff --git a/lib/ssl/src/tls_socket.erl b/lib/ssl/src/tls_socket.erl index a391bc53de..6c32e6fa04 100644 --- a/lib/ssl/src/tls_socket.erl +++ b/lib/ssl/src/tls_socket.erl @@ -32,6 +32,7 @@ emulated_socket_options/2, get_emulated_opts/1, set_emulated_opts/2, get_all_opts/1, handle_call/3, handle_cast/2, handle_info/2, code_change/3]). +-export([update_active_n/2]). -record(state, { emulated_opts, @@ -45,18 +46,20 @@ send(Transport, Socket, Data) -> Transport:send(Socket, Data). -listen(Transport, Port, #config{transport_info = {Transport, _, _, _}, +listen(Transport, Port, #config{transport_info = {Transport, _, _, _, _}, inet_user = Options, ssl = SslOpts, emulated = EmOpts} = Config) -> case Transport:listen(Port, Options ++ internal_inet_values()) of {ok, ListenSocket} -> {ok, Tracker} = inherit_tracker(ListenSocket, EmOpts, SslOpts), - {ok, #sslsocket{pid = {ListenSocket, Config#config{emulated = Tracker}}}}; + Socket = #sslsocket{pid = {ListenSocket, Config#config{emulated = Tracker}}}, + check_active_n(EmOpts, Socket), + {ok, Socket}; Err = {error, _} -> Err end. -accept(ListenSocket, #config{transport_info = {Transport,_,_,_} = CbInfo, +accept(ListenSocket, #config{transport_info = {Transport,_,_,_,_} = CbInfo, connection_cb = ConnectionCb, ssl = SslOpts, emulated = Tracker}, Timeout) -> @@ -77,7 +80,7 @@ accept(ListenSocket, #config{transport_info = {Transport,_,_,_} = CbInfo, {error, Reason} end. -upgrade(Socket, #config{transport_info = {Transport,_,_,_}= CbInfo, +upgrade(Socket, #config{transport_info = {Transport,_,_,_,_}= CbInfo, ssl = SslOptions, emulated = EmOpts, connection_cb = ConnectionCb}, Timeout) -> ok = setopts(Transport, Socket, tls_socket:internal_inet_values()), @@ -95,7 +98,7 @@ connect(Address, Port, #config{transport_info = CbInfo, inet_user = UserOpts, ssl = SslOpts, emulated = EmOpts, inet_ssl = SocketOpts, connection_cb = ConnetionCb}, Timeout) -> - {Transport, _, _, _} = CbInfo, + {Transport, _, _, _, _} = CbInfo, try Transport:connect(Address, Port, SocketOpts, Timeout) of {ok, Socket} -> ssl_connection:connect(ConnetionCb, Address, Port, Socket, @@ -117,14 +120,16 @@ socket(Pids, Transport, Socket, ConnectionCb, Tracker) -> #sslsocket{pid = Pids, %% "The name "fd" is keept for backwards compatibility fd = {Transport, Socket, ConnectionCb, Tracker}}. -setopts(gen_tcp, #sslsocket{pid = {ListenSocket, #config{emulated = Tracker}}}, Options) -> +setopts(gen_tcp, Socket = #sslsocket{pid = {ListenSocket, #config{emulated = Tracker}}}, Options) -> {SockOpts, EmulatedOpts} = split_options(Options), ok = set_emulated_opts(Tracker, EmulatedOpts), + check_active_n(EmulatedOpts, Socket), inet:setopts(ListenSocket, SockOpts); -setopts(_, #sslsocket{pid = {ListenSocket, #config{transport_info = {Transport,_,_,_}, +setopts(_, Socket = #sslsocket{pid = {ListenSocket, #config{transport_info = {Transport,_,_,_,_}, emulated = Tracker}}}, Options) -> {SockOpts, EmulatedOpts} = split_options(Options), ok = set_emulated_opts(Tracker, EmulatedOpts), + check_active_n(EmulatedOpts, Socket), Transport:setopts(ListenSocket, SockOpts); %%% Following clauses will not be called for emulated options, they are handled in the connection process setopts(gen_tcp, Socket, Options) -> @@ -132,6 +137,31 @@ setopts(gen_tcp, Socket, Options) -> setopts(Transport, Socket, Options) -> Transport:setopts(Socket, Options). +check_active_n(EmulatedOpts, Socket = #sslsocket{pid = {_, #config{emulated = Tracker}}}) -> + %% We check the resulting options to send an ssl_passive message if necessary. + case proplists:lookup(active, EmulatedOpts) of + %% The provided value is out of bound. + {_, N} when is_integer(N), N < -32768 -> + throw(einval); + {_, N} when is_integer(N), N > 32767 -> + throw(einval); + {_, N} when is_integer(N) -> + case get_emulated_opts(Tracker, [active]) of + [{_, false}] -> + self() ! {ssl_passive, Socket}, + ok; + %% The result of the addition is out of bound. + [{_, A}] when is_integer(A), A < -32768 -> + throw(einval); + [{_, A}] when is_integer(A), A > 32767 -> + throw(einval); + _ -> + ok + end; + _ -> + ok + end. + getopts(gen_tcp, #sslsocket{pid = {ListenSocket, #config{emulated = Tracker}}}, Options) -> {SockOptNames, EmulatedOptNames} = split_options(Options), EmulatedOpts = get_emulated_opts(Tracker, EmulatedOptNames), @@ -209,7 +239,7 @@ start_link(Port, SockOpts, SslOpts) -> init([Port, Opts, SslOpts]) -> process_flag(trap_exit, true), true = link(Port), - {ok, #state{emulated_opts = Opts, port = Port, ssl_opts = SslOpts}}. + {ok, #state{emulated_opts = do_set_emulated_opts(Opts, []), port = Port, ssl_opts = SslOpts}}. %%-------------------------------------------------------------------- -spec handle_call(msg(), from(), #state{}) -> {reply, reply(), #state{}}. @@ -304,9 +334,24 @@ split_options([Name | Opts], Emu, SocketOptNames, EmuOptNames) -> do_set_emulated_opts([], Opts) -> Opts; +do_set_emulated_opts([{active, N0} | Rest], Opts) when is_integer(N0) -> + N = update_active_n(N0, proplists:get_value(active, Opts, false)), + do_set_emulated_opts(Rest, [{active, N} | proplists:delete(active, Opts)]); do_set_emulated_opts([{Name,_} = Opt | Rest], Opts) -> do_set_emulated_opts(Rest, [Opt | proplists:delete(Name, Opts)]). +update_active_n(New, Current) -> + if + is_integer(Current), New + Current =< 0 -> + false; + is_integer(Current) -> + New + Current; + New =< 0 -> + false; + true -> + New + end. + get_socket_opts(_, [], _) -> []; get_socket_opts(ListenSocket, SockOptNames, Cb) -> @@ -366,6 +411,9 @@ validate_inet_option(header, Value) when not is_integer(Value) -> throw({error, {options, {header,Value}}}); validate_inet_option(active, Value) + when Value >= -32768, Value =< 32767 -> + ok; +validate_inet_option(active, Value) when Value =/= true, Value =/= false, Value =/= once -> throw({error, {options, {active,Value}}}); validate_inet_option(_, _) -> diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index f109d85e49..cbc0c2e70b 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -167,6 +167,7 @@ api_tests() -> accept_pool, prf, socket_options, + active_n, cipher_suites, handshake_continue, handshake_continue_timeout, @@ -246,6 +247,7 @@ error_handling_tests()-> [close_transport_accept, recv_active, recv_active_once, + recv_active_n, recv_error_handling, call_in_error_state, close_in_error_state, @@ -277,7 +279,18 @@ tls13_test_group() -> tls_record_1_3_encode_decode, tls13_finished_verify_data, tls13_1_RTT_handshake, - tls13_basic_ssl_s_client]. + tls13_basic_ssl_server_openssl_client, + tls13_custom_groups_ssl_server_openssl_client, + tls13_hello_retry_request_ssl_server_openssl_client, + tls13_client_auth_empty_cert_alert_ssl_server_openssl_client, + tls13_client_auth_empty_cert_ssl_server_openssl_client, + tls13_client_auth_ssl_server_openssl_client, + tls13_hrr_client_auth_empty_cert_alert_ssl_server_openssl_client, + tls13_hrr_client_auth_empty_cert_ssl_server_openssl_client, + tls13_hrr_client_auth_ssl_server_openssl_client, + tls13_unsupported_sign_algo_client_auth_ssl_server_openssl_client, + tls13_unsupported_sign_algo_cert_client_auth_ssl_server_openssl_client, + tls13_connection_information]. %%-------------------------------------------------------------------- init_per_suite(Config0) -> @@ -1991,7 +2004,7 @@ recv_active(Config) when is_list(Config) -> %%-------------------------------------------------------------------- recv_active_once() -> - [{doc,"Test recv on active socket"}]. + [{doc,"Test recv on active (once) socket"}]. recv_active_once(Config) when is_list(Config) -> ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), @@ -2016,6 +2029,178 @@ recv_active_once(Config) when is_list(Config) -> ssl_test_lib:close(Client). %%-------------------------------------------------------------------- +recv_active_n() -> + [{doc,"Test recv on active (n) socket"}]. + +recv_active_n(Config) when is_list(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + Server = + ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, try_recv_active_once, []}}, + {options, [{active, 1} | ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + Client = + ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, try_recv_active_once, []}}, + {options, [{active, 1} | ClientOpts]}]), + + ssl_test_lib:check_result(Server, ok, Client, ok), + + ssl_test_lib:close(Server), + ssl_test_lib:close(Client). + +%%-------------------------------------------------------------------- +%% Test case adapted from gen_tcp_misc_SUITE. +active_n() -> + [{doc,"Test {active,N} option"}]. + +active_n(Config) when is_list(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + Port = ssl_test_lib:inet_port(node()), + N = 3, + LS = ok(ssl:listen(Port, [{active,N}|ServerOpts])), + [{active,N}] = ok(ssl:getopts(LS, [active])), + active_n_common(LS, N), + Self = self(), + spawn_link(fun() -> + S0 = ok(ssl:transport_accept(LS)), + {ok, S} = ssl:handshake(S0), + ok = ssl:setopts(S, [{active,N}]), + [{active,N}] = ok(ssl:getopts(S, [active])), + ssl:controlling_process(S, Self), + Self ! {server, S} + end), + C = ok(ssl:connect("localhost", Port, [{active,N}|ClientOpts])), + [{active,N}] = ok(ssl:getopts(C, [active])), + S = receive + {server, S0} -> S0 + after + 1000 -> + exit({error, connect}) + end, + active_n_common(C, N), + active_n_common(S, N), + ok = ssl:setopts(C, [{active,N}]), + ok = ssl:setopts(S, [{active,N}]), + ReceiveMsg = fun(Socket, Msg) -> + receive + {ssl,Socket,Msg} -> + ok; + {ssl,Socket,Begin} -> + receive + {ssl,Socket,End} -> + Msg = Begin ++ End, + ok + after 1000 -> + exit(timeout) + end + after 1000 -> + exit(timeout) + end + end, + repeat(3, fun(I) -> + Msg = "message "++integer_to_list(I), + ok = ssl:send(C, Msg), + ReceiveMsg(S, Msg), + ok = ssl:send(S, Msg), + ReceiveMsg(C, Msg) + end), + receive + {ssl_passive,S} -> + [{active,false}] = ok(ssl:getopts(S, [active])) + after + 1000 -> + exit({error,ssl_passive}) + end, + receive + {ssl_passive,C} -> + [{active,false}] = ok(ssl:getopts(C, [active])) + after + 1000 -> + exit({error,ssl_passive}) + end, + LS2 = ok(ssl:listen(0, [{active,0}])), + receive + {ssl_passive,LS2} -> + [{active,false}] = ok(ssl:getopts(LS2, [active])) + after + 1000 -> + exit({error,ssl_passive}) + end, + ok = ssl:close(LS2), + ok = ssl:close(C), + ok = ssl:close(S), + ok = ssl:close(LS), + ok. + +active_n_common(S, N) -> + ok = ssl:setopts(S, [{active,-N}]), + receive + {ssl_passive, S} -> ok + after + 1000 -> + error({error,ssl_passive_failure}) + end, + [{active,false}] = ok(ssl:getopts(S, [active])), + ok = ssl:setopts(S, [{active,0}]), + receive + {ssl_passive, S} -> ok + after + 1000 -> + error({error,ssl_passive_failure}) + end, + ok = ssl:setopts(S, [{active,32767}]), + {error,{options,_}} = ssl:setopts(S, [{active,1}]), + {error,{options,_}} = ssl:setopts(S, [{active,-32769}]), + ok = ssl:setopts(S, [{active,-32768}]), + receive + {ssl_passive, S} -> ok + after + 1000 -> + error({error,ssl_passive_failure}) + end, + [{active,false}] = ok(ssl:getopts(S, [active])), + ok = ssl:setopts(S, [{active,N}]), + ok = ssl:setopts(S, [{active,true}]), + [{active,true}] = ok(ssl:getopts(S, [active])), + receive + _ -> error({error,active_n}) + after + 0 -> + ok + end, + ok = ssl:setopts(S, [{active,N}]), + ok = ssl:setopts(S, [{active,once}]), + [{active,once}] = ok(ssl:getopts(S, [active])), + receive + _ -> error({error,active_n}) + after + 0 -> + ok + end, + {error,{options,_}} = ssl:setopts(S, [{active,32768}]), + ok = ssl:setopts(S, [{active,false}]), + [{active,false}] = ok(ssl:getopts(S, [active])), + ok. + +ok({ok,V}) -> V. + +repeat(N, Fun) -> + repeat(N, N, Fun). + +repeat(N, T, Fun) when is_integer(N), N > 0 -> + Fun(T-N), + repeat(N-1, T, Fun); +repeat(_, _, _) -> + ok. + +%%-------------------------------------------------------------------- dh_params() -> [{doc,"Test to specify DH-params file in server."}]. @@ -3902,7 +4087,7 @@ tls_tcp_error_propagation_in_active_mode(Config) when is_list(Config) -> [_, _,_, _, Prop] = StatusInfo, State = ssl_test_lib:state(Prop), StaticEnv = element(2, State), - Socket = element(10, StaticEnv), + Socket = element(11, StaticEnv), %% Fake tcp error Pid ! {tcp_error, Socket, etimedout}, @@ -5341,10 +5526,10 @@ tls13_finished_verify_data(_Config) -> FinishedKey = tls_v1:finished_key(BaseKey, sha256), VerifyData = tls_v1:finished_verify_data(FinishedKey, sha256, Messages). -tls13_basic_ssl_s_client() -> +tls13_basic_ssl_server_openssl_client() -> [{doc,"Test TLS 1.3 basic connection between ssl server and openssl s_client"}]. -tls13_basic_ssl_s_client(Config) -> +tls13_basic_ssl_server_openssl_client(Config) -> ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), %% Set versions @@ -5363,6 +5548,330 @@ tls13_basic_ssl_s_client(Config) -> ssl_test_lib:close(Server), ssl_test_lib:close_port(Client). +tls13_custom_groups_ssl_server_openssl_client() -> + [{doc,"Test that ssl server can select a common group for key-exchange"}]. + +tls13_custom_groups_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {supported_groups, [x448, secp256r1, secp384r1]}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + ClientOpts = [{groups,"P-384:P-256:X25519"}|ClientOpts0], + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + +tls13_hello_retry_request_ssl_server_openssl_client() -> + [{doc,"Test that ssl server can request a new group when the client's first key share" + "is not supported"}]. + +tls13_hello_retry_request_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {supported_groups, [x448, x25519]}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + ClientOpts = [{groups,"P-256:X25519"}|ClientOpts0], + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + +tls13_client_auth_empty_cert_alert_ssl_server_openssl_client() -> + [{doc,"TLS 1.3: Test client authentication when client sends an empty certificate and fail_if_no_peer_cert is set to true."}]. + +tls13_client_auth_empty_cert_alert_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + %% Delete Client Cert and Key + ClientOpts1 = proplists:delete(certfile, ClientOpts0), + ClientOpts = proplists:delete(keyfile, ClientOpts1), + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + {fail_if_no_peer_cert, true}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, + {error, + {tls_alert, + {certificate_required, + "received SERVER ALERT: Fatal - Certificate required - certificate_required"}}}), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + +tls13_client_auth_empty_cert_ssl_server_openssl_client() -> + [{doc,"TLS 1.3: Test client authentication when client sends an empty certificate and fail_if_no_peer_cert is set to false."}]. + +tls13_client_auth_empty_cert_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + %% Delete Client Cert and Key + ClientOpts1 = proplists:delete(certfile, ClientOpts0), + ClientOpts = proplists:delete(keyfile, ClientOpts1), + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + {fail_if_no_peer_cert, false}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +tls13_client_auth_ssl_server_openssl_client() -> + [{doc,"TLS 1.3: Test client authentication."}]. + +tls13_client_auth_ssl_server_openssl_client(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + {fail_if_no_peer_cert, true}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +tls13_hrr_client_auth_empty_cert_alert_ssl_server_openssl_client() -> + [{doc,"TLS 1.3 (HelloRetryRequest): Test client authentication when client sends an empty certificate and fail_if_no_peer_cert is set to true."}]. + +tls13_hrr_client_auth_empty_cert_alert_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + %% Delete Client Cert and Key + ClientOpts1 = proplists:delete(certfile, ClientOpts0), + ClientOpts2 = proplists:delete(keyfile, ClientOpts1), + ClientOpts = [{groups,"P-256:X25519"}|ClientOpts2], + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + {fail_if_no_peer_cert, true}, + {supported_groups, [x448, x25519]}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, + {error, + {tls_alert, + {certificate_required, + "received SERVER ALERT: Fatal - Certificate required - certificate_required"}}}), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +tls13_hrr_client_auth_empty_cert_ssl_server_openssl_client() -> + [{doc,"TLS 1.3 (HelloRetryRequest): Test client authentication when client sends an empty certificate and fail_if_no_peer_cert is set to false."}]. + +tls13_hrr_client_auth_empty_cert_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + %% Delete Client Cert and Key + ClientOpts1 = proplists:delete(certfile, ClientOpts0), + ClientOpts2 = proplists:delete(keyfile, ClientOpts1), + ClientOpts = [{groups,"P-256:X25519"}|ClientOpts2], + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + {fail_if_no_peer_cert, false}, + {supported_groups, [x448, x25519]}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +tls13_hrr_client_auth_ssl_server_openssl_client() -> + [{doc,"TLS 1.3 (HelloRetryRequest): Test client authentication."}]. + +tls13_hrr_client_auth_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ClientOpts = [{groups,"P-256:X25519"}|ClientOpts0], + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + {fail_if_no_peer_cert, true}, + {supported_groups, [x448, x25519]}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +tls13_unsupported_sign_algo_client_auth_ssl_server_openssl_client() -> + [{doc,"TLS 1.3: Test client authentication with unsupported signature_algorithm"}]. + +tls13_unsupported_sign_algo_client_auth_ssl_server_openssl_client(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + %% Skip rsa_pkcs1_sha256! + {signature_algs, [rsa_pkcs1_sha384, rsa_pkcs1_sha512]}, + {fail_if_no_peer_cert, true}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result( + Server, + {error, + {tls_alert, + {insufficient_security, + "received SERVER ALERT: Fatal - Insufficient Security - " + "\"No suitable signature algorithm\""}}}), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +%% Triggers Client Alert as openssl s_client does not have a certificate with a +%% signature algorithm supported by the server (signature_algorithms_cert extension +%% of CertificateRequest does not contain the algorithm of the client certificate). +tls13_unsupported_sign_algo_cert_client_auth_ssl_server_openssl_client() -> + [{doc,"TLS 1.3: Test client authentication with unsupported signature_algorithm_cert"}]. + +tls13_unsupported_sign_algo_cert_client_auth_ssl_server_openssl_client(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {log_level, debug}, + {verify, verify_peer}, + {signature_algs, [rsa_pkcs1_sha256, rsa_pkcs1_sha384, rsa_pss_rsae_sha256]}, + %% Skip rsa_pkcs1_sha256! + {signature_algs_cert, [rsa_pkcs1_sha384, rsa_pkcs1_sha512]}, + {fail_if_no_peer_cert, true}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result( + Server, + {error, + {tls_alert, + {illegal_parameter, + "received CLIENT ALERT: Fatal - Illegal Parameter"}}}), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +tls13_connection_information() -> + [{doc,"Test the API function ssl:connection_information/1 in a TLS 1.3 connection"}]. + +tls13_connection_information(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, connection_information_result, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index c921dcae4c..7f8e81dbd8 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -1110,11 +1110,26 @@ start_basic_client(openssl, Version, Port, ClientOpts) -> Cert = proplists:get_value(certfile, ClientOpts), Key = proplists:get_value(keyfile, ClientOpts), CA = proplists:get_value(cacertfile, ClientOpts), + Groups0 = proplists:get_value(groups, ClientOpts), Exe = "openssl", - Args = ["s_client", "-verify", "2", "-port", integer_to_list(Port), + Args0 = ["s_client", "-verify", "2", "-port", integer_to_list(Port), ssl_test_lib:version_flag(Version), - "-cert", Cert, "-CAfile", CA, - "-key", Key, "-host","localhost", "-msg", "-debug"], + "-CAfile", CA, "-host", "localhost", "-msg", "-debug"], + Args1 = + case Groups0 of + undefined -> + Args0; + G -> + Args0 ++ ["-groups", G] + end, + Args = + case {Cert, Key} of + {C, K} when C =:= undefined orelse + K =:= undefined -> + Args1; + {C, K} -> + Args1 ++ ["-cert", C, "-key", K] + end, OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), true = port_command(OpenSslPort, "Hello world"), diff --git a/lib/ssl/vsn.mk b/lib/ssl/vsn.mk index 3527062a8a..c4bcc1560c 100644 --- a/lib/ssl/vsn.mk +++ b/lib/ssl/vsn.mk @@ -1 +1 @@ -SSL_VSN = 9.1.2 +SSL_VSN = 9.2.1 diff --git a/lib/stdlib/doc/src/beam_lib.xml b/lib/stdlib/doc/src/beam_lib.xml index 8bb4cf9101..bb44ca3201 100644 --- a/lib/stdlib/doc/src/beam_lib.xml +++ b/lib/stdlib/doc/src/beam_lib.xml @@ -470,6 +470,18 @@ CryptoKeyFun(clear) -> term()</code> </func> <func> + <name name="strip" arity="2" since=""/> + <fsummary>Remove chunks not needed by the loader from a BEAM file. + </fsummary> + <desc> + <p>Removes all chunks from a BEAM + file except those needed by the loader or passed in. In particular, + the debug information (chunk <c>debug_info</c> and <c>abstract_code</c>) + is removed.</p> + </desc> + </func> + + <func> <name name="strip_files" arity="1" since=""/> <fsummary>Removes chunks not needed by the loader from BEAM files. </fsummary> @@ -483,6 +495,19 @@ CryptoKeyFun(clear) -> term()</code> </func> <func> + <name name="strip_files" arity="2" since=""/> + <fsummary>Removes chunks not needed by the loader from BEAM files. + </fsummary> + <desc> + <p>Removes all chunks except + those needed by the loader or passed in from BEAM files. In particular, + the debug information (chunk <c>debug_info</c> and <c>abstract_code</c>) + is removed. The returned list contains one element for each + specified filename, in the same order as in <c>Files</c>.</p> + </desc> + </func> + + <func> <name name="strip_release" arity="1" since=""/> <fsummary>Remove chunks not needed by the loader from all BEAM files of a release.</fsummary> @@ -497,6 +522,20 @@ CryptoKeyFun(clear) -> term()</code> </func> <func> + <name name="strip_release" arity="2" since=""/> + <fsummary>Remove chunks not needed by the loader from all BEAM files of + a release.</fsummary> + <desc> + <p>Removes all chunks + except those needed by the loader or passed in from the BEAM files of a + release. <c><anno>Dir</anno></c> is to be the installation root + directory. For example, the current OTP release can be + stripped with the call + <c>beam_lib:strip_release(code:root_dir())</c>.</p> + </desc> + </func> + + <func> <name name="version" arity="1" since=""/> <fsummary>Read the module version of the BEAM file.</fsummary> <desc> diff --git a/lib/stdlib/doc/src/ets.xml b/lib/stdlib/doc/src/ets.xml index d2ac6a75b1..2cb677785d 100644 --- a/lib/stdlib/doc/src/ets.xml +++ b/lib/stdlib/doc/src/ets.xml @@ -642,12 +642,11 @@ Error: fun containing local Erlang function calls <p><marker id="info_2_safe_fixed_monotonic_time"/></p> <p><c>Item=safe_fixed|safe_fixed_monotonic_time, Value={FixationTime,Info}|false</c></p> - <p>If the table has been fixed using + <p>If the table is fixed using <seealso marker="#safe_fixtable/2"> <c>safe_fixtable/2</c></seealso>, the call returns a tuple where <c>FixationTime</c> is the - time when the table was first fixed by a process, which either - is or is not one of the processes it is fixed by now.</p> + last time when the table changed from unfixed to fixed.</p> <p>The format and value of <c>FixationTime</c> depends on <c>Item</c>:</p> <taglist> @@ -679,8 +678,15 @@ Error: fun containing local Erlang function calls table is fixed by now. <c>RefCount</c> is the value of the reference counter and it keeps track of how many times the table has been fixed by the process.</p> - <p>If the table never has been fixed, the call returns - <c>false</c>.</p> + <p>Table fixations are not limited to <seealso marker="#safe_fixtable/2"> + <c>safe_fixtable/2</c></seealso>. Temporary fixations may also + be done by for example <seealso marker="#traversal">traversing + functions</seealso> like <c>select</c> and <c>match</c>. Such + table fixations are automatically released before the + corresponding functions returns, but they may be seen by a + concurrent call to + <c>ets:info(T,safe_fixed|safe_fixed_monotonic_time)</c>.</p> + <p>If the table is not fixed at all, the call returns <c>false</c>.</p> </item> <item> <p><c>Item=stats, Value=tuple()</c></p> diff --git a/lib/stdlib/doc/src/gen_statem.xml b/lib/stdlib/doc/src/gen_statem.xml index 0cad5929ca..d4a4ba268b 100644 --- a/lib/stdlib/doc/src/gen_statem.xml +++ b/lib/stdlib/doc/src/gen_statem.xml @@ -320,12 +320,13 @@ erlang:'!' -----> Module:StateName/3 </p> <p> There is also a server start option - <seealso marker="#type-hibernate_after_opt"> + <seealso marker="#type-enter_loop_opt"> <c>{hibernate_after, Timeout}</c> </seealso> for - <seealso marker="#start/3"><c>start/3,4</c></seealso> or - <seealso marker="#start_link/3"><c>start_link/3,4</c></seealso> + <seealso marker="#start/3"><c>start/3,4</c></seealso>, + <seealso marker="#start_link/3"><c>start_link/3,4</c></seealso> or + <seealso marker="#enter_loop/4"><c>enter_loop/4,5,6</c></seealso>, that may be used to automatically hibernate the server. </p> </description> diff --git a/lib/stdlib/doc/src/notes.xml b/lib/stdlib/doc/src/notes.xml index dee7136eb1..23c3f6e981 100644 --- a/lib/stdlib/doc/src/notes.xml +++ b/lib/stdlib/doc/src/notes.xml @@ -31,6 +31,57 @@ </header> <p>This document describes the changes made to the STDLIB application.</p> +<section><title>STDLIB 3.8</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Fix a bug in the Erlang Pretty Printer: long atom + names in combination with <c><<>></c> could + cause a crash. </p> + <p> + Own Id: OTP-15592 Aux Id: ERL-818 </p> + </item> + <item> + <p> Fix bugs that could cause wrong results or bad + performance when formatting lists of characters using the + control sequences <c>p</c> or <c>P</c> and limiting the + output with the option <c>chars_limit</c>. </p> + <p> + Own Id: OTP-15639</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Improved ETS documentation about safe table traversal and + the partially bound key optimization for + <c>ordered_set</c>.</p> + <p> + Own Id: OTP-15545 Aux Id: PR-2103, PR-2139 </p> + </item> + <item> + <p> Optimize <c>calendar:gregorian_days_to_date/1</c>. + </p> + <p> + Own Id: OTP-15572 Aux Id: PR-2121 </p> + </item> + <item> + <p> Optimize functions + <c>calendar:rfc3339_to_system_time()</c> and + <c>calendar:system_time_to_rfc3339()</c>. </p> + <p> + Own Id: OTP-15630</p> + </item> + </list> + </section> + +</section> + <section><title>STDLIB 3.7.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/stdlib/src/array.erl b/lib/stdlib/src/array.erl index 939b1fb488..1504326c61 100644 --- a/lib/stdlib/src/array.erl +++ b/lib/stdlib/src/array.erl @@ -126,11 +126,12 @@ %% per write than base 10, but the speedup is only 21%.) -define(DEFAULT, undefined). --define(LEAFSIZE, 10). % the "base" --define(NODESIZE, ?LEAFSIZE). % (no reason to have a different size) +-define(LEAFSIZE, 10). % the "base" (assumed to be > 1) +-define(NODESIZE, ?LEAFSIZE). % must not be LEAFSIZE-1; keep same as leaf -define(NODEPATTERN(S), {_,_,_,_,_,_,_,_,_,_,S}). % NODESIZE+1 elements! --define(NEW_NODE(S), % beware of argument duplication! - setelement((?NODESIZE+1),erlang:make_tuple((?NODESIZE+1),(S)),(S))). +-define(NEW_NODE(E,S), % general case (currently unused) + setelement((?NODESIZE+1),erlang:make_tuple((?NODESIZE+1),(E)),(S))). +-define(NEW_NODE(S), erlang:make_tuple((?NODESIZE+1),(S))). % when E = S -define(NEW_LEAF(D), erlang:make_tuple(?LEAFSIZE,(D))). -define(NODELEAFS, ?NODESIZE*?LEAFSIZE). @@ -605,7 +606,7 @@ grow(I, E, M) -> grow_1(I, E, M). grow_1(I, E, M) when I >= M -> - grow(I, setelement(1, ?NEW_NODE(M), E), ?extend(M)); + grow_1(I, setelement(1, ?NEW_NODE(M), E), ?extend(M)); grow_1(_I, E, M) -> {E, M}. @@ -1631,12 +1632,11 @@ foldl_test_() -> ?_assert(foldl(Sum, 0, from_list(lists:seq(0,10))) =:= 55), ?_assert(foldl(Reverse, [], from_list(lists:seq(0,1000))) =:= lists:reverse(lists:seq(0,1000))), - ?_assert({999,[N0*100+1+2,N0*2+1+1,0]} =:= - foldl(Vals, {0,[]}, + ?_assertEqual({N0*100+1-2,[N0*100+1+2,N0*2+1+1,0]}, + foldl(Vals, {0,[]}, set(N0*100+1,2, set(N0*2+1,1, set(0,0,new()))))) - ]. -endif. @@ -1786,12 +1786,11 @@ foldr_test_() -> ?_assert(foldr(Sum, 0, from_list(lists:seq(0,10))) =:= 55), ?_assert(foldr(List, [], from_list(lists:seq(0,1000))) =:= lists:seq(0,1000)), - ?_assert({999,[0,N0*2+1+1,N0*100+1+2]} =:= - foldr(Vals, {0,[]}, + ?_assertEqual({N0*100+1-2,[0,N0*2+1+1,N0*100+1+2]}, + foldr(Vals, {0,[]}, set(N0*100+1,2, set(N0*2+1,1, set(0,0,new()))))) - ]. -endif. diff --git a/lib/stdlib/src/beam_lib.erl b/lib/stdlib/src/beam_lib.erl index 3386cfcbe6..aa992f17ab 100644 --- a/lib/stdlib/src/beam_lib.erl +++ b/lib/stdlib/src/beam_lib.erl @@ -32,8 +32,12 @@ all_chunks/1, diff_dirs/2, strip/1, + strip/2, strip_files/1, + strip_files/2, strip_release/1, + strip_release/2, + significant_chunks/0, build_module/1, version/1, md5/1, @@ -188,7 +192,16 @@ diff_dirs(Dir1, Dir2) -> Beam2 :: beam(). strip(FileName) -> - try strip_file(FileName) + strip(FileName, []). + +-spec strip(Beam1, AdditionalChunks) -> + {'ok', {module(), Beam2}} | {'error', 'beam_lib', info_rsn()} when + Beam1 :: beam(), + AdditionalChunks :: [chunkid()], + Beam2 :: beam(). + +strip(FileName, AdditionalChunks) -> + try strip_file(FileName, AdditionalChunks) catch Error -> Error end. -spec strip_files(Files) -> @@ -196,8 +209,17 @@ strip(FileName) -> Files :: [beam()], Beam :: beam(). -strip_files(Files) when is_list(Files) -> - try strip_fils(Files) +strip_files(Files) -> + strip_files(Files, []). + +-spec strip_files(Files, AdditionalChunks) -> + {'ok', [{module(), Beam}]} | {'error', 'beam_lib', info_rsn()} when + Files :: [beam()], + AdditionalChunks :: [chunkid()], + Beam :: beam(). + +strip_files(Files, AdditionalChunks) when is_list(Files) -> + try strip_fils(Files, AdditionalChunks) catch Error -> Error end. -spec strip_release(Dir) -> @@ -207,7 +229,17 @@ strip_files(Files) when is_list(Files) -> Reason :: {'not_a_directory', term()} | info_rsn(). strip_release(Root) -> - catch strip_rel(Root). + strip_release(Root, []). + +-spec strip_release(Dir, AdditionalChunks) -> + {'ok', [{module(), file:filename()}]} + | {'error', 'beam_lib', Reason} when + Dir :: atom() | file:filename(), + AdditionalChunks :: [chunkid()], + Reason :: {'not_a_directory', term()} | info_rsn(). + +strip_release(Root, AdditionalChunks) -> + catch strip_rel(Root, AdditionalChunks). -spec version(Beam) -> {'ok', {module(), [Version :: term()]}} | @@ -401,17 +433,17 @@ cmp_lists([{Id, C1} | R1], [{Id, C2} | R2]) -> cmp_lists(_, _) -> error(different_chunks). -strip_rel(Root) -> +strip_rel(Root, AdditionalChunks) -> ok = assert_directory(Root), - strip_fils(filelib:wildcard(filename:join(Root, "lib/*/ebin/*.beam"))). + strip_fils(filelib:wildcard(filename:join(Root, "lib/*/ebin/*.beam")), AdditionalChunks). %% -> {ok, [{Mod, BinaryOrFileName}]} | throw(Error) -strip_fils(Files) -> - {ok, [begin {ok, Reply} = strip_file(F), Reply end || F <- Files]}. +strip_fils(Files, AdditionalChunks) -> + {ok, [begin {ok, Reply} = strip_file(F, AdditionalChunks), Reply end || F <- Files]}. %% -> {ok, {Mod, FileName}} | {ok, {Mod, binary()}} | throw(Error) -strip_file(File) -> - {ok, {Mod, Chunks}} = read_significant_chunks(File, significant_chunks()), +strip_file(File, AdditionalChunks) -> + {ok, {Mod, Chunks}} = read_significant_chunks(File, AdditionalChunks ++ significant_chunks()), {ok, Stripped0} = build_module(Chunks), Stripped = compress(Stripped0), case File of diff --git a/lib/stdlib/src/calendar.erl b/lib/stdlib/src/calendar.erl index 3a083d9fda..3a8fe2211b 100644 --- a/lib/stdlib/src/calendar.erl +++ b/lib/stdlib/src/calendar.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -357,13 +357,17 @@ rfc3339_to_system_time(DateTimeString) -> rfc3339_to_system_time(DateTimeString, Options) -> Unit = proplists:get_value(unit, Options, second), %% _T is the character separating the date and the time: - {DateStr, [_T|TimeStr]} = lists:split(10, DateTimeString), - {TimeStr2, TimeStr3} = lists:split(8, TimeStr), - {ok, [Hour, Min, Sec], []} = io_lib:fread("~d:~d:~d", TimeStr2), - {ok, [Year, Month, Day], []} = io_lib:fread("~d-~d-~d", DateStr), + [Y1, Y2, Y3, Y4, $-, Mon1, Mon2, $-, D1, D2, _T, + H1, H2, $:, Min1, Min2, $:, S1, S2 | TimeStr] = DateTimeString, + Hour = list_to_integer([H1, H2]), + Min = list_to_integer([Min1, Min2]), + Sec = list_to_integer([S1, S2]), + Year = list_to_integer([Y1, Y2, Y3, Y4]), + Month = list_to_integer([Mon1, Mon2]), + Day = list_to_integer([D1, D2]), DateTime = {{Year, Month, Day}, {Hour, Min, Sec}}, IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end, - {FractionStr, UtcOffset} = lists:splitwith(IsFractionChar, TimeStr3), + {FractionStr, UtcOffset} = lists:splitwith(IsFractionChar, TimeStr), Time = datetime_to_system_time(DateTime), Secs = Time - offset_adjustment(Time, second, UtcOffset), check(DateTimeString, Options, Secs), @@ -451,8 +455,9 @@ system_time_to_rfc3339(Time, Options) -> DateTime = system_time_to_datetime(Secs), {{Year, Month, Day}, {Hour, Min, Sec}} = DateTime, FractionStr = fraction_str(Factor, AdjustedTime), - flat_fwrite("~4.10.0B-~2.10.0B-~2.10.0B~c~2.10.0B:~2.10.0B:~2.10.0B~s~s", - [Year, Month, Day, T, Hour, Min, Sec, FractionStr, Offset]). + L = [pad4(Year), "-", pad2(Month), "-", pad2(Day), [T], + pad2(Hour), ":", pad2(Min), ":", pad2(Sec), FractionStr, Offset], + lists:append(L). %% time_difference(T1, T2) = Tdiff %% @@ -680,7 +685,7 @@ offset(OffsetOption, Secs0) when OffsetOption =:= ""; Secs = abs(Secs0), Hour = Secs div 3600, Min = (Secs rem 3600) div 60, - io_lib:fwrite("~c~2.10.0B:~2.10.0B", [Sign, Hour, Min]); + [Sign | lists:append([pad2(Hour), ":", pad2(Min)])]; offset(OffsetOption, _Secs) -> OffsetOption. @@ -695,8 +700,10 @@ offset_string_adjustment(_Time, _Unit, "Z") -> 0; offset_string_adjustment(_Time, _Unit, "z") -> 0; -offset_string_adjustment(_Time, _Unit, [Sign|Tz]) -> - {ok, [Hour, Min], []} = io_lib:fread("~d:~d", Tz), +offset_string_adjustment(_Time, _Unit, Tz) -> + [Sign, H1, H2, $:, M1, M2] = Tz, + Hour = list_to_integer([H1, H2]), + Min = list_to_integer([M1, M2]), Adjustment = 3600 * Hour + 60 * Min, case Sign of $- -> -Adjustment; @@ -704,8 +711,9 @@ offset_string_adjustment(_Time, _Unit, [Sign|Tz]) -> end. local_offset(SystemTime, Unit) -> - LocalTime = system_time_to_local_time(SystemTime, Unit), + %% Not optimized for special cases. UniversalTime = system_time_to_universal_time(SystemTime, Unit), + LocalTime = erlang:universaltime_to_localtime(UniversalTime), LocalSecs = datetime_to_gregorian_seconds(LocalTime), UniversalSecs = datetime_to_gregorian_seconds(UniversalTime), LocalSecs - UniversalSecs. @@ -714,7 +722,8 @@ fraction_str(1, _Time) -> ""; fraction_str(Factor, Time) -> Fraction = Time rem Factor, - io_lib:fwrite(".~*..0B", [log10(Factor), abs(Fraction)]). + S = integer_to_list(abs(Fraction)), + [$. | pad(log10(Factor) - length(S), S)]. fraction(second, _) -> 0; @@ -735,5 +744,21 @@ log10(1000) -> 3; log10(1000000) -> 6; log10(1000000000) -> 9. -flat_fwrite(F, S) -> - lists:flatten(io_lib:fwrite(F, S)). +pad(0, S) -> + S; +pad(I, S) -> + [$0 | pad(I - 1, S)]. + +pad2(N) when N < 10 -> + [$0 | integer_to_list(N)]; +pad2(N) -> + integer_to_list(N). + +pad4(N) when N < 10 -> + [$0, $0, $0 | integer_to_list(N)]; +pad4(N) when N < 100 -> + [$0, $0 | integer_to_list(N)]; +pad4(N) when N < 1000 -> + [$0 | integer_to_list(N)]; +pad4(N) -> + integer_to_list(N). diff --git a/lib/stdlib/src/io_lib_pretty.erl b/lib/stdlib/src/io_lib_pretty.erl index 5483ea87b5..8f2fd7ea8f 100644 --- a/lib/stdlib/src/io_lib_pretty.erl +++ b/lib/stdlib/src/io_lib_pretty.erl @@ -721,7 +721,7 @@ printable_list(_L, 1, _T, _Enc) -> printable_list(L, _D, T, latin1) when T < 0 -> io_lib:printable_latin1_list(L); printable_list(L, _D, T, Enc) when T >= 0 -> - case slice(L, tsub(T, 2)) of + case slice(L, tsub(T, 2), Enc) of false -> false; {prefix, Prefix} when Enc =:= latin1 -> @@ -737,20 +737,46 @@ printable_list(L, _D, T, Enc) when T >= 0 -> printable_list(L, _D, T, _Uni) when T < 0-> io_lib:printable_list(L). -slice(L, N) -> - try io_lib:chars_length(L) =< N of - true -> +slice(L, N, latin1) -> + try lists:split(N, L) of + {_, []} -> all; - false -> - case string:slice(L, 0, N) of - "" -> - false; - Prefix -> - {prefix, Prefix} + {[], _} -> + false; + {L1, _} -> + {prefix, L1} + catch + _:_ -> + all + end; +slice(L, N, _Uni) -> + %% Be careful not to traverse more of L than necessary. + try string:slice(L, 0, N) of + "" -> + false; + Prefix -> + %% Assume no binaries are introduced by string:slice(). + case is_flat(L, lists:flatlength(Prefix)) of + true -> + case string:equal(Prefix, L) of + true -> + all; + false -> + {prefix, Prefix} + end; + false -> + false end catch _:_ -> false end. +is_flat(_L, 0) -> + true; +is_flat([C|Cs], N) when is_integer(C) -> + is_flat(Cs, N - 1); +is_flat(_, _N) -> + false. + printable_bin0(Bin, D, T, Enc) -> Len = case D >= 0 of true -> diff --git a/lib/stdlib/src/stdlib.appup.src b/lib/stdlib/src/stdlib.appup.src index 9e5d6a3bd8..37ea97c353 100644 --- a/lib/stdlib/src/stdlib.appup.src +++ b/lib/stdlib/src/stdlib.appup.src @@ -40,7 +40,8 @@ {<<"^3\\.6$">>,[restart_new_emulator]}, {<<"^3\\.6\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.7$">>,[restart_new_emulator]}, - {<<"^3\\.7\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}], + {<<"^3\\.7\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^3\\.7\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], [{<<"^3\\.4$">>,[restart_new_emulator]}, {<<"^3\\.4\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.4\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -54,4 +55,5 @@ {<<"^3\\.6$">>,[restart_new_emulator]}, {<<"^3\\.6\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.7$">>,[restart_new_emulator]}, - {<<"^3\\.7\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}]}. + {<<"^3\\.7\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^3\\.7\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. diff --git a/lib/stdlib/src/string.erl b/lib/stdlib/src/string.erl index 2939e78d9d..1f8bdc5432 100644 --- a/lib/stdlib/src/string.erl +++ b/lib/stdlib/src/string.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1247,18 +1247,20 @@ split_1(Bin, [_C|_]=Needle, Start, Where, Curr0, Acc) -> end end. -lexemes_m([CP|_]=Cs0, {GCs,CPs,_}=Seps, Ts) when is_integer(CP) -> +lexemes_m([CP|_]=Cs0, {GCs,CPs,_}=Seps0, Ts) when is_integer(CP) -> case lists:member(CP, CPs) of true -> [GC|Cs2] = unicode_util:gc(Cs0), case lists:member(GC, GCs) of true -> - lexemes_m(Cs2, Seps, Ts); + lexemes_m(Cs2, Seps0, Ts); false -> + Seps = search_compile(Seps0), {Lexeme,Rest} = lexeme_pick(Cs0, Seps, []), lexemes_m(Rest, Seps, [Lexeme|Ts]) end; false -> + Seps = search_compile(Seps0), {Lexeme,Rest} = lexeme_pick(Cs0, Seps, []), lexemes_m(Rest, Seps, [Lexeme|Ts]) end; diff --git a/lib/stdlib/test/beam_lib_SUITE.erl b/lib/stdlib/test/beam_lib_SUITE.erl index 6418dc7eb6..4b2694320e 100644 --- a/lib/stdlib/test/beam_lib_SUITE.erl +++ b/lib/stdlib/test/beam_lib_SUITE.erl @@ -35,7 +35,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, - normal/1, error/1, cmp/1, cmp_literals/1, strip/1, otp_6711/1, + normal/1, error/1, cmp/1, cmp_literals/1, strip/1, strip_add_chunks/1, otp_6711/1, building/1, md5/1, encrypted_abstr/1, encrypted_abstr_file/1]). -export([init_per_testcase/2, end_per_testcase/2]). @@ -45,7 +45,7 @@ suite() -> {timetrap,{minutes,2}}]. all() -> - [error, normal, cmp, cmp_literals, strip, otp_6711, + [error, normal, cmp, cmp_literals, strip, strip_add_chunks, otp_6711, building, md5, encrypted_abstr, encrypted_abstr_file]. groups() -> @@ -401,6 +401,69 @@ strip(Conf) when is_list(Conf) -> Source5D1, BeamFile5D1]), ok. +strip_add_chunks(Conf) when is_list(Conf) -> + PrivDir = ?privdir, + {SourceD1, BeamFileD1} = make_beam(PrivDir, simple, member), + {Source2D1, BeamFile2D1} = make_beam(PrivDir, simple2, concat), + {Source3D1, BeamFile3D1} = make_beam(PrivDir, make_fun, make_fun), + {Source4D1, BeamFile4D1} = make_beam(PrivDir, constant, constant), + {Source5D1, BeamFile5D1} = make_beam(PrivDir, lines, lines), + + NoOfTables = erlang:system_info(ets_count), + P0 = pps(), + + %% strip binary + verify(not_a_beam_file, beam_lib:strip(<<>>)), + {ok, B1} = file:read_file(BeamFileD1), + {ok, {simple, NB1}} = beam_lib:strip(B1), + + BId1 = chunk_ids(B1), + NBId1 = chunk_ids(NB1), + true = length(BId1) > length(NBId1), + compare_chunks(B1, NB1, NBId1), + + %% Keep all the extra chunks + ExtraChunks = ["Abst" , "Dbgi" , "Attr" , "CInf" , "LocT" , "Atom" ], + {ok, {simple, AB1}} = beam_lib:strip(B1, ExtraChunks), + ABId1 = chunk_ids(AB1), + true = length(BId1) == length(ABId1), + compare_chunks(B1, AB1, ABId1), + + %% strip file - Keep extra chunks + verify(file_error, beam_lib:strip(foo)), + {ok, {simple, _}} = beam_lib:strip(BeamFileD1, ExtraChunks), + compare_chunks(B1, BeamFileD1, ABId1), + + %% strip_files + {ok, B2} = file:read_file(BeamFile2D1), + {ok, [{simple,_},{simple2,_}]} = beam_lib:strip_files([B1, B2], ExtraChunks), + {ok, [{simple,_},{simple2,_},{make_fun,_},{constant,_}]} = + beam_lib:strip_files([BeamFileD1, BeamFile2D1, BeamFile3D1, BeamFile4D1], ExtraChunks), + + %% check that each module can be loaded. + {module, simple} = code:load_abs(filename:rootname(BeamFileD1)), + {module, simple2} = code:load_abs(filename:rootname(BeamFile2D1)), + {module, make_fun} = code:load_abs(filename:rootname(BeamFile3D1)), + {module, constant} = code:load_abs(filename:rootname(BeamFile4D1)), + + %% check that line number information is still present after stripping + {module, lines} = code:load_abs(filename:rootname(BeamFile5D1)), + {'EXIT',{badarith,[{lines,t,1,Info}|_]}} = (catch lines:t(atom)), + false = code:purge(lines), + true = code:delete(lines), + {ok, {lines,BeamFile5D1}} = beam_lib:strip(BeamFile5D1), + {module, lines} = code:load_abs(filename:rootname(BeamFile5D1)), + {'EXIT',{badarith,[{lines,t,1,Info}|_]}} = (catch lines:t(atom)), + + true = (P0 == pps()), + NoOfTables = erlang:system_info(ets_count), + + delete_files([SourceD1, BeamFileD1, + Source2D1, BeamFile2D1, + Source3D1, BeamFile3D1, + Source4D1, BeamFile4D1, + Source5D1, BeamFile5D1]), + ok. otp_6711(Conf) when is_list(Conf) -> {'EXIT',{function_clause,_}} = (catch {a, beam_lib:info(3)}), @@ -729,6 +792,7 @@ make_beam(Dir, Module, F) -> FileBase = filename:join(Dir, atom_to_list(Module)), Source = FileBase ++ ".erl", BeamFile = FileBase ++ ".beam", + file:delete(BeamFile), simple_file(Source, Module, F), {ok, _} = compile:file(Source, [{outdir,Dir}, debug_info, report]), {Source, BeamFile}. diff --git a/lib/stdlib/test/binary_module_SUITE.erl b/lib/stdlib/test/binary_module_SUITE.erl index e0811f19cf..9b2033ec4a 100644 --- a/lib/stdlib/test/binary_module_SUITE.erl +++ b/lib/stdlib/test/binary_module_SUITE.erl @@ -755,8 +755,9 @@ list_to_bin(Config) when is_list(Config) -> copy(Config) when is_list(Config) -> <<1,2,3>> = binary:copy(<<1,2,3>>), RS = random_string({1,10000}), - RS = RS2 = binary:copy(RS), - false = erts_debug:same(RS,RS2), + RS2 = binary:copy(RS), + true = RS =:= RS2, + false = erts_debug:same(RS, RS2), <<>> = ?MASK_ERROR(binary:copy(<<1,2,3>>,0)), badarg = ?MASK_ERROR(binary:copy(<<1,2,3:3>>,2)), badarg = ?MASK_ERROR(binary:copy([],0)), diff --git a/lib/stdlib/test/calendar_SUITE.erl b/lib/stdlib/test/calendar_SUITE.erl index c6d9dbca4a..224c0d5625 100644 --- a/lib/stdlib/test/calendar_SUITE.erl +++ b/lib/stdlib/test/calendar_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2018. All Rights Reserved. +%% Copyright Ericsson AB 1997-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 22c77aa172..87ca9bd32c 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -56,9 +56,11 @@ -export([t_match_spec_run/1]). -export([t_bucket_disappears/1]). -export([t_named_select/1]). +-export([select_fixtab_owner_change/1]). -export([otp_5340/1]). -export([otp_6338/1]). -export([otp_6842_select_1000/1]). +-export([select_mbuf_trapping/1]). -export([otp_7665/1]). -export([meta_wb/1]). -export([grow_shrink/1, grow_pseudo_deleted/1, shrink_pseudo_deleted/1]). @@ -68,7 +70,10 @@ -export([smp_insert/1, smp_fixed_delete/1, smp_unfix_fix/1, smp_select_delete/1, smp_ordered_iteration/1, smp_select_replace/1, otp_8166/1, otp_8732/1, delete_unfix_race/1]). --export([throughput_benchmark/0, test_throughput_benchmark/1]). +-export([throughput_benchmark/0, + throughput_benchmark/1, + test_throughput_benchmark/1, + long_throughput_benchmark/1]). -export([exit_large_table_owner/1, exit_many_large_table_owner/1, exit_many_tables_owner/1, @@ -91,6 +96,7 @@ -include_lib("stdlib/include/ms_transform.hrl"). % ets:fun2ms -include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). -define(m(A,B), assert_eq(A,B)). -define(heap_binary_size, 64). @@ -129,9 +135,10 @@ all() -> t_insert_list, t_test_ms, t_select_delete, t_select_replace, t_select_replace_next_bug, t_ets_dets, memory, t_select_reverse, t_bucket_disappears, - t_named_select, + t_named_select, select_fixtab_owner_change, select_fail, t_insert_new, t_repair_continuation, otp_5340, otp_6338, otp_6842_select_1000, otp_7665, + select_mbuf_trapping, otp_8732, meta_wb, grow_shrink, grow_pseudo_deleted, shrink_pseudo_deleted, {group, meta_smp}, smp_insert, smp_fixed_delete, smp_unfix_fix, smp_select_replace, @@ -148,7 +155,8 @@ all() -> take, whereis_table, delete_unfix_race, - test_throughput_benchmark]. + test_throughput_benchmark, + {group, benchmark}]. groups() -> [{new, [], @@ -176,7 +184,9 @@ groups() -> {meta_smp, [], [meta_lookup_unnamed_read, meta_lookup_unnamed_write, meta_lookup_named_read, meta_lookup_named_write, - meta_newdel_unnamed, meta_newdel_named]}]. + meta_newdel_unnamed, meta_newdel_named]}, + {benchmark, [], + [long_throughput_benchmark]}]. init_per_suite(Config) -> erts_debug:set_internal_state(available_internal_state, true), @@ -189,9 +199,61 @@ end_per_suite(_Config) -> catch erts_debug:set_internal_state(available_internal_state, false), ok. +init_per_group(benchmark, Config) -> + P = self(), + %% Spawn owner of ETS table that is alive until end_per_group is run + EtsProcess = + spawn( + fun()-> + Tab = ets:new(ets_benchmark_result_summary_tab, [public]), + P ! {the_table, Tab}, + receive + kill -> ok + end + end), + Tab = receive {the_table, T} -> T end, + CounterNames = [nr_of_benchmarks, + total_throughput, + nr_of_set_benchmarks, + total_throughput_set, + nr_of_ordered_set_benchmarks, + total_throughput_ordered_set], + lists:foreach(fun(CtrName) -> + ets:insert(Tab, {CtrName, 0.0}) + end, + CounterNames), + [{ets_benchmark_result_summary_tab, Tab}, + {ets_benchmark_result_summary_tab_process, EtsProcess} | Config]; init_per_group(_GroupName, Config) -> Config. +end_per_group(benchmark, Config) -> + T = proplists:get_value(ets_benchmark_result_summary_tab, Config), + EtsProcess = proplists:get_value(ets_benchmark_result_summary_tab_process, Config), + Report = + fun(NOfBenchmarksCtr, TotThroughoutCtr, Name) -> + Average = + ets:lookup_element(T, TotThroughoutCtr, 2) / + ets:lookup_element(T, NOfBenchmarksCtr, 2), + io:format("~p ~p~n", [Name, Average]), + ct_event:notify( + #event{name = benchmark_data, + data = [{suite,"ets_bench"}, + {name, Name}, + {value, Average}]}) + end, + Report(nr_of_benchmarks, + total_throughput, + "Average Throughput"), + Report(nr_of_set_benchmarks, + total_throughput_set, + "Average Throughput Set"), + Report(nr_of_ordered_set_benchmarks, + total_throughput_ordered_set, + "Average Throughput Ordered Set"), + ets:delete(T), + EtsProcess ! kill, + Config; end_per_group(_GroupName, Config) -> Config. @@ -247,7 +309,64 @@ t_named_select_do(Opts) -> verify_etsmem(EtsMem). +%% Verify select and friends release fixtab as they should +%% even when owneship is changed between traps. +select_fixtab_owner_change(_Config) -> + T = ets:new(xxx, [protected]), + NKeys = 2000, + [ets:insert(T,{K,K band 7}) || K <- lists:seq(1,NKeys)], + + %% Buddy and Papa will ping-pong table ownership between them + %% and the aim is to give Buddy the table when he is + %% in the middle of a yielding select* call. + {Buddy,_} = spawn_opt(fun() -> sfoc_buddy_loop(T, 1, undefined) end, + [link,monitor]), + + sfoc_papa_loop(T, Buddy), + + receive {'DOWN', _, process, Buddy, _} -> ok end, + ets:delete(T), + ok. + +sfoc_buddy_loop(T, I, State0) -> + receive + {'ETS-TRANSFER', T, Papa, _} -> + ets:give_away(T, Papa, State0), + case State0 of + done -> + ok; + _ -> + State1 = sfoc_traverse(T, I, State0), + %% Verify no fixation left + {I, false} = {I, ets:info(T, safe_fixed_monotonic_time)}, + sfoc_buddy_loop(T, I+1, State1) + end + end. + +sfoc_papa_loop(T, Buddy) -> + ets:give_away(T, Buddy, "Catch!"), + receive + {'ETS-TRANSFER', T, Buddy, State} -> + case State of + done -> + ok; + _ -> + sfoc_papa_loop(T, Buddy) + end + end. +sfoc_traverse(T, 1, S) -> + ets:select(T, [{{'$1',7}, [], ['$1']}]), S; +sfoc_traverse(T, 2, S) -> + 0 = ets:select_count(T, [{{'$1',7}, [], [false]}]), S; +sfoc_traverse(T, 3, _) -> + Limit = ets:info(T, size) div 2, + {_, Continuation} = ets:select(T, [{{'$1',7}, [], ['$1']}], + Limit), + Continuation; +sfoc_traverse(_T, 4, Continuation) -> + _ = ets:select(Continuation), + done. %% Check ets:match_spec_run/2. t_match_spec_run(Config) when is_list(Config) -> @@ -5292,6 +5411,61 @@ otp_6338(Config) when is_list(Config) -> end), ok. +%% OTP-15660: Verify select not doing excessive trapping +%% when process have mbuf heap fragments. +select_mbuf_trapping(Config) when is_list(Config) -> + select_mbuf_trapping_do(set), + select_mbuf_trapping_do(ordered_set). + +select_mbuf_trapping_do(Type) -> + T = ets:new(xxx, [Type]), + NKeys = 50, + [ets:insert(T, {K, value}) || K <- lists:seq(1,NKeys)], + + {priority, Prio} = process_info(self(), priority), + Tracee = self(), + [SchedTracer] + = start_loopers(1, Prio, + fun (SC) -> + receive + {trace, Tracee, out, _} -> + SC+1; + done -> + Tracee ! {schedule_count, SC}, + exit(normal) + end + end, + 0), + + erlang:garbage_collect(), + 1 = erlang:trace(self(), true, [running,{tracer,SchedTracer}]), + + %% Artificially create an mbuf heap fragment + MbufTerm = "Frag me up", + MbufTerm = erts_debug:set_internal_state(mbuf, MbufTerm), + + Keys = ets:select(T, [{{'$1', value}, [], ['$1']}]), + NKeys = length(Keys), + + 1 = erlang:trace(self(), false, [running]), + Ref = erlang:trace_delivered(Tracee), + receive + {trace_delivered, Tracee, Ref} -> + SchedTracer ! done + end, + receive + {schedule_count, N} -> + io:format("~p context switches: ~p", [Type,N]), + if + N < 3 -> ok; + true -> ct:fail(failed) + end + end, + true = ets:delete(T), + ok. + + + %% Elements could come in the wrong order in a bag if a rehash occurred. otp_5340(Config) when is_list(Config) -> repeat_for_opts(fun otp_5340_do/1). @@ -6415,8 +6589,8 @@ whereis_table(Config) when is_list(Config) -> ok. -%% The following work functions are used by -%% throughput_benchmark/4. They are declared on the top level beacuse +%% The following help functions are used by +%% throughput_benchmark. They are declared on the top level beacuse %% declaring them as function local funs cause a scalability issue. get_op([{_,O}], _RandNum) -> O; @@ -6451,10 +6625,131 @@ prefill_table_loop(T, RS0, N, ObjFun) -> ets:insert(T, ObjFun(Key)), prefill_table_loop(T, RS1, N-1, ObjFun). -throughput_benchmark() -> - throughput_benchmark(false, not_set, not_set). +-record(ets_throughput_bench_config, + {benchmark_duration_ms = 3000, + recover_time_ms = 1000, + thread_counts = not_set, + key_ranges = [1000000], + scenarios = + [ + [ + {0.5, insert}, + {0.5, delete} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.8, lookup} + ], + [ + {0.01, insert}, + {0.01, delete}, + {0.98, lookup} + ], + [ + {1.0, lookup} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq10} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq100} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq1000} + ], + [ + {1.0, nextseq1000} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.7999, lookup}, + {0.0001, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.799999, lookup}, + {0.000001, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, partial_select1000} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.7999, lookup}, + {0.0001, partial_select1000} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.799999, lookup}, + {0.000001, partial_select1000} + ] + ], + table_types = + [ + [ordered_set, public], + [ordered_set, public, {write_concurrency, true}], + [ordered_set, public, {read_concurrency, true}], + [ordered_set, public, {write_concurrency, true}, {read_concurrency, true}], + [set, public], + [set, public, {write_concurrency, true}], + [set, public, {read_concurrency, true}], + [set, public, {write_concurrency, true}, {read_concurrency, true}] + ], + etsmem_fun = fun() -> ok end, + verify_etsmem_fun = fun(_) -> true end, + notify_res_fun = fun(_Name, _Throughput) -> ok end, + print_result_paths_fun = + fun(ResultPath, _LatestResultPath) -> + Comment = + io_lib:format("<a href=\"file:///~s\">Result visualization</a>",[ResultPath]), + {comment, Comment} + end + }). + +stdout_notify_res(ResultPath, LatestResultPath) -> + io:format("Result Location: /~s~n", [ResultPath]), + io:format("Latest Result Location: ~s~n", [LatestResultPath]). -throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> +throughput_benchmark() -> + throughput_benchmark( + #ets_throughput_bench_config{ + print_result_paths_fun = fun stdout_notify_res/2}). + +throughput_benchmark( + #ets_throughput_bench_config{ + benchmark_duration_ms = BenchmarkDurationMs, + recover_time_ms = RecoverTimeMs, + thread_counts = ThreadCountsOpt, + key_ranges = KeyRanges, + scenarios = Scenarios, + table_types = TableTypes, + etsmem_fun = ETSMemFun, + verify_etsmem_fun = VerifyETSMemFun, + notify_res_fun = NotifyResFun, + print_result_paths_fun = PrintResultPathsFun}) -> NrOfSchedulers = erlang:system_info(schedulers), %% Definitions of operations that are supported by the benchmark NextSeqOp = @@ -6519,7 +6814,7 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> fun(T,KeyRange) -> NextSeqOp(T,KeyRange,1000) end, selectAll => fun(T,_KeyRange) -> - case -1 =:= ets:select_count(T, ets:fun2ms(fun(X) -> true end)) of + case -1 =:= ets:select_count(T, ets:fun2ms(fun(_X) -> true end)) of true -> io:format("Will never be printed"); false -> ok end @@ -6568,11 +6863,28 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> false -> ok end end, + DataHolder = + fun DataHolderFun(Data)-> + receive + {get_data, Pid} -> Pid ! {ets_bench_data, Data}; + D -> DataHolderFun([Data,D]) + end + end, + DataHolderPid = spawn_link(fun()-> DataHolder([]) end), + PrintData = + fun (Str, List) -> + io:format(Str, List), + DataHolderPid ! io_lib:format(Str, List) + end, + GetData = + fun () -> + DataHolderPid ! {get_data, self()}, + receive {ets_bench_data, Data} -> Data end + end, %% Function that runs a benchmark instance and returns the number %% of operations that were performed RunBenchmark = - fun(NrOfProcs, TableConfig, Scenario, - Range, Duration, RecoverTime) -> + fun({NrOfProcs, TableConfig, Scenario, Range, Duration}) -> ProbHelpTab = CalculateOpsProbHelpTab(Scenario, 0), Table = ets:new(t, TableConfig), Nobj = Range div 2, @@ -6580,16 +6892,15 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> Nobj = ets:info(Table, size), SafeFixTableIfRequired(Table, Scenario, true), ParentPid = self(), + Worker = + fun() -> + receive start -> ok end, + WorksDone = + do_work(0, Table, ProbHelpTab, Range, Operations), + ParentPid ! WorksDone + end, ChildPids = - lists:map( - fun(_N) -> - spawn(fun() -> - receive start -> ok end, - WorksDone = - do_work(0, Table, ProbHelpTab, Range, Operations), - ParentPid ! WorksDone - end) - end, lists:seq(1, NrOfProcs)), + lists:map(fun(_N) ->spawn_link(Worker)end, lists:seq(1, NrOfProcs)), lists:foreach(fun(Pid) -> Pid ! start end, ChildPids), timer:sleep(Duration), lists:foreach(fun(Pid) -> Pid ! stop end, ChildPids), @@ -6601,185 +6912,194 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> end, 0, ChildPids), SafeFixTableIfRequired(Table, Scenario, false), ets:delete(Table), - timer:sleep(RecoverTime), TotalWorksDone end, - %% - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %%%% Benchmark Configuration %%%%%%%%%%%%%%%%%%%%%%%% - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% - %% Change the following variables to configure the benchmark runs - ThreadCounts = - case TestMode of - true -> [1, NrOfSchedulers]; - false -> CalculateThreadCounts([1]) - end, - KeyRanges = % Sizes of the key ranges - case TestMode of - true -> [50000]; - false -> [1000000] + RunBenchmarkInSepProcess = + fun(ParameterTuple) -> + P = self(), + spawn_link(fun()-> P ! {bench_result, RunBenchmark(ParameterTuple)} end), + Result = receive {bench_result, Res} -> Res end, + timer:sleep(RecoverTimeMs), + Result end, - Duration = - case BenchmarkRunMs of % Duration of a benchmark run in milliseconds - not_set -> 30000; - _ -> BenchmarkRunMs + RunBenchmarkAndReport = + fun(ThreadCount, + TableType, + Scenario, + KeyRange, + Duration) -> + Result = RunBenchmarkInSepProcess({ThreadCount, + TableType, + Scenario, + KeyRange, + Duration}), + Throughput = Result/(Duration/1000.0), + PrintData("; ~f",[Throughput]), + Name = io_lib:format("Scenario: ~w, Key Range Size: ~w, " + "# of Processes: ~w, Table Type: ~w", + [Scenario, KeyRange, ThreadCount, TableType]), + NotifyResFun(Name, Throughput) end, - TimeMsToSleepAfterEachBenchmarkRun = - case RecoverTimeMs of - not_set -> 1000; - _ -> RecoverTimeMs + ThreadCounts = + case ThreadCountsOpt of + not_set -> + CalculateThreadCounts([1]); + _ -> ThreadCountsOpt end, - TableTypes = % The table types that will be benchmarked - [ - [ordered_set, public], - [ordered_set, public, {write_concurrency, true}], - [ordered_set, public, {read_concurrency, true}], - [ordered_set, public, {write_concurrency, true}, {read_concurrency, true}], - [set, public], - [set, public, {write_concurrency, true}], - [set, public, {read_concurrency, true}], - [set, public, {write_concurrency, true}, {read_concurrency, true}] - ], - Scenarios = % Benchmark scenarios (the fractions should add up to approximately 1.0) - [ - [ - {0.5, insert}, - {0.5, delete} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.8, lookup} - ], - [ - {0.01, insert}, - {0.01, delete}, - {0.98, lookup} - ], - [ - {1.0, lookup} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.4, lookup}, - {0.4, nextseq10} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.4, lookup}, - {0.4, nextseq100} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.4, lookup}, - {0.4, nextseq1000} - ], - [ - {1.0, nextseq1000} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.79, lookup}, - {0.01, selectAll} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.7999, lookup}, - {0.0001, selectAll} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.799999, lookup}, - {0.000001, selectAll} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.79, lookup}, - {0.01, partial_select1000} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.7999, lookup}, - {0.0001, partial_select1000} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.799999, lookup}, - {0.000001, partial_select1000} - ] - ], - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %%%% End of Benchmark Configuration %%%%%%%%%%%%%%%% - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% Prepare for memory check - EtsMem = case TestMode of - true -> etsmem(); - false -> ok - end, %% Run the benchmark - io:format("# Each instance of the benchmark runs for ~w seconds:~n", [Duration/1000]), - io:format("# The result of a benchmark instance is presented as a number representing~n"), - io:format("# the number of operations performed per second:~n~n~n"), - io:format("# To plot graphs for the results below:~n"), - io:format("# 1. Open \"$ERL_TOP/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html\" in a web browser~n"), - io:format("# 2. Copy the lines between \"#BENCHMARK STARTED$\" and \"#BENCHMARK ENDED$\" below~n"), - io:format("# 3. Paste the lines copied in step 2 to the text box in the browser window opened in~n"), - io:format("# step 1 and press the Render button~n~n"), - io:format("#BENCHMARK STARTED$~n"), + PrintData("# Each instance of the benchmark runs for ~w seconds:~n", [BenchmarkDurationMs/1000]), + PrintData("# The result of a benchmark instance is presented as a number representing~n",[]), + PrintData("# the number of operations performed per second:~n~n~n",[]), + PrintData("# To plot graphs for the results below:~n",[]), + PrintData("# 1. Open \"$ERL_TOP/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html\" in a web browser~n",[]), + PrintData("# 2. Copy the lines between \"#BENCHMARK STARTED$\" and \"#BENCHMARK ENDED$\" below~n",[]), + PrintData("# 3. Paste the lines copied in step 2 to the text box in the browser window opened in~n",[]), + PrintData("# step 1 and press the Render button~n~n",[]), + PrintData("#BENCHMARK STARTED$~n",[]), + EtsMem = ETSMemFun(), %% The following loop runs all benchmark scenarios and prints the results (i.e, operations/second) lists:foreach( fun(KeyRange) -> lists:foreach( fun(Scenario) -> - io:format("Scenario: ~s | Key Range Size: ~w$~n", - [RenderScenario(Scenario, ""), - KeyRange]), + PrintData("Scenario: ~s | Key Range Size: ~w$~n", + [RenderScenario(Scenario, ""), KeyRange]), lists:foreach( fun(ThreadCount) -> - io:format("; ~w",[ThreadCount]) + PrintData("; ~w",[ThreadCount]) end, ThreadCounts), - io:format("$~n",[]), + PrintData("$~n",[]), lists:foreach( fun(TableType) -> - io:format("~w ",[TableType]), + PrintData("~w ",[TableType]), lists:foreach( fun(ThreadCount) -> - Result = RunBenchmark(ThreadCount, + RunBenchmarkAndReport(ThreadCount, TableType, Scenario, KeyRange, - Duration, - TimeMsToSleepAfterEachBenchmarkRun), - io:format("; ~f",[Result/(Duration/1000.0)]) + BenchmarkDurationMs) end, ThreadCounts), - io:format("$~n",[]) + PrintData("$~n",[]) end, TableTypes) end, Scenarios) end, KeyRanges), - io:format("~n#BENCHMARK ENDED$~n~n"), - case TestMode of - true -> verify_etsmem(EtsMem); - false -> ok - end. + PrintData("~n#BENCHMARK ENDED$~n~n",[]), + VerifyETSMemFun(EtsMem), + DataDir = filename:join(filename:dirname(code:which(?MODULE)), "ets_SUITE_data"), + TemplatePath = filename:join(DataDir, "visualize_throughput.html"), + {ok, Template} = file:read_file(TemplatePath), + OutputData = string:replace(Template, "#bench_data_placeholder", GetData()), + OutputPath1 = filename:join(DataDir, "ets_bench_result.html"), + {{Year, Month, Day}, {Hour, Minute, Second}} = calendar:now_to_datetime(erlang:timestamp()), + StrTime = lists:flatten(io_lib:format("~4..0w-~2..0w-~2..0wT~2..0w:~2..0w:~2..0w",[Year,Month,Day,Hour,Minute,Second])), + OutputPath2 = filename:join(DataDir, io_lib:format("ets_bench_result_~s.html", [StrTime])), + file:write_file(OutputPath1, OutputData), + file:write_file(OutputPath2, OutputData), + PrintResultPathsFun(OutputPath2, OutputPath1). test_throughput_benchmark(Config) when is_list(Config) -> - throughput_benchmark(true, 100, 0). - + throughput_benchmark( + #ets_throughput_bench_config{ + benchmark_duration_ms = 100, + recover_time_ms = 0, + thread_counts = [1, erlang:system_info(schedulers)], + key_ranges = [50000], + etsmem_fun = fun etsmem/0, + verify_etsmem_fun = fun verify_etsmem/1}). + +long_throughput_benchmark(Config) when is_list(Config) -> + N = erlang:system_info(schedulers), + throughput_benchmark( + #ets_throughput_bench_config{ + benchmark_duration_ms = 3000, + recover_time_ms = 1000, + thread_counts = [1, N div 2, N], + key_ranges = [1000000], + scenarios = + [ + [ + {0.5, insert}, + {0.5, delete} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.8, lookup} + ], + [ + {0.01, insert}, + {0.01, delete}, + {0.98, lookup} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq100} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, partial_select1000} + ] + ], + table_types = + [ + [ordered_set, public, {write_concurrency, true}, {read_concurrency, true}], + [set, public, {write_concurrency, true}, {read_concurrency, true}] + ], + etsmem_fun = fun etsmem/0, + verify_etsmem_fun = fun verify_etsmem/1, + notify_res_fun = + fun(Name, Throughput) -> + SummaryTable = + proplists:get_value(ets_benchmark_result_summary_tab, Config), + AddToSummaryCounter = + case SummaryTable of + undefined -> + fun(_, _) -> + ok + end; + Tab -> + fun(CounterName, ToAdd) -> + OldVal = ets:lookup_element(Tab, CounterName, 2), + NewVal = OldVal + ToAdd, + ets:insert(Tab, {CounterName, NewVal}) + end + end, + Record = + fun(NoOfBenchsCtr, TotThrputCtr) -> + AddToSummaryCounter(NoOfBenchsCtr, 1), + AddToSummaryCounter(TotThrputCtr, Throughput) + end, + Record(nr_of_benchmarks, total_throughput), + case string:find(Name, "ordered_set") of + nomatch -> + Record(nr_of_set_benchmarks, total_throughput_set); + _ -> + Record(nr_of_ordered_set_benchmarks, + total_throughput_ordered_set) + end, + ct_event:notify( + #event{name = benchmark_data, + data = [{suite,"ets_bench"}, + {name, Name}, + {value,Throughput}]}) + end + }). add_lists(L1,L2) -> add_lists(L1,L2,[]). diff --git a/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html b/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html index a2c61aa938..27d6849c60 100644 --- a/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html +++ b/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html @@ -42,7 +42,7 @@ </p> Paste the generated data in the field below and press the Render button: <br> - <textarea id="dataField" rows="4" cols="50"></textarea> + <textarea id="dataField" rows="4" cols="50">#bench_data_placeholder</textarea> <br> <input type="checkbox" id="barPlot"> Bar Plot <br> @@ -56,13 +56,13 @@ <br> <input type="checkbox" class="showCheck" value="[ordered_set,public,{write_concurrency,true},{read_concurrency,true}]" checked> Show <code>[ordered_set,public,{write_concurrency,true},{read_concurrency,true}]</code> <br> - <input type="checkbox" class="showCheck" value="[set,public]"> Show <code>[set,public]</code> + <input type="checkbox" class="showCheck" value="[set,public]" checked> Show <code>[set,public]</code> <br> - <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true}]"> Show <code>[set,public,{write_concurrency,true}]</code> + <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true}]" checked> Show <code>[set,public,{write_concurrency,true}]</code> <br> - <input type="checkbox" class="showCheck" value="[set,public,{read_concurrency,true}]"> Show <code>[set,public,{read_concurrency,true}]</code> + <input type="checkbox" class="showCheck" value="[set,public,{read_concurrency,true}]" checked> Show <code>[set,public,{read_concurrency,true}]</code> <br> - <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true},{read_concurrency,true}]"> Show <code>[set,public,{write_concurrency,true},{read_concurrency,true}]</code> + <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true},{read_concurrency,true}]" checked> Show <code>[set,public,{write_concurrency,true},{read_concurrency,true}]</code> <br> <button id="renderButton" type="button">Render</button> diff --git a/lib/stdlib/test/io_SUITE.erl b/lib/stdlib/test/io_SUITE.erl index f097552e8c..824f5d19f2 100644 --- a/lib/stdlib/test/io_SUITE.erl +++ b/lib/stdlib/test/io_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2018. All Rights Reserved. +%% Copyright Ericsson AB 1999-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -32,7 +32,7 @@ io_with_huge_message_queue/1, format_string/1, maps/1, coverage/1, otp_14178_unicode_atoms/1, otp_14175/1, otp_14285/1, limit_term/1, otp_14983/1, otp_15103/1, otp_15076/1, - otp_15159/1]). + otp_15159/1, otp_15639/1]). -export([pretty/2, trf/3]). @@ -64,7 +64,8 @@ all() -> io_lib_print_binary_depth_one, otp_10302, otp_10755, otp_10836, io_lib_width_too_small, io_with_huge_message_queue, format_string, maps, coverage, otp_14178_unicode_atoms, otp_14175, - otp_14285, limit_term, otp_14983, otp_15103, otp_15076, otp_15159]. + otp_14285, limit_term, otp_14983, otp_15103, otp_15076, otp_15159, + otp_15639]. %% Error cases for output. error_1(Config) when is_list(Config) -> @@ -2647,3 +2648,35 @@ otp_15076(_Config) -> {'EXIT', {badarg, _}} = (catch io_lib:build_text(L)), {'EXIT', {badarg, _}} = (catch io_lib:build_text(L, [])), ok. + +otp_15639(_Config) -> + L = lists:duplicate(10, "a"), + LOpts = [{encoding, latin1}, {chars_limit, 10}], + UOpts = [{encoding, unicode}, {chars_limit, 10}], + "[[...]|...]" = pretty(L, LOpts), + "[[...]|...]" = pretty(L, UOpts), + "[\"a\",[...]|...]" = + pretty(L, [{chars_limit, 12}, {encoding, latin1}]), + "[\"a\",[...]|...]" = + pretty(L, [{chars_limit, 12}, {encoding, unicode}]), + + %% Latin-1 + "\"12345678\"" = pretty("12345678", LOpts), + "\"12345678\"..." = pretty("123456789", LOpts), + "\"\\r\\n123456\"..." = pretty("\r\n1234567", LOpts), + "\"\\r1234567\"..." = pretty("\r12345678", LOpts), + "\"\\r\\n123456\"..." = pretty("\r\n12345678", LOpts), + "\"12345678\"..." = pretty("12345678"++[x], LOpts), + "[49,50|...]" = pretty("1234567"++[x], LOpts), + "[49,x]" = pretty("1"++[x], LOpts), + "[[...]|...]" = pretty(["1","2","3","4","5","6","7","8"], LOpts), + %% Unicode + "\"12345678\"" = pretty("12345678", UOpts), + "\"12345678\"..." = pretty("123456789", UOpts), + "\"\\r\\n1234567\"" = pretty("\r\n1234567", UOpts), + "\"\\r1234567\"..." = pretty("\r12345678", UOpts), + "\"\\r\\n1234567\"..." = pretty("\r\n12345678", UOpts), + "[49,50|...]" = pretty("12345678"++[x], UOpts), + "\"12345678\"..." = pretty("123456789"++[x], UOpts), + "[[...]|...]" = pretty(["1","2","3","4","5","6","7","8"], UOpts), + ok. diff --git a/lib/stdlib/test/qlc_SUITE.erl b/lib/stdlib/test/qlc_SUITE.erl index d7354438f9..2354a08f78 100644 --- a/lib/stdlib/test/qlc_SUITE.erl +++ b/lib/stdlib/test/qlc_SUITE.erl @@ -3163,19 +3163,13 @@ lookup2(Config) when is_list(Config) -> [a] = lookup_keys(Q) end, [{a},{b},{c}])">>, - {cres, - <<"etsc(fun(E) -> + <<"etsc(fun(E) -> Q = qlc:q([X || {X}=Y <- ets:table(E), element(2, Y) == b, X =:= 1]), [] = qlc:e(Q), false = lookup_keys(Q) - end, [{1,b},{2,3}])">>, - %% {warnings,[{2,sys_core_fold,nomatch_guard}, - %% {3,qlc,nomatch_filter}, - %% {3,sys_core_fold,{eval_failure,badarg}}]}}, - {warnings,[{2,sys_core_fold,nomatch_guard}, - {3,sys_core_fold,{eval_failure,badarg}}]}}, + end, [{1,b},{2,3}])">>, <<"etsc(fun(E) -> Q = qlc:q([X || {X} <- ets:table(E), element(1,{X}) =:= 1]), diff --git a/lib/stdlib/test/stdlib.spec b/lib/stdlib/test/stdlib.spec index 4de7c1a0eb..bf64eae2c7 100644 --- a/lib/stdlib/test/stdlib.spec +++ b/lib/stdlib/test/stdlib.spec @@ -2,3 +2,6 @@ {skip_groups,"../stdlib_test",stdlib_bench_SUITE, [binary,base64,gen_server,gen_statem,unicode], "Benchmark only"}. +{skip_groups,"../stdlib_test",ets_SUITE, + [benchmark], + "Benchmark only"}. diff --git a/lib/stdlib/test/stdlib_bench.spec b/lib/stdlib/test/stdlib_bench.spec index 7a0da811a0..6d665f22b6 100644 --- a/lib/stdlib/test/stdlib_bench.spec +++ b/lib/stdlib/test/stdlib_bench.spec @@ -8,3 +8,4 @@ {skip_groups,"../stdlib_test",stdlib_bench_SUITE, [gen_server_comparison,gen_statem_comparison], "Not a benchmark"}. +{groups,"../stdlib_test",ets_SUITE,[benchmark]}. diff --git a/lib/stdlib/test/string_SUITE.erl b/lib/stdlib/test/string_SUITE.erl index 251e09121c..6afe9e7a76 100644 --- a/lib/stdlib/test/string_SUITE.erl +++ b/lib/stdlib/test/string_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2018. All Rights Reserved. +%% Copyright Ericsson AB 2004-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -52,7 +52,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}, - {timetrap,{minutes,1}}]. + {timetrap,{minutes,2}}]. all() -> [{group, chardata}, {group, list_string}]. @@ -737,10 +737,10 @@ meas(Config) -> case ct:get_timetrap_info() of {_,{_,Scale}} when Scale > 1 -> {skip,{will_not_run_in_debug,Scale}}; - _ -> % No scaling, run at most 1.5 min + _ -> % No scaling, run at most 2 mins Tester = spawn(Exec), receive {test_done, Tester} -> ok - after 90000 -> + after 120000 -> io:format("Timelimit reached stopping~n",[]), exit(Tester, die) end, @@ -754,19 +754,22 @@ do_measure(DataDir) -> io:format("~p~n",[byte_size(Bin)]), Do = fun(Name, Func, Mode) -> {N, Mean, Stddev, _} = time_func(Func, Mode, Bin, 20), - io:format("~15w ~6w ~6.2fms ±~5.2fms #~.2w gc included~n", + io:format("~15w ~15w ~8.2fms ±~6.2fms #~.2w gc included~n", [Name, Mode, Mean/1000, Stddev/1000, N]) end, Do2 = fun(Name, Func, Mode) -> {N, Mean, Stddev, _} = time_func(Func, binary, <<>>, 20), - io:format("~15w ~6w ~6.2fms ±~5.2fms #~.2w gc included~n", + io:format("~15w ~15w ~8.2fms ±~6.2fms #~.2w gc included~n", [Name, Mode, Mean/1000, Stddev/1000, N]) end, + %% lefty_list means a list balanced to the left, like + %% [[[30],31],32]. Only some functions check such lists. + Modes = [list, lefty_list, binary, {many_lists,1}, {many_lists, 4}], io:format("----------------------~n"), Do(old_tokens, fun(Str) -> string:tokens(Str, [$\n,$\r]) end, list), Tokens = {lexemes, fun(Str) -> string:lexemes(Str, [$\n,$\r]) end}, - [Do(Name,Fun,Mode) || {Name,Fun} <- [Tokens], Mode <- [list, binary]], + [Do(Name,Fun,Mode) || {Name,Fun} <- [Tokens], Mode <- Modes], S0 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxy.....", S0B = <<"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxy.....">>, @@ -824,17 +827,17 @@ do_measure(DataDir) -> io:format("--~n",[]), NthTokens = {nth_lexemes, fun(Str) -> string:nth_lexeme(Str, 18000, [$\n,$\r]) end}, - [Do(Name,Fun,Mode) || {Name,Fun} <- [NthTokens], Mode <- [list, binary]], + [Do(Name,Fun,Mode) || {Name,Fun} <- [NthTokens], Mode <- Modes], Do2(take_t, repeat(fun() -> string:take(S0, [$.,$y], false, trailing) end), list), Do2(take_t, repeat(fun() -> string:take(S0B, [$.,$y], false, trailing) end), binary), Do2(take_tc, repeat(fun() -> string:take(S0, [$x], true, trailing) end), list), Do2(take_tc, repeat(fun() -> string:take(S0B, [$x], true, trailing) end), binary), Length = {length, fun(Str) -> string:length(Str) end}, - [Do(Name,Fun,Mode) || {Name,Fun} <- [Length], Mode <- [list, binary]], + [Do(Name,Fun,Mode) || {Name,Fun} <- [Length], Mode <- Modes], Reverse = {reverse, fun(Str) -> string:reverse(Str) end}, - [Do(Name,Fun,Mode) || {Name,Fun} <- [Reverse], Mode <- [list, binary]], + [Do(Name,Fun,Mode) || {Name,Fun} <- [Reverse], Mode <- Modes], ok. @@ -1064,7 +1067,33 @@ time_func(N,Sum,SumSq, _, _, Res, _) -> {N, Mean, Stdev, Res}. mode(binary, Bin) -> Bin; -mode(list, Bin) -> unicode:characters_to_list(Bin). +mode(list, Bin) -> unicode:characters_to_list(Bin); +mode(lefty_list, Bin) -> + L = unicode:characters_to_list(Bin), + to_left(L); +mode({many_lists, N}, Bin) -> + group(unicode:characters_to_list(Bin), N). + +group([], _N) -> + []; +group(L, N) -> + try lists:split(N, L) of + {L1, L2} -> + [L1 | group(L2, N)] + catch + _:_ -> + [L] + end. + +to_left([]) -> + []; +to_left([H|L]) -> + to_left([H], L). + +to_left(V, []) -> + V; +to_left(V, [H|L]) -> + to_left([V,H], L). %% %% Old string lists Test cases starts here. diff --git a/lib/stdlib/uc_spec/gen_unicode_mod.escript b/lib/stdlib/uc_spec/gen_unicode_mod.escript index 70eec1a6f2..8636c69a0d 100755..100644 --- a/lib/stdlib/uc_spec/gen_unicode_mod.escript +++ b/lib/stdlib/uc_spec/gen_unicode_mod.escript @@ -4,7 +4,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2017. All Rights Reserved. +%% Copyright Ericsson AB 2017-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -460,17 +460,73 @@ gen_cp(Fd) -> " maybe_improper_list() | {error, unicode:chardata()}.\n"), io:put_chars(Fd, "cp([C|_]=L) when is_integer(C) -> L;\n"), io:put_chars(Fd, "cp([List]) -> cp(List);\n"), - io:put_chars(Fd, "cp([List|R]) ->\n"), - io:put_chars(Fd, " case cp(List) of\n"), - io:put_chars(Fd, " [] -> cp(R);\n"), - io:put_chars(Fd, " [CP] -> [CP|R];\n"), - io:put_chars(Fd, " [C|R0] -> [C|[R0|R]];\n"), - io:put_chars(Fd, " {error,Error} -> {error,[Error|R]}\n"), - io:put_chars(Fd, " end;\n"), + io:put_chars(Fd, "cp([List|R]) -> cpl(List, R);\n"), io:put_chars(Fd, "cp([]) -> [];\n"), io:put_chars(Fd, "cp(<<C/utf8, R/binary>>) -> [C|R];\n"), io:put_chars(Fd, "cp(<<>>) -> [];\n"), - io:put_chars(Fd, "cp(<<R/binary>>) -> {error,R}.\n\n"), + io:put_chars(Fd, "cp(<<R/binary>>) -> {error,R}.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl([C], R) when is_integer(C) -> [C|cpl_1_cont(R)];\n"), + io:put_chars(Fd, "cpl([C|T], R) when is_integer(C) -> [C|cpl_cont(T, R)];\n"), + io:put_chars(Fd, "cpl([List], R) -> cpl(List, R);\n"), + io:put_chars(Fd, "cpl([List|T], R) -> cpl(List, [T|R]);\n"), + io:put_chars(Fd, "cpl([], R) -> cp(R);\n"), + io:put_chars(Fd, "cpl(<<C/utf8, T/binary>>, R) -> [C,T|R];\n"), + io:put_chars(Fd, "cpl(<<>>, R) -> cp(R);\n"), + io:put_chars(Fd, "cpl(<<B/binary>>, R) -> {error,[B|R]}.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "%%%\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_cont([C|T], R) when is_integer(C) -> [C|cpl_cont2(T, R)];\n"), + io:put_chars(Fd, "cpl_cont([L], R) -> cpl_cont(L, R);\n"), + io:put_chars(Fd, "cpl_cont([L|T], R) -> cpl_cont(L, [T|R]);\n"), + io:put_chars(Fd, "cpl_cont([], R) -> cpl_1_cont(R);\n"), + io:put_chars(Fd, "cpl_cont(T, R) -> [T|R].\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_cont2([C|T], R) when is_integer(C) -> [C|cpl_cont3(T, R)];\n"), + io:put_chars(Fd, "cpl_cont2([L], R) -> cpl_cont2(L, R);\n"), + io:put_chars(Fd, "cpl_cont2([L|T], R) -> cpl_cont2(L, [T|R]);\n"), + io:put_chars(Fd, "cpl_cont2([], R) -> cpl_1_cont2(R);\n"), + io:put_chars(Fd, "cpl_cont2(T, R) -> [T|R].\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_cont3([C], R) when is_integer(C) -> [C|R];\n"), + io:put_chars(Fd, "cpl_cont3([C|T], R) when is_integer(C) -> [C,T|R];\n"), + io:put_chars(Fd, "cpl_cont3([L], R) -> cpl_cont3(L, R);\n"), + io:put_chars(Fd, "cpl_cont3([L|T], R) -> cpl_cont3(L, [T|R]);\n"), + io:put_chars(Fd, "cpl_cont3([], R) -> cpl_1_cont3(R);\n"), + io:put_chars(Fd, "cpl_cont3(T, R) -> [T|R].\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "%%%\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_1_cont([C|T]) when is_integer(C) -> [C|cpl_1_cont2(T)];\n"), + io:put_chars(Fd, "cpl_1_cont([L]) -> cpl_1_cont(L);\n"), + io:put_chars(Fd, "cpl_1_cont([L|T]) -> cpl_cont(L, T);\n"), + io:put_chars(Fd, "cpl_1_cont(T) -> T.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_1_cont2([C|T]) when is_integer(C) -> [C|cpl_1_cont3(T)];\n"), + io:put_chars(Fd, "cpl_1_cont2([L]) -> cpl_1_cont2(L);\n"), + io:put_chars(Fd, "cpl_1_cont2([L|T]) -> cpl_cont2(L, T);\n"), + io:put_chars(Fd, "cpl_1_cont2(T) -> T.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_1_cont3([C|_]=T) when is_integer(C) -> T;\n"), + io:put_chars(Fd, "cpl_1_cont3([L]) -> cpl_1_cont3(L);\n"), + io:put_chars(Fd, "cpl_1_cont3([L|T]) -> cpl_cont3(L, T);\n"), + io:put_chars(Fd, "cpl_1_cont3(T) -> T.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "%%%\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cp_no_bin([C|_]=L) when is_integer(C) -> L;\n"), + io:put_chars(Fd, "cp_no_bin([List]) -> cp_no_bin(List);\n"), + io:put_chars(Fd, "cp_no_bin([List|R]) -> cp_no_binl(List, R);\n"), + io:put_chars(Fd, "cp_no_bin([]) -> [];\n"), + io:put_chars(Fd, "cp_no_bin(_) -> binary_found.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cp_no_binl([C], R) when is_integer(C) -> [C|cpl_1_cont(R)];\n"), + io:put_chars(Fd, "cp_no_binl([C|T], R) when is_integer(C) -> [C|cpl_cont(T, R)];\n"), + io:put_chars(Fd, "cp_no_binl([List], R) -> cp_no_binl(List, R);\n"), + io:put_chars(Fd, "cp_no_binl([List|T], R) -> cp_no_binl(List, [T|R]);\n"), + io:put_chars(Fd, "cp_no_binl([], R) -> cp_no_bin(R);\n"), + io:put_chars(Fd, "cp_no_binl(_, _) -> binary_found.\n\n"), ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -481,11 +537,26 @@ gen_gc(Fd, GBP) -> "-spec gc(String::unicode:chardata()) ->" " maybe_improper_list() | {error, unicode:chardata()}.\n"), io:put_chars(Fd, - "gc([CP1, CP2|_]=T)\n" - " when CP1 < 256, CP2 < 256, CP1 =/= $\r -> %% Ascii Fast path\n" - " T;\n" + "gc([]=R) -> R;\n" + "gc([CP]=R) when is_integer(CP) -> R;\n" + "gc([$\\r=CP|R0]) ->\n" + " case cp(R0) of % Don't break CRLF\n" + " [$\\n|R1] -> [[$\\r,$\\n]|R1];\n" + " T -> [CP|T]\n" + " end;\n" + "gc([CP1|T1]=T) when CP1 < 256 ->\n" + " case T1 of\n" + " [CP2|_] when CP2 < 256 -> T; %% Ascii Fast path\n" + " _ -> %% Keep the tail binary.\n" + " case cp_no_bin(T1) of\n" + " [CP2|_]=T3 when CP2 < 256 -> [CP1|T3]; %% Asciii Fast path\n" + " binary_found -> gc_1(T);\n" + " T4 -> gc_1([CP1|T4])\n" + " end\n" + " end;\n" + "gc(<<>>) -> [];\n" "gc(<<CP1/utf8, Rest/binary>>) ->\n" - " if CP1 < 256, CP1 =/= $\r ->\n" + " if CP1 < 256, CP1 =/= $\\r ->\n" " case Rest of\n" " <<CP2/utf8, _/binary>> when CP2 < 256 -> %% Ascii Fast path\n" " [CP1|Rest];\n" @@ -493,13 +564,12 @@ gen_gc(Fd, GBP) -> " end;\n" " true -> gc_1([CP1|Rest])\n" " end;\n" + "gc([CP|_]=T) when is_integer(CP) -> gc_1(T);\n" "gc(Str) ->\n" - " gc_1(cp(Str)).\n\n" - "gc_1([$\\r|R0] = R) ->\n" - " case cp(R0) of % Don't break CRLF\n" - " [$\\n|R1] -> [[$\\r,$\\n]|R1];\n" - " _ -> R\n" - " end;\n" + " case cp(Str) of\n" + " {error,_}=Error -> Error;\n" + " CPs -> gc(CPs)\n" + " end.\n" ), GenExtP = fun(Range) -> io:format(Fd, "gc_1~s gc_ext_pict(R1,[CP]);\n", [gen_clause(Range)]) end, @@ -507,7 +577,12 @@ gen_gc(Fd, GBP) -> %% Pick codepoints below 256 (some data knowledge here) {ExtendedPictographicLow,ExtendedPictographicHigh} = lists:splitwith(fun({Start,undefined}) -> Start < 256 end,ExtendedPictographic0), - + io:put_chars(Fd, + "\ngc_1([$\\r|R0] = R) ->\n" + " case cp(R0) of % Don't break CRLF\n" + " [$\\n|R1] -> [[$\\r,$\\n]|R1];\n" + " _ -> R\n" + " end;\n"), io:put_chars(Fd, "\n%% Handle control\n"), GenControl = fun(Range) -> io:format(Fd, "gc_1~s R0;\n", [gen_clause(Range)]) end, CRs0 = merge_ranges(maps:get(cr, GBP) ++ maps:get(lf, GBP) ++ maps:get(control, GBP), false), @@ -516,7 +591,14 @@ gen_gc(Fd, GBP) -> %%GenControl(R1),GenControl(R2),GenControl(R3), io:put_chars(Fd, "\n%% Optimize Latin-1\n"), [GenExtP(CP) || CP <- merge_ranges(ExtendedPictographicLow)], - io:format(Fd, "gc_1([CP|R]) when CP < 256 -> gc_extend(R,CP);\n\n", []), + + io:format(Fd, + "gc_1([CP|R]=R0) when CP < 256 ->\n" + " case R of\n" + " [CP2|_] when CP2 < 256 -> R0;\n" + " _ -> gc_extend(cp(R), R, CP)\n" + " end;\n", + []), io:put_chars(Fd, "\n%% Continue control\n"), [GenControl(CP) || CP <- Crs], %% One clause per CP @@ -540,7 +622,7 @@ gen_gc(Fd, GBP) -> io:put_chars(Fd, "gc_1([CP|_]=R0) when 44000 < CP, CP < 56000 -> gc_h_lv_lvt(R0, []);\n"), io:put_chars(Fd, "\n%% Handle Regional\n"), - GenRegional = fun(Range) -> io:format(Fd, "gc_1~s gc_regional(R1,[CP]);\n", [gen_clause(Range)]) end, + GenRegional = fun(Range) -> io:format(Fd, "gc_1~s gc_regional(R1,CP);\n", [gen_clause(Range)]) end, [GenRegional(CP) || CP <- merge_ranges(maps:get(regional_indicator,GBP))], %% io:put_chars(Fd, "%% Handle E_Base\n"), %% GenEBase = fun(Range) -> io:format(Fd, "gc_1~s gc_e_cont(R1,[CP]);\n", [gen_clause(Range)]) end, @@ -552,9 +634,7 @@ gen_gc(Fd, GBP) -> io:put_chars(Fd, "%% Handle extended_pictographic\n"), [GenExtP(CP) || CP <- merge_ranges(ExtendedPictographicHigh)], io:put_chars(Fd, "\n%% default clauses\n"), - io:put_chars(Fd, "gc_1([CP|R]) -> gc_extend(R, CP);\n"), - io:put_chars(Fd, "gc_1([]) -> [];\n"), - io:put_chars(Fd, "gc_1({error,_}=Error) -> Error.\n\n"), + io:put_chars(Fd, "gc_1([CP|R]) -> gc_extend(cp(R), R, CP).\n\n"), io:put_chars(Fd, "%% Handle Prepend\n"), io:put_chars(Fd, @@ -581,31 +661,24 @@ gen_gc(Fd, GBP) -> "%% To simplify binary handling in libraries the tail should be kept binary\n" "%% and not a lookahead CP\n" ), - io:put_chars(Fd, "gc_extend(T, Acc) ->\n" - " gc_extend(cp(T), T, Acc).\n\n"), io:put_chars(Fd, - "gc_extend([CP|T], T0, Acc0) ->\n" + "gc_extend([CP|T], T0, CP0) ->\n" " case is_extend(CP) of\n" - " false ->\n" - " case Acc0 of\n" - " [Acc] -> [Acc|T0];\n" - " [_|_]=Acc -> [lists:reverse(Acc)|T0];\n" - " Acc -> [Acc|T0]\n" - " end;\n" - " _TrueOrZWJ ->\n" - " case Acc0 of\n" - " [_|_] -> gc_extend(T, [CP|Acc0]);\n" - " Acc -> gc_extend(T, [CP,Acc])\n" - " end\n" + " false -> [CP0|T0]; % losing work done on T\n" + " _TrueOrZWJ -> gc_extend2(cp(T), T, [CP,CP0])\n" " end;\n" - "gc_extend([], _, Acc0) ->\n" - " case Acc0 of\n" - " [_]=Acc -> Acc;\n" - " [_|_]=Acc -> [lists:reverse(Acc)];\n" - " Acc -> [Acc]\n" + "gc_extend([], _, CP) -> [CP];\n" + "gc_extend({error,R}, _, CP) -> [CP|R].\n\n"), + io:put_chars(Fd, + "gc_extend2([CP|T], T0, Acc) ->\n" + " case is_extend(CP) of\n" + " false -> [lists:reverse(Acc)|T0]; % losing work done on T\n" + " _TrueOrZWJ -> gc_extend2(cp(T), T, [CP|Acc])\n" " end;\n" - "gc_extend({error,R}, T, Acc0) ->\n" - " gc_extend([], T, Acc0) ++ [R].\n\n" + "gc_extend2([], _, Acc) ->\n" + " [lists:reverse(Acc)];\n" + "gc_extend2({error,R}, _, Acc) ->\n" + " [lists:reverse(Acc)] ++ [R].\n\n" ), [ZWJ] = maps:get(zwj, GBP), GenExtend = fun(R) when R =:= ZWJ -> io:format(Fd, "is_extend~s zwj;\n", [gen_single_clause(ZWJ)]); @@ -660,10 +733,10 @@ gen_gc(Fd, GBP) -> %% -------------------- io:put_chars(Fd, "%% Handle Regional\n"), [{RLess,RLarge}] = merge_ranges(maps:get(regional_indicator,GBP)), - io:put_chars(Fd,"gc_regional(R0, Acc) ->\n" + io:put_chars(Fd,"gc_regional(R0, CP0) ->\n" " case cp(R0) of\n"), - io:format(Fd, " [CP|R1] when ~w =< CP,CP =< ~w-> gc_extend(R1,[CP|Acc]);~n",[RLess, RLarge]), - io:put_chars(Fd," R1 -> gc_extend(R1, R0, Acc)\n" + io:format(Fd, " [CP|R1] when ~w =< CP,CP =< ~w-> gc_extend2(cp(R1),R1,[CP,CP0]);~n",[RLess, RLarge]), + io:put_chars(Fd," R1 -> gc_extend(R1, R0, CP0)\n" " end.\n\n"), %% Special hangul @@ -685,16 +758,23 @@ gen_gc(Fd, GBP) -> GenHangulV_2 = fun(Range) -> io:format(Fd, "~8c~s gc_h_T(R1,[CP|Acc]);\n", [$\s,gen_case_clause(Range)]) end, [GenHangulV_2(CP) || CP <- merge_ranges(maps:get(t,GBP))], - io:put_chars(Fd, " R1 -> gc_extend(R1, R0, Acc)\n end.\n\n"), - + io:put_chars(Fd, + " R1 ->\n" + " case Acc of\n" + " [CP] -> gc_extend(R1, R0, CP);\n" + " _ -> gc_extend2(R1, R0, Acc)\n" + " end\n end.\n\n"), io:put_chars(Fd, "%% Handle Hangul T\n"), io:put_chars(Fd, "gc_h_T(R0, Acc) ->\n case cp(R0) of\n"), GenHangulT_1 = fun(Range) -> io:format(Fd, "~8c~s gc_h_T(R1,[CP|Acc]);\n", [$\s,gen_case_clause(Range)]) end, [GenHangulT_1(CP) || CP <- merge_ranges(maps:get(t,GBP))], - io:put_chars(Fd, " R1 -> gc_extend(R1, R0, Acc)\n end.\n\n"), - - io:put_chars(Fd, "gc_h_lv_lvt({error,_}=Error, Acc) -> gc_extend(Error, [], Acc);\n"), + io:put_chars(Fd, + " R1 ->\n" + " case Acc of\n" + " [CP] -> gc_extend(R1, R0, CP);\n" + " _ -> gc_extend2(R1, R0, Acc)\n" + " end\n end.\n\n"), io:put_chars(Fd, "%% Handle Hangul LV\n"), GenHangulLV = fun(Range) -> io:format(Fd, "gc_h_lv_lvt~s gc_h_V(R1,[CP|Acc]);\n", [gen_clause2(Range)]) end, @@ -703,8 +783,10 @@ gen_gc(Fd, GBP) -> GenHangulLVT = fun(Range) -> io:format(Fd, "gc_h_lv_lvt~s gc_h_T(R1,[CP|Acc]);\n", [gen_clause2(Range)]) end, [GenHangulLVT(CP) || CP <- merge_ranges(maps:get(lvt,GBP))], - io:put_chars(Fd, "gc_h_lv_lvt([CP|R], []) -> gc_extend(R, CP);\n"), %% From gc_1/1 - io:put_chars(Fd, "gc_h_lv_lvt(R, Acc) -> gc_extend(R, Acc).\n\n"), + io:put_chars(Fd, "gc_h_lv_lvt([CP|R], []) -> gc_extend(cp(R), R, CP);\n"), %% From gc_1/1 + io:put_chars(Fd, "%% Also handles error tuples\n"), + io:put_chars(Fd, "gc_h_lv_lvt(R, [CP]) -> gc_extend(R, R, CP);\n"), + io:put_chars(Fd, "gc_h_lv_lvt(R, Acc) -> gc_extend2(R, R, Acc).\n\n"), ok. gen_compose_pairs(Fd, ExclData, Data) -> diff --git a/lib/stdlib/vsn.mk b/lib/stdlib/vsn.mk index d46173497b..cbefd6590a 100644 --- a/lib/stdlib/vsn.mk +++ b/lib/stdlib/vsn.mk @@ -1 +1 @@ -STDLIB_VSN = 3.7.1 +STDLIB_VSN = 3.8 diff --git a/lib/syntax_tools/doc/src/notes.xml b/lib/syntax_tools/doc/src/notes.xml index dc13fe474b..772f5e6e04 100644 --- a/lib/syntax_tools/doc/src/notes.xml +++ b/lib/syntax_tools/doc/src/notes.xml @@ -32,6 +32,20 @@ <p>This document describes the changes made to the Syntax_Tools application.</p> +<section><title>Syntax_Tools 2.1.7</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Fix pretty-printing of type funs. </p> + <p> + Own Id: OTP-15519 Aux Id: ERL-815 </p> + </item> + </list> + </section> + +</section> + <section><title>Syntax_Tools 2.1.6</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/syntax_tools/vsn.mk b/lib/syntax_tools/vsn.mk index 8959ebbd04..538c71dc24 100644 --- a/lib/syntax_tools/vsn.mk +++ b/lib/syntax_tools/vsn.mk @@ -1 +1 @@ -SYNTAX_TOOLS_VSN = 2.1.6 +SYNTAX_TOOLS_VSN = 2.1.7 diff --git a/lib/tools/c_src/Makefile.in b/lib/tools/c_src/Makefile.in index cfe91917f8..289322b6fa 100644 --- a/lib/tools/c_src/Makefile.in +++ b/lib/tools/c_src/Makefile.in @@ -29,7 +29,6 @@ CC=@CC@ LD=@LD@ AR=@AR@ RANLIB=@RANLIB@ -RM=@RM@ MKDIR=@MKDIR@ INSTALL=@INSTALL@ INSTALL_DIR=@INSTALL_DIR@ @@ -158,9 +157,9 @@ $(ERTS_LIB): docs: clean: - $(RM) -rf ../obj/* - $(RM) -rf ../bin/* - $(RM) -f ./*~ + $(RM) -r ../obj/* + $(RM) -r ../bin/* + $(RM) ./*~ .PHONY: all erts_lib docs clean diff --git a/lib/tools/doc/src/notes.xml b/lib/tools/doc/src/notes.xml index a6781dfdb3..28f8346a19 100644 --- a/lib/tools/doc/src/notes.xml +++ b/lib/tools/doc/src/notes.xml @@ -31,6 +31,45 @@ </header> <p>This document describes the changes made to the Tools application.</p> +<section><title>Tools 3.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Minor fixes for <c>make clean</c>.</p> + <p> + Own Id: OTP-15657</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + In the HTML file generated by + <c>cover:analyse_to_file/1,2</c>, a link is now added to + the line number. This makes it easier to share pointers + to specific lines.</p> + <p> + Own Id: OTP-15541</p> + </item> + <item> + <p> + Uncovered lines are now marked with a sad face, + <c>:-(</c>, in the HTML output from + <c>cover:analyse_to_file/1,2</c>. This is to make these + lines easier to find by search.</p> + <p> + Own Id: OTP-15542</p> + </item> + </list> + </section> + +</section> + <section><title>Tools 3.0.2</title> <section><title>Improvements and New Features</title> diff --git a/lib/tools/vsn.mk b/lib/tools/vsn.mk index bb8305e9f1..5700885549 100644 --- a/lib/tools/vsn.mk +++ b/lib/tools/vsn.mk @@ -1 +1 @@ -TOOLS_VSN = 3.0.2 +TOOLS_VSN = 3.1 diff --git a/lib/wx/doc/src/notes.xml b/lib/wx/doc/src/notes.xml index 1061e73138..33d02e22ba 100644 --- a/lib/wx/doc/src/notes.xml +++ b/lib/wx/doc/src/notes.xml @@ -32,6 +32,25 @@ <p>This document describes the changes made to the wxErlang application.</p> +<section><title>Wx 1.8.7</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Improved support for wxWidgets 3.1.3 which have changed + <c>wxFONTWEIGTH</c>, also added <c>wxGCDC</c> and + <c>wxDisplay</c> modules.</p> + <p> + Fixed a crash on Mojave and check for events more often.</p> + <p> + Own Id: OTP-15587</p> + </item> + </list> + </section> + +</section> + <section><title>Wx 1.8.6</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/wx/vsn.mk b/lib/wx/vsn.mk index d241a7a1b4..dac219fa98 100644 --- a/lib/wx/vsn.mk +++ b/lib/wx/vsn.mk @@ -1 +1 @@ -WX_VSN = 1.8.6 +WX_VSN = 1.8.7 |