diff options
Diffstat (limited to 'lib')
202 files changed, 7902 insertions, 3139 deletions
diff --git a/lib/Makefile b/lib/Makefile index 6605c6145c..ea2827bef0 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -58,10 +58,10 @@ else SUB_DIRECTORIES = hipe parsetools asn1/src else ifdef TERTIARY_BOOTSTRAP - SUB_DIRECTORIES = snmp sasl jinterface syntax_tools wx + SUB_DIRECTORIES = snmp sasl erl_interface jinterface syntax_tools wx else ifdef DOC_BOOTSTRAP - SUB_DIRECTORIES = xmerl edoc erl_docgen + SUB_DIRECTORIES = xmerl edoc erl_docgen public_key else # Not bootstrap build SUB_DIRECTORIES = $(ERTS_APPLICATIONS) \ $(ERLANG_APPLICATIONS) \ diff --git a/lib/common_test/doc/src/notes.xml b/lib/common_test/doc/src/notes.xml index 2f53f1c29e..c8e0722a0f 100644 --- a/lib/common_test/doc/src/notes.xml +++ b/lib/common_test/doc/src/notes.xml @@ -33,6 +33,66 @@ <file>notes.xml</file> </header> +<section><title>Common_Test 1.17</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + A bug caused <c>ct:encrypt_config_file/3</c> and + <c>ct:decrypt_config_file/3</c> to fail with + <c>badmatch</c> if input parameter <c>KeyOrFile</c> was + <c>{key,string()}</c>. This is now corrected.</p> + <p> + Own Id: OTP-15540</p> + </item> + <item> + <p> + The status of a test case which failed with timetrap + timeout in <c>end_per_testcase</c> could not be modified + by returning <c>{fail,Reason}</c> from a + <c>post_end_per_testcase</c> hook function. This is now + corrected.</p> + <p> + Own Id: OTP-15584 Aux Id: ERIERL-282 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + A new variant of the <c>newline</c> option to + <c>ct_telnet:cmd/3</c> and <c>ct_telnet:send/3</c> is + added, which allows to specify a string to append as + newline indicator on a command. By default, the value is + "\n", but in some cases it is required to be "\r\n", + which this option allows.</p> + <p> + A faulty regular expression given as parameter to + <c>ct_telnet:expect/2,3</c> would earlier crash and look + like an internal error in common_test. A better error + indication is now given, but the test case will still + fail.</p> + <p> + Own Id: OTP-15229 Aux Id: ERIERL-203 </p> + </item> + <item> + <p> + Since the yang RFC allows more than one top element of + config data in an <c>edit-config</c> element, + <c>ct_netconfc:edit_config/3,4,5</c> can now take a list + of XML elements.</p> + <p> + Own Id: OTP-15298</p> + </item> + </list> + </section> + +</section> + <section><title>Common_Test 1.16.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/common_test/test_server/ts_erl_config.erl b/lib/common_test/test_server/ts_erl_config.erl index c2852131d6..f3972bea4e 100644 --- a/lib/common_test/test_server/ts_erl_config.erl +++ b/lib/common_test/test_server/ts_erl_config.erl @@ -197,28 +197,24 @@ system_include(Root, Vars) -> " -I" ++ quote(filename:nativename(filename:join([Root, "erts", "emulator", SysDir]))). erl_interface(Vars,OsType) -> - {Incl, TargetIncl, {LibPath, MkIncl}} = + {Incl, {LibPath, MkIncl}} = case lib_dir(Vars, erl_interface) of {error, bad_name} -> throw({cannot_find_app, erl_interface}); Dir -> - BaseIncl = filename:join(Dir, "include"), - case erl_root(Vars) of - {installed, _Root} -> - {BaseIncl, - [], - {filename:join(Dir, "lib"), - filename:join([Dir, "src", "eidefs.mk"])}}; - {srctree, _Root, Target} -> - Obj = case is_debug_build() of - true -> "obj.debug"; - false -> "obj" - end, - {BaseIncl, - filename:join(BaseIncl, Target), - {filename:join([Dir, Obj, Target]), - filename:join([Dir, "src", Target, "eidefs.mk"])}} - end + {filename:join(Dir, "include"), + case erl_root(Vars) of + {installed, _Root} -> + {filename:join(Dir, "lib"), + filename:join([Dir, "src", "eidefs.mk"])}; + {srctree, _Root, Target} -> + Obj = case is_debug_build() of + true -> "obj.debug"; + false -> "obj" + end, + {filename:join([Dir, Obj, Target]), + filename:join([Dir, "src", Target, "eidefs.mk"])} + end} end, Lib = link_library("erl_interface",OsType), Lib1 = link_library("ei",OsType), @@ -264,10 +260,6 @@ erl_interface(Vars,OsType) -> {erl_interface_eilib_drv, quote(filename:join(LibPath, Lib1Drv))}, {erl_interface_threadlib, ThreadLib}, {erl_interface_include, quote(filename:nativename(Incl))}, - {erl_interface_target_include, case TargetIncl of - [] -> []; - _ -> "-I" ++ quote(filename:nativename(TargetIncl)) - end}, {erl_interface_mk_include, quote(filename:nativename(MkIncl))} | Vars]. diff --git a/lib/common_test/vsn.mk b/lib/common_test/vsn.mk index fd5d4a57aa..23eb8d9656 100644 --- a/lib/common_test/vsn.mk +++ b/lib/common_test/vsn.mk @@ -1 +1 @@ -COMMON_TEST_VSN = 1.16.1 +COMMON_TEST_VSN = 1.17 diff --git a/lib/compiler/doc/src/notes.xml b/lib/compiler/doc/src/notes.xml index 02e6203137..d45dfef8f3 100644 --- a/lib/compiler/doc/src/notes.xml +++ b/lib/compiler/doc/src/notes.xml @@ -32,6 +32,38 @@ <p>This document describes the changes made to the Compiler application.</p> +<section><title>Compiler 7.3.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>An expression such as <c>(A / B) band 16#ff</c> would + crash the compiler.</p> + <p> + Own Id: OTP-15518 Aux Id: ERL-829 </p> + </item> + <item> + <p>There could be an incorrect warning when the + <c>tuple_calls</c> option was given. The generated code + would be correct. Here is an example of code that would + trigger the warning:</p> + <p><c>(list_to_atom("prefix_" ++ + atom_to_list(suffix))):doit(X)</c>.</p> + <p> + Own Id: OTP-15552 Aux Id: ERL-838 </p> + </item> + <item> + <p>Optimize (again) Dialyzer's handling of + left-associative use of <c>andalso</c> and <c>orelse</c> + in guards.</p> + <p> + Own Id: OTP-15577 Aux Id: ERL-851, PR-2141, PR-1944 </p> + </item> + </list> + </section> + +</section> + <section><title>Compiler 7.3.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/compiler/scripts/smoke b/lib/compiler/scripts/smoke index 2429f104c0..ae31c923b8 100755 --- a/lib/compiler/scripts/smoke +++ b/lib/compiler/scripts/smoke @@ -54,6 +54,7 @@ setup_mix() -> ElixirBin = filename:join([SmokeDir,"elixir","bin"]), PATH = ElixirBin ++ ":" ++ os:getenv("PATH"), os:putenv("PATH", PATH), + mix("local.hex --force"), mix("local.rebar --force"), ok. diff --git a/lib/compiler/scripts/smoke-mix.exs b/lib/compiler/scripts/smoke-mix.exs index 82ae3370fe..ba0815e465 100644 --- a/lib/compiler/scripts/smoke-mix.exs +++ b/lib/compiler/scripts/smoke-mix.exs @@ -25,6 +25,14 @@ defmodule Smoke.MixProject do [ {:bear, "~> 0.8.7"}, {:cloudi_core, "~> 1.7"}, + {:cloudi_service_monitoring, "~> 1.7"}, + {:cloudi_service_tcp, "~> 1.7"}, + {:cloudi_service_queue, "~> 1.7"}, + {:cloudi_service_udp, "~> 1.7"}, + {:cloudi_service_map_reduce, "~> 1.7"}, + {:cloudi_service_api_requests, "~> 1.7"}, + {:cloudi_service_router, "~> 1.7"}, + {:cloudi_service_request_rate, "~> 1.7"}, {:concuerror, "~> 0.20.0"}, {:cowboy, "~> 2.6.1"}, {:ecto, "~> 3.0.6"}, diff --git a/lib/compiler/src/beam_ssa_opt.erl b/lib/compiler/src/beam_ssa_opt.erl index bcf55f3fda..90c0d3cf16 100644 --- a/lib/compiler/src/beam_ssa_opt.erl +++ b/lib/compiler/src/beam_ssa_opt.erl @@ -911,6 +911,11 @@ ssa_opt_float({#st{ssa=Linear0,cnt=Count0}=St, FuncDb}) -> {Linear,Count} = float_opt(Linear0, Count0, Fs), {St#st{ssa=Linear,cnt=Count}, FuncDb}. +float_blk_is_in_guard(#b_blk{last=#b_br{fail=F}}, #fs{non_guards=NonGuards}) -> + not gb_sets:is_member(F, NonGuards); +float_blk_is_in_guard(#b_blk{}, #fs{}) -> + false. + float_non_guards([{L,#b_blk{is=Is}}|Bs]) -> case Is of [#b_set{op=landingpad}|_] -> @@ -920,21 +925,18 @@ float_non_guards([{L,#b_blk{is=Is}}|Bs]) -> end; float_non_guards([]) -> [?BADARG_BLOCK]. -float_opt([{L,#b_blk{last=#b_br{fail=F}}=Blk}|Bs0], - Count0, #fs{non_guards=NonGuards}=Fs) -> - case gb_sets:is_member(F, NonGuards) of +float_opt([{L,Blk}|Bs0], Count0, Fs) -> + case float_blk_is_in_guard(Blk, Fs) of true -> - %% This block is not inside a guard. - %% We can do the optimization. - float_opt_1(L, Blk, Bs0, Count0, Fs); - false -> %% This block is inside a guard. Don't do %% any floating point optimizations. {Bs,Count} = float_opt(Bs0, Count0, Fs), - {[{L,Blk}|Bs],Count} + {[{L,Blk}|Bs],Count}; + false -> + %% This block is not inside a guard. + %% We can do the optimization. + float_opt_1(L, Blk, Bs0, Count0, Fs) end; -float_opt([{L,Blk}|Bs], Count, Fs) -> - float_opt_1(L, Blk, Bs, Count, Fs); float_opt([], Count, _Fs) -> {[],Count}. @@ -1010,10 +1012,14 @@ float_conv([{L,#b_blk{is=Is0}=Blk0}|Bs0], Fail, Count0) -> float_maybe_flush(Blk0, #fs{s=cleared,fail=Fail,bs=Blocks}=Fs0, Count0) -> #b_blk{last=#b_br{bool=#b_var{},succ=Succ}=Br} = Blk0, - #b_blk{is=Is} = map_get(Succ, Blocks), + + %% If the success block starts with a floating point operation, we can + %% defer flushing to that block as long as it isn't a guard. + #b_blk{is=Is} = SuccBlk = map_get(Succ, Blocks), + SuccIsGuard = float_blk_is_in_guard(SuccBlk, Fs0), + case Is of - [#b_set{anno=#{float_op:=_}}|_] -> - %% The next operation is also a floating point operation. + [#b_set{anno=#{float_op:=_}}|_] when not SuccIsGuard -> %% No flush needed. {[],Blk0,Fs0,Count0}; _ -> diff --git a/lib/compiler/src/beam_validator.erl b/lib/compiler/src/beam_validator.erl index 296c095be2..4fba3fa1c6 100644 --- a/lib/compiler/src/beam_validator.erl +++ b/lib/compiler/src/beam_validator.erl @@ -579,7 +579,11 @@ valfun_1({get_tuple_element,Src,N,Dst}, Vst) -> Type = get_element_type(Index, Src, Vst), extract_term(Type, {bif,element}, [Index, Src], Dst, Vst); valfun_1({jump,{f,Lbl}}, Vst) -> - kill_state(branch_state(Lbl, Vst)); + branch(Lbl, Vst, + fun(SuccVst) -> + %% The next instruction is never executed. + kill_state(SuccVst) + end); valfun_1(I, Vst) -> valfun_2(I, Vst). @@ -589,14 +593,17 @@ init_try_catch_branch(Tag, Dst, Fail, Vst0) -> St = St0#st{ct=[[Fail]|Fails]}, Vst = Vst0#vst{current=St}, - complex_test(Fail, - fun(CatchVst) -> - #vst{current=#st{ys=Ys}} = CatchVst, - maps:fold(fun init_catch_handler_1/3, CatchVst, Ys) - end, - fun(SuccVst) -> - SuccVst - end, Vst). + branch(Fail, Vst, + fun(CatchVst) -> + #vst{current=#st{ys=Ys}} = CatchVst, + maps:fold(fun init_catch_handler_1/3, CatchVst, Ys) + end, + fun(SuccVst) -> + %% All potentially-throwing instructions after this + %% one will implicitly branch to the fail label; + %% see valfun_2/2 + SuccVst + end). %% Set the initial state at the try/catch label. Assume that Y registers %% contain terms or try/catch tags. @@ -607,21 +614,27 @@ init_catch_handler_1(Reg, uninitialized, Vst) -> init_catch_handler_1(_, _, Vst) -> Vst. -%% Update branched state if necessary and try next set of instructions. -valfun_2(I, #vst{current=#st{ct=[]}}=Vst) -> - valfun_3(I, Vst); valfun_2(I, #vst{current=#st{ct=[[Fail]|_]}}=Vst) when is_integer(Fail) -> - %% Update branched state. + %% We have an active try/catch tag and we can jump there from this + %% instruction, so we need to update the branched state of the try/catch + %% handler. valfun_3(I, branch_state(Fail, Vst)); +valfun_2(I, #vst{current=#st{ct=[]}}=Vst) -> + valfun_3(I, Vst); valfun_2(_, _) -> error(ambiguous_catch_try_state). %% Handle the remaining floating point instructions here. %% Floating point. -valfun_3({fconv,Src,{fr,_}=Dst}, Vst0) -> - assert_term(Src, Vst0), - Vst = update_type(fun meet/2, number, Src, Vst0), - set_freg(Dst, Vst); +valfun_3({fconv,Src,{fr,_}=Dst}, Vst) -> + assert_term(Src, Vst), + + %% An exception is raised on error, hence branching to 0. + branch(0, Vst, + fun(SuccVst0) -> + SuccVst = update_type(fun meet/2, number, Src, SuccVst0), + set_freg(Dst, SuccVst) + end); valfun_3({bif,fadd,_,[_,_]=Ss,Dst}, Vst) -> float_op(Ss, Dst, Vst); valfun_3({bif,fdiv,_,[_,_]=Ss,Dst}, Vst) -> @@ -682,67 +695,87 @@ valfun_4({call_ext_last,_,_,_}, #vst{current=#st{numy=NumY}}) -> valfun_4({make_fun2,_,_,_,Live}, Vst) -> call(make_fun, Live, Vst); %% Other BIFs -valfun_4({bif,element,{f,Fail},[Pos,Tuple],Dst}, Vst0) -> - PosType = get_term_type(Pos, Vst0), - ElementType = get_element_type(PosType, Tuple, Vst0), - InferredType = {tuple,[get_tuple_size(PosType)],#{}}, - Vst1 = branch_state(Fail, Vst0), - Vst2 = update_type(fun meet/2, InferredType, Tuple, Vst1), - Vst = update_type(fun meet/2, {integer,[]}, Pos, Vst2), - extract_term(ElementType, {bif,element}, [Pos,Tuple], Dst, Vst); +valfun_4({bif,element,{f,Fail},[Pos,Src],Dst}, Vst) -> + branch(Fail, Vst, + fun(SuccVst0) -> + PosType = get_term_type(Pos, SuccVst0), + TupleType = {tuple,[get_tuple_size(PosType)],#{}}, + + SuccVst1 = update_type(fun meet/2, TupleType, + Src, SuccVst0), + SuccVst = update_type(fun meet/2, {integer,[]}, + Pos, SuccVst1), + + ElementType = get_element_type(PosType, Src, SuccVst), + extract_term(ElementType, {bif,element}, [Pos,Src], + Dst, SuccVst) + end); valfun_4({bif,raise,{f,0},Src,_Dst}, Vst) -> validate_src(Src, Vst), kill_state(Vst); valfun_4(raw_raise=I, Vst) -> call(I, 3, Vst); -valfun_4({bif,Op,{f,Fail},[Cons]=Ss,Dst}, Vst0) - when Op =:= hd; Op =:= tl -> - validate_src(Ss, Vst0), - Vst = type_test(Fail, cons, Cons, Vst0), - Type = bif_return_type(Op, Ss, Vst), - extract_term(Type, {bif,Op}, Ss, Dst, Vst); -valfun_4({bif,Op,{f,Fail},Ss,Dst}, Vst0) -> - validate_src(Ss, Vst0), - Vst1 = branch_state(Fail, Vst0), - - %% Infer argument types. Note that we can't type_test in the general case - %% as the BIF could fail for reasons other than bad argument types. - ArgTypes = bif_arg_types(Op, Ss), - Vst = foldl(fun({Arg, T}, Vsti) -> - update_type(fun meet/2, T, Arg, Vsti) - end, Vst1, zip(Ss, ArgTypes)), - - Type = bif_return_type(Op, Ss, Vst), - extract_term(Type, {bif,Op}, Ss, Dst, Vst); +valfun_4({bif,Op,{f,Fail},[Src]=Ss,Dst}, Vst) when Op =:= hd; Op =:= tl -> + assert_term(Src, Vst), + branch(Fail, Vst, + fun(FailVst) -> + update_type(fun subtract/2, cons, Src, FailVst) + end, + fun(SuccVst0) -> + SuccVst = update_type(fun meet/2, cons, Src, SuccVst0), + extract_term(term, {bif,Op}, Ss, Dst, SuccVst) + end); +valfun_4({bif,Op,{f,Fail},Ss,Dst}, Vst) -> + validate_src(Ss, Vst), + branch(Fail, Vst, + fun(SuccVst0) -> + %% Infer argument types. Note that we can't subtract + %% types as the BIF could fail for reasons other than + %% bad argument types. + ArgTypes = bif_arg_types(Op, Ss), + SuccVst = foldl(fun({Arg, T}, V) -> + update_type(fun meet/2, T, Arg, V) + end, SuccVst0, zip(Ss, ArgTypes)), + Type = bif_return_type(Op, Ss, SuccVst), + extract_term(Type, {bif,Op}, Ss, Dst, SuccVst) + end); valfun_4({gc_bif,Op,{f,Fail},Live,Ss,Dst}, #vst{current=St0}=Vst0) -> validate_src(Ss, Vst0), verify_live(Live, Vst0), verify_y_init(Vst0), + + %% Heap allocations and X registers are killed regardless of whether we + %% fail or not, as we may fail after GC. St = kill_heap_allocation(St0), - Vst1 = Vst0#vst{current=St}, - Vst2 = branch_state(Fail, Vst1), + Vst = prune_x_regs(Live, Vst0#vst{current=St}), + + branch(Fail, Vst, + fun(SuccVst0) -> + ArgTypes = bif_arg_types(Op, Ss), + SuccVst = foldl(fun({Arg, T}, V) -> + update_type(fun meet/2, T, Arg, V) + end, SuccVst0, zip(Ss, ArgTypes)), - ArgTypes = bif_arg_types(Op, Ss), - Vst3 = foldl(fun({Arg, T}, Vsti) -> - update_type(fun meet/2, T, Arg, Vsti) - end, Vst2, zip(Ss, ArgTypes)), + Type = bif_return_type(Op, Ss, SuccVst), - Type = bif_return_type(Op, Ss, Vst3), - Vst = prune_x_regs(Live, Vst3), - extract_term(Type, {gc_bif,Op}, Ss, Dst, Vst, Vst0); + %% We're passing Vst0 as the original because the + %% registers were pruned before the branch. + extract_term(Type, {gc_bif,Op}, Ss, Dst, SuccVst, Vst0) + end); valfun_4(return, #vst{current=#st{numy=none}}=Vst) -> assert_durable_term({x,0}, Vst), kill_state(Vst); valfun_4(return, #vst{current=#st{numy=NumY}}) -> error({stack_frame,NumY}); -valfun_4({loop_rec,{f,Fail},Dst}, Vst0) -> - %% This term may not be part of the root set until - %% remove_message/0 is executed. If control transfers - %% to the loop_rec_end/1 instruction, no part of - %% this term must be stored in a Y register. - Vst1 = branch_state(Fail, Vst0), - {Ref, Vst} = new_value(term, loop_rec, [], Vst1), - mark_fragile(Dst, set_reg_vref(Ref, Dst, Vst)); +valfun_4({loop_rec,{f,Fail},Dst}, Vst) -> + %% This term may not be part of the root set until remove_message/0 is + %% executed. If control transfers to the loop_rec_end/1 instruction, no + %% part of this term must be stored in a Y register. + branch(Fail, Vst, + fun(SuccVst0) -> + {Ref, SuccVst} = new_value(term, loop_rec, [], SuccVst0), + mark_fragile(Dst, set_reg_vref(Ref, Dst, SuccVst)) + end); valfun_4({wait,_}, Vst) -> verify_y_init(Vst), kill_state(Vst); @@ -769,14 +802,14 @@ valfun_4({set_tuple_element,Src,Tuple,N}, Vst) -> Es = set_element_type({integer,I}, get_term_type(Src, Vst), Es0), override_type({tuple, Sz, Es}, Tuple, Vst); %% Match instructions. -valfun_4({select_val,Src,{f,Fail},{list,Choices}}, Vst0) -> - assert_term(Src, Vst0), +valfun_4({select_val,Src,{f,Fail},{list,Choices}}, Vst) -> + assert_term(Src, Vst), assert_choices(Choices), - select_val_branches(Fail, Src, Choices, Vst0); + validate_select_val(Fail, Choices, Src, Vst); valfun_4({select_tuple_arity,Tuple,{f,Fail},{list,Choices}}, Vst) -> assert_type(tuple, Tuple, Vst), assert_arities(Choices), - select_arity_branches(Fail, Choices, Tuple, Vst); + validate_select_tuple_arity(Fail, Choices, Tuple, Vst); %% New bit syntax matching instructions. valfun_4({test,bs_start_match3,{f,Fail},Live,[Src],Dst}, Vst) -> @@ -785,17 +818,17 @@ valfun_4({test,bs_start_match2,{f,Fail},Live,[Src,Slots],Dst}, Vst) -> validate_bs_start_match(Fail, Live, bsm_match_state(Slots), Src, Dst, Vst); valfun_4({test,bs_match_string,{f,Fail},[Ctx,_,_]}, Vst) -> bsm_validate_context(Ctx, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, fun(V) -> V end); valfun_4({test,bs_skip_bits2,{f,Fail},[Ctx,Src,_,_]}, Vst) -> bsm_validate_context(Ctx, Vst), assert_term(Src, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, fun(V) -> V end); valfun_4({test,bs_test_tail2,{f,Fail},[Ctx,_]}, Vst) -> bsm_validate_context(Ctx, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, fun(V) -> V end); valfun_4({test,bs_test_unit,{f,Fail},[Ctx,_]}, Vst) -> bsm_validate_context(Ctx, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, fun(V) -> V end); valfun_4({test,bs_skip_utf8,{f,Fail},[Ctx,Live,_]}, Vst) -> validate_bs_skip_utf(Fail, Ctx, Live, Vst); valfun_4({test,bs_skip_utf16,{f,Fail},[Ctx,Live,_]}, Vst) -> @@ -830,6 +863,10 @@ valfun_4({bs_set_position, Ctx, Pos}, Vst) -> Vst; %% Other test instructions. +valfun_4({test,has_map_fields,{f,Lbl},Src,{list,List}}, Vst) -> + assert_type(map, Src, Vst), + assert_unique_map_keys(List), + branch(Lbl, Vst, fun(V) -> V end); valfun_4({test,is_atom,{f,Lbl},[Src]}, Vst) -> type_test(Lbl, {atom,[]}, Src, Vst); valfun_4({test,is_binary,{f,Lbl},[Src]}, Vst) -> @@ -850,70 +887,68 @@ valfun_4({test,is_number,{f,Lbl},[Src]}, Vst) -> type_test(Lbl, number, Src, Vst); valfun_4({test,is_list,{f,Lbl},[Src]}, Vst) -> type_test(Lbl, list, Src, Vst); +valfun_4({test,is_map,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, map, Src, Vst); valfun_4({test,is_nil,{f,Lbl},[Src]}, Vst) -> %% is_nil is an exact check against the 'nil' value, and should not be %% treated as a simple type test. assert_term(Src, Vst), - complex_test(Lbl, - fun(FailVst) -> - update_ne_types(Src, nil, FailVst) - end, - fun(SuccVst) -> - update_eq_types(Src, nil, SuccVst) - end, Vst); -valfun_4({test,is_map,{f,Lbl},[Src]}, Vst) -> - case Src of - {Tag,_} when Tag =:= x; Tag =:= y -> - type_test(Lbl, map, Src, Vst); - {literal,Map} when is_map(Map) -> - Vst; - _ -> - assert_term(Src, Vst), - kill_state(Vst) - end; -valfun_4({test,test_arity,{f,Lbl},[Tuple,Sz]}, Vst0) when is_integer(Sz) -> - assert_type(tuple, Tuple, Vst0), - Vst = branch_state(Lbl, Vst0), - update_type(fun meet/2, {tuple,Sz,#{}}, Tuple, Vst); -valfun_4({test,is_tagged_tuple,{f,Lbl},[Src,Sz,Atom]}, Vst0) -> - assert_term(Src, Vst0), - Vst = branch_state(Lbl, Vst0), - update_type(fun meet/2, {tuple,Sz,#{ {integer,1} => Atom }}, Src, Vst); -valfun_4({test,has_map_fields,{f,Lbl},Src,{list,List}}, Vst) -> - assert_type(map, Src, Vst), - assert_unique_map_keys(List), - branch_state(Lbl, Vst); + branch(Lbl, Vst, + fun(FailVst) -> + update_ne_types(Src, nil, FailVst) + end, + fun(SuccVst) -> + update_eq_types(Src, nil, SuccVst) + end); +valfun_4({test,test_arity,{f,Lbl},[Tuple,Sz]}, Vst) when is_integer(Sz) -> + assert_type(tuple, Tuple, Vst), + Type = {tuple, Sz, #{}}, + type_test(Lbl, Type, Tuple, Vst); +valfun_4({test,is_tagged_tuple,{f,Lbl},[Src,Sz,Atom]}, Vst) -> + assert_term(Src, Vst), + Type = {tuple, Sz, #{ {integer,1} => Atom }}, + type_test(Lbl, Type, Src, Vst); valfun_4({test,is_eq_exact,{f,Lbl},[Src,Val]=Ss}, Vst) -> validate_src(Ss, Vst), - complex_test(Lbl, - fun(FailVst) -> - update_ne_types(Src, Val, FailVst) - end, - fun(SuccVst) -> - update_eq_types(Src, Val, SuccVst) - end, Vst); + branch(Lbl, Vst, + fun(FailVst) -> + update_ne_types(Src, Val, FailVst) + end, + fun(SuccVst) -> + update_eq_types(Src, Val, SuccVst) + end); valfun_4({test,is_ne_exact,{f,Lbl},[Src,Val]=Ss}, Vst) -> validate_src(Ss, Vst), - complex_test(Lbl, - fun(FailVst) -> - update_eq_types(Src, Val, FailVst) - end, - fun(SuccVst) -> - update_ne_types(Src, Val, SuccVst) - end, Vst); + branch(Lbl, Vst, + fun(FailVst) -> + update_eq_types(Src, Val, FailVst) + end, + fun(SuccVst) -> + update_ne_types(Src, Val, SuccVst) + end); valfun_4({test,_Op,{f,Lbl},Src}, Vst) -> + %% is_pid, is_reference, et cetera. validate_src(Src, Vst), - branch_state(Lbl, Vst); + branch(Lbl, Vst, fun(V) -> V end); valfun_4({bs_add,{f,Fail},[A,B,_],Dst}, Vst) -> assert_term(A, Vst), assert_term(B, Vst), - create_term({integer,[]}, bs_add, [A, B], Dst, branch_state(Fail, Vst)); + branch(Fail, Vst, + fun(SuccVst) -> + create_term({integer,[]}, bs_add, [A, B], Dst, SuccVst) + end); valfun_4({bs_utf8_size,{f,Fail},A,Dst}, Vst) -> assert_term(A, Vst), - create_term({integer,[]}, bs_utf8_size, [A], Dst, branch_state(Fail, Vst)); + branch(Fail, Vst, + fun(SuccVst) -> + create_term({integer,[]}, bs_utf8_size, [A], Dst, SuccVst) + end); valfun_4({bs_utf16_size,{f,Fail},A,Dst}, Vst) -> assert_term(A, Vst), - create_term({integer,[]}, bs_utf16_size, [A], Dst, branch_state(Fail, Vst)); + branch(Fail, Vst, + fun(SuccVst) -> + create_term({integer,[]}, bs_utf16_size, [A], Dst, SuccVst) + end); valfun_4({bs_init2,{f,Fail},Sz,Heap,Live,_,Dst}, Vst0) -> verify_live(Live, Vst0), verify_y_init(Vst0), @@ -923,10 +958,12 @@ valfun_4({bs_init2,{f,Fail},Sz,Heap,Live,_,Dst}, Vst0) -> true -> assert_term(Sz, Vst0) end, - Vst1 = heap_alloc(Heap, Vst0), - Vst2 = branch_state(Fail, Vst1), - Vst = prune_x_regs(Live, Vst2), - create_term(binary, bs_init2, [], Dst, Vst, Vst0); + Vst = heap_alloc(Heap, Vst0), + branch(Fail, Vst, + fun(SuccVst0) -> + SuccVst = prune_x_regs(Live, SuccVst0), + create_term(binary, bs_init2, [], Dst, SuccVst, SuccVst0) + end); valfun_4({bs_init_bits,{f,Fail},Sz,Heap,Live,_,Dst}, Vst0) -> verify_live(Live, Vst0), verify_y_init(Vst0), @@ -936,47 +973,71 @@ valfun_4({bs_init_bits,{f,Fail},Sz,Heap,Live,_,Dst}, Vst0) -> true -> assert_term(Sz, Vst0) end, - Vst1 = heap_alloc(Heap, Vst0), - Vst2 = branch_state(Fail, Vst1), - Vst = prune_x_regs(Live, Vst2), - create_term(binary, bs_init_bits, [], Dst, Vst); + Vst = heap_alloc(Heap, Vst0), + branch(Fail, Vst, + fun(SuccVst0) -> + SuccVst = prune_x_regs(Live, SuccVst0), + create_term(binary, bs_init_bits, [], Dst, SuccVst) + end); valfun_4({bs_append,{f,Fail},Bits,Heap,Live,_Unit,Bin,_Flags,Dst}, Vst0) -> verify_live(Live, Vst0), verify_y_init(Vst0), assert_term(Bits, Vst0), assert_term(Bin, Vst0), - Vst1 = heap_alloc(Heap, Vst0), - Vst2 = branch_state(Fail, Vst1), - Vst = prune_x_regs(Live, Vst2), - create_term(binary, bs_append, [Bin], Dst, Vst, Vst0); -valfun_4({bs_private_append,{f,Fail},Bits,_Unit,Bin,_Flags,Dst}, Vst0) -> - assert_term(Bits, Vst0), - assert_term(Bin, Vst0), - Vst = branch_state(Fail, Vst0), - create_term(binary, bs_private_append, [Bin], Dst, Vst); + Vst = heap_alloc(Heap, Vst0), + branch(Fail, Vst, + fun(SuccVst0) -> + SuccVst = prune_x_regs(Live, SuccVst0), + create_term(binary, bs_append, [Bin], Dst, SuccVst, SuccVst0) + end); +valfun_4({bs_private_append,{f,Fail},Bits,_Unit,Bin,_Flags,Dst}, Vst) -> + assert_term(Bits, Vst), + assert_term(Bin, Vst), + branch(Fail, Vst, + fun(SuccVst) -> + create_term(binary, bs_private_append, [Bin], Dst, SuccVst) + end); valfun_4({bs_put_string,Sz,_}, Vst) when is_integer(Sz) -> Vst; valfun_4({bs_put_binary,{f,Fail},Sz,_,_,Src}, Vst) -> assert_term(Sz, Vst), assert_term(Src, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, + fun(SuccVst) -> + update_type(fun meet/2, binary, Src, SuccVst) + end); valfun_4({bs_put_float,{f,Fail},Sz,_,_,Src}, Vst) -> assert_term(Sz, Vst), assert_term(Src, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, + fun(SuccVst) -> + update_type(fun meet/2, {float,[]}, Src, SuccVst) + end); valfun_4({bs_put_integer,{f,Fail},Sz,_,_,Src}, Vst) -> assert_term(Sz, Vst), assert_term(Src, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, + fun(SuccVst) -> + update_type(fun meet/2, {integer,[]}, Src, SuccVst) + end); valfun_4({bs_put_utf8,{f,Fail},_,Src}, Vst) -> assert_term(Src, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, + fun(SuccVst) -> + update_type(fun meet/2, {integer,[]}, Src, SuccVst) + end); valfun_4({bs_put_utf16,{f,Fail},_,Src}, Vst) -> assert_term(Src, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, + fun(SuccVst) -> + update_type(fun meet/2, {integer,[]}, Src, SuccVst) + end); valfun_4({bs_put_utf32,{f,Fail},_,Src}, Vst) -> assert_term(Src, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, + fun(SuccVst) -> + update_type(fun meet/2, {integer,[]}, Src, SuccVst) + end); %% Map instructions. valfun_4({put_map_assoc=Op,{f,Fail},Src,Dst,Live,{list,List}}, Vst) -> verify_put_map(Op, Fail, Src, Dst, Live, List, Vst); @@ -991,15 +1052,15 @@ verify_get_map(Fail, Src, List, Vst0) -> assert_not_literal(Src), %OTP 22. assert_type(map, Src, Vst0), - complex_test(Fail, - fun(FailVst) -> - clobber_map_vals(List, Src, FailVst) - end, - fun(SuccVst) -> - Keys = extract_map_keys(List), - assert_unique_map_keys(Keys), - extract_map_vals(List, Src, SuccVst, SuccVst) - end, Vst0). + branch(Fail, Vst0, + fun(FailVst) -> + clobber_map_vals(List, Src, FailVst) + end, + fun(SuccVst) -> + Keys = extract_map_keys(List), + assert_unique_map_keys(Keys), + extract_map_vals(List, Src, SuccVst, SuccVst) + end). %% get_map_elements may leave its destinations in an inconsistent state when %% the fail label is taken. Consider the following: @@ -1033,13 +1094,16 @@ verify_put_map(Op, Fail, Src, Dst, Live, List, Vst0) -> assert_type(map, Src, Vst0), verify_live(Live, Vst0), verify_y_init(Vst0), - [assert_term(Term, Vst0) || Term <- List], - Vst1 = heap_alloc(0, Vst0), - Vst2 = branch_state(Fail, Vst1), - Vst = prune_x_regs(Live, Vst2), - Keys = extract_map_keys(List), - assert_unique_map_keys(Keys), - create_term(map, Op, [Src], Dst, Vst, Vst0). + _ = [assert_term(Term, Vst0) || Term <- List], + Vst = heap_alloc(0, Vst0), + + branch(Fail, Vst, + fun(SuccVst0) -> + SuccVst = prune_x_regs(Live, SuccVst0), + Keys = extract_map_keys(List), + assert_unique_map_keys(Keys), + create_term(map, Op, [Src], Dst, SuccVst, SuccVst0) + end). %% %% Common code for validating bs_start_match* instructions. @@ -1052,40 +1116,60 @@ validate_bs_start_match(Fail, Live, Type, Src, Dst, Vst) -> %% #ms{} can represent either a match context or a term, so we have to mark %% the source as a term if it fails with a match context as an input. This %% hack is only needed until we get proper union types. - complex_test(Fail, - fun(FailVst) -> - case get_raw_type(Src, FailVst) of - #ms{} -> override_type(term, Src, FailVst); - _ -> FailVst - end - end, - fun(SuccVst0) -> - SuccVst1 = update_type(fun meet/2, binary, Src, SuccVst0), - SuccVst = prune_x_regs(Live, SuccVst1), - extract_term(Type, bs_start_match, [Src], Dst, - SuccVst, SuccVst0) - end, Vst). + branch(Fail, Vst, + fun(FailVst) -> + case get_movable_term_type(Src, FailVst) of + #ms{} -> override_type(term, Src, FailVst); + _ -> FailVst + end + end, + fun(SuccVst0) -> + SuccVst1 = update_type(fun meet/2, binary, + Src, SuccVst0), + SuccVst = prune_x_regs(Live, SuccVst1), + extract_term(Type, bs_start_match, [Src], Dst, + SuccVst, SuccVst0) + end). %% %% Common code for validating bs_get* instructions. %% -validate_bs_get(Op, Fail, Ctx, Live, Type, Dst, Vst0) -> - bsm_validate_context(Ctx, Vst0), - verify_live(Live, Vst0), - verify_y_init(Vst0), - Vst1 = prune_x_regs(Live, Vst0), - Vst = branch_state(Fail, Vst1), - extract_term(Type, Op, [Ctx], Dst, Vst, Vst0). +validate_bs_get(Op, Fail, Ctx, Live, Type, Dst, Vst) -> + bsm_validate_context(Ctx, Vst), + verify_live(Live, Vst), + verify_y_init(Vst), + + branch(Fail, Vst, + fun(SuccVst0) -> + SuccVst = prune_x_regs(Live, SuccVst0), + extract_term(Type, Op, [Ctx], Dst, SuccVst, SuccVst0) + end). %% %% Common code for validating bs_skip_utf* instructions. %% -validate_bs_skip_utf(Fail, Ctx, Live, Vst0) -> - bsm_validate_context(Ctx, Vst0), - verify_y_init(Vst0), - verify_live(Live, Vst0), - Vst = prune_x_regs(Live, Vst0), - branch_state(Fail, Vst). +validate_bs_skip_utf(Fail, Ctx, Live, Vst) -> + bsm_validate_context(Ctx, Vst), + verify_y_init(Vst), + verify_live(Live, Vst), + + branch(Fail, Vst, + fun(SuccVst) -> + prune_x_regs(Live, SuccVst) + end). + +%% +%% Common code for is_$type instructions. +%% +type_test(Fail, Type, Reg, Vst) -> + assert_term(Reg, Vst), + branch(Fail, Vst, + fun(FailVst) -> + update_type(fun subtract/2, Type, Reg, FailVst) + end, + fun(SuccVst) -> + update_type(fun meet/2, Type, Reg, SuccVst) + end). %% %% Special state handling for setelement/3 and set_tuple_element/3 instructions. @@ -1156,9 +1240,8 @@ verify_local_args(-1, _Lbl, _CtxIds, _Vst) -> ok; verify_local_args(X, Lbl, CtxIds, Vst) -> Reg = {x, X}, - assert_movable(Reg, Vst), assert_not_fragile(Reg, Vst), - case get_raw_type(Reg, Vst) of + case get_movable_term_type(Reg, Vst) of #ms{id=Id}=Type -> case CtxIds of #{ Id := Other } -> @@ -1347,7 +1430,7 @@ assert_arities(_) -> error(bad_tuple_arity_list). %%% float_op(Ss, Dst, Vst0) -> - [assert_freg_set(S, Vst0) || S <- Ss], + _ = [assert_freg_set(S, Vst0) || S <- Ss], assert_fls(cleared, Vst0), Vst = set_fls(cleared, Vst0), set_freg(Dst, Vst). @@ -1424,7 +1507,7 @@ bsm_validate_context(Reg, Vst) -> ok. bsm_get_context({Kind,_}=Reg, Vst) when Kind =:= x; Kind =:= y-> - case get_raw_type(Reg, Vst) of + case get_movable_term_type(Reg, Vst) of #ms{}=Ctx -> Ctx; _ -> error({no_bsm_context,Reg}) end; @@ -1459,44 +1542,46 @@ bsm_restore(Reg, SavePoint, Vst) -> _ -> error({illegal_restore,SavePoint,range}) end. -select_val_branches(Fail, Src, Choices, Vst0) -> - Vst = svb_1(Choices, Src, Vst0), - kill_state(branch_state(Fail, Vst)). - -svb_1([Val,{f,L}|T], Src, Vst0) -> - Vst = complex_test(L, - fun(BranchVst) -> - update_eq_types(Src, Val, BranchVst) - end, - fun(FailVst) -> - update_ne_types(Src, Val, FailVst) - end, Vst0), - svb_1(T, Src, Vst); -svb_1([], _, Vst) -> - Vst. - -select_arity_branches(Fail, List, Tuple, Vst0) -> - Type = get_term_type(Tuple, Vst0), - Vst = sab_1(List, Tuple, Type, Vst0), - kill_state(branch_state(Fail, Vst)). - -sab_1([Sz,{f,L}|T], Tuple, {tuple,[_],Es}=Type0, Vst0) -> - #vst{current=St0} = Vst0, - Vst1 = update_type(fun meet/2, {tuple,Sz,Es}, Tuple, Vst0), - Vst2 = branch_state(L, Vst1), - Vst = Vst2#vst{current=St0}, - - sab_1(T, Tuple, Type0, Vst); -sab_1([Sz,{f,L}|T], Tuple, {tuple,Sz,_Es}=Type, Vst0) -> - %% The type is already correct. (This test is redundant.) - Vst = branch_state(L, Vst0), - sab_1(T, Tuple, Type, Vst); -sab_1([_,{f,_}|T], Tuple, Type, Vst) -> - %% We already have an established different exact size for the tuple. - %% This label can't possibly be reached. - sab_1(T, Tuple, Type, Vst); -sab_1([], _, _, #vst{}=Vst) -> - Vst. +validate_select_val(_Fail, _Choices, _Src, #vst{current=none}=Vst) -> + %% We've already branched on all of Src's possible values, so we know we + %% can't reach the fail label or any of the remaining choices. + Vst; +validate_select_val(Fail, [Val,{f,L}|T], Src, Vst0) -> + Vst = branch(L, Vst0, + fun(BranchVst) -> + update_eq_types(Src, Val, BranchVst) + end, + fun(FailVst) -> + update_ne_types(Src, Val, FailVst) + end), + validate_select_val(Fail, T, Src, Vst); +validate_select_val(Fail, [], _, Vst) -> + branch(Fail, Vst, + fun(SuccVst) -> + %% The next instruction is never executed. + kill_state(SuccVst) + end). + +validate_select_tuple_arity(_Fail, _Choices, _Src, #vst{current=none}=Vst) -> + %% We've already branched on all of Src's possible values, so we know we + %% can't reach the fail label or any of the remaining choices. + Vst; +validate_select_tuple_arity(Fail, [Arity,{f,L}|T], Tuple, Vst0) -> + Type = {tuple, Arity, #{}}, + Vst = branch(L, Vst0, + fun(BranchVst) -> + update_type(fun meet/2, Type, Tuple, BranchVst) + end, + fun(FailVst) -> + update_type(fun subtract/2, Type, Tuple, FailVst) + end), + validate_select_tuple_arity(Fail, T, Tuple, Vst); +validate_select_tuple_arity(Fail, [], _, #vst{}=Vst) -> + branch(Fail, Vst, + fun(SuccVst) -> + %% The next instruction is never executed. + kill_state(SuccVst) + end). infer_types({Kind,_}=Reg, Vst) when Kind =:= x; Kind =:= y -> infer_types(get_reg_vref(Reg, Vst), Vst); @@ -1510,8 +1595,11 @@ infer_types(_, #vst{}) -> infer_types_1(#value{op={bif,'=:='},args=[LHS,RHS]}) -> fun({atom,true}, S) -> - Infer = infer_types(RHS, S), - Infer(LHS, S); + %% Either side might contain something worth inferring, so we need + %% to check them both. + Infer_L = infer_types(RHS, S), + Infer_R = infer_types(LHS, S), + Infer_R(RHS, Infer_L(LHS, S)); (_, S) -> S end; infer_types_1(#value{op={bif,element},args=[{integer,Index}=Key,Tuple]}) -> @@ -1628,26 +1716,6 @@ resolve_args([Lit | Args], Vst) -> resolve_args([], _Vst) -> []. -%% Helper functions for tests that alter state on both the success and fail -%% branches, keeping the states from tainting each other. -complex_test(Fail, FailFun, SuccFun, Vst0) -> - #vst{current=St0} = Vst0, - Vst1 = FailFun(Vst0), - Vst2 = branch_state(Fail, Vst1), - Vst = Vst2#vst{current=St0}, - SuccFun(Vst). - -%% Helper function for simple "is_type" tests. -type_test(Fail, Type, Reg, Vst) -> - assert_term(Reg, Vst), - complex_test(Fail, - fun(FailVst) -> - update_type(fun subtract/2, Type, Reg, FailVst) - end, - fun(SuccVst) -> - update_type(fun meet/2, Type, Reg, SuccVst) - end, Vst). - %% Overrides the type of Reg. This is ugly but a necessity for certain %% destructive operations. override_type(Type, Reg, Vst) -> @@ -1655,7 +1723,7 @@ override_type(Type, Reg, Vst) -> %% This is used when linear code finds out more and more information about a %% type, so that the type gets more specialized. -update_type(Merge, Type0, #value_ref{}=Ref, Vst) -> +update_type(Merge, With, #value_ref{}=Ref, Vst) -> %% If the old type can't be merged with the new one, the type information %% is inconsistent and we know that some instructions will never be %% executed at run-time. For example: @@ -1664,25 +1732,54 @@ update_type(Merge, Type0, #value_ref{}=Ref, Vst) -> %% {test,is_tuple,Fail,[Reg]}. %% {test,test_arity,Fail,[Reg,5]}. %% - %% Note that the test_arity instruction can never be reached, so we use the - %% new type instead of 'none'. - Type = case Merge(get_raw_type(Ref, Vst), Type0) of - none -> Type0; - T -> T - end, - set_type(Type, Ref, Vst); -update_type(Merge, Type, {Kind,_}=Reg, Vst) when Kind =:= x; Kind =:= y -> - update_type(Merge, Type, get_reg_vref(Reg, Vst), Vst); -update_type(_Merge, _Type, Literal, Vst) -> + %% Note that the test_arity instruction can never be reached, so we need to + %% kill the state to avoid raising an error when we encounter it. + %% + %% Simply returning `kill_state(Vst)` is unsafe however as we might be in + %% the middle of an instruction, and altering the rest of the validator + %% (eg. prune_x_regs/2) to no-op on dead states is prone to error. + %% + %% We therefore throw a 'type_conflict' error instead, which causes + %% validation to fail unless we're in a context where such errors can be + %% handled, such as in a branch handler. + Current = get_raw_type(Ref, Vst), + case Merge(Current, With) of + none -> throw({type_conflict, Current, With}); + Type -> set_type(Type, Ref, Vst) + end; +update_type(Merge, With, {Kind,_}=Reg, Vst) when Kind =:= x; Kind =:= y -> + update_type(Merge, With, get_reg_vref(Reg, Vst), Vst); +update_type(Merge, With, Literal, Vst) -> assert_literal(Literal), - Vst. + %% Literals always retain their type, but we still need to bail on type + %% conflicts. + case Merge(Literal, With) of + none -> throw({type_conflict, Literal, With}); + _Type -> Vst + end. update_ne_types(LHS, RHS, Vst) -> - update_type(fun subtract/2, get_term_type(RHS, Vst), LHS, Vst). + %% While updating types on equality is fairly straightforward, inequality + %% is a bit trickier since all we know is that the *value* of LHS differs + %% from RHS, so we can't blindly subtract their types. + %% + %% Consider `number =/= {integer,[]}`; all we know is that LHS isn't equal + %% to some *specific integer* of unknown value, and if we were to subtract + %% {integer,[]} we would erroneously infer that the new type is {float,[]}. + %% + %% Therefore, we only subtract when we know that RHS has a specific value. + RType = get_term_type(RHS, Vst), + case is_literal(RType) of + true -> update_type(fun subtract/2, RType, LHS, Vst); + false -> Vst + end. update_eq_types(LHS, RHS, Vst0) -> - Infer = infer_types(LHS, Vst0), - Vst1 = Infer(RHS, Vst0), + %% Either side might contain something worth inferring, so we need + %% to check them both. + Infer_L = infer_types(RHS, Vst0), + Infer_R = infer_types(LHS, Vst0), + Vst1 = Infer_R(RHS, Infer_L(LHS, Vst0)), T1 = get_term_type(LHS, Vst1), T2 = get_term_type(RHS, Vst1), @@ -1786,19 +1883,27 @@ assert_term(Src, Vst) -> ok. assert_movable(Src, Vst) -> - _ = get_move_term_type(Src, Vst), + _ = get_movable_term_type(Src, Vst), ok. -assert_literal(nil) -> ok; -assert_literal({atom,A}) when is_atom(A) -> ok; -assert_literal({float,F}) when is_float(F) -> ok; -assert_literal({integer,I}) when is_integer(I) -> ok; -assert_literal({literal,_L}) -> ok; -assert_literal(T) -> error({literal_required,T}). +assert_literal(Src) -> + case is_literal(Src) of + true -> ok; + false -> error({literal_required,Src}) + end. + +assert_not_literal(Src) -> + case is_literal(Src) of + true -> error({literal_not_allowed,Src}); + false -> ok + end. -assert_not_literal({x,_}) -> ok; -assert_not_literal({y,_}) -> ok; -assert_not_literal(Literal) -> error({literal_not_allowed,Literal}). +is_literal(nil) -> true; +is_literal({atom,A}) when is_atom(A) -> true; +is_literal({float,F}) when is_float(F) -> true; +is_literal({integer,I}) when is_integer(I) -> true; +is_literal({literal,_L}) -> true; +is_literal(_) -> false. %% The possible types. %% @@ -2050,6 +2155,7 @@ assert_tuple_elements(Limit, Es) -> %% Subtract Type2 from Type2. Example: %% subtract(list, nil) -> cons +subtract(Same, Same) -> none; subtract(list, nil) -> cons; subtract(list, cons) -> nil; subtract(number, {integer,[]}) -> {float,[]}; @@ -2082,11 +2188,10 @@ assert_type(Needed, Actual) -> get_element_type(Key, Src, Vst) -> get_element_type_1(Key, get_term_type(Src, Vst)). -get_element_type_1({integer,Index}=Key, {tuple,Sz,Es}) -> +get_element_type_1({integer,_}=Key, {tuple,_Sz,Es}) -> case Es of #{ Key := Type } -> Type; - #{} when Index =< Sz -> term; - #{} -> none + #{} -> term end; get_element_type_1(_Index, _Type) -> term. @@ -2103,7 +2208,7 @@ get_tuple_size({integer,Sz}) -> Sz; get_tuple_size(_) -> 0. validate_src(Ss, Vst) when is_list(Ss) -> - [assert_term(S, Vst) || S <- Ss], + _ = [assert_term(S, Vst) || S <- Ss], ok. %% get_term_type(Src, ValidatorState) -> Type @@ -2111,22 +2216,23 @@ validate_src(Ss, Vst) when is_list(Ss) -> %% a standard Erlang type (no catch/try tags or match contexts). get_term_type(Src, Vst) -> - case get_move_term_type(Src, Vst) of + case get_movable_term_type(Src, Vst) of #ms{} -> error({match_context,Src}); Type -> Type end. -%% get_move_term_type(Src, ValidatorState) -> Type +%% get_movable_term_type(Src, ValidatorState) -> Type %% Get the type of the source Src. The returned type Type will be %% a standard Erlang type (no catch/try tags). Match contexts are OK. -get_move_term_type(Src, Vst) -> +get_movable_term_type(Src, Vst) -> case get_raw_type(Src, Vst) of initialized -> error({unassigned,Src}); uninitialized -> error({uninitialized_reg,Src}); {catchtag,_} -> error({catchtag,Src}); {trytag,_} -> error({trytag,Src}); tuple_in_progress -> error({tuple_in_progress,Src}); + {literal,_}=Lit -> get_literal_type(Lit); Type -> Type end. @@ -2145,7 +2251,8 @@ get_tag_type(Src, _) -> error({invalid_tag_register,Src}). %% get_raw_type(Src, ValidatorState) -> Type -%% Return the type of a register without doing any validity checks. +%% Return the type of a register without doing any validity checks or +%% conversions. get_raw_type({x,X}=Src, #vst{current=#st{xs=Xs}}=Vst) when is_integer(X) -> check_limit(Src), case Xs of @@ -2192,10 +2299,53 @@ glt_1(T) when is_tuple(T) -> glt_1(L) -> {literal, L}. +%%% +%%% Branch tracking +%%% + +%% Forks the execution flow, with the provided funs returning the new state of +%% their respective branch; the "fail" fun returns the state where the branch +%% is taken, and the "success" fun returns the state where it's not. +%% +%% If either path is known not to be taken at runtime (eg. due to a type +%% conflict), it will simply be discarded. +-spec branch(Lbl :: label(), + Original :: #vst{}, + FailFun :: BranchFun, + SuccFun :: BranchFun) -> #vst{} when + BranchFun :: fun((#vst{}) -> #vst{}). +branch(Lbl, Vst0, FailFun, SuccFun) -> + #vst{current=St0} = Vst0, + try FailFun(Vst0) of + Vst1 -> + Vst2 = branch_state(Lbl, Vst1), + Vst = Vst2#vst{current=St0}, + try SuccFun(Vst) of + V -> V + catch + {type_conflict, _, _} -> + %% The instruction is guaranteed to fail; kill the state. + kill_state(Vst) + end + catch + {type_conflict, _, _} -> + %% This instruction is guaranteed not to fail, so we run the + %% success branch *without* catching type conflicts to avoid hiding + %% errors in the validator itself; one of the branches must + %% succeed. + SuccFun(Vst0) + end. + +%% A shorthand version of branch/4 for when the state is only altered on +%% success. +branch(Fail, Vst, SuccFun) -> + branch(Fail, Vst, fun(V) -> V end, SuccFun). + +%% Directly branches off the state. This is an "internal" operation that should +%% be used sparingly. branch_state(0, #vst{}=Vst) -> - %% If the instruction fails, the stack may be scanned - %% looking for a catch tag. Therefore the Y registers - %% must be initialized at this point. + %% If the instruction fails, the stack may be scanned looking for a catch + %% tag. Therefore the Y registers must be initialized at this point. verify_y_init(Vst), Vst; branch_state(L, #vst{current=St,branched=B,ref_ctr=Counter0}=Vst) -> diff --git a/lib/compiler/test/beam_except_SUITE.erl b/lib/compiler/test/beam_except_SUITE.erl index 9380fe06c8..8e3b373d29 100644 --- a/lib/compiler/test/beam_except_SUITE.erl +++ b/lib/compiler/test/beam_except_SUITE.erl @@ -84,9 +84,16 @@ coverage(_) -> {'EXIT',{function_clause, [{?MODULE,fc,[y],[File,{line,2}]}|_]}} = (catch fc(y)), - {'EXIT',{function_clause, - [{?MODULE,fc,[[a,b,c]],[File,{line,6}]}|_]}} = - (catch fc([a,b,c])), + case ?MODULE of + beam_except_no_opt_SUITE -> + %% There will be a different stack fram in + %% unoptimized code. + ok; + _ -> + {'EXIT',{function_clause, + [{?MODULE,fc,[[a,b,c]],[File,{line,6}]}|_]}} = + (catch fc([a,b,c])) + end, {'EXIT',{undef,[{erlang,error,[a,b,c],_}|_]}} = (catch erlang:error(a, b, c)), diff --git a/lib/compiler/test/beam_validator_SUITE.erl b/lib/compiler/test/beam_validator_SUITE.erl index 265da43f9d..de5a3c2873 100644 --- a/lib/compiler/test/beam_validator_SUITE.erl +++ b/lib/compiler/test/beam_validator_SUITE.erl @@ -34,7 +34,8 @@ undef_label/1,illegal_instruction/1,failing_gc_guard_bif/1, map_field_lists/1,cover_bin_opt/1, val_dsetel/1,bad_tuples/1,bad_try_catch_nesting/1, - receive_stacked/1,aliased_types/1]). + receive_stacked/1,aliased_types/1,type_conflict/1, + infer_on_eq/1]). -include_lib("common_test/include/ct.hrl"). @@ -63,7 +64,8 @@ groups() -> undef_label,illegal_instruction,failing_gc_guard_bif, map_field_lists,cover_bin_opt,val_dsetel, bad_tuples,bad_try_catch_nesting, - receive_stacked,aliased_types]}]. + receive_stacked,aliased_types,type_conflict, + infer_on_eq]}]. init_per_suite(Config) -> test_lib:recompile(?MODULE), @@ -156,8 +158,8 @@ call_last(Config) when is_list(Config) -> merge_undefined(Config) when is_list(Config) -> Errors = do_val(merge_undefined, Config), [{{t,handle_call,2}, - {{call_ext,1,{extfunc,erlang,exit,1}}, - 10, + {{call_ext,2,{extfunc,debug,filter,2}}, + 22, {uninitialized_reg,{y,_}}}}] = Errors, ok. @@ -630,6 +632,53 @@ aliased_types_3(Bug) -> hd(List) end. + +%% ERL-867; validation proceeded after a type conflict, causing incorrect types +%% to be joined. + +-record(r, { e1 = e1, e2 = e2 }). + +type_conflict(Config) when is_list(Config) -> + {e1, e2} = type_conflict_1(#r{}), + ok. + +type_conflict_1(C) -> + Src = id(C#r.e2), + TRes = try id(Src) of + R -> R + catch + %% C:R can never match, yet it assumed that the type of 'C' was + %% an atom from here on. + C:R -> R + end, + {C#r.e1, TRes}. + +%% ERL-886; validation failed to infer types on both sides of '=:=' + +infer_on_eq(Config) when is_list(Config) -> + {ok, gurka} = infer_on_eq_1(id({gurka})), + {ok, gaffel} = infer_on_eq_2(id({gaffel})), + {ok, elefant} = infer_on_eq_3(id({elefant})), + {ok, myra} = infer_on_eq_4(id({myra})), + ok. + +infer_on_eq_1(T) -> + 1 = erlang:tuple_size(T), + {ok, erlang:element(1, T)}. + +infer_on_eq_2(T) -> + Size = erlang:tuple_size(T), + Size = 1, + {ok, erlang:element(1, T)}. + +infer_on_eq_3(T) -> + true = 1 =:= erlang:tuple_size(T), + {ok, erlang:element(1, T)}. + +infer_on_eq_4(T) -> + true = erlang:tuple_size(T) =:= 1, + {ok, erlang:element(1, T)}. + %%%------------------------------------------------------------------------- transform_remove(Remove, Module) -> diff --git a/lib/compiler/test/beam_validator_SUITE_data/merge_undefined.S b/lib/compiler/test/beam_validator_SUITE_data/merge_undefined.S index 481d55045d..aa344807e4 100644 --- a/lib/compiler/test/beam_validator_SUITE_data/merge_undefined.S +++ b/lib/compiler/test/beam_validator_SUITE_data/merge_undefined.S @@ -15,8 +15,9 @@ {select_val,{x,0},{f,1},{list,[{atom,gurka},{f,3},{atom,delete},{f,4}]}}. {label,3}. {allocate_heap,2,6,2}. - %% The Y registers are not initialized here. {test,is_eq_exact,{f,5},[{x,0},{atom,ok}]}. + %% This is unreachable since {x,0} is known not to be 'ok'. We should not + %% fail with "uninitialized y registers" on erlang:exit/1 {move,{atom,nisse},{x,0}}. {call_ext,1,{extfunc,erlang,exit,1}}. {label,4}. @@ -29,6 +30,7 @@ {call_ext,2,{extfunc,io,format,2}}. {test,is_ne_exact,{f,6},[{x,0},{atom,ok}]}. {label,5}. + %% The Y registers are not initialized here. {move,{atom,logReader},{x,1}}. {move,{atom,console},{x,0}}. {call_ext,2,{extfunc,debug,filter,2}}. diff --git a/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.S b/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.S index 5b974119c6..a878204d16 100644 --- a/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.S +++ b/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.S @@ -240,7 +240,7 @@ {y,0}}. {'%',{no_bin_opt,{binary_used_in,{test,is_binary,{f,34},[{y,0}]}}, [63,{file,"receive_stacked.erl"}]}}. - {test,is_binary,{f,34},[{y,0}]}. + {test,is_eq_exact,{f,34},[{y,0},{literal,<<0,1,2,3>>}]}. remove_message. {move,{integer,42},{x,0}}. {line,[{location,"receive_stacked.erl",64}]}. @@ -283,7 +283,7 @@ {y,0}}. {'%',{no_bin_opt,{[{x,1},{y,0}],{loop_rec_end,{f,38}},not_handled}, [70,{file,"receive_stacked.erl"}]}}. - {test,is_binary,{f,39},[{x,0}]}. + {test,is_eq_exact,{f,39},[{x,0},{literal,<<0,1,2,3>>}]}. remove_message. {move,{integer,42},{x,0}}. {line,[{location,"receive_stacked.erl",71}]}. diff --git a/lib/compiler/test/float_SUITE.erl b/lib/compiler/test/float_SUITE.erl index 831e8279aa..0fa8070dc8 100644 --- a/lib/compiler/test/float_SUITE.erl +++ b/lib/compiler/test/float_SUITE.erl @@ -21,15 +21,16 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, pending/1,bif_calls/1,math_functions/1,mixed_float_and_int/1, - subtract_number_type/1]). + subtract_number_type/1,float_followed_by_guard/1]). -include_lib("common_test/include/ct.hrl"). suite() -> [{ct_hooks,[ts_install_cth]}]. -all() -> +all() -> [pending, bif_calls, math_functions, - mixed_float_and_int, subtract_number_type]. + mixed_float_and_int, subtract_number_type, + float_followed_by_guard]. groups() -> []. @@ -187,5 +188,21 @@ fact(0, P) -> P; fact(1, P) -> P; fact(N, P) -> fact(N-1, P*N). +float_followed_by_guard(Config) when is_list(Config) -> + true = ffbg_1(5, 1), + false = ffbg_1(1, 5), + ok. + +ffbg_1(A, B0) -> + %% This is a non-guard block followed by a *guard block* that starts with a + %% floating point operation, and the compiler erroneously assumed that it + %% was safe to skip fcheckerror because the next block started with a float + %% op. + B = id(B0) / 1.0, + if + A - B > 0.0 -> true; + A - B =< 0.0 -> false + end. + id(I) -> I. diff --git a/lib/compiler/vsn.mk b/lib/compiler/vsn.mk index efedb414ad..a523627384 100644 --- a/lib/compiler/vsn.mk +++ b/lib/compiler/vsn.mk @@ -1 +1 @@ -COMPILER_VSN = 7.3.1 +COMPILER_VSN = 7.3.2 diff --git a/lib/crypto/c_src/Makefile.in b/lib/crypto/c_src/Makefile.in index e1e7f71538..b6a65d7488 100644 --- a/lib/crypto/c_src/Makefile.in +++ b/lib/crypto/c_src/Makefile.in @@ -77,9 +77,7 @@ CRYPTO_OBJS = $(OBJDIR)/crypto$(TYPEMARKER).o \ $(OBJDIR)/algorithms$(TYPEMARKER).o \ $(OBJDIR)/api_ng$(TYPEMARKER).o \ $(OBJDIR)/atoms$(TYPEMARKER).o \ - $(OBJDIR)/block$(TYPEMARKER).o \ $(OBJDIR)/bn$(TYPEMARKER).o \ - $(OBJDIR)/chacha20$(TYPEMARKER).o \ $(OBJDIR)/cipher$(TYPEMARKER).o \ $(OBJDIR)/cmac$(TYPEMARKER).o \ $(OBJDIR)/dh$(TYPEMARKER).o \ @@ -98,7 +96,6 @@ CRYPTO_OBJS = $(OBJDIR)/crypto$(TYPEMARKER).o \ $(OBJDIR)/pkey$(TYPEMARKER).o \ $(OBJDIR)/poly1305$(TYPEMARKER).o \ $(OBJDIR)/rand$(TYPEMARKER).o \ - $(OBJDIR)/rc4$(TYPEMARKER).o \ $(OBJDIR)/rsa$(TYPEMARKER).o \ $(OBJDIR)/srp$(TYPEMARKER).o CALLBACK_OBJS = $(OBJDIR)/crypto_callback$(TYPEMARKER).o diff --git a/lib/crypto/c_src/aes.c b/lib/crypto/c_src/aes.c index ee2bb70fb7..4b01e629f9 100644 --- a/lib/crypto/c_src/aes.c +++ b/lib/crypto/c_src/aes.c @@ -166,156 +166,7 @@ ERL_NIF_TERM aes_ige_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv } -#ifdef HAVE_EVP_AES_CTR -ERL_NIF_TERM aes_ctr_stream_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Key, IVec) */ - ErlNifBinary key_bin, ivec_bin; - struct evp_cipher_ctx *ctx = NULL; - const EVP_CIPHER *cipher; - ERL_NIF_TERM ret; - - ASSERT(argc == 2); - - if (!enif_inspect_iolist_as_binary(env, argv[0], &key_bin)) - goto bad_arg; - if (!enif_inspect_binary(env, argv[1], &ivec_bin)) - goto bad_arg; - if (ivec_bin.size != 16) - goto bad_arg; - - switch (key_bin.size) - { - case 16: - cipher = EVP_aes_128_ctr(); - break; - case 24: - cipher = EVP_aes_192_ctr(); - break; - case 32: - cipher = EVP_aes_256_ctr(); - break; - default: - goto bad_arg; - } - - if ((ctx = enif_alloc_resource(evp_cipher_ctx_rtype, sizeof(struct evp_cipher_ctx))) == NULL) - goto err; - if ((ctx->ctx = EVP_CIPHER_CTX_new()) == NULL) - goto err; - - if (EVP_CipherInit_ex(ctx->ctx, cipher, NULL, - key_bin.data, ivec_bin.data, 1) != 1) - goto err; - - if (EVP_CIPHER_CTX_set_padding(ctx->ctx, 0) != 1) - goto err; - - ret = enif_make_resource(env, ctx); - goto done; - - bad_arg: - return enif_make_badarg(env); - - err: - ret = enif_make_badarg(env); - - done: - if (ctx) - enif_release_resource(ctx); - return ret; -} - -ERL_NIF_TERM aes_ctr_stream_encrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Context, Data) */ - struct evp_cipher_ctx *ctx = NULL, *new_ctx = NULL; - ErlNifBinary data_bin; - ERL_NIF_TERM ret, cipher_term; - unsigned char *out; - int outl = 0; - - ASSERT(argc == 2); - - if (!enif_get_resource(env, argv[0], evp_cipher_ctx_rtype, (void**)&ctx)) - goto bad_arg; - if (!enif_inspect_iolist_as_binary(env, argv[1], &data_bin)) - goto bad_arg; - if (data_bin.size > INT_MAX) - goto bad_arg; - - if ((new_ctx = enif_alloc_resource(evp_cipher_ctx_rtype, sizeof(struct evp_cipher_ctx))) == NULL) - goto err; - if ((new_ctx->ctx = EVP_CIPHER_CTX_new()) == NULL) - goto err; - - if (EVP_CIPHER_CTX_copy(new_ctx->ctx, ctx->ctx) != 1) - goto err; - - if ((out = enif_make_new_binary(env, data_bin.size, &cipher_term)) == NULL) - goto err; - - if (EVP_CipherUpdate(new_ctx->ctx, out, &outl, data_bin.data, (int)data_bin.size) != 1) - goto err; - ASSERT(outl >= 0 && (size_t)outl == data_bin.size); - - ret = enif_make_tuple2(env, enif_make_resource(env, new_ctx), cipher_term); - CONSUME_REDS(env,data_bin); - goto done; - - bad_arg: - return enif_make_badarg(env); - - err: - ret = enif_make_badarg(env); - - done: - if (new_ctx) - enif_release_resource(new_ctx); - return ret; -} - -#else /* if not HAVE_EVP_AES_CTR */ - -ERL_NIF_TERM aes_ctr_stream_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Key, IVec) */ - ASSERT(argc == 2); - - return aes_ctr_stream_init_compat(env, argv[0], argv[1]); -} - - -ERL_NIF_TERM aes_ctr_stream_init_compat(ErlNifEnv* env, const ERL_NIF_TERM key_term, const ERL_NIF_TERM iv_term) -{ - ErlNifBinary key_bin, ivec_bin; - ERL_NIF_TERM ecount_bin; - unsigned char *outp; - - if (!enif_inspect_iolist_as_binary(env, key_term, &key_bin)) - goto bad_arg; - if (key_bin.size != 16 && key_bin.size != 24 && key_bin.size != 32) - goto bad_arg; - if (!enif_inspect_binary(env, iv_term, &ivec_bin)) - goto bad_arg; - if (ivec_bin.size != 16) - goto bad_arg; - if ((outp = enif_make_new_binary(env, AES_BLOCK_SIZE, &ecount_bin)) == NULL) - goto err; - memset(outp, 0, AES_BLOCK_SIZE); - - return enif_make_tuple4(env, key_term, iv_term, ecount_bin, enif_make_int(env, 0)); - - bad_arg: - err: - return enif_make_badarg(env); -} - -ERL_NIF_TERM aes_ctr_stream_encrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - ASSERT(argc == 2); - - return aes_ctr_stream_encrypt_compat(env, argv[0], argv[1]); -} - - +#if !defined(HAVE_EVP_AES_CTR) ERL_NIF_TERM aes_ctr_stream_encrypt_compat(ErlNifEnv* env, const ERL_NIF_TERM state_arg, const ERL_NIF_TERM data_arg) {/* ({Key, IVec, ECount, Num}, Data) */ ErlNifBinary key_bin, ivec_bin, text_bin, ecount_bin; diff --git a/lib/crypto/c_src/aes.h b/lib/crypto/c_src/aes.h index 527d041410..c0b2b91f8d 100644 --- a/lib/crypto/c_src/aes.h +++ b/lib/crypto/c_src/aes.h @@ -27,10 +27,7 @@ ERL_NIF_TERM aes_cfb_8_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[] ERL_NIF_TERM aes_cfb_128_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM aes_ige_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -ERL_NIF_TERM aes_ctr_stream_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -ERL_NIF_TERM aes_ctr_stream_encrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); #if !defined(HAVE_EVP_AES_CTR) -ERL_NIF_TERM aes_ctr_stream_init_compat(ErlNifEnv* env, const ERL_NIF_TERM key_term, const ERL_NIF_TERM iv_term); ERL_NIF_TERM aes_ctr_stream_encrypt_compat(ErlNifEnv* env, const ERL_NIF_TERM state_arg, const ERL_NIF_TERM data_arg); #endif diff --git a/lib/crypto/c_src/api_ng.c b/lib/crypto/c_src/api_ng.c index c4114d1626..6a833a0984 100644 --- a/lib/crypto/c_src/api_ng.c +++ b/lib/crypto/c_src/api_ng.c @@ -25,199 +25,532 @@ /* * A unified set of functions for encryption/decryption. * - * EXPERIMENTAL!! - * */ ERL_NIF_TERM ng_crypto_update(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM ng_crypto_one_shot(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -/* Try better error messages in new functions */ -#define ERROR_Term(Env, ReasonTerm) enif_make_tuple2((Env), atom_error, (ReasonTerm)) -#define ERROR_Str(Env, ReasonString) ERROR_Term((Env), enif_make_string((Env),(ReasonString),(ERL_NIF_LATIN1))) +/* All nif functions return a valid value or throws an exception */ +#define EXCP(Env, Class, Str) enif_raise_exception((Env), \ + enif_make_tuple2((Env), (Class), \ + enif_make_string((Env),(Str),(ERL_NIF_LATIN1)) )) -/* Initializes state for (de)encryption - */ -ERL_NIF_TERM ng_crypto_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Cipher, Key, IVec, Encrypt) % if no IV for the Cipher, set IVec = <<>> - */ - ErlNifBinary key_bin, ivec_bin; - unsigned char *iv = NULL; - struct evp_cipher_ctx *ctx; - const struct cipher_type_t *cipherp; - const EVP_CIPHER *cipher; - ERL_NIF_TERM enc_flg_arg, ret; - int enc; - unsigned iv_len; - - enc_flg_arg = argv[argc-1]; - if (enc_flg_arg == atom_true) - enc = 1; - else if (enc_flg_arg == atom_false) - enc = 0; - else if (enc_flg_arg == atom_undefined) +#define EXCP_NOTSUP(Env, Str) EXCP((Env), atom_notsup, (Str)) +#define EXCP_BADARG(Env, Str) EXCP((Env), atom_badarg, (Str)) +#define EXCP_ERROR(Env, Str) EXCP((Env), atom_error, (Str)) + + +#ifdef HAVE_ECB_IVEC_BUG + /* <= 0.9.8l returns faulty ivec length */ +# define GET_IV_LEN(Ciph) ((Ciph)->flags & ECB_BUG_0_9_8L) ? 0 : EVP_CIPHER_iv_length((Ciph)->cipher.p) +#else +# define GET_IV_LEN(Ciph) EVP_CIPHER_iv_length((Ciph)->cipher.p) +#endif + +/*************************************************************************/ +/* Get the arguments for the initialization of the EVP_CIPHER_CTX. Check */ +/* them and initialize that context. */ +/*************************************************************************/ +static int get_init_args(ErlNifEnv* env, + struct evp_cipher_ctx *ctx_res, + const ERL_NIF_TERM cipher_arg, + const ERL_NIF_TERM key_arg, + const ERL_NIF_TERM ivec_arg, + const ERL_NIF_TERM encflg_arg, + const struct cipher_type_t **cipherp, + ERL_NIF_TERM *return_term) +{ + int ivec_len; + ErlNifBinary key_bin; + ErlNifBinary ivec_bin; + int encflg; + + ctx_res->ctx = NULL; /* For testing if *ctx should be freed after errors */ + + /* Fetch the flag telling if we are going to encrypt (=true) or decrypt (=false) */ + if (encflg_arg == atom_true) + encflg = 1; + else if (encflg_arg == atom_false) + encflg = 0; + else if (encflg_arg == atom_undefined) /* For compat funcs in crypto.erl */ - enc = -1; + encflg = -1; else - return ERROR_Str(env, "Bad enc flag"); + { + *return_term = EXCP_BADARG(env, "Bad enc flag"); + goto err; + } - if (!enif_inspect_binary(env, argv[1], &key_bin)) - return ERROR_Str(env, "Bad key"); + /* Fetch the key */ + if (!enif_inspect_iolist_as_binary(env, key_arg, &key_bin)) + { + *return_term = EXCP_BADARG(env, "Bad key"); + goto err; + } - if (!(cipherp = get_cipher_type(argv[0], key_bin.size))) - return ERROR_Str(env, "Unknown cipher or bad key size"); + /* Fetch cipher type */ + if (!enif_is_atom(env, cipher_arg)) + { + *return_term = EXCP_BADARG(env, "Cipher id is not an atom"); + goto err; + } - if (FORBIDDEN_IN_FIPS(cipherp)) - return enif_raise_exception(env, atom_notsup); + if (!(*cipherp = get_cipher_type(cipher_arg, key_bin.size))) + { + if (!get_cipher_type_no_key(cipher_arg)) + *return_term = EXCP_BADARG(env, "Unknown cipher"); + else + *return_term = EXCP_BADARG(env, "Bad key size"); + goto err; + } - if (enc == -1) - return atom_undefined; + if (FORBIDDEN_IN_FIPS(*cipherp)) + { + *return_term = EXCP_NOTSUP(env, "Forbidden in FIPS"); + goto err; + } - if (!(cipher = cipherp->cipher.p)) { + /* Get ivec_len for this cipher (if we found one) */ #if !defined(HAVE_EVP_AES_CTR) - if (cipherp->flags & AES_CTR_COMPAT) - return aes_ctr_stream_init_compat(env, argv[1], argv[2]); - else + /* This code is for historic OpenSSL where EVP_aes_*_ctr is not defined.... */ + if ((*cipherp)->cipher.p) { + /* Not aes_ctr compatibility code since EVP_* + was defined and assigned to (*cipherp)->cipher.p */ + ivec_len = GET_IV_LEN(*cipherp); + } else { + /* No EVP_* was found */ + if ((*cipherp)->flags & AES_CTR_COMPAT) + /* Use aes_ctr compatibility code later */ + ivec_len = 16; + else { + /* Unsupported crypto */ + *return_term = EXCP_NOTSUP(env, "Cipher not supported in this libcrypto version"); + goto err; + } + } +#else + /* Normal code */ + if (!((*cipherp)->cipher.p)) { + *return_term = EXCP_NOTSUP(env, "Cipher not supported in this libcrypto version"); + goto err; + } + ivec_len = GET_IV_LEN(*cipherp); #endif - return enif_raise_exception(env, atom_notsup); + + /* (*cipherp)->cipher.p != NULL and ivec_len has a value */ + + /* Fetch IV */ + if (ivec_len && (ivec_arg != atom_undefined)) { + if (!enif_inspect_iolist_as_binary(env, ivec_arg, &ivec_bin)) + { + *return_term = EXCP_BADARG(env, "Bad iv type"); + goto err; + } + + if (ivec_len != ivec_bin.size) + { + *return_term = EXCP_BADARG(env, "Bad iv size"); + goto err; + } } -#ifdef HAVE_ECB_IVEC_BUG - if (cipherp->flags & ECB_BUG_0_9_8L) - iv_len = 0; /* <= 0.9.8l returns faulty ivec length */ - else + ctx_res->iv_len = ivec_len; + +#if !defined(HAVE_EVP_AES_CTR) + if (!((*cipherp)->cipher.p) + && ((*cipherp)->flags & AES_CTR_COMPAT) + ) { + /* Must use aes_ctr compatibility code */ + ERL_NIF_TERM ecount_bin; + unsigned char *outp; + if ((outp = enif_make_new_binary(env, AES_BLOCK_SIZE, &ecount_bin)) == NULL) { + *return_term = EXCP_ERROR(env, "Can't allocate ecount_bin"); + goto err; + } + memset(outp, 0, AES_BLOCK_SIZE); + + ctx_res->env = enif_alloc_env(); + if (!ctx_res->env) { + *return_term = EXCP_ERROR(env, "Can't allocate env"); + goto err; + } + ctx_res->state = + enif_make_copy(ctx_res->env, + enif_make_tuple4(env, key_arg, ivec_arg, ecount_bin, enif_make_int(env, 0))); + goto success; + } else { + /* Flag for subsequent calls that no aes_ctr compatibility code should be called */ + ctx_res->state = atom_undefined; + ctx_res->env = NULL; + } #endif - iv_len = EVP_CIPHER_iv_length(cipher); - if (iv_len) { - if (!enif_inspect_binary(env, argv[2], &ivec_bin)) - return ERROR_Str(env, "Bad iv type"); + /* Initialize the EVP_CIPHER_CTX */ - if (iv_len != ivec_bin.size) - return ERROR_Str(env, "Bad iv size"); + ctx_res->ctx = EVP_CIPHER_CTX_new(); + if (! ctx_res->ctx) + { + *return_term = EXCP_ERROR(env, "Can't allocate context"); + goto err; + } - iv = ivec_bin.data; - } + if (!EVP_CipherInit_ex(ctx_res->ctx, (*cipherp)->cipher.p, NULL, NULL, NULL, encflg)) + { + *return_term = EXCP_ERROR(env, "Can't initialize context, step 1"); + goto err; + } - if ((ctx = enif_alloc_resource(evp_cipher_ctx_rtype, sizeof(struct evp_cipher_ctx))) == NULL) - return ERROR_Str(env, "Can't allocate resource"); + if (!EVP_CIPHER_CTX_set_key_length(ctx_res->ctx, (int)key_bin.size)) + { + *return_term = EXCP_ERROR(env, "Can't initialize context, key_length"); + goto err; + } - ctx->ctx = EVP_CIPHER_CTX_new(); - if (! ctx->ctx) - return ERROR_Str(env, "Can't allocate context"); - if (!EVP_CipherInit_ex(ctx->ctx, cipher, NULL, NULL, NULL, enc)) { - enif_release_resource(ctx); - return ERROR_Str(env, "Can't initialize context, step 1"); + if (EVP_CIPHER_type((*cipherp)->cipher.p) == NID_rc2_cbc) { + if (key_bin.size > INT_MAX / 8) { + *return_term = EXCP_BADARG(env, "To large rc2_cbc key"); + goto err; + } + if (!EVP_CIPHER_CTX_ctrl(ctx_res->ctx, EVP_CTRL_SET_RC2_KEY_BITS, (int)key_bin.size * 8, NULL)) { + *return_term = EXCP_ERROR(env, "ctrl rc2_cbc key"); + goto err; + } } - if (!EVP_CIPHER_CTX_set_key_length(ctx->ctx, (int)key_bin.size)) { - enif_release_resource(ctx); - return ERROR_Str(env, "Can't initialize context, key_length"); - } + if (ivec_arg == atom_undefined || ivec_len == 0) + { + if (!EVP_CipherInit_ex(ctx_res->ctx, NULL, NULL, key_bin.data, NULL, -1)) { + *return_term = EXCP_ERROR(env, "Can't initialize key"); + goto err; + } + } + else + if (!EVP_CipherInit_ex(ctx_res->ctx, NULL, NULL, key_bin.data, ivec_bin.data, -1)) + { + *return_term = EXCP_ERROR(env, "Can't initialize key or iv"); + goto err; + } - if (EVP_CIPHER_type(cipher) == NID_rc2_cbc) { - if (key_bin.size > INT_MAX / 8) { - enif_release_resource(ctx); - return ERROR_Str(env, "To large rc2_cbc key"); + EVP_CIPHER_CTX_set_padding(ctx_res->ctx, 0); + + *return_term = atom_ok; + +#if !defined(HAVE_EVP_AES_CTR) + success: +#endif + return 1; + + err: + if (ctx_res->ctx) EVP_CIPHER_CTX_free(ctx_res->ctx); + return 0; +} + +/*************************************************************************/ +/* Get the arguments for the EVP_CipherUpdate function, and call it. */ +/*************************************************************************/ + +static int get_update_args(ErlNifEnv* env, + struct evp_cipher_ctx *ctx_res, + const ERL_NIF_TERM indata_arg, + ERL_NIF_TERM *return_term) +{ + ErlNifBinary in_data_bin, out_data_bin; + int out_len, block_size; + + if (!enif_inspect_binary(env, indata_arg, &in_data_bin) ) + { + *return_term = EXCP_BADARG(env, "Bad 2:nd arg"); + goto err; } - if (!EVP_CIPHER_CTX_ctrl(ctx->ctx, EVP_CTRL_SET_RC2_KEY_BITS, (int)key_bin.size * 8, NULL)) { - enif_release_resource(ctx); - return ERROR_Str(env, "ctrl rc2_cbc key"); + + ASSERT(in_data_bin.size <= INT_MAX); + +#if !defined(HAVE_EVP_AES_CTR) + if (ctx_res->state != atom_undefined) { + ERL_NIF_TERM state0, newstate_and_outdata; + const ERL_NIF_TERM *tuple_argv; + int tuple_argc; + + state0 = enif_make_copy(env, ctx_res->state); + + if (enif_get_tuple(env, state0, &tuple_argc, &tuple_argv) && (tuple_argc == 4)) { + /* A compatibility state term */ + /* encrypt and decrypt is performed by calling the same function */ + newstate_and_outdata = aes_ctr_stream_encrypt_compat(env, state0, indata_arg); + + if (enif_get_tuple(env, newstate_and_outdata, &tuple_argc, &tuple_argv) && (tuple_argc == 2)) { + /* newstate_and_outdata = {NewState, OutData} */ + ctx_res->state = enif_make_copy(ctx_res->env, tuple_argv[0]); + /* Return the OutData (from the newstate_and_outdata tuple) only: */ + *return_term = tuple_argv[1]; + } } + } else +#endif + { + block_size = EVP_CIPHER_CTX_block_size(ctx_res->ctx); + + if (!enif_alloc_binary((size_t)in_data_bin.size+block_size, &out_data_bin)) + { + *return_term = EXCP_ERROR(env, "Can't allocate outdata"); + goto err; + } + + if (!EVP_CipherUpdate(ctx_res->ctx, out_data_bin.data, &out_len, in_data_bin.data, in_data_bin.size)) + { + *return_term = EXCP_ERROR(env, "Can't update"); + goto err; + } + + if (!enif_realloc_binary(&out_data_bin, (size_t)out_len)) + { + *return_term = EXCP_ERROR(env, "Can't reallocate"); + goto err; + } + + CONSUME_REDS(env, in_data_bin); + /* return the result text as a binary: */ + *return_term = enif_make_binary(env, &out_data_bin); } - if (!EVP_CipherInit_ex(ctx->ctx, NULL, NULL, key_bin.data, iv, enc)) { - enif_release_resource(ctx); - return ERROR_Str(env, "Can't initialize key and/or iv"); - } + /* success: */ + return 1; - EVP_CIPHER_CTX_set_padding(ctx->ctx, 0); + err: + return 0; +} - ret = enif_make_resource(env, ctx); - enif_release_resource(ctx); +/*************************************************************************/ +/* Initialize the state for (de/en)cryption */ +/*************************************************************************/ + +ERL_NIF_TERM ng_crypto_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Cipher, Key, IVec, Encrypt) % if no IV for the Cipher, set IVec = <<>> + */ + struct evp_cipher_ctx *ctx_res = NULL; + const struct cipher_type_t *cipherp; + ERL_NIF_TERM ret; + int encflg; + + if (enif_is_atom(env, argv[0])) { + if ((ctx_res = enif_alloc_resource(evp_cipher_ctx_rtype, sizeof(struct evp_cipher_ctx))) == NULL) + return EXCP_ERROR(env, "Can't allocate resource"); + + if (!get_init_args(env, ctx_res, argv[0], argv[1], argv[2], argv[argc-1], + &cipherp, &ret)) + /* Error msg in &ret */ + goto ret; + + ret = enif_make_resource(env, ctx_res); + if(ctx_res) enif_release_resource(ctx_res); + + } else if (enif_get_resource(env, argv[0], evp_cipher_ctx_rtype, (void**)&ctx_res)) { + /* Fetch the flag telling if we are going to encrypt (=true) or decrypt (=false) */ + if (argv[3] == atom_true) + encflg = 1; + else if (argv[3] == atom_false) + encflg = 0; + else { + ret = EXCP_BADARG(env, "Bad enc flag"); + goto ret; + } + if (ctx_res->ctx) { + /* It is *not* a ctx_res for the compatibility handling of non-EVP aes_ctr */ + if (!EVP_CipherInit_ex(ctx_res->ctx, NULL, NULL, NULL, NULL, encflg)) { + ret = EXCP_ERROR(env, "Can't initialize encflag"); + goto ret; + } + } + ret = argv[0]; + } else { + ret = EXCP_BADARG(env, "Bad 1:st arg"); + goto ret; + } + + ret: return ret; } -ERL_NIF_TERM ng_crypto_update(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Context, Data) - (Context, Data, IV) */ - struct evp_cipher_ctx *ctx; - ErlNifBinary in_data_bin, ivec_bin, out_data_bin; - int out_len, block_size; -#if !defined(HAVE_EVP_AES_CTR) - const ERL_NIF_TERM *state_term; - int state_arity; +/*************************************************************************/ +/* Encrypt/decrypt */ +/*************************************************************************/ - if (enif_get_tuple(env, argv[0], &state_arity, &state_term) && (state_arity == 4)) { - return aes_ctr_stream_encrypt_compat(env, argv[0], argv[1]); - } +#if !defined(HAVE_EVP_CIPHER_CTX_COPY) +/* + The EVP_CIPHER_CTX_copy is not available in older cryptolibs although + the function is needed. + Instead of implement it in-place, we have a copy here as a compatibility + function +*/ + +int EVP_CIPHER_CTX_copy(EVP_CIPHER_CTX *out, const EVP_CIPHER_CTX *in); + +int EVP_CIPHER_CTX_copy(EVP_CIPHER_CTX *out, const EVP_CIPHER_CTX *in) +{ + if ((in == NULL) || (in->cipher == NULL)) + { + return 0; + } +#ifdef HAS_ENGINE_SUPPORT + /* Make sure it's safe to copy a cipher context using an ENGINE */ + if (in->engine && !ENGINE_init(in->engine)) + return 0; #endif - if (!enif_get_resource(env, argv[0], evp_cipher_ctx_rtype, (void**)&ctx)) - return ERROR_Str(env, "Bad 1:st arg"); - - if (!enif_inspect_binary(env, argv[1], &in_data_bin) ) - return ERROR_Str(env, "Bad 2:nd arg"); + EVP_CIPHER_CTX_cleanup(out); + memcpy(out,in,sizeof *out); - /* arg[1] was checked by the caller */ - ASSERT(in_data_bin.size =< INT_MAX); + if (in->cipher_data && in->cipher->ctx_size) + { + out->cipher_data=OPENSSL_malloc(in->cipher->ctx_size); + if (!out->cipher_data) + return 0; + memcpy(out->cipher_data,in->cipher_data,in->cipher->ctx_size); + } - block_size = EVP_CIPHER_CTX_block_size(ctx->ctx); - if (in_data_bin.size % (size_t)block_size != 0) - return ERROR_Str(env, "Data not a multiple of block size"); +#if defined(EVP_CIPH_CUSTOM_COPY) && defined(EVP_CTRL_COPY) + if (in->cipher->flags & EVP_CIPH_CUSTOM_COPY) + return in->cipher->ctrl((EVP_CIPHER_CTX *)in, EVP_CTRL_COPY, 0, out); +#endif + return 1; +} +/****** End of compatibility function ******/ +#endif - if (argc==3) { - if (!enif_inspect_iolist_as_binary(env, argv[2], &ivec_bin)) - return ERROR_Str(env, "Not binary IV"); - - if (ivec_bin.size > INT_MAX) - return ERROR_Str(env, "Too big IV"); - - if (!EVP_CipherInit_ex(ctx->ctx, NULL, NULL, NULL, ivec_bin.data, -1)) - return ERROR_Str(env, "Can't set IV"); - } - if (!enif_alloc_binary((size_t)in_data_bin.size+block_size, &out_data_bin)) - return ERROR_Str(env, "Can't allocate outdata"); +ERL_NIF_TERM ng_crypto_update(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Context, Data [, IV]) */ + struct evp_cipher_ctx *ctx_res; + ERL_NIF_TERM ret; + + if (!enif_get_resource(env, argv[0], evp_cipher_ctx_rtype, (void**)&ctx_res)) + return EXCP_BADARG(env, "Bad 1:st arg"); + + if (argc == 3) { + struct evp_cipher_ctx ctx_res_copy; + ErlNifBinary ivec_bin; - if (!EVP_CipherUpdate(ctx->ctx, out_data_bin.data, &out_len, in_data_bin.data, in_data_bin.size)) - return ERROR_Str(env, "Can't update"); + memcpy(&ctx_res_copy, ctx_res, sizeof ctx_res_copy); +#if !defined(HAVE_EVP_AES_CTR) + if (ctx_res_copy.state == atom_undefined) + /* Not going to use aes_ctr compat functions */ +#endif + { + ctx_res_copy.ctx = EVP_CIPHER_CTX_new(); - if (!enif_realloc_binary(&out_data_bin, (size_t)out_len)) - return ERROR_Str(env, "Can't reallocate"); + if (!EVP_CIPHER_CTX_copy(ctx_res_copy.ctx, ctx_res->ctx)) { + ret = EXCP_ERROR(env, "Can't copy ctx_res"); + goto err; + } + } - CONSUME_REDS(env, in_data_bin); - return enif_make_binary(env, &out_data_bin); + ctx_res = &ctx_res_copy; + + if (!enif_inspect_iolist_as_binary(env, argv[2], &ivec_bin)) + { + ret = EXCP_BADARG(env, "Bad iv type"); + goto err; + } + + if (ctx_res_copy.iv_len != ivec_bin.size) + { + ret = EXCP_BADARG(env, "Bad iv size"); + goto err; + } + +#if !defined(HAVE_EVP_AES_CTR) + if ((ctx_res_copy.state != atom_undefined) ) { + /* replace the iv in state with argv[2] */ + ERL_NIF_TERM state0; + const ERL_NIF_TERM *tuple_argv; + int tuple_argc; + state0 = enif_make_copy(env, ctx_res_copy.state); + if (enif_get_tuple(env, state0, &tuple_argc, &tuple_argv) && (tuple_argc == 4)) { + /* A compatibility state term */ + ctx_res_copy.state = enif_make_tuple4(env, tuple_argv[0], argv[2], tuple_argv[2], tuple_argv[3]); + } + } else +#endif + if (!EVP_CipherInit_ex(ctx_res_copy.ctx, NULL, NULL, NULL, ivec_bin.data, -1)) + { + ret = EXCP_ERROR(env, "Can't set iv"); + goto err; + } + + get_update_args(env, &ctx_res_copy, argv[1], &ret); + } else + get_update_args(env, ctx_res, argv[1], &ret); + + err: + return ret; /* Both success and error */ } ERL_NIF_TERM ng_crypto_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Context, Data) - (Context, Data, IV) */ - int i; +{/* (Context, Data [, IV]) */ ErlNifBinary data_bin; - ERL_NIF_TERM new_argv[3]; - ASSERT(argc =< 3); + ASSERT(argc <= 3); - if (!enif_inspect_iolist_as_binary(env, argv[1], &data_bin)) - return ERROR_Str(env, "iodata expected as data"); + if (!enif_inspect_binary(env, argv[1], &data_bin)) + return EXCP_BADARG(env, "expected binary as data"); if (data_bin.size > INT_MAX) - return ERROR_Str(env, "to long data"); - - for (i=0; i<argc; i++) new_argv[i] = argv[i]; - new_argv[1] = enif_make_binary(env, &data_bin); + return EXCP_BADARG(env, "to long data"); /* Run long jobs on a dirty scheduler to not block the current emulator thread */ if (data_bin.size > MAX_BYTES_TO_NIF) { return enif_schedule_nif(env, "ng_crypto_update", ERL_NIF_DIRTY_JOB_CPU_BOUND, - ng_crypto_update, argc, new_argv); + ng_crypto_update, argc, argv); } - return ng_crypto_update(env, argc, new_argv); + return ng_crypto_update(env, argc, argv); +} + +/*************************************************************************/ +/* One shot */ +/*************************************************************************/ + +ERL_NIF_TERM ng_crypto_one_shot(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Cipher, Key, IVec, Data, Encrypt) */ + struct evp_cipher_ctx ctx_res; + const struct cipher_type_t *cipherp; + ERL_NIF_TERM ret; + + if (!get_init_args(env, &ctx_res, argv[0], argv[1], argv[2], argv[4], &cipherp, &ret)) + goto ret; + + get_update_args(env, &ctx_res, argv[3], &ret); + + ret: + if (ctx_res.ctx) + EVP_CIPHER_CTX_free(ctx_res.ctx); + return ret; } +ERL_NIF_TERM ng_crypto_one_shot_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Cipher, Key, IVec, Data, Encrypt) % if no IV for the Cipher, set IVec = <<>> + */ + ErlNifBinary data_bin; + + ASSERT(argc == 5); + + if (!enif_inspect_binary(env, argv[3], &data_bin)) + return EXCP_BADARG(env, "expected binary as data"); + + if (data_bin.size > INT_MAX) + return EXCP_BADARG(env, "to long data"); + + /* Run long jobs on a dirty scheduler to not block the current emulator thread */ + if (data_bin.size > MAX_BYTES_TO_NIF) { + return enif_schedule_nif(env, "ng_crypto_one_shot", + ERL_NIF_DIRTY_JOB_CPU_BOUND, + ng_crypto_one_shot, argc, argv); + } + + return ng_crypto_one_shot(env, argc, argv); +} diff --git a/lib/crypto/c_src/api_ng.h b/lib/crypto/c_src/api_ng.h index a3b40fe7fc..5c7d9af3c5 100644 --- a/lib/crypto/c_src/api_ng.h +++ b/lib/crypto/c_src/api_ng.h @@ -25,5 +25,6 @@ ERL_NIF_TERM ng_crypto_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM ng_crypto_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM ng_crypto_one_shot_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); #endif /* E_AES_H__ */ diff --git a/lib/crypto/c_src/atoms.c b/lib/crypto/c_src/atoms.c index 2e417da7f4..114e3c1985 100644 --- a/lib/crypto/c_src/atoms.c +++ b/lib/crypto/c_src/atoms.c @@ -33,6 +33,7 @@ ERL_NIF_TERM atom_undefined; ERL_NIF_TERM atom_ok; ERL_NIF_TERM atom_none; ERL_NIF_TERM atom_notsup; +ERL_NIF_TERM atom_badarg; ERL_NIF_TERM atom_digest; #ifdef FIPS_SUPPORT ERL_NIF_TERM atom_enabled; @@ -41,6 +42,18 @@ ERL_NIF_TERM atom_not_enabled; ERL_NIF_TERM atom_not_supported; #endif +ERL_NIF_TERM atom_type; +ERL_NIF_TERM atom_size; +ERL_NIF_TERM atom_block_size; +ERL_NIF_TERM atom_key_length; +ERL_NIF_TERM atom_iv_length; +ERL_NIF_TERM atom_mode; +ERL_NIF_TERM atom_ecb_mode; +ERL_NIF_TERM atom_cbc_mode; +ERL_NIF_TERM atom_cfb_mode; +ERL_NIF_TERM atom_ofb_mode; +ERL_NIF_TERM atom_stream_cipher; + #if defined(HAVE_EC) ERL_NIF_TERM atom_prime_field; ERL_NIF_TERM atom_characteristic_two_field; @@ -138,8 +151,21 @@ int init_atoms(ErlNifEnv *env, const ERL_NIF_TERM fips_mode, const ERL_NIF_TERM atom_ok = enif_make_atom(env,"ok"); atom_none = enif_make_atom(env,"none"); atom_notsup = enif_make_atom(env,"notsup"); + atom_badarg = enif_make_atom(env,"badarg"); atom_digest = enif_make_atom(env,"digest"); + atom_type = enif_make_atom(env,"type"); + atom_size = enif_make_atom(env,"size"); + atom_block_size = enif_make_atom(env,"block_size"); + atom_key_length = enif_make_atom(env,"key_length"); + atom_iv_length = enif_make_atom(env,"iv_length"); + atom_mode = enif_make_atom(env,"mode"); + atom_ecb_mode = enif_make_atom(env,"ecb_mode"); + atom_cbc_mode = enif_make_atom(env,"cbc_mode"); + atom_cfb_mode = enif_make_atom(env,"cfb_mode"); + atom_ofb_mode = enif_make_atom(env,"ofb_mode"); + atom_stream_cipher = enif_make_atom(env,"stream_cipher"); + #if defined(HAVE_EC) atom_prime_field = enif_make_atom(env,"prime_field"); atom_characteristic_two_field = enif_make_atom(env,"characteristic_two_field"); diff --git a/lib/crypto/c_src/atoms.h b/lib/crypto/c_src/atoms.h index f15523d865..fc46d838aa 100644 --- a/lib/crypto/c_src/atoms.h +++ b/lib/crypto/c_src/atoms.h @@ -37,6 +37,7 @@ extern ERL_NIF_TERM atom_undefined; extern ERL_NIF_TERM atom_ok; extern ERL_NIF_TERM atom_none; extern ERL_NIF_TERM atom_notsup; +extern ERL_NIF_TERM atom_badarg; extern ERL_NIF_TERM atom_digest; #ifdef FIPS_SUPPORT extern ERL_NIF_TERM atom_enabled; @@ -45,6 +46,18 @@ extern ERL_NIF_TERM atom_not_enabled; extern ERL_NIF_TERM atom_not_supported; #endif +extern ERL_NIF_TERM atom_type; +extern ERL_NIF_TERM atom_size; +extern ERL_NIF_TERM atom_block_size; +extern ERL_NIF_TERM atom_key_length; +extern ERL_NIF_TERM atom_iv_length; +extern ERL_NIF_TERM atom_mode; +extern ERL_NIF_TERM atom_ecb_mode; +extern ERL_NIF_TERM atom_cbc_mode; +extern ERL_NIF_TERM atom_cfb_mode; +extern ERL_NIF_TERM atom_ofb_mode; +extern ERL_NIF_TERM atom_stream_cipher; + #if defined(HAVE_EC) extern ERL_NIF_TERM atom_prime_field; extern ERL_NIF_TERM atom_characteristic_two_field; diff --git a/lib/crypto/c_src/block.c b/lib/crypto/c_src/block.c deleted file mode 100644 index 0a4fd72623..0000000000 --- a/lib/crypto/c_src/block.c +++ /dev/null @@ -1,149 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2010-2018. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -#include "block.h" -#include "aes.h" -#include "cipher.h" - -ERL_NIF_TERM block_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Type, Key, Ivec, Text, IsEncrypt) or (Type, Key, Text, IsEncrypt) */ - const struct cipher_type_t *cipherp; - const EVP_CIPHER *cipher; - ErlNifBinary key, ivec, text; - EVP_CIPHER_CTX *ctx = NULL; - ERL_NIF_TERM ret; - unsigned char *out; - int ivec_size, out_size = 0; - int cipher_len; - - ASSERT(argc == 4 || argc == 5); - - if (!enif_inspect_iolist_as_binary(env, argv[1], &key)) - goto bad_arg; - if (key.size > INT_MAX) - goto bad_arg; - if ((cipherp = get_cipher_type(argv[0], key.size)) == NULL) - goto bad_arg; - if (cipherp->flags & (NON_EVP_CIPHER | AEAD_CIPHER)) - goto bad_arg; - if (!enif_inspect_iolist_as_binary(env, argv[argc - 2], &text)) - goto bad_arg; - if (text.size > INT_MAX) - goto bad_arg; - - if (FORBIDDEN_IN_FIPS(cipherp)) - return enif_raise_exception(env, atom_notsup); - if ((cipher = cipherp->cipher.p) == NULL) - return enif_raise_exception(env, atom_notsup); - - if (cipherp->flags & AES_CFBx) { - if (argv[0] == atom_aes_cfb8 - && (key.size == 24 || key.size == 32)) { - /* Why do EVP_CIPHER_CTX_set_key_length() fail on these key sizes? - * Fall back on low level API - */ - return aes_cfb_8_crypt(env, argc-1, argv+1); - } - else if (argv[0] == atom_aes_cfb128 - && (key.size == 24 || key.size == 32)) { - /* Why do EVP_CIPHER_CTX_set_key_length() fail on these key sizes? - * Fall back on low level API - */ - return aes_cfb_128_crypt_nif(env, argc-1, argv+1); - } - } - - ivec_size = EVP_CIPHER_iv_length(cipher); - -#ifdef HAVE_ECB_IVEC_BUG - if (cipherp->flags & ECB_BUG_0_9_8L) - ivec_size = 0; /* 0.9.8l returns faulty ivec_size */ -#endif - - if (ivec_size < 0) - goto bad_arg; - - if ((cipher_len = EVP_CIPHER_block_size(cipher)) < 0) - goto bad_arg; - if (text.size % (size_t)cipher_len != 0) - goto bad_arg; - - if (ivec_size == 0) { - if (argc != 4) - goto bad_arg; - } else { - if (argc != 5) - goto bad_arg; - if (!enif_inspect_iolist_as_binary(env, argv[2], &ivec)) - goto bad_arg; - if (ivec.size != (size_t)ivec_size) - goto bad_arg; - } - - if ((out = enif_make_new_binary(env, text.size, &ret)) == NULL) - goto err; - if ((ctx = EVP_CIPHER_CTX_new()) == NULL) - goto err; - - if (!EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL, - (argv[argc - 1] == atom_true))) - goto err; - if (!EVP_CIPHER_CTX_set_key_length(ctx, (int)key.size)) - goto err; - - if (EVP_CIPHER_type(cipher) == NID_rc2_cbc) { - if (key.size > INT_MAX / 8) - goto err; - if (!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_SET_RC2_KEY_BITS, (int)key.size * 8, NULL)) - goto err; - } - - if (!EVP_CipherInit_ex(ctx, NULL, NULL, key.data, - ivec_size ? ivec.data : NULL, -1)) - goto err; - if (!EVP_CIPHER_CTX_set_padding(ctx, 0)) - goto err; - - /* OpenSSL 0.9.8h asserts text.size > 0 */ - if (text.size > 0) { - if (!EVP_CipherUpdate(ctx, out, &out_size, text.data, (int)text.size)) - goto err; - if (ASSERT(out_size == text.size), 0) - goto err; - if (!EVP_CipherFinal_ex(ctx, out + out_size, &out_size)) - goto err; - } - - ASSERT(out_size == 0); - CONSUME_REDS(env, text); - goto done; - - bad_arg: - ret = enif_make_badarg(env); - goto done; - - err: - ret = enif_raise_exception(env, atom_notsup); - - done: - if (ctx) - EVP_CIPHER_CTX_free(ctx); - return ret; -} diff --git a/lib/crypto/c_src/block.h b/lib/crypto/c_src/block.h deleted file mode 100644 index cc5e78ce12..0000000000 --- a/lib/crypto/c_src/block.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2010-2018. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -#ifndef E_BLOCK_H__ -#define E_BLOCK_H__ 1 - -#include "common.h" - -ERL_NIF_TERM block_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); - -#endif /* E_BLOCK_H__ */ diff --git a/lib/crypto/c_src/chacha20.c b/lib/crypto/c_src/chacha20.c deleted file mode 100644 index cfcc395dca..0000000000 --- a/lib/crypto/c_src/chacha20.c +++ /dev/null @@ -1,124 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2010-2018. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -#include "chacha20.h" -#include "cipher.h" - -ERL_NIF_TERM chacha20_stream_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Key, IV) */ -#if defined(HAVE_CHACHA20) - ErlNifBinary key_bin, ivec_bin; - struct evp_cipher_ctx *ctx = NULL; - const EVP_CIPHER *cipher; - ERL_NIF_TERM ret; - - ASSERT(argc == 2); - - if (!enif_inspect_iolist_as_binary(env, argv[0], &key_bin)) - goto bad_arg; - if (key_bin.size != 32) - goto bad_arg; - if (!enif_inspect_binary(env, argv[1], &ivec_bin)) - goto bad_arg; - if (ivec_bin.size != 16) - goto bad_arg; - - cipher = EVP_chacha20(); - - if ((ctx = enif_alloc_resource(evp_cipher_ctx_rtype, sizeof(struct evp_cipher_ctx))) == NULL) - goto err; - if ((ctx->ctx = EVP_CIPHER_CTX_new()) == NULL) - goto err; - - if (EVP_CipherInit_ex(ctx->ctx, cipher, NULL, - key_bin.data, ivec_bin.data, 1) != 1) - goto err; - if (EVP_CIPHER_CTX_set_padding(ctx->ctx, 0) != 1) - goto err; - - ret = enif_make_resource(env, ctx); - goto done; - - bad_arg: - return enif_make_badarg(env); - - err: - ret = enif_make_badarg(env); - - done: - if (ctx) - enif_release_resource(ctx); - return ret; - -#else - return enif_raise_exception(env, atom_notsup); -#endif -} - -ERL_NIF_TERM chacha20_stream_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (State, Data) */ -#if defined(HAVE_CHACHA20) - struct evp_cipher_ctx *ctx = NULL, *new_ctx = NULL; - ErlNifBinary data_bin; - ERL_NIF_TERM ret, cipher_term; - unsigned char *out; - int outl = 0; - - ASSERT(argc == 2); - - if (!enif_get_resource(env, argv[0], evp_cipher_ctx_rtype, (void**)&ctx)) - goto bad_arg; - if (!enif_inspect_iolist_as_binary(env, argv[1], &data_bin)) - goto bad_arg; - if (data_bin.size > INT_MAX) - goto bad_arg; - - if ((new_ctx = enif_alloc_resource(evp_cipher_ctx_rtype, sizeof(struct evp_cipher_ctx))) == NULL) - goto err; - if ((new_ctx->ctx = EVP_CIPHER_CTX_new()) == NULL) - goto err; - - if (EVP_CIPHER_CTX_copy(new_ctx->ctx, ctx->ctx) != 1) - goto err; - if ((out = enif_make_new_binary(env, data_bin.size, &cipher_term)) == NULL) - goto err; - if (EVP_CipherUpdate(new_ctx->ctx, out, &outl, data_bin.data, (int)data_bin.size) != 1) - goto err; - ASSERT(outl >= 0 && (size_t)outl == data_bin.size); - - ret = enif_make_tuple2(env, enif_make_resource(env, new_ctx), cipher_term); - CONSUME_REDS(env, data_bin); - goto done; - - bad_arg: - return enif_make_badarg(env); - - err: - ret = enif_make_badarg(env); - - done: - if (new_ctx) - enif_release_resource(new_ctx); - return ret; - -#else - return enif_raise_exception(env, atom_notsup); -#endif -} diff --git a/lib/crypto/c_src/chacha20.h b/lib/crypto/c_src/chacha20.h deleted file mode 100644 index 7e2ccae2bb..0000000000 --- a/lib/crypto/c_src/chacha20.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2010-2018. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -#ifndef E_CHACHA20_H__ -#define E_CHACHA20_H__ 1 - -#include "common.h" - -ERL_NIF_TERM chacha20_stream_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -ERL_NIF_TERM chacha20_stream_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); - -#endif /* E_CHACHA20_H__ */ diff --git a/lib/crypto/c_src/cipher.c b/lib/crypto/c_src/cipher.c index f8e44b228a..5c57898c50 100644 --- a/lib/crypto/c_src/cipher.c +++ b/lib/crypto/c_src/cipher.c @@ -67,14 +67,26 @@ static struct cipher_type_t cipher_types[] = {{"aes_cfb8"}, {&EVP_aes_192_cfb8}, 24, NO_FIPS_CIPHER | AES_CFBx}, {{"aes_cfb8"}, {&EVP_aes_256_cfb8}, 32, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_128_cfb8"}, {&EVP_aes_128_cfb8}, 16, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_192_cfb8"}, {&EVP_aes_192_cfb8}, 24, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_256_cfb8"}, {&EVP_aes_256_cfb8}, 32, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_cfb128"}, {&EVP_aes_128_cfb128}, 16, NO_FIPS_CIPHER | AES_CFBx}, {{"aes_cfb128"}, {&EVP_aes_192_cfb128}, 24, NO_FIPS_CIPHER | AES_CFBx}, {{"aes_cfb128"}, {&EVP_aes_256_cfb128}, 32, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_128_cfb128"}, {&EVP_aes_128_cfb128}, 16, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_192_cfb128"}, {&EVP_aes_192_cfb128}, 24, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_256_cfb128"}, {&EVP_aes_256_cfb128}, 32, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_ecb"}, {&EVP_aes_128_ecb}, 16, ECB_BUG_0_9_8L}, {{"aes_ecb"}, {&EVP_aes_192_ecb}, 24, ECB_BUG_0_9_8L}, {{"aes_ecb"}, {&EVP_aes_256_ecb}, 32, ECB_BUG_0_9_8L}, + {{"aes_128_ecb"}, {&EVP_aes_128_ecb}, 16, ECB_BUG_0_9_8L}, + {{"aes_192_ecb"}, {&EVP_aes_192_ecb}, 24, ECB_BUG_0_9_8L}, + {{"aes_256_ecb"}, {&EVP_aes_256_ecb}, 32, ECB_BUG_0_9_8L}, + #if defined(HAVE_EVP_AES_CTR) {{"aes_128_ctr"}, {&EVP_aes_128_ctr}, 16, 0}, {{"aes_192_ctr"}, {&EVP_aes_192_ctr}, 24, 0}, @@ -86,7 +98,9 @@ static struct cipher_type_t cipher_types[] = {{"aes_128_ctr"}, {NULL}, 16, AES_CTR_COMPAT}, {{"aes_192_ctr"}, {NULL}, 24, AES_CTR_COMPAT}, {{"aes_256_ctr"}, {NULL}, 32, AES_CTR_COMPAT}, - {{"aes_ctr"}, {NULL}, 0, AES_CTR_COMPAT}, + {{"aes_ctr"}, {NULL}, 16, AES_CTR_COMPAT}, + {{"aes_ctr"}, {NULL}, 24, AES_CTR_COMPAT}, + {{"aes_ctr"}, {NULL}, 32, AES_CTR_COMPAT}, #endif #if defined(HAVE_CHACHA20) @@ -150,6 +164,11 @@ static void evp_cipher_ctx_dtor(ErlNifEnv* env, struct evp_cipher_ctx* ctx) { if (ctx->ctx) EVP_CIPHER_CTX_free(ctx->ctx); + +#if !defined(HAVE_EVP_AES_CTR) + if (ctx->env) + enif_free_env(ctx->env); +#endif } int init_cipher_ctx(ErlNifEnv *env) { @@ -207,6 +226,87 @@ int cmp_cipher_types(const void *keyp, const void *elemp) { } +ERL_NIF_TERM cipher_info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Type) */ + const struct cipher_type_t *cipherp; + const EVP_CIPHER *cipher; + ERL_NIF_TERM ret, ret_mode; + unsigned type; + unsigned long mode; + + if ((cipherp = get_cipher_type_no_key(argv[0])) == NULL) + return enif_make_badarg(env); + + if (FORBIDDEN_IN_FIPS(cipherp)) + return enif_raise_exception(env, atom_notsup); + if ((cipher = cipherp->cipher.p) == NULL) + return enif_raise_exception(env, atom_notsup); + + ret = enif_make_new_map(env); + + type = EVP_CIPHER_type(cipher); + enif_make_map_put(env, ret, atom_type, + type == NID_undef ? atom_undefined : enif_make_int(env, type), + &ret); + + enif_make_map_put(env, ret, atom_key_length, + enif_make_int(env, EVP_CIPHER_key_length(cipher)), &ret); + enif_make_map_put(env, ret, atom_iv_length, + enif_make_int(env, EVP_CIPHER_iv_length(cipher)), &ret); + enif_make_map_put(env, ret, atom_block_size, + enif_make_int(env, EVP_CIPHER_block_size(cipher)), &ret); + + mode = EVP_CIPHER_mode(cipher); + switch (mode) { + case EVP_CIPH_ECB_MODE: + ret_mode = atom_ecb_mode; + break; + + case EVP_CIPH_CBC_MODE: + ret_mode = atom_cbc_mode; + break; + + case EVP_CIPH_CFB_MODE: + ret_mode = atom_cfb_mode; + break; + + case EVP_CIPH_OFB_MODE: + ret_mode = atom_ofb_mode; + break; + + case EVP_CIPH_STREAM_CIPHER: + ret_mode = atom_stream_cipher; + break; + + default: + ret_mode = atom_undefined; + break; + } + + enif_make_map_put(env, ret, atom_mode, ret_mode, &ret); + + return ret; +} + +const struct cipher_type_t* get_cipher_type_no_key(ERL_NIF_TERM type) +{ + struct cipher_type_t key; + + key.type.atom = type; + + return bsearch(&key, cipher_types, num_cipher_types, sizeof(cipher_types[0]), cmp_cipher_types_no_key); +} + +int cmp_cipher_types_no_key(const void *keyp, const void *elemp) { + const struct cipher_type_t *key = keyp; + const struct cipher_type_t *elem = elemp; + + if (key->type.atom < elem->type.atom) return -1; + else if (key->type.atom > elem->type.atom) return 1; + else /* key->type.atom == elem->type.atom */ return 0; +} + + ERL_NIF_TERM cipher_types_as_list(ErlNifEnv* env) { struct cipher_type_t* p; diff --git a/lib/crypto/c_src/cipher.h b/lib/crypto/c_src/cipher.h index 6b43afea99..b94873940f 100644 --- a/lib/crypto/c_src/cipher.h +++ b/lib/crypto/c_src/cipher.h @@ -59,14 +59,23 @@ struct cipher_type_t { extern ErlNifResourceType* evp_cipher_ctx_rtype; struct evp_cipher_ctx { EVP_CIPHER_CTX* ctx; + int iv_len; +#if !defined(HAVE_EVP_AES_CTR) + ErlNifEnv* env; + ERL_NIF_TERM state; +#endif }; +ERL_NIF_TERM cipher_info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + int init_cipher_ctx(ErlNifEnv *env); void init_cipher_types(ErlNifEnv* env); +const struct cipher_type_t* get_cipher_type_no_key(ERL_NIF_TERM type); const struct cipher_type_t* get_cipher_type(ERL_NIF_TERM type, size_t key_len); int cmp_cipher_types(const void *keyp, const void *elemp); +int cmp_cipher_types_no_key(const void *keyp, const void *elemp); ERL_NIF_TERM cipher_types_as_list(ErlNifEnv* env); diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c index 06439c34b2..4aed06a489 100644 --- a/lib/crypto/c_src/crypto.c +++ b/lib/crypto/c_src/crypto.c @@ -29,9 +29,7 @@ #include "aes.h" #include "algorithms.h" #include "api_ng.h" -#include "block.h" #include "bn.h" -#include "chacha20.h" #include "cipher.h" #include "cmac.h" #include "dh.h" @@ -50,7 +48,6 @@ #include "pkey.h" #include "poly1305.h" #include "rand.h" -#include "rc4.h" #include "rsa.h" #include "srp.h" @@ -67,6 +64,7 @@ static ErlNifFunc nif_funcs[] = { {"info_fips", 0, info_fips, 0}, {"enable_fips_mode", 1, enable_fips_mode, 0}, {"algorithms", 0, algorithms, 0}, + {"hash_info", 1, hash_info_nif, 0}, {"hash_nif", 2, hash_nif, 0}, {"hash_init_nif", 1, hash_init_nif, 0}, {"hash_update_nif", 2, hash_update_nif, 0}, @@ -78,22 +76,17 @@ static ErlNifFunc nif_funcs[] = { {"hmac_final_nif", 1, hmac_final_nif, 0}, {"hmac_final_nif", 2, hmac_final_nif, 0}, {"cmac_nif", 3, cmac_nif, 0}, - {"block_crypt_nif", 5, block_crypt_nif, 0}, - {"block_crypt_nif", 4, block_crypt_nif, 0}, + {"cipher_info_nif", 1, cipher_info_nif, 0}, {"aes_ige_crypt_nif", 4, aes_ige_crypt_nif, 0}, - {"aes_ctr_stream_init", 2, aes_ctr_stream_init, 0}, - {"aes_ctr_stream_encrypt", 2, aes_ctr_stream_encrypt, 0}, - {"aes_ctr_stream_decrypt", 2, aes_ctr_stream_encrypt, 0}, {"ng_crypto_init_nif", 4, ng_crypto_init_nif, 0}, {"ng_crypto_update_nif", 2, ng_crypto_update_nif, 0}, {"ng_crypto_update_nif", 3, ng_crypto_update_nif, 0}, + {"ng_crypto_one_shot_nif", 5, ng_crypto_one_shot_nif, 0}, {"strong_rand_bytes_nif", 1, strong_rand_bytes_nif, 0}, {"strong_rand_range_nif", 1, strong_rand_range_nif, 0}, {"rand_uniform_nif", 2, rand_uniform_nif, 0}, {"mod_exp_nif", 4, mod_exp_nif, 0}, {"do_exor", 2, do_exor, 0}, - {"rc4_set_key", 1, rc4_set_key, 0}, - {"rc4_encrypt_with_state", 2, rc4_encrypt_with_state, 0}, {"pkey_sign_nif", 5, pkey_sign_nif, 0}, {"pkey_verify_nif", 6, pkey_verify_nif, 0}, {"pkey_crypt_nif", 6, pkey_crypt_nif, 0}, @@ -115,10 +108,6 @@ static ErlNifFunc nif_funcs[] = { {"aead_encrypt", 6, aead_encrypt, 0}, {"aead_decrypt", 6, aead_decrypt, 0}, - {"chacha20_stream_init", 2, chacha20_stream_init, 0}, - {"chacha20_stream_encrypt", 2, chacha20_stream_crypt, 0}, - {"chacha20_stream_decrypt", 2, chacha20_stream_crypt, 0}, - {"poly1305_nif", 2, poly1305_nif, 0}, {"engine_by_id_nif", 1, engine_by_id_nif, 0}, diff --git a/lib/crypto/c_src/hash.c b/lib/crypto/c_src/hash.c index 457e9d071a..0a9f64acef 100644 --- a/lib/crypto/c_src/hash.c +++ b/lib/crypto/c_src/hash.c @@ -61,6 +61,32 @@ int init_hash_ctx(ErlNifEnv* env) { #endif } +ERL_NIF_TERM hash_info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Type) */ + struct digest_type_t *digp = NULL; + const EVP_MD *md; + ERL_NIF_TERM ret; + + ASSERT(argc == 1); + + if ((digp = get_digest_type(argv[0])) == NULL) + return enif_make_badarg(env); + + if ((md = digp->md.p) == NULL) + return atom_notsup; + + ret = enif_make_new_map(env); + + enif_make_map_put(env, ret, atom_type, + enif_make_int(env, EVP_MD_type(md)), &ret); + enif_make_map_put(env, ret, atom_size, + enif_make_int(env, EVP_MD_size(md)), &ret); + enif_make_map_put(env, ret, atom_block_size, + enif_make_int(env, EVP_MD_block_size(md)), &ret); + + return ret; +} + ERL_NIF_TERM hash_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {/* (Type, Data) */ struct digest_type_t *digp = NULL; diff --git a/lib/crypto/c_src/hash.h b/lib/crypto/c_src/hash.h index 8bae07f39a..92a25cedb7 100644 --- a/lib/crypto/c_src/hash.h +++ b/lib/crypto/c_src/hash.h @@ -25,6 +25,7 @@ int init_hash_ctx(ErlNifEnv *env); +ERL_NIF_TERM hash_info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM hash_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM hash_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM hash_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); diff --git a/lib/crypto/c_src/openssl_config.h b/lib/crypto/c_src/openssl_config.h index 45144a0c25..46868cb987 100644 --- a/lib/crypto/c_src/openssl_config.h +++ b/lib/crypto/c_src/openssl_config.h @@ -109,6 +109,7 @@ #ifndef HAS_LIBRESSL # if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) # define HAS_EVP_PKEY_CTX +# define HAVE_EVP_CIPHER_CTX_COPY # endif #endif @@ -203,12 +204,17 @@ #if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,1,0) # ifndef HAS_LIBRESSL -# define HAVE_CHACHA20 # define HAVE_CHACHA20_POLY1305 # define HAVE_RSA_OAEP_MD # endif #endif +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION(1,1,0,'d') +# ifndef HAS_LIBRESSL +# define HAVE_CHACHA20 +# endif +#endif + // OPENSSL_VERSION_NUMBER >= 1.1.1-pre8 #if OPENSSL_VERSION_NUMBER >= (PACKED_OPENSSL_VERSION_PLAIN(1,1,1)-7) # ifndef HAS_LIBRESSL diff --git a/lib/crypto/c_src/pkey.c b/lib/crypto/c_src/pkey.c index 393358d173..638bb588fa 100644 --- a/lib/crypto/c_src/pkey.c +++ b/lib/crypto/c_src/pkey.c @@ -719,6 +719,11 @@ enif_get_atom(env,argv[1],buf,1024,ERL_NIF_LATIN1); printf("hash=%s ",buf); if (pkey) EVP_PKEY_free(pkey); +#ifdef HAVE_EDDSA + if (mdctx) + EVP_MD_CTX_free(mdctx); +#endif + return ret; } diff --git a/lib/crypto/c_src/poly1305.c b/lib/crypto/c_src/poly1305.c index db3433dce3..76579c0a29 100644 --- a/lib/crypto/c_src/poly1305.c +++ b/lib/crypto/c_src/poly1305.c @@ -85,6 +85,6 @@ ERL_NIF_TERM poly1305_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) return ret; #else - return atom_notsup; + return enif_raise_exception(env, atom_notsup); #endif } diff --git a/lib/crypto/c_src/rc4.c b/lib/crypto/c_src/rc4.c deleted file mode 100644 index e423661097..0000000000 --- a/lib/crypto/c_src/rc4.c +++ /dev/null @@ -1,92 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2010-2018. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -#include "rc4.h" - -ERL_NIF_TERM rc4_set_key(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Key) */ -#ifndef OPENSSL_NO_RC4 - ErlNifBinary key; - ERL_NIF_TERM ret; - RC4_KEY *rc4_key; - - CHECK_NO_FIPS_MODE(); - - ASSERT(argc == 1); - - if (!enif_inspect_iolist_as_binary(env, argv[0], &key)) - goto bad_arg; - if (key.size > INT_MAX) - goto bad_arg; - - if ((rc4_key = (RC4_KEY*)enif_make_new_binary(env, sizeof(RC4_KEY), &ret)) == NULL) - goto err; - - RC4_set_key(rc4_key, (int)key.size, key.data); - return ret; - - bad_arg: - err: - return enif_make_badarg(env); - -#else - return enif_raise_exception(env, atom_notsup); -#endif -} - -ERL_NIF_TERM rc4_encrypt_with_state(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (State, Data) */ -#ifndef OPENSSL_NO_RC4 - ErlNifBinary state, data; - RC4_KEY* rc4_key; - ERL_NIF_TERM new_state, new_data; - unsigned char *outp; - - CHECK_NO_FIPS_MODE(); - - ASSERT(argc == 2); - - if (!enif_inspect_iolist_as_binary(env, argv[0], &state)) - goto bad_arg; - if (state.size != sizeof(RC4_KEY)) - goto bad_arg; - if (!enif_inspect_iolist_as_binary(env, argv[1], &data)) - goto bad_arg; - - if ((rc4_key = (RC4_KEY*)enif_make_new_binary(env, sizeof(RC4_KEY), &new_state)) == NULL) - goto err; - if ((outp = enif_make_new_binary(env, data.size, &new_data)) == NULL) - goto err; - - memcpy(rc4_key, state.data, sizeof(RC4_KEY)); - RC4(rc4_key, data.size, data.data, outp); - - CONSUME_REDS(env, data); - return enif_make_tuple2(env, new_state, new_data); - - bad_arg: - err: - return enif_make_badarg(env); - -#else - return enif_raise_exception(env, atom_notsup); -#endif -} - diff --git a/lib/crypto/c_src/rc4.h b/lib/crypto/c_src/rc4.h deleted file mode 100644 index 28bf674253..0000000000 --- a/lib/crypto/c_src/rc4.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2010-2018. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -#ifndef E_RC4_H__ -#define E_RC4_H__ 1 - -#include "common.h" - -ERL_NIF_TERM rc4_set_key(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -ERL_NIF_TERM rc4_encrypt_with_state(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); - -#endif /* E_RC4_H__ */ diff --git a/lib/crypto/doc/src/notes.xml b/lib/crypto/doc/src/notes.xml index 0a3f68ade2..c0b302734e 100644 --- a/lib/crypto/doc/src/notes.xml +++ b/lib/crypto/doc/src/notes.xml @@ -31,6 +31,38 @@ </header> <p>This document describes the changes made to the Crypto application.</p> +<section><title>Crypto 4.4.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixes a bug that caused <c>crypto:sign</c> and + <c>crypto:verify</c> to return the error message + <c>badarg</c> instead of <c>notsup</c> in one case. That + case was when signing or verifying with eddsa keys (that + is, ed15519 or ed448), but only when FIPS was supported + and enabled.</p> + <p> + Own Id: OTP-15634</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Added a crypto benchmark test suite.</p> + <p> + Own Id: OTP-15447</p> + </item> + </list> + </section> + +</section> + <section><title>Crypto 4.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index fe8390c5b8..5cf34f8069 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -24,6 +24,7 @@ -export([start/0, stop/0, info_lib/0, info_fips/0, supports/0, enable_fips_mode/1, version/0, bytes_to_integer/1]). +-export([cipher_info/1, hash_info/1]). -export([hash/2, hash_init/1, hash_update/2, hash_final/1]). -export([sign/4, sign/5, verify/5, verify/6]). -export([generate_key/2, generate_key/3, compute_key/4]). @@ -39,24 +40,27 @@ -export([rand_plugin_uniform/2]). -export([rand_cache_plugin_next/1]). -export([rand_uniform/2]). --export([block_encrypt/3, block_decrypt/3, block_encrypt/4, block_decrypt/4]). -export([next_iv/2, next_iv/3]). --export([stream_init/2, stream_init/3, stream_encrypt/2, stream_decrypt/2]). -export([public_encrypt/4, private_decrypt/4]). -export([private_encrypt/4, public_decrypt/4]). -export([privkey_to_pubkey/2]). -export([ec_curve/1, ec_curves/0]). -export([rand_seed/1]). -%% Experiment --export([crypto_init/4, - crypto_update/2, crypto_update/3, - %% Emulates old api: - crypto_stream_init/2, crypto_stream_init/3, - crypto_stream_encrypt/2, - crypto_stream_decrypt/2, - crypto_block_encrypt/3, crypto_block_encrypt/4, - crypto_block_decrypt/3, crypto_block_decrypt/4 +%% Old interface. Now implemented with the New interface +-export([stream_init/2, stream_init/3, + stream_encrypt/2, + stream_decrypt/2, + block_encrypt/3, block_encrypt/4, + block_decrypt/3, block_decrypt/4 + ]). + +%% New interface +-export([crypto_init/4, crypto_init/3, + crypto_update/2, + crypto_one_shot/5, + crypto_init_dyn_iv/3, + crypto_update_dyn_iv/3 ]). @@ -403,6 +407,11 @@ enable_fips_mode(_) -> ?nif_stub. -define(HASH_HASH_ALGORITHM, sha1() | sha2() | sha3() | blake2() | ripemd160 | compatibility_only_hash() ). +-spec hash_info(Type) -> map() when Type :: ?HASH_HASH_ALGORITHM. + +hash_info(Type) -> + notsup_to_error(hash_info_nif(Type)). + -spec hash(Type, Data) -> Digest when Type :: ?HASH_HASH_ALGORITHM, Data :: iodata(), Digest :: binary(). @@ -527,12 +536,24 @@ poly1305(Key, Data) -> %%%================================================================ %%% -%%% Encrypt/decrypt +%%% Encrypt/decrypt, The "Old API" %%% %%%================================================================ -%%%---- Block ciphers +-define(COMPAT(CALL), + try CALL + catch + error:{E,_Reason} when E==notsup ; E==badarg -> + error(E) + end). + +-spec cipher_info(Type) -> map() when Type :: block_cipher_with_iv() + | aead_cipher() + | block_cipher_without_iv(). +cipher_info(Type) -> + cipher_info_nif(Type). +%%%---- Block ciphers %%%---------------------------------------------------------------- -spec block_encrypt(Type::block_cipher_with_iv(), Key::key()|des3_key(), Ivec::binary(), PlainText::iodata()) -> binary(); (Type::aead_cipher(), Key::iodata(), Ivec::binary(), {AAD::binary(), PlainText::iodata()}) -> @@ -544,11 +565,6 @@ poly1305(Key, Data) -> block_encrypt(Type, Key, Ivec, Data) -> do_block_encrypt(alias(Type), Key, Ivec, Data). -do_block_encrypt(Type, Key0, Ivec, Data) when Type =:= des_ede3_cbc; - Type =:= des_ede3_cfb -> - Key = check_des3_key(Key0), - block_crypt_nif(Type, Key, Ivec, Data, true); - do_block_encrypt(Type, Key, Ivec, PlainText) when Type =:= aes_ige256 -> notsup_to_error(aes_ige_crypt_nif(Key, Ivec, PlainText, true)); @@ -565,14 +581,13 @@ do_block_encrypt(Type, Key, Ivec, Data) when Type =:= aes_gcm; end; do_block_encrypt(Type, Key, Ivec, PlainText) -> - block_crypt_nif(Type, Key, Ivec, PlainText, true). - + ?COMPAT(crypto_one_shot(Type, Key, Ivec, PlainText, true)). -spec block_encrypt(Type::block_cipher_without_iv(), Key::key(), PlainText::iodata()) -> binary(). block_encrypt(Type, Key, PlainText) -> - block_crypt_nif(alias(Type), Key, PlainText, true). + ?COMPAT(crypto_one_shot(Type, Key, <<>>, PlainText, true)). %%%---------------------------------------------------------------- %%%---------------------------------------------------------------- @@ -583,11 +598,6 @@ block_encrypt(Type, Key, PlainText) -> block_decrypt(Type, Key, Ivec, Data) -> do_block_decrypt(alias(Type), Key, Ivec, Data). -do_block_decrypt(Type, Key0, Ivec, Data) when Type =:= des_ede3_cbc; - Type =:= des_ede3_cfb -> - Key = check_des3_key(Key0), - block_crypt_nif(Type, Key, Ivec, Data, false); - do_block_decrypt(aes_ige256, Key, Ivec, Data) -> notsup_to_error(aes_ige_crypt_nif(Key, Ivec, Data, false)); @@ -597,14 +607,80 @@ do_block_decrypt(Type, Key, Ivec, {AAD, Data, Tag}) when Type =:= aes_gcm; aead_decrypt(Type, Key, Ivec, AAD, Data, Tag); do_block_decrypt(Type, Key, Ivec, Data) -> - block_crypt_nif(Type, Key, Ivec, Data, false). - + ?COMPAT(crypto_one_shot(Type, Key, Ivec, Data, false)). -spec block_decrypt(Type::block_cipher_without_iv(), Key::key(), Data::iodata()) -> binary(). block_decrypt(Type, Key, Data) -> - block_crypt_nif(alias(Type), Key, Data, false). + ?COMPAT(crypto_one_shot(Type, Key, <<>>, Data, false)). + +%%%-------- Stream ciphers API + +-opaque stream_state() :: {stream_cipher(), + crypto_state() | {crypto_state(),flg_undefined} + }. + +-type stream_cipher() :: stream_cipher_iv() | stream_cipher_no_iv() . +-type stream_cipher_no_iv() :: rc4 . +-type stream_cipher_iv() :: aes_ctr + | aes_128_ctr + | aes_192_ctr + | aes_256_ctr + | chacha20 . + +%%%---- stream_init +-spec stream_init(Type, Key, IVec) -> State | no_return() + when Type :: stream_cipher_iv(), + Key :: iodata(), + IVec ::binary(), + State :: stream_state() . +stream_init(Type, Key, IVec) when is_binary(IVec) -> + Ref = ?COMPAT(ng_crypto_init_nif(alias(Type), + iolist_to_binary(Key), iolist_to_binary(IVec), + undefined) + ), + {Type, {Ref,flg_undefined}}. + + +-spec stream_init(Type, Key) -> State | no_return() + when Type :: stream_cipher_no_iv(), + Key :: iodata(), + State :: stream_state() . +stream_init(rc4 = Type, Key) -> + Ref = ?COMPAT(ng_crypto_init_nif(alias(Type), + iolist_to_binary(Key), <<>>, + undefined) + ), + {Type, {Ref,flg_undefined}}. + +%%%---- stream_encrypt +-spec stream_encrypt(State, PlainText) -> {NewState, CipherText} | no_return() + when State :: stream_state(), + PlainText :: iodata(), + NewState :: stream_state(), + CipherText :: iodata() . +stream_encrypt(State, Data) -> + crypto_stream_emulate(State, Data, true). + +%%%---- stream_decrypt +-spec stream_decrypt(State, CipherText) -> {NewState, PlainText} | no_return() + when State :: stream_state(), + CipherText :: iodata(), + NewState :: stream_state(), + PlainText :: iodata() . +stream_decrypt(State, Data) -> + crypto_stream_emulate(State, Data, false). + +%%%-------- helpers +crypto_stream_emulate({Cipher,{Ref0,flg_undefined}}, Data, EncryptFlag) when is_reference(Ref0) -> + ?COMPAT(begin + Ref = ng_crypto_init_nif(Ref0, <<>>, <<>>, EncryptFlag), + {{Cipher,Ref}, crypto_update(Ref, Data)} + end); + +crypto_stream_emulate({Cipher,Ref}, Data, _) when is_reference(Ref) -> + ?COMPAT({{Cipher,Ref}, crypto_update(Ref, Data)}). %%%---------------------------------------------------------------- -spec next_iv(Type:: cbc_cipher(), Data) -> NextIVec when % Type :: cbc_cipher(), %des_cbc | des3_cbc | aes_cbc | aes_ige, @@ -633,59 +709,155 @@ next_iv(des_cfb, Data, IVec) -> next_iv(Type, Data, _Ivec) -> next_iv(Type, Data). -%%%---- Stream ciphers +%%%================================================================ +%%% +%%% Encrypt/decrypt, The "New API" +%%% +%%%================================================================ --opaque stream_state() :: {stream_cipher(), reference()}. +-opaque crypto_state() :: reference() . --type stream_cipher() :: stream_cipher_iv() | stream_cipher_no_iv() . --type stream_cipher_no_iv() :: rc4 . --type stream_cipher_iv() :: aes_ctr - | aes_128_ctr - | aes_192_ctr - | aes_256_ctr - | chacha20 . --spec stream_init(Type, Key, IVec) -> State when Type :: stream_cipher_iv(), - Key :: iodata(), - IVec :: binary(), - State :: stream_state() . -stream_init(aes_ctr, Key, Ivec) -> - {aes_ctr, aes_ctr_stream_init(Key, Ivec)}; -stream_init(aes_128_ctr, Key, Ivec) -> - {aes_ctr, aes_ctr_stream_init(Key, Ivec)}; -stream_init(aes_192_ctr, Key, Ivec) -> - {aes_ctr, aes_ctr_stream_init(Key, Ivec)}; -stream_init(aes_256_ctr, Key, Ivec) -> - {aes_ctr, aes_ctr_stream_init(Key, Ivec)}; -stream_init(chacha20, Key, Ivec) -> - {chacha20, chacha20_stream_init(Key,Ivec)}. - --spec stream_init(Type, Key) -> State when Type :: stream_cipher_no_iv(), - Key :: iodata(), - State :: stream_state() . -stream_init(rc4, Key) -> - {rc4, notsup_to_error(rc4_set_key(Key))}. - --spec stream_encrypt(State, PlainText) -> {NewState, CipherText} - when State :: stream_state(), - PlainText :: iodata(), - NewState :: stream_state(), - CipherText :: iodata() . -stream_encrypt(State, Data0) -> - Data = iolist_to_binary(Data0), - MaxByts = max_bytes(), - stream_crypt(fun do_stream_encrypt/2, State, Data, erlang:byte_size(Data), MaxByts, []). +%%%---------------------------------------------------------------- +%%% +%%% Create and initialize a new state for encryption or decryption +%%% --spec stream_decrypt(State, CipherText) -> {NewState, PlainText} - when State :: stream_state(), - CipherText :: iodata(), - NewState :: stream_state(), - PlainText :: iodata() . -stream_decrypt(State, Data0) -> - Data = iolist_to_binary(Data0), - MaxByts = max_bytes(), - stream_crypt(fun do_stream_decrypt/2, State, Data, erlang:byte_size(Data), MaxByts, []). +-spec crypto_init(Cipher, Key, EncryptFlag) -> State | ng_crypto_error() + when Cipher :: block_cipher_without_iv() + | stream_cipher_no_iv(), + Key :: iodata(), + EncryptFlag :: boolean(), + State :: crypto_state() . +crypto_init(Cipher, Key, EncryptFlag) -> + %% The IV is supposed to be supplied by calling crypto_update/3 + ng_crypto_init_nif(alias(Cipher), iolist_to_binary(Key), <<>>, EncryptFlag). + + +-spec crypto_init(Cipher, Key, IV, EncryptFlag) -> State | ng_crypto_error() + when Cipher :: stream_cipher_iv() + | block_cipher_with_iv(), + Key :: iodata(), + IV :: iodata(), + EncryptFlag :: boolean(), + State :: crypto_state() . +crypto_init(Cipher, Key, IV, EncryptFlag) -> + ng_crypto_init_nif(alias(Cipher), iolist_to_binary(Key), iolist_to_binary(IV), EncryptFlag). + + + +%%%---------------------------------------------------------------- +-spec crypto_init_dyn_iv(Cipher, Key, EncryptFlag) -> State | ng_crypto_error() + when Cipher :: stream_cipher_iv() + | block_cipher_with_iv(), + Key :: iodata(), + EncryptFlag :: boolean(), + State :: crypto_state() . +crypto_init_dyn_iv(Cipher, Key, EncryptFlag) -> + %% The IV is supposed to be supplied by calling crypto_update/3 + ng_crypto_init_nif(alias(Cipher), iolist_to_binary(Key), undefined, EncryptFlag). + +%%%---------------------------------------------------------------- +%%% +%%% Encrypt/decrypt a sequence of bytes. The sum of the sizes +%%% of all blocks must be an integer multiple of the crypto's +%%% blocksize. +%%% + +-spec crypto_update(State, Data) -> Result | ng_crypto_error() + when State :: crypto_state(), + Data :: iodata(), + Result :: binary() . +crypto_update(State, Data0) -> + case iolist_to_binary(Data0) of + <<>> -> + <<>>; % Known to fail on OpenSSL 0.9.8h + Data -> + ng_crypto_update_nif(State, Data) + end. + + +%%%---------------------------------------------------------------- +-spec crypto_update_dyn_iv(State, Data, IV) -> Result | ng_crypto_error() + when State :: crypto_state(), + Data :: iodata(), + IV :: iodata(), + Result :: binary() . +crypto_update_dyn_iv(State, Data0, IV) -> + %% When State is from State = crypto_init(Cipher, Key, undefined, EncryptFlag) + case iolist_to_binary(Data0) of + <<>> -> + <<>>; % Known to fail on OpenSSL 0.9.8h + Data -> + ng_crypto_update_nif(State, Data, iolist_to_binary(IV)) + end. + +%%%---------------------------------------------------------------- +%%% +%%% Encrypt/decrypt one set bytes. +%%% The size must be an integer multiple of the crypto's blocksize. +%%% + +-spec crypto_one_shot(Cipher, Key, IV, Data, EncryptFlag) -> Result | ng_crypto_error() + when Cipher :: stream_cipher() + | block_cipher_with_iv() + | block_cipher_without_iv(), + Key :: iodata(), + IV :: iodata() | undefined, + Data :: iodata(), + EncryptFlag :: boolean(), + Result :: binary() . +crypto_one_shot(Cipher, Key, undefined, Data, EncryptFlag) -> + crypto_one_shot(Cipher, Key, <<>>, Data, EncryptFlag); + +crypto_one_shot(Cipher, Key, IV, Data0, EncryptFlag) -> + case iolist_to_binary(Data0) of + <<>> -> + <<>>; % Known to fail on OpenSSL 0.9.8h + Data -> + ng_crypto_one_shot_nif(alias(Cipher), + iolist_to_binary(Key), iolist_to_binary(IV), Data, + EncryptFlag) + end. + +%%%---------------------------------------------------------------- +%%% NIFs + +-type ng_crypto_error() :: no_return() . + +-spec ng_crypto_init_nif(atom(), binary(), binary()|undefined, boolean()|undefined ) -> crypto_state() | ng_crypto_error() + ; (crypto_state(), <<>>, <<>>, boolean()) -> crypto_state() | ng_crypto_error(). +ng_crypto_init_nif(_Cipher, _Key, _IVec, _EncryptFlg) -> ?nif_stub. + + +-spec ng_crypto_update_nif(crypto_state(), binary()) -> binary() | ng_crypto_error() . +ng_crypto_update_nif(_State, _Data) -> ?nif_stub. + +-spec ng_crypto_update_nif(crypto_state(), binary(), binary()) -> binary() | ng_crypto_error() . +ng_crypto_update_nif(_State, _Data, _IV) -> ?nif_stub. + + +-spec ng_crypto_one_shot_nif(atom(), binary(), binary(), binary(), boolean() ) -> binary() | ng_crypto_error(). +ng_crypto_one_shot_nif(_Cipher, _Key, _IVec, _Data, _EncryptFlg) -> ?nif_stub. + +%%%---------------------------------------------------------------- +%%% Cipher aliases +%%% +prepend_cipher_aliases(L) -> + [des3_cbc, des_ede3, des_ede3_cbf, des3_cbf, des3_cfb, aes_cbc128, aes_cbc256 | L]. +%%%---- des_ede3_cbc +alias(des3_cbc) -> des_ede3_cbc; +alias(des_ede3) -> des_ede3_cbc; +%%%---- des_ede3_cfb +alias(des_ede3_cbf) -> des_ede3_cfb; +alias(des3_cbf) -> des_ede3_cfb; +alias(des3_cfb) -> des_ede3_cfb; +%%%---- aes_*_cbc +alias(aes_cbc128) -> aes_128_cbc; +alias(aes_cbc256) -> aes_256_cbc; + +alias(Alg) -> Alg. %%%================================================================ %%% @@ -1726,6 +1898,7 @@ hash_update(State0, Data, _, MaxBytes) -> State = notsup_to_error(hash_update_nif(State0, Increment)), hash_update(State, Rest, erlang:byte_size(Rest), MaxBytes). +hash_info_nif(_Hash) -> ?nif_stub. hash_nif(_Hash, _Data) -> ?nif_stub. hash_init_nif(_Hash) -> ?nif_stub. hash_update_nif(_State, _Data) -> ?nif_stub. @@ -1770,18 +1943,7 @@ poly1305_nif(_Key, _Data) -> ?nif_stub. %% CIPHERS -------------------------------------------------------------------- -block_crypt_nif(_Type, _Key, _Ivec, _Text, _IsEncrypt) -> ?nif_stub. -block_crypt_nif(_Type, _Key, _Text, _IsEncrypt) -> ?nif_stub. - -check_des3_key(Key) -> - case lists:map(fun erlang:iolist_to_binary/1, Key) of - ValidKey = [B1, B2, B3] when byte_size(B1) =:= 8, - byte_size(B2) =:= 8, - byte_size(B3) =:= 8 -> - ValidKey; - _ -> - error(badarg) - end. +cipher_info_nif(_Type) -> ?nif_stub. %% %% AES - in Galois/Counter Mode (GCM) @@ -1799,59 +1961,7 @@ aead_decrypt(_Type, _Key, _Ivec, _AAD, _In, _Tag) -> ?nif_stub. aes_ige_crypt_nif(_Key, _IVec, _Data, _IsEncrypt) -> ?nif_stub. - -%% Stream ciphers -------------------------------------------------------------------- - -stream_crypt(Fun, State, Data, Size, MaxByts, []) when Size =< MaxByts -> - Fun(State, Data); -stream_crypt(Fun, State0, Data, Size, MaxByts, Acc) when Size =< MaxByts -> - {State, Cipher} = Fun(State0, Data), - {State, list_to_binary(lists:reverse([Cipher | Acc]))}; -stream_crypt(Fun, State0, Data, _, MaxByts, Acc) -> - <<Increment:MaxByts/binary, Rest/binary>> = Data, - {State, CipherText} = Fun(State0, Increment), - stream_crypt(Fun, State, Rest, erlang:byte_size(Rest), MaxByts, [CipherText | Acc]). - -do_stream_encrypt({aes_ctr, State0}, Data) -> - {State, Cipher} = aes_ctr_stream_encrypt(State0, Data), - {{aes_ctr, State}, Cipher}; -do_stream_encrypt({rc4, State0}, Data) -> - {State, Cipher} = rc4_encrypt_with_state(State0, Data), - {{rc4, State}, Cipher}; -do_stream_encrypt({chacha20, State0}, Data) -> - {State, Cipher} = chacha20_stream_encrypt(State0, Data), - {{chacha20, State}, Cipher}. - -do_stream_decrypt({aes_ctr, State0}, Data) -> - {State, Text} = aes_ctr_stream_decrypt(State0, Data), - {{aes_ctr, State}, Text}; -do_stream_decrypt({rc4, State0}, Data) -> - {State, Text} = rc4_encrypt_with_state(State0, Data), - {{rc4, State}, Text}; -do_stream_decrypt({chacha20, State0}, Data) -> - {State, Cipher} = chacha20_stream_decrypt(State0, Data), - {{chacha20, State}, Cipher}. - - -%% -%% AES - in counter mode (CTR) with state maintained for multi-call streaming -%% -aes_ctr_stream_init(_Key, _IVec) -> ?nif_stub. -aes_ctr_stream_encrypt(_State, _Data) -> ?nif_stub. -aes_ctr_stream_decrypt(_State, _Cipher) -> ?nif_stub. - -%% -%% RC4 - symmetric stream cipher -%% -rc4_set_key(_Key) -> ?nif_stub. -rc4_encrypt_with_state(_State, _Data) -> ?nif_stub. - -%% -%% CHACHA20 - stream cipher -%% -chacha20_stream_init(_Key, _IVec) -> ?nif_stub. -chacha20_stream_encrypt(_State, _Data) -> ?nif_stub. -chacha20_stream_decrypt(_State, _Data) -> ?nif_stub. +%%%================================================================ %% Secure remote password ------------------------------------------------------------------- @@ -2217,176 +2327,3 @@ check_otp_test_engine(LibDir) -> end. -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%%% -%%% Experimental NG -%%% - -%%% -> {ok,State::ref()} | {error,Reason} - --opaque crypto_state() :: reference() | {any(),any(),any(),any()}. - - -%%%---------------------------------------------------------------- -%%% -%%% Create and initialize a new state for encryption or decryption -%%% - --spec crypto_init(Cipher, Key, IV, EncryptFlag) -> {ok,State} | {error,term()} | undefined - when Cipher :: stream_cipher() - | block_cipher_with_iv() - | block_cipher_without_iv() , - Key :: iodata(), - IV :: binary(), - EncryptFlag :: boolean() | undefined, - State :: crypto_state() . - -crypto_init(Cipher, Key, IV, EncryptFlag) when is_atom(Cipher), - is_binary(Key), - is_binary(IV), - is_atom(EncryptFlag) -> - case ng_crypto_init_nif(alias(Cipher), Key, IV, EncryptFlag) of - {error,Error} -> - {error,Error}; - undefined -> % For compatibility function crypto_stream_init/3 - undefined; - Ref when is_reference(Ref) -> - {ok,Ref}; - State when is_tuple(State), - size(State)==4 -> - {ok,State} % compatibility with old cryptolibs < 1.0.1 - end. - - -%%%---------------------------------------------------------------- -%%% -%%% Encrypt/decrypt a sequence of bytes. The sum of the sizes -%%% of all blocks must be an integer multiple of the crypto's -%%% blocksize. -%%% - --spec crypto_update(State, Data) -> {ok,Result} | {error,term()} - when State :: crypto_state(), - Data :: iodata(), - Result :: binary() | {crypto_state(),binary()}. -crypto_update(State, Data) -> - mk_ret(ng_crypto_update_nif(State, Data)). - -%%%---------------------------------------------------------------- -%%% -%%% Encrypt/decrypt a sequence of bytes but change the IV first. -%%% Not applicable for all modes. -%%% - --spec crypto_update(State, Data, IV) -> {ok,Result} | {error,term()} - when State :: crypto_state(), - Data :: iodata(), - IV :: binary(), - Result :: binary() | {crypto_state(),binary()}. -crypto_update(State, Data, IV) -> - mk_ret(ng_crypto_update_nif(State, Data, IV)). - -%%%---------------------------------------------------------------- -%%% Helpers -mk_ret(R) -> mk_ret(R, []). - -mk_ret({error,Error}, _) -> - {error,Error}; -mk_ret(Bin, Acc) when is_binary(Bin) -> - {ok, iolist_to_binary(lists:reverse([Bin|Acc]))}; -mk_ret({State1,Bin}, Acc) when is_tuple(State1), - size(State1) == 4, - is_binary(Bin) -> - %% compatibility with old cryptolibs < 1.0.1 - {ok, {State1, iolist_to_binary(lists:reverse([Bin|Acc]))}}. - -%%%---------------------------------------------------------------- -%%% NIFs -ng_crypto_init_nif(_Cipher, _Key, _IVec, _EncryptFlg) -> ?nif_stub. -ng_crypto_update_nif(_State, _Data) -> ?nif_stub. -ng_crypto_update_nif(_State, _Data, _IV) -> ?nif_stub. - -%%%================================================================ -%%% Compatibility functions to be called by "old" api functions. - -%%%-------------------------------- -%%%---- block encrypt/decrypt -crypto_block_encrypt(Cipher, Key, Data) -> crypto_block_encrypt(Cipher, Key, <<>>, Data). -crypto_block_decrypt(Cipher, Key, Data) -> crypto_block_decrypt(Cipher, Key, <<>>, Data). - -crypto_block_encrypt(Cipher, Key, Ivec, Data) -> crypto_block(Cipher, Key, Ivec, Data, true). -crypto_block_decrypt(Cipher, Key, Ivec, Data) -> crypto_block(Cipher, Key, Ivec, Data, false). - -%% AEAD: use old funcs - -%%%---- helper -crypto_block(Cipher, Key, IV, Data, EncryptFlag) -> - case crypto_init(Cipher, iolist_to_binary(Key), iolist_to_binary(IV), EncryptFlag) of - {ok, Ref} -> - case crypto_update(Ref, Data) of - {ok, {_,Bin}} when is_binary(Bin) -> Bin; - {ok, Bin} when is_binary(Bin) -> Bin; - {error,_} -> error(badarg) - end; - - {error,_} -> error(badarg) - end. - -%%%-------------------------------- -%%%---- stream init, encrypt/decrypt - -crypto_stream_init(Cipher, Key) -> - crypto_stream_init(Cipher, Key, <<>>). - -crypto_stream_init(Cipher, Key0, IV0) -> - Key = iolist_to_binary(Key0), - IV = iolist_to_binary(IV0), - %% First check the argumensts: - case crypto_init(Cipher, Key, IV, undefined) of - undefined -> - {Cipher, {Key, IV}}; - {error,_} -> - {error,badarg} - end. - -crypto_stream_encrypt(State, PlainText) -> - crypto_stream_emulate(State, PlainText, true). - -crypto_stream_decrypt(State, CryptoText) -> - crypto_stream_emulate(State, CryptoText, false). - - -%%%---- helper -crypto_stream_emulate({Cipher,{Key,IV}}, Data, EncryptFlag) -> - case crypto_init(Cipher, Key, IV, EncryptFlag) of - {ok,State} -> - crypto_stream_emulate({Cipher,State}, Data, EncryptFlag); - {error,_} -> - error(badarg) - end; -crypto_stream_emulate({Cipher,State}, Data, _) -> - case crypto_update(State, Data) of - {ok, {State1,Bin}} when is_binary(Bin) -> {{Cipher,State1},Bin}; - {ok,Bin} when is_binary(Bin) -> {{Cipher,State},Bin}; - {error,_} -> error(badarg) - end. - - -%%%================================================================ - -prepend_cipher_aliases(L) -> - [des3_cbc, des_ede3, des_ede3_cbf, des3_cbf, des3_cfb, aes_cbc128, aes_cbc256 | L]. - - -%%%---- des_ede3_cbc -alias(des3_cbc) -> des_ede3_cbc; -alias(des_ede3) -> des_ede3_cbc; -%%%---- des_ede3_cfb -alias(des_ede3_cbf) -> des_ede3_cfb; -alias(des3_cbf) -> des_ede3_cfb; -alias(des3_cfb) -> des_ede3_cfb; -%%%---- aes_*_cbc -alias(aes_cbc128) -> aes_128_cbc; -alias(aes_cbc256) -> aes_256_cbc; - -alias(Alg) -> Alg. diff --git a/lib/crypto/test/crypto_SUITE.erl b/lib/crypto/test/crypto_SUITE.erl index ab6d88deb2..7dbbde68e9 100644 --- a/lib/crypto/test/crypto_SUITE.erl +++ b/lib/crypto/test/crypto_SUITE.erl @@ -9,7 +9,7 @@ %% %% http://www.apache.org/licenses/LICENSE-2.0 %% -%% Unless required by applicable law or agreed to in writing, software + %% distributed under the License is distributed on an "AS IS" BASIS, %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and @@ -40,57 +40,73 @@ all() -> rand_uniform, rand_threads, rand_plugin, - rand_plugin_s + rand_plugin_s, + cipher_info, + hash_info ]. groups() -> - [{non_fips, [], [{group, md4}, + [{non_fips, [], [ + {group, blake2b}, + {group, blake2s}, + {group, dss}, + {group, ecdsa}, + {group, ed25519}, + {group, ed448}, + {group, rsa}, + + {group, md4}, {group, md5}, {group, ripemd160}, - {group, sha}, {group, sha224}, {group, sha256}, {group, sha384}, - {group, sha512}, {group, sha3_224}, {group, sha3_256}, {group, sha3_384}, {group, sha3_512}, - {group, blake2b}, - {group, blake2s}, - {group, rsa}, - {group, dss}, - {group, ecdsa}, - {group, ed25519}, - {group, ed448}, + {group, sha512}, + {group, sha}, + {group, dh}, {group, ecdh}, {group, srp}, - {group, des_cbc}, - {group, des_cfb}, - {group, des3_cbc}, - {group, des3_cbf}, - {group, des3_cfb}, - {group, des_ede3}, - {group, blowfish_cbc}, - {group, blowfish_ecb}, - {group, blowfish_cfb64}, - {group, blowfish_ofb64}, - {group, aes_cbc128}, - {group, aes_cfb8}, - {group, aes_cfb128}, - {group, aes_cbc256}, - {group, aes_ige256}, - {group, rc2_cbc}, - {group, rc4}, - {group, aes_ctr}, + + {group, aes_cbc}, {group, aes_ccm}, {group, aes_gcm}, {group, chacha20_poly1305}, {group, chacha20}, + {group, des3_cfb}, + {group, aes_cbc128}, + {group, aes_cbc256}, + {group, aes_cfb128}, + {group, aes_cfb8}, + {group, aes_ctr}, + {group, aes_ige256}, + {group, blowfish_cbc}, + {group, blowfish_cfb64}, + {group, blowfish_ecb}, + {group, blowfish_ofb64}, + {group, des3_cbc}, + {group, des3_cbf}, + {group, des_cbc}, + {group, des_cfb}, + {group, des_ede3}, {group, poly1305}, - {group, aes_cbc}]}, - {fips, [], [{group, no_md4}, + {group, rc2_cbc}, + {group, rc4} + ]}, + {fips, [], [ + {group, no_blake2b}, + {group, no_blake2s}, + {group, dss}, + {group, ecdsa}, + {group, no_ed25519}, + {group, no_ed448}, + {group, rsa}, + + {group, no_md4}, {group, no_md5}, {group, no_ripemd160}, {group, sha}, @@ -98,37 +114,36 @@ groups() -> {group, sha256}, {group, sha384}, {group, sha512}, - {group, rsa}, - {group, dss}, - {group, ecdsa}, - {group, no_ed25519}, - {group, no_ed448}, + {group, dh}, {group, ecdh}, {group, no_srp}, - {group, no_des_cbc}, - {group, no_des_cfb}, - {group, des3_cbc}, - {group, des3_cbf}, + + {group, aes_cbc}, + {group, aes_ccm}, + {group, aes_gcm}, + {group, no_chacha20_poly1305}, + {group, no_chacha20}, {group, des3_cfb}, - {group, des_ede3}, - {group, no_blowfish_cbc}, - {group, no_blowfish_ecb}, - {group, no_blowfish_cfb64}, - {group, no_blowfish_ofb64}, {group, aes_cbc128}, - {group, no_aes_cfb8}, - {group, no_aes_cfb128}, {group, aes_cbc256}, + {group, no_aes_cfb128}, + {group, no_aes_cfb8}, + {group, aes_ctr}, {group, no_aes_ige256}, + {group, no_blowfish_cbc}, + {group, no_blowfish_cfb64}, + {group, no_blowfish_ecb}, + {group, no_blowfish_ofb64}, + {group, des3_cbc}, + {group, des3_cbf}, + {group, no_des_cbc}, + {group, no_des_cfb}, + {group, des_ede3}, + {group, no_poly1305}, {group, no_rc2_cbc}, - {group, no_rc4}, - {group, aes_ctr}, - {group, aes_ccm}, - {group, aes_gcm}, - {group, no_chacha20_poly1305}, - {group, no_chacha20}, - {group, aes_cbc}]}, + {group, no_rc4} + ]}, {md4, [], [hash]}, {md5, [], [hash, hmac]}, {ripemd160, [], [hash]}, @@ -143,6 +158,8 @@ groups() -> {sha3_512, [], [hash, hmac]}, {blake2b, [], [hash, hmac]}, {blake2s, [], [hash, hmac]}, + {no_blake2b, [], [no_hash, no_hmac]}, + {no_blake2s, [], [no_hash, no_hmac]}, {rsa, [], [sign_verify, public_encrypt, private_encrypt, @@ -164,31 +181,32 @@ groups() -> compute_bug]}, {ecdh, [], [use_all_elliptic_curves, compute, generate]}, {srp, [], [generate_compute]}, - {des_cbc, [], [block]}, - {des_cfb, [], [block]}, - {des3_cbc,[], [block]}, - {des_ede3,[], [block]}, - {des3_cbf,[], [block]}, - {des3_cfb,[], [block]}, - {rc2_cbc,[], [block]}, - {aes_cbc128,[], [block, cmac]}, - {aes_cfb8,[], [block]}, - {aes_cfb128,[], [block]}, - {aes_cbc256,[], [block, cmac]}, - {aes_ecb,[], [block]}, + {des_cbc, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {des_cfb, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {des3_cbc,[], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {des_ede3,[], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {des3_cbf,[], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {des3_cfb,[], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {rc2_cbc,[], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_cbc128,[], [block, api_ng, api_ng_one_shot, api_ng_tls, cmac]}, + {aes_cfb8,[], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_cfb128,[], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_cbc256,[], [block, api_ng, api_ng_one_shot, api_ng_tls, cmac]}, + {aes_ecb,[], [block, api_ng, api_ng_one_shot, api_ng_tls]}, {aes_ige256,[], [block]}, - {blowfish_cbc, [], [block]}, - {blowfish_ecb, [], [block]}, - {blowfish_cfb64, [], [block]}, - {blowfish_ofb64,[], [block]}, - {rc4, [], [stream]}, - {aes_ctr, [], [stream]}, + {blowfish_cbc, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {blowfish_ecb, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {blowfish_cfb64, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {blowfish_ofb64,[], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {rc4, [], [stream, api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_ctr, [], [stream, api_ng, api_ng_one_shot, api_ng_tls]}, {aes_ccm, [], [aead]}, {aes_gcm, [], [aead]}, {chacha20_poly1305, [], [aead]}, - {chacha20, [], [stream]}, + {chacha20, [], [stream, api_ng, api_ng_one_shot, api_ng_tls]}, {poly1305, [], [poly1305]}, - {aes_cbc, [], [block]}, + {no_poly1305, [], [no_poly1305]}, + {aes_cbc, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, {no_aes_cfb8,[], [no_support, no_block]}, {no_aes_cfb128,[], [no_support, no_block]}, {no_md4, [], [no_support, no_hash]}, @@ -410,11 +428,19 @@ poly1305(Config) -> end, proplists:get_value(poly1305, Config)). %%-------------------------------------------------------------------- +no_poly1305() -> + [{doc, "Test disabled poly1305 function"}]. +no_poly1305(Config) -> + Type = ?config(type, Config), + Key = <<133,214,190,120,87,85,109,51,127,68,82,254,66,213,6,168,1, + 3,128,138,251,13,178,253,74,191,246,175,65,73,245,27>>, + Txt = <<"Cryptographic Forum Research Group">>, + notsup(fun crypto:poly1305/2, [Key,Txt]). + +%%-------------------------------------------------------------------- block() -> [{doc, "Test block ciphers"}]. block(Config) when is_list(Config) -> - Fips = proplists:get_bool(fips, Config), - Type = ?config(type, Config), Blocks = lazy_eval(proplists:get_value(block, Config)), lists:foreach(fun block_cipher/1, Blocks), lists:foreach(fun block_cipher/1, block_iolistify(Blocks)), @@ -437,6 +463,156 @@ no_block(Config) when is_list(Config) -> notsup(fun crypto:block_encrypt/N, Args), notsup(fun crypto:block_decrypt/N, Args). %%-------------------------------------------------------------------- +api_ng() -> + [{doc, "Test new api"}]. + +api_ng(Config) when is_list(Config) -> + Blocks = lazy_eval(proplists:get_value(block, Config, [])), + Streams = lazy_eval(proplists:get_value(stream, Config, [])), + lists:foreach(fun api_ng_cipher_increment/1, Blocks++Streams). + + +api_ng_cipher_increment({Type, Key, PlainTexts}=_X) -> + ct:log("~p",[_X]), + api_ng_cipher_increment({Type, Key, <<>>, PlainTexts}); + +api_ng_cipher_increment({Type, Key, IV, PlainTexts}=_X) -> + ct:log("~p",[_X]), + api_ng_cipher_increment({Type, Key, IV, PlainTexts, undefined}); + +api_ng_cipher_increment({Type, Key, IV, PlainText0, ExpectedEncText}=_X) -> + ct:log("~p",[_X]), + PlainTexts = iolistify(PlainText0), + RefEnc = crypto:crypto_init(Type, Key, IV, true), + RefDec = crypto:crypto_init(Type, Key, IV, false), + EncTexts = api_ng_cipher_increment_loop(RefEnc, PlainTexts), + Enc = iolist_to_binary(EncTexts), + case ExpectedEncText of + undefined -> + ok; + Enc -> + ok; + _ -> + ct:log("encode~nIn: ~p~nExpected: ~p~nEnc: ~p~n", [{Type,Key,IV,PlainTexts}, ExpectedEncText, Enc]), + ct:fail("api_ng_cipher_increment (encode)",[]) + end, + Plain = iolist_to_binary(PlainTexts), + case iolist_to_binary(api_ng_cipher_increment_loop(RefDec, EncTexts)) of + Plain -> + ok; + OtherPT -> + ct:log("decode~nIn: ~p~nExpected: ~p~nDec: ~p~n", [{Type,Key,IV,EncTexts}, Plain, OtherPT]), + ct:fail("api_ng_cipher_increment (encode)",[]) + end. + + +api_ng_cipher_increment_loop(Ref, InTexts) -> + lists:map(fun(Txt) -> + try crypto:crypto_update(Ref, Txt) + of + Bin when is_binary(Bin) -> + Bin + catch + error:Error -> + ct:pal("Txt = ~p",[Txt]), + ct:fail("~p",[Error]) + end + end, InTexts). + +%%-------------------------------------------------------------------- +api_ng_one_shot() -> + [{doc, "Test new api"}]. + +api_ng_one_shot(Config) when is_list(Config) -> + Blocks = lazy_eval(proplists:get_value(block, Config, [])), + Streams = lazy_eval(proplists:get_value(stream, Config, [])), + lists:foreach(fun do_api_ng_one_shot/1, Blocks++Streams). + +do_api_ng_one_shot({Type, Key, PlainTexts}=_X) -> + ct:log("~p",[_X]), + do_api_ng_one_shot({Type, Key, <<>>, PlainTexts}); + +do_api_ng_one_shot({Type, Key, IV, PlainTexts}=_X) -> + ct:log("~p",[_X]), + do_api_ng_one_shot({Type, Key, IV, PlainTexts, undefined}); + +do_api_ng_one_shot({Type, Key, IV, PlainText0, ExpectedEncText}=_X) -> + ct:log("~p",[_X]), + PlainText = iolist_to_binary(PlainText0), + EncTxt = crypto:crypto_one_shot(Type, Key, IV, PlainText, true), + case ExpectedEncText of + undefined -> + ok; + EncTxt -> + ok; + _ -> + ct:log("encode~nIn: ~p~nExpected: ~p~nEnc: ~p~n", [{Type,Key,IV,PlainText}, ExpectedEncText, EncTxt]), + ct:fail("api_ng_one_shot (encode)",[]) + end, + case crypto:crypto_one_shot(Type, Key, IV, EncTxt, false) of + PlainText -> + ok; + OtherPT -> + ct:log("decode~nIn: ~p~nExpected: ~p~nDec: ~p~n", [{Type,Key,IV,EncTxt}, PlainText, OtherPT]), + ct:fail("api_ng_one_shot (decode)",[]) + end. + +%%-------------------------------------------------------------------- +api_ng_tls() -> + [{doc, "Test special tls api"}]. + +api_ng_tls(Config) when is_list(Config) -> + Blocks = lazy_eval(proplists:get_value(block, Config, [])), + Streams = lazy_eval(proplists:get_value(stream, Config, [])), + lists:foreach(fun do_api_ng_tls/1, Blocks++Streams). + + +do_api_ng_tls({Type, Key, PlainTexts}=_X) -> + ct:log("~p",[_X]), + do_api_ng_tls({Type, Key, <<>>, PlainTexts}); + +do_api_ng_tls({Type, Key, IV, PlainTexts}=_X) -> + ct:log("~p",[_X]), + do_api_ng_tls({Type, Key, IV, PlainTexts, undefined}); + +do_api_ng_tls({Type, Key, IV, PlainText0, ExpectedEncText}=_X) -> + ct:log("~p",[_X]), + PlainText = iolist_to_binary(PlainText0), + Renc = crypto:crypto_init_dyn_iv(Type, Key, true), + Rdec = crypto:crypto_init_dyn_iv(Type, Key, false), + EncTxt = crypto:crypto_update_dyn_iv(Renc, PlainText, IV), + case ExpectedEncText of + undefined -> + ok; + EncTxt -> + %% Now check that the state is NOT updated: + case crypto:crypto_update_dyn_iv(Renc, PlainText, IV) of + EncTxt -> + ok; + EncTxt2 -> + ct:log("2nd encode~nIn: ~p~nExpected: ~p~nEnc: ~p~n", [{Type,Key,IV,PlainText}, EncTxt, EncTxt2]), + ct:fail("api_ng_tls (second encode)",[]) + end; + OtherEnc -> + ct:log("1st encode~nIn: ~p~nExpected: ~p~nEnc: ~p~n", [{Type,Key,IV,PlainText}, ExpectedEncText, OtherEnc]), + ct:fail("api_ng_tls (encode)",[]) + end, + case crypto:crypto_update_dyn_iv(Rdec, EncTxt, IV) of + PlainText -> + %% Now check that the state is NOT updated: + case crypto:crypto_update_dyn_iv(Rdec, EncTxt, IV) of + PlainText -> + ok; + PlainText2 -> + ct:log("2nd decode~nIn: ~p~nExpected: ~p~nDec: ~p~n", [{Type,Key,IV,EncTxt}, PlainText, PlainText2]), + ct:fail("api_ng_tls (second decode)",[]) + end; + OtherPT -> + ct:log("1st decode~nIn: ~p~nExpected: ~p~nDec: ~p~n", [{Type,Key,IV,EncTxt}, PlainText, OtherPT]), + ct:fail("api_ng_tlst (decode)",[]) + end. + +%%-------------------------------------------------------------------- no_aead() -> [{doc, "Test disabled aead ciphers"}]. no_aead(Config) when is_list(Config) -> @@ -666,6 +842,23 @@ rand_plugin_s(Config) when is_list(Config) -> rand_plugin_aux(explicit_state). %%-------------------------------------------------------------------- +cipher_info() -> + [{doc, "crypto cipher_info testing"}]. +cipher_info(Config) when is_list(Config) -> + #{type := _,key_length := _,iv_length := _, + block_size := _,mode := _} = crypto:cipher_info(aes_128_cbc), + {'EXIT',_} = (catch crypto:cipher_info(not_a_cipher)), + ok. + +%%-------------------------------------------------------------------- +hash_info() -> + [{doc, "crypto hash_info testing"}]. +hash_info(Config) when is_list(Config) -> + #{type := _,size := _,block_size := _} = crypto:hash_info(sha256), + {'EXIT',_} = (catch crypto:hash_info(not_a_hash)), + ok. + +%%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- hash(_, [], []) -> @@ -755,6 +948,7 @@ cmac_check({Type, Key, Text, Size, CMac}) -> ct:fail({{crypto, cmac, [Type, Key, Text, Size]}, {expected, ExpCMac}, {got, Other}}) end. + block_cipher({Type, Key, PlainText}) -> Plain = iolist_to_binary(PlainText), CipherText = crypto:block_encrypt(Type, Key, PlainText), @@ -832,46 +1026,51 @@ block_cipher_increment(Type, Key, IV0, IV, [PlainText | PlainTexts], Plain, Ciph stream_cipher({Type, Key, PlainText}) -> Plain = iolist_to_binary(PlainText), - State = crypto:stream_init(Type, Key), - {_, CipherText} = crypto:stream_encrypt(State, PlainText), - case crypto:stream_decrypt(State, CipherText) of + StateE = crypto:stream_init(Type, Key), + StateD = crypto:stream_init(Type, Key), + {_, CipherText} = crypto:stream_encrypt(StateE, PlainText), + case crypto:stream_decrypt(StateD, CipherText) of {_, Plain} -> ok; Other -> - ct:fail({{crypto, stream_decrypt, [State, CipherText]}, {expected, PlainText}, {got, Other}}) + ct:fail({{crypto, stream_decrypt, [StateD, CipherText]}, {expected, PlainText}, {got, Other}}) end; stream_cipher({Type, Key, IV, PlainText}) -> Plain = iolist_to_binary(PlainText), - State = crypto:stream_init(Type, Key, IV), - {_, CipherText} = crypto:stream_encrypt(State, PlainText), - case crypto:stream_decrypt(State, CipherText) of + StateE = crypto:stream_init(Type, Key, IV), + StateD = crypto:stream_init(Type, Key, IV), + {_, CipherText} = crypto:stream_encrypt(StateE, PlainText), + case crypto:stream_decrypt(StateD, CipherText) of {_, Plain} -> ok; Other -> - ct:fail({{crypto, stream_decrypt, [State, CipherText]}, {expected, PlainText}, {got, Other}}) + ct:fail({{crypto, stream_decrypt, [StateD, CipherText]}, {expected, PlainText}, {got, Other}}) end; stream_cipher({Type, Key, IV, PlainText, CipherText}) -> Plain = iolist_to_binary(PlainText), - State = crypto:stream_init(Type, Key, IV), - case crypto:stream_encrypt(State, PlainText) of + StateE = crypto:stream_init(Type, Key, IV), + StateD = crypto:stream_init(Type, Key, IV), + case crypto:stream_encrypt(StateE, PlainText) of {_, CipherText} -> ok; {_, Other0} -> - ct:fail({{crypto, stream_encrypt, [State, Type, Key, IV, Plain]}, {expected, CipherText}, {got, Other0}}) + ct:fail({{crypto, stream_encrypt, [StateE, Type, Key, IV, Plain]}, {expected, CipherText}, {got, Other0}}) end, - case crypto:stream_decrypt(State, CipherText) of + case crypto:stream_decrypt(StateD, CipherText) of {_, Plain} -> ok; Other1 -> - ct:fail({{crypto, stream_decrypt, [State, CipherText]}, {expected, PlainText}, {got, Other1}}) + ct:fail({{crypto, stream_decrypt, [StateD, CipherText]}, {expected, PlainText}, {got, Other1}}) end. stream_cipher_incment({Type, Key, PlainTexts}) -> - State = crypto:stream_init(Type, Key), - stream_cipher_incment_loop(State, State, PlainTexts, [], iolist_to_binary(PlainTexts)); + StateE = crypto:stream_init(Type, Key), + StateD = crypto:stream_init(Type, Key), + stream_cipher_incment_loop(StateE, StateD, PlainTexts, [], iolist_to_binary(PlainTexts)); stream_cipher_incment({Type, Key, IV, PlainTexts}) -> - State = crypto:stream_init(Type, Key, IV), - stream_cipher_incment_loop(State, State, PlainTexts, [], iolist_to_binary(PlainTexts)); + StateE = crypto:stream_init(Type, Key, IV), + StateD = crypto:stream_init(Type, Key, IV), + stream_cipher_incment_loop(StateE, StateD, PlainTexts, [], iolist_to_binary(PlainTexts)); stream_cipher_incment({Type, Key, IV, PlainTexts, _CipherText}) -> stream_cipher_incment({Type, Key, IV, PlainTexts}). diff --git a/lib/crypto/vsn.mk b/lib/crypto/vsn.mk index 6a91244715..deba17fb66 100644 --- a/lib/crypto/vsn.mk +++ b/lib/crypto/vsn.mk @@ -1 +1 @@ -CRYPTO_VSN = 4.4 +CRYPTO_VSN = 4.4.1 diff --git a/lib/debugger/doc/src/Makefile b/lib/debugger/doc/src/Makefile index 56d6085e9c..49b5a4be57 100644 --- a/lib/debugger/doc/src/Makefile +++ b/lib/debugger/doc/src/Makefile @@ -114,8 +114,7 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_docs_spec: docs $(INSTALL_DIR) "$(RELSYSDIR)/doc/pdf" $(INSTALL_DATA) $(TOP_PDF_FILE) "$(RELSYSDIR)/doc/pdf" - $(INSTALL_DIR) "$(RELSYSDIR)/doc/html" - ($(CP) -rf $(HTMLDIR) "$(RELSYSDIR)/doc") + $(INSTALL_DIR_DATA) $(HTMLDIR) "$(RELSYSDIR)/doc/html" $(INSTALL_DATA) $(INFO_FILE) "$(RELSYSDIR)" $(INSTALL_DIR) "$(RELEASE_PATH)/man/man3" $(INSTALL_DATA) $(MAN3DIR)/* "$(RELEASE_PATH)/man/man3" diff --git a/lib/dialyzer/doc/src/notes.xml b/lib/dialyzer/doc/src/notes.xml index 3cf776e566..bc422c43a0 100644 --- a/lib/dialyzer/doc/src/notes.xml +++ b/lib/dialyzer/doc/src/notes.xml @@ -32,6 +32,36 @@ <p>This document describes the changes made to the Dialyzer application.</p> +<section><title>Dialyzer 3.3.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Fix a bug that caused Dialyzer to crash when + analyzing a contract with a module name differing from + the analyzed module's name. The bug was introduced in + Erlang/OTP 18. </p> + <p> + Own Id: OTP-15562 Aux Id: ERL-845 </p> + </item> + <item> + <p> Fix a bug in the handling of the <c>Key</c> argument + of <c>lists:{keysearch, keyfind, keymember}</c>. </p> + <p> + Own Id: OTP-15570</p> + </item> + <item> + <p>Optimize (again) Dialyzer's handling of + left-associative use of <c>andalso</c> and <c>orelse</c> + in guards.</p> + <p> + Own Id: OTP-15577 Aux Id: ERL-851, PR-2141, PR-1944 </p> + </item> + </list> + </section> + +</section> + <section><title>Dialyzer 3.3.1</title> <section><title>Improvements and New Features</title> diff --git a/lib/dialyzer/vsn.mk b/lib/dialyzer/vsn.mk index 98ab533a58..7221993963 100644 --- a/lib/dialyzer/vsn.mk +++ b/lib/dialyzer/vsn.mk @@ -1 +1 @@ -DIALYZER_VSN = 3.3.1 +DIALYZER_VSN = 3.3.2 diff --git a/lib/diameter/doc/src/diameter.xml b/lib/diameter/doc/src/diameter.xml index 0a0194af2d..85522c99b2 100644 --- a/lib/diameter/doc/src/diameter.xml +++ b/lib/diameter/doc/src/diameter.xml @@ -1,7 +1,9 @@ <?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE erlref SYSTEM "erlref.dtd" [ - <!ENTITY spawn_opt + <!ENTITY spawn_opt2 '<seealso marker="erts:erlang#spawn_opt-2">erlang:spawn_opt/2</seealso>'> + <!ENTITY spawn_opt5 + '<seealso marker="erts:erlang#spawn_opt-5">erlang:spawn_opt/5</seealso>'> <!ENTITY nodes '<seealso marker="erts:erlang#nodes-0">erlang:nodes/0</seealso>'> <!ENTITY make_ref @@ -21,7 +23,7 @@ <copyright> <year>2011</year> -<year>2017</year> +<year>2019</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -1384,12 +1386,22 @@ the same peer.</p> </item> <tag> -<marker id="spawn_opt"/><c>{spawn_opt, [term()]}</c></tag> +<marker id="spawn_opt"/><c>{spawn_opt, [term()] | {M,F,A}}</c></tag> <item> <p> -Options passed to &spawn_opt; when spawning a process for an -incoming Diameter request. -Options <c>monitor</c> and <c>link</c> are ignored.</p> +An options list passed to &spawn_opt2; to spawn a handler process for an +incoming Diameter request on the local node, or an MFA that returns +the pid of a handler process.</p> + +<p> +Options <c>monitor</c> and <c>link</c> are ignored in the list-valued +case. +An MFA is applied with an additional term prepended to its argument +list, and should return either the pid of the handler process that +invokes <c>diameter_traffic:request/1</c> on the term in order to +process the request, or the atom <c>discard</c>. +The handler process need not be local, but diameter must be started on +the remote node.</p> <p> Defaults to the empty list.</p> diff --git a/lib/diameter/doc/src/diameter_transport.xml b/lib/diameter/doc/src/diameter_transport.xml index 67fd54bc56..0a8ef321c6 100644 --- a/lib/diameter/doc/src/diameter_transport.xml +++ b/lib/diameter/doc/src/diameter_transport.xml @@ -14,7 +14,8 @@ <erlref> <header> <copyright> -<year>2011</year><year>2016</year> +<year>2011</year> +<year>2019</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -174,10 +175,13 @@ its parent.</p> <taglist> -<tag><c>{diameter, {send, &message;}}</c></tag> +<tag><c>{diameter, {send, &message; | false}}</c></tag> <item> <p> -An outbound Diameter message.</p> +An outbound Diameter message. +The atom <c>false</c> can only be received when request +acknowledgements have been requests: see the <c>ack</c> message +below.</p> </item> <tag><c>{diameter, {close, Pid}}</c></tag> @@ -246,6 +250,27 @@ A <c>LocalAddr</c> list has the same semantics as one returned from &start;.</p> </item> +<tag><c>{diameter, ack}</c></tag> +<item> +<p> +Request acknowledgements of unanswered requests. +A transport process should send this once before passing incoming +Diameter messages into diameter. +As a result, every Diameter request passed into diameter with a +<c>recv</c> message (below) will be answered with a +<c>send</c> message (above), either a &message; for the transport +process to send or the atom <c>false</c> if the request has been +discarded or otherwise not answered.</p> + +<p> +This is to allow a transport process to keep count of the number +of incoming request messages that have not yet been answered or +discarded, to allow it to regulate the amount of incoming traffic. +Both diameter_tcp and diameter_sctp request acknowledgements when a +<c>message_cb</c> is configured, turning send/recv message into +callbacks that can be used to regulate traffic.</p> +</item> + <tag><c>{diameter, {recv, &message;}}</c></tag> <item> <p> diff --git a/lib/diameter/doc/src/notes.xml b/lib/diameter/doc/src/notes.xml index cc92bd99f0..5777225ae7 100644 --- a/lib/diameter/doc/src/notes.xml +++ b/lib/diameter/doc/src/notes.xml @@ -43,6 +43,41 @@ first.</p> <!-- ===================================================================== --> +<section><title>diameter 2.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix failure of incoming answer message with faulty + Experimental-Result-Code. Failure to decode the AVP + resulted in an uncaught exception, with no no + handle_answer/error callback as a consequence.</p> + <p> + Own Id: OTP-15569 Aux Id: ERIERL-302 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Add spawn_opt MFA configuration to allow a callback to + spawn a handler process for an incoming Diameter request + on an an arbitrary node. Module diameter_dist provides a + route_session/2 that can be used to distribute requests + based on Session-Id, although this module is currently + only documented in the module itself and may change.</p> + <p> + Own Id: OTP-15398</p> + </item> + </list> + </section> + +</section> + <section><title>diameter 2.1.6</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/diameter/src/base/diameter.erl b/lib/diameter/src/base/diameter.erl index b90b794611..7f172e1fa1 100644 --- a/lib/diameter/src/base/diameter.erl +++ b/lib/diameter/src/base/diameter.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% Copyright Ericsson AB 2010-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -365,7 +365,7 @@ call(SvcName, App, Message) -> | {connect_timer, 'Unsigned32'()} | {watchdog_timer, 'Unsigned32'() | {module(), atom(), list()}} | {watchdog_config, [{okay|suspect, non_neg_integer()}]} - | {spawn_opt, list()}. + | {spawn_opt, list() | mfa()}. %% Options passed to start_service/2 diff --git a/lib/diameter/src/base/diameter_callback.erl b/lib/diameter/src/base/diameter_callback.erl index d04a416bef..3bcf550cd8 100644 --- a/lib/diameter/src/base/diameter_callback.erl +++ b/lib/diameter/src/base/diameter_callback.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% Copyright Ericsson AB 2010-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -70,7 +70,7 @@ -module(diameter_callback). -%% Default callbacks when no aleternate is specified. +%% Default callbacks when no alternate is specified. -export([peer_up/3, peer_down/3, pick_peer/4, diff --git a/lib/diameter/src/base/diameter_dist.erl b/lib/diameter/src/base/diameter_dist.erl new file mode 100644 index 0000000000..5c29ea95a4 --- /dev/null +++ b/lib/diameter/src/base/diameter_dist.erl @@ -0,0 +1,525 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(diameter_dist). + +-behaviour(gen_server). + +%% +%% Implements callbacks that can be configured as a spawn_opt +%% transport configuration, to be able to distribute incoming Diameter +%% requests to handler processes (local or remote) in various ways. +%% + +%% spawn_opt callbacks +-export([spawn_local/2, + spawn_local/1, + route_session/2, + route_session/1]). + +%% signal availability for handling incoming requests to route_sesssion/2 +-export([attach/1, + detach/1]). + +%% consistent hashing +-export([hash/3, %% for use as default MFA in route_session/2 options map + hash/2]). %% arbitrary key/values + +-include_lib("diameter/include/diameter.hrl"). + +%% server start +-export([start_link/0]). + +%% gen_server callbacks +-export([init/1, + handle_info/2, + handle_cast/2, + handle_call/3, + code_change/3, + terminate/2]). + +-type request() :: tuple(). %% callback argument from diameter_traffic + +-define(SERVER, ?MODULE). %% server monitoring node connections + +%% Maps a node name binary to the corresponding atom. Used by +%% route_session/2 to map the optional value of a Session-Id to +%% node(). +-define(NODE_TABLE, diameter_dist_node). + +%% Maps a diameter:service_name() to a node() that has called attach/1 +%% to declare its willingness to handle incoming requests for the +%% service. Use by route_session/2 in case the optional value mapping +%% has failed. +-define(SERVICE_TABLE, diameter_dist_service). + +-define(B(A), atom_to_binary(A, utf8)). +-define(ORCOND(List), list_to_tuple(['orelse', false | List])). +-define(HASH(T), erlang:phash2(T, 16#100000000)). + +%% spawn_local/2 +%% +%% Callback that is equivalent to an options list. That is, the +%% following are equivalent when passed as options to +%% diameter:add_transport/2. +%% +%% {spawn_opt, Opts} +%% {spawn_opt, {diameter_dist, spawn_local, [Opts]}} + +-spec spawn_local(ReqT :: request(), Opts :: list()) + -> pid(). + +spawn_local(ReqT, Opts) -> + spawn_opt(diameter_traffic, request, [ReqT], Opts). + +%% spawn_local/1 + +spawn_local(ReqT) -> + spawn_local(ReqT, []). + +%% route_session/2 +%% +%% Callback that maps the Session-Id of an incoming request to a +%% handler node. +%% +%% With an options list, maps an id whose optional value is the name +%% of a connected node to the same node, to handle the case that the +%% session id has been returned from diameter:session_id/1; otherwise +%% to a node that has called diameter_dist:attach/1 using the +%% consistent hashing provided by hash/3, or to the local node() if a +%% session id could not be extracted or there are no attached nodes. A +%% handler process is spawned on the selected node using +%% erlang:spawn_opt/4. +%% +%% Different behaviour can be configured by supplying an options map +%% of the following form: +%% +%% #{search => non_neg_integer(), +%% id => [binary()], +%% default => discard | local | mfa(), +%% dispatch => list() | mfa()} +%% +%% The search member limits the number of AVPs that are examined in +%% the message (from the front), to avoid searching entire message in +%% case it's known that peers follow RFC 6733's recommendation that +%% Session-Id be placed at the head of a message. The default is to +%% search the entire message. +%% +%% The id member restricts the optional value mapping to session ids +%% whose DiamterIdentity is one of those specified. Set this to the +%% list of Diameter identities advertised by the service in question +%% (typically one) to ensure that only locally generated session ids +%% are mapped; or to the empty list to disable the mapping. +%% +%% The default member determines where to handle a message whose +%% Session-Id isn't found or whose optional value isn't mapped to the +%% name of a connected node. The atom local says the local node, an +%% MFA is invoked on Session-Id | false, the name of the diameter +%% service, and the message binary, and should return either a node() +%% or false to discard the message. Defaults to {diameter_dist, hash, []}. +%% +%% The dispatch member determines how the pid() of the request handler +%% process is retrieved. An MFA is applied to a previously selected +%% node(), and the module, function, and arguments list to apply in +%% the handler process to handle the request, the MFA being supplied +%% by diameter, and returns pid() | discard. A list is equivalent to +%% {erlang, spawn_opt, []}. Defaults to []. +%% +%% This can be used with search = 0 to route on something other than +%% Session-Id, but this is probably no simpler than just implementing +%% an own spawn_opt callback. (Except with the default dispatch possibly.) +%% +%% Note that if the peer is also implemented with OTP diameter and +%% generating session ids with diameter:session_id/1 then +%% route_session/2 can map an optional value to a local node that +%% happens to have the same name as one of the peer's nodes. This +%% could lead to an uneven distribution; for example, if the peer +%% nodes are a subset of the local nodes. In practice, it's typically +%% known if it's peers or the local node originating sessions; if the +%% former then setting id = [] disables the optional value mapping, if +%% the latter then setting default = local disables the hashing. +-spec route_session(ReqT :: request(), Opts) + -> discard + | pid() + when Opts :: pos_integer() %% aka #{search => N} + | list() %% aka #{dispatch => Opts} + | #{search => non_neg_integer(), %% limit number of examined AVPs + id => [binary()], %% restrict optional value map on DiamIdent + default => local %% handle locally + | discard + | mfa(), %% return node() | false + dispatch => list() %% spawn options + | mfa()}. %% (Node, M, F, A) -> pid() | discard + +route_session(ReqT, Opts) -> + {_, Bin} = Info = diameter_traffic:request_info(ReqT), + Sid = session_id(avps(Bin), search(Opts)), + Node = default(node_of_session_id(Sid, Opts), Sid, Opts, Info), + dispatch(Node, ReqT, dispatch(Opts)). + +%% avps/1 + +avps(<<_:20/binary, Bin/binary>>) -> + Bin; + +avps(_) -> + false. + +%% dispatch/3 + +dispatch(false, _, _) -> + discard; + +dispatch(Node, ReqT, {M,F,A}) -> + apply(M, F, [Node, diameter_traffic, request, [ReqT] | A]); + +dispatch(Node, ReqT, Opts) -> + spawn_opt(Node, diameter_traffic, request, [ReqT], Opts). + +%% route_session/1 + +route_session(ReqT) -> + route_session(ReqT, []). + +%% node_of_session_id/2 +%% +%% Return the node name encoded as optional value in a Session-Id, +%% assuming the id has been created with diameter:session_id/0. Lookup +%% the node name to ensure we don't convert arbitrary binaries to +%% atom. + +node_of_session_id([Id, _, _, Bin], #{id := Ids}) -> + lists:member(Id, Ids) andalso nodemap(Bin); + +node_of_session_id([_, _, _, Bin], _) -> + nodemap(Bin); + +node_of_session_id(_, _) -> + false. + +%% nodemap/1 + +nodemap(Bin) -> + try + ets:lookup_element(?NODE_TABLE, Bin, 2) + catch + error: badarg -> false + end. + +%% session_id/2 + +session_id(_, 0) -> %% give up + false; + +%% Session-Id = Command Code 263, V-bit = 0. +session_id(<<263:32, 0:1, _:7, Len:24, _/binary>> = Bin, _) -> + case Bin of + <<Avp:Len/binary, _/binary>> -> + <<_:8/binary, Sid/binary>> = Avp, + split(Sid); + _ -> + false + end; + +%% Jump to the next AVP. This is potentially costly for a message with +%% many AVPs and no Session-Id, which an attacker is prone to send. +%% 8.8 or RFC 6733 says that Session-Id SHOULD (but not MUST) appear +%% immediately following the Diameter Header, so there is no +%% guarantee. +session_id(<<_:40, Len:24, _/binary>> = Bin, N) -> + Pad = (4 - (Len rem 4)) rem 4, + case Bin of + <<_:Len/binary, _:Pad/binary, Rest/binary>> -> + session_id(Rest, if N == infinity -> N; true -> N-1 end); + _ -> + false + end; + +session_id(_, _) -> + false. + +%% split/1 +%% +%% Split a Session-Id at no more than three semicolons: the optional +%% value (if any) follows the third. binary:split/2 does better than +%% matching character by character, especially when the pattern is +%% compiled. + +split(Bin) -> + split(3, Bin, pattern()). + +%% split/3 + +split(0, Bin, _) -> + [Bin]; + +split(N, Bin, Pattern) -> + [H|T] = binary:split(Bin, Pattern), + [H | case T of + [] -> + T; + [Rest] -> + split(N-1, Rest, Pattern) + end]. + +%% pattern/0 +%% +%% Since this is being called in a watchdog process, compile the +%% pattern once and maintain it in the process dictionary. + +pattern() -> + case get(?MODULE) of + undefined -> + CP = binary:compile_pattern(<<$;>>), %% tuple + put(?MODULE, CP), + CP; + CP -> + CP + end. + +%% dispatch/1 + +dispatch(#{} = Opts) -> + maps:get(dispatch, Opts, []); + +dispatch(Opts) + when is_list(Opts) -> + Opts; + +dispatch(_) -> + []. + +%% search/1 +%% +%% Bound number of AVPs examined when looking for Session-Id. + +search(#{search := N}) + when is_integer(N), 0 =< N -> + N; + +search(N) + when is_integer(N), 0 =< N -> + N; + +search(_) -> + infinity. + +%% default/3 +%% +%% Choose a node when Session-Id lookup has failed. + +default(false, _, #{default := discard}, _) -> + false; + +default(false, _, #{default := local}, _) -> + node(); + +default(false, Sid, #{default := {M,F,A}}, Info) -> + {ServiceName, Bin} = Info, + apply(M, F, [Sid, ServiceName, Bin | A]); %% node() | false + +default(false, Sid, _, Info) -> %% aka {?MODULE, hash, []} + {ServiceName, Bin} = Info, + hash(Sid, ServiceName, Bin); + +default(Node, _, _, _) -> + Node. + +%% =========================================================================== + +%% hash/3 +%% +%% Consistent hashing of Session-Id to an attached node, or the local +%% node if Session-Id = false or no attached nodes. + +hash(Sid, ServiceName, _) -> + case false /= Sid andalso attached(ServiceName) of + [_|_] = Nodes -> + hash(Sid, Nodes); + _ -> + node() + end. + +%% hash/2 +%% +%% Consistent hashing on arbitrary key/values. Returns false if the +%% list is empty. + +%% No key or no values. +hash(_, []) -> + false; + +%% Not much choice. +hash(_, [Value]) -> + Value; + +%% Hash on a circle and choose the closest predecessor. +hash(Key, Values) -> + Hash = ?HASH(Key), + tl(lists:foldl(fun(V,A) -> + choose(Hash, [?HASH({Key, V}) | V], A) + end, + false, %% < list() + Values)). + +%% choose/3 + +choose(Hash, [Hash1 | _] = T, [Hash2 | _]) + when Hash1 =< Hash, Hash < Hash2 -> + T; + +choose(Hash, [Hash1 | _], [Hash2 | _] = T) + when Hash2 =< Hash, Hash < Hash1 -> + T; + +choose(_, T1, T2) -> + max(T1, T2). + +%% =========================================================================== + +%% attach/1 +%% +%% Register the local node as a handler of incoming requests for the +%% specified services when using the route_session/2 spawn_opt +%% callback. + +attach(ServiceNames) -> + abcast({attach, node(), ServiceNames}). + +%% detach/1 +%% +%% Deregister the local node as a handler of incoming requests. + +detach(ServiceNames) -> + abcast({detach, node(), ServiceNames}). + +%% abcast/1 + +abcast(T) -> + gen_server:abcast([node() | nodes()], ?SERVER, T), + ok. + +%% attached/1 + +attached(ServiceName) -> + try + ets:lookup_element(?SERVICE_TABLE, ServiceName, 2) + catch + error: badarg -> [] + end. + +%% cast/2 + +cast(Node, T) -> + gen_server:cast({?SERVER, Node}, T). + +%% attach/2 + +attach(Node, S) -> + case sets:to_list(S) of + [] -> + ok; + Services -> + cast(Node, {attach, node(), Services}) + end. + +%% =========================================================================== + +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, _Args = [], _Opts = []). + +%% init/1 +%% +%% Maintain [node() | nodes()] in a table that maps from binary-valued +%% names, so we can lookup the corresponding atoms rather than convert +%% binaries that aren't necessarily node names. + +init([]) -> + ets:new(?NODE_TABLE, [set, named_table]), + ets:new(?SERVICE_TABLE, [bag, named_table]), + ok = net_kernel:monitor_nodes(true, [{node_type, all}, nodedown_reason]), + ets:insert(?NODE_TABLE, [{?B(N), N} || N <- [node() | nodes()]]), + abcast({attach, node()}), + {ok, sets:new()}. + +%% handle_call/3 + +handle_call(_, _From, S) -> + {reply, nok, S}. + +%% handle_cast/2 + +%% Remote node is asking which services the local node wants to handle. +handle_cast({attach, Node}, S) + when Node /= node() -> + attach(Node, S), + {noreply, S}; + +%% Node wants to handle incoming requests ... +handle_cast({attach, Node, ServiceNames}, S) -> + ets:insert(?SERVICE_TABLE, [{N, Node} || N <- ServiceNames]), + {noreply, case node() of + Node -> + sets:union(S, sets:from_list(ServiceNames)); + _ -> + S + end}; + +%% ... or not. +handle_cast({detach, Node, ServiceNames}, S) -> + ets:select_delete(?SERVICE_TABLE, [{{'$1', Node}, + [?ORCOND([{'==', '$1', {const, N}} + || N <- ServiceNames])], + [true]}]), + {noreply, case node() of + Node -> + sets:subtract(S, sets:from_list(ServiceNames)); + _ -> + S + end}; + +handle_cast(_, S) -> + {noreply, S}. + +%% handle_info/2 + +handle_info({nodeup, Node, _}, S) -> + ets:insert(?NODE_TABLE, {?B(Node), Node}), + cast(Node, {attach, node()}), %% ask which services remote node handles + attach(Node, S), %% say which service local node handles + {noreply, S}; + +handle_info({nodedown, Node, _}, S) -> + ets:delete(?NODE_TABLE, ?B(Node)), + ets:select_delete(?SERVICE_TABLE, [{{'_', Node}, [], [true]}]), + {noreply, S}; + +handle_info(_, S) -> + {noreply, S}. + +%% terminate/2 + +terminate(_, _) -> + ok. + +%% code_change/3 + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. diff --git a/lib/diameter/src/base/diameter_misc_sup.erl b/lib/diameter/src/base/diameter_misc_sup.erl index 343688be23..fec5a41b5c 100644 --- a/lib/diameter/src/base/diameter_misc_sup.erl +++ b/lib/diameter/src/base/diameter_misc_sup.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -35,6 +35,7 @@ diameter_stats, %% statistics counter management diameter_reg, %% service/property publishing diameter_peer, %% remote peer manager + diameter_dist, %% request distribution diameter_config]). %% configuration/restart %% start_link/0 diff --git a/lib/diameter/src/base/diameter_traffic.erl b/lib/diameter/src/base/diameter_traffic.erl index 2d3e4a2ac9..8423e30269 100644 --- a/lib/diameter/src/base/diameter_traffic.erl +++ b/lib/diameter/src/base/diameter_traffic.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2017. All Rights Reserved. +%% Copyright Ericsson AB 2013-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -42,8 +42,12 @@ peer_up/1, peer_down/1]). +%% towards diameter_dist +-export([request_info/1]). + %% internal -export([send/1, %% send from remote node + request/1, %% process request in handler process init/1]). %% monitor process start -include_lib("diameter/include/diameter.hrl"). @@ -232,7 +236,7 @@ incr_rc(Dir, Pkt, TPid, MsgDict, AppDict, Dict0) -> -spec receive_message(pid(), Route, #diameter_packet{}, module(), RecvData) -> pid() %% request handler | boolean() %% answer, known request or not - | discard %% request discarded by MFA + | discard %% request discarded when Route :: {Handler, RequestRef, TPid} | Ack, RecvData :: {[SpawnOpt], #recvdata{}}, @@ -252,7 +256,8 @@ receive_message(TPid, Route, Pkt, Dict0, RecvData) -> recv(true, Ack, TPid, Pkt, Dict0, T) when is_boolean(Ack) -> {Opts, RecvData} = T, - spawn_request(Ack, TPid, Pkt, Dict0, RecvData, Opts); + AppT = find_app(TPid, Pkt, RecvData), + ack(Ack, TPid, spawn_request(AppT, Opts, Ack, TPid, Pkt, Dict0, RecvData)); %% ... answer to known request ... recv(false, {Pid, Ref, TPid}, _, Pkt, Dict0, _) -> @@ -274,58 +279,73 @@ recv(false, false, TPid, Pkt, _, _) -> incr(TPid, {{unknown, 0}, recv, discarded}), false. -%% spawn_request/6 +%% spawn_request/7 + +spawn_request(false, _, _, _, _, _, _) -> %% no transport + discard; -%% An MFA should return a pid() or the atom 'discard'. The latter -%% results in an acknowledgment back to the transport process when -%% appropriate, to ensure that send/recv callbacks can count -%% outstanding requests. Acknowledgement is implicit if the +%% An MFA should return the pid() of a process in which the argument +%% fun in applied, or the atom 'discard' if the fun is not applied. +%% The latter results in an acknowledgment back to the transport +%% process when appropriate, to ensure that send/recv callbacks can +%% count outstanding requests. Acknowledgement is implicit if the %% handler process dies (in a handle_request callback for example). -spawn_request(Ack, TPid, Pkt, Dict0, RecvData, {M,F,A}) -> - ReqF = fun() -> - ack(Ack, TPid, recv_request(Ack, TPid, Pkt, Dict0, RecvData)) - end, - ack(Ack, TPid, apply(M, F, [ReqF | A])); +spawn_request(AppT, {M,F,A}, Ack, TPid, Pkt, Dict0, RecvData) -> + %% Term to pass to request/1 in an appropriate process. Module + %% diameter_dist implements callbacks. + ReqT = {Pkt, AppT, Ack, TPid, Dict0, RecvData}, + apply(M, F, [ReqT | A]); %% A spawned process acks implicitly when it dies, so there's no need %% to handle 'discard'. -spawn_request(Ack, TPid, Pkt, Dict0, RecvData, Opts) -> +spawn_request(AppT, Opts, Ack, TPid, Pkt, Dict0, RecvData) -> spawn_opt(fun() -> - recv_request(Ack, TPid, Pkt, Dict0, RecvData) + recv_request(Ack, TPid, Pkt, Dict0, RecvData, AppT) end, Opts). +%% request_info/1 +%% +%% Limited request information for diameter_dist. + +request_info({Pkt, _AppT, _Ack, _TPid, _Dict0, RecvData} = _ReqT) -> + {RecvData#recvdata.service_name, Pkt#diameter_packet.bin}. + +%% request/1 +%% +%% Called from a handler process chosen by a transport spawn_opt MFA +%% to process an incoming request. + +request({Pkt, AppT, Ack, TPid, Dict0, RecvData} = _ReqT) -> + ack(Ack, TPid, recv_request(Ack, TPid, Pkt, Dict0, RecvData, AppT)). + %% ack/3 ack(Ack, TPid, RC) -> - RC == discard andalso Ack andalso (TPid ! {send, false}), + RC == discard + andalso Ack + andalso (TPid ! {send, false}), RC. %% --------------------------------------------------------------------------- -%% recv_request/5 +%% recv_request/6 %% --------------------------------------------------------------------------- -spec recv_request(Ack :: boolean(), TPid :: pid(), #diameter_packet{}, Dict0 :: module(), - #recvdata{}) + #recvdata{}, + AppT :: {#diameter_app{}, #diameter_caps{}} + | #diameter_caps{}) %% no suitable app -> ok %% answer was sent - | discard %% or not - | false. %% no transport - -recv_request(Ack, - TPid, - #diameter_packet{header = #diameter_header{application_id = Id}} - = Pkt, - Dict0, - #recvdata{peerT = PeerT, - apps = Apps, - counters = Count} - = RecvData) -> + | discard. %% or not + +recv_request(Ack, TPid, Pkt, Dict0, RecvData, AppT) -> Ack andalso (TPid ! {handler, self()}), - case diameter_service:find_incoming_app(PeerT, TPid, Id, Apps) of + case AppT of {#diameter_app{id = Aid, dictionary = AppDict} = App, Caps} -> + Count = RecvData#recvdata.counters, Count andalso incr(recv, Pkt, TPid, AppDict), DecPkt = decode(Aid, AppDict, RecvData, Pkt), Count andalso incr_error(recv, DecPkt, TPid, AppDict), @@ -349,11 +369,20 @@ recv_request(Ack, Dict0, RecvData, DecPkt, - [[]]); - false = No -> %% transport has gone down - No + [[]]) end. +%% find_app/3 +%% +%% Lookup the application of a received Diameter request on the node +%% on which it's received. + +find_app(TPid, + #diameter_packet{header = #diameter_header{application_id = Id}}, + #recvdata{peerT = PeerT, + apps = Apps}) -> + diameter_service:find_incoming_app(PeerT, TPid, Id, Apps). + %% decode/4 decode(Id, Dict, #recvdata{codec = Opts}, Pkt) -> diff --git a/lib/diameter/src/diameter.appup.src b/lib/diameter/src/diameter.appup.src index 4e6b983bac..52263633fb 100644 --- a/lib/diameter/src/diameter.appup.src +++ b/lib/diameter/src/diameter.appup.src @@ -60,7 +60,8 @@ {"2.1.3", [{restart_application, diameter}]}, %% 20.2 {"2.1.4", [{restart_application, diameter}]}, %% 20.3 {"2.1.4.1", [{restart_application, diameter}]}, %% 20.3.8.19 - {"2.1.5", [{update, diameter_peer_fsm}]} %% 21.0 + {"2.1.5", [{restart_application, diameter}]}, %% 21.0 + {"2.1.6", [{restart_application, diameter}]} %% 21.1 ], [ {"0.9", [{restart_application, diameter}]}, @@ -102,6 +103,7 @@ {"2.1.3", [{restart_application, diameter}]}, {"2.1.4", [{restart_application, diameter}]}, {"2.1.4.1", [{restart_application, diameter}]}, - {"2.1.5", [{update, diameter_peer_fsm}]} + {"2.1.5", [{restart_application, diameter}]}, + {"2.1.6", [{restart_application, diameter}]} ] }. diff --git a/lib/diameter/src/modules.mk b/lib/diameter/src/modules.mk index bb86de016a..d16292bb88 100644 --- a/lib/diameter/src/modules.mk +++ b/lib/diameter/src/modules.mk @@ -1,7 +1,7 @@ # %CopyrightBegin% # -# Copyright Ericsson AB 2010-2017. All Rights Reserved. +# Copyright Ericsson AB 2010-2019. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -40,6 +40,7 @@ RT_MODULES = \ base/diameter_config \ base/diameter_config_sup \ base/diameter_codec \ + base/diameter_dist \ base/diameter_gen \ base/diameter_lib \ base/diameter_misc_sup \ diff --git a/lib/diameter/test/diameter_dist_SUITE.erl b/lib/diameter/test/diameter_dist_SUITE.erl new file mode 100644 index 0000000000..b2e4c35b9a --- /dev/null +++ b/lib/diameter/test/diameter_dist_SUITE.erl @@ -0,0 +1,332 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +%% Tests of traffic between two Diameter nodes, the server being +%% spread across three Erlang nodes. +%% + +-module(diameter_dist_SUITE). + +-export([suite/0, + all/0]). + +%% testcases +-export([enslave/1, enslave/0, + ping/1, + start/1, + connect/1, + send/1, + stop/1, stop/0]). + +%% diameter callbacks +-export([peer_up/3, + peer_down/3, + pick_peer/4, + prepare_request/3, + prepare_retransmit/3, + handle_answer/4, + handle_error/4, + handle_request/3]). + +-export([call/1]). + +-include("diameter.hrl"). +-include("diameter_gen_base_rfc6733.hrl"). + +%% =========================================================================== + +-define(util, diameter_util). + +-define(CLIENT, 'CLIENT'). +-define(SERVER, 'SERVER'). +-define(REALM, "erlang.org"). +-define(DICT, diameter_gen_base_rfc6733). +-define(ADDR, {127,0,0,1}). + +%% Config for diameter:start_service/2. +-define(SERVICE(Host), + [{'Origin-Host', Host ++ [$.|?REALM]}, + {'Origin-Realm', ?REALM}, + {'Host-IP-Address', [?ADDR]}, + {'Vendor-Id', 12345}, + {'Product-Name', "OTP/diameter"}, + {'Auth-Application-Id', [?DICT:id()]}, + {'Origin-State-Id', origin()}, + {spawn_opt, {diameter_dist, route_session, [#{id => []}]}}, + {sequence, fun sequence/0}, + {string_decode, false}, + {application, [{dictionary, ?DICT}, + {module, ?MODULE}, + {request_errors, callback}, + {answer_errors, callback}]}]). + +-define(SUCCESS, 2001). +-define(BUSY, 3004). +-define(LOGOUT, ?'DIAMETER_BASE_TERMINATION-CAUSE_LOGOUT'). +-define(MOVED, ?'DIAMETER_BASE_TERMINATION-CAUSE_USER_MOVED'). +-define(TIMEOUT, ?'DIAMETER_BASE_TERMINATION-CAUSE_SESSION_TIMEOUT'). + +-define(L, atom_to_list). +-define(A, list_to_atom). + +%% The order here is significant and causes the server to listen +%% before the clients connect. The server listens on the first node, +%% and distributes requests to the other two. +-define(NODES, [{server0, ?SERVER}, + {server1, ?SERVER}, + {server2, ?SERVER}, + {client, ?CLIENT}]). + +%% Options to ct_slave:start/2. +-define(TIMEOUTS, [{T, 15000} || T <- [boot_timeout, + init_timeout, + start_timeout]]). + +%% =========================================================================== + +suite() -> + [{timetrap, {seconds, 60}}]. + +all() -> + [enslave, + ping, + start, + connect, + send, + stop]. + +%% =========================================================================== +%% start/stop testcases + +%% enslave/1 +%% +%% Start four slave nodes, three to implement a Diameter server, +%% one to implement a client. + +enslave() -> + [{timetrap, {seconds, 30*length(?NODES)}}]. + +enslave(Config) -> + Here = filename:dirname(code:which(?MODULE)), + Ebin = filename:join([Here, "..", "ebin"]), + Dirs = [Here, Ebin], + Nodes = [{N,S} || {M,S} <- ?NODES, N <- [slave(M, Dirs)]], + ?util:write_priv(Config, nodes, [{N,S} || {{N,ok},S} <- Nodes]), + [] = [{T,S} || {{_,E} = T, S} <- Nodes, E /= ok]. + +slave(Name, Dirs) -> + add_pathsa(Dirs, ct_slave:start(Name, ?TIMEOUTS)). + +add_pathsa(Dirs, {ok, Node}) -> + {Node, rpc:call(Node, code, add_pathsa, [Dirs])}; +add_pathsa(_, No) -> + {No, error}. + +%% ping/1 +%% +%% Ensure the server nodes are connected so that diameter_dist can attach. + +ping({S, Nodes}) -> + ?SERVER = S, + [N || {N,_} <- Nodes, + node() /= N, + pang <- [net_adm:ping(N)]]; + +ping(Config) -> + Nodes = lists:droplast(?util:read_priv(Config, nodes)), + [] = [{N,RC} || {N,S} <- Nodes, + RC <- [rpc:call(N, ?MODULE, ping, [{S,Nodes}])], + RC /= []]. + +%% start/1 +%% +%% Start diameter services. + +start(SvcName) + when is_atom(SvcName) -> + ok = diameter:start(), + ok = diameter:start_service(SvcName, ?SERVICE((?L(SvcName)))); + +start(Config) -> + Nodes = ?util:read_priv(Config, nodes), + [] = [{N,RC} || {N,S} <- Nodes, + RC <- [rpc:call(N, ?MODULE, start, [S])], + RC /= ok]. + +sequence() -> + sequence(sname()). + +sequence(client) -> + {0,32}; +sequence(Server) -> + "server" ++ N = ?L(Server), + {list_to_integer(N), 30}. + +origin() -> + origin(sname()). + +origin(client) -> + 99; +origin(Server) -> + "server" ++ N = ?L(Server), + list_to_integer(N). + +%% connect/1 +%% +%% Establish one connection from the client, terminated on the first +%% server node, the others handling requests. + +connect({?SERVER, Config, [{Node, _} | _]}) -> + if Node == node() -> %% server0 + ?util:write_priv(Config, lref, {Node, ?util:listen(?SERVER, tcp)}); + true -> + diameter_dist:attach([?SERVER]) + end, + ok; + +connect({?CLIENT, Config, _}) -> + ?util:connect(?CLIENT, tcp, ?util:read_priv(Config, lref)), + ok; + +connect(Config) -> + Nodes = ?util:read_priv(Config, nodes), + [] = [{N,RC} || {N,S} <- Nodes, + RC <- [rpc:call(N, ?MODULE, connect, [{S, Config, Nodes}])], + RC /= ok]. + +%% stop/1 +%% +%% Stop the slave nodes. + +stop() -> + [{timetrap, {seconds, 30*length(?NODES)}}]. + +stop(_Config) -> + [] = [{N,E} || {N,_} <- ?NODES, + {error, _, _} = E <- [ct_slave:stop(N)]]. + +%% =========================================================================== +%% traffic testcases + +%% send/1 +%% +%% Send 100 requests and ensure the node name sent as User-Name isn't +%% the node terminating transport. + +send(Config) -> + send(Config, 100, dict:new()). + +%% send/2 + +send(Config, 0, Dict) -> + [{Server0, _} | _] = ?util:read_priv(Config, nodes) , + Node = atom_to_binary(Server0, utf8), + {false, _} = {dict:is_key(Node, Dict), dict:to_list(Dict)}; + +send(Config, N, Dict) -> + #diameter_base_STA{'Result-Code' = ?SUCCESS, + 'User-Name' = [ServerNode]} + = send(Config, str(?LOGOUT)), + true = is_binary(ServerNode), + send(Config, N-1, dict:update_counter(ServerNode, 1, Dict)). + +%% =========================================================================== + +str(Cause) -> + #diameter_base_STR{'Destination-Realm' = ?REALM, + 'Auth-Application-Id' = ?DICT:id(), + 'Termination-Cause' = Cause}. + +%% send/2 + +send(Config, Req) -> + {Node, _} = lists:last(?util:read_priv(Config, nodes)), + rpc:call(Node, ?MODULE, call, [Req]). + +%% call/1 + +call(Req) -> + diameter:call(?CLIENT, ?DICT, Req, []). + +%% sname/0 + +sname() -> + ?A(hd(string:tokens(?L(node()), "@"))). + +%% =========================================================================== +%% diameter callbacks + +%% peer_up/3 + +peer_up(_SvcName, _Peer, State) -> + State. + +%% peer_down/3 + +peer_down(_SvcName, _Peer, State) -> + State. + +%% pick_peer/4 + +pick_peer([Peer], [], ?CLIENT, _State) -> + {ok, Peer}. + +%% prepare_request/3 + +prepare_request(Pkt, ?CLIENT, {_Ref, Caps}) -> + #diameter_packet{msg = Req} + = Pkt, + #diameter_caps{origin_host = {OH, _}, + origin_realm = {OR, _}} + = Caps, + {send, Req#diameter_base_STR{'Origin-Host' = OH, + 'Origin-Realm' = OR, + 'Session-Id' = diameter:session_id(OH)}}. + +%% prepare_retransmit/3 + +prepare_retransmit(_, ?CLIENT, _) -> + discard. + +%% handle_answer/5 + +handle_answer(Pkt, _Req, ?CLIENT, _Peer) -> + #diameter_packet{msg = Rec, errors = []} = Pkt, + Rec. + +%% handle_error/5 + +handle_error(Reason, _Req, ?CLIENT, _Peer) -> + {error, Reason}. + +%% handle_request/3 + +handle_request(Pkt, ?SERVER, {_, Caps}) -> + #diameter_packet{msg = #diameter_base_STR{'Session-Id' = SId}} + = Pkt, + #diameter_caps{origin_host = {OH, _}, + origin_realm = {OR, _}} + = Caps, + {reply, #diameter_base_STA{'Result-Code' = ?SUCCESS, + 'Session-Id' = SId, + 'Origin-Host' = OH, + 'Origin-Realm' = OR, + 'User-Name' = [atom_to_binary(node(), utf8)]}}. diff --git a/lib/diameter/test/diameter_distribution_SUITE.erl b/lib/diameter/test/diameter_distribution_SUITE.erl index 5146f68ff1..5fe02284ae 100644 --- a/lib/diameter/test/diameter_distribution_SUITE.erl +++ b/lib/diameter/test/diameter_distribution_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2016. All Rights Reserved. +%% Copyright Ericsson AB 2013-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -76,6 +76,7 @@ {share_peers, peers()}, {use_shared_peers, peers()}, {restrict_connections, false}, + {spawn_opt, {diameter_dist, spawn_local, []}}, {sequence, fun sequence/0}, {application, [{dictionary, ?DICT}, {module, ?MODULE}, @@ -125,7 +126,7 @@ all() -> %% enslave/1 %% %% Start four slave nodes, one to implement a Diameter server, -%% two three to implement a client. +%% three to implement a client. enslave() -> [{timetrap, {seconds, 30*length(?NODES)}}]. @@ -331,6 +332,8 @@ prepare_request(Pkt, ?CLIENT, {_Ref, Caps}, {_, client0}) -> 'Origin-Realm' = OR, 'Session-Id' = diameter:session_id(OH)}}. +%% prepare_retransmit/4 + prepare_retransmit(Pkt, ?CLIENT, _, {_, client0}) -> #diameter_packet{msg = #diameter_base_STR{'Termination-Cause' = ?MOVED}} = Pkt, %% assert diff --git a/lib/diameter/test/diameter_pool_SUITE.erl b/lib/diameter/test/diameter_pool_SUITE.erl index 97c16940ff..a36a4fa17a 100644 --- a/lib/diameter/test/diameter_pool_SUITE.erl +++ b/lib/diameter/test/diameter_pool_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2015-2017. All Rights Reserved. +%% Copyright Ericsson AB 2015-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -51,6 +51,7 @@ {'Auth-Application-Id', [0]}, %% common {'Acct-Application-Id', [3]}, %% accounting {restrict_connections, false}, + {spawn_opt, {diameter_dist, route_session, []}}, {application, [{alias, common}, {dictionary, diameter_gen_base_rfc6733}, {module, diameter_callback}]}, diff --git a/lib/diameter/test/diameter_traffic_SUITE.erl b/lib/diameter/test/diameter_traffic_SUITE.erl index 434aef01dd..47b00c25a2 100644 --- a/lib/diameter/test/diameter_traffic_SUITE.erl +++ b/lib/diameter/test/diameter_traffic_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2018. All Rights Reserved. +%% Copyright Ericsson AB 2010-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -539,8 +539,7 @@ add_transports(Config) -> ++ [{unordered, unordered()} || T == sctp], [{capabilities_cb, fun capx/2}, {pool_size, 8} - | server_apps()] - ++ [{spawn_opt, {erlang, spawn, []}} || CS]), + | server_apps()]), Cs = [?util:connect(CN, [T, {sender, CS} | client_opts(T)], LRef, diff --git a/lib/diameter/test/modules.mk b/lib/diameter/test/modules.mk index 0c73adca12..90b0a25d5f 100644 --- a/lib/diameter/test/modules.mk +++ b/lib/diameter/test/modules.mk @@ -1,7 +1,7 @@ # %CopyrightBegin% # -# Copyright Ericsson AB 2010-2017. All Rights Reserved. +# Copyright Ericsson AB 2010-2019. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -31,6 +31,7 @@ MODULES = \ diameter_codec_test \ diameter_config_SUITE \ diameter_compiler_SUITE \ + diameter_dist_SUITE \ diameter_distribution_SUITE \ diameter_dpr_SUITE \ diameter_event_SUITE \ diff --git a/lib/diameter/vsn.mk b/lib/diameter/vsn.mk index 8c75c9e55e..a900e8f28e 100644 --- a/lib/diameter/vsn.mk +++ b/lib/diameter/vsn.mk @@ -17,5 +17,5 @@ # %CopyrightEnd% APPLICATION = diameter -DIAMETER_VSN = 2.1.6 +DIAMETER_VSN = 2.2 APP_VSN = $(APPLICATION)-$(DIAMETER_VSN)$(PRE_VSN) diff --git a/lib/edoc/doc/src/notes.xml b/lib/edoc/doc/src/notes.xml index e818887eb8..145856bcaa 100644 --- a/lib/edoc/doc/src/notes.xml +++ b/lib/edoc/doc/src/notes.xml @@ -32,6 +32,22 @@ <p>This document describes the changes made to the EDoc application.</p> +<section><title>Edoc 0.10</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> Print a helpful message explaining that adding + <c>{preprocess, true}</c> can help if reading a source + file fails. </p> + <p> + Own Id: OTP-15605 Aux Id: ERL-841 </p> + </item> + </list> + </section> + +</section> + <section><title>Edoc 0.9.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/edoc/vsn.mk b/lib/edoc/vsn.mk index 0b3636f030..b6e1422623 100644 --- a/lib/edoc/vsn.mk +++ b/lib/edoc/vsn.mk @@ -1 +1 @@ -EDOC_VSN = 0.9.4 +EDOC_VSN = 0.10 diff --git a/lib/erl_docgen/doc/src/notes.xml b/lib/erl_docgen/doc/src/notes.xml index 97c842a324..54f0a36b27 100644 --- a/lib/erl_docgen/doc/src/notes.xml +++ b/lib/erl_docgen/doc/src/notes.xml @@ -31,7 +31,31 @@ </header> <p>This document describes the changes made to the <em>erl_docgen</em> application.</p> - <section><title>Erl_Docgen 0.8.1</title> + <section><title>Erl_Docgen 0.9</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + The HTML reference documentation shows the OTP version + where modules and functions were first introduced. + Versions older than R13B04 are not shown.</p> + <p> + Own Id: OTP-15460</p> + </item> + <item> + <p> + Make html documentation of C functions with many + arguments more readable.</p> + <p> + Own Id: OTP-15637 Aux Id: PR-2160 </p> + </item> + </list> + </section> + +</section> + +<section><title>Erl_Docgen 0.8.1</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/erl_docgen/priv/css/otp_doc.css b/lib/erl_docgen/priv/css/otp_doc.css index 89b278215c..17d9f8dd56 100644 --- a/lib/erl_docgen/priv/css/otp_doc.css +++ b/lib/erl_docgen/priv/css/otp_doc.css @@ -74,7 +74,7 @@ a:visited { color: #1b6ec2; text-decoration: none } /* Invisible table for function specs, * just to get since-version out in right margin */ -.func-table, .func-tr, .func-td, .func-since-td { +.func-table, .func-tr, .func-td, .cfunc-td, .func-since-td { width: 200%; border: 0; padding: 0; @@ -86,7 +86,13 @@ a:visited { color: #1b6ec2; text-decoration: none } } .func-td { - width: 50% + width: 50%; +} + +.cfunc-td { + width: 50%; + padding-left: 100px; + text-indent: -100px; } .func-since-td { diff --git a/lib/erl_docgen/priv/xsl/db_html.xsl b/lib/erl_docgen/priv/xsl/db_html.xsl index c5150d447c..c9be926e1e 100644 --- a/lib/erl_docgen/priv/xsl/db_html.xsl +++ b/lib/erl_docgen/priv/xsl/db_html.xsl @@ -2119,17 +2119,10 @@ <xsl:when test="ancestor::cref"> <table class="func-table"> <tr class="func-tr"> - <td class="func-td"> + <td class="cfunc-td"> <span class="bold_code bc-7"> <xsl:call-template name="title_link"> <xsl:with-param name="link" select="substring-before(nametext, '(')"/> - <xsl:with-param name="title"> - <xsl:value-of select="ret"/> - <xsl:call-template name="maybe-space-after-ret"> - <xsl:with-param name="s" select="ret"/> - </xsl:call-template> - <xsl:value-of select="nametext"/> - </xsl:with-param> </xsl:call-template> </span> </td> @@ -2199,18 +2192,6 @@ </xsl:template> - <xsl:template name="maybe-space-after-ret"> - <xsl:param name="s"/> - <xsl:variable name="last_char" - select="substring($s, string-length($s), 1)"/> - <xsl:choose> - <xsl:when test="$last_char != '*'"> - <xsl:text> </xsl:text> - </xsl:when> - </xsl:choose> - </xsl:template> - - <!-- Type --> <xsl:template match="type"> <xsl:param name="partnum"/> @@ -2264,7 +2245,7 @@ </xsl:template> <xsl:template name="title_link"> - <xsl:param name="title"/> + <xsl:param name="title" select="'APPLY'"/> <xsl:param name="link" select="erl:to-link(title)"/> <xsl:param name="ghlink" select="ancestor-or-self::*[@ghlink][position() = 1]/@ghlink"/> <xsl:variable name="id" select="concat(concat($link,'-'), generate-id(.))"/> @@ -2274,7 +2255,16 @@ <xsl:with-param name="id" select="$id"/> <xsl:with-param name="ghlink" select="$ghlink"/> </xsl:call-template> - <a class="title_link" name="{$link}" href="#{$link}"><xsl:value-of select="$title"/></a> + <a class="title_link" name="{$link}" href="#{$link}"> + <xsl:choose> + <xsl:when test="$title = 'APPLY'"> + <xsl:apply-templates/> <!-- like <ret> and <nametext> --> + </xsl:when> + <xsl:otherwise> + <xsl:value-of select="$title"/> + </xsl:otherwise> + </xsl:choose> + </a> </span> </xsl:template> @@ -2704,4 +2694,46 @@ <xsl:value-of select="normalize-space(.)"/> </xsl:template> + <xsl:template match="ret"> + <xsl:value-of select="."/> + <xsl:variable name="last_char" select="substring(., string-length(.), 1)"/> + <xsl:if test="$last_char != '*'"> + <xsl:text> </xsl:text> + </xsl:if> + </xsl:template> + + <xsl:template match="nametext"> + <xsl:value-of select="substring-before(.,'(')"/> + <xsl:text>(</xsl:text> + <xsl:variable name="arglist" select="substring-after(.,'(')"/> + <xsl:choose> + <xsl:when test="$arglist = ')' or $arglist = 'void)'"> + <xsl:value-of select="$arglist"/> + </xsl:when> + <xsl:otherwise> + <br/> + <xsl:call-template name="cfunc-arglist"> + <xsl:with-param name="text" select="$arglist"/> + </xsl:call-template> + </xsl:otherwise> + </xsl:choose> + </xsl:template> + + <!-- Format C function argument list with <br> after comma --> + <xsl:template name="cfunc-arglist"> + <xsl:param name="text"/> + <xsl:variable name="line" select="normalize-space($text)"/> + <xsl:choose> + <xsl:when test="contains($line,',')"> + <xsl:value-of select="substring-before($line,',')"/>,<br/> + <xsl:call-template name="cfunc-arglist"> + <xsl:with-param name="text" select="substring-after($line,',')"/> + </xsl:call-template> + </xsl:when> + <xsl:otherwise> + <xsl:value-of select="$line"/> + </xsl:otherwise> + </xsl:choose> + </xsl:template> + </xsl:stylesheet> diff --git a/lib/erl_docgen/vsn.mk b/lib/erl_docgen/vsn.mk index 3b2f6db6a1..6321f229dd 100644 --- a/lib/erl_docgen/vsn.mk +++ b/lib/erl_docgen/vsn.mk @@ -1 +1 @@ -ERL_DOCGEN_VSN = 0.8.1 +ERL_DOCGEN_VSN = 0.9 diff --git a/lib/erl_interface/configure.in b/lib/erl_interface/configure.in index 53c2f7069c..f0e9b2eb3f 100644 --- a/lib/erl_interface/configure.in +++ b/lib/erl_interface/configure.in @@ -61,7 +61,7 @@ fi TARGET=$host AC_SUBST(TARGET) -AC_CONFIG_HEADER([src/$host/config.h:config.h.in include/$host/ei_config.h:include/ei_config.h.in]) +AC_CONFIG_HEADER([src/$host/config.h:config.h.in]) dnl ---------------------------------------------------------------------- dnl Optional features @@ -184,14 +184,6 @@ AC_TRY_COMPILE([#include <sys/types.h> AC_DEFINE(HAVE_SOCKLEN_T, [], [Define if you have the `socklen_t' type])], [AC_MSG_RESULT(no)]) -AC_MSG_CHECKING([for deprecated attribute]) -AC_TRY_COMPILE([], -[void my_function(void) __attribute__((deprecated));], -[AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_DEPRECATED_ATTRIBUTE, [], [Define to 1 if you have the `deprecated' attribute])], -[AC_MSG_RESULT(no)]) - - # Checks for library functions. AC_FUNC_ALLOCA dnl AC_FUNC_FORK diff --git a/lib/erl_interface/doc/src/ei.xml b/lib/erl_interface/doc/src/ei.xml index 2bdb390644..f081ca926a 100644 --- a/lib/erl_interface/doc/src/ei.xml +++ b/lib/erl_interface/doc/src/ei.xml @@ -736,7 +736,7 @@ ei_encode_tuple_header(buf, &i, 0);</pre> </func> <func> - <name since="OTP @OTP-15442@"><ret>int</ret><nametext>ei_init(void)</nametext></name> + <name since="OTP 21.3"><ret>int</ret><nametext>ei_init(void)</nametext></name> <fsummary>Initialize the ei library.</fsummary> <desc> <p>Initialize the <c>ei</c> library. This function should be called once diff --git a/lib/erl_interface/doc/src/ei_connect.xml b/lib/erl_interface/doc/src/ei_connect.xml index df40973270..795f1249b3 100644 --- a/lib/erl_interface/doc/src/ei_connect.xml +++ b/lib/erl_interface/doc/src/ei_connect.xml @@ -412,7 +412,7 @@ typedef struct { </func> <func> - <name since="OTP @OTP-15442@"><ret>int</ret><nametext>ei_close_connection(int fd)</nametext></name> + <name since="OTP 21.3"><ret>int</ret><nametext>ei_close_connection(int fd)</nametext></name> <fsummary>Close a connection.</fsummary> <desc> <p>Closes a previously opened connection or listen socket.</p> @@ -472,9 +472,9 @@ fd = ei_xconnect(&ec, &addr, ALIVE); <func> <name since=""><ret>int</ret><nametext>ei_connect_init(ei_cnode* ec, const char* this_node_name, const char *cookie, short creation)</nametext></name> - <name since="OTP @OTP-15442@"><ret>int</ret><nametext>ei_connect_init_ussi(ei_cnode* ec, const char* this_node_name, const char *cookie, short creation, ei_socket_callbacks *cbs, int cbs_sz, void *setup_context)</nametext></name> + <name since="OTP 21.3"><ret>int</ret><nametext>ei_connect_init_ussi(ei_cnode* ec, const char* this_node_name, const char *cookie, short creation, ei_socket_callbacks *cbs, int cbs_sz, void *setup_context)</nametext></name> <name since=""><ret>int</ret><nametext>ei_connect_xinit(ei_cnode* ec, const char *thishostname, const char *thisalivename, const char *thisnodename, Erl_IpAddr thisipaddr, const char *cookie, short creation)</nametext></name> - <name since="OTP @OTP-15442@"><ret>int</ret><nametext>ei_connect_xinit_ussi(ei_cnode* ec, const char *thishostname, const char *thisalivename, const char *thisnodename, Erl_IpAddr thisipaddr, const char *cookie, short creation, ei_socket_callbacks *cbs, int cbs_sz, void *setup_context)</nametext></name> + <name since="OTP 21.3"><ret>int</ret><nametext>ei_connect_xinit_ussi(ei_cnode* ec, const char *thishostname, const char *thisalivename, const char *thisnodename, Erl_IpAddr thisipaddr, const char *cookie, short creation, ei_socket_callbacks *cbs, int cbs_sz, void *setup_context)</nametext></name> <fsummary>Initialize for a connection.</fsummary> <desc> <p>Initializes the <c>ec</c> structure, to @@ -595,8 +595,8 @@ if (ei_connect_init(&ec, "madonna", "cookie...", n++) < 0) { </func> <func> - <name since="OTP @OTP-15442@"><ret>int</ret><nametext>ei_listen(ei_cnode *ec, int *port, int backlog)</nametext></name> - <name since="OTP @OTP-15442@"><ret>int</ret><nametext>ei_xlisten(ei_cnode *ec, Erl_IpAddr adr, int *port, int backlog)</nametext></name> + <name since="OTP 21.3"><ret>int</ret><nametext>ei_listen(ei_cnode *ec, int *port, int backlog)</nametext></name> + <name since="OTP 21.3"><ret>int</ret><nametext>ei_xlisten(ei_cnode *ec, Erl_IpAddr adr, int *port, int backlog)</nametext></name> <fsummary>Create a listen socket.</fsummary> <desc> <p>Used by a server process to setup a listen socket which diff --git a/lib/erl_interface/doc/src/notes.xml b/lib/erl_interface/doc/src/notes.xml index 07ddd82718..32d28b853b 100644 --- a/lib/erl_interface/doc/src/notes.xml +++ b/lib/erl_interface/doc/src/notes.xml @@ -31,6 +31,43 @@ </header> <p>This document describes the changes made to the Erl_interface application.</p> +<section><title>Erl_Interface 3.11.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed two bugs in the <c>erl_call</c> program. A missing + initialization (introduced in <c>erl_interface-3.11</c>) + which either caused a crash or failure to connect to or + start a node, and an incorrectly calculated timeout which + could cause failure to start an erlang node. These bugs + only caused failures on some platforms.</p> + <p> + Own Id: OTP-15676 Aux Id: OTP-15442, ERL-881 </p> + </item> + </list> + </section> + +</section> + +<section><title>Erl_Interface 3.11</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Support for plugin of a <seealso + marker="ei_connect#ussi">user supplied socket + implementation</seealso> has been added.</p> + <p> + Own Id: OTP-15442 Aux Id: ERIERL-258 </p> + </item> + </list> + </section> + +</section> + <section><title>Erl_Interface 3.10.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/erl_interface/include/ei.h b/lib/erl_interface/include/ei.h index 10a6e5bf50..aa2a49098f 100644 --- a/lib/erl_interface/include/ei.h +++ b/lib/erl_interface/include/ei.h @@ -47,21 +47,26 @@ typedef LONG_PTR ssize_t; /* Sigh... */ # include <netdb.h> #endif -#ifndef EI_INCLUDED_CONFIG_H__ -#include "ei_config.h" +#ifdef __has_attribute +#if __has_attribute(deprecated) +#define EI_HAVE_DEPRECATED_ATTR__ 1 +#else +#undef EI_HAVE_DEPRECATED_ATTR__ +#endif #endif -#if defined(HAVE_DEPRECATED_ATTRIBUTE) && !defined(EI_NO_DEPR_WARN) -#define EI_HAVE_DEPRECATED_ATTR__ 1 +#ifdef EI_NO_DEPR_WARN +#undef EI_HAVE_DEPRECATED_ATTR__ +#endif + +#ifdef EI_HAVE_DEPRECATED_ATTR__ #define EI_DEPRECATED_ATTR_NAME deprecated #define EI_DEPRECATED_ATTR __attribute__((EI_DEPRECATED_ATTR_NAME)) #else -#undef EI_HAVE_DEPRECATED_ATTR__ #define EI_DEPRECATED_ATTR_NAME #define EI_DEPRECATED_ATTR #endif - /* -------------------------------------------------------------------- */ /* Defines part of API */ /* -------------------------------------------------------------------- */ diff --git a/lib/erl_interface/include/ei_config.h.in b/lib/erl_interface/include/ei_config.h.in deleted file mode 100644 index dfcf676bed..0000000000 --- a/lib/erl_interface/include/ei_config.h.in +++ /dev/null @@ -1,3 +0,0 @@ - -/* Define to 1 if you have the `deprecated' attribute */ -#undef HAVE_DEPRECATED_ATTRIBUTE diff --git a/lib/erl_interface/src/Makefile b/lib/erl_interface/src/Makefile index 800557fbfe..00c49f1622 100644 --- a/lib/erl_interface/src/Makefile +++ b/lib/erl_interface/src/Makefile @@ -27,7 +27,9 @@ include $(ERL_TOP)/make/output.mk include $(ERL_TOP)/make/target.mk debug opt shared purify quantify purecov gcov: +ifndef TERTIARY_BOOTSTRAP $(make_verbose)$(MAKE) -f $(TARGET)/Makefile TYPE=$@ +endif clean depend docs release release_docs tests release_tests check xmllint: $(make_verbose)$(MAKE) -f $(TARGET)/Makefile $@ diff --git a/lib/erl_interface/src/Makefile.in b/lib/erl_interface/src/Makefile.in index 1b19a85f1e..b2600f0fab 100644 --- a/lib/erl_interface/src/Makefile.in +++ b/lib/erl_interface/src/Makefile.in @@ -118,7 +118,7 @@ else AR_FLAGS=rcv endif -INCFLAGS = -I. -I../include -I../include/$(TARGET) -Iconnect -Iencode -Idecode -Imisc -Iepmd \ +INCFLAGS = -I. -I../include -Iconnect -Iencode -Idecode -Imisc -Iepmd \ -Iregistry -I$(TARGET) ifeq ($(USING_VC),yes) @@ -322,8 +322,7 @@ HEADERS = \ ../include/ei.h \ ../include/ei_connect.h \ ../include/eicode.h \ - ../include/erl_interface.h \ - ../include/$(TARGET)/ei_config.h + ../include/erl_interface.h EISOURCES = \ $(CONNECTSRC) \ diff --git a/lib/erl_interface/src/misc/eidef.h b/lib/erl_interface/src/misc/eidef.h index 083814c6e9..f38824d826 100644 --- a/lib/erl_interface/src/misc/eidef.h +++ b/lib/erl_interface/src/misc/eidef.h @@ -25,7 +25,6 @@ /* Common definitions used in ei user interface */ -#define EI_INCLUDED_CONFIG_H__ #include "config.h" /* Central include of config.h */ /* vxWorks.h needs to be before stddef.h */ diff --git a/lib/erl_interface/src/prog/ei_fake_prog.c b/lib/erl_interface/src/prog/ei_fake_prog.c index c7a16dc7c4..158464b385 100644 --- a/lib/erl_interface/src/prog/ei_fake_prog.c +++ b/lib/erl_interface/src/prog/ei_fake_prog.c @@ -98,11 +98,18 @@ int main(void) EI_ULONGLONG ulonglongx = 0; #endif erlang_char_encoding enc; + ei_socket_callbacks cbs; intx = erl_errno; + ei_init(); + + ei_close_connection(intx); + ei_connect_init(&xec, charp, charp, creation); + ei_connect_init_ussi(&xec, charp, charp, creation, &cbs, sizeof(cbs), NULL); ei_connect_xinit (&xec, charp, charp, charp, thisipaddr, charp, creation); + ei_connect_xinit_ussi(&xec, charp, charp, charp, thisipaddr, charp, creation, &cbs, sizeof(cbs), NULL); ei_connect(&xec, charp); ei_xconnect (&xec, thisipaddr, charp); @@ -121,6 +128,8 @@ int main(void) ei_publish(&xec, intx); ei_accept(&xec, intx, &conp); ei_unpublish(&xec); + ei_listen(&xec, intp, intx); + ei_xlisten(&xec, thisipaddr, intp, intx); ei_thisnodename(&xec); ei_thishostname(&xec); @@ -187,7 +196,7 @@ int main(void) ei_decode_char(charp, intp, charp); ei_decode_string(charp, intp, charp); ei_decode_atom(charp, intp, charp); - ei_decode_atom_as(charp, intp, charp, MAXATOMLEN_UTF8, ERLANG_WHATEVER, &enc, &enc); + ei_decode_atom_as(charp, intp, charp, MAXATOMLEN_UTF8, ERLANG_UTF8, &enc, &enc); ei_decode_binary(charp, intp, (void *)0, longp); ei_decode_fun(charp, intp, &efun); free_fun(&efun); diff --git a/lib/erl_interface/src/prog/erl_call.c b/lib/erl_interface/src/prog/erl_call.c index 52ad6885e8..ab91157035 100644 --- a/lib/erl_interface/src/prog/erl_call.c +++ b/lib/erl_interface/src/prog/erl_call.c @@ -88,10 +88,6 @@ #include "ei_resolve.h" #include "erl_start.h" /* FIXME remove dependency */ -#ifdef __WIN32__ -static void initWinSock(void); -#endif - /* * Some nice global variables * (I don't think "nice" is the right word actually... -gordon) @@ -157,6 +153,8 @@ int erl_call(int argc, char **argv) char* progname = argv[0]; ei_cnode ec; + ei_init(); + /* Get the command line options */ while (i < argc) { if (argv[i][0] != '-') { @@ -317,14 +315,6 @@ int erl_call(int argc, char **argv) struct in_addr h_ipadr; char* ct; -#ifdef __WIN32__ - /* - * FIXME Extremly ugly, but needed to get ei_gethostbyname() below - * to work. - */ - initWinSock(); -#endif - /* gethostname requires len to be max(hostname) + 1 */ if (gethostname(h_hostname, EI_MAXHOSTNAMELEN+1) < 0) { fprintf(stderr,"erl_call: failed to get host name: %d\n", errno); @@ -857,46 +847,6 @@ static void usage(const char *progname) { exit(0); } - -/*************************************************************************** - * - * OS specific functions - * - ***************************************************************************/ - -#ifdef __WIN32__ -/* - * FIXME This should not be here. This is a quick fix to make erl_call - * work at all on Windows NT. - */ -static void initWinSock(void) -{ - WORD wVersionRequested; - WSADATA wsaData; - int err; - static int initialized; - - wVersionRequested = MAKEWORD(1, 1); - if (!initialized) { - initialized = 1; - err = WSAStartup(wVersionRequested, &wsaData); - - if (err != 0) { - fprintf(stderr,"erl_call: " - "Can't initialize windows sockets: %d\n", err); - } - - if ( LOBYTE( wsaData.wVersion ) != 1 || - HIBYTE( wsaData.wVersion ) != 1 ) { - fprintf(stderr,"erl_call: This version of " - "windows sockets not supported\n"); - WSACleanup(); - } - } -} -#endif - - /*************************************************************************** * * Utility functions diff --git a/lib/erl_interface/src/prog/erl_start.c b/lib/erl_interface/src/prog/erl_start.c index ba495ac818..b7aa451946 100644 --- a/lib/erl_interface/src/prog/erl_start.c +++ b/lib/erl_interface/src/prog/erl_start.c @@ -657,7 +657,7 @@ static int wait_for_erlang(int sockd, int magic, struct timeval *timeout) gettimeofday(&now,NULL); to.tv_sec = stop_time.tv_sec - now.tv_sec; to.tv_usec = stop_time.tv_usec - now.tv_usec; - while ((to.tv_usec <= 0) && (to.tv_sec >= 0)) { + while ((to.tv_usec < 0) && (to.tv_sec > 0)) { to.tv_usec += 1000000; to.tv_sec--; } diff --git a/lib/erl_interface/test/Makefile b/lib/erl_interface/test/Makefile index 94f4b422d6..f8f2ef0156 100644 --- a/lib/erl_interface/test/Makefile +++ b/lib/erl_interface/test/Makefile @@ -33,6 +33,7 @@ MODULES= \ ei_format_SUITE \ ei_print_SUITE \ ei_tmo_SUITE \ + erl_call_SUITE \ erl_connect_SUITE \ erl_global_SUITE \ erl_eterm_SUITE \ diff --git a/lib/erl_interface/test/all_SUITE_data/Makefile.src b/lib/erl_interface/test/all_SUITE_data/Makefile.src index 57e522fd3e..4f27b097c8 100644 --- a/lib/erl_interface/test/all_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/all_SUITE_data/Makefile.src @@ -21,8 +21,8 @@ include @erl_interface_mk_include@ CC0 = @CC@ CC = .@DS@gccifier@exe@ -CC"$(CC0)" -CFLAGS0 = @CFLAGS@ -I@erl_interface_include@ @erl_interface_target_include@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ +CFLAGS0 = @CFLAGS@ -I@erl_interface_include@ +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ EI_COMMON_OBJS = runner@obj@ ei_runner@obj@ ALL_OBJS = gccifier@exe@ $(EI_COMMON_OBJS) diff --git a/lib/erl_interface/test/ei_accept_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_accept_SUITE_data/Makefile.src index 76dc84221f..10ef437f8b 100644 --- a/lib/erl_interface/test/ei_accept_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_accept_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_ACCEPT_OBJS = ei_accept_test@obj@ ei_accept_test_decl@obj@ EIACCNODE_OBJS = eiaccnode@obj@ diff --git a/lib/erl_interface/test/ei_connect_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_connect_SUITE_data/Makefile.src index d1694e607d..c2d8261dd8 100644 --- a/lib/erl_interface/test/ei_connect_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_connect_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBERL) $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_CONNECT_OBJS = ei_connect_test@obj@ ei_connect_test_decl@obj@ EINODE_OBJS = einode@obj@ diff --git a/lib/erl_interface/test/ei_decode_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_decode_SUITE_data/Makefile.src index 76591d893c..e678914a40 100644 --- a/lib/erl_interface/test/ei_decode_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_decode_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_DECODE_OBJS = ei_decode_test@obj@ ei_decode_test_decl@obj@ all: ei_decode_test@exe@ diff --git a/lib/erl_interface/test/ei_decode_encode_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_decode_encode_SUITE_data/Makefile.src index 3f5fa4f295..853fe9ddeb 100644 --- a/lib/erl_interface/test/ei_decode_encode_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_decode_encode_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_DECODE_ENCODE_OBJS = ei_decode_encode_test@obj@ ei_decode_encode_test_decl@obj@ all: ei_decode_encode_test@exe@ diff --git a/lib/erl_interface/test/ei_encode_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_encode_SUITE_data/Makefile.src index 489382d85e..3b2cab7af4 100644 --- a/lib/erl_interface/test/ei_encode_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_encode_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_ENCODE_OBJS = ei_encode_test@obj@ ei_encode_test_decl@obj@ all: ei_encode_test@exe@ diff --git a/lib/erl_interface/test/ei_format_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_format_SUITE_data/Makefile.src index 9e5a271db6..b89dcae45a 100644 --- a/lib/erl_interface/test/ei_format_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_format_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_FORMAT_OBJS = ei_format_test@obj@ ei_format_test_decl@obj@ all: ei_format_test@exe@ diff --git a/lib/erl_interface/test/ei_print_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_print_SUITE_data/Makefile.src index 354011f1a5..150c11b99c 100644 --- a/lib/erl_interface/test/ei_print_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_print_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_PRINT_OBJS = ei_print_test@obj@ ei_print_test_decl@obj@ all: ei_print_test@exe@ diff --git a/lib/erl_interface/test/ei_tmo_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_tmo_SUITE_data/Makefile.src index 76a9c6a606..b4ee361939 100644 --- a/lib/erl_interface/test/ei_tmo_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_tmo_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_TMO_OBJS = ei_tmo_test@obj@ ei_tmo_test_decl@obj@ all: ei_tmo_test@exe@ diff --git a/lib/erl_interface/test/erl_call_SUITE.erl b/lib/erl_interface/test/erl_call_SUITE.erl new file mode 100644 index 0000000000..9e2b2e4251 --- /dev/null +++ b/lib/erl_interface/test/erl_call_SUITE.erl @@ -0,0 +1,96 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +-module(erl_call_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([all/0, smoke/1]). + +all() -> + [smoke]. + +smoke(Config) when is_list(Config) -> + ErlCall = find_erl_call(), + NameSwitch = case net_kernel:longnames() of + true -> + "-name"; + false -> + "-sname" + end, + Name = atom_to_list(?MODULE) + ++ "-" + ++ integer_to_list(erlang:system_time(microsecond)), + + ArgsList = ["-s", "-a", "erlang node", NameSwitch, Name], + io:format("erl_call: \"~ts\"\n~nargs list: ~p~n", [ErlCall, ArgsList]), + CmdRes = get_smoke_port_res(open_port({spawn_executable, ErlCall}, + [{args, ArgsList}, eof]), []), + io:format("CmdRes: ~p~n", [CmdRes]), + + [_, Hostname] = string:lexemes(atom_to_list(node()), "@"), + NodeName = list_to_atom(Name ++ "@" ++ Hostname), + io:format("NodeName: ~p~n~n", [NodeName]), + + pong = net_adm:ping(NodeName), + rpc:cast(NodeName, erlang, halt, []), + NodeName = list_to_atom(string:trim(CmdRes, both, "'")), + ok. + +% +% Utility functions... +% + +find_erl_call() -> + ErlCallName = case os:type() of + {win32, _} -> "erl_call.exe"; + _ -> "erl_call" + end, + LibDir = code:lib_dir(erl_interface), + InstalledErlCall = filename:join([LibDir, "bin", ErlCallName]), + TargetDir = erlang:system_info(system_architecture), + TargetErlCall = filename:join([LibDir, "bin", TargetDir, ErlCallName]), + + try + lists:foreach(fun (F) -> + io:format("Checking: \"~ts\"~n", [F]), + case file:read_file_info(F) of + {ok, _} -> + throw(F); + _ -> + ok + end + end, + [InstalledErlCall, TargetErlCall]), + exit({missing, erl_call}) + catch + throw:ErlCall -> + ErlCall + end. + +get_smoke_port_res(Port, Acc) when is_port(Port) -> + receive + {Port, {data, Data}} -> + get_smoke_port_res(Port, [Acc|Data]); + {Port, eof} -> + lists:flatten(Acc) + end. + diff --git a/lib/erl_interface/test/erl_connect_SUITE_data/Makefile.src b/lib/erl_interface/test/erl_connect_SUITE_data/Makefile.src index 19b076794a..ff4c382c97 100644 --- a/lib/erl_interface/test/erl_connect_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/erl_connect_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/runner@obj@ \ $(LIBERL) $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data OBJS = erl_connect_test@obj@ erl_connect_test_decl@obj@ all: erl_connect_test@exe@ diff --git a/lib/erl_interface/test/erl_eterm_SUITE_data/Makefile.src b/lib/erl_interface/test/erl_eterm_SUITE_data/Makefile.src index cd4de56d7c..4b1ddf77b6 100644 --- a/lib/erl_interface/test/erl_eterm_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/erl_eterm_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/runner@obj@ \ $(LIBERL) $(LIBEI) @erl_interface_sock_libs@ @LIBS@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data ETERM_OBJS = eterm_test@obj@ eterm_test_decl@obj@ CNODE_OBJS = cnode@obj@ PRINT_OBJS = print_term@obj@ diff --git a/lib/erl_interface/test/erl_ext_SUITE_data/Makefile.src b/lib/erl_interface/test/erl_ext_SUITE_data/Makefile.src index 50b60637cd..fe8caebbd6 100644 --- a/lib/erl_interface/test/erl_ext_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/erl_ext_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/runner@obj@ \ $(LIBERL) $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EXT_OBJS = ext_test@obj@ ext_test_decl@obj@ all: ext_test@exe@ diff --git a/lib/erl_interface/test/erl_format_SUITE_data/Makefile.src b/lib/erl_interface/test/erl_format_SUITE_data/Makefile.src index 7d51cd6007..2ba59ab651 100644 --- a/lib/erl_interface/test/erl_format_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/erl_format_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/runner@obj@ \ $(LIBERL) $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data FORMAT_OBJS = format_test@obj@ format_test_decl@obj@ all: format_test@exe@ diff --git a/lib/erl_interface/test/erl_global_SUITE_data/Makefile.src b/lib/erl_interface/test/erl_global_SUITE_data/Makefile.src index 9f2a8619ac..1c1530d1b6 100644 --- a/lib/erl_interface/test/erl_global_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/erl_global_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/runner@obj@ \ $(LIBERL) $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data OBJS = erl_global_test@obj@ erl_global_test_decl@obj@ all: erl_global_test@exe@ diff --git a/lib/erl_interface/test/erl_match_SUITE_data/Makefile.src b/lib/erl_interface/test/erl_match_SUITE_data/Makefile.src index 56ed1df203..156214a269 100644 --- a/lib/erl_interface/test/erl_match_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/erl_match_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/runner@obj@ \ $(LIBERL) $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data MATCH_OBJS = match_test@obj@ match_test_decl@obj@ all: match_test@exe@ diff --git a/lib/erl_interface/test/port_call_SUITE_data/Makefile.src b/lib/erl_interface/test/port_call_SUITE_data/Makefile.src index e09e0fe175..0f97ce9f70 100644 --- a/lib/erl_interface/test/port_call_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/port_call_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBERL = @erl_interface_lib_drv@ LIBEI = @erl_interface_eilib_drv@ SHLIB_EXTRA_LDLIBS = $(LIBERL) $(LIBEI) @erl_interface_threadlib@ -SHLIB_EXTRA_CFLAGS = -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +SHLIB_EXTRA_CFLAGS = -I@erl_interface_include@ -I../all_SUITE_data all: port_call_drv@dll@ diff --git a/lib/erl_interface/vsn.mk b/lib/erl_interface/vsn.mk index 06ef907d6c..dae6052d55 100644 --- a/lib/erl_interface/vsn.mk +++ b/lib/erl_interface/vsn.mk @@ -1,2 +1,2 @@ -EI_VSN = 3.10.4 +EI_VSN = 3.11.1 ERL_INTERFACE_VSN = $(EI_VSN) diff --git a/lib/ftp/doc/src/notes.xml b/lib/ftp/doc/src/notes.xml index 01c1f88cf1..61da079900 100644 --- a/lib/ftp/doc/src/notes.xml +++ b/lib/ftp/doc/src/notes.xml @@ -33,7 +33,23 @@ <file>notes.xml</file> </header> - <section><title>Ftp 1.0.1</title> + <section><title>Ftp 1.0.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed timing related bug that could make ftp functions + behave badly.</p> + <p> + Own Id: OTP-15659 Aux Id: ERIERL-316 </p> + </item> + </list> + </section> + +</section> + +<section><title>Ftp 1.0.1</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/ftp/src/ftp.erl b/lib/ftp/src/ftp.erl index 40f6b53fa3..18cd8c7524 100644 --- a/lib/ftp/src/ftp.erl +++ b/lib/ftp/src/ftp.erl @@ -1065,9 +1065,9 @@ handle_call({_, {open, ip_comm, Opts, {CtrlOpts, DataPassOpts, DataActOpts}}}, F end end; -handle_call({_, {open, tls_upgrade, TLSOptions}}, From, State) -> - _ = send_ctrl_message(State, mk_cmd("AUTH TLS", [])), - activate_ctrl_connection(State), +handle_call({_, {open, tls_upgrade, TLSOptions}}, From, State0) -> + _ = send_ctrl_message(State0, mk_cmd("AUTH TLS", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From, caller = open, tls_options = TLSOptions}}; handle_call({_, {user, User, Password}}, From, @@ -1081,17 +1081,17 @@ handle_call({_, {user, User, Password, Acc}}, From, handle_call({_, {account, Acc}}, From, State)-> handle_user_account(Acc, State#state{client = From}); -handle_call({_, pwd}, From, #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("PWD", [])), - activate_ctrl_connection(State), +handle_call({_, pwd}, From, #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("PWD", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From, caller = pwd}}; handle_call({_, lpwd}, From, #state{ldir = LDir} = State) -> {reply, {ok, LDir}, State#state{client = From}}; -handle_call({_, {cd, Dir}}, From, #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("CWD ~s", [Dir])), - activate_ctrl_connection(State), +handle_call({_, {cd, Dir}}, From, #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("CWD ~s", [Dir])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From, caller = cd}}; handle_call({_,{lcd, Dir}}, _From, #state{ldir = LDir0} = State) -> @@ -1108,41 +1108,41 @@ handle_call({_, {dir, Len, Dir}}, {_Pid, _} = From, setup_data_connection(State#state{caller = {dir, Dir, Len}, client = From}); handle_call({_, {rename, CurrFile, NewFile}}, From, - #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("RNFR ~s", [CurrFile])), - activate_ctrl_connection(State), + #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("RNFR ~s", [CurrFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {rename, NewFile}, client = From}}; handle_call({_, {delete, File}}, {_Pid, _} = From, - #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("DELE ~s", [File])), - activate_ctrl_connection(State), + #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("DELE ~s", [File])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From}}; -handle_call({_, {mkdir, Dir}}, From, #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("MKD ~s", [Dir])), - activate_ctrl_connection(State), +handle_call({_, {mkdir, Dir}}, From, #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("MKD ~s", [Dir])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From}}; -handle_call({_,{rmdir, Dir}}, From, #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("RMD ~s", [Dir])), - activate_ctrl_connection(State), +handle_call({_,{rmdir, Dir}}, From, #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("RMD ~s", [Dir])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From}}; -handle_call({_,{type, Type}}, From, #state{chunk = false} = State) -> +handle_call({_,{type, Type}}, From, #state{chunk = false} = State0) -> case Type of ascii -> - _ = send_ctrl_message(State, mk_cmd("TYPE A", [])), - activate_ctrl_connection(State), + _ = send_ctrl_message(State0, mk_cmd("TYPE A", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = type, type = ascii, client = From}}; binary -> - _ = send_ctrl_message(State, mk_cmd("TYPE I", [])), - activate_ctrl_connection(State), + _ = send_ctrl_message(State0, mk_cmd("TYPE I", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = type, type = binary, client = From}}; _ -> - {reply, {error, etype}, State} + {reply, {error, etype}, State0} end; handle_call({_,{recv, RemoteFile, LocalFile}}, From, @@ -1181,8 +1181,8 @@ handle_call({_, recv_chunk}, _From, #state{chunk = true, } = State0) -> %% The ftp:recv_chunk call was the last event we waited for, finnish and clean up ?DBG("recv_chunk_closing ftp:recv_chunk, last event",[]), - activate_ctrl_connection(State0), - {reply, ok, State0#state{caller = undefined, + State = activate_ctrl_connection(State0), + {reply, ok, State#state{caller = undefined, chunk = false, client = undefined}}; @@ -1238,18 +1238,18 @@ handle_call({_, {transfer_chunk, Bin}}, _, #state{chunk = true} = State) -> handle_call({_, {transfer_chunk, _}}, _, #state{chunk = false} = State) -> {reply, {error, echunk}, State}; -handle_call({_, chunk_end}, From, #state{chunk = true} = State) -> - close_data_connection(State), - activate_ctrl_connection(State), +handle_call({_, chunk_end}, From, #state{chunk = true} = State0) -> + close_data_connection(State0), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From, dsock = undefined, caller = end_chunk_transfer, chunk = false}}; handle_call({_, chunk_end}, _, #state{chunk = false} = State) -> {reply, {error, echunk}, State}; -handle_call({_, {quote, Cmd}}, From, #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd(Cmd, [])), - activate_ctrl_connection(State), +handle_call({_, {quote, Cmd}}, From, #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd(Cmd, [])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From, caller = quote}}; handle_call({_, _Req}, _From, #state{csock = CSock} = State) @@ -1325,38 +1325,38 @@ handle_info({Trpt, Socket, Data}, #state{dsock = {Trpt,Socket}} = State0) when T Data/binary>>}}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, - caller = {recv_file, Fd}} = State) + caller = {recv_file, Fd}} = State0) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> file_close(Fd), - progress_report({transfer_size, 0}, State), - activate_ctrl_connection(State), + progress_report({transfer_size, 0}, State0), + State = activate_ctrl_connection(State0), ?DBG("Data channel close",[]), {noreply, State#state{dsock = undefined, data = <<>>}}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, client = Client, - caller = recv_chunk} = State) + caller = recv_chunk} = State0) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> ?DBG("Data channel close recv_chunk",[]), - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{dsock = undefined, caller = #recv_chunk_closing{dconn_closed = true, client_called_us = Client =/= undefined} }}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, caller = recv_bin, - data = Data} = State) + data = Data} = State0) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> ?DBG("Data channel close",[]), - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{dsock = undefined, data = <<>>, caller = {recv_bin, Data}}}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, data = Data, - caller = {handle_dir_result, Dir}} - = State) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> + caller = {handle_dir_result, Dir}} + = State0) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> ?DBG("Data channel close",[]), - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{dsock = undefined, caller = {handle_dir_result, Dir, Data}, % data = <<?CR,?LF>>}}; @@ -1377,7 +1377,7 @@ handle_info({Transport, Socket, Data}, #state{csock = {Transport, Socket}, client = From, ctrl_data = {CtrlData, AccLines, LineStatus}} - = State) -> + = State0) -> ?DBG('--ctrl ~p ----> ~s~p~n',[Socket,<<CtrlData/binary, Data/binary>>,State]), case ftp_response:parse_lines(<<CtrlData/binary, Data/binary>>, AccLines, LineStatus) of @@ -1387,21 +1387,21 @@ handle_info({Transport, Socket, Data}, #state{csock = {Transport, Socket}, case Caller of quote -> gen_server:reply(From, string:tokens(Lines, [?CR, ?LF])), - {noreply, State#state{client = undefined, - caller = undefined, - latest_ctrl_response = Lines, - ctrl_data = {NextMsgData, [], - start}}}; + {noreply, State0#state{client = undefined, + caller = undefined, + latest_ctrl_response = Lines, + ctrl_data = {NextMsgData, [], + start}}}; _ -> ?DBG(' ...handle_ctrl_result(~p,...) ctrl_data=~p~n',[CtrlResult,{NextMsgData, [], start}]), handle_ctrl_result(CtrlResult, - State#state{latest_ctrl_response = Lines, - ctrl_data = - {NextMsgData, [], start}}) + State0#state{latest_ctrl_response = Lines, + ctrl_data = + {NextMsgData, [], start}}) end; {continue, NewCtrlData} -> ?DBG(' ...Continue... ctrl_data=~p~n',[NewCtrlData]), - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{ctrl_data = NewCtrlData}} end; @@ -1567,19 +1567,19 @@ start_link(Opts, GenServerOptions) -> %%% Help functions to handle_call and/or handle_ctrl_result %%-------------------------------------------------------------------------- %% User handling -handle_user(User, Password, Acc, State) -> - _ = send_ctrl_message(State, mk_cmd("USER ~s", [User])), - activate_ctrl_connection(State), +handle_user(User, Password, Acc, State0) -> + _ = send_ctrl_message(State0, mk_cmd("USER ~s", [User])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {handle_user, Password, Acc}}}. -handle_user_passwd(Password, Acc, State) -> - _ = send_ctrl_message(State, mk_cmd("PASS ~s", [Password])), - activate_ctrl_connection(State), +handle_user_passwd(Password, Acc, State0) -> + _ = send_ctrl_message(State0, mk_cmd("PASS ~s", [Password])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {handle_user_passwd, Acc}}}. -handle_user_account(Acc, State) -> - _ = send_ctrl_message(State, mk_cmd("ACCT ~s", [Acc])), - activate_ctrl_connection(State), +handle_user_account(Acc, State0) -> + _ = send_ctrl_message(State0, mk_cmd("ACCT ~s", [Acc])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = handle_user_account}}. @@ -1594,9 +1594,9 @@ handle_ctrl_result({tls_upgrade, _}, #state{csock = {tcp, Socket}, ?DBG('<--ctrl ssl:connect(~p, ~p)~n~p~n',[Socket,TLSOptions,State0]), case ssl:connect(Socket, TLSOptions, Timeout) of {ok, TLSSocket} -> - State = State0#state{csock = {ssl,TLSSocket}}, - _ = send_ctrl_message(State, mk_cmd("PBSZ 0", [])), - activate_ctrl_connection(State), + State1 = State0#state{csock = {ssl,TLSSocket}}, + _ = send_ctrl_message(State1, mk_cmd("PBSZ 0", [])), + State = activate_ctrl_connection(State1), {noreply, State#state{tls_upgrading_data_connection = {true, pbsz}} }; {error, _} = Error -> gen_server:reply(From, {Error, self()}), @@ -1605,9 +1605,9 @@ handle_ctrl_result({tls_upgrade, _}, #state{csock = {tcp, Socket}, tls_upgrading_data_connection = false}} end; -handle_ctrl_result({pos_compl, _}, #state{tls_upgrading_data_connection = {true, pbsz}} = State) -> - _ = send_ctrl_message(State, mk_cmd("PROT P", [])), - activate_ctrl_connection(State), +handle_ctrl_result({pos_compl, _}, #state{tls_upgrading_data_connection = {true, pbsz}} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("PROT P", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{tls_upgrading_data_connection = {true, prot}}}; handle_ctrl_result({pos_compl, _}, #state{tls_upgrading_data_connection = {true, prot}, @@ -1801,10 +1801,10 @@ handle_ctrl_result({pos_compl, _}, #state{caller = {handle_dir_result, Dir, handle_ctrl_result({pos_compl, Lines}, #state{caller = {handle_dir_data, Dir, DirData}} = - State) -> + State0) -> OldDir = pwd_result(Lines), - _ = send_ctrl_message(State, mk_cmd("CWD ~s", [Dir])), - activate_ctrl_connection(State), + _ = send_ctrl_message(State0, mk_cmd("CWD ~s", [Dir])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {handle_dir_data_second_phase, OldDir, DirData}}}; handle_ctrl_result({Status, _}, @@ -1818,9 +1818,9 @@ handle_ctrl_result(S={_Status, _}, handle_ctrl_result({pos_compl, _}, #state{caller = {handle_dir_data_second_phase, OldDir, - DirData}} = State) -> - _ = send_ctrl_message(State, mk_cmd("CWD ~s", [OldDir])), - activate_ctrl_connection(State), + DirData}} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("CWD ~s", [OldDir])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {handle_dir_data_third_phase, DirData}}}; handle_ctrl_result({Status, _}, #state{caller = {handle_dir_data_second_phase, _, _}} @@ -1840,9 +1840,9 @@ handle_ctrl_result(Status={epath, _}, #state{caller = {dir,_}} = State) -> %%-------------------------------------------------------------------------- %% File renaming handle_ctrl_result({pos_interm, _}, #state{caller = {rename, NewFile}} - = State) -> - _ = send_ctrl_message(State, mk_cmd("RNTO ~s", [NewFile])), - activate_ctrl_connection(State), + = State0) -> + _ = send_ctrl_message(State0, mk_cmd("RNTO ~s", [NewFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = rename_second_phase}}; handle_ctrl_result({Status, _}, @@ -1916,10 +1916,10 @@ handle_ctrl_result({pos_compl, _}, #state{client = From, %% The pos_compl was the last event we waited for, finnish and clean up ?DBG("recv_chunk_closing pos_compl, last event",[]), gen_server:reply(From, ok), - activate_ctrl_connection(State0), - {noreply, State0#state{caller = undefined, - chunk = false, - client = undefined}}; + State = activate_ctrl_connection(State0), + {noreply, State#state{caller = undefined, + chunk = false, + client = undefined}}; handle_ctrl_result({pos_compl, _}, #state{caller = #recv_chunk_closing{}=R} = State0) -> @@ -2013,71 +2013,71 @@ ctrl_result_response(_, #state{client = From} = State, ErrorMsg) -> {noreply, State#state{client = undefined, caller = undefined}}. %%-------------------------------------------------------------------------- -handle_caller(#state{caller = {dir, Dir, Len}} = State) -> +handle_caller(#state{caller = {dir, Dir, Len}} = State0) -> Cmd = case Len of short -> "NLST"; long -> "LIST" end, _ = case Dir of "" -> - send_ctrl_message(State, mk_cmd(Cmd, "")); + send_ctrl_message(State0, mk_cmd(Cmd, "")); _ -> - send_ctrl_message(State, mk_cmd(Cmd ++ " ~s", [Dir])) + send_ctrl_message(State0, mk_cmd(Cmd ++ " ~s", [Dir])) end, - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {dir, Dir}}}; -handle_caller(#state{caller = {recv_bin, RemoteFile}} = State) -> - _ = send_ctrl_message(State, mk_cmd("RETR ~s", [RemoteFile])), - activate_ctrl_connection(State), +handle_caller(#state{caller = {recv_bin, RemoteFile}} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("RETR ~s", [RemoteFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = recv_bin}}; handle_caller(#state{caller = {start_chunk_transfer, Cmd, RemoteFile}} = - State) -> - _ = send_ctrl_message(State, mk_cmd("~s ~s", [Cmd, RemoteFile])), - activate_ctrl_connection(State), + State0) -> + _ = send_ctrl_message(State0, mk_cmd("~s ~s", [Cmd, RemoteFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = start_chunk_transfer}}; -handle_caller(#state{caller = {recv_file, RemoteFile, Fd}} = State) -> - _ = send_ctrl_message(State, mk_cmd("RETR ~s", [RemoteFile])), - activate_ctrl_connection(State), +handle_caller(#state{caller = {recv_file, RemoteFile, Fd}} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("RETR ~s", [RemoteFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {recv_file, Fd}}}; handle_caller(#state{caller = {transfer_file, {Cmd, LocalFile, RemoteFile}}, - ldir = LocalDir, client = From} = State) -> + ldir = LocalDir, client = From} = State0) -> case file_open(filename:absname(LocalFile, LocalDir), read) of {ok, Fd} -> - _ = send_ctrl_message(State, mk_cmd("~s ~s", [Cmd, RemoteFile])), - activate_ctrl_connection(State), + _ = send_ctrl_message(State0, mk_cmd("~s ~s", [Cmd, RemoteFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {transfer_file, Fd}}}; {error, _} -> gen_server:reply(From, {error, epath}), - {noreply, State#state{client = undefined, caller = undefined, - dsock = undefined}} + {noreply, State0#state{client = undefined, caller = undefined, + dsock = undefined}} end; handle_caller(#state{caller = {transfer_data, {Cmd, Bin, RemoteFile}}} = - State) -> - _ = send_ctrl_message(State, mk_cmd("~s ~s", [Cmd, RemoteFile])), - activate_ctrl_connection(State), + State0) -> + _ = send_ctrl_message(State0, mk_cmd("~s ~s", [Cmd, RemoteFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {transfer_data, Bin}}}. %% ----------- FTP SERVER COMMUNICATION ------------------------- %% Connect to FTP server at Host (default is TCP port 21) %% in order to establish a control connection. -setup_ctrl_connection(Host, Port, Timeout, #state{sockopts_ctrl = SockOpts} = State) -> +setup_ctrl_connection(Host, Port, Timeout, #state{sockopts_ctrl = SockOpts} = State0) -> MsTime = erlang:monotonic_time(), - case connect(Host, Port, SockOpts, Timeout, State) of + case connect(Host, Port, SockOpts, Timeout, State0) of {ok, IpFam, CSock} -> - NewState = State#state{csock = {tcp, CSock}, ipfamily = IpFam}, - activate_ctrl_connection(NewState), + State1 = State0#state{csock = {tcp, CSock}, ipfamily = IpFam}, + State = activate_ctrl_connection(State1), case Timeout - millisec_passed(MsTime) of Timeout2 when (Timeout2 >= 0) -> - {ok, NewState#state{caller = open}, Timeout2}; + {ok, State#state{caller = open}, Timeout2}; _ -> %% Oups: Simulate timeout - {ok, NewState#state{caller = open}, 0} + {ok, State#state{caller = open}, 0} end; Error -> Error @@ -2087,7 +2087,7 @@ setup_data_connection(#state{mode = active, caller = Caller, csock = CSock, sockopts_data_active = SockOpts, - ftp_extension = FtpExt} = State) -> + ftp_extension = FtpExt} = State0) -> case (catch sockname(CSock)) of {ok, {{_, _, _, _, _, _, _, _} = IP0, _}} -> IP = proplists:get_value(ip, SockOpts, IP0), @@ -2098,8 +2098,8 @@ setup_data_connection(#state{mode = active, {ok, {_, Port}} = sockname({tcp,LSock}), IpAddress = inet_parse:ntoa(IP), Cmd = mk_cmd("EPRT |2|~s|~p|", [IpAddress, Port]), - _ = send_ctrl_message(State, Cmd), - activate_ctrl_connection(State), + _ = send_ctrl_message(State0, Cmd), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {setup_data_connection, {LSock, Caller}}}}; {ok, {{_,_,_,_} = IP0, _}} -> @@ -2112,37 +2112,37 @@ setup_data_connection(#state{mode = active, false -> {IP1, IP2, IP3, IP4} = IP, {Port1, Port2} = {Port div 256, Port rem 256}, - send_ctrl_message(State, + send_ctrl_message(State0, mk_cmd("PORT ~w,~w,~w,~w,~w,~w", [IP1, IP2, IP3, IP4, Port1, Port2])); true -> IpAddress = inet_parse:ntoa(IP), Cmd = mk_cmd("EPRT |1|~s|~p|", [IpAddress, Port]), - send_ctrl_message(State, Cmd) + send_ctrl_message(State0, Cmd) end, - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {setup_data_connection, {LSock, Caller}}}} end; setup_data_connection(#state{mode = passive, ipfamily = inet6, - caller = Caller} = State) -> - _ = send_ctrl_message(State, mk_cmd("EPSV", [])), - activate_ctrl_connection(State), + caller = Caller} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("EPSV", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {setup_data_connection, Caller}}}; setup_data_connection(#state{mode = passive, ipfamily = inet, caller = Caller, - ftp_extension = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("PASV", [])), - activate_ctrl_connection(State), + ftp_extension = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("PASV", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {setup_data_connection, Caller}}}; setup_data_connection(#state{mode = passive, ipfamily = inet, caller = Caller, - ftp_extension = true} = State) -> - _ = send_ctrl_message(State, mk_cmd("EPSV", [])), - activate_ctrl_connection(State), + ftp_extension = true} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("EPSV", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {setup_data_connection, Caller}}}. connect(Host, Port, SockOpts, Timeout, #state{ipfamily = inet = IpFam}) -> @@ -2248,14 +2248,15 @@ send_message({tcp, Socket}, Message) -> send_message({ssl, Socket}, Message) -> ssl:send(Socket, Message). -activate_ctrl_connection(#state{csock = CSock, ctrl_data = {<<>>, _, _}}) -> - activate_connection(CSock); -activate_ctrl_connection(#state{csock = CSock}) -> +activate_ctrl_connection(#state{csock = CSock, ctrl_data = {<<>>, _, _}} = State) -> + activate_connection(CSock), + State; +activate_ctrl_connection(#state{csock = CSock} = State0) -> activate_connection(CSock), %% We have already received at least part of the next control message, %% that has been saved in ctrl_data, process this first. - self() ! {socket_type(CSock), unwrap_socket(CSock), <<>>}, - ok. + {noreply, State} = handle_info({socket_type(CSock), unwrap_socket(CSock), <<>>}, State0), + State. activate_data_connection(#state{dsock = DSock} = State) -> activate_connection(DSock), @@ -2290,22 +2291,22 @@ close_connection({ssl, Socket}) -> ignore_return_value( ssl:close(Socket) ). %% ------------ FILE HANDLING ---------------------------------------- send_file(#state{tls_upgrading_data_connection = {true, CTRL, _}} = State, Fd) -> {noreply, State#state{tls_upgrading_data_connection = {true, CTRL, ?MODULE, send_file, Fd}}}; -send_file(State, Fd) -> +send_file(State0, Fd) -> case file_read(Fd) of {ok, N, Bin} when N > 0 -> - send_data_message(State, Bin), - progress_report({binary, Bin}, State), - send_file(State, Fd); + send_data_message(State0, Bin), + progress_report({binary, Bin}, State0), + send_file(State0, Fd); {ok, _, _} -> file_close(Fd), - close_data_connection(State), - progress_report({transfer_size, 0}, State), - activate_ctrl_connection(State), + close_data_connection(State0), + progress_report({transfer_size, 0}, State0), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = transfer_file_second_phase, dsock = undefined}}; {error, Reason} -> - gen_server:reply(State#state.client, {error, Reason}), - {stop, normal, State#state{client = undefined}} + gen_server:reply(State0#state.client, {error, Reason}), + {stop, normal, State0#state{client = undefined}} end. file_open(File, Option) -> @@ -2347,10 +2348,10 @@ cast(GenServer, Msg) -> send_bin(#state{tls_upgrading_data_connection = {true, CTRL, _}} = State, Bin) -> State#state{tls_upgrading_data_connection = {true, CTRL, ?MODULE, send_bin, Bin}}; -send_bin(State, Bin) -> - send_data_message(State, Bin), - close_data_connection(State), - activate_ctrl_connection(State), +send_bin(State0, Bin) -> + send_data_message(State0, Bin), + close_data_connection(State0), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = transfer_data_second_phase, dsock = undefined}}. diff --git a/lib/ftp/test/ftp_SUITE.erl b/lib/ftp/test/ftp_SUITE.erl index 7c87d5cbdb..0b070ee8cb 100644 --- a/lib/ftp/test/ftp_SUITE.erl +++ b/lib/ftp/test/ftp_SUITE.erl @@ -96,6 +96,7 @@ ftp_tests()-> recv_chunk, recv_chunk_twice, recv_chunk_three_times, + recv_chunk_delay, type, quote, error_elogin, @@ -669,9 +670,9 @@ recv_chunk(Config0) -> Contents = list_to_binary( lists:duplicate(1000, lists:seq(0,255)) ), Config = set_state([reset, {mkfile,File,Contents}], Config0), Pid = proplists:get_value(ftp, Config), - {{error, "ftp:recv_chunk_start/2 not called"},_} = recv_chunk(Pid, <<>>), + {error, "ftp:recv_chunk_start/2 not called"} = do_recv_chunk(Pid), ok = ftp:recv_chunk_start(Pid, id2ftp(File,Config)), - {ok, ReceivedContents, _Ncunks} = recv_chunk(Pid, <<>>), + {ok, ReceivedContents} = do_recv_chunk(Pid), find_diff(ReceivedContents, Contents). recv_chunk_twice() -> @@ -683,11 +684,11 @@ recv_chunk_twice(Config0) -> Contents2 = crypto:strong_rand_bytes(1200), Config = set_state([reset, {mkfile,File1,Contents1}, {mkfile,File2,Contents2}], Config0), Pid = proplists:get_value(ftp, Config), - {{error, "ftp:recv_chunk_start/2 not called"},_} = recv_chunk(Pid, <<>>), + {error, "ftp:recv_chunk_start/2 not called"} = do_recv_chunk(Pid), ok = ftp:recv_chunk_start(Pid, id2ftp(File1,Config)), - {ok, ReceivedContents1, _Ncunks1} = recv_chunk(Pid, <<>>), + {ok, ReceivedContents1} = do_recv_chunk(Pid), ok = ftp:recv_chunk_start(Pid, id2ftp(File2,Config)), - {ok, ReceivedContents2, _Ncunks2} = recv_chunk(Pid, <<>>), + {ok, ReceivedContents2} = do_recv_chunk(Pid), find_diff(ReceivedContents1, Contents1), find_diff(ReceivedContents2, Contents2). @@ -704,46 +705,56 @@ recv_chunk_three_times(Config0) -> Config = set_state([reset, {mkfile,File1,Contents1}, {mkfile,File2,Contents2}, {mkfile,File3,Contents3}], Config0), Pid = proplists:get_value(ftp, Config), - {{error, "ftp:recv_chunk_start/2 not called"},_} = recv_chunk(Pid, <<>>), + {error, "ftp:recv_chunk_start/2 not called"} = do_recv_chunk(Pid), + ok = ftp:recv_chunk_start(Pid, id2ftp(File3,Config)), + {ok, ReceivedContents3} = do_recv_chunk(Pid), + ok = ftp:recv_chunk_start(Pid, id2ftp(File1,Config)), - {ok, ReceivedContents1, Nchunks1} = recv_chunk(Pid, <<>>), + {ok, ReceivedContents1} = do_recv_chunk(Pid), ok = ftp:recv_chunk_start(Pid, id2ftp(File2,Config)), - {ok, ReceivedContents2, _Nchunks2} = recv_chunk(Pid, <<>>), - - ok = ftp:recv_chunk_start(Pid, id2ftp(File3,Config)), - {ok, ReceivedContents3, _Nchunks3} = recv_chunk(Pid, <<>>, 10000, 0, Nchunks1), + {ok, ReceivedContents2} = do_recv_chunk(Pid), find_diff(ReceivedContents1, Contents1), find_diff(ReceivedContents2, Contents2), find_diff(ReceivedContents3, Contents3). - +do_recv_chunk(Pid) -> + recv_chunk(Pid, <<>>). recv_chunk(Pid, Acc) -> - recv_chunk(Pid, Acc, 0, 0, undefined). - - - -%% ExpectNchunks :: integer() | undefined -recv_chunk(Pid, Acc, DelayMilliSec, N, ExpectNchunks) when N+1 < ExpectNchunks -> - %% for all I in integer(), I < undefined - recv_chunk1(Pid, Acc, DelayMilliSec, N, ExpectNchunks); - -recv_chunk(Pid, Acc, DelayMilliSec, N, ExpectNchunks) -> - %% N >= ExpectNchunks-1 - timer:sleep(DelayMilliSec), - recv_chunk1(Pid, Acc, DelayMilliSec, N, ExpectNchunks). + case ftp:recv_chunk(Pid) of + ok -> + {ok, Acc}; + {ok, Bin} -> + recv_chunk(Pid, <<Acc/binary, Bin/binary>>); + Error -> + Error + end. +recv_chunk_delay(Config0) when is_list(Config0) -> + File1 = "big_file1.txt", + Contents = list_to_binary(lists:duplicate(1000, lists:seq(0,255))), + Config = set_state([reset, {mkfile,File1,Contents}], Config0), + Pid = proplists:get_value(ftp, Config), + ok = ftp:recv_chunk_start(Pid, id2ftp(File1,Config)), + {ok, ReceivedContents} = delay_recv_chunk(Pid), + find_diff(ReceivedContents, Contents). -recv_chunk1(Pid, Acc, DelayMilliSec, N, ExpectNchunks) -> - ct:log("Call ftp:recv_chunk",[]), +delay_recv_chunk(Pid) -> + delay_recv_chunk(Pid, <<>>). +delay_recv_chunk(Pid, Acc) -> + ct:pal("Recived size ~p", [byte_size(Acc)]), case ftp:recv_chunk(Pid) of - ok -> {ok, Acc, N}; - {ok, Bin} -> recv_chunk(Pid, <<Acc/binary, Bin/binary>>, DelayMilliSec, N+1, ExpectNchunks); - Error -> {Error, N} - end. + ok -> + {ok, Acc}; + {ok, Bin} -> + ct:sleep(100), + delay_recv_chunk(Pid, <<Acc/binary, Bin/binary>>); + Error -> + Error + end. %%------------------------------------------------------------------------- type() -> diff --git a/lib/ftp/vsn.mk b/lib/ftp/vsn.mk index d5d6c45b28..9f14658099 100644 --- a/lib/ftp/vsn.mk +++ b/lib/ftp/vsn.mk @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = ftp -FTP_VSN = 1.0.1 +FTP_VSN = 1.0.2 PRE_VSN = APP_VSN = "$(APPLICATION)-$(FTP_VSN)$(PRE_VSN)" diff --git a/lib/hipe/doc/src/notes.xml b/lib/hipe/doc/src/notes.xml index e9cdf42018..9a803cb9df 100644 --- a/lib/hipe/doc/src/notes.xml +++ b/lib/hipe/doc/src/notes.xml @@ -31,6 +31,21 @@ </header> <p>This document describes the changes made to HiPE.</p> +<section><title>Hipe 3.18.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Fix a bug in the handling of the <c>Key</c> argument + of <c>lists:{keysearch, keyfind, keymember}</c>. </p> + <p> + Own Id: OTP-15570</p> + </item> + </list> + </section> + +</section> + <section><title>Hipe 3.18.2</title> <section><title>Improvements and New Features</title> diff --git a/lib/hipe/vsn.mk b/lib/hipe/vsn.mk index 12d621bf01..39565d721f 100644 --- a/lib/hipe/vsn.mk +++ b/lib/hipe/vsn.mk @@ -1 +1 @@ -HIPE_VSN = 3.18.2 +HIPE_VSN = 3.18.3 diff --git a/lib/inets/doc/src/notes.xml b/lib/inets/doc/src/notes.xml index 31dae6317e..91dd9cd6ed 100644 --- a/lib/inets/doc/src/notes.xml +++ b/lib/inets/doc/src/notes.xml @@ -33,7 +33,43 @@ <file>notes.xml</file> </header> - <section><title>Inets 7.0.5</title> + <section><title>Inets 7.0.6</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix the internal handling of the option + erl_script_timeout in httpd. When httpd was started with + explicit erl_script_timeout, the value of the option was + converted to milliseconds before storage. Subsequent + calls to httpd:info/1 returned the input value multiplied + by 1000.</p> + <p> + This change fixes the handing of erl_script_timeout by + storing the timeout in seconds and converting to + milliseconds before usage.</p> + <p> + Own Id: OTP-15669 Aux Id: ERIERL-321 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Enhance documentation</p> + <p> + Own Id: OTP-15508 Aux Id: ERL-816 </p> + </item> + </list> + </section> + +</section> + +<section><title>Inets 7.0.5</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/inets/src/http_server/mod_esi.erl b/lib/inets/src/http_server/mod_esi.erl index 443b7ee564..f495f12f03 100644 --- a/lib/inets/src/http_server/mod_esi.erl +++ b/lib/inets/src/http_server/mod_esi.erl @@ -33,7 +33,7 @@ -include("httpd_internal.hrl"). -define(VMODULE,"ESI"). --define(DEFAULT_ERL_TIMEOUT,15000). +-define(DEFAULT_ERL_TIMEOUT,15). %%%========================================================================= @@ -174,7 +174,7 @@ store({erl_script_alias, Value}, _) -> {error, {wrong_type, {erl_script_alias, Value}}}; store({erl_script_timeout, TimeoutSec}, _) when is_integer(TimeoutSec) andalso (TimeoutSec >= 0) -> - {ok, {erl_script_timeout, TimeoutSec * 1000}}; + {ok, {erl_script_timeout, TimeoutSec}}; store({erl_script_timeout, Value}, _) -> {error, {wrong_type, {erl_script_timeout, Value}}}; store({erl_script_nocache, Value} = Conf, _) @@ -500,7 +500,7 @@ kill_esi_delivery_process(Pid) -> erl_script_timeout(Db) -> - httpd_util:lookup(Db, erl_script_timeout, ?DEFAULT_ERL_TIMEOUT). + httpd_util:lookup(Db, erl_script_timeout, ?DEFAULT_ERL_TIMEOUT * 1000). script_elements(FuncAndInput, Input) -> case input_type(FuncAndInput) of diff --git a/lib/inets/test/httpd_SUITE.erl b/lib/inets/test/httpd_SUITE.erl index 5b6740fba3..fcb9ad7905 100644 --- a/lib/inets/test/httpd_SUITE.erl +++ b/lib/inets/test/httpd_SUITE.erl @@ -78,7 +78,8 @@ all() -> {group, http_rel_path_script_alias}, {group, http_not_sup}, {group, https_not_sup}, - mime_types_format + mime_types_format, + erl_script_timeout_option ]. groups() -> @@ -1777,6 +1778,18 @@ mime_types_format(Config) when is_list(Config) -> {"hqx","application/mac-binhex40"}]} = httpd_conf:load_mime_types(MimeTypes). +erl_script_timeout_option(Config) when is_list(Config) -> + inets:start(), + {ok, Pid} = inets:start(httpd, [{erl_script_timeout, 215}, + {server_name, "test"}, + {port,0}, + {server_root, "."}, + {document_root, "."}]), + Info = httpd:info(Pid), + 215 = proplists:get_value(erl_script_timeout, Info), + inets:stop(). + + %%-------------------------------------------------------------------- %% Internal functions ----------------------------------- %%-------------------------------------------------------------------- diff --git a/lib/inets/vsn.mk b/lib/inets/vsn.mk index 921161dce1..b7ddf39ebd 100644 --- a/lib/inets/vsn.mk +++ b/lib/inets/vsn.mk @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = inets -INETS_VSN = 7.0.5 +INETS_VSN = 7.0.6 PRE_VSN = APP_VSN = "$(APPLICATION)-$(INETS_VSN)$(PRE_VSN)" diff --git a/lib/jinterface/doc/src/Makefile b/lib/jinterface/doc/src/Makefile index d80bb38ec1..f5cba9d074 100644 --- a/lib/jinterface/doc/src/Makefile +++ b/lib/jinterface/doc/src/Makefile @@ -137,6 +137,7 @@ clean clean_docs: jdoc:$(JAVA_SRC_FILES) (cd ../../java_src;$(JAVADOC) -sourcepath . -d $(JAVADOC_DEST) \ -windowtitle $(JAVADOC_TITLE) $(JAVADOC_PKGS)) + touch jdoc man: @@ -152,15 +153,8 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_docs_spec: docs $(INSTALL_DIR) "$(RELSYSDIR)/doc/pdf" $(INSTALL_DATA) $(TOP_PDF_FILE) "$(RELSYSDIR)/doc/pdf" - $(INSTALL_DIR) "$(RELSYSDIR)/doc/html" - $(INSTALL_DIR) "$(RELSYSDIR)/doc/html/java/$(JAVA_PKG_PATH)" $(INSTALL_DATA) $(INFO_FILE) "$(RELSYSDIR)" - ($(CP) -rf ../html "$(RELSYSDIR)/doc") - -# $(INSTALL_DATA) $(GIF_FILES) $(EXTRA_FILES) $(HTML_FILES) \ -# "$(RELSYSDIR)/doc/html" -# $(INSTALL_DATA) $(JAVA_EXTRA_FILES) "$(RELSYSDIR)/doc/html/java" -# $(INSTALL_DATA) $(TOP_HTML_FILES) "$(RELSYSDIR)/doc" + $(INSTALL_DIR_DATA) $(HTMLDIR) "$(RELSYSDIR)/doc/html" release_spec: diff --git a/lib/kernel/doc/src/application.xml b/lib/kernel/doc/src/application.xml index 5170502581..83a83ebad2 100644 --- a/lib/kernel/doc/src/application.xml +++ b/lib/kernel/doc/src/application.xml @@ -238,8 +238,8 @@ Nodes = [cp1@cave, {cp2@cave, cp3@cave}]</code> </desc> </func> <func> - <name name="set_env" arity="1" since="OTP @OTP-15642@"/> - <name name="set_env" arity="2" since="OTP @OTP-15642@"/> + <name name="set_env" arity="1" since="OTP 21.3"/> + <name name="set_env" arity="2" since="OTP 21.3"/> <fsummary>Sets the configuration parameters of multiple applications.</fsummary> <desc> <p>Sets the configuration <c><anno>Config</anno></c> for multiple diff --git a/lib/kernel/doc/src/kernel_app.xml b/lib/kernel/doc/src/kernel_app.xml index 15dbdb47dc..7f9609d5c1 100644 --- a/lib/kernel/doc/src/kernel_app.xml +++ b/lib/kernel/doc/src/kernel_app.xml @@ -369,6 +369,14 @@ MaxT = TickTime + TickTime / 4</code> performed. This option ensures that <c>global</c> is synchronized.</p> </item> + <tag><c>start_distribution = true | false</c></tag> + <item> + <p>Starts all distribution services, such as <c>rpc</c>, + <c>global</c>, and <c>net_kernel</c> if the parameter is + <c>true</c>. This parameter is to be set to <c>false</c> + for systems who want to disable all distribution functionality.</p> + <p>Defaults to <c>true</c>.</p> + </item> <tag><c>start_dist_ac = true | false</c></tag> <item> <p>Starts the <c>dist_ac</c> server if the parameter is @@ -510,11 +518,13 @@ MaxT = TickTime + TickTime / 4</code> parameters for Logger are not set.</p> <taglist> <tag><c>error_logger</c></tag> - <item>Replaced by setting the type of the default - <seealso marker="logger_std_h#type"><c>logger_std_h</c></seealso> - to the same value. Example: + <item>Replaced by setting the <seealso + marker="logger_std_h#type"><c>type</c></seealso>, and possibly + <seealso marker="logger_std_h#file"><c>file</c></seealso> and + <seealso marker="logger_std_h#modes"><c>modes</c></seealso> + parameters of the default <c>logger_std_h</c> handler. Example: <code type="none"> -erl -kernel logger '[{handler,default,logger_std_h,#{config=>#{type=>{file,"/tmp/erlang.log"}}}}]' +erl -kernel logger '[{handler,default,logger_std_h,#{config=>#{file=>"/tmp/erlang.log"}}}]' </code> </item> <tag><c>error_logger_format_depth</c></tag> diff --git a/lib/kernel/doc/src/logger.xml b/lib/kernel/doc/src/logger.xml index e6448e144e..2a060ce1f9 100644 --- a/lib/kernel/doc/src/logger.xml +++ b/lib/kernel/doc/src/logger.xml @@ -66,7 +66,7 @@ logger:error("error happened because: ~p", [Reason]). % Without macro [{kernel, [{logger, [{handler, default, logger_std_h, - #{config => #{type => {file, "path/to/file.log"}}}}]}]}]. + #{config => #{file => "path/to/file.log"}}}]}]}]. </code> <p> For more information about: @@ -1119,7 +1119,7 @@ logger:set_proxy_config(maps:merge(Old, Config)). </func> <func> - <name name="timestamp" arity="0" since="OTP @OTP-15625@"/> + <name name="timestamp" arity="0" since="OTP 21.3"/> <fsummary>Return a timestamp to insert in meta data for a log event.</fsummary> <desc> @@ -1351,5 +1351,3 @@ logger:set_proxy_config(maps:merge(Old, Config)). </p> </section> </erlref> - - diff --git a/lib/kernel/doc/src/logger_chapter.xml b/lib/kernel/doc/src/logger_chapter.xml index 03b9edcf8f..8458ffa042 100644 --- a/lib/kernel/doc/src/logger_chapter.xml +++ b/lib/kernel/doc/src/logger_chapter.xml @@ -801,7 +801,7 @@ logger:debug(#{got => connection_request, id => Id, state => State}, [{kernel, [{logger, [{handler, default, logger_std_h, % {handler, HandlerId, Module, - #{config => #{type => {file,"log/erlang.log"}}}} % Config} + #{config => #{file => "log/erlang.log"}}} % Config} ]}]}]. </code> <p>Modify the default handler to print each log event as a @@ -831,10 +831,10 @@ logger:debug(#{got => connection_request, id => Id, state => State}, [{logger, [{handler, default, logger_std_h, #{level => error, - config => #{type => {file, "log/erlang.log"}}}}, + config => #{file => "log/erlang.log"}}}, {handler, info, logger_std_h, #{level => debug, - config => #{type => {file, "log/debug.log"}}}} + config => #{file => "log/debug.log"}}} ]}]}]. </code> </section> @@ -1004,10 +1004,10 @@ ok</pre> <p>Then, add a new handler which prints to file. You can use the handler module <seealso marker="logger_std_h"><c>logger_std_h</c></seealso>, - and specify type <c>{file,File}</c>.:</p> + and configure it to log to file:</p> <pre> -4> <input>Config = #{config => #{type => {file,"./info.log"}}, level => info}.</input> -#{config => #{type => {file,"./info.log"}},level => info} +4> <input>Config = #{config => #{file => "./info.log"}, level => info}.</input> +#{config => #{file => "./info.log"},level => info} 5> <input>logger:add_handler(myhandler, logger_std_h, Config).</input> ok</pre> <p>Since <c>filter_default</c> defaults to <c>log</c>, this @@ -1246,7 +1246,7 @@ do_log(Fd, LogEvent, #{formatter := {FModule, FConfig}}) -> <p>A configuration example:</p> <code type="none"> logger:add_handler(my_standard_h, logger_std_h, - #{config => #{type => {file,"./system_info.log"}, + #{config => #{file => "./system_info.log", sync_mode_qlen => 100, drop_mode_qlen => 1000, flush_qlen => 2000}}). diff --git a/lib/kernel/doc/src/logger_std_h.xml b/lib/kernel/doc/src/logger_std_h.xml index fcd180abd6..5ed1a2f210 100644 --- a/lib/kernel/doc/src/logger_std_h.xml +++ b/lib/kernel/doc/src/logger_std_h.xml @@ -55,30 +55,105 @@ is stored in a sub map with the key <c>config</c>, and can contain the following parameters:</p> <taglist> - <tag><marker id="type"/><c>type</c></tag> + <tag><marker id="type"/><c>type = standard_io | standard_error | file</c></tag> <item> - <p>This has the value <c>standard_io</c>, <c>standard_error</c>, - <c>{file,LogFileName}</c>, or <c>{file,LogFileName,LogFileOpts}</c>.</p> - <p>If <c>LogFileOpts</c> is specified, it replaces the default - list of options used when opening the log file. The default - list is <c>[raw,append,delayed_write]</c>. One reason to do - so can be to change <c>append</c> to, for - example, <c>write</c>, ensuring that the old log is - truncated when a node is restarted. See the reference manual - for <seealso marker="file#open-2"><c>file:open/2</c></seealso> - for more information about file options.</p> + <p>Specifies the log destination.</p> + <p>The value is set when the handler is added, and it can not + be changed in runtime.</p> + <p>Defaults to <c>standard_io</c>, unless + parameter <seealso marker="#file"><c>file</c></seealso> is + given, in which case it defaults to <c>file</c>.</p> + </item> + <tag><marker id="file"/><c>file = </c><seealso marker="file#type-filename"><c>file:filename()</c></seealso></tag> + <item> + <p>This specifies the name of the log file when the handler is + of type <c>file</c>.</p> + <p>The value is set when the handler is added, and it can not + be changed in runtime.</p> + <p>Defaults to the same name as the handler identity, in the + current directory.</p> + </item> + <tag><marker id="modes"/><c>modes = [</c><seealso marker="file#type-mode"><c>file:mode()</c></seealso><c>]</c></tag> + <item> + <p>This specifies the file modes to use when opening the log + file, + see <seealso marker="file#open-2"><c>file:open/2</c></seealso>. + If <c>modes</c> are not specified, the default list used + is <c>[raw,append,delayed_write]</c>. If <c>modes</c> are + specified, the list replaces the default modes list with the + following adjustments:</p> + <list> + <item> + If <c>raw</c> is not found in the list, it is added. + </item> + <item> + If none of <c>write</c>, <c>append</c> or <c>exclusive</c> is + found in the list, <c>append</c> is added.</item> + <item>If none of <c>delayed_write</c> + or <c>{delayed_write,Size,Delay}</c> is found in the + list, <c>delayed_write</c> is added.</item> + </list> <p>Log files are always UTF-8 encoded. The encoding can not be - changed by setting the option <c>{encoding,Encoding}</c> - in <c>LogFileOpts</c>.</p> - <p>Notice that the standard handler does not have support for - circular logging. Use the disk_log handler, - <seealso marker="logger_disk_log_h"><c>logger_disk_log_h</c></seealso>, - for this.</p> + changed by setting the mode <c>{encoding,Encoding}</c>.</p> <p>The value is set when the handler is added, and it can not be changed in runtime.</p> - <p>Defaults to <c>standard_io</c>.</p> + <p>Defaults to <c>[raw,append,delayed_write]</c>.</p> + </item> + <tag><marker id="max_no_bytes"/><c>max_no_bytes = pos_integer() | infinity</c></tag> + <item> + <p>This parameter specifies if the log file should be rotated + or not. The value <c>infinity</c> means the log file will + grow indefinitely, while an integer value specifies at which + file size (bytes) the file is rotated.</p> + <p>Defaults to <c>infinity</c>.</p> + </item> + <tag><marker id="max_no_files"/><c>max_no_files = non_neg_integer()</c></tag> + <item> + <p>This parameter specifies the number of rotated log file + archives to keep. This has meaning only + if <seealso marker="#max_no_bytes"><c>max_no_bytes</c></seealso> + is set to an integer value.</p> + <p>The log archives are + named <c>FileName.0</c>, <c>FileName.1</c>, + ... <c>FileName.N</c>, where <c>FileName</c> is the name of + the current log file. <c>FileName.0</c> is the newest of the + archives. The maximum value for <c>N</c> is the value + of <c>max_no_files</c> minus 1.</p> + <p>Notice that setting this value to <c>0</c> does not turn of + rotation. It only specifies that no archives are kept.</p> + <p>Defaults to <c>0</c>.</p> + </item> + <tag><marker id="compress_on_rotate"/><c>compress_on_rotate = boolean()</c></tag> + <item> + <p>This parameter specifies if the rotated log file archives + shall be compressed or not. If set to <c>true</c>, all + archives are compressed with <c>gzip</c>, and renamed + to <c>FileName.N.gz</c></p> + <p><c>compress_on_rotate</c> has no meaning if <seealso + marker="#max_no_bytes"><c>max_no_bytes</c></seealso> has the + value <c>infinity</c>.</p> + <p>Defaults to <c>false</c>.</p> + </item> + <tag><marker id="file_check"/><c>file_check = non_neg_integer()</c></tag> + <item> + <p>When <c>logger_std_h</c> logs to a file, it reads the file + information of the log file prior to each write + operation. This is to make sure the file still exists and + has the same inode as when it was opened. This implies some + performance loss, but ensures that no log events are lost in + the case when the file has been removed or renamed by an + external actor.</p> + <p>In order to allow minimizing the performance loss, the + <c>file_check</c> parameter can be set to a positive integer + value, <c>N</c>. The handler will then skip reading the file + information prior to writing, as long as no more + than <c>N</c> milliseconds have passed since it was last + read.</p> + <p>Notice that the risk of loosing log events grows when + the <c>file_check</c> value grows.</p> + <p>Defaults to 0.</p> </item> - <tag><c>filesync_repeat_interval</c></tag> + <tag><c>filesync_repeat_interval = pos_integer() | no_repeat</c></tag> <item> <p>This value, in milliseconds, specifies how often the handler does a file sync operation to write buffered data to disk. The handler attempts @@ -97,12 +172,13 @@ standard handler and the disk_log handler, and are documented in the <seealso marker="logger_chapter#overload_protection"><c>User's Guide</c> </seealso>.</p> - <p>Notice that if changing the configuration of the handler in runtime, - the <c>type</c> parameter must not be modified.</p> + <p>Notice that if changing the configuration of the handler in + runtime, the <c>type</c>, <c>file</c>, or <c>modes</c> parameters + must not be modified.</p> <p>Example of adding a standard handler:</p> <code type="none"> logger:add_handler(my_standard_h, logger_std_h, - #{config => #{type => {file,"./system_info.log"}, + #{config => #{file => "./system_info.log", filesync_repeat_interval => 1000}}). </code> <p>To set the default handler, that starts initially with @@ -110,7 +186,7 @@ logger:add_handler(my_standard_h, logger_std_h, change the Kernel default logger configuration. Example:</p> <code type="none"> erl -kernel logger '[{handler,default,logger_std_h, - #{config => #{type => {file,"./log.log"}}}}]' + #{config => #{file => "./log.log"}}}]' </code> <p>An example of how to replace the standard handler with a disk_log handler at startup is found in the diff --git a/lib/kernel/doc/src/notes.xml b/lib/kernel/doc/src/notes.xml index 021ecfa40d..0c187eb19f 100644 --- a/lib/kernel/doc/src/notes.xml +++ b/lib/kernel/doc/src/notes.xml @@ -31,6 +31,204 @@ </header> <p>This document describes the changes made to the Kernel application.</p> +<section><title>Kernel 6.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + If for example the <c>/etc/hosts</c> did not come into + existence until after the kernel application had started, + its content was never read. This bug has now been + corrected.</p> + <p> + Own Id: OTP-14702 Aux Id: PR-2066 </p> + </item> + <item> + <p> + Fix bug where doing <c>seq_trace:reset_trace()</c> while + another process was doing a garbage collection could + cause the run-time system to segfault.</p> + <p> + Own Id: OTP-15490</p> + </item> + <item> + <p> + Fix <c>erl_epmd:port_please</c> spec to include + <c>atom()</c> and <c>string()</c>.</p> + <p> + Own Id: OTP-15557 Aux Id: PR-2117 </p> + </item> + <item> + <p> + The Logger handler logger_std_h now keeps track of the + inode of its log file in order to re-open the file if the + inode changes. This may happen, for instance, if the log + file is opened and saved by an editor.</p> + <p> + Own Id: OTP-15578 Aux Id: ERL-850 </p> + </item> + <item> + <p> + When user specific file modes are given to the logger + handler <c>logger_std_h</c>, they were earlier accepted + without any control. This is now changes, so Logger will + adjust the file modes as follows:</p> + <p> + - If <c>raw</c> is not found in the list, it is + added.<br/> - If none of <c>write</c>, <c>append</c> or + <c>exclusive</c> are found in the list, <c>append</c> is + added.<br/> - If none of <c>delayed_write</c> or + <c>{delayed_write,Size,Delay}</c> are found in the list, + <c>delayed_write</c> is added.</p> + <p> + Own Id: OTP-15602</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + The standard logger handler, <c>logger_std_h</c>, now has + a new internal feature for log rotation. The rotation + scheme is as follows:</p> + <p> + The log file to which the handler currently writes always + has the same name, i.e. the name which is configured for + the handler. The archived files have the same name, but + with extension ".N", where N is an integer. The newest + archive has extension ".0", and the oldest archive has + the highest number. </p> + <p> + The size at which the log file is rotated, and the number + of archive files that are kept, is specified with the + handler specific configuration parameters + <c>max_no_bytes</c> and <c>max_no_files</c> respectively. </p> + <p> + Archives can be compressed, in which case they get a + ".gz" file extension after the integer. Compression is + specified with the handler specific configuration + parameter <c>compress_on_rotate</c>.</p> + <p> + Own Id: OTP-15479</p> + </item> + <item> + <p> + The new functions <c>logger:i/0</c> and <c>logger:i/1</c> + are added. These provide the same information as + <c>logger:get_config/0</c> and other + <c>logger:get_*_config</c> functions, but the information + is more logically sorted and more readable.</p> + <p> + Own Id: OTP-15600</p> + </item> + <item> + <p> + Logger is now protected against overload due to massive + amounts of log events from the emulator or from remote + nodes.</p> + <p> + Own Id: OTP-15601</p> + </item> + <item> + <p> + Logger now uses os:system_time/1 instead of + erlang:system_time/1 to generate log event timestamps.</p> + <p> + Own Id: OTP-15625</p> + </item> + <item> + <p> + Add functions <c>application:set_env/1,2</c> and + <c>application:set_env/2</c>. These take a list of + application configuration parameters, and the behaviour + is equivalent to calling <c>application:set_env/4</c> + individually for each application/key combination, except + it is more efficient.</p> + <p> + <c>set_env/1,2</c> warns about duplicated applications or + keys. The warning is also emitted during boot, if + applications or keys are duplicated within one + configuration file, e.g. sys.config.</p> + <p> + Own Id: OTP-15642 Aux Id: PR-2164 </p> + </item> + <item> + <p> + Handler specific configuration parameters for the + standard handler <c>logger_std_h</c> are changed to be + more intuitive and more similar to the disk_log handler.</p> + <p> + Earlier there was only one parameter, <c>type</c>, which + could have the values <c>standard_io</c>, + <c>standard_error</c>, <c>{file,FileName}</c> or + <c>{file,FileName,Modes}</c>.</p> + <p> + This is now changed, so the following parameters are + allowed:</p> + <p> + <c>type = standard_io | standard_error | file</c><br/> + <c>file = file:filename()</c><br/> <c>modes = + [file:mode()]</c></p> + <p> + All parameters are optional. <c>type</c> defaults to + <c>standard_io</c>, unless a file name is given, in which + case it defaults to <c>file</c>. If <c>type</c> is set to + <c>file</c>, the file name defaults to the same as the + handler id.</p> + <p> + The potential incompatibility is that + <c>logger:get_config/0</c> and + <c>logger:get_handler_config/1</c> now returns the new + parameters, even if the configuration was set with the + old variant, e.g. <c>#{type=>{file,FileName}}</c>.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-15662</p> + </item> + <item> + <p> + The new configuration parameter <c>file_check</c> is + added to the Logger handler <c>logger_std_h</c>. This + parameter specifies how long (in milliseconds) the + handler may wait before checking if the log file still + exists and the inode is the same as when it was opened.</p> + <p> + The default value is 0, which means that this check is + done prior to each write operation. Setting a higher + number may improve performance, but adds the risk of + loosing log events.</p> + <p> + Own Id: OTP-15663</p> + </item> + </list> + </section> + +</section> + +<section><title>Kernel 6.2.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Setting the <c>recbuf</c> size of an inet socket the + <c>buffer</c> is also automatically increased. Fix a bug + where the auto adjustment of inet buffer size would be + triggered even if an explicit inet buffer size had + already been set.</p> + <p> + Own Id: OTP-15651 Aux Id: ERIERL-304 </p> + </item> + </list> + </section> + +</section> + <section><title>Kernel 6.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/kernel/src/kernel.app.src b/lib/kernel/src/kernel.app.src index 4b48f6cd1d..8fe6bdd1ca 100644 --- a/lib/kernel/src/kernel.app.src +++ b/lib/kernel/src/kernel.app.src @@ -147,6 +147,6 @@ {logger_sasl_compatible, false} ]}, {mod, {kernel, []}}, - {runtime_dependencies, ["erts-10.1", "stdlib-3.5", "sasl-3.0"]} + {runtime_dependencies, ["erts-10.2.5", "stdlib-3.5", "sasl-3.0"]} ] }. diff --git a/lib/kernel/src/kernel.appup.src b/lib/kernel/src/kernel.appup.src index ccf0a82ced..8fa3f5c588 100644 --- a/lib/kernel/src/kernel.appup.src +++ b/lib/kernel/src/kernel.appup.src @@ -40,7 +40,10 @@ {<<"^6\\.0\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^6\\.1$">>,[restart_new_emulator]}, {<<"^6\\.1\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], + {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^6\\.2$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], [{<<"^5\\.3$">>,[restart_new_emulator]}, {<<"^5\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^5\\.3\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -54,4 +57,7 @@ {<<"^6\\.0\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^6\\.1$">>,[restart_new_emulator]}, {<<"^6\\.1\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. + {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^6\\.2$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. diff --git a/lib/kernel/src/kernel.erl b/lib/kernel/src/kernel.erl index c68d04e279..111d103df2 100644 --- a/lib/kernel/src/kernel.erl +++ b/lib/kernel/src/kernel.erl @@ -145,26 +145,11 @@ init([]) -> {ok, {SupFlags, [Code, File, StdError, User, LoggerSup, Config, RefC, SafeSup]}}; _ -> - Rpc = #{id => rex, - start => {rpc, start_link, []}, - restart => permanent, - shutdown => 2000, - type => worker, - modules => [rpc]}, - - Global = #{id => global_name_server, - start => {global, start_link, []}, - restart => permanent, - shutdown => 2000, - type => worker, - modules => [global]}, - - GlGroup = #{id => global_group, - start => {global_group,start_link,[]}, - restart => permanent, - shutdown => 2000, - type => worker, - modules => [global_group]}, + DistChildren = + case application:get_env(kernel, start_distribution) of + {ok, false} -> []; + _ -> start_distribution() + end, InetDb = #{id => inet_db, start => {inet_db, start_link, []}, @@ -173,13 +158,6 @@ init([]) -> type => worker, modules => [inet_db]}, - NetSup = #{id => net_sup, - start => {erl_distribution, start_link, []}, - restart => permanent, - shutdown => infinity, - type => supervisor, - modules => [erl_distribution]}, - SigSrv = #{id => erl_signal_server, start => {gen_event, start_link, [{local, erl_signal_server}]}, restart => permanent, @@ -187,14 +165,11 @@ init([]) -> type => worker, modules => dynamic}, - DistAC = start_dist_ac(), - Timer = start_timer(), {ok, {SupFlags, - [Code, Rpc, Global, InetDb | DistAC] ++ - [NetSup, GlGroup, File, SigSrv, - StdError, User, Config, RefC, SafeSup, LoggerSup] ++ Timer}} + [Code, InetDb | DistChildren] ++ + [File, SigSrv, StdError, User, Config, RefC, SafeSup, LoggerSup] ++ Timer}} end; init(safe) -> SupFlags = #{strategy => one_for_one, @@ -213,6 +188,39 @@ init(safe) -> {ok, {SupFlags, Boot ++ DiskLog ++ Pg2}}. +start_distribution() -> + Rpc = #{id => rex, + start => {rpc, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [rpc]}, + + Global = #{id => global_name_server, + start => {global, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [global]}, + + DistAC = start_dist_ac(), + + NetSup = #{id => net_sup, + start => {erl_distribution, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [erl_distribution]}, + + GlGroup = #{id => global_group, + start => {global_group,start_link,[]}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [global_group]}, + + [Rpc, Global | DistAC] ++ [NetSup, GlGroup]. + start_dist_ac() -> Spec = [#{id => dist_ac, start => {dist_ac,start_link,[]}, diff --git a/lib/kernel/src/logger_formatter.erl b/lib/kernel/src/logger_formatter.erl index ded89bac9f..8696adbd72 100644 --- a/lib/kernel/src/logger_formatter.erl +++ b/lib/kernel/src/logger_formatter.erl @@ -64,7 +64,7 @@ format(#{level:=Level,msg:=Msg0,meta:=Meta},Config0) Config; Size0 -> Size = - case Size0 - string:length([B,A]) of + case Size0 - io_lib:chars_length([B,A]) of S when S>=0 -> S; _ -> 0 end, @@ -75,7 +75,11 @@ format(#{level:=Level,msg:=Msg0,meta:=Meta},Config0) true -> %% Trim leading and trailing whitespaces, and replace %% newlines with ", " - re:replace(string:trim(MsgStr0),",?\r?\n\s*",", ", + T = lists:reverse( + trim( + lists:reverse( + trim(MsgStr0,false)),true)), + re:replace(T,",?\r?\n\s*",", ", [{return,list},global,unicode]); _false -> MsgStr0 @@ -83,7 +87,26 @@ format(#{level:=Level,msg:=Msg0,meta:=Meta},Config0) true -> "" end, - truncate([B,MsgStr,A],maps:get(max_size,Config)). + truncate(B,MsgStr,A,maps:get(max_size,Config)). + +trim([H|T],Rev) when H==$\s; H==$\r; H==$\n -> + trim(T,Rev); +trim([H|T],false) when is_list(H) -> + case trim(H,false) of + [] -> + trim(T,false); + TrimmedH -> + [TrimmedH|T] + end; +trim([H|T],true) when is_list(H) -> + case trim(lists:reverse(H),true) of + [] -> + trim(T,true); + TrimmedH -> + [lists:reverse(TrimmedH)|T] + end; +trim(String,_) -> + String. do_format(Level,Data,[level|Format],Config) -> [to_string(level,Level,Config)|do_format(Level,Data,Format,Config)]; @@ -239,21 +262,47 @@ chardata_to_list(Chardata) -> throw(Error) end. -truncate(String,unlimited) -> - String; -truncate(String,Size) -> - Length = string:length(String), +truncate(B,Msg,A,unlimited) -> + [B,Msg,A]; +truncate(B,Msg,A,Size) -> + String = [B,Msg,A], + Length = io_lib:chars_length(String), if Length>Size -> - case lists:reverse(lists:flatten(String)) of - [$\n|_] -> - string:slice(String,0,Size-4)++"...\n"; + {Last,FlatString} = + case A of + [] -> + case Msg of + [] -> + {get_last(B),lists:flatten(B)}; + _ -> + {get_last(Msg),lists:flatten([B,Msg])} + end; + _ -> + {get_last(A),lists:flatten(String)} + end, + case Last of + $\n-> + lists:sublist(FlatString,1,Size-4)++"...\n"; _ -> - string:slice(String,0,Size-3)++"..." + lists:sublist(FlatString,1,Size-3)++"..." end; true -> String end. +get_last(L) -> + get_first(lists:reverse(L)). + +get_first([]) -> + error; +get_first([C|_]) when is_integer(C) -> + C; +get_first([L|Rest]) when is_list(L) -> + case get_last(L) of + error -> get_first(Rest); + First -> First + end. + %% SysTime is the system time in microseconds format_time(SysTime,#{time_offset:=Offset,time_designator:=Des}) when is_integer(SysTime) -> diff --git a/lib/kernel/src/logger_h_common.erl b/lib/kernel/src/logger_h_common.erl index 4b5e0a7dd0..16946ff97c 100644 --- a/lib/kernel/src/logger_h_common.erl +++ b/lib/kernel/src/logger_h_common.erl @@ -142,8 +142,9 @@ changing_config(SetOrUpdate, maps:with(?OLP_KEYS,NewHConfig0)), case logger_olp:set_opts(Olp,NewOlpOpts) of ok -> - maybe_set_repeated_filesync(Olp,OldCommonConfig, - NewCommonConfig), + logger_olp:cast(Olp, {config_changed, + NewCommonConfig, + NewHandlerConfig}), ReadOnly = maps:with(?READ_ONLY_KEYS,OldHConfig), NewHConfig = maps:merge( @@ -281,11 +282,24 @@ handle_cast(repeated_filesync, State#{handler_state => HS, last_op => sync} end, {noreply,set_repeated_filesync(State1)}; - -handle_cast({set_repeated_filesync,FSyncInt},State) -> - State1 = State#{filesync_repeat_interval=>FSyncInt}, - State2 = set_repeated_filesync(cancel_repeated_filesync(State1)), - {noreply, State2}. +handle_cast({config_changed, CommonConfig, HConfig}, + State = #{id := Name, + module := Module, + handler_state := HandlerState, + filesync_repeat_interval := OldFSyncInt}) -> + State1 = + case maps:get(filesync_repeat_interval,CommonConfig) of + OldFSyncInt -> + State; + FSyncInt -> + set_repeated_filesync( + cancel_repeated_filesync( + State#{filesync_repeat_interval=>FSyncInt})) + end, + HS = try Module:config_changed(Name, HConfig, HandlerState) + catch error:undef -> HandlerState + end, + {noreply, State1#{handler_state => HS}}. handle_info(Info, #{id := Name, module := Module, handler_state := HandlerState} = State) -> @@ -447,10 +461,3 @@ cancel_repeated_filesync(State) -> end. error_notify(Term) -> ?internal_log(error, Term). - -maybe_set_repeated_filesync(_Olp, - #{filesync_repeat_interval:=FSyncInt}, - #{filesync_repeat_interval:=FSyncInt}) -> - ok; -maybe_set_repeated_filesync(Olp,_,#{filesync_repeat_interval:=FSyncInt}) -> - logger_olp:cast(Olp,{set_repeated_filesync,FSyncInt}). diff --git a/lib/kernel/src/logger_olp.erl b/lib/kernel/src/logger_olp.erl index 009280a9c9..8365383fe2 100644 --- a/lib/kernel/src/logger_olp.erl +++ b/lib/kernel/src/logger_olp.erl @@ -515,10 +515,11 @@ check_load(State = #{id:=_Name, mode_ref := ModeRef, mode := Mode, end, State1 = ?update_other(drops,DROPS,_NewDrops,State), State2 = ?update_max_qlen(QLen,State1), - State3 = maybe_notify_mode_change(Mode1,State2), + State3 = ?update_max_mem(Mem,State2), + State4 = maybe_notify_mode_change(Mode1,State3), {Mode1, QLen, Mem, ?update_other(flushes,FLUSHES,_NewFlushes, - State3#{last_qlen => QLen})}. + State4#{last_qlen => QLen})}. limit_burst(#{burst_limit_enable := false}=State) -> {true,State}; diff --git a/lib/kernel/src/logger_olp.hrl b/lib/kernel/src/logger_olp.hrl index 9b4f5ebf27..d68b5c048d 100644 --- a/lib/kernel/src/logger_olp.hrl +++ b/lib/kernel/src/logger_olp.hrl @@ -114,12 +114,16 @@ flushes => 0, flushed => 0, drops => 0, burst_drops => 0, casts => 0, calls => 0, writes => 0, max_qlen => 0, max_time => 0, - freq => {TIME,0,0}} end). + max_mem => 0, freq => {TIME,0,0}} end). -define(update_max_qlen(QLEN, STATE), begin #{max_qlen := QLEN0} = STATE, STATE#{max_qlen => ?max(QLEN0,QLEN)} end). + -define(update_max_mem(MEM, STATE), + begin #{max_mem := MEM0} = STATE, + STATE#{max_mem => ?max(MEM0,MEM)} end). + -define(update_calls_or_casts(CALL_OR_CAST, INC, STATE), case CALL_OR_CAST of cast -> @@ -154,6 +158,7 @@ -else. % DEFAULT! -define(merge_with_stats(STATE), STATE). -define(update_max_qlen(_QLEN, STATE), STATE). + -define(update_max_mem(_MEM, STATE), STATE). -define(update_calls_or_casts(_CALL_OR_CAST, _INC, STATE), STATE). -define(update_max_time(_TIME, STATE), STATE). -define(update_other(_OTHER, _VAR, _INCVAL, STATE), STATE). diff --git a/lib/kernel/src/logger_std_h.erl b/lib/kernel/src/logger_std_h.erl index 65f5b3876e..c8f1acfca4 100644 --- a/lib/kernel/src/logger_std_h.erl +++ b/lib/kernel/src/logger_std_h.erl @@ -29,7 +29,7 @@ -export([filesync/1]). %% logger_h_common callbacks --export([init/2, check_config/4, reset_state/2, +-export([init/2, check_config/4, config_changed/3, reset_state/2, filesync/3, write/4, handle_info/3, terminate/3]). %% logger callbacks @@ -105,85 +105,169 @@ filter_config(Config) -> %%%=================================================================== %%% logger_h_common callbacks %%%=================================================================== -init(Name, #{type := Type}) -> - case open_log_file(Name, Type) of +init(Name, Config) -> + MyConfig = maps:with([type,file,modes,file_check,max_no_bytes, + max_no_files,compress_on_rotate],Config), + case file_ctrl_start(Name, MyConfig) of {ok,FileCtrlPid} -> - {ok,#{type=>Type,file_ctrl_pid=>FileCtrlPid}}; + {ok,MyConfig#{file_ctrl_pid=>FileCtrlPid}}; Error -> Error end. -check_config(_Name,set,undefined,NewHConfig) -> - check_config(maps:merge(get_default_config(),NewHConfig)); -check_config(_Name,SetOrUpdate,OldHConfig,NewHConfig0) -> - WriteOnce = maps:with([type],OldHConfig), +check_config(Name,set,undefined,NewHConfig) -> + check_h_config(merge_default_config(Name,normalize_config(NewHConfig))); +check_config(Name,SetOrUpdate,OldHConfig,NewHConfig0) -> + WriteOnce = maps:with([type,file,modes],OldHConfig), Default = case SetOrUpdate of set -> %% Do not reset write-once fields to defaults - maps:merge(get_default_config(),WriteOnce); + merge_default_config(Name,WriteOnce); update -> OldHConfig end, - NewHConfig = maps:merge(Default, NewHConfig0), + NewHConfig = maps:merge(Default, normalize_config(NewHConfig0)), %% Fail if write-once fields are changed - case maps:with([type],NewHConfig) of + case maps:with([type,file,modes],NewHConfig) of WriteOnce -> - check_config(NewHConfig); + check_h_config(NewHConfig); Other -> {error,{illegal_config_change,?MODULE,WriteOnce,Other}} end. -check_config(#{type:=Type}=HConfig) -> - case check_h_config(maps:to_list(HConfig)) of - ok when is_atom(Type) -> - {ok,HConfig#{filesync_repeat_interval=>no_repeat}}; +check_h_config(HConfig) -> + case check_h_config(maps:get(type,HConfig),maps:to_list(HConfig)) of ok -> - {ok,HConfig}; + {ok,fix_file_opts(HConfig)}; {error,{Key,Value}} -> {error,{invalid_config,?MODULE,#{Key=>Value}}} end. -check_h_config([{type,Type} | Config]) when Type == standard_io; - Type == standard_error -> - check_h_config(Config); -check_h_config([{type,{file,File}} | Config]) when is_list(File) -> - check_h_config(Config); -check_h_config([{type,{file,File,Modes}} | Config]) when is_list(File), - is_list(Modes) -> - check_h_config(Config); -check_h_config([Other | _]) -> +check_h_config(Type,[{type,Type} | Config]) when Type =:= standard_io; + Type =:= standard_error; + Type =:= file -> + check_h_config(Type,Config); +check_h_config(file,[{file,File} | Config]) when is_list(File) -> + check_h_config(file,Config); +check_h_config(file,[{modes,Modes} | Config]) when is_list(Modes) -> + check_h_config(file,Config); +check_h_config(file,[{max_no_bytes,Size} | Config]) + when (is_integer(Size) andalso Size>0) orelse Size=:=infinity -> + check_h_config(file,Config); +check_h_config(file,[{max_no_files,Num} | Config]) when is_integer(Num), Num>=0 -> + check_h_config(file,Config); +check_h_config(file,[{compress_on_rotate,Bool} | Config]) when is_boolean(Bool) -> + check_h_config(file,Config); +check_h_config(file,[{file_check,FileCheck} | Config]) + when is_integer(FileCheck), FileCheck>=0 -> + check_h_config(file,Config); +check_h_config(_Type,[Other | _]) -> {error,Other}; -check_h_config([]) -> +check_h_config(_Type,[]) -> ok. -get_default_config() -> - #{type => standard_io}. +normalize_config(#{type:={file,File}}=HConfig) -> + HConfig#{type=>file,file=>File}; +normalize_config(#{type:={file,File,Modes}}=HConfig) -> + HConfig#{type=>file,file=>File,modes=>Modes}; +normalize_config(HConfig) -> + HConfig. + +merge_default_config(Name,#{type:=Type}=HConfig) -> + merge_default_config(Name,Type,HConfig); +merge_default_config(Name,#{file:=_}=HConfig) -> + merge_default_config(Name,file,HConfig); +merge_default_config(Name,HConfig) -> + merge_default_config(Name,standard_io,HConfig). + +merge_default_config(Name,Type,HConfig) -> + maps:merge(get_default_config(Name,Type),HConfig). + +get_default_config(Name,file) -> + #{type => file, + file => atom_to_list(Name), + modes => [raw,append], + file_check => 0, + max_no_bytes => infinity, + max_no_files => 0, + compress_on_rotate => false}; +get_default_config(_Name,Type) -> + #{type => Type}. + +fix_file_opts(#{modes:=Modes}=HConfig) -> + HConfig#{modes=>fix_modes(Modes)}; +fix_file_opts(HConfig) -> + HConfig#{filesync_repeat_interval=>no_repeat}. + +fix_modes(Modes) -> + %% Ensure write|append|exclusive + Modes1 = + case [M || M <- Modes, + lists:member(M,[write,append,exclusive])] of + [] -> [append|Modes]; + _ -> Modes + end, + %% Ensure raw + Modes2 = + case lists:member(raw,Modes) of + false -> [raw|Modes1]; + true -> Modes1 + end, + %% Ensure delayed_write + case lists:partition(fun(delayed_write) -> true; + ({delayed_write,_,_}) -> true; + (_) -> false + end, Modes2) of + {[],_} -> + [delayed_write|Modes2]; + _ -> + Modes2 + end. -filesync(_Name, _Mode, #{type := Type}=State) when is_atom(Type) -> - {ok,State}; -filesync(_Name, async, #{file_ctrl_pid := FileCtrlPid} = State) -> - ok = file_ctrl_filesync_async(FileCtrlPid), - {ok,State}; -filesync(_Name, sync, #{file_ctrl_pid := FileCtrlPid} = State) -> - Result = file_ctrl_filesync_sync(FileCtrlPid), +config_changed(_Name, + #{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}, + #{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}=State) -> + State; +config_changed(_Name, + #{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}, + #{file_ctrl_pid := FileCtrlPid} = State) -> + FileCtrlPid ! {update_config,#{file_check=>FileCheck, + max_no_bytes=>Size, + max_no_files=>Count, + compress_on_rotate=>Compress}}, + State#{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}; +config_changed(_Name,_NewHConfig,State) -> + State. + +filesync(_Name, SyncAsync, #{file_ctrl_pid := FileCtrlPid} = State) -> + Result = file_ctrl_filesync(SyncAsync, FileCtrlPid), {Result,State}. -write(_Name, async, Bin, #{file_ctrl_pid:=FileCtrlPid} = State) -> - ok = file_write_async(FileCtrlPid, Bin), - {ok,State}; -write(_Name, sync, Bin, #{file_ctrl_pid:=FileCtrlPid} = State) -> - Result = file_write_sync(FileCtrlPid, Bin), +write(_Name, SyncAsync, Bin, #{file_ctrl_pid:=FileCtrlPid} = State) -> + Result = file_write(SyncAsync, FileCtrlPid, Bin), {Result,State}. reset_state(_Name, State) -> State. -handle_info(_Name, {'EXIT',Pid,Why}, #{type := FileInfo, file_ctrl_pid := Pid}) -> +handle_info(_Name, {'EXIT',Pid,Why}, #{file_ctrl_pid := Pid}=State) -> %% file_ctrl_pid died, file error, terminate handler - exit({error,{write_failed,FileInfo,Why}}); + exit({error,{write_failed,maps:with([type,file,modes],State),Why}}); handle_info(_, _, State) -> State. @@ -211,27 +295,33 @@ terminate(_Name, _Reason, #{file_ctrl_pid:=FWPid}) -> %%%----------------------------------------------------------------- %%% -open_log_file(HandlerName, FileInfo) -> - case file_ctrl_start(HandlerName, FileInfo) of - OK = {ok,_FileCtrlPid} -> OK; - Error -> Error - end. - -do_open_log_file({file,FileName}) -> - do_open_log_file({file,FileName,[raw,append,delayed_write]}); - -do_open_log_file({file,FileName,[]}) -> - do_open_log_file({file,FileName,[raw,append,delayed_write]}); - -do_open_log_file({file,FileName,Modes}) -> +open_log_file(HandlerName,#{type:=file, + file:=FileName, + modes:=Modes, + file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}) -> try case filelib:ensure_dir(FileName) of ok -> case file:open(FileName, Modes) of {ok, Fd} -> {ok,#file_info{inode=INode}} = - file:read_file_info(FileName), - {ok, {Fd, INode}}; + file:read_file_info(FileName,[raw]), + UpdateModes = [append | Modes--[write,append,exclusive]], + State0 = #{handler_name=>HandlerName, + file_name=>FileName, + modes=>UpdateModes, + file_check=>FileCheck, + fd=>Fd, + inode=>INode, + last_check=>timestamp(), + synced=>false, + write_res=>ok, + sync_res=>ok}, + State = update_rotation({Size,Count,Compress},State0), + {ok,State}; Error -> Error end; @@ -242,21 +332,23 @@ do_open_log_file({file,FileName,Modes}) -> _:Reason -> {error,Reason} end. -close_log_file(Std) when Std == standard_io; Std == standard_error -> - ok; -close_log_file({Fd,_}) -> +close_log_file(#{fd:=Fd}) -> _ = file:datasync(Fd), - _ = file:close(Fd). + _ = file:close(Fd), + ok; +close_log_file(_) -> + ok. + %%%----------------------------------------------------------------- %%% File control process -file_ctrl_start(HandlerName, FileInfo) -> +file_ctrl_start(HandlerName, HConfig) -> Starter = self(), FileCtrlPid = spawn_link(fun() -> - file_ctrl_init(HandlerName, FileInfo, Starter) + file_ctrl_init(HandlerName, HConfig, Starter) end), receive {FileCtrlPid,ok} -> @@ -271,18 +363,16 @@ file_ctrl_start(HandlerName, FileInfo) -> file_ctrl_stop(Pid) -> Pid ! stop. -file_write_async(Pid, Bin) -> +file_write(async, Pid, Bin) -> Pid ! {log,Bin}, - ok. - -file_write_sync(Pid, Bin) -> + ok; +file_write(sync, Pid, Bin) -> file_ctrl_call(Pid, {log,Bin}). -file_ctrl_filesync_async(Pid) -> +file_ctrl_filesync(async, Pid) -> Pid ! filesync, - ok. - -file_ctrl_filesync_sync(Pid) -> + ok; +file_ctrl_filesync(sync, Pid) -> file_ctrl_call(Pid, filesync). file_ctrl_call(Pid, Msg) -> @@ -299,98 +389,255 @@ file_ctrl_call(Pid, Msg) -> {error,{no_response,Pid}} end. -file_ctrl_init(HandlerName, FileInfo, Starter) when is_tuple(FileInfo) -> +file_ctrl_init(HandlerName, + #{type:=file, + file:=FileName} = HConfig, + Starter) -> process_flag(message_queue_data, off_heap), - FileName = element(2, FileInfo), - case do_open_log_file(FileInfo) of - {ok,File} -> + case open_log_file(HandlerName,HConfig) of + {ok,State} -> Starter ! {self(),ok}, - file_ctrl_loop(File, FileName, false, ok, ok, HandlerName); + file_ctrl_loop(State); {error,Reason} -> Starter ! {self(),{error,{open_failed,FileName,Reason}}} end; -file_ctrl_init(HandlerName, StdDev, Starter) -> +file_ctrl_init(HandlerName, #{type:=StdDev}, Starter) -> Starter ! {self(),ok}, - file_ctrl_loop(StdDev, StdDev, false, ok, ok, HandlerName). + file_ctrl_loop(#{handler_name=>HandlerName,dev=>StdDev}). -file_ctrl_loop(File, DevName, Synced, - PrevWriteResult, PrevSyncResult, HandlerName) -> +file_ctrl_loop(State) -> receive %% asynchronous event {log,Bin} -> - File1 = ensure(File, DevName), - Result = write_to_dev(File1, Bin, DevName, - PrevWriteResult, HandlerName), - file_ctrl_loop(File1, DevName, false, - Result, PrevSyncResult, HandlerName); + State1 = write_to_dev(Bin,State), + file_ctrl_loop(State1); %% synchronous event {{log,Bin},{From,MRef}} -> - File1 = ensure(File, DevName), - Result = write_to_dev(File1, Bin, DevName, - PrevWriteResult, HandlerName), + State1 = ensure_file(State), + State2 = write_to_dev(Bin,State1), From ! {MRef,ok}, - file_ctrl_loop(File1, DevName, false, - Result, PrevSyncResult, HandlerName); + file_ctrl_loop(State2); filesync -> - File1 = ensure(File, DevName), - Result = sync_dev(File1, DevName, Synced, - PrevSyncResult, HandlerName), - file_ctrl_loop(File1, DevName, true, - PrevWriteResult, Result, HandlerName); + State1 = sync_dev(State), + file_ctrl_loop(State1); {filesync,{From,MRef}} -> - File1 = ensure(File, DevName), - Result = sync_dev(File1, DevName, Synced, - PrevSyncResult, HandlerName), + State1 = ensure_file(State), + State2 = sync_dev(State1), From ! {MRef,ok}, - file_ctrl_loop(File1, DevName, true, - PrevWriteResult, Result, HandlerName); + file_ctrl_loop(State2); + + {update_config,#{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}} -> + State1 = update_rotation({Size,Count,Compress},State), + file_ctrl_loop(State1#{file_check=>FileCheck}); stop -> - _ = close_log_file(File), + close_log_file(State), stopped end. +maybe_ensure_file(#{file_check:=0}=State) -> + ensure_file(State); +maybe_ensure_file(#{last_check:=T0,file_check:=CheckInt}=State) + when is_integer(CheckInt) -> + T = timestamp(), + if T-T0 > CheckInt -> ensure_file(State); + true -> State + end; +maybe_ensure_file(State) -> + State. + %% In order to play well with tools like logrotate, we need to be able %% to re-create the file if it has disappeared (e.g. if rotated by %% logrotate) -ensure(Fd,DevName) when is_atom(DevName) -> - Fd; -ensure({Fd,INode},FileName) -> - case file:read_file_info(FileName) of - {ok,#file_info{inode=INode}} -> - {Fd,INode}; +ensure_file(#{fd:=Fd0,inode:=INode0,file_name:=FileName,modes:=Modes}=State) -> + case file:read_file_info(FileName,[raw]) of + {ok,#file_info{inode=INode0}} -> + State#{last_check=>timestamp()}; _ -> - _ = file:close(Fd), - _ = file:close(Fd), % delayed_write cause close not to close - case do_open_log_file({file,FileName}) of - {ok,File} -> - File; + close_log_file(Fd0), + case file:open(FileName,Modes) of + {ok,Fd} -> + {ok,#file_info{inode=INode}} = + file:read_file_info(FileName,[raw]), + State#{fd=>Fd,inode=>INode, + last_check=>timestamp(), + synced=>true,sync_res=>ok}; Error -> exit({could_not_reopen_file,Error}) end - end. + end; +ensure_file(State) -> + State. -write_to_dev(DevName, Bin, _DevName, _PrevWriteResult, _HandlerName) - when is_atom(DevName) -> - io:put_chars(DevName, Bin); -write_to_dev({Fd,_}, Bin, FileName, PrevWriteResult, HandlerName) -> +write_to_dev(Bin,#{dev:=DevName}=State) -> + io:put_chars(DevName, Bin), + State; +write_to_dev(Bin, State) -> + State1 = #{fd:=Fd} = maybe_ensure_file(State), Result = ?file_write(Fd, Bin), - maybe_notify_error(write,Result,PrevWriteResult,FileName,HandlerName). + State2 = maybe_rotate_file(Bin,State1), + maybe_notify_error(write,Result,State2), + State2#{synced=>false,write_res=>Result}. -sync_dev(_, _FileName, true, PrevSyncResult, _HandlerName) -> - PrevSyncResult; -sync_dev({Fd,_}, FileName, false, PrevSyncResult, HandlerName) -> +sync_dev(#{synced:=false}=State) -> + State1 = #{fd:=Fd} = maybe_ensure_file(State), Result = ?file_datasync(Fd), - maybe_notify_error(filesync,Result,PrevSyncResult,FileName,HandlerName). + maybe_notify_error(filesync,Result,State1), + State1#{synced=>true,sync_res=>Result}; +sync_dev(State) -> + State. -maybe_notify_error(_Op, ok, _PrevResult, _FileName, _HandlerName) -> +update_rotation({infinity,_,_},State) -> + maybe_remove_archives(0,State), + maps:remove(rotation,State); +update_rotation({Size,Count,Compress},#{file_name:=FileName} = State) -> + maybe_remove_archives(Count,State), + {ok,#file_info{size=CurrSize}} = file:read_file_info(FileName,[raw]), + State1 = State#{rotation=>#{size=>Size, + count=>Count, + compress=>Compress, + curr_size=>CurrSize}}, + maybe_update_compress(0,State1), + maybe_rotate_file(0,State1). + +maybe_remove_archives(Count,#{file_name:=FileName}=State) -> + Archive = rot_file_name(FileName,Count,false), + CompressedArchive = rot_file_name(FileName,Count,true), + case {file:read_file_info(Archive,[raw]), + file:read_file_info(CompressedArchive,[raw])} of + {{error,enoent},{error,enoent}} -> + ok; + _ -> + _ = file:delete(Archive), + _ = file:delete(CompressedArchive), + maybe_remove_archives(Count+1,State) + end. + +maybe_update_compress(Count,#{rotation:=#{count:=Count}}) -> + ok; +maybe_update_compress(N,#{file_name:=FileName, + rotation:=#{compress:=Compress}}=State) -> + Archive = rot_file_name(FileName,N,not Compress), + case file:read_file_info(Archive,[raw]) of + {ok,_} when Compress -> + compress_file(Archive); + {ok,_} -> + decompress_file(Archive); + _ -> + ok + end, + maybe_update_compress(N+1,State). + +maybe_rotate_file(Bin,#{rotation:=_}=State) when is_binary(Bin) -> + maybe_rotate_file(byte_size(Bin),State); +maybe_rotate_file(AddSize,#{rotation:=#{size:=RotSize, + curr_size:=CurrSize}=Rotation}=State) -> + NewSize = CurrSize + AddSize, + if NewSize>RotSize -> + rotate_file(State#{rotation=>Rotation#{curr_size=>NewSize}}); + true -> + State#{rotation=>Rotation#{curr_size=>NewSize}} + end; +maybe_rotate_file(_Bin,State) -> + State. + +rotate_file(#{fd:=Fd0,file_name:=FileName,modes:=Modes,rotation:=Rotation}=State) -> + State1 = sync_dev(State), + _ = file:close(Fd0), + _ = file:close(Fd0), + rotate_files(FileName,maps:get(count,Rotation),maps:get(compress,Rotation)), + case file:open(FileName,Modes) of + {ok,Fd} -> + {ok,#file_info{inode=INode}} = file:read_file_info(FileName,[raw]), + State1#{fd=>Fd,inode=>INode,rotation=>Rotation#{curr_size=>0}}; + Error -> + exit({could_not_reopen_file,Error}) + end. + +rotate_files(FileName,0,_Compress) -> + _ = file:delete(FileName), + ok; +rotate_files(FileName,1,Compress) -> + FileName0 = FileName++".0", + _ = file:rename(FileName,FileName0), + if Compress -> compress_file(FileName0); + true -> ok + end, ok; -maybe_notify_error(_Op, PrevResult, PrevResult, _FileName, _HandlerName) -> +rotate_files(FileName,Count,Compress) -> + _ = file:rename(rot_file_name(FileName,Count-2,Compress), + rot_file_name(FileName,Count-1,Compress)), + rotate_files(FileName,Count-1,Compress). + +rot_file_name(FileName,Count,false) -> + FileName ++ "." ++ integer_to_list(Count); +rot_file_name(FileName,Count,true) -> + rot_file_name(FileName,Count,false) ++ ".gz". + +compress_file(FileName) -> + {ok,In} = file:open(FileName,[read,binary]), + {ok,Out} = file:open(FileName++".gz",[write]), + Z = zlib:open(), + zlib:deflateInit(Z, default, deflated, 31, 8, default), + compress_data(Z,In,Out), + zlib:deflateEnd(Z), + zlib:close(Z), + _ = file:close(In), + _ = file:close(Out), + _ = file:delete(FileName), + ok. + +compress_data(Z,In,Out) -> + case file:read(In,100000) of + {ok,Data} -> + Compressed = zlib:deflate(Z, Data), + _ = file:write(Out,Compressed), + compress_data(Z,In,Out); + eof -> + Compressed = zlib:deflate(Z, <<>>, finish), + _ = file:write(Out,Compressed), + ok + end. + +decompress_file(FileName) -> + {ok,In} = file:open(FileName,[read,binary]), + {ok,Out} = file:open(filename:rootname(FileName,".gz"),[write]), + Z = zlib:open(), + zlib:inflateInit(Z, 31), + decompress_data(Z,In,Out), + zlib:inflateEnd(Z), + zlib:close(Z), + _ = file:close(In), + _ = file:close(Out), + _ = file:delete(FileName), + ok. + +decompress_data(Z,In,Out) -> + case file:read(In,1000) of + {ok,Data} -> + Decompressed = zlib:inflate(Z, Data), + _ = file:write(Out,Decompressed), + decompress_data(Z,In,Out); + eof -> + ok + end. + +maybe_notify_error(_Op, ok, _State) -> + ok; +maybe_notify_error(Op, Result, #{write_res:=WR,sync_res:=SR}) + when (Op==write andalso Result==WR) orelse + (Op==filesync andalso Result==SR) -> %% don't report same error twice - PrevResult; -maybe_notify_error(Op, Error, _PrevResult, FileName, HandlerName) -> + ok; +maybe_notify_error(Op, Error, #{handler_name:=HandlerName,file_name:=FileName}) -> logger_h_common:error_notify({HandlerName,Op,FileName,Error}), - Error. + ok. + +timestamp() -> + erlang:monotonic_time(millisecond). diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl index 64e0b9d8dd..99fecbe970 100644 --- a/lib/kernel/test/code_SUITE.erl +++ b/lib/kernel/test/code_SUITE.erl @@ -140,6 +140,11 @@ end_per_testcase(on_load_embedded, Config) -> LinkName = proplists:get_value(link_name, Config), _ = del_link(LinkName), end_per_testcase(Config); +end_per_testcase(upgrade, Config) -> + %% Make sure tracing is turned off even if the test times out. + erlang:trace_pattern({error_handler,undefined_function,3}, false, [global]), + erlang:trace(self(), false, [call]), + end_per_testcase(Config); end_per_testcase(_Func, Config) -> end_per_testcase(Config). @@ -1556,6 +1561,11 @@ on_load_update_code_1(3, Mod) -> %% Test -on_load while trace feature 'on_load' is enabled (OTP-14612) on_load_trace_on_load(Config) -> + %% 'on_load' enables tracing for all newly loaded modules, so we make a dry + %% run to ensure that ancillary modules like 'merl' won't be loaded during + %% the actual test. + on_load_update(Config), + Papa = self(), Tracer = spawn_link(fun F() -> receive M -> Papa ! M end, F() end), {tracer,[]} = erlang:trace_info(self(),tracer), diff --git a/lib/kernel/test/gen_sctp_SUITE.erl b/lib/kernel/test/gen_sctp_SUITE.erl index a0ae792ba9..e4c489bd10 100644 --- a/lib/kernel/test/gen_sctp_SUITE.erl +++ b/lib/kernel/test/gen_sctp_SUITE.erl @@ -1459,11 +1459,11 @@ do_open_and_connect(ServerAddresses, AddressToConnectTo) -> do_open_and_connect(ServerAddresses, AddressToConnectTo, Fun). %% do_open_and_connect(ServerAddresses, AddressToConnectTo, Fun) -> - ServerFamily = get_family_by_addrs(ServerAddresses), + {ServerFamily, ServerOpts} = get_family_by_addrs(ServerAddresses), io:format("Serving ~p addresses: ~p~n", [ServerFamily, ServerAddresses]), S1 = ok(gen_sctp:open(0, [{ip,Addr} || Addr <- ServerAddresses] ++ - [ServerFamily])), + [ServerFamily|ServerOpts])), ok = gen_sctp:listen(S1, true), P1 = ok(inet:port(S1)), ClientFamily = get_family_by_addr(AddressToConnectTo), @@ -1493,9 +1493,9 @@ do_open_and_connect(ServerAddresses, AddressToConnectTo, Fun) -> %% If at least one of the addresses is an ipv6 address, return inet6, else inet. get_family_by_addrs(Addresses) -> case lists:usort([get_family_by_addr(Addr) || Addr <- Addresses]) of - [inet, inet6] -> inet6; - [inet] -> inet; - [inet6] -> inet6 + [inet, inet6] -> {inet6, [{ipv6_v6only, false}]}; + [inet] -> {inet, []}; + [inet6] -> {inet6, []} end. get_family_by_addr(Addr) when tuple_size(Addr) =:= 4 -> inet; diff --git a/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c b/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c index b91dca61d4..96938f9071 100644 --- a/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c +++ b/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c @@ -30,6 +30,7 @@ #define sock_close(s) closesocket(s) #else #include <sys/socket.h> +#include <unistd.h> #define sock_close(s) close(s) #endif diff --git a/lib/kernel/test/gen_tcp_misc_SUITE.erl b/lib/kernel/test/gen_tcp_misc_SUITE.erl index 52edfaee29..edf30448c4 100644 --- a/lib/kernel/test/gen_tcp_misc_SUITE.erl +++ b/lib/kernel/test/gen_tcp_misc_SUITE.erl @@ -2086,8 +2086,39 @@ test_pktoptions(Family, Spec, CheckConnect, OSType, OSVer) -> %%% {ok,<<"hi">>} = gen_tcp:recv(S1, 2, Timeout), %% %% Verify returned remote options - {ok,[{pktoptions,OptsVals1}]} = inet:getopts(S1, [pktoptions]), - {ok,[{pktoptions,OptsVals2}]} = inet:getopts(S2, [pktoptions]), + VerifyRemOpts = + fun(S, Role) -> + case inet:getopts(S, [pktoptions]) of + {ok, [{pktoptions, PktOpts1}]} -> + PktOpts1; + {ok, UnexpOK1} -> + io:format("Unexpected OK (~w): " + "~n ~p" + "~n", [Role, UnexpOK1]), + exit({unexpected_getopts_ok, + Role, + Spec, + TrueRecvOpts, + OptsVals, + OptsValsDefault, + UnexpOK1}); + {error, UnexpERR1} -> + io:format("Unexpected ERROR (~w): " + "~n ~p" + "~n", [Role, UnexpERR1]), + exit({unexpected_getopts_failure, + Role, + Spec, + TrueRecvOpts, + OptsVals, + OptsValsDefault, + UnexpERR1}) + end + end, + OptsVals1 = VerifyRemOpts(S1, dest), + OptsVals2 = VerifyRemOpts(S2, orig), + %% {ok,[{pktoptions,OptsVals1}]} = inet:getopts(S1, [pktoptions]), + %% {ok,[{pktoptions,OptsVals2}]} = inet:getopts(S2, [pktoptions]), (Result1 = sets_eq(OptsVals1, OptsVals)) orelse io:format( "Accept differs: ~p neq ~p~n", [OptsVals1,OptsVals]), @@ -3430,7 +3461,7 @@ wait(Mref) -> %% OTP-15536 %% Test that send error works correctly for delay_send -delay_send_error(Config) -> +delay_send_error(_Config) -> {ok, LS} = gen_tcp:listen(0, [{reuseaddr, true}, {packet, 1}, {active, false}]), {ok,{{0,0,0,0},PortNum}}=inet:sockname(LS), P = spawn_link( diff --git a/lib/kernel/test/kernel.spec b/lib/kernel/test/kernel.spec index 62afc9f97b..eaa17f3a59 100644 --- a/lib/kernel/test/kernel.spec +++ b/lib/kernel/test/kernel.spec @@ -2,3 +2,4 @@ {config, "../test_server/ts.unix.config"}. {suites,"../kernel_test", all}. +{skip_suites,"../kernel_test",[logger_stress_SUITE],"Benchmarks only"}. diff --git a/lib/kernel/test/kernel_config_SUITE.erl b/lib/kernel/test/kernel_config_SUITE.erl index 9207025a2c..57c44c2498 100644 --- a/lib/kernel/test/kernel_config_SUITE.erl +++ b/lib/kernel/test/kernel_config_SUITE.erl @@ -21,7 +21,8 @@ -include_lib("common_test/include/ct.hrl"). --export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2, sync/1]). +-export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2, + start_distribution_false/1, sync/1]). -export([init_per_suite/1, end_per_suite/1]). @@ -30,7 +31,7 @@ suite() -> {timetrap,{minutes,2}}]. all() -> - [sync]. + [sync, start_distribution_false]. groups() -> []. @@ -59,12 +60,9 @@ from(H, [H | T]) -> T; from(H, [_ | T]) -> from(H, T); from(_, []) -> []. -%%----------------------------------------------------------------- -%% Test suite for sync_nodes. This is quite tricky. -%% +%% Test sync_nodes. This is quite tricky. %% Should be started in a CC view with: %% erl -sname XXX where XX not in [cp1, cp2] -%%----------------------------------------------------------------- sync(Conf) when is_list(Conf) -> %% Write a config file Dir = proplists:get_value(priv_dir,Conf), @@ -106,9 +104,18 @@ wait_for_node(Node) -> _Other -> wait_for_node(Node) end. - stop_node(Node) -> M = list_to_atom(lists:concat([Node, [$@], from($@,atom_to_list(node()))])), rpc:cast(M, erlang, halt, []). + +start_distribution_false(Config) when is_list(Config) -> + %% When distribution is disabled, -sname/-name has no effect + Str = os:cmd(ct:get_progname() + ++ " -kernel start_distribution false" + ++ " -sname no_distribution" + ++ " -eval \"erlang:display(node())\"" + ++ " -noshell -s erlang halt"), + "'nonode@nohost'" ++ _ = Str, + ok. diff --git a/lib/kernel/test/logger_SUITE.erl b/lib/kernel/test/logger_SUITE.erl index 70bb775db8..035e5d8974 100644 --- a/lib/kernel/test/logger_SUITE.erl +++ b/lib/kernel/test/logger_SUITE.erl @@ -1048,8 +1048,11 @@ kernel_config(Config) -> ok = rpc:call(Node,logger,internal_init_logger,[]), ok = rpc:call(Node,logger,add_handlers,[kernel]), #{primary:=#{filter_default:=log,filters:=[]}, - handlers:=[#{id:=default,filters:=DF,config:=#{type:={file,F}}}], + handlers:=[#{id:=default,filters:=DF, + config:=#{type:=file,file:=F,modes:=Modes}}], module_levels:=[]} = rpc:call(Node,logger,get_config,[]), + [append,delayed_write,raw] = lists:sort(Modes), + %% Same, but using 'logger' parameter instead of 'error_logger' ok = rpc:call(Node,logger,remove_handler,[default]),% so it can be added again @@ -1060,26 +1063,27 @@ kernel_config(Config) -> ok = rpc:call(Node,logger,internal_init_logger,[]), ok = rpc:call(Node,logger,add_handlers,[kernel]), #{primary:=#{filter_default:=log,filters:=[]}, - handlers:=[#{id:=default,filters:=DF,config:=#{type:={file,F}}}], + handlers:=[#{id:=default,filters:=DF, + config:=#{type:=file,file:=F,modes:=Modes}}], module_levels:=[]} = rpc:call(Node,logger,get_config,[]), %% Same, but with type={file,File,Modes} ok = rpc:call(Node,logger,remove_handler,[default]),% so it can be added again ok = rpc:call(Node,application,unset_env,[kernel,error_logger]), - M = [raw,write,delayed_write], + M = [raw,write], ok = rpc:call(Node,application,set_env,[kernel,logger, [{handler,default,logger_std_h, #{config=>#{type=>{file,F,M}}}}]]), ok = rpc:call(Node,logger,internal_init_logger,[]), ok = rpc:call(Node,logger,add_handlers,[kernel]), #{primary:=#{filter_default:=log,filters:=[]}, - handlers:=[#{id:=default,filters:=DF,config:=#{type:={file,F,M}}}], + handlers:=[#{id:=default,filters:=DF, + config:=#{type:=file,file:=F,modes:=[delayed_write|M]}}], module_levels:=[]} = rpc:call(Node,logger,get_config,[]), %% Same, but with disk_log handler ok = rpc:call(Node,logger,remove_handler,[default]),% so it can be added again ok = rpc:call(Node,application,unset_env,[kernel,error_logger]), - M = [raw,write,delayed_write], ok = rpc:call(Node,application,set_env,[kernel,logger, [{handler,default,logger_disk_log_h, #{config=>#{file=>F}}}]]), diff --git a/lib/kernel/test/logger_std_h_SUITE.erl b/lib/kernel/test/logger_std_h_SUITE.erl index b2c2c8ba67..0c5516f82b 100644 --- a/lib/kernel/test/logger_std_h_SUITE.erl +++ b/lib/kernel/test/logger_std_h_SUITE.erl @@ -112,6 +112,8 @@ all() -> add_remove_instance_standard_error, add_remove_instance_file1, add_remove_instance_file2, + add_remove_instance_file3, + add_remove_instance_file4, default_formatter, filter_config, errors, @@ -142,7 +144,12 @@ all() -> restart_after, handler_requests_under_load, recreate_deleted_log, - reopen_changed_log + reopen_changed_log, + rotate_size, + rotate_size_compressed, + rotate_size_reopen, + rotation_opts, + rotation_opts_restart_handler ]. add_remove_instance_tty(_Config) -> @@ -179,10 +186,27 @@ add_remove_instance_file2(Config) -> add_remove_instance_file2(cleanup,_Config) -> logger_std_h_remove(). -add_remove_instance_file(Log, Type) -> +add_remove_instance_file3(_Config) -> + Log = atom_to_list(?MODULE), + StdHConfig = #{type=>file}, + add_remove_instance_file(Log, StdHConfig). +add_remove_instance_file3(cleanup,_Config) -> + logger_std_h_remove(). + +add_remove_instance_file4(Config) -> + Dir = ?config(priv_dir,Config), + Log = filename:join(Dir,"stdlog4.txt"), + StdHConfig = #{file=>Log,modes=>[]}, + add_remove_instance_file(Log, StdHConfig). +add_remove_instance_file4(cleanup,_Config) -> + logger_std_h_remove(). + +add_remove_instance_file(Log, Type) when not is_map(Type) -> + add_remove_instance_file(Log,#{type=>Type}); +add_remove_instance_file(Log, StdHConfig) when is_map(StdHConfig) -> ok = logger:add_handler(?MODULE, logger_std_h, - #{config => #{type => Type}, + #{config => StdHConfig, filter_default=>stop, filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), formatter=>{?MODULE,self()}}), @@ -257,9 +281,10 @@ errors(Config) -> end, {error, - {handler_not_added,{open_failed,Log,_}}} = + {handler_not_added, + {invalid_config,logger_std_h,#{modes:=bad_file_opt}}}} = logger:add_handler(myh3,logger_std_h, - #{config=>#{type=>{file,Log,[bad_file_opt]}}}), + #{config=>#{type=>{file,Log,bad_file_opt}}}), ok = logger:notice(?msg). @@ -607,24 +632,51 @@ reconfig(cleanup, _Config) -> file_opts(Config) -> Dir = ?config(priv_dir,Config), Log = filename:join(Dir, lists:concat([?FUNCTION_NAME,".log"])), - BadFileOpts = [raw], - BadType = {file,Log,BadFileOpts}, - {error,{handler_not_added,{open_failed,Log,enoent}}} = - logger:add_handler(?MODULE, logger_std_h, - #{config => #{type => BadType}}), + MissingOpts = [raw], + Type1 = {file,Log,MissingOpts}, + ok = logger:add_handler(?MODULE, logger_std_h, + #{config => #{type => Type1}}), + {ok,#{config:=#{type:=file,file:=Log,modes:=Modes1}}} = + logger:get_handler_config(?MODULE), + [append,delayed_write,raw] = lists:sort(Modes1), + ok = logger:remove_handler(?MODULE), OkFileOpts = [raw,append], OkType = {file,Log,OkFileOpts}, ok = logger:add_handler(?MODULE, logger_std_h, - #{config => #{type => OkType}, + #{config => #{type => OkType}, % old format filter_default=>log, filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), formatter=>{?MODULE,self()}}), - #{cb_state := #{handler_state := #{type := OkType}}} = + ModOpts = [delayed_write|OkFileOpts], + #{cb_state := #{handler_state := #{type:=file, + file:=Log, + modes:=ModOpts}}} = logger_olp:info(h_proc_name()), - {ok,#{config := #{type := OkType}}} = logger:get_handler_config(?MODULE), + {ok,#{config := #{type:=file, + file:=Log, + modes:=ModOpts}}} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + + ok = logger:add_handler(?MODULE, + logger_std_h, + #{config => #{type => file, + file => Log, + modes => OkFileOpts}, % new format + filter_default=>log, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter=>{?MODULE,self()}}), + + #{cb_state := #{handler_state := #{type:=file, + file:=Log, + modes:=ModOpts}}} = + logger_olp:info(h_proc_name()), + {ok,#{config := #{type:=file, + file:=Log, + modes:=ModOpts}}} = + logger:get_handler_config(?MODULE), logger:notice(M1=?msg,?domain), ?check(M1), B1 = ?bin(M1), @@ -640,13 +692,14 @@ sync(Config) -> Type = {file,Log}, ok = logger:add_handler(?MODULE, logger_std_h, - #{config => #{type => Type}, + #{config => #{type => Type, + file_check => 10000}, filter_default=>log, filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), formatter=>{?MODULE,nl}}), %% check repeated filesync happens - start_tracer([{logger_std_h, write_to_dev, 5}, + start_tracer([{logger_std_h, write_to_dev, 2}, {file, datasync, 1}], [{logger_std_h, write_to_dev, <<"first\n">>}, {file,datasync}]), @@ -656,7 +709,7 @@ sync(Config) -> check_tracer(filesync_rep_int()*2), %% check that explicit filesync is only done once - start_tracer([{logger_std_h, write_to_dev, 5}, + start_tracer([{logger_std_h, write_to_dev, 2}, {file, datasync, 1}], [{logger_std_h, write_to_dev, <<"second\n">>}, {file,datasync}, @@ -675,7 +728,7 @@ sync(Config) -> #{filesync_repeat_interval => no_repeat}), no_repeat = maps:get(filesync_repeat_interval, maps:get(cb_state, logger_olp:info(h_proc_name()))), - start_tracer([{logger_std_h, write_to_dev, 5}, + start_tracer([{logger_std_h, write_to_dev, 2}, {file, datasync, 1}], [{logger_std_h, write_to_dev, <<"third\n">>}, {file,datasync}, @@ -1285,6 +1338,331 @@ reopen_changed_log(Config) -> reopen_changed_log(cleanup, _Config) -> ok = stop_handler(?MODULE). +rotate_size(Config) -> + {Log,_HConfig,_StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + ok = logger:update_handler_config(?MODULE,#{config=>#{max_no_bytes=>1000, + max_no_files=>2}}), + + Str = lists:duplicate(19,$a), + [logger:notice(Str,?domain) || _ <- lists:seq(1,50)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=1000}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + + logger:notice(Str,?domain), + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".0"), + {error,enoent} = file:read_file_info(Log++".1"), + + [logger:notice(Str,?domain) || _ <- lists:seq(1,51)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".0"), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".1"), + {error,enoent} = file:read_file_info(Log++".2"), + + [logger:notice(Str,?domain) || _ <- lists:seq(1,50)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=1000}} = file:read_file_info(Log), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".0"), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".1"), + {error,enoent} = file:read_file_info(Log++".2"), + + logger:notice("bbbb",?domain), + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=1005}} = file:read_file_info(Log++".0"), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".1"), + {error,enoent} = file:read_file_info(Log++".2"), + + ok. +rotate_size(cleanup,_Config) -> + ok = stop_handler(?MODULE). + +rotate_size_compressed(Config) -> + {Log,_HConfig,_StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + ok = logger:update_handler_config(?MODULE, + #{config=>#{max_no_bytes=>1000, + max_no_files=>2, + compress_on_rotate=>true}}), + Str = lists:duplicate(19,$a), + [logger:notice(Str,?domain) || _ <- lists:seq(1,50)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=1000}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + {error,enoent} = file:read_file_info(Log++".0.gz"), + + logger:notice(Str,?domain), + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".0.gz"), + {error,enoent} = file:read_file_info(Log++".1"), + {error,enoent} = file:read_file_info(Log++".1.gz"), + + [logger:notice(Str,?domain) || _ <- lists:seq(1,51)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".0.gz"), + {error,enoent} = file:read_file_info(Log++".1"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".1.gz"), + {error,enoent} = file:read_file_info(Log++".2"), + {error,enoent} = file:read_file_info(Log++".2.gz"), + + [logger:notice(Str,?domain) || _ <- lists:seq(1,50)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=1000}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".0.gz"), + {error,enoent} = file:read_file_info(Log++".1"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".1.gz"), + {error,enoent} = file:read_file_info(Log++".2"), + {error,enoent} = file:read_file_info(Log++".2.gz"), + + logger:notice("bbbb",?domain), + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + {ok,#file_info{size=38}} = file:read_file_info(Log++".0.gz"), + {error,enoent} = file:read_file_info(Log++".1"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".1.gz"), + {error,enoent} = file:read_file_info(Log++".2"), + {error,enoent} = file:read_file_info(Log++".2.gz"), + + ok. +rotate_size_compressed(cleanup,_Config) -> + ok = stop_handler(?MODULE). + +rotate_size_reopen(Config) -> + {Log,_HConfig,_StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + ok = logger:update_handler_config(?MODULE,#{config=>#{max_no_bytes=>1000, + max_no_files=>2}}), + + Str = lists:duplicate(19,$a), + [logger:notice(Str,?domain) || _ <- lists:seq(1,40)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=800}} = file:read_file_info(Log), + + {ok,HConfig} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + ok = logger:add_handler(?MODULE,maps:get(module,HConfig),HConfig), + {ok,#file_info{size=800}} = file:read_file_info(Log), + + [logger:notice(Str,?domain) || _ <- lists:seq(1,40)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=580}} = file:read_file_info(Log), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".0"), + ok. +rotate_size_reopen(cleanup,_Config) -> + ok = stop_handler(?MODULE). + +rotation_opts(Config) -> + {Log,_HConfig,StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + #{max_no_bytes:=infinity, + max_no_files:=0, + compress_on_rotate:=false} = StdHConfig, + + %% Test bad rotation config + {error,{invalid_config,_,_}} = + logger:update_handler_config(?MODULE,config,#{max_no_bytes=>0}), + {error,{invalid_config,_,_}} = + logger:update_handler_config(?MODULE,config,#{max_no_files=>infinity}), + {error,{invalid_config,_,_}} = + logger:update_handler_config(?MODULE,config, + #{compress_on_rotate=>undefined}), + + + %% Test good rotation config - start with no rotation + Str = lists:duplicate(19,$a), + [logger:notice(Str,?domain) || _ <- lists:seq(1,10)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=200}} = file:read_file_info(Log), + [] = filelib:wildcard(Log++".*"), + + %% Turn on rotation, check that existing file is rotated since its + %% size exceeds max_no_bytes + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_bytes=>100, + max_no_files=>2}), + timer:sleep(100), % give some time to execute config_changed + {ok,#file_info{size=0}} = file:read_file_info(Log), + Log0 = Log++".0", + {ok,#file_info{size=200}} = file:read_file_info(Log0), + [Log0] = filelib:wildcard(Log++".*"), + + %% Fill all logs + [logger:notice(Str,?domain) || _ <- lists:seq(1,13)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=20}} = file:read_file_info(Log), + {ok,#file_info{size=120}} = file:read_file_info(Log0), + {ok,#file_info{size=120}} = file:read_file_info(Log++".1"), + [_,_] = filelib:wildcard(Log++".*"), + + %% Extend size and count and check that nothing changes with existing files + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_bytes=>200, + max_no_files=>3}), + timer:sleep(100), % give some time to execute config_changed + {ok,#file_info{size=20}} = file:read_file_info(Log), + {ok,#file_info{size=120}} = file:read_file_info(Log0), + {ok,#file_info{size=120}} = file:read_file_info(Log++".1"), + [_,_] = filelib:wildcard(Log++".*"), + + %% Add more log events and see that extended size and count works + [logger:notice(Str,?domain) || _ <- lists:seq(1,10)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=220}} = file:read_file_info(Log0), + {ok,#file_info{size=120}} = file:read_file_info(Log++".1"), + {ok,#file_info{size=120}} = file:read_file_info(Log++".2"), + [_,_,_] = filelib:wildcard(Log++".*"), + + %% Reduce count and check that archive files that exceed the new + %% count are moved + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_files=>1}), + timer:sleep(100), % give some time to execute config_changed + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=220}} = file:read_file_info(Log0), + [Log0] = filelib:wildcard(Log++".*"), + + %% Extend size and count again, and turn on compression. Check + %% that archives are compressed + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_bytes=>100, + max_no_files=>2, + compress_on_rotate=>true}), + timer:sleep(100), % give some time to execute config_changed + {ok,#file_info{size=0}} = file:read_file_info(Log), + Log0gz = Log0++".gz", + {ok,#file_info{size=29}} = file:read_file_info(Log0gz), + [Log0gz] = filelib:wildcard(Log++".*"), + + %% Fill all logs + [logger:notice(Str,?domain) || _ <- lists:seq(1,13)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=20}} = file:read_file_info(Log), + {ok,#file_info{size=29}} = file:read_file_info(Log0gz), + {ok,#file_info{size=29}} = file:read_file_info(Log++".1.gz"), + [_,_] = filelib:wildcard(Log++".*"), + + %% Reduce count and turn off compression. Check that archives that + %% exceeds the new count are removed, and the rest are + %% uncompressed. + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_files=>1, + compress_on_rotate=>false}), + timer:sleep(100), % give some time to execute config_changed + {ok,#file_info{size=20}} = file:read_file_info(Log), + {ok,#file_info{size=120}} = file:read_file_info(Log0), + [Log0] = filelib:wildcard(Log++".*"), + + %% Check that config and handler state agree on the current rotation settings + {ok,#{config:=#{max_no_bytes:=100, + max_no_files:=1, + compress_on_rotate:=false}}} = + logger:get_handler_config(?MODULE), + #{cb_state:=#{handler_state:=#{max_no_bytes:=100, + max_no_files:=1, + compress_on_rotate:=false}}} = + logger_olp:info(h_proc_name()), + ok. +rotation_opts(cleanup,_Config) -> + ok = stop_handler(?MODULE). + +rotation_opts_restart_handler(Config) -> + {Log,_HConfig,_StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_bytes=>100, + max_no_files=>2}), + + %% Fill all logs + Str = lists:duplicate(19,$a), + [logger:notice(Str,?domain) || _ <- lists:seq(1,15)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=60}} = file:read_file_info(Log), + {ok,#file_info{size=120}} = file:read_file_info(Log++".0"), + {ok,#file_info{size=120}} = file:read_file_info(Log++".1"), + [_,_] = filelib:wildcard(Log++".*"), + + %% Stop/start handler and turn off rotation. Check that archives are removed. + {ok,#{config:=StdHConfig1}=HConfig1} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + ok = logger:add_handler( + ?MODULE,logger_std_h, + HConfig1#{config=>StdHConfig1#{max_no_bytes=>infinity}}), + timer:sleep(100), + {ok,#file_info{size=60}} = file:read_file_info(Log), + [] = filelib:wildcard(Log++".*"), + + %% Add some log events and check that file is no longer rotated. + [logger:notice(Str,?domain) || _ <- lists:seq(1,10)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=260}} = file:read_file_info(Log), + [] = filelib:wildcard(Log++".*"), + + %% Stop/start handler and trun on rotation. Check that file is rotated. + {ok,#{config:=StdHConfig2}=HConfig2} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + ok = logger:add_handler( + ?MODULE,logger_std_h, + HConfig2#{config=>StdHConfig2#{max_no_bytes=>100, + max_no_files=>2}}), + timer:sleep(100), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=260}} = file:read_file_info(Log++".0"), + [_] = filelib:wildcard(Log++".*"), + + %% Fill all logs + [logger:notice(Str,?domain) || _ <- lists:seq(1,10)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=80}} = file:read_file_info(Log), + {ok,#file_info{size=120}} = file:read_file_info(Log++".0"), + {ok,#file_info{size=260}} = file:read_file_info(Log++".1"), + + %% Stop/start handler, reduce count and turn on compression. Check + %% that excess archives are removed, and the rest compressed. + {ok,#{config:=StdHConfig3}=HConfig3} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + ok = logger:add_handler( + ?MODULE,logger_std_h, + HConfig3#{config=>StdHConfig3#{max_no_bytes=>75, + max_no_files=>1, + compress_on_rotate=>true}}), + timer:sleep(100), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=29}} = file:read_file_info(Log++".0.gz"), + [_] = filelib:wildcard(Log++".*"), + + %% Stop/start handler and turn off compression. Check that achives + %% are decompressed. + {ok,#{config:=StdHConfig4}=HConfig4} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + ok = logger:add_handler( + ?MODULE,logger_std_h, + HConfig4#{config=>StdHConfig4#{compress_on_rotate=>false}}), + timer:sleep(100), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=80}} = file:read_file_info(Log++".0"), + [_] = filelib:wildcard(Log++".*"), + + ok. +rotation_opts_restart_handler(cleanup,_Config) -> + ok = stop_handler(?MODULE). + %%%----------------------------------------------------------------- %%% send_requests(TO, Reqs = [{Mod,Func,Args,Res}|Rs]) -> @@ -1305,8 +1683,8 @@ start_handler(Name, TTY, _Config) when TTY == standard_io; ok = logger:add_handler(Name, logger_std_h, #{config => #{type => TTY}, - filter_default=>log, - filters=>?DEFAULT_HANDLER_FILTERS([Name]), + filter_default=>stop, + filters=>filter_only_this_domain(Name), formatter=>{?MODULE,op}}), {ok,HConfig = #{config := StdHConfig}} = logger:get_handler_config(Name), {HConfig,StdHConfig}; @@ -1320,12 +1698,17 @@ start_handler(Name, FuncName, Config) -> ok = logger:add_handler(Name, logger_std_h, #{config => #{type => Type}, - filter_default=>log, - filters=>?DEFAULT_HANDLER_FILTERS([Name]), + filter_default=>stop, + filters=>filter_only_this_domain(Name), formatter=>{?MODULE,op}}), {ok,HConfig = #{config := StdHConfig}} = logger:get_handler_config(Name), {Log,HConfig,StdHConfig}. + +filter_only_this_domain(Name) -> + [{remote_gl,{fun logger_filters:remote_gl/2,stop}}, + {domain,{fun logger_filters:domain/2,{log,super,[Name]}}}]. + stop_handler(Name) -> R = logger:remove_handler(Name), ct:pal("Handler ~p stopped! Result: ~p", [Name,R]), @@ -1658,7 +2041,7 @@ tpl([]) -> tracer({trace,_,call,{logger_h_common,handle_cast,[Op|_]}}, {Pid,[{Mod,Func,Op}|Expected]}) -> maybe_tracer_done(Pid,Expected,{Mod,Func,Op}); -tracer({trace,_,call,{Mod=logger_std_h,Func=write_to_dev,[_,Data,_,_,_]}}, +tracer({trace,_,call,{Mod=logger_std_h,Func=write_to_dev,[Data,_]}}, {Pid,[{Mod,Func,Data}|Expected]}) -> maybe_tracer_done(Pid,Expected,{Mod,Func,Data}); tracer({trace,_,call,{Mod,Func,_}}, {Pid,[{Mod,Func}|Expected]}) -> @@ -1742,4 +2125,3 @@ filesync_rep_int() -> file_delete(Log) -> file:delete(Log). - diff --git a/lib/kernel/test/logger_stress_SUITE.erl b/lib/kernel/test/logger_stress_SUITE.erl index 4072e8c86a..1a278fb1b2 100644 --- a/lib/kernel/test/logger_stress_SUITE.erl +++ b/lib/kernel/test/logger_stress_SUITE.erl @@ -67,6 +67,10 @@ all() -> reject_events, std_handler, disk_log_handler, + std_handler_time, + std_handler_time_big, + disk_log_handler_time, + disk_log_handler_time_big, emulator_events, remote_events, remote_to_disk_log, @@ -123,6 +127,20 @@ std_handler(cleanup,_Config) -> _ = file:delete("default.log"), ok. +%% Disable overload protection and just print a lot - measure time. +%% The IOPS reported is the number of log events written per millisecond. +std_handler_time(Config) -> + measure_handler_time(logger_std_h,#{type=>{file,"default.log"}},Config). +std_handler_time(cleanup,_Config) -> + _ = file:delete("default.log"), + ok. + +std_handler_time_big(Config) -> + measure_handler_time_big(logger_std_h,#{type=>{file,"default.log"}},Config). +std_handler_time_big(cleanup,_Config) -> + _ = file:delete("default.log"), + ok. + %% Cascading failure that produce gen_server and proc_lib reports - %% how many of the produced log events are actually written to a log %% with logger_disk_log_h wrap file handler. @@ -140,6 +158,20 @@ disk_log_handler(cleanup,_Config) -> [_ = file:delete(F) || F <- Files], ok. +%% Disable overload protection and just print a lot - measure time. +%% The IOPS reported is the number of log events written per millisecond. +disk_log_handler_time(Config) -> + measure_handler_time(logger_disk_log_h,#{type=>halt},Config). +disk_log_handler_time(cleanup,_Config) -> + _ = file:delete("default"), + ok. + +disk_log_handler_time_big(Config) -> + measure_handler_time_big(logger_disk_log_h,#{type=>halt},Config). +disk_log_handler_time_big(cleanup,_Config) -> + _ = file:delete("default"), + ok. + %% Cascading failure that produce log events from the emulator - how %% many of the produced log events pass through the proxy. emulator_events(Config) -> @@ -221,6 +253,68 @@ remote_emulator_to_disk_log(cleanup,_Config) -> %%%----------------------------------------------------------------- %%% Internal functions +measure_handler_time(Module,HCfg,Config) -> + measure_handler_time(Module,100000,small_fa(),millisecond,HCfg,#{},Config). + +measure_handler_time_big(Module,HCfg,Config) -> + FCfg = #{chars_limit=>4096, max_size=>1024}, + measure_handler_time(Module,100,big_fa(),second,HCfg,FCfg,Config). + +measure_handler_time(Module,N,FA,Unit,HCfg,FCfg,Config) -> + {ok,_,Node} = + logger_test_lib:setup( + Config, + [{logger, + [{handler,default,Module, + #{formatter => {logger_formatter, + maps:merge(#{legacy_header=>false, + single_line=>true},FCfg)}, + config=>maps:merge(#{sync_mode_qlen => N+1, + drop_mode_qlen => N+1, + flush_qlen => N+1, + burst_limit_enable => false, + filesync_repeat_interval => no_repeat}, + HCfg)}}]}]), + %% HPid = rpc:call(Node,erlang,whereis,[?name_to_reg_name(Module,default)]), + %% {links,[_,FCPid]} = rpc:call(Node,erlang,process_info,[HPid,links]), + T0 = erlang:monotonic_time(millisecond), + ok = rpc:call(Node,?MODULE,nlogs_wait,[N,FA,Module]), + %% ok = rpc:call(Node,fprof,apply,[fun ?MODULE:nlogs_wait/2,[N div 10,FA,Module],[{procs,[HPid,FCPid]}]]), + T1 = erlang:monotonic_time(millisecond), + T = T1-T0, + M = case Unit of + millisecond -> 1; + second -> 1000 + end, + IOPS = M*N/T, + ct:pal("N: ~p~nT: ~p~nIOPS: ~.2f events pr ~w",[N,T,IOPS,Unit]), + %% Stats = rpc:call(Node,logger_olp,info,[?name_to_reg_name(Module,default)]), + %% ct:pal("Stats: ~p",[Stats]), + ct_event:notify(#event{name = benchmark_data, + data = [{value,IOPS}]}), + {comment,io_lib:format("~.2f events written pr ~w",[IOPS,Unit])}. + +nlogs_wait(N,{F,A},Module) -> + group_leader(whereis(user),self()), + [?LOG_NOTICE(F,A) || _ <- lists:seq(1,N)], + wait(Module). + +wait(Module) -> + case Module:filesync(default) of + {error,handler_busy} -> + wait(Module); + ok -> + ok + end. + +small_fa() -> + Str = "\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "[\\]^_`abcdefghijklmnopqr", + {"~ts",[Str]}. + +big_fa() -> + {"~p",[lists:duplicate(1048576,"a")]}. + nlogs(N) -> group_leader(whereis(user),self()), Str = "\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ" diff --git a/lib/kernel/vsn.mk b/lib/kernel/vsn.mk index 4b43c6ae9d..7bebe1ba70 100644 --- a/lib/kernel/vsn.mk +++ b/lib/kernel/vsn.mk @@ -1 +1 @@ -KERNEL_VSN = 6.2 +KERNEL_VSN = 6.3 diff --git a/lib/mnesia/doc/src/notes.xml b/lib/mnesia/doc/src/notes.xml index 8fc3610bb6..01d1666b8d 100644 --- a/lib/mnesia/doc/src/notes.xml +++ b/lib/mnesia/doc/src/notes.xml @@ -39,7 +39,22 @@ thus constitutes one section in this document. The title of each section is the version number of Mnesia.</p> - <section><title>Mnesia 4.15.5</title> + <section><title>Mnesia 4.15.6</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Avoid overload warnings caused by a race condition.</p> + <p> + Own Id: OTP-15619 Aux Id: ERIERL-310 </p> + </item> + </list> + </section> + +</section> + +<section><title>Mnesia 4.15.5</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/mnesia/src/mnesia.erl b/lib/mnesia/src/mnesia.erl index 77afb8250c..02bc884e36 100644 --- a/lib/mnesia/src/mnesia.erl +++ b/lib/mnesia/src/mnesia.erl @@ -160,7 +160,7 @@ {'sync_transaction', Retries::non_neg_integer()}. -type table() :: atom(). -type storage_type() :: 'ram_copies' | 'disc_copies' | 'disc_only_copies'. --type index_attr() :: atom() | non_neg_integer(). +-type index_attr() :: atom() | non_neg_integer() | {atom()}. -type write_locks() :: 'write' | 'sticky_write'. -type read_locks() :: 'read'. -type lock_kind() :: write_locks() | read_locks(). @@ -1277,6 +1277,14 @@ match_object(Tid, Ts, Tab, Pat, LockKind) match_object(_Tid, _Ts, Tab, Pat, _LockKind) -> abort({bad_type, Tab, Pat}). +add_written_index(Store, Pos, Tab, Key, Objs) when is_integer(Pos) -> + Pat = setelement(Pos, val({Tab, wild_pattern}), Key), + add_written_match(Store, Pat, Tab, Objs); +add_written_index(Store, Pos, Tab, Key, Objs) when is_tuple(Pos) -> + IxF = mnesia_index:index_vals_f(val({Tab, storage_type}), Tab, Pos), + Ops = find_ops(Store, Tab, '_'), + add_ix_match(Ops, Objs, IxF, Key, val({Tab, setorbag})). + add_written_match(S, Pat, Tab, Objs) -> Ops = find_ops(S, Tab, Pat), FixedRes = add_match(Ops, Objs, val({Tab, setorbag})), @@ -1303,6 +1311,46 @@ add_match([{_Oid, Val, write}|R], Objs, bag) -> add_match([{Oid, Val, write}|R], Objs, set) -> add_match(R, [Val | deloid(Oid,Objs)],set). +add_ix_match([], Objs, _IxF, _Key, _Type) -> + Objs; +add_ix_match(Written, Objs, IxF, Key, ordered_set) -> + %% Must use keysort which is stable + add_ordered_match(lists:keysort(1, ix_filter_ops(IxF, Key, Written)), Objs, []); +add_ix_match([{Oid, _, delete}|R], Objs, IxF, Key, Type) -> + add_ix_match(R, deloid(Oid, Objs), IxF, Key, Type); +add_ix_match([{_Oid, Val, delete_object}|R], Objs, IxF, Key, Type) -> + case ix_match(Val, IxF, Key) of + true -> + add_ix_match(R, lists:delete(Val, Objs), IxF, Key, Type); + false -> + add_ix_match(R, Objs, IxF, Key, Type) + end; +add_ix_match([{_Oid, Val, write}|R], Objs, IxF, Key, bag) -> + case ix_match(Val, IxF, Key) of + true -> + add_ix_match(R, [Val | lists:delete(Val, Objs)], IxF, Key, bag); + false -> + add_ix_match(R, Objs, IxF, Key, bag) + end; +add_ix_match([{Oid, Val, write}|R], Objs, IxF, Key, set) -> + case ix_match(Val, IxF, Key) of + true -> + add_ix_match(R, [Val | deloid(Oid,Objs)],IxF,Key,set); + false -> + add_ix_match(R, Objs, IxF, Key, set) + end. + +ix_match(Val, IxF, Key) -> + lists:member(Key, IxF(Val)). + +ix_filter_ops(IxF, Key, Ops) -> + lists:filter( + fun({_Oid, Obj, write}) -> + ix_match(Obj, IxF, Key); + (_) -> + true + end, Ops). + %% For ordered_set only !! add_ordered_match(Written = [{{_, Key}, _, _}|_], [Obj|Objs], Acc) when Key > element(2, Obj) -> @@ -1641,6 +1689,16 @@ index_match_object(Tid, Ts, Tab, Pat, Attr, LockKind) dirty_index_match_object(Tab, Pat, Attr); % Should be optimized? tid -> case mnesia_schema:attr_tab_to_pos(Tab, Attr) of + {_} -> + case LockKind of + read -> + Store = Ts#tidstore.store, + mnesia_locker:rlock_table(Tid, Store, Tab), + Objs = dirty_match_object(Tab, Pat), + add_written_match(Store, Pat, Tab, Objs); + _ -> + abort({bad_type, Tab, LockKind}) + end; Pos when Pos =< tuple_size(Pat) -> case LockKind of read -> @@ -1688,8 +1746,8 @@ index_read(Tid, Ts, Tab, Key, Attr, LockKind) false -> Store = Ts#tidstore.store, Objs = mnesia_index:read(Tid, Store, Tab, Key, Pos), - Pat = setelement(Pos, val({Tab, wild_pattern}), Key), - add_written_match(Store, Pat, Tab, Objs); + add_written_index( + Ts#tidstore.store, Pos, Tab, Key, Objs); true -> abort({bad_type, Tab, Attr, Key}) end; @@ -1825,7 +1883,7 @@ remote_dirty_match_object(Tab, Pat) -> false -> mnesia_lib:db_match_object(Tab, Pat); true -> - PosList = val({Tab, index}), + PosList = regular_indexes(Tab), remote_dirty_match_object(Tab, Pat, PosList) end. @@ -1857,7 +1915,7 @@ remote_dirty_select(Tab, Spec) -> false -> mnesia_lib:db_select(Tab, Spec); true -> - PosList = val({Tab, index}), + PosList = regular_indexes(Tab), remote_dirty_select(Tab, Spec, PosList) end; _ -> @@ -1924,6 +1982,8 @@ dirty_index_match_object(Pat, _Attr) -> dirty_index_match_object(Tab, Pat, Attr) when is_atom(Tab), Tab /= schema, is_tuple(Pat), tuple_size(Pat) > 2 -> case mnesia_schema:attr_tab_to_pos(Tab, Attr) of + {_} -> + dirty_match_object(Tab, Pat); Pos when Pos =< tuple_size(Pat) -> case has_var(element(2, Pat)) of false -> @@ -3254,3 +3314,7 @@ put_activity_id(Activity) -> mnesia_tm:put_activity_id(Activity). put_activity_id(Activity,Fun) -> mnesia_tm:put_activity_id(Activity,Fun). + +regular_indexes(Tab) -> + PosList = val({Tab, index}), + [P || P <- PosList, is_integer(P)]. diff --git a/lib/mnesia/src/mnesia_index.erl b/lib/mnesia/src/mnesia_index.erl index 098265d5fc..6f1c21e3b9 100644 --- a/lib/mnesia/src/mnesia_index.erl +++ b/lib/mnesia/src/mnesia_index.erl @@ -1,8 +1,8 @@ %% %% %CopyrightBegin% -%% +%% %% Copyright Ericsson AB 1996-2018. All Rights Reserved. -%% +%% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. %% You may obtain a copy of the License at @@ -14,7 +14,7 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%% +%% %% %CopyrightEnd% %% @@ -37,7 +37,7 @@ db_match_erase/2, get_index_table/2, get_index_table/3, - + tab2filename/2, init_index/2, init_indecies/3, @@ -45,6 +45,7 @@ del_transient/3, del_index_table/3, + index_vals_f/3, index_info/2, ext_index_instances/1]). @@ -60,9 +61,14 @@ read(Tid, Store, Tab, IxKey, Pos) -> ResList = mnesia_locker:ixrlock(Tid, Store, Tab, IxKey, Pos), %% Remove all tuples which don't include Ixkey, happens when Tab is a bag case val({Tab, setorbag}) of - bag -> + bag when is_integer(Pos) -> mnesia_lib:key_search_all(IxKey, Pos, ResList); - _ -> + bag when is_tuple(Pos) -> + TabStorage = val({Tab, storage_type}), + ValsF = index_vals_f(TabStorage, Tab, Pos), + [Obj || Obj <- ResList, + lists:member(IxKey, ValsF(Obj))]; + _ -> ResList end. @@ -136,7 +142,7 @@ del_object_index2([], _, _Storage, _Tab, _K, _Obj) -> ok; del_object_index2([{{Pos, Type}, Ixt} | Tail], SoB, Storage, Tab, K, Obj) -> ValsF = index_vals_f(Storage, Tab, Pos), case SoB of - bag -> + bag -> del_object_bag(Type, ValsF, Tab, K, Obj, Ixt); _ -> %% If set remove the tuple in index table del_ixes(Type, Ixt, ValsF, Obj, K) @@ -197,7 +203,7 @@ merge([], _, _, Ack) -> realkeys(Tab, Pos, IxKey) -> Index = get_index_table(Tab, Pos), db_get(Index, IxKey). % a list on the form [{IxKey, RealKey1} , .... - + dirty_select(Tab, Spec, Pos) when is_integer(Pos) -> %% Assume that we are on the node where the replica is %% Returns the records without applying the match spec @@ -233,7 +239,7 @@ dirty_read2(Tab, IxKey, Pos) -> end, Acc, mnesia_lib:db_get(Storage, Tab, K)) end, [], Keys)). -pick_index([{{{Pfx,_},IxType}, Ixt}|_], _Tab, {_} = Pfx) -> +pick_index([{{{Pfx,_,_},IxType}, Ixt}|_], _Tab, {_} = Pfx) -> {IxType, Ixt}; pick_index([{{Pos,IxType}, Ixt}|_], _Tab, Pos) -> {IxType, Ixt}; @@ -242,7 +248,7 @@ pick_index([_|T], Tab, Pos) -> pick_index([], Tab, Pos) -> mnesia:abort({no_exist, Tab, {index, Pos}}). - + %%%%%%% Creation, Init and deletion routines for index tables %% We can have several indexes on the same table @@ -387,12 +393,12 @@ init_ext_index(Tab, Storage, Alias, Mod, [{Pos,Type} | Tail]) -> create_fun(Cont, Tab, Pos) -> IxF = index_vals_f(disc_only_copies, Tab, Pos), fun(read) -> - Data = + Data = case Cont of {start, KeysPerChunk} -> mnesia_lib:db_init_chunk( disc_only_copies, Tab, KeysPerChunk); - '$end_of_table' -> + '$end_of_table' -> '$end_of_table'; _Else -> mnesia_lib:db_chunk(disc_only_copies, Cont) @@ -462,7 +468,7 @@ add_index_info(Tab, SetOrBag, IxElem) -> %% Check later if mnesia_tm is sensitive about the order mnesia_lib:set({Tab, index_info}, IndexInfo), mnesia_lib:set({Tab, index}, index_positions(IndexInfo)), - mnesia_lib:set({Tab, commit_work}, + mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit([IndexInfo | Commit])); {value, Old} -> %% We could check for consistency here @@ -470,7 +476,7 @@ add_index_info(Tab, SetOrBag, IxElem) -> mnesia_lib:set({Tab, index_info}, Index), mnesia_lib:set({Tab, index}, index_positions(Index)), NewC = lists:keyreplace(index, 1, Commit, Index), - mnesia_lib:set({Tab, commit_work}, + mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)) end. @@ -488,19 +494,19 @@ del_index_info(Tab, Pos) -> element(1,P)=/=Pos end, Old#index.pos_list) of - [] -> + [] -> IndexInfo = index_info(Old#index.setorbag,[]), mnesia_lib:set({Tab, index_info}, IndexInfo), mnesia_lib:set({Tab, index}, index_positions(IndexInfo)), NewC = lists:keydelete(index, 1, Commit), - mnesia_lib:set({Tab, commit_work}, + mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)); New -> Index = Old#index{pos_list = New}, mnesia_lib:set({Tab, index_info}, Index), mnesia_lib:set({Tab, index}, index_positions(Index)), NewC = lists:keyreplace(index, 1, Commit, Index), - mnesia_lib:set({Tab, commit_work}, + mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)) end end. @@ -537,7 +543,7 @@ db_match_erase({{ext,_,_} = Ext, Ixt}, Pat) -> mnesia_lib:db_match_erase(Ext, Ixt, Pat); db_match_erase({dets, Ixt}, Pat) -> ok = dets:match_delete(Ixt, Pat). - + db_select({ram, Ixt}, Pat) -> ets:select(Ixt, Pat); db_select({{ext,_,_} = Ext, Ixt}, Pat) -> @@ -545,7 +551,7 @@ db_select({{ext,_,_} = Ext, Ixt}, Pat) -> db_select({dets, Ixt}, Pat) -> dets:select(Ixt, Pat). - + get_index_table(Tab, Pos) -> get_index_table(Tab, val({Tab, storage_type}), Pos). diff --git a/lib/mnesia/test/Makefile b/lib/mnesia/test/Makefile index 5b61b1af65..b43bc82801 100644 --- a/lib/mnesia/test/Makefile +++ b/lib/mnesia/test/Makefile @@ -53,7 +53,8 @@ MODULES= \ mnesia_measure_test \ mnesia_cost \ mnesia_dbn_meters \ - ext_test + ext_test \ + mnesia_index_plugin_test DocExamplesDir := ../doc/src/ diff --git a/lib/mnesia/test/mnesia_SUITE.erl b/lib/mnesia/test/mnesia_SUITE.erl index 24c1def6da..b41bf22efa 100644 --- a/lib/mnesia/test/mnesia_SUITE.erl +++ b/lib/mnesia/test/mnesia_SUITE.erl @@ -69,12 +69,13 @@ groups() -> %% covered. [{light, [], [{group, install}, {group, nice}, {group, evil}, - {group, mnesia_frag_test, light}, {group, qlc}, + {group, mnesia_frag_test, light}, {group, qlc}, {group, index_plugins}, {group, registry}, {group, config}, {group, examples}]}, {install, [], [{mnesia_install_test, all}]}, {nice, [], [{mnesia_nice_coverage_test, all}]}, {evil, [], [{mnesia_evil_coverage_test, all}]}, {qlc, [], [{mnesia_qlc_test, all}]}, + {index_plugins, [], [{mnesia_index_plugin_test, all}]}, {registry, [], [{mnesia_registry_test, all}]}, {config, [], [{mnesia_config_test, all}]}, {examples, [], [{mnesia_examples_test, all}]}, diff --git a/lib/mnesia/test/mnesia_index_plugin_test.erl b/lib/mnesia/test/mnesia_index_plugin_test.erl new file mode 100644 index 0000000000..44fe047c50 --- /dev/null +++ b/lib/mnesia/test/mnesia_index_plugin_test.erl @@ -0,0 +1,261 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +-module(mnesia_index_plugin_test). +-author('[email protected]'). + +-export([init_per_testcase/2, end_per_testcase/2, + init_per_group/2, end_per_group/2, + init_per_suite/1, end_per_suite/1, + all/0, groups/0]). + +-export([ + add_rm_plugin/1, + tab_with_plugin_index/1, + tab_with_multiple_plugin_indexes/1, + ix_match_w_plugin/1, + ix_match_w_plugin_ordered/1, + ix_match_w_plugin_bag/1 + ]). + +-export([ix_prefixes/3, % test plugin + ix_prefixes2/3]). % test plugin 2 + +-include("mnesia_test_lib.hrl"). + +init_per_suite(Conf) -> + Conf. + +end_per_suite(Conf) -> + Conf. + +init_per_testcase(Func, Conf) -> + mnesia_test_lib:init_per_testcase(Func, Conf). + +end_per_testcase(Func, Conf) -> + mnesia_test_lib:end_per_testcase(Func, Conf). + +all() -> + [add_rm_plugin, + tab_with_plugin_index, + tab_with_multiple_plugin_indexes, + ix_match_w_plugin, + ix_match_w_plugin_ordered, + ix_match_w_plugin_bag]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + +add_rm_plugin(suite) -> []; +add_rm_plugin(Config) when is_list(Config) -> + [N1, N2] = Nodes = ?acquire_nodes(2, Config), + ok = add_plugin(), + ok = rpc_check_plugin(N1), + ok = rpc_check_plugin(N2), + ok = add_plugin2(), + ok = del_plugin(), + ok = del_plugin2(), + ok = add_plugin(), + ok = add_plugin2(), + ok = del_plugin(), + ok = del_plugin2(), + ?verify_mnesia(Nodes, []). + +-define(PLUGIN1, {{pfx},?MODULE,ix_prefixes}). +-define(PLUGIN2, {{pfx2},?MODULE,ix_prefixes2}). + +add_plugin() -> + {atomic, ok} = mnesia_schema:add_index_plugin({pfx}, ?MODULE, ix_prefixes), + [?PLUGIN1] = mnesia_schema:index_plugins(), + ok. + +add_plugin2() -> + {atomic, ok} = mnesia_schema:add_index_plugin({pfx2}, ?MODULE, ix_prefixes2), + [?PLUGIN1, ?PLUGIN2] = lists:sort(mnesia_schema:index_plugins()), + ok. + +del_plugin() -> + {atomic, ok} = mnesia_schema:delete_index_plugin({pfx}), + [?PLUGIN2] = mnesia_schema:index_plugins(), + ok. + +del_plugin2() -> + {atomic, ok} = mnesia_schema:delete_index_plugin({pfx2}), + [] = mnesia_schema:index_plugins(), + ok. + +rpc_check_plugin(N) -> + [?PLUGIN1] = + rpc:call(N, mnesia_schema, index_plugins, []), + ok. + +tab_with_plugin_index(suite) -> []; +tab_with_plugin_index(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + {atomic, ok} = mnesia:create_table(t, [{attributes, [k,v1,v2]}, + {index, [{{pfx}, ordered}, + {v1, ordered}, + v2]}]), + [ok,ok,ok,ok] = + [mnesia:dirty_write({t, K, V1, V2}) + || {K,V1,V2} <- [{1,a,"123"}, + {2,b,"12345"}, + {3,c,"6789"}, + {4,d,nil}]], + [{t,1,a,"123"},{t,2,b,"12345"}] = + mnesia:dirty_index_read(t,<<"123">>,{pfx}), + [{t,3,c,"6789"}] = + mnesia:dirty_index_read(t,"6789",v2), + [{t,1,a,"123"}] = + mnesia:dirty_match_object({t,'_',a,"123"}), + [{t,1,a,"123"}] = + mnesia:dirty_select(t, [{ {t,'_',a,"123"}, [], ['$_']}]), + mnesia:dirty_delete(t,2), + [{t,1,a,"123"}] = + mnesia:dirty_index_read(t,<<"123">>,{pfx}), + ?verify_mnesia(Nodes, []). + +tab_with_multiple_plugin_indexes(suite) -> []; +tab_with_multiple_plugin_indexes(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + ok = add_plugin2(), + {atomic, ok} = + mnesia:create_table(u, [{attributes, [k,v1,v2]}, + {index, [{{pfx}, ordered}, + {{pfx2}, ordered}]}]), + [ok,ok,ok,ok] = + [mnesia:dirty_write({u, K, V1, V2}) + || {K,V1,V2} <- [{1,a,"123"}, + {2,b,"12345"}, + {3,c,"6789"}, + {4,d,nil}]], + [{u,1,a,"123"},{u,2,b,"12345"}] = + mnesia:dirty_index_read(u,<<"123">>,{pfx}), + [{u,1,a,"123"},{u,2,b,"12345"}] = + mnesia:dirty_index_read(u,<<"321">>,{pfx2}), + ?verify_mnesia(Nodes, []). + +ix_match_w_plugin(suite) -> []; +ix_match_w_plugin(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + {atomic, ok} = mnesia:create_table(im1, [{attributes, [k, v1, v2]}, + {index, [{{pfx}, ordered}, + {v1, ordered}]}]), + fill_and_test_index_match(im1, set), + ?verify_mnesia(Nodes, []). + + +ix_match_w_plugin_ordered(suite) -> []; +ix_match_w_plugin_ordered(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + {atomic, ok} = mnesia:create_table(im2, [{attributes, [k, v1, v2]}, + {type, ordered_set}, + {index, [{{pfx}, ordered}, + {v1, ordered}]}]), + fill_and_test_index_match(im2, ordered_set), + ?verify_mnesia(Nodes, []). + +ix_match_w_plugin_bag(suite) -> []; +ix_match_w_plugin_bag(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + {atomic, ok} = mnesia:create_table(im3, [{attributes, [k, v1, v2]}, + {type, bag}, + {index, [{{pfx}, ordered}, + {v1, ordered}]}]), + fill_and_test_index_match(im3, bag), + ?verify_mnesia(Nodes, []). + +fill_and_test_index_match(Tab, Type) -> + [ok,ok,ok,ok,ok,ok,ok,ok,ok] = + [mnesia:dirty_write({Tab, K, V1, V2}) + || {K,V1,V2} <- [{1,a,"123"}, + {2,b,"12345"}, + {3,c,"123"}, + {4,d,nil}, + {5,e,nil}, + {6,f,nil}, + {7,g,nil}, %% overwritten if not bag + {7,g,"234"}, + {8,h,"123"}]], + mnesia:activity( + transaction, + fun() -> + ok = mnesia:write({Tab, 1, aa, "1234"}), %% replaces if not bag + ok = mnesia:delete({Tab, 2}), + ok = mnesia:delete({Tab, 4}), + ok = mnesia:write({Tab, 6, ff, nil}), + ok = mnesia:write({Tab, 7, gg, "123"}), + ok = mnesia:write({Tab, 100, x, nil}), + ok = mnesia:delete_object({Tab,3,c,"123"}), + ok = mnesia:delete_object({Tab,5,e,nil}), + Res = mnesia:index_read(Tab, <<"123">>, {pfx}), + SetRes = [{Tab,1,aa,"1234"}, {Tab,7,gg,"123"}, {Tab,8,h,"123"}], + case Type of + set -> + SetRes = lists:sort(Res); + ordered_set -> + SetRes = Res; + bag -> + [{Tab,1,a,"123"}, {Tab,1,aa,"1234"}, + {Tab,7,gg,"123"}, {Tab,8,h,"123"}] = lists:sort(Res) + end + end). + +%% ============================================================ +%% +ix_prefixes(_Tab, _Pos, Obj) -> + lists:foldl( + fun(V, Acc) when is_list(V) -> + try Pfxs = prefixes(list_to_binary(V)), + Pfxs ++ Acc + catch + error:_ -> + Acc + end; + (V, Acc) when is_binary(V) -> + Pfxs = prefixes(V), + Pfxs ++ Acc; + (_, Acc) -> + Acc + end, [], tl(tuple_to_list(Obj))). + +ix_prefixes2(Tab, Pos, Obj) -> + [rev(P) || P <- ix_prefixes(Tab, Pos, Obj)]. + +rev(B) when is_binary(B) -> + list_to_binary(lists:reverse(binary_to_list(B))). + +prefixes(<<P:3/binary, _/binary>>) -> + [P]; +prefixes(_) -> + []. diff --git a/lib/mnesia/test/mt.erl b/lib/mnesia/test/mt.erl index 5a981bf539..037d6adb38 100644 --- a/lib/mnesia/test/mt.erl +++ b/lib/mnesia/test/mt.erl @@ -67,6 +67,7 @@ alias(recovery) -> mnesia_recovery_test; alias(registry) -> mnesia_registry_test; alias(suite) -> mnesia_SUITE; alias(trans) -> mnesia_trans_access_test; +alias(ixp) -> mnesia_index_plugin_test; alias(Other) -> Other. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/mnesia/vsn.mk b/lib/mnesia/vsn.mk index 1cfb35c774..781a4830a0 100644 --- a/lib/mnesia/vsn.mk +++ b/lib/mnesia/vsn.mk @@ -1 +1 @@ -MNESIA_VSN = 4.15.5 +MNESIA_VSN = 4.15.6 diff --git a/lib/observer/doc/src/notes.xml b/lib/observer/doc/src/notes.xml index 22035af982..2d914f8c61 100644 --- a/lib/observer/doc/src/notes.xml +++ b/lib/observer/doc/src/notes.xml @@ -32,6 +32,45 @@ <p>This document describes the changes made to the Observer application.</p> +<section><title>Observer 2.9</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Since Logger was introduced in OTP-21.0, menu choice + <em>Log > Toggle Log View</em> in observer would cause + a crash unless an <c>error_logger</c> event handler was + explicitly installed. This is now corrected.</p> + <p> + Own Id: OTP-15553 Aux Id: ERL-848 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Since persistent_term was introduced, observer would + sometimes crash when expanding a term from a process + state. This is now corrected.</p> + <p> + Own Id: OTP-15493 Aux Id: ERL-810 </p> + </item> + <item> + <p> + Add <c>OBSERVER_SCALE</c> environment variable for HiDPI + support.</p> + <p> + Own Id: OTP-15586 Aux Id: PR-2105 </p> + </item> + </list> + </section> + +</section> + <section><title>Observer 2.8.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/observer/vsn.mk b/lib/observer/vsn.mk index 5ce0aca589..0e9c8b302c 100644 --- a/lib/observer/vsn.mk +++ b/lib/observer/vsn.mk @@ -1 +1 @@ -OBSERVER_VSN = 2.8.2 +OBSERVER_VSN = 2.9 diff --git a/lib/odbc/doc/src/notes.xml b/lib/odbc/doc/src/notes.xml index dba7663bb9..696fcaa479 100644 --- a/lib/odbc/doc/src/notes.xml +++ b/lib/odbc/doc/src/notes.xml @@ -32,7 +32,22 @@ <p>This document describes the changes made to the odbc application. </p> - <section><title>ODBC 2.12.2</title> + <section><title>ODBC 2.12.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Enhance error handling to avoid stack corruption</p> + <p> + Own Id: OTP-15667 Aux Id: ERL-808, PR-2065 </p> + </item> + </list> + </section> + +</section> + +<section><title>ODBC 2.12.2</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/odbc/vsn.mk b/lib/odbc/vsn.mk index bb21016fad..ff023e666b 100644 --- a/lib/odbc/vsn.mk +++ b/lib/odbc/vsn.mk @@ -1 +1 @@ -ODBC_VSN = 2.12.2 +ODBC_VSN = 2.12.3 diff --git a/lib/public_key/asn1/OTP-PKIX.asn1 b/lib/public_key/asn1/OTP-PKIX.asn1 index 9bcd99fba3..ff3250b383 100644 --- a/lib/public_key/asn1/OTP-PKIX.asn1 +++ b/lib/public_key/asn1/OTP-PKIX.asn1 @@ -233,9 +233,13 @@ countryName ATTRIBUTE-TYPE-AND-VALUE-CLASS ::= { -- regarding how to handle and sometimes accept incorrect certificates -- we define and use the type below instead of X520countryName + -- We accept utf8String encoding of the US-ASCII + -- country name code and the mix up with other country code systems + -- that uses three characters instead of two. + OTP-X520countryname ::= CHOICE { - printableString PrintableString (SIZE (2)), - utf8String UTF8String (SIZE (2)) + printableString PrintableString (SIZE (2..3)), + utf8String UTF8String (SIZE (2..3)) } serialNumber ATTRIBUTE-TYPE-AND-VALUE-CLASS ::= { diff --git a/lib/public_key/doc/src/notes.xml b/lib/public_key/doc/src/notes.xml index b3e6023c41..f6bc0dc797 100644 --- a/lib/public_key/doc/src/notes.xml +++ b/lib/public_key/doc/src/notes.xml @@ -35,6 +35,21 @@ <file>notes.xml</file> </header> +<section><title>Public_Key 1.6.5</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Add export of dialyzer type</p> + <p> + Own Id: OTP-15624</p> + </item> + </list> + </section> + +</section> + <section><title>Public_Key 1.6.4</title> <section><title>Improvements and New Features</title> diff --git a/lib/public_key/doc/src/public_key.xml b/lib/public_key/doc/src/public_key.xml index 9fcedf6ef9..fb81ea68a4 100644 --- a/lib/public_key/doc/src/public_key.xml +++ b/lib/public_key/doc/src/public_key.xml @@ -423,7 +423,7 @@ <p>Available options:</p> <taglist> - <tag>{verify_fun, fun()}</tag> + <tag>{verify_fun, {fun(), InitialUserState::term()}</tag> <item> <p>The fun must be defined as:</p> diff --git a/lib/public_key/src/pubkey_pbe.erl b/lib/public_key/src/pubkey_pbe.erl index 806f7c5b0f..e6bcedd1b1 100644 --- a/lib/public_key/src/pubkey_pbe.erl +++ b/lib/public_key/src/pubkey_pbe.erl @@ -42,15 +42,14 @@ encode(Data, Password, "DES-CBC" = Cipher, KeyDevParams) -> {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), crypto:block_encrypt(des_cbc, Key, IV, pbe_pad(Data, KeyDevParams)); - encode(Data, Password, "DES-EDE3-CBC" = Cipher, KeyDevParams) -> {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), <<Key1:8/binary, Key2:8/binary, Key3:8/binary>> = Key, crypto:block_encrypt(des3_cbc, [Key1, Key2, Key3], IV, pbe_pad(Data)); - encode(Data, Password, "RC2-CBC" = Cipher, KeyDevParams) -> {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), crypto:block_encrypt(rc2_cbc, Key, IV, pbe_pad(Data, KeyDevParams)). + %%-------------------------------------------------------------------- -spec decode(binary(), string(), string(), term()) -> binary(). %% @@ -59,21 +58,20 @@ encode(Data, Password, "RC2-CBC" = Cipher, KeyDevParams) -> decode(Data, Password,"DES-CBC"= Cipher, KeyDevParams) -> {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), crypto:block_decrypt(des_cbc, Key, IV, Data); - decode(Data, Password,"DES-EDE3-CBC" = Cipher, KeyDevParams) -> {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), <<Key1:8/binary, Key2:8/binary, Key3:8/binary>> = Key, crypto:block_decrypt(des3_cbc, [Key1, Key2, Key3], IV, Data); - decode(Data, Password,"RC2-CBC"= Cipher, KeyDevParams) -> {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), crypto:block_decrypt(rc2_cbc, Key, IV, Data); +decode(Data, Password,"AES-128-CBC"= Cipher, KeyDevParams) -> + {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), + crypto:block_decrypt(aes_cbc128, Key, IV, Data); +decode(Data, Password,"AES-256-CBC"= Cipher, KeyDevParams) -> + {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), + crypto:block_decrypt(aes_cbc256, Key, IV, Data). -decode(Data, Password,"AES-128-CBC"= Cipher, IV) -> - %% PKCS5_SALT_LEN is 8 bytes - <<Salt:8/binary,_/binary>> = IV, - {Key, _} = password_to_key_and_iv(Password, Cipher, Salt), - crypto:block_decrypt(aes_cbc128, Key, IV, Data). %%-------------------------------------------------------------------- -spec pbdkdf1(string(), iodata(), integer(), atom()) -> binary(). @@ -131,13 +129,15 @@ password_to_key_and_iv(Password, _Cipher, {#'PBEParameter'{salt = Salt, <<Key:8/binary, IV:8/binary, _/binary>> = pbdkdf1(Password, Salt, Count, Hash), {Key, IV}; -password_to_key_and_iv(Password, Cipher, Salt) -> - KeyLen = derived_key_length(Cipher, undefined), +password_to_key_and_iv(Password, Cipher, KeyDevParams) -> + %% PKCS5_SALT_LEN is 8 bytes + <<Salt:8/binary,_/binary>> = KeyDevParams, + KeyLen = derived_key_length(Cipher, undefined), <<Key:KeyLen/binary, _/binary>> = pem_encrypt(<<>>, Password, Salt, ceiling(KeyLen div 16), <<>>, md5), %% Old PEM encryption does not use standard encryption method - %% pbdkdf1 and uses then salt as IV - {Key, Salt}. + %% pbdkdf1 + {Key, KeyDevParams}. pem_encrypt(_, _, _, 0, Acc, _) -> Acc; pem_encrypt(Prev, Password, Salt, Count, Acc, Hash) -> @@ -267,7 +267,9 @@ derived_key_length(Cipher,_) when (Cipher == ?'des-EDE3-CBC') or (Cipher == "DES-EDE3-CBC") -> 24; derived_key_length(Cipher,_) when (Cipher == "AES-128-CBC") -> - 16. + 16; +derived_key_length(Cipher,_) when (Cipher == "AES-256-CBC") -> + 32. cipher(#'PBES2-params_encryptionScheme'{algorithm = ?'desCBC'}) -> "DES-CBC"; diff --git a/lib/public_key/src/pubkey_pem.erl b/lib/public_key/src/pubkey_pem.erl index d7e5bc3ad8..0fd1453f7c 100644 --- a/lib/public_key/src/pubkey_pem.erl +++ b/lib/public_key/src/pubkey_pem.erl @@ -101,10 +101,10 @@ encode_pem_entry({'PrivateKeyInfo', Der, EncParams}) -> EncDer = encode_encrypted_private_keyinfo(Der, EncParams), StartStr = pem_start('EncryptedPrivateKeyInfo'), [StartStr, "\n", b64encode_and_split(EncDer), "\n", pem_end(StartStr) ,"\n\n"]; -encode_pem_entry({Type, Der, {Cipher, Salt}}) -> +encode_pem_entry({Type, Decrypted, {Cipher, Salt}}) -> StartStr = pem_start(Type), [StartStr,"\n", pem_decrypt(),"\n", pem_decrypt_info(Cipher, Salt),"\n\n", - b64encode_and_split(Der), "\n", pem_end(StartStr) ,"\n\n"]. + b64encode_and_split(Decrypted), "\n", pem_end(StartStr) ,"\n\n"]. decode_pem_entries([], Entries) -> lists:reverse(Entries); diff --git a/lib/public_key/test/pbe_SUITE.erl b/lib/public_key/test/pbe_SUITE.erl index 523c9e2515..1136267411 100644 --- a/lib/public_key/test/pbe_SUITE.erl +++ b/lib/public_key/test/pbe_SUITE.erl @@ -37,7 +37,7 @@ all() -> [ pbdkdf1, pbdkdf2, - old_enc, + old_pbe, pbes1, pbes2]. @@ -197,23 +197,11 @@ pbdkdf2(Config) when is_list(Config) -> = pubkey_pbe:pbdkdf2("pass\0word", "sa\0lt", 4096, 16, fun crypto:hmac/4, sha, 20). -old_enc() -> - [{doc,"Tests encode/decode RSA key encrypted with different ciphers using old PEM encryption scheme"}]. -old_enc(Config) when is_list(Config) -> - Datadir = proplists:get_value(data_dir, Config), - %% key generated with ssh-keygen -N hello_aes -f old_aes_128_cbc_enc_key.pem - {ok, PemAesCbc} = file:read_file(filename:join(Datadir, "old_aes_128_cbc_enc_key.pem")), - - PemAesCbcEntry = public_key:pem_decode(PemAesCbc), - ct:print("Pem entry: ~p" , [PemAesCbcEntry]), - [{'RSAPrivateKey', _, {"AES-128-CBC",_}} = PubAesCbcEntry] = PemAesCbcEntry, - #'RSAPrivateKey'{} = public_key:pem_entry_decode(PubAesCbcEntry, "hello_aes"). - pbes1() -> [{doc,"Tests encode/decode EncryptedPrivateKeyInfo encrypted with different ciphers using PBES1"}]. pbes1(Config) when is_list(Config) -> decode_encode_key_file("pbes1_des_cbc_md5_enc_key.pem", "password", "DES-CBC", Config). - + pbes2() -> [{doc,"Tests encode/decode EncryptedPrivateKeyInfo encrypted with different ciphers using PBES2"}]. pbes2(Config) when is_list(Config) -> @@ -225,13 +213,33 @@ pbes2(Config) when is_list(Config) -> false -> ok end. +old_pbe() -> + [{doc,"Tests encode/decode with old format used before PBE"}]. +old_pbe(Config) when is_list(Config) -> + Datadir = proplists:get_value(data_dir, Config), + % key generated with ssh-keygen -N hello_aes -f old_aes_128_cbc.pem + {ok, PemAes128Cbc} = file:read_file(filename:join(Datadir, "old_aes_128_cbc.pem")), + + PemAes128CbcEntries = public_key:pem_decode(PemAes128Cbc), + ct:print("Pem entry: ~p" , [PemAes128CbcEntries]), + [{'RSAPrivateKey', _, {"AES-128-CBC",_}} = Aes128CbcEntry] = PemAes128CbcEntries, + #'RSAPrivateKey'{} = Key = public_key:pem_entry_decode(Aes128CbcEntry, "hello_aes"), + + %% Converted with openssl rsa -in old_aes_128_cbc.pem -out old_aes_256_cbc.pem -aes256 + {ok, PemAes256Cbc} = file:read_file(filename:join(Datadir, "old_aes_256_cbc.pem")), + + PemAes256CbcEntries = public_key:pem_decode(PemAes256Cbc), + ct:print("Pem entry: ~p" , [PemAes256CbcEntries]), + [{'RSAPrivateKey', _, {"AES-256-CBC",_}} = Aes256CbcEntry] = PemAes256CbcEntries, + Key = public_key:pem_entry_decode(Aes256CbcEntry, "hello_aes"). + decode_encode_key_file(File, Password, Cipher, Config) -> Datadir = proplists:get_value(data_dir, Config), {ok, PemKey} = file:read_file(filename:join(Datadir, File)), PemEntry = public_key:pem_decode(PemKey), - ct:print("Pem entry: ~p" , [PemEntry]), + ct:pal("Pem entry: ~p" , [PemEntry]), [{Asn1Type, _, {Cipher,_} = CipherInfo} = PubEntry] = PemEntry, #'RSAPrivateKey'{} = KeyInfo = public_key:pem_entry_decode(PubEntry, Password), PemKey1 = public_key:pem_encode([public_key:pem_entry_encode(Asn1Type, KeyInfo, {CipherInfo, Password})]), diff --git a/lib/public_key/test/pbe_SUITE_data/old_aes_128_cbc_enc_key.pem b/lib/public_key/test/pbe_SUITE_data/old_aes_128_cbc.pem index 34c7543f30..34c7543f30 100644 --- a/lib/public_key/test/pbe_SUITE_data/old_aes_128_cbc_enc_key.pem +++ b/lib/public_key/test/pbe_SUITE_data/old_aes_128_cbc.pem diff --git a/lib/public_key/test/pbe_SUITE_data/old_aes_256_cbc.pem b/lib/public_key/test/pbe_SUITE_data/old_aes_256_cbc.pem new file mode 100644 index 0000000000..e6aec2869d --- /dev/null +++ b/lib/public_key/test/pbe_SUITE_data/old_aes_256_cbc.pem @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-256-CBC,ABDA22398E511E9465983E1A50706044 + +XhIcPOb6pTWL++pgeeTH5rsx0tackhllVqyXyOfbYMBJnVFRhQ/V/1MDg3Jt4wD1 +Nerhcv5srHeiwmf+vwXwDFOzvFzLVM1jFMUJe/2XloYFX4TBiLZAF/zekQA3uPY6 +DKJuBuO5vVSZ0VlxGpu3jphIAwxbssfZkZmryCP3b1/oX5Y3Em/wEWW3RduaeWFu +Z2nTsPH3yNmHkuqFF3cq0aZs+VxtjcuYo0gbkN5hNVgOoOVxIBzw7DsgBAkdXvr3 +LRCMGg7Y2pVthA963s0xkN37XtZEiYbydoLnzHlW/Kx5QSvED8/Fn94Lcdy5NsQN +7nYWWgYZRH39Wnsi0BrTZv399U5rBe7DnpStKWPn2Sa8Bdu3CX2oLajM1cZjAp1X +y6vuasK7SoZ8rWpcsHQpV1HyNBTl/uRU5nrYc/KGD8l2x7tdNidGx2Hey/6O4H5v +rxL1TB4PlzDYwCsReuyLbEjyuZQ10j0SK/SFzikHuvD2IEpM/oUaPdVFqcW8/sjw +VhcsupAf4EXfsi2dJBylmDjI2W7h9XwBDLzQN+69DtkBmvimE5CguTITf8MAHQQ6 +Q1vYF2ij7W675tj9ulksRPaMxSsI03luai/Jrieh0mPqGEenGEEC5QU0XPOvsfyw +GMYaBUbdYrMpbHM3wFPxM2wRlXf4gX5BhZKRGZX7OaEs54pfglyQtTPuZmD0VcAp +EWHq70G9mbuBlhbMX1rKAomuDmIvgLeLRUpAFf59Qr8KoJSLD1S8TJB3uEPk6i39 +4GnylbpmqS4gv/OIc6WTeOeUZTAD3A77HBwSlELPk3/s1d/MLyfciYClOBEuZ6NB +FXEKCGCEC786zJA678gLEaa2XPNkEM+2gjzNFqtYMIn5ehAq/HRRsFvW+wkTbee4 +z+qe5HbVKAQ3EOfbidvYrDaGd7HvHVG8zosl+O61iIFs04lLEMDFXBIdvIgEncOK +Rq3yXdpBKMg89aoZLniaPobSvuvdjNOMzW6EKlb5FKZduCiR68MEZ+rLHYHTwE3W +Z5+TCbrbV2F6WQpq3zqnB14wGu8igEb5Veq+N2vMkx4iTMTUyCty1SwIjj4NidM1 +dJM7Ighdal6tQ6hIwbDfpIPsY4eGH/UrdVZ0SkxuDR2s76cZ8nFX3lJ/BNwTZLKo +IqAC4NjUOv3ID+0Q6Lz+sxLCi5pLYUf0E+s4pgi1BYAOu+BF3GwxyqnqVoq/Fs5D +LXxuY0946YM+WcrYzke4mq3MPx6QQYj04H5KJ2mzxtnbZJrfLF23PVRVhvgKSjyV +I3/zgJ16fV2H/fb26oCpTNbb11pQvhorkLwdvpwtM+go7dJGebAi1762Nbj/CqnW +fbBPxPRvNPZn6pEodJ/L/APhvGv1K7eC9THj66H7Kmeoq8Lz74idhywP9I3QS0ZO +15ORbTDjuiRYNJPxxu79A3/tWMUlprJ9ljhI/0DXRB0M3UGic52D/32Q64I7eewy +qRNS/3C3ejDShIRBDFTdDkM3s/42LySXJjmjU9bpZY4POQ3kOaJb3EzSvbzTyXzu +3FiHvDQY+b8XwbxtE/kTMaAPQZ7TtWOao7SRi7J94MvCQ5/tbakFP2suM8psnigC +-----END RSA PRIVATE KEY----- diff --git a/lib/public_key/vsn.mk b/lib/public_key/vsn.mk index 5e2643f0ea..11c06fb158 100644 --- a/lib/public_key/vsn.mk +++ b/lib/public_key/vsn.mk @@ -1 +1 @@ -PUBLIC_KEY_VSN = 1.6.4 +PUBLIC_KEY_VSN = 1.6.5 diff --git a/lib/runtime_tools/doc/src/notes.xml b/lib/runtime_tools/doc/src/notes.xml index 810bb8207c..58a2a66c4b 100644 --- a/lib/runtime_tools/doc/src/notes.xml +++ b/lib/runtime_tools/doc/src/notes.xml @@ -32,6 +32,21 @@ <p>This document describes the changes made to the Runtime_Tools application.</p> +<section><title>Runtime_Tools 1.13.2</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Update of systemtap trace example scripts.</p> + <p> + Own Id: OTP-15670</p> + </item> + </list> + </section> + +</section> + <section><title>Runtime_Tools 1.13.1</title> <section><title>Improvements and New Features</title> diff --git a/lib/runtime_tools/vsn.mk b/lib/runtime_tools/vsn.mk index aa3d702997..fa2f338ec2 100644 --- a/lib/runtime_tools/vsn.mk +++ b/lib/runtime_tools/vsn.mk @@ -1 +1 @@ -RUNTIME_TOOLS_VSN = 1.13.1 +RUNTIME_TOOLS_VSN = 1.13.2 diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index 0bc4baf5eb..17f14bdea2 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -30,6 +30,23 @@ <file>notes.xml</file> </header> +<section><title>Ssh 4.7.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + SSH sftp daemon now accepts an SSH_FXP_STAT message + encoded according to the wrong sftp version. Some clients + sends such messages.</p> + <p> + Own Id: OTP-15498 Aux Id: ERL-822, PR-2077 </p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.7.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssh/src/ssh_transport.erl b/lib/ssh/src/ssh_transport.erl index 9ff20454cd..1f4e281a30 100644 --- a/lib/ssh/src/ssh_transport.erl +++ b/lib/ssh/src/ssh_transport.erl @@ -162,14 +162,14 @@ supported_algorithms(cipher) -> select_crypto_supported( [ {'[email protected]', [{ciphers,chacha20}, {macs,poly1305}]}, - {'[email protected]', [{ciphers,{aes_gcm,256}}]}, - {'aes256-ctr', [{ciphers,{aes_ctr,256}}]}, - {'aes192-ctr', [{ciphers,{aes_ctr,192}}]}, - {'[email protected]', [{ciphers,{aes_gcm,128}}]}, - {'aes128-ctr', [{ciphers,{aes_ctr,128}}]}, - {'AEAD_AES_256_GCM', [{ciphers,{aes_gcm,256}}]}, - {'AEAD_AES_128_GCM', [{ciphers,{aes_gcm,128}}]}, - {'aes128-cbc', [{ciphers,aes_cbc128}]}, + {'[email protected]', [{ciphers,aes_256_gcm}]}, + {'aes256-ctr', [{ciphers,aes_256_ctr}]}, + {'aes192-ctr', [{ciphers,aes_192_ctr}]}, + {'[email protected]', [{ciphers,aes_128_gcm}]}, + {'aes128-ctr', [{ciphers,aes_128_ctr}]}, + {'AEAD_AES_256_GCM', [{ciphers,aes_256_gcm}]}, + {'AEAD_AES_128_GCM', [{ciphers,aes_128_gcm}]}, + {'aes128-cbc', [{ciphers,aes_128_cbc}]}, {'3des-cbc', [{ciphers,des3_cbc}]} ] )); @@ -179,8 +179,8 @@ supported_algorithms(mac) -> [{'hmac-sha2-256', [{macs,hmac}, {hashs,sha256}]}, {'hmac-sha2-512', [{macs,hmac}, {hashs,sha512}]}, {'hmac-sha1', [{macs,hmac}, {hashs,sha}]}, - {'AEAD_AES_128_GCM', [{ciphers,{aes_gcm,128}}]}, - {'AEAD_AES_256_GCM', [{ciphers,{aes_gcm,256}}]} + {'AEAD_AES_128_GCM', [{ciphers,aes_128_gcm}]}, + {'AEAD_AES_256_GCM', [{ciphers,aes_256_gcm}]} ] )); supported_algorithms(compression) -> @@ -1256,11 +1256,6 @@ get_length(aead, EncryptedBuffer, Ssh) -> end. -pkt_type('AEAD_AES_128_GCM') -> aead; -pkt_type('AEAD_AES_256_GCM') -> aead; -pkt_type('[email protected]') -> aead; -pkt_type(_) -> common. - payload(<<PacketLen:32, PaddingLen:8, PayloadAndPadding/binary>>) -> PayloadLen = PacketLen - PaddingLen - 1, <<Payload:PayloadLen/binary, _/binary>> = PayloadAndPadding, @@ -1323,162 +1318,115 @@ verify(PlainText, HashAlg, Sig, Key, _) -> %%% Unit: bytes --record(cipher_data, { - key_bytes, - iv_bytes, - block_bytes - }). +-record(cipher, { + impl, + key_bytes, + iv_bytes, + block_bytes, + pkt_type = common + }). %%% Start of a more parameterized crypto handling. cipher('AEAD_AES_128_GCM') -> - #cipher_data{key_bytes = 16, - iv_bytes = 12, - block_bytes = 16}; + #cipher{key_bytes = 16, + iv_bytes = 12, + block_bytes = 16, + pkt_type = aead}; cipher('AEAD_AES_256_GCM') -> - #cipher_data{key_bytes = 32, - iv_bytes = 12, - block_bytes = 16}; + #cipher{key_bytes = 32, + iv_bytes = 12, + block_bytes = 16, + pkt_type = aead}; cipher('3des-cbc') -> - #cipher_data{key_bytes = 24, - iv_bytes = 8, - block_bytes = 8}; + #cipher{impl = des3_cbc, + key_bytes = 24, + iv_bytes = 8, + block_bytes = 8}; cipher('aes128-cbc') -> - #cipher_data{key_bytes = 16, - iv_bytes = 16, - block_bytes = 16}; + #cipher{impl = aes_cbc, + key_bytes = 16, + iv_bytes = 16, + block_bytes = 16}; cipher('aes128-ctr') -> - #cipher_data{key_bytes = 16, - iv_bytes = 16, - block_bytes = 16}; + #cipher{impl = aes_128_ctr, + key_bytes = 16, + iv_bytes = 16, + block_bytes = 16}; cipher('aes192-ctr') -> - #cipher_data{key_bytes = 24, - iv_bytes = 16, - block_bytes = 16}; + #cipher{impl = aes_192_ctr, + key_bytes = 24, + iv_bytes = 16, + block_bytes = 16}; cipher('aes256-ctr') -> - #cipher_data{key_bytes = 32, - iv_bytes = 16, - block_bytes = 16}; + #cipher{impl = aes_256_ctr, + key_bytes = 32, + iv_bytes = 16, + block_bytes = 16}; cipher('[email protected]') -> % FIXME: Verify!! - #cipher_data{key_bytes = 32, - iv_bytes = 12, - block_bytes = 8}. - + #cipher{key_bytes = 32, + iv_bytes = 12, + block_bytes = 8, + pkt_type = aead}; + +cipher(_) -> + #cipher{}. + + +pkt_type(SshCipher) -> (cipher(SshCipher))#cipher.pkt_type. + +decrypt_magic(server) -> {"A", "C"}; +decrypt_magic(client) -> {"B", "D"}. + +encrypt_magic(client) -> decrypt_magic(server); +encrypt_magic(server) -> decrypt_magic(client). + encrypt_init(#ssh{encrypt = none} = Ssh) -> {ok, Ssh}; -encrypt_init(#ssh{encrypt = '[email protected]', role = client} = Ssh) -> + +encrypt_init(#ssh{encrypt = '[email protected]', role = Role} = Ssh) -> %% [email protected] uses two independent crypto streams, one (chacha20) %% for the length used in stream mode, and the other (chacha20-poly1305) as AEAD for %% the payload and to MAC the length||payload. %% See draft-josefsson-ssh-chacha20-poly1305-openssh-00 - <<K2:32/binary,K1:32/binary>> = hash(Ssh, "C", 512), + {_, KeyMagic} = encrypt_magic(Role), + <<K2:32/binary,K1:32/binary>> = hash(Ssh, KeyMagic, 8*64), {ok, Ssh#ssh{encrypt_keys = {K1,K2} % encrypt_block_size = 16, %default = 8. What to set it to? 64 (openssl chacha.h) % ctx and iv is setup for each packet }}; -encrypt_init(#ssh{encrypt = '[email protected]', role = server} = Ssh) -> - <<K2:32/binary,K1:32/binary>> = hash(Ssh, "D", 512), - {ok, Ssh#ssh{encrypt_keys = {K1,K2} - % encrypt_block_size = 16, %default = 8. What to set it to? - }}; -encrypt_init(#ssh{encrypt = 'AEAD_AES_128_GCM', role = client} = Ssh) -> - IV = hash(Ssh, "A", 12*8), - <<K:16/binary>> = hash(Ssh, "C", 128), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = 'AEAD_AES_128_GCM', role = server} = Ssh) -> - IV = hash(Ssh, "B", 12*8), - <<K:16/binary>> = hash(Ssh, "D", 128), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = 'AEAD_AES_256_GCM', role = client} = Ssh) -> - IV = hash(Ssh, "A", 12*8), - <<K:32/binary>> = hash(Ssh, "C", 256), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = 'AEAD_AES_256_GCM', role = server} = Ssh) -> - IV = hash(Ssh, "B", 12*8), - <<K:32/binary>> = hash(Ssh, "D", 256), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = '3des-cbc', role = client} = Ssh) -> - IV = hash(Ssh, "A", 64), - <<K1:8/binary, K2:8/binary, K3:8/binary>> = hash(Ssh, "C", 192), - {ok, Ssh#ssh{encrypt_keys = {K1,K2,K3}, - encrypt_block_size = 8, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = '3des-cbc', role = server} = Ssh) -> - IV = hash(Ssh, "B", 64), - <<K1:8/binary, K2:8/binary, K3:8/binary>> = hash(Ssh, "D", 192), - {ok, Ssh#ssh{encrypt_keys = {K1,K2,K3}, - encrypt_block_size = 8, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = 'aes128-cbc', role = client} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:16/binary>> = hash(Ssh, "C", 128), + +encrypt_init(#ssh{encrypt = SshCipher, role = Role} = Ssh) when SshCipher == 'AEAD_AES_128_GCM'; + SshCipher == 'AEAD_AES_256_GCM' -> + {IvMagic, KeyMagic} = encrypt_magic(Role), + #cipher{key_bytes = KeyBytes, + iv_bytes = IvBytes, + block_bytes = BlockBytes} = cipher(SshCipher), + IV = hash(Ssh, IvMagic, 8*IvBytes), + K = hash(Ssh, KeyMagic, 8*KeyBytes), {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, + encrypt_block_size = BlockBytes, encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = 'aes128-cbc', role = server} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:16/binary>> = hash(Ssh, "D", 128), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = 'aes128-ctr', role = client} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:16/binary>> = hash(Ssh, "C", 128), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = State}}; -encrypt_init(#ssh{encrypt = 'aes192-ctr', role = client} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:24/binary>> = hash(Ssh, "C", 192), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = State}}; -encrypt_init(#ssh{encrypt = 'aes256-ctr', role = client} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:32/binary>> = hash(Ssh, "C", 256), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = State}}; -encrypt_init(#ssh{encrypt = 'aes128-ctr', role = server} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:16/binary>> = hash(Ssh, "D", 128), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = State}}; -encrypt_init(#ssh{encrypt = 'aes192-ctr', role = server} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:24/binary>> = hash(Ssh, "D", 192), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = State}}; -encrypt_init(#ssh{encrypt = 'aes256-ctr', role = server} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:32/binary>> = hash(Ssh, "D", 256), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = State}}. + +encrypt_init(#ssh{encrypt = SshCipher, role = Role} = Ssh) -> + {IvMagic, KeyMagic} = encrypt_magic(Role), + #cipher{impl = CryptoCipher, + key_bytes = KeyBytes, + iv_bytes = IvBytes, + block_bytes = BlockBytes} = cipher(SshCipher), + IV = hash(Ssh, IvMagic, 8*IvBytes), + K = hash(Ssh, KeyMagic, 8*KeyBytes), + Ctx0 = crypto:crypto_init(CryptoCipher, K, IV, true), + {ok, Ssh#ssh{encrypt_block_size = BlockBytes, + encrypt_ctx = Ctx0}}. encrypt_final(Ssh) -> {ok, Ssh#ssh{encrypt = none, @@ -1487,249 +1435,126 @@ encrypt_final(Ssh) -> encrypt_ctx = undefined }}. + encrypt(#ssh{encrypt = none} = Ssh, Data) -> {Ssh, Data}; + encrypt(#ssh{encrypt = '[email protected]', encrypt_keys = {K1,K2}, send_sequence = Seq} = Ssh, <<LenData:4/binary, PayloadData/binary>>) -> %% Encrypt length IV1 = <<0:8/unit:8, Seq:8/unit:8>>, - {_,EncLen} = crypto:stream_encrypt(crypto:stream_init(chacha20, K1, IV1), - LenData), + EncLen = crypto:crypto_one_shot(chacha20, K1, IV1, LenData, true), %% Encrypt payload IV2 = <<1:8/little-unit:8, Seq:8/unit:8>>, - {_,EncPayloadData} = crypto:stream_encrypt(crypto:stream_init(chacha20, K2, IV2), - PayloadData), - + EncPayloadData = crypto:crypto_one_shot(chacha20, K2, IV2, PayloadData, true), %% MAC tag - {_,PolyKey} = crypto:stream_encrypt(crypto:stream_init(chacha20, K2, <<0:8/unit:8,Seq:8/unit:8>>), - <<0:32/unit:8>>), + PolyKey = crypto:crypto_one_shot(chacha20, K2, <<0:8/unit:8,Seq:8/unit:8>>, <<0:32/unit:8>>, true), EncBytes = <<EncLen/binary,EncPayloadData/binary>>, Ctag = crypto:poly1305(PolyKey, EncBytes), %% Result {Ssh, {EncBytes,Ctag}}; -encrypt(#ssh{encrypt = 'AEAD_AES_128_GCM', - encrypt_keys = K, + +encrypt(#ssh{encrypt = SshCipher, + encrypt_keys = K, encrypt_ctx = IV0} = Ssh, - <<LenData:4/binary, PayloadData/binary>>) -> - {Ctext,Ctag} = crypto:block_encrypt(aes_gcm, K, IV0, {LenData,PayloadData}), - IV = next_gcm_iv(IV0), - {Ssh#ssh{encrypt_ctx = IV}, {<<LenData/binary,Ctext/binary>>,Ctag}}; -encrypt(#ssh{encrypt = 'AEAD_AES_256_GCM', - encrypt_keys = K, - encrypt_ctx = IV0} = Ssh, - <<LenData:4/binary, PayloadData/binary>>) -> + <<LenData:4/binary, PayloadData/binary>>) when SshCipher == 'AEAD_AES_128_GCM' ; + SshCipher == 'AEAD_AES_256_GCM' -> {Ctext,Ctag} = crypto:block_encrypt(aes_gcm, K, IV0, {LenData,PayloadData}), IV = next_gcm_iv(IV0), {Ssh#ssh{encrypt_ctx = IV}, {<<LenData/binary,Ctext/binary>>,Ctag}}; -encrypt(#ssh{encrypt = '3des-cbc', - encrypt_keys = {K1,K2,K3}, - encrypt_ctx = IV0} = Ssh, Data) -> - Enc = crypto:block_encrypt(des3_cbc, [K1,K2,K3], IV0, Data), - IV = crypto:next_iv(des3_cbc, Enc), - {Ssh#ssh{encrypt_ctx = IV}, Enc}; -encrypt(#ssh{encrypt = 'aes128-cbc', - encrypt_keys = K, - encrypt_ctx = IV0} = Ssh, Data) -> - Enc = crypto:block_encrypt(aes_cbc128, K,IV0,Data), - IV = crypto:next_iv(aes_cbc, Enc), - {Ssh#ssh{encrypt_ctx = IV}, Enc}; -encrypt(#ssh{encrypt = 'aes128-ctr', - encrypt_ctx = State0} = Ssh, Data) -> - {State, Enc} = crypto:stream_encrypt(State0,Data), - {Ssh#ssh{encrypt_ctx = State}, Enc}; -encrypt(#ssh{encrypt = 'aes192-ctr', - encrypt_ctx = State0} = Ssh, Data) -> - {State, Enc} = crypto:stream_encrypt(State0,Data), - {Ssh#ssh{encrypt_ctx = State}, Enc}; -encrypt(#ssh{encrypt = 'aes256-ctr', - encrypt_ctx = State0} = Ssh, Data) -> - {State, Enc} = crypto:stream_encrypt(State0,Data), - {Ssh#ssh{encrypt_ctx = State}, Enc}. - +encrypt(#ssh{encrypt_ctx = Ctx0} = Ssh, Data) -> + Enc = crypto:crypto_update(Ctx0, Data), + {Ssh, Enc}. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Decryption %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% decrypt_init(#ssh{decrypt = none} = Ssh) -> {ok, Ssh}; -decrypt_init(#ssh{decrypt = '[email protected]', role = client} = Ssh) -> - <<K2:32/binary,K1:32/binary>> = hash(Ssh, "D", 512), - {ok, Ssh#ssh{decrypt_keys = {K1,K2} - }}; -decrypt_init(#ssh{decrypt = '[email protected]', role = server} = Ssh) -> - <<K2:32/binary,K1:32/binary>> = hash(Ssh, "C", 512), + +decrypt_init(#ssh{decrypt = '[email protected]', role = Role} = Ssh) -> + {_, KeyMagic} = decrypt_magic(Role), + <<K2:32/binary,K1:32/binary>> = hash(Ssh, KeyMagic, 8*64), {ok, Ssh#ssh{decrypt_keys = {K1,K2} }}; -decrypt_init(#ssh{decrypt = 'AEAD_AES_128_GCM', role = client} = Ssh) -> - IV = hash(Ssh, "B", 12*8), - <<K:16/binary>> = hash(Ssh, "D", 128), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = IV}}; -decrypt_init(#ssh{decrypt = 'AEAD_AES_128_GCM', role = server} = Ssh) -> - IV = hash(Ssh, "A", 12*8), - <<K:16/binary>> = hash(Ssh, "C", 128), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = IV}}; -decrypt_init(#ssh{decrypt = 'AEAD_AES_256_GCM', role = client} = Ssh) -> - IV = hash(Ssh, "B", 12*8), - <<K:32/binary>> = hash(Ssh, "D", 256), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = IV}}; -decrypt_init(#ssh{decrypt = 'AEAD_AES_256_GCM', role = server} = Ssh) -> - IV = hash(Ssh, "A", 12*8), - <<K:32/binary>> = hash(Ssh, "C", 256), + +decrypt_init(#ssh{decrypt = SshCipher, role = Role} = Ssh) when SshCipher == 'AEAD_AES_128_GCM'; + SshCipher == 'AEAD_AES_256_GCM' -> + {IvMagic, KeyMagic} = decrypt_magic(Role), + #cipher{key_bytes = KeyBytes, + iv_bytes = IvBytes, + block_bytes = BlockBytes} = cipher(SshCipher), + IV = hash(Ssh, IvMagic, 8*IvBytes), + K = hash(Ssh, KeyMagic, 8*KeyBytes), {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, + decrypt_block_size = BlockBytes, decrypt_ctx = IV}}; -decrypt_init(#ssh{decrypt = '3des-cbc', role = client} = Ssh) -> - {IV, KD} = {hash(Ssh, "B", 64), - hash(Ssh, "D", 192)}, - <<K1:8/binary, K2:8/binary, K3:8/binary>> = KD, - {ok, Ssh#ssh{decrypt_keys = {K1,K2,K3}, decrypt_ctx = IV, - decrypt_block_size = 8}}; -decrypt_init(#ssh{decrypt = '3des-cbc', role = server} = Ssh) -> - {IV, KD} = {hash(Ssh, "A", 64), - hash(Ssh, "C", 192)}, - <<K1:8/binary, K2:8/binary, K3:8/binary>> = KD, - {ok, Ssh#ssh{decrypt_keys = {K1, K2, K3}, decrypt_ctx = IV, - decrypt_block_size = 8}}; -decrypt_init(#ssh{decrypt = 'aes128-cbc', role = client} = Ssh) -> - {IV, KD} = {hash(Ssh, "B", 128), - hash(Ssh, "D", 128)}, - <<K:16/binary>> = KD, - {ok, Ssh#ssh{decrypt_keys = K, decrypt_ctx = IV, - decrypt_block_size = 16}}; -decrypt_init(#ssh{decrypt = 'aes128-cbc', role = server} = Ssh) -> - {IV, KD} = {hash(Ssh, "A", 128), - hash(Ssh, "C", 128)}, - <<K:16/binary>> = KD, - {ok, Ssh#ssh{decrypt_keys = K, decrypt_ctx = IV, - decrypt_block_size = 16}}; -decrypt_init(#ssh{decrypt = 'aes128-ctr', role = client} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:16/binary>> = hash(Ssh, "D", 128), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = State}}; -decrypt_init(#ssh{decrypt = 'aes192-ctr', role = client} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:24/binary>> = hash(Ssh, "D", 192), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = State}}; -decrypt_init(#ssh{decrypt = 'aes256-ctr', role = client} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:32/binary>> = hash(Ssh, "D", 256), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = State}}; -decrypt_init(#ssh{decrypt = 'aes128-ctr', role = server} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:16/binary>> = hash(Ssh, "C", 128), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = State}}; -decrypt_init(#ssh{decrypt = 'aes192-ctr', role = server} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:24/binary>> = hash(Ssh, "C", 192), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = State}}; -decrypt_init(#ssh{decrypt = 'aes256-ctr', role = server} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:32/binary>> = hash(Ssh, "C", 256), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = State}}. - +decrypt_init(#ssh{decrypt = SshCipher, role = Role} = Ssh) -> + {IvMagic, KeyMagic} = decrypt_magic(Role), + #cipher{impl = CryptoCipher, + key_bytes = KeyBytes, + iv_bytes = IvBytes, + block_bytes = BlockBytes} = cipher(SshCipher), + IV = hash(Ssh, IvMagic, 8*IvBytes), + K = hash(Ssh, KeyMagic, 8*KeyBytes), + Ctx0 = crypto:crypto_init(CryptoCipher, K, IV, false), + {ok, Ssh#ssh{decrypt_block_size = BlockBytes, + decrypt_ctx = Ctx0}}. + decrypt_final(Ssh) -> {ok, Ssh#ssh {decrypt = none, decrypt_keys = undefined, decrypt_ctx = undefined, decrypt_block_size = 8}}. + decrypt(Ssh, <<>>) -> {Ssh, <<>>}; + decrypt(#ssh{decrypt = '[email protected]', decrypt_keys = {K1,_K2}, recv_sequence = Seq} = Ssh, {length,EncryptedLen}) -> - {_State,PacketLenBin} = - crypto:stream_decrypt(crypto:stream_init(chacha20, K1, <<0:8/unit:8, Seq:8/unit:8>>), - EncryptedLen), + PacketLenBin = crypto:crypto_one_shot(chacha20, K1, <<0:8/unit:8, Seq:8/unit:8>>, EncryptedLen, false), {Ssh, PacketLenBin}; + decrypt(#ssh{decrypt = '[email protected]', decrypt_keys = {_K1,K2}, recv_sequence = Seq} = Ssh, {AAD,Ctext,Ctag}) -> %% The length is already decoded and used to divide the input %% Check the mac (important that it is timing-safe): - {_,PolyKey} = - crypto:stream_encrypt(crypto:stream_init(chacha20, K2, <<0:8/unit:8,Seq:8/unit:8>>), - <<0:32/unit:8>>), + PolyKey = crypto:crypto_one_shot(chacha20, K2, <<0:8/unit:8,Seq:8/unit:8>>, <<0:32/unit:8>>, false), case equal_const_time(Ctag, crypto:poly1305(PolyKey, <<AAD/binary,Ctext/binary>>)) of true -> %% MAC is ok, decode IV2 = <<1:8/little-unit:8, Seq:8/unit:8>>, - {_,PlainText} = - crypto:stream_decrypt(crypto:stream_init(chacha20,K2,IV2), Ctext), + PlainText = crypto:crypto_one_shot(chacha20, K2, IV2, Ctext, false), {Ssh, PlainText}; false -> {Ssh,error} end; + decrypt(#ssh{decrypt = none} = Ssh, Data) -> {Ssh, Data}; -decrypt(#ssh{decrypt = 'AEAD_AES_128_GCM', - decrypt_keys = K, - decrypt_ctx = IV0} = Ssh, Data = {_AAD,_Ctext,_Ctag}) -> - Dec = crypto:block_decrypt(aes_gcm, K, IV0, Data), % Dec = PlainText | error - IV = next_gcm_iv(IV0), - {Ssh#ssh{decrypt_ctx = IV}, Dec}; -decrypt(#ssh{decrypt = 'AEAD_AES_256_GCM', + +decrypt(#ssh{decrypt = SshCipher, decrypt_keys = K, - decrypt_ctx = IV0} = Ssh, Data = {_AAD,_Ctext,_Ctag}) -> + decrypt_ctx = IV0} = Ssh, Data = {_AAD,_Ctext,_Ctag}) when SshCipher == 'AEAD_AES_128_GCM' ; + SshCipher == 'AEAD_AES_256_GCM' -> Dec = crypto:block_decrypt(aes_gcm, K, IV0, Data), % Dec = PlainText | error IV = next_gcm_iv(IV0), {Ssh#ssh{decrypt_ctx = IV}, Dec}; -decrypt(#ssh{decrypt = '3des-cbc', decrypt_keys = Keys, - decrypt_ctx = IV0} = Ssh, Data) -> - {K1, K2, K3} = Keys, - Dec = crypto:block_decrypt(des3_cbc, [K1,K2,K3], IV0, Data), - IV = crypto:next_iv(des3_cbc, Data), - {Ssh#ssh{decrypt_ctx = IV}, Dec}; -decrypt(#ssh{decrypt = 'aes128-cbc', decrypt_keys = Key, - decrypt_ctx = IV0} = Ssh, Data) -> - Dec = crypto:block_decrypt(aes_cbc128, Key,IV0,Data), - IV = crypto:next_iv(aes_cbc, Data), - {Ssh#ssh{decrypt_ctx = IV}, Dec}; -decrypt(#ssh{decrypt = 'aes128-ctr', - decrypt_ctx = State0} = Ssh, Data) -> - {State, Enc} = crypto:stream_decrypt(State0,Data), - {Ssh#ssh{decrypt_ctx = State}, Enc}; -decrypt(#ssh{decrypt = 'aes192-ctr', - decrypt_ctx = State0} = Ssh, Data) -> - {State, Enc} = crypto:stream_decrypt(State0,Data), - {Ssh#ssh{decrypt_ctx = State}, Enc}; -decrypt(#ssh{decrypt = 'aes256-ctr', - decrypt_ctx = State0} = Ssh, Data) -> - {State, Enc} = crypto:stream_decrypt(State0,Data), - {Ssh#ssh{decrypt_ctx = State}, Enc}. +decrypt(#ssh{decrypt_ctx = Ctx0} = Ssh, Data) -> + Dec = crypto:crypto_update(Ctx0, Data), + {Ssh, Dec}. next_gcm_iv(<<Fixed:32, InvCtr:64>>) -> <<Fixed:32, (InvCtr+1):64>>. - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Compression %% @@ -2058,9 +1883,9 @@ compute_key(Algorithm, OthersPublic, MyPrivate, Args) -> dh_bits(#alg{encrypt = Encrypt, send_mac = SendMac}) -> C = cipher(Encrypt), - 8 * lists:max([C#cipher_data.key_bytes, - C#cipher_data.block_bytes, - C#cipher_data.iv_bytes, + 8 * lists:max([C#cipher.key_bytes, + C#cipher.block_bytes, + C#cipher.iv_bytes, mac_key_bytes(SendMac) ]). @@ -2091,40 +1916,13 @@ select_crypto_supported(L) -> crypto_supported(Conditions, Supported) -> lists:all( fun({Tag,CryptoName}) when is_atom(CryptoName) -> - crypto_name_supported(Tag,CryptoName,Supported); - ({Tag,{Name,Len}}) when is_integer(Len) -> - crypto_name_supported(Tag,Name,Supported) andalso - len_supported(Name,Len) + crypto_name_supported(Tag,CryptoName,Supported) end, Conditions). crypto_name_supported(Tag, CryptoName, Supported) -> - Vs = case proplists:get_value(Tag,Supported,[]) of - [] when Tag == curves -> crypto:ec_curves(); - L -> L - end, + Vs = proplists:get_value(Tag,Supported,[]), lists:member(CryptoName, Vs). -len_supported(Name, Len) -> - try - case Name of - aes_ctr -> - {_, <<_/binary>>} = - %% Test encryption - crypto:stream_encrypt(crypto:stream_init(Name, <<0:Len>>, <<0:128>>), <<"">>); - aes_gcm -> - {<<_/binary>>, <<_/binary>>} = - crypto:block_encrypt(Name, - _Key = <<0:Len>>, - _IV = <<0:12/unsigned-unit:8>>, - {<<"AAD">>,"PT"}) - end - of - _ -> true - catch - _:_ -> false - end. - - same(Algs) -> [{client2server,Algs}, {server2client,Algs}]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/ssh/test/ssh_protocol_SUITE.erl b/lib/ssh/test/ssh_protocol_SUITE.erl index 3e3e151781..b12ddfeef6 100644 --- a/lib/ssh/test/ssh_protocol_SUITE.erl +++ b/lib/ssh/test/ssh_protocol_SUITE.erl @@ -38,7 +38,11 @@ -define(EXTRA_KEX, 'diffie-hellman-group1-sha1'). -define(CIPHERS, ['aes256-ctr','aes192-ctr','aes128-ctr','aes128-cbc','3des-cbc']). --define(DEFAULT_CIPHERS, [{client2server,?CIPHERS}, {server2client,?CIPHERS}]). +-define(DEFAULT_CIPHERS, (fun() -> Ciphs = filter_supported(cipher, ?CIPHERS), + [{client2server,Ciphs}, {server2client,Ciphs}] + end)() + ). + -define(v(Key, Config), proplists:get_value(Key, Config)). -define(v(Key, Config, Default), proplists:get_value(Key, Config, Default)). diff --git a/lib/ssh/vsn.mk b/lib/ssh/vsn.mk index 2890d7fe5b..0f9eee887c 100644 --- a/lib/ssh/vsn.mk +++ b/lib/ssh/vsn.mk @@ -1,4 +1,4 @@ #-*-makefile-*- ; force emacs to enter makefile-mode -SSH_VSN = 4.7.3 +SSH_VSN = 4.7.4 APP_VSN = "ssh-$(SSH_VSN)" diff --git a/lib/ssl/doc/src/notes.xml b/lib/ssl/doc/src/notes.xml index 82eb8ff700..a511cb4db3 100644 --- a/lib/ssl/doc/src/notes.xml +++ b/lib/ssl/doc/src/notes.xml @@ -27,6 +27,117 @@ </header> <p>This document describes the changes made to the SSL application.</p> +<section><title>SSL 9.2.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The timeout for a passive receive was sometimes not + cancelled and later caused a server crash. This bug has + now been corrected.</p> + <p> + Own Id: OTP-14701 Aux Id: ERL-883, ERL-884 </p> + </item> + <item> + <p> + Add tag for passive message (active N) in cb_info to + retain transport transparency.</p> + <p> + Own Id: OTP-15679 Aux Id: ERL-861 </p> + </item> + </list> + </section> + +</section> + +<section><title>SSL 9.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix bug that an incorrect return value for gen_statem + could be created when alert was a result of handling + renegotiation info extension</p> + <p> + Own Id: OTP-15502</p> + </item> + <item> + <p> + Correct check for 3des_ede_cbc, could cause ssl to claim + to support 3des_ede_cbc when cryptolib does not.</p> + <p> + Own Id: OTP-15539</p> + </item> + <item> + <p> + Improved DTLS error handling, avoids unexpected + connection failure in rare cases.</p> + <p> + Own Id: OTP-15561</p> + </item> + <item> + <p> + Corrected active once emulation bug that could cause the + ssl_closed meassage to not be sent. Bug introduced by + OTP-15449</p> + <p> + Own Id: OTP-15666 Aux Id: ERIERL-316, </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Add client option {reuse_session, SessionID::binary()} + that can be used together with new option value + {reuse_sessions, save}. This makes it possible to reuse a + session from a specific connection establishment.</p> + <p> + Own Id: OTP-15369</p> + </item> + <item> + <p> + The Reason part of of the error return from the functions + connect and handshake has a better and documented format. + This will sometimes differ from previous returned + reasons, however those where only documented as term() + and should for that reason not be relied on.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-15423</p> + </item> + <item> + <p> + Refactor of state handling to improve TLS application + data throughput and reduce CPU overhead</p> + <p> + Own Id: OTP-15445</p> + </item> + <item> + <p> + The SSL code has been optimized in many small ways to + reduce CPU load for encryption/decryption, especially for + Erlang's distribution protocol over TLS.</p> + <p> + Own Id: OTP-15529</p> + </item> + <item> + <p> + Add support for active N</p> + <p> + Own Id: OTP-15665 Aux Id: ERL-811, PR-2072 </p> + </item> + </list> + </section> + +</section> + <section><title>SSL 9.1.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index b145aac6ab..37bf9033a1 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -101,16 +101,21 @@ <datatype> <name name="transport_option"/> <desc> - <p>Defaults to <c>{gen_tcp, tcp, tcp_closed, tcp_error}</c> - for TLS and <c>{gen_udp, udp, udp_closed, udp_error}</c> for - DTLS. Can be used to customize the transport layer. The tag - values should be the values used by the underlying transport - in its active mode messages. For TLS the callback module must implement a - reliable transport protocol, behave as <c>gen_tcp</c>, and have functions - corresponding to <c>inet:setopts/2</c>, <c>inet:getopts/2</c>, - <c>inet:peername/1</c>, <c>inet:sockname/1</c>, and <c>inet:port/1</c>. - The callback <c>gen_tcp</c> is treated specially and calls <c>inet</c> - directly. For DTLS this feature must be considered exprimental. + <p>Defaults to <c>{gen_tcp, tcp, tcp_closed, tcp_error, + tcp_passive}</c> for TLS (for backward compatibility a four + tuple will be converted to a five tuple with the last element + "second_element"_passive) and <c>{gen_udp, udp, udp_closed, + udp_error}</c> for DTLS (might also be changed to five tuple in + the future). Can be used to customize the transport layer. The + tag values should be the values used by the underlying + transport in its active mode messages. For TLS the callback + module must implement a reliable transport protocol, behave as + <c>gen_tcp</c>, and have functions corresponding to + <c>inet:setopts/2</c>, <c>inet:getopts/2</c>, + <c>inet:peername/1</c>, <c>inet:sockname/1</c>, and + <c>inet:port/1</c>. The callback <c>gen_tcp</c> is treated + specially and calls <c>inet</c> directly. For DTLS this + feature must be considered exprimental. </p> </desc> </datatype> @@ -1658,17 +1663,6 @@ fun(srp, Username :: string(), UserState :: term()) -> </func> <func> - <name since="OTP 22.0">set_log_level(Level) -> ok | {error, Reason}</name> - <fsummary>Sets log level for the SSL application.</fsummary> - <type> - <v>Level = atom()</v> - </type> - <desc> - <p>Sets log level for the SSL application.</p> - </desc> - </func> - - <func> <name since="OTP R14B">shutdown(SslSocket, How) -> ok | {error, Reason}</name> <fsummary>Immediately closes a socket.</fsummary> <type> diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index 5a2d31ffc2..bfa349c8d8 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -55,8 +55,7 @@ format_error/1, renegotiate/1, prf/5, negotiated_protocol/1, connection_information/1, connection_information/2]). %% Misc --export([handle_options/2, tls_version/1, new_ssl_options/3, suite_to_str/1, - set_log_level/1]). +-export([handle_options/2, tls_version/1, new_ssl_options/3, suite_to_str/1]). -deprecated({ssl_accept, 1, eventually}). -deprecated({ssl_accept, 2, eventually}). @@ -96,7 +95,9 @@ -type active_msgs() :: {ssl, sslsocket(), Data::binary() | list()} | {ssl_closed, sslsocket()} | {ssl_error, sslsocket(), Reason::term()} | {ssl_passive, sslsocket()}. -type transport_option() :: {cb_info, {CallbackModule::atom(), DataTag::atom(), - ClosedTag::atom(), ErrTag::atom()}}. + ClosedTag::atom(), ErrTag::atom()}} | + {cb_info, {CallbackModule::atom(), DataTag::atom(), + ClosedTag::atom(), ErrTag::atom(), PassiveTag::atom()}}. -type host() :: hostname() | ip_address(). -type hostname() :: string(). -type ip_address() :: inet:ip_address(). @@ -422,9 +423,9 @@ connect(Socket, SslOptions) when is_port(Socket) -> timeout() | list()) -> {ok, #sslsocket{}} | {error, reason()}. connect(Socket, SslOptions0, Timeout) when is_port(Socket), - (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> - {Transport,_,_,_} = proplists:get_value(cb_info, SslOptions0, - {gen_tcp, tcp, tcp_closed, tcp_error}), + (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> + CbInfo = handle_option(cb_info, SslOptions0, default_cb_info(tls)), + Transport = element(1, CbInfo), EmulatedOptions = tls_socket:emulated_options(), {ok, SocketValues} = tls_socket:getopts(Transport, Socket, EmulatedOptions), try handle_options(SslOptions0 ++ SocketValues, client) of @@ -572,8 +573,8 @@ handshake(#sslsocket{pid = [Pid|_], fd = {_, _, _}} = Socket, SslOpts, Timeout) end; handshake(Socket, SslOptions, Timeout) when is_port(Socket), (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> - {Transport,_,_,_} = - proplists:get_value(cb_info, SslOptions, {gen_tcp, tcp, tcp_closed, tcp_error}), + CbInfo = handle_option(cb_info, SslOptions, default_cb_info(tls)), + Transport = element(1, CbInfo), EmulatedOptions = tls_socket:emulated_options(), {ok, SocketValues} = tls_socket:getopts(Transport, Socket, EmulatedOptions), ConnetionCb = connection_cb(SslOptions), @@ -625,7 +626,7 @@ close(#sslsocket{pid = [Pid|_]}) when is_pid(Pid) -> ssl_connection:close(Pid, {close, ?DEFAULT_TIMEOUT}); close(#sslsocket{pid = {dtls, #config{dtls_handler = {Pid, _}}}}) -> dtls_packet_demux:close(Pid); -close(#sslsocket{pid = {ListenSocket, #config{transport_info={Transport,_, _, _}}}}) -> +close(#sslsocket{pid = {ListenSocket, #config{transport_info={Transport,_,_,_,_}}}}) -> Transport:close(ListenSocket). %%-------------------------------------------------------------------- @@ -641,7 +642,7 @@ close(#sslsocket{pid = [TLSPid|_]}, close(#sslsocket{pid = [TLSPid|_]}, Timeout) when is_pid(TLSPid), (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> ssl_connection:close(TLSPid, {close, Timeout}); -close(#sslsocket{pid = {ListenSocket, #config{transport_info={Transport,_, _, _}}}}, _) -> +close(#sslsocket{pid = {ListenSocket, #config{transport_info={Transport,_,_,_,_}}}}, _) -> Transport:close(ListenSocket). %%-------------------------------------------------------------------- @@ -657,7 +658,8 @@ send(#sslsocket{pid = {_, #config{transport_info={_, udp, _, _}}}}, _) -> {error,enotconn}; %% Emulate connection behaviour send(#sslsocket{pid = {dtls,_}}, _) -> {error,enotconn}; %% Emulate connection behaviour -send(#sslsocket{pid = {ListenSocket, #config{transport_info={Transport, _, _, _}}}}, Data) -> +send(#sslsocket{pid = {ListenSocket, #config{transport_info = Info}}}, Data) -> + Transport = element(1, Info), Transport:send(ListenSocket, Data). %% {error,enotconn} %%-------------------------------------------------------------------- @@ -675,7 +677,8 @@ recv(#sslsocket{pid = [Pid|_]}, Length, Timeout) when is_pid(Pid), recv(#sslsocket{pid = {dtls,_}}, _, _) -> {error,enotconn}; recv(#sslsocket{pid = {Listen, - #config{transport_info = {Transport, _, _, _}}}}, _,_) when is_port(Listen)-> + #config{transport_info = Info}}},_,_) when is_port(Listen)-> + Transport = element(1, Info), Transport:recv(Listen, 0). %% {error,enotconn} %%-------------------------------------------------------------------- @@ -690,7 +693,7 @@ controlling_process(#sslsocket{pid = {dtls, _}}, NewOwner) when is_pid(NewOwner) -> ok; %% Meaningless but let it be allowed to conform with TLS controlling_process(#sslsocket{pid = {Listen, - #config{transport_info = {Transport, _, _, _}}}}, + #config{transport_info = {Transport,_,_,_,_}}}}, NewOwner) when is_port(Listen), is_pid(NewOwner) -> %% Meaningless but let it be allowed to conform with normal sockets @@ -733,13 +736,13 @@ connection_information(#sslsocket{pid = [Pid|_]}, Items) when is_pid(Pid) -> %% %% Description: same as inet:peername/1. %%-------------------------------------------------------------------- -peername(#sslsocket{pid = [Pid|_], fd = {Transport, Socket, _}}) when is_pid(Pid)-> +peername(#sslsocket{pid = [Pid|_], fd = {Transport, Socket,_}}) when is_pid(Pid)-> dtls_socket:peername(Transport, Socket); -peername(#sslsocket{pid = [Pid|_], fd = {Transport, Socket, _, _}}) when is_pid(Pid)-> +peername(#sslsocket{pid = [Pid|_], fd = {Transport, Socket,_,_}}) when is_pid(Pid)-> tls_socket:peername(Transport, Socket); -peername(#sslsocket{pid = {dtls, #config{dtls_handler = {_Pid, _}}}}) -> +peername(#sslsocket{pid = {dtls, #config{dtls_handler = {_Pid,_}}}}) -> dtls_socket:peername(dtls, undefined); -peername(#sslsocket{pid = {ListenSocket, #config{transport_info = {Transport,_,_,_}}}}) -> +peername(#sslsocket{pid = {ListenSocket, #config{transport_info = {Transport,_,_,_,_}}}}) -> tls_socket:peername(Transport, ListenSocket); %% Will return {error, enotconn} peername(#sslsocket{pid = {dtls,_}}) -> {error,enotconn}. @@ -931,7 +934,7 @@ getopts(#sslsocket{pid = {dtls, #config{transport_info = {Transport,_,_,_}}}} = _:Error -> {error, {options, {socket_options, OptionTags, Error}}} end; -getopts(#sslsocket{pid = {_, #config{transport_info = {Transport,_,_,_}}}} = ListenSocket, +getopts(#sslsocket{pid = {_, #config{transport_info = {Transport,_,_,_,_}}}} = ListenSocket, OptionTags) when is_list(OptionTags) -> try tls_socket:getopts(Transport, ListenSocket, OptionTags) of {ok, _} = Result -> @@ -988,7 +991,7 @@ setopts(#sslsocket{pid = {dtls, #config{transport_info = {Transport,_,_,_}}}} = _:Error -> {error, {options, {socket_options, Options, Error}}} end; -setopts(#sslsocket{pid = {_, #config{transport_info = {Transport,_,_,_}}}} = ListenSocket, Options) when is_list(Options) -> +setopts(#sslsocket{pid = {_, #config{transport_info = {Transport,_,_,_,_}}}} = ListenSocket, Options) when is_list(Options) -> try tls_socket:setopts(Transport, ListenSocket, Options) of ok -> ok; @@ -1032,8 +1035,9 @@ getstat(#sslsocket{pid = [Pid|_], fd = {Transport, Socket, _, _}}, Options) when %% %% Description: Same as gen_tcp:shutdown/2 %%-------------------------------------------------------------------- -shutdown(#sslsocket{pid = {Listen, #config{transport_info = {Transport,_, _, _}}}}, +shutdown(#sslsocket{pid = {Listen, #config{transport_info = Info}}}, How) when is_port(Listen) -> + Transport = element(1, Info), Transport:shutdown(Listen, How); shutdown(#sslsocket{pid = {dtls,_}},_) -> {error, enotconn}; @@ -1045,13 +1049,13 @@ shutdown(#sslsocket{pid = [Pid|_]}, How) when is_pid(Pid) -> %% %% Description: Same as inet:sockname/1 %%-------------------------------------------------------------------- -sockname(#sslsocket{pid = {Listen, #config{transport_info = {Transport, _, _, _}}}}) when is_port(Listen) -> +sockname(#sslsocket{pid = {Listen, #config{transport_info = {Transport,_,_,_,_}}}}) when is_port(Listen) -> tls_socket:sockname(Transport, Listen); sockname(#sslsocket{pid = {dtls, #config{dtls_handler = {Pid, _}}}}) -> dtls_packet_demux:sockname(Pid); -sockname(#sslsocket{pid = [Pid|_], fd = {Transport, Socket, _}}) when is_pid(Pid) -> +sockname(#sslsocket{pid = [Pid|_], fd = {Transport, Socket,_}}) when is_pid(Pid) -> dtls_socket:sockname(Transport, Socket); -sockname(#sslsocket{pid = [Pid| _], fd = {Transport, Socket, _, _}}) when is_pid(Pid) -> +sockname(#sslsocket{pid = [Pid| _], fd = {Transport, Socket,_,_}}) when is_pid(Pid) -> tls_socket:sockname(Transport, Socket). %%--------------------------------------------------------------- @@ -1167,32 +1171,6 @@ suite_to_str(Cipher) -> ssl_cipher_format:suite_to_str(Cipher). -%%-------------------------------------------------------------------- --spec set_log_level(atom()) -> ok | {error, term()}. -%% -%% Description: Set log level for the SSL application -%%-------------------------------------------------------------------- -set_log_level(Level) -> - case application:get_all_key(ssl) of - {ok, PropList} -> - Modules = proplists:get_value(modules, PropList), - set_module_level(Modules, Level); - undefined -> - {error, ssl_not_started} - end. - -set_module_level(Modules, Level) -> - Fun = fun (Module) -> - ok = logger:set_module_level(Module, Level) - end, - try lists:map(Fun, Modules) of - _ -> - ok - catch - error:{badmatch, Error} -> - Error - end. - %%%-------------------------------------------------------------- %%% Internal functions %%%-------------------------------------------------------------------- @@ -1211,7 +1189,7 @@ supported_suites(all, Version) -> supported_suites(anonymous, Version) -> ssl_cipher:anonymous_suites(Version). -do_listen(Port, #config{transport_info = {Transport, _, _, _}} = Config, tls_connection) -> +do_listen(Port, #config{transport_info = {Transport, _, _, _,_}} = Config, tls_connection) -> tls_socket:listen(Transport, Port, Config); do_listen(Port, Config, dtls_connection) -> @@ -1381,7 +1359,7 @@ handle_options(Opts0, Role, Host) -> log_level = handle_option(log_level, Opts, LogLevel) }, - CbInfo = proplists:get_value(cb_info, Opts, default_cb_info(Protocol)), + CbInfo = handle_option(cb_info, Opts, default_cb_info(Protocol)), SslOptions = [protocol, versions, verify, verify_fun, partial_chain, fail_if_no_peer_cert, verify_client_once, depth, cert, certfile, key, keyfile, @@ -1425,6 +1403,10 @@ handle_option(sni_fun, Opts, Default) -> _ -> throw({error, {conflict_options, [sni_fun, sni_hosts]}}) end; +handle_option(cb_info, Opts, Default) -> + CbInfo = proplists:get_value(cb_info, Opts, Default), + true = validate_option(cb_info, CbInfo), + handle_cb_info(CbInfo, Default); handle_option(OptionName, Opts, Default) -> validate_option(OptionName, proplists:get_value(OptionName, Opts, Default)). @@ -1659,9 +1641,29 @@ validate_option(handshake, full = Value) -> Value; validate_option(customize_hostname_check, Value) when is_list(Value) -> Value; +validate_option(cb_info, {V1, V2, V3, V4}) when is_atom(V1), + is_atom(V2), + is_atom(V3), + is_atom(V4) + -> + true; +validate_option(cb_info, {V1, V2, V3, V4, V5}) when is_atom(V1), + is_atom(V2), + is_atom(V3), + is_atom(V4), + is_atom(V5) + -> + true; +validate_option(cb_info, _) -> + false; validate_option(Opt, Value) -> throw({error, {options, {Opt, Value}}}). +handle_cb_info({V1, V2, V3, V4}, {_,_,_,_,_}) -> + {V1,V2,V3,V4, list_to_atom(atom_to_list(V2) ++ "passive")}; +handle_cb_info(CbInfo, _) -> + CbInfo. + handle_hashsigns_option(Value, Version) when is_list(Value) andalso Version >= {3, 4} -> case tls_v1:signature_schemes(Version, Value) of @@ -2132,7 +2134,7 @@ default_option_role(_,_,_) -> default_cb_info(tls) -> - {gen_tcp, tcp, tcp_closed, tcp_error}; + {gen_tcp, tcp, tcp_closed, tcp_error, tcp_passive}; default_cb_info(dtls) -> {gen_udp, udp, udp_closed, udp_error}. diff --git a/lib/ssl/src/ssl_alert.erl b/lib/ssl/src/ssl_alert.erl index e17476f33b..06b1b005a5 100644 --- a/lib/ssl/src/ssl_alert.erl +++ b/lib/ssl/src/ssl_alert.erl @@ -161,6 +161,8 @@ description_txt(?INSUFFICIENT_SECURITY) -> "Insufficient Security"; description_txt(?INTERNAL_ERROR) -> "Internal Error"; +description_txt(?INAPPROPRIATE_FALLBACK) -> + "Inappropriate Fallback"; description_txt(?USER_CANCELED) -> "User Canceled"; description_txt(?NO_RENEGOTIATION) -> @@ -179,8 +181,6 @@ description_txt(?BAD_CERTIFICATE_HASH_VALUE) -> "Bad Certificate Hash Value"; description_txt(?UNKNOWN_PSK_IDENTITY) -> "Unknown Psk Identity"; -description_txt(?INAPPROPRIATE_FALLBACK) -> - "Inappropriate Fallback"; description_txt(?CERTIFICATE_REQUIRED) -> "Certificate required"; description_txt(?NO_APPLICATION_PROTOCOL) -> @@ -232,10 +232,14 @@ description_atom(?INSUFFICIENT_SECURITY) -> insufficient_security; description_atom(?INTERNAL_ERROR) -> internal_error; +description_atom(?INAPPROPRIATE_FALLBACK) -> + inappropriate_fallback; description_atom(?USER_CANCELED) -> user_canceled; description_atom(?NO_RENEGOTIATION) -> no_renegotiation; +description_atom(?MISSING_EXTENSION) -> + missing_extension; description_atom(?UNSUPPORTED_EXTENSION) -> unsupported_extension; description_atom(?CERTIFICATE_UNOBTAINABLE) -> @@ -248,9 +252,9 @@ description_atom(?BAD_CERTIFICATE_HASH_VALUE) -> bad_certificate_hash_value; description_atom(?UNKNOWN_PSK_IDENTITY) -> unknown_psk_identity; -description_atom(?INAPPROPRIATE_FALLBACK) -> - inappropriate_fallback; +description_atom(?CERTIFICATE_REQUIRED) -> + certificate_required; description_atom(?NO_APPLICATION_PROTOCOL) -> no_application_protocol; description_atom(_) -> - 'unsupported/unkonwn_alert'. + 'unsupported/unknown_alert'. diff --git a/lib/ssl/src/ssl_app.erl b/lib/ssl/src/ssl_app.erl index 2a5047c75c..9e6d676bef 100644 --- a/lib/ssl/src/ssl_app.erl +++ b/lib/ssl/src/ssl_app.erl @@ -44,11 +44,11 @@ start_logger() -> formatter => {ssl_logger, #{}}}, Filter = {fun logger_filters:domain/2,{log,sub,[otp,ssl]}}, logger:add_handler(ssl_handler, logger_std_h, Config), - logger:add_handler_filter(ssl_handler, filter_non_ssl, Filter). + logger:add_handler_filter(ssl_handler, filter_non_ssl, Filter), + logger:set_application_level(ssl, debug). %% %% Description: Stop SSL logger stop_logger() -> + logger:unset_application_level(ssl), logger:remove_handler(ssl_handler). - - diff --git a/lib/ssl/src/ssl_cipher.erl b/lib/ssl/src/ssl_cipher.erl index 6e751f9ceb..fe8736d2df 100644 --- a/lib/ssl/src/ssl_cipher.erl +++ b/lib/ssl/src/ssl_cipher.erl @@ -45,7 +45,7 @@ random_bytes/1, calc_mac_hash/4, calc_mac_hash/6, is_stream_ciphersuite/1, signature_scheme/1, scheme_to_components/1, hash_size/1, effective_key_bits/1, - key_material/1]). + key_material/1, signature_algorithm_to_scheme/1]). %% RFC 8446 TLS 1.3 -export([generate_client_shares/1, generate_server_share/1, add_zero_padding/2]). @@ -900,6 +900,18 @@ scheme_to_components(rsa_pss_pss_sha512) -> {sha512, rsa_pss_pss, undefined}; scheme_to_components(rsa_pkcs1_sha1) -> {sha1, rsa_pkcs1, undefined}; scheme_to_components(ecdsa_sha1) -> {sha1, ecdsa, undefined}. + +%% TODO: Add support for EC and RSA-SSA signatures +signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?sha1WithRSAEncryption}) -> + rsa_pkcs1_sha1; +signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?sha256WithRSAEncryption}) -> + rsa_pkcs1_sha256; +signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?sha384WithRSAEncryption}) -> + rsa_pkcs1_sha384; +signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?sha512WithRSAEncryption}) -> + rsa_pkcs1_sha512. + + %% RFC 5246: 6.2.3.2. CBC Block Cipher %% %% Implementation note: Canvel et al. [CBCTIME] have demonstrated a diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index 422c4c94ae..1e97fe046b 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -385,7 +385,8 @@ handle_alert(#alert{level = ?FATAL} = Alert, StateName, log_alert(SslOpts#ssl_options.log_level, Role, Connection:protocol_name(), StateName, Alert#alert{role = opposite_role(Role)}), Pids = Connection:pids(State), - alert_user(Pids, Transport, Tracker, Socket, StateName, Opts, Pid, From, Alert, Role, Connection), + alert_user(Pids, Transport, Tracker, Socket, StateName, Opts, Pid, From, Alert, + opposite_role(Role), Connection), {stop, {shutdown, normal}, State}; handle_alert(#alert{level = ?WARNING, description = ?CLOSE_NOTIFY} = Alert, @@ -2736,6 +2737,11 @@ ssl_options_list([Key | Keys], [Value | Values], Acc) -> handle_active_option(false, connection = StateName, To, Reply, State) -> hibernate_after(StateName, State, [{reply, To, Reply}]); +handle_active_option(_, connection = StateName, To, _Reply, #state{connection_env = #connection_env{terminated = true}, + user_data_buffer = {_,0,_}} = State) -> + handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY, all_data_deliverd), StateName, + State#state{start_or_recv_from = To}), + {stop,{shutdown, peer_close}, State}; handle_active_option(_, connection = StateName0, To, Reply, #state{static_env = #static_env{protocol_cb = Connection}, user_data_buffer = {_,0,_}} = State0) -> case Connection:next_event(StateName0, no_record, State0) of diff --git a/lib/ssl/src/ssl_connection.hrl b/lib/ssl/src/ssl_connection.hrl index 201164949a..ff7207a8ce 100644 --- a/lib/ssl/src/ssl_connection.hrl +++ b/lib/ssl/src/ssl_connection.hrl @@ -40,6 +40,7 @@ data_tag :: atom(), % ex tcp. close_tag :: atom(), % ex tcp_closed error_tag :: atom(), % ex tcp_error + passive_tag :: atom(), % ex tcp_passive host :: string() | inet:ip_address(), port :: integer(), socket :: port() | tuple(), %% TODO: dtls socket diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index 6b1e3b6e07..6c95a7edf8 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -39,7 +39,7 @@ -type oid() :: tuple(). -type public_key_params() :: #'Dss-Parms'{} | {namedCurve, oid()} | #'ECParameters'{} | term(). -type public_key_info() :: {oid(), #'RSAPublicKey'{} | integer() | #'ECPoint'{}, public_key_params()}. --type ssl_handshake_history() :: {[binary()], [binary()]}. +-type ssl_handshake_history() :: {iodata(), iodata()}. -type ssl_handshake() :: #server_hello{} | #server_hello_done{} | #certificate{} | #certificate_request{} | #client_key_exchange{} | #finished{} | #certificate_verify{} | @@ -76,10 +76,13 @@ handle_client_hello_extensions/9, %% Returns server hello extensions handle_server_hello_extensions/9, select_curve/2, select_curve/3, select_hashsign/4, select_hashsign/5, - select_hashsign_algs/3, empty_extensions/2, add_server_share/2 + select_hashsign_algs/3, empty_extensions/2, add_server_share/3 ]). --export([get_cert_params/1]). +-export([get_cert_params/1, + server_name/3, + validation_fun_and_state/9, + handle_path_validation_error/7]). %%==================================================================== %% Create handshake messages @@ -1150,12 +1153,18 @@ maybe_add_key_share(HelloExtensions, KeyShare) -> HelloExtensions#{key_share => #key_share_client_hello{ client_shares = ClientShares}}. -add_server_share(Extensions, KeyShare) -> +add_server_share(server_hello, Extensions, KeyShare) -> #key_share_server_hello{server_share = ServerShare0} = KeyShare, %% Keep only public keys ServerShare = kse_remove_private_key(ServerShare0), Extensions#{key_share => #key_share_server_hello{ - server_share = ServerShare}}. + server_share = ServerShare}}; +add_server_share(hello_retry_request, Extensions, + #key_share_server_hello{ + server_share = #key_share_entry{group = Group}}) -> + Extensions#{key_share => #key_share_hello_retry_request{ + selected_group = Group}}. + kse_remove_private_key(#key_share_entry{ group = Group, diff --git a/lib/ssl/src/ssl_handshake.hrl b/lib/ssl/src/ssl_handshake.hrl index d4233bea9b..b248edcaa9 100644 --- a/lib/ssl/src/ssl_handshake.hrl +++ b/lib/ssl/src/ssl_handshake.hrl @@ -47,7 +47,9 @@ srp_username, is_resumable, time_stamp, - ecc + ecc, %% TLS 1.3 Group + sign_alg, %% TLS 1.3 Signature Algorithm + dh_public_value %% TLS 1.3 DH Public Value from peer }). -define(NUM_OF_SESSION_ID_BYTES, 32). % TSL 1.1 & SSL 3 diff --git a/lib/ssl/src/ssl_logger.erl b/lib/ssl/src/ssl_logger.erl index b82b3937a1..f497315235 100644 --- a/lib/ssl/src/ssl_logger.erl +++ b/lib/ssl/src/ssl_logger.erl @@ -181,6 +181,11 @@ parse_handshake(Direction, #hello_request{} = HelloRequest) -> [header_prefix(Direction)]), Message = io_lib:format("~p", [?rec_info(hello_request, HelloRequest)]), {Header, Message}; +parse_handshake(Direction, #certificate_request_1_3{} = CertificateRequest) -> + Header = io_lib:format("~s Handshake, CertificateRequest", + [header_prefix(Direction)]), + Message = io_lib:format("~p", [?rec_info(certificate_request_1_3, CertificateRequest)]), + {Header, Message}; parse_handshake(Direction, #certificate_1_3{} = Certificate) -> Header = io_lib:format("~s Handshake, Certificate", [header_prefix(Direction)]), diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index 8eb9e56375..fde73cdef1 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -98,7 +98,7 @@ %% Setup %%==================================================================== start_fsm(Role, Host, Port, Socket, {#ssl_options{erl_dist = false},_, Tracker} = Opts, - User, {CbModule, _,_, _} = CbInfo, + User, {CbModule, _,_, _, _} = CbInfo, Timeout) -> try {ok, Sender} = tls_sender:start(), @@ -112,7 +112,7 @@ start_fsm(Role, Host, Port, Socket, {#ssl_options{erl_dist = false},_, Tracker} end; start_fsm(Role, Host, Port, Socket, {#ssl_options{erl_dist = true},_, Tracker} = Opts, - User, {CbModule, _,_, _} = CbInfo, + User, {CbModule, _,_, _, _} = CbInfo, Timeout) -> try {ok, Sender} = tls_sender:start([{spawn_opt, ?DIST_CNTRL_SPAWN_OPTS}]), @@ -251,13 +251,28 @@ next_event(StateName, Record, State, Actions) -> %%% TLS record protocol level application data messages - -handle_protocol_record(#ssl_tls{type = ?APPLICATION_DATA, fragment = Data}, StateName0, State0) -> +handle_protocol_record(#ssl_tls{type = ?APPLICATION_DATA, fragment = Data}, StateName, + #state{start_or_recv_from = From, + socket_options = #socket_options{active = false}} = State0) when From =/= undefined -> + case ssl_connection:read_application_data(Data, State0) of + {stop, _, _} = Stop-> + Stop; + {Record, #state{start_or_recv_from = Caller} = State1} -> + TimerAction = case Caller of + undefined -> %% Passive recv complete cancel timer + [{{timeout, recv}, infinity, timeout}]; + _ -> + [] + end, + {next_state, StateName, State, Actions} = next_event(StateName, Record, State1, TimerAction), + ssl_connection:hibernate_after(StateName, State, Actions) + end; +handle_protocol_record(#ssl_tls{type = ?APPLICATION_DATA, fragment = Data}, StateName, State0) -> case ssl_connection:read_application_data(Data, State0) of {stop, _, _} = Stop-> Stop; {Record, State1} -> - case next_event(StateName0, Record, State1) of + case next_event(StateName, Record, State1) of {next_state, StateName, State, Actions} -> ssl_connection:hibernate_after(StateName, State, Actions); {stop, _, _} = Stop -> @@ -308,9 +323,7 @@ handle_protocol_record(#ssl_tls{type = ?ALERT, fragment = EncAlerts}, StateName, handle_alerts(Alerts, {next_state, StateName, State}); [] -> ssl_connection:handle_own_alert(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, empty_alert), - Version, StateName, State); - #alert{} = Alert -> - ssl_connection:handle_own_alert(Alert, Version, StateName, State) + Version, StateName, State) catch _:_ -> ssl_connection:handle_own_alert(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, alert_decode_error), @@ -941,7 +954,7 @@ code_change(_OldVsn, StateName, State, _) -> %%% Internal functions %%-------------------------------------------------------------------- initial_state(Role, Sender, Host, Port, Socket, {SSLOptions, SocketOptions, Tracker}, User, - {CbModule, DataTag, CloseTag, ErrorTag}) -> + {CbModule, DataTag, CloseTag, ErrorTag, PassiveTag}) -> #ssl_options{beast_mitigation = BeastMitigation, erl_dist = IsErlDist} = SSLOptions, ConnectionStates = tls_record:init_connection_states(Role, BeastMitigation), @@ -965,6 +978,7 @@ initial_state(Role, Sender, Host, Port, Socket, {SSLOptions, SocketOptions, Trac data_tag = DataTag, close_tag = CloseTag, error_tag = ErrorTag, + passive_tag = PassiveTag, host = Host, port = Port, socket = Socket, @@ -1061,8 +1075,9 @@ handle_info({Protocol, _, Data}, StateName, ssl_connection:handle_normal_shutdown(Alert, StateName, State0), {stop, {shutdown, own_alert}, State0} end; -handle_info({tcp_passive, Socket}, StateName, - #state{static_env = #static_env{socket = Socket}, +handle_info({PassiveTag, Socket}, StateName, + #state{static_env = #static_env{socket = Socket, + passive_tag = PassiveTag}, protocol_specific = PS } = State) -> next_event(StateName, no_record, @@ -1135,6 +1150,7 @@ encode_handshake(Handshake, Version, ConnectionStates0, Hist0) -> encode_change_cipher(#change_cipher_spec{}, Version, ConnectionStates) -> tls_record:encode_change_cipher_spec(Version, ConnectionStates). +-spec decode_alerts(binary()) -> list(). decode_alerts(Bin) -> ssl_alert:decode(Bin). diff --git a/lib/ssl/src/tls_connection_1_3.erl b/lib/ssl/src/tls_connection_1_3.erl index de786d0875..701a5860c2 100644 --- a/lib/ssl/src/tls_connection_1_3.erl +++ b/lib/ssl/src/tls_connection_1_3.erl @@ -110,51 +110,75 @@ %% gen_statem helper functions -export([start/4, negotiated/4, + wait_cert/4, + wait_cv/4, wait_finished/4 ]). -start(internal, - #client_hello{} = Hello, - #state{connection_states = _ConnectionStates0, - ssl_options = #ssl_options{ciphers = _ServerCiphers, - signature_algs = _ServerSignAlgs, - signature_algs_cert = _SignatureSchemes, %% TODO: Check?? - supported_groups = _ServerGroups0, - versions = _Versions} = SslOpts, - session = #session{own_certificate = Cert}} = State0, - _Module) -> - Env = #{cert => Cert}, - case tls_handshake_1_3:handle_client_hello(Hello, SslOpts, Env) of +start(internal, #change_cipher_spec{}, State0, _Module) -> + {Record, State} = tls_connection:next_record(State0), + tls_connection:next_event(?FUNCTION_NAME, Record, State); +start(internal, #client_hello{} = Hello, State0, _Module) -> + case tls_handshake_1_3:do_start(Hello, State0) of #alert{} = Alert -> ssl_connection:handle_own_alert(Alert, {3,4}, start, State0); - M -> - %% update connection_states with cipher - State = update_state(State0, M), - {next_state, negotiated, State, [{next_event, internal, M}]} - - end. + {State, start} -> + {next_state, start, State, []}; + {State, negotiated} -> + {next_state, negotiated, State, [{next_event, internal, start_handshake}]} + end; +start(Type, Msg, State, Connection) -> + ssl_connection:handle_common_event(Type, Msg, ?FUNCTION_NAME, State, Connection). -negotiated(internal, Map, State0, _Module) -> - case tls_handshake_1_3:do_negotiated(Map, State0) of +negotiated(internal, #change_cipher_spec{}, State0, _Module) -> + {Record, State} = tls_connection:next_record(State0), + tls_connection:next_event(?FUNCTION_NAME, Record, State); +negotiated(internal, Message, State0, _Module) -> + case tls_handshake_1_3:do_negotiated(Message, State0) of #alert{} = Alert -> ssl_connection:handle_own_alert(Alert, {3,4}, negotiated, State0); - State -> - {next_state, wait_finished, State, []} - + {State, NextState} -> + {next_state, NextState, State, []} end. -wait_finished(internal, - #change_cipher_spec{} = ChangeCipherSpec, State0, _Module) -> - case tls_handshake_1_3:do_wait_finished(ChangeCipherSpec, State0) of - #alert{} = Alert -> - ssl_connection:handle_own_alert(Alert, {3,4}, wait_finished, State0); - State1 -> +wait_cert(internal, #change_cipher_spec{}, State0, _Module) -> + {Record, State} = tls_connection:next_record(State0), + tls_connection:next_event(?FUNCTION_NAME, Record, State); +wait_cert(internal, + #certificate_1_3{} = Certificate, State0, _Module) -> + case tls_handshake_1_3:do_wait_cert(Certificate, State0) of + {#alert{} = Alert, State} -> + ssl_connection:handle_own_alert(Alert, {3,4}, wait_cert, State); + {State1, NextState} -> + {Record, State} = tls_connection:next_record(State1), + tls_connection:next_event(NextState, Record, State) + end; +wait_cert(Type, Msg, State, Connection) -> + ssl_connection:handle_common_event(Type, Msg, ?FUNCTION_NAME, State, Connection). + + +wait_cv(internal, #change_cipher_spec{}, State0, _Module) -> + {Record, State} = tls_connection:next_record(State0), + tls_connection:next_event(?FUNCTION_NAME, Record, State); +wait_cv(internal, + #certificate_verify_1_3{} = CertificateVerify, State0, _Module) -> + case tls_handshake_1_3:do_wait_cv(CertificateVerify, State0) of + {#alert{} = Alert, State} -> + ssl_connection:handle_own_alert(Alert, {3,4}, wait_cv, State); + {State1, NextState} -> {Record, State} = tls_connection:next_record(State1), - tls_connection:next_event(?FUNCTION_NAME, Record, State) + tls_connection:next_event(NextState, Record, State) end; +wait_cv(Type, Msg, State, Connection) -> + ssl_connection:handle_common_event(Type, Msg, ?FUNCTION_NAME, State, Connection). + + +wait_finished(internal, #change_cipher_spec{}, State0, _Module) -> + {Record, State} = tls_connection:next_record(State0), + tls_connection:next_event(?FUNCTION_NAME, Record, State); wait_finished(internal, #finished{} = Finished, State0, Module) -> case tls_handshake_1_3:do_wait_finished(Finished, State0) of @@ -166,24 +190,3 @@ wait_finished(internal, end; wait_finished(Type, Msg, State, Connection) -> ssl_connection:handle_common_event(Type, Msg, ?FUNCTION_NAME, State, Connection). - - -update_state(#state{connection_states = ConnectionStates0, - connection_env = CEnv, - session = Session} = State, - #{cipher := Cipher, - key_share := KeyShare, - session_id := SessionId}) -> - #{security_parameters := SecParamsR0} = PendingRead = - maps:get(pending_read, ConnectionStates0), - #{security_parameters := SecParamsW0} = PendingWrite = - maps:get(pending_write, ConnectionStates0), - SecParamsR = ssl_cipher:security_parameters_1_3(SecParamsR0, Cipher), - SecParamsW = ssl_cipher:security_parameters_1_3(SecParamsW0, Cipher), - ConnectionStates = - ConnectionStates0#{pending_read => PendingRead#{security_parameters => SecParamsR}, - pending_write => PendingWrite#{security_parameters => SecParamsW}}, - State#state{connection_states = ConnectionStates, - key_share = KeyShare, - session = Session#session{session_id = SessionId}, - connection_env = CEnv#connection_env{negotiated_version = {3,4}}}. diff --git a/lib/ssl/src/tls_handshake_1_3.erl b/lib/ssl/src/tls_handshake_1_3.erl index 6a6de4b988..1e8b046c1e 100644 --- a/lib/ssl/src/tls_handshake_1_3.erl +++ b/lib/ssl/src/tls_handshake_1_3.erl @@ -36,38 +36,51 @@ %% Encode -export([encode_handshake/1, decode_handshake/2]). -%% Handshake --export([handle_client_hello/3]). - %% Create handshake messages -export([certificate/5, certificate_verify/4, encrypted_extensions/0, server_hello/4]). --export([do_negotiated/2, +-export([do_start/2, + do_negotiated/2, + do_wait_cert/2, + do_wait_cv/2, do_wait_finished/2]). %%==================================================================== %% Create handshake messages %%==================================================================== -server_hello(SessionId, KeyShare, ConnectionStates, _Map) -> +server_hello(MsgType, SessionId, KeyShare, ConnectionStates) -> #{security_parameters := SecParams} = ssl_record:pending_connection_state(ConnectionStates, read), - Extensions = server_hello_extensions(KeyShare), + Extensions = server_hello_extensions(MsgType, KeyShare), #server_hello{server_version = {3,3}, %% legacy_version cipher_suite = SecParams#security_parameters.cipher_suite, compression_method = 0, %% legacy attribute - random = SecParams#security_parameters.server_random, + random = server_hello_random(MsgType, SecParams), session_id = SessionId, extensions = Extensions }. -server_hello_extensions(KeyShare) -> +server_hello_extensions(MsgType, KeyShare) -> SupportedVersions = #server_hello_selected_version{selected_version = {3,4}}, Extensions = #{server_hello_selected_version => SupportedVersions}, - ssl_handshake:add_server_share(Extensions, KeyShare). + ssl_handshake:add_server_share(MsgType, Extensions, KeyShare). + +server_hello_random(server_hello, #security_parameters{server_random = Random}) -> + Random; +%% For reasons of backward compatibility with middleboxes (see +%% Appendix D.4), the HelloRetryRequest message uses the same structure +%% as the ServerHello, but with Random set to the special value of the +%% SHA-256 of "HelloRetryRequest": +%% +%% CF 21 AD 74 E5 9A 61 11 BE 1D 8C 02 1E 65 B8 91 +%% C2 A2 11 16 7A BB 8C 5E 07 9E 09 E2 C8 A8 33 9C +server_hello_random(hello_retry_request, _) -> + crypto:hash(sha256, "HelloRetryRequest"). + %% TODO: implement support for encrypted_extensions encrypted_extensions() -> @@ -75,6 +88,37 @@ encrypted_extensions() -> extensions = #{} }. + +certificate_request(SignAlgs0, SignAlgsCert0) -> + %% Input arguments contain TLS 1.2 algorithms due to backward compatibility + %% reasons. These {Hash, Algo} tuples must be filtered before creating the + %% the extensions. + SignAlgs = filter_tls13_algs(SignAlgs0), + SignAlgsCert = filter_tls13_algs(SignAlgsCert0), + Extensions0 = add_signature_algorithms(#{}, SignAlgs), + Extensions = add_signature_algorithms_cert(Extensions0, SignAlgsCert), + #certificate_request_1_3{ + certificate_request_context = <<>>, + extensions = Extensions}. + + +add_signature_algorithms(Extensions, SignAlgs) -> + Extensions#{signature_algorithms => + #signature_algorithms{signature_scheme_list = SignAlgs}}. + + +add_signature_algorithms_cert(Extensions, undefined) -> + Extensions; +add_signature_algorithms_cert(Extensions, SignAlgsCert) -> + Extensions#{signature_algorithms_cert => + #signature_algorithms{signature_scheme_list = SignAlgsCert}}. + + +filter_tls13_algs(undefined) -> undefined; +filter_tls13_algs(Algo) -> + lists:filter(fun is_atom/1, Algo). + + %% TODO: use maybe monad for error handling! %% enum { %% X509(0), @@ -132,7 +176,7 @@ certificate_verify(PrivateKey, SignatureScheme, %% Digital signatures use the hash function defined by the selected signature %% scheme. - case digitally_sign(THash, <<"TLS 1.3, server CertificateVerify">>, + case sign(THash, <<"TLS 1.3, server CertificateVerify">>, HashAlgo, PrivateKey) of {ok, Signature} -> {ok, #certificate_verify_1_3{ @@ -340,7 +384,7 @@ certificate_entry(DER) -> %% 79 %% 00 %% 0101010101010101010101010101010101010101010101010101010101010101 -digitally_sign(THash, Context, HashAlgo, PrivateKey) -> +sign(THash, Context, HashAlgo, PrivateKey) -> Content = build_content(Context, THash), %% The length of the Salt MUST be equal to the length of the output @@ -357,24 +401,41 @@ digitally_sign(THash, Context, HashAlgo, PrivateKey) -> end. +verify(THash, Context, HashAlgo, Signature, PublicKey) -> + Content = build_content(Context, THash), + + %% The length of the Salt MUST be equal to the length of the output + %% of the digest algorithm: rsa_pss_saltlen = -1 + try public_key:verify(Content, HashAlgo, Signature, PublicKey, + [{rsa_padding, rsa_pkcs1_pss_padding}, + {rsa_pss_saltlen, -1}, + {rsa_mgf1_md, HashAlgo}]) of + Result -> + {ok, Result} + catch + error:badarg -> + {error, badarg} + end. + + build_content(Context, THash) -> Prefix = binary:copy(<<32>>, 64), <<Prefix/binary,Context/binary,?BYTE(0),THash/binary>>. + %%==================================================================== %% Handle handshake messages %%==================================================================== -handle_client_hello(#client_hello{cipher_suites = ClientCiphers, - session_id = SessionId, - extensions = Extensions} = _Hello, - #ssl_options{ciphers = ServerCiphers, - signature_algs = ServerSignAlgs, - signature_algs_cert = _SignatureSchemes, %% TODO: Check?? - supported_groups = ServerGroups0} = _SslOpts, - Env) -> - Cert = maps:get(cert, Env, undefined), +do_start(#client_hello{cipher_suites = ClientCiphers, + session_id = SessionId, + extensions = Extensions} = _Hello, + #state{connection_states = _ConnectionStates0, + ssl_options = #ssl_options{ciphers = ServerCiphers, + signature_algs = ServerSignAlgs, + supported_groups = ServerGroups0}, + session = #session{own_certificate = Cert}} = State0) -> ClientGroups0 = maps:get(elliptic_curves, Extensions, undefined), ClientGroups = get_supported_groups(ClientGroups0), @@ -398,11 +459,9 @@ handle_client_hello(#client_hello{cipher_suites = ClientCiphers, %% and a signature algorithm/certificate pair to authenticate itself to %% the client. Cipher = Maybe(select_cipher_suite(ClientCiphers, ServerCiphers)), - Group = Maybe(select_server_group(ServerGroups, ClientGroups)), + Groups = Maybe(select_common_groups(ServerGroups, ClientGroups)), Maybe(validate_key_share(ClientGroups, ClientShares)), - ClientPubKey = Maybe(get_client_public_key(Group, ClientShares)), - {PublicKeyAlgo, SignAlgo, SignHash} = get_certificate_params(Cert), %% Check if client supports signature algorithm of server certificate @@ -411,14 +470,25 @@ handle_client_hello(#client_hello{cipher_suites = ClientCiphers, %% Select signature algorithm (used in CertificateVerify message). SelectedSignAlg = Maybe(select_sign_algo(PublicKeyAlgo, ClientSignAlgs, ServerSignAlgs)), + %% Select client public key. If no public key found in ClientShares or + %% ClientShares is empty, trigger HelloRetryRequest as we were able + %% to find an acceptable set of parameters but the ClientHello does not + %% contain sufficient information. + {Group, ClientPubKey} = get_client_public_key(Groups, ClientShares), + %% Generate server_share KeyShare = ssl_cipher:generate_server_share(Group), - _Ret = #{cipher => Cipher, - group => Group, - sign_alg => SelectedSignAlg, - client_share => ClientPubKey, - key_share => KeyShare, - session_id => SessionId} + + State1 = update_start_state(State0, Cipher, KeyShare, SessionId, + Group, SelectedSignAlg, ClientPubKey), + + %% 4.1.4. Hello Retry Request + %% + %% The server will send this message in response to a ClientHello + %% message if it is able to find an acceptable set of parameters but the + %% ClientHello does not contain sufficient information to proceed with + %% the handshake. + Maybe(send_hello_retry_request(State1, ClientPubKey, KeyShare, SessionId)) %% TODO: %% - session handling @@ -430,41 +500,38 @@ handle_client_hello(#client_hello{cipher_suites = ClientCiphers, ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_groups); {Ref, illegal_parameter} -> ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER); - {Ref, {hello_retry_request, _Group0}} -> - %% TODO - ?ALERT_REC(?FATAL, ?INTERNAL_ERROR, "hello_retry_request not implemented"); {Ref, no_suitable_cipher} -> ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_cipher); {Ref, {insufficient_security, no_suitable_signature_algorithm}} -> - ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_signature_algorithm); + ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, "No suitable signature algorithm"); {Ref, {insufficient_security, no_suitable_public_key}} -> ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_public_key) end. -do_negotiated(#{client_share := ClientKey, - group := SelectedGroup, - sign_alg := SignatureScheme - } = Map, - #state{connection_states = ConnectionStates0, - session = #session{session_id = SessionId, - own_certificate = OwnCert}, - ssl_options = #ssl_options{} = _SslOpts, - key_share = KeyShare, - handshake_env = #handshake_env{tls_handshake_history = _HHistory0}, - connection_env = #connection_env{private_key = CertPrivateKey}, - static_env = #static_env{ - cert_db = CertDbHandle, - cert_db_ref = CertDbRef, - socket = _Socket, - transport_cb = _Transport} - } = State0) -> +do_negotiated(start_handshake, + #state{connection_states = ConnectionStates0, + session = #session{session_id = SessionId, + own_certificate = OwnCert, + ecc = SelectedGroup, + sign_alg = SignatureScheme, + dh_public_value = ClientKey}, + ssl_options = #ssl_options{} = SslOpts, + key_share = KeyShare, + handshake_env = #handshake_env{tls_handshake_history = _HHistory0}, + connection_env = #connection_env{private_key = CertPrivateKey}, + static_env = #static_env{ + cert_db = CertDbHandle, + cert_db_ref = CertDbRef, + socket = _Socket, + transport_cb = _Transport} + } = State0) -> {Ref,Maybe} = maybe(), try %% Create server_hello %% Extensions: supported_versions, key_share, (pre_shared_key) - ServerHello = server_hello(SessionId, KeyShare, ConnectionStates0, Map), + ServerHello = server_hello(server_hello, SessionId, KeyShare, ConnectionStates0), {State1, _} = tls_connection:send_handshake(ServerHello, State0), @@ -479,59 +546,71 @@ do_negotiated(#{client_share := ClientKey, %% Encode EncryptedExtensions State4 = tls_connection:queue_handshake(EncryptedExtensions, State3), + %% Create and send CertificateRequest ({verify, verify_peer}) + {State5, NextState} = maybe_send_certificate_request(State4, SslOpts), + %% Create Certificate Certificate = certificate(OwnCert, CertDbHandle, CertDbRef, <<>>, server), %% Encode Certificate - State5 = tls_connection:queue_handshake(Certificate, State4), + State6 = tls_connection:queue_handshake(Certificate, State5), %% Create CertificateVerify CertificateVerify = Maybe(certificate_verify(CertPrivateKey, SignatureScheme, - State5, server)), + State6, server)), %% Encode CertificateVerify - State6 = tls_connection:queue_handshake(CertificateVerify, State5), + State7 = tls_connection:queue_handshake(CertificateVerify, State6), %% Create Finished - Finished = finished(State6), + Finished = finished(State7), %% Encode Finished - State7 = tls_connection:queue_handshake(Finished, State6), + State8 = tls_connection:queue_handshake(Finished, State7), %% Send first flight - {State8, _} = tls_connection:send_handshake_flight(State7), + {State9, _} = tls_connection:send_handshake_flight(State8), - State8 + {State9, NextState} catch - {Ref, {state_not_implemented, State}} -> - %% TODO - ?ALERT_REC(?FATAL, ?INTERNAL_ERROR, {state_not_implemented, State}) + {Ref, badarg} -> + ?ALERT_REC(?FATAL, ?INTERNAL_ERROR, {digitally_sign, badarg}) end. -do_wait_finished(#change_cipher_spec{}, - #state{connection_states = _ConnectionStates0, - session = #session{session_id = _SessionId, - own_certificate = _OwnCert}, - ssl_options = #ssl_options{} = _SslOpts, - key_share = _KeyShare, - handshake_env = #handshake_env{tls_handshake_history = _HHistory0}, - static_env = #static_env{ - cert_db = _CertDbHandle, - cert_db_ref = _CertDbRef, - socket = _Socket, - transport_cb = _Transport} - } = State0) -> - %% {Ref,Maybe} = maybe(), - +do_wait_cert(#certificate_1_3{} = Certificate, State0) -> + {Ref,Maybe} = maybe(), try + Maybe(process_client_certificate(Certificate, State0)) + catch + {Ref, {certificate_required, State}} -> + {?ALERT_REC(?FATAL, ?CERTIFICATE_REQUIRED, certificate_required), State}; + {Ref, {{certificate_unknown, Reason}, State}} -> + {?ALERT_REC(?FATAL, ?CERTIFICATE_UNKNOWN, Reason), State}; + {Ref, {{internal_error, Reason}, State}} -> + {?ALERT_REC(?FATAL, ?INTERNAL_ERROR, Reason), State}; + {Ref, {{handshake_failure, Reason}, State}} -> + {?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, Reason), State}; + {#alert{} = Alert, State} -> + {Alert, State} + end. - State0 +do_wait_cv(#certificate_verify_1_3{} = CertificateVerify, State0) -> + {Ref,Maybe} = maybe(), + try + Maybe(verify_signature_algorithm(State0, CertificateVerify)), + Maybe(verify_certificate_verify(State0, CertificateVerify)) catch - {_Ref, {state_not_implemented, State}} -> - ?ALERT_REC(?FATAL, ?INTERNAL_ERROR, {state_not_implemented, State}) - end; + {Ref, {{bad_certificate, Reason}, State}} -> + {?ALERT_REC(?FATAL, ?BAD_CERTIFICATE, {bad_certificate, Reason}), State}; + {Ref, {badarg, State}} -> + {?ALERT_REC(?FATAL, ?INTERNAL_ERROR, {verify, badarg}), State}; + {Ref, {{handshake_failure, Reason}, State}} -> + {?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, {handshake_failure, Reason}), State} + end. + + do_wait_finished(#finished{verify_data = VerifyData}, #state{connection_states = _ConnectionStates0, session = #session{session_id = _SessionId, @@ -559,16 +638,19 @@ do_wait_finished(#finished{verify_data = VerifyData}, catch {Ref, decrypt_error} -> - ?ALERT_REC(?FATAL, ?DECRYPT_ERROR, decrypt_error); - {_, {state_not_implemented, State}} -> - %% TODO - ?ALERT_REC(?FATAL, ?INTERNAL_ERROR, {state_not_implemented, State}) + ?ALERT_REC(?FATAL, ?DECRYPT_ERROR, decrypt_error) end. %% TODO: Remove this function! -%% not_implemented(State) -> -%% {error, {state_not_implemented, State}}. +%% not_implemented(State, Reason) -> +%% {error, {not_implemented, State, Reason}}. +%% +%% not_implemented(update_secrets, State0, Reason) -> +%% State1 = calculate_traffic_secrets(State0), +%% State = ssl_record:step_encryption_state(State1), +%% {error, {not_implemented, State, Reason}}. + %% Recipients of Finished messages MUST verify that the contents are @@ -597,6 +679,188 @@ compare_verify_data(_, _) -> {error, decrypt_error}. +send_hello_retry_request(#state{connection_states = ConnectionStates0} = State0, + no_suitable_key, KeyShare, SessionId) -> + ServerHello = server_hello(hello_retry_request, SessionId, KeyShare, ConnectionStates0), + {State1, _} = tls_connection:send_handshake(ServerHello, State0), + + %% TODO: Fix handshake history! + State2 = replace_ch1_with_message_hash(State1), + + {ok, {State2, start}}; +send_hello_retry_request(State0, _, _, _) -> + %% Suitable key found. + {ok, {State0, negotiated}}. + + +maybe_send_certificate_request(State, #ssl_options{verify = verify_none}) -> + {State, wait_finished}; +maybe_send_certificate_request(State, #ssl_options{ + verify = verify_peer, + signature_algs = SignAlgs, + signature_algs_cert = SignAlgsCert}) -> + CertificateRequest = certificate_request(SignAlgs, SignAlgsCert), + {tls_connection:queue_handshake(CertificateRequest, State), wait_cert}. + + +process_client_certificate(#certificate_1_3{ + certificate_request_context = <<>>, + certificate_list = []}, + #state{ssl_options = + #ssl_options{ + fail_if_no_peer_cert = false}} = State) -> + {ok, {State, wait_finished}}; +process_client_certificate(#certificate_1_3{ + certificate_request_context = <<>>, + certificate_list = []}, + #state{ssl_options = + #ssl_options{ + fail_if_no_peer_cert = true}} = State0) -> + + %% At this point the client believes that the connection is up and starts using + %% its traffic secrets. In order to be able send an proper Alert to the client + %% the server should also change its connection state and use the traffic + %% secrets. + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {error, {certificate_required, State}}; +process_client_certificate(#certificate_1_3{certificate_list = Certs0}, + #state{ssl_options = + #ssl_options{signature_algs = SignAlgs, + signature_algs_cert = SignAlgsCert} = SslOptions, + static_env = + #static_env{ + role = Role, + host = Host, + cert_db = CertDbHandle, + cert_db_ref = CertDbRef, + crl_db = CRLDbHandle}} = State0) -> + %% TODO: handle extensions! + + %% Remove extensions from list of certificates! + Certs = convert_certificate_chain(Certs0), + case is_supported_signature_algorithm(Certs, SignAlgs, SignAlgsCert) of + true -> + case validate_certificate_chain(Certs, CertDbHandle, CertDbRef, + SslOptions, CRLDbHandle, Role, Host) of + {ok, {PeerCert, PublicKeyInfo}} -> + State = store_peer_cert(State0, PeerCert, PublicKeyInfo), + {ok, {State, wait_cv}}; + {error, Reason} -> + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {error, {Reason, State}}; + #alert{} = Alert -> + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {Alert, State} + end; + false -> + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {error, {{handshake_failure, + "Client certificate uses unsupported signature algorithm"}, State}} + end. + + +%% TODO: check whole chain! +is_supported_signature_algorithm(Certs, SignAlgs, undefined) -> + is_supported_signature_algorithm(Certs, SignAlgs); +is_supported_signature_algorithm(Certs, _, SignAlgsCert) -> + is_supported_signature_algorithm(Certs, SignAlgsCert). +%% +is_supported_signature_algorithm([BinCert|_], SignAlgs0) -> + #'OTPCertificate'{signatureAlgorithm = SignAlg} = + public_key:pkix_decode_cert(BinCert, otp), + SignAlgs = filter_tls13_algs(SignAlgs0), + Scheme = ssl_cipher:signature_algorithm_to_scheme(SignAlg), + lists:member(Scheme, SignAlgs). + + +validate_certificate_chain(Certs, CertDbHandle, CertDbRef, SslOptions, CRLDbHandle, Role, Host) -> + ServerName = ssl_handshake:server_name(SslOptions#ssl_options.server_name_indication, Host, Role), + [PeerCert | ChainCerts ] = Certs, + try + {TrustedCert, CertPath} = + ssl_certificate:trusted_cert_and_path(Certs, CertDbHandle, CertDbRef, + SslOptions#ssl_options.partial_chain), + ValidationFunAndState = + ssl_handshake:validation_fun_and_state(SslOptions#ssl_options.verify_fun, Role, + CertDbHandle, CertDbRef, ServerName, + SslOptions#ssl_options.customize_hostname_check, + SslOptions#ssl_options.crl_check, CRLDbHandle, CertPath), + Options = [{max_path_length, SslOptions#ssl_options.depth}, + {verify_fun, ValidationFunAndState}], + %% TODO: Validate if Certificate is using a supported signature algorithm + %% (signature_algs_cert)! + case public_key:pkix_path_validation(TrustedCert, CertPath, Options) of + {ok, {PublicKeyInfo,_}} -> + {ok, {PeerCert, PublicKeyInfo}}; + {error, Reason} -> + ssl_handshake:handle_path_validation_error(Reason, PeerCert, ChainCerts, + SslOptions, Options, + CertDbHandle, CertDbRef) + end + catch + error:{badmatch,{asn1, Asn1Reason}} -> + %% ASN-1 decode of certificate somehow failed + {error, {certificate_unknown, {failed_to_decode_certificate, Asn1Reason}}}; + error:OtherReason -> + {error, {internal_error, {unexpected_error, OtherReason}}} + end. + + +store_peer_cert(#state{session = Session, + handshake_env = HsEnv} = State, PeerCert, PublicKeyInfo) -> + State#state{session = Session#session{peer_certificate = PeerCert}, + handshake_env = HsEnv#handshake_env{public_key_info = PublicKeyInfo}}. + + +convert_certificate_chain(Certs) -> + Fun = fun(#certificate_entry{data = Data}) -> + {true, Data}; + (_) -> + false + end, + lists:filtermap(Fun, Certs). + + +%% 4.4.1. The Transcript Hash +%% +%% As an exception to this general rule, when the server responds to a +%% ClientHello with a HelloRetryRequest, the value of ClientHello1 is +%% replaced with a special synthetic handshake message of handshake type +%% "message_hash" containing Hash(ClientHello1). I.e., +%% +%% Transcript-Hash(ClientHello1, HelloRetryRequest, ... Mn) = +%% Hash(message_hash || /* Handshake type */ +%% 00 00 Hash.length || /* Handshake message length (bytes) */ +%% Hash(ClientHello1) || /* Hash of ClientHello1 */ +%% HelloRetryRequest || ... || Mn) +%% +%% NOTE: Hash.length is used in practice (openssl) and not message length! +%% It is most probably a fault in the RFC. +replace_ch1_with_message_hash(#state{connection_states = ConnectionStates, + handshake_env = + #handshake_env{ + tls_handshake_history = + {[HRR,CH1|HHistory], LM}} = HSEnv} = State0) -> + #{security_parameters := SecParamsR} = + ssl_record:pending_connection_state(ConnectionStates, read), + #security_parameters{prf_algorithm = HKDFAlgo} = SecParamsR, + MessageHash = message_hash(CH1, HKDFAlgo), + State0#state{handshake_env = + HSEnv#handshake_env{ + tls_handshake_history = + {[HRR,MessageHash|HHistory], LM}}}. + + +message_hash(ClientHello1, HKDFAlgo) -> + [?MESSAGE_HASH, + 0,0,ssl_cipher:hash_size(HKDFAlgo), + crypto:hash(HKDFAlgo, ClientHello1)]. + + calculate_handshake_secrets(ClientKey, SelectedGroup, KeyShare, #state{connection_states = ConnectionStates, handshake_env = @@ -648,10 +912,8 @@ calculate_traffic_secrets(#state{connection_states = ConnectionStates, MasterSecret = tls_v1:key_schedule(master_secret, HKDFAlgo, HandshakeSecret), - {Messages0, _} = HHistory, - - %% Drop Client Finish - [_|Messages] = Messages0, + %% Get the correct list messages for the handshake context. + Messages = get_handshake_context(HHistory), %% Calculate [sender]_application_traffic_secret_0 ClientAppTrafficSecret0 = @@ -680,6 +942,11 @@ get_private_key(#key_share_entry{ {_, PrivateKey}}) -> PrivateKey. +%% TODO: implement EC keys +get_public_key({?'rsaEncryption', PublicKey, _}) -> + PublicKey. + + %% X25519, X448 calculate_shared_secret(OthersKey, MyKey, Group) when is_binary(OthersKey) andalso is_binary(MyKey) andalso @@ -721,6 +988,28 @@ update_connection_state(ConnectionState = #{security_parameters := SecurityParam cipher_state => cipher_init(Key, IV, FinishedKey)}. +update_start_state(#state{connection_states = ConnectionStates0, + connection_env = CEnv, + session = Session} = State, + Cipher, KeyShare, SessionId, + Group, SelectedSignAlg, ClientPubKey) -> + #{security_parameters := SecParamsR0} = PendingRead = + maps:get(pending_read, ConnectionStates0), + #{security_parameters := SecParamsW0} = PendingWrite = + maps:get(pending_write, ConnectionStates0), + SecParamsR = ssl_cipher:security_parameters_1_3(SecParamsR0, Cipher), + SecParamsW = ssl_cipher:security_parameters_1_3(SecParamsW0, Cipher), + ConnectionStates = + ConnectionStates0#{pending_read => PendingRead#{security_parameters => SecParamsR}, + pending_write => PendingWrite#{security_parameters => SecParamsW}}, + State#state{connection_states = ConnectionStates, + key_share = KeyShare, + session = Session#session{session_id = SessionId, + ecc = Group, + sign_alg = SelectedSignAlg, + dh_public_value = ClientPubKey}, + connection_env = CEnv#connection_env{negotiated_version = {3,4}}}. + cipher_init(Key, IV, FinishedKey) -> #cipher_state{key = Key, @@ -729,18 +1018,139 @@ cipher_init(Key, IV, FinishedKey) -> tag_len = 16}. +%% Get handshake context for verification of CertificateVerify. +%% +%% Verify CertificateVerify: +%% ClientHello (client) (1) +%% ServerHello (server) (2) +%% EncryptedExtensions (server) (8) +%% CertificateRequest (server) (13) +%% Certificate (server) (11) +%% CertificateVerify (server) (15) +%% Finished (server) (20) +%% Certificate (client) (11) +%% CertificateVerify (client) (15) - Drop! Not included in calculations! +get_handshake_context_cv({[<<15,_/binary>>|Messages], _}) -> + Messages. + + +%% Get handshake context for traffic key calculation. +%% +%% Client is authenticated with certificate: +%% ClientHello (client) (1) +%% ServerHello (server) (2) +%% EncryptedExtensions (server) (8) +%% CertificateRequest (server) (13) +%% Certificate (server) (11) +%% CertificateVerify (server) (15) +%% Finished (server) (20) +%% Certificate (client) (11) - Drop! Not included in calculations! +%% CertificateVerify (client) (15) - Drop! Not included in calculations! +%% Finished (client) (20) - Drop! Not included in calculations! +%% +%% Client is authenticated but sends empty certificate: +%% ClientHello (client) (1) +%% ServerHello (server) (2) +%% EncryptedExtensions (server) (8) +%% CertificateRequest (server) (13) +%% Certificate (server) (11) +%% CertificateVerify (server) (15) +%% Finished (server) (20) +%% Certificate (client) (11) - Drop! Not included in calculations! +%% Finished (client) (20) - Drop! Not included in calculations! +%% +%% Client is not authenticated: +%% ClientHello (client) (1) +%% ServerHello (server) (2) +%% EncryptedExtensions (server) (8) +%% Certificate (server) (11) +%% CertificateVerify (server) (15) +%% Finished (server) (20) +%% Finished (client) (20) - Drop! Not included in calculations! +%% +%% Drop all client messages from the front of the iolist using the property that +%% incoming messages are binaries. +get_handshake_context({Messages, _}) -> + get_handshake_context(Messages); +get_handshake_context([H|T]) when is_binary(H) -> + get_handshake_context(T); +get_handshake_context(L) -> + L. + + +%% If sent by a client, the signature algorithm used in the signature +%% MUST be one of those present in the supported_signature_algorithms +%% field of the "signature_algorithms" extension in the +%% CertificateRequest message. +verify_signature_algorithm(#state{ssl_options = + #ssl_options{ + signature_algs = ServerSignAlgs}} = State0, + #certificate_verify_1_3{algorithm = ClientSignAlg}) -> + case lists:member(ClientSignAlg, ServerSignAlgs) of + true -> + ok; + false -> + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {error, {{handshake_failure, + "CertificateVerify uses unsupported signature algorithm"}, State}} + end. + + +verify_certificate_verify(#state{connection_states = ConnectionStates, + handshake_env = + #handshake_env{ + public_key_info = PublicKeyInfo, + tls_handshake_history = HHistory}} = State0, + #certificate_verify_1_3{ + algorithm = SignatureScheme, + signature = Signature}) -> + #{security_parameters := SecParamsR} = + ssl_record:pending_connection_state(ConnectionStates, write), + #security_parameters{prf_algorithm = HKDFAlgo} = SecParamsR, + + {HashAlgo, _, _} = + ssl_cipher:scheme_to_components(SignatureScheme), + + Messages = get_handshake_context_cv(HHistory), + + Context = lists:reverse(Messages), + + %% Transcript-Hash uses the HKDF hash function defined by the cipher suite. + THash = tls_v1:transcript_hash(Context, HKDFAlgo), + + PublicKey = get_public_key(PublicKeyInfo), + + %% Digital signatures use the hash function defined by the selected signature + %% scheme. + case verify(THash, <<"TLS 1.3, client CertificateVerify">>, + HashAlgo, Signature, PublicKey) of + {ok, true} -> + {ok, {State0, wait_finished}}; + {ok, false} -> + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {error, {{handshake_failure, "Failed to verify CertificateVerify"}, State}}; + {error, badarg} -> + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {error, {badarg, State}} + end. + + %% If there is no overlap between the received %% "supported_groups" and the groups supported by the server, then the %% server MUST abort the handshake with a "handshake_failure" or an %% "insufficient_security" alert. -select_server_group(_, []) -> +select_common_groups(_, []) -> {error, {insufficient_security, no_suitable_groups}}; -select_server_group(ServerGroups, [C|ClientGroups]) -> - case lists:member(C, ServerGroups) of - true -> - {ok, C}; - false -> - select_server_group(ServerGroups, ClientGroups) +select_common_groups(ServerGroups, ClientGroups) -> + Fun = fun(E) -> lists:member(E, ClientGroups) end, + case lists:filter(Fun, ServerGroups) of + [] -> + {error, {insufficient_security, no_suitable_groups}}; + L -> + {ok, L} end. @@ -771,20 +1181,36 @@ validate_key_share([_|ClientGroups], [_|_] = ClientShares) -> validate_key_share(ClientGroups, ClientShares). -get_client_public_key(Group, ClientShares) -> +get_client_public_key([Group|_] = Groups, ClientShares) -> + get_client_public_key(Groups, ClientShares, Group). +%% +get_client_public_key(_, [], PreferredGroup) -> + {PreferredGroup, no_suitable_key}; +get_client_public_key([], _, PreferredGroup) -> + {PreferredGroup, no_suitable_key}; +get_client_public_key([Group|Groups], ClientShares, PreferredGroup) -> case lists:keysearch(Group, 2, ClientShares) of {value, {_, _, ClientPublicKey}} -> - {ok, ClientPublicKey}; + {Group, ClientPublicKey}; false -> - %% 4.1.4. Hello Retry Request - %% - %% The server will send this message in response to a ClientHello - %% message if it is able to find an acceptable set of parameters but the - %% ClientHello does not contain sufficient information to proceed with - %% the handshake. - {error, {hello_retry_request, Group}} + get_client_public_key(Groups, ClientShares, PreferredGroup) end. + +%% get_client_public_key(Group, ClientShares) -> +%% case lists:keysearch(Group, 2, ClientShares) of +%% {value, {_, _, ClientPublicKey}} -> +%% ClientPublicKey; +%% false -> +%% %% 4.1.4. Hello Retry Request +%% %% +%% %% The server will send this message in response to a ClientHello +%% %% message if it is able to find an acceptable set of parameters but the +%% %% ClientHello does not contain sufficient information to proceed with +%% %% the handshake. +%% no_suitable_key +%% end. + select_cipher_suite([], _) -> {error, no_suitable_cipher}; select_cipher_suite([Cipher|ClientCiphers], ServerCiphers) -> diff --git a/lib/ssl/src/tls_record_1_3.erl b/lib/ssl/src/tls_record_1_3.erl index 05acc08392..97331e1510 100644 --- a/lib/ssl/src/tls_record_1_3.erl +++ b/lib/ssl/src/tls_record_1_3.erl @@ -124,6 +124,20 @@ decode_cipher_text(#ssl_tls{type = ?OPAQUE_TYPE, {decode_inner_plaintext(PlainFragment), ConnectionStates} end; + +%% RFC8446 - TLS 1.3 (OpenSSL compatibility) +%% Handle unencrypted Alerts from openssl s_client when server's +%% connection states are already stepped into traffic encryption. +%% (E.g. openssl s_client receives a CertificateRequest with +%% a signature_algorithms_cert extension that does not contain +%% the signature algorithm of the client's certificate.) +decode_cipher_text(#ssl_tls{type = ?ALERT, + version = ?LEGACY_VERSION, + fragment = <<2,47>>}, + ConnectionStates0) -> + {#ssl_tls{type = ?ALERT, + version = {3,4}, %% Internally use real version + fragment = <<2,47>>}, ConnectionStates0}; %% RFC8446 - TLS 1.3 %% D.4. Middlebox Compatibility Mode %% - If not offering early data, the client sends a dummy @@ -139,7 +153,6 @@ decode_cipher_text(#ssl_tls{type = ?CHANGE_CIPHER_SPEC, {#ssl_tls{type = ?CHANGE_CIPHER_SPEC, version = {3,4}, %% Internally use real version fragment = <<1>>}, ConnectionStates0}; - decode_cipher_text(#ssl_tls{type = Type, version = ?LEGACY_VERSION, fragment = CipherFragment}, diff --git a/lib/ssl/src/tls_socket.erl b/lib/ssl/src/tls_socket.erl index c3c41d3e12..6c32e6fa04 100644 --- a/lib/ssl/src/tls_socket.erl +++ b/lib/ssl/src/tls_socket.erl @@ -46,7 +46,7 @@ send(Transport, Socket, Data) -> Transport:send(Socket, Data). -listen(Transport, Port, #config{transport_info = {Transport, _, _, _}, +listen(Transport, Port, #config{transport_info = {Transport, _, _, _, _}, inet_user = Options, ssl = SslOpts, emulated = EmOpts} = Config) -> case Transport:listen(Port, Options ++ internal_inet_values()) of @@ -59,7 +59,7 @@ listen(Transport, Port, #config{transport_info = {Transport, _, _, _}, Err end. -accept(ListenSocket, #config{transport_info = {Transport,_,_,_} = CbInfo, +accept(ListenSocket, #config{transport_info = {Transport,_,_,_,_} = CbInfo, connection_cb = ConnectionCb, ssl = SslOpts, emulated = Tracker}, Timeout) -> @@ -80,7 +80,7 @@ accept(ListenSocket, #config{transport_info = {Transport,_,_,_} = CbInfo, {error, Reason} end. -upgrade(Socket, #config{transport_info = {Transport,_,_,_}= CbInfo, +upgrade(Socket, #config{transport_info = {Transport,_,_,_,_}= CbInfo, ssl = SslOptions, emulated = EmOpts, connection_cb = ConnectionCb}, Timeout) -> ok = setopts(Transport, Socket, tls_socket:internal_inet_values()), @@ -98,7 +98,7 @@ connect(Address, Port, #config{transport_info = CbInfo, inet_user = UserOpts, ssl = SslOpts, emulated = EmOpts, inet_ssl = SocketOpts, connection_cb = ConnetionCb}, Timeout) -> - {Transport, _, _, _} = CbInfo, + {Transport, _, _, _, _} = CbInfo, try Transport:connect(Address, Port, SocketOpts, Timeout) of {ok, Socket} -> ssl_connection:connect(ConnetionCb, Address, Port, Socket, @@ -125,7 +125,7 @@ setopts(gen_tcp, Socket = #sslsocket{pid = {ListenSocket, #config{emulated = Tra ok = set_emulated_opts(Tracker, EmulatedOpts), check_active_n(EmulatedOpts, Socket), inet:setopts(ListenSocket, SockOpts); -setopts(_, Socket = #sslsocket{pid = {ListenSocket, #config{transport_info = {Transport,_,_,_}, +setopts(_, Socket = #sslsocket{pid = {ListenSocket, #config{transport_info = {Transport,_,_,_,_}, emulated = Tracker}}}, Options) -> {SockOpts, EmulatedOpts} = split_options(Options), ok = set_emulated_opts(Tracker, EmulatedOpts), diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index 0873a4fbcd..ff5638ff34 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -279,7 +279,17 @@ tls13_test_group() -> tls_record_1_3_encode_decode, tls13_finished_verify_data, tls13_1_RTT_handshake, - tls13_basic_ssl_s_client]. + tls13_basic_ssl_server_openssl_client, + tls13_custom_groups_ssl_server_openssl_client, + tls13_hello_retry_request_ssl_server_openssl_client, + tls13_client_auth_empty_cert_alert_ssl_server_openssl_client, + tls13_client_auth_empty_cert_ssl_server_openssl_client, + tls13_client_auth_ssl_server_openssl_client, + tls13_hrr_client_auth_empty_cert_alert_ssl_server_openssl_client, + tls13_hrr_client_auth_empty_cert_ssl_server_openssl_client, + tls13_hrr_client_auth_ssl_server_openssl_client, + tls13_unsupported_sign_algo_client_auth_ssl_server_openssl_client, + tls13_unsupported_sign_algo_cert_client_auth_ssl_server_openssl_client]. %%-------------------------------------------------------------------- init_per_suite(Config0) -> @@ -4076,7 +4086,7 @@ tls_tcp_error_propagation_in_active_mode(Config) when is_list(Config) -> [_, _,_, _, Prop] = StatusInfo, State = ssl_test_lib:state(Prop), StaticEnv = element(2, State), - Socket = element(10, StaticEnv), + Socket = element(11, StaticEnv), %% Fake tcp error Pid ! {tcp_error, Socket, etimedout}, @@ -5515,10 +5525,10 @@ tls13_finished_verify_data(_Config) -> FinishedKey = tls_v1:finished_key(BaseKey, sha256), VerifyData = tls_v1:finished_verify_data(FinishedKey, sha256, Messages). -tls13_basic_ssl_s_client() -> +tls13_basic_ssl_server_openssl_client() -> [{doc,"Test TLS 1.3 basic connection between ssl server and openssl s_client"}]. -tls13_basic_ssl_s_client(Config) -> +tls13_basic_ssl_server_openssl_client(Config) -> ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), %% Set versions @@ -5537,6 +5547,307 @@ tls13_basic_ssl_s_client(Config) -> ssl_test_lib:close(Server), ssl_test_lib:close_port(Client). +tls13_custom_groups_ssl_server_openssl_client() -> + [{doc,"Test that ssl server can select a common group for key-exchange"}]. + +tls13_custom_groups_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {supported_groups, [x448, secp256r1, secp384r1]}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + ClientOpts = [{groups,"P-384:P-256:X25519"}|ClientOpts0], + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + +tls13_hello_retry_request_ssl_server_openssl_client() -> + [{doc,"Test that ssl server can request a new group when the client's first key share" + "is not supported"}]. + +tls13_hello_retry_request_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {supported_groups, [x448, x25519]}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + ClientOpts = [{groups,"P-256:X25519"}|ClientOpts0], + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + +tls13_client_auth_empty_cert_alert_ssl_server_openssl_client() -> + [{doc,"TLS 1.3: Test client authentication when client sends an empty certificate and fail_if_no_peer_cert is set to true."}]. + +tls13_client_auth_empty_cert_alert_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + %% Delete Client Cert and Key + ClientOpts1 = proplists:delete(certfile, ClientOpts0), + ClientOpts = proplists:delete(keyfile, ClientOpts1), + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + {fail_if_no_peer_cert, true}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, + {error, + {tls_alert, + {certificate_required, + "received SERVER ALERT: Fatal - Certificate required - certificate_required"}}}), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + +tls13_client_auth_empty_cert_ssl_server_openssl_client() -> + [{doc,"TLS 1.3: Test client authentication when client sends an empty certificate and fail_if_no_peer_cert is set to false."}]. + +tls13_client_auth_empty_cert_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + %% Delete Client Cert and Key + ClientOpts1 = proplists:delete(certfile, ClientOpts0), + ClientOpts = proplists:delete(keyfile, ClientOpts1), + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + {fail_if_no_peer_cert, false}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +tls13_client_auth_ssl_server_openssl_client() -> + [{doc,"TLS 1.3: Test client authentication."}]. + +tls13_client_auth_ssl_server_openssl_client(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + {fail_if_no_peer_cert, true}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +tls13_hrr_client_auth_empty_cert_alert_ssl_server_openssl_client() -> + [{doc,"TLS 1.3 (HelloRetryRequest): Test client authentication when client sends an empty certificate and fail_if_no_peer_cert is set to true."}]. + +tls13_hrr_client_auth_empty_cert_alert_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + %% Delete Client Cert and Key + ClientOpts1 = proplists:delete(certfile, ClientOpts0), + ClientOpts2 = proplists:delete(keyfile, ClientOpts1), + ClientOpts = [{groups,"P-256:X25519"}|ClientOpts2], + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + {fail_if_no_peer_cert, true}, + {supported_groups, [x448, x25519]}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, + {error, + {tls_alert, + {certificate_required, + "received SERVER ALERT: Fatal - Certificate required - certificate_required"}}}), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +tls13_hrr_client_auth_empty_cert_ssl_server_openssl_client() -> + [{doc,"TLS 1.3 (HelloRetryRequest): Test client authentication when client sends an empty certificate and fail_if_no_peer_cert is set to false."}]. + +tls13_hrr_client_auth_empty_cert_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + %% Delete Client Cert and Key + ClientOpts1 = proplists:delete(certfile, ClientOpts0), + ClientOpts2 = proplists:delete(keyfile, ClientOpts1), + ClientOpts = [{groups,"P-256:X25519"}|ClientOpts2], + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + {fail_if_no_peer_cert, false}, + {supported_groups, [x448, x25519]}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +tls13_hrr_client_auth_ssl_server_openssl_client() -> + [{doc,"TLS 1.3 (HelloRetryRequest): Test client authentication."}]. + +tls13_hrr_client_auth_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ClientOpts = [{groups,"P-256:X25519"}|ClientOpts0], + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + {fail_if_no_peer_cert, true}, + {supported_groups, [x448, x25519]}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +tls13_unsupported_sign_algo_client_auth_ssl_server_openssl_client() -> + [{doc,"TLS 1.3: Test client authentication with unsupported signature_algorithm"}]. + +tls13_unsupported_sign_algo_client_auth_ssl_server_openssl_client(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + %% Skip rsa_pkcs1_sha256! + {signature_algs, [rsa_pkcs1_sha384, rsa_pkcs1_sha512]}, + {fail_if_no_peer_cert, true}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result( + Server, + {error, + {tls_alert, + {insufficient_security, + "received SERVER ALERT: Fatal - Insufficient Security - " + "\"No suitable signature algorithm\""}}}), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +%% Triggers Client Alert as openssl s_client does not have a certificate with a +%% signature algorithm supported by the server (signature_algorithms_cert extension +%% of CertificateRequest does not contain the algorithm of the client certificate). +tls13_unsupported_sign_algo_cert_client_auth_ssl_server_openssl_client() -> + [{doc,"TLS 1.3: Test client authentication with unsupported signature_algorithm_cert"}]. + +tls13_unsupported_sign_algo_cert_client_auth_ssl_server_openssl_client(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {log_level, debug}, + {verify, verify_peer}, + {signature_algs, [rsa_pkcs1_sha256, rsa_pkcs1_sha384, rsa_pss_rsae_sha256]}, + %% Skip rsa_pkcs1_sha256! + {signature_algs_cert, [rsa_pkcs1_sha384, rsa_pkcs1_sha512]}, + {fail_if_no_peer_cert, true}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result( + Server, + {error, + {tls_alert, + {illegal_parameter, + "received CLIENT ALERT: Fatal - Illegal Parameter"}}}), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index c921dcae4c..7f8e81dbd8 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -1110,11 +1110,26 @@ start_basic_client(openssl, Version, Port, ClientOpts) -> Cert = proplists:get_value(certfile, ClientOpts), Key = proplists:get_value(keyfile, ClientOpts), CA = proplists:get_value(cacertfile, ClientOpts), + Groups0 = proplists:get_value(groups, ClientOpts), Exe = "openssl", - Args = ["s_client", "-verify", "2", "-port", integer_to_list(Port), + Args0 = ["s_client", "-verify", "2", "-port", integer_to_list(Port), ssl_test_lib:version_flag(Version), - "-cert", Cert, "-CAfile", CA, - "-key", Key, "-host","localhost", "-msg", "-debug"], + "-CAfile", CA, "-host", "localhost", "-msg", "-debug"], + Args1 = + case Groups0 of + undefined -> + Args0; + G -> + Args0 ++ ["-groups", G] + end, + Args = + case {Cert, Key} of + {C, K} when C =:= undefined orelse + K =:= undefined -> + Args1; + {C, K} -> + Args1 ++ ["-cert", C, "-key", K] + end, OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), true = port_command(OpenSslPort, "Hello world"), diff --git a/lib/ssl/vsn.mk b/lib/ssl/vsn.mk index 3527062a8a..c4bcc1560c 100644 --- a/lib/ssl/vsn.mk +++ b/lib/ssl/vsn.mk @@ -1 +1 @@ -SSL_VSN = 9.1.2 +SSL_VSN = 9.2.1 diff --git a/lib/stdlib/doc/src/beam_lib.xml b/lib/stdlib/doc/src/beam_lib.xml index 8bb4cf9101..bb44ca3201 100644 --- a/lib/stdlib/doc/src/beam_lib.xml +++ b/lib/stdlib/doc/src/beam_lib.xml @@ -470,6 +470,18 @@ CryptoKeyFun(clear) -> term()</code> </func> <func> + <name name="strip" arity="2" since=""/> + <fsummary>Remove chunks not needed by the loader from a BEAM file. + </fsummary> + <desc> + <p>Removes all chunks from a BEAM + file except those needed by the loader or passed in. In particular, + the debug information (chunk <c>debug_info</c> and <c>abstract_code</c>) + is removed.</p> + </desc> + </func> + + <func> <name name="strip_files" arity="1" since=""/> <fsummary>Removes chunks not needed by the loader from BEAM files. </fsummary> @@ -483,6 +495,19 @@ CryptoKeyFun(clear) -> term()</code> </func> <func> + <name name="strip_files" arity="2" since=""/> + <fsummary>Removes chunks not needed by the loader from BEAM files. + </fsummary> + <desc> + <p>Removes all chunks except + those needed by the loader or passed in from BEAM files. In particular, + the debug information (chunk <c>debug_info</c> and <c>abstract_code</c>) + is removed. The returned list contains one element for each + specified filename, in the same order as in <c>Files</c>.</p> + </desc> + </func> + + <func> <name name="strip_release" arity="1" since=""/> <fsummary>Remove chunks not needed by the loader from all BEAM files of a release.</fsummary> @@ -497,6 +522,20 @@ CryptoKeyFun(clear) -> term()</code> </func> <func> + <name name="strip_release" arity="2" since=""/> + <fsummary>Remove chunks not needed by the loader from all BEAM files of + a release.</fsummary> + <desc> + <p>Removes all chunks + except those needed by the loader or passed in from the BEAM files of a + release. <c><anno>Dir</anno></c> is to be the installation root + directory. For example, the current OTP release can be + stripped with the call + <c>beam_lib:strip_release(code:root_dir())</c>.</p> + </desc> + </func> + + <func> <name name="version" arity="1" since=""/> <fsummary>Read the module version of the BEAM file.</fsummary> <desc> diff --git a/lib/stdlib/doc/src/ets.xml b/lib/stdlib/doc/src/ets.xml index d2ac6a75b1..2cb677785d 100644 --- a/lib/stdlib/doc/src/ets.xml +++ b/lib/stdlib/doc/src/ets.xml @@ -642,12 +642,11 @@ Error: fun containing local Erlang function calls <p><marker id="info_2_safe_fixed_monotonic_time"/></p> <p><c>Item=safe_fixed|safe_fixed_monotonic_time, Value={FixationTime,Info}|false</c></p> - <p>If the table has been fixed using + <p>If the table is fixed using <seealso marker="#safe_fixtable/2"> <c>safe_fixtable/2</c></seealso>, the call returns a tuple where <c>FixationTime</c> is the - time when the table was first fixed by a process, which either - is or is not one of the processes it is fixed by now.</p> + last time when the table changed from unfixed to fixed.</p> <p>The format and value of <c>FixationTime</c> depends on <c>Item</c>:</p> <taglist> @@ -679,8 +678,15 @@ Error: fun containing local Erlang function calls table is fixed by now. <c>RefCount</c> is the value of the reference counter and it keeps track of how many times the table has been fixed by the process.</p> - <p>If the table never has been fixed, the call returns - <c>false</c>.</p> + <p>Table fixations are not limited to <seealso marker="#safe_fixtable/2"> + <c>safe_fixtable/2</c></seealso>. Temporary fixations may also + be done by for example <seealso marker="#traversal">traversing + functions</seealso> like <c>select</c> and <c>match</c>. Such + table fixations are automatically released before the + corresponding functions returns, but they may be seen by a + concurrent call to + <c>ets:info(T,safe_fixed|safe_fixed_monotonic_time)</c>.</p> + <p>If the table is not fixed at all, the call returns <c>false</c>.</p> </item> <item> <p><c>Item=stats, Value=tuple()</c></p> diff --git a/lib/stdlib/doc/src/gen_statem.xml b/lib/stdlib/doc/src/gen_statem.xml index 0cad5929ca..d4a4ba268b 100644 --- a/lib/stdlib/doc/src/gen_statem.xml +++ b/lib/stdlib/doc/src/gen_statem.xml @@ -320,12 +320,13 @@ erlang:'!' -----> Module:StateName/3 </p> <p> There is also a server start option - <seealso marker="#type-hibernate_after_opt"> + <seealso marker="#type-enter_loop_opt"> <c>{hibernate_after, Timeout}</c> </seealso> for - <seealso marker="#start/3"><c>start/3,4</c></seealso> or - <seealso marker="#start_link/3"><c>start_link/3,4</c></seealso> + <seealso marker="#start/3"><c>start/3,4</c></seealso>, + <seealso marker="#start_link/3"><c>start_link/3,4</c></seealso> or + <seealso marker="#enter_loop/4"><c>enter_loop/4,5,6</c></seealso>, that may be used to automatically hibernate the server. </p> </description> diff --git a/lib/stdlib/doc/src/notes.xml b/lib/stdlib/doc/src/notes.xml index dee7136eb1..23c3f6e981 100644 --- a/lib/stdlib/doc/src/notes.xml +++ b/lib/stdlib/doc/src/notes.xml @@ -31,6 +31,57 @@ </header> <p>This document describes the changes made to the STDLIB application.</p> +<section><title>STDLIB 3.8</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Fix a bug in the Erlang Pretty Printer: long atom + names in combination with <c><<>></c> could + cause a crash. </p> + <p> + Own Id: OTP-15592 Aux Id: ERL-818 </p> + </item> + <item> + <p> Fix bugs that could cause wrong results or bad + performance when formatting lists of characters using the + control sequences <c>p</c> or <c>P</c> and limiting the + output with the option <c>chars_limit</c>. </p> + <p> + Own Id: OTP-15639</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Improved ETS documentation about safe table traversal and + the partially bound key optimization for + <c>ordered_set</c>.</p> + <p> + Own Id: OTP-15545 Aux Id: PR-2103, PR-2139 </p> + </item> + <item> + <p> Optimize <c>calendar:gregorian_days_to_date/1</c>. + </p> + <p> + Own Id: OTP-15572 Aux Id: PR-2121 </p> + </item> + <item> + <p> Optimize functions + <c>calendar:rfc3339_to_system_time()</c> and + <c>calendar:system_time_to_rfc3339()</c>. </p> + <p> + Own Id: OTP-15630</p> + </item> + </list> + </section> + +</section> + <section><title>STDLIB 3.7.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/stdlib/src/array.erl b/lib/stdlib/src/array.erl index 939b1fb488..1504326c61 100644 --- a/lib/stdlib/src/array.erl +++ b/lib/stdlib/src/array.erl @@ -126,11 +126,12 @@ %% per write than base 10, but the speedup is only 21%.) -define(DEFAULT, undefined). --define(LEAFSIZE, 10). % the "base" --define(NODESIZE, ?LEAFSIZE). % (no reason to have a different size) +-define(LEAFSIZE, 10). % the "base" (assumed to be > 1) +-define(NODESIZE, ?LEAFSIZE). % must not be LEAFSIZE-1; keep same as leaf -define(NODEPATTERN(S), {_,_,_,_,_,_,_,_,_,_,S}). % NODESIZE+1 elements! --define(NEW_NODE(S), % beware of argument duplication! - setelement((?NODESIZE+1),erlang:make_tuple((?NODESIZE+1),(S)),(S))). +-define(NEW_NODE(E,S), % general case (currently unused) + setelement((?NODESIZE+1),erlang:make_tuple((?NODESIZE+1),(E)),(S))). +-define(NEW_NODE(S), erlang:make_tuple((?NODESIZE+1),(S))). % when E = S -define(NEW_LEAF(D), erlang:make_tuple(?LEAFSIZE,(D))). -define(NODELEAFS, ?NODESIZE*?LEAFSIZE). @@ -605,7 +606,7 @@ grow(I, E, M) -> grow_1(I, E, M). grow_1(I, E, M) when I >= M -> - grow(I, setelement(1, ?NEW_NODE(M), E), ?extend(M)); + grow_1(I, setelement(1, ?NEW_NODE(M), E), ?extend(M)); grow_1(_I, E, M) -> {E, M}. @@ -1631,12 +1632,11 @@ foldl_test_() -> ?_assert(foldl(Sum, 0, from_list(lists:seq(0,10))) =:= 55), ?_assert(foldl(Reverse, [], from_list(lists:seq(0,1000))) =:= lists:reverse(lists:seq(0,1000))), - ?_assert({999,[N0*100+1+2,N0*2+1+1,0]} =:= - foldl(Vals, {0,[]}, + ?_assertEqual({N0*100+1-2,[N0*100+1+2,N0*2+1+1,0]}, + foldl(Vals, {0,[]}, set(N0*100+1,2, set(N0*2+1,1, set(0,0,new()))))) - ]. -endif. @@ -1786,12 +1786,11 @@ foldr_test_() -> ?_assert(foldr(Sum, 0, from_list(lists:seq(0,10))) =:= 55), ?_assert(foldr(List, [], from_list(lists:seq(0,1000))) =:= lists:seq(0,1000)), - ?_assert({999,[0,N0*2+1+1,N0*100+1+2]} =:= - foldr(Vals, {0,[]}, + ?_assertEqual({N0*100+1-2,[0,N0*2+1+1,N0*100+1+2]}, + foldr(Vals, {0,[]}, set(N0*100+1,2, set(N0*2+1,1, set(0,0,new()))))) - ]. -endif. diff --git a/lib/stdlib/src/beam_lib.erl b/lib/stdlib/src/beam_lib.erl index 3386cfcbe6..aa992f17ab 100644 --- a/lib/stdlib/src/beam_lib.erl +++ b/lib/stdlib/src/beam_lib.erl @@ -32,8 +32,12 @@ all_chunks/1, diff_dirs/2, strip/1, + strip/2, strip_files/1, + strip_files/2, strip_release/1, + strip_release/2, + significant_chunks/0, build_module/1, version/1, md5/1, @@ -188,7 +192,16 @@ diff_dirs(Dir1, Dir2) -> Beam2 :: beam(). strip(FileName) -> - try strip_file(FileName) + strip(FileName, []). + +-spec strip(Beam1, AdditionalChunks) -> + {'ok', {module(), Beam2}} | {'error', 'beam_lib', info_rsn()} when + Beam1 :: beam(), + AdditionalChunks :: [chunkid()], + Beam2 :: beam(). + +strip(FileName, AdditionalChunks) -> + try strip_file(FileName, AdditionalChunks) catch Error -> Error end. -spec strip_files(Files) -> @@ -196,8 +209,17 @@ strip(FileName) -> Files :: [beam()], Beam :: beam(). -strip_files(Files) when is_list(Files) -> - try strip_fils(Files) +strip_files(Files) -> + strip_files(Files, []). + +-spec strip_files(Files, AdditionalChunks) -> + {'ok', [{module(), Beam}]} | {'error', 'beam_lib', info_rsn()} when + Files :: [beam()], + AdditionalChunks :: [chunkid()], + Beam :: beam(). + +strip_files(Files, AdditionalChunks) when is_list(Files) -> + try strip_fils(Files, AdditionalChunks) catch Error -> Error end. -spec strip_release(Dir) -> @@ -207,7 +229,17 @@ strip_files(Files) when is_list(Files) -> Reason :: {'not_a_directory', term()} | info_rsn(). strip_release(Root) -> - catch strip_rel(Root). + strip_release(Root, []). + +-spec strip_release(Dir, AdditionalChunks) -> + {'ok', [{module(), file:filename()}]} + | {'error', 'beam_lib', Reason} when + Dir :: atom() | file:filename(), + AdditionalChunks :: [chunkid()], + Reason :: {'not_a_directory', term()} | info_rsn(). + +strip_release(Root, AdditionalChunks) -> + catch strip_rel(Root, AdditionalChunks). -spec version(Beam) -> {'ok', {module(), [Version :: term()]}} | @@ -401,17 +433,17 @@ cmp_lists([{Id, C1} | R1], [{Id, C2} | R2]) -> cmp_lists(_, _) -> error(different_chunks). -strip_rel(Root) -> +strip_rel(Root, AdditionalChunks) -> ok = assert_directory(Root), - strip_fils(filelib:wildcard(filename:join(Root, "lib/*/ebin/*.beam"))). + strip_fils(filelib:wildcard(filename:join(Root, "lib/*/ebin/*.beam")), AdditionalChunks). %% -> {ok, [{Mod, BinaryOrFileName}]} | throw(Error) -strip_fils(Files) -> - {ok, [begin {ok, Reply} = strip_file(F), Reply end || F <- Files]}. +strip_fils(Files, AdditionalChunks) -> + {ok, [begin {ok, Reply} = strip_file(F, AdditionalChunks), Reply end || F <- Files]}. %% -> {ok, {Mod, FileName}} | {ok, {Mod, binary()}} | throw(Error) -strip_file(File) -> - {ok, {Mod, Chunks}} = read_significant_chunks(File, significant_chunks()), +strip_file(File, AdditionalChunks) -> + {ok, {Mod, Chunks}} = read_significant_chunks(File, AdditionalChunks ++ significant_chunks()), {ok, Stripped0} = build_module(Chunks), Stripped = compress(Stripped0), case File of diff --git a/lib/stdlib/src/io_lib_pretty.erl b/lib/stdlib/src/io_lib_pretty.erl index 5483ea87b5..8f2fd7ea8f 100644 --- a/lib/stdlib/src/io_lib_pretty.erl +++ b/lib/stdlib/src/io_lib_pretty.erl @@ -721,7 +721,7 @@ printable_list(_L, 1, _T, _Enc) -> printable_list(L, _D, T, latin1) when T < 0 -> io_lib:printable_latin1_list(L); printable_list(L, _D, T, Enc) when T >= 0 -> - case slice(L, tsub(T, 2)) of + case slice(L, tsub(T, 2), Enc) of false -> false; {prefix, Prefix} when Enc =:= latin1 -> @@ -737,20 +737,46 @@ printable_list(L, _D, T, Enc) when T >= 0 -> printable_list(L, _D, T, _Uni) when T < 0-> io_lib:printable_list(L). -slice(L, N) -> - try io_lib:chars_length(L) =< N of - true -> +slice(L, N, latin1) -> + try lists:split(N, L) of + {_, []} -> all; - false -> - case string:slice(L, 0, N) of - "" -> - false; - Prefix -> - {prefix, Prefix} + {[], _} -> + false; + {L1, _} -> + {prefix, L1} + catch + _:_ -> + all + end; +slice(L, N, _Uni) -> + %% Be careful not to traverse more of L than necessary. + try string:slice(L, 0, N) of + "" -> + false; + Prefix -> + %% Assume no binaries are introduced by string:slice(). + case is_flat(L, lists:flatlength(Prefix)) of + true -> + case string:equal(Prefix, L) of + true -> + all; + false -> + {prefix, Prefix} + end; + false -> + false end catch _:_ -> false end. +is_flat(_L, 0) -> + true; +is_flat([C|Cs], N) when is_integer(C) -> + is_flat(Cs, N - 1); +is_flat(_, _N) -> + false. + printable_bin0(Bin, D, T, Enc) -> Len = case D >= 0 of true -> diff --git a/lib/stdlib/src/stdlib.appup.src b/lib/stdlib/src/stdlib.appup.src index 9e5d6a3bd8..37ea97c353 100644 --- a/lib/stdlib/src/stdlib.appup.src +++ b/lib/stdlib/src/stdlib.appup.src @@ -40,7 +40,8 @@ {<<"^3\\.6$">>,[restart_new_emulator]}, {<<"^3\\.6\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.7$">>,[restart_new_emulator]}, - {<<"^3\\.7\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}], + {<<"^3\\.7\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^3\\.7\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], [{<<"^3\\.4$">>,[restart_new_emulator]}, {<<"^3\\.4\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.4\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -54,4 +55,5 @@ {<<"^3\\.6$">>,[restart_new_emulator]}, {<<"^3\\.6\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.7$">>,[restart_new_emulator]}, - {<<"^3\\.7\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}]}. + {<<"^3\\.7\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^3\\.7\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. diff --git a/lib/stdlib/src/string.erl b/lib/stdlib/src/string.erl index 2939e78d9d..1f8bdc5432 100644 --- a/lib/stdlib/src/string.erl +++ b/lib/stdlib/src/string.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1247,18 +1247,20 @@ split_1(Bin, [_C|_]=Needle, Start, Where, Curr0, Acc) -> end end. -lexemes_m([CP|_]=Cs0, {GCs,CPs,_}=Seps, Ts) when is_integer(CP) -> +lexemes_m([CP|_]=Cs0, {GCs,CPs,_}=Seps0, Ts) when is_integer(CP) -> case lists:member(CP, CPs) of true -> [GC|Cs2] = unicode_util:gc(Cs0), case lists:member(GC, GCs) of true -> - lexemes_m(Cs2, Seps, Ts); + lexemes_m(Cs2, Seps0, Ts); false -> + Seps = search_compile(Seps0), {Lexeme,Rest} = lexeme_pick(Cs0, Seps, []), lexemes_m(Rest, Seps, [Lexeme|Ts]) end; false -> + Seps = search_compile(Seps0), {Lexeme,Rest} = lexeme_pick(Cs0, Seps, []), lexemes_m(Rest, Seps, [Lexeme|Ts]) end; diff --git a/lib/stdlib/test/beam_lib_SUITE.erl b/lib/stdlib/test/beam_lib_SUITE.erl index 6418dc7eb6..4b2694320e 100644 --- a/lib/stdlib/test/beam_lib_SUITE.erl +++ b/lib/stdlib/test/beam_lib_SUITE.erl @@ -35,7 +35,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, - normal/1, error/1, cmp/1, cmp_literals/1, strip/1, otp_6711/1, + normal/1, error/1, cmp/1, cmp_literals/1, strip/1, strip_add_chunks/1, otp_6711/1, building/1, md5/1, encrypted_abstr/1, encrypted_abstr_file/1]). -export([init_per_testcase/2, end_per_testcase/2]). @@ -45,7 +45,7 @@ suite() -> {timetrap,{minutes,2}}]. all() -> - [error, normal, cmp, cmp_literals, strip, otp_6711, + [error, normal, cmp, cmp_literals, strip, strip_add_chunks, otp_6711, building, md5, encrypted_abstr, encrypted_abstr_file]. groups() -> @@ -401,6 +401,69 @@ strip(Conf) when is_list(Conf) -> Source5D1, BeamFile5D1]), ok. +strip_add_chunks(Conf) when is_list(Conf) -> + PrivDir = ?privdir, + {SourceD1, BeamFileD1} = make_beam(PrivDir, simple, member), + {Source2D1, BeamFile2D1} = make_beam(PrivDir, simple2, concat), + {Source3D1, BeamFile3D1} = make_beam(PrivDir, make_fun, make_fun), + {Source4D1, BeamFile4D1} = make_beam(PrivDir, constant, constant), + {Source5D1, BeamFile5D1} = make_beam(PrivDir, lines, lines), + + NoOfTables = erlang:system_info(ets_count), + P0 = pps(), + + %% strip binary + verify(not_a_beam_file, beam_lib:strip(<<>>)), + {ok, B1} = file:read_file(BeamFileD1), + {ok, {simple, NB1}} = beam_lib:strip(B1), + + BId1 = chunk_ids(B1), + NBId1 = chunk_ids(NB1), + true = length(BId1) > length(NBId1), + compare_chunks(B1, NB1, NBId1), + + %% Keep all the extra chunks + ExtraChunks = ["Abst" , "Dbgi" , "Attr" , "CInf" , "LocT" , "Atom" ], + {ok, {simple, AB1}} = beam_lib:strip(B1, ExtraChunks), + ABId1 = chunk_ids(AB1), + true = length(BId1) == length(ABId1), + compare_chunks(B1, AB1, ABId1), + + %% strip file - Keep extra chunks + verify(file_error, beam_lib:strip(foo)), + {ok, {simple, _}} = beam_lib:strip(BeamFileD1, ExtraChunks), + compare_chunks(B1, BeamFileD1, ABId1), + + %% strip_files + {ok, B2} = file:read_file(BeamFile2D1), + {ok, [{simple,_},{simple2,_}]} = beam_lib:strip_files([B1, B2], ExtraChunks), + {ok, [{simple,_},{simple2,_},{make_fun,_},{constant,_}]} = + beam_lib:strip_files([BeamFileD1, BeamFile2D1, BeamFile3D1, BeamFile4D1], ExtraChunks), + + %% check that each module can be loaded. + {module, simple} = code:load_abs(filename:rootname(BeamFileD1)), + {module, simple2} = code:load_abs(filename:rootname(BeamFile2D1)), + {module, make_fun} = code:load_abs(filename:rootname(BeamFile3D1)), + {module, constant} = code:load_abs(filename:rootname(BeamFile4D1)), + + %% check that line number information is still present after stripping + {module, lines} = code:load_abs(filename:rootname(BeamFile5D1)), + {'EXIT',{badarith,[{lines,t,1,Info}|_]}} = (catch lines:t(atom)), + false = code:purge(lines), + true = code:delete(lines), + {ok, {lines,BeamFile5D1}} = beam_lib:strip(BeamFile5D1), + {module, lines} = code:load_abs(filename:rootname(BeamFile5D1)), + {'EXIT',{badarith,[{lines,t,1,Info}|_]}} = (catch lines:t(atom)), + + true = (P0 == pps()), + NoOfTables = erlang:system_info(ets_count), + + delete_files([SourceD1, BeamFileD1, + Source2D1, BeamFile2D1, + Source3D1, BeamFile3D1, + Source4D1, BeamFile4D1, + Source5D1, BeamFile5D1]), + ok. otp_6711(Conf) when is_list(Conf) -> {'EXIT',{function_clause,_}} = (catch {a, beam_lib:info(3)}), @@ -729,6 +792,7 @@ make_beam(Dir, Module, F) -> FileBase = filename:join(Dir, atom_to_list(Module)), Source = FileBase ++ ".erl", BeamFile = FileBase ++ ".beam", + file:delete(BeamFile), simple_file(Source, Module, F), {ok, _} = compile:file(Source, [{outdir,Dir}, debug_info, report]), {Source, BeamFile}. diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 22c77aa172..87ca9bd32c 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -56,9 +56,11 @@ -export([t_match_spec_run/1]). -export([t_bucket_disappears/1]). -export([t_named_select/1]). +-export([select_fixtab_owner_change/1]). -export([otp_5340/1]). -export([otp_6338/1]). -export([otp_6842_select_1000/1]). +-export([select_mbuf_trapping/1]). -export([otp_7665/1]). -export([meta_wb/1]). -export([grow_shrink/1, grow_pseudo_deleted/1, shrink_pseudo_deleted/1]). @@ -68,7 +70,10 @@ -export([smp_insert/1, smp_fixed_delete/1, smp_unfix_fix/1, smp_select_delete/1, smp_ordered_iteration/1, smp_select_replace/1, otp_8166/1, otp_8732/1, delete_unfix_race/1]). --export([throughput_benchmark/0, test_throughput_benchmark/1]). +-export([throughput_benchmark/0, + throughput_benchmark/1, + test_throughput_benchmark/1, + long_throughput_benchmark/1]). -export([exit_large_table_owner/1, exit_many_large_table_owner/1, exit_many_tables_owner/1, @@ -91,6 +96,7 @@ -include_lib("stdlib/include/ms_transform.hrl"). % ets:fun2ms -include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). -define(m(A,B), assert_eq(A,B)). -define(heap_binary_size, 64). @@ -129,9 +135,10 @@ all() -> t_insert_list, t_test_ms, t_select_delete, t_select_replace, t_select_replace_next_bug, t_ets_dets, memory, t_select_reverse, t_bucket_disappears, - t_named_select, + t_named_select, select_fixtab_owner_change, select_fail, t_insert_new, t_repair_continuation, otp_5340, otp_6338, otp_6842_select_1000, otp_7665, + select_mbuf_trapping, otp_8732, meta_wb, grow_shrink, grow_pseudo_deleted, shrink_pseudo_deleted, {group, meta_smp}, smp_insert, smp_fixed_delete, smp_unfix_fix, smp_select_replace, @@ -148,7 +155,8 @@ all() -> take, whereis_table, delete_unfix_race, - test_throughput_benchmark]. + test_throughput_benchmark, + {group, benchmark}]. groups() -> [{new, [], @@ -176,7 +184,9 @@ groups() -> {meta_smp, [], [meta_lookup_unnamed_read, meta_lookup_unnamed_write, meta_lookup_named_read, meta_lookup_named_write, - meta_newdel_unnamed, meta_newdel_named]}]. + meta_newdel_unnamed, meta_newdel_named]}, + {benchmark, [], + [long_throughput_benchmark]}]. init_per_suite(Config) -> erts_debug:set_internal_state(available_internal_state, true), @@ -189,9 +199,61 @@ end_per_suite(_Config) -> catch erts_debug:set_internal_state(available_internal_state, false), ok. +init_per_group(benchmark, Config) -> + P = self(), + %% Spawn owner of ETS table that is alive until end_per_group is run + EtsProcess = + spawn( + fun()-> + Tab = ets:new(ets_benchmark_result_summary_tab, [public]), + P ! {the_table, Tab}, + receive + kill -> ok + end + end), + Tab = receive {the_table, T} -> T end, + CounterNames = [nr_of_benchmarks, + total_throughput, + nr_of_set_benchmarks, + total_throughput_set, + nr_of_ordered_set_benchmarks, + total_throughput_ordered_set], + lists:foreach(fun(CtrName) -> + ets:insert(Tab, {CtrName, 0.0}) + end, + CounterNames), + [{ets_benchmark_result_summary_tab, Tab}, + {ets_benchmark_result_summary_tab_process, EtsProcess} | Config]; init_per_group(_GroupName, Config) -> Config. +end_per_group(benchmark, Config) -> + T = proplists:get_value(ets_benchmark_result_summary_tab, Config), + EtsProcess = proplists:get_value(ets_benchmark_result_summary_tab_process, Config), + Report = + fun(NOfBenchmarksCtr, TotThroughoutCtr, Name) -> + Average = + ets:lookup_element(T, TotThroughoutCtr, 2) / + ets:lookup_element(T, NOfBenchmarksCtr, 2), + io:format("~p ~p~n", [Name, Average]), + ct_event:notify( + #event{name = benchmark_data, + data = [{suite,"ets_bench"}, + {name, Name}, + {value, Average}]}) + end, + Report(nr_of_benchmarks, + total_throughput, + "Average Throughput"), + Report(nr_of_set_benchmarks, + total_throughput_set, + "Average Throughput Set"), + Report(nr_of_ordered_set_benchmarks, + total_throughput_ordered_set, + "Average Throughput Ordered Set"), + ets:delete(T), + EtsProcess ! kill, + Config; end_per_group(_GroupName, Config) -> Config. @@ -247,7 +309,64 @@ t_named_select_do(Opts) -> verify_etsmem(EtsMem). +%% Verify select and friends release fixtab as they should +%% even when owneship is changed between traps. +select_fixtab_owner_change(_Config) -> + T = ets:new(xxx, [protected]), + NKeys = 2000, + [ets:insert(T,{K,K band 7}) || K <- lists:seq(1,NKeys)], + + %% Buddy and Papa will ping-pong table ownership between them + %% and the aim is to give Buddy the table when he is + %% in the middle of a yielding select* call. + {Buddy,_} = spawn_opt(fun() -> sfoc_buddy_loop(T, 1, undefined) end, + [link,monitor]), + + sfoc_papa_loop(T, Buddy), + + receive {'DOWN', _, process, Buddy, _} -> ok end, + ets:delete(T), + ok. + +sfoc_buddy_loop(T, I, State0) -> + receive + {'ETS-TRANSFER', T, Papa, _} -> + ets:give_away(T, Papa, State0), + case State0 of + done -> + ok; + _ -> + State1 = sfoc_traverse(T, I, State0), + %% Verify no fixation left + {I, false} = {I, ets:info(T, safe_fixed_monotonic_time)}, + sfoc_buddy_loop(T, I+1, State1) + end + end. + +sfoc_papa_loop(T, Buddy) -> + ets:give_away(T, Buddy, "Catch!"), + receive + {'ETS-TRANSFER', T, Buddy, State} -> + case State of + done -> + ok; + _ -> + sfoc_papa_loop(T, Buddy) + end + end. +sfoc_traverse(T, 1, S) -> + ets:select(T, [{{'$1',7}, [], ['$1']}]), S; +sfoc_traverse(T, 2, S) -> + 0 = ets:select_count(T, [{{'$1',7}, [], [false]}]), S; +sfoc_traverse(T, 3, _) -> + Limit = ets:info(T, size) div 2, + {_, Continuation} = ets:select(T, [{{'$1',7}, [], ['$1']}], + Limit), + Continuation; +sfoc_traverse(_T, 4, Continuation) -> + _ = ets:select(Continuation), + done. %% Check ets:match_spec_run/2. t_match_spec_run(Config) when is_list(Config) -> @@ -5292,6 +5411,61 @@ otp_6338(Config) when is_list(Config) -> end), ok. +%% OTP-15660: Verify select not doing excessive trapping +%% when process have mbuf heap fragments. +select_mbuf_trapping(Config) when is_list(Config) -> + select_mbuf_trapping_do(set), + select_mbuf_trapping_do(ordered_set). + +select_mbuf_trapping_do(Type) -> + T = ets:new(xxx, [Type]), + NKeys = 50, + [ets:insert(T, {K, value}) || K <- lists:seq(1,NKeys)], + + {priority, Prio} = process_info(self(), priority), + Tracee = self(), + [SchedTracer] + = start_loopers(1, Prio, + fun (SC) -> + receive + {trace, Tracee, out, _} -> + SC+1; + done -> + Tracee ! {schedule_count, SC}, + exit(normal) + end + end, + 0), + + erlang:garbage_collect(), + 1 = erlang:trace(self(), true, [running,{tracer,SchedTracer}]), + + %% Artificially create an mbuf heap fragment + MbufTerm = "Frag me up", + MbufTerm = erts_debug:set_internal_state(mbuf, MbufTerm), + + Keys = ets:select(T, [{{'$1', value}, [], ['$1']}]), + NKeys = length(Keys), + + 1 = erlang:trace(self(), false, [running]), + Ref = erlang:trace_delivered(Tracee), + receive + {trace_delivered, Tracee, Ref} -> + SchedTracer ! done + end, + receive + {schedule_count, N} -> + io:format("~p context switches: ~p", [Type,N]), + if + N < 3 -> ok; + true -> ct:fail(failed) + end + end, + true = ets:delete(T), + ok. + + + %% Elements could come in the wrong order in a bag if a rehash occurred. otp_5340(Config) when is_list(Config) -> repeat_for_opts(fun otp_5340_do/1). @@ -6415,8 +6589,8 @@ whereis_table(Config) when is_list(Config) -> ok. -%% The following work functions are used by -%% throughput_benchmark/4. They are declared on the top level beacuse +%% The following help functions are used by +%% throughput_benchmark. They are declared on the top level beacuse %% declaring them as function local funs cause a scalability issue. get_op([{_,O}], _RandNum) -> O; @@ -6451,10 +6625,131 @@ prefill_table_loop(T, RS0, N, ObjFun) -> ets:insert(T, ObjFun(Key)), prefill_table_loop(T, RS1, N-1, ObjFun). -throughput_benchmark() -> - throughput_benchmark(false, not_set, not_set). +-record(ets_throughput_bench_config, + {benchmark_duration_ms = 3000, + recover_time_ms = 1000, + thread_counts = not_set, + key_ranges = [1000000], + scenarios = + [ + [ + {0.5, insert}, + {0.5, delete} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.8, lookup} + ], + [ + {0.01, insert}, + {0.01, delete}, + {0.98, lookup} + ], + [ + {1.0, lookup} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq10} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq100} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq1000} + ], + [ + {1.0, nextseq1000} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.7999, lookup}, + {0.0001, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.799999, lookup}, + {0.000001, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, partial_select1000} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.7999, lookup}, + {0.0001, partial_select1000} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.799999, lookup}, + {0.000001, partial_select1000} + ] + ], + table_types = + [ + [ordered_set, public], + [ordered_set, public, {write_concurrency, true}], + [ordered_set, public, {read_concurrency, true}], + [ordered_set, public, {write_concurrency, true}, {read_concurrency, true}], + [set, public], + [set, public, {write_concurrency, true}], + [set, public, {read_concurrency, true}], + [set, public, {write_concurrency, true}, {read_concurrency, true}] + ], + etsmem_fun = fun() -> ok end, + verify_etsmem_fun = fun(_) -> true end, + notify_res_fun = fun(_Name, _Throughput) -> ok end, + print_result_paths_fun = + fun(ResultPath, _LatestResultPath) -> + Comment = + io_lib:format("<a href=\"file:///~s\">Result visualization</a>",[ResultPath]), + {comment, Comment} + end + }). + +stdout_notify_res(ResultPath, LatestResultPath) -> + io:format("Result Location: /~s~n", [ResultPath]), + io:format("Latest Result Location: ~s~n", [LatestResultPath]). -throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> +throughput_benchmark() -> + throughput_benchmark( + #ets_throughput_bench_config{ + print_result_paths_fun = fun stdout_notify_res/2}). + +throughput_benchmark( + #ets_throughput_bench_config{ + benchmark_duration_ms = BenchmarkDurationMs, + recover_time_ms = RecoverTimeMs, + thread_counts = ThreadCountsOpt, + key_ranges = KeyRanges, + scenarios = Scenarios, + table_types = TableTypes, + etsmem_fun = ETSMemFun, + verify_etsmem_fun = VerifyETSMemFun, + notify_res_fun = NotifyResFun, + print_result_paths_fun = PrintResultPathsFun}) -> NrOfSchedulers = erlang:system_info(schedulers), %% Definitions of operations that are supported by the benchmark NextSeqOp = @@ -6519,7 +6814,7 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> fun(T,KeyRange) -> NextSeqOp(T,KeyRange,1000) end, selectAll => fun(T,_KeyRange) -> - case -1 =:= ets:select_count(T, ets:fun2ms(fun(X) -> true end)) of + case -1 =:= ets:select_count(T, ets:fun2ms(fun(_X) -> true end)) of true -> io:format("Will never be printed"); false -> ok end @@ -6568,11 +6863,28 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> false -> ok end end, + DataHolder = + fun DataHolderFun(Data)-> + receive + {get_data, Pid} -> Pid ! {ets_bench_data, Data}; + D -> DataHolderFun([Data,D]) + end + end, + DataHolderPid = spawn_link(fun()-> DataHolder([]) end), + PrintData = + fun (Str, List) -> + io:format(Str, List), + DataHolderPid ! io_lib:format(Str, List) + end, + GetData = + fun () -> + DataHolderPid ! {get_data, self()}, + receive {ets_bench_data, Data} -> Data end + end, %% Function that runs a benchmark instance and returns the number %% of operations that were performed RunBenchmark = - fun(NrOfProcs, TableConfig, Scenario, - Range, Duration, RecoverTime) -> + fun({NrOfProcs, TableConfig, Scenario, Range, Duration}) -> ProbHelpTab = CalculateOpsProbHelpTab(Scenario, 0), Table = ets:new(t, TableConfig), Nobj = Range div 2, @@ -6580,16 +6892,15 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> Nobj = ets:info(Table, size), SafeFixTableIfRequired(Table, Scenario, true), ParentPid = self(), + Worker = + fun() -> + receive start -> ok end, + WorksDone = + do_work(0, Table, ProbHelpTab, Range, Operations), + ParentPid ! WorksDone + end, ChildPids = - lists:map( - fun(_N) -> - spawn(fun() -> - receive start -> ok end, - WorksDone = - do_work(0, Table, ProbHelpTab, Range, Operations), - ParentPid ! WorksDone - end) - end, lists:seq(1, NrOfProcs)), + lists:map(fun(_N) ->spawn_link(Worker)end, lists:seq(1, NrOfProcs)), lists:foreach(fun(Pid) -> Pid ! start end, ChildPids), timer:sleep(Duration), lists:foreach(fun(Pid) -> Pid ! stop end, ChildPids), @@ -6601,185 +6912,194 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> end, 0, ChildPids), SafeFixTableIfRequired(Table, Scenario, false), ets:delete(Table), - timer:sleep(RecoverTime), TotalWorksDone end, - %% - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %%%% Benchmark Configuration %%%%%%%%%%%%%%%%%%%%%%%% - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% - %% Change the following variables to configure the benchmark runs - ThreadCounts = - case TestMode of - true -> [1, NrOfSchedulers]; - false -> CalculateThreadCounts([1]) - end, - KeyRanges = % Sizes of the key ranges - case TestMode of - true -> [50000]; - false -> [1000000] + RunBenchmarkInSepProcess = + fun(ParameterTuple) -> + P = self(), + spawn_link(fun()-> P ! {bench_result, RunBenchmark(ParameterTuple)} end), + Result = receive {bench_result, Res} -> Res end, + timer:sleep(RecoverTimeMs), + Result end, - Duration = - case BenchmarkRunMs of % Duration of a benchmark run in milliseconds - not_set -> 30000; - _ -> BenchmarkRunMs + RunBenchmarkAndReport = + fun(ThreadCount, + TableType, + Scenario, + KeyRange, + Duration) -> + Result = RunBenchmarkInSepProcess({ThreadCount, + TableType, + Scenario, + KeyRange, + Duration}), + Throughput = Result/(Duration/1000.0), + PrintData("; ~f",[Throughput]), + Name = io_lib:format("Scenario: ~w, Key Range Size: ~w, " + "# of Processes: ~w, Table Type: ~w", + [Scenario, KeyRange, ThreadCount, TableType]), + NotifyResFun(Name, Throughput) end, - TimeMsToSleepAfterEachBenchmarkRun = - case RecoverTimeMs of - not_set -> 1000; - _ -> RecoverTimeMs + ThreadCounts = + case ThreadCountsOpt of + not_set -> + CalculateThreadCounts([1]); + _ -> ThreadCountsOpt end, - TableTypes = % The table types that will be benchmarked - [ - [ordered_set, public], - [ordered_set, public, {write_concurrency, true}], - [ordered_set, public, {read_concurrency, true}], - [ordered_set, public, {write_concurrency, true}, {read_concurrency, true}], - [set, public], - [set, public, {write_concurrency, true}], - [set, public, {read_concurrency, true}], - [set, public, {write_concurrency, true}, {read_concurrency, true}] - ], - Scenarios = % Benchmark scenarios (the fractions should add up to approximately 1.0) - [ - [ - {0.5, insert}, - {0.5, delete} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.8, lookup} - ], - [ - {0.01, insert}, - {0.01, delete}, - {0.98, lookup} - ], - [ - {1.0, lookup} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.4, lookup}, - {0.4, nextseq10} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.4, lookup}, - {0.4, nextseq100} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.4, lookup}, - {0.4, nextseq1000} - ], - [ - {1.0, nextseq1000} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.79, lookup}, - {0.01, selectAll} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.7999, lookup}, - {0.0001, selectAll} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.799999, lookup}, - {0.000001, selectAll} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.79, lookup}, - {0.01, partial_select1000} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.7999, lookup}, - {0.0001, partial_select1000} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.799999, lookup}, - {0.000001, partial_select1000} - ] - ], - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %%%% End of Benchmark Configuration %%%%%%%%%%%%%%%% - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% Prepare for memory check - EtsMem = case TestMode of - true -> etsmem(); - false -> ok - end, %% Run the benchmark - io:format("# Each instance of the benchmark runs for ~w seconds:~n", [Duration/1000]), - io:format("# The result of a benchmark instance is presented as a number representing~n"), - io:format("# the number of operations performed per second:~n~n~n"), - io:format("# To plot graphs for the results below:~n"), - io:format("# 1. Open \"$ERL_TOP/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html\" in a web browser~n"), - io:format("# 2. Copy the lines between \"#BENCHMARK STARTED$\" and \"#BENCHMARK ENDED$\" below~n"), - io:format("# 3. Paste the lines copied in step 2 to the text box in the browser window opened in~n"), - io:format("# step 1 and press the Render button~n~n"), - io:format("#BENCHMARK STARTED$~n"), + PrintData("# Each instance of the benchmark runs for ~w seconds:~n", [BenchmarkDurationMs/1000]), + PrintData("# The result of a benchmark instance is presented as a number representing~n",[]), + PrintData("# the number of operations performed per second:~n~n~n",[]), + PrintData("# To plot graphs for the results below:~n",[]), + PrintData("# 1. Open \"$ERL_TOP/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html\" in a web browser~n",[]), + PrintData("# 2. Copy the lines between \"#BENCHMARK STARTED$\" and \"#BENCHMARK ENDED$\" below~n",[]), + PrintData("# 3. Paste the lines copied in step 2 to the text box in the browser window opened in~n",[]), + PrintData("# step 1 and press the Render button~n~n",[]), + PrintData("#BENCHMARK STARTED$~n",[]), + EtsMem = ETSMemFun(), %% The following loop runs all benchmark scenarios and prints the results (i.e, operations/second) lists:foreach( fun(KeyRange) -> lists:foreach( fun(Scenario) -> - io:format("Scenario: ~s | Key Range Size: ~w$~n", - [RenderScenario(Scenario, ""), - KeyRange]), + PrintData("Scenario: ~s | Key Range Size: ~w$~n", + [RenderScenario(Scenario, ""), KeyRange]), lists:foreach( fun(ThreadCount) -> - io:format("; ~w",[ThreadCount]) + PrintData("; ~w",[ThreadCount]) end, ThreadCounts), - io:format("$~n",[]), + PrintData("$~n",[]), lists:foreach( fun(TableType) -> - io:format("~w ",[TableType]), + PrintData("~w ",[TableType]), lists:foreach( fun(ThreadCount) -> - Result = RunBenchmark(ThreadCount, + RunBenchmarkAndReport(ThreadCount, TableType, Scenario, KeyRange, - Duration, - TimeMsToSleepAfterEachBenchmarkRun), - io:format("; ~f",[Result/(Duration/1000.0)]) + BenchmarkDurationMs) end, ThreadCounts), - io:format("$~n",[]) + PrintData("$~n",[]) end, TableTypes) end, Scenarios) end, KeyRanges), - io:format("~n#BENCHMARK ENDED$~n~n"), - case TestMode of - true -> verify_etsmem(EtsMem); - false -> ok - end. + PrintData("~n#BENCHMARK ENDED$~n~n",[]), + VerifyETSMemFun(EtsMem), + DataDir = filename:join(filename:dirname(code:which(?MODULE)), "ets_SUITE_data"), + TemplatePath = filename:join(DataDir, "visualize_throughput.html"), + {ok, Template} = file:read_file(TemplatePath), + OutputData = string:replace(Template, "#bench_data_placeholder", GetData()), + OutputPath1 = filename:join(DataDir, "ets_bench_result.html"), + {{Year, Month, Day}, {Hour, Minute, Second}} = calendar:now_to_datetime(erlang:timestamp()), + StrTime = lists:flatten(io_lib:format("~4..0w-~2..0w-~2..0wT~2..0w:~2..0w:~2..0w",[Year,Month,Day,Hour,Minute,Second])), + OutputPath2 = filename:join(DataDir, io_lib:format("ets_bench_result_~s.html", [StrTime])), + file:write_file(OutputPath1, OutputData), + file:write_file(OutputPath2, OutputData), + PrintResultPathsFun(OutputPath2, OutputPath1). test_throughput_benchmark(Config) when is_list(Config) -> - throughput_benchmark(true, 100, 0). - + throughput_benchmark( + #ets_throughput_bench_config{ + benchmark_duration_ms = 100, + recover_time_ms = 0, + thread_counts = [1, erlang:system_info(schedulers)], + key_ranges = [50000], + etsmem_fun = fun etsmem/0, + verify_etsmem_fun = fun verify_etsmem/1}). + +long_throughput_benchmark(Config) when is_list(Config) -> + N = erlang:system_info(schedulers), + throughput_benchmark( + #ets_throughput_bench_config{ + benchmark_duration_ms = 3000, + recover_time_ms = 1000, + thread_counts = [1, N div 2, N], + key_ranges = [1000000], + scenarios = + [ + [ + {0.5, insert}, + {0.5, delete} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.8, lookup} + ], + [ + {0.01, insert}, + {0.01, delete}, + {0.98, lookup} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq100} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, partial_select1000} + ] + ], + table_types = + [ + [ordered_set, public, {write_concurrency, true}, {read_concurrency, true}], + [set, public, {write_concurrency, true}, {read_concurrency, true}] + ], + etsmem_fun = fun etsmem/0, + verify_etsmem_fun = fun verify_etsmem/1, + notify_res_fun = + fun(Name, Throughput) -> + SummaryTable = + proplists:get_value(ets_benchmark_result_summary_tab, Config), + AddToSummaryCounter = + case SummaryTable of + undefined -> + fun(_, _) -> + ok + end; + Tab -> + fun(CounterName, ToAdd) -> + OldVal = ets:lookup_element(Tab, CounterName, 2), + NewVal = OldVal + ToAdd, + ets:insert(Tab, {CounterName, NewVal}) + end + end, + Record = + fun(NoOfBenchsCtr, TotThrputCtr) -> + AddToSummaryCounter(NoOfBenchsCtr, 1), + AddToSummaryCounter(TotThrputCtr, Throughput) + end, + Record(nr_of_benchmarks, total_throughput), + case string:find(Name, "ordered_set") of + nomatch -> + Record(nr_of_set_benchmarks, total_throughput_set); + _ -> + Record(nr_of_ordered_set_benchmarks, + total_throughput_ordered_set) + end, + ct_event:notify( + #event{name = benchmark_data, + data = [{suite,"ets_bench"}, + {name, Name}, + {value,Throughput}]}) + end + }). add_lists(L1,L2) -> add_lists(L1,L2,[]). diff --git a/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html b/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html index a2c61aa938..27d6849c60 100644 --- a/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html +++ b/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html @@ -42,7 +42,7 @@ </p> Paste the generated data in the field below and press the Render button: <br> - <textarea id="dataField" rows="4" cols="50"></textarea> + <textarea id="dataField" rows="4" cols="50">#bench_data_placeholder</textarea> <br> <input type="checkbox" id="barPlot"> Bar Plot <br> @@ -56,13 +56,13 @@ <br> <input type="checkbox" class="showCheck" value="[ordered_set,public,{write_concurrency,true},{read_concurrency,true}]" checked> Show <code>[ordered_set,public,{write_concurrency,true},{read_concurrency,true}]</code> <br> - <input type="checkbox" class="showCheck" value="[set,public]"> Show <code>[set,public]</code> + <input type="checkbox" class="showCheck" value="[set,public]" checked> Show <code>[set,public]</code> <br> - <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true}]"> Show <code>[set,public,{write_concurrency,true}]</code> + <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true}]" checked> Show <code>[set,public,{write_concurrency,true}]</code> <br> - <input type="checkbox" class="showCheck" value="[set,public,{read_concurrency,true}]"> Show <code>[set,public,{read_concurrency,true}]</code> + <input type="checkbox" class="showCheck" value="[set,public,{read_concurrency,true}]" checked> Show <code>[set,public,{read_concurrency,true}]</code> <br> - <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true},{read_concurrency,true}]"> Show <code>[set,public,{write_concurrency,true},{read_concurrency,true}]</code> + <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true},{read_concurrency,true}]" checked> Show <code>[set,public,{write_concurrency,true},{read_concurrency,true}]</code> <br> <button id="renderButton" type="button">Render</button> diff --git a/lib/stdlib/test/io_SUITE.erl b/lib/stdlib/test/io_SUITE.erl index f097552e8c..824f5d19f2 100644 --- a/lib/stdlib/test/io_SUITE.erl +++ b/lib/stdlib/test/io_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2018. All Rights Reserved. +%% Copyright Ericsson AB 1999-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -32,7 +32,7 @@ io_with_huge_message_queue/1, format_string/1, maps/1, coverage/1, otp_14178_unicode_atoms/1, otp_14175/1, otp_14285/1, limit_term/1, otp_14983/1, otp_15103/1, otp_15076/1, - otp_15159/1]). + otp_15159/1, otp_15639/1]). -export([pretty/2, trf/3]). @@ -64,7 +64,8 @@ all() -> io_lib_print_binary_depth_one, otp_10302, otp_10755, otp_10836, io_lib_width_too_small, io_with_huge_message_queue, format_string, maps, coverage, otp_14178_unicode_atoms, otp_14175, - otp_14285, limit_term, otp_14983, otp_15103, otp_15076, otp_15159]. + otp_14285, limit_term, otp_14983, otp_15103, otp_15076, otp_15159, + otp_15639]. %% Error cases for output. error_1(Config) when is_list(Config) -> @@ -2647,3 +2648,35 @@ otp_15076(_Config) -> {'EXIT', {badarg, _}} = (catch io_lib:build_text(L)), {'EXIT', {badarg, _}} = (catch io_lib:build_text(L, [])), ok. + +otp_15639(_Config) -> + L = lists:duplicate(10, "a"), + LOpts = [{encoding, latin1}, {chars_limit, 10}], + UOpts = [{encoding, unicode}, {chars_limit, 10}], + "[[...]|...]" = pretty(L, LOpts), + "[[...]|...]" = pretty(L, UOpts), + "[\"a\",[...]|...]" = + pretty(L, [{chars_limit, 12}, {encoding, latin1}]), + "[\"a\",[...]|...]" = + pretty(L, [{chars_limit, 12}, {encoding, unicode}]), + + %% Latin-1 + "\"12345678\"" = pretty("12345678", LOpts), + "\"12345678\"..." = pretty("123456789", LOpts), + "\"\\r\\n123456\"..." = pretty("\r\n1234567", LOpts), + "\"\\r1234567\"..." = pretty("\r12345678", LOpts), + "\"\\r\\n123456\"..." = pretty("\r\n12345678", LOpts), + "\"12345678\"..." = pretty("12345678"++[x], LOpts), + "[49,50|...]" = pretty("1234567"++[x], LOpts), + "[49,x]" = pretty("1"++[x], LOpts), + "[[...]|...]" = pretty(["1","2","3","4","5","6","7","8"], LOpts), + %% Unicode + "\"12345678\"" = pretty("12345678", UOpts), + "\"12345678\"..." = pretty("123456789", UOpts), + "\"\\r\\n1234567\"" = pretty("\r\n1234567", UOpts), + "\"\\r1234567\"..." = pretty("\r12345678", UOpts), + "\"\\r\\n1234567\"..." = pretty("\r\n12345678", UOpts), + "[49,50|...]" = pretty("12345678"++[x], UOpts), + "\"12345678\"..." = pretty("123456789"++[x], UOpts), + "[[...]|...]" = pretty(["1","2","3","4","5","6","7","8"], UOpts), + ok. diff --git a/lib/stdlib/test/stdlib.spec b/lib/stdlib/test/stdlib.spec index 4de7c1a0eb..bf64eae2c7 100644 --- a/lib/stdlib/test/stdlib.spec +++ b/lib/stdlib/test/stdlib.spec @@ -2,3 +2,6 @@ {skip_groups,"../stdlib_test",stdlib_bench_SUITE, [binary,base64,gen_server,gen_statem,unicode], "Benchmark only"}. +{skip_groups,"../stdlib_test",ets_SUITE, + [benchmark], + "Benchmark only"}. diff --git a/lib/stdlib/test/stdlib_bench.spec b/lib/stdlib/test/stdlib_bench.spec index 7a0da811a0..6d665f22b6 100644 --- a/lib/stdlib/test/stdlib_bench.spec +++ b/lib/stdlib/test/stdlib_bench.spec @@ -8,3 +8,4 @@ {skip_groups,"../stdlib_test",stdlib_bench_SUITE, [gen_server_comparison,gen_statem_comparison], "Not a benchmark"}. +{groups,"../stdlib_test",ets_SUITE,[benchmark]}. diff --git a/lib/stdlib/test/string_SUITE.erl b/lib/stdlib/test/string_SUITE.erl index 251e09121c..6afe9e7a76 100644 --- a/lib/stdlib/test/string_SUITE.erl +++ b/lib/stdlib/test/string_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2018. All Rights Reserved. +%% Copyright Ericsson AB 2004-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -52,7 +52,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}, - {timetrap,{minutes,1}}]. + {timetrap,{minutes,2}}]. all() -> [{group, chardata}, {group, list_string}]. @@ -737,10 +737,10 @@ meas(Config) -> case ct:get_timetrap_info() of {_,{_,Scale}} when Scale > 1 -> {skip,{will_not_run_in_debug,Scale}}; - _ -> % No scaling, run at most 1.5 min + _ -> % No scaling, run at most 2 mins Tester = spawn(Exec), receive {test_done, Tester} -> ok - after 90000 -> + after 120000 -> io:format("Timelimit reached stopping~n",[]), exit(Tester, die) end, @@ -754,19 +754,22 @@ do_measure(DataDir) -> io:format("~p~n",[byte_size(Bin)]), Do = fun(Name, Func, Mode) -> {N, Mean, Stddev, _} = time_func(Func, Mode, Bin, 20), - io:format("~15w ~6w ~6.2fms ±~5.2fms #~.2w gc included~n", + io:format("~15w ~15w ~8.2fms ±~6.2fms #~.2w gc included~n", [Name, Mode, Mean/1000, Stddev/1000, N]) end, Do2 = fun(Name, Func, Mode) -> {N, Mean, Stddev, _} = time_func(Func, binary, <<>>, 20), - io:format("~15w ~6w ~6.2fms ±~5.2fms #~.2w gc included~n", + io:format("~15w ~15w ~8.2fms ±~6.2fms #~.2w gc included~n", [Name, Mode, Mean/1000, Stddev/1000, N]) end, + %% lefty_list means a list balanced to the left, like + %% [[[30],31],32]. Only some functions check such lists. + Modes = [list, lefty_list, binary, {many_lists,1}, {many_lists, 4}], io:format("----------------------~n"), Do(old_tokens, fun(Str) -> string:tokens(Str, [$\n,$\r]) end, list), Tokens = {lexemes, fun(Str) -> string:lexemes(Str, [$\n,$\r]) end}, - [Do(Name,Fun,Mode) || {Name,Fun} <- [Tokens], Mode <- [list, binary]], + [Do(Name,Fun,Mode) || {Name,Fun} <- [Tokens], Mode <- Modes], S0 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxy.....", S0B = <<"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxy.....">>, @@ -824,17 +827,17 @@ do_measure(DataDir) -> io:format("--~n",[]), NthTokens = {nth_lexemes, fun(Str) -> string:nth_lexeme(Str, 18000, [$\n,$\r]) end}, - [Do(Name,Fun,Mode) || {Name,Fun} <- [NthTokens], Mode <- [list, binary]], + [Do(Name,Fun,Mode) || {Name,Fun} <- [NthTokens], Mode <- Modes], Do2(take_t, repeat(fun() -> string:take(S0, [$.,$y], false, trailing) end), list), Do2(take_t, repeat(fun() -> string:take(S0B, [$.,$y], false, trailing) end), binary), Do2(take_tc, repeat(fun() -> string:take(S0, [$x], true, trailing) end), list), Do2(take_tc, repeat(fun() -> string:take(S0B, [$x], true, trailing) end), binary), Length = {length, fun(Str) -> string:length(Str) end}, - [Do(Name,Fun,Mode) || {Name,Fun} <- [Length], Mode <- [list, binary]], + [Do(Name,Fun,Mode) || {Name,Fun} <- [Length], Mode <- Modes], Reverse = {reverse, fun(Str) -> string:reverse(Str) end}, - [Do(Name,Fun,Mode) || {Name,Fun} <- [Reverse], Mode <- [list, binary]], + [Do(Name,Fun,Mode) || {Name,Fun} <- [Reverse], Mode <- Modes], ok. @@ -1064,7 +1067,33 @@ time_func(N,Sum,SumSq, _, _, Res, _) -> {N, Mean, Stdev, Res}. mode(binary, Bin) -> Bin; -mode(list, Bin) -> unicode:characters_to_list(Bin). +mode(list, Bin) -> unicode:characters_to_list(Bin); +mode(lefty_list, Bin) -> + L = unicode:characters_to_list(Bin), + to_left(L); +mode({many_lists, N}, Bin) -> + group(unicode:characters_to_list(Bin), N). + +group([], _N) -> + []; +group(L, N) -> + try lists:split(N, L) of + {L1, L2} -> + [L1 | group(L2, N)] + catch + _:_ -> + [L] + end. + +to_left([]) -> + []; +to_left([H|L]) -> + to_left([H], L). + +to_left(V, []) -> + V; +to_left(V, [H|L]) -> + to_left([V,H], L). %% %% Old string lists Test cases starts here. diff --git a/lib/stdlib/uc_spec/gen_unicode_mod.escript b/lib/stdlib/uc_spec/gen_unicode_mod.escript index 70eec1a6f2..8636c69a0d 100755..100644 --- a/lib/stdlib/uc_spec/gen_unicode_mod.escript +++ b/lib/stdlib/uc_spec/gen_unicode_mod.escript @@ -4,7 +4,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2017. All Rights Reserved. +%% Copyright Ericsson AB 2017-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -460,17 +460,73 @@ gen_cp(Fd) -> " maybe_improper_list() | {error, unicode:chardata()}.\n"), io:put_chars(Fd, "cp([C|_]=L) when is_integer(C) -> L;\n"), io:put_chars(Fd, "cp([List]) -> cp(List);\n"), - io:put_chars(Fd, "cp([List|R]) ->\n"), - io:put_chars(Fd, " case cp(List) of\n"), - io:put_chars(Fd, " [] -> cp(R);\n"), - io:put_chars(Fd, " [CP] -> [CP|R];\n"), - io:put_chars(Fd, " [C|R0] -> [C|[R0|R]];\n"), - io:put_chars(Fd, " {error,Error} -> {error,[Error|R]}\n"), - io:put_chars(Fd, " end;\n"), + io:put_chars(Fd, "cp([List|R]) -> cpl(List, R);\n"), io:put_chars(Fd, "cp([]) -> [];\n"), io:put_chars(Fd, "cp(<<C/utf8, R/binary>>) -> [C|R];\n"), io:put_chars(Fd, "cp(<<>>) -> [];\n"), - io:put_chars(Fd, "cp(<<R/binary>>) -> {error,R}.\n\n"), + io:put_chars(Fd, "cp(<<R/binary>>) -> {error,R}.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl([C], R) when is_integer(C) -> [C|cpl_1_cont(R)];\n"), + io:put_chars(Fd, "cpl([C|T], R) when is_integer(C) -> [C|cpl_cont(T, R)];\n"), + io:put_chars(Fd, "cpl([List], R) -> cpl(List, R);\n"), + io:put_chars(Fd, "cpl([List|T], R) -> cpl(List, [T|R]);\n"), + io:put_chars(Fd, "cpl([], R) -> cp(R);\n"), + io:put_chars(Fd, "cpl(<<C/utf8, T/binary>>, R) -> [C,T|R];\n"), + io:put_chars(Fd, "cpl(<<>>, R) -> cp(R);\n"), + io:put_chars(Fd, "cpl(<<B/binary>>, R) -> {error,[B|R]}.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "%%%\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_cont([C|T], R) when is_integer(C) -> [C|cpl_cont2(T, R)];\n"), + io:put_chars(Fd, "cpl_cont([L], R) -> cpl_cont(L, R);\n"), + io:put_chars(Fd, "cpl_cont([L|T], R) -> cpl_cont(L, [T|R]);\n"), + io:put_chars(Fd, "cpl_cont([], R) -> cpl_1_cont(R);\n"), + io:put_chars(Fd, "cpl_cont(T, R) -> [T|R].\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_cont2([C|T], R) when is_integer(C) -> [C|cpl_cont3(T, R)];\n"), + io:put_chars(Fd, "cpl_cont2([L], R) -> cpl_cont2(L, R);\n"), + io:put_chars(Fd, "cpl_cont2([L|T], R) -> cpl_cont2(L, [T|R]);\n"), + io:put_chars(Fd, "cpl_cont2([], R) -> cpl_1_cont2(R);\n"), + io:put_chars(Fd, "cpl_cont2(T, R) -> [T|R].\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_cont3([C], R) when is_integer(C) -> [C|R];\n"), + io:put_chars(Fd, "cpl_cont3([C|T], R) when is_integer(C) -> [C,T|R];\n"), + io:put_chars(Fd, "cpl_cont3([L], R) -> cpl_cont3(L, R);\n"), + io:put_chars(Fd, "cpl_cont3([L|T], R) -> cpl_cont3(L, [T|R]);\n"), + io:put_chars(Fd, "cpl_cont3([], R) -> cpl_1_cont3(R);\n"), + io:put_chars(Fd, "cpl_cont3(T, R) -> [T|R].\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "%%%\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_1_cont([C|T]) when is_integer(C) -> [C|cpl_1_cont2(T)];\n"), + io:put_chars(Fd, "cpl_1_cont([L]) -> cpl_1_cont(L);\n"), + io:put_chars(Fd, "cpl_1_cont([L|T]) -> cpl_cont(L, T);\n"), + io:put_chars(Fd, "cpl_1_cont(T) -> T.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_1_cont2([C|T]) when is_integer(C) -> [C|cpl_1_cont3(T)];\n"), + io:put_chars(Fd, "cpl_1_cont2([L]) -> cpl_1_cont2(L);\n"), + io:put_chars(Fd, "cpl_1_cont2([L|T]) -> cpl_cont2(L, T);\n"), + io:put_chars(Fd, "cpl_1_cont2(T) -> T.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_1_cont3([C|_]=T) when is_integer(C) -> T;\n"), + io:put_chars(Fd, "cpl_1_cont3([L]) -> cpl_1_cont3(L);\n"), + io:put_chars(Fd, "cpl_1_cont3([L|T]) -> cpl_cont3(L, T);\n"), + io:put_chars(Fd, "cpl_1_cont3(T) -> T.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "%%%\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cp_no_bin([C|_]=L) when is_integer(C) -> L;\n"), + io:put_chars(Fd, "cp_no_bin([List]) -> cp_no_bin(List);\n"), + io:put_chars(Fd, "cp_no_bin([List|R]) -> cp_no_binl(List, R);\n"), + io:put_chars(Fd, "cp_no_bin([]) -> [];\n"), + io:put_chars(Fd, "cp_no_bin(_) -> binary_found.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cp_no_binl([C], R) when is_integer(C) -> [C|cpl_1_cont(R)];\n"), + io:put_chars(Fd, "cp_no_binl([C|T], R) when is_integer(C) -> [C|cpl_cont(T, R)];\n"), + io:put_chars(Fd, "cp_no_binl([List], R) -> cp_no_binl(List, R);\n"), + io:put_chars(Fd, "cp_no_binl([List|T], R) -> cp_no_binl(List, [T|R]);\n"), + io:put_chars(Fd, "cp_no_binl([], R) -> cp_no_bin(R);\n"), + io:put_chars(Fd, "cp_no_binl(_, _) -> binary_found.\n\n"), ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -481,11 +537,26 @@ gen_gc(Fd, GBP) -> "-spec gc(String::unicode:chardata()) ->" " maybe_improper_list() | {error, unicode:chardata()}.\n"), io:put_chars(Fd, - "gc([CP1, CP2|_]=T)\n" - " when CP1 < 256, CP2 < 256, CP1 =/= $\r -> %% Ascii Fast path\n" - " T;\n" + "gc([]=R) -> R;\n" + "gc([CP]=R) when is_integer(CP) -> R;\n" + "gc([$\\r=CP|R0]) ->\n" + " case cp(R0) of % Don't break CRLF\n" + " [$\\n|R1] -> [[$\\r,$\\n]|R1];\n" + " T -> [CP|T]\n" + " end;\n" + "gc([CP1|T1]=T) when CP1 < 256 ->\n" + " case T1 of\n" + " [CP2|_] when CP2 < 256 -> T; %% Ascii Fast path\n" + " _ -> %% Keep the tail binary.\n" + " case cp_no_bin(T1) of\n" + " [CP2|_]=T3 when CP2 < 256 -> [CP1|T3]; %% Asciii Fast path\n" + " binary_found -> gc_1(T);\n" + " T4 -> gc_1([CP1|T4])\n" + " end\n" + " end;\n" + "gc(<<>>) -> [];\n" "gc(<<CP1/utf8, Rest/binary>>) ->\n" - " if CP1 < 256, CP1 =/= $\r ->\n" + " if CP1 < 256, CP1 =/= $\\r ->\n" " case Rest of\n" " <<CP2/utf8, _/binary>> when CP2 < 256 -> %% Ascii Fast path\n" " [CP1|Rest];\n" @@ -493,13 +564,12 @@ gen_gc(Fd, GBP) -> " end;\n" " true -> gc_1([CP1|Rest])\n" " end;\n" + "gc([CP|_]=T) when is_integer(CP) -> gc_1(T);\n" "gc(Str) ->\n" - " gc_1(cp(Str)).\n\n" - "gc_1([$\\r|R0] = R) ->\n" - " case cp(R0) of % Don't break CRLF\n" - " [$\\n|R1] -> [[$\\r,$\\n]|R1];\n" - " _ -> R\n" - " end;\n" + " case cp(Str) of\n" + " {error,_}=Error -> Error;\n" + " CPs -> gc(CPs)\n" + " end.\n" ), GenExtP = fun(Range) -> io:format(Fd, "gc_1~s gc_ext_pict(R1,[CP]);\n", [gen_clause(Range)]) end, @@ -507,7 +577,12 @@ gen_gc(Fd, GBP) -> %% Pick codepoints below 256 (some data knowledge here) {ExtendedPictographicLow,ExtendedPictographicHigh} = lists:splitwith(fun({Start,undefined}) -> Start < 256 end,ExtendedPictographic0), - + io:put_chars(Fd, + "\ngc_1([$\\r|R0] = R) ->\n" + " case cp(R0) of % Don't break CRLF\n" + " [$\\n|R1] -> [[$\\r,$\\n]|R1];\n" + " _ -> R\n" + " end;\n"), io:put_chars(Fd, "\n%% Handle control\n"), GenControl = fun(Range) -> io:format(Fd, "gc_1~s R0;\n", [gen_clause(Range)]) end, CRs0 = merge_ranges(maps:get(cr, GBP) ++ maps:get(lf, GBP) ++ maps:get(control, GBP), false), @@ -516,7 +591,14 @@ gen_gc(Fd, GBP) -> %%GenControl(R1),GenControl(R2),GenControl(R3), io:put_chars(Fd, "\n%% Optimize Latin-1\n"), [GenExtP(CP) || CP <- merge_ranges(ExtendedPictographicLow)], - io:format(Fd, "gc_1([CP|R]) when CP < 256 -> gc_extend(R,CP);\n\n", []), + + io:format(Fd, + "gc_1([CP|R]=R0) when CP < 256 ->\n" + " case R of\n" + " [CP2|_] when CP2 < 256 -> R0;\n" + " _ -> gc_extend(cp(R), R, CP)\n" + " end;\n", + []), io:put_chars(Fd, "\n%% Continue control\n"), [GenControl(CP) || CP <- Crs], %% One clause per CP @@ -540,7 +622,7 @@ gen_gc(Fd, GBP) -> io:put_chars(Fd, "gc_1([CP|_]=R0) when 44000 < CP, CP < 56000 -> gc_h_lv_lvt(R0, []);\n"), io:put_chars(Fd, "\n%% Handle Regional\n"), - GenRegional = fun(Range) -> io:format(Fd, "gc_1~s gc_regional(R1,[CP]);\n", [gen_clause(Range)]) end, + GenRegional = fun(Range) -> io:format(Fd, "gc_1~s gc_regional(R1,CP);\n", [gen_clause(Range)]) end, [GenRegional(CP) || CP <- merge_ranges(maps:get(regional_indicator,GBP))], %% io:put_chars(Fd, "%% Handle E_Base\n"), %% GenEBase = fun(Range) -> io:format(Fd, "gc_1~s gc_e_cont(R1,[CP]);\n", [gen_clause(Range)]) end, @@ -552,9 +634,7 @@ gen_gc(Fd, GBP) -> io:put_chars(Fd, "%% Handle extended_pictographic\n"), [GenExtP(CP) || CP <- merge_ranges(ExtendedPictographicHigh)], io:put_chars(Fd, "\n%% default clauses\n"), - io:put_chars(Fd, "gc_1([CP|R]) -> gc_extend(R, CP);\n"), - io:put_chars(Fd, "gc_1([]) -> [];\n"), - io:put_chars(Fd, "gc_1({error,_}=Error) -> Error.\n\n"), + io:put_chars(Fd, "gc_1([CP|R]) -> gc_extend(cp(R), R, CP).\n\n"), io:put_chars(Fd, "%% Handle Prepend\n"), io:put_chars(Fd, @@ -581,31 +661,24 @@ gen_gc(Fd, GBP) -> "%% To simplify binary handling in libraries the tail should be kept binary\n" "%% and not a lookahead CP\n" ), - io:put_chars(Fd, "gc_extend(T, Acc) ->\n" - " gc_extend(cp(T), T, Acc).\n\n"), io:put_chars(Fd, - "gc_extend([CP|T], T0, Acc0) ->\n" + "gc_extend([CP|T], T0, CP0) ->\n" " case is_extend(CP) of\n" - " false ->\n" - " case Acc0 of\n" - " [Acc] -> [Acc|T0];\n" - " [_|_]=Acc -> [lists:reverse(Acc)|T0];\n" - " Acc -> [Acc|T0]\n" - " end;\n" - " _TrueOrZWJ ->\n" - " case Acc0 of\n" - " [_|_] -> gc_extend(T, [CP|Acc0]);\n" - " Acc -> gc_extend(T, [CP,Acc])\n" - " end\n" + " false -> [CP0|T0]; % losing work done on T\n" + " _TrueOrZWJ -> gc_extend2(cp(T), T, [CP,CP0])\n" " end;\n" - "gc_extend([], _, Acc0) ->\n" - " case Acc0 of\n" - " [_]=Acc -> Acc;\n" - " [_|_]=Acc -> [lists:reverse(Acc)];\n" - " Acc -> [Acc]\n" + "gc_extend([], _, CP) -> [CP];\n" + "gc_extend({error,R}, _, CP) -> [CP|R].\n\n"), + io:put_chars(Fd, + "gc_extend2([CP|T], T0, Acc) ->\n" + " case is_extend(CP) of\n" + " false -> [lists:reverse(Acc)|T0]; % losing work done on T\n" + " _TrueOrZWJ -> gc_extend2(cp(T), T, [CP|Acc])\n" " end;\n" - "gc_extend({error,R}, T, Acc0) ->\n" - " gc_extend([], T, Acc0) ++ [R].\n\n" + "gc_extend2([], _, Acc) ->\n" + " [lists:reverse(Acc)];\n" + "gc_extend2({error,R}, _, Acc) ->\n" + " [lists:reverse(Acc)] ++ [R].\n\n" ), [ZWJ] = maps:get(zwj, GBP), GenExtend = fun(R) when R =:= ZWJ -> io:format(Fd, "is_extend~s zwj;\n", [gen_single_clause(ZWJ)]); @@ -660,10 +733,10 @@ gen_gc(Fd, GBP) -> %% -------------------- io:put_chars(Fd, "%% Handle Regional\n"), [{RLess,RLarge}] = merge_ranges(maps:get(regional_indicator,GBP)), - io:put_chars(Fd,"gc_regional(R0, Acc) ->\n" + io:put_chars(Fd,"gc_regional(R0, CP0) ->\n" " case cp(R0) of\n"), - io:format(Fd, " [CP|R1] when ~w =< CP,CP =< ~w-> gc_extend(R1,[CP|Acc]);~n",[RLess, RLarge]), - io:put_chars(Fd," R1 -> gc_extend(R1, R0, Acc)\n" + io:format(Fd, " [CP|R1] when ~w =< CP,CP =< ~w-> gc_extend2(cp(R1),R1,[CP,CP0]);~n",[RLess, RLarge]), + io:put_chars(Fd," R1 -> gc_extend(R1, R0, CP0)\n" " end.\n\n"), %% Special hangul @@ -685,16 +758,23 @@ gen_gc(Fd, GBP) -> GenHangulV_2 = fun(Range) -> io:format(Fd, "~8c~s gc_h_T(R1,[CP|Acc]);\n", [$\s,gen_case_clause(Range)]) end, [GenHangulV_2(CP) || CP <- merge_ranges(maps:get(t,GBP))], - io:put_chars(Fd, " R1 -> gc_extend(R1, R0, Acc)\n end.\n\n"), - + io:put_chars(Fd, + " R1 ->\n" + " case Acc of\n" + " [CP] -> gc_extend(R1, R0, CP);\n" + " _ -> gc_extend2(R1, R0, Acc)\n" + " end\n end.\n\n"), io:put_chars(Fd, "%% Handle Hangul T\n"), io:put_chars(Fd, "gc_h_T(R0, Acc) ->\n case cp(R0) of\n"), GenHangulT_1 = fun(Range) -> io:format(Fd, "~8c~s gc_h_T(R1,[CP|Acc]);\n", [$\s,gen_case_clause(Range)]) end, [GenHangulT_1(CP) || CP <- merge_ranges(maps:get(t,GBP))], - io:put_chars(Fd, " R1 -> gc_extend(R1, R0, Acc)\n end.\n\n"), - - io:put_chars(Fd, "gc_h_lv_lvt({error,_}=Error, Acc) -> gc_extend(Error, [], Acc);\n"), + io:put_chars(Fd, + " R1 ->\n" + " case Acc of\n" + " [CP] -> gc_extend(R1, R0, CP);\n" + " _ -> gc_extend2(R1, R0, Acc)\n" + " end\n end.\n\n"), io:put_chars(Fd, "%% Handle Hangul LV\n"), GenHangulLV = fun(Range) -> io:format(Fd, "gc_h_lv_lvt~s gc_h_V(R1,[CP|Acc]);\n", [gen_clause2(Range)]) end, @@ -703,8 +783,10 @@ gen_gc(Fd, GBP) -> GenHangulLVT = fun(Range) -> io:format(Fd, "gc_h_lv_lvt~s gc_h_T(R1,[CP|Acc]);\n", [gen_clause2(Range)]) end, [GenHangulLVT(CP) || CP <- merge_ranges(maps:get(lvt,GBP))], - io:put_chars(Fd, "gc_h_lv_lvt([CP|R], []) -> gc_extend(R, CP);\n"), %% From gc_1/1 - io:put_chars(Fd, "gc_h_lv_lvt(R, Acc) -> gc_extend(R, Acc).\n\n"), + io:put_chars(Fd, "gc_h_lv_lvt([CP|R], []) -> gc_extend(cp(R), R, CP);\n"), %% From gc_1/1 + io:put_chars(Fd, "%% Also handles error tuples\n"), + io:put_chars(Fd, "gc_h_lv_lvt(R, [CP]) -> gc_extend(R, R, CP);\n"), + io:put_chars(Fd, "gc_h_lv_lvt(R, Acc) -> gc_extend2(R, R, Acc).\n\n"), ok. gen_compose_pairs(Fd, ExclData, Data) -> diff --git a/lib/stdlib/vsn.mk b/lib/stdlib/vsn.mk index d46173497b..cbefd6590a 100644 --- a/lib/stdlib/vsn.mk +++ b/lib/stdlib/vsn.mk @@ -1 +1 @@ -STDLIB_VSN = 3.7.1 +STDLIB_VSN = 3.8 diff --git a/lib/syntax_tools/doc/src/notes.xml b/lib/syntax_tools/doc/src/notes.xml index dc13fe474b..772f5e6e04 100644 --- a/lib/syntax_tools/doc/src/notes.xml +++ b/lib/syntax_tools/doc/src/notes.xml @@ -32,6 +32,20 @@ <p>This document describes the changes made to the Syntax_Tools application.</p> +<section><title>Syntax_Tools 2.1.7</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Fix pretty-printing of type funs. </p> + <p> + Own Id: OTP-15519 Aux Id: ERL-815 </p> + </item> + </list> + </section> + +</section> + <section><title>Syntax_Tools 2.1.6</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/syntax_tools/vsn.mk b/lib/syntax_tools/vsn.mk index 8959ebbd04..538c71dc24 100644 --- a/lib/syntax_tools/vsn.mk +++ b/lib/syntax_tools/vsn.mk @@ -1 +1 @@ -SYNTAX_TOOLS_VSN = 2.1.6 +SYNTAX_TOOLS_VSN = 2.1.7 diff --git a/lib/tools/c_src/Makefile.in b/lib/tools/c_src/Makefile.in index cfe91917f8..289322b6fa 100644 --- a/lib/tools/c_src/Makefile.in +++ b/lib/tools/c_src/Makefile.in @@ -29,7 +29,6 @@ CC=@CC@ LD=@LD@ AR=@AR@ RANLIB=@RANLIB@ -RM=@RM@ MKDIR=@MKDIR@ INSTALL=@INSTALL@ INSTALL_DIR=@INSTALL_DIR@ @@ -158,9 +157,9 @@ $(ERTS_LIB): docs: clean: - $(RM) -rf ../obj/* - $(RM) -rf ../bin/* - $(RM) -f ./*~ + $(RM) -r ../obj/* + $(RM) -r ../bin/* + $(RM) ./*~ .PHONY: all erts_lib docs clean diff --git a/lib/tools/doc/src/notes.xml b/lib/tools/doc/src/notes.xml index a6781dfdb3..28f8346a19 100644 --- a/lib/tools/doc/src/notes.xml +++ b/lib/tools/doc/src/notes.xml @@ -31,6 +31,45 @@ </header> <p>This document describes the changes made to the Tools application.</p> +<section><title>Tools 3.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Minor fixes for <c>make clean</c>.</p> + <p> + Own Id: OTP-15657</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + In the HTML file generated by + <c>cover:analyse_to_file/1,2</c>, a link is now added to + the line number. This makes it easier to share pointers + to specific lines.</p> + <p> + Own Id: OTP-15541</p> + </item> + <item> + <p> + Uncovered lines are now marked with a sad face, + <c>:-(</c>, in the HTML output from + <c>cover:analyse_to_file/1,2</c>. This is to make these + lines easier to find by search.</p> + <p> + Own Id: OTP-15542</p> + </item> + </list> + </section> + +</section> + <section><title>Tools 3.0.2</title> <section><title>Improvements and New Features</title> diff --git a/lib/tools/vsn.mk b/lib/tools/vsn.mk index bb8305e9f1..5700885549 100644 --- a/lib/tools/vsn.mk +++ b/lib/tools/vsn.mk @@ -1 +1 @@ -TOOLS_VSN = 3.0.2 +TOOLS_VSN = 3.1 diff --git a/lib/wx/doc/src/notes.xml b/lib/wx/doc/src/notes.xml index 1061e73138..33d02e22ba 100644 --- a/lib/wx/doc/src/notes.xml +++ b/lib/wx/doc/src/notes.xml @@ -32,6 +32,25 @@ <p>This document describes the changes made to the wxErlang application.</p> +<section><title>Wx 1.8.7</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Improved support for wxWidgets 3.1.3 which have changed + <c>wxFONTWEIGTH</c>, also added <c>wxGCDC</c> and + <c>wxDisplay</c> modules.</p> + <p> + Fixed a crash on Mojave and check for events more often.</p> + <p> + Own Id: OTP-15587</p> + </item> + </list> + </section> + +</section> + <section><title>Wx 1.8.6</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/wx/vsn.mk b/lib/wx/vsn.mk index d241a7a1b4..dac219fa98 100644 --- a/lib/wx/vsn.mk +++ b/lib/wx/vsn.mk @@ -1 +1 @@ -WX_VSN = 1.8.6 +WX_VSN = 1.8.7 |