diff options
Diffstat (limited to 'lib')
36 files changed, 967 insertions, 417 deletions
diff --git a/lib/compiler/src/sys_core_fold.erl b/lib/compiler/src/sys_core_fold.erl index 08b02101a6..4922953407 100644 --- a/lib/compiler/src/sys_core_fold.erl +++ b/lib/compiler/src/sys_core_fold.erl @@ -734,7 +734,7 @@ call(#c_call{args=As}=Call, #c_literal{val=M}=M0, #c_literal{val=N}=N0, Sub) -> false -> case sys_core_fold_lists:call(Call, M, N, As) of none -> - call_1(Call, M, N, As, Sub); + call_1(Call, M0, N0, As, Sub); Core -> expr(Core, Sub) end @@ -1130,7 +1130,13 @@ clause_1(#c_clause{guard=G0,body=B0}=Cl, Ps1, Cexpr, Ctxt, Sub1) -> %% %% case A of NewVar when true -> ... %% - sub_set_var(Var, Cexpr, Sub2); + case cerl:is_c_fname(Cexpr) of + false -> + sub_set_var(Var, Cexpr, Sub2); + true -> + %% We must not copy funs, and especially not into guards. + Sub2 + end; _ -> Sub2 end, diff --git a/lib/compiler/test/guard_SUITE.erl b/lib/compiler/test/guard_SUITE.erl index 6302f82f29..429d6b79e0 100644 --- a/lib/compiler/test/guard_SUITE.erl +++ b/lib/compiler/test/guard_SUITE.erl @@ -87,6 +87,7 @@ misc(Config) when is_list(Config) -> {ok,buf,<<>>} = get_data({o,true,0}, 42, buf), {ok,buf,<<>>} = get_data({o,false,0}, 0, buf), error = get_data({o,false,0}, 42, buf), + ok. @@ -343,6 +344,11 @@ complex_semicolon(Config) when is_list(Config) -> ok = csemi7(#{a=>1}, 3, 3), ok = csemi7(#{a=>1, b=>3}, 0, 0), + %% 8: Make sure that funs cannot be copied into guards. + ok = csemi8(true), + error = csemi8(false), + error = csemi8(42), + ok. csemi1(Type, Val) when is_list(Val), Type == float; @@ -457,6 +463,13 @@ csemi6(_, _) -> error. csemi7(A, B, C) when A#{a:=B} > #{a=>1}; abs(C) > 2 -> ok; csemi7(_, _, _) -> error. +csemi8(Together) -> + case fun csemi8/1 of + Typically when Together; Typically, Together -> ok; + _ -> error + end. + + comma(Config) when is_list(Config) -> %% ',' combinations of literal true/false. diff --git a/lib/dialyzer/test/small_SUITE_data/src/ms.erl b/lib/dialyzer/test/small_SUITE_data/src/ms.erl new file mode 100644 index 0000000000..47a5e886cf --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/ms.erl @@ -0,0 +1,8 @@ +-module(ms). +-export([t/0]). + +-include_lib("stdlib/include/ms_transform.hrl"). + +t() -> + MS = dbg:fun2ms(fun(All) -> message(All) end), + erlang:trace_pattern({m, f, '_'}, MS). diff --git a/lib/eldap/doc/src/eldap.xml b/lib/eldap/doc/src/eldap.xml index 43873e44e2..f2c7889e58 100644 --- a/lib/eldap/doc/src/eldap.xml +++ b/lib/eldap/doc/src/eldap.xml @@ -197,7 +197,7 @@ </type> <desc> <p> Add an entry. The entry must not exist.</p> - <pre> + <code> add(Handle, "cn=Bill Valentine, ou=people, o=Example Org, dc=example, dc=com", [{"objectclass", ["person"]}, @@ -205,7 +205,7 @@ {"sn", ["Valentine"]}, {"telephoneNumber", ["545 555 00"]}] ) - </pre> + </code> </desc> </func> <func> @@ -216,9 +216,9 @@ </type> <desc> <p> Delete an entry.</p> - <pre> + <code> delete(Handle, "cn=Bill Valentine, ou=people, o=Example Org, dc=example, dc=com") - </pre> + </code> </desc> </func> @@ -259,11 +259,11 @@ </type> <desc> <p> Modify an entry.</p> - <pre> + <code> modify(Handle, "cn=Bill Valentine, ou=people, o=Example Org, dc=example, dc=com", [eldap:mod_replace("telephoneNumber", ["555 555 00"]), eldap:mod_add("description", ["LDAP Hacker"]) ]) - </pre> + </code> </desc> </func> <func> @@ -320,10 +320,10 @@ whether the current RDN should be removed from the attribute list after the after operation. <c>NewSupDN</c> is the new parent that the RDN shall be moved to. If the old parent should remain as parent, <c>NewSupDN</c> shall be "".</p> - <pre> + <code> modify_dn(Handle, "cn=Bill Valentine, ou=people, o=Example Org, dc=example, dc=com ", "cn=Bill Jr Valentine", true, "") - </pre> + </code> </desc> </func> <func> @@ -342,10 +342,10 @@ Default values: scope is <c>wholeSubtree()</c>, deref is <c>derefAlways()</c>, types_only is <c>false</c> and timeout is <c>0</c> (meaning infinity). </p> - <pre> + <code> Filter = eldap:substrings("cn", [{any,"V"}]), search(Handle, [{base, "dc=example, dc=com"}, {filter, Filter}, {attributes, ["cn"]}]), - </pre> + </code> <p>The <c>timeout</c> option in the <c>SearchOptions</c> is for the ldap server, while the timeout in <seealso marker="#open/2">eldap:open/2</seealso> is used for each individual request in the search operation. @@ -454,7 +454,7 @@ </type> <desc> <p>Creates an extensible match filter. For example, </p> <code> - eldap:extensibleMatch("Bar", [{type,"sn"}, {matchingRule,"caseExactMatch"}])) + eldap:extensibleMatch("Bar", [{type,"sn"}, {matchingRule,"caseExactMatch"}])) </code> <p>creates a filter which performs a <c>caseExactMatch</c> on the attribute <c>sn</c> and matches with the value <c>"Bar"</c>. The default value of <c>dnAttributes</c> is <c>false</c>.</p> </desc> </func> diff --git a/lib/hipe/icode/hipe_beam_to_icode.erl b/lib/hipe/icode/hipe_beam_to_icode.erl index 224aacd8d7..3386523206 100644 --- a/lib/hipe/icode/hipe_beam_to_icode.erl +++ b/lib/hipe/icode/hipe_beam_to_icode.erl @@ -763,32 +763,10 @@ trans_fun([{test,bs_test_unit,{f,Lbl},[Ms,Unit]}| [MsVar], [], Env, Instructions); trans_fun([{test,bs_match_string,{f,Lbl},[Ms,BitSize,Bin]}| Instructions], Env) -> - True = mk_label(new), - FalseLabName = map_label(Lbl), - TrueLabName = hipe_icode:label_name(True), + %% the current match buffer MsVar = mk_var(Ms), - TmpVar = mk_var(new), - ByteSize = BitSize div 8, - ExtraBits = BitSize rem 8, - WordSize = hipe_rtl_arch:word_size(), - if ExtraBits =:= 0 -> - trans_op_call({hipe_bs_primop,{bs_match_string,Bin,ByteSize}}, Lbl, - [MsVar], [MsVar], Env, Instructions); - BitSize =< ((WordSize * 8) - 5) -> - <<Int:BitSize, _/bits>> = Bin, - {I1,Env1} = trans_one_op_call({hipe_bs_primop,{bs_get_integer,BitSize,0}}, Lbl, - [MsVar], [TmpVar, MsVar], Env), - I2 = hipe_icode:mk_type([TmpVar], {integer,Int}, TrueLabName, FalseLabName), - I1 ++ [I2,True] ++ trans_fun(Instructions, Env1); - true -> - <<RealBin:ByteSize/binary, Int:ExtraBits, _/bits>> = Bin, - {I1,Env1} = trans_one_op_call({hipe_bs_primop,{bs_match_string,RealBin,ByteSize}}, Lbl, - [MsVar], [MsVar], Env), - {I2,Env2} = trans_one_op_call({hipe_bs_primop,{bs_get_integer,ExtraBits,0}}, Lbl, - [MsVar], [TmpVar, MsVar], Env1), - I3 = hipe_icode:mk_type([TmpVar], {integer,Int}, TrueLabName, FalseLabName), - I1 ++ I2 ++ [I3,True] ++ trans_fun(Instructions, Env2) - end; + Primop = {hipe_bs_primop, {bs_match_string, Bin, BitSize}}, + trans_op_call(Primop, Lbl, [MsVar], [MsVar], Env, Instructions); trans_fun([{bs_context_to_binary,Var}|Instructions], Env) -> %% the current match buffer IVars = [trans_arg(Var)], diff --git a/lib/hipe/icode/hipe_icode_primops.erl b/lib/hipe/icode/hipe_icode_primops.erl index cee37b6a57..2a141c514e 100644 --- a/lib/hipe/icode/hipe_icode_primops.erl +++ b/lib/hipe/icode/hipe_icode_primops.erl @@ -287,8 +287,8 @@ pp(Dev, Op) -> io:format(Dev, "bs_start_match<~w>", [Max]); {{bs_start_match, Type}, Max} -> io:format(Dev, "bs_start_match<~w,~w>", [Type,Max]); - {bs_match_string, String, SizeInBytes} -> - io:format(Dev, "bs_match_string<~w, ~w>", [String, SizeInBytes]); + {bs_match_string, String, SizeInBits} -> + io:format(Dev, "bs_match_string<~w, ~w>", [String, SizeInBits]); {bs_get_integer, Size, Flags} -> io:format(Dev, "bs_get_integer<~w, ~w>", [Size, Flags]); {bs_get_float, Size, Flags} -> @@ -596,10 +596,10 @@ type(Primop, Args) -> erl_types:t_subtract(Type, erl_types:t_matchstate()), erl_types:t_matchstate_slot( erl_types:t_inf(Type, erl_types:t_matchstate()), 0)); - {hipe_bs_primop, {bs_match_string,_,Bytes}} -> + {hipe_bs_primop, {bs_match_string,_,Bits}} -> [MatchState] = Args, BinType = erl_types:t_matchstate_present(MatchState), - NewBinType = match_bin(erl_types:t_bitstr(0, Bytes*8), BinType), + NewBinType = match_bin(erl_types:t_bitstr(0, Bits), BinType), erl_types:t_matchstate_update_present(NewBinType, MatchState); {hipe_bs_primop, {bs_test_unit,Unit}} -> [MatchState] = Args, diff --git a/lib/hipe/rtl/hipe_rtl_binary.erl b/lib/hipe/rtl/hipe_rtl_binary.erl index fb9c0c196d..9b400f4c93 100644 --- a/lib/hipe/rtl/hipe_rtl_binary.erl +++ b/lib/hipe/rtl/hipe_rtl_binary.erl @@ -19,7 +19,7 @@ %%% %CopyrightEnd% %%% %%%------------------------------------------------------------------- -%%% File : hipe_rtl_binary_2.erl +%%% File : hipe_rtl_binary.erl %%% Author : Per Gustafsson <[email protected]> %%% Description : %%% diff --git a/lib/hipe/rtl/hipe_rtl_binary_match.erl b/lib/hipe/rtl/hipe_rtl_binary_match.erl index 528672b893..d999cd2743 100644 --- a/lib/hipe/rtl/hipe_rtl_binary_match.erl +++ b/lib/hipe/rtl/hipe_rtl_binary_match.erl @@ -270,24 +270,23 @@ gen_rtl({bs_save, Slot}, [NewMs], [Ms], TrueLblName, _FalseLblName) -> set_field_from_term({matchstate, {saveoffset, Slot}}, Ms, Offset), hipe_rtl:mk_goto(TrueLblName)]; %% ----- bs_match_string ----- -gen_rtl({bs_match_string, String, ByteSize}, Dst, [Ms], +gen_rtl({bs_match_string, String, BitSize}, Dst, [Ms], TrueLblName, FalseLblName) -> {[Offset, BinSize, Base], Instrs} = extract_matchstate_vars([offset, binsize, base], Ms), [SuccessLbl, ALbl, ULbl] = create_lbls(3), [NewOffset, BitOffset] = create_gcsafe_regs(2), - Unit = hipe_rtl_arch:word_size() - 1, - Loops = ByteSize div Unit, - Init = + Unit = (hipe_rtl_arch:word_size() - 1) * ?BYTE_SIZE, + Init = [Instrs, opt_update_ms(Dst, Ms), - check_size(Offset, hipe_rtl:mk_imm(ByteSize*?BYTE_SIZE), BinSize, + check_size(Offset, hipe_rtl:mk_imm(BitSize), BinSize, NewOffset, hipe_rtl:label_name(SuccessLbl), FalseLblName), SuccessLbl], SplitCode = [hipe_rtl:mk_alub(BitOffset, Offset, 'and', hipe_rtl:mk_imm(?LOW_BITS), eq, hipe_rtl:label_name(ALbl), hipe_rtl:label_name(ULbl))], - Loops = ByteSize div Unit, + Loops = BitSize div Unit, SkipSize = Loops * Unit, {ACode1, UCode1} = case Loops of @@ -297,9 +296,9 @@ gen_rtl({bs_match_string, String, ByteSize}, Dst, [Ms], create_loops(Loops, Unit, String, Base, Offset, BitOffset, FalseLblName) end, - <<_:SkipSize/binary, RestString/binary>> = String, + <<_:SkipSize/bits, RestString/bits>> = String, {ACode2, UCode2} = - case ByteSize rem Unit of + case BitSize rem Unit of 0 -> {[], []}; Rem -> @@ -393,12 +392,12 @@ validate_unicode_retract_c_code(Src, Ms, TrueLblName, FalseLblName) -> create_loops(Loops, Unit, String, Base, Offset, BitOffset, FalseLblName) -> [Reg] = create_gcsafe_regs(1), AlignedFun = fun(Value) -> - [get_int_to_reg(Reg, Unit*?BYTE_SIZE, Base, Offset, 'srl', + [get_int_to_reg(Reg, Unit, Base, Offset, 'srl', {unsigned, big}), update_and_test(Reg, Unit, Offset, Value, FalseLblName)] end, UnAlignedFun = fun(Value) -> - [get_unaligned_int_to_reg(Reg, Unit*?BYTE_SIZE, + [get_unaligned_int_to_reg(Reg, Unit, Base, Offset, BitOffset, 'srl', {unsigned, big})| update_and_test(Reg, Unit, Offset, Value, FalseLblName)] @@ -406,31 +405,31 @@ create_loops(Loops, Unit, String, Base, Offset, BitOffset, FalseLblName) -> {create_loops(Loops, Unit, String, AlignedFun), create_loops(Loops, Unit, String, UnAlignedFun)}. -create_rests(Rem, String, Base, Offset, BitOffset, FalseLblName) -> +create_rests(RemBits, String, Base, Offset, BitOffset, FalseLblName) -> [Reg] = create_gcsafe_regs(1), AlignedFun = fun(Value) -> - [get_int_to_reg(Reg, Rem*?BYTE_SIZE, Base, Offset, 'srl', + [get_int_to_reg(Reg, RemBits, Base, Offset, 'srl', {unsigned, big})| just_test(Reg, Value, FalseLblName)] end, UnAlignedFun = fun(Value) -> - [get_unaligned_int_to_reg(Reg, Rem*?BYTE_SIZE, + [get_unaligned_int_to_reg(Reg, RemBits, Base, Offset, BitOffset, 'srl', {unsigned, big})| just_test(Reg, Value, FalseLblName)] end, - {create_loops(1, Rem, String, AlignedFun), - create_loops(1, Rem, String, UnAlignedFun)}. + {create_loops(1, RemBits, String, AlignedFun), + create_loops(1, RemBits, String, UnAlignedFun)}. create_loops(0, _Unit, _String, _IntFun) -> []; create_loops(N, Unit, String, IntFun) -> - {Value, RestString} = get_value(Unit,String), + {Value, RestString} = get_value(Unit, String), [IntFun(Value), create_loops(N-1, Unit, RestString, IntFun)]. update_and_test(Reg, Unit, Offset, Value, FalseLblName) -> - [add_to_offset(Offset, Offset, hipe_rtl:mk_imm(Unit*?BYTE_SIZE), FalseLblName), + [add_to_offset(Offset, Offset, hipe_rtl:mk_imm(Unit), FalseLblName), just_test(Reg, Value, FalseLblName)]. just_test(Reg, Value, FalseLblName) -> @@ -439,8 +438,8 @@ just_test(Reg, Value, FalseLblName) -> hipe_rtl:label_name(ContLbl), FalseLblName), ContLbl]. -get_value(N,String) -> - <<I:N/integer-unit:8, Rest/binary>> = String, +get_value(N, String) -> + <<I:N, Rest/bits>> = String, {I, Rest}. make_int_gc_code(I) when is_integer(I) -> diff --git a/lib/hipe/test/bs_SUITE_data/bs_pmatch_bugs.erl b/lib/hipe/test/bs_SUITE_data/bs_pmatch_bugs.erl index b280705a47..d9f3278b45 100644 --- a/lib/hipe/test/bs_SUITE_data/bs_pmatch_bugs.erl +++ b/lib/hipe/test/bs_SUITE_data/bs_pmatch_bugs.erl @@ -9,6 +9,7 @@ test() -> <<49,50,51>> = lex_digits1(Bin, 1, []), <<49,50,51>> = lex_digits2(Bin, 1, []), ok = var_bind_bug(<<1, 2, 3, 4, 5, 6, 7, 8>>), + ok = bs_match_string_bug(), ok. %%-------------------------------------------------------------------- @@ -65,3 +66,50 @@ var_bind_bug(<<A:1/binary, B:8/integer, _C:B/binary, _Rest/binary>>) -> B -> wrong; _ -> ok end. + +%%-------------------------------------------------------------------- +%% From: Andreas Schultz +%% Date: 2/11/2016 +%% +%% Either HiPE is messing up binary matches in some cases or I'm not +%% seeing the problem. ... <SNIP PROGRAM - CLEANED UP VERSION BELOW> +%% With Erlang 19.1.3 the HiPE compiled version behaves differently +%% than the non-HiPE version: ... <SNIP TEST RUNS> +%% So, do I do something wrong here or is this a legitimate HiPE bug? +%% +%% Yes, this was a legitimate HiPE bug: The BEAM to ICode tranaslation +%% of the bs_match_string instruction, written long ago for binaries +%% (i.e., with byte-sized strings), tried to do a `clever' translation +%% of even bit-sized strings using a HiPE primop that took a `Size' +%% argument expressed in *bytes*. ICode is not really the place to do +%% such a thing, and moreover there is really no reason for the HiPE +%% primop not to take a Size argument expressed in *bits* instead. +%% The bug was fixed by changing the `Size' argument to be in bits, +%% postponing the translation of the bs_match_string primop until RTL +%% and doing a proper translation using bit-sized quantities there. +%%-------------------------------------------------------------------- + +bs_match_string_bug() -> + ok = test0(<<50>>), + Bin = data(), + ok = test1(Bin), + ok = test2(Bin), + ok. + +%% Minimal test case showing the problem matching with strings +test0(<<6:5, 0:1, 0:2>>) -> weird; +test0(<<6:5, _:1, _:2>>) -> ok; +test0(_) -> default. + +data() -> <<50,16,0>>. + +%% This was the problematic test case in HiPE: 'default' was returned +test1(<<1:3, 1:1, _:1, 0:1, 0:1, 0:1, _/binary>>) -> weird; +test1(<<1:3, 1:1, _:1, _:1, _:1, _:1, _/binary>>) -> ok; +test1(_) -> default. + +%% This variation of test1/1 above worked OK, even in HiPE +test2(<<1:3, 1:1, _:1, A:1, B:1, C:1, _/binary>>) + when A =:= 1; B =:= 1; C =:= 1 -> ok; +test2(<<1:3, 1:1, _:1, 0:1, 0:1, 0:1, _/binary>>) -> weird; +test2(_) -> default. diff --git a/lib/inets/src/ftp/ftp.erl b/lib/inets/src/ftp/ftp.erl index 6868b75eff..911f5b71a7 100644 --- a/lib/inets/src/ftp/ftp.erl +++ b/lib/inets/src/ftp/ftp.erl @@ -1096,7 +1096,7 @@ init(Options) -> erlang:monitor(process, Client), %% Make sure inet is started - inet_db:start(), + _ = inet_db:start(), %% Where are we {ok, Dir} = file:get_cwd(), @@ -1106,15 +1106,17 @@ init(Options) -> trace -> dbg:tracer(), dbg:p(all, [call]), - dbg:tpl(ftp, [{'_', [], [{return_trace}]}]), - dbg:tpl(ftp_response, [{'_', [], [{return_trace}]}]), - dbg:tpl(ftp_progress, [{'_', [], [{return_trace}]}]); + {ok, _} = dbg:tpl(ftp, [{'_', [], [{return_trace}]}]), + {ok, _} = dbg:tpl(ftp_response, [{'_', [], [{return_trace}]}]), + {ok, _} = dbg:tpl(ftp_progress, [{'_', [], [{return_trace}]}]), + ok; debug -> dbg:tracer(), dbg:p(all, [call]), - dbg:tp(ftp, [{'_', [], [{return_trace}]}]), - dbg:tp(ftp_response, [{'_', [], [{return_trace}]}]), - dbg:tp(ftp_progress, [{'_', [], [{return_trace}]}]); + {ok, _} = dbg:tp(ftp, [{'_', [], [{return_trace}]}]), + {ok, _} = dbg:tp(ftp_response, [{'_', [], [{return_trace}]}]), + {ok, _} = dbg:tp(ftp_progress, [{'_', [], [{return_trace}]}]), + ok; _ -> %% Keep silent ok @@ -1296,8 +1298,7 @@ handle_call({_,{rmdir, Dir}}, From, #state{chunk = false} = State) -> activate_ctrl_connection(State), {noreply, State#state{client = From}}; -handle_call({_,{type, Type}}, From, #state{chunk = false} - = State) -> +handle_call({_,{type, Type}}, From, #state{chunk = false} = State) -> case Type of ascii -> send_ctrl_message(State, mk_cmd("TYPE A", [])), @@ -1455,7 +1456,7 @@ handle_info({Trpt, Socket, Data}, #state{dsock = {Trpt,Socket}, caller = {recv_file, Fd}} = State0) when Trpt==tcp;Trpt==ssl -> ?DBG('L~p --data ~p ----> ~s~p~n',[?LINE,Socket,Data,State0]), - file_write(binary_to_list(Data), Fd), + ok = file_write(binary_to_list(Data), Fd), progress_report({binary, Data}, State0), State = activate_data_connection(State0), {noreply, State}; @@ -1474,16 +1475,19 @@ handle_info({Trpt, Socket, Data}, #state{dsock = {Trpt,Socket}} = State0) when T Data/binary>>}}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, - caller = {recv_file, Fd}} - = State) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> - file_close(Fd), + caller = {recv_file, Fd}} = State) + when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> + case file_close(Fd) of + ok -> ok; + {error,einval} -> ok + end, progress_report({transfer_size, 0}, State), activate_ctrl_connection(State), {noreply, State#state{dsock = undefined, data = <<>>}}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, client = From, - caller = recv_chunk} - = State) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> + caller = recv_chunk} = State) + when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> gen_server:reply(From, ok), {noreply, State#state{dsock = undefined, client = undefined, data = <<>>, caller = undefined, @@ -2062,7 +2066,10 @@ handle_ctrl_result({pos_prel, _}, #state{caller = {recv_file, _}} = State0) -> end; handle_ctrl_result({Status, _}, #state{caller = {recv_file, Fd}} = State) -> - file_close(Fd), + case file_close(Fd) of + ok -> ok; + {error, einval} -> ok + end, close_data_connection(State), ctrl_result_response(Status, State#state{dsock = undefined}, {error, epath}); @@ -2221,16 +2228,16 @@ setup_data_connection(#state{mode = active, {ok, Port} = inet:port(LSock), case FtpExt of false -> - {IP1, IP2, IP3, IP4} = IP, - {Port1, Port2} = {Port div 256, Port rem 256}, - send_ctrl_message(State, - mk_cmd("PORT ~w,~w,~w,~w,~w,~w", - [IP1, IP2, IP3, IP4, Port1, Port2])); - true -> - IpAddress = inet_parse:ntoa(IP), - Cmd = mk_cmd("EPRT |1|~s|~p|", [IpAddress, Port]), - send_ctrl_message(State, Cmd) - end, + {IP1, IP2, IP3, IP4} = IP, + {Port1, Port2} = {Port div 256, Port rem 256}, + send_ctrl_message(State, + mk_cmd("PORT ~w,~w,~w,~w,~w,~w", + [IP1, IP2, IP3, IP4, Port1, Port2])); + true -> + IpAddress = inet_parse:ntoa(IP), + Cmd = mk_cmd("EPRT |1|~s|~p|", [IpAddress, Port]), + send_ctrl_message(State, Cmd) + end, activate_ctrl_connection(State), {noreply, State#state{caller = {setup_data_connection, {LSock, Caller}}}} @@ -2338,7 +2345,7 @@ accept_data_connection(#state{mode = passive} = State) -> send_ctrl_message(_S=#state{csock = Socket, verbose = Verbose}, Message) -> verbose(lists:flatten(Message),Verbose,send), ?DBG('<--ctrl ~p ---- ~s~p~n',[Socket,Message,_S]), - send_message(Socket, Message). + ok = send_message(Socket, Message). send_data_message(_S=#state{dsock = Socket}, Message) -> ?DBG('<==data ~p ==== ~s~n~p~n',[Socket,Message,_S]), @@ -2360,26 +2367,26 @@ send_message({ssl, Socket}, Message) -> ssl:send(Socket, Message). activate_ctrl_connection(#state{csock = Socket, ctrl_data = {<<>>, _, _}}) -> - activate_connection(Socket); + ok = activate_connection(Socket); activate_ctrl_connection(#state{csock = Socket}) -> - activate_connection(Socket), + ok = activate_connection(Socket), %% We have already received at least part of the next control message, %% that has been saved in ctrl_data, process this first. - self() ! {socket_type(Socket), unwrap_socket(Socket), <<>>}. + self() ! {socket_type(Socket), unwrap_socket(Socket), <<>>}, + ok. unwrap_socket({tcp,Socket}) -> Socket; -unwrap_socket({ssl,Socket}) -> Socket; -unwrap_socket(Socket) -> Socket. +unwrap_socket({ssl,Socket}) -> Socket. socket_type({tcp,_Socket}) -> tcp; socket_type({ssl,_Socket}) -> ssl. activate_data_connection(#state{dsock = Socket} = State) -> - activate_connection(Socket), + ok = activate_connection(Socket), State. -activate_connection({tcp, Socket}) -> inet:setopts(Socket, [{active, once}]); -activate_connection({ssl, Socket}) -> ssl:setopts(Socket, [{active, once}]). +activate_connection({tcp, Socket}) -> inet:setopts(Socket, [{active, once}]); +activate_connection({ssl, Socket}) -> ssl:setopts(Socket, [{active, once}]). close_ctrl_connection(#state{csock = undefined}) -> ok; close_ctrl_connection(#state{csock = Socket}) -> close_connection(Socket). @@ -2387,21 +2394,21 @@ close_ctrl_connection(#state{csock = Socket}) -> close_connection(Socket). close_data_connection(#state{dsock = undefined}) -> ok; close_data_connection(#state{dsock = Socket}) -> close_connection(Socket). -close_connection({lsock,Socket}) -> gen_tcp:close(Socket); +close_connection({lsock,Socket}) -> gen_tcp:close(Socket); close_connection({tcp, Socket}) -> gen_tcp:close(Socket); close_connection({ssl, Socket}) -> ssl:close(Socket). -%% ------------ FILE HANDELING ---------------------------------------- +%% ------------ FILE HANDLING ---------------------------------------- send_file(#state{tls_upgrading_data_connection = {true, CTRL, _}} = State, Fd) -> {noreply, State#state{tls_upgrading_data_connection = {true, CTRL, ?MODULE, send_file, Fd}}}; send_file(State, Fd) -> case file_read(Fd) of - {ok, N, Bin} when N > 0-> + {ok, N, Bin} when N > 0 -> send_data_message(State, Bin), progress_report({binary, Bin}, State), send_file(State, Fd); {ok, _, _} -> - file_close(Fd), + ok = file_close(Fd), close_data_connection(State), progress_report({transfer_size, 0}, State), activate_ctrl_connection(State), @@ -2508,7 +2515,7 @@ progress_report(stop, #state{progress = ProgressPid}) -> ftp_progress:stop(ProgressPid); progress_report({binary, Data}, #state{progress = ProgressPid}) -> ftp_progress:report(ProgressPid, {transfer_size, size(Data)}); -progress_report(Report, #state{progress = ProgressPid}) -> +progress_report(Report, #state{progress = ProgressPid}) -> ftp_progress:report(ProgressPid, Report). diff --git a/lib/inets/src/ftp/ftp_progress.erl b/lib/inets/src/ftp/ftp_progress.erl index 68185a222d..a6263e5cd7 100644 --- a/lib/inets/src/ftp/ftp_progress.erl +++ b/lib/inets/src/ftp/ftp_progress.erl @@ -36,11 +36,11 @@ -include_lib("kernel/include/file.hrl"). -record(progress, { - file, % string() - cb_module, % atom() - cb_function, % atom() - init_progress_term, % term() - current_progress_term % term() + file :: string() | 'undefined', + cb_module :: module(), + cb_function :: atom(), + init_progress_term :: term(), + current_progress_term :: term() }). %%%========================================================================= @@ -53,13 +53,15 @@ %% Description: Starts the progress report process unless progress reporting %% should not be performed. %%-------------------------------------------------------------------------- +-type options() :: 'ignore' | {module(), atom(), term()}. +-spec start_link(options()) -> 'ignore' | pid(). start_link(ignore) -> ignore; start_link(Options) -> spawn_link(?MODULE, init, [Options]). %%-------------------------------------------------------------------------- -%% report_progress(Pid, Report) -> _ +%% report_progress(Pid, Report) -> ok %% Pid = pid() %% Report = {local_file, File} | {remote_file, File} | %% {transfer_size, Size} @@ -68,17 +70,23 @@ start_link(Options) -> %% Description: Reports progress to the reporting process that calls the %% user defined callback function. %%-------------------------------------------------------------------------- +-type report() :: {'local_file', string()} | {'remote_file', string()} + | {'transfer_size', non_neg_integer()}. +-spec report(pid(), report()) -> 'ok'. report(Pid, Report) -> - Pid ! {progress_report, Report}. + Pid ! {progress_report, Report}, + ok. %%-------------------------------------------------------------------------- -%% stop(Pid) -> _ +%% stop(Pid) -> ok %% Pid = pid() %% %% Description: %%-------------------------------------------------------------------------- +-spec stop(pid()) -> 'ok'. stop(Pid) -> - Pid ! stop. + Pid ! stop, + ok. %%%========================================================================= %%% Internal functions diff --git a/lib/kernel/doc/src/file.xml b/lib/kernel/doc/src/file.xml index 09497482cf..b674b3ca93 100644 --- a/lib/kernel/doc/src/file.xml +++ b/lib/kernel/doc/src/file.xml @@ -1477,8 +1477,8 @@ f.txt: {person, "kalle", 25}. <tag><c>16#400</c></tag> <item><p>set group id on execution</p></item> </taglist> - <p>On Unix platforms, the following bits - can also be set:</p> + <p>On Unix platforms, other bits than those listed above + may be set.</p> </item> <tag><c>links = integer() >= 0</c></tag> <item> @@ -2042,8 +2042,8 @@ f.txt: {person, "kalle", 25}. <tag><c>16#400</c></tag> <item><p>Set group id on execution</p></item> </taglist> - <p>On Unix platforms, the following bits - can also be set.</p> + <p>On Unix platforms, other bits than those listed above + may be set.</p> </item> <tag><c>uid = integer() >= 0</c></tag> <item> diff --git a/lib/kernel/src/kernel.app.src b/lib/kernel/src/kernel.app.src index 56d1699656..d184223524 100644 --- a/lib/kernel/src/kernel.app.src +++ b/lib/kernel/src/kernel.app.src @@ -118,6 +118,6 @@ {applications, []}, {env, [{error_logger, tty}]}, {mod, {kernel, []}}, - {runtime_dependencies, ["erts-8.0", "stdlib-3.0", "sasl-3.0"]} + {runtime_dependencies, ["erts-8.1", "stdlib-3.0", "sasl-3.0"]} ] }. diff --git a/lib/mnesia/test/mnesia_test_lib.erl b/lib/mnesia/test/mnesia_test_lib.erl index 6e84a27ec9..0fabdc7929 100644 --- a/lib/mnesia/test/mnesia_test_lib.erl +++ b/lib/mnesia/test/mnesia_test_lib.erl @@ -263,6 +263,7 @@ slave_start_link(Host, Name, Retries) -> Path = code:get_path(), ok = rpc:call(NewNode, file, set_cwd, [Cwd]), true = rpc:call(NewNode, code, set_path, [Path]), + ok = rpc:call(NewNode, error_logger, tty, [false]), spawn_link(NewNode, ?MODULE, slave_sup, []), rpc:multicall([node() | nodes()], global, sync, []), {ok, NewNode}; diff --git a/lib/ssh/src/ssh_channel.erl b/lib/ssh/src/ssh_channel.erl index 426e2f5125..85b31f3669 100644 --- a/lib/ssh/src/ssh_channel.erl +++ b/lib/ssh/src/ssh_channel.erl @@ -261,7 +261,7 @@ handle_info({ssh_cm, _, _} = Msg, #state{cm = ConnectionManager, adjust_window(Msg), {noreply, State#state{channel_state = ChannelState}, Timeout}; {stop, ChannelId, ChannelState} -> - ssh_connection:close(ConnectionManager, ChannelId), + catch ssh_connection:close(ConnectionManager, ChannelId), {stop, normal, State#state{close_sent = true, channel_state = ChannelState}} end; diff --git a/lib/ssh/src/ssh_dbg.erl b/lib/ssh/src/ssh_dbg.erl index bd6bc0335b..ce5596e0f9 100644 --- a/lib/ssh/src/ssh_dbg.erl +++ b/lib/ssh/src/ssh_dbg.erl @@ -113,7 +113,12 @@ setup_tracer(Write, MangleArg) -> ok. %%%---------------------------------------------------------------- -shrink_bin(B) when is_binary(B), size(B)>100 -> {'*** SHRINKED BIN',size(B),element(1,split_binary(B,20)),'***'}; +shrink_bin(B) when is_binary(B), size(B)>100 -> {'*** SHRINKED BIN', + size(B), + element(1,split_binary(B,20)), + '...', + element(2,split_binary(B,size(B)-20)) + }; shrink_bin(L) when is_list(L) -> lists:map(fun shrink_bin/1, L); shrink_bin(T) when is_tuple(T) -> list_to_tuple(shrink_bin(tuple_to_list(T))); shrink_bin(X) -> X. diff --git a/lib/ssh/test/ssh_basic_SUITE.erl b/lib/ssh/test/ssh_basic_SUITE.erl index 51e0d5196b..0a0ab5cdf7 100644 --- a/lib/ssh/test/ssh_basic_SUITE.erl +++ b/lib/ssh/test/ssh_basic_SUITE.erl @@ -315,9 +315,9 @@ init_per_testcase(TC, Config) when TC==shell_no_unicode ; {user_passwords, [{"foo", "bar"}]}]), ct:sleep(500), IO = ssh_test_lib:start_io_server(), - Shell = ssh_test_lib:start_shell(Port, IO, UserDir, - [{silently_accept_hosts, true}, - {user,"foo"},{password,"bar"}]), + Shell = ssh_test_lib:start_shell(Port, IO, [{user_dir,UserDir}, + {silently_accept_hosts, true}, + {user,"foo"},{password,"bar"}]), ct:log("IO=~p, Shell=~p, self()=~p",[IO,Shell,self()]), ct:log("file:native_name_encoding() = ~p,~nio:getopts() = ~p", [file:native_name_encoding(),io:getopts()]), @@ -343,14 +343,15 @@ end_per_testcase(TC, Config) when TC==shell_no_unicode ; TC==shell_unicode_string -> case proplists:get_value(sftpd, Config) of {Pid, _, _} -> - ssh:stop_daemon(Pid), - ssh:stop(); + catch ssh:stop_daemon(Pid); _ -> - ssh:stop() - end; + ok + end, + end_per_testcase(Config); end_per_testcase(_TestCase, Config) -> end_per_testcase(Config). -end_per_testcase(_Config) -> + +end_per_testcase(_Config) -> ssh:stop(), ok. @@ -524,7 +525,7 @@ shell(Config) when is_list(Config) -> ct:sleep(500), IO = ssh_test_lib:start_io_server(), - Shell = ssh_test_lib:start_shell(Port, IO, UserDir), + Shell = ssh_test_lib:start_shell(Port, IO, [{user_dir,UserDir}]), receive {'EXIT', _, _} -> ct:fail(no_ssh_connection); @@ -562,10 +563,10 @@ exec_key_differs(Config, UserPKAlgs) -> ct:sleep(500), IO = ssh_test_lib:start_io_server(), - Shell = ssh_test_lib:start_shell(Port, IO, UserDir, - [{preferred_algorithms,[{public_key,['ssh-rsa']}]}, - {pref_public_key_algs,UserPKAlgs} - ]), + Shell = ssh_test_lib:start_shell(Port, IO, [{user_dir,UserDir}, + {preferred_algorithms,[{public_key,['ssh-rsa']}]}, + {pref_public_key_algs,UserPKAlgs} + ]), receive @@ -596,9 +597,9 @@ exec_key_differs_fail(Config) when is_list(Config) -> ct:sleep(500), IO = ssh_test_lib:start_io_server(), - ssh_test_lib:start_shell(Port, IO, UserDir, - [{preferred_algorithms,[{public_key,['ssh-rsa']}]}, - {pref_public_key_algs,['ssh-dss']}]), + ssh_test_lib:start_shell(Port, IO, [{user_dir,UserDir}, + {preferred_algorithms,[{public_key,['ssh-rsa']}]}, + {pref_public_key_algs,['ssh-dss']}]), receive {'EXIT', _, _} -> ok; diff --git a/lib/ssh/test/ssh_options_SUITE.erl b/lib/ssh/test/ssh_options_SUITE.erl index 61883c0647..4cc12cbcbe 100644 --- a/lib/ssh/test/ssh_options_SUITE.erl +++ b/lib/ssh/test/ssh_options_SUITE.erl @@ -540,10 +540,18 @@ connectfun_disconnectfun_server(Config) -> {disconnect,Ref,R} -> ct:log("Disconnect result: ~p",[R]), ssh:stop_daemon(Pid) - after 2000 -> + after 5000 -> + receive + X -> ct:log("received ~p",[X]) + after 0 -> ok + end, {fail, "No disconnectfun action"} end - after 2000 -> + after 5000 -> + receive + X -> ct:log("received ~p",[X]) + after 0 -> ok + end, {fail, "No connectfun action"} end. @@ -649,7 +657,7 @@ disconnectfun_option_server(Config) -> ct:log("Server detected disconnect: ~p",[Reason]), ssh:stop_daemon(Pid), ok - after 3000 -> + after 5000 -> receive X -> ct:log("received ~p",[X]) after 0 -> ok @@ -974,7 +982,14 @@ ssh_connect_negtimeout(Config, Parallel) -> ct:sleep(round(Factor * NegTimeOut)), case inet:sockname(Socket) of - {ok,_} -> ct:fail("Socket not closed"); + {ok,_} -> + %% Give it another chance... + ct:log("Sleep more...",[]), + ct:sleep(round(Factor * NegTimeOut)), + case inet:sockname(Socket) of + {ok,_} -> ct:fail("Socket not closed"); + {error,_} -> ok + end; {error,_} -> ok end. @@ -1003,7 +1018,7 @@ ssh_connect_nonegtimeout_connected(Config, Parallel) -> ct:sleep(500), IO = ssh_test_lib:start_io_server(), - Shell = ssh_test_lib:start_shell(Port, IO, UserDir), + Shell = ssh_test_lib:start_shell(Port, IO, [{user_dir,UserDir}]), receive Error = {'EXIT', _, _} -> ct:log("~p",[Error]), diff --git a/lib/ssh/test/ssh_renegotiate_SUITE.erl b/lib/ssh/test/ssh_renegotiate_SUITE.erl index b10ec3707f..74bbc291b2 100644 --- a/lib/ssh/test/ssh_renegotiate_SUITE.erl +++ b/lib/ssh/test/ssh_renegotiate_SUITE.erl @@ -92,11 +92,11 @@ rekey(Config) -> ConnectionRef = ssh_test_lib:std_connect(Config, Host, Port, [{rekey_limit, 0}]), - Kex1 = get_kex_init(ConnectionRef), + Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), receive after ?REKEY_DATA_TMO -> %%By this time rekeying would have been done - Kex2 = get_kex_init(ConnectionRef), + Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), false = (Kex2 == Kex1), ssh:close(ConnectionRef), ssh:stop_daemon(Pid) @@ -120,31 +120,31 @@ rekey_limit(Config) -> {max_random_length_padding,0}]), {ok, SftpPid} = ssh_sftp:start_channel(ConnectionRef), - Kex1 = get_kex_init(ConnectionRef), + Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), timer:sleep(?REKEY_DATA_TMO), - Kex1 = get_kex_init(ConnectionRef), + Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), Data = lists:duplicate(159000,1), ok = ssh_sftp:write_file(SftpPid, DataFile, Data), timer:sleep(?REKEY_DATA_TMO), - Kex2 = get_kex_init(ConnectionRef), + Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), false = (Kex2 == Kex1), timer:sleep(?REKEY_DATA_TMO), - Kex2 = get_kex_init(ConnectionRef), + Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), ok = ssh_sftp:write_file(SftpPid, DataFile, "hi\n"), timer:sleep(?REKEY_DATA_TMO), - Kex2 = get_kex_init(ConnectionRef), + Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), false = (Kex2 == Kex1), timer:sleep(?REKEY_DATA_TMO), - Kex2 = get_kex_init(ConnectionRef), + Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), ssh_sftp:stop_channel(SftpPid), ssh:close(ConnectionRef), @@ -169,7 +169,7 @@ renegotiate1(Config) -> ConnectionRef = ssh_test_lib:std_connect(Config, Host, RPort, [{max_random_length_padding,0}]), {ok, SftpPid} = ssh_sftp:start_channel(ConnectionRef), - Kex1 = get_kex_init(ConnectionRef), + Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), {ok, Handle} = ssh_sftp:open(SftpPid, DataFile, [write]), @@ -181,7 +181,7 @@ renegotiate1(Config) -> timer:sleep(2000), - Kex2 = get_kex_init(ConnectionRef), + Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), false = (Kex2 == Kex1), @@ -208,7 +208,7 @@ renegotiate2(Config) -> ConnectionRef = ssh_test_lib:std_connect(Config, Host, RPort, [{max_random_length_padding,0}]), {ok, SftpPid} = ssh_sftp:start_channel(ConnectionRef), - Kex1 = get_kex_init(ConnectionRef), + Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), {ok, Handle} = ssh_sftp:open(SftpPid, DataFile, [write]), @@ -223,7 +223,7 @@ renegotiate2(Config) -> timer:sleep(2000), - Kex2 = get_kex_init(ConnectionRef), + Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), false = (Kex2 == Kex1), @@ -235,19 +235,3 @@ renegotiate2(Config) -> %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- -%% get_kex_init - helper function to get key_exchange_init_msg -get_kex_init(Conn) -> - %% First, validate the key exchange is complete (StateName == connected) - {{connected,_},S} = sys:get_state(Conn), - %% Next, walk through the elements of the #state record looking - %% for the #ssh_msg_kexinit record. This method is robust against - %% changes to either record. The KEXINIT message contains a cookie - %% unique to each invocation of the key exchange procedure (RFC4253) - SL = tuple_to_list(S), - case lists:keyfind(ssh_msg_kexinit, 1, SL) of - false -> - throw(not_found); - KexInit -> - KexInit - end. - diff --git a/lib/ssh/test/ssh_test_lib.erl b/lib/ssh/test/ssh_test_lib.erl index 6fd401d182..f93237f3e7 100644 --- a/lib/ssh/test/ssh_test_lib.erl +++ b/lib/ssh/test/ssh_test_lib.erl @@ -127,24 +127,19 @@ std_simple_exec(Host, Port, Config, Opts) -> ssh:close(ConnectionRef). -start_shell(Port, IOServer, UserDir) -> - start_shell(Port, IOServer, UserDir, []). - -start_shell(Port, IOServer, UserDir, Options) -> - spawn_link(?MODULE, init_shell, [Port, IOServer, [{user_dir, UserDir}|Options]]). - start_shell(Port, IOServer) -> - spawn_link(?MODULE, init_shell, [Port, IOServer, []]). + start_shell(Port, IOServer, []). -init_shell(Port, IOServer, UserDir) -> - Host = hostname(), - Options = [{user_interaction, false}, {silently_accept_hosts, - true}] ++ UserDir, - group_leader(IOServer, self()), - loop_shell(Host, Port, Options). +start_shell(Port, IOServer, ExtraOptions) -> + spawn_link( + fun() -> + Host = hostname(), + Options = [{user_interaction, false}, + {silently_accept_hosts,true} | ExtraOptions], + group_leader(IOServer, self()), + ssh:shell(Host, Port, Options) + end). -loop_shell(Host, Port, Options) -> - ssh:shell(Host, Port, Options). start_io_server() -> spawn_link(?MODULE, init_io_server, [self()]). @@ -802,3 +797,40 @@ busy_wait(Nus, T0) -> end. %%%---------------------------------------------------------------- +%% get_kex_init - helper function to get key_exchange_init_msg + +get_kex_init(Conn) -> + Ref = make_ref(), + {ok,TRef} = timer:send_after(15000, {reneg_timeout,Ref}), + get_kex_init(Conn, Ref, TRef). + +get_kex_init(Conn, Ref, TRef) -> + %% First, validate the key exchange is complete (StateName == connected) + case sys:get_state(Conn) of + {{connected,_}, S} -> + timer:cancel(TRef), + %% Next, walk through the elements of the #state record looking + %% for the #ssh_msg_kexinit record. This method is robust against + %% changes to either record. The KEXINIT message contains a cookie + %% unique to each invocation of the key exchange procedure (RFC4253) + SL = tuple_to_list(S), + case lists:keyfind(ssh_msg_kexinit, 1, SL) of + false -> + throw(not_found); + KexInit -> + KexInit + end; + + {OtherState, S} -> + ct:log("Not in 'connected' state: ~p",[OtherState]), + receive + {reneg_timeout,Ref} -> + ct:log("S = ~p", [S]), + ct:fail(reneg_timeout) + after 0 -> + timer:sleep(100), % If renegotiation is complete we do not + % want to exit on the reneg_timeout + get_kex_init(Conn, Ref, TRef) + end + end. + diff --git a/lib/ssh/test/ssh_to_openssh_SUITE.erl b/lib/ssh/test/ssh_to_openssh_SUITE.erl index f481e9c1ce..2c7fe7898f 100644 --- a/lib/ssh/test/ssh_to_openssh_SUITE.erl +++ b/lib/ssh/test/ssh_to_openssh_SUITE.erl @@ -29,6 +29,7 @@ -define(TIMEOUT, 50000). -define(SSH_DEFAULT_PORT, 22). +-define(REKEY_DATA_TMO, 65000). %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- @@ -55,7 +56,8 @@ groups() -> erlang_client_openssh_server_publickey_rsa, erlang_client_openssh_server_password, erlang_client_openssh_server_kexs, - erlang_client_openssh_server_nonexistent_subsystem + erlang_client_openssh_server_nonexistent_subsystem, + erlang_client_openssh_server_renegotiate ]}, {erlang_server, [], [erlang_server_openssh_client_public_key_dsa, erlang_server_openssh_client_public_key_rsa, @@ -105,6 +107,11 @@ init_per_testcase(erlang_server_openssh_client_public_key_rsa, Config) -> chk_key(sshc, 'ssh-rsa', ".ssh/id_rsa", Config); init_per_testcase(erlang_client_openssh_server_publickey_dsa, Config) -> chk_key(sshd, 'ssh-dss', ".ssh/id_dsa", Config); +init_per_testcase(erlang_server_openssh_client_renegotiate, Config) -> + case os:type() of + {unix,_} -> ssh:start(), Config; + Type -> {skip, io_lib:format("Unsupported test on ~p",[Type])} + end; init_per_testcase(_TestCase, Config) -> ssh:start(), Config. @@ -146,7 +153,7 @@ erlang_shell_client_openssh_server(Config) when is_list(Config) -> IO = ssh_test_lib:start_io_server(), Shell = ssh_test_lib:start_shell(?SSH_DEFAULT_PORT, IO), IO ! {input, self(), "echo Hej\n"}, - receive_hej(), + receive_data("Hej"), IO ! {input, self(), "exit\n"}, receive_logout(), receive_normal_exit(Shell). @@ -393,24 +400,34 @@ erlang_server_openssh_client_renegotiate(Config) -> SystemDir = proplists:get_value(data_dir, Config), PrivDir = proplists:get_value(priv_dir, Config), KnownHosts = filename:join(PrivDir, "known_hosts"), + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, {public_key_alg, PubKeyAlg}, {failfun, fun ssh_test_lib:failfun/2}]), +%% catch ssh_dbg:messages(fun(String,_D) -> ct:log(String) end), ct:sleep(500), + RenegLimitK = 3, DataFile = filename:join(PrivDir, "renegotiate_openssh_client.data"), - Data = lists:duplicate(32000, $a), + Data = lists:duplicate(trunc(1.1*RenegLimitK*1024), $a), ok = file:write_file(DataFile, Data), Cmd = "ssh -p " ++ integer_to_list(Port) ++ " -o UserKnownHostsFile=" ++ KnownHosts ++ - " -o RekeyLimit=20K" ++ + " -o RekeyLimit=" ++ integer_to_list(RenegLimitK) ++"K" ++ " " ++ Host ++ " < " ++ DataFile, OpenSsh = ssh_test_lib:open_port({spawn, Cmd}), Expect = fun({data,R}) -> - try lists:prefix(binary_to_list(R), Data) + try + NonAlphaChars = [C || C<-lists:seq(1,255), + not lists:member(C,lists:seq($a,$z)), + not lists:member(C,lists:seq($A,$Z)) + ], + Lines = string:tokens(binary_to_list(R), NonAlphaChars), + lists:any(fun(L) -> length(L)>1 andalso lists:prefix(L, Data) end, + Lines) catch _:_ -> false end; @@ -419,9 +436,61 @@ erlang_server_openssh_client_renegotiate(Config) -> end, ssh_test_lib:rcv_expected(Expect, OpenSsh, ?TIMEOUT), + %% Unfortunatly we can't check that there has been a renegotiation, just trust OpenSSH. ssh:stop_daemon(Pid). %%-------------------------------------------------------------------- +erlang_client_openssh_server_renegotiate(_Config) -> + process_flag(trap_exit, true), + + IO = ssh_test_lib:start_io_server(), + Ref = make_ref(), + Parent = self(), + + catch ssh_dbg:messages(fun(X,_) -> ct:log(X) end), + Shell = + spawn_link( + fun() -> + Host = ssh_test_lib:hostname(), + Options = [{user_interaction, false}, + {silently_accept_hosts,true}], + group_leader(IO, self()), + {ok, ConnRef} = ssh:connect(Host, ?SSH_DEFAULT_PORT, Options), + case ssh_connection:session_channel(ConnRef, infinity) of + {ok,ChannelId} -> + success = ssh_connection:ptty_alloc(ConnRef, ChannelId, []), + Args = [{channel_cb, ssh_shell}, + {init_args,[ConnRef, ChannelId]}, + {cm, ConnRef}, {channel_id, ChannelId}], + {ok, State} = ssh_channel:init([Args]), + Parent ! {ok, Ref, ConnRef}, + ssh_channel:enter_loop(State); + Error -> + Parent ! {error, Ref, Error} + end, + receive + nothing -> ok + end + end), + + receive + {error, Ref, Error} -> + ct:fail("Error=~p",[Error]); + {ok, Ref, ConnectionRef} -> + IO ! {input, self(), "echo Hej1\n"}, + receive_data("Hej1"), + Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), + ssh_connection_handler:renegotiate(ConnectionRef), + IO ! {input, self(), "echo Hej2\n"}, + receive_data("Hej2"), + Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), + IO ! {input, self(), "exit\n"}, + receive_logout(), + receive_normal_exit(Shell), + true = (Kex1 =/= Kex2) + end. + +%%-------------------------------------------------------------------- erlang_client_openssh_server_password() -> [{doc, "Test client password option"}]. erlang_client_openssh_server_password(Config) when is_list(Config) -> @@ -476,27 +545,24 @@ erlang_client_openssh_server_nonexistent_subsystem(Config) when is_list(Config) %%-------------------------------------------------------------------- %%% Internal functions ----------------------------------------------- %%-------------------------------------------------------------------- -receive_hej() -> +receive_data(Data) -> receive - <<"Hej", _binary>> = Hej -> - ct:log("Expected result: ~p~n", [Hej]); - <<"Hej\n", _binary>> = Hej -> - ct:log("Expected result: ~p~n", [Hej]); - <<"Hej\r\n", _/binary>> = Hej -> - ct:log("Expected result: ~p~n", [Hej]); - Info -> - Lines = binary:split(Info, [<<"\r\n">>], [global]), - case lists:member(<<"Hej">>, Lines) of + Info when is_binary(Info) -> + Lines = string:tokens(binary_to_list(Info), "\r\n "), + case lists:member(Data, Lines) of true -> ct:log("Expected result found in lines: ~p~n", [Lines]), ok; false -> ct:log("Extra info: ~p~n", [Info]), - receive_hej() - end + receive_data(Data) + end; + Other -> + ct:log("Unexpected: ~p",[Other]), + receive_data(Data) after 30000 -> ct:fail("timeout ~p:~p",[?MODULE,?LINE]) - end. + end. receive_logout() -> receive diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index 68f2f97b6e..edc7e0d8b2 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -170,6 +170,14 @@ <tag><c>SNIfun::fun()</c></tag> <item><p><c>= fun(ServerName :: string()) -> [ssl_option()]</c></p></item> + <tag><c>named_curve() =</c></tag> + <item><p><c>sect571r1 | sect571k1 | secp521r1 | brainpoolP512r1 + | sect409k1 | sect409r1 | brainpoolP384r1 | secp384r1 + | sect283k1 | sect283r1 | brainpoolP256r1 | secp256k1 | secp256r1 + | sect239k1 | sect233k1 | sect233r1 | secp224k1 | secp224r1 + | sect193r1 | sect193r2 | secp192k1 | secp192r1 | sect163k1 + | sect163r1 | sect163r2 | secp160k1 | secp160r1 | secp160r2</c></p></item> + </taglist> </section> @@ -217,6 +225,11 @@ Anonymous cipher suites are supported for testing purposes only and are not be used when security matters.</p></item> + <tag><c>{eccs, [named_curve()]}</c></tag> + <item><p> Allows to specify the order of preference for named curves + and to restrict their usage when using a cipher suite supporting them. + </p></item> + <tag><c>{secure_renegotiate, boolean()}</c></tag> <item><p>Specifies if to reject renegotiation attempt that does not live up to @@ -751,6 +764,11 @@ fun(srp, Username :: string(), UserState :: term()) -> (the default), use the client's preference. </item> + <tag><c>{honor_ecc_order, boolean()}</c></tag> + <item>If true, use the server's preference for ECC curve selection. If false + (the default), use the client's preference. + </item> + <tag><c>{signature_algs, [{hash(), ecdsa | rsa | dsa}]}</c></tag> <item><p> The algorithms specified by this option will be the ones accepted by the server in a signature algorithm @@ -804,6 +822,17 @@ fun(srp, Username :: string(), UserState :: term()) -> </func> <func> + <name>eccs() -></name> + <name>eccs(protocol()) -> [named_curve()]</name> + <fsummary>Returns a list of supported ECCs.</fsummary> + + <desc><p>Returns a list of supported ECCs. <c>eccs()</c> + is equivalent to calling <c>eccs(Protocol)</c> with all + supported protocols and then deduplicating the output.</p> + </desc> + </func> + + <func> <name>clear_pem_cache() -> ok </name> <fsummary> Clears the pem cache</fsummary> @@ -898,7 +927,7 @@ fun(srp, Username :: string(), UserState :: term()) -> <fsummary>Returns all the connection information. </fsummary> <type> - <v>Item = protocol | cipher_suite | sni_hostname | atom()</v> + <v>Item = protocol | cipher_suite | sni_hostname | ecc | atom()</v> <d>Meaningful atoms, not specified above, are the ssl option names.</d> <v>Result = [{Item::atom(), Value::term()}]</v> <v>Reason = term()</v> diff --git a/lib/ssl/doc/src/ssl_distribution.xml b/lib/ssl/doc/src/ssl_distribution.xml index 1150043e76..61f88e3860 100644 --- a/lib/ssl/doc/src/ssl_distribution.xml +++ b/lib/ssl/doc/src/ssl_distribution.xml @@ -43,7 +43,7 @@ Erlang node distributed, <c>net_kernel</c> uses this module to set up listen ports and connections.</p> - <p>In the SSL application, an exra distribution + <p>In the SSL application, an extra distribution module, <c>inet_tls_dist</c>, can be used as an alternative. All distribution connections will use SSL and all participating Erlang nodes in a distributed system must use diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index 27b753af2e..aa62ab8865 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -38,7 +38,7 @@ getopts/2, setopts/2, getstat/1, getstat/2 ]). %% SSL/TLS protocol handling --export([cipher_suites/0, cipher_suites/1, +-export([cipher_suites/0, cipher_suites/1, eccs/0, eccs/1, connection_info/1, versions/0, session_info/1, format_error/1, renegotiate/1, prf/5, negotiated_protocol/1, negotiated_next_protocol/1, connection_information/1, connection_information/2]). @@ -420,6 +420,33 @@ cipher_suites(all) -> [ssl_cipher:erl_suite_definition(Suite) || Suite <- available_suites(all)]. %%-------------------------------------------------------------------- +-spec eccs() -> tls_v1:curves(). +%% Description: returns all supported curves across all versions +%%-------------------------------------------------------------------- +eccs() -> + Curves = tls_v1:ecc_curves(all), % only tls_v1 has named curves right now + eccs_filter_supported(Curves). + +%%-------------------------------------------------------------------- +-spec eccs(tls_record:tls_version() | tls_record:tls_atom_version()) -> + tls_v1:curves(). +%% Description: returns the curves supported for a given version of +%% ssl/tls. +%%-------------------------------------------------------------------- +eccs({3,0}) -> + []; +eccs({3,_}) -> + Curves = tls_v1:ecc_curves(all), + eccs_filter_supported(Curves); +eccs(AtomVersion) when is_atom(AtomVersion) -> + eccs(tls_record:protocol_version(AtomVersion)). + +eccs_filter_supported(Curves) -> + CryptoCurves = crypto:ec_curves(), + lists:filter(fun(Curve) -> proplists:get_bool(Curve, CryptoCurves) end, + Curves). + +%%-------------------------------------------------------------------- -spec getopts(#sslsocket{}, [gen_tcp:option_name()]) -> {ok, [gen_tcp:option()]} | {error, reason()}. %% @@ -647,6 +674,8 @@ do_connect(Address, Port, end. %% Handle extra ssl options given to ssl_accept +-spec handle_options([any()], #ssl_options{}) -> #ssl_options{} + ; ([any()], client | server) -> {ok, #config{}}. handle_options(Opts0, #ssl_options{protocol = Protocol, cacerts = CaCerts0, cacertfile = CaCertFile0} = InheritedSslOpts) -> RecordCB = record_cb(Protocol), @@ -725,6 +754,8 @@ handle_options(Opts0, Role) -> srp_identity = handle_option(srp_identity, Opts, undefined), ciphers = handle_cipher_option(proplists:get_value(ciphers, Opts, []), RecordCb:highest_protocol_version(Versions)), + eccs = handle_eccs_option(proplists:get_value(eccs, Opts, eccs()), + RecordCb:highest_protocol_version(Versions)), signature_algs = handle_hashsigns_option(proplists:get_value(signature_algs, Opts, default_option_role(server, tls_v1:default_signature_algs(Versions), Role)), @@ -755,6 +786,9 @@ handle_options(Opts0, Role) -> honor_cipher_order = handle_option(honor_cipher_order, Opts, default_option_role(server, false, Role), server, Role), + honor_ecc_order = handle_option(honor_ecc_order, Opts, + default_option_role(server, false, Role), + server, Role), protocol = proplists:get_value(protocol, Opts, tls), padding_check = proplists:get_value(padding_check, Opts, true), beast_mitigation = handle_option(beast_mitigation, Opts, one_n_minus_one), @@ -780,7 +814,7 @@ handle_options(Opts0, Role) -> alpn_preferred_protocols, next_protocols_advertised, client_preferred_next_protocols, log_alert, server_name_indication, honor_cipher_order, padding_check, crl_check, crl_cache, - fallback, signature_algs, beast_mitigation, v2_hello_compatible], + fallback, signature_algs, eccs, honor_ecc_order, beast_mitigation, v2_hello_compatible], SockOpts = lists:foldl(fun(Key, PropList) -> proplists:delete(Key, PropList) @@ -1010,6 +1044,8 @@ validate_option(sni_fun, Fun) when is_function(Fun) -> Fun; validate_option(honor_cipher_order, Value) when is_boolean(Value) -> Value; +validate_option(honor_ecc_order, Value) when is_boolean(Value) -> + Value; validate_option(padding_check, Value) when is_boolean(Value) -> Value; validate_option(fallback, Value) when is_boolean(Value) -> @@ -1164,6 +1200,14 @@ binary_cipher_suites(Version, Ciphers0) -> Ciphers = [ssl_cipher:openssl_suite(C) || C <- string:tokens(Ciphers0, ":")], binary_cipher_suites(Version, Ciphers). +handle_eccs_option(Value, {_Major, Minor}) when is_list(Value) -> + try tls_v1:ecc_curves(Minor, Value) of + Curves -> #elliptic_curves{elliptic_curve_list = Curves} + catch + exit:_ -> throw({error, {options, {eccs, Value}}}); + error:_ -> throw({error, {options, {eccs, Value}}}) + end. + unexpected_format(Error) -> lists:flatten(io_lib:format("Unexpected error: ~p", [Error])). @@ -1334,6 +1378,14 @@ new_ssl_options([{server_name_indication, Value} | Rest], #ssl_options{} = Opts, new_ssl_options(Rest, Opts#ssl_options{server_name_indication = validate_option(server_name_indication, Value)}, RecordCB); new_ssl_options([{honor_cipher_order, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> new_ssl_options(Rest, Opts#ssl_options{honor_cipher_order = validate_option(honor_cipher_order, Value)}, RecordCB); +new_ssl_options([{honor_ecc_order, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> + new_ssl_options(Rest, Opts#ssl_options{honor_ecc_order = validate_option(honor_ecc_order, Value)}, RecordCB); +new_ssl_options([{eccs, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> + new_ssl_options(Rest, + Opts#ssl_options{eccs = + handle_eccs_option(Value, RecordCB:highest_protocol_version()) + }, + RecordCB); new_ssl_options([{signature_algs, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> new_ssl_options(Rest, Opts#ssl_options{signature_algs = diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index 08fca76123..b6e4d5b433 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -1172,14 +1172,23 @@ handle_alert(#alert{level = ?WARNING} = Alert, StateName, %%% Internal functions %%-------------------------------------------------------------------- connection_info(#state{sni_hostname = SNIHostname, - session = #session{cipher_suite = CipherSuite}, + session = #session{cipher_suite = CipherSuite, ecc = ECCCurve}, protocol_cb = Connection, negotiated_version = {_,_} = Version, ssl_options = Opts}) -> RecordCB = record_cb(Connection), + CipherSuiteDef = ssl_cipher:erl_suite_definition(CipherSuite), + IsNamedCurveSuite = lists:member(element(1,CipherSuiteDef), + [ecdh_ecdsa, ecdhe_ecdsa, ecdh_anon]), + CurveInfo = case ECCCurve of + {namedCurve, Curve} when IsNamedCurveSuite -> + [{ecc, {named_curve, pubkey_cert_records:namedCurves(Curve)}}]; + _ -> + [] + end, [{protocol, RecordCB:protocol_version(Version)}, - {cipher_suite, ssl_cipher:erl_suite_definition(CipherSuite)}, - {sni_hostname, SNIHostname}] ++ ssl_options_list(Opts). + {cipher_suite, CipherSuiteDef}, + {sni_hostname, SNIHostname} | CurveInfo] ++ ssl_options_list(Opts). do_server_hello(Type, #hello_extensions{next_protocol_negotiation = NextProtocols} = ServerHelloExt, @@ -1741,12 +1750,13 @@ calculate_secret(#server_dh_params{dh_p = Prime, dh_g = Base, Connection, certify, certify); calculate_secret(#server_ecdh_params{curve = ECCurve, public = ECServerPubKey}, - State, Connection) -> + State=#state{session=Session}, Connection) -> ECDHKeys = public_key:generate_key(ECCurve), PremasterSecret = ssl_handshake:premaster_secret(#'ECPoint'{point = ECServerPubKey}, ECDHKeys), calculate_master_secret(PremasterSecret, - State#state{diffie_hellman_keys = ECDHKeys}, + State#state{diffie_hellman_keys = ECDHKeys, + session = Session#session{ecc = ECCurve}}, Connection, certify, certify); calculate_secret(#server_psk_params{ diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index 5b51ac0916..4acc745c5f 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -70,7 +70,7 @@ %% Extensions handling -export([client_hello_extensions/6, handle_client_hello_extensions/9, %% Returns server hello extensions - handle_server_hello_extensions/9, select_curve/2 + handle_server_hello_extensions/9, select_curve/2, select_curve/3 ]). %% MISC @@ -120,11 +120,13 @@ server_hello_done() -> #server_hello_done{}. client_hello_extensions(Host, Version, CipherSuites, - #ssl_options{signature_algs = SupportedHashSigns, versions = AllVersions} = SslOpts, ConnectionStates, Renegotiation) -> + #ssl_options{signature_algs = SupportedHashSigns, + eccs = SupportedECCs, + versions = AllVersions} = SslOpts, ConnectionStates, Renegotiation) -> {EcPointFormats, EllipticCurves} = case advertises_ec_ciphers(lists:map(fun ssl_cipher:suite_definition/1, CipherSuites)) of true -> - client_ecc_extensions(tls_v1, Version); + client_ecc_extensions(SupportedECCs); false -> {undefined, undefined} end, @@ -1169,8 +1171,9 @@ select_session(SuggestedSessionId, CipherSuites, HashSigns, Compressions, Port, {resumed, Resumed} end. -supported_ecc({Major, Minor} = Version) when ((Major == 3) and (Minor >= 1)) orelse (Major > 3) -> - Curves = tls_v1:ecc_curves(Version), +%% Deprecated? +supported_ecc({Major, Minor}) when ((Major == 3) and (Minor >= 1)) orelse (Major > 3) -> + Curves = tls_v1:ecc_curves(Minor), #elliptic_curves{elliptic_curve_list = Curves}; supported_ecc(_) -> #elliptic_curves{elliptic_curve_list = []}. @@ -1454,12 +1457,12 @@ srp_user(#ssl_options{srp_identity = {UserName, _}}) -> srp_user(_) -> undefined. -client_ecc_extensions(Module, Version) -> +client_ecc_extensions(SupportedECCs) -> CryptoSupport = proplists:get_value(public_keys, crypto:supports()), case proplists:get_bool(ecdh, CryptoSupport) of true -> EcPointFormats = #ec_point_formats{ec_point_format_list = [?ECPOINT_UNCOMPRESSED]}, - EllipticCurves = #elliptic_curves{elliptic_curve_list = Module:ecc_curves(Version)}, + EllipticCurves = SupportedECCs, {EcPointFormats, EllipticCurves}; _ -> {undefined, undefined} @@ -1493,22 +1496,34 @@ advertises_ec_ciphers([{ecdh_anon, _,_,_} | _]) -> true; advertises_ec_ciphers([_| Rest]) -> advertises_ec_ciphers(Rest). -select_curve(#elliptic_curves{elliptic_curve_list = ClientCurves}, - #elliptic_curves{elliptic_curve_list = ServerCurves}) -> - select_curve(ClientCurves, ServerCurves); -select_curve(undefined, _) -> + +select_curve(Client, Server) -> + select_curve(Client, Server, false). + +select_curve(#elliptic_curves{elliptic_curve_list = ClientCurves}, + #elliptic_curves{elliptic_curve_list = ServerCurves}, + ServerOrder) -> + case ServerOrder of + false -> + select_shared_curve(ClientCurves, ServerCurves); + true -> + select_shared_curve(ServerCurves, ClientCurves) + end; +select_curve(undefined, _, _) -> %% Client did not send ECC extension use default curve if %% ECC cipher is negotiated - {namedCurve, ?secp256r1}; -select_curve(_, []) -> + {namedCurve, ?secp256r1}. + +select_shared_curve([], _) -> no_curve; -select_curve(Curves, [Curve| Rest]) -> +select_shared_curve([Curve | Rest], Curves) -> case lists:member(Curve, Curves) of true -> {namedCurve, Curve}; false -> - select_curve(Curves, Rest) + select_shared_curve(Rest, Curves) end. + %% RFC 6066, Section 3: Currently, the only server names supported are %% DNS hostnames sni(_, disable) -> diff --git a/lib/ssl/src/ssl_internal.hrl b/lib/ssl/src/ssl_internal.hrl index c19c1787ff..487d1fa096 100644 --- a/lib/ssl/src/ssl_internal.hrl +++ b/lib/ssl/src/ssl_internal.hrl @@ -140,6 +140,8 @@ crl_check :: boolean() | peer | best_effort, crl_cache, signature_algs, + eccs, + honor_ecc_order :: boolean(), v2_hello_compatible :: boolean() }). diff --git a/lib/ssl/src/tls_handshake.erl b/lib/ssl/src/tls_handshake.erl index a2486bf752..2bd103c18a 100644 --- a/lib/ssl/src/tls_handshake.erl +++ b/lib/ssl/src/tls_handshake.erl @@ -160,13 +160,15 @@ handle_client_hello(Version, #client_hello{session_id = SugesstedId, extensions = #hello_extensions{elliptic_curves = Curves, signature_algs = ClientHashSigns} = HelloExt}, #ssl_options{versions = Versions, - signature_algs = SupportedHashSigns} = SslOpts, + signature_algs = SupportedHashSigns, + eccs = SupportedECCs, + honor_ecc_order = ECCOrder} = SslOpts, {Port, Session0, Cache, CacheCb, ConnectionStates0, Cert, _}, Renegotiation) -> case tls_record:is_acceptable_version(Version, Versions) of true -> AvailableHashSigns = ssl_handshake:available_signature_algs( ClientHashSigns, SupportedHashSigns, Cert, Version), - ECCCurve = ssl_handshake:select_curve(Curves, ssl_handshake:supported_ecc(Version)), + ECCCurve = ssl_handshake:select_curve(Curves, SupportedECCs, ECCOrder), {Type, #session{cipher_suite = CipherSuite} = Session1} = ssl_handshake:select_session(SugesstedId, CipherSuites, AvailableHashSigns, Compressions, Port, Session0#session{ecc = ECCCurve}, Version, diff --git a/lib/ssl/src/tls_v1.erl b/lib/ssl/src/tls_v1.erl index 711db77708..7f24ce5192 100644 --- a/lib/ssl/src/tls_v1.erl +++ b/lib/ssl/src/tls_v1.erl @@ -31,9 +31,18 @@ -export([master_secret/4, finished/5, certificate_verify/3, mac_hash/7, setup_keys/8, suites/1, prf/5, - ecc_curves/1, oid_to_enum/1, enum_to_oid/1, + ecc_curves/1, ecc_curves/2, oid_to_enum/1, enum_to_oid/1, default_signature_algs/1, signature_algs/2]). +-type named_curve() :: sect571r1 | sect571k1 | secp521r1 | brainpoolP512r1 | + sect409k1 | sect409r1 | brainpoolP384r1 | secp384r1 | + sect283k1 | sect283r1 | brainpoolP256r1 | secp256k1 | secp256r1 | + sect239k1 | sect233k1 | sect233r1 | secp224k1 | secp224r1 | + sect193r1 | sect193r2 | secp192k1 | secp192r1 | sect163k1 | + sect163r1 | sect163r2 | secp160k1 | secp160r1 | secp160r2. +-type curves() :: [named_curve()]. +-export_type([curves/0, named_curve/0]). + %%==================================================================== %% Internal application API %%==================================================================== @@ -399,13 +408,20 @@ is_pair(Hash, rsa, Hashs) -> lists:member(Hash, AtLeastMd5). %% list ECC curves in prefered order -ecc_curves(_Minor) -> - TLSCurves = [sect571r1,sect571k1,secp521r1,brainpoolP512r1, - sect409k1,sect409r1,brainpoolP384r1,secp384r1, - sect283k1,sect283r1,brainpoolP256r1,secp256k1,secp256r1, - sect239k1,sect233k1,sect233r1,secp224k1,secp224r1, - sect193r1,sect193r2,secp192k1,secp192r1,sect163k1, - sect163r1,sect163r2,secp160k1,secp160r1,secp160r2], +-spec ecc_curves(1..3 | all) -> [named_curve()]. +ecc_curves(all) -> + [sect571r1,sect571k1,secp521r1,brainpoolP512r1, + sect409k1,sect409r1,brainpoolP384r1,secp384r1, + sect283k1,sect283r1,brainpoolP256r1,secp256k1,secp256r1, + sect239k1,sect233k1,sect233r1,secp224k1,secp224r1, + sect193r1,sect193r2,secp192k1,secp192r1,sect163k1, + sect163r1,sect163r2,secp160k1,secp160r1,secp160r2]; +ecc_curves(Minor) -> + TLSCurves = ecc_curves(all), + ecc_curves(Minor, TLSCurves). + +-spec ecc_curves(1..3, [named_curve()]) -> [named_curve()]. +ecc_curves(_Minor, TLSCurves) -> CryptoCurves = crypto:ec_curves(), lists:foldr(fun(Curve, Curves) -> case proplists:get_bool(Curve, CryptoCurves) of @@ -414,6 +430,7 @@ ecc_curves(_Minor) -> end end, [], TLSCurves). + %% ECC curves from draft-ietf-tls-ecc-12.txt (Oct. 17, 2005) oid_to_enum(?sect163k1) -> 1; oid_to_enum(?sect163r1) -> 2; diff --git a/lib/ssl/test/ssl_ECC_SUITE.erl b/lib/ssl/test/ssl_ECC_SUITE.erl index 258922d128..bd0c630d41 100644 --- a/lib/ssl/test/ssl_ECC_SUITE.erl +++ b/lib/ssl/test/ssl_ECC_SUITE.erl @@ -46,7 +46,7 @@ groups() -> {'tlsv1', [], all_versions_groups()}, {'erlang_server', [], key_cert_combinations()}, {'erlang_client', [], key_cert_combinations()}, - {'erlang', [], key_cert_combinations() ++ misc()} + {'erlang', [], key_cert_combinations() ++ misc() ++ ecc_negotiation()} ]. all_versions_groups ()-> @@ -68,6 +68,23 @@ key_cert_combinations() -> misc()-> [client_ecdsa_server_ecdsa_with_raw_key]. +ecc_negotiation() -> + [ecc_default_order, + ecc_default_order_custom_curves, + ecc_client_order, + ecc_client_order_custom_curves, + ecc_unknown_curve, + client_ecdh_server_ecdh_ecc_server_custom, + client_rsa_server_ecdh_ecc_server_custom, + client_ecdh_server_rsa_ecc_server_custom, + client_rsa_server_rsa_ecc_server_custom, + client_ecdsa_server_ecdsa_ecc_server_custom, + client_ecdsa_server_rsa_ecc_server_custom, + client_rsa_server_ecdsa_ecc_server_custom, + client_ecdsa_server_ecdsa_ecc_client_custom, + client_rsa_server_ecdsa_ecc_client_custom + ]. + %%-------------------------------------------------------------------- init_per_suite(Config0) -> end_per_suite(Config0), @@ -218,6 +235,132 @@ client_ecdsa_server_ecdsa_with_raw_key(Config) when is_list(Config) -> check_result(Server, SType, Client, CType), close(Server, Client). +ecc_default_order(Config) -> + COpts = proplists:get_value(client_ecdsa_opts, Config), + SOpts = proplists:get_value(server_ecdsa_opts, Config), + ECCOpts = [], + case supported_eccs([{eccs, [sect571r1]}]) of + true -> ecc_test(sect571r1, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +ecc_default_order_custom_curves(Config) -> + COpts = proplists:get_value(client_ecdsa_opts, Config), + SOpts = proplists:get_value(server_ecdsa_opts, Config), + ECCOpts = [{eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(sect571r1, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +ecc_client_order(Config) -> + COpts = proplists:get_value(client_ecdsa_opts, Config), + SOpts = proplists:get_value(server_ecdsa_opts, Config), + ECCOpts = [{honor_ecc_order, false}], + case supported_eccs([{eccs, [sect571r1]}]) of + true -> ecc_test(sect571r1, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +ecc_client_order_custom_curves(Config) -> + COpts = proplists:get_value(client_ecdsa_opts, Config), + SOpts = proplists:get_value(server_ecdsa_opts, Config), + ECCOpts = [{honor_ecc_order, false}, {eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(sect571r1, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +ecc_unknown_curve(Config) -> + COpts = proplists:get_value(client_ecdsa_opts, Config), + SOpts = proplists:get_value(server_ecdsa_opts, Config), + ECCOpts = [{eccs, ['123_fake_curve']}], + ecc_test_error(COpts, SOpts, [], ECCOpts, Config). + +%% We can only expect to see a named curve on a conn with +%% a server supporting ecdsa. Otherwise the curve is selected +%% but not used and communicated to the client? +client_ecdh_server_ecdh_ecc_server_custom(Config) -> + COpts = proplists:get_value(client_ecdh_rsa_opts, Config), + SOpts = proplists:get_value(server_ecdh_rsa_opts, Config), + ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(undefined, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +client_ecdh_server_rsa_ecc_server_custom(Config) -> + COpts = proplists:get_value(client_ecdh_rsa_opts, Config), + SOpts = proplists:get_value(server_opts, Config), + ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(undefined, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +client_rsa_server_ecdh_ecc_server_custom(Config) -> + COpts = proplists:get_value(client_opts, Config), + SOpts = proplists:get_value(server_ecdh_rsa_opts, Config), + ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(undefined, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +client_rsa_server_rsa_ecc_server_custom(Config) -> + COpts = proplists:get_value(client_opts, Config), + SOpts = proplists:get_value(server_opts, Config), + ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(undefined, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +client_ecdsa_server_ecdsa_ecc_server_custom(Config) -> + COpts = proplists:get_value(client_ecdsa_opts, Config), + SOpts = proplists:get_value(server_ecdsa_opts, Config), + ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(secp256r1, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +client_ecdsa_server_rsa_ecc_server_custom(Config) -> + COpts = proplists:get_value(client_ecdsa_opts, Config), + SOpts = proplists:get_value(server_opts, Config), + ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(undefined, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +client_rsa_server_ecdsa_ecc_server_custom(Config) -> + COpts = proplists:get_value(client_opts, Config), + SOpts = proplists:get_value(server_ecdsa_opts, Config), + ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(secp256r1, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +client_ecdsa_server_ecdsa_ecc_client_custom(Config) -> + COpts = proplists:get_value(client_ecdsa_opts, Config), + SOpts = proplists:get_value(server_ecdsa_opts, Config), + ECCOpts = [{eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(secp256r1, COpts, SOpts, ECCOpts, [], Config); + false -> {skip, "unsupported named curves"} + end. + +client_rsa_server_ecdsa_ecc_client_custom(Config) -> + COpts = proplists:get_value(client_opts, Config), + SOpts = proplists:get_value(server_ecdsa_opts, Config), + ECCOpts = [{eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(secp256r1, COpts, SOpts, ECCOpts, [], Config); + false -> {skip, "unsupported named curves"} + end. + %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- @@ -244,6 +387,30 @@ basic_test(ClientCert, ClientKey, ClientCA, ServerCert, ServerKey, ServerCA, Con check_result(Server, SType, Client, CType), close(Server, Client). +ecc_test(Expect, COpts, SOpts, CECCOpts, SECCOpts, Config) -> + CCA = proplists:get_value(cacertfile, COpts), + CCert = proplists:get_value(certfile, COpts), + CKey = proplists:get_value(keyfile, COpts), + SCA = proplists:get_value(cacertfile, SOpts), + SCert = proplists:get_value(certfile, SOpts), + SKey = proplists:get_value(keyfile, SOpts), + {Server, Port} = start_server_ecc(erlang, CCA, SCA, SCert, SKey, Expect, SECCOpts, Config), + Client = start_client_ecc(erlang, Port, SCA, CCA, CCert, CKey, Expect, CECCOpts, Config), + ssl_test_lib:check_result(Server, ok, Client, ok), + close(Server, Client). + +ecc_test_error(COpts, SOpts, CECCOpts, SECCOpts, Config) -> + CCA = proplists:get_value(cacertfile, COpts), + CCert = proplists:get_value(certfile, COpts), + CKey = proplists:get_value(keyfile, COpts), + SCA = proplists:get_value(cacertfile, SOpts), + SCert = proplists:get_value(certfile, SOpts), + SKey = proplists:get_value(keyfile, SOpts), + {Server, Port} = start_server_ecc_error(erlang, CCA, SCA, SCert, SKey, SECCOpts, Config), + Client = start_client_ecc_error(erlang, Port, SCA, CCA, CCert, CKey, CECCOpts, Config), + Error = {error, {tls_alert, "insufficient security"}}, + ssl_test_lib:check_result(Server, Error, Client, Error). + start_client(openssl, Port, PeerCA, OwnCa, Cert, Key, _Config) -> CA = new_openssl_ca("openssl_client_ca", PeerCA, OwnCa), Version = tls_record:protocol_version(tls_record:highest_protocol_version([])), @@ -267,6 +434,31 @@ start_client(erlang, Port, PeerCA, OwnCa, Cert, Key, Config) -> {cacertfile, CA}, {certfile, Cert}, {keyfile, Key}]}]). +start_client_ecc(erlang, Port, PeerCA, OwnCa, Cert, Key, Expect, ECCOpts, Config) -> + CA = new_ca("erlang_client_ca", PeerCA, OwnCa), + {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), + ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, check_ecc, [client, Expect]}}, + {options, + ECCOpts ++ + [{verify, verify_peer}, + {cacertfile, CA}, + {certfile, Cert}, {keyfile, Key}]}]). + +start_client_ecc_error(erlang, Port, PeerCA, OwnCa, Cert, Key, ECCOpts, Config) -> + CA = new_ca("erlang_client_ca", PeerCA, OwnCa), + {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), + ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {options, + ECCOpts ++ + [{verify, verify_peer}, + {cacertfile, CA}, + {certfile, Cert}, {keyfile, Key}]}]). + start_server(openssl, PeerCA, OwnCa, Cert, Key, _Config) -> CA = new_openssl_ca("openssl_server_ca", PeerCA, OwnCa), Port = ssl_test_lib:inet_port(node()), @@ -290,6 +482,7 @@ start_server(erlang, PeerCA, OwnCa, Cert, Key, Config) -> [{verify, verify_peer}, {cacertfile, CA}, {certfile, Cert}, {keyfile, Key}]}]), {Server, ssl_test_lib:inet_port(Server)}. + start_server_with_raw_key(erlang, PeerCA, OwnCa, Cert, Key, Config) -> CA = new_ca("erlang_server_ca", PeerCA, OwnCa), {_, ServerNode, _} = ssl_test_lib:run_where(Config), @@ -303,6 +496,29 @@ start_server_with_raw_key(erlang, PeerCA, OwnCa, Cert, Key, Config) -> {certfile, Cert}, {key, Key}]}]), {Server, ssl_test_lib:inet_port(Server)}. +start_server_ecc(erlang, PeerCA, OwnCa, Cert, Key, Expect, ECCOpts, Config) -> + CA = new_ca("erlang_server_ca", PeerCA, OwnCa), + {_, ServerNode, _} = ssl_test_lib:run_where(Config), + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, check_ecc, [server, Expect]}}, + {options, + ECCOpts ++ + [{verify, verify_peer}, {cacertfile, CA}, + {certfile, Cert}, {keyfile, Key}]}]), + {Server, ssl_test_lib:inet_port(Server)}. + +start_server_ecc_error(erlang, PeerCA, OwnCa, Cert, Key, ECCOpts, Config) -> + CA = new_ca("erlang_server_ca", PeerCA, OwnCa), + {_, ServerNode, _} = ssl_test_lib:run_where(Config), + Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, + {from, self()}, + {options, + ECCOpts ++ + [{verify, verify_peer}, {cacertfile, CA}, + {certfile, Cert}, {keyfile, Key}]}]), + {Server, ssl_test_lib:inet_port(Server)}. + check_result(Server, erlang, Client, erlang) -> ssl_test_lib:check_result(Server, ok, Client, ok); check_result(Server, erlang, _, _) -> @@ -362,3 +578,17 @@ new_openssl_ca(FileName, CA, OwnCa) -> file:write_file(FileName, Pem) end, FileName. + +supported_eccs(Opts) -> + ToCheck = proplists:get_value(eccs, Opts, []), + Supported = ssl:eccs(), + lists:all(fun(Curve) -> lists:member(Curve, Supported) end, ToCheck). + +check_ecc(SSL, Role, Expect) -> + {ok, Data} = ssl:connection_information(SSL), + case lists:keyfind(ecc, 1, Data) of + {ecc, {named_curve, Expect}} -> ok; + false when Expect =:= undefined -> ok; + Other -> {error, Role, Expect, Other} + end. + diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index 1be43c56c4..f8dea736ae 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -150,6 +150,7 @@ api_tests() -> peercert_with_client_cert, sockname, versions, + eccs, controlling_process, getstat, close_with_timeout, @@ -456,6 +457,15 @@ init_per_testcase(accept_pool, Config) -> init_per_testcase(controller_dies, Config) -> ct:timetrap({seconds, 10}), Config; +init_per_testcase(eccs, Config) -> + case ssl:eccs() of + [] -> + {skip, "named curves not supported"}; + [_|_] -> + ssl_test_lib:ct_log_supported_protocol_versions(Config), + ct:timetrap({seconds, 5}), + Config + end; init_per_testcase(_TestCase, Config) -> ssl_test_lib:ct_log_supported_protocol_versions(Config), ct:timetrap({seconds, 5}), @@ -1504,6 +1514,25 @@ versions(Config) when is_list(Config) -> [_|_] = Versions = ssl:versions(), ct:log("~p~n", [Versions]). + +%%-------------------------------------------------------------------- +eccs() -> + [{doc, "Test API functions eccs/0 and eccs/1"}]. + +eccs(Config) when is_list(Config) -> + [_|_] = All = ssl:eccs(), + [] = SSL3 = ssl:eccs({3,0}), + [_|_] = Tls = ssl:eccs({3,1}), + [_|_] = Tls1 = ssl:eccs({3,2}), + [_|_] = Tls2 = ssl:eccs({3,3}), + [] = SSL3 = ssl:eccs(sslv3), + [_|_] = Tls = ssl:eccs(tlsv1), + [_|_] = Tls1 = ssl:eccs('tlsv1.1'), + [_|_] = Tls2 = ssl:eccs('tlsv1.2'), + %% ordering is currently unverified by the test + true = lists:sort(All) =:= lists:usort(SSL3 ++ Tls ++ Tls1 ++ Tls2), + ok. + %%-------------------------------------------------------------------- send_recv() -> [{doc,""}]. diff --git a/lib/stdlib/doc/src/shell_default.xml b/lib/stdlib/doc/src/shell_default.xml index 81c99bce10..75bf89ba8d 100644 --- a/lib/stdlib/doc/src/shell_default.xml +++ b/lib/stdlib/doc/src/shell_default.xml @@ -51,7 +51,7 @@ <p>In command one, module <seealso marker="lists"><c>lists</c></seealso> is called. In command two, no module name is specified. The shell searches module <c>user_default</c> followed by module <c>shell_default</c> for - function <c>foo/1</c>.</p> + function <c>c/1</c>.</p> <p><c>shell_default</c> is intended for "system wide" customizations to the shell. <c>user_default</c> is intended for diff --git a/lib/stdlib/test/base64_SUITE.erl b/lib/stdlib/test/base64_SUITE.erl index 9176a3664a..d0abe5c961 100644 --- a/lib/stdlib/test/base64_SUITE.erl +++ b/lib/stdlib/test/base64_SUITE.erl @@ -23,9 +23,7 @@ -include_lib("common_test/include/ct.hrl"). %% Test server specific exports --export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, - init_per_group/2,end_per_group/2, - init_per_testcase/2, end_per_testcase/2]). +-export([all/0, suite/0, groups/0, group/1]). %% Test cases must be exported. -export([base64_encode/1, base64_decode/1, base64_otp_5635/1, @@ -33,41 +31,26 @@ mime_decode_to_string/1, roundtrip_1/1, roundtrip_2/1, roundtrip_3/1, roundtrip_4/1]). -init_per_testcase(_, Config) -> - Config. - -end_per_testcase(_, _Config) -> - ok. - %%------------------------------------------------------------------------- %% Test cases starts here. %%------------------------------------------------------------------------- + suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap,{minutes,4}}]. -all() -> +all() -> [base64_encode, base64_decode, base64_otp_5635, base64_otp_6279, big, illegal, mime_decode, mime_decode_to_string, {group, roundtrip}]. -groups() -> +groups() -> [{roundtrip, [parallel], [roundtrip_1, roundtrip_2, roundtrip_3, roundtrip_4]}]. -init_per_suite(Config) -> - Config. - -end_per_suite(_Config) -> - ok. - -init_per_group(_GroupName, Config) -> - Config. - -end_per_group(_GroupName, Config) -> - Config. - - +group(roundtrip) -> + %% valgrind needs a lot of time + [{timetrap,{minutes,10}}]. %%------------------------------------------------------------------------- %% Test base64:encode/1. @@ -78,9 +61,9 @@ base64_encode(Config) when is_list(Config) -> %% One pad <<"SGVsbG8gV29ybGQ=">> = base64:encode(<<"Hello World">>), %% No pad - "QWxhZGRpbjpvcGVuIHNlc2Ft" = + "QWxhZGRpbjpvcGVuIHNlc2Ft" = base64:encode_to_string("Aladdin:open sesam"), - + "MDEyMzQ1Njc4OSFAIzBeJiooKTs6PD4sLiBbXXt9" = base64:encode_to_string(<<"0123456789!@#0^&*();:<>,. []{}">>), ok. @@ -93,7 +76,7 @@ base64_decode(Config) when is_list(Config) -> %% One pad <<"Hello World">> = base64:decode(<<"SGVsbG8gV29ybGQ=">>), %% No pad - <<"Aladdin:open sesam">> = + <<"Aladdin:open sesam">> = base64:decode("QWxhZGRpbjpvcGVuIHNlc2Ft"), Alphabet = list_to_binary(lists:seq(0, 255)), @@ -208,7 +191,7 @@ mime_decode_to_string(Config) when is_list(Config) -> %% One pad to ignore, followed by more text "Hello World!!" = base64:mime_decode_to_string(<<"SGVsb)(G8gV29ybGQ=h IQ= =">>), %% No pad - "Aladdin:open sesam" = + "Aladdin:open sesam" = base64:mime_decode_to_string("QWxhZGRpbjpvcG¤\")(VuIHNlc2Ft"), %% Encoded base 64 strings may be divided by non base 64 chars. %% In this cases whitespaces. @@ -314,7 +297,7 @@ interleaved_ws_roundtrip_1([], Base64List, Bin, List) -> random_byte_list(0, Acc) -> Acc; -random_byte_list(N, Acc) -> +random_byte_list(N, Acc) -> random_byte_list(N-1, [rand:uniform(255)|Acc]). make_big_binary(N) -> diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index b02d17bdb6..00e02a06cc 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -19,7 +19,7 @@ %% -module(ets_SUITE). --export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2]). -export([default/1,setbag/1,badnew/1,verybadnew/1,named/1,keypos2/1, privacy/1,privacy_owner/2]). @@ -31,15 +31,14 @@ -export([match_delete3/1]). -export([firstnext/1,firstnext_concurrent/1]). -export([slot/1]). --export([ match1/1, match2/1, match_object/1, match_object2/1]). --export([ dups/1, misc1/1, safe_fixtable/1, info/1, tab2list/1]). --export([ tab2file/1, tab2file2/1, tabfile_ext1/1, - tabfile_ext2/1, tabfile_ext3/1, tabfile_ext4/1, badfile/1]). --export([ heavy_lookup/1, heavy_lookup_element/1, heavy_concurrent/1]). --export([ lookup_element_mult/1]). --export([]). +-export([match1/1, match2/1, match_object/1, match_object2/1]). +-export([dups/1, misc1/1, safe_fixtable/1, info/1, tab2list/1]). +-export([tab2file/1, tab2file2/1, tabfile_ext1/1, + tabfile_ext2/1, tabfile_ext3/1, tabfile_ext4/1, badfile/1]). +-export([heavy_lookup/1, heavy_lookup_element/1, heavy_concurrent/1]). +-export([lookup_element_mult/1]). -export([foldl_ordered/1, foldr_ordered/1, foldl/1, foldr/1, fold_empty/1]). --export([t_delete_object/1, t_init_table/1, t_whitebox/1, +-export([t_delete_object/1, t_init_table/1, t_whitebox/1, t_delete_all_objects/1, t_insert_list/1, t_test_ms/1, t_select_delete/1,t_ets_dets/1]). @@ -61,8 +60,7 @@ -export([otp_7665/1]). -export([meta_wb/1]). -export([grow_shrink/1, grow_pseudo_deleted/1, shrink_pseudo_deleted/1]). --export([ - meta_lookup_unnamed_read/1, meta_lookup_unnamed_write/1, +-export([meta_lookup_unnamed_read/1, meta_lookup_unnamed_write/1, meta_lookup_named_read/1, meta_lookup_named_write/1, meta_newdel_unnamed/1, meta_newdel_named/1]). -export([smp_insert/1, smp_fixed_delete/1, smp_unfix_fix/1, smp_select_delete/1, @@ -95,7 +93,7 @@ rename_do/1, rename_unnamed_do/1, interface_equality_do/1, ordered_match_do/1, ordered_do/1, privacy_do/1, empty_do/1, badinsert_do/1, time_lookup_do/1, lookup_order_do/1, lookup_element_mult_do/1, delete_tab_do/1, delete_elem_do/1, - match_delete_do/1, match_delete3_do/1, firstnext_do/1, + match_delete_do/1, match_delete3_do/1, firstnext_do/1, slot_do/1, match1_do/1, match2_do/1, match_object_do/1, match_object2_do/1, misc1_do/1, safe_fixtable_do/1, info_do/1, dups_do/1, heavy_lookup_do/1, heavy_lookup_element_do/1, member_do/1, otp_5340_do/1, otp_7665_do/1, meta_wb_do/1, @@ -129,7 +127,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap,{minutes,5}}]. -all() -> +all() -> [{group, new}, {group, insert}, {group, lookup}, {group, delete}, firstnext, firstnext_concurrent, slot, {group, match}, t_match_spec_run, @@ -161,7 +159,7 @@ all() -> memory_check_summary]. % MUST BE LAST -groups() -> +groups() -> [{new, [], [default, setbag, badnew, verybadnew, named, keypos2, privacy]}, @@ -249,6 +247,7 @@ t_bucket_disappears_do(Opts) -> %% Check ets:match_spec_run/2. t_match_spec_run(Config) when is_list(Config) -> + ct:timetrap({minutes,30}), %% valgrind needs a lot init_externals(), EtsMem = etsmem(), @@ -709,7 +708,7 @@ adjust_xmem([_T1,_T2,_T3,_T4], {A0,B0,C0,D0} = _Mem0) -> {A0+TabDiff, B0+TabDiff, C0+TabDiff, D0+TabDiff}. %% Misc. whitebox tests -t_whitebox(Config) when is_list(Config) -> +t_whitebox(Config) when is_list(Config) -> EtsMem = etsmem(), repeat_for_opts(whitebox_1), repeat_for_opts(whitebox_1), @@ -1050,6 +1049,7 @@ do_reverse_chunked({L,C},Acc) -> %% Test the ets:select_delete/2 and ets:select_count/2 BIFs. t_select_delete(Config) when is_list(Config) -> + ct:timetrap({minutes,30}), %% valgrind needs a lot EtsMem = etsmem(), Tables = fill_sets_int(10000) ++ fill_sets_int(10000,[{write_concurrency,true}]), lists:foreach @@ -1495,15 +1495,15 @@ update_element(Config) when is_list(Config) -> verify_etsmem(EtsMem). update_element_opts(Opts) -> - TupleCases = [{{key,val}, 1 ,2}, - {{val,key}, 2, 1}, - {{key,val}, 1 ,[2]}, + TupleCases = [{{key,val}, 1 ,2}, + {{val,key}, 2, 1}, + {{key,val}, 1 ,[2]}, {{key,val,val}, 1, [2,3]}, {{val,key,val,val}, 2, [3,4,1]}, {{val,val,key,val}, 3, [1,4,1,2]}, % update pos1 twice {{val,val,val,key}, 4, [2,1,2,3]}],% update pos2 twice - lists:foreach(fun({Tuple,KeyPos,UpdPos}) -> update_element_opts(Tuple,KeyPos,UpdPos,Opts) end, + lists:foreach(fun({Tuple,KeyPos,UpdPos}) -> update_element_opts(Tuple,KeyPos,UpdPos,Opts) end, TupleCases), update_element_neg(Opts). @@ -1519,9 +1519,9 @@ update_element_opts(Tuple,KeyPos,UpdPos,Opts) -> true = ets:delete(OrdSet), ok. -update_element(T,Tuple,KeyPos,UpdPos) -> +update_element(T,Tuple,KeyPos,UpdPos) -> KeyList = [17,"seventeen",<<"seventeen">>,{17},list_to_binary(lists:seq(1,100)),make_ref(), self()], - lists:foreach(fun(Key) -> + lists:foreach(fun(Key) -> TupleWithKey = setelement(KeyPos,Tuple,Key), update_element_do(T,TupleWithKey,Key,UpdPos) end, @@ -1556,29 +1556,29 @@ update_element_do(Tab,Tuple,Key,UpdPos) -> {Pos, element(ToIx+1,Values)} % single {pos,value} arg end, - UpdateF = fun(ToIx,Rand) -> - PosValArg = PosValArgF(ToIx,[],UpdPos,Rand,PosValArgF), - %%io:format("update_element(~p)~n",[PosValArg]), - ArgHash = erlang:phash2({Tab,Key,PosValArg}), - true = ets:update_element(Tab, Key, PosValArg), - ArgHash = erlang:phash2({Tab,Key,PosValArg}), - NewTuple = update_tuple(PosValArg,Tuple), - [NewTuple] = ets:lookup(Tab,Key) + UpdateF = fun(ToIx,Rand) -> + PosValArg = PosValArgF(ToIx,[],UpdPos,Rand,PosValArgF), + %%io:format("update_element(~p)~n",[PosValArg]), + ArgHash = erlang:phash2({Tab,Key,PosValArg}), + true = ets:update_element(Tab, Key, PosValArg), + ArgHash = erlang:phash2({Tab,Key,PosValArg}), + NewTuple = update_tuple(PosValArg,Tuple), + [NewTuple] = ets:lookup(Tab,Key) end, - LoopF = fun(_FromIx, Incr, _Times, Checksum, _MeF) when Incr >= Length -> + LoopF = fun(_FromIx, Incr, _Times, Checksum, _MeF) when Incr >= Length -> Checksum; % done - (FromIx, Incr, 0, Checksum, MeF) -> + (FromIx, Incr, 0, Checksum, MeF) -> MeF(FromIx, Incr+1, Length, Checksum, MeF); - (FromIx, Incr, Times, Checksum, MeF) -> + (FromIx, Incr, Times, Checksum, MeF) -> ToIx = (FromIx + Incr) rem Length, UpdateF(ToIx,Checksum), - if + if Incr =:= 0 -> UpdateF(ToIx,Checksum); % extra update to same value true -> true - end, + end, MeF(ToIx, Incr, Times-1, Checksum+ToIx+1, MeF) end, @@ -1622,7 +1622,7 @@ update_element_neg_do(T) -> Object = {key, 0, "Hej"}, true = ets:insert(T,Object), - UpdateF = fun(Arg3) -> + UpdateF = fun(Arg3) -> ArgHash = erlang:phash2({T,key,Arg3}), {'EXIT',{badarg,_}} = (catch ets:update_element(T,key,Arg3)), ArgHash = erlang:phash2({T,key,Arg3}), @@ -1697,7 +1697,7 @@ update_counter_for(T) -> true = ets:lookup(T, b) =:= [setelement(1, NewObj, b)], ets:delete(T, b), Myself(NewObj,Times-1,Arg3,Myself) - end, + end, LoopF = fun(Obj, Times, Arg3) -> %%io:format("Loop start:\nObj = ~p\nArg3=~p\n",[Obj,Arg3]), @@ -1806,7 +1806,7 @@ uc_mimic(Obj, [Pits|Tail], Acc) -> uc_adder(Init, {_Pos, Add}) -> Init + Add; -uc_adder(Init, {_Pos, Add, Thres, Warp}) -> +uc_adder(Init, {_Pos, Add, Thres, Warp}) -> case Init + Add of X when X > Thres, Add > 0 -> Warp; @@ -1838,7 +1838,7 @@ update_counter_neg_for(T) -> Object = {key,0,false,1}, true = ets:insert(T,Object), - UpdateF = fun(Arg3) -> + UpdateF = fun(Arg3) -> ArgHash = erlang:phash2({T,key,Arg3}), {'EXIT',{badarg,_}} = (catch ets:update_counter(T,key,Arg3)), ArgHash = erlang:phash2({T,key,Arg3}), @@ -1978,15 +1978,16 @@ fixtable_next_do(Opts) -> verify_etsmem(EtsMem). do_fixtable_next(Tab) -> - F = fun(X,T,FF) -> case X of - 0 -> true; - _ -> - ets:insert(T, {X, - integer_to_list(X), - X rem 10}), - FF(X-1,T,FF) - end - end, + F = fun(X,T,FF) -> + case X of + 0 -> true; + _ -> + ets:insert(T, {X, + integer_to_list(X), + X rem 10}), + FF(X-1,T,FF) + end + end, F(100,Tab,F), ets:safe_fixtable(Tab,true), First = ets:first(Tab), @@ -2001,7 +2002,7 @@ do_fixtable_next(Tab) -> %% Check inserts of deleted keys in fixed bags. fixtable_insert(Config) when is_list(Config) -> - Combos = [[Type,{write_concurrency,WC}] || Type<- [bag,duplicate_bag], + Combos = [[Type,{write_concurrency,WC}] || Type<- [bag,duplicate_bag], WC <- [false,true]], lists:foreach(fun(Opts) -> fixtable_insert_do(Opts) end, Combos), @@ -2117,7 +2118,7 @@ heir_do(Opts) -> %% Different types of heir data and link/monitor relations TestFun = fun(Arg) -> {EtsMem,Arg} end, - Combos = [{Data,Mode} || Data<-[foo_data, <<"binary">>, + Combos = [{Data,Mode} || Data<-[foo_data, <<"binary">>, lists:seq(1,10), {17,TestFun,self()}, "The busy heir"], Mode<-[none,link,monitor]], @@ -2157,7 +2158,7 @@ heir_do(Opts) -> Founder4 ! {go, Heir4}, {'DOWN', MrefH4, process, Heir4, normal} = receive_any(), erts_debug:set_internal_state(next_pid, NextPidIx), - DoppelGanger = spawn_monitor_with_pid(Heir4, + DoppelGanger = spawn_monitor_with_pid(Heir4, fun()-> die_please = receive_any() end), Founder4 ! die_please, {'DOWN', MrefF4, process, Founder4, normal} = receive_any(), @@ -2170,12 +2171,12 @@ heir_do(Opts) -> failed -> io:format("Failed to spawn process with pid ~p\n", [Heir4]), true % try again - end + end end), verify_etsmem(EtsMem). -heir_founder(Master, HeirData, Opts) -> +heir_founder(Master, HeirData, Opts) -> {go,Heir} = receive_any(), HeirTpl = case Heir of none -> {heir,none}; @@ -2248,7 +2249,7 @@ heir_1(HeirData,Mode,Opts) -> {'DOWN', Mref, process, Heir, normal} = receive_any(). %% Test ets:give_way/3. -give_away(Config) when is_list(Config) -> +give_away(Config) when is_list(Config) -> repeat_for_opts(give_away_do). give_away_do(Opts) -> @@ -2387,7 +2388,7 @@ bad_table(Config) when is_list(Config) -> ok. bad_table_do(Opts, DummyFile) -> - Parent = self(), + Parent = self(), {Pid,Mref} = my_spawn_opt(fun()-> ets_new(priv,[private,named_table | Opts]), Priv = ets_new(priv,[private | Opts]), ets_new(prot,[protected,named_table | Opts]), @@ -2442,7 +2443,7 @@ bad_table_do(Opts, DummyFile) -> ], Info = {Opts, Priv, Prot}, lists:foreach(fun(Op) -> bad_table_op(Info, Op) end, - OpList), + OpList), Pid ! die_please, {'DOWN', Mref, process, Pid, normal} = receive_any(), ok. @@ -2577,14 +2578,14 @@ interface_equality_do(Opts) -> Set = ets_new(set,[set | Opts]), OrderedSet = ets_new(ordered_set,[ordered_set | Opts]), F = fun(X,T,FF) -> case X of - 0 -> true; - _ -> - ets:insert(T, {X, - integer_to_list(X), - X rem 10}), - FF(X-1,T,FF) - end - end, + 0 -> true; + _ -> + ets:insert(T, {X, + integer_to_list(X), + X rem 10}), + FF(X-1,T,FF) + end + end, F(100,Set,F), F(100,OrderedSet,F), equal_results(ets, insert, Set, OrderedSet, [{a,"a"}]), @@ -2653,20 +2654,20 @@ ordered_match_do(Opts) -> F(3000,T1,F), [[3,3],[3,3],[3,3]] = ets:match(T1, {'_','_','$1','$2',3}), F2 = fun(X,Rem,Res,FF) -> case X of - 0 -> []; - _ -> + 0 -> []; + _ -> case X rem Rem of Res -> FF(X-1,Rem,Res,FF) ++ [{X, - integer_to_list(X), + integer_to_list(X), X rem 10, X rem 100, X rem 1000}]; _ -> FF(X-1,Rem,Res,FF) end - end + end end, OL1 = F2(3000,100,2,F2), OL1 = ets:match_object(T1, {'_','_','_',2,'_'}), @@ -2744,7 +2745,7 @@ pick_all_backwards(T) -> %% Small test case for both set and bag type ets tables. -setbag(Config) when is_list(Config) -> +setbag(Config) when is_list(Config) -> EtsMem = etsmem(), Set = ets_new(set,[set]), Bag = ets_new(bag,[bag]), @@ -2821,7 +2822,7 @@ privacy_do(Opts) -> privacy_check(pub,prot,priv), - Owner ! {shift,1,{pub,prot,priv}}, + Owner ! {shift,1,{pub,prot,priv}}, receive {Pub1,Prot1,Priv1} -> ok = privacy_check(Pub1,Prot1,Priv1), @@ -2960,7 +2961,7 @@ badlookup(Config) when is_list(Config) -> verify_etsmem(EtsMem). %% Test that lookup returns objects in order of insertion for bag and dbag. -lookup_order(Config) when is_list(Config) -> +lookup_order(Config) when is_list(Config) -> EtsMem = etsmem(), repeat_for_opts(lookup_order_do, [write_concurrency,[bag,duplicate_bag]]), verify_etsmem(EtsMem), @@ -2982,7 +2983,7 @@ lookup_order_2(Opts, Fixed) -> case Fixed of true -> ets:safe_fixtable(T,true); false -> ok - end, + end, S10 = {T,[],key}, S20 = check_insert(S10,A), S30 = check_insert(S20,B), @@ -2994,7 +2995,7 @@ lookup_order_2(Opts, Fixed) -> S80 = check_delete(S70,D2b), S90 = check_insert(S80,D2a), SA0 = check_delete(S90,D3a), - SB0 = check_delete(SA0,D3b), + SB0 = check_delete(SA0,D3b), check_insert_new(SB0,D3b), true = ets:delete(T) @@ -3007,7 +3008,7 @@ check_insert({T,List0,Key},Val) -> ets:insert(T,{Key,Val}), List1 = case (ets:info(T,type) =:= bag andalso lists:member({Key,Val},List0)) of - true -> List0; + true -> List0; false -> [{Key,Val} | List0] end, check_check({T,List1,Key}). @@ -3040,8 +3041,6 @@ check_check(S={T,List,Key}) -> Items = length(List), S. - - fill_tab(Tab,Val) -> ets:insert(Tab,{key,Val}), ets:insert(Tab,{{a,144},Val}), @@ -3069,13 +3068,11 @@ lookup_element_mult_do(Opts) -> verify_etsmem(EtsMem). lem_data() -> - [ - {service,'eddie2@boromir',{150,236,14,103},httpd88,self()}, + [{service,'eddie2@boromir',{150,236,14,103},httpd88,self()}, {service,'eddie2@boromir',{150,236,14,103},httpd80,self()}, {service,'eddie3@boromir',{150,236,14,107},httpd88,self()}, {service,'eddie3@boromir',{150,236,14,107},httpd80,self()}, - {service,'eddie4@boromir',{150,236,14,108},httpd88,self()} - ]. + {service,'eddie4@boromir',{150,236,14,108},httpd88,self()}]. lem_crash(T) -> L = ets:lookup_element(T, 'eddie2@boromir', 3), @@ -3126,6 +3123,7 @@ delete_tab_do(Opts) -> %% Check that ets:delete/1 works and that other processes can run. delete_large_tab(Config) when is_list(Config) -> + ct:timetrap({minutes,30}), %% valgrind needs a lot Data = [{erlang:phash2(I, 16#ffffff),I} || I <- lists:seq(1, 200000)], EtsMem = etsmem(), repeat_for_opts(fun(Opts) -> delete_large_tab_do(Opts,Data) end), @@ -3148,7 +3146,7 @@ delete_large_tab_1(Name, Flags, Data, Fix) -> lists:foreach(fun({K,_}) -> ets:delete(Tab, K) end, Data) end, - {priority, Prio} = process_info(self(), priority), + {priority, Prio} = process_info(self(), priority), Deleter = self(), [SchedTracer] = start_loopers(1, @@ -3195,7 +3193,7 @@ delete_large_tab_1(Name, Flags, Data, Fix) -> %% Delete a large name table and try to create a new table with %% the same name in another process. -delete_large_named_table(Config) when is_list(Config) -> +delete_large_named_table(Config) when is_list(Config) -> Data = [{erlang:phash2(I, 16#ffffff),I} || I <- lists:seq(1, 200000)], EtsMem = etsmem(), repeat_for_opts(fun(Opts) -> delete_large_named_table_do(Opts,Data) end), @@ -3566,7 +3564,7 @@ dyn_lookup(T) -> dyn_lookup(T, ets:first(T)). dyn_lookup(_T, '$end_of_table') -> []; dyn_lookup(T, K) -> - NextKey=ets:next(T,K), + NextKey = ets:next(T,K), case ets:next(T,K) of NextKey -> dyn_lookup(T, NextKey); @@ -4085,9 +4083,9 @@ tabfile_ext2_do(Opts,Config) -> Name = make_ref(), [ets:insert(T,{X,integer_to_list(X)}) || X <- L], ok = ets:tab2file(T,FName,[{extended_info,[md5sum]}]), - true = lists:sort(ets:tab2list(T)) =:= + true = lists:sort(ets:tab2list(T)) =:= lists:sort(ets:tab2list(element(2,ets:file2tab(FName)))), - true = lists:sort(ets:tab2list(T)) =:= + true = lists:sort(ets:tab2list(T)) =:= lists:sort(ets:tab2list( element(2,ets:file2tab(FName,[{verify,true}])))), {ok, Name} = disk_log:open([{name,Name},{file,FName}]), @@ -4102,9 +4100,9 @@ tabfile_ext2_do(Opts,Config) -> ets:tab2list( element(2,ets:file2tab(FName2)))), {error,checksum_error} = ets:file2tab(FName2,[{verify,true}]), - {value,{extended_info,[md5sum]}} = + {value,{extended_info,[md5sum]}} = lists:keysearch(extended_info,1,element(2,ets:tabfile_info(FName2))), - {value,{extended_info,[md5sum]}} = + {value,{extended_info,[md5sum]}} = lists:keysearch(extended_info,1,element(2,ets:tabfile_info(FName))), file:delete(FName), file:delete(FName2), @@ -4149,15 +4147,14 @@ tabfile_ext4(Config) when is_list(Config) -> Name2 = make_ref(), [ets:insert(TL,{X,integer_to_list(X)}) || X <- LL], ok = ets:tab2file(TL,FName,[{extended_info,[md5sum]}]), - {ok, Name2} = disk_log:open([{name, Name2}, {file, FName}, + {ok, Name2} = disk_log:open([{name, Name2}, {file, FName}, {mode, read_only}]), {C,[_|_]} = disk_log:chunk(Name2,start), {_,[_|_]} = disk_log:chunk(Name2,C), disk_log:close(Name2), - true = lists:sort(ets:tab2list(TL)) =:= + true = lists:sort(ets:tab2list(TL)) =:= lists:sort(ets:tab2list(element(2,ets:file2tab(FName)))), - Res = [ - begin + Res = [begin {ok,FD} = file:open(FName,[binary,read,write]), {ok, Bin} = file:pread(FD,0,1000), <<B1:N/binary,Ch:8,B2/binary>> = Bin, @@ -4167,7 +4164,7 @@ tabfile_ext4(Config) when is_list(Config) -> ok = file:close(FD), X = case ets:file2tab(FName) of {ok,TL2} -> - true = lists:sort(ets:tab2list(TL)) =/= + true = lists:sort(ets:tab2list(TL)) =/= lists:sort(ets:tab2list(TL2)); _ -> totally_broken @@ -4175,7 +4172,7 @@ tabfile_ext4(Config) when is_list(Config) -> {error,Y} = ets:file2tab(FName,[{verify,true}]), ets:tab2file(TL,FName,[{extended_info,[md5sum]}]), {X,Y} - end || N <- lists:seq(500,600) ], + end || N <- lists:seq(500,600)], io:format("~p~n",[Res]), file:delete(FName), ok. @@ -4404,16 +4401,14 @@ member_do(Opts) -> build_table(L1,L2,Num) -> - T = ets_new(xxx, [ordered_set] - ), + T = ets_new(xxx, [ordered_set]), lists:foreach( fun(X1) -> lists:foreach( fun(X2) -> F = fun(FF,N) -> - ets:insert(T,{{X1,X2,N}, - X1, X2, N}), - case N of + ets:insert(T,{{X1,X2,N}, X1, X2, N}), + case N of 0 -> ok; _ -> @@ -4426,16 +4421,14 @@ build_table(L1,L2,Num) -> T. build_table2(L1,L2,Num) -> - T = ets_new(xxx, [ordered_set] - ), + T = ets_new(xxx, [ordered_set]), lists:foreach( fun(X1) -> lists:foreach( fun(X2) -> F = fun(FF,N) -> - ets:insert(T,{{N,X1,X2}, - N, X1, X2}), - case N of + ets:insert(T,{{N,X1,X2}, N, X1, X2}), + case N of 0 -> ok; _ -> @@ -4726,7 +4719,7 @@ del_one_by_one_dbag_3(T,From,To) -> N = (ets:info(T,size) + 1), Obj2 = {From, integer_to_list(From)}, ets:delete_object(T,Obj2), - N = (ets:info(T,size) + 2) + N = (ets:info(T,size) + 2) end, Next = if From < To -> @@ -4773,14 +4766,14 @@ gen_dets_filename(Config,N) -> filename:join(proplists:get_value(priv_dir,Config), "testdets_" ++ integer_to_list(N) ++ ".dets"). -otp_6842_select_1000(Config) when is_list(Config) -> +otp_6842_select_1000(Config) when is_list(Config) -> Tab = ets_new(xxx,[ordered_set]), [ets:insert(Tab,{X,X}) || X <- lists:seq(1,10000)], AllTrue = lists:duplicate(10,true), AllTrue = [ length( element(1, - ets:select(Tab,[{'_',[],['$_']}],X*1000))) =:= + ets:select(Tab,[{'_',[],['$_']}],X*1000))) =:= X*1000 || X <- lists:seq(1,10) ], Sequences = [[1000,1000,1000,1000,1000,1000,1000,1000,1000,1000], [2000,2000,2000,2000,2000], @@ -4806,7 +4799,13 @@ check_seq(A,B,C) -> false. otp_6338(Config) when is_list(Config) -> - L = binary_to_term(<<131,108,0,0,0,2,104,2,108,0,0,0,2,103,100,0,19,112,112,98,49,95,98,115,49,50,64,98,108,97,100,101,95,48,95,53,0,0,33,50,0,0,0,4,1,98,0,0,23,226,106,100,0,4,101,120,105,116,104,2,108,0,0,0,2,104,2,100,0,3,115,98,109,100,0,19,112,112,98,50,95,98,115,49,50,64,98,108,97,100,101,95,48,95,56,98,0,0,18,231,106,100,0,4,114,101,99,118,106>>), + L = binary_to_term(<<131,108,0,0,0,2,104,2,108,0,0,0,2,103,100,0,19,112,112, + 98,49,95,98,115,49,50,64,98,108,97,100,101,95,48,95,53, + 0,0,33,50,0,0,0,4,1,98,0,0,23,226,106,100,0,4,101,120, + 105,116,104,2,108,0,0,0,2,104,2,100,0,3,115,98,109,100, + 0,19,112,112,98,50,95,98,115,49,50,64,98,108,97,100, + 101,95,48,95,56,98,0,0,18,231,106,100,0,4,114,101,99, + 118,106>>), T = ets_new(xxx,[ordered_set]), lists:foreach(fun(X) -> ets:insert(T,X) end,L), [[4839,recv]] = ets:match(T,{[{sbm,ppb2_bs12@blade_0_8},'$1'],'$2'}), @@ -4825,7 +4824,7 @@ otp_5340_do(Opts) -> ets:delete(T). w(_,0, _) -> ok; -w(T,N, Id) -> +w(T,N, Id) -> ets:insert(T, {N, Id}), w(T,N-1,Id). @@ -4915,7 +4914,7 @@ meta_wb_new(Name, _, Tabs, Opts) -> case (catch ets_new(Name,[named_table|Opts])) of Name -> false = lists:member(Name, Tabs), - [Name | Tabs]; + [Name | Tabs]; {'EXIT',{badarg,_}} -> true = lists:member(Name, Tabs), Tabs @@ -5090,7 +5089,7 @@ meta_lookup_unnamed_read(Config) when is_list(Config) -> Tab end, ExecF = fun(Tab) -> [{key,data}] = ets:lookup(Tab,key), - Tab + Tab end, FiniF = fun(Tab) -> true = ets:delete(Tab) end, @@ -5114,7 +5113,7 @@ meta_lookup_named_read(Config) when is_list(Config) -> Tab end, ExecF = fun(Tab) -> [{key,data}] = ets:lookup(Tab,key), - Tab + Tab end, FiniF = fun(Tab) -> true = ets:delete(Tab) end, @@ -5173,9 +5172,9 @@ smp_fixed_delete_do() -> ets:safe_fixtable(T,true), Buckets = num_of_buckets(T), InitF = fun([ProcN,NumOfProcs|_]) -> {ProcN,NumOfProcs} end, - ExecF = fun({Key,_}) when Key > NumOfObjs -> + ExecF = fun({Key,_}) when Key > NumOfObjs -> [end_of_work]; - ({Key,Increment}) -> + ({Key,Increment}) -> true = ets:delete(T,Key), {Key+Increment,Increment} end, @@ -5204,7 +5203,7 @@ smp_unfix_fix_do() -> T = ets_new(foo,[public,{write_concurrency,true}]), %%Mem = ets:info(T,memory), NumOfObjs = 100000, - Deleted = 50000, + Deleted = 50000, filltabint(T,NumOfObjs), ets:safe_fixtable(T,true), Buckets = num_of_buckets(T), @@ -5217,7 +5216,7 @@ smp_unfix_fix_do() -> true = ets:info(T,fixed), Deleted = get_kept_objects(T), - {Child, Mref} = + {Child, Mref} = my_spawn_opt( fun()-> true = ets:info(T,fixed), @@ -5276,22 +5275,19 @@ otp_8166_do(WC) -> NumOfObjs = 3000, %% Need more than 1000 live objects for match_object to trap one time Deleted = NumOfObjs div 2, filltabint(T,NumOfObjs), - {ReaderPid, ReaderMref} = - my_spawn_opt(fun()-> otp_8166_reader(T,NumOfObjs) end, - [link, monitor, {scheduler,2}]), - {ZombieCrPid, ZombieCrMref} = - my_spawn_opt(fun()-> otp_8166_zombie_creator(T,Deleted) end, - [link, monitor, {scheduler,3}]), + {ReaderPid, ReaderMref} = my_spawn_opt(fun()-> otp_8166_reader(T,NumOfObjs) end, + [link, monitor, {scheduler,2}]), + {ZombieCrPid, ZombieCrMref} = my_spawn_opt(fun()-> otp_8166_zombie_creator(T,Deleted) end, + [link, monitor, {scheduler,3}]), repeat(fun() -> ZombieCrPid ! {loop, self()}, zombies_created = receive_any(), otp_8166_trapper(T, 10, ZombieCrPid) - end, - 100), + end, 100), ReaderPid ! quit, {'DOWN', ReaderMref, process, ReaderPid, normal} = receive_any(), - ZombieCrPid ! quit, + ZombieCrPid ! quit, {'DOWN', ZombieCrMref, process, ZombieCrPid, normal} = receive_any(), false = ets:info(T,fixed), 0 = get_kept_objects(T), @@ -5301,7 +5297,7 @@ otp_8166_do(WC) -> %% Keep reading the table otp_8166_reader(T, NumOfObjs) -> - repeat_while(fun(0) -> + repeat_while(fun(0) -> receive quit -> {false,done} after 0 -> {true,NumOfObjs} end; @@ -5315,14 +5311,14 @@ otp_8166_reader(T, NumOfObjs) -> otp_8166_trapper(T, Try, ZombieCrPid) -> [] = ets:match_object(T,{'_',"Pink Unicorn"}), case {ets:info(T,fixed),Try} of - {true,1} -> + {true,1} -> io:format("failed to provoke unsafe unfix, give up...\n",[]), ZombieCrPid ! unfix; - {true,_} -> + {true,_} -> io:format("trapper too fast, trying again...\n",[]), otp_8166_trapper(T, Try-1, ZombieCrPid); {false,_} -> done - end. + end. %% Fixate table and create some pseudo-deleted objects (zombies) @@ -5342,7 +5338,7 @@ otp_8166_zombie_creator(T,Deleted) -> repeat_while(fun() -> case ets:info(T,safe_fixed_monotonic_time) of {_,[_P1,_P2]} -> false; - _ -> + _ -> receive unfix -> false after 0 -> true end @@ -5399,7 +5395,7 @@ smp_select_delete(Config) when is_list(Config) -> Mod = 17, Zeros = erlang:make_tuple(Mod,0), InitF = fun(_) -> Zeros end, - ExecF = fun(Diffs0) -> + ExecF = fun(Diffs0) -> case rand:uniform(20) of 1 -> Mod = 17, @@ -5421,7 +5417,7 @@ smp_select_delete(Config) when is_list(Config) -> Diffs1; false -> Diffs0 end - end + end end, FiniF = fun(Result) -> Result end, Results = run_workers_do(InitF,ExecF,FiniF,20000), @@ -5432,7 +5428,7 @@ smp_select_delete(Config) when is_list(Config) -> 0, TotCnts), io:format("LeftInTab = ~p\n",[LeftInTab]), LeftInTab = ets:info(T,size), - lists:foldl(fun(Cnt,Eq) -> + lists:foldl(fun(Cnt,Eq) -> WasCnt = ets:select_count(T, [{{'_', '$1'}, [{'=:=', {'rem', '$1', Mod}, Eq}], @@ -5440,7 +5436,7 @@ smp_select_delete(Config) when is_list(Config) -> io:format("~p: ~p =?= ~p\n",[Eq,Cnt,WasCnt]), Cnt = WasCnt, Eq+1 - end, + end, 0, TotCnts), verify_table_load(T), LeftInTab = ets:select_delete(T, [{{'$1','$1'}, [], [true]}]), @@ -5478,8 +5474,8 @@ types_do(Opts) -> %% OTP-9932: Memory overwrite when inserting large integers in compressed bag. %% Will crash with segv on 64-bit opt if not fixed. otp_9932(Config) when is_list(Config) -> - T = ets:new(xxx, [bag, compressed]), - Fun = fun(N) -> + T = ets:new(xxx, [bag, compressed]), + Fun = fun(N) -> Key = {1316110174588445 bsl N,1316110174588583 bsl N}, S = {Key, Key}, true = ets:insert(T, S), @@ -5495,9 +5491,9 @@ otp_9932(Config) when is_list(Config) -> %% write_concurrency table. otp_9423(Config) when is_list(Config) -> InitF = fun(_) -> {0,0} end, - ExecF = fun({S,F}) -> - receive - stop -> + ExecF = fun({S,F}) -> + receive + stop -> io:format("~p got stop\n", [self()]), [end_of_work | {"Succeded=",S,"Failed=",F}] after 0 -> @@ -5593,12 +5589,12 @@ take(Config) when is_list(Config) -> %% Utility functions: %% -add_lists(L1,L2) -> +add_lists(L1,L2) -> add_lists(L1,L2,[]). add_lists([],[],Acc) -> lists:reverse(Acc); add_lists([E1|T1], [E2|T2], Acc) -> - add_lists(T1, T2, [E1+E2 | Acc]). + add_lists(T1, T2, [E1+E2 | Acc]). run_workers(InitF,ExecF,FiniF,Laps) -> run_workers(InitF,ExecF,FiniF,Laps, 0). @@ -5644,9 +5640,9 @@ worker_loop(infinite, ExecF, State) -> worker_loop(N, ExecF, State) -> worker_loop(N-1,ExecF,ExecF(State)). -wait_pids(Pids) -> +wait_pids(Pids) -> wait_pids(Pids,[]). -wait_pids([],Acc) -> +wait_pids([],Acc) -> Acc; wait_pids(Pids, Acc) -> receive @@ -5683,7 +5679,7 @@ etsmem() -> wait_for_memory_deallocations(), AllTabs = lists:map(fun(T) -> {T,ets:info(T,name),ets:info(T,size), - ets:info(T,memory),ets:info(T,type)} + ets:info(T,memory),ets:info(T,type)} end, ets:all()), EtsAllocInfo = erlang:system_info({allocator,ets_alloc}), @@ -5895,7 +5891,7 @@ receive_any() -> receive_any_spinning() -> receive_any_spinning(1000000). receive_any_spinning(Loops) -> - receive_any_spinning(Loops,Loops,1). + receive_any_spinning(Loops,Loops,1). receive_any_spinning(Loops,0,Tries) -> receive M -> io:format("Spinning process ~p got msg ~p after ~p tries\n", [self(),M,Tries]), diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl index cb778c96d4..02b7cb10c2 100644 --- a/lib/stdlib/test/rand_SUITE.erl +++ b/lib/stdlib/test/rand_SUITE.erl @@ -18,7 +18,11 @@ %% %CopyrightEnd% -module(rand_SUITE). --export([all/0, suite/0,groups/0]). +-compile({nowarn_deprecated_function,[{random,seed,1}, + {random,uniform_s,1}, + {random,uniform_s,2}]}). + +-export([all/0, suite/0, groups/0, group/1]). -export([interval_int/1, interval_float/1, seed/1, api_eq/1, reference/1, @@ -47,18 +51,22 @@ groups() -> [{basic_stats, [parallel], [basic_stats_uniform_1, basic_stats_uniform_2, basic_stats_normal]}]. +group(basic_stats) -> + %% valgrind needs a lot of time + [{timetrap,{minutes,10}}]. + %% A simple helper to test without test_server during dev test() -> Tests = all(), lists:foreach(fun(Test) -> - try - ok = ?MODULE:Test([]), - io:format("~p: ok~n", [Test]) - catch _:Reason -> - io:format("Failed: ~p: ~p ~p~n", - [Test, Reason, erlang:get_stacktrace()]) - end - end, Tests). + try + ok = ?MODULE:Test([]), + io:format("~p: ok~n", [Test]) + catch _:Reason -> + io:format("Failed: ~p: ~p ~p~n", + [Test, Reason, erlang:get_stacktrace()]) + end + end, Tests). algs() -> [exs64, exsplus, exs1024]. diff --git a/lib/tools/src/fprof.erl b/lib/tools/src/fprof.erl index 1291a3e5ec..d1a4624419 100644 --- a/lib/tools/src/fprof.erl +++ b/lib/tools/src/fprof.erl @@ -1702,6 +1702,12 @@ trace_handler({trace_ts, Pid, send, _OtherPid, _Msg, TS} = Trace, dump_stack(Dump, get(Pid), Trace), TS; %% +%% send_to_non_existing_process +trace_handler({trace_ts, Pid, send_to_non_existing_process, _OtherPid, _Msg, TS} = Trace, + _Table, _, Dump) -> + dump_stack(Dump, get(Pid), Trace), + TS; +%% %% 'receive' trace_handler({trace_ts, Pid, 'receive', _Msg, TS} = Trace, _Table, _, Dump) -> |