diff options
Diffstat (limited to 'lib')
262 files changed, 6242 insertions, 3765 deletions
diff --git a/lib/asn1/src/asn1ct_gen_per.erl b/lib/asn1/src/asn1ct_gen_per.erl index 82e9326294..c09b0f47d1 100644 --- a/lib/asn1/src/asn1ct_gen_per.erl +++ b/lib/asn1/src/asn1ct_gen_per.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2017. All Rights Reserved. +%% Copyright Ericsson AB 1997-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -47,14 +47,20 @@ dialyzer_suppressions(#gen{erule=per,aligned=Aligned}) -> false -> uper; true -> per end, - case asn1ct_func:is_used({Mod,complete,1}) of + suppress({Mod,complete,1}), + suppress({per_common,to_bitstring,2}), + emit([" ok.",nl]). + +suppress({M,F,A}=MFA) -> + case asn1ct_func:is_used(MFA) of false -> ok; true -> - emit([" _ = complete(Arg),",nl]) - end, - emit([" ok.",nl]). - + Args = + [lists:concat(["element(",I,", Arg)"]) + || I <- lists:seq(1, A)], + emit([" ",{call,M,F,Args},com,nl]) + end. gen_encode(Erules,Type) when is_record(Type,typedef) -> gen_encode_user(Erules,Type). diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 69e371a30f..fd7fa07b81 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -872,8 +872,8 @@ fail(Reason) -> try exit({test_case_failed,Reason}) catch - Class:R -> - case erlang:get_stacktrace() of + Class:R:S -> + case S of [{?MODULE,fail,1,_}|Stk] -> ok; Stk -> ok end, @@ -894,8 +894,8 @@ fail(Format, Args) -> try exit({test_case_failed,lists:flatten(Str)}) catch - Class:R -> - case erlang:get_stacktrace() of + Class:R:S -> + case S of [{?MODULE,fail,2,_}|Stk] -> ok; Stk -> ok end, diff --git a/lib/common_test/src/ct_hooks.erl b/lib/common_test/src/ct_hooks.erl index f0592a40be..bb33f0243b 100644 --- a/lib/common_test/src/ct_hooks.erl +++ b/lib/common_test/src/ct_hooks.erl @@ -233,8 +233,7 @@ call([{Hook, call_id, NextFun} | Rest], Config, Meta, Hooks) -> Rest ++ [{NewId, call_init}, {NewId,NextFun}]} end, call(resort(NewRest,NewHooks,Meta), Config, Meta, NewHooks) - catch Error:Reason -> - Trace = erlang:get_stacktrace(), + catch Error:Reason:Trace -> ct_logs:log("Suite Hook","Failed to start a CTH: ~tp:~tp", [Error,{Reason,Trace}]), call([], {fail,"Failed to start CTH" @@ -422,8 +421,7 @@ catch_apply(M,F,A, Default, Fallback) -> catch_apply(M,F,A) -> try erlang:apply(M,F,A) - catch _:Reason -> - Trace = erlang:get_stacktrace(), + catch _:Reason:Trace -> ct_logs:log("Suite Hook","Call to CTH failed: ~w:~tp", [error,{Reason,Trace}]), throw({error_in_cth_call, diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 8c401cf3f5..afc5fee4f7 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -442,11 +442,9 @@ script_start2(Opts = #opts{vts = undefined, TestSpecData -> execute_all_specs(TestSpecData, Opts, Args, []) catch - throw:{error,Reason} -> - StackTrace = erlang:get_stacktrace(), + throw:{error,Reason}:StackTrace -> {error,{invalid_testspec,{Reason,StackTrace}}}; - _:Reason -> - StackTrace = erlang:get_stacktrace(), + _:Reason:StackTrace -> {error,{invalid_testspec,{Reason,StackTrace}}} end; [] -> @@ -1211,11 +1209,9 @@ run_spec_file(Relaxed, TestSpecData -> run_all_specs(TestSpecData, Opts, StartOpts, []) catch - throw:{error,CTReason} -> - StackTrace = erlang:get_stacktrace(), + throw:{error,CTReason}:StackTrace -> exit({error,{invalid_testspec,{CTReason,StackTrace}}}); - _:CTReason -> - StackTrace = erlang:get_stacktrace(), + _:CTReason:StackTrace -> exit({error,{invalid_testspec,{CTReason,StackTrace}}}) end. diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index bb445bb0d2..bd3755722f 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -1425,7 +1425,12 @@ skip_groups1(Suite,Groups,Cmt,Suites0) -> GrAndCases1 = GrAndCases0 ++ SkipGroups, insert_in_order({Suite,GrAndCases1},Suites0,replace); false -> - insert_in_order({Suite,SkipGroups},Suites0,replace) + case Suites0 of + [{all,_}=All|Skips]-> + [All|Skips++[{Suite,SkipGroups}]]; + _ -> + insert_in_order({Suite,SkipGroups},Suites0,replace) + end end. skip_cases(Node,Dir,Suite,Cases,Cmt,Tests,false) when is_list(Cases) -> diff --git a/lib/common_test/src/ct_util.erl b/lib/common_test/src/ct_util.erl index 10a06d5c88..e904bb1e7c 100644 --- a/lib/common_test/src/ct_util.erl +++ b/lib/common_test/src/ct_util.erl @@ -209,11 +209,10 @@ do_start(Parent, Mode, LogDir, Verbosity) -> self() ! {{stop,{self(),{user_error,CTHReason}}}, {Parent,make_ref()}} catch - _:CTHReason -> + _:CTHReason:StackTrace -> ErrorInfo = if is_atom(CTHReason) -> io_lib:format("{~tp,~tp}", - [CTHReason, - erlang:get_stacktrace()]); + [CTHReason, StackTrace]); true -> CTHReason end, diff --git a/lib/common_test/src/test_server.erl b/lib/common_test/src/test_server.erl index 7250041e13..9469619aa9 100644 --- a/lib/common_test/src/test_server.erl +++ b/lib/common_test/src/test_server.erl @@ -1340,13 +1340,12 @@ do_init_per_testcase(Mod, Args) -> {skip,Reason}; exit:{Skip,Reason} when Skip =:= skip; Skip =:= skipped -> {skip,Reason}; - throw:Other -> - set_loc(erlang:get_stacktrace()), + throw:Other:Stk -> + set_loc(Stk), Line = get_loc(), print_init_conf_result(Line,"thrown",Other), {skip,{failed,{Mod,init_per_testcase,Other}}}; - _:Reason0 -> - Stk = erlang:get_stacktrace(), + _:Reason0:Stk -> Reason = {Reason0,Stk}, set_loc(Stk), Line = get_loc(), @@ -1395,20 +1394,19 @@ do_end_per_testcase(Mod,EndFunc,Func,Conf) -> _ -> ok catch - throw:Other -> + throw:Other:Stk -> Comment0 = case read_comment() of "" -> ""; Cmt -> Cmt ++ test_server_ctrl:xhtml("<br>", "<br />") end, - set_loc(erlang:get_stacktrace()), + set_loc(Stk), comment(io_lib:format("~ts<font color=\"red\">" "WARNING: ~w thrown!" "</font>\n",[Comment0,EndFunc])), print_end_tc_warning(EndFunc,Other,"thrown",get_loc()), {failed,{Mod,end_per_testcase,Other}}; - Class:Reason -> - Stk = erlang:get_stacktrace(), + Class:Reason:Stk -> set_loc(Stk), Why = case Class of exit -> {'EXIT',Reason}; @@ -1550,8 +1548,7 @@ ts_tc(M, F, A) -> throw:{skipped, Reason} -> {skip, Reason}; exit:{skip, Reason} -> {skip, Reason}; exit:{skipped, Reason} -> {skip, Reason}; - Type:Reason -> - Stk = erlang:get_stacktrace(), + Type:Reason:Stk -> set_loc(Stk), case Type of throw -> @@ -1740,8 +1737,8 @@ fail(Reason) -> try exit({suite_failed,Reason}) catch - Class:R -> - case erlang:get_stacktrace() of + Class:R:Stacktrace -> + case Stacktrace of [{?MODULE,fail,1,_}|Stk] -> ok; Stk -> ok end, @@ -1763,8 +1760,8 @@ fail() -> try exit(suite_failed) catch - Class:R -> - case erlang:get_stacktrace() of + Class:R:Stacktrace -> + case Stacktrace of [{?MODULE,fail,0,_}|Stk] -> ok; Stk -> ok end, @@ -2043,15 +2040,15 @@ call_user_timetrap(Func, Sup) when is_function(Func) -> try Func() of Result -> Sup ! {self(),Result} - catch _:Error -> - exit({Error,erlang:get_stacktrace()}) + catch _:Error:Stk -> + exit({Error,Stk}) end; call_user_timetrap({M,F,A}, Sup) -> try apply(M,F,A) of Result -> Sup ! {self(),Result} - catch _:Error -> - exit({Error,erlang:get_stacktrace()}) + catch _:Error:Stk -> + exit({Error,Stk}) end. save_user_timetrap(TCPid, UserTTSup, StartTime) -> diff --git a/lib/common_test/src/test_server_sup.erl b/lib/common_test/src/test_server_sup.erl index 6ddbf1ad27..139621141d 100644 --- a/lib/common_test/src/test_server_sup.erl +++ b/lib/common_test/src/test_server_sup.erl @@ -781,8 +781,8 @@ framework_call(Callback,Func,Args,DefaultReturn) -> catch exit:Why -> EH(Why); - error:Why -> - EH({Why,erlang:get_stacktrace()}); + error:Why:Stacktrace -> + EH({Why,Stacktrace}); throw:Why -> EH(Why) end; diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl index 63bf9be134..975f6cafcb 100644 --- a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl +++ b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl @@ -231,8 +231,7 @@ data_for_channel(CM, Ch, Data, State) -> {ok, NewState} end catch - Class:Reason -> - Stacktrace = erlang:get_stacktrace(), + Class:Reason:Stacktrace -> error_logger:error_report([{?MODULE, data_for_channel}, {request, Data}, {buffer, State#session.buffer}, diff --git a/lib/common_test/test/ct_unicode_SUITE_data/unicode_atoms_SUITE.erl b/lib/common_test/test/ct_unicode_SUITE_data/unicode_atoms_SUITE.erl index 993452500e..041c13cdbd 100644 --- a/lib/common_test/test/ct_unicode_SUITE_data/unicode_atoms_SUITE.erl +++ b/lib/common_test/test/ct_unicode_SUITE_data/unicode_atoms_SUITE.erl @@ -77,7 +77,7 @@ all() -> 'fail_αβ_4'(_Config) -> ct:log("This is test case ~tw",[?FUNCTION_NAME]), - S = try throw(ok) catch throw:ok -> erlang:get_stacktrace() end, + S = try throw(ok) catch throw:ok:Stacktrace -> Stacktrace end, erlang:raise(error,{error,testcase,?FUNCTION_NAME},S), ok. diff --git a/lib/common_test/test_server/ts_install_cth.erl b/lib/common_test/test_server/ts_install_cth.erl index 5d325b1115..35a5ec916c 100644 --- a/lib/common_test/test_server/ts_install_cth.erl +++ b/lib/common_test/test_server/ts_install_cth.erl @@ -108,8 +108,7 @@ pre_init_per_suite(_Suite,Config,State) -> {add_node_name(Config, State), State} catch error:{badmatch,{error,enoent}} -> {add_node_name(Config, State), State}; - Error:Reason -> - Stack = erlang:get_stacktrace(), + Error:Reason:Stack -> ct:pal("~p failed! ~p:{~p,~p}",[?MODULE,Error,Reason,Stack]), {{fail,{?MODULE,{Error,Reason, Stack}}},State} end. diff --git a/lib/compiler/src/beam_a.erl b/lib/compiler/src/beam_a.erl index 7df2edd714..91acb19971 100644 --- a/lib/compiler/src/beam_a.erl +++ b/lib/compiler/src/beam_a.erl @@ -61,6 +61,14 @@ rename_instrs([{'%live',_}|Is]) -> %% Ignore old type of live annotation. Only happens when compiling %% from very old .S files. rename_instrs(Is); +rename_instrs([{get_list,S,D1,D2}|Is]) -> + %% Only happens when compiling from old .S files. + if + D1 =:= S -> + [{get_tl,S,D2},{get_hd,S,D1}|rename_instrs(Is)]; + true -> + [{get_hd,S,D1},{get_tl,S,D2}|rename_instrs(Is)] + end; rename_instrs([I|Is]) -> [rename_instr(I)|rename_instrs(Is)]; rename_instrs([]) -> []. diff --git a/lib/compiler/src/beam_asm.erl b/lib/compiler/src/beam_asm.erl index 453e00fce3..fa919ca862 100644 --- a/lib/compiler/src/beam_asm.erl +++ b/lib/compiler/src/beam_asm.erl @@ -407,7 +407,17 @@ encode_arg({atom, Atom}, Dict0) when is_atom(Atom) -> {Index, Dict} = beam_dict:atom(Atom, Dict0), {encode(?tag_a, Index), Dict}; encode_arg({integer, N}, Dict) -> - {encode(?tag_i, N), Dict}; + %% Conservatily assume that all integers whose absolute + %% value is greater than 1 bsl 128 will be bignums in + %% the runtime system. + if + N >= 1 bsl 128 -> + encode_arg({literal, N}, Dict); + N =< -(1 bsl 128) -> + encode_arg({literal, N}, Dict); + true -> + {encode(?tag_i, N), Dict} + end; encode_arg(nil, Dict) -> {encode(?tag_a, 0), Dict}; encode_arg({f, W}, Dict) -> diff --git a/lib/compiler/src/beam_block.erl b/lib/compiler/src/beam_block.erl index d0536e0669..47a2be8ab5 100644 --- a/lib/compiler/src/beam_block.erl +++ b/lib/compiler/src/beam_block.erl @@ -36,13 +36,11 @@ module({Mod,Exp,Attr,Fs0,Lc}, Opts) -> function({function,Name,Arity,CLabel,Is0}, Blockify) -> try %% Collect basic blocks and optimize them. - Is2 = case Blockify of - true -> - Is1 = blockify(Is0), - embed_lines(Is1); - false -> - Is0 + Is1 = case Blockify of + false -> Is0; + true -> blockify(Is0) end, + Is2 = embed_lines(Is1), Is3 = local_cse(Is2), Is4 = beam_utils:anno_defs(Is3), Is5 = move_allocates(Is4), @@ -110,7 +108,8 @@ collect({put_tuple,A,D}) -> {set,[D],[],{put_tuple,A}}; collect({put,S}) -> {set,[],[S],put}; collect({get_tuple_element,S,I,D}) -> {set,[D],[S],{get_tuple_element,I}}; collect({set_tuple_element,S,D,I}) -> {set,[],[S,D],{set_tuple_element,I}}; -collect({get_list,S,D1,D2}) -> {set,[D1,D2],[S],get_list}; +collect({get_hd,S,D}) -> {set,[D],[S],get_hd}; +collect({get_tl,S,D}) -> {set,[D],[S],get_tl}; collect(remove_message) -> {set,[],[],remove_message}; collect({put_map,F,Op,S,D,R,{list,Puts}}) -> {set,[D],[S|Puts],{alloc,R,{put_map,Op,F}}}; @@ -138,6 +137,11 @@ embed_lines([{block,B2},{line,_}=Line,{block,B1}|T], Acc) -> embed_lines([{block,B1},{line,_}=Line|T], Acc) -> B = {block,[{set,[],[],Line}|B1]}, embed_lines([B|T], Acc); +embed_lines([{block,B2},{block,B1}|T], Acc) -> + %% This can only happen when beam_block is run for + %% the second time. + B = {block,B1++B2}, + embed_lines([B|T], Acc); embed_lines([I|Is], Acc) -> embed_lines(Is, [I|Acc]); embed_lines([], Acc) -> Acc. @@ -205,7 +209,7 @@ move_allocates([]) -> []. move_allocates_1([{'%anno',_}|Is], Acc) -> move_allocates_1(Is, Acc); -move_allocates_1([I|Is], [{set,[],[],{alloc,Live0,Info}}|Acc]=Acc0) -> +move_allocates_1([I|Is], [{set,[],[],{alloc,Live0,Info0}}|Acc]=Acc0) -> case alloc_may_pass(I) of false -> move_allocates_1(Is, [I|Acc0]); @@ -214,6 +218,7 @@ move_allocates_1([I|Is], [{set,[],[],{alloc,Live0,Info}}|Acc]=Acc0) -> not_possible -> move_allocates_1(Is, [I|Acc0]); Live when is_integer(Live) -> + Info = safe_info(Info0), A = {set,[],[],{alloc,Live,Info}}, move_allocates_1(Is, [A,I|Acc]) end @@ -222,12 +227,20 @@ move_allocates_1([I|Is], Acc) -> move_allocates_1(Is, [I|Acc]); move_allocates_1([], Acc) -> Acc. +alloc_may_pass({set,_,[{fr,_}],fmove}) -> false; alloc_may_pass({set,_,_,{alloc,_,_}}) -> false; alloc_may_pass({set,_,_,{set_tuple_element,_}}) -> false; alloc_may_pass({set,_,_,put_list}) -> false; alloc_may_pass({set,_,_,put}) -> false; alloc_may_pass({set,_,_,_}) -> true. +safe_info({nozero,Stack,Heap,_}) -> + %% nozero is not safe if the allocation instruction is moved + %% upwards past an instruction that may throw an exception + %% (such as element/2). + {zero,Stack,Heap,[]}; +safe_info(Info) -> Info. + %% opt([Instruction]) -> [Instruction] %% Optimize the instruction stream inside a basic block. @@ -251,6 +264,16 @@ opt([{set,[D1],[{integer,Idx1},Reg],{bif,element,{f,L}}}=I1, {set,[D2],[{integer,Idx2},Reg],{bif,element,{f,L}}}=I2|Is]) when Idx1 < Idx2, D1 =/= D2, D1 =/= Reg, D2 =/= Reg -> opt([I2,I1|Is]); +opt([{set,Hd0,Cons,get_hd}=GetHd, + {set,Tl0,Cons,get_tl}=GetTl|Is0]) -> + case {opt_moves(Hd0, [GetTl|Is0]),opt_moves(Tl0, [GetHd|Is0])} of + {{Hd0,Is},{Tl0,_}} -> + [GetHd|opt(Is)]; + {{Hd,Is},{Tl0,_}} -> + [{set,Hd,Cons,get_hd}|opt(Is)]; + {{_,_},{Tl,Is}} -> + [{set,Tl,Cons,get_tl}|opt(Is)] + end; opt([{set,Ds0,Ss,Op}|Is0]) -> {Ds,Is} = opt_moves(Ds0, Is0), [{set,Ds,Ss,Op}|opt(Is)]; @@ -266,17 +289,6 @@ opt_moves([D0]=Ds, Is0) -> case opt_move(D0, Is0) of not_possible -> {Ds,Is0}; {D1,Is} -> {[D1],Is} - end; -opt_moves([X0,Y0], Is0) -> - {X,Is2} = case opt_move(X0, Is0) of - not_possible -> {X0,Is0}; - {Y0,_} -> {X0,Is0}; - {_X1,_Is1} = XIs1 -> XIs1 - end, - case opt_move(Y0, Is2) of - not_possible -> {[X,Y0],Is2}; - {X,_} -> {[X,Y0],Is2}; - {Y,Is} -> {[X,Y],Is} end. %% opt_move(Dest, [Instruction]) -> {UpdatedDest,[Instruction]} | not_possible @@ -619,7 +631,13 @@ cse_find(Expr, Es) -> end. cse_expr({set,[D],Ss,{bif,N,_}}) -> - {ok,D,{{bif,N},Ss}}; + case D of + {fr,_} -> + %% There are too many things that can go wrong. + none; + _ -> + {ok,D,{{bif,N},Ss}} + end; cse_expr({set,[D],Ss,{alloc,_,{gc_bif,N,_}}}) -> {ok,D,{{gc_bif,N},Ss}}; cse_expr({set,[D],Ss,put_list}) -> diff --git a/lib/compiler/src/beam_clean.erl b/lib/compiler/src/beam_clean.erl index e094c2c320..7ddf9fa2e2 100644 --- a/lib/compiler/src/beam_clean.erl +++ b/lib/compiler/src/beam_clean.erl @@ -24,7 +24,7 @@ -export([module/2]). -export([bs_clean_saves/1]). -export([clean_labels/1]). --import(lists, [foldl/3,reverse/1,filter/2]). +-import(lists, [foldl/3,reverse/1]). -spec module(beam_utils:module_code(), [compile:option()]) -> {'ok',beam_utils:module_code()}. @@ -303,8 +303,21 @@ maybe_remove_lines(Fs, Opts) -> end. remove_lines([{function,N,A,Lbl,Is0}|T]) -> - Is = filter(fun({line,_}) -> false; - (_) -> true - end, Is0), + Is = remove_lines_fun(Is0), [{function,N,A,Lbl,Is}|remove_lines(T)]; remove_lines([]) -> []. + +remove_lines_fun([{line,_}|Is]) -> + remove_lines_fun(Is); +remove_lines_fun([{block,Bl0}|Is]) -> + Bl = remove_lines_block(Bl0), + [{block,Bl}|remove_lines_fun(Is)]; +remove_lines_fun([I|Is]) -> + [I|remove_lines_fun(Is)]; +remove_lines_fun([]) -> []. + +remove_lines_block([{set,_,_,{line,_}}|Is]) -> + remove_lines_block(Is); +remove_lines_block([I|Is]) -> + [I|remove_lines_block(Is)]; +remove_lines_block([]) -> []. diff --git a/lib/compiler/src/beam_disasm.erl b/lib/compiler/src/beam_disasm.erl index 50b76d7f29..a68c4b5367 100644 --- a/lib/compiler/src/beam_disasm.erl +++ b/lib/compiler/src/beam_disasm.erl @@ -1090,6 +1090,10 @@ resolve_inst({build_stacktrace,[]},_,_,_) -> build_stacktrace; resolve_inst({raw_raise,[]},_,_,_) -> raw_raise; +resolve_inst({get_hd,[Src,Dst]},_,_,_) -> + {get_hd,Src,Dst}; +resolve_inst({get_tl,[Src,Dst]},_,_,_) -> + {get_tl,Src,Dst}; %% %% Catches instructions that are not yet handled. diff --git a/lib/compiler/src/beam_flatten.erl b/lib/compiler/src/beam_flatten.erl index a4d45a4ca6..c60211f516 100644 --- a/lib/compiler/src/beam_flatten.erl +++ b/lib/compiler/src/beam_flatten.erl @@ -50,6 +50,9 @@ norm_block([{set,[],[],{alloc,R,Alloc}}|Is], Acc0) -> Acc -> norm_block(Is, Acc) end; +norm_block([{set,[D1],[S],get_hd},{set,[D2],[S],get_tl}|Is], Acc) -> + I = {get_list,S,D1,D2}, + norm_block(Is, [I|Acc]); norm_block([I|Is], Acc) -> norm_block(Is, [norm(I)|Acc]); norm_block([], Acc) -> Acc. @@ -64,12 +67,14 @@ norm({set,[D],[],{put_tuple,A}}) -> {put_tuple,A,D}; norm({set,[],[S],put}) -> {put,S}; norm({set,[D],[S],{get_tuple_element,I}}) -> {get_tuple_element,S,I,D}; norm({set,[],[S,D],{set_tuple_element,I}}) -> {set_tuple_element,S,D,I}; -norm({set,[D1,D2],[S],get_list}) -> {get_list,S,D1,D2}; +norm({set,[D],[S],get_hd}) -> {get_hd,S,D}; +norm({set,[D],[S],get_tl}) -> {get_tl,S,D}; norm({set,[D],[S|Puts],{alloc,R,{put_map,Op,F}}}) -> {put_map,F,Op,S,D,R,{list,Puts}}; norm({set,[],[],remove_message}) -> remove_message; norm({set,[],[],fclearerror}) -> fclearerror; -norm({set,[],[],fcheckerror}) -> {fcheckerror,{f,0}}. +norm({set,[],[],fcheckerror}) -> {fcheckerror,{f,0}}; +norm({set,[],[],{line,_}=Line}) -> Line. norm_allocate({_Zero,nostack,Nh,[]}, Regs) -> [{test_heap,Nh,Regs}]; diff --git a/lib/compiler/src/beam_split.erl b/lib/compiler/src/beam_split.erl index d041f18806..52dd89b5bb 100644 --- a/lib/compiler/src/beam_split.erl +++ b/lib/compiler/src/beam_split.erl @@ -50,8 +50,9 @@ split_block([{set,[R],[_,_,_]=As,{bif,is_record,{f,Lbl}}}|Is], Bl, Acc) -> split_block(Is, [], [{bif,is_record,{f,Lbl},As,R}|make_block(Bl, Acc)]); split_block([{set,[R],As,{bif,N,{f,Lbl}=Fail}}|Is], Bl, Acc) when Lbl =/= 0 -> split_block(Is, [], [{bif,N,Fail,As,R}|make_block(Bl, Acc)]); -split_block([{set,[R],As,{bif,raise,{f,_}=Fail}}|Is], Bl, Acc) -> - split_block(Is, [], [{bif,raise,Fail,As,R}|make_block(Bl, Acc)]); +split_block([{set,[],[],{line,_}=Line}, + {set,[R],As,{bif,raise,{f,_}=Fail}}|Is], Bl, Acc) -> + split_block(Is, [], [{bif,raise,Fail,As,R},Line|make_block(Bl, Acc)]); split_block([{set,[R],As,{alloc,Live,{gc_bif,N,{f,Lbl}=Fail}}}|Is], Bl, Acc) when Lbl =/= 0 -> split_block(Is, [], [{gc_bif,N,Fail,Live,As,R}|make_block(Bl, Acc)]); @@ -61,8 +62,6 @@ split_block([{set,[D],[S|Puts],{alloc,R,{put_map,Op,{f,Lbl}=Fail}}}|Is], make_block(Bl, Acc)]); split_block([{set,[R],[],{try_catch,Op,L}}|Is], Bl, Acc) -> split_block(Is, [], [{Op,R,L}|make_block(Bl, Acc)]); -split_block([{set,[],[],{line,_}=Line}|Is], Bl, Acc) -> - split_block(Is, [], [Line|make_block(Bl, Acc)]); split_block([I|Is], Bl, Acc) -> split_block(Is, [I|Bl], Acc); split_block([], Bl, Acc) -> make_block(Bl, Acc). diff --git a/lib/compiler/src/beam_type.erl b/lib/compiler/src/beam_type.erl index b2fabed2c5..b83ed17b55 100644 --- a/lib/compiler/src/beam_type.erl +++ b/lib/compiler/src/beam_type.erl @@ -477,8 +477,6 @@ update({set,[D],[S1,S2],{alloc,_,{gc_bif,Op,{f,0}}}}, Ts0) -> update({set,[],_Src,_Op}, Ts0) -> Ts0; update({set,[D],_Src,_Op}, Ts0) -> tdb_update([{D,kill}], Ts0); -update({set,[D1,D2],_Src,_Op}, Ts0) -> - tdb_update([{D1,kill},{D2,kill}], Ts0); update({kill,D}, Ts) -> tdb_update([{D,kill}], Ts); @@ -943,10 +941,10 @@ merge_type_info({tuple,SzKind1,Sz1,[]}, {tuple,_SzKind2,_Sz2,First}=Tuple2) -> merge_type_info({tuple,SzKind1,Sz1,First}, Tuple2); merge_type_info({tuple,_SzKind1,_Sz1,First}=Tuple1, {tuple,SzKind2,Sz2,_}) -> merge_type_info(Tuple1, {tuple,SzKind2,Sz2,First}); -merge_type_info(integer, {integer,_}=Int) -> - Int; -merge_type_info({integer,_}=Int, integer) -> - Int; +merge_type_info(integer, {integer,_}) -> + integer; +merge_type_info({integer,_}, integer) -> + integer; merge_type_info({integer,{Min1,Max1}}, {integer,{Min2,Max2}}) -> {integer,{max(Min1, Min2),min(Max1, Max2)}}; merge_type_info({binary,U1}, {binary,U2}) -> diff --git a/lib/compiler/src/beam_utils.erl b/lib/compiler/src/beam_utils.erl index 5333925589..814cfb8265 100644 --- a/lib/compiler/src/beam_utils.erl +++ b/lib/compiler/src/beam_utils.erl @@ -118,7 +118,7 @@ is_killed(R, Is, D) -> St = #live{lbl=D,res=gb_trees:empty()}, case check_liveness(R, Is, St) of {killed,_} -> true; - {exit_not_used,_} -> true; + {exit_not_used,_} -> false; {_,_} -> false end. @@ -131,7 +131,7 @@ is_killed_at(R, Lbl, D) when is_integer(Lbl) -> St0 = #live{lbl=D,res=gb_trees:empty()}, case check_liveness_at(R, Lbl, St0) of {killed,_} -> true; - {exit_not_used,_} -> true; + {exit_not_used,_} -> false; {_,_} -> false end. @@ -148,7 +148,7 @@ is_not_used(R, Is, D) -> St = #live{lbl=D,res=gb_trees:empty()}, case check_liveness(R, Is, St) of {used,_} -> false; - {exit_not_used,_} -> false; + {exit_not_used,_} -> true; {_,_} -> true end. @@ -440,8 +440,11 @@ check_liveness(R, [{bs_init,_,_,Live,Ss,Dst}|Is], St) -> case member(R, Ss) of true -> {used,St}; false -> + %% If the exception is taken, the stack may + %% be scanned. Therefore the register is not + %% guaranteed to be killed. if - R =:= Dst -> {killed,St}; + R =:= Dst -> {not_used,St}; true -> not_used(check_liveness(R, Is, St)) end end @@ -602,8 +605,11 @@ check_liveness(R, [{test_heap,N,Live}|Is], St) -> check_liveness(R, [{allocate_zero,N,Live}|Is], St) -> I = {block,[{set,[],[],{alloc,Live,{zero,N,0,[]}}}]}, check_liveness(R, [I|Is], St); -check_liveness(R, [{get_list,S,D1,D2}|Is], St) -> - I = {block,[{set,[D1,D2],[S],get_list}]}, +check_liveness(R, [{get_hd,S,D}|Is], St) -> + I = {block,[{set,[D],[S],get_hd}]}, + check_liveness(R, [I|Is], St); +check_liveness(R, [{get_tl,S,D}|Is], St) -> + I = {block,[{set,[D],[S],get_tl}]}, check_liveness(R, [I|Is], St); check_liveness(R, [remove_message|Is], St) -> check_liveness(R, Is, St); @@ -732,8 +738,8 @@ check_liveness_block_1(R, Ss, Ds, Op, Is, St0) -> end end. -check_liveness_block_2(R, {gc_bif,_Op,{f,Lbl}}, _Ss, St) -> - check_liveness_block_3(R, Lbl, St); +check_liveness_block_2(R, {gc_bif,Op,{f,Lbl}}, Ss, St) -> + check_liveness_block_3(R, Lbl, {Op,length(Ss)}, St); check_liveness_block_2(R, {bif,Op,{f,Lbl}}, Ss, St) -> Arity = length(Ss), case erl_internal:comp_op(Op, Arity) orelse @@ -741,16 +747,23 @@ check_liveness_block_2(R, {bif,Op,{f,Lbl}}, Ss, St) -> true -> {killed,St}; false -> - check_liveness_block_3(R, Lbl, St) + check_liveness_block_3(R, Lbl, {Op,length(Ss)}, St) end; check_liveness_block_2(R, {put_map,_Op,{f,Lbl}}, _Ss, St) -> - check_liveness_block_3(R, Lbl, St); + check_liveness_block_3(R, Lbl, {unsafe,0}, St); check_liveness_block_2(_, _, _, St) -> {killed,St}. -check_liveness_block_3(_, 0, St) -> +check_liveness_block_3({x,_}, 0, _FA, St) -> {killed,St}; -check_liveness_block_3(R, Lbl, St0) -> +check_liveness_block_3({y,_}, 0, {F,A}, St) -> + %% If the exception is thrown, the stack may be scanned, + %% thus implicitly using the y register. + case erl_bifs:is_safe(erlang, F, A) of + true -> {killed,St}; + false -> {used,St} + end; +check_liveness_block_3(R, Lbl, _FA, St0) -> check_liveness_at(R, Lbl, St0). index_labels_1([{label,Lbl}|Is0], Acc) -> @@ -991,47 +1004,52 @@ live_opt([{recv_mark,_}=I|Is], Regs, D, Acc) -> live_opt([], _, _, Acc) -> Acc. -live_opt_block([{set,Ds,Ss,Op0}|Is], Regs0, D, Acc) -> - Regs1 = x_live(Ss, x_dead(Ds, Regs0)), - {Op, Regs} = live_opt_block_op(Op0, Regs1, D), - I = {set, Ds, Ss, Op}, - - case Ds of - [{x,X}] -> - case (not is_live(X, Regs0)) andalso Op =:= move of - true -> - live_opt_block(Is, Regs0, D, Acc); - false -> - live_opt_block(Is, Regs, D, [I|Acc]) - end; - _ -> - live_opt_block(Is, Regs, D, [I|Acc]) +live_opt_block([{set,[{x,X}]=Ds,Ss,move}=I|Is], Regs0, D, Acc) -> + Regs = x_live(Ss, x_dead(Ds, Regs0)), + case is_live(X, Regs0) of + true -> + live_opt_block(Is, Regs, D, [I|Acc]); + false -> + %% Useless move, will never be used. + live_opt_block(Is, Regs, D, Acc) end; -live_opt_block([{'%anno',_}|Is], Regs, D, Acc) -> - live_opt_block(Is, Regs, D, Acc); -live_opt_block([], Regs, _, Acc) -> {Acc,Regs}. - -live_opt_block_op({alloc,Live0,AllocOp}, Regs0, D) -> - Regs = - case AllocOp of - {Kind, _N, Fail} when Kind =:= gc_bif; Kind =:= put_map -> - live_join_label(Fail, D, Regs0); - _ -> - Regs0 - end, +live_opt_block([{set,Ds,Ss,{alloc,Live0,AllocOp}}|Is], Regs0, D, Acc) -> + %% Calculate liveness from the point of view of the GC. + %% There will never be a GC if the instruction fails, so we should + %% ignore the failure branch. + GcRegs1 = x_dead(Ds, Regs0), + GcRegs = x_live(Ss, GcRegs1), + Live = live_regs(GcRegs), %% The life-time analysis used by the code generator is sometimes too %% conservative, so it may be possible to lower the number of live %% registers based on the exact liveness information. The main benefit is %% that more optimizations that depend on liveness information (such as the - %% beam_bool and beam_dead passes) may be applied. - Live = live_regs(Regs), - true = Live =< Live0, - {{alloc,Live,AllocOp}, live_call(Live)}; -live_opt_block_op({bif,_N,Fail} = Op, Regs, D) -> - {Op, live_join_label(Fail, D, Regs)}; -live_opt_block_op(Op, Regs, _D) -> - {Op, Regs}. + %% beam_dead pass) may be applied. + true = Live =< Live0, %Assertion. + I = {set,Ds,Ss,{alloc,Live,AllocOp}}, + + %% Calculate liveness from the point of view of the preceding instruction. + %% The liveness is the union of live registers in the GC and the live + %% registers at the failure label. + Regs1 = live_call(Live), + Regs = live_join_alloc(AllocOp, D, Regs1), + live_opt_block(Is, Regs, D, [I|Acc]); +live_opt_block([{set,Ds,Ss,{bif,_,Fail}}=I|Is], Regs0, D, Acc) -> + Regs1 = x_dead(Ds, Regs0), + Regs2 = x_live(Ss, Regs1), + Regs = live_join_label(Fail, D, Regs2), + live_opt_block(Is, Regs, D, [I|Acc]); +live_opt_block([{set,Ds,Ss,_}=I|Is], Regs0, D, Acc) -> + Regs = x_live(Ss, x_dead(Ds, Regs0)), + live_opt_block(Is, Regs, D, [I|Acc]); +live_opt_block([{'%anno',_}|Is], Regs, D, Acc) -> + live_opt_block(Is, Regs, D, Acc); +live_opt_block([], Regs, _, Acc) -> {Acc,Regs}. + +live_join_alloc({Kind,_Name,Fail}, D, Regs) when Kind =:= gc_bif; Kind =:= put_map -> + live_join_label(Fail, D, Regs); +live_join_alloc(_, _, Regs) -> Regs. live_join_labels([{f,L}|T], D, Regs0) when L =/= 0 -> Regs = gb_trees:get(L, D) bor Regs0, diff --git a/lib/compiler/src/beam_validator.erl b/lib/compiler/src/beam_validator.erl index f8bf935132..c30ab34ac7 100644 --- a/lib/compiler/src/beam_validator.erl +++ b/lib/compiler/src/beam_validator.erl @@ -591,6 +591,12 @@ valfun_4({get_list,Src,D1,D2}, Vst0) -> assert_type(cons, Src, Vst0), Vst = set_type_reg(term, D1, Vst0), set_type_reg(term, D2, Vst); +valfun_4({get_hd,Src,Dst}, Vst) -> + assert_type(cons, Src, Vst), + set_type_reg(term, Dst, Vst); +valfun_4({get_tl,Src,Dst}, Vst) -> + assert_type(cons, Src, Vst), + set_type_reg(term, Dst, Vst); valfun_4({get_tuple_element,Src,I,Dst}, Vst) -> assert_type({tuple_element,I+1}, Src, Vst), set_type_reg(term, Dst, Vst); @@ -1147,6 +1153,7 @@ set_type_y(Type, {y,Y}=Reg, #vst{current=#st{y=Ys0}=St}=Vst) {value,_} -> gb_trees:update(Y, Type, Ys0) end, + check_try_catch_tags(Type, Y, Ys0), Vst#vst{current=St#st{y=Ys}}; set_type_y(Type, Reg, #vst{}) -> error({invalid_store,Reg,Type}). @@ -1154,6 +1161,29 @@ set_catch_end({y,Y}, #vst{current=#st{y=Ys0}=St}=Vst) -> Ys = gb_trees:update(Y, initialized, Ys0), Vst#vst{current=St#st{y=Ys}}. +check_try_catch_tags(Type, LastY, Ys) -> + case is_try_catch_tag(Type) of + false -> + ok; + true -> + %% Every catch or try/catch must use a lower Y register + %% number than any enclosing catch or try/catch. That will + %% ensure that when the stack is scanned when an + %% exception occurs, the innermost try/catch tag is found + %% first. + Bad = [{{y,Y},Tag} || {Y,Tag} <- gb_trees:to_list(Ys), + Y < LastY, is_try_catch_tag(Tag)], + case Bad of + [] -> + ok; + [_|_] -> + error({bad_try_catch_nesting,{y,LastY},Bad}) + end + end. + +is_try_catch_tag({catchtag,_}) -> true; +is_try_catch_tag({trytag,_}) -> true; +is_try_catch_tag(_) -> false. is_reg_defined({x,_}=Reg, Vst) -> is_type_defined_x(Reg, Vst); is_reg_defined({y,_}=Reg, Vst) -> is_type_defined_y(Reg, Vst); @@ -1343,7 +1373,12 @@ branch_arities([Sz,{f,L}|T], Tuple, #vst{current=St}=Vst0) Vst = branch_state(L, Vst1), branch_arities(T, Tuple, Vst#vst{current=St}). -branch_state(0, #vst{}=Vst) -> Vst; +branch_state(0, #vst{}=Vst) -> + %% If the instruction fails, the stack may be scanned + %% looking for a catch tag. Therefore the Y registers + %% must be initialized at this point. + verify_y_init(Vst), + Vst; branch_state(L, #vst{current=St,branched=B}=Vst) -> Vst#vst{ branched=case gb_trees:is_defined(L, B) of diff --git a/lib/compiler/src/beam_z.erl b/lib/compiler/src/beam_z.erl index 1c56b95a9e..6c3a6995d7 100644 --- a/lib/compiler/src/beam_z.erl +++ b/lib/compiler/src/beam_z.erl @@ -24,18 +24,20 @@ -export([module/2]). --import(lists, [dropwhile/2]). +-import(lists, [dropwhile/2,map/2]). -spec module(beam_utils:module_code(), [compile:option()]) -> {'ok',beam_asm:module_code()}. -module({Mod,Exp,Attr,Fs0,Lc}, _Opt) -> - Fs = [function(F) || F <- Fs0], +module({Mod,Exp,Attr,Fs0,Lc}, Opts) -> + NoGetHdTl = proplists:get_bool(no_get_hd_tl, Opts), + Fs = [function(F, NoGetHdTl) || F <- Fs0], {ok,{Mod,Exp,Attr,Fs,Lc}}. -function({function,Name,Arity,CLabel,Is0}) -> +function({function,Name,Arity,CLabel,Is0}, NoGetHdTl) -> try - Is = undo_renames(Is0), + Is1 = undo_renames(Is0), + Is = maybe_eliminate_get_hd_tl(Is1, NoGetHdTl), {function,Name,Arity,CLabel,Is} catch Class:Error:Stack -> @@ -65,6 +67,10 @@ undo_renames([{bif,raise,_,_,_}=I|Is0]) -> (_) -> true end, Is0), [I|undo_renames(Is)]; +undo_renames([{get_hd,Src,Dst1},{get_tl,Src,Dst2}|Is]) -> + [{get_list,Src,Dst1,Dst2}|undo_renames(Is)]; +undo_renames([{get_tl,Src,Dst2},{get_hd,Src,Dst1}|Is]) -> + [{get_list,Src,Dst1,Dst2}|undo_renames(Is)]; undo_renames([I|Is]) -> [undo_rename(I)|undo_renames(Is)]; undo_renames([]) -> []. @@ -107,3 +113,17 @@ undo_rename({get_map_elements,Fail,Src,{list,List}}) -> undo_rename({select,I,Reg,Fail,List}) -> {I,Reg,Fail,{list,List}}; undo_rename(I) -> I. + +%%% +%%% Eliminate get_hd/get_tl instructions if requested by +%%% the no_get_hd_tl option. +%%% + +maybe_eliminate_get_hd_tl(Is, true) -> + map(fun({get_hd,Cons,Hd}) -> + {get_list,Cons,Hd,{x,1022}}; + ({get_tl,Cons,Tl}) -> + {get_list,Cons,{x,1022},Tl}; + (I) -> I + end, Is); +maybe_eliminate_get_hd_tl(Is, false) -> Is. diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index 1409c358c2..c6a0056a70 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -219,13 +219,15 @@ expand_opt(report, Os) -> expand_opt(return, Os) -> [return_errors,return_warnings|Os]; expand_opt(r16, Os) -> - [no_record_opt,no_utf8_atoms|Os]; + [no_get_hd_tl,no_record_opt,no_utf8_atoms|Os]; expand_opt(r17, Os) -> - [no_record_opt,no_utf8_atoms|Os]; + [no_get_hd_tl,no_record_opt,no_utf8_atoms|Os]; expand_opt(r18, Os) -> - [no_record_opt,no_utf8_atoms|Os]; + [no_get_hd_tl,no_record_opt,no_utf8_atoms|Os]; expand_opt(r19, Os) -> - [no_record_opt,no_utf8_atoms|Os]; + [no_get_hd_tl,no_record_opt,no_utf8_atoms|Os]; +expand_opt(r20, Os) -> + [no_get_hd_tl,no_record_opt,no_utf8_atoms|Os]; expand_opt({debug_info_key,_}=O, Os) -> [encrypt_debug_info,O|Os]; expand_opt(no_float_opt, Os) -> diff --git a/lib/compiler/src/genop.tab b/lib/compiler/src/genop.tab index d59bb241a8..a47d4e8cf7 100755 --- a/lib/compiler/src/genop.tab +++ b/lib/compiler/src/genop.tab @@ -564,3 +564,13 @@ BEAM_FORMAT_NUMBER=0 ## exception, but store the atom 'badarg' in x(0) and execute the ## next instruction. 161: raw_raise/0 + +## @spec get_hd Source Head +## @doc Get the head (or car) part of a list (a cons cell) from Source and +## put it into the register Head. +162: get_hd/2 + +## @spec get_tl Source Tail +## @doc Get the tail (or cdr) part of a list (a cons cell) from Source and +## put it into the register Tail. +163: get_tl/2 diff --git a/lib/compiler/src/v3_codegen.erl b/lib/compiler/src/v3_codegen.erl index a96d58a903..a8f4926e55 100644 --- a/lib/compiler/src/v3_codegen.erl +++ b/lib/compiler/src/v3_codegen.erl @@ -1495,28 +1495,34 @@ select_extract_map(Src, Vs, Fail, I, Vdb, Bef, St) -> {Code, Aft, St}. -select_extract_cons(Src, [#k_var{name=Hd}, #k_var{name=Tl}], I, Vdb, Bef, St) -> - {Es,Aft} = case {vdb_find(Hd, Vdb), vdb_find(Tl, Vdb)} of - {{_,_,Lhd}, {_,_,Ltl}} when Lhd =< I, Ltl =< I -> - %% Both head and tail are dead. No need to generate - %% any instruction. - {[], Bef}; - _ -> - %% At least one of head and tail will be used, - %% but we must always fetch both. We will call - %% clear_dead/2 to allow reuse of the register - %% in case only of them is used. - - Reg0 = put_reg(Tl, put_reg(Hd, Bef#sr.reg)), - Int0 = Bef#sr{reg=Reg0}, - Rsrc = fetch_var(Src, Int0), - Rhd = fetch_reg(Hd, Reg0), - Rtl = fetch_reg(Tl, Reg0), - Int1 = clear_dead(Int0, I, Vdb), - {[{get_list,Rsrc,Rhd,Rtl}], Int1} - end, - {Es,Aft,St}. - +select_extract_cons(Src, [#k_var{name=Hd},#k_var{name=Tl}], I, Vdb, Bef, St) -> + Rsrc = fetch_var(Src, Bef), + Int = clear_dead(Bef, I, Vdb), + {{_,_,Lhd},{_,_,Ltl}} = {vdb_find(Hd, Vdb),vdb_find(Tl, Vdb)}, + case {Lhd =< I, Ltl =< I} of + {true,true} -> + %% Both dead. + {[],Bef,St}; + {true,false} -> + %% Head dead. + Reg0 = put_reg(Tl, Bef#sr.reg), + Aft = Int#sr{reg=Reg0}, + Rtl = fetch_reg(Tl, Reg0), + {[{get_tl,Rsrc,Rtl}],Aft,St}; + {false,true} -> + %% Tail dead. + Reg0 = put_reg(Hd, Bef#sr.reg), + Aft = Int#sr{reg=Reg0}, + Rhd = fetch_reg(Hd, Reg0), + {[{get_hd,Rsrc,Rhd}],Aft,St}; + {false,false} -> + %% Both used. + Reg0 = put_reg(Tl, put_reg(Hd, Bef#sr.reg)), + Aft = Bef#sr{reg=Reg0}, + Rhd = fetch_reg(Hd, Reg0), + Rtl = fetch_reg(Tl, Reg0), + {[{get_hd,Rsrc,Rhd},{get_tl,Rsrc,Rtl}],Aft,St} + end. guard_clause_cg(#k_guard_clause{anno=#l{vdb=Vdb},guard=G,body=B}, Fail, Bef, St0) -> {Gis,Int,St1} = guard_cg(G, Fail, Vdb, Bef, St0), diff --git a/lib/compiler/test/beam_block_SUITE.erl b/lib/compiler/test/beam_block_SUITE.erl index fac18789e0..38ead96cc8 100644 --- a/lib/compiler/test/beam_block_SUITE.erl +++ b/lib/compiler/test/beam_block_SUITE.erl @@ -22,7 +22,7 @@ -export([all/0,suite/0,groups/0,init_per_suite/1,end_per_suite/1, init_per_group/2,end_per_group/2, get_map_elements/1,otp_7345/1,move_opt_across_gc_bif/1, - erl_202/1,repro/1,local_cse/1]). + erl_202/1,repro/1,local_cse/1,second_block_pass/1]). %% The only test for the following functions is that %% the code compiles and is accepted by beam_validator. @@ -41,7 +41,8 @@ groups() -> move_opt_across_gc_bif, erl_202, repro, - local_cse + local_cse, + second_block_pass ]}]. init_per_suite(Config) -> @@ -295,6 +296,15 @@ local_cse_4() -> do_local_cse_4(X, Y, Z) -> {X,Y,Z}. +%% Tests previously found bugs when running beam_block the second time. + +second_block_pass(_Config) -> + [#{dts:=5.0}] = second_1([#{dts => 10.0}], 2.0), + ok. + +second_1(Fs, TS) -> + [F#{dts=>DTS / TS} || #{dts:=DTS} = F <- Fs]. + %%% %%% Common functions. %%% diff --git a/lib/compiler/test/beam_type_SUITE.erl b/lib/compiler/test/beam_type_SUITE.erl index dfbf2aa4a0..e33df809ff 100644 --- a/lib/compiler/test/beam_type_SUITE.erl +++ b/lib/compiler/test/beam_type_SUITE.erl @@ -67,6 +67,15 @@ integers(_Config) -> college = do_integers_3(), + zero = do_integers_4(<<0:1>>, 0), + one = do_integers_4(<<1:1>>, 0), + other = do_integers_4(<<1:1>>, 2), + + zero = do_integers_5(0, 0), + one = do_integers_5(0, 1), + two = do_integers_5(0, 2), + three = do_integers_5(0, 3), + ok. do_integers_1(B0) -> @@ -89,6 +98,30 @@ do_integers_3() -> 1 -> 0 end. +do_integers_4(<<X:1,T/bits>>, C) -> + %% Binary matching gives the range 0-1 for X. + %% The range for `X bor C` is unknown. It must not be inherited + %% from X. (`X bor C` will reuse the register used for X.) + case X bor C of + 0 -> do_integers_4(T, C, zero); + 1 -> do_integers_4(T, C, one); + _ -> do_integers_4(T, C, other) + end. + +do_integers_4(_, _, Res) -> + Res. + +do_integers_5(X0, Y0) -> + %% X and Y will use the same register. + X = X0 band 1, + Y = Y0 band 3, + case Y of + 0 -> zero; + 1 -> one; + 2 -> two; + 3 -> three + end. + coverage(_Config) -> {'EXIT',{badarith,_}} = (catch id(1) bsl 0.5), {'EXIT',{badarith,_}} = (catch id(2.0) bsl 2), diff --git a/lib/compiler/test/beam_validator_SUITE.erl b/lib/compiler/test/beam_validator_SUITE.erl index 63a13281a8..b8fff7b100 100644 --- a/lib/compiler/test/beam_validator_SUITE.erl +++ b/lib/compiler/test/beam_validator_SUITE.erl @@ -33,7 +33,7 @@ state_after_fault_in_catch/1,no_exception_in_catch/1, undef_label/1,illegal_instruction/1,failing_gc_guard_bif/1, map_field_lists/1,cover_bin_opt/1, - val_dsetel/1,bad_tuples/1]). + val_dsetel/1,bad_tuples/1,bad_try_catch_nesting/1]). -include_lib("common_test/include/ct.hrl"). @@ -62,7 +62,7 @@ groups() -> state_after_fault_in_catch,no_exception_in_catch, undef_label,illegal_instruction,failing_gc_guard_bif, map_field_lists,cover_bin_opt,val_dsetel, - bad_tuples]}]. + bad_tuples,bad_try_catch_nesting]}]. init_per_suite(Config) -> Config. @@ -523,6 +523,14 @@ bad_tuples(Config) -> ok. +bad_try_catch_nesting(Config) -> + Errors = do_val(bad_try_catch_nesting, Config), + [{{bad_try_catch_nesting,main,2}, + {{'try',{y,2},{f,3}}, + 7, + {bad_try_catch_nesting,{y,2},[{{y,1},{trytag,[5]}}]}}}] = Errors, + ok. + %%%------------------------------------------------------------------------- transform_remove(Remove, Module) -> diff --git a/lib/compiler/test/beam_validator_SUITE_data/bad_try_catch_nesting.S b/lib/compiler/test/beam_validator_SUITE_data/bad_try_catch_nesting.S new file mode 100644 index 0000000000..9f1b21a17b --- /dev/null +++ b/lib/compiler/test/beam_validator_SUITE_data/bad_try_catch_nesting.S @@ -0,0 +1,64 @@ +{module, bad_try_catch_nesting}. %% version = 0 + +{exports, [{main,2},{module_info,0},{module_info,1}]}. + +{attributes, []}. + +{labels, 11}. + + +{function, main, 2, 2}. + {label,1}. + {line,[{location,"bad_try_catch_nesting.erl",4}]}. + {func_info,{atom,bad_try_catch_nesting},{atom,main},2}. + {label,2}. + {allocate_zero,3,2}. + {'try',{y,1},{f,5}}. + {move,{x,1},{y,0}}. + {'try',{y,2},{f,3}}. + {line,[{location,"bad_try_catch_nesting.erl",7}]}. + {call_fun,0}. + {try_end,{y,2}}. + {jump,{f,4}}. + {label,3}. + {try_case,{y,2}}. + {test,is_ne_exact,{f,4},[{x,0},{atom,error}]}. + {line,[]}. + {bif,raise,{f,0},[{x,2},{x,1}],{x,0}}. + {label,4}. + {move,{y,0},{x,0}}. + {kill,{y,0}}. + {line,[{location,"bad_try_catch_nesting.erl",12}]}. + {call_fun,0}. + {try_end,{y,1}}. + {deallocate,3}. + return. + {label,5}. + {try_case,{y,1}}. + {test,is_eq_exact,{f,6},[{x,0},{atom,throw}]}. + {deallocate,3}. + return. + {label,6}. + {line,[]}. + {bif,raise,{f,0},[{x,2},{x,1}],{x,0}}. + + +{function, module_info, 0, 8}. + {label,7}. + {line,[]}. + {func_info,{atom,bad_try_catch_nesting},{atom,module_info},0}. + {label,8}. + {move,{atom,bad_try_catch_nesting},{x,0}}. + {line,[]}. + {call_ext_only,1,{extfunc,erlang,get_module_info,1}}. + + +{function, module_info, 1, 10}. + {label,9}. + {line,[]}. + {func_info,{atom,bad_try_catch_nesting},{atom,module_info},1}. + {label,10}. + {move,{x,0},{x,1}}. + {move,{atom,bad_try_catch_nesting},{x,0}}. + {line,[]}. + {call_ext_only,2,{extfunc,erlang,get_module_info,2}}. diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl index 35c11d894d..eee5bc733f 100644 --- a/lib/compiler/test/compile_SUITE.erl +++ b/lib/compiler/test/compile_SUITE.erl @@ -1455,19 +1455,21 @@ env_compiler_options(_Config) -> bc_options(Config) -> DataDir = proplists:get_value(data_dir, Config), - 101 = highest_opcode(DataDir, small_float, [no_line_info]), + 101 = highest_opcode(DataDir, small_float, [no_get_hd_tl,no_line_info]), 103 = highest_opcode(DataDir, big, - [no_record_opt,no_line_info,no_stack_trimming]), + [no_get_hd_tl,no_record_opt, + no_line_info,no_stack_trimming]), - 125 = highest_opcode(DataDir, small_float, [no_line_info,no_float_opt]), + 125 = highest_opcode(DataDir, small_float, + [no_get_hd_tl,no_line_info,no_float_opt]), 132 = highest_opcode(DataDir, small, - [no_record_opt,no_float_opt,no_line_info]), + [no_get_hd_tl,no_record_opt,no_float_opt,no_line_info]), - 136 = highest_opcode(DataDir, big, [no_record_opt,no_line_info]), + 136 = highest_opcode(DataDir, big, [no_get_hd_tl,no_record_opt,no_line_info]), - 153 = highest_opcode(DataDir, big, [no_record_opt]), + 153 = highest_opcode(DataDir, big, [no_get_hd_tl,no_record_opt]), 153 = highest_opcode(DataDir, big, [r16]), 153 = highest_opcode(DataDir, big, [r17]), 153 = highest_opcode(DataDir, big, [r18]), @@ -1478,9 +1480,10 @@ bc_options(Config) -> 158 = highest_opcode(DataDir, small_maps, [r17]), 158 = highest_opcode(DataDir, small_maps, [r18]), 158 = highest_opcode(DataDir, small_maps, [r19]), + 158 = highest_opcode(DataDir, small_maps, [r20]), 158 = highest_opcode(DataDir, small_maps, []), - 159 = highest_opcode(DataDir, big, []), + 163 = highest_opcode(DataDir, big, []), ok. diff --git a/lib/compiler/test/compile_SUITE_data/big.erl b/lib/compiler/test/compile_SUITE_data/big.erl index 2e54ee8660..1db07755a1 100644 --- a/lib/compiler/test/compile_SUITE_data/big.erl +++ b/lib/compiler/test/compile_SUITE_data/big.erl @@ -741,3 +741,7 @@ snmp_access(suite) -> debug_support(suite) -> [ info, schema, schema, kill, lkill ]. +%% Cover translation of get_hd/2 to get_list/3 when option no_get_hd_tl +%% is given. +cover_get_hd([Hd|_]) -> + Hd. diff --git a/lib/compiler/test/misc_SUITE.erl b/lib/compiler/test/misc_SUITE.erl index d93c5dda1e..4e39f4663e 100644 --- a/lib/compiler/test/misc_SUITE.erl +++ b/lib/compiler/test/misc_SUITE.erl @@ -359,9 +359,7 @@ integer_encoding_1(Config) -> io:put_chars(Src, "t(Last) ->[\n"), io:put_chars(Data, "[\n"), - do_integer_encoding(-(id(1) bsl 10000), Src, Data), - do_integer_encoding(id(1) bsl 10000, Src, Data), - do_integer_encoding(1024, 0, Src, Data), + do_integer_encoding(137, 0, Src, Data), _ = [begin B = 1 bsl I, do_integer_encoding(-B-1, Src, Data), @@ -370,7 +368,7 @@ integer_encoding_1(Config) -> do_integer_encoding(B-1, Src, Data), do_integer_encoding(B, Src, Data), do_integer_encoding(B+1, Src, Data) - end || I <- lists:seq(1, 128)], + end || I <- lists:seq(1, 130)], io:put_chars(Src, "Last].\n\n"), ok = file:close(Src), io:put_chars(Data, "0].\n\n"), @@ -384,8 +382,6 @@ integer_encoding_1(Config) -> %% Compare lists. List = Mod:t(0), {ok,[List]} = file:consult(DataFile), - OneBsl10000 = id(1) bsl 10000, - [-(1 bsl 10000),OneBsl10000|_] = List, %% Cleanup. file:delete(SrcFile), @@ -404,7 +400,3 @@ do_integer_encoding(I, Src, Data) -> Str = integer_to_list(I), io:put_chars(Src, [Str,",\n"]), io:put_chars(Data, [Str,",\n"]). - - -id(I) -> I. - diff --git a/lib/crypto/test/engine_SUITE.erl b/lib/crypto/test/engine_SUITE.erl index f206f967c7..f410542f72 100644 --- a/lib/crypto/test/engine_SUITE.erl +++ b/lib/crypto/test/engine_SUITE.erl @@ -72,7 +72,12 @@ groups() -> init_per_suite(Config) -> try crypto:start() of ok -> - Config; + case crypto:info_lib() of + [{_,_, <<"OpenSSL 1.0.1s-freebsd 1 Mar 2016">>}] -> + {skip, "Problem with engine on OpenSSL 1.0.1s-freebsd"}; + _ -> + Config + end; {error,{already_started,crypto}} -> Config catch _:_ -> diff --git a/lib/debugger/src/dbg_debugged.erl b/lib/debugger/src/dbg_debugged.erl index e142af4ae0..8b64a60a9c 100644 --- a/lib/debugger/src/dbg_debugged.erl +++ b/lib/debugger/src/dbg_debugged.erl @@ -31,32 +31,25 @@ %% Called via the error handler. %%-------------------------------------------------------------------- eval(Mod, Func, Args) -> - SaveStacktrace = erlang:get_stacktrace(), Meta = dbg_ieval:eval(Mod, Func, Args), Mref = erlang:monitor(process, Meta), - msg_loop(Meta, Mref, SaveStacktrace). + msg_loop(Meta, Mref). %%==================================================================== %% Internal functions %%==================================================================== -msg_loop(Meta, Mref, SaveStacktrace) -> +msg_loop(Meta, Mref) -> receive %% Evaluated function has returned a value {sys, Meta, {ready, Val}} -> erlang:demonitor(Mref, [flush]), - - %% Restore original stacktrace and return the value - try erlang:raise(throw, stack, SaveStacktrace) - catch - throw:stack -> - case Val of - {dbg_apply,M,F,A} -> - apply(M, F, A); - _ -> - Val - end + case Val of + {dbg_apply,M,F,A} -> + apply(M, F, A); + _ -> + Val end; %% Evaluated function raised an (uncaught) exception @@ -74,32 +67,25 @@ msg_loop(Meta, Mref, SaveStacktrace) -> Meta ! {self(), rec_acked}, ok end, - msg_loop(Meta, Mref, SaveStacktrace); + msg_loop(Meta, Mref); %% Meta needs something evaluated within context of real process {sys, Meta, {command,Command}} -> Reply = handle_command(Command), Meta ! {sys, self(), Reply}, - msg_loop(Meta, Mref, SaveStacktrace); + msg_loop(Meta, Mref); %% Meta has terminated %% Must be due to int:stop() (or -heaven forbid- a debugger bug) {'DOWN', Mref, _, _, Reason} -> - - %% Restore original stacktrace and return a dummy value - try erlang:raise(throw, stack, SaveStacktrace) - catch - throw:stack -> - {interpreter_terminated, Reason} - end + {interpreter_terminated, Reason} end. handle_command(Command) -> try reply(Command) - catch Class:Reason -> - Stacktrace = stacktrace_f(erlang:get_stacktrace()), - {exception,{Class,Reason,Stacktrace}} + catch Class:Reason:Stacktrace -> + {exception,{Class,Reason,stacktrace_f(Stacktrace)}} end. reply({apply,M,F,As}) -> diff --git a/lib/debugger/src/dbg_ieval.erl b/lib/debugger/src/dbg_ieval.erl index 8009d62629..b0db8228e5 100644 --- a/lib/debugger/src/dbg_ieval.erl +++ b/lib/debugger/src/dbg_ieval.erl @@ -924,8 +924,7 @@ expr({dbg,Line,raise,As0}, Bs0, #ieval{level=Le}=Ieval0) -> trace(return, {Le,Error}), {value,Error,Bs} catch - _:_ -> - Stk = erlang:get_stacktrace(), %Possibly truncated. + _:_:Stk -> %Possibly truncated. StkFun = fun(_) -> Stk end, do_exception(Class, Reason, StkFun, Bs, Ieval) end; diff --git a/lib/debugger/src/dbg_wx_mon.erl b/lib/debugger/src/dbg_wx_mon.erl index a32a6894b8..00aee62a87 100644 --- a/lib/debugger/src/dbg_wx_mon.erl +++ b/lib/debugger/src/dbg_wx_mon.erl @@ -119,9 +119,9 @@ init(CallingPid, Mode, SFile) -> init2(CallingPid, Mode, SFile, GS) catch exit:stop -> stop; - Error:Reason -> + Error:Reason:Stacktrace -> io:format("~p: Crashed {~p,~p} in~n ~p", - [?MODULE, Error, Reason, erlang:get_stacktrace()]) + [?MODULE, Error, Reason, Stacktrace]) end end. diff --git a/lib/debugger/src/dbg_wx_trace.erl b/lib/debugger/src/dbg_wx_trace.erl index b1e0e03b4c..505d53005f 100644 --- a/lib/debugger/src/dbg_wx_trace.erl +++ b/lib/debugger/src/dbg_wx_trace.erl @@ -95,9 +95,9 @@ start(Pid, Env, Parent, TraceWin, BackTrace, Strings) -> catch _:stop -> exit(stop); - E:R -> + E:R:S -> io:format("TraceWin Crashed ~p~n",[E]), - io:format(" ~p in ~p~n",[R, erlang:get_stacktrace()]), + io:format(" ~p in ~p~n",[R, S]), exit(R) end; error -> diff --git a/lib/dialyzer/src/dialyzer_cl_parse.erl b/lib/dialyzer/src/dialyzer_cl_parse.erl index 80c10183cf..f21eaed087 100644 --- a/lib/dialyzer/src/dialyzer_cl_parse.erl +++ b/lib/dialyzer/src/dialyzer_cl_parse.erl @@ -41,8 +41,8 @@ start() -> Ret catch throw:{dialyzer_cl_parse_error, Msg} -> {error, Msg}; - _:R -> - Msg = io_lib:format("~tp\n~tp\n", [R, erlang:get_stacktrace()]), + _:R:S -> + Msg = io_lib:format("~tp\n~tp\n", [R, S]), {error, lists:flatten(Msg)} end. diff --git a/lib/dialyzer/src/dialyzer_dataflow.erl b/lib/dialyzer/src/dialyzer_dataflow.erl index 384912f983..c5f93a3392 100644 --- a/lib/dialyzer/src/dialyzer_dataflow.erl +++ b/lib/dialyzer/src/dialyzer_dataflow.erl @@ -299,7 +299,7 @@ traverse(Tree, Map, State) -> match_fail -> t_none(); raise -> t_none(); bs_init_writable -> t_from_term(<<>>); - build_stacktrace -> t_list(); + build_stacktrace -> erl_bif_types:type(erlang, build_stacktrace, 0); Other -> erlang:error({'Unsupported primop', Other}) end, {State, Map, Type}; @@ -1236,6 +1236,13 @@ handle_tuple(Tree, Map, State) -> State2 = state__add_warning(State1, ?WARN_OPAQUE, Tree, Msg), {State2, Map1, t_none()}; + {error, record, ErrorPat, ErrorType, _} -> + Msg = {record_match, + [format_patterns(ErrorPat), + format_type(ErrorType, State1)]}, + State2 = state__add_warning(State1, ?WARN_MATCHING, + Tree, Msg), + {State2, Map1, t_none()}; {Map2, ETypes} -> {State1, Map2, t_tuple(ETypes)} end @@ -3437,19 +3444,19 @@ state__fun_info(Fun, #state{callgraph = CG, fun_tab = FunTab, plt = PLT}) -> {Fun, Sig, Contract, LocalRet}. forward_args(Fun, ArgTypes, #state{work = Work, fun_tab = FunTab} = State) -> - {OldArgTypes, OldOut, Fixpoint} = + {NewArgTypes, OldOut, Fixpoint} = case dict:find(Fun, FunTab) of - {ok, {not_handled, {OldArgTypes0, OldOut0}}} -> - {OldArgTypes0, OldOut0, false}; + {ok, {not_handled, {_OldArgTypesAreNone, OldOut0}}} -> + {ArgTypes, OldOut0, false}; {ok, {OldArgTypes0, OldOut0}} -> - {OldArgTypes0, OldOut0, - t_is_subtype(t_product(ArgTypes), t_product(OldArgTypes0))} + NewArgTypes0 = [t_sup(X, Y) || + {X, Y} <- lists:zip(ArgTypes, OldArgTypes0)], + {NewArgTypes0, OldOut0, + t_is_equal(t_product(NewArgTypes0), t_product(OldArgTypes0))} end, case Fixpoint of true -> State; false -> - NewArgTypes = [t_sup(X, Y) || - {X, Y} <- lists:zip(ArgTypes, OldArgTypes)], NewWork = add_work(Fun, Work), ?debug("~tw: forwarding args ~ts\n", [state__lookup_name(Fun, State), diff --git a/lib/dialyzer/src/dialyzer_plt.erl b/lib/dialyzer/src/dialyzer_plt.erl index 95c8b5ebce..2af4534396 100644 --- a/lib/dialyzer/src/dialyzer_plt.erl +++ b/lib/dialyzer/src/dialyzer_plt.erl @@ -531,17 +531,19 @@ compute_md5_from_files(Files) -> lists:keysort(1, [{F, compute_md5_from_file(F)} || F <- Files]). compute_md5_from_file(File) -> - case filelib:is_regular(File) of - false -> + case beam_lib:all_chunks(File) of + {ok, _, Chunks} -> + %% We cannot use beam_lib:md5 because it does not consider + %% the debug_info chunk, where typespecs are likely stored. + %% So we consider almost all chunks except the useless ones. + Filtered = [[ID, Chunk] || {ID, Chunk} <- Chunks, ID =/= "CInf", ID =/= "Docs"], + erlang:md5(lists:sort(Filtered)); + {error, beam_lib, {file_error, _, enoent}} -> Msg = io_lib:format("Not a regular file: ~ts\n", [File]), throw({dialyzer_error, Msg}); - true -> - case dialyzer_utils:get_core_from_beam(File) of - {error, Error} -> - throw({dialyzer_error, Error}); - {ok, Core} -> - erlang:md5(term_to_binary(Core)) - end + {error, beam_lib, _} -> + Msg = io_lib:format("Could not compute MD5 for .beam: ~ts\n", [File]), + throw({dialyzer_error, Msg}) end. init_diff_list(RemoveFiles, AddFiles) -> diff --git a/lib/dialyzer/src/dialyzer_typesig.erl b/lib/dialyzer/src/dialyzer_typesig.erl index b1f6a54503..dede475f98 100644 --- a/lib/dialyzer/src/dialyzer_typesig.erl +++ b/lib/dialyzer/src/dialyzer_typesig.erl @@ -418,7 +418,11 @@ traverse(Tree, DefinedVars, State) -> match_fail -> throw(error); raise -> throw(error); bs_init_writable -> {State, t_from_term(<<>>)}; - build_stacktrace -> {State, t_list()}; + build_stacktrace -> + V = mk_var(Tree), + Type = erl_bif_types:type(erlang, build_stacktrace, 0), + State1 = state__store_conj(V, sub, Type, State), + {State1, V}; Other -> erlang:error({'Unsupported primop', Other}) end; 'receive' -> @@ -1896,9 +1900,8 @@ solver(Solver, SolveFun) -> ?debug("Solver ~w returned unexpected result:\n ~P\n", [Solver, _R, 60]), throw(error) - catch E:R -> - io:format("Solver ~w failed: ~w:~p\n ~tp\n", - [Solver, E, R, erlang:get_stacktrace()]), + catch E:R:S -> + io:format("Solver ~w failed: ~w:~p\n ~tp\n", [Solver, E, R, S]), throw(error) end. diff --git a/lib/dialyzer/src/typer.erl b/lib/dialyzer/src/typer.erl index 16b9c8a94a..9d3d9ce438 100644 --- a/lib/dialyzer/src/typer.erl +++ b/lib/dialyzer/src/typer.erl @@ -164,9 +164,9 @@ get_type_info(#analysis{callgraph = CallGraph, CodeServer), Analysis#analysis{callgraph = StrippedCallGraph, trust_plt = NewPlt} catch - error:What -> + error:What:Stacktrace -> fatal_error(io_lib:format("Analysis failed with message: ~tp", - [{What, erlang:get_stacktrace()}])); + [{What, Stacktrace}])); throw:{dialyzer_succ_typing_error, Msg} -> fatal_error(io_lib:format("Analysis failed with message: ~ts", [Msg])) end. diff --git a/lib/dialyzer/test/behaviour_SUITE_data/src/proper/proper_typeserver.erl b/lib/dialyzer/test/behaviour_SUITE_data/src/proper/proper_typeserver.erl index b16075763f..12f6532c0c 100644 --- a/lib/dialyzer/test/behaviour_SUITE_data/src/proper/proper_typeserver.erl +++ b/lib/dialyzer/test/behaviour_SUITE_data/src/proper/proper_typeserver.erl @@ -539,7 +539,7 @@ apply_spec_test({Mod,Fun,_Arity}=MFA, {_Domain,Range}, SpecTimeout, FalsePositiv try apply(Mod, Fun, Args) of X -> {ok, X} catch - X:Y -> {X, Y} + X:Y:S -> {{X, Y}, S} end, case Result of {ok, Z} -> @@ -551,15 +551,15 @@ apply_spec_test({Mod,Fun,_Arity}=MFA, {_Domain,Range}, SpecTimeout, FalsePositiv false -> false end; - Exception when is_function(FalsePositiveMFAs) -> + {Exception, S2} when is_function(FalsePositiveMFAs) -> case FalsePositiveMFAs(MFA, Args, Exception) of true -> true; false -> - error(Exception, erlang:get_stacktrace()) + error(Exception, S2) end; - Exception -> - error(Exception, erlang:get_stacktrace()) + {Exception, S3} -> + error(Exception, S3) end end). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/para_bug/same.erl b/lib/dialyzer/test/opaque_SUITE_data/src/para_bug/same.erl new file mode 100644 index 0000000000..44149f4199 --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/para_bug/same.erl @@ -0,0 +1,15 @@ +-module(same). + +-export([baz/1]). + +-record(bar, { + a :: same_type:st(integer()), + b :: same_type:st(atom()) + }). + +baz(Bar) -> + _ = wrap_find(0, Bar#bar.a), + wrap_find(0, Bar#bar.b). + +wrap_find(K, D) -> + same_type:t(K, D). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/para_bug/same_type.erl b/lib/dialyzer/test/opaque_SUITE_data/src/para_bug/same_type.erl new file mode 100644 index 0000000000..855a5d30be --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/para_bug/same_type.erl @@ -0,0 +1,13 @@ +-module(same_type). + +-export([t/2]). + +-export_type([st/1]). + +%% When unopaqued all specializations of st/1 are equal. +-opaque st(_A) :: {st, tuple()}. + +-spec t(_, st(_)) -> _. + +t(K, V) -> + {K, V}. diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/proper/proper_typeserver.erl b/lib/dialyzer/test/opaque_SUITE_data/src/proper/proper_typeserver.erl index 1677b4efb8..529f9fba72 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/src/proper/proper_typeserver.erl +++ b/lib/dialyzer/test/opaque_SUITE_data/src/proper/proper_typeserver.erl @@ -533,7 +533,7 @@ apply_spec_test({Mod,Fun,_Arity}=MFA, {_Domain,Range}, SpecTimeout, FalsePositiv try apply(Mod,Fun,Args) of X -> {ok, X} catch - X:Y -> {X, Y} + X:Y:S -> {{X, Y}, S} end, case Result of {ok, Z} -> @@ -545,15 +545,15 @@ apply_spec_test({Mod,Fun,_Arity}=MFA, {_Domain,Range}, SpecTimeout, FalsePositiv false -> false end; - Exception when is_function(FalsePositiveMFAs) -> + {Exception, S2} when is_function(FalsePositiveMFAs) -> case FalsePositiveMFAs(MFA, Args, Exception) of true -> true; false -> - error(Exception, erlang:get_stacktrace()) + error(Exception, S2) end; - Exception -> - error(Exception, erlang:get_stacktrace()) + {Exception, S3} -> + error(Exception, S3) end end). diff --git a/lib/dialyzer/test/options1_SUITE_data/src/compiler/beam_validator.erl b/lib/dialyzer/test/options1_SUITE_data/src/compiler/beam_validator.erl index 8fe43163f6..ea92613781 100644 --- a/lib/dialyzer/test/options1_SUITE_data/src/compiler/beam_validator.erl +++ b/lib/dialyzer/test/options1_SUITE_data/src/compiler/beam_validator.erl @@ -174,7 +174,7 @@ validate_error(Error, Name, Ar) -> -endif. validate_error_1(Error, Name, Ar) -> {{'_',Name,Ar}, - {internal_error,'_',{Error,erlang:get_stacktrace()}}}. + {internal_error,'_',{Error,[]}}}. -record(st, %Emulation state {x=init_regs(0, term), %x register info. diff --git a/lib/dialyzer/test/small_SUITE_data/results/record_match b/lib/dialyzer/test/small_SUITE_data/results/record_match new file mode 100644 index 0000000000..a0dd6f560a --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/record_match @@ -0,0 +1,3 @@ + +record_match.erl:16: Function select/0 has no local return +record_match.erl:17: Matching of pattern {'b_literal', 'undefined'} tagged with a record name violates the declared type of #b_local{} | #b_remote{} diff --git a/lib/dialyzer/test/small_SUITE_data/results/stacktrace b/lib/dialyzer/test/small_SUITE_data/results/stacktrace new file mode 100644 index 0000000000..fd60881953 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/stacktrace @@ -0,0 +1,5 @@ + +stacktrace.erl:11: The pattern {'a', 'b'} can never match the type [{atom(),atom(),[any()] | byte(),[{'file',string()} | {'line',pos_integer()}]}] +stacktrace.erl:19: The pattern ['a', 'b'] can never match the type [{atom(),atom(),[any()] | byte(),[{'file',string()} | {'line',pos_integer()}]}] +stacktrace.erl:44: The pattern {'a', 'b'} can never match the type [{atom(),atom(),[any()] | byte(),[{'file',string()} | {'line',pos_integer()}]}] +stacktrace.erl:53: The pattern ['a', 'b'] can never match the type [{atom(),atom(),[any()] | byte(),[{'file',string()} | {'line',pos_integer()}]}] diff --git a/lib/dialyzer/test/small_SUITE_data/src/record_match.erl b/lib/dialyzer/test/small_SUITE_data/src/record_match.erl new file mode 100644 index 0000000000..8e9b91937f --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/record_match.erl @@ -0,0 +1,17 @@ +-module(record_match). + +-export([select/0]). + +-record(b_literal, {val}). +-record(b_remote, {mod,name,arity}). +-record(b_local, {name,arity}). + +-type b_remote() :: #b_remote{}. +-type b_local() :: #b_local{}. + +-type argument() :: b_remote() | b_local(). + +-record(b_set, {args=[] :: [argument()]}). + +select() -> + #b_set{args=[#b_remote{},#b_literal{}]}. diff --git a/lib/dialyzer/test/small_SUITE_data/src/stacktrace.erl b/lib/dialyzer/test/small_SUITE_data/src/stacktrace.erl new file mode 100644 index 0000000000..de79e710e9 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/stacktrace.erl @@ -0,0 +1,73 @@ +-module(stacktrace). + +%% Check the stacktrace variable introduced in Erlang/OTP 21.0 + +-export([t1/0, t2/0, t3/0, t4/0, s1/0, s2/0, s3/0, s4/0]). + +t1() -> + try foo:bar() + catch + E:P:S -> + {a,b} = S, % can never match + {E, P} + end. + +t2() -> + try foo:bar() + catch + E:P:S -> + [a,b] = S, % can never match + {E, P} + end. + +t3() -> + try foo:bar() + catch + E:P:S -> + [{m,f,[],[]}] = S, + {E, P} + end. + +t4() -> + try foo:bar() + catch + E:P:S -> + [{m,f,1,[{file,"tjo"},{line,95}]}] = S, + {E, P} + end. + +s1() -> + try foo:bar() + catch + E:P -> + S = erlang:get_stacktrace(), + {a,b} = S, % can never match + {E, P} + end. + +s2() -> + try foo:bar() + catch + E:P -> + S = erlang:get_stacktrace(), + [a,b] = S, % can never match + {E, P} + end. + +s3() -> + try foo:bar() + catch + E:P -> + S = erlang:get_stacktrace(), + [{m,f,[],[]}] = S, + {E, P} + end. + +s4() -> + try foo:bar() + catch + E:P -> + S = erlang:get_stacktrace(), + [{m,f,1,[{file,"tjo"},{line,95}]}] = S, + {E, P} + end. diff --git a/lib/diameter/doc/src/diameter.xml b/lib/diameter/doc/src/diameter.xml index 6b84b22eb5..6bc7d147c0 100644 --- a/lib/diameter/doc/src/diameter.xml +++ b/lib/diameter/doc/src/diameter.xml @@ -1865,8 +1865,8 @@ An example return value with for a client service with Origin-Host {raddr,{127,0,0,1}}, {rport,3868}, {reuseaddr,true}]}]}, - {watchdog,{<0.66.0>,{1346,171491,996448},okay}}, - {peer,{<0.67.0>,{1346,171491,999906}}}, + {watchdog,{<0.66.0>,-576460736368485571,okay}}, + {peer,{<0.67.0>,-576460736357885808}}, {apps,[{0,common}]}, {caps,[{origin_host,{"client.example.com","server.example.com"}}, {origin_realm,{"example.com","example.com"}}, @@ -1946,8 +1946,8 @@ connection might look as follows.</p> {transport_config,[{reuseaddr,true}, {ip,{127,0,0,1}}, {port,3868}]}]}, - {accept,[[{watchdog,{<0.56.0>,{1346,171481,226895},okay}}, - {peer,{<0.58.0>,{1346,171491,999511}}}, + {accept,[[{watchdog,{<0.56.0>,-576460739249514012,okay}}, + {peer,{<0.58.0>,-576460638229179167}}, {apps,[{0,common}]}, {caps,[{origin_host,{"server.example.com","client.example.com"}}, {origin_realm,{"example.com","example.com"}}, @@ -1976,7 +1976,7 @@ connection might look as follows.</p> {send_max,148}, {send_avg,87}, {send_pend,0}]}]}], - [{watchdog,{<0.72.0>,{1346,171491,998404},initial}}]]}, + [{watchdog,{<0.72.0>,-576460638229717546,initial}}]]}, {statistics,[{{{0,280,0},recv},7}, {{{0,280,1},send},7}, {{{0,280,0},recv,{'Result-Code',2001}},7}, @@ -2024,8 +2024,8 @@ A return value for the server above might look as follows.</p> {transport_config,[{reuseaddr,true}, {ip,{127,0,0,1}}, {port,3868}]}]}, - {watchdog,{<0.56.0>,{1346,171481,226895},okay}}, - {peer,{<0.58.0>,{1346,171491,999511}}}, + {watchdog,{<0.56.0>,-576460739249514012,okay}}, + {peer,{<0.58.0>,-576460638229179167}}, {apps,[{0,common}]}, {caps,[{origin_host,{"server.example.com","client.example.com"}}, {origin_realm,{"example.com","example.com"}}, diff --git a/lib/diameter/src/base/diameter_reg.erl b/lib/diameter/src/base/diameter_reg.erl index 5b7cfab31a..c1762a07e3 100644 --- a/lib/diameter/src/base/diameter_reg.erl +++ b/lib/diameter/src/base/diameter_reg.erl @@ -246,8 +246,11 @@ handle_call({add, Uniq, Key}, {Pid, _}, S) -> handle_call({remove, Key}, {Pid, _}, S) -> Rec = {Key, Pid}, - ets:delete_object(?TABLE, Rec), - {reply, true, notify(remove, Rec, S)}; + {reply, true, try + notify(remove, Rec, S) + after + ets:delete_object(?TABLE, Rec) + end}; handle_call({wait, Pat}, {Pid, _} = From, S) -> NS = add_monitor(Pid, S), @@ -370,10 +373,12 @@ send({_,_} = From, add, Rec) -> down(Pid, #state{monitors = Ps} = S) -> Recs = match('_', Pid), - ets:match_delete(?TABLE, {'_', Pid}), - lists:foldl(fun(R,NS) -> notify(remove, R, NS) end, - flush(Pid, S#state{monitors = sets:del_element(Pid, Ps)}), - Recs). + Acc0 = flush(Pid, S#state{monitors = sets:del_element(Pid, Ps)}), + try + lists:foldl(fun(R,NS) -> notify(remove, R, NS) end, Acc0, Recs) + after + ets:match_delete(?TABLE, {'_', Pid}) + end. %% flush/3 diff --git a/lib/diameter/src/base/diameter_service.erl b/lib/diameter/src/base/diameter_service.erl index 31dd92f878..cbe66ef27a 100644 --- a/lib/diameter/src/base/diameter_service.erl +++ b/lib/diameter/src/base/diameter_service.erl @@ -151,7 +151,7 @@ apps :: match([{0..16#FFFFFFFF, diameter:app_alias()}] %% {Id, Alias} | [diameter:app_alias()]), %% remote caps :: match(#diameter_caps{}), - started = diameter_lib:now(), %% at process start or sharing + started = diameter_lib:now(), %% at connection_up watchdog :: match(pid() %% key into watchdogT | undefined)}). %% undefined if remote @@ -554,15 +554,25 @@ terminate(Reason, #state{service_name = Name, local = {PeerT, _, _}} = S) -> %% wait for watchdog state changes to take care of if. That this %% takes place after deleting the state entry ensures that the %% resulting failover by request processes accomplishes nothing. - ets:foldl(fun(#peer{pid = TPid}, _) -> - diameter_traffic:peer_down(TPid) - end, - ok, - PeerT), + ets:foldl(fun peer_down/2, ok, PeerT), shutdown == Reason %% application shutdown andalso shutdown(application, S). +%% peer_down/1 +%% +%% Entries with watchdog state SUSPECT are already down: ignore the +%% expected failure. This assumes the current implementation, but +%% double the number of lookups (in the typical case) could be the +%% greater evil if there are many peer connections. + +peer_down(#peer{pid = TPid}, _) -> + try + diameter_traffic:peer_down(TPid) + catch + error: {badmatch, []} -> ok + end. + %% --------------------------------------------------------------------------- %% # code_change/3 %% --------------------------------------------------------------------------- diff --git a/lib/diameter/src/diameter.appup.src b/lib/diameter/src/diameter.appup.src index 7da59f8b25..05a8c9378e 100644 --- a/lib/diameter/src/diameter.appup.src +++ b/lib/diameter/src/diameter.appup.src @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% Copyright Ericsson AB 2010-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -54,10 +54,10 @@ {"1.12.1", [{restart_application, diameter}]}, %% 19.1 {"1.12.2", [{restart_application, diameter}]}, %% 19.3 {"2.0", [{restart_application, diameter}]}, %% 20.0 - {"2.1", [{load_module, diameter_gen}, %% 20.1 - {update, diameter_reg, {advanced, "2.1"}}]}, - {"2.1.1", [{load_module, diameter_gen}]}, %% 20.1.2 - {"2.1.2", []} %% 20.1.3 + {"2.1", [{restart_application, diameter}]}, %% 20.1 + {"2.1.1", [{restart_application, diameter}]}, %% 20.1.2 + {"2.1.2", [{restart_application, diameter}]}, %% 20.1.3 + {"2.1.3", [{restart_application, diameter}]} %% 20.2 ], [ {"0.9", [{restart_application, diameter}]}, @@ -94,7 +94,8 @@ {"1.12.2", [{restart_application, diameter}]}, {"2.0", [{restart_application, diameter}]}, {"2.1", [{restart_application, diameter}]}, - {"2.1.1", [{load_module, diameter_gen}]}, - {"2.1.2", []} + {"2.1.1", [{restart_application, diameter}]}, + {"2.1.2", [{restart_application, diameter}]}, + {"2.1.3", [{restart_application, diameter}]} ] }. diff --git a/lib/diameter/vsn.mk b/lib/diameter/vsn.mk index 0c852d75cd..b0fb4ada28 100644 --- a/lib/diameter/vsn.mk +++ b/lib/diameter/vsn.mk @@ -1,6 +1,6 @@ # %CopyrightBegin% # -# Copyright Ericsson AB 2010-2017. All Rights Reserved. +# Copyright Ericsson AB 2010-2018. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,5 +17,5 @@ # %CopyrightEnd% APPLICATION = diameter -DIAMETER_VSN = 2.1.3 +DIAMETER_VSN = 2.1.4 APP_VSN = $(APPLICATION)-$(DIAMETER_VSN)$(PRE_VSN) diff --git a/lib/et/src/et_wx_contents_viewer.erl b/lib/et/src/et_wx_contents_viewer.erl index bca517317e..7ca41da3e8 100644 --- a/lib/et/src/et_wx_contents_viewer.erl +++ b/lib/et/src/et_wx_contents_viewer.erl @@ -93,8 +93,8 @@ start_link(Options) -> end, {ok, Pid} catch - error:Reason -> - {error, {'EXIT', Reason, erlang:get_stacktrace()}} + error:Reason:Stacktrace -> + {error, {'EXIT', Reason, Stacktrace}} end; {error, Reason} -> {error, Reason} diff --git a/lib/eunit/src/eunit_lib.erl b/lib/eunit/src/eunit_lib.erl index d1bd160ea1..771541354c 100644 --- a/lib/eunit/src/eunit_lib.erl +++ b/lib/eunit/src/eunit_lib.erl @@ -202,13 +202,13 @@ format_exception_test_() -> "\nymmud:rorre"++_, lists:reverse(lists:flatten( format_exception(try erlang:error(dummy) - catch C:R -> {C, R, erlang:get_stacktrace()} + catch C:R:S -> {C, R, S} end)))), ?_assertMatch( "\nymmud:rorre"++_, lists:reverse(lists:flatten( format_exception(try erlang:error(dummy, [a]) - catch C:R -> {C, R, erlang:get_stacktrace()} + catch C:R:S -> {C, R, S} end))))]. -endif. diff --git a/lib/eunit/src/eunit_listener.erl b/lib/eunit/src/eunit_listener.erl index e652c5b2f6..75aa05c543 100644 --- a/lib/eunit/src/eunit_listener.erl +++ b/lib/eunit/src/eunit_listener.erl @@ -137,8 +137,7 @@ call(F, As, St) when is_atom(F) -> try apply(St#state.callback, F, As) of Substate -> St#state{state = Substate} catch - Class:Term -> - Trace = erlang:get_stacktrace(), + Class:Term:Trace -> if F =/= terminate -> call(terminate, [{error, {Class, Term, Trace}}, St#state.state], St); diff --git a/lib/eunit/src/eunit_proc.erl b/lib/eunit/src/eunit_proc.erl index e075005238..96bdcf88b6 100644 --- a/lib/eunit/src/eunit_proc.erl +++ b/lib/eunit/src/eunit_proc.erl @@ -628,7 +628,7 @@ io_request({put_chars, M, F, As}, Buf) -> try apply(M, F, As) of Chars -> {ok, [Chars | Buf]} catch - C:T -> {{error, {C,T,erlang:get_stacktrace()}}, Buf} + C:T:S -> {{error, {C,T,S}}, Buf} end; io_request({put_chars, _Enc, Chars}, Buf) -> io_request({put_chars, Chars}, Buf); diff --git a/lib/eunit/src/eunit_test.erl b/lib/eunit/src/eunit_test.erl index 6036537178..6fe85ae70a 100644 --- a/lib/eunit/src/eunit_test.erl +++ b/lib/eunit/src/eunit_test.erl @@ -39,11 +39,11 @@ %% somewhat, but you can't have everything.) Note that we assume that %% this particular module is the boundary between eunit and user code. -get_stacktrace() -> - get_stacktrace([]). +get_stacktrace(Trace) -> + get_stacktrace(Trace, []). -get_stacktrace(Ts) -> - eunit_lib:uniq(prune_trace(erlang:get_stacktrace(), Ts)). +get_stacktrace(Trace, Ts) -> + eunit_lib:uniq(prune_trace(Trace, Ts)). -dialyzer({no_match, prune_trace/2}). prune_trace([{eunit_data, _, _} | Rest], Tail) -> @@ -75,8 +75,8 @@ run_testfun(F) -> {eunit_internal, Term} -> %% Internally generated: re-throw Term (lose the trace) throw(Term); - Class:Reason -> - {error, {Class, Reason, get_stacktrace()}} + Class:Reason:Trace -> + {error, {Class, Reason, Trace}} end. @@ -272,7 +272,7 @@ mf_wrapper(M, F) -> fun () -> try M:F() catch - error:undef -> + error:undef:Trace -> %% Check if it was M:F/0 that was undefined case erlang:module_loaded(M) of false -> @@ -282,14 +282,14 @@ mf_wrapper(M, F) -> false -> fail({no_such_function, {M,F,0}}); true -> - rethrow(error, undef, [{M,F,0}]) + rethrow(error, undef, Trace, [{M,F,0}]) end end end end. -rethrow(Class, Reason, Trace) -> - erlang:raise(Class, Reason, get_stacktrace(Trace)). +rethrow(Class, Reason, Trace, Ts) -> + erlang:raise(Class, Reason, get_stacktrace(Trace, Ts)). fail(Term) -> throw({eunit_internal, Term}). @@ -332,12 +332,14 @@ enter_context(Setup, Cleanup, Instantiate, Callback) -> T -> case eunit_lib:is_not_test(T) of true -> - catch throw(error), % generate a stack trace + {_, Stacktrace} = + erlang:process_info(self(), + current_stacktrace), {module,M} = erlang:fun_info(Instantiate, module), {name,N} = erlang:fun_info(Instantiate, name), {arity,A} = erlang:fun_info(Instantiate, arity), context_error({bad_instantiator, {{M,N,A},T}}, - error, badarg); + error, Stacktrace, badarg); false -> ok end, @@ -346,21 +348,22 @@ enter_context(Setup, Cleanup, Instantiate, Callback) -> %% Always run cleanup; client may be an idiot try Cleanup(R) catch - Class:Term -> - context_error(cleanup_failed, Class, Term) + Class:Term:Trace -> + context_error(cleanup_failed, + Class, Trace, Term) end end catch - Class:Term -> - context_error(instantiation_failed, Class, Term) + Class:Term:Trace -> + context_error(instantiation_failed, Class, Trace, Term) end catch - Class:Term -> - context_error(setup_failed, Class, Term) + Class:Term:Trace -> + context_error(setup_failed, Class, Trace, Term) end. -context_error(Type, Class, Term) -> - throw({context_error, Type, {Class, Term, get_stacktrace()}}). +context_error(Type, Class, Trace, Term) -> + throw({context_error, Type, {Class, Term, get_stacktrace(Trace)}}). %% This generates single setup/cleanup functions from a list of tuples %% on the form {Tag, Setup, Cleanup}, where the setup function always @@ -378,8 +381,8 @@ multi_setup([{Tag, S, C} | Es], CleanupPrev) -> try C(R) of _ -> CleanupPrev(Rs) catch - Class:Term -> - throw({Tag, {Class, Term, get_stacktrace()}}) + Class:Term:Trace -> + throw({Tag, {Class, Term, Trace}}) end end, {SetupRest, CleanupAll} = multi_setup(Es, Cleanup), @@ -388,9 +391,9 @@ multi_setup([{Tag, S, C} | Es], CleanupPrev) -> R -> SetupRest([R|Rs]) catch - Class:Term -> + Class:Term:Trace -> CleanupPrev(Rs), - throw({Tag, {Class, Term, get_stacktrace()}}) + throw({Tag, {Class, Term, Trace}}) end end, CleanupAll}; diff --git a/lib/hipe/cerl/erl_bif_types.erl b/lib/hipe/cerl/erl_bif_types.erl index 5fda857bf1..bfffb8db41 100644 --- a/lib/hipe/cerl/erl_bif_types.erl +++ b/lib/hipe/cerl/erl_bif_types.erl @@ -585,6 +585,13 @@ type(erlang, float, 1, Xs, Opaques) -> %% Guard bif, needs to be here. type(erlang, floor, 1, Xs, Opaques) -> strict(erlang, floor, 1, Xs, fun (_) -> t_integer() end, Opaques); +%% Primop, needs to be somewhere. +type(erlang, build_stacktrace, 0, _, _Opaques) -> + t_list(t_tuple([t_module(), + t_atom(), + t_sup([t_arity(),t_list()]), + t_list(t_sup([t_tuple([t_atom('file'),t_string()]), + t_tuple([t_atom('line'),t_pos_integer()])]))])); %% Guard bif, needs to be here. type(erlang, hd, 1, Xs, Opaques) -> strict(erlang, hd, 1, Xs, fun ([X]) -> t_cons_hd(X) end, Opaques); @@ -2336,6 +2343,9 @@ arg_types(erlang, float, 1) -> %% Guard bif, needs to be here. arg_types(erlang, floor, 1) -> [t_number()]; +%% Primop, needs to be somewhere. +arg_types(erlang, build_stacktrace, 0) -> + []; %% Guard bif, needs to be here. arg_types(erlang, hd, 1) -> [t_cons()]; diff --git a/lib/hipe/icode/hipe_beam_to_icode.erl b/lib/hipe/icode/hipe_beam_to_icode.erl index 6e66ec057c..f429d40272 100644 --- a/lib/hipe/icode/hipe_beam_to_icode.erl +++ b/lib/hipe/icode/hipe_beam_to_icode.erl @@ -605,6 +605,16 @@ trans_fun([{get_list,List,Head,Tail}|Instructions], Env) -> ?error_msg("hd and tl regs identical in get_list~n",[]), erlang:error(not_handled) end; +%%--- get_hd --- +trans_fun([{get_hd,List,Head}|Instructions], Env) -> + TransList = [trans_arg(List)], + I = hipe_icode:mk_primop([mk_var(Head)],unsafe_hd,TransList), + [I | trans_fun(Instructions,Env)]; +%%--- get_tl --- +trans_fun([{get_tl,List,Tail}|Instructions], Env) -> + TransList = [trans_arg(List)], + I = hipe_icode:mk_primop([mk_var(Tail)],unsafe_tl,TransList), + [I | trans_fun(Instructions,Env)]; %%--- get_tuple_element --- trans_fun([{get_tuple_element,Xreg,Index,Dst}|Instructions], Env) -> I = hipe_icode:mk_primop([mk_var(Dst)], diff --git a/lib/hipe/icode/hipe_icode_inline_bifs.erl b/lib/hipe/icode/hipe_icode_inline_bifs.erl index 7a6947f190..16a95991e7 100644 --- a/lib/hipe/icode/hipe_icode_inline_bifs.erl +++ b/lib/hipe/icode/hipe_icode_inline_bifs.erl @@ -24,8 +24,9 @@ %% Currently inlined BIFs: %% and, or, xor, not, <, >, >=, =<, ==, /=, =/=, =:= -%% is_atom, is_boolean, is_binary, is_float, is_function, -%% is_integer, is_list, is_pid, is_port, is_reference, is_tuple +%% is_atom, is_binary, is_bitstring, is_boolean, is_float, +%% is_function, is_integer, is_list, is_map, is_number, +%% is_pid, is_port, is_reference, is_tuple -module(hipe_icode_inline_bifs). @@ -116,17 +117,20 @@ try_type_tests(I) -> I. is_type_test(Name) -> case Name of - is_integer -> {true, integer}; + is_atom -> {true, atom}; + is_binary -> {true, binary}; + is_bitstring -> {true, bitstr}; + is_boolean -> {true, boolean}; is_float -> {true, float}; - is_tuple -> {true, tuple}; - is_binary -> {true, binary}; + is_function -> {true, function}; + is_integer -> {true, integer}; is_list -> {true, list}; + is_map -> {true, map}; + is_number -> {true, number}; is_pid -> {true, pid}; - is_atom -> {true, atom}; - is_boolean -> {true, boolean}; - is_function -> {true, function}; - is_reference -> {true, reference}; is_port -> {true, port}; + is_reference -> {true, reference}; + is_tuple -> {true, tuple}; _ -> false end. diff --git a/lib/hipe/main/hipe_main.erl b/lib/hipe/main/hipe_main.erl index 4b5eb4c63e..6e48f0cffd 100644 --- a/lib/hipe/main/hipe_main.erl +++ b/lib/hipe/main/hipe_main.erl @@ -410,9 +410,9 @@ icode_to_rtl(MFA, Icode, Options, Servers) -> hipe_llvm_liveness:analyze(RtlCfg4) end, pp(RtlCfg5, MFA, rtl, pp_rtl, Options, Servers), - case proplists:get_bool(verify_gcsafe, Options) of - false -> ok; - true -> + case proplists:get_bool(no_verify_gcsafe, Options) of + true -> ok; + false -> ok = hipe_rtl_verify_gcsafe:check(RtlCfg5) end, LinearRTL1 = hipe_rtl_cfg:linearize(RtlCfg5), diff --git a/lib/hipe/rtl/hipe_rtl_verify_gcsafe.erl b/lib/hipe/rtl/hipe_rtl_verify_gcsafe.erl index c3f20bfec1..01d7e89ccd 100644 --- a/lib/hipe/rtl/hipe_rtl_verify_gcsafe.erl +++ b/lib/hipe/rtl/hipe_rtl_verify_gcsafe.erl @@ -76,6 +76,7 @@ safe_primop(bs_allocate) -> true; safe_primop(bs_reallocate) -> true; safe_primop(bs_utf8_size) -> true; safe_primop(bs_get_utf8) -> true; +safe_primop(bs_put_utf8) -> true; safe_primop(bs_utf16_size) -> true; safe_primop(bs_get_utf16) -> true; safe_primop(bs_validate_unicode_retract) -> true; diff --git a/lib/hipe/x86/hipe_rtl_to_x86.erl b/lib/hipe/x86/hipe_rtl_to_x86.erl index 31e4f6e4ac..22947da148 100644 --- a/lib/hipe/x86/hipe_rtl_to_x86.erl +++ b/lib/hipe/x86/hipe_rtl_to_x86.erl @@ -646,7 +646,7 @@ conv_imm(Opnd, Map) -> is_imm64(Value) when is_integer(Value) -> (Value < -(1 bsl (32 - 1))) or (Value > (1 bsl (32 - 1)) - 1); is_imm64({_,atom}) -> false; % Atoms are 32 bits. -is_imm64({_,c_const}) -> false; % c_consts are 32 bits. +is_imm64({_,c_const}) -> true; % c_consts are 64 bits. is_imm64({_,_}) -> true . % Other relocs are 64 bits. -else. conv_imm(Opnd, Map) -> @@ -777,6 +777,18 @@ conv_fconv(Dst, Src) -> %%% Finalise the conversion of a 2-address FP operation. +-ifdef(HIPE_AMD64). +conv_fp_unary(Dst, Src, 'fchs') -> + Tmp = new_untagged_temp(), + case same_opnd(Dst, Src) of + true -> + []; + _ -> + [hipe_x86:mk_fmove(Src, Dst)] + end ++ + mk_load_address(c_const, hipe_x86:mk_imm({sse2_fnegate_mask, c_const}), Tmp) ++ + [hipe_x86:mk_fp_binop('xorpd', hipe_x86:mk_mem(Tmp, hipe_x86:mk_imm(0), double), Dst)]. +-else. conv_fp_unary(Dst, Src, FpUnOp) -> case same_opnd(Dst, Src) of true -> @@ -785,6 +797,7 @@ conv_fp_unary(Dst, Src, FpUnOp) -> [hipe_x86:mk_fmove(Src, Dst), hipe_x86:mk_fp_unop(FpUnOp, Dst)] end. +-endif. conv_fp_unop(RtlFpUnOp) -> case RtlFpUnOp of @@ -854,13 +867,8 @@ mk_jmp_switch(Index, JTabLab, Labels) -> %%% Finalise the translation of a load_address instruction. -ifdef(HIPE_AMD64). -mk_load_address(Type, Src, Dst) -> - case Type of - c_const -> % 32 bits - [hipe_x86:mk_move(Src, Dst)]; - _ -> - [hipe_x86:mk_move64(Src, Dst)] - end. +mk_load_address(_Type, Src, Dst) -> + [hipe_x86:mk_move64(Src, Dst)]. -else. mk_load_address(_Type, Src, Dst) -> [hipe_x86:mk_move(Src, Dst)]. diff --git a/lib/hipe/x86/hipe_x86_assemble.erl b/lib/hipe/x86/hipe_x86_assemble.erl index 50919bdf4e..9d2586a14d 100644 --- a/lib/hipe/x86/hipe_x86_assemble.erl +++ b/lib/hipe/x86/hipe_x86_assemble.erl @@ -735,6 +735,7 @@ resolve_sse2_op(Op) -> fdiv -> divsd; fmul -> mulsd; fsub -> subsd; + xorpd -> xorpd; _ -> exit({?MODULE, unknown_sse2_operator, Op}) end. diff --git a/lib/inets/doc/src/http_client.xml b/lib/inets/doc/src/http_client.xml index 212958f17f..15e383ec77 100644 --- a/lib/inets/doc/src/http_client.xml +++ b/lib/inets/doc/src/http_client.xml @@ -97,27 +97,32 @@ 7 > {ok, {{NewVersion, 200, NewReasonPhrase}, NewHeaders, NewBody}} = httpc:request(get, {"http://www.erlang.org", [{"connection", "close"}]}, [], []).</code> - + <p>This sends an HTTP request over a unix domain socket (experimental):</p> + <code type="erl"> + 8 > httpc:set_options([{ipfamily, local}, + {unix_socket,"/tmp/unix_socket/consul_http.sock"}]). + 9 > {ok, {{NewVersion, 200, NewReasonPhrase}, NewHeaders, NewBody}} = + httpc:request(put, {"http:///v1/kv/foo", [], [], "hello"}, [], []).</code> <p>Start an HTTP client profile:</p> <code><![CDATA[ - 8 > {ok, Pid} = inets:start(httpc, [{profile, foo}]). + 10 > {ok, Pid} = inets:start(httpc, [{profile, foo}]). {ok, <0.45.0>} ]]></code> <p>The new profile has no proxy settings, so the connection is refused:</p> <code type="erl"> - 9 > httpc:request("http://www.erlang.org", foo). + 11 > httpc:request("http://www.erlang.org", foo). {error, econnrefused}</code> <p>Stop the HTTP client profile:</p> <code type="erl"> - 10 > inets:stop(httpc, foo). + 12 > inets:stop(httpc, foo). ok</code> <p>Alternative way to stop the HTTP client profile:</p> <code type="erl"> - 10 > inets:stop(httpc, Pid). + 13 > inets:stop(httpc, Pid). ok</code> </section> diff --git a/lib/inets/doc/src/httpc.xml b/lib/inets/doc/src/httpc.xml index 58714328c5..14662f257c 100644 --- a/lib/inets/doc/src/httpc.xml +++ b/lib/inets/doc/src/httpc.xml @@ -210,7 +210,8 @@ ip | port | socket_opts | - verbose</v> + verbose | + unix_socket</v> <v>Profile = profile() | pid()</v> <d>When started <c>stand_alone</c> only the pid can used.</d> <v>Values = [{option_item(), term()}]</v> @@ -297,8 +298,8 @@ {full_result, boolean()} | {headers_as_is, boolean() | {socket_opts, socket_opts()} | - {receiver, receiver()}, - {ipv6_host_with_brackets, boolean()}}</v> + {receiver, receiver()} | + {ipv6_host_with_brackets, boolean()}</v> <v>stream_to() = none | self | {self, once} | filename()</v> <v>socket_opts() = [socket_opt()]</v> <v>receiver() = pid() | function()/1 | {Module, Function, Args}</v> @@ -533,7 +534,8 @@ <v>| {ip, IpAddress}</v> <v>| {port, Port}</v> <v>| {socket_opts, socket_opts()}</v> - <v>| {verbose, VerboseMode}</v> + <v>| {verbose, VerboseMode}</v> + <v>| {unix_socket, UnixSocket}</v> <v>Proxy = {Hostname, Port}</v> <v>Hostname = string()</v> <d>Example: "localhost" or "foo.bar.se"</d> @@ -576,7 +578,7 @@ If option <c>verify</c> is used, function <c>store_cookies/2</c> has to be called for the cookies to be saved. Default is <c>disabled</c>.</d> - <v>IpFamily = inet | inet6 </v> + <v>IpFamily = inet | inet6 | local</v> <d>Default is <c>inet</c>.</d> <v>IpAddress = ip_address()</v> <d>If the host has several network interfaces, this option specifies @@ -601,6 +603,12 @@ It is a debug feature.</d> <v>Profile = profile() | pid()</v> <d>When started <c>stand_alone</c> only the pid can be used.</d> + <v>UnixSocket = path()</v> + <d> + Experimental option for sending HTTP requests over a unix domain socket. The value + of <c>unix_socket</c> shall be the full path to a unix domain socket file with read/write + permissions for the erlang process. Default is <c>undefined</c>. + </d> </type> <desc> <p>Sets options to be used for subsequent requests.</p> diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl index 821eb7f02f..a73503a5ce 100644 --- a/lib/inets/src/http_client/httpc.erl +++ b/lib/inets/src/http_client/httpc.erl @@ -171,6 +171,7 @@ request(Method, HTTPOptions, Options, Profile) when (Method =:= options) orelse (Method =:= get) orelse + (Method =:= put) orelse (Method =:= head) orelse (Method =:= delete) orelse (Method =:= trace) andalso @@ -531,6 +532,7 @@ handle_request(Method, Url, Stream = proplists:get_value(stream, Options), Receiver = proplists:get_value(receiver, Options), SocketOpts = proplists:get_value(socket_opts, Options), + UnixSocket = proplists:get_value(unix_socket, Options), BracketedHost = proplists:get_value(ipv6_host_with_brackets, Options), @@ -558,6 +560,7 @@ handle_request(Method, Url, headers_as_is = headers_as_is(Headers0, Options), socket_opts = SocketOpts, started = Started, + unix_socket = UnixSocket, ipv6_host_with_brackets = BracketedHost}, case httpc_manager:request(Request, profile_name(Profile)) of {ok, RequestId} -> @@ -823,7 +826,7 @@ request_options_defaults() -> error end, - VerifyBrackets = VerifyBoolean, + VerifyBrackets = VerifyBoolean, [ {sync, true, VerifySync}, @@ -894,11 +897,36 @@ request_options_sanity_check(Opts) -> end, ok. -validate_options(Options) -> - (catch validate_options(Options, [])). - -validate_options([], ValidateOptions) -> - {ok, lists:reverse(ValidateOptions)}; +validate_ipfamily_unix_socket(Options0) -> + IpFamily = proplists:get_value(ipfamily, Options0, inet), + UnixSocket = proplists:get_value(unix_socket, Options0, undefined), + Options1 = proplists:delete(ipfamily, Options0), + Options2 = proplists:delete(ipfamily, Options1), + validate_ipfamily_unix_socket(IpFamily, UnixSocket, Options2, + [{ipfamily, IpFamily}, {unix_socket, UnixSocket}]). +%% +validate_ipfamily_unix_socket(local, undefined, _Options, _Acc) -> + bad_option(unix_socket, undefined); +validate_ipfamily_unix_socket(IpFamily, UnixSocket, _Options, _Acc) + when IpFamily =/= local, UnixSocket =/= undefined -> + bad_option(ipfamily, IpFamily); +validate_ipfamily_unix_socket(IpFamily, UnixSocket, Options, Acc) -> + validate_ipfamily(IpFamily), + validate_unix_socket(UnixSocket), + {Options, Acc}. + + +validate_options(Options0) -> + try + {Options, Acc} = validate_ipfamily_unix_socket(Options0), + validate_options(Options, Acc) + catch + error:Reason -> + {error, Reason} + end. +%% +validate_options([], ValidOptions) -> + {ok, lists:reverse(ValidOptions)}; validate_options([{proxy, Proxy} = Opt| Tail], Acc) -> validate_proxy(Proxy), @@ -958,6 +986,10 @@ validate_options([{verbose, Value} = Opt| Tail], Acc) -> validate_verbose(Value), validate_options(Tail, [Opt | Acc]); +validate_options([{unix_socket, Value} = Opt| Tail], Acc) -> + validate_unix_socket(Value), + validate_options(Tail, [Opt | Acc]); + validate_options([{_, _} = Opt| _], _Acc) -> {error, {not_an_option, Opt}}. @@ -1026,7 +1058,8 @@ validate_ipv6(BadValue) -> bad_option(ipv6, BadValue). validate_ipfamily(Value) - when (Value =:= inet) orelse (Value =:= inet6) orelse (Value =:= inet6fb4) -> + when (Value =:= inet) orelse (Value =:= inet6) orelse + (Value =:= inet6fb4) orelse (Value =:= local) -> Value; validate_ipfamily(BadValue) -> bad_option(ipfamily, BadValue). @@ -1056,6 +1089,15 @@ validate_verbose(Value) validate_verbose(BadValue) -> bad_option(verbose, BadValue). +validate_unix_socket(Value) + when (Value =:= undefined) -> + Value; +validate_unix_socket(Value) + when is_list(Value) andalso length(Value) > 0 -> + Value; +validate_unix_socket(BadValue) -> + bad_option(unix_socket, BadValue). + bad_option(Option, BadValue) -> throw({error, {bad_option, Option, BadValue}}). diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index 1482f4f922..9b09832eb8 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -711,9 +711,9 @@ do_handle_info({'EXIT', _, _}, State = #state{request = undefined}) -> %% can retry requests in the pipeline. do_handle_info({'EXIT', _, _}, State) -> {noreply, State#state{status = close}}. - + call(Msg, Pid) -> - try gen_server:call(Pid, Msg) + try gen_server:call(Pid, Msg, infinity) catch exit:{noproc, _} -> {error, closed}; @@ -754,6 +754,7 @@ connect(SocketType, ToAddress, #options{ipfamily = IpFamily, ip = FromAddress, port = FromPort, + unix_socket = UnixSocket, socket_opts = Opts0}, Timeout) -> Opts1 = case FromPort of @@ -789,6 +790,16 @@ connect(SocketType, ToAddress, OK -> OK end; + local -> + Opts3 = [IpFamily | Opts2], + SocketAddr = {local, UnixSocket}, + case http_transport:connect(SocketType, {SocketAddr, 0}, Opts3, Timeout) of + {error, Reason} -> + {error, {failed_connect, [{to_address, SocketAddr}, + {IpFamily, Opts3, Reason}]}}; + Else -> + Else + end; _ -> Opts3 = [IpFamily | Opts2], case http_transport:connect(SocketType, ToAddress, Opts3, Timeout) of @@ -800,9 +811,23 @@ connect(SocketType, ToAddress, end end. -connect_and_send_first_request(Address, Request, #state{options = Options} = State) -> +handle_unix_socket_options(#request{unix_socket = UnixSocket}, Options) + when UnixSocket =:= undefined -> + Options; + +handle_unix_socket_options(#request{unix_socket = UnixSocket}, + Options = #options{ipfamily = IpFamily}) -> + case IpFamily of + local -> + Options#options{unix_socket = UnixSocket}; + Else -> + error({badarg, [{ipfamily, Else}, {unix_socket, UnixSocket}]}) + end. + +connect_and_send_first_request(Address, Request, #state{options = Options0} = State) -> SocketType = socket_type(Request), ConnTimeout = (Request#request.settings)#http_options.connect_timeout, + Options = handle_unix_socket_options(Request, Options0), case connect(SocketType, Address, Options, ConnTimeout) of {ok, Socket} -> ClientClose = @@ -841,9 +866,10 @@ connect_and_send_first_request(Address, Request, #state{options = Options} = Sta {ok, State#state{request = Request}} end. -connect_and_send_upgrade_request(Address, Request, #state{options = Options} = State) -> +connect_and_send_upgrade_request(Address, Request, #state{options = Options0} = State) -> ConnTimeout = (Request#request.settings)#http_options.connect_timeout, SocketType = ip_comm, + Options = handle_unix_socket_options(Request, Options0), case connect(SocketType, Address, Options, ConnTimeout) of {ok, Socket} -> SessionType = httpc_manager:session_type(Options), @@ -1685,9 +1711,8 @@ update_session(ProfileName, #session{id = SessionId} = Session, Pos, Value) -> insert_session(Session2, ProfileName); error:badarg -> {stop, normal}; - T:E -> + T:E:Stacktrace -> %% Unexpected this must be an error! - Stacktrace = erlang:get_stacktrace(), error_logger:error_msg("Failed updating session: " "~n ProfileName: ~p" "~n SessionId: ~p" diff --git a/lib/inets/src/http_client/httpc_internal.hrl b/lib/inets/src/http_client/httpc_internal.hrl index 5f8c70f28d..c5fe439722 100644 --- a/lib/inets/src/http_client/httpc_internal.hrl +++ b/lib/inets/src/http_client/httpc_internal.hrl @@ -83,10 +83,11 @@ max_sessions = ?HTTP_MAX_TCP_SESSIONS, cookies = disabled, % enabled | disabled | verify verbose = false, % boolean(), - ipfamily = inet, % inet | inet6 | inet6fb4 + ipfamily = inet, % inet | inet6 | inet6fb4 | local ip = default, % specify local interface port = default, % specify local port - socket_opts = [] % other socket options + socket_opts = [], % other socket options + unix_socket = undefined % Local unix socket } ). -type options() :: #options{}. @@ -115,6 +116,7 @@ % request timer :: undefined | reference(), socket_opts, % undefined | [socket_option()] + unix_socket, % undefined | string() ipv6_host_with_brackets % boolean() } ). diff --git a/lib/inets/src/http_client/httpc_manager.erl b/lib/inets/src/http_client/httpc_manager.erl index ffdf1603b3..7b8d7875de 100644 --- a/lib/inets/src/http_client/httpc_manager.erl +++ b/lib/inets/src/http_client/httpc_manager.erl @@ -553,7 +553,8 @@ handle_cast({set_options, Options}, State = #state{options = OldOptions}) -> ip = get_ip(Options, OldOptions), port = get_port(Options, OldOptions), verbose = get_verbose(Options, OldOptions), - socket_opts = get_socket_opts(Options, OldOptions) + socket_opts = get_socket_opts(Options, OldOptions), + unix_socket = get_unix_socket_opts(Options, OldOptions) }, case {OldOptions#options.verbose, NewOptions#options.verbose} of {Same, Same} -> @@ -963,7 +964,10 @@ get_option(ip, #options{ip = IP}) -> get_option(port, #options{port = Port}) -> Port; get_option(socket_opts, #options{socket_opts = SocketOpts}) -> - SocketOpts. + SocketOpts; +get_option(unix_socket, #options{unix_socket = UnixSocket}) -> + UnixSocket. + get_proxy(Opts, #options{proxy = Default}) -> proplists:get_value(proxy, Opts, Default). @@ -1016,6 +1020,8 @@ get_verbose(Opts, #options{verbose = Default}) -> get_socket_opts(Opts, #options{socket_opts = Default}) -> proplists:get_value(socket_opts, Opts, Default). +get_unix_socket_opts(Opts, #options{unix_socket = Default}) -> + proplists:get_value(unix_socket, Opts, Default). handle_verbose(debug) -> dbg:p(self(), [call]), diff --git a/lib/inets/src/http_server/httpd_response.erl b/lib/inets/src/http_server/httpd_response.erl index 6b9053fda6..57ce162922 100644 --- a/lib/inets/src/http_server/httpd_response.erl +++ b/lib/inets/src/http_server/httpd_response.erl @@ -84,14 +84,14 @@ traverse_modules(ModData,[Module|Rest]) -> {proceed, NewData} -> traverse_modules(ModData#mod{data = NewData}, Rest) catch - T:E -> + T:E:Stacktrace -> String = lists:flatten( io_lib:format("module traverse failed: ~p:do => " "~n Error Type: ~p" "~n Error: ~p" "~n Stack trace: ~p", - [Module, T, E, ?STACK()])), + [Module, T, E, Stacktrace])), httpd_util:error_log(ModData#mod.config_db, String), send_status(ModData, 500, none), done diff --git a/lib/inets/src/inets_app/inets.appup.src b/lib/inets/src/inets_app/inets.appup.src index fdf4cc6e07..a86413147c 100644 --- a/lib/inets/src/inets_app/inets.appup.src +++ b/lib/inets/src/inets_app/inets.appup.src @@ -18,14 +18,10 @@ %% %CopyrightEnd% {"%VSN%", [ - {<<"6.4.3">>, [{load_module, httpd_esi, - soft_purge, soft_purge, []}]}, {<<"6\\..*">>,[{restart_application, inets}]}, {<<"5\\..*">>,[{restart_application, inets}]} ], [ - {<<"6.4.3">>, [{load_module, httpd_esi, - soft_purge, soft_purge, []}]}, {<<"6\\..*">>,[{restart_application, inets}]}, {<<"5\\..*">>,[{restart_application, inets}]} ] diff --git a/lib/inets/src/inets_app/inets_internal.hrl b/lib/inets/src/inets_app/inets_internal.hrl index 079b415b56..e0f59bba5f 100644 --- a/lib/inets/src/inets_app/inets_internal.hrl +++ b/lib/inets/src/inets_app/inets_internal.hrl @@ -22,8 +22,6 @@ -ifndef(inets_internal_hrl). -define(inets_internal_hrl, true). --define(STACK(), erlang:get_stacktrace()). - %% Various trace macros -define(report(Severity, Label, Service, Content), diff --git a/lib/inets/test/http_test_lib.erl b/lib/inets/test/http_test_lib.erl index 38e9e4976e..4e119cce04 100644 --- a/lib/inets/test/http_test_lib.erl +++ b/lib/inets/test/http_test_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2015-2015. All Rights Reserved. +%% Copyright Ericsson AB 2015-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -55,6 +55,25 @@ dummy_server_init(Caller, ip_comm, Inet, Extra) -> ]]}, [], ContentCb, Conf, ListenSocket); +dummy_server_init(Caller, unix_socket, Inet, Extra) -> + ContentCb = proplists:get_value(content_cb, Extra), + UnixSocket = proplists:get_value(unix_socket, Extra), + SocketAddr = {local, UnixSocket}, + BaseOpts = [binary, {packet, 0}, {reuseaddr,true}, {active, false}, {nodelay, true}, + {ifaddr, SocketAddr}], + Conf = proplists:get_value(conf, Extra), + {ok, ListenSocket} = gen_tcp:listen(0, [Inet | BaseOpts]), + {ok, Port} = inet:port(ListenSocket), + Caller ! {port, Port}, + dummy_ipcomm_server_loop({httpd_request, parse, [[{max_uri, ?HTTP_MAX_URI_SIZE}, + {max_header, ?HTTP_MAX_HEADER_SIZE}, + {max_version,?HTTP_MAX_VERSION_STRING}, + {max_method, ?HTTP_MAX_METHOD_STRING}, + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH}, + {customize, httpd_custom} + ]]}, + [], ContentCb, Conf, ListenSocket); + dummy_server_init(Caller, ssl, Inet, Extra) -> ContentCb = proplists:get_value(content_cb, Extra), SSLOptions = proplists:get_value(ssl, Extra), diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 0533b9ab70..38705372c9 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -37,6 +37,10 @@ -define(TLS_URL_START, "https://"). -define(NOT_IN_USE_PORT, 8997). +%% Using hardcoded file path to keep it below 107 charaters +%% (maximum length supported by erlang) +-define(UNIX_SOCKET, "/tmp/inets_httpc_SUITE.sock"). + -record(sslsocket, {fd = nil, pid = nil}). %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- @@ -50,6 +54,8 @@ all() -> [ {group, http}, {group, sim_http}, + {group, http_internal}, + {group, http_unix_socket}, {group, https}, {group, sim_https}, {group, misc} @@ -62,6 +68,8 @@ groups() -> %% and it shall be the last test case in the suite otherwise cookie %% will fail. {sim_http, [], only_simulated() ++ [process_leak_on_keepalive]}, + {http_internal, [], real_requests_esi()}, + {http_unix_socket, [], simulated_unix_socket()}, {https, [], real_requests()}, {sim_https, [], only_simulated()}, {misc, [], misc()} @@ -97,6 +105,12 @@ real_requests()-> invalid_body ]. +real_requests_esi() -> + [slow_connection]. + +simulated_unix_socket() -> + [unix_domain_socket]. + only_simulated() -> [ cookie, @@ -182,15 +196,29 @@ init_per_group(Group, Config0) when Group =:= sim_https; Group =:= https-> _:_ -> {skip, "Crypto did not start"} end; - +init_per_group(http_unix_socket = Group, Config0) -> + case os:type() of + {win32,_} -> + {skip, "Unix Domain Sockets are not supported on Windows"}; + _ -> + file:delete(?UNIX_SOCKET), + start_apps(Group), + Config = proplists:delete(port, Config0), + Port = server_start(Group, server_config(Group, Config)), + [{port, Port} | Config] + end; init_per_group(Group, Config0) -> start_apps(Group), Config = proplists:delete(port, Config0), Port = server_start(Group, server_config(Group, Config)), [{port, Port} | Config]. +end_per_group(http_unix_socket,_Config) -> + file:delete(?UNIX_SOCKET), + ok; end_per_group(_, _Config) -> ok. + do_init_per_group(Group, Config0) -> Config = proplists:delete(port, Config0), Port = server_start(Group, server_config(Group, Config)), @@ -1245,7 +1273,40 @@ stream_fun_server_close(Config) when is_list(Config) -> after 13000 -> ct:fail(did_not_receive_close) end. - + +%%-------------------------------------------------------------------- +slow_connection() -> + [{doc, "Test that a request on a slow keep-alive connection won't crash the httpc_manager"}]. +slow_connection(Config) when is_list(Config) -> + BodyFun = fun(0) -> eof; + (LenLeft) -> timer:sleep(1000), + {ok, lists:duplicate(10, "1"), LenLeft - 10} + end, + Request = {url(group_name(Config), "/httpc_SUITE:esi_post", Config), + [{"content-length", "100"}], + "text/plain", + {BodyFun, 100}}, + {ok, _} = httpc:request(post, Request, [], []), + %% Second request causes a crash if gen_server timeout is not set to infinity + %% in httpc_handler. + {ok, _} = httpc:request(post, Request, [], []). + +%%------------------------------------------------------------------------- +unix_domain_socket() -> + [{"doc, Test HTTP requests over unix domain sockets"}]. +unix_domain_socket(Config) when is_list(Config) -> + + URL = "http:///v1/kv/foo", + + {ok,[{unix_socket,?UNIX_SOCKET}]} = + httpc:get_options([unix_socket]), + {ok, {{_,200,_}, [_ | _], _}} + = httpc:request(put, {URL, [], [], ""}, [], []), + {ok, {{_,200,_}, [_ | _], _}} + = httpc:request(get, {URL, []}, [], []). + + + %%-------------------------------------------------------------------- %% Internal Functions ------------------------------------------------ %%-------------------------------------------------------------------- @@ -1339,6 +1400,8 @@ url(https, End, Config) -> ?TLS_URL_START ++ Host ++ ":" ++ integer_to_list(Port) ++ End; url(sim_http, End, Config) -> url(http, End, Config); +url(http_internal, End, Config) -> + url(http, End, Config); url(sim_https, End, Config) -> url(https, End, Config). url(http, UserInfo, End, Config) -> @@ -1358,19 +1421,28 @@ group_name(Config) -> server_start(sim_http, _) -> Inet = inet_version(), - ok = httpc:set_options([{ipfamily, Inet}]), + ok = httpc:set_options([{ipfamily, Inet},{unix_socket, undefined}]), {_Pid, Port} = http_test_lib:dummy_server(ip_comm, Inet, [{content_cb, ?MODULE}]), Port; server_start(sim_https, SslConfig) -> Inet = inet_version(), - ok = httpc:set_options([{ipfamily, Inet}]), + ok = httpc:set_options([{ipfamily, Inet},{unix_socket, undefined}]), {_Pid, Port} = http_test_lib:dummy_server(ssl, Inet, [{ssl, SslConfig}, {content_cb, ?MODULE}]), Port; +server_start(http_unix_socket, Config) -> + Inet = local, + Socket = proplists:get_value(unix_socket, Config), + ok = httpc:set_options([{ipfamily, Inet},{unix_socket, Socket}]), + {_Pid, Port} = http_test_lib:dummy_server(unix_socket, Inet, [{content_cb, ?MODULE}, + {unix_socket, Socket}]), + Port; + server_start(_, HttpdConfig) -> {ok, Pid} = inets:start(httpd, HttpdConfig), Serv = inets:services_info(), + ok = httpc:set_options([{ipfamily, inet_version()},{unix_socket, undefined}]), {value, {_, _, Info}} = lists:keysearch(Pid, 2, Serv), proplists:get_value(port, Info). @@ -1385,14 +1457,31 @@ server_config(http, Config) -> {mime_type, "text/plain"}, {script_alias, {"/cgi-bin/", filename:join(ServerRoot, "cgi-bin") ++ "/"}} ]; - +server_config(http_internal, Config) -> + ServerRoot = proplists:get_value(server_root, Config), + [{port, 0}, + {server_name,"httpc_test"}, + {server_root, ServerRoot}, + {document_root, proplists:get_value(doc_root, Config)}, + {bind_address, any}, + {ipfamily, inet_version()}, + {mime_type, "text/plain"}, + {erl_script_alias, {"", [httpc_SUITE]}} + ]; server_config(https, Config) -> [{socket_type, {essl, ssl_config(Config)}} | server_config(http, Config)]; server_config(sim_https, Config) -> ssl_config(Config); +server_config(http_unix_socket, _Config) -> + Socket = ?UNIX_SOCKET, + [{unix_socket, Socket}]; + server_config(_, _) -> []. +esi_post(Sid, _Env, _Input) -> + mod_esi:deliver(Sid, ["OK"]). + start_apps(https) -> inets_test_lib:start_apps([crypto, public_key, ssl]); start_apps(sim_https) -> @@ -2131,6 +2220,19 @@ handle_uri(_,"/delay_close.html",_,_,Socket,_) -> handle_uri("HEAD",_,_,_,_,_) -> "HTTP/1.1 200 ok\r\n" ++ "Content-Length:0\r\n\r\n"; +handle_uri("PUT","/v1/kv/foo",_,_,_,_) -> + "HTTP/1.1 200 OK\r\n" ++ + "Date: Tue, 20 Feb 2018 14:39:08 GMT\r\n" ++ + "Content-Length: 5\r\n\r\n" ++ + "Content-Type: application/json\r\n\r\n" ++ + "true\n"; +handle_uri("GET","/v1/kv/foo",_,_,_,_) -> + "HTTP/1.1 200 OK\r\n" ++ + "Date: Tue, 20 Feb 2018 14:39:08 GMT\r\n" ++ + "Content-Length: 24\r\n" ++ + "Content-Type: application/json\r\n\r\n" ++ + "[{\"Value\": \"aGVsbG8=\"}]\n"; + handle_uri(_,_,_,_,_,DefaultResponse) -> DefaultResponse. diff --git a/lib/inets/vsn.mk b/lib/inets/vsn.mk index 05cf4f6cc3..1fad9afe33 100644 --- a/lib/inets/vsn.mk +++ b/lib/inets/vsn.mk @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = inets -INETS_VSN = 6.4.5 +INETS_VSN = 6.5 PRE_VSN = APP_VSN = "$(APPLICATION)-$(INETS_VSN)$(PRE_VSN)" diff --git a/lib/kernel/doc/src/code.xml b/lib/kernel/doc/src/code.xml index c94f612c01..bd95819636 100644 --- a/lib/kernel/doc/src/code.xml +++ b/lib/kernel/doc/src/code.xml @@ -34,26 +34,28 @@ <p>This module contains the interface to the Erlang <em>code server</em>, which deals with the loading of compiled code into a running Erlang runtime system.</p> - <p>The runtime system can be started in <em>embedded</em> or - <em>interactive</em> mode. Which one is decided by command-line + <p>The runtime system can be started in <em>interactive</em> or + <em>embedded</em> mode. Which one is decided by the command-line flag <c>-mode</c>:</p> <pre> % <input>erl -mode interactive</input></pre> <p>The modes are as follows:</p> <list type="bulleted"> <item> - <p>In embedded mode, all code is loaded during system startup - according to the boot script. (Code can also be loaded later - by explicitly ordering the code server to do so).</p> - </item> - <item> <p>In interactive mode, which is default, only some code is loaded - during system startup, basically the modules needed by the runtime + during system startup, basically the modules needed by the runtime system. Other code is dynamically loaded when first referenced. When a call to a function in a certain module is made, and the module is not loaded, the code server searches for and tries to load the module.</p> </item> + <item> + <p>In embedded mode, modules are not auto loaded. Trying to use + a module that has not been loaded results in an error. This mode is + recommended when the boot script loads all modules, as it is + typically done in OTP releases. (Code can still be loaded later + by explicitly ordering the code server to do so).</p> + </item> </list> <p>To prevent accidentally reloading of modules affecting the Erlang runtime system, directories <c>kernel</c>, <c>stdlib</c>, diff --git a/lib/kernel/doc/src/file.xml b/lib/kernel/doc/src/file.xml index 8477b0e148..1b72769ce3 100644 --- a/lib/kernel/doc/src/file.xml +++ b/lib/kernel/doc/src/file.xml @@ -981,8 +981,7 @@ f.txt: {person, "kalle", 25}. </item> <tag><c>eisdir</c></tag> <item> - <p>The named file is not a regular file. It can be a - directory, a FIFO, or a device.</p> + <p>The named file is a directory.</p> </item> <tag><c>enotdir</c></tag> <item> diff --git a/lib/kernel/doc/src/notes.xml b/lib/kernel/doc/src/notes.xml index d7f224c38e..65fe9b9c07 100644 --- a/lib/kernel/doc/src/notes.xml +++ b/lib/kernel/doc/src/notes.xml @@ -31,6 +31,26 @@ </header> <p>This document describes the changes made to the Kernel application.</p> +<section><title>Kernel 5.4.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Add <c>os:cmd/2</c> that takes an options map as the + second argument.</p> + <p> + Add <c>max_size</c> as an option to <c>os:cmd/2</c> that + control the maximum size of the result that + <c>os:cmd/2</c> will return.</p> + <p> + Own Id: OTP-14823</p> + </item> + </list> + </section> + +</section> + <section><title>Kernel 5.4.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/kernel/doc/src/os.xml b/lib/kernel/doc/src/os.xml index c27182ff0b..ef416ed233 100644 --- a/lib/kernel/doc/src/os.xml +++ b/lib/kernel/doc/src/os.xml @@ -103,7 +103,7 @@ </desc> </datatype> <datatype> - <name name="command_input"/> + <name name="os_command"/> <desc> <p>All characters needs to be valid characters on the specific OS using @@ -116,19 +116,31 @@ </p> </desc> </datatype> + <datatype> + <name name="os_command_opts"/> + <desc> + <p>Options for <seealso marker="#cmd/2"><c>os:cmd/2</c></seealso></p> + <taglist> + <tag><c>max_size</c></tag> + <item> + <p>The maximum size of the data returned by the <c>os:cmd</c> call. + See the <seealso marker="#cmd/2"><c>os:cmd/2</c></seealso> + documentation for more details.</p> + </item> + </taglist> + </desc> + </datatype> </datatypes> - + <funcs> <func> <name name="cmd" arity="1"/> + <name name="cmd" arity="2"/> <fsummary>Execute a command in a shell of the target OS.</fsummary> <desc> <p>Executes <c><anno>Command</anno></c> in a command shell of the - target OS, - captures the standard output of the command, and returns this - result as a string. This function is a replacement of - the previous function <c>unix:cmd/1</c>; they are equivalent on a - Unix platform.</p> + target OS, captures the standard output of the command, + and returns this result as a string.</p> <warning><p>Previous implementation used to allow all characters as long as they were integer values greater than or equal to zero. This sometimes lead to unwanted results since null characters @@ -142,6 +154,21 @@ DirOut = os:cmd("dir"), % on Win32 platform</code> called from another program (for example, <c>os:cmd/1</c>) can differ, compared with the standard output of the command when called directly from an OS command shell.</p> + <p><c>os:cmd/2</c> was added in kernel-5.5 (OTP-20.2.1). It makes it + possible to pass an options map as the second argument in order to + control the behaviour of <c>os:cmd</c>. The possible options are: + </p> + <taglist> + <tag><c>max_size</c></tag> + <item> + <p>The maximum size of the data returned by the <c>os:cmd</c> call. + This option is a safety feature that should be used when the command + executed can return a very large, possibly infinite, result.</p> + <code type="none"> +> os:cmd("cat /dev/zero", #{ max_size => 20 }). +[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]</code> + </item> + </taglist> </desc> </func> diff --git a/lib/kernel/doc/src/rpc.xml b/lib/kernel/doc/src/rpc.xml index adec2d9520..fab616e630 100644 --- a/lib/kernel/doc/src/rpc.xml +++ b/lib/kernel/doc/src/rpc.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2016</year> + <year>1996</year><year>2018</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -217,7 +217,7 @@ <list type="bulleted"> <item>A list of the nodes that do not exist</item> <item>A list of the nodes where the server does not exist</item> - <item>A list of the nodes where the server terminatd before sending + <item>A list of the nodes where the server terminated before sending any reply.</item> </list> </desc> @@ -268,8 +268,9 @@ on the specified nodes and collects the answers. It returns <c>{<anno>ResL</anno>, <anno>BadNodes</anno>}</c>, where <c><anno>BadNodes</anno></c> is a list - of the nodes that terminated or timed out during computation, - and <c><anno>ResL</anno></c> is a list of the return values. + of the nodes that do not exist, + and <c><anno>ResL</anno></c> is a list of the return values, + or <c>{badrpc, <anno>Reason</anno>}</c> for failing calls. <c><anno>Timeout</anno></c> is a time (integer) in milliseconds, or <c>infinity</c>.</p> <p>The following example is useful when new object code is to @@ -347,7 +348,7 @@ <func> <name name="pmap" arity="3"/> - <fsummary>Parallell evaluation of mapping a function over a + <fsummary>Parallel evaluation of mapping a function over a list.</fsummary> <desc> <p>Evaluates <c>apply(<anno>Module</anno>, <anno>Function</anno>, diff --git a/lib/kernel/src/code_server.erl b/lib/kernel/src/code_server.erl index 418b0c50e1..f5a890cb95 100644 --- a/lib/kernel/src/code_server.erl +++ b/lib/kernel/src/code_server.erl @@ -340,8 +340,7 @@ handle_call(all_loaded, _From, S) -> {reply,all_loaded(Db),S}; handle_call({get_object_code,Mod}, _From, St) when is_atom(Mod) -> - Path = St#state.path, - case mod_to_bin(Path, Mod) of + case get_object_code(St, Mod) of {_,Bin,FName} -> {reply,{Mod,Bin,FName},St}; Error -> {reply,Error,St} end; @@ -1182,19 +1181,28 @@ load_file(Mod, From, St0) -> end, handle_pending_on_load(Action, Mod, From, St0). -load_file_1(Mod, From, #state{path=Path}=St) -> - case mod_to_bin(Path, Mod) of +load_file_1(Mod, From, St) -> + case get_object_code(St, Mod) of error -> {reply,{error,nofile},St}; {Mod,Binary,File} -> try_load_module_1(File, Mod, Binary, From, St) end. -mod_to_bin([Dir|Tail], Mod) -> - File = filename:append(Dir, atom_to_list(Mod) ++ objfile_extension()), +get_object_code(#state{path=Path}, Mod) when is_atom(Mod) -> + ModStr = atom_to_list(Mod), + case erl_prim_loader:is_basename(ModStr) of + true -> + mod_to_bin(Path, Mod, ModStr ++ objfile_extension()); + false -> + error + end. + +mod_to_bin([Dir|Tail], Mod, ModFile) -> + File = filename:append(Dir, ModFile), case erl_prim_loader:get_file(File) of error -> - mod_to_bin(Tail, Mod); + mod_to_bin(Tail, Mod, ModFile); {ok,Bin,_} -> case filename:pathtype(File) of absolute -> @@ -1203,10 +1211,9 @@ mod_to_bin([Dir|Tail], Mod) -> {Mod,Bin,absname(File)} end end; -mod_to_bin([], Mod) -> +mod_to_bin([], Mod, ModFile) -> %% At last, try also erl_prim_loader's own method - File = to_list(Mod) ++ objfile_extension(), - case erl_prim_loader:get_file(File) of + case erl_prim_loader:get_file(ModFile) of error -> error; % No more alternatives ! {ok,Bin,FName} -> diff --git a/lib/kernel/src/dist_util.erl b/lib/kernel/src/dist_util.erl index fb9f7fd7eb..5ea0ca991f 100644 --- a/lib/kernel/src/dist_util.erl +++ b/lib/kernel/src/dist_util.erl @@ -539,8 +539,8 @@ do_setnode(#hs_data{other_node = Node, socket = Socket, "no table space left for node ~w ** ~n", [Node]), ?shutdown(Node); - error:Other -> - exit({Other, erlang:get_stacktrace()}) + error:Other:Stacktrace -> + exit({Other, Stacktrace}) end; _ -> error_msg("** Distribution connection error, " diff --git a/lib/kernel/src/error_handler.erl b/lib/kernel/src/error_handler.erl index 59ca8e690d..a9582c6225 100644 --- a/lib/kernel/src/error_handler.erl +++ b/lib/kernel/src/error_handler.erl @@ -106,8 +106,8 @@ crash(M, F, A) -> crash(Tuple) -> try erlang:error(undef) catch - error:undef -> - Stk = [Tuple|tl(erlang:get_stacktrace())], + error:undef:Stacktrace -> + Stk = [Tuple|tl(Stacktrace)], erlang:raise(error, undef, Stk) end. diff --git a/lib/kernel/src/file.erl b/lib/kernel/src/file.erl index d05199897f..c2df1ee288 100644 --- a/lib/kernel/src/file.erl +++ b/lib/kernel/src/file.erl @@ -1382,8 +1382,8 @@ eval_stream2({ok,Form,EndLine}, Fd, H, Last, E, Bs0) -> try erl_eval:exprs(Form, Bs0) of {value,V,Bs} -> eval_stream(Fd, H, EndLine, {V}, E, Bs) - catch Class:Reason -> - Error = {EndLine,?MODULE,{Class,Reason,erlang:get_stacktrace()}}, + catch Class:Reason:StackTrace -> + Error = {EndLine,?MODULE,{Class,Reason,StackTrace}}, eval_stream(Fd, H, EndLine, Last, [Error|E], Bs0) end; eval_stream2({error,What,EndLine}, Fd, H, Last, E, Bs) -> diff --git a/lib/kernel/src/hipe_unified_loader.erl b/lib/kernel/src/hipe_unified_loader.erl index f8199fcf71..fd06f0f7d8 100644 --- a/lib/kernel/src/hipe_unified_loader.erl +++ b/lib/kernel/src/hipe_unified_loader.erl @@ -275,6 +275,7 @@ needs_trampolines(Architecture) -> arm -> true; powerpc -> true; ppc64 -> true; + amd64 -> true; _ -> false end. diff --git a/lib/kernel/src/os.erl b/lib/kernel/src/os.erl index fbc046c8f9..77c883f57f 100644 --- a/lib/kernel/src/os.erl +++ b/lib/kernel/src/os.erl @@ -21,11 +21,11 @@ %% Provides a common operating system interface. --export([type/0, version/0, cmd/1, find_executable/1, find_executable/2]). +-export([type/0, version/0, cmd/1, cmd/2, find_executable/1, find_executable/2]). -include("file.hrl"). --export_type([env_var_name/0, env_var_value/0, env_var_name_value/0, command_input/0]). +-export_type([env_var_name/0, env_var_value/0, env_var_name_value/0]). -export([getenv/0, getenv/1, getenv/2, putenv/2, unsetenv/1]). @@ -35,14 +35,17 @@ perf_counter/1, set_env_var/2, set_signal/2, system_time/0, system_time/1, timestamp/0, unset_env_var/1]). +-type os_command() :: atom() | io_lib:chars(). +-type os_command_opts() :: #{ max_size => non_neg_integer() | infinity }. + +-export_type([os_command/0, os_command_opts/0]). + -type env_var_name() :: nonempty_string(). -type env_var_value() :: string(). -type env_var_name_value() :: nonempty_string(). --type command_input() :: atom() | io_lib:chars(). - -spec list_env_vars() -> [{env_var_name(), env_var_value()}]. list_env_vars() -> erlang:nif_error(undef). @@ -260,14 +263,20 @@ extensions() -> %% Executes the given command in the default shell for the operating system. -spec cmd(Command) -> string() when - Command :: os:command_input(). + Command :: os_command(). cmd(Cmd) -> + cmd(Cmd, #{ }). + +-spec cmd(Command, Options) -> string() when + Command :: os_command(), + Options :: os_command_opts(). +cmd(Cmd, Opts) -> {SpawnCmd, SpawnOpts, SpawnInput, Eot} = mk_cmd(os:type(), validate(Cmd)), Port = open_port({spawn, SpawnCmd}, [binary, stderr_to_stdout, stream, in, hide | SpawnOpts]), MonRef = erlang:monitor(port, Port), true = port_command(Port, SpawnInput), - Bytes = get_data(Port, MonRef, Eot, []), + Bytes = get_data(Port, MonRef, Eot, [], 0, maps:get(max_size, Opts, infinity)), demonitor(MonRef, [flush]), String = unicode:characters_to_list(Bytes), if %% Convert to unicode list if possible otherwise return bytes @@ -332,12 +341,13 @@ validate2([List|Rest]) when is_list(List) -> validate2(List), validate2(Rest). -get_data(Port, MonRef, Eot, Sofar) -> +get_data(Port, MonRef, Eot, Sofar, Size, Max) -> receive {Port, {data, Bytes}} -> - case eot(Bytes, Eot) of + case eot(Bytes, Eot, Size, Max) of more -> - get_data(Port, MonRef, Eot, [Sofar,Bytes]); + get_data(Port, MonRef, Eot, [Sofar, Bytes], + Size + byte_size(Bytes), Max); Last -> catch port_close(Port), flush_until_down(Port, MonRef), @@ -348,13 +358,16 @@ get_data(Port, MonRef, Eot, Sofar) -> iolist_to_binary(Sofar) end. -eot(_Bs, <<>>) -> +eot(Bs, <<>>, Size, Max) when Size + byte_size(Bs) < Max -> more; -eot(Bs, Eot) -> +eot(Bs, <<>>, Size, Max) -> + binary:part(Bs, {0, Max - Size}); +eot(Bs, Eot, Size, Max) -> case binary:match(Bs, Eot) of - nomatch -> more; - {Pos, _} -> - binary:part(Bs,{0, Pos}) + {Pos, _} when Size + Pos < Max -> + binary:part(Bs,{0, Pos}); + _ -> + eot(Bs, <<>>, Size, Max) end. %% When port_close returns we know that all the diff --git a/lib/kernel/src/rpc.erl b/lib/kernel/src/rpc.erl index 0e0b7dffa3..b04aa9030b 100644 --- a/lib/kernel/src/rpc.erl +++ b/lib/kernel/src/rpc.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2017. All Rights Reserved. +%% Copyright Ericsson AB 1996-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -498,7 +498,7 @@ start_monitor(Node, Name) -> Module :: module(), Function :: atom(), Args :: [term()], - ResL :: [term()], + ResL :: [Res :: term() | {'badrpc', Reason :: term()}], BadNodes :: [node()]. multicall(M, F, A) -> @@ -509,14 +509,14 @@ multicall(M, F, A) -> Module :: module(), Function :: atom(), Args :: [term()], - ResL :: [term()], + ResL :: [Res :: term() | {'badrpc', Reason :: term()}], BadNodes :: [node()]; (Module, Function, Args, Timeout) -> {ResL, BadNodes} when Module :: module(), Function :: atom(), Args :: [term()], Timeout :: timeout(), - ResL :: [term()], + ResL :: [Res :: term() | {'badrpc', Reason :: term()}], BadNodes :: [node()]. multicall(Nodes, M, F, A) when is_list(Nodes) -> @@ -531,7 +531,7 @@ multicall(M, F, A, Timeout) -> Function :: atom(), Args :: [term()], Timeout :: timeout(), - ResL :: [term()], + ResL :: [Res :: term() | {'badrpc', Reason :: term()}], BadNodes :: [node()]. multicall(Nodes, M, F, A, infinity) diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl index 569753155f..902196def2 100644 --- a/lib/kernel/test/code_SUITE.erl +++ b/lib/kernel/test/code_SUITE.erl @@ -931,37 +931,34 @@ purge_stacktrace(Config) when is_list(Config) -> code:purge(code_b_test), try code_b_test:call(fun(b) -> ok end, a) catch - error:function_clause -> + error:function_clause:Stacktrace -> code:load_file(code_b_test), - case erlang:get_stacktrace() of + case Stacktrace of [{?MODULE,_,[a],_}, {code_b_test,call,2,_}, {?MODULE,purge_stacktrace,1,_}|_] -> - false = code:purge(code_b_test), - [] = erlang:get_stacktrace() + false = code:purge(code_b_test) end end, try code_b_test:call(nofun, 2) catch - error:function_clause -> + error:function_clause:Stacktrace2 -> code:load_file(code_b_test), - case erlang:get_stacktrace() of + case Stacktrace2 of [{code_b_test,call,[nofun,2],_}, {?MODULE,purge_stacktrace,1,_}|_] -> - false = code:purge(code_b_test), - [] = erlang:get_stacktrace() + false = code:purge(code_b_test) end end, Args = [erlang,error,[badarg]], try code_b_test:call(erlang, error, [badarg,Args]) catch - error:badarg -> + error:badarg:Stacktrace3 -> code:load_file(code_b_test), - case erlang:get_stacktrace() of + case Stacktrace3 of [{code_b_test,call,Args,_}, {?MODULE,purge_stacktrace,1,_}|_] -> - false = code:purge(code_b_test), - [] = erlang:get_stacktrace() + false = code:purge(code_b_test) end end, ok. diff --git a/lib/kernel/test/erl_distribution_SUITE.erl b/lib/kernel/test/erl_distribution_SUITE.erl index bbfaa9d147..f6791adf86 100644 --- a/lib/kernel/test/erl_distribution_SUITE.erl +++ b/lib/kernel/test/erl_distribution_SUITE.erl @@ -95,7 +95,11 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. - +init_per_testcase(TC, Config) when TC == hostnames; + TC == nodenames -> + file:make_dir("hostnames_nodedir"), + file:write_file("hostnames_nodedir/ignore_core_files",""), + Config; init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Config. @@ -251,7 +255,7 @@ test_node(Name, Illigal) -> end, net_kernel:monitor_nodes(true), BinCommand = unicode:characters_to_binary(Command, utf8), - Prt = open_port({spawn, BinCommand}, [stream]), + Prt = open_port({spawn, BinCommand}, [stream,{cd,"hostnames_nodedir"}]), Node = list_to_atom(Name), receive {nodeup, Node} -> @@ -459,9 +463,9 @@ run_remote_test([FuncStr, TestNodeStr | Args]) -> 1 end catch - C:E -> + C:E:S -> io:format("Node ~p got EXCEPTION ~p:~p\nat ~p\n", - [node(), C, E, erlang:get_stacktrace()]), + [node(), C, E, S]), 2 end, io:format("Node ~p doing halt(~p).\n",[node(), Status]), diff --git a/lib/kernel/test/file_SUITE.erl b/lib/kernel/test/file_SUITE.erl index 0cb8087a76..9a77454432 100644 --- a/lib/kernel/test/file_SUITE.erl +++ b/lib/kernel/test/file_SUITE.erl @@ -399,11 +399,11 @@ read_write_0(Str, {Func, ReadFun}, Options) -> io:format("~p:~p: ~p ERROR: ~ts vs~n ~w~n - ~p~n", [?MODULE, Line, Func, Str, ReadBytes, Options]), exit({error, ?LINE}); - error:What -> + error:What:Stacktrace -> io:format("~p:??: ~p ERROR: ~p from~n ~w~n ~p~n", [?MODULE, Func, What, Str, Options]), - io:format("\t~p~n", [erlang:get_stacktrace()]), + io:format("\t~p~n", [Stacktrace]), exit({error, ?LINE}) end. diff --git a/lib/kernel/test/gen_sctp_SUITE.erl b/lib/kernel/test/gen_sctp_SUITE.erl index 620ab235a0..9dde00652c 100644 --- a/lib/kernel/test/gen_sctp_SUITE.erl +++ b/lib/kernel/test/gen_sctp_SUITE.erl @@ -1038,8 +1038,7 @@ do_from_other_process(Fun) -> Result -> Parent ! {Ref,Result} catch - Class:Reason -> - Stacktrace = erlang:get_stacktrace(), + Class:Reason:Stacktrace -> Parent ! {Ref,Class,Reason,Stacktrace} end end), @@ -1617,8 +1616,7 @@ s_start(Socket, Timeout, Parent) -> try s_loop(Socket, Timeout, Parent, Handler, gb_trees:empty()) catch - Class:Reason -> - Stacktrace = erlang:get_stacktrace(), + Class:Reason:Stacktrace -> io:format(?MODULE_STRING":socket exception ~w:~w at~n" "~p.~n", [Class,Reason,Stacktrace]), erlang:raise(Class, Reason, Stacktrace) diff --git a/lib/kernel/test/gen_tcp_api_SUITE.erl b/lib/kernel/test/gen_tcp_api_SUITE.erl index 12d22519ce..0fe44e8a88 100644 --- a/lib/kernel/test/gen_tcp_api_SUITE.erl +++ b/lib/kernel/test/gen_tcp_api_SUITE.erl @@ -605,9 +605,9 @@ ok({ok,V}) -> V; ok(NotOk) -> try throw(not_ok) catch - Thrown -> + throw:Thrown:Stacktrace -> erlang:raise( - error, {Thrown, NotOk}, tl(erlang:get_stacktrace())) + error, {Thrown, NotOk}, tl(Stacktrace)) end. get_localaddr() -> diff --git a/lib/kernel/test/gen_udp_SUITE.erl b/lib/kernel/test/gen_udp_SUITE.erl index 96e495505a..6a50239c2a 100644 --- a/lib/kernel/test/gen_udp_SUITE.erl +++ b/lib/kernel/test/gen_udp_SUITE.erl @@ -755,9 +755,9 @@ ok({ok,V}) -> V; ok(NotOk) -> try throw(not_ok) catch - Thrown -> + throw:Thrown:Stacktrace -> erlang:raise( - error, {Thrown, NotOk}, tl(erlang:get_stacktrace())) + error, {Thrown, NotOk}, tl(Stacktrace)) end. diff --git a/lib/kernel/test/inet_SUITE.erl b/lib/kernel/test/inet_SUITE.erl index 77ac743a6e..2e5f8c7d2c 100644 --- a/lib/kernel/test/inet_SUITE.erl +++ b/lib/kernel/test/inet_SUITE.erl @@ -1104,11 +1104,9 @@ ifaddrs([{If,Opts}|IOs]) -> #ifopts{flags=F} = Ifopts = check_ifopts(Opts, #ifopts{name=If}), case F of {flags,Flags} -> - case lists:member(up, Flags) of - true -> - Ifopts#ifopts.addrs; - false -> - [] + case lists:member(running, Flags) of + true -> Ifopts#ifopts.addrs; + false -> [] end ++ ifaddrs(IOs); undefined -> ifaddrs(IOs) diff --git a/lib/kernel/test/inet_res_SUITE.erl b/lib/kernel/test/inet_res_SUITE.erl index 6691ad9c06..2a5b8d0044 100644 --- a/lib/kernel/test/inet_res_SUITE.erl +++ b/lib/kernel/test/inet_res_SUITE.erl @@ -217,10 +217,10 @@ proxy_start(TC, {NS,P}) -> spawn_link( fun () -> try proxy_start(TC, NS, P, Parent, Tag) - catch C:X -> + catch C:X:Stacktrace -> io:format( "~w: ~w:~p ~p~n", - [self(),C,X,erlang:get_stacktrace()]) + [self(),C,X,Stacktrace]) end end), receive {started,Tag,Port} -> diff --git a/lib/kernel/test/os_SUITE.erl b/lib/kernel/test/os_SUITE.erl index 8056321448..591fbb2125 100644 --- a/lib/kernel/test/os_SUITE.erl +++ b/lib/kernel/test/os_SUITE.erl @@ -26,7 +26,8 @@ null_in_command/1, space_in_name/1, bad_command/1, find_executable/1, unix_comment_in_command/1, deep_list_command/1, large_output_command/1, background_command/0, background_command/1, - message_leak/1, close_stdin/0, close_stdin/1, perf_counter_api/1]). + message_leak/1, close_stdin/0, close_stdin/1, max_size_command/1, + perf_counter_api/1]). -include_lib("common_test/include/ct.hrl"). @@ -39,7 +40,7 @@ all() -> space_in_name, bad_command, find_executable, unix_comment_in_command, deep_list_command, large_output_command, background_command, message_leak, - close_stdin, perf_counter_api]. + close_stdin, max_size_command, perf_counter_api]. groups() -> []. @@ -322,6 +323,19 @@ close_stdin(Config) -> "-1" = os:cmd(Fds). +max_size_command(_Config) -> + + Res20 = os:cmd("cat /dev/zero", #{ max_size => 20 }), + 20 = length(Res20), + + Res0 = os:cmd("cat /dev/zero", #{ max_size => 0 }), + 0 = length(Res0), + + Res32768 = os:cmd("cat /dev/zero", #{ max_size => 32768 }), + 32768 = length(Res32768), + + ResHello = string:trim(os:cmd("echo hello", #{ max_size => 20 })), + 5 = length(ResHello). %% Test that the os:perf_counter api works as expected perf_counter_api(_Config) -> diff --git a/lib/kernel/vsn.mk b/lib/kernel/vsn.mk index 106bda01ca..91261e1d55 100644 --- a/lib/kernel/vsn.mk +++ b/lib/kernel/vsn.mk @@ -1 +1 @@ -KERNEL_VSN = 5.4.1 +KERNEL_VSN = 5.4.2 diff --git a/lib/mnesia/src/mnesia.erl b/lib/mnesia/src/mnesia.erl index 95f7d4afc1..6d87544bd2 100644 --- a/lib/mnesia/src/mnesia.erl +++ b/lib/mnesia/src/mnesia.erl @@ -177,8 +177,8 @@ %% Local function in order to avoid external function call val(Var) -> - case ?catch_val(Var) of - {'EXIT', _} -> mnesia_lib:other_val(Var); + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> mnesia_lib:other_val(Var, Stacktrace); Value -> Value end. diff --git a/lib/mnesia/src/mnesia.hrl b/lib/mnesia/src/mnesia.hrl index da7e662288..e36222d098 100644 --- a/lib/mnesia/src/mnesia.hrl +++ b/lib/mnesia/src/mnesia.hrl @@ -47,6 +47,10 @@ -define(catch_val(Var), (try ?ets_lookup_element(mnesia_gvar, Var, 2) catch error:_ -> {'EXIT', {badarg, []}} end)). +-define(catch_val_and_stack(Var), + (try ?ets_lookup_element(mnesia_gvar, Var, 2) + catch error:_:_Stacktrace -> {'EXIT', _Stacktrace} end)). + %% It's important that counter is first, since we compare tid's -record(tid, diff --git a/lib/mnesia/src/mnesia_bup.erl b/lib/mnesia/src/mnesia_bup.erl index 34f16f178b..9dda340333 100644 --- a/lib/mnesia/src/mnesia_bup.erl +++ b/lib/mnesia/src/mnesia_bup.erl @@ -90,9 +90,9 @@ iterate(Mod, Fun, Opaque, Acc) -> catch throw:Err -> close_read(R2), Err; - _:Reason -> + _:Reason:Stacktrace -> close_read(R2), - {error, {Reason, erlang:get_stacktrace()}} + {error, {Reason, Stacktrace}} end catch throw:{error,_} = Err -> Err @@ -198,9 +198,9 @@ do_read_schema_section(R) -> try {R3, RawSchema} = safe_apply(R2, read, [R2#restore.bup_data]), do_read_schema_section(R3, verify_header(RawSchema), []) - catch T:E -> + catch T:E:S -> close_read(R2), - erlang:raise(T,E,erlang:get_stacktrace()) + erlang:raise(T,E,S) end. do_read_schema_section(R, {ok, B, C, []}, Acc) -> diff --git a/lib/mnesia/src/mnesia_checkpoint.erl b/lib/mnesia/src/mnesia_checkpoint.erl index 8112378ffd..8290237908 100644 --- a/lib/mnesia/src/mnesia_checkpoint.erl +++ b/lib/mnesia/src/mnesia_checkpoint.erl @@ -1269,9 +1269,9 @@ system_code_change(Cp, _Module, _OldVsn, _Extra) -> %%%%%%%%%%%%%%%%%%%%%%%%%% +%% Local function in order to avoid external function call val(Var) -> - case ?catch_val(Var) of - {'EXIT', _} -> mnesia_lib:other_val(Var); - _VaLuE_ -> _VaLuE_ + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> mnesia_lib:other_val(Var, Stacktrace); + Value -> Value end. - diff --git a/lib/mnesia/src/mnesia_controller.erl b/lib/mnesia/src/mnesia_controller.erl index 77013489b3..77e67a59db 100644 --- a/lib/mnesia/src/mnesia_controller.erl +++ b/lib/mnesia/src/mnesia_controller.erl @@ -185,9 +185,10 @@ max_loaders() -> worker_res }). +%% Local function in order to avoid external function call val(Var) -> - case ?catch_val(Var) of - {'EXIT', _} -> mnesia_lib:other_val(Var); + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> mnesia_lib:other_val(Var, Stacktrace); Value -> Value end. diff --git a/lib/mnesia/src/mnesia_dumper.erl b/lib/mnesia/src/mnesia_dumper.erl index f0ed7aef4a..31bcc1451f 100644 --- a/lib/mnesia/src/mnesia_dumper.erl +++ b/lib/mnesia/src/mnesia_dumper.erl @@ -191,8 +191,7 @@ do_perform_dump(Cont, InPlace, InitBy, Regulator, OldVersion) -> try insert_recs(Recs, InPlace, InitBy, Regulator, OldVersion) of Version -> do_perform_dump(C2, InPlace, InitBy, Regulator, Version) - catch _:R when R =/= fatal -> - ST = erlang:get_stacktrace(), + catch _:R:ST when R =/= fatal -> Reason = {"Transaction log dump error: ~tp~n", [{R, ST}]}, close_files(InPlace, {error, Reason}, InitBy), exit(Reason) @@ -325,8 +324,7 @@ perform_update(Tid, SchemaOps, _DumperMode, _UseDir) -> ?eval_debug_fun({?MODULE, post_dump}, [InitBy]), close_files(InPlace, ok, InitBy), ok - catch _:Reason when Reason =/= fatal -> - ST = erlang:get_stacktrace(), + catch _:Reason:ST when Reason =/= fatal -> Error = {error, {"Schema update error", {Reason, ST}}}, close_files(InPlace, Error, InitBy), fatal("Schema update error ~tp ~tp", [{Reason,ST}, SchemaOps]) @@ -1471,8 +1469,9 @@ regulate(RegulatorPid) -> {regulated, RegulatorPid} -> ok end. +%% Local function in order to avoid external function call val(Var) -> - case ?catch_val(Var) of - {'EXIT', _} -> mnesia_lib:other_val(Var); + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> mnesia_lib:other_val(Var, Stacktrace); Value -> Value end. diff --git a/lib/mnesia/src/mnesia_frag.erl b/lib/mnesia/src/mnesia_frag.erl index c39f30e140..63fb4981fe 100644 --- a/lib/mnesia/src/mnesia_frag.erl +++ b/lib/mnesia/src/mnesia_frag.erl @@ -1157,9 +1157,10 @@ remove_node(Node, Cs) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Helpers +%% Local function in order to avoid external function call val(Var) -> - case ?catch_val(Var) of - {'EXIT', _} -> mnesia_lib:other_val(Var); + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> mnesia_lib:other_val(Var, Stacktrace); Value -> Value end. diff --git a/lib/mnesia/src/mnesia_lib.erl b/lib/mnesia/src/mnesia_lib.erl index 53fdd76de8..56f6926df0 100644 --- a/lib/mnesia/src/mnesia_lib.erl +++ b/lib/mnesia/src/mnesia_lib.erl @@ -116,7 +116,7 @@ lock_table/1, mkcore/1, not_active_here/1, - other_val/1, + other_val/2, overload_read/0, overload_read/1, overload_set/2, @@ -435,8 +435,8 @@ validate_record(Tab, Obj) -> %% val(Var) -> - case ?catch_val(Var) of - {'EXIT', _} -> other_val(Var); + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> other_val(Var, Stacktrace); _VaLuE_ -> _VaLuE_ end. @@ -446,9 +446,9 @@ set(Var, Val) -> unset(Var) -> ?ets_delete(mnesia_gvar, Var). -other_val(Var) -> +other_val(Var, Stacktrace) -> case other_val_1(Var) of - error -> pr_other(Var); + error -> pr_other(Var, Stacktrace); Val -> Val end. @@ -460,8 +460,8 @@ other_val_1(Var) -> _ -> error end. --spec pr_other(_) -> no_return(). -pr_other(Var) -> +-spec pr_other(_, _) -> no_return(). +pr_other(Var, Stacktrace) -> Why = case is_running() of no -> {node_not_running, node()}; @@ -469,7 +469,7 @@ pr_other(Var) -> end, verbose("~p (~tp) val(mnesia_gvar, ~tw) -> ~p ~tp ~n", [self(), process_info(self(), registered_name), - Var, Why, erlang:get_stacktrace()]), + Var, Why, Stacktrace]), mnesia:abort(Why). %% Some functions for list valued variables diff --git a/lib/mnesia/src/mnesia_loader.erl b/lib/mnesia/src/mnesia_loader.erl index 4c6336cb73..bbceb9ba66 100644 --- a/lib/mnesia/src/mnesia_loader.erl +++ b/lib/mnesia/src/mnesia_loader.erl @@ -34,9 +34,10 @@ -include("mnesia.hrl"). +%% Local function in order to avoid external function call val(Var) -> - case ?catch_val(Var) of - {'EXIT', _} -> mnesia_lib:other_val(Var); + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> mnesia_lib:other_val(Var, Stacktrace); Value -> Value end. @@ -535,7 +536,7 @@ init_table(Tab, _, Fun, _DetsInfo,_) -> try true = ets:init_table(Tab, Fun), ok - catch _:Else -> {Else, erlang:get_stacktrace()} + catch _:Else:Stacktrace -> {Else, Stacktrace} end. @@ -777,9 +778,9 @@ do_send_table(Pid, Tab, Storage, RemoteS) -> throw:receiver_died -> cleanup_tab_copier(Pid, Storage, Tab), ok; - error:Reason -> %% Prepare failed + error:Reason:Stacktrace -> %% Prepare failed cleanup_tab_copier(Pid, Storage, Tab), - {error, {tab_copier, Tab, {Reason, erlang:get_stacktrace()}}} + {error, {tab_copier, Tab, {Reason, Stacktrace}}} after unlink(whereis(mnesia_tm)) end. diff --git a/lib/mnesia/src/mnesia_locker.erl b/lib/mnesia/src/mnesia_locker.erl index 073b48abc0..710bf38cb5 100644 --- a/lib/mnesia/src/mnesia_locker.erl +++ b/lib/mnesia/src/mnesia_locker.erl @@ -97,10 +97,11 @@ init(Parent) -> end, loop(#state{supervisor = Parent}). +%% Local function in order to avoid external function call val(Var) -> - case ?catch_val(Var) of - {'EXIT', _} -> mnesia_lib:other_val(Var); - _VaLuE_ -> _VaLuE_ + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> mnesia_lib:other_val(Var, Stacktrace); + Value -> Value end. reply(From, R) -> diff --git a/lib/mnesia/src/mnesia_recover.erl b/lib/mnesia/src/mnesia_recover.erl index d792070332..475bf3c327 100644 --- a/lib/mnesia/src/mnesia_recover.erl +++ b/lib/mnesia/src/mnesia_recover.erl @@ -177,10 +177,10 @@ disconnect(Node) -> log_decision(D) -> cast({log_decision, D}). +%% Local function in order to avoid external function call val(Var) -> - case ?catch_val(Var) of - {'EXIT', _Reason} -> - mnesia_lib:other_val(Var); + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> mnesia_lib:other_val(Var, Stacktrace); Value -> Value end. diff --git a/lib/mnesia/src/mnesia_schema.erl b/lib/mnesia/src/mnesia_schema.erl index 71952af31c..de6c3fb4ee 100644 --- a/lib/mnesia/src/mnesia_schema.erl +++ b/lib/mnesia/src/mnesia_schema.erl @@ -181,9 +181,10 @@ exit_on_error({error, Reason}) -> exit_on_error(GoodRes) -> GoodRes. +%% Local function in order to avoid external function call val(Var) -> - case ?catch_val(Var) of - {'EXIT', _} -> mnesia_lib:other_val(Var); + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> mnesia_lib:other_val(Var, Stacktrace); Value -> Value end. @@ -2695,10 +2696,10 @@ prepare_op(_Tid, {op, transform, Fun, TabDef}, _WaitFor) -> Objs -> mnesia_lib:db_fixtable(Storage, Tab, false), {true, Objs, mandatory} - catch _:Reason -> + catch _:Reason:Stacktrace -> mnesia_lib:db_fixtable(Storage, Tab, false), mnesia_lib:important("Transform function failed: '~tp' in '~tp'", - [Reason, erlang:get_stacktrace()]), + [Reason, Stacktrace]), exit({"Bad transform function", Tab, Fun, node(), Reason}) end end; diff --git a/lib/mnesia/src/mnesia_tm.erl b/lib/mnesia/src/mnesia_tm.erl index ebf580d09e..eaebdf6d02 100644 --- a/lib/mnesia/src/mnesia_tm.erl +++ b/lib/mnesia/src/mnesia_tm.erl @@ -121,10 +121,11 @@ init(Parent) -> proc_lib:init_ack(Parent, {ok, self()}), doit_loop(#state{supervisor = Parent}). +%% Local function in order to avoid external function call val(Var) -> - case ?catch_val(Var) of - {'EXIT', _} -> mnesia_lib:other_val(Var); - _VaLuE_ -> _VaLuE_ + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> mnesia_lib:other_val(Var, Stacktrace); + Value -> Value end. reply({From,Ref}, R) -> @@ -597,9 +598,9 @@ recover_coordinator(Tid, Etabs) -> false -> %% When killed before store havn't been copied to ok %% to the new nested trans store. end - catch _:Reason -> + catch _:Reason:Stacktrace -> dbg_out("Recovery of coordinator ~p failed: ~tp~n", - [Tid, {Reason, erlang:get_stacktrace()}]), + [Tid, {Reason, Stacktrace}]), Protocol = asym_trans, tell_outcome(Tid, Protocol, node(), CheckNodes, TellNodes) end, @@ -825,8 +826,7 @@ execute_transaction(Fun, Args, Factor, Retries, Type) -> catch throw:Value -> %% User called throw Reason = {aborted, {throw, Value}}, return_abort(Fun, Args, Reason); - error:Reason -> - ST = erlang:get_stacktrace(), + error:Reason:ST -> check_exit(Fun, Args, Factor, Retries, {Reason,ST}, Type); _:Reason -> check_exit(Fun, Args, Factor, Retries, Reason, Type) @@ -1796,14 +1796,13 @@ do_update(Tid, Storage, [Op | Ops], OldRes) -> try do_update_op(Tid, Storage, Op) of ok -> do_update(Tid, Storage, Ops, OldRes); NewRes -> do_update(Tid, Storage, Ops, NewRes) - catch _:Reason -> + catch _:Reason:ST -> %% This may only happen when we recently have %% deleted our local replica, changed storage_type %% or transformed table %% BUGBUG: Updates may be lost if storage_type is changed. %% Determine actual storage type and try again. %% BUGBUG: Updates may be lost if table is transformed. - ST = erlang:get_stacktrace(), verbose("do_update in ~w failed: ~tp -> {'EXIT', ~tp}~n", [Tid, Op, {Reason, ST}]), do_update(Tid, Storage, Ops, OldRes) @@ -1914,11 +1913,10 @@ commit_clear([H|R], Tid, Storage, Tab, K, Obj) do_snmp(_, []) -> ok; do_snmp(Tid, [Head|Tail]) -> try mnesia_snmp_hook:update(Head) - catch _:Reason -> + catch _:Reason:ST -> %% This should only happen when we recently have %% deleted our local replica or recently deattached %% the snmp table - ST = erlang:get_stacktrace(), verbose("do_snmp in ~w failed: ~tp -> {'EXIT', ~tp}~n", [Tid, Head, {Reason, ST}]) end, diff --git a/lib/mnesia/test/mnesia_test_lib.erl b/lib/mnesia/test/mnesia_test_lib.erl index 78dbe7ffde..db0e7beac6 100644 --- a/lib/mnesia/test/mnesia_test_lib.erl +++ b/lib/mnesia/test/mnesia_test_lib.erl @@ -470,9 +470,9 @@ get_suite(Mod, {group, Suite}) -> {_, _, TCList} = lists:keyfind(Suite, 1, Groups), TCList catch - _:Reason -> + _:Reason:Stacktrace -> io:format("Not implemented ~p ~p (~p ~p)~n", - [Mod,Suite,Reason, erlang:get_stacktrace()]), + [Mod,Suite,Reason,Stacktrace]), 'NYI' end; get_suite(Mod, all) -> diff --git a/lib/mnesia/test/mnesia_test_lib.hrl b/lib/mnesia/test/mnesia_test_lib.hrl index ba7eb10ea2..90a3c686b0 100644 --- a/lib/mnesia/test/mnesia_test_lib.hrl +++ b/lib/mnesia/test/mnesia_test_lib.hrl @@ -55,25 +55,25 @@ ?error("Not Matching Actual result was:~n ~p~n",[_AR_0]), {fail,_AR_0} catch - exit:{aborted, _ER_1} when + exit:{aborted, _ER_1}:Stacktrace when element(1, _ER_1) =:= node_not_running; element(1, _ER_1) =:= bad_commit; element(1, _ER_1) =:= cyclic -> %% Need to re-raise these to restart transaction - erlang:raise(exit, {aborted, _ER_1}, erlang:get_stacktrace()); - exit:_AR_1 -> + erlang:raise(exit, {aborted, _ER_1}, Stacktrace); + exit:_AR_1:Stacktrace -> case fun(_AR_EXIT_) -> {'EXIT', _AR_EXIT_} end(_AR_1) of _AR_2 = ExpectedRes -> ?verbose("ok, ~n Result as expected:~p~n",[_AR_2]), {success,_AR_2}; _AR_2 -> ?error("Not Matching Actual result was:~n ~p~n ~p~n", - [_AR_2, erlang:get_stacktrace()]), + [_AR_2, Stacktrace]), {fail,_AR_2} end; - _T1_:_AR_1 -> + _T1_:_AR_1:Stacktrace -> ?error("Not Matching Actual result was:~n ~p~n ~p~n", - [{_T1_,_AR_1}, erlang:get_stacktrace()]), + [{_T1_,_AR_1}, Stacktrace]), {fail,{_T1_,_AR_1}} end end()). diff --git a/lib/observer/src/cdv_port_cb.erl b/lib/observer/src/cdv_port_cb.erl index b5cbe8132d..6bb8f07a74 100644 --- a/lib/observer/src/cdv_port_cb.erl +++ b/lib/observer/src/cdv_port_cb.erl @@ -34,7 +34,8 @@ -define(COL_CONN, ?COL_ID+1). -define(COL_NAME, ?COL_CONN+1). -define(COL_CTRL, ?COL_NAME+1). --define(COL_SLOT, ?COL_CTRL+1). +-define(COL_QUEUE, ?COL_CTRL+1). +-define(COL_SLOT, ?COL_QUEUE+1). @@ -44,6 +45,7 @@ col_to_elem(?COL_ID) -> #port.id; col_to_elem(?COL_CONN) -> #port.connected; col_to_elem(?COL_NAME) -> #port.name; col_to_elem(?COL_CTRL) -> #port.controls; +col_to_elem(?COL_QUEUE) -> #port.queue; col_to_elem(?COL_SLOT) -> #port.slot. col_spec() -> @@ -51,6 +53,7 @@ col_spec() -> {"Connected", ?wxLIST_FORMAT_LEFT, 120}, {"Name", ?wxLIST_FORMAT_LEFT, 150}, {"Controls", ?wxLIST_FORMAT_LEFT, 200}, + {"Queue", ?wxLIST_FORMAT_RIGHT, 100}, {"Slot", ?wxLIST_FORMAT_RIGHT, 50}]. get_info(_) -> @@ -96,9 +99,17 @@ format(D) -> info_fields() -> [{"Overview", [{"Name", name}, + {"State", state}, + {"Task Flags", task_flags}, {"Connected", {click,connected}}, {"Slot", slot}, - {"Controls", controls}]}, + {"Controls", controls}, + {"Input bytes", input}, + {"Output bytes", output}, + {"Queue bytes", queue}, + {"Port data", port_data}]}, {scroll_boxes, [{"Links",1,{click,links}}, - {"Monitors",1,{click,monitors}}]}]. + {"Monitors",1,{click,monitors}}, + {"Suspended",1,{click,suspended}} + ]}]. diff --git a/lib/observer/src/cdv_proc_cb.erl b/lib/observer/src/cdv_proc_cb.erl index f10650bbb7..0ea23dd7cb 100644 --- a/lib/observer/src/cdv_proc_cb.erl +++ b/lib/observer/src/cdv_proc_cb.erl @@ -149,6 +149,10 @@ info_fields() -> {"Old Heap", old_heap}, {"Heap Unused", heap_unused}, {"Old Heap Unused", old_heap_unused}, + {"Binary vheap", bin_vheap}, + {"Old Binary vheap", old_bin_vheap}, + {"Binary vheap unused", bin_vheap_unused}, + {"Old Binary vheap unused", old_bin_vheap_unused}, {"Number of Heap Fragements", num_heap_frag}, {"Heap Fragment Data",heap_frag_data}, {"New Heap Start", new_heap_start}, diff --git a/lib/observer/src/cdv_sched_cb.erl b/lib/observer/src/cdv_sched_cb.erl index 192aaf31a7..d2696a276f 100644 --- a/lib/observer/src/cdv_sched_cb.erl +++ b/lib/observer/src/cdv_sched_cb.erl @@ -31,7 +31,8 @@ %% Columns -define(COL_ID, 0). --define(COL_PROC, ?COL_ID+1). +-define(COL_TYPE, ?COL_ID+1). +-define(COL_PROC, ?COL_TYPE+1). -define(COL_PORT, ?COL_PROC+1). -define(COL_RQL, ?COL_PORT+1). -define(COL_PQL, ?COL_RQL+1). @@ -39,6 +40,7 @@ %% Callbacks for cdv_virtual_list_wx col_to_elem(id) -> col_to_elem(?COL_ID); col_to_elem(?COL_ID) -> #sched.name; +col_to_elem(?COL_TYPE) -> #sched.type; col_to_elem(?COL_PROC) -> #sched.process; col_to_elem(?COL_PORT) -> #sched.port; col_to_elem(?COL_RQL) -> #sched.run_q; @@ -46,6 +48,7 @@ col_to_elem(?COL_PQL) -> #sched.port_q. col_spec() -> [{"Id", ?wxLIST_FORMAT_RIGHT, 50}, + {"Type", ?wxLIST_FORMAT_CENTER, 100}, {"Current Process", ?wxLIST_FORMAT_CENTER, 130}, {"Current Port", ?wxLIST_FORMAT_CENTER, 130}, {"Run Queue Length", ?wxLIST_FORMAT_RIGHT, 180}, @@ -73,7 +76,8 @@ detail_pages() -> [{"Scheduler Information", fun init_gen_page/2}]. init_gen_page(Parent, Info0) -> - Fields = info_fields(), + Type = proplists:get_value(type, Info0), + Fields = info_fields(Type), Details = proplists:get_value(details, Info0), Info = if is_map(Details) -> Info0 ++ maps:to_list(Details); true -> Info0 @@ -81,15 +85,16 @@ init_gen_page(Parent, Info0) -> cdv_info_wx:start_link(Parent,{Fields,Info,[]}). %%% Internal -info_fields() -> +info_fields(Type) -> [{"Scheduler Overview", [{"Id", id}, + {"Type", type}, {"Current Process",process}, {"Current Port", port}, {"Sleep Info Flags", sleep_info}, {"Sleep Aux Work", sleep_aux} ]}, - {"Run Queues", + {run_queues_header(Type), [{"Flags", runq_flags}, {"Priority Max Length", runq_max}, {"Priority High Length", runq_high}, @@ -116,3 +121,8 @@ info_fields() -> {" ", {currp_stack, 11}} ]} ]. + +run_queues_header(normal) -> + "Run Queues"; +run_queues_header(DirtyX) -> + "Run Queues (common for all '" ++ atom_to_list(DirtyX) ++ "' schedulers)". diff --git a/lib/observer/src/crashdump_viewer.erl b/lib/observer/src/crashdump_viewer.erl index bba97624a7..d0c14db486 100644 --- a/lib/observer/src/crashdump_viewer.erl +++ b/lib/observer/src/crashdump_viewer.erl @@ -116,6 +116,10 @@ -define(allocator,allocator). -define(atoms,atoms). -define(binary,binary). +-define(dirty_cpu_scheduler,dirty_cpu_scheduler). +-define(dirty_cpu_run_queue,dirty_cpu_run_queue). +-define(dirty_io_scheduler,dirty_io_scheduler). +-define(dirty_io_run_queue,dirty_io_run_queue). -define(ende,ende). -define(erl_crash_dump,erl_crash_dump). -define(ets,ets). @@ -1222,6 +1226,18 @@ all_procinfo(Fd,Fun,Proc,WS,LineHead) -> "OldHeap unused" -> Bytes = list_to_integer(bytes(Fd))*WS, get_procinfo(Fd,Fun,Proc#proc{old_heap_unused=Bytes},WS); + "BinVHeap" -> + Bytes = list_to_integer(bytes(Fd))*WS, + get_procinfo(Fd,Fun,Proc#proc{bin_vheap=Bytes},WS); + "OldBinVHeap" -> + Bytes = list_to_integer(bytes(Fd))*WS, + get_procinfo(Fd,Fun,Proc#proc{old_bin_vheap=Bytes},WS); + "BinVHeap unused" -> + Bytes = list_to_integer(bytes(Fd))*WS, + get_procinfo(Fd,Fun,Proc#proc{bin_vheap_unused=Bytes},WS); + "OldBinVHeap unused" -> + Bytes = list_to_integer(bytes(Fd))*WS, + get_procinfo(Fd,Fun,Proc#proc{old_bin_vheap_unused=Bytes},WS); "New heap start" -> get_procinfo(Fd,Fun,Proc#proc{new_heap_start=bytes(Fd)},WS); "New heap top" -> @@ -1632,6 +1648,10 @@ port_to_tuple("#Port<"++Port) -> get_portinfo(Fd,Port) -> case line_head(Fd) of + "State" -> + get_portinfo(Fd,Port#port{state=bytes(Fd)}); + "Task Flags" -> + get_portinfo(Fd,Port#port{task_flags=bytes(Fd)}); "Slot" -> %% stored as integer so we can sort on it get_portinfo(Fd,Port#port{slot=list_to_integer(bytes(Fd))}); @@ -1656,6 +1676,10 @@ get_portinfo(Fd,Port) -> {Pid,Pid++" ("++Ref++")"} end || Mon <- Monitors0], get_portinfo(Fd,Port#port{monitors=Monitors}); + "Suspended" -> + Pids = split_pid_list_no_space(bytes(Fd)), + Suspended = [{Pid,Pid} || Pid <- Pids], + get_portinfo(Fd,Port#port{suspended=Suspended}); "Port controls linked-in driver" -> Str = lists:flatten(["Linked in driver: " | string(Fd)]), get_portinfo(Fd,Port#port{controls=Str}); @@ -1671,6 +1695,15 @@ get_portinfo(Fd,Port) -> "Port is UNIX fd not opened by emulator" -> Str = lists:flatten(["UNIX fd not opened by emulator: "| string(Fd)]), get_portinfo(Fd,Port#port{controls=Str}); + "Input" -> + get_portinfo(Fd,Port#port{input=list_to_integer(bytes(Fd))}); + "Output" -> + get_portinfo(Fd,Port#port{output=list_to_integer(bytes(Fd))}); + "Queue" -> + get_portinfo(Fd,Port#port{queue=list_to_integer(bytes(Fd))}); + "Port Data" -> + get_portinfo(Fd,Port#port{port_data=string(Fd)}); + "=" ++ _next_tag -> Port; Other -> @@ -2503,73 +2536,142 @@ get_indextableinfo1(Fd,IndexTable) -> %%----------------------------------------------------------------- %% Page with scheduler table information schedulers(File) -> - case lookup_index(?scheduler) of - [] -> - []; - Schedulers -> - Fd = open(File), - R = lists:map(fun({Name,Start}) -> - get_schedulerinfo(Fd,Name,Start) - end, - Schedulers), - close(Fd), - R - end. + Fd = open(File), -get_schedulerinfo(Fd,Name,Start) -> + Schds0 = case lookup_index(?scheduler) of + [] -> + []; + Normals -> + [{Normals, #sched{type=normal}}] + end, + Schds1 = case lookup_index(?dirty_cpu_scheduler) of + [] -> + Schds0; + DirtyCpus -> + [{DirtyCpus, get_dirty_runqueue(Fd, ?dirty_cpu_run_queue)} + | Schds0] + end, + Schds2 = case lookup_index(?dirty_io_scheduler) of + [] -> + Schds1; + DirtyIos -> + [{DirtyIos, get_dirty_runqueue(Fd, ?dirty_io_run_queue)} + | Schds1] + end, + + R = schedulers1(Fd, Schds2, []), + close(Fd), + R. + +schedulers1(_Fd, [], Acc) -> + Acc; +schedulers1(Fd, [{Scheds,Sched0} | Tail], Acc0) -> + Acc1 = lists:foldl(fun({Name,Start}, AccIn) -> + [get_schedulerinfo(Fd,Name,Start,Sched0) | AccIn] + end, + Acc0, + Scheds), + schedulers1(Fd, Tail, Acc1). + +get_schedulerinfo(Fd,Name,Start,Sched0) -> pos_bof(Fd,Start), - get_schedulerinfo1(Fd,#sched{name=Name}). + get_schedulerinfo1(Fd,Sched0#sched{name=list_to_integer(Name)}). -get_schedulerinfo1(Fd,Sched=#sched{details=Ds}) -> +sched_type(?dirty_cpu_run_queue) -> dirty_cpu; +sched_type(?dirty_io_run_queue) -> dirty_io. + +get_schedulerinfo1(Fd, Sched) -> + case get_schedulerinfo2(Fd, Sched) of + {more, Sched2} -> + get_schedulerinfo1(Fd, Sched2); + {done, Sched2} -> + Sched2 + end. + +get_schedulerinfo2(Fd, Sched=#sched{details=Ds}) -> case line_head(Fd) of "Current Process" -> - get_schedulerinfo1(Fd,Sched#sched{process=bytes(Fd, "None")}); + {more, Sched#sched{process=bytes(Fd, "None")}}; "Current Port" -> - get_schedulerinfo1(Fd,Sched#sched{port=bytes(Fd, "None")}); + {more, Sched#sched{port=bytes(Fd, "None")}}; + + "Scheduler Sleep Info Flags" -> + {more, Sched#sched{details=Ds#{sleep_info=>bytes(Fd, "None")}}}; + "Scheduler Sleep Info Aux Work" -> + {more, Sched#sched{details=Ds#{sleep_aux=>bytes(Fd, "None")}}}; + + "Current Process State" -> + {more, Sched#sched{details=Ds#{currp_state=>bytes(Fd)}}}; + "Current Process Internal State" -> + {more, Sched#sched{details=Ds#{currp_int_state=>bytes(Fd)}}}; + "Current Process Program counter" -> + {more, Sched#sched{details=Ds#{currp_prg_cnt=>string(Fd)}}}; + "Current Process CP" -> + {more, Sched#sched{details=Ds#{currp_cp=>string(Fd)}}}; + "Current Process Limited Stack Trace" -> + %% If there shall be last in scheduler information block + {done, Sched#sched{details=get_limited_stack(Fd, 0, Ds)}}; + + "=" ++ _next_tag -> + {done, Sched}; + + Other -> + case Sched#sched.type of + normal -> + get_runqueue_info2(Fd, Other, Sched); + _ -> + unexpected(Fd,Other,"dirty scheduler information"), + {done, Sched} + end + end. + +get_dirty_runqueue(Fd, Tag) -> + case lookup_index(Tag) of + [{_, Start}] -> + pos_bof(Fd,Start), + get_runqueue_info1(Fd,#sched{type=sched_type(Tag)}); + [] -> + #sched{} + end. + +get_runqueue_info1(Fd, Sched) -> + case get_runqueue_info2(Fd, line_head(Fd), Sched) of + {more, Sched2} -> + get_runqueue_info1(Fd, Sched2); + {done, Sched2} -> + Sched2 + end. + +get_runqueue_info2(Fd, LineHead, Sched=#sched{details=Ds}) -> + case LineHead of "Run Queue Max Length" -> RQMax = list_to_integer(bytes(Fd)), RQ = RQMax + Sched#sched.run_q, - get_schedulerinfo1(Fd,Sched#sched{run_q=RQ, details=Ds#{runq_max=>RQMax}}); + {more, Sched#sched{run_q=RQ, details=Ds#{runq_max=>RQMax}}}; "Run Queue High Length" -> RQHigh = list_to_integer(bytes(Fd)), RQ = RQHigh + Sched#sched.run_q, - get_schedulerinfo1(Fd,Sched#sched{run_q=RQ, details=Ds#{runq_high=>RQHigh}}); + {more, Sched#sched{run_q=RQ, details=Ds#{runq_high=>RQHigh}}}; "Run Queue Normal Length" -> RQNorm = list_to_integer(bytes(Fd)), RQ = RQNorm + Sched#sched.run_q, - get_schedulerinfo1(Fd,Sched#sched{run_q=RQ, details=Ds#{runq_norm=>RQNorm}}); + {more, Sched#sched{run_q=RQ, details=Ds#{runq_norm=>RQNorm}}}; "Run Queue Low Length" -> RQLow = list_to_integer(bytes(Fd)), RQ = RQLow + Sched#sched.run_q, - get_schedulerinfo1(Fd,Sched#sched{run_q=RQ, details=Ds#{runq_low=>RQLow}}); + {more, Sched#sched{run_q=RQ, details=Ds#{runq_low=>RQLow}}}; "Run Queue Port Length" -> RQ = list_to_integer(bytes(Fd)), - get_schedulerinfo1(Fd,Sched#sched{port_q=RQ}); - - "Scheduler Sleep Info Flags" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{sleep_info=>bytes(Fd, "None")}}); - "Scheduler Sleep Info Aux Work" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{sleep_aux=>bytes(Fd, "None")}}); + {more, Sched#sched{port_q=RQ}}; "Run Queue Flags" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{runq_flags=>bytes(Fd, "None")}}); + {more, Sched#sched{details=Ds#{runq_flags=>bytes(Fd, "None")}}}; - "Current Process State" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{currp_state=>bytes(Fd)}}); - "Current Process Internal State" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{currp_int_state=>bytes(Fd)}}); - "Current Process Program counter" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{currp_prg_cnt=>string(Fd)}}); - "Current Process CP" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{currp_cp=>string(Fd)}}); - "Current Process Limited Stack Trace" -> - %% If there shall be last in scheduler information block - Sched#sched{details=get_limited_stack(Fd, 0, Ds)}; "=" ++ _next_tag -> - Sched; + {done, Sched}; Other -> unexpected(Fd,Other,"scheduler information"), - Sched + {done, Sched} end. get_limited_stack(Fd, N, Ds) -> @@ -3000,6 +3102,10 @@ tag_to_atom("allocated_areas") -> ?allocated_areas; tag_to_atom("allocator") -> ?allocator; tag_to_atom("atoms") -> ?atoms; tag_to_atom("binary") -> ?binary; +tag_to_atom("dirty_cpu_scheduler") -> ?dirty_cpu_scheduler; +tag_to_atom("dirty_cpu_run_queue") -> ?dirty_cpu_run_queue; +tag_to_atom("dirty_io_scheduler") -> ?dirty_io_scheduler; +tag_to_atom("dirty_io_run_queue") -> ?dirty_io_run_queue; tag_to_atom("end") -> ?ende; tag_to_atom("erl_crash_dump") -> ?erl_crash_dump; tag_to_atom("ets") -> ?ets; diff --git a/lib/observer/src/crashdump_viewer.hrl b/lib/observer/src/crashdump_viewer.hrl index 6a93a089fd..252e19379d 100644 --- a/lib/observer/src/crashdump_viewer.hrl +++ b/lib/observer/src/crashdump_viewer.hrl @@ -80,6 +80,10 @@ old_heap, heap_unused, old_heap_unused, + bin_vheap, + old_bin_vheap, + bin_vheap_unused, + old_bin_vheap_unused, new_heap_start, new_heap_top, stack_top, @@ -95,19 +99,27 @@ -record(port, {id, + state, + task_flags=0, slot, connected, links, name, monitors, - controls}). + suspended, + controls, + input, + output, + queue, + port_data}). -record(sched, {name, + type, process, port, run_q=0, - port_q=0, + port_q, details=#{} }). diff --git a/lib/observer/src/observer_alloc_wx.erl b/lib/observer/src/observer_alloc_wx.erl index 7f4b3dd484..e0f319b6cc 100644 --- a/lib/observer/src/observer_alloc_wx.erl +++ b/lib/observer/src/observer_alloc_wx.erl @@ -79,8 +79,8 @@ init([Notebook, Parent, Config]) -> max = #{} } } - catch _:Err -> - io:format("~p crashed ~tp: ~tp~n",[?MODULE, Err, erlang:get_stacktrace()]), + catch _:Err:Stacktrace -> + io:format("~p crashed ~tp: ~tp~n",[?MODULE, Err, Stacktrace]), {stop, Err} end. diff --git a/lib/observer/src/observer_perf_wx.erl b/lib/observer/src/observer_perf_wx.erl index 5adfadb16e..eddb9327f3 100644 --- a/lib/observer/src/observer_perf_wx.erl +++ b/lib/observer/src/observer_perf_wx.erl @@ -86,8 +86,8 @@ init([Notebook, Parent, Config]) -> secs=maps:get(secs, Config, ?DISP_SECONDS)} }, {Panel, State0} - catch _:Err -> - io:format("~p crashed ~tp: ~tp~n",[?MODULE, Err, erlang:get_stacktrace()]), + catch _:Err:Stacktrace -> + io:format("~p crashed ~tp: ~tp~n",[?MODULE, Err, Stacktrace]), {stop, Err} end. diff --git a/lib/observer/src/observer_traceoptions_wx.erl b/lib/observer/src/observer_traceoptions_wx.erl index fbcf6d7fe9..ff8c40ce77 100644 --- a/lib/observer/src/observer_traceoptions_wx.erl +++ b/lib/observer/src/observer_traceoptions_wx.erl @@ -536,7 +536,7 @@ ms_from_string(Str) -> {error, List} -> throw([[Error, $\n] || {_, Error} <- List]) end catch error:_Reason -> - %% io:format("Bad term: ~ts~n ~tp in ~tp~n", [Str, _Reason, erlang:get_stacktrace()]), + %% io:format("Bad term: ~ts~n ~tp in ~tp~n", [Str, _Reason, Stacktrace]), throw("Invalid term") end. diff --git a/lib/observer/test/crashdump_viewer_SUITE.erl b/lib/observer/test/crashdump_viewer_SUITE.erl index 32773a779e..41ca3f3ce9 100644 --- a/lib/observer/test/crashdump_viewer_SUITE.erl +++ b/lib/observer/test/crashdump_viewer_SUITE.erl @@ -820,10 +820,10 @@ dump_with_size_limit_reached(DataDir,Rel,DumpName,Max) -> "-env ERL_CRASH_DUMP_BYTES " ++ integer_to_list(Bytes)), {ok,#file_info{size=Size}} = file:read_file_info(CD), - if Size < Bytes -> + if Size =< Bytes -> %% This means that the dump was actually smaller than the %% randomly selected truncation size, so we'll just do it - %% again with a smaller numer + %% again with a smaller number ok = file:delete(CD), dump_with_size_limit_reached(DataDir,Rel,DumpName,Size-3); true -> diff --git a/lib/parsetools/include/yeccpre.hrl b/lib/parsetools/include/yeccpre.hrl index 91d6cd49a6..53e8124aaa 100644 --- a/lib/parsetools/include/yeccpre.hrl +++ b/lib/parsetools/include/yeccpre.hrl @@ -56,8 +56,7 @@ return_error(Line, Message) -> yeccpars0(Tokens, Tzr, State, States, Vstack) -> try yeccpars1(Tokens, Tzr, State, States, Vstack) catch - error: Error -> - Stacktrace = erlang:get_stacktrace(), + error: Error: Stacktrace -> try yecc_error_type(Error, Stacktrace) of Desc -> erlang:raise(error, {yecc_bug, ?CODE_VERSION, Desc}, diff --git a/lib/parsetools/src/yeccparser.erl b/lib/parsetools/src/yeccparser.erl index 6f6f66d56c..45d91a83a3 100644 --- a/lib/parsetools/src/yeccparser.erl +++ b/lib/parsetools/src/yeccparser.erl @@ -72,8 +72,7 @@ return_error(Line, Message) -> yeccpars0(Tokens, Tzr, State, States, Vstack) -> try yeccpars1(Tokens, Tzr, State, States, Vstack) catch - error: Error -> - Stacktrace = erlang:get_stacktrace(), + error: Error: Stacktrace -> try yecc_error_type(Error, Stacktrace) of Desc -> erlang:raise(error, {yecc_bug, ?CODE_VERSION, Desc}, @@ -167,21 +166,20 @@ yecctoken_location(Token) -> end. -compile({nowarn_unused_function, yecctoken2string/1}). -yecctoken2string({atom, _, A}) -> io_lib:write(A); +yecctoken2string({atom, _, A}) -> io_lib:write_atom(A); yecctoken2string({integer,_,N}) -> io_lib:write(N); yecctoken2string({float,_,F}) -> io_lib:write(F); yecctoken2string({char,_,C}) -> io_lib:write_char(C); yecctoken2string({var,_,V}) -> io_lib:format("~s", [V]); yecctoken2string({string,_,S}) -> io_lib:write_string(S); yecctoken2string({reserved_symbol, _, A}) -> io_lib:write(A); -yecctoken2string({_Cat, _, Val}) -> io_lib:format("~p",[Val]); +yecctoken2string({_Cat, _, Val}) -> io_lib:format("~tp", [Val]); yecctoken2string({dot, _}) -> "'.'"; -yecctoken2string({'$end', _}) -> - []; +yecctoken2string({'$end', _}) -> []; yecctoken2string({Other, _}) when is_atom(Other) -> - io_lib:write(Other); + io_lib:write_atom(Other); yecctoken2string(Other) -> - io_lib:write(Other). + io_lib:format("~tp", [Other]). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/parsetools/test/yecc_SUITE.erl b/lib/parsetools/test/yecc_SUITE.erl index a7166b91ed..2d3b15326b 100644 --- a/lib/parsetools/test/yecc_SUITE.erl +++ b/lib/parsetools/test/yecc_SUITE.erl @@ -1674,8 +1674,7 @@ format_error(Message) -> yeccpars0(Tokens, MFA) -> try yeccpars1(Tokens, MFA, 0, [], []) catch - error: Error -> - Stacktrace = erlang:get_stacktrace(), + error: Error : Stacktrace -> try yecc_error_type(Error, Stacktrace) of {syntax_error, Token} -> yeccerror(Token); diff --git a/lib/reltool/src/reltool_app_win.erl b/lib/reltool/src/reltool_app_win.erl index 663144861f..335b40fa74 100644 --- a/lib/reltool/src/reltool_app_win.erl +++ b/lib/reltool/src/reltool_app_win.erl @@ -102,8 +102,8 @@ init(Parent, WxEnv, Xref, C, AppName) -> try do_init(Parent, WxEnv, Xref, C, AppName) catch - error:Reason -> - exit({Reason, erlang:get_stacktrace()}) + error:Reason:Stacktrace -> + exit({Reason, Stacktrace}) end. do_init(Parent, WxEnv, Xref, C, AppName) -> diff --git a/lib/reltool/src/reltool_mod_win.erl b/lib/reltool/src/reltool_mod_win.erl index dcd802a5de..8272d864c2 100644 --- a/lib/reltool/src/reltool_mod_win.erl +++ b/lib/reltool/src/reltool_mod_win.erl @@ -107,8 +107,8 @@ init(Parent, WxEnv, Xref, RelPid, C, ModName) -> try do_init(Parent, WxEnv, Xref, RelPid, C, ModName) catch - error:Reason -> - exit({Reason, erlang:get_stacktrace()}) + error:Reason:Stacktrace -> + exit({Reason, Stacktrace}) end. do_init(Parent, WxEnv, Xref, RelPid, C, ModName) -> diff --git a/lib/reltool/src/reltool_server.erl b/lib/reltool/src/reltool_server.erl index 853191c696..66ee1a3742 100644 --- a/lib/reltool/src/reltool_server.erl +++ b/lib/reltool/src/reltool_server.erl @@ -135,8 +135,8 @@ init([{parent,Parent}|_] = Options) -> catch throw:{error,Reason} -> proc_lib:init_ack(Parent,{error,Reason}); - error:Reason -> - exit({Reason, erlang:get_stacktrace()}) + error:Reason:Stacktrace -> + exit({Reason, Stacktrace}) end. do_init(Options) -> diff --git a/lib/reltool/src/reltool_sys_win.erl b/lib/reltool/src/reltool_sys_win.erl index 92df270752..d87b1dc91f 100644 --- a/lib/reltool/src/reltool_sys_win.erl +++ b/lib/reltool/src/reltool_sys_win.erl @@ -135,9 +135,9 @@ init(Options) -> try do_init(Options) catch - error:Reason -> - io:format("~tp: ~tp~n",[Reason, erlang:get_stacktrace()]), - exit({Reason, erlang:get_stacktrace()}) + error:Reason:Stacktrace -> + io:format("~tp: ~tp~n",[Reason, Stacktrace]), + exit({Reason, Stacktrace}) end. do_init([{safe_config, Safe}, {parent, Parent} | Options]) -> diff --git a/lib/runtime_tools/doc/src/Makefile b/lib/runtime_tools/doc/src/Makefile index ec19a4ce59..a9b0056a93 100644 --- a/lib/runtime_tools/doc/src/Makefile +++ b/lib/runtime_tools/doc/src/Makefile @@ -41,7 +41,13 @@ RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) # Target Specs # ---------------------------------------------------- XML_APPLICATION_FILES = ref_man.xml -XML_REF3_FILES = dbg.xml dyntrace.xml erts_alloc_config.xml system_information.xml msacc.xml +XML_REF3_FILES = \ + dbg.xml \ + dyntrace.xml \ + erts_alloc_config.xml \ + system_information.xml \ + msacc.xml \ + scheduler.xml XML_REF6_FILES = runtime_tools_app.xml XML_PART_FILES = part.xml diff --git a/lib/runtime_tools/doc/src/notes.xml b/lib/runtime_tools/doc/src/notes.xml index 93e3e26fda..74300ba3fc 100644 --- a/lib/runtime_tools/doc/src/notes.xml +++ b/lib/runtime_tools/doc/src/notes.xml @@ -32,6 +32,23 @@ <p>This document describes the changes made to the Runtime_Tools application.</p> +<section><title>Runtime_Tools 1.12.4</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + New family of <c>erts_alloc</c> strategies: Age Order + First Fit. Similar to "address order", but instead the + oldest possible carrier is always chosen for allocation.</p> + <p> + Own Id: OTP-14917 Aux Id: ERIERL-88 </p> + </item> + </list> + </section> + +</section> + <section><title>Runtime_Tools 1.12.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/runtime_tools/doc/src/ref_man.xml b/lib/runtime_tools/doc/src/ref_man.xml index d2fb7a29af..eb3a6f0f5c 100644 --- a/lib/runtime_tools/doc/src/ref_man.xml +++ b/lib/runtime_tools/doc/src/ref_man.xml @@ -37,6 +37,7 @@ <xi:include href="dyntrace.xml"/> <xi:include href="erts_alloc_config.xml"/> <xi:include href="msacc.xml"/> + <xi:include href="scheduler.xml"/> <xi:include href="system_information.xml"/> </application> diff --git a/lib/runtime_tools/doc/src/scheduler.xml b/lib/runtime_tools/doc/src/scheduler.xml new file mode 100644 index 0000000000..dd8bf73bae --- /dev/null +++ b/lib/runtime_tools/doc/src/scheduler.xml @@ -0,0 +1,135 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title></title> + <prepared></prepared> + <responsible></responsible> + <docno>1</docno> + <approved></approved> + <checked></checked> + <date></date> + <rev></rev> + <file>scheduler.xml</file> + </header> + <module>scheduler</module> + <modulesummary>Measure scheduler utilization</modulesummary> + <description> + <p>This module contains utility functions for easier measurement and + calculation of scheduler utilization, otherwise obtained from calling the + more primitive <seealso marker="erts:erlang#statistics_scheduler_wall_time"> + <c>statistics(scheduler_wall_time)</c></seealso>.</p> + <p>The simplest usage is to call <seealso marker="#utilization-1"> + <c>scheduler:utilization(Seconds)</c></seealso>.</p> + </description> + + <datatypes> + <datatype> + <name name="sched_sample"/> + </datatype> + <datatype> + <name name="sched_type"/> + </datatype> + <datatype> + <name name="sched_id"/> + </datatype> + <datatype> + <name name="sched_util_result"/> + <desc> + <p>A list of tuples containing results for individual schedulers + as well as aggregated averages. <c>Util</c> is the scheduler utilization + as a floating point value between 0.0 and 1.0. <c>Percent</c> is the + same utilization as a more human readable string expressed in percent.</p> + <taglist> + <tag><c>{normal, SchedulerId, Util, Percent}</c></tag> + <item>Scheduler utilization of a normal scheduler with number + <c>SchedulerId</c>.</item> + <tag><c>{cpu, SchedulerId, Util, Percent}</c></tag> + <item>Scheduler utilization of a dirty-cpu scheduler with number + <c>SchedulerId</c>.</item> + <tag><c>{io, SchedulerId, Util, Percent}</c></tag> + <item>Scheduler utilization of a dirty-io scheduler with number + <c>SchedulerId</c>. This tuple will only exist if both samples were + taken with <seealso marker="#sample_all-0"><c>sample_all/0</c></seealso>.</item> + <tag><c>{total, Util, Percent}</c></tag> + <item>Total utilization of all normal and dirty-cpu schedulers.</item> + <tag><c>{weighted, Util, Percent}</c></tag> + <item>Total utilization of all normal and dirty-cpu schedulers, + weighted against maximum amount of available CPU time.</item> + </taglist> + </desc> + </datatype> + </datatypes> + + <funcs> + + <func> + <name name="sample" arity="0"/> + <fsummary>Get scheduler utilization sample.</fsummary> + <desc> + <p>Return a scheduler utilization sample for normal and dirty-cpu + schedulers.</p> + </desc> + </func> + + <func> + <name name="sample_all" arity="0"/> + <fsummary>Get scheduler utilization sample.</fsummary> + <desc> + <p>Return a scheduler utilization sample for all schedulers, + including dirty-io schedulers.</p> + </desc> + </func> + + <func> + <name name="utilization" arity="1" clause_i="1"/> + <fsummary>Measure scheduler utilizations during a period of time.</fsummary> + <desc> + <p>Measure utilization for normal and dirty-cpu schedulers during + <c><anno>Seconds</anno></c> seconds, and then return the result.</p> + </desc> + </func> + + <func> + <name name="utilization" arity="1" clause_i="2"/> + <fsummary>Measure scheduler utilizations since sample.</fsummary> + <desc> + <p>Calculate scheduler utilizations for the time interval from when + <c><anno>Sample</anno></c> was taken and "now". The same as calling + <c>scheduler:utilization(Sample, scheduler:sample_all())</c>.</p> + </desc> + </func> + + <func> + <name name="utilization" arity="2"/> + <fsummary>Measure scheduler utilizations between two samples.</fsummary> + <desc> + <p>Calculates scheduler utilizations for the time interval between + the two samples obtained from calling + <seealso marker="#sample-0"><c>sample/0</c></seealso> or + <seealso marker="#sample_all-0"><c>sample_all/0</c></seealso>.</p> + </desc> + </func> + + </funcs> + </erlref> diff --git a/lib/runtime_tools/doc/src/specs.xml b/lib/runtime_tools/doc/src/specs.xml index 978bd39e55..33fe7fa370 100644 --- a/lib/runtime_tools/doc/src/specs.xml +++ b/lib/runtime_tools/doc/src/specs.xml @@ -2,4 +2,5 @@ <specs xmlns:xi="http://www.w3.org/2001/XInclude"> <xi:include href="../specs/specs_system_information.xml"/> <xi:include href="../specs/specs_msacc.xml"/> + <xi:include href="../specs/specs_scheduler.xml"/> </specs> diff --git a/lib/runtime_tools/src/Makefile b/lib/runtime_tools/src/Makefile index 5a99c6e240..6faa9c2e35 100644 --- a/lib/runtime_tools/src/Makefile +++ b/lib/runtime_tools/src/Makefile @@ -45,6 +45,7 @@ MODULES= \ system_information \ observer_backend \ ttb_autostart\ + scheduler\ msacc HRL_FILES= ../include/observer_backend.hrl diff --git a/lib/runtime_tools/src/erts_alloc_config.erl b/lib/runtime_tools/src/erts_alloc_config.erl index 342363d08d..6c1a945484 100644 --- a/lib/runtime_tools/src/erts_alloc_config.erl +++ b/lib/runtime_tools/src/erts_alloc_config.erl @@ -265,7 +265,13 @@ strategy_str(aoff) -> strategy_str(aoffcbf) -> "Address order first fit carrier best fit"; strategy_str(aoffcaobf) -> - "Address order first fit carrier adress order best fit". + "Address order first fit carrier adress order best fit"; +strategy_str(ageffcaoff) -> + "Age order first fit carrier address order first fit"; +strategy_str(ageffcbf) -> + "Age order first fit carrier best fit"; +strategy_str(ageffcaobf) -> + "Age order first fit carrier adress order best fit". default_acul(A, S) -> case carrier_migration_support(S) of diff --git a/lib/runtime_tools/src/observer_backend.erl b/lib/runtime_tools/src/observer_backend.erl index a1edde8516..7ec3b38930 100644 --- a/lib/runtime_tools/src/observer_backend.erl +++ b/lib/runtime_tools/src/observer_backend.erl @@ -279,7 +279,7 @@ get_table_list(mnesia, Opts) -> end, [Tab|Acc] catch _:_What -> - %% io:format("Skipped ~p: ~p ~p ~n",[Id, _What, erlang:get_stacktrace()]), + %% io:format("Skipped ~p: ~p ~p ~n",[Id, _What, Stacktrace]), Acc end end, diff --git a/lib/runtime_tools/src/runtime_tools.app.src b/lib/runtime_tools/src/runtime_tools.app.src index 449532e5c4..09a9b447c2 100644 --- a/lib/runtime_tools/src/runtime_tools.app.src +++ b/lib/runtime_tools/src/runtime_tools.app.src @@ -23,6 +23,7 @@ {modules, [appmon_info, dbg,observer_backend,runtime_tools, runtime_tools_sup,erts_alloc_config, ttb_autostart,dyntrace,system_information, + scheduler, msacc]}, {registered, [runtime_tools_sup]}, {applications, [kernel, stdlib]}, diff --git a/lib/runtime_tools/src/scheduler.erl b/lib/runtime_tools/src/scheduler.erl new file mode 100644 index 0000000000..c896b671ac --- /dev/null +++ b/lib/runtime_tools/src/scheduler.erl @@ -0,0 +1,152 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% @doc Utility functions for easier measurement of scheduler utilization +%% using erlang:statistics(scheduler_wall_time). + +-module(scheduler). + +-export([sample/0, + sample_all/0, + utilization/1, + utilization/2]). + +-export_type([sched_sample/0]). + + +-opaque sched_sample() :: + {scheduler_wall_time | scheduler_wall_time_all, + [{sched_type(), sched_id(), ActiveTime::integer(), TotalTime::integer()}]}. + +-type sched_type() :: normal | cpu | io. + +-type sched_id() :: integer(). + +-spec sample() -> sched_sample(). +sample() -> + sample(scheduler_wall_time). + +-spec sample_all() -> sched_sample(). +sample_all() -> + sample(scheduler_wall_time_all). + +sample(Stats) -> + case erlang:statistics(Stats) of + undefined -> + erlang:system_flag(scheduler_wall_time, true), + sample(Stats); + + List -> + Sorted = lists:sort(List), + Tagged = lists:map(fun({I, A, T}) -> {sched_tag(I), I, A, T} end, + Sorted), + {Stats, Tagged} + end. + +-type sched_util_result() :: + [{sched_type(), sched_id(), float(), string()} | + {total, float(), string()} | + {weighted, float(), string()}]. + +-spec utilization(Seconds) -> sched_util_result() when + Seconds :: pos_integer(); + (Sample) -> sched_util_result() when + Sample :: sched_sample(). +utilization(Seconds) when is_integer(Seconds), Seconds > 0 -> + OldFlag = erlang:system_flag(scheduler_wall_time, true), + T0 = sample(), + receive after Seconds*1000 -> ok end, + T1 = sample(), + case OldFlag of + false -> + erlang:system_flag(scheduler_wall_time, OldFlag); + true -> + ok + end, + utilization(T0,T1); + +utilization({Stats, _}=T0) when Stats =:= scheduler_wall_time; + Stats =:= scheduler_wall_time_all -> + utilization(T0, sample(Stats)). + +-spec utilization(Sample1, Sample2) -> sched_util_result() when + Sample1 :: sched_sample(), + Sample2 :: sched_sample(). +utilization({Stats, Ts0}, {Stats, Ts1}) -> + Diffs = lists:map(fun({{Tag, I, A0, T0}, {Tag, I, A1, T1}}) -> + {Tag, I, (A1 - A0), (T1 - T0)} + end, + lists:zip(Ts0,Ts1)), + + {Lst0, {A, T, N}} = lists:foldl(fun({Tag, I, Adiff, Tdiff}, {Lst, Acc}) -> + R = safe_div(Adiff, Tdiff), + {[{Tag, I, R, percent(R)} | Lst], + acc(Tag, Adiff, Tdiff, Acc)} + end, + {[], {0, 0, 0}}, + Diffs), + + Total = safe_div(A, T), + Lst1 = lists:reverse(Lst0), + Lst2 = case erlang:system_info(logical_processors_available) of + unknown -> Lst1; + LPA -> + Weighted = Total * (N / LPA), + [{weighted, Weighted, percent(Weighted)} | Lst1] + end, + [{total, Total, percent(Total)} | Lst2]; + +utilization({scheduler_wall_time, _}=T0, + {scheduler_wall_time_all, Ts1}) -> + utilization(T0, {scheduler_wall_time, remove_io(Ts1)}); + +utilization({scheduler_wall_time_all, Ts0}, + {scheduler_wall_time, _}=T1) -> + utilization({scheduler_wall_time, remove_io(Ts0)}, T1). + +%% Do not include dirty-io in totals +acc(io, _, _, Acc) -> + Acc; +acc(Tag, Adiff, Tdiff, {Asum, Tsum, N}) when Tag =:= normal; Tag =:= cpu -> + {Adiff+Asum, Tdiff+Tsum, N+1}. + + +remove_io(Ts) -> + lists:filter(fun({io,_,_,_}) -> false; + (_) -> true end, + Ts). + +safe_div(A, B) -> + if B == 0.0 -> 0.0; + true -> A / B + end. + +sched_tag(Nr) -> + Normal = erlang:system_info(schedulers), + Cpu = Normal + erlang:system_info(dirty_cpu_schedulers), + case Nr of + _ when Nr =< Normal -> normal; + _ when Nr =< Cpu -> cpu; + _ -> io + end. + + +percent(F) -> + float_to_list(F*100, [{decimals,1}]) ++ [$%]. diff --git a/lib/runtime_tools/src/system_information.erl b/lib/runtime_tools/src/system_information.erl index 119d7cc3d4..136ee55b54 100644 --- a/lib/runtime_tools/src/system_information.erl +++ b/lib/runtime_tools/src/system_information.erl @@ -488,10 +488,10 @@ to_fd(Fd) -> ok = file:write(Fd, io_lib:format(Format, Args)) end, - EmitChunk("{system_information_version, ~p}.~n" + EmitChunk("{system_information_version, ~w}.~n" "{system_information,[" - "{init_arguments,~p}," - "{code_paths,~p},", + "{init_arguments,~w}," + "{code_paths,~w},", [?REPORT_FILE_VSN, init:get_arguments(), code:get_path()]), @@ -499,12 +499,12 @@ to_fd(Fd) -> emit_code_info(EmitChunk), EmitChunk( "," %% Note the leading comma! - "{system_info,~p}," - "{erts_compile_info,~p}," - "{beam_dynamic_libraries,~p}," - "{environment_erts,~p}," - "{environment,~p}," - "{sanity_check,~p}" + "{system_info,~w}," + "{erts_compile_info,~w}," + "{beam_dynamic_libraries,~w}," + "{environment_erts,~w}," + "{environment,~w}," + "{sanity_check,~w}" "]}.~n", [erlang_system_info(), erlang:system_info(compile_info), @@ -533,11 +533,11 @@ emit_application_info(EmitChunk, Path) -> Description = proplists:get_value(description, Info, []), Version = proplists:get_value(vsn, Info, []), - EmitChunk("{application, {~p,[" - "{description,~p}," - "{vsn,~p}," - "{path,~p}," - "{runtime_dependencies,~p},", + EmitChunk("{application, {~w,[" + "{description,~w}," + "{vsn,~w}," + "{path,~w}," + "{runtime_dependencies,~w},", [App, Description, Version, Path, RtDeps]), emit_module_info_from_path(EmitChunk, Path), EmitChunk("]}}", []) @@ -545,7 +545,7 @@ emit_application_info(EmitChunk, Path) -> emit_code_path_info(EmitChunk, Path) -> EmitChunk("{code, [" - "{path, ~p},", [Path]), + "{path, ~w},", [Path]), emit_module_info_from_path(EmitChunk, Path), EmitChunk("]}", []). @@ -573,11 +573,11 @@ emit_module_info(EmitChunk, Beam) -> _ -> true end, - EmitChunk("{~p,[" - "{loaded,~p}," - "{native,~p}," - "{compiler,~p}," - "{md5,~p}" + EmitChunk("{~w,[" + "{loaded,~w}," + "{native,~w}," + "{compiler,~w}," + "{md5,~w}" "]}", [Mod, Loaded, Native, CompilerVersion, hexstring(Md5)]). diff --git a/lib/runtime_tools/test/Makefile b/lib/runtime_tools/test/Makefile index 61377ea09e..de37b2570d 100644 --- a/lib/runtime_tools/test/Makefile +++ b/lib/runtime_tools/test/Makefile @@ -9,6 +9,7 @@ MODULES = \ system_information_SUITE \ dbg_SUITE \ erts_alloc_config_SUITE \ + scheduler_SUITE \ msacc_SUITE ERL_FILES= $(MODULES:%=%.erl) diff --git a/lib/runtime_tools/test/scheduler_SUITE.erl b/lib/runtime_tools/test/scheduler_SUITE.erl new file mode 100644 index 0000000000..1c80253371 --- /dev/null +++ b/lib/runtime_tools/test/scheduler_SUITE.erl @@ -0,0 +1,104 @@ +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% + +-module(scheduler_SUITE). + +-export([suite/0, all/0]). + +%% Test cases +-export([basic/1]). + +all() -> [basic]. + + +suite() -> [{ct_hooks,[ts_install_cth]}]. + + +basic(_Config) -> + S1 = scheduler:sample(), + S2 = scheduler:sample_all(), + + check(scheduler:utilization(1)), + + check(scheduler:utilization(S1)), + check(scheduler:utilization(S2)), + check(scheduler:utilization(S1, scheduler:sample())), + check(scheduler:utilization(S2, scheduler:sample())), + + S3 = scheduler:sample_all(), + U13 = scheduler:utilization(S1, S3), + U13 = scheduler:utilization(S1, remove_io(S3)), + check(U13), + + U23all = scheduler:utilization(S2, S3), + check(U23all), + U23 = scheduler:utilization(S2, remove_io(S3)), + U23 = scheduler:utilization(remove_io(S2), S3), + U23 = remove_io(U23all), + check(U23), + + ok. + + +check([{total, Tf, Ts} | List]=U) -> + io:format("\nU = ~p\n", [U]), + check_values(Tf, Ts, true), + + SchdList = case hd(List) of + {weighted, Wf, Ws} -> + check_values(Wf, Ws, false), + tl(List); + _ -> + unknown = erlang:system_info(logical_processors_available), + List + end, + + lists:foreach(fun({Type, Id, F, S}) when ((Type =:= normal) or (Type =:= cpu) or (Type =:= io)), + is_integer(Id) -> + check_values(F, S, true) + end, + SchdList), + ok. + +check_values(F, S, Max100) -> + true = is_float(F), + true = F >= 0.0, + + $% = lists:last(S), + Sf = list_to_float(lists:droplast(S)), + true = Sf >= 0.0, + true = case Max100 of + true -> + true = F =< 1.0, + true = Sf =< 100.0; + false -> + true + end, + MaxDiff = 0.055555555555555555, %% change to 0.05 when float_to_list/2 is fixed + true = abs(F*100 - Sf) =< MaxDiff, + ok. + + +remove_io({scheduler_wall_time_all,Lst}) -> + {scheduler_wall_time, remove_io(Lst)}; +remove_io(Lst) -> + lists:filter(fun({io,_,_,_}) -> false; + (_) -> true end, + Lst). diff --git a/lib/runtime_tools/vsn.mk b/lib/runtime_tools/vsn.mk index 872bd5db1d..0dc6a48570 100644 --- a/lib/runtime_tools/vsn.mk +++ b/lib/runtime_tools/vsn.mk @@ -1 +1 @@ -RUNTIME_TOOLS_VSN = 1.12.3 +RUNTIME_TOOLS_VSN = 1.12.4 diff --git a/lib/sasl/doc/src/release_handler.xml b/lib/sasl/doc/src/release_handler.xml index 8f073807fb..975188f489 100644 --- a/lib/sasl/doc/src/release_handler.xml +++ b/lib/sasl/doc/src/release_handler.xml @@ -61,6 +61,7 @@ <list type="bulleted"> <item>A release upgrade file, <c>relup</c></item> <item>A system configuration file, <c>sys.config</c></item> + <item>A system configuration source file, <c>sys.config.src</c></item> </list> <p>The <c>relup</c> file contains instructions for how to upgrade to, or downgrade from, this version of the release.</p> @@ -819,4 +820,3 @@ release_handler:set_unpacked(RelFile, [{myapp,"1.0","/home/user"},...]). <seealso marker="systools"><c>systools(3)</c></seealso></p> </section> </erlref> - diff --git a/lib/sasl/doc/src/systools.xml b/lib/sasl/doc/src/systools.xml index e7c3c499da..4842c732b1 100644 --- a/lib/sasl/doc/src/systools.xml +++ b/lib/sasl/doc/src/systools.xml @@ -349,10 +349,11 @@ myapp-1/ebin/myapp.app the release version as specified in <c>Name.rel</c>.</p> <p><c>releases/RelVsn</c> contains the boot script <c>Name.boot</c> renamed to <c>start.boot</c> and, if found, - the files <c>relup</c> and <c>sys.config</c>. These files + the files <c>relup</c> and <c>sys.config</c> or <c>sys.config.src</c>. These files are searched for in the same directory as <c>Name.rel</c>, in the current working directory, and in any directories - specified using option <c>path</c>.</p> + specified using option <c>path</c>. In the case of <c>sys.config</c> + it is not included if <c>sys.config.src</c> is found.</p> <p>If the release package is to contain a new Erlang runtime system, the <c>bin</c> directory of the specified runtime system <c>{erts,Dir}</c> is copied to <c>erts-ErtsVsn/bin</c>.</p> @@ -397,4 +398,3 @@ myapp-1/ebin/myapp.app <seealso marker="script"><c>script(4)</c></seealso></p> </section> </erlref> - diff --git a/lib/sasl/src/systools_make.erl b/lib/sasl/src/systools_make.erl index 9d960b7361..fa3182cc08 100644 --- a/lib/sasl/src/systools_make.erl +++ b/lib/sasl/src/systools_make.erl @@ -310,6 +310,7 @@ add_apply_upgrade(Script,Args) -> %% RelVsn/start.boot %% relup %% sys.config +%% sys.config.src %% erts-EVsn[/bin] %%----------------------------------------------------------------- @@ -1552,6 +1553,7 @@ create_kernel_procs(Appls) -> %% RelVsn/start.boot %% relup %% sys.config +%% sys.config.src %% erts-EVsn[/bin] %% %% The VariableN.tar.gz files can also be stored as own files not @@ -1707,14 +1709,18 @@ add_system_files(Tar, RelName, Release, Path1) -> add_to_tar(Tar, Relup, filename:join(RelVsnDir, "relup")) end, - case lookup_file("sys.config", Path) of - false -> - ignore; - Sys -> - check_sys_config(Sys), - add_to_tar(Tar, Sys, filename:join(RelVsnDir, "sys.config")) + case lookup_file("sys.config.src", Path) of + false -> + case lookup_file("sys.config", Path) of + false -> + ignore; + Sys -> + check_sys_config(Sys), + add_to_tar(Tar, Sys, filename:join(RelVsnDir, "sys.config")) + end; + SysSrc -> + add_to_tar(Tar, SysSrc, filename:join(RelVsnDir, "sys.config.src")) end, - ok. lookup_file(Name, [Dir|Path]) -> diff --git a/lib/sasl/test/release_handler_SUITE.erl b/lib/sasl/test/release_handler_SUITE.erl index 824820c214..edd2efdf05 100644 --- a/lib/sasl/test/release_handler_SUITE.erl +++ b/lib/sasl/test/release_handler_SUITE.erl @@ -2580,15 +2580,15 @@ check_gg_info(Node,OtherAlive,OtherDead,Synced,N) -> GGI = rpc:call(Node, global_group, info, []), GI = rpc:call(Node, global, info,[]), try do_check_gg_info(OtherAlive,OtherDead,Synced,GGI,GI) - catch _:E when N==0 -> + catch _:E:Stacktrace when N==0 -> ?t:format("~nERROR: check_gg_info failed for ~p:~n~p~n" "when GGI was: ~p~nand GI was: ~p~n", - [Node,{E,erlang:get_stacktrace()},GGI,GI]), + [Node,{E,Stacktrace},GGI,GI]), ?t:fail("check_gg_info failed"); - _:E -> + _:E:Stacktrace -> ?t:format("~nWARNING: check_gg_info failed for ~p:~n~p~n" "when GGI was: ~p~nand GI was: ~p~n", - [Node,{E,erlang:get_stacktrace()},GGI,GI]), + [Node,{E,Stacktrace},GGI,GI]), timer:sleep(1000), check_gg_info(Node,OtherAlive,OtherDead,Synced,N-1) end. diff --git a/lib/sasl/test/systools_SUITE.erl b/lib/sasl/test/systools_SUITE.erl index 07748d975f..c8b2f31120 100644 --- a/lib/sasl/test/systools_SUITE.erl +++ b/lib/sasl/test/systools_SUITE.erl @@ -67,7 +67,7 @@ groups() -> otp_3065_circular_dependenies, included_and_used_sort_script]}, {tar, [], [tar_options, normal_tar, no_mod_vsn_tar, system_files_tar, - invalid_system_files_tar, variable_tar, + system_src_file_tar, invalid_system_files_tar, variable_tar, src_tests_tar, var_tar, exref_tar, link_tar, no_sasl_tar, otp_9507_path_ebin]}, {relup, [], @@ -945,12 +945,47 @@ system_files_tar(Config) -> ok. + system_files_tar(cleanup,Config) -> Dir = ?privdir, file:delete(filename:join(Dir,"sys.config")), file:delete(filename:join(Dir,"relup")), ok. +%% make_tar: Check that sys.config.src and not sys.config is included +system_src_file_tar(Config) -> + {ok, OldDir} = file:get_cwd(), + + {LatestDir, LatestName} = create_script(latest,Config), + + DataDir = filename:absname(?copydir), + LibDir = fname([DataDir, d_normal, lib]), + P = [fname([LibDir, 'db-2.1', ebin]), + fname([LibDir, 'fe-3.1', ebin])], + + ok = file:set_cwd(LatestDir), + + %% Add dummy sys.config and sys.config.src + ok = file:write_file("sys.config.src","[${SOMETHING}].\n"), + ok = file:write_file("sys.config","[].\n"), + + {ok, _, _} = systools:make_script(LatestName, [silent, {path, P}]), + ok = systools:make_tar(LatestName, [{path, P}]), + ok = check_tar(fname(["releases","LATEST","sys.config.src"]), LatestName), + {error, _} = check_tar(fname(["releases","LATEST","sys.config"]), LatestName), + {ok, _, _} = systools:make_tar(LatestName, [{path, P}, silent]), + ok = check_tar(fname(["releases","LATEST","sys.config.src"]), LatestName), + {error, _} = check_tar(fname(["releases","LATEST","sys.config"]), LatestName), + + ok = file:set_cwd(OldDir), + + ok. + +system_src_file_tar(cleanup,Config) -> + Dir = ?privdir, + file:delete(filename:join(Dir,"sys.config")), + file:delete(filename:join(Dir,"sys.config.src")), + ok. %% make_tar: Check that make_tar fails if relup or sys.config exist %% but do not have valid content diff --git a/lib/snmp/doc/src/snmp_impl_example_agent.xml b/lib/snmp/doc/src/snmp_impl_example_agent.xml index a86006a0a7..e576fa51f3 100644 --- a/lib/snmp/doc/src/snmp_impl_example_agent.xml +++ b/lib/snmp/doc/src/snmp_impl_example_agent.xml @@ -47,6 +47,7 @@ EX1-MIB DEFINITIONS ::= BEGIN IMPORTS + experimental FROM RFC1155-SMI RowStatus FROM STANDARD-MIB DisplayString FROM RFC1213-MIB OBJECT-TYPE FROM RFC-1212 @@ -81,7 +82,7 @@ EX1-MIB DEFINITIONS ::= BEGIN FriendsEntry ::= SEQUENCE { - fIndex + fIndex INTEGER, fName DisplayString, @@ -105,6 +106,7 @@ EX1-MIB DEFINITIONS ::= BEGIN DESCRIPTION "Name of friend" ::= { friendsEntry 2 } + fAddress OBJECT-TYPE SYNTAX DisplayString (SIZE (0..255)) ACCESS read-write @@ -112,6 +114,7 @@ EX1-MIB DEFINITIONS ::= BEGIN DESCRIPTION "Address of friend" ::= { friendsEntry 3 } + fStatus OBJECT-TYPE SYNTAX RowStatus ACCESS read-write @@ -119,12 +122,13 @@ EX1-MIB DEFINITIONS ::= BEGIN DESCRIPTION "The status of this conceptual row." ::= { friendsEntry 4 } + fTrap TRAP-TYPE ENTERPRISE example1 VARIABLES { myName, fIndex } DESCRIPTION - "This trap is sent when something happens to - the friend specified by fIndex." + "This trap is sent when something happens to + the friend specified by fIndex." ::= 1 END </code> diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index 3a2f55a487..f5cb3ec254 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -30,6 +30,41 @@ <file>notes.xml</file> </header> +<section><title>Ssh 4.6.5</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Adjusted supervisor timeouts</p> + <p> + Own Id: OTP-14907</p> + </item> + <item> + <p> + Remove ERROR messages for slow process exits</p> + <p> + Own Id: OTP-14930</p> + </item> + </list> + </section> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Add option <c>save_accepted_host</c> to + <c>ssh:connection</c>. This option, if set to false, + inhibits saving host keys to e.g the file + <c>known_hosts</c>.</p> + <p> + Own Id: OTP-14935</p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.6.4</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -60,7 +95,6 @@ </list> </section> - <section><title>Improvements and New Features</title> <list> <item> @@ -90,7 +124,6 @@ </section> <section><title>Ssh 4.6.2</title> - <section><title>Fixed Bugs and Malfunctions</title> <list> <item> @@ -370,6 +403,40 @@ </section> + +<section><title>Ssh 4.4.2.2</title> + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Default exec is disabled when a user-defined shell is + enabled.</p> + <p> + Own Id: OTP-14881</p> + </item> + </list> + </section> +</section> + + +<section><title>Ssh 4.4.2.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Trailing white space was removed at end of the + hello-string. This caused interoperability problems with + some other ssh-implementations (e.g OpenSSH 7.3p1 on + Solaris 11)</p> + <p> + Own Id: OTP-14763 Aux Id: ERIERL-74 </p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.4.2</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -740,6 +807,93 @@ </section> +<section><title>Ssh 4.2.2.5</title> + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Default exec is disabled when a user-defined shell is + enabled.</p> + <p> + Own Id: OTP-14881</p> + </item> + </list> + </section> +</section> + + +<section><title>Ssh 4.2.2.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Trailing white space was removed at end of the + hello-string. This caused interoperability problems with + some other ssh-implementations (e.g OpenSSH 7.3p1 on + Solaris 11)</p> + <p> + Own Id: OTP-14763 Aux Id: ERIERL-74 </p> + </item> + </list> + </section> + +</section> + +<section><title>Ssh 4.2.2.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The key exchange algorithm + diffie-hellman-group-exchange-sha* has a server-option + <c>{dh_gex_limits,{Min,Max}}</c>. There was a hostkey + signature validation error on the client side if the + option was used and the <c>Min</c> or the <c>Max</c> + differed from the corresponding values obtained from the + client.</p> + <p> + This bug is now corrected.</p> + <p> + Own Id: OTP-14166</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Key exchange algorithms + diffie-hellman-group-exchange-sha* optimized, up to a + factor of 11 for the slowest ( = biggest and safest) one.</p> + <p> + Own Id: OTP-14169 Aux Id: seq-13261 </p> + </item> + </list> + </section> + +</section> + +<section><title>Ssh 4.2.2.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Upgrade of an established client connection could crash + because the ssh client supervisors children had wrong + type. This is fixed now.</p> + <p> + Own Id: OTP-13782 Aux Id: seq13158 </p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.2.2.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssh/doc/src/ssh.xml b/lib/ssh/doc/src/ssh.xml index 337f4094cc..acf94ff6af 100644 --- a/lib/ssh/doc/src/ssh.xml +++ b/lib/ssh/doc/src/ssh.xml @@ -227,6 +227,18 @@ </item> </list> </item> + + <tag><c><![CDATA[{save_accepted_host, boolean()}]]></c></tag> + <item> + <p>If <c>true</c>, the client saves an accepted host key to avoid the + accept question the next time the same host is connected. If the option + <c>key_cb</c> is not present, the key is saved in the file "known_hosts". + </p> + <p>If <c>false</c>, the key is not saved and the key will still be unknown + at the next access of the same host. + </p> + </item> + <tag><c><![CDATA[{user_interaction, boolean()}]]></c></tag> <item> <p>If <c>false</c>, disables the client to connect to the server diff --git a/lib/ssh/doc/src/ssh_sftp.xml b/lib/ssh/doc/src/ssh_sftp.xml index ed7fbf9cf3..129426a6d5 100644 --- a/lib/ssh/doc/src/ssh_sftp.xml +++ b/lib/ssh/doc/src/ssh_sftp.xml @@ -464,11 +464,16 @@ <v>FileInfo = record()</v> </type> <desc> - <p>Returns a <c><![CDATA[file_info]]></c> record from the file specified by + <p>Returns a <c><![CDATA[file_info]]></c> record from the file system object specified by <c><![CDATA[Name]]></c> or <c><![CDATA[Handle]]></c>. See <seealso marker="kernel:file#read_file_info-2">file:read_file_info/2</seealso> for information about the record. </p> + <p> + Depending on the underlying OS:es links might be followed and info on the final file, directory + etc is returned. See <seealso marker="#read_link_info-2">ssh_sftp::read_link_info/2</seealso> + on how to get information on links instead. + </p> </desc> </func> diff --git a/lib/ssh/src/ssh.erl b/lib/ssh/src/ssh.erl index 032d87bdad..25d537c624 100644 --- a/lib/ssh/src/ssh.erl +++ b/lib/ssh/src/ssh.erl @@ -184,7 +184,6 @@ channel_info(ConnectionRef, ChannelId, Options) -> daemon(Port) -> daemon(Port, []). - daemon(Socket, UserOptions) when is_port(Socket) -> try #{} = Options = ssh_options:handle_options(server, UserOptions), @@ -267,8 +266,6 @@ daemon(Host0, Port0, UserOptions0) when 0 =< Port0, Port0 =< 65535, daemon(_, _, _) -> {error, badarg}. - - %%-------------------------------------------------------------------- -spec daemon_info(daemon_ref()) -> ok_error( [{atom(), term()}] ). diff --git a/lib/ssh/src/ssh.hrl b/lib/ssh/src/ssh.hrl index 3dee1c5521..4711f54fb5 100644 --- a/lib/ssh/src/ssh.hrl +++ b/lib/ssh/src/ssh.hrl @@ -35,6 +35,8 @@ -define(DEFAULT_TRANSPORT, {tcp, gen_tcp, tcp_closed} ). +-define(DEFAULT_SHELL, {shell, start, []} ). + -define(MAX_RND_PADDING_LEN, 15). -define(SUPPORTED_AUTH_METHODS, "publickey,keyboard-interactive,password"). diff --git a/lib/ssh/src/ssh_acceptor_sup.erl b/lib/ssh/src/ssh_acceptor_sup.erl index a24664793b..fc564a359b 100644 --- a/lib/ssh/src/ssh_acceptor_sup.erl +++ b/lib/ssh/src/ssh_acceptor_sup.erl @@ -86,10 +86,7 @@ child_spec(Address, Port, Profile, Options) -> Timeout = ?GET_INTERNAL_OPT(timeout, Options, ?DEFAULT_TIMEOUT), #{id => id(Address, Port, Profile), start => {ssh_acceptor, start_link, [Port, Address, Options, Timeout]}, - restart => transient, - shutdown => 5500, %brutal_kill, - type => worker, - modules => [ssh_acceptor] + restart => transient % because a crashed listener could be replaced by a new one }. id(Address, Port, Profile) -> diff --git a/lib/ssh/src/ssh_channel_sup.erl b/lib/ssh/src/ssh_channel_sup.erl index 6b01dc334d..8444533fd1 100644 --- a/lib/ssh/src/ssh_channel_sup.erl +++ b/lib/ssh/src/ssh_channel_sup.erl @@ -26,7 +26,7 @@ -behaviour(supervisor). --export([start_link/1, start_child/2]). +-export([start_link/1, start_child/5]). %% Supervisor callback -export([init/1]). @@ -37,7 +37,14 @@ start_link(Args) -> supervisor:start_link(?MODULE, [Args]). -start_child(Sup, ChildSpec) -> +start_child(Sup, Callback, Id, Args, Exec) -> + ChildSpec = + #{id => make_ref(), + start => {ssh_channel, start_link, [self(), Id, Callback, Args, Exec]}, + restart => temporary, + type => worker, + modules => [ssh_channel] + }, supervisor:start_child(Sup, ChildSpec). %%%========================================================================= diff --git a/lib/ssh/src/ssh_cli.erl b/lib/ssh/src/ssh_cli.erl index 62854346b0..958c342f5f 100644 --- a/lib/ssh/src/ssh_cli.erl +++ b/lib/ssh/src/ssh_cli.erl @@ -127,7 +127,8 @@ handle_ssh_msg({ssh_cm, ConnectionHandler, cm = ConnectionHandler}}; handle_ssh_msg({ssh_cm, ConnectionHandler, - {exec, ChannelId, WantReply, Cmd}}, #state{exec=undefined} = State) -> + {exec, ChannelId, WantReply, Cmd}}, #state{exec=undefined, + shell=?DEFAULT_SHELL} = State) -> {Reply, Status} = exec(Cmd), write_chars(ConnectionHandler, ChannelId, io_lib:format("~p\n", [Reply])), @@ -136,6 +137,15 @@ handle_ssh_msg({ssh_cm, ConnectionHandler, ssh_connection:exit_status(ConnectionHandler, ChannelId, Status), ssh_connection:send_eof(ConnectionHandler, ChannelId), {stop, ChannelId, State#state{channel = ChannelId, cm = ConnectionHandler}}; + +handle_ssh_msg({ssh_cm, ConnectionHandler, + {exec, ChannelId, WantReply, _Cmd}}, #state{exec = undefined} = State) -> + write_chars(ConnectionHandler, ChannelId, 1, "Prohibited.\n"), + ssh_connection:reply_request(ConnectionHandler, WantReply, success, ChannelId), + ssh_connection:exit_status(ConnectionHandler, ChannelId, 255), + ssh_connection:send_eof(ConnectionHandler, ChannelId), + {stop, ChannelId, State#state{channel = ChannelId, cm = ConnectionHandler}}; + handle_ssh_msg({ssh_cm, ConnectionHandler, {exec, ChannelId, WantReply, Cmd}}, State) -> NewState = start_shell(ConnectionHandler, Cmd, State), @@ -453,11 +463,14 @@ move_cursor(From, To, #ssh_pty{width=Width, term=Type}) -> %% %%% make sure that there is data to send %% %%% before calling ssh_connection:send write_chars(ConnectionHandler, ChannelId, Chars) -> + write_chars(ConnectionHandler, ChannelId, ?SSH_EXTENDED_DATA_DEFAULT, Chars). + +write_chars(ConnectionHandler, ChannelId, Type, Chars) -> case has_chars(Chars) of false -> ok; true -> ssh_connection:send(ConnectionHandler, ChannelId, - ?SSH_EXTENDED_DATA_DEFAULT, + Type, Chars) end. diff --git a/lib/ssh/src/ssh_connection.erl b/lib/ssh/src/ssh_connection.erl index 7e9ee78fd2..946ae2967b 100644 --- a/lib/ssh/src/ssh_connection.erl +++ b/lib/ssh/src/ssh_connection.erl @@ -812,22 +812,20 @@ start_channel(Cb, Id, Args, SubSysSup, Opts) -> start_channel(Cb, Id, Args, SubSysSup, undefined, Opts). start_channel(Cb, Id, Args, SubSysSup, Exec, Opts) -> - ChildSpec = child_spec(Cb, Id, Args, Exec), ChannelSup = ssh_subsystem_sup:channel_supervisor(SubSysSup), - assert_limit_num_channels_not_exceeded(ChannelSup, Opts), - ssh_channel_sup:start_child(ChannelSup, ChildSpec). + case max_num_channels_not_exceeded(ChannelSup, Opts) of + true -> + ssh_channel_sup:start_child(ChannelSup, Cb, Id, Args, Exec); + false -> + throw(max_num_channels_exceeded) + end. -assert_limit_num_channels_not_exceeded(ChannelSup, Opts) -> +max_num_channels_not_exceeded(ChannelSup, Opts) -> MaxNumChannels = ?GET_OPT(max_channels, Opts), NumChannels = length([x || {_,_,worker,[ssh_channel]} <- supervisor:which_children(ChannelSup)]), - if - %% Note that NumChannels is BEFORE starting a new one - NumChannels < MaxNumChannels -> - ok; - true -> - throw(max_num_channels_exceeded) - end. + %% Note that NumChannels is BEFORE starting a new one + NumChannels < MaxNumChannels. %%-------------------------------------------------------------------- %%% Internal functions @@ -874,14 +872,6 @@ check_subsystem(SsName, Options) -> Value end. -child_spec(Callback, Id, Args, Exec) -> - Name = make_ref(), - StartFunc = {ssh_channel, start_link, [self(), Id, Callback, Args, Exec]}, - Restart = temporary, - Shutdown = 3600, - Type = worker, - {Name, StartFunc, Restart, Shutdown, Type, [ssh_channel]}. - start_cli(#connection{cli_spec = no_cli}, _) -> {error, cli_disabled}; start_cli(#connection{options = Options, diff --git a/lib/ssh/src/ssh_connection_handler.erl b/lib/ssh/src/ssh_connection_handler.erl index 0ca960ef96..e11d3adee4 100644 --- a/lib/ssh/src/ssh_connection_handler.erl +++ b/lib/ssh/src/ssh_connection_handler.erl @@ -1174,17 +1174,25 @@ handle_event({call,_}, _, StateName, _) when not ?CONNECTED(StateName) -> handle_event({call,From}, {request, ChannelPid, ChannelId, Type, Data, Timeout}, StateName, D0) when ?CONNECTED(StateName) -> - D = handle_request(ChannelPid, ChannelId, Type, Data, true, From, D0), - %% Note reply to channel will happen later when reply is recived from peer on the socket - start_channel_request_timer(ChannelId, From, Timeout), - {keep_state, cache_request_idle_timer_check(D)}; + case handle_request(ChannelPid, ChannelId, Type, Data, true, From, D0) of + {error,Error} -> + {keep_state, D0, {reply,From,{error,Error}}}; + D -> + %% Note reply to channel will happen later when reply is recived from peer on the socket + start_channel_request_timer(ChannelId, From, Timeout), + {keep_state, cache_request_idle_timer_check(D)} + end; handle_event({call,From}, {request, ChannelId, Type, Data, Timeout}, StateName, D0) when ?CONNECTED(StateName) -> - D = handle_request(ChannelId, Type, Data, true, From, D0), - %% Note reply to channel will happen later when reply is recived from peer on the socket - start_channel_request_timer(ChannelId, From, Timeout), - {keep_state, cache_request_idle_timer_check(D)}; + case handle_request(ChannelId, Type, Data, true, From, D0) of + {error,Error} -> + {keep_state, D0, {reply,From,{error,Error}}}; + D -> + %% Note reply to channel will happen later when reply is recived from peer on the socket + start_channel_request_timer(ChannelId, From, Timeout), + {keep_state, cache_request_idle_timer_check(D)} + end; handle_event({call,From}, {data, ChannelId, Type, Data, Timeout}, StateName, D0) when ?CONNECTED(StateName) -> @@ -1460,13 +1468,12 @@ terminate(shutdown, StateName, State0) -> State0), finalize_termination(StateName, State); -%% terminate({shutdown,Msg}, StateName, State0) when is_record(Msg,ssh_msg_disconnect)-> -%% State = send_msg(Msg, State0), -%% finalize_termination(StateName, Msg, State); - terminate({shutdown,_R}, StateName, State) -> finalize_termination(StateName, State); +terminate(kill, StateName, State) -> + finalize_termination(StateName, State); + terminate(Reason, StateName, State0) -> %% Others, e.g undef, {badmatch,_} log_error(Reason), @@ -1774,21 +1781,31 @@ is_usable_user_pubkey(A, Ssh) -> %%%---------------------------------------------------------------- handle_request(ChannelPid, ChannelId, Type, Data, WantReply, From, D) -> case ssh_channel:cache_lookup(cache(D), ChannelId) of - #channel{remote_id = Id} = Channel -> + #channel{remote_id = Id, + sent_close = false} = Channel -> update_sys(cache(D), Channel, Type, ChannelPid), send_msg(ssh_connection:channel_request_msg(Id, Type, WantReply, Data), add_request(WantReply, ChannelId, From, D)); - undefined -> - D + + _ when WantReply==true -> + {error,closed}; + + _ -> + D end. handle_request(ChannelId, Type, Data, WantReply, From, D) -> case ssh_channel:cache_lookup(cache(D), ChannelId) of - #channel{remote_id = Id} -> + #channel{remote_id = Id, + sent_close = false} -> send_msg(ssh_connection:channel_request_msg(Id, Type, WantReply, Data), add_request(WantReply, ChannelId, From, D)); - undefined -> - D + + _ when WantReply==true -> + {error,closed}; + + _ -> + D end. %%%---------------------------------------------------------------- diff --git a/lib/ssh/src/ssh_connection_sup.erl b/lib/ssh/src/ssh_connection_sup.erl index 60ee8b7c73..2e8450090a 100644 --- a/lib/ssh/src/ssh_connection_sup.erl +++ b/lib/ssh/src/ssh_connection_sup.erl @@ -52,10 +52,7 @@ init(_) -> }, ChildSpecs = [#{id => undefined, % As simple_one_for_one is used. start => {ssh_connection_handler, start_link, []}, - restart => temporary, - shutdown => 4000, - type => worker, - modules => [ssh_connection_handler] + restart => temporary % because there is no way to restart a crashed connection } ], {ok, {SupFlags,ChildSpecs}}. diff --git a/lib/ssh/src/ssh_options.erl b/lib/ssh/src/ssh_options.erl index 68c99743ee..1e10f72956 100644 --- a/lib/ssh/src/ssh_options.erl +++ b/lib/ssh/src/ssh_options.erl @@ -268,7 +268,7 @@ default(server) -> }, {shell, def} => - #{default => {shell, start, []}, + #{default => ?DEFAULT_SHELL, chk => fun({M,F,A}) -> is_atom(M) andalso is_atom(F) andalso is_list(A); (V) -> check_function1(V) orelse check_function2(V) end, @@ -439,6 +439,12 @@ default(client) -> class => user_options }, + {save_accepted_host, def} => + #{default => true, + chk => fun erlang:is_boolean/1, + class => user_options + }, + {pref_public_key_algs, def} => #{default => ssh_transport:default_algorithms(public_key), chk => fun check_pref_public_key_algs/1, diff --git a/lib/ssh/src/ssh_subsystem_sup.erl b/lib/ssh/src/ssh_subsystem_sup.erl index 8db051095c..77da240a66 100644 --- a/lib/ssh/src/ssh_subsystem_sup.erl +++ b/lib/ssh/src/ssh_subsystem_sup.erl @@ -74,18 +74,14 @@ ssh_connection_child_spec(Role, Address, Port, _Profile, Options) -> #{id => id(Role, ssh_connection_sup, Address, Port), start => {ssh_connection_sup, start_link, [Options]}, restart => temporary, - shutdown => 5000, - type => supervisor, - modules => [ssh_connection_sup] + type => supervisor }. ssh_channel_child_spec(Role, Address, Port, _Profile, Options) -> #{id => id(Role, ssh_channel_sup, Address, Port), start => {ssh_channel_sup, start_link, [Options]}, restart => temporary, - shutdown => infinity, - type => supervisor, - modules => [ssh_channel_sup] + type => supervisor }. id(Role, Sup, Address, Port) -> diff --git a/lib/ssh/src/ssh_sup.erl b/lib/ssh/src/ssh_sup.erl index eaec7a54e4..8183016ba5 100644 --- a/lib/ssh/src/ssh_sup.erl +++ b/lib/ssh/src/ssh_sup.erl @@ -36,15 +36,14 @@ init(_) -> intensity => 10, period => 3600 }, - ChildSpecs = [#{id => Module, - start => {Module, start_link, []}, - restart => permanent, - shutdown => 4000, %brutal_kill, - type => supervisor, - modules => [Module] + ChildSpecs = [#{id => sshd_sup, + start => {sshd_sup, start_link, []}, + type => supervisor + }, + #{id => sshc_sup, + start => {sshc_sup, start_link, []}, + type => supervisor } - || Module <- [sshd_sup, - sshc_sup] ], {ok, {SupFlags,ChildSpecs}}. diff --git a/lib/ssh/src/ssh_system_sup.erl b/lib/ssh/src/ssh_system_sup.erl index e70abf59c2..17f990c5d8 100644 --- a/lib/ssh/src/ssh_system_sup.erl +++ b/lib/ssh/src/ssh_system_sup.erl @@ -63,9 +63,7 @@ init([Address, Port, Profile, Options]) -> [#{id => id(ssh_acceptor_sup, Address, Port, Profile), start => {ssh_acceptor_sup, start_link, [Address, Port, Profile, Options]}, restart => transient, - shutdown => infinity, - type => supervisor, - modules => [ssh_acceptor_sup] + type => supervisor }]; _ -> [] @@ -124,9 +122,8 @@ start_subsystem(SystemSup, Role, Address, Port, Profile, Options) -> #{id => make_ref(), start => {ssh_subsystem_sup, start_link, [Role, Address, Port, Profile, Options]}, restart => temporary, - shutdown => infinity, - type => supervisor, - modules => [ssh_subsystem_sup]}, + type => supervisor + }, supervisor:start_child(SystemSup, SubsystemSpec). stop_subsystem(SystemSup, SubSys) -> diff --git a/lib/ssh/src/ssh_transport.erl b/lib/ssh/src/ssh_transport.erl index ad9efc4755..975053d301 100644 --- a/lib/ssh/src/ssh_transport.erl +++ b/lib/ssh/src/ssh_transport.erl @@ -889,10 +889,13 @@ known_host_key(#ssh{opts = Opts, key_cb = {KeyCb,KeyCbOpts}, peer = {PeerName,_} {_,true} -> ok; {_,false} -> + DoAdd = ?GET_OPT(save_accepted_host, Opts), case accepted_host(Ssh, PeerName, Public, Opts) of - true -> + true when DoAdd == true -> {_,R} = add_host_key(KeyCb, PeerName, Public, [{key_cb_private,KeyCbOpts}|UserOpts]), R; + true when DoAdd == false -> + ok; false -> {error, rejected_by_user}; {error,E} -> diff --git a/lib/ssh/src/sshc_sup.erl b/lib/ssh/src/sshc_sup.erl index 133b2c6450..fd4d8a3c07 100644 --- a/lib/ssh/src/sshc_sup.erl +++ b/lib/ssh/src/sshc_sup.erl @@ -60,10 +60,7 @@ init(_) -> }, ChildSpecs = [#{id => undefined, % As simple_one_for_one is used. start => {ssh_connection_handler, start_link, []}, - restart => temporary, - shutdown => 4000, - type => worker, - modules => [ssh_connection_handler] + restart => temporary % because there is no way to restart a crashed connection } ], {ok, {SupFlags,ChildSpecs}}. diff --git a/lib/ssh/src/sshd_sup.erl b/lib/ssh/src/sshd_sup.erl index c23e65d955..779a861a54 100644 --- a/lib/ssh/src/sshd_sup.erl +++ b/lib/ssh/src/sshd_sup.erl @@ -90,10 +90,8 @@ init(_) -> child_spec(Address, Port, Profile, Options) -> #{id => id(Address, Port, Profile), start => {ssh_system_sup, start_link, [Address, Port, Profile, Options]}, - restart => temporary, - shutdown => infinity, - type => supervisor, - modules => [ssh_system_sup] + restart => temporary, + type => supervisor }. id(Address, Port, Profile) -> diff --git a/lib/ssh/test/ssh_basic_SUITE.erl b/lib/ssh/test/ssh_basic_SUITE.erl index 202b0afe57..365f25fabb 100644 --- a/lib/ssh/test/ssh_basic_SUITE.erl +++ b/lib/ssh/test/ssh_basic_SUITE.erl @@ -60,7 +60,7 @@ login_bad_pwd_no_retry5/1, misc_ssh_options/1, openssh_zlib_basic_test/1, - packet_size_zero/1, + packet_size/1, pass_phrase/1, peername_sockname/1, send/1, @@ -111,7 +111,7 @@ all() -> double_close, daemon_opt_fd, multi_daemon_opt_fd, - packet_size_zero, + packet_size, ssh_info_print, {group, login_bad_pwd_no_retry}, shell_exit_status @@ -764,11 +764,11 @@ cli(Config) when is_list(Config) -> {ok, ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), ssh_connection:shell(ConnectionRef, ChannelId), - ok = ssh_connection:send(ConnectionRef, ChannelId, <<"q">>), + ssh_connection:send(ConnectionRef, ChannelId, <<"q">>), receive {ssh_cm, ConnectionRef, {data,0,0, <<"\r\nYou are accessing a dummy, type \"q\" to exit\r\n\n">>}} -> - ok = ssh_connection:send(ConnectionRef, ChannelId, <<"q">>) + ssh_connection:send(ConnectionRef, ChannelId, <<"q">>) after 30000 -> ct:fail("timeout ~p:~p",[?MODULE,?LINE]) end, @@ -1104,7 +1104,7 @@ multi_daemon_opt_fd(Config) -> end || {S,Pid,C} <- Tests]. %%-------------------------------------------------------------------- -packet_size_zero(Config) -> +packet_size(Config) -> SystemDir = proplists:get_value(data_dir, Config), PrivDir = proplists:get_value(priv_dir, Config), UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth @@ -1119,21 +1119,31 @@ packet_size_zero(Config) -> {user_interaction, false}, {user, "vego"}, {password, "morot"}]), - - {ok,Chan} = ssh_connection:session_channel(Conn, 1000, _MaxPacketSize=0, 60000), - ok = ssh_connection:shell(Conn, Chan), + lists:foreach( + fun(MaxPacketSize) -> + ct:log("Try max_packet_size=~p",[MaxPacketSize]), + {ok,Ch} = ssh_connection:session_channel(Conn, 1000, MaxPacketSize, 60000), + ok = ssh_connection:shell(Conn, Ch), + rec(Server, Conn, Ch, MaxPacketSize) + end, [0, 1, 10, 25]), ssh:close(Conn), - ssh:stop_daemon(Server), + ssh:stop_daemon(Server). +rec(Server, Conn, Ch, MaxSz) -> receive - {ssh_cm,Conn,{data,Chan,_Type,_Msg1}} = M -> - ct:log("Got ~p",[M]), - ct:fail(doesnt_obey_max_packet_size_0) - after 5000 -> - ok - end. - + {ssh_cm,Conn,{data,Ch,_,M}} when size(M) =< MaxSz -> + ct:log("~p: ~p",[MaxSz,M]), + rec(Server, Conn, Ch, MaxSz); + {ssh_cm,Conn,{data,Ch,_,_}} = M -> + ct:log("Max pkt size=~p. Got ~p",[MaxSz,M]), + ssh:close(Conn), + ssh:stop_daemon(Server), + ct:fail("Does not obey max_packet_size=~p",[MaxSz]) + after + 2000 -> ok + end. + %%-------------------------------------------------------------------- shell_no_unicode(Config) -> new_do_shell(proplists:get_value(io,Config), @@ -1491,7 +1501,7 @@ new_do_shell(IO, N, Ops=[{Order,Arg}|More]) -> ct:fail("*** Expected ~p, but got ~p",[string:strip(ExpStr),RecStr]) end after 30000 -> - ct:log("Meassage queue of ~p:~n~p", + ct:log("Message queue of ~p:~n~p", [self(), erlang:process_info(self(), messages)]), case Order of expect -> ct:fail("timeout, expected ~p",[string:strip(Arg)]); diff --git a/lib/ssh/test/ssh_compat_SUITE.erl b/lib/ssh/test/ssh_compat_SUITE.erl index 74ab5aca3a..f7eda1dc08 100644 --- a/lib/ssh/test/ssh_compat_SUITE.erl +++ b/lib/ssh/test/ssh_compat_SUITE.erl @@ -32,7 +32,8 @@ -compile(export_all). -define(USER,"sshtester"). --define(PWD, "foobar"). +-define(PASSWD, "foobar"). +-define(BAD_PASSWD, "NOT-"?PASSWD). -define(DOCKER_PFX, "ssh_compat_suite-ssh"). %%-------------------------------------------------------------------- @@ -44,25 +45,22 @@ suite() -> {timetrap,{seconds,40}}]. all() -> - [{group,G} || G <- vers()]. +%% [check_docker_present] ++ + [{group,G} || G <- ssh_image_versions()]. groups() -> - [{G, [], tests()} || G <- vers()]. - -tests() -> - [login_with_password_otp_is_client, - login_with_password_otp_is_server, - login_with_keyboard_interactive_otp_is_client, - login_with_keyboard_interactive_otp_is_server, - login_with_all_public_keys_otp_is_client, - login_with_all_public_keys_otp_is_server, - all_algorithms_otp_is_client, - all_algorithms_otp_is_server + [{otp_client, [], [login_otp_is_client, + all_algorithms_sftp_exec_reneg_otp_is_client, + send_recv_big_with_renegotiate_otp_is_client + ]}, + {otp_server, [], [login_otp_is_server, + all_algorithms_sftp_exec_reneg_otp_is_server + ]} | + [{G, [], [{group,otp_client}, {group,otp_server}]} || G <- ssh_image_versions()] ]. - -vers() -> +ssh_image_versions() -> try %% Find all useful containers in such a way that undefined command, too low %% priviliges, no containers and containers found give meaningful result: @@ -94,28 +92,60 @@ end_per_suite(Config) -> %%% os:cmd("docker rm $(docker ps -aq -f status=exited)"), %% Remove dangling images: %%% os:cmd("docker rmi $(docker images -f dangling=true -q)"), + catch ssh:stop(), Config. +init_per_group(otp_server, Config) -> + case proplists:get_value(common_remote_client_algs, Config) of + undefined -> + SSHver = proplists:get_value(ssh_version, Config, ""), + {skip,"No "++SSHver++ " client found in docker"}; + _ -> + Config + end; + +init_per_group(otp_client, Config) -> + Config; -init_per_group(G, Config) -> - case lists:member(G, vers()) of +init_per_group(G, Config0) -> + case lists:member(G, ssh_image_versions()) of true -> + %% This group is for one of the images + Vssh = atom_to_list(G), + Cmnt = io_lib:format("+++ ~s +++",[Vssh]), + ct:comment("~s",[Cmnt]), try start_docker(G) of {ok,ID} -> - ct:log("==> ~p",[G]), - [Vssh|VsslRest] = string:tokens(atom_to_list(G), "-"), - Vssl = lists:flatten(lists:join($-,VsslRest)), - ct:comment("+++ ~s + ~s +++",[Vssh,Vssl]), + ct:log("==> ~p started",[G]), %% Find the algorithms that both client and server supports: {IP,Port} = ip_port([{id,ID}]), - try common_algs([{id,ID}|Config], IP, Port) of - {ok, RemoteServerCommon, RemoteClientCommon} -> - [{ssh_version,Vssh},{ssl_version,Vssl}, - {id,ID}, - {common_server_algs,RemoteServerCommon}, - {common_client_algs,RemoteClientCommon} - |Config]; + ct:log("Try contact ~p:~p",[IP,Port]), + Config1 = [{id,ID}, + {ssh_version,Vssh} + | Config0], + try common_algs(Config1, IP, Port) of + {ok, ServerHello, RemoteServerCommon, ClientHello, RemoteClientCommon} -> + case chk_hellos([ServerHello,ClientHello], Cmnt) of + Cmnt -> + ok; + NewCmnt -> + ct:comment("~s",[NewCmnt]) + end, + AuthMethods = + %% This should be obtained by quering the peer, but that + %% is a bit hard. It is possible with ssh_protocol_SUITE + %% techniques, but it can wait. + case Vssh of + "dropbear" ++ _ -> + [password, publickey]; + _ -> + [password, 'keyboard-interactive', publickey] + end, + [{common_remote_server_algs,RemoteServerCommon}, + {common_remote_client_algs,RemoteClientCommon}, + {common_authmethods,AuthMethods} + |Config1]; Other -> ct:log("Error in init_per_group: ~p",[Other]), stop_docker(ID), @@ -138,188 +168,301 @@ init_per_group(G, Config) -> end; false -> - Config + Config0 end. -end_per_group(_, Config) -> - catch stop_docker(proplists:get_value(id,Config)), - Config. +end_per_group(G, Config) -> + case lists:member(G, ssh_image_versions()) of + true -> + catch stop_docker(proplists:get_value(id,Config)); + false -> + ok + end. %%-------------------------------------------------------------------- %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- -login_with_password_otp_is_client(Config) -> - {IP,Port} = ip_port(Config), - {ok,C} = ssh:connect(IP, Port, [{auth_methods,"password"}, - {user,?USER}, - {password,?PWD}, - {user_dir, new_dir(Config)}, - {silently_accept_hosts,true}, - {user_interaction,false} - ]), - ssh:close(C). - -%%-------------------------------------------------------------------- -login_with_password_otp_is_server(Config) -> - {Server, Host, HostPort} = - ssh_test_lib:daemon(0, - [{auth_methods,"password"}, - {system_dir, setup_local_hostdir('ssh-rsa',Config)}, - {user_dir, new_dir(Config)}, - {user_passwords, [{?USER,?PWD}]}, - {failfun, fun ssh_test_lib:failfun/2} - ]), - R = exec_from_docker(Config, Host, HostPort, - "'lists:concat([\"Answer=\",1+2]).\r\n'", - [<<"Answer=3">>], - ""), - ssh:stop_daemon(Server), - R. - -%%-------------------------------------------------------------------- -login_with_keyboard_interactive_otp_is_client(Config) -> - {DockerIP,DockerPort} = ip_port(Config), - {ok,C} = ssh:connect(DockerIP, DockerPort, - [{auth_methods,"keyboard-interactive"}, - {user,?USER}, - {password,?PWD}, - {user_dir, new_dir(Config)}, - {silently_accept_hosts,true}, - {user_interaction,false} - ]), - ssh:close(C). - -%%-------------------------------------------------------------------- -login_with_keyboard_interactive_otp_is_server(Config) -> - {Server, Host, HostPort} = - ssh_test_lib:daemon(0, - [{auth_methods,"keyboard-interactive"}, - {system_dir, setup_local_hostdir('ssh-rsa',Config)}, - {user_dir, new_dir(Config)}, - {user_passwords, [{?USER,?PWD}]}, - {failfun, fun ssh_test_lib:failfun/2} - ]), - R = exec_from_docker(Config, Host, HostPort, - "'lists:concat([\"Answer=\",1+3]).\r\n'", - [<<"Answer=4">>], - ""), - ssh:stop_daemon(Server), - R. +check_docker_present(_Config) -> + ct:log("This testcase is just to show in Monitor that we have a test host with docker installed",[]), + {fail, "Test is OK: just showing docker is available"}. %%-------------------------------------------------------------------- -login_with_all_public_keys_otp_is_client(Config) -> - CommonAlgs = [{public_key_from_host,A} - || {public_key,A} <- proplists:get_value(common_server_algs, Config)], - {DockerIP,DockerPort} = ip_port(Config), - chk_all_algos(CommonAlgs, Config, - fun(_Tag,Alg) -> - ssh:connect(DockerIP, DockerPort, - [{auth_methods, "publickey"}, - {user, ?USER}, - {user_dir, setup_remote_auth_keys_and_local_priv(Alg, Config)}, - {silently_accept_hosts,true}, - {user_interaction,false} - ]) +login_otp_is_client(Config) -> + {IP,Port} = ip_port(Config), + PublicKeyAlgs = [A || {public_key,A} <- proplists:get_value(common_remote_server_algs, Config)], + CommonAuths = + [{AuthMethod,Alg} || AuthMethod <- proplists:get_value(common_authmethods, Config), + Alg <- case AuthMethod of + publickey -> + PublicKeyAlgs; + _ -> + [' '] + end + ], + + chk_all_algos(?FUNCTION_NAME, CommonAuths, Config, + fun(AuthMethod,Alg) -> + {Opts,Dir} = + case AuthMethod of + publickey -> + {[], setup_remote_auth_keys_and_local_priv(Alg, Config)}; + _ -> + {[{password,?PASSWD}], new_dir(Config)} + end, + ssh:connect(IP, Port, [{auth_methods, atom_to_list(AuthMethod)}, + {user,?USER}, + {user_dir, Dir}, + {silently_accept_hosts,true}, + {user_interaction,false} + | Opts + ]) end). + %%-------------------------------------------------------------------- -login_with_all_public_keys_otp_is_server(Config) -> - CommonAlgs = [{public_key_to_host,A} - || {public_key,A} <- proplists:get_value(common_client_algs, Config)], - UserDir = new_dir(Config), - {Server, Host, HostPort} = - ssh_test_lib:daemon(0, - [{auth_methods, "publickey"}, - {system_dir, setup_local_hostdir('ssh-rsa',Config)}, - {user_dir, UserDir}, - {user_passwords, [{?USER,?PWD}]}, - {failfun, fun ssh_test_lib:failfun/2} - ]), - - R = chk_all_algos(CommonAlgs, Config, - fun(_Tag,Alg) -> - setup_remote_priv_and_local_auth_keys(Alg, clear_dir(UserDir), Config), - exec_from_docker(Config, Host, HostPort, - "'lists:concat([\"Answer=\",1+4]).\r\n'", - [<<"Answer=5">>], - "") - end), - ssh:stop_daemon(Server), - R. +login_otp_is_server(Config) -> + PublicKeyAlgs = [A || {public_key,A} <- proplists:get_value(common_remote_client_algs, Config)], + CommonAuths = + [{AuthMethod,Alg} || AuthMethod <- proplists:get_value(common_authmethods, Config), + Alg <- case AuthMethod of + publickey -> + PublicKeyAlgs; + _ -> + [' '] + end + ], + SysDir = setup_local_hostdir(hd(PublicKeyAlgs), Config), + chk_all_algos(?FUNCTION_NAME, CommonAuths, Config, + fun(AuthMethod,Alg) -> + {Opts,UsrDir} = + case AuthMethod of + publickey -> + {[{user_passwords, [{?USER,?BAD_PASSWD}]}], + setup_remote_priv_and_local_auth_keys(Alg, Config) + }; + _ -> + {[{user_passwords, [{?USER,?PASSWD}]}], + new_dir(Config) + } + end, + {Server, Host, HostPort} = + ssh_test_lib:daemon(0, + [{auth_methods, atom_to_list(AuthMethod)}, + {system_dir, SysDir}, + {user_dir, UsrDir}, + {failfun, fun ssh_test_lib:failfun/2} + | Opts + ]), + R = exec_from_docker(Config, Host, HostPort, + "'lists:concat([\"Answer=\",1+3]).\r\n'", + [<<"Answer=4">>], + ""), + ssh:stop_daemon(Server), + R + end). %%-------------------------------------------------------------------- -all_algorithms_otp_is_client(Config) -> - CommonAlgs = proplists:get_value(common_server_algs, Config), +all_algorithms_sftp_exec_reneg_otp_is_client(Config) -> + CommonAlgs = proplists:get_value(common_remote_server_algs, Config), {IP,Port} = ip_port(Config), - chk_all_algos(CommonAlgs, Config, + chk_all_algos(?FUNCTION_NAME, CommonAlgs, Config, fun(Tag, Alg) -> - ssh:connect(IP, Port, [{user,?USER}, - {password,?PWD}, - {auth_methods, "password"}, - {user_dir, new_dir(Config)}, - {preferred_algorithms, [{Tag,[Alg]}]}, - {silently_accept_hosts,true}, - {user_interaction,false} + ConnRes = + ssh:connect(IP, Port, + [{user,?USER}, + {password,?PASSWD}, + {auth_methods, "password"}, + {user_dir, new_dir(Config)}, + {preferred_algorithms, [{Tag,[Alg]}]}, + {silently_accept_hosts,true}, + {user_interaction,false} + ]) , + test_erl_client_reneg(ConnRes, % Seems that max 10 channels may be open in sshd + [{exec,1}, + {sftp,5}, + {no_subsyst,1}, + {setenv, 1}, + {sftp_async,1} ]) end). %%-------------------------------------------------------------------- -all_algorithms_otp_is_server(Config) -> - CommonAlgs = proplists:get_value(common_client_algs, Config), +all_algorithms_sftp_exec_reneg_otp_is_server(Config) -> + CommonAlgs = proplists:get_value(common_remote_client_algs, Config), UserDir = setup_remote_priv_and_local_auth_keys('ssh-rsa', Config), - chk_all_algos(CommonAlgs, Config, + chk_all_algos(?FUNCTION_NAME, CommonAlgs, Config, fun(Tag,Alg) -> HostKeyAlg = case Tag of public_key -> Alg; _ -> 'ssh-rsa' end, + SftpRootDir = new_dir(Config), + %% ct:log("Rootdir = ~p",[SftpRootDir]), {Server, Host, HostPort} = ssh_test_lib:daemon(0, [{preferred_algorithms, [{Tag,[Alg]}]}, {system_dir, setup_local_hostdir(HostKeyAlg, Config)}, {user_dir, UserDir}, - {user_passwords, [{?USER,?PWD}]}, - {failfun, fun ssh_test_lib:failfun/2} + {user_passwords, [{?USER,?PASSWD}]}, + {failfun, fun ssh_test_lib:failfun/2}, + {subsystems, + [ssh_sftpd:subsystem_spec([{cwd,SftpRootDir}, + {root,SftpRootDir}]), + {"echo_10",{ssh_echo_server,[10,[{dbg,true}]]}} + ]} ]), - R = exec_from_docker(Config, Host, HostPort, - "hi_there.\r\n", - [<<"hi_there">>], - ""), + R = do([fun() -> + exec_from_docker(Config, Host, HostPort, + "hi_there.\r\n", + [<<"hi_there">>], + "") + end, + fun() -> + sftp_tests_erl_server(Config, Host, HostPort, SftpRootDir, UserDir) + end + ]), ssh:stop_daemon(Server), R end). %%-------------------------------------------------------------------- +send_recv_big_with_renegotiate_otp_is_client(Config) -> + %% Connect to the remote openssh server: + {IP,Port} = ip_port(Config), + {ok,C} = ssh:connect(IP, Port, [{user,?USER}, + {password,?PASSWD}, + {user_dir, setup_remote_auth_keys_and_local_priv('ssh-rsa', Config)}, + {silently_accept_hosts,true}, + {user_interaction,false} + ]), + + %% Open a channel and exec the Linux 'cat' command at the openssh side. + %% This 'cat' will read stdin and write to stdout until an eof is read from stdin. + {ok, Ch1} = ssh_connection:session_channel(C, infinity), + success = ssh_connection:exec(C, Ch1, "cat", infinity), + + %% Build big binary + HalfSizeBytes = 100*1000*1000, + Data = << <<X:32>> || X <- lists:seq(1, HalfSizeBytes div 4)>>, + + %% Send the data. Must spawn a process to avoid deadlock. The client will block + %% until all is sent through the send window. But the server will stop receiveing + %% when the servers send-window towards the client is full. + %% Since the client can't receive before the server has received all but 655k from the client + %% ssh_connection:send/4 is blocking... + spawn_link( + fun() -> + ct:comment("Sending ~p Mbytes with renegotiation in the middle",[2*byte_size(Data)/1000000]), + %% ct:log("sending first ~p bytes",[byte_size(Data)]), + ok = ssh_connection:send(C, Ch1, Data, 10000), + %% ct:log("Init renegotiation test",[]), + Kex1 = renegotiate_test(init, C), + %% ct:log("sending next ~p bytes",[byte_size(Data)]), + ok = ssh_connection:send(C, Ch1, Data, 10000), + %% ct:log("Finnish renegotiation test",[]), + renegotiate_test(Kex1, C), + %% ct:log("sending eof",[]), + ok = ssh_connection:send_eof(C, Ch1) + %%, ct:log("READY, sent ~p bytes",[2*byte_size(Data)]) + end), + + {eof,ReceivedData} = + loop_until(fun({eof,_}) -> true; + (_ ) -> false + end, + fun(Acc) -> + %%ct:log("Get more ~p",[ ExpectedSize-byte_size(Acc) ]), + receive + {ssh_cm, C, {eof,Ch}} when Ch==Ch1 -> + %% ct:log("eof received",[]), + {eof,Acc}; + + {ssh_cm, C, {data,Ch,0,B}} when Ch==Ch1, + is_binary(B) -> + %% ct:log("(1) Received ~p bytes (total ~p), missing ~p bytes", + %% [byte_size(B), + %% byte_size(B)+byte_size(Acc), + %% 2*byte_size(Data)-(byte_size(B)+byte_size(Acc))]), + ssh_connection:adjust_window(C, Ch1, byte_size(B)), + <<Acc/binary, B/binary>> + end + end, + <<>>), + + ExpectedData = <<Data/binary, Data/binary>>, + case ReceivedData of + ExpectedData -> + %% ct:log("Correct data returned",[]), + %% receive close messages + loop_until(fun(Left) -> %% ct:log("Expect: ~p",[Left]), + Left == [] + end, + fun([Next|Rest]) -> + receive + {ssh_cm,C,Next} -> Rest + end + end, + [%% Already received: {eof, Ch1}, + {exit_status,Ch1,0}, + {closed,Ch1}] + ), + ok; + _ when is_binary(ReceivedData) -> + ct:fail("~p bytes echoed but ~p expected", [byte_size(ReceivedData), 2*byte_size(Data)]) + end. + +%%-------------------------------------------------------------------- %% Utilities --------------------------------------------------------- %%-------------------------------------------------------------------- -exec_from_docker(WhatEver, {0,0,0,0}, HostPort, Command, Expects, ExtraSshArg) -> - exec_from_docker(WhatEver, host_ip(), HostPort, Command, Expects, ExtraSshArg); +%%-------------------------------------------------------------------- +%% +%% A practical meta function +%% +loop_until(CondFun, DoFun, Acc) -> + case CondFun(Acc) of + true -> + Acc; + false -> + loop_until(CondFun, DoFun, DoFun(Acc)) + end. + +%%-------------------------------------------------------------------- +%% +%% Exec the Command in the docker. Add the arguments ExtraSshArg in the +%% ssh command. +%% +%% If Expects is returned, then return 'ok', else return {fail,Msg}. +%% exec_from_docker(Config, HostIP, HostPort, Command, Expects, ExtraSshArg) when is_binary(hd(Expects)), is_list(Config) -> {DockerIP,DockerPort} = ip_port(Config), {ok,C} = ssh:connect(DockerIP, DockerPort, [{user,?USER}, - {password,?PWD}, + {password,?PASSWD}, {user_dir, new_dir(Config)}, {silently_accept_hosts,true}, {user_interaction,false} ]), - R = exec_from_docker(C, HostIP, HostPort, Command, Expects, ExtraSshArg), + R = exec_from_docker(C, HostIP, HostPort, Command, Expects, ExtraSshArg, Config), ssh:close(C), - R; - -exec_from_docker(C, HostIP, HostPort, Command, Expects, ExtraSshArg) when is_binary(hd(Expects)) -> - SSH_from_docker = - lists:concat(["sshpass -p ",?PWD," ", - "/buildroot/ssh/bin/ssh -p ",HostPort," -o 'CheckHostIP=no' -o 'StrictHostKeyChecking=no' ", - ExtraSshArg," ", - inet_parse:ntoa(HostIP)," " - ]), - ExecCommand = SSH_from_docker ++ Command, - R = exec(C, ExecCommand), - case R of - {ok,{ExitStatus,Result}} when ExitStatus == 0 -> + R. + +exec_from_docker(C, DestIP, DestPort, Command, Expects, ExtraSshArg, Config) when is_binary(hd(Expects)) -> + ExecCommand = + lists:concat( + ["sshpass -p ",?PASSWD," " + | case proplists:get_value(ssh_version,Config) of + "dropbear" ++ _ -> + ["dbclient -y -y -p ",DestPort," ",ExtraSshArg," ",iptoa(DestIP)," "]; + + _ -> %% OpenSSH or compatible + ["/buildroot/ssh/bin/ssh -o 'CheckHostIP=no' -o 'StrictHostKeyChecking=no' ", + ExtraSshArg," -p ",DestPort," ",iptoa(DestIP)," "] + end]) ++ Command, + + case exec(C, ExecCommand) of + {ok,{ExitStatus,Result}} = R when ExitStatus == 0 -> case binary:match(Result, Expects) of nomatch -> ct:log("Result of~n ~s~nis~n ~p",[ExecCommand,R]), @@ -327,28 +470,26 @@ exec_from_docker(C, HostIP, HostPort, Command, Expects, ExtraSshArg) when is_bin _ -> ok end; - {ok,_} -> + {ok,_} = R -> ct:log("Result of~n ~s~nis~n ~p",[ExecCommand,R]), {fail, "Exit status =/= 0"}; - _ -> + R -> ct:log("Result of~n ~s~nis~n ~p",[ExecCommand,R]), {fail, "Couldn't login to host"} end. - - exec(C, Cmd) -> - ct:log("~s",[Cmd]), + %% ct:log("~s",[Cmd]), {ok,Ch} = ssh_connection:session_channel(C, 10000), success = ssh_connection:exec(C, Ch, Cmd, 10000), - exec_result(C, Ch). + result_of_exec(C, Ch). -exec_result(C, Ch) -> - exec_result(C, Ch, undefined, <<>>). +result_of_exec(C, Ch) -> + result_of_exec(C, Ch, undefined, <<>>). -exec_result(C, Ch, ExitStatus, Acc) -> +result_of_exec(C, Ch, ExitStatus, Acc) -> receive {ssh_cm,C,{closed,Ch}} -> %%ct:log("CHAN ~p got *closed*",[Ch]), @@ -356,29 +497,37 @@ exec_result(C, Ch, ExitStatus, Acc) -> {ssh_cm,C,{exit_status,Ch,ExStat}} when ExitStatus == undefined -> %%ct:log("CHAN ~p got *exit status ~p*",[Ch,ExStat]), - exec_result(C, Ch, ExStat, Acc); + result_of_exec(C, Ch, ExStat, Acc); {ssh_cm,C,{data,Ch,_,Data}=_X} when ExitStatus == undefined -> %%ct:log("CHAN ~p got ~p",[Ch,_X]), - exec_result(C, Ch, ExitStatus, <<Acc/binary, Data/binary>>); + result_of_exec(C, Ch, ExitStatus, <<Acc/binary, Data/binary>>); _Other -> %%ct:log("OTHER: ~p",[_Other]), - exec_result(C, Ch, ExitStatus, Acc) + result_of_exec(C, Ch, ExitStatus, Acc) after 5000 -> - %%ct:log("NO MORE, received so far:~n~s",[Acc]), + ct:log("NO MORE, received so far:~n~s",[Acc]), {error, timeout} end. -chk_all_algos(CommonAlgs, Config, DoTestFun) when is_function(DoTestFun,2) -> +%%-------------------------------------------------------------------- +%% +%% Loop through all {Tag,Alg} pairs in CommonAlgs, call DoTestFun(Tag,Alg) which +%% returns one of {ok,C}, ok, or Other. +%% +%% The chk_all_algos returns 'ok' or {fail,FaledAlgosList} +%% + +chk_all_algos(FunctionName, CommonAlgs, Config, DoTestFun) when is_function(DoTestFun,2) -> ct:comment("~p algorithms",[length(CommonAlgs)]), %% Check each algorithm Failed = lists:foldl( fun({Tag,Alg}, FailedAlgos) -> - ct:log("Try ~p",[Alg]), + %% ct:log("Try ~p",[Alg]), case DoTestFun(Tag,Alg) of {ok,C} -> ssh:close(C), @@ -387,10 +536,10 @@ chk_all_algos(CommonAlgs, Config, DoTestFun) when is_function(DoTestFun,2) -> FailedAlgos; Other -> ct:log("FAILED! ~p ~p: ~p",[Tag,Alg,Other]), - [Alg|FailedAlgos] + [{Alg,Other}|FailedAlgos] end end, [], CommonAlgs), - ct:pal("~s", [format_result_table_use_all_algos(Config, CommonAlgs, Failed)]), + ct:pal("~s", [format_result_table_use_all_algos(FunctionName, Config, CommonAlgs, Failed)]), case Failed of [] -> ok; @@ -398,6 +547,41 @@ chk_all_algos(CommonAlgs, Config, DoTestFun) when is_function(DoTestFun,2) -> {fail, Failed} end. + + +%%%---------------------------------------------------------------- +%%% +%%% Call all Funs as Fun() which returns 'ok', {ok,C} or Other. +%%% do/1 returns 'ok' or the first encountered value that is not +%%% successful. +%%% + +do(Funs) -> + do(Funs, 1). + +do([Fun|Funs], N) -> + case Fun() of + ok -> + %% ct:log("Fun ~p ok",[N]), + do(Funs, N-1); + {ok,C} -> + %% ct:log("Fun ~p {ok,C}",[N]), + ssh:close(C), + do(Funs, N-1); + Other -> + ct:log("Fun ~p FAILED:~n~p",[N, Other]), + Other + end; + +do([], _) -> + %% ct:log("All Funs ok",[]), + ok. + +%%-------------------------------------------------------------------- +%% +%% Functions to set up local and remote host's and user's keys and directories +%% + setup_local_hostdir(KeyAlg, Config) -> setup_local_hostdir(KeyAlg, new_dir(Config), Config). setup_local_hostdir(KeyAlg, HostDir, Config) -> @@ -428,7 +612,7 @@ setup_remote_auth_keys_and_local_priv(KeyAlg, IP, Port, UserDir, Config) -> ok = file:write_file(DstFile++".pub", Publ), %% Remote auth_methods with public key {ok,Ch,Cc} = ssh_sftp:start_channel(IP, Port, [{user, ?USER }, - {password, ?PWD }, + {password, ?PASSWD }, {auth_methods, "password"}, {silently_accept_hosts,true}, {user_interaction,false} @@ -460,7 +644,7 @@ setup_remote_priv_and_local_auth_keys(KeyAlg, IP, Port, UserDir, Config) -> ok = file:write_file(AuthKeyFile, Publ), %% Remote private and public key {ok,Ch,Cc} = ssh_sftp:start_channel(IP, Port, [{user, ?USER }, - {password, ?PWD }, + {password, ?PASSWD }, {auth_methods, "password"}, {silently_accept_hosts,true}, {user_interaction,false} @@ -485,6 +669,7 @@ priv_pub_keys(KeySubDir, Type, Config, KeyAlg) -> {ok, {Priv,Publ}}. +%%%---------------- The default filenames src_filename(user, 'ssh-rsa' ) -> "id_rsa"; src_filename(user, 'rsa-sha2-256' ) -> "id_rsa"; src_filename(user, 'rsa-sha2-512' ) -> "id_rsa"; @@ -516,7 +701,11 @@ dst_filename(host, 'ecdsa-sha2-nistp384') -> "ssh_host_ecdsa_key"; dst_filename(host, 'ecdsa-sha2-nistp521') -> "ssh_host_ecdsa_key". -format_result_table_use_all_algos(Config, CommonAlgs, Failed) -> +%%-------------------------------------------------------------------- +%% +%% Format the result table for chk_all_algos/4 +%% +format_result_table_use_all_algos(FunctionName, Config, CommonAlgs, Failed) -> %% Write a nice table with the result AlgHead = 'Algorithm', AlgWidth = lists:max([length(atom_to_list(A)) || {_,A} <- CommonAlgs]), @@ -529,23 +718,25 @@ format_result_table_use_all_algos(Config, CommonAlgs, Failed) -> end, {io_lib:format('~s ~*s ~s~n', [Tag, -AlgWidth, A, - case lists:member(A,Failed) of - true -> "<<<< FAIL <<<<"; - false-> "(ok)" + case proplists:get_value(A,Failed) of + undefined -> "(ok)"; + Err -> io_lib:format("<<<< FAIL <<<< ~p",[Err]) end]), T} end, undefined, CommonAlgs), Vssh = proplists:get_value(ssh_version,Config,""), - Vssl = proplists:get_value(ssl_version,Config,""), - io_lib:format("~nResults, Peer versions: ~s and ~s~n" + io_lib:format("~nResults of ~p, Peer version: ~s~n~n" "Tag ~*s Result~n" "=====~*..=s=======~n~s" - ,[Vssh,Vssl, - -AlgWidth,AlgHead, + ,[FunctionName, Vssh, + -AlgWidth, AlgHead, AlgWidth, "", ResultTable]). - +%%-------------------------------------------------------------------- +%% +%% Docker handling: start_docker/1 and stop_docker/1 +%% start_docker(Ver) -> Cmnd = lists:concat(["docker run -itd --rm -p 1234 ",?DOCKER_PFX,":",Ver]), Id0 = os:cmd(Cmnd), @@ -572,6 +763,10 @@ is_docker_sha(L) -> (_) -> false end, L). +%%-------------------------------------------------------------------- +%% +%% Misc docker info functions + ip_port(Config) -> {_Ver,{IP,Port},_} = proplists:get_value(id,Config), {IP,Port}. @@ -590,6 +785,23 @@ ip(Id) -> {ok,IP} = inet:parse_address(IPstr), IP. +%%-------------------------------------------------------------------- +%% +%% Normalize the host returned from ssh_test_lib + +iptoa({0,0,0,0}) -> inet_parse:ntoa(host_ip()); +iptoa(IP) -> inet_parse:ntoa(IP). + +host_ip() -> + {ok,Name} = inet:gethostname(), + {ok,#hostent{h_addr_list = [IP|_]}} = inet_res:gethostbyname(Name), + IP. + +%%-------------------------------------------------------------------- +%% +%% Create a new fresh directory or clear an existing one +%% + new_dir(Config) -> PrivDir = proplists:get_value(priv_dir, Config), SubDirName = integer_to_list(erlang:system_time()), @@ -626,20 +838,34 @@ delete_all_contents(Dir) -> end end, Fs). +%%-------------------------------------------------------------------- +%% +%% Find the intersection of algoritms for otp ssh and the docker ssh. +%% Returns {ok, ServerHello, Server, ClientHello, Client} where Server are the algorithms common +%% with the docker server and analogous for Client. +%% +%% Client may be undefined if no usable client is found. +%% +%% Both Server and Client are lists of {Tag,AlgName}. +%% + common_algs(Config, IP, Port) -> case remote_server_algs(IP, Port) of - {ok, {RemoteHelloBin, RemoteServerKexInit}} -> + {ok, {ServerHello, RemoteServerKexInit}} -> + RemoteServerAlgs = kexint_msg2default_algorithms(RemoteServerKexInit), + Server = find_common_algs(RemoteServerAlgs, + use_algorithms(ServerHello)), + ct:log("Remote server:~n~p~n~p",[ServerHello, RemoteServerAlgs]), case remote_client_algs(Config) of - {ok,{_Hello,RemoteClientKexInit}} -> - RemoteServerAlgs = kexint_msg2default_algorithms(RemoteServerKexInit), - Server = find_common_algs(RemoteServerAlgs, - use_algorithms(RemoteHelloBin)), + {ok,{ClientHello,RemoteClientKexInit}} -> RemoteClientAlgs = kexint_msg2default_algorithms(RemoteClientKexInit), Client = find_common_algs(RemoteClientAlgs, - use_algorithms(RemoteHelloBin)), - ct:log("Docker server algorithms:~n ~p~n~nDocker client algorithms:~n ~p", - [RemoteServerAlgs,RemoteClientAlgs]), - {ok, Server, Client}; + use_algorithms(ClientHello)), + ct:log("Remote client:~n~p~n~p",[ClientHello, RemoteClientAlgs]), + {ok, ServerHello, Server, ClientHello, Client}; + {error,_} =TO -> + ct:log("Remote client algs can't be found: ~p",[TO]), + {ok, ServerHello, Server, undefined, undefined}; Other -> Other end; @@ -648,6 +874,24 @@ common_algs(Config, IP, Port) -> end. +chk_hellos(Hs, Str) -> + lists:foldl( + fun(H, Acc) -> + try binary:split(H, <<"-">>, [global]) + of + %% [<<"SSH">>,<<"2.0">>|_] -> + %% Acc; + [<<"SSH">>,OldVer = <<"1.",_/binary>>|_] -> + io_lib:format("~s, Old SSH ver ~s",[Acc,OldVer]); + _ -> + Acc + catch + _:_ -> + Acc + end + end, Str, Hs). + + find_common_algs(Remote, Local) -> [{T,V} || {T,Vs} <- ssh_test_lib:extract_algos( ssh_test_lib:intersection(Remote, @@ -685,12 +929,18 @@ kexint_msg2default_algorithms(#ssh_msg_kexinit{kex_algorithms = Kex, {server2client,ssh_test_lib:to_atoms(CompS2C)}]}]. - +%%-------------------------------------------------------------------- +%% +%% Find the algorithms supported by the remote server +%% +%% Connect with tcp to the server, send a hello and read the returned +%% server hello and kexinit message. +%% remote_server_algs(IP, Port) -> case try_gen_tcp_connect(IP, Port, 5) of {ok,S} -> ok = gen_tcp:send(S, "SSH-2.0-CheckAlgs\r\n"), - receive_hello(S, <<>>); + receive_hello(S); {error,Error} -> {error,Error} end. @@ -709,6 +959,13 @@ try_gen_tcp_connect(_, _, _) -> {error, "No contact"}. +%%-------------------------------------------------------------------- +%% +%% Find the algorithms supported by the remote client +%% +%% Set up a fake ssh server and make the remote client connect to it. Use +%% hello message and the kexinit message. +%% remote_client_algs(Config) -> Parent = self(), Ref = make_ref(), @@ -719,7 +976,7 @@ remote_client_algs(Config) -> Parent ! {addr,Ref,IP,Port}, {ok,S} = gen_tcp:accept(Sl), ok = gen_tcp:send(S, "SSH-2.0-CheckAlgs\r\n"), - Parent ! {Ref,receive_hello(S, <<>>)} + Parent ! {Ref,receive_hello(S)} end), receive {addr,Ref,IP,Port} -> @@ -732,14 +989,28 @@ remote_client_algs(Config) -> receive {Ref, Result} -> Result - after 15000 -> - {error, timeout2} + after 5000 -> + {error, {timeout,2}} end - after 15000 -> - {error, timeout1} + after 5000 -> + {error, {timeout,1}} end. +%%% Receive a few packets from the remote server or client and find what is supported: + +receive_hello(S) -> + try + receive_hello(S, <<>>) + of + Result -> + Result + catch + Class:Error -> + ST = erlang:get_stacktrace(), + {error, {Class,Error,ST}} + end. + receive_hello(S, Ack) -> %% The Ack is to collect bytes until the full message is received @@ -747,20 +1018,19 @@ receive_hello(S, Ack) -> {tcp, S, Bin0} when is_binary(Bin0) -> case binary:split(<<Ack/binary, Bin0/binary>>, [<<"\r\n">>,<<"\r">>,<<"\n">>]) of [Hello = <<"SSH-2.0-",_/binary>>, NextPacket] -> - ct:log("Got 2.0 hello (~p), ~p bytes to next msg",[Hello,size(NextPacket)]), + %% ct:log("Got 2.0 hello (~p), ~p bytes to next msg",[Hello,size(NextPacket)]), {ok, {Hello, receive_kexinit(S, NextPacket)}}; [Hello = <<"SSH-1.99-",_/binary>>, NextPacket] -> - ct:comment("Old SSH ~s",["1.99"]), - ct:log("Got 1.99 hello (~p), ~p bytes to next msg",[Hello,size(NextPacket)]), + %% ct:log("Got 1.99 hello (~p), ~p bytes to next msg",[Hello,size(NextPacket)]), {ok, {Hello, receive_kexinit(S, NextPacket)}}; [Bin] when size(Bin) < 256 -> - ct:log("Got part of hello (~p chars):~n~s~n~s",[size(Bin),Bin, - [io_lib:format('~2.16.0b ',[C]) - || C <- binary_to_list(Bin0) - ] - ]), + %% ct:log("Got part of hello (~p chars):~n~s~n~s",[size(Bin),Bin, + %% [io_lib:format('~2.16.0b ',[C]) + %% || C <- binary_to_list(Bin0) + %% ] + %% ]), receive_hello(S, Bin0); _ -> @@ -804,11 +1074,326 @@ receive_kexinit(S, Ack) -> throw(timeout) end. +%%%---------------------------------------------------------------- +%%% Test of sftp from the OpenSSH client side +%%% +sftp_tests_erl_server(Config, ServerIP, ServerPort, ServerRootDir, UserDir) -> + try + Cmnds = prepare_local_directory(ServerRootDir), + call_sftp_in_docker(Config, ServerIP, ServerPort, Cmnds, UserDir), + check_local_directory(ServerRootDir) + catch + Class:Error -> + ST = erlang:get_stacktrace(), + {error, {Class,Error,ST}} + end. -host_ip() -> - {ok,Name} = inet:gethostname(), - {ok,#hostent{h_addr_list = [IP|_]}} = inet_res:gethostbyname(Name), - IP. +prepare_local_directory(ServerRootDir) -> + file:write_file(filename:join(ServerRootDir,"tst1"), + <<"Some test text">> + ), + ["get tst1", + "put tst1 tst2", + "put tst1 tst3", + "rename tst1 ex_tst1", + "rm tst3", + "mkdir mydir", + "cd mydir", + "put tst1 file_1", + "put tst1 unreadable_file", + "chmod 222 unreadable_file", + "exit"]. + +check_local_directory(ServerRootDir) -> + case lists:sort(ok(file:list_dir(ServerRootDir)) -- [".",".."]) of + ["ex_tst1","mydir","tst2"] -> + {ok,Expect} = file:read_file(filename:join(ServerRootDir,"ex_tst1")), + case file:read_file(filename:join(ServerRootDir,"tst2")) of + {ok,Expect} -> + case lists:sort(ok(file:list_dir(filename:join(ServerRootDir,"mydir"))) -- [".",".."]) of + ["file_1","unreadable_file"] -> + case file:read_file(filename:join([ServerRootDir,"mydir","file_1"])) of + {ok,Expect} -> + case file:read_file(filename:join([ServerRootDir,"mydir","unreadable_file"])) of + {error,_} -> + ok; + {ok,_} -> + {error, {could_read_unreadable,"mydir/unreadable_file"}} + end; + {ok,Other} -> + ct:log("file_1:~n~s~nExpected:~n~s",[Other,Expect]), + {error, {bad_contents_in_file,"mydir/file_1"}} + end; + Other -> + ct:log("Directory ~s~n~p",[filename:join(ServerRootDir,"mydir"),Other]), + {error,{bad_dir_contents,"mydir"}} + end; + {ok,Other} -> + ct:log("tst2:~n~s~nExpected:~n~s",[Other,Expect]), + {error, {bad_contents_in_file,"tst2"}} + end; + ["tst1"] -> + {error,{missing_file,"tst2"}}; + Other -> + ct:log("Directory ~s~n~p",[ServerRootDir,Other]), + {error,{bad_dir_contents,"/"}} + end. + +call_sftp_in_docker(Config, ServerIP, ServerPort, Cmnds, UserDir) -> + {DockerIP,DockerPort} = ip_port(Config), + {ok,C} = ssh:connect(DockerIP, DockerPort, + [{user,?USER}, + {password,?PASSWD}, + {user_dir, UserDir}, + {silently_accept_hosts,true}, + {user_interaction,false} + ]), + + %% Make commands for "expect" in the docker: + PreExpectCmnds = ["spawn /buildroot/ssh/bin/sftp -oPort="++integer_to_list(ServerPort)++ + " -oCheckHostIP=no -oStrictHostKeyChecking=no " ++ + iptoa(ServerIP)++"\n" + ], + PostExpectCmnds= [], + ExpectCmnds = + PreExpectCmnds ++ + ["expect \"sftp>\" {send \""++Cmnd++"\n\"}\n" || Cmnd <- Cmnds] ++ + PostExpectCmnds, + + %% Make an commands file in the docker + {ok,Ch} = ssh_sftp:start_channel(C, [{timeout,10000}]), + ok = ssh_sftp:write_file(Ch, "commands", erlang:iolist_to_binary(ExpectCmnds)), + ok = ssh_sftp:stop_channel(Ch), + + %% Call expect in the docker + {ok, Ch1} = ssh_connection:session_channel(C, infinity), + Kex1 = renegotiate_test(init, C), + success = ssh_connection:exec(C, Ch1, "expect commands", infinity), + + renegotiate_test(Kex1, C), + recv_log_msgs(C, Ch1), + + %% Done. + ssh:close(C). + +recv_log_msgs(C, Ch) -> + receive + {ssh_cm,C,{closed,Ch}} -> + %% ct:log("Channel closed ~p",[{closed,1}]), + ok; + {ssh_cm,C,{data,Ch,1,Msg}} -> + ct:log("*** ERROR from docker:~n~s",[Msg]), + recv_log_msgs(C, Ch); + {ssh_cm,C,_Msg} -> + %% ct:log("Got ~p",[_Msg]), + recv_log_msgs(C, Ch) + end. +%%%---------------------------------------------------------------- +%%%---------------------------------------------------------------- +%%% +%%% Tests from the Erlang client side +%%% +%%%---------------------------------------------------------------- +%%%---------------------------------------------------------------- +test_erl_client_reneg({ok,C}, Spec) -> + %% Start the test processes on the connection C: + Parent = self(), + Pids = [spawn( + fun() -> + Parent ! {self(), TestType, Id, one_test_erl_client(TestType,Id,C)} + end + ) + || {TestType,N} <- Spec, + Id <- lists:seq(1,N)], + + Kex1 = renegotiate_test(init, C), + + %% Collect the results: + case lists:filter( + fun(R) -> R=/=ok end, + [receive + {Pid,_TestType,_Id,ok} -> + %% ct:log("Test ~p:~p passed!", [_TestType,_Id]), + ok; + {Pid,TestType,Id,OtherResult} -> + ct:log("~p:~p ~p ~p~n~p",[?MODULE,?LINE,TestType,Id,OtherResult]), + {error,TestType,Id} + end || Pid <- Pids]) + of + [] -> + renegotiate_test(Kex1, C), + {ok,C}; + Other -> + renegotiate_test(Kex1, C), + Other + end; + +test_erl_client_reneg(Error, _) -> + Error. + + +one_test_erl_client(exec, Id, C) -> + {ok, Ch} = ssh_connection:session_channel(C, infinity), + success = ssh_connection:exec(C, Ch, "echo Hi there", 5000), + case loop_until(fun({eof,_}) -> true; + (_ ) -> false + end, + fun(Acc) -> + receive + {ssh_cm, C, {eof,Ch}} -> + {eof,Acc}; + {ssh_cm, C, {data,Ch,0,B}} when is_binary(B) -> + <<Acc/binary, B/binary>> + end + end, + <<>>) of + {eof,<<"Hi there\n">>} -> + ok; + Other -> + ct:pal("exec Got other ~p", [Other]), + {error, {exec,Id,bad_msg,Other,undefined}} + end; + +one_test_erl_client(no_subsyst, Id, C) -> + {ok, Ch} = ssh_connection:session_channel(C, infinity), + case ssh_connection:subsystem(C, Ch, "foo", infinity) of + failure -> + ok; + Other -> + ct:pal("no_subsyst Got other ~p", [Other]), + {error, {no_subsyst,Id,bad_ret,Other,undefined}} + end; + +one_test_erl_client(setenv, Id, C) -> + {ok, Ch} = ssh_connection:session_channel(C, infinity), + Var = "ENV_TEST", + Value = lists:concat(["env_test_",Id,"_",erlang:system_time()]), + Env = case ssh_connection:setenv(C, Ch, Var, Value, infinity) of + success -> binary_to_list(Value++"\n"); + failure -> <<"\n">> + end, + success = ssh_connection:exec(C, Ch, "echo $"++Var, 5000), + case loop_until(fun({eof,_}) -> true; + (_ ) -> false + end, + fun(Acc) -> + receive + {ssh_cm, C, {eof,Ch}} -> + {eof,Acc}; + {ssh_cm, C, {data,Ch,0,B}} when is_binary(B) -> + <<Acc/binary, B/binary>> + end + end, + <<>>) of + {eof,Env} -> + ok; + Other -> + ct:pal("setenv Got other ~p", [Other]), + {error, {setenv,Id,bad_msg,Other,undefined}} + end; + +one_test_erl_client(SFTP, Id, C) when SFTP==sftp ; SFTP==sftp_async -> + try + {ok,Ch} = ssh_sftp:start_channel(C, [{timeout,10000}]), + %% A new fresh name of a new file tree: + RootDir = lists:concat(["r_",Id,"_",erlang:system_time()]), + %% Check that it does not exist: + false = lists:member(RootDir, ok(ssh_sftp:list_dir(Ch, "."))), + %% Create it: + ok = ssh_sftp:make_dir(Ch, RootDir), + {ok, #file_info{type=directory, access=read_write}} = ssh_sftp:read_file_info(Ch, RootDir), + R = do_sftp_tests_erl_client(SFTP, C, Ch, Id, RootDir), + catch ssh_sftp:stop_channel(Ch), + R + catch + Class:Error -> + ST = erlang:get_stacktrace(), + {error, {SFTP,Id,Class,Error,ST}} + end. + + + +do_sftp_tests_erl_client(sftp_async, _C, Ch, _Id, RootDir) -> + FileName1 = "boring_name", + F1 = filename:join(RootDir, FileName1), + %% Open a new handle and start writing: + {ok,Handle1} = ssh_sftp:open(Ch, F1, [write,binary]), + {async,Aref1} = ssh_sftp:awrite(Ch, Handle1, <<0:250000/unsigned-unit:8>>), + wait_for_async_result(Aref1); + +do_sftp_tests_erl_client(sftp, _C, Ch, _Id, RootDir) -> + FileName0 = "f0", + F0 = filename:join(RootDir, FileName0), + + %% Create and write a file: + ok = ssh_sftp:write_file(Ch, + F0 = filename:join(RootDir, FileName0), + Data0 = mkbin(1234,240)), + {ok,Data0} = ssh_sftp:read_file(Ch, F0), + {ok, #file_info{type=regular, access=read_write, size=1234}} = ssh_sftp:read_file_info(Ch, F0), + + %% Re-write: + {ok,Handle0} = ssh_sftp:open(Ch, F0, [write,read,binary]), + ok = ssh_sftp:pwrite(Ch, Handle0, 16, Data0_1=mkbin(10,255)), + + <<B1:16/binary, _:10/binary, B2:(1234-26)/binary>> = Data0, + FileContents = <<B1:16/binary, Data0_1:10/binary, B2:(1234-26)/binary>>, + + <<_:1/binary, Part:25/binary, _/binary>> = FileContents, + {ok, Part} = ssh_sftp:pread(Ch, Handle0, 1, 25), + + %% Check: + {ok, FileContents} = ssh_sftp:pread(Ch, Handle0, 0, 1234), + ok = ssh_sftp:close(Ch, Handle0), + + %% Check in another way: + {ok, FileContents} = ssh_sftp:read_file(Ch, F0), + + %% Remove write access rights and check that it can't be written: + ok = ssh_sftp:write_file_info(Ch, F0, #file_info{mode=8#400}), %read}), + {ok, #file_info{type=regular, access=read}} = ssh_sftp:read_file_info(Ch, F0), + {error,permission_denied} = ssh_sftp:write_file(Ch, F0, mkbin(10,14)), + + %% Test deletion of file and dir: + [FileName0] = ok(ssh_sftp:list_dir(Ch, RootDir)) -- [".", ".."], + ok = ssh_sftp:delete(Ch, F0), + [] = ok(ssh_sftp:list_dir(Ch, RootDir)) -- [".", ".."], + ok = ssh_sftp:del_dir(Ch, RootDir), + false = lists:member(RootDir, ok(ssh_sftp:list_dir(Ch, "."))), + ok. + + +wait_for_async_result(Aref) -> + receive + {async_reply, Aref, Result} -> + Result + after + 60000 -> + timeout + end. + + +mkbin(Size, Byte) -> + list_to_binary(lists:duplicate(Size,Byte)). + +ok({ok,X}) -> X. + +%%%---------------------------------------------------------------- +renegotiate_test(init, ConnectionRef) -> + Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), + ssh_connection_handler:renegotiate(ConnectionRef), + %%ct:log("Renegotiate test initiated!",[]), + Kex1; + +renegotiate_test(Kex1, ConnectionRef) -> + case ssh_test_lib:get_kex_init(ConnectionRef) of + Kex1 -> + ct:log("Renegotiate test failed, Kex1 == Kex2!",[]), + error(renegotiate_failed); + _ -> + %% ct:log("Renegotiate test passed!",[]), + ok + end. diff --git a/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-dropbear-ssh b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-dropbear-ssh new file mode 100755 index 0000000000..85973081d0 --- /dev/null +++ b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-dropbear-ssh @@ -0,0 +1,28 @@ +#!/bin/sh + +# ./create-dropbear-ssh + +# This way of fetching the tar-file separate from the docker commands makes +# http-proxy handling way easier. The wget command handles the $https_proxy +# variable while the docker command must have /etc/docker/something changed +# and the docker server restarted. That is not possible without root access. + +# Make a Dockerfile. This method simplifies env variable handling considerably: +cat - > TempDockerFile <<EOF + + FROM ubuntubuildbase + + WORKDIR /buildroot + + RUN apt-get -y update + RUN apt-get -y upgrade + RUN apt-get -y install openssh-sftp-server +%% RUN echo 81 | apt-get -y install dropbear + +EOF + +# Build the image: +docker build -t ssh_compat_suite-ssh-dropbear -f ./TempDockerFile . + +# Cleaning +rm -fr ./TempDockerFile $TMP diff --git a/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-dropbear-ssh-run b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-dropbear-ssh-run new file mode 100755 index 0000000000..d98c0cfaa3 --- /dev/null +++ b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-dropbear-ssh-run @@ -0,0 +1,27 @@ +#!/bin/sh + +# ./create-dropbear-ssh-run + +VER=v2016.72 + +# This way of fetching the tar-file separate from the docker commands makes +# http-proxy handling way easier. The wget command handles the $https_proxy +# variable while the docker command must have /etc/docker/something changed +# and the docker server restarted. That is not possible without root access. + +# Make a Dockerfile. This method simplifies env variable handling considerably: +cat - > TempDockerFile <<EOF + + FROM ssh_compat_suite-ssh-dropbear-installed:${VER} + + WORKDIR /buildroot + + CMD dropbear -F -p 1234 + +EOF + +# Build the image: +docker build -t ssh_compat_suite-ssh:dropbear${VER} -f ./TempDockerFile . + +# Cleaning +rm -fr ./TempDockerFile $TMP diff --git a/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-ssh-image b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-ssh-image index 983c57b18b..2e08408841 100755 --- a/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-ssh-image +++ b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-ssh-image @@ -47,7 +47,8 @@ cat - > TempDockerFile <<EOF RUN ./configure --without-pie \ --prefix=/buildroot/ssh \ --with-ssl-dir=/buildroot/ssl \ - --with-pam + --with-pam \ + LDFLAGS=-Wl,-R/buildroot/ssl/lib RUN make RUN make install RUN echo UsePAM yes >> /buildroot/ssh/etc/sshd_config diff --git a/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-ssl-image b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-ssl-image index 66f8358b8a..4ab2a8bddc 100755 --- a/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-ssl-image +++ b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-ssl-image @@ -23,6 +23,16 @@ case "$1" in ;; esac +case $1$2 in + openssl0.9.8[a-l]) + CONFIG_FLAGS=no-asm + ;; + *) + CONFIG_FLAGS= + ;; +esac + + # This way of fetching the tar-file separate from the docker commands makes # http-proxy handling way easier. The wget command handles the $https_proxy # variable while the docker command must have /etc/docker/something changed @@ -42,10 +52,10 @@ cat - > TempDockerFile <<EOF WORKDIR ${FAM}-${VER} - RUN ./config --prefix=/buildroot/ssl + RUN ./config --prefix=/buildroot/ssl ${CONFIG_FLAGS} RUN make - RUN make install + RUN make install_sw RUN echo Built ${FAM}-${VER} EOF diff --git a/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create_all b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create_all index 16b9c21d9f..0dcf8cb570 100755 --- a/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create_all +++ b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create_all @@ -3,19 +3,21 @@ UBUNTU_VERSION=16.04 SSH_SSL_VERSIONS=(\ - openssh 4.4p1 openssl 0.9.8zh \ - openssh 4.5p1 openssl 0.9.8zh \ - openssh 5.0p1 openssl 0.9.8zh \ - openssh 6.2p2 openssl 0.9.8zh \ - openssh 6.3p1 openssl 0.9.8zh \ - \ - openssh 7.1p1 openssl 1.0.0t \ - \ - openssh 7.1p1 openssl 1.0.1p \ - \ - openssh 6.6p1 openssl 1.0.2n \ - openssh 7.1p1 openssl 1.0.2n \ - openssh 7.6p1 openssl 1.0.2n \ + openssh 4.4p1 openssl 0.9.8c \ + openssh 4.5p1 openssl 0.9.8m \ + openssh 5.0p1 openssl 0.9.8za \ + openssh 6.2p2 openssl 0.9.8c \ + openssh 6.3p1 openssl 0.9.8zh \ + \ + openssh 7.1p1 openssl 1.0.0a \ + \ + openssh 7.1p1 openssl 1.0.1p \ + \ + openssh 6.6p1 openssl 1.0.2n \ + openssh 7.1p1 openssl 1.0.2n \ + openssh 7.6p1 openssl 1.0.2n \ + \ + openssh 7.6p1 libressl 2.6.4 \ ) if [ "x$1" == "x-b" ] diff --git a/lib/ssh/test/ssh_connection_SUITE.erl b/lib/ssh/test/ssh_connection_SUITE.erl index ba4518cfe6..9587c0c251 100644 --- a/lib/ssh/test/ssh_connection_SUITE.erl +++ b/lib/ssh/test/ssh_connection_SUITE.erl @@ -45,6 +45,8 @@ all() -> {group, openssh}, small_interrupted_send, interrupted_send, + exec_erlang_term, + exec_erlang_term_non_default_shell, start_shell, start_shell_exec, start_shell_exec_fun, @@ -85,6 +87,7 @@ init_per_suite(Config) -> ?CHECK_CRYPTO(Config). end_per_suite(Config) -> + catch ssh:stop(), Config. %%-------------------------------------------------------------------- @@ -542,6 +545,79 @@ start_shell_exec(Config) when is_list(Config) -> ssh:stop_daemon(Pid). %%-------------------------------------------------------------------- +exec_erlang_term(Config) when is_list(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = proplists:get_value(data_dir, Config), + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"} + ]), + + ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_interaction, true}, + {user_dir, UserDir}]), + + {ok, ChannelId0} = ssh_connection:session_channel(ConnectionRef, infinity), + + success = ssh_connection:exec(ConnectionRef, ChannelId0, + "1+2.", infinity), + TestResult = + receive + {ssh_cm, ConnectionRef, {data, _ChannelId, 0, <<"3",_/binary>>}} = R -> + ct:log("Got expected ~p",[R]); + Other -> + ct:log("Got unexpected ~p",[Other]) + after 5000 -> + {fail,"Exec Timeout"} + end, + + ssh:close(ConnectionRef), + ssh:stop_daemon(Pid), + TestResult. + +%%-------------------------------------------------------------------- +exec_erlang_term_non_default_shell(Config) when is_list(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = proplists:get_value(data_dir, Config), + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {shell, fun(U, H) -> start_our_shell(U, H) end} + ]), + + ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_interaction, true}, + {user_dir, UserDir} + ]), + + {ok, ChannelId0} = ssh_connection:session_channel(ConnectionRef, infinity), + + success = ssh_connection:exec(ConnectionRef, ChannelId0, + "1+2.", infinity), + TestResult = + receive + {ssh_cm, ConnectionRef, {data, _ChannelId, 0, <<"3",_/binary>>}} = R -> + ct:log("Got unexpected ~p",[R]), + {fail,"Could exec erlang term although non-erlang shell"}; + Other -> + ct:log("Got expected ~p",[Other]) + after 5000 -> + {fail, "Exec Timeout"} + end, + + ssh:close(ConnectionRef), + ssh:stop_daemon(Pid), + TestResult. + +%%-------------------------------------------------------------------- start_shell_exec_fun() -> [{doc, "start shell to exec command"}]. @@ -800,6 +876,8 @@ stop_listener(Config) when is_list(Config) -> ssh:stop_daemon(Pid0), ssh:stop_daemon(Pid1); Error -> + ssh:close(ConnectionRef0), + ssh:stop_daemon(Pid0), ct:fail({unexpected, Error}) end. @@ -819,11 +897,22 @@ start_subsystem_on_closed_channel(Config) -> {user_interaction, false}, {user_dir, UserDir}]), - {ok, ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), - ok = ssh_connection:close(ConnectionRef, ChannelId), + {ok, ChannelId1} = ssh_connection:session_channel(ConnectionRef, infinity), + ok = ssh_connection:close(ConnectionRef, ChannelId1), + {error, closed} = ssh_connection:ptty_alloc(ConnectionRef, ChannelId1, []), + {error, closed} = ssh_connection:subsystem(ConnectionRef, ChannelId1, "echo_n", 5000), + {error, closed} = ssh_connection:exec(ConnectionRef, ChannelId1, "testing1.\n", 5000), + {error, closed} = ssh_connection:send(ConnectionRef, ChannelId1, "exit().\n", 5000), - {error, closed} = ssh_connection:subsystem(ConnectionRef, ChannelId, "echo_n", infinity), + %% Test that there could be a gap between close and an operation (Bugfix OTP-14939): + {ok, ChannelId2} = ssh_connection:session_channel(ConnectionRef, infinity), + ok = ssh_connection:close(ConnectionRef, ChannelId2), + timer:sleep(2000), + {error, closed} = ssh_connection:ptty_alloc(ConnectionRef, ChannelId2, []), + {error, closed} = ssh_connection:subsystem(ConnectionRef, ChannelId2, "echo_n", 5000), + {error, closed} = ssh_connection:exec(ConnectionRef, ChannelId2, "testing1.\n", 5000), + {error, closed} = ssh_connection:send(ConnectionRef, ChannelId2, "exit().\n", 5000), ssh:close(ConnectionRef), ssh:stop_daemon(Pid). diff --git a/lib/ssh/test/ssh_engine_SUITE.erl b/lib/ssh/test/ssh_engine_SUITE.erl index daf93891e9..c131a70973 100644 --- a/lib/ssh/test/ssh_engine_SUITE.erl +++ b/lib/ssh/test/ssh_engine_SUITE.erl @@ -55,16 +55,22 @@ basic_tests() -> init_per_suite(Config) -> ssh:start(), ?CHECK_CRYPTO( - case load_engine() of - {ok,E} -> - [{engine,E}|Config]; - {error, notsup} -> - {skip, "Engine not supported on this OpenSSL version"}; - {error, bad_engine_id} -> - {skip, "Dynamic Engine not supported"}; - Other -> - ct:log("Engine load failed: ~p",[Other]), - {fail, "Engine load failed"} + case crypto:info_lib() of + [{_,_, <<"OpenSSL 1.0.1s-freebsd 1 Mar 2016">>}] -> + {skip, "Strange Engine stuff"}; + + _ -> + case load_engine() of + {ok,E} -> + [{engine,E}|Config]; + {error, notsup} -> + {skip, "Engine not supported on this OpenSSL version"}; + {error, bad_engine_id} -> + {skip, "Dynamic Engine not supported"}; + Other -> + ct:log("Engine load failed: ~p",[Other]), + {fail, "Engine load failed"} + end end ). diff --git a/lib/ssh/test/ssh_options_SUITE.erl b/lib/ssh/test/ssh_options_SUITE.erl index 144ec7f8fd..12a85c40aa 100644 --- a/lib/ssh/test/ssh_options_SUITE.erl +++ b/lib/ssh/test/ssh_options_SUITE.erl @@ -70,7 +70,8 @@ hostkey_fingerprint_check_sha256/1, hostkey_fingerprint_check_sha384/1, hostkey_fingerprint_check_sha512/1, - hostkey_fingerprint_check_list/1 + hostkey_fingerprint_check_list/1, + save_accepted_host_option/1 ]). %%% Common test callbacks @@ -124,6 +125,7 @@ all() -> id_string_own_string_server, id_string_own_string_server_trail_space, id_string_random_server, + save_accepted_host_option, {group, hardening_tests} ]. @@ -206,32 +208,23 @@ end_per_group(_, Config) -> %%-------------------------------------------------------------------- init_per_testcase(_TestCase, Config) -> ssh:start(), - Config. - -end_per_testcase(TestCase, Config) when TestCase == server_password_option; - TestCase == server_userpassword_option; - TestCase == server_pwdfun_option; - TestCase == server_pwdfun_4_option -> + %% Create a clean user_dir UserDir = filename:join(proplists:get_value(priv_dir, Config), nopubkey), ssh_test_lib:del_dirs(UserDir), - end_per_testcase(Config); -end_per_testcase(_TestCase, Config) -> - end_per_testcase(Config). + file:make_dir(UserDir), + [{user_dir,UserDir}|Config]. -end_per_testcase(_Config) -> +end_per_testcase(_TestCase, Config) -> ssh:stop(), ok. %%-------------------------------------------------------------------- %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- -%%-------------------------------------------------------------------- %%% validate to server that uses the 'password' option server_password_option(Config) when is_list(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, {user_dir, UserDir}, @@ -262,12 +255,10 @@ server_password_option(Config) when is_list(Config) -> %%% validate to server that uses the 'password' option server_userpassword_option(Config) when is_list(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, PrivDir}, + {user_dir, UserDir}, {user_passwords, [{"vego", "morot"}]}]), ConnectionRef = @@ -297,15 +288,13 @@ server_userpassword_option(Config) when is_list(Config) -> %%-------------------------------------------------------------------- %%% validate to server that uses the 'pwdfun' option server_pwdfun_option(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), CHKPWD = fun("foo",Pwd) -> Pwd=="bar"; (_,_) -> false end, {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, PrivDir}, + {user_dir, UserDir}, {pwdfun,CHKPWD}]), ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, @@ -335,9 +324,7 @@ server_pwdfun_option(Config) -> %%-------------------------------------------------------------------- %%% validate to server that uses the 'pwdfun/4' option server_pwdfun_4_option(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), PWDFUN = fun("foo",Pwd,{_,_},undefined) -> Pwd=="bar"; ("fie",Pwd,{_,_},undefined) -> {Pwd=="bar",new_state}; @@ -345,7 +332,7 @@ server_pwdfun_4_option(Config) -> (_,_,_,_) -> false end, {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, PrivDir}, + {user_dir, UserDir}, {pwdfun,PWDFUN}]), ConnectionRef1 = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, @@ -395,9 +382,7 @@ server_pwdfun_4_option(Config) -> %%-------------------------------------------------------------------- server_pwdfun_4_option_repeat(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), %% Test that the state works Parent = self(), @@ -406,7 +391,7 @@ server_pwdfun_4_option_repeat(Config) -> (_,P,_,S) -> Parent!{P,S}, {false,S+1} end, {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, PrivDir}, + {user_dir, UserDir}, {auth_methods,"keyboard-interactive"}, {pwdfun,PWDFUN}]), @@ -490,9 +475,7 @@ user_dir_option(Config) -> %%-------------------------------------------------------------------- %%% validate client that uses the 'ssh_msg_debug_fun' option ssh_msg_debug_fun_option_client(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, @@ -530,9 +513,7 @@ ssh_msg_debug_fun_option_client(Config) -> %%-------------------------------------------------------------------- connectfun_disconnectfun_server(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -576,9 +557,7 @@ connectfun_disconnectfun_server(Config) -> %%-------------------------------------------------------------------- connectfun_disconnectfun_client(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -607,9 +586,7 @@ connectfun_disconnectfun_client(Config) -> %%-------------------------------------------------------------------- %%% validate client that uses the 'ssh_msg_debug_fun' option ssh_msg_debug_fun_option_server(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -651,9 +628,7 @@ ssh_msg_debug_fun_option_server(Config) -> %%-------------------------------------------------------------------- disconnectfun_option_server(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -686,9 +661,7 @@ disconnectfun_option_server(Config) -> %%-------------------------------------------------------------------- disconnectfun_option_client(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -720,9 +693,7 @@ disconnectfun_option_client(Config) -> %%-------------------------------------------------------------------- unexpectedfun_option_server(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -763,9 +734,7 @@ unexpectedfun_option_server(Config) -> %%-------------------------------------------------------------------- unexpectedfun_option_client(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -840,14 +809,9 @@ supported_hash(HashAlg) -> really_do_hostkey_fingerprint_check(Config, HashAlg) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDirServer = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDirServer), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), - UserDirClient = - ssh_test_lib:create_random_dir(Config), % Ensure no 'known_hosts' disturbs - %% All host key fingerprints. Trust that public_key has checked the ssh_hostkey_fingerprint %% function since that function is used by the ssh client... FPs0 = [case HashAlg of @@ -873,7 +837,7 @@ really_do_hostkey_fingerprint_check(Config, HashAlg) -> %% Start daemon with the public keys that we got fingerprints from {Pid, Host0, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, UserDirServer}, + {user_dir, UserDir}, {password, "morot"}]), Host = ssh_test_lib:ntoa(Host0), FP_check_fun = fun(PeerName, FP) -> @@ -896,7 +860,8 @@ really_do_hostkey_fingerprint_check(Config, HashAlg) -> end}, {user, "foo"}, {password, "morot"}, - {user_dir, UserDirClient}, + {user_dir, UserDir}, + {save_accepted_host, false}, % Ensure no 'known_hosts' disturbs {user_interaction, false}]), ssh:stop_daemon(Pid). @@ -987,9 +952,7 @@ ms_passed(T0) -> %%-------------------------------------------------------------------- ssh_daemon_minimal_remote_max_packet_size_option(Config) -> SystemDir = proplists:get_value(data_dir, Config), - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), {Server, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, {user_dir, UserDir}, @@ -1314,6 +1277,33 @@ try_to_connect(Connect, Host, Port, Pid, Tref, N) -> end. %%-------------------------------------------------------------------- +save_accepted_host_option(Config) -> + UserDir = proplists:get_value(user_dir, Config), + KnownHosts = filename:join(UserDir, "known_hosts"), + SysDir = proplists:get_value(data_dir, Config), + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {user_passwords, [{"vego", "morot"}]} + ]), + {error,enoent} = file:read_file(KnownHosts), + + {ok,_C1} = ssh:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "vego"}, + {password, "morot"}, + {user_interaction, false}, + {save_accepted_host, false}, + {user_dir, UserDir}]), + {error,enoent} = file:read_file(KnownHosts), + + {ok,_C2} = ssh:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "vego"}, + {password, "morot"}, + {user_interaction, false}, + {user_dir, UserDir}]), + {ok,_} = file:read_file(KnownHosts), + ssh:stop_daemon(Pid). + +%%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- diff --git a/lib/ssh/test/ssh_sup_SUITE.erl b/lib/ssh/test/ssh_sup_SUITE.erl index 3920a1c592..1df55834b1 100644 --- a/lib/ssh/test/ssh_sup_SUITE.erl +++ b/lib/ssh/test/ssh_sup_SUITE.erl @@ -42,7 +42,9 @@ suite() -> all() -> [default_tree, sshc_subtree, sshd_subtree, sshd_subtree_profile, - killed_acceptor_restarts]. + killed_acceptor_restarts, + shell_channel_tree + ]. groups() -> []. @@ -199,7 +201,7 @@ killed_acceptor_restarts(Config) -> Port2 = ssh_test_lib:daemon_port(DaemonPid2), true = (Port /= Port2), - ct:pal("~s",[lists:flatten(ssh_info:string())]), + ct:log("~s",[lists:flatten(ssh_info:string())]), {ok,[{AccPid,ListenAddr,Port}]} = acceptor_pid(DaemonPid), {ok,[{AccPid2,ListenAddr,Port2}]} = acceptor_pid(DaemonPid2), @@ -216,11 +218,14 @@ killed_acceptor_restarts(Config) -> %% Make acceptor restart: exit(AccPid, kill), + ?wait_match(undefined, process_info(AccPid)), %% Check it is a new acceptor: - {ok,[{AccPid1,ListenAddr,Port}]} = acceptor_pid(DaemonPid), - true = (AccPid /= AccPid1), - true = (AccPid2 /= AccPid1), + ?wait_match({ok,[{AccPid1,ListenAddr,Port}]}, AccPid1=/=AccPid, + acceptor_pid(DaemonPid), + AccPid1, + 500, 30), + AccPid1 =/= AccPid2, %% Connect second client and check it is alive: {ok,C2} = ssh:connect("localhost", Port, [{silently_accept_hosts, true}, @@ -230,21 +235,113 @@ killed_acceptor_restarts(Config) -> {user_dir, UserDir}]), [{client_version,_}] = ssh:connection_info(C2,[client_version]), - ct:pal("~s",[lists:flatten(ssh_info:string())]), + ct:log("~s",[lists:flatten(ssh_info:string())]), %% Check first client is still alive: [{client_version,_}] = ssh:connection_info(C1,[client_version]), ok = ssh:stop_daemon(DaemonPid2), - timer:sleep(15000), + ?wait_match(undefined, process_info(DaemonPid2), 1000, 30), [{client_version,_}] = ssh:connection_info(C1,[client_version]), [{client_version,_}] = ssh:connection_info(C2,[client_version]), ok = ssh:stop_daemon(DaemonPid), - timer:sleep(15000), + ?wait_match(undefined, process_info(DaemonPid), 1000, 30), {error,closed} = ssh:connection_info(C1,[client_version]), {error,closed} = ssh:connection_info(C2,[client_version]). + +%%------------------------------------------------------------------------- +shell_channel_tree(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = proplists:get_value(data_dir, Config), + TimeoutShell = + fun() -> + io:format("TimeoutShell started!~n",[]), + timer:sleep(5000), + ct:log("~p TIMEOUT!",[self()]) + end, + {Daemon, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {shell, fun(_User) -> + spawn(TimeoutShell) + end + } + ]), + ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_interaction, true}, + {user_dir, UserDir}]), + + [ChannelSup|_] = Sups0 = chk_empty_con_daemon(Daemon), + {ok, ChannelId0} = ssh_connection:session_channel(ConnectionRef, infinity), + ok = ssh_connection:shell(ConnectionRef,ChannelId0), + + ?wait_match([{_, GroupPid,worker,[ssh_channel]}], + supervisor:which_children(ChannelSup), + [GroupPid]), + {links,GroupLinks} = erlang:process_info(GroupPid, links), + [ShellPid] = GroupLinks--[ChannelSup], + ct:log("GroupPid = ~p, ShellPid = ~p",[GroupPid,ShellPid]), + + receive + {ssh_cm,ConnectionRef, {data, ChannelId0, 0, <<"TimeoutShell started!\r\n">>}} -> + receive + %%---- wait for the subsystem to terminate + {ssh_cm,ConnectionRef,{closed,ChannelId0}} -> + ct:log("Subsystem terminated",[]), + case {chk_empty_con_daemon(Daemon), + process_info(GroupPid), + process_info(ShellPid)} of + {Sups0, undefined, undefined} -> + %% SUCCESS + ssh:stop_daemon(Daemon); + {Sups0, _, undefined} -> + ssh:stop_daemon(Daemon), + ct:fail("Group proc lives!"); + {Sups0, undefined, _} -> + ssh:stop_daemon(Daemon), + ct:fail("Shell proc lives!"); + _ -> + ssh:stop_daemon(Daemon), + ct:fail("Sup tree changed!") + end + after 10000 -> + ssh:close(ConnectionRef), + ssh:stop_daemon(Daemon), + ct:fail("CLI Timeout") + end + after 10000 -> + ssh:close(ConnectionRef), + ssh:stop_daemon(Daemon), + ct:fail("CLI Timeout") + end. + + +chk_empty_con_daemon(Daemon) -> + ?wait_match([{_,SubSysSup, supervisor,[ssh_subsystem_sup]}, + {{ssh_acceptor_sup,_,_,_}, AccSup, supervisor,[ssh_acceptor_sup]}], + supervisor:which_children(Daemon), + [SubSysSup,AccSup]), + ?wait_match([{{server,ssh_connection_sup, _,_}, + ConnectionSup, supervisor, + [ssh_connection_sup]}, + {{server,ssh_channel_sup,_ ,_}, + ChannelSup,supervisor, + [ssh_channel_sup]}], + supervisor:which_children(SubSysSup), + [ConnectionSup,ChannelSup]), + ?wait_match([{{ssh_acceptor_sup,_,_,_},_,worker,[ssh_acceptor]}], + supervisor:which_children(AccSup)), + ?wait_match([{_, _, worker,[ssh_connection_handler]}], + supervisor:which_children(ConnectionSup)), + ?wait_match([], supervisor:which_children(ChannelSup)), + [ChannelSup, ConnectionSup, SubSysSup, AccSup]. + %%------------------------------------------------------------------------- %% Help functions %%------------------------------------------------------------------------- diff --git a/lib/ssh/test/ssh_test_lib.erl b/lib/ssh/test/ssh_test_lib.erl index 2d7bf75847..57ae2dbac2 100644 --- a/lib/ssh/test/ssh_test_lib.erl +++ b/lib/ssh/test/ssh_test_lib.erl @@ -56,7 +56,9 @@ daemon(Host, Port, Options) -> ct:log("~p:~p Calling ssh:daemon(~p, ~p, ~p)",[?MODULE,?LINE,Host,Port,Options]), case ssh:daemon(Host, Port, Options) of {ok, Pid} -> - {ok,L} = ssh:daemon_info(Pid), + R = ssh:daemon_info(Pid), + ct:log("~p:~p ssh:daemon_info(~p) ->~n ~p",[?MODULE,?LINE,Pid,R]), + {ok,L} = R, ListenPort = proplists:get_value(port, L), ListenIP = proplists:get_value(ip, L), {Pid, ListenIP, ListenPort}; @@ -199,15 +201,17 @@ init_io_server(TestCase) -> loop_io_server(TestCase, Buff0) -> receive - {input, TestCase, Line} -> + {input, TestCase, Line} = _INP -> + %%ct:log("io_server ~p:~p ~p got ~p",[?MODULE,?LINE,self(),_INP]), loop_io_server(TestCase, Buff0 ++ [Line]); - {io_request, From, ReplyAs, Request} -> + {io_request, From, ReplyAs, Request} = _REQ-> + %%ct:log("io_server ~p:~p ~p got ~p",[?MODULE,?LINE,self(),_REQ]), {ok, Reply, Buff} = io_request(Request, TestCase, From, ReplyAs, Buff0), io_reply(From, ReplyAs, Reply), loop_io_server(TestCase, Buff); {'EXIT',_, _} = _Exit -> -%% ct:log("ssh_test_lib:loop_io_server/2 got ~p",[_Exit]), + ct:log("ssh_test_lib:loop_io_server/2 got ~p",[_Exit]), ok after 30000 -> ct:fail("timeout ~p:~p",[?MODULE,?LINE]) diff --git a/lib/ssh/test/ssh_test_lib.hrl b/lib/ssh/test/ssh_test_lib.hrl index eaf856e6e8..4b6579bd71 100644 --- a/lib/ssh/test/ssh_test_lib.hrl +++ b/lib/ssh/test/ssh_test_lib.hrl @@ -16,12 +16,12 @@ %%------------------------------------------------------------------------- %% Help macro %%------------------------------------------------------------------------- --define(wait_match(Pattern, FunctionCall, Bind, Timeout, Ntries), +-define(wait_match(Pattern, Guard, FunctionCall, Bind, Timeout, Ntries), Bind = (fun() -> F = fun(N, F1) -> case FunctionCall of - Pattern -> Bind; + Pattern when Guard -> Bind; _ when N>0 -> ct:pal("Must sleep ~p ms at ~p:~p",[Timeout,?MODULE,?LINE]), timer:sleep(Timeout), @@ -34,6 +34,9 @@ end)() ). +-define(wait_match(Pattern, FunctionCall, Bind, Timeout, Ntries), + ?wait_match(Pattern, true, FunctionCall, Bind, Timeout, Ntries)). + -define(wait_match(Pattern, FunctionCall, Timeout, Ntries), ?wait_match(Pattern, FunctionCall, ok, Timeout, Ntries)). -define(wait_match(Pattern, FunctionCall, Bind), ?wait_match(Pattern, FunctionCall, Bind, 500, 10) ). diff --git a/lib/ssh/test/ssh_to_openssh_SUITE.erl b/lib/ssh/test/ssh_to_openssh_SUITE.erl index b20764ce47..9df404d7ed 100644 --- a/lib/ssh/test/ssh_to_openssh_SUITE.erl +++ b/lib/ssh/test/ssh_to_openssh_SUITE.erl @@ -48,19 +48,9 @@ all() -> end. groups() -> - [{erlang_client, [], [erlang_shell_client_openssh_server, - erlang_client_openssh_server_exec_compressed, - erlang_client_openssh_server_setenv, - erlang_client_openssh_server_publickey_dsa, - erlang_client_openssh_server_publickey_rsa, - erlang_client_openssh_server_password, - erlang_client_openssh_server_kexs, - erlang_client_openssh_server_nonexistent_subsystem, - erlang_client_openssh_server_renegotiate + [{erlang_client, [], [erlang_shell_client_openssh_server ]}, - {erlang_server, [], [erlang_server_openssh_client_public_key_dsa, - erlang_server_openssh_client_public_key_rsa, - erlang_server_openssh_client_renegotiate + {erlang_server, [], [erlang_server_openssh_client_renegotiate ]} ]. @@ -100,15 +90,6 @@ end_per_group(_, Config) -> Config. -init_per_testcase(erlang_server_openssh_client_public_key_dsa, Config) -> - chk_key(sshc, 'ssh-dss', ".ssh/id_dsa", Config); -init_per_testcase(erlang_server_openssh_client_public_key_rsa, Config) -> - chk_key(sshc, 'ssh-rsa', ".ssh/id_rsa", Config); -init_per_testcase(erlang_client_openssh_server_publickey_dsa, Config) -> - chk_key(sshd, 'ssh-dss', ".ssh/id_dsa", Config); -init_per_testcase(erlang_client_openssh_server_publickey_rsa, Config) -> - chk_key(sshd, 'ssh-rsa', ".ssh/id_rsa", Config); - init_per_testcase(erlang_server_openssh_client_renegotiate, Config) -> case os:type() of {unix,_} -> ssh:start(), Config; @@ -122,27 +103,6 @@ end_per_testcase(_TestCase, _Config) -> ssh:stop(), ok. - -chk_key(Pgm, Name, File, Config) -> - case ssh_test_lib:openssh_supports(Pgm, public_key, Name) of - false -> - {skip,lists:concat(["openssh client does not support ",Name])}; - true -> - {ok,[[Home]]} = init:get_argument(home), - KeyFile = filename:join(Home, File), - case file:read_file(KeyFile) of - {ok, Pem} -> - case public_key:pem_decode(Pem) of - [{_,_, not_encrypted}] -> - init_per_testcase('__default__',Config); - _ -> - {skip, {error, "Has pass phrase can not be used by automated test case"}} - end; - _ -> - {skip, lists:concat(["no ~/",File])} - end - end. - %%-------------------------------------------------------------------- %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- @@ -160,219 +120,6 @@ erlang_shell_client_openssh_server(Config) when is_list(Config) -> receive_logout(), receive_normal_exit(Shell). -%-------------------------------------------------------------------- -erlang_client_openssh_server_exec() -> - [{doc, "Test api function ssh_connection:exec"}]. - -erlang_client_openssh_server_exec(Config) when is_list(Config) -> - ConnectionRef = ssh_test_lib:connect(?SSH_DEFAULT_PORT, [{silently_accept_hosts, true}, - {user_interaction, false}]), - {ok, ChannelId0} = ssh_connection:session_channel(ConnectionRef, infinity), - success = ssh_connection:exec(ConnectionRef, ChannelId0, - "echo testing", infinity), - Data0 = {ssh_cm, ConnectionRef, {data, ChannelId0, 0, <<"testing\n">>}}, - case ssh_test_lib:receive_exec_result(Data0) of - expected -> - ssh_test_lib:receive_exec_end(ConnectionRef, ChannelId0); - {unexpected_msg,{ssh_cm, ConnectionRef, {exit_status, ChannelId0, 0}} - = ExitStatus0} -> - ct:log("0: Collected data ~p", [ExitStatus0]), - ssh_test_lib:receive_exec_result(Data0, - ConnectionRef, ChannelId0); - Other0 -> - ct:fail(Other0) - end, - - {ok, ChannelId1} = ssh_connection:session_channel(ConnectionRef, infinity), - success = ssh_connection:exec(ConnectionRef, ChannelId1, - "echo testing1", infinity), - Data1 = {ssh_cm, ConnectionRef, {data, ChannelId1, 0, <<"testing1\n">>}}, - case ssh_test_lib:receive_exec_result(Data1) of - expected -> - ssh_test_lib:receive_exec_end(ConnectionRef, ChannelId1); - {unexpected_msg,{ssh_cm, ConnectionRef, {exit_status, ChannelId1, 0}} - = ExitStatus1} -> - ct:log("0: Collected data ~p", [ExitStatus1]), - ssh_test_lib:receive_exec_result(Data1, - ConnectionRef, ChannelId1); - Other1 -> - ct:fail(Other1) - end. - -%%-------------------------------------------------------------------- -erlang_client_openssh_server_exec_compressed() -> - [{doc, "Test that compression option works"}]. - -erlang_client_openssh_server_exec_compressed(Config) when is_list(Config) -> - CompressAlgs = [zlib, '[email protected]',none], - case ssh_test_lib:ssh_supports(CompressAlgs, compression) of - {false,L} -> - {skip, io_lib:format("~p compression is not supported",[L])}; - - true -> - ConnectionRef = ssh_test_lib:connect(?SSH_DEFAULT_PORT, [{silently_accept_hosts, true}, - {user_interaction, false}, - {preferred_algorithms, - [{compression,CompressAlgs}]}]), - {ok, ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), - success = ssh_connection:exec(ConnectionRef, ChannelId, - "echo testing", infinity), - Data = {ssh_cm, ConnectionRef, {data, ChannelId, 0, <<"testing\n">>}}, - case ssh_test_lib:receive_exec_result(Data) of - expected -> - ssh_test_lib:receive_exec_end(ConnectionRef, ChannelId); - {unexpected_msg,{ssh_cm, ConnectionRef, - {exit_status, ChannelId, 0}} = ExitStatus} -> - ct:log("0: Collected data ~p", [ExitStatus]), - ssh_test_lib:receive_exec_result(Data, ConnectionRef, ChannelId); - Other -> - ct:fail(Other) - end - end. - -%%-------------------------------------------------------------------- -erlang_client_openssh_server_kexs() -> - [{doc, "Test that we can connect with different KEXs."}]. - -erlang_client_openssh_server_kexs(Config) when is_list(Config) -> - KexAlgos = try proplists:get_value(kex, proplists:get_value(common_algs,Config)) - catch _:_ -> [] - end, - comment(KexAlgos), - case KexAlgos of - [] -> {skip, "No common kex algorithms"}; - _ -> - Success = - lists:foldl( - fun(Kex, Acc) -> - ConnectionRef = - ssh_test_lib:connect(?SSH_DEFAULT_PORT, [{silently_accept_hosts, true}, - {user_interaction, false}, - {preferred_algorithms, - [{kex,[Kex]}]}]), - - {ok, ChannelId} = - ssh_connection:session_channel(ConnectionRef, infinity), - success = - ssh_connection:exec(ConnectionRef, ChannelId, - "echo testing", infinity), - - ExpectedData = {ssh_cm, ConnectionRef, {data, ChannelId, 0, <<"testing\n">>}}, - case ssh_test_lib:receive_exec_result(ExpectedData) of - expected -> - ssh_test_lib:receive_exec_end(ConnectionRef, ChannelId), - Acc; - {unexpected_msg,{ssh_cm, ConnectionRef, - {exit_status, ChannelId, 0}} = ExitStatus} -> - ct:log("0: Collected data ~p", [ExitStatus]), - ssh_test_lib:receive_exec_result(ExpectedData, ConnectionRef, ChannelId), - Acc; - Other -> - ct:log("~p failed: ~p",[Kex,Other]), - false - end - end, true, KexAlgos), - case Success of - true -> - ok; - false -> - {fail, "Kex failed for one or more algos"} - end - end. - -%%-------------------------------------------------------------------- -erlang_client_openssh_server_setenv() -> - [{doc, "Test api function ssh_connection:setenv"}]. - -erlang_client_openssh_server_setenv(Config) when is_list(Config) -> - ConnectionRef = - ssh_test_lib:connect(?SSH_DEFAULT_PORT, [{silently_accept_hosts, true}, - {user_interaction, false}]), - {ok, ChannelId} = - ssh_connection:session_channel(ConnectionRef, infinity), - Env = case ssh_connection:setenv(ConnectionRef, ChannelId, - "ENV_TEST", "testing_setenv", - infinity) of - success -> - <<"tesing_setenv\n">>; - failure -> - <<"\n">> - end, - success = ssh_connection:exec(ConnectionRef, ChannelId, - "echo $ENV_TEST", infinity), - Data = {ssh_cm, ConnectionRef, {data, ChannelId, 0, Env}}, - case ssh_test_lib:receive_exec_result(Data) of - expected -> - ssh_test_lib:receive_exec_end(ConnectionRef, ChannelId); - {unexpected_msg,{ssh_cm, ConnectionRef, - {data,0,1, UnxpectedData}}} -> - %% Some os may return things as - %% ENV_TEST: Undefined variable.\n" - ct:log("UnxpectedData: ~p", [UnxpectedData]), - ssh_test_lib:receive_exec_end(ConnectionRef, ChannelId); - {unexpected_msg,{ssh_cm, ConnectionRef, {exit_status, ChannelId, 0}} - = ExitStatus} -> - ct:log("0: Collected data ~p", [ExitStatus]), - ssh_test_lib:receive_exec_result(Data, - ConnectionRef, ChannelId); - Other -> - ct:fail(Other) - end. - -%%-------------------------------------------------------------------- - -%% setenv not meaningfull on erlang ssh daemon! - -%%-------------------------------------------------------------------- -erlang_client_openssh_server_publickey_rsa(Config) -> - erlang_client_openssh_server_publickey_X(Config, 'ssh-rsa'). - -erlang_client_openssh_server_publickey_dsa(Config) -> - erlang_client_openssh_server_publickey_X(Config, 'ssh-dss'). - - -erlang_client_openssh_server_publickey_X(_Config, Alg) -> - ConnectionRef = - ssh_test_lib:connect(?SSH_DEFAULT_PORT, - [{pref_public_key_algs, [Alg]}, - {user_interaction, false}, - {auth_methods, "publickey"}, - silently_accept_hosts]), - {ok, Channel} = - ssh_connection:session_channel(ConnectionRef, infinity), - ok = ssh_connection:close(ConnectionRef, Channel), - ok = ssh:close(ConnectionRef). - -%%-------------------------------------------------------------------- -erlang_server_openssh_client_public_key_dsa() -> - [{timetrap, {seconds,(?TIMEOUT div 1000)+10}}]. -erlang_server_openssh_client_public_key_dsa(Config) when is_list(Config) -> - erlang_server_openssh_client_public_key_X(Config, 'ssh-dss'). - -erlang_server_openssh_client_public_key_rsa() -> - [{timetrap, {seconds,(?TIMEOUT div 1000)+10}}]. -erlang_server_openssh_client_public_key_rsa(Config) when is_list(Config) -> - erlang_server_openssh_client_public_key_X(Config, 'ssh-rsa'). - - -erlang_server_openssh_client_public_key_X(Config, Alg) -> - SystemDir = proplists:get_value(data_dir, Config), - PrivDir = proplists:get_value(priv_dir, Config), - KnownHosts = filename:join(PrivDir, "known_hosts"), - {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, - {preferred_algorithms,[{public_key, [Alg]}]}, - {auth_methods, "publickey"}, - {failfun, fun ssh_test_lib:failfun/2}]), - ct:sleep(500), - - Cmd = ssh_test_lib:open_sshc_cmd(Host, Port, - [" -o UserKnownHostsFile=", KnownHosts, - " -o StrictHostKeyChecking=no"], - "1+1."), - OpenSsh = ssh_test_lib:open_port({spawn, Cmd}), - ssh_test_lib:rcv_expected({data,<<"2\n">>}, OpenSsh, ?TIMEOUT), - ssh:stop_daemon(Pid). - %%-------------------------------------------------------------------- %% Test that the Erlang/OTP server can renegotiate with openSSH erlang_server_openssh_client_renegotiate(Config) -> @@ -430,108 +177,6 @@ erlang_server_openssh_client_renegotiate(Config) -> end. %%-------------------------------------------------------------------- -erlang_client_openssh_server_renegotiate(_Config) -> - process_flag(trap_exit, true), - IO = ssh_test_lib:start_io_server(), - Ref = make_ref(), - Parent = self(), - - Shell = - spawn_link( - fun() -> - Host = ssh_test_lib:hostname(), - Options = [{user_interaction, false}, - {silently_accept_hosts,true}], - group_leader(IO, self()), - {ok, ConnRef} = ssh:connect(Host, ?SSH_DEFAULT_PORT, Options), - ct:log("Parent = ~p, IO = ~p, Shell = ~p, ConnRef = ~p~n",[Parent, IO, self(), ConnRef]), - case ssh_connection:session_channel(ConnRef, infinity) of - {ok,ChannelId} -> - success = ssh_connection:ptty_alloc(ConnRef, ChannelId, []), - Args = [{channel_cb, ssh_shell}, - {init_args,[ConnRef, ChannelId]}, - {cm, ConnRef}, {channel_id, ChannelId}], - {ok, State} = ssh_channel:init([Args]), - Parent ! {ok, Ref, ConnRef}, - ssh_channel:enter_loop(State); - Error -> - Parent ! {error, Ref, Error} - end, - receive - nothing -> ok - end - end), - - receive - {error, Ref, Error} -> - ct:fail("Error=~p",[Error]); - {ok, Ref, ConnectionRef} -> - IO ! {input, self(), "echo Hej1\n"}, - receive_data("Hej1", ConnectionRef), - Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), - ssh_connection_handler:renegotiate(ConnectionRef), - IO ! {input, self(), "echo Hej2\n"}, - receive_data("Hej2", ConnectionRef), - Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), - IO ! {input, self(), "exit\n"}, - receive_logout(), - receive_normal_exit(Shell), - true = (Kex1 =/= Kex2) - end. - -%%-------------------------------------------------------------------- -erlang_client_openssh_server_password() -> - [{doc, "Test client password option"}]. -erlang_client_openssh_server_password(Config) when is_list(Config) -> - %% to make sure we don't public-key-auth - UserDir = proplists:get_value(data_dir, Config), - {error, Reason0} = - ssh:connect(any, ?SSH_DEFAULT_PORT, [{silently_accept_hosts, true}, - {user, "foo"}, - {password, "morot"}, - {user_interaction, false}, - {user_dir, UserDir}]), - - ct:log("Test of user foo that does not exist. " - "Error msg: ~p~n", [Reason0]), - - User = string:strip(os:cmd("whoami"), right, $\n), - - case length(string:tokens(User, " ")) of - 1 -> - {error, Reason1} = - ssh:connect(any, ?SSH_DEFAULT_PORT, - [{silently_accept_hosts, true}, - {user, User}, - {password, "foo"}, - {user_interaction, false}, - {user_dir, UserDir}]), - ct:log("Test of wrong Pasword. " - "Error msg: ~p~n", [Reason1]); - _ -> - ct:log("Whoami failed reason: ~n", []) - end. - -%%-------------------------------------------------------------------- - -erlang_client_openssh_server_nonexistent_subsystem() -> - [{doc, "Test client password option"}]. -erlang_client_openssh_server_nonexistent_subsystem(Config) when is_list(Config) -> - - ConnectionRef = ssh_test_lib:connect(?SSH_DEFAULT_PORT, - [{user_interaction, false}, - silently_accept_hosts]), - - {ok, ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), - - failure = ssh_connection:subsystem(ConnectionRef, ChannelId, "foo", infinity). - -%%-------------------------------------------------------------------- -% -%% Not possible to send password with openssh without user interaction -%% -%%-------------------------------------------------------------------- -%%-------------------------------------------------------------------- %%% Internal functions ----------------------------------------------- %%-------------------------------------------------------------------- receive_data(Data, Conn) -> diff --git a/lib/ssh/vsn.mk b/lib/ssh/vsn.mk index 004db6e3a2..480e955ec4 100644 --- a/lib/ssh/vsn.mk +++ b/lib/ssh/vsn.mk @@ -1,5 +1,4 @@ #-*-makefile-*- ; force emacs to enter makefile-mode -SSH_VSN = 4.6.4 - +SSH_VSN = 4.6.5 APP_VSN = "ssh-$(SSH_VSN)" diff --git a/lib/ssl/doc/src/notes.xml b/lib/ssl/doc/src/notes.xml index 79176f5edf..bdf8711b2f 100644 --- a/lib/ssl/doc/src/notes.xml +++ b/lib/ssl/doc/src/notes.xml @@ -307,6 +307,21 @@ </section> </section> +<section><title>SSL 8.1.3.1.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix alert handling so that unexpected messages are logged + and alerted correctly</p> + <p> + Own Id: OTP-14929</p> + </item> + </list> + </section> +</section> + <section><title>SSL 8.1.3.1</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index 80b639155b..4f72114ae9 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -141,23 +141,25 @@ <tag><c>sslsocket() =</c></tag> <item><p>opaque()</p></item> - <tag><marker id="type-protocol"/><c> protocol_versions() =</c></tag> + <tag><marker id="type-protocol"/><c> protocol_version() =</c></tag> <item><p><c> ssl_tls_protocol() | dtls_protocol() </c></p></item> - <tag><marker id="type-protocol"/><c> ssl_tls_protocol() =</c></tag> <item><p><c>sslv3 | tlsv1 | 'tlsv1.1' | 'tlsv1.2'</c></p></item> <tag><marker id="type-protocol"/><c> dtls_protocol() =</c></tag> <item><p><c>'dtlsv1' | 'dtlsv1.2'</c></p></item> <tag><c>ciphers() =</c></tag> - <item><p><c>= [ciphersuite()] | string()</c></p> - <p>According to old API.</p></item> + <item><p><c>= [ciphersuite()]</c></p> + <p>Tuples and string formats accepted by versions + before ssl-8.2.4 will be converted for backwards compatibility</p></item> <tag><c>ciphersuite() =</c></tag> - - <item><p><c>{key_exchange(), cipher(), MAC::hash()} | - {key_exchange(), cipher(), MAC::hash(), PRF::hash()}</c></p></item> + <item><p><c> + #{key_exchange := key_exchange(), + cipher := cipher(), + mac := MAC::hash() | aead, + prf := PRF::hash() | default_prf} </c></p></item> <tag><c>key_exchange()=</c></tag> <item><p><c>rsa | dhe_dss | dhe_rsa | dh_anon | psk | dhe_psk @@ -174,6 +176,12 @@ <tag><c>prf_random() =</c></tag> <item><p><c>client_random | server_random</c></p></item> + <tag><c>cipher_filters() =</c></tag> + <item><p><c> [{key_exchange | cipher | mac | prf, algo_filter()}])</c></p></item> + + <tag><c>algo_filter() =</c></tag> + <item><p>fun(key_exchange() | cipher() | hash() | aead | default_prf) -> true | false </p></item> + <tag><c>srp_param_type() =</c></tag> <item><p><c>srp_1024 | srp_1536 | srp_2048 | srp_3072 | srp_4096 | srp_6144 | srp_8192</c></p></item> @@ -465,7 +473,8 @@ marker="public_key:public_key#pkix_path_validation-3">public_key:pkix_path_valid marker="public_key:public_key#pkix_path_validation-3">public_key:pkix_path_validation/3</seealso> with the selected CA as trusted anchor and the rest of the chain.</p></item> - <tag><c>{versions, [protocol_versions()]}</c></tag> + <tag><c>{versions, [protocol_version()]}</c></tag> + <item><p>TLS protocol versions supported by started clients and servers. This option overrides the application environment option <c>protocol_version</c> and <c>dtls_protocol_version</c>. If the environment option is not set, it defaults @@ -838,23 +847,55 @@ fun(srp, Username :: string(), UserState :: term()) -> </section> <funcs> + + <func> + <name>append_cipher_suites(Deferred, Suites) -> ciphers() </name> + <fsummary></fsummary> + <type> + <v>Deferred = ciphers() | cipher_filters() </v> + <v>Suites = ciphers() </v> + </type> + <desc><p>Make <c>Deferred</c> suites become the least preferred + suites, that is put them at the end of the cipher suite list + <c>Suites</c> after removing them from <c>Suites</c> if + present. <c>Deferred</c> may be a list of cipher suits or a + list of filters in which case the filters are use on <c>Suites</c> to + extract the Deferred cipher list.</p> + </desc> + </func> + <func> <name>cipher_suites() -></name> - <name>cipher_suites(Type) -> ciphers()</name> - <name>cipher_suites(Type, protocol_version()) -> ciphers()</name> + <name>cipher_suites(Type) -> old_ciphers()</name> <fsummary>Returns a list of supported cipher suites.</fsummary> <type> <v>Type = erlang | openssl | all</v> </type> - <desc><p>Returns a list of supported cipher suites. + <desc> + <p>Returns a list of supported cipher suites. + This function will become deprecated in OTP 21, and replaced + by <seealso marker="#cipher_suites-2">ssl:cipher-suites/2</seealso> <c>cipher_suites()</c> is equivalent to <c>cipher_suites(erlang).</c> Type <c>openssl</c> is provided for backwards compatibility with the old SSL, which used OpenSSL. <c>cipher_suites(all)</c> returns all available cipher suites. The cipher suites not present in <c>cipher_suites(erlang)</c> but included in <c>cipher_suites(all)</c> are not used unless explicitly configured - by the user. If the version option is not specified, the highest supported - TLS version will be used to determine the supported cipher suites</p> + by the user.</p> + </desc> + </func> + + <func> + <name>cipher_suites(Supported, Version) -> ciphers()</name> + <fsummary>Returns a list of all default or + all supported cipher suites.</fsummary> + <type> + <v> Supported = default | all | anonymous </v> + <v> Version = protocol_version() </v> + </type> + <desc><p>Returns all default or all supported (except anonymous), + or all anonymous cipher suites for a + TLS version</p> </desc> </func> @@ -1019,6 +1060,21 @@ fun(srp, Username :: string(), UserState :: term()) -> </desc> </func> + <func> + <name>filter_cipher_suites(Suites, Filters) -> ciphers()</name> + <fsummary></fsummary> + <type> + <v> Suites = ciphers()</v> + <v> Filters = cipher_filters()</v> + </type> + <desc><p>Removes cipher suites if any of the filter functions + returns false for any part of the cipher suite. This function + also calls default filter functions to make sure the cipher + suites are supported by crypto. If no filter function is supplied for some + part the default behaviour is fun(Algorithm) -> true.</p> + </desc> + </func> + <func> <name>format_error(Reason) -> string()</name> <fsummary>Returns an error string.</fsummary> @@ -1116,6 +1172,22 @@ fun(srp, Username :: string(), UserState :: term()) -> <p>Returns the address and port number of the peer.</p> </desc> </func> + + <func> + <name>prepend_cipher_suites(Preferred, Suites) -> ciphers()</name> + <fsummary></fsummary> + <type> + <v>Preferred = ciphers() | cipher_filters() </v> + <v>Suites = ciphers() </v> + </type> + <desc><p>Make <c>Preferred</c> suites become the most preferred + suites that is put them at the head of the cipher suite list + <c>Suites</c> after removing them from <c>Suites</c> if + present. <c>Preferred</c> may be a list of cipher suits or a + list of filters in which case the filters are use on <c>Suites</c> to + extract the preferred cipher list. </p> + </desc> + </func> <func> <name>prf(Socket, Secret, Label, Seed, WantedLength) -> {ok, binary()} | {error, reason()}</name> diff --git a/lib/ssl/doc/src/using_ssl.xml b/lib/ssl/doc/src/using_ssl.xml index c369c3c133..3ef33df719 100644 --- a/lib/ssl/doc/src/using_ssl.xml +++ b/lib/ssl/doc/src/using_ssl.xml @@ -153,7 +153,51 @@ ok</code> </section> </section> - <section> + <section> + <title>Customizing cipher suits</title> + + <p>Fetch default cipher suite list for an TLS/DTLS version. Change default + to all to get all possible cipher suites.</p> + <code type="erl">1> Default = ssl:cipher_suites(default, 'tlsv1.2'). + [#{cipher => aes_256_gcm,key_exchange => ecdhe_ecdsa, + mac => aead,prf => sha384}, ....] +</code> + + <p>In OTP 20 it is desirable to remove all cipher suites + that uses rsa kexchange (removed from default in 21) </p> + <code type="erl">2> NoRSA = + ssl:filter_cipher_suites(Default, + [{key_exchange, fun(rsa) -> false; + (_) -> true end}]). + [...] + </code> + + <p> Pick just a few suites </p> + <code type="erl"> 3> Suites = + ssl:filter_cipher_suites(Default, + [{key_exchange, fun(ecdh_ecdsa) -> true; + (_) -> false end}, + {cipher, fun(aes_128_cbc) ->true; + (_) ->false end}]). + [#{cipher => aes_128_cbc,key_exchange => ecdh_ecdsa, + mac => sha256,prf => sha256}, + #{cipher => aes_128_cbc,key_exchange => ecdh_ecdsa,mac => sha, + prf => default_prf}] + </code> + + <p> Make some particular suites the most preferred, or least + preferred by changing prepend to append.</p> + <code type="erl"> 4>ssl:prepend_cipher_suites(Suites, Default). + [#{cipher => aes_128_cbc,key_exchange => ecdh_ecdsa, + mac => sha256,prf => sha256}, + #{cipher => aes_128_cbc,key_exchange => ecdh_ecdsa,mac => sha, + prf => default_prf}, + #{cipher => aes_256_cbc,key_exchange => ecdhe_ecdsa, + mac => sha384,prf => sha384}, ...] + </code> + </section> + + <section> <title>Using an Engine Stored Key</title> <p>Erlang ssl application is able to use private keys provided diff --git a/lib/ssl/src/dtls_v1.erl b/lib/ssl/src/dtls_v1.erl index 51ee8ec047..0f6344b6f7 100644 --- a/lib/ssl/src/dtls_v1.erl +++ b/lib/ssl/src/dtls_v1.erl @@ -21,7 +21,7 @@ -include("ssl_cipher.hrl"). --export([suites/1, all_suites/1, hmac_hash/3, ecc_curves/1, +-export([suites/1, all_suites/1, anonymous_suites/1,hmac_hash/3, ecc_curves/1, corresponding_tls_version/1, corresponding_dtls_version/1, cookie_secret/0, cookie_timeout/0]). @@ -40,6 +40,12 @@ all_suites(Version) -> end, ssl_cipher:all_suites(corresponding_tls_version(Version))). +anonymous_suites(Version) -> + lists:filter(fun(Cipher) -> + is_acceptable_cipher(ssl_cipher:suite_definition(Cipher)) + end, + ssl_cipher:anonymous_suites(corresponding_tls_version(Version))). + hmac_hash(MacAlg, MacSecret, Value) -> tls_v1:hmac_hash(MacAlg, MacSecret, Value). diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index a298012f26..0b035d31be 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -39,7 +39,9 @@ ]). %% SSL/TLS protocol handling --export([cipher_suites/0, cipher_suites/1, cipher_suites/2, eccs/0, eccs/1, versions/0, +-export([cipher_suites/0, cipher_suites/1, cipher_suites/2, filter_cipher_suites/2, + prepend_cipher_suites/2, append_cipher_suites/2, + eccs/0, eccs/1, versions/0, format_error/1, renegotiate/1, prf/5, negotiated_protocol/1, connection_information/1, connection_information/2]). %% Misc @@ -379,35 +381,91 @@ negotiated_protocol(#sslsocket{pid = Pid}) -> cipher_suites() -> cipher_suites(erlang). %%-------------------------------------------------------------------- --spec cipher_suites(erlang | openssl | all) -> [ssl_cipher:old_erl_cipher_suite() | string()]. +-spec cipher_suites(erlang | openssl | all) -> + [ssl_cipher:old_erl_cipher_suite() | string()]. %% Description: Returns all supported cipher suites. %%-------------------------------------------------------------------- cipher_suites(erlang) -> - Version = tls_record:highest_protocol_version([]), - cipher_suites(erlang, Version); + [ssl_cipher:erl_suite_definition(Suite) || Suite <- available_suites(default)]; + cipher_suites(openssl) -> - Version = tls_record:highest_protocol_version([]), - cipher_suites(openssl, Version); + [ssl_cipher:openssl_suite_name(Suite) || + Suite <- available_suites(default)]; + cipher_suites(all) -> - Version = tls_record:highest_protocol_version([]), - cipher_suites(all, Version). + [ssl_cipher:erl_suite_definition(Suite) || Suite <- available_suites(all)]. %%-------------------------------------------------------------------- --spec cipher_suites(erlang | openssl | all, tls_record:tls_version() | - dtls_record:dtls_version()) -> [ssl_cipher:old_erl_cipher_suite() | string()]. -%% Description: Returns all supported cipher suites. +-spec cipher_suites(default | all | anonymous, tls_record:tls_version() | dtls_record:dtls_version() | + tls_record:tls_atom_version() | dtls_record:dtls_atom_version()) -> + [ssl_cipher:erl_cipher_suite()]. +%% Description: Returns all default and all supported cipher suites for a +%% TLS/DTLS version %%-------------------------------------------------------------------- -cipher_suites(Type, Version) when Version == 'dtlsv1'; - Version == 'dtlsv1.2' -> - cipher_suites(Type, dtls_record:protocol_version(Version)); -cipher_suites(Type, Version) when is_atom(Version) -> - cipher_suites(Type, tls_record:protocol_version(Version)); -cipher_suites(erlang, Version) -> - [ssl_cipher:erl_suite_definition(Suite) || Suite <- available_suites(default, Version)]; -cipher_suites(openssl, Version) -> - [ssl_cipher:openssl_suite_name(Suite) || Suite <- available_suites(default, Version)]; -cipher_suites(all, Version) -> - [ssl_cipher:erl_suite_definition(Suite) || Suite <- available_suites(all, Version)]. +cipher_suites(Base, Version) when Version == 'tlsv1.2'; + Version == 'tlsv1.1'; + Version == tlsv1; + Version == sslv3 -> + cipher_suites(Base, tls_record:protocol_version(Version)); +cipher_suites(Base, Version) when Version == 'dtlsv1.2'; + Version == 'dtlsv1'-> + cipher_suites(Base, dtls_record:protocol_version(Version)); +cipher_suites(Base, Version) -> + [ssl_cipher:suite_definition(Suite) || Suite <- supported_suites(Base, Version)]. + +%%-------------------------------------------------------------------- +-spec filter_cipher_suites([ssl_cipher:erl_cipher_suite()], + [{key_exchange | cipher | mac | prf, fun()}] | []) -> + [ssl_cipher:erl_cipher_suite()]. +%% Description: Removes cipher suites if any of the filter functions returns false +%% for any part of the cipher suite. This function also calls default filter functions +%% to make sure the cipher suite are supported by crypto. +%%-------------------------------------------------------------------- +filter_cipher_suites(Suites, Filters0) -> + #{key_exchange_filters := KexF, + cipher_filters := CipherF, + mac_filters := MacF, + prf_filters := PrfF} + = ssl_cipher:crypto_support_filters(), + Filters = #{key_exchange_filters => add_filter(proplists:get_value(key_exchange, Filters0), KexF), + cipher_filters => add_filter(proplists:get_value(cipher, Filters0), CipherF), + mac_filters => add_filter(proplists:get_value(mac, Filters0), MacF), + prf_filters => add_filter(proplists:get_value(prf, Filters0), PrfF)}, + ssl_cipher:filter_suites(Suites, Filters). +%%-------------------------------------------------------------------- +-spec prepend_cipher_suites([ssl_cipher:erl_cipher_suite()] | + [{key_exchange | cipher | mac | prf, fun()}], + [ssl_cipher:erl_cipher_suite()]) -> + [ssl_cipher:erl_cipher_suite()]. +%% Description: Make <Preferred> suites become the most prefered +%% suites that is put them at the head of the cipher suite list +%% and remove them from <Suites> if present. <Preferred> may be a +%% list of cipher suits or a list of filters in which case the +%% filters are use on Suites to extract the the preferred +%% cipher list. +%% -------------------------------------------------------------------- +prepend_cipher_suites([First | _] = Preferred, Suites0) when is_map(First) -> + Suites = Preferred ++ (Suites0 -- Preferred), + Suites; +prepend_cipher_suites(Filters, Suites) -> + Preferred = filter_cipher_suites(Suites, Filters), + Preferred ++ (Suites -- Preferred). +%%-------------------------------------------------------------------- +-spec append_cipher_suites(Deferred :: [ssl_cipher:erl_cipher_suite()] | + [{key_exchange | cipher | mac | prf, fun()}], + [ssl_cipher:erl_cipher_suite()]) -> + [ssl_cipher:erl_cipher_suite()]. +%% Description: Make <Deferred> suites suites become the +%% least prefered suites that is put them at the end of the cipher suite list +%% and removed them from <Suites> if present. +%% +%%-------------------------------------------------------------------- +append_cipher_suites([First | _] = Deferred, Suites0) when is_map(First)-> + Suites = (Suites0 -- Deferred) ++ Deferred, + Suites; +append_cipher_suites(Filters, Suites) -> + Deferred = filter_cipher_suites(Suites, Filters), + (Suites -- Deferred) ++ Deferred. %%-------------------------------------------------------------------- -spec eccs() -> tls_v1:curves(). @@ -661,14 +719,21 @@ tls_version({254, _} = Version) -> %%%-------------------------------------------------------------- %%% Internal functions %%%-------------------------------------------------------------------- - %% Possible filters out suites not supported by crypto -available_suites(default, Version) -> +available_suites(default) -> + Version = tls_record:highest_protocol_version([]), ssl_cipher:filter_suites(ssl_cipher:suites(Version)); - -available_suites(all, Version) -> +available_suites(all) -> + Version = tls_record:highest_protocol_version([]), ssl_cipher:filter_suites(ssl_cipher:all_suites(Version)). +supported_suites(default, Version) -> + ssl_cipher:suites(Version); +supported_suites(all, Version) -> + ssl_cipher:all_suites(Version); +supported_suites(anonymous, Version) -> + ssl_cipher:anonymous_suites(Version). + do_listen(Port, #config{transport_info = {Transport, _, _, _}} = Config, tls_connection) -> tls_socket:listen(Transport, Port, Config); @@ -1178,17 +1243,21 @@ handle_cipher_option(Value, Version) when is_list(Value) -> binary_cipher_suites(Version, []) -> %% Defaults to all supported suites that does %% not require explicit configuration - ssl_cipher:filter_suites(ssl_cipher:suites(tls_version(Version))); + default_binary_suites(Version); +binary_cipher_suites(Version, [Map|_] = Ciphers0) when is_map(Map) -> + Ciphers = [ssl_cipher:suite(C) || C <- Ciphers0], + binary_cipher_suites(Version, Ciphers); binary_cipher_suites(Version, [Tuple|_] = Ciphers0) when is_tuple(Tuple) -> Ciphers = [ssl_cipher:suite(tuple_to_map(C)) || C <- Ciphers0], binary_cipher_suites(Version, Ciphers); binary_cipher_suites(Version, [Cipher0 | _] = Ciphers0) when is_binary(Cipher0) -> - All = ssl_cipher:all_suites(tls_version(Version)), + All = ssl_cipher:all_suites(Version) ++ + ssl_cipher:anonymous_suites(Version), case [Cipher || Cipher <- Ciphers0, lists:member(Cipher, All)] of [] -> %% Defaults to all supported suites that does %% not require explicit configuration - ssl_cipher:filter_suites(ssl_cipher:suites(tls_version(Version))); + default_binary_suites(Version); Ciphers -> Ciphers end; @@ -1201,6 +1270,9 @@ binary_cipher_suites(Version, Ciphers0) -> Ciphers = [ssl_cipher:openssl_suite(C) || C <- string:lexemes(Ciphers0, ":")], binary_cipher_suites(Version, Ciphers). +default_binary_suites(Version) -> + ssl_cipher:filter_suites(ssl_cipher:suites(Version)). + tuple_to_map({Kex, Cipher, Mac}) -> #{key_exchange => Kex, cipher => Cipher, @@ -1209,9 +1281,19 @@ tuple_to_map({Kex, Cipher, Mac}) -> tuple_to_map({Kex, Cipher, Mac, Prf}) -> #{key_exchange => Kex, cipher => Cipher, - mac => Mac, + mac => tuple_to_map_mac(Cipher, Mac), prf => Prf}. +%% Backwards compatible +tuple_to_map_mac(aes_128_gcm, _) -> + aead; +tuple_to_map_mac(aes_256_gcm, _) -> + aead; +tuple_to_map_mac(chacha20_poly1305, _) -> + aead; +tuple_to_map_mac(_, MAC) -> + MAC. + handle_eccs_option(Value, Version) when is_list(Value) -> {_Major, Minor} = tls_version(Version), try tls_v1:ecc_curves(Minor, Value) of @@ -1490,3 +1572,8 @@ reject_alpn_next_prot_options([Opt| AlpnNextOpts], Opts) -> false -> reject_alpn_next_prot_options(AlpnNextOpts, Opts) end. + +add_filter(undefined, Filters) -> + Filters; +add_filter(Filter, Filters) -> + [Filter | Filters]. diff --git a/lib/ssl/src/ssl_cipher.erl b/lib/ssl/src/ssl_cipher.erl index 7c5cff3665..a83ce42455 100644 --- a/lib/ssl/src/ssl_cipher.erl +++ b/lib/ssl/src/ssl_cipher.erl @@ -36,9 +36,11 @@ -export([security_parameters/2, security_parameters/3, suite_definition/1, erl_suite_definition/1, cipher_init/3, decipher/6, cipher/5, decipher_aead/6, cipher_aead/6, - suite/1, suites/1, all_suites/1, - ec_keyed_suites/0, chacha_suites/1, anonymous_suites/1, psk_suites/1, srp_suites/0, - rc4_suites/1, des_suites/1, rsa_suites/1, openssl_suite/1, openssl_suite_name/1, filter/2, filter_suites/1, + suite/1, suites/1, all_suites/1, crypto_support_filters/0, + ec_keyed_suites/0, chacha_suites/1, anonymous_suites/1, psk_suites/1, psk_suites_anon/1, + srp_suites/0, srp_suites_anon/0, + rc4_suites/1, des_suites/1, rsa_suites/1, openssl_suite/1, openssl_suite_name/1, + filter/2, filter_suites/1, filter_suites/2, hash_algorithm/1, sign_algorithm/1, is_acceptable_hash/2, is_fallback/1, random_bytes/1, calc_mac_hash/4, is_stream_ciphersuite/1]). @@ -53,7 +55,7 @@ -type key_algo() :: null | rsa | dhe_rsa | dhe_dss | ecdhe_ecdsa| ecdh_ecdsa | ecdh_rsa| srp_rsa| srp_dss | psk | dhe_psk | rsa_psk | dh_anon | ecdh_anon | srp_anon. -type erl_cipher_suite() :: #{key_exchange := key_algo(), cipher := cipher(), - mac := hash(), + mac := hash() | aead, prf := hash() | default_prf %% Old cipher suites, version dependent }. -type old_erl_cipher_suite() :: {key_algo(), cipher(), hash()} % Pre TLS 1.2 @@ -94,7 +96,7 @@ security_parameters(Version, CipherSuite, SecParams) -> expanded_key_material_length = expanded_key_material(Cipher), key_material_length = key_material(Cipher), iv_size = iv_size(Cipher), - mac_algorithm = hash_algorithm(Hash), + mac_algorithm = mac_algorithm(Hash), prf_algorithm = prf_algorithm(PrfHashAlg, Version), hash_size = hash_size(Hash)}. @@ -321,12 +323,12 @@ suites({_, Minor}) -> all_suites({3, _} = Version) -> suites(Version) ++ chacha_suites(Version) - ++ anonymous_suites(Version) ++ psk_suites(Version) ++ srp_suites() ++ rc4_suites(Version) ++ des_suites(Version) ++ rsa_suites(Version); + all_suites(Version) -> dtls_v1:all_suites(Version). %%-------------------------------------------------------------------- @@ -350,12 +352,12 @@ chacha_suites(_) -> %% if explicitly set by user. Intended only for testing. %%-------------------------------------------------------------------- anonymous_suites({3, N}) -> - anonymous_suites(N); + srp_suites_anon() ++ anonymous_suites(N); anonymous_suites({254, _} = Version) -> - anonymous_suites(dtls_v1:corresponding_tls_version(Version)) - -- [?TLS_DH_anon_WITH_RC4_128_MD5]; + dtls_v1:anonymous_suites(Version); anonymous_suites(N) when N >= 3 -> + psk_suites_anon(N) ++ [?TLS_DH_anon_WITH_AES_128_GCM_SHA256, ?TLS_DH_anon_WITH_AES_256_GCM_SHA384, ?TLS_DH_anon_WITH_AES_128_CBC_SHA256, @@ -364,20 +366,20 @@ anonymous_suites(N) ?TLS_ECDH_anon_WITH_AES_256_CBC_SHA, ?TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA, ?TLS_DH_anon_WITH_RC4_128_MD5]; - -anonymous_suites(2) -> +anonymous_suites(2 = N) -> + psk_suites_anon(N) ++ [?TLS_ECDH_anon_WITH_AES_128_CBC_SHA, ?TLS_ECDH_anon_WITH_AES_256_CBC_SHA, ?TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA, ?TLS_DH_anon_WITH_DES_CBC_SHA, ?TLS_DH_anon_WITH_RC4_128_MD5]; - anonymous_suites(N) when N == 0; N == 1 -> - [?TLS_DH_anon_WITH_RC4_128_MD5, - ?TLS_DH_anon_WITH_3DES_EDE_CBC_SHA, - ?TLS_DH_anon_WITH_DES_CBC_SHA - ]. + psk_suites_anon(N) ++ + [?TLS_DH_anon_WITH_RC4_128_MD5, + ?TLS_DH_anon_WITH_3DES_EDE_CBC_SHA, + ?TLS_DH_anon_WITH_DES_CBC_SHA + ]. %%-------------------------------------------------------------------- -spec psk_suites(ssl_record:ssl_version() | integer()) -> [cipher_suite()]. @@ -390,41 +392,52 @@ psk_suites({3, N}) -> psk_suites(N) when N >= 3 -> [ - ?TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384, - ?TLS_DHE_PSK_WITH_AES_256_GCM_SHA384, ?TLS_RSA_PSK_WITH_AES_256_GCM_SHA384, + ?TLS_RSA_PSK_WITH_AES_256_CBC_SHA384, + ?TLS_RSA_PSK_WITH_AES_128_GCM_SHA256, + ?TLS_RSA_PSK_WITH_AES_128_CBC_SHA256 + ] ++ psk_suites(0); +psk_suites(_) -> + [?TLS_RSA_PSK_WITH_AES_256_CBC_SHA, + ?TLS_RSA_PSK_WITH_AES_128_CBC_SHA, + ?TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA, + ?TLS_RSA_PSK_WITH_RC4_128_SHA]. + +%%-------------------------------------------------------------------- +-spec psk_suites_anon(ssl_record:ssl_version() | integer()) -> [cipher_suite()]. +%% +%% Description: Returns a list of the anonymous PSK cipher suites, only supported +%% if explicitly set by user. +%%-------------------------------------------------------------------- +psk_suites_anon({3, N}) -> + psk_suites_anon(N); +psk_suites_anon(N) + when N >= 3 -> + [ + ?TLS_DHE_PSK_WITH_AES_256_GCM_SHA384, ?TLS_PSK_WITH_AES_256_GCM_SHA384, ?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384, ?TLS_DHE_PSK_WITH_AES_256_CBC_SHA384, - ?TLS_RSA_PSK_WITH_AES_256_CBC_SHA384, ?TLS_PSK_WITH_AES_256_CBC_SHA384, ?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256, ?TLS_DHE_PSK_WITH_AES_128_GCM_SHA256, - ?TLS_RSA_PSK_WITH_AES_128_GCM_SHA256, ?TLS_PSK_WITH_AES_128_GCM_SHA256, ?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256, ?TLS_DHE_PSK_WITH_AES_128_CBC_SHA256, - ?TLS_RSA_PSK_WITH_AES_128_CBC_SHA256, ?TLS_PSK_WITH_AES_128_CBC_SHA256 - ] ++ psk_suites(0); -psk_suites(_) -> - [?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA, - ?TLS_DHE_PSK_WITH_AES_256_CBC_SHA, - ?TLS_RSA_PSK_WITH_AES_256_CBC_SHA, + ] ++ psk_suites_anon(0); +psk_suites_anon(_) -> + [?TLS_DHE_PSK_WITH_AES_256_CBC_SHA, ?TLS_PSK_WITH_AES_256_CBC_SHA, ?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA, ?TLS_DHE_PSK_WITH_AES_128_CBC_SHA, - ?TLS_RSA_PSK_WITH_AES_128_CBC_SHA, ?TLS_PSK_WITH_AES_128_CBC_SHA, ?TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA, ?TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA, - ?TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA, ?TLS_PSK_WITH_3DES_EDE_CBC_SHA, ?TLS_ECDHE_PSK_WITH_RC4_128_SHA, ?TLS_DHE_PSK_WITH_RC4_128_SHA, - ?TLS_RSA_PSK_WITH_RC4_128_SHA, ?TLS_PSK_WITH_RC4_128_SHA]. - %%-------------------------------------------------------------------- -spec srp_suites() -> [cipher_suite()]. %% @@ -432,15 +445,24 @@ psk_suites(_) -> %% if explicitly set by user. %%-------------------------------------------------------------------- srp_suites() -> - [?TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA, - ?TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA, + [?TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA, ?TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA, - ?TLS_SRP_SHA_WITH_AES_128_CBC_SHA, ?TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA, ?TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA, - ?TLS_SRP_SHA_WITH_AES_256_CBC_SHA, ?TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA, ?TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA]. + +%%-------------------------------------------------------------------- +-spec srp_suites_anon() -> [cipher_suite()]. +%% +%% Description: Returns a list of the SRP anonymous cipher suites, only supported +%% if explicitly set by user. +%%-------------------------------------------------------------------- +srp_suites_anon() -> + [?TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA, + ?TLS_SRP_SHA_WITH_AES_128_CBC_SHA, + ?TLS_SRP_SHA_WITH_AES_256_CBC_SHA]. + %%-------------------------------------------------------------------- -spec rc4_suites(Version::ssl_record:ssl_version() | integer()) -> [cipher_suite()]. %% @@ -750,32 +772,32 @@ suite_definition(?TLS_RSA_PSK_WITH_NULL_SHA) -> suite_definition(?TLS_PSK_WITH_AES_128_GCM_SHA256) -> #{key_exchange => psk, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_PSK_WITH_AES_256_GCM_SHA384) -> #{key_exchange => psk, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_DHE_PSK_WITH_AES_128_GCM_SHA256) -> #{key_exchange => dhe_psk, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_DHE_PSK_WITH_AES_256_GCM_SHA384) -> #{key_exchange => dhe_psk, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_RSA_PSK_WITH_AES_128_GCM_SHA256) -> #{key_exchange => rsa_psk, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_RSA_PSK_WITH_AES_256_GCM_SHA384) -> #{key_exchange => rsa_psk, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_PSK_WITH_AES_128_CBC_SHA256) -> #{key_exchange => psk, @@ -1115,42 +1137,42 @@ suite_definition(?TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384) -> suite_definition(?TLS_RSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => rsa, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_RSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => rsa, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_DHE_RSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => dhe_rsa, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_DHE_RSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => dhe_rsa, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_DH_RSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => dh_rsa, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_DH_RSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => dh_rsa, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_DHE_DSS_WITH_AES_128_GCM_SHA256) -> #{key_exchange => dhe_dss, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_DHE_DSS_WITH_AES_256_GCM_SHA384) -> #{key_exchange => dhe_dss, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_DH_DSS_WITH_AES_128_GCM_SHA256) -> #{key_exchange => dh_dss, @@ -1160,74 +1182,74 @@ suite_definition(?TLS_DH_DSS_WITH_AES_128_GCM_SHA256) -> suite_definition(?TLS_DH_DSS_WITH_AES_256_GCM_SHA384) -> #{key_exchange => dh_dss, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_DH_anon_WITH_AES_128_GCM_SHA256) -> #{key_exchange => dh_anon, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_DH_anon_WITH_AES_256_GCM_SHA384) -> #{key_exchange => dh_anon, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; %% RFC 5289 ECC AES-GCM Cipher Suites suite_definition(?TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => ecdhe_ecdsa, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => ecdhe_ecdsa, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => ecdh_ecdsa, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => ecdh_ecdsa, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => ecdhe_rsa, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => ecdhe_rsa, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => ecdh_rsa, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => ecdh_rsa, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; %% draft-agl-tls-chacha20poly1305-04 Chacha20/Poly1305 Suites suite_definition(?TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256) -> #{key_exchange => ecdhe_rsa, cipher => chacha20_poly1305, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256) -> #{key_exchange => ecdhe_ecdsa, cipher => chacha20_poly1305, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256) -> #{key_exchange => dhe_rsa, cipher => chacha20_poly1305, - mac => null, + mac => aead, prf => sha256}. %%-------------------------------------------------------------------- @@ -1428,32 +1450,32 @@ suite(#{key_exchange := rsa_psk, %%% TLS 1.2 PSK Cipher Suites RFC 5487 suite(#{key_exchange := psk, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_PSK_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := psk, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_PSK_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := dhe_psk, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_DHE_PSK_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := dhe_psk, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_DHE_PSK_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := rsa_psk, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_RSA_PSK_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := rsa_psk, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_RSA_PSK_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := psk, @@ -1739,119 +1761,119 @@ suite(#{key_exchange := ecdh_rsa, %% RFC 5288 AES-GCM Cipher Suites suite(#{key_exchange := rsa, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_RSA_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := rsa, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_RSA_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := dhe_rsa, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_DHE_RSA_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := dhe_rsa, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_DHE_RSA_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := dh_rsa, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_DH_RSA_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := dh_rsa, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_DH_RSA_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := dhe_dss, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_DHE_DSS_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := dhe_dss, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_DHE_DSS_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := dh_dss, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_DH_DSS_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := dh_dss, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_DH_DSS_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := dh_anon, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_DH_anon_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := dh_anon, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_DH_anon_WITH_AES_256_GCM_SHA384; %% RFC 5289 ECC AES-GCM Cipher Suites suite(#{key_exchange := ecdhe_ecdsa, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := ecdhe_ecdsa, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := ecdh_ecdsa, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := ecdh_ecdsa, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := ecdhe_rsa, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := ecdhe_rsa, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := ecdh_rsa, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := ecdh_rsa, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384; %% draft-agl-tls-chacha20poly1305-04 Chacha20/Poly1305 Suites suite(#{key_exchange := ecdhe_rsa, cipher := chacha20_poly1305, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256; suite(#{key_exchange := ecdhe_ecdsa, cipher := chacha20_poly1305, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256; suite(#{key_exchange := dhe_rsa, cipher := chacha20_poly1305, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256. @@ -2011,9 +2033,9 @@ openssl_suite("ECDH-RSA-AES256-GCM-SHA384") -> ?TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384. %%-------------------------------------------------------------------- --spec openssl_suite_name(cipher_suite()) -> openssl_cipher_suite(). +-spec openssl_suite_name(cipher_suite()) -> openssl_cipher_suite() | erl_cipher_suite(). %% -%% Description: Return openssl cipher suite name. +%% Description: Return openssl cipher suite name if possible %%------------------------------------------------------------------- openssl_suite_name(?TLS_DHE_RSA_WITH_AES_256_CBC_SHA) -> "DHE-RSA-AES256-SHA"; @@ -2223,38 +2245,74 @@ filter(DerCert, Ciphers) -> {_, ecdsa} -> Ciphers1 -- rsa_signed_suites() end. - %%-------------------------------------------------------------------- --spec filter_suites([cipher_suite()]) -> [cipher_suite()]. +-spec filter_suites([erl_cipher_suite()] | [cipher_suite()], map()) -> + [erl_cipher_suite()] | [cipher_suite()]. +%% +%% Description: Filter suites using supplied filter funs +%%------------------------------------------------------------------- +filter_suites(Suites, Filters) -> + ApplyFilters = fun(Suite) -> + filter_suite(Suite, Filters) + end, + lists:filter(ApplyFilters, Suites). + +filter_suite(#{key_exchange := KeyExchange, + cipher := Cipher, + mac := Hash, + prf := Prf}, + #{key_exchange_filters := KeyFilters, + cipher_filters := CipherFilters, + mac_filters := HashFilters, + prf_filters := PrfFilters}) -> + all_filters(KeyExchange, KeyFilters) andalso + all_filters(Cipher, CipherFilters) andalso + all_filters(Hash, HashFilters) andalso + all_filters(Prf, PrfFilters); +filter_suite(Suite, Filters) -> + filter_suite(suite_definition(Suite), Filters). + +%%-------------------------------------------------------------------- +-spec filter_suites([erl_cipher_suite()] | [cipher_suite()]) -> + [erl_cipher_suite()] | [cipher_suite()]. %% %% Description: Filter suites for algorithms supported by crypto. %%------------------------------------------------------------------- -filter_suites(Suites = [Value|_]) when is_map(Value) -> - Algos = crypto:supports(), - Hashs = proplists:get_value(hashs, Algos), - lists:filter(fun(#{key_exchange := KeyExchange, - cipher := Cipher, - mac := Hash, - prf := Prf}) -> - is_acceptable_keyexchange(KeyExchange, proplists:get_value(public_keys, Algos)) andalso - is_acceptable_cipher(Cipher, proplists:get_value(ciphers, Algos)) andalso - is_acceptable_hash(Hash, Hashs) andalso - is_acceptable_prf(Prf, Hashs) - end, Suites); - filter_suites(Suites) -> + Filters = crypto_support_filters(), + filter_suites(Suites, Filters). + +all_filters(_, []) -> + true; +all_filters(Value, [Filter| Rest]) -> + case Filter(Value) of + true -> + all_filters(Value, Rest); + false -> + false + end. +crypto_support_filters() -> Algos = crypto:supports(), Hashs = proplists:get_value(hashs, Algos), - lists:filter(fun(Suite) -> - #{key_exchange := KeyExchange, - cipher := Cipher, - mac := Hash, - prf := Prf} = suite_definition(Suite), - is_acceptable_keyexchange(KeyExchange, proplists:get_value(public_keys, Algos)) andalso - is_acceptable_cipher(Cipher, proplists:get_value(ciphers, Algos)) andalso - is_acceptable_hash(Hash, Hashs) andalso - is_acceptable_prf(Prf, Hashs) - end, Suites). + #{key_exchange_filters => + [fun(KeyExchange) -> + is_acceptable_keyexchange(KeyExchange, + proplists:get_value(public_keys, Algos)) + end], + cipher_filters => + [fun(Cipher) -> + is_acceptable_cipher(Cipher, + proplists:get_value(ciphers, Algos)) + end], + mac_filters => + [fun(Hash) -> + is_acceptable_hash(Hash, Hashs) + end], + prf_filters => + [fun(Prf) -> + is_acceptable_prf(Prf, + proplists:get_value(hashs, Algos)) + end]}. is_acceptable_keyexchange(KeyExchange, _Algos) when KeyExchange == psk; KeyExchange == null -> @@ -2473,6 +2531,11 @@ prf_algorithm(default_prf, {3, _}) -> prf_algorithm(Algo, _) -> hash_algorithm(Algo). +mac_algorithm(aead) -> + aead; +mac_algorithm(Algo) -> + hash_algorithm(Algo). + hash_algorithm(null) -> ?NULL; hash_algorithm(md5) -> ?MD5; hash_algorithm(sha) -> ?SHA; %% Only sha always refers to "SHA-1" @@ -2503,6 +2566,10 @@ sign_algorithm(Other) when is_integer(Other) andalso ((Other >= 224) and (Other hash_size(null) -> 0; +%% The AEAD MAC hash size is not used in the context +%% of calculating the master secret. See RFC 5246 Section 6.2.3.3. +hash_size(aead) -> + 0; hash_size(md5) -> 16; hash_size(sha) -> diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index d046145dff..2031735a71 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -1148,8 +1148,8 @@ handle_common_event(internal, #change_cipher_spec{type = <<1>>}, StateName, StateName, State); handle_common_event(_Type, Msg, StateName, #state{negotiated_version = Version} = State, _) -> - Alert = ?ALERT_REC(?FATAL,?UNEXPECTED_MESSAGE), - handle_own_alert(Alert, Version, {StateName, Msg}, State). + Alert = ?ALERT_REC(?FATAL,?UNEXPECTED_MESSAGE, {unexpected_msg, Msg}), + handle_own_alert(Alert, Version, StateName, State). handle_call({application_data, _Data}, _, _, _, _) -> %% In renegotiation priorities handshake, send data when handshake is finished diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index 0c55af9174..09160e2f9c 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -774,9 +774,8 @@ decode_suites('3_bytes', Dec) -> %%==================================================================== available_suites(UserSuites, Version) -> - lists:filtermap(fun(Suite) -> - lists:member(Suite, ssl_cipher:all_suites(Version)) - end, UserSuites). + VersionSuites = ssl_cipher:all_suites(Version) ++ ssl_cipher:anonymous_suites(Version), + lists:filtermap(fun(Suite) -> lists:member(Suite, VersionSuites) end, UserSuites). available_suites(ServerCert, UserSuites, Version, undefined, Curve) -> ssl_cipher:filter(ServerCert, available_suites(UserSuites, Version)) @@ -1056,7 +1055,9 @@ select_curve(undefined, _, _) -> %% %% Description: Handles signature_algorithms hello extension (server) %%-------------------------------------------------------------------- -select_hashsign(_, undefined, _, _, _Version) -> +select_hashsign(_, _, KeyExAlgo, _, _Version) when KeyExAlgo == dh_anon; + KeyExAlgo == ecdh_anon; + KeyExAlgo == srp_anon -> {null, anon}; %% The signature_algorithms extension was introduced with TLS 1.2. Ignore it if we have %% negotiated a lower version. diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index f13bd53a7c..2781203557 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -163,7 +163,8 @@ api_tests() -> server_name_indication_option, accept_pool, prf, - socket_options + socket_options, + cipher_suites ]. api_tests_tls() -> @@ -207,7 +208,7 @@ tls_cipher_tests() -> rc4_ecdsa_cipher_suites]. cipher_tests() -> - [cipher_suites, + [old_cipher_suites, cipher_suites_mix, ciphers_rsa_signed_certs, ciphers_rsa_signed_certs_openssl_names, @@ -704,8 +705,6 @@ secret_connection_info(Config) when is_list(Config) -> ct:log("Testcase ~p, Client ~p Server ~p ~n", [self(), Client, Server]), - - Version = ssl_test_lib:protocol_version(Config), ssl_test_lib:check_result(Server, true, Client, true), @@ -1130,11 +1129,16 @@ fallback(Config) when is_list(Config) -> %%-------------------------------------------------------------------- cipher_format() -> - [{doc, "Test that cipher conversion from tuples to binarys works"}]. + [{doc, "Test that cipher conversion from maps | tuples | stings to binarys works"}]. cipher_format(Config) when is_list(Config) -> - {ok, Socket} = ssl:listen(0, [{ciphers, ssl:cipher_suites()}]), - ssl:close(Socket). - + {ok, Socket0} = ssl:listen(0, [{ciphers, ssl:cipher_suites(default, 'tlsv1.2')}]), + ssl:close(Socket0), + %% Legacy + {ok, Socket1} = ssl:listen(0, [{ciphers, ssl:cipher_suites()}]), + ssl:close(Socket1), + {ok, Socket2} = ssl:listen(0, [{ciphers, ssl:cipher_suites(openssl)}]), + ssl:close(Socket2). + %%-------------------------------------------------------------------- peername() -> @@ -1285,20 +1289,76 @@ sockname_result(S) -> ssl:sockname(S). %%-------------------------------------------------------------------- + cipher_suites() -> - [{doc,"Test API function cipher_suites/0"}]. + [{doc,"Test API function cipher_suites/2, filter_cipher_suites/2" + " and prepend|append_cipher_suites/2"}]. cipher_suites(Config) when is_list(Config) -> - MandatoryCipherSuiteTLS1_0TLS1_1 = {rsa,'3des_ede_cbc',sha}, - MandatoryCipherSuiteTLS1_0TLS1_2 = {rsa,'aes_128_cbc',sha} , - [_|_] = Suites = ssl:cipher_suites(), - AllSuites = ssl:cipher_suites(all), - %% The mandantory suites will no longer be supported by default - %% due to security reasons - true = lists:member(MandatoryCipherSuiteTLS1_0TLS1_1, AllSuites), - true = lists:member(MandatoryCipherSuiteTLS1_0TLS1_2, AllSuites), + MandatoryCipherSuiteTLS1_0TLS1_1 = #{key_exchange => rsa, + cipher => '3des_ede_cbc', + mac => sha, + prf => default_prf}, + MandatoryCipherSuiteTLS1_0TLS1_2 = #{key_exchange =>rsa, + cipher => 'aes_128_cbc', + mac => sha, + prf => default_prf}, + Version = ssl_test_lib:protocol_version(Config), + All = [_|_] = ssl:cipher_suites(all, Version), + Default = [_|_] = ssl:cipher_suites(default, Version), + Anonymous = [_|_] = ssl:cipher_suites(anonymous, Version), + true = length(Default) < length(All), + Filters = [{key_exchange, + fun(dhe_rsa) -> + true; + (_) -> + false + end + }, + {cipher, + fun(aes_256_cbc) -> + true; + (_) -> + false + end + }, + {mac, + fun(sha) -> + true; + (_) -> + false + end + } + ], + Cipher = #{cipher => aes_256_cbc, + key_exchange => dhe_rsa, + mac => sha, + prf => default_prf}, + [Cipher] = ssl:filter_cipher_suites(All, Filters), + [Cipher | Rest0] = ssl:prepend_cipher_suites([Cipher], Default), + [Cipher | Rest0] = ssl:prepend_cipher_suites(Filters, Default), + true = lists:member(Cipher, Default), + false = lists:member(Cipher, Rest0), + [Cipher | Rest1] = lists:reverse(ssl:append_cipher_suites([Cipher], Default)), + [Cipher | Rest1] = lists:reverse(ssl:append_cipher_suites(Filters, Default)), + true = lists:member(Cipher, Default), + false = lists:member(Cipher, Rest1), + [] = lists:dropwhile(fun(X) -> not lists:member(X, Default) end, Anonymous), + [] = lists:dropwhile(fun(X) -> not lists:member(X, All) end, Anonymous), + true = lists:member(MandatoryCipherSuiteTLS1_0TLS1_1, All), + true = lists:member(MandatoryCipherSuiteTLS1_0TLS1_2, All). + +%%-------------------------------------------------------------------- + +old_cipher_suites() -> + [{doc,"Test API function cipher_suites/0"}]. + +old_cipher_suites(Config) when is_list(Config) -> + MandatoryCipherSuite = {rsa, '3des_ede_cbc', sha}, + [_|_] = Suites = ssl:cipher_suites(), Suites = ssl:cipher_suites(erlang), - [_|_] =ssl:cipher_suites(openssl). + [_|_] = ssl:cipher_suites(openssl), + true = lists:member(MandatoryCipherSuite, ssl:cipher_suites(all)). %%-------------------------------------------------------------------- cipher_suites_mix() -> @@ -3800,9 +3860,23 @@ rizzo() -> vunrable to Rizzo/Dungon attack"}]. rizzo(Config) when is_list(Config) -> - Ciphers = [X || X ={_,Y,_} <- ssl:cipher_suites(), Y =/= rc4_128], Prop = proplists:get_value(tc_group_properties, Config), Version = proplists:get_value(name, Prop), + NVersion = ssl_test_lib:protocol_version(Config, tuple), + Ciphers = ssl:filter_cipher_suites(ssl:cipher_suites(all, NVersion), + [{key_exchange, + fun(Alg) when Alg == ecdh_rsa; Alg == ecdhe_rsa-> + true; + (_) -> + false + end}, + {cipher, + fun(rc4_128) -> + false; + (_) -> + true + end}]), + run_send_recv_rizzo(Ciphers, Config, Version, {?MODULE, send_recv_result_active_rizzo, []}). %%-------------------------------------------------------------------- @@ -3814,8 +3888,13 @@ no_rizzo_rc4(Config) when is_list(Config) -> Version = proplists:get_value(name, Prop), NVersion = ssl_test_lib:protocol_version(Config, tuple), %% Test uses RSA certs - Ciphers = ssl_test_lib:rc4_suites(NVersion) -- [{ecdhe_ecdsa,rc4_128,sha}, - {ecdh_ecdsa,rc4_128,sha}], + Ciphers = ssl:filter_cipher_suites(ssl_test_lib:rc4_suites(NVersion), + [{key_exchange, + fun(Alg) when Alg == ecdh_rsa; Alg == ecdhe_rsa-> + true; + (_) -> + false + end}]), run_send_recv_rizzo(Ciphers, Config, Version, {?MODULE, send_recv_result_active_no_rizzo, []}). @@ -3826,10 +3905,21 @@ rizzo_one_n_minus_one(Config) when is_list(Config) -> Prop = proplists:get_value(tc_group_properties, Config), Version = proplists:get_value(name, Prop), NVersion = ssl_test_lib:protocol_version(Config, tuple), - AllSuites = ssl_test_lib:available_suites(NVersion), - Ciphers = [X || X ={_,Y,_} <- AllSuites, Y =/= rc4_128], + Ciphers = ssl:filter_cipher_suites(ssl:cipher_suites(all, NVersion), + [{key_exchange, + fun(Alg) when Alg == ecdh_rsa; Alg == ecdhe_rsa-> + true; + (_) -> + false + end}, + {cipher, + fun(rc4_128) -> + false; + (_) -> + true + end}]), run_send_recv_rizzo(Ciphers, Config, Version, - {?MODULE, send_recv_result_active_rizzo, []}). + {?MODULE, send_recv_result_active_rizzo, []}). rizzo_zero_n() -> [{doc,"Test that the 0/n-split mitigation of Rizzo/Dungon attack can be explicitly selected"}]. @@ -3838,8 +3928,13 @@ rizzo_zero_n(Config) when is_list(Config) -> Prop = proplists:get_value(tc_group_properties, Config), Version = proplists:get_value(name, Prop), NVersion = ssl_test_lib:protocol_version(Config, tuple), - AllSuites = ssl_test_lib:available_suites(NVersion), - Ciphers = [X || X ={_,Y,_} <- AllSuites, Y =/= rc4_128], + Ciphers = ssl:filter_cipher_suites(ssl:cipher_suites(default, NVersion), + [{cipher, + fun(rc4_128) -> + false; + (_) -> + true + end}]), run_send_recv_rizzo(Ciphers, Config, Version, {?MODULE, send_recv_result_active_no_rizzo, []}). @@ -3847,9 +3942,16 @@ rizzo_disabled() -> [{doc,"Test that the mitigation of Rizzo/Dungon attack can be explicitly disabled"}]. rizzo_disabled(Config) when is_list(Config) -> - Ciphers = [X || X ={_,Y,_} <- ssl:cipher_suites(), Y =/= rc4_128], Prop = proplists:get_value(tc_group_properties, Config), Version = proplists:get_value(name, Prop), + NVersion = ssl_test_lib:protocol_version(Config, tuple), + Ciphers = ssl:filter_cipher_suites(ssl:cipher_suites(default, NVersion), + [{cipher, + fun(rc4_128) -> + false; + (_) -> + true + end}]), run_send_recv_rizzo(Ciphers, Config, Version, {?MODULE, send_recv_result_active_no_rizzo, []}). @@ -4624,19 +4726,21 @@ rizzo_test(Cipher, Config, Version, Mfa) -> [{Cipher, Error}] end. -client_server_opts({KeyAlgo,_,_}, Config) +client_server_opts(#{key_exchange := KeyAlgo}, Config) when KeyAlgo == rsa orelse KeyAlgo == dhe_rsa orelse - KeyAlgo == ecdhe_rsa -> + KeyAlgo == ecdhe_rsa orelse + KeyAlgo == rsa_psk orelse + KeyAlgo == srp_rsa -> {ssl_test_lib:ssl_options(client_opts, Config), ssl_test_lib:ssl_options(server_opts, Config)}; -client_server_opts({KeyAlgo,_,_}, Config) when KeyAlgo == dss orelse KeyAlgo == dhe_dss -> +client_server_opts(#{key_exchange := KeyAlgo}, Config) when KeyAlgo == dss orelse KeyAlgo == dhe_dss -> {ssl_test_lib:ssl_options(client_dsa_opts, Config), ssl_test_lib:ssl_options(server_dsa_opts, Config)}; -client_server_opts({KeyAlgo,_,_}, Config) when KeyAlgo == ecdh_ecdsa orelse KeyAlgo == ecdhe_ecdsa -> +client_server_opts(#{key_exchange := KeyAlgo}, Config) when KeyAlgo == ecdh_ecdsa orelse KeyAlgo == ecdhe_ecdsa -> {ssl_test_lib:ssl_options(client_opts, Config), ssl_test_lib:ssl_options(server_ecdsa_opts, Config)}; -client_server_opts({KeyAlgo,_,_}, Config) when KeyAlgo == ecdh_rsa -> +client_server_opts(#{key_exchange := KeyAlgo}, Config) when KeyAlgo == ecdh_rsa -> {ssl_test_lib:ssl_options(client_opts, Config), ssl_test_lib:ssl_options(server_ecdh_rsa_opts, Config)}. diff --git a/lib/ssl/test/ssl_engine_SUITE.erl b/lib/ssl/test/ssl_engine_SUITE.erl index bc221d35fd..71891356e8 100644 --- a/lib/ssl/test/ssl_engine_SUITE.erl +++ b/lib/ssl/test/ssl_engine_SUITE.erl @@ -39,23 +39,28 @@ init_per_suite(Config) -> catch crypto:stop(), try crypto:start() of ok -> - ssl_test_lib:clean_start(), - case crypto:get_test_engine() of - {ok, EngineName} -> - try crypto:engine_load(<<"dynamic">>, - [{<<"SO_PATH">>, EngineName}, - <<"LOAD">>], - []) of - {ok, Engine} -> - [{engine, Engine} |Config]; - {error, Reason} -> - ct:pal("Reason ~p", [Reason]), - {skip, "No dynamic engine support"} - catch error:notsup -> - {skip, "No engine support in OpenSSL"} - end; - {error, notexist} -> - {skip, "Test engine not found"} + case crypto:info_lib() of + [{_,_, <<"OpenSSL 1.0.1s-freebsd 1 Mar 2016">>}] -> + {skip, "Problem with engine on OpenSSL 1.0.1s-freebsd"}; + _ -> + ssl_test_lib:clean_start(), + case crypto:get_test_engine() of + {ok, EngineName} -> + try crypto:engine_load(<<"dynamic">>, + [{<<"SO_PATH">>, EngineName}, + <<"LOAD">>], + []) of + {ok, Engine} -> + [{engine, Engine} |Config]; + {error, Reason} -> + ct:pal("Reason ~p", [Reason]), + {skip, "No dynamic engine support"} + catch error:notsup -> + {skip, "No engine support in OpenSSL"} + end; + {error, notexist} -> + {skip, "Test engine not found"} + end end catch _:_ -> {skip, "Crypto did not start"} diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index 94d10b2f9b..26ef311615 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -1024,59 +1024,50 @@ string_regex_filter(Str, Search) when is_list(Str) -> string_regex_filter(_Str, _Search) -> false. -anonymous_suites({3,_ } = Version) -> - [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:filter_suites(ssl_cipher:anonymous_suites(Version))]; -anonymous_suites(DTLSVersion) -> - Version = dtls_v1:corresponding_tls_version(DTLSVersion), - [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:filter_suites(ssl_cipher:anonymous_suites(Version)), - not ssl_cipher:is_stream_ciphersuite(tuple_to_map(ssl_cipher:erl_suite_definition(S)))]. - -psk_suites({3,_ } = Version) -> - [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:filter_suites(ssl_cipher:psk_suites(Version))]; -psk_suites(DTLSVersion) -> - Version = dtls_v1:corresponding_tls_version(DTLSVersion), - [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:filter_suites(ssl_cipher:psk_suites(Version)), - not ssl_cipher:is_stream_ciphersuite(tuple_to_map(ssl_cipher:erl_suite_definition(S)))]. - -psk_anon_suites({3,_ } = Version) -> - [Suite || Suite <- psk_suites(Version), is_psk_anon_suite(Suite)]; -psk_anon_suites(DTLSVersion) -> - Version = dtls_v1:corresponding_tls_version(DTLSVersion), - [Suite || Suite <- psk_suites(Version), is_psk_anon_suite(Suite), - not ssl_cipher:is_stream_ciphersuite(tuple_to_map(Suite))]. +anonymous_suites(Version) -> + ssl:filter_cipher_suites([ssl_cipher:suite_definition(S) || S <- ssl_cipher:anonymous_suites(Version)],[]). +psk_suites(Version) -> + ssl:filter_cipher_suites([ssl_cipher:suite_definition(S) || S <- ssl_cipher:psk_suites(Version)], []). + +psk_anon_suites(Version) -> + ssl:filter_cipher_suites([ssl_cipher:suite_definition(S) || S <- ssl_cipher:psk_suites_anon(Version)], + [{key_exchange, + fun(psk) -> + true; + (psk_dhe) -> + true; + (_) -> + false + end}]). + srp_suites() -> - [ssl_cipher:erl_suite_definition(Suite) || - Suite <- - ssl_cipher:filter_suites([tuple_to_map(S) || - S <- [{srp_anon,'3des_ede_cbc', sha}, - {srp_rsa, '3des_ede_cbc', sha}, - {srp_anon, aes_128_cbc, sha}, - {srp_rsa, aes_128_cbc, sha}, - {srp_anon, aes_256_cbc, sha}, - {srp_rsa, aes_256_cbc, sha}]])]. + ssl:filter_cipher_suites([ssl_cipher:suite_definition(S) || S <- ssl_cipher:srp_suites()], + [{key_exchange, + fun(srp_rsa) -> + true; + (_) -> + false + end}]). srp_anon_suites() -> - [ssl_cipher:erl_suite_definition(Suite) || - Suite <- - ssl_cipher:filter_suites([tuple_to_map(S) || - S <-[{srp_anon, '3des_ede_cbc', sha}, - {srp_anon, aes_128_cbc, sha}, - {srp_anon, aes_256_cbc, sha}]])]. + ssl:filter_cipher_suites([ssl_cipher:suite_definition(S) || S <- ssl_cipher:srp_suites_anon()], + []). srp_dss_suites() -> - [ssl_cipher:erl_suite_definition(Suite) || - Suite <- - ssl_cipher:filter_suites([tuple_to_map(S) || - S <- [{srp_dss, '3des_ede_cbc', sha}, - {srp_dss, aes_128_cbc, sha}, - {srp_dss, aes_256_cbc, sha}]])]. - + ssl:filter_cipher_suites([ssl_cipher:suite_definition(S) || S <- ssl_cipher:srp_suites()], + [{key_exchange, + fun(srp_dss) -> + true; + (_) -> + false + end}]). chacha_suites(Version) -> [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:filter_suites(ssl_cipher:chacha_suites(Version))]. + rc4_suites(Version) -> - [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:filter_suites(ssl_cipher:rc4_suites(Version))]. + ssl:filter_cipher_suites([ssl_cipher:suite_definition(S) || S <-ssl_cipher:rc4_suites(Version)], []). des_suites(Version) -> - [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:filter_suites(ssl_cipher:des_suites(Version))]. + ssl:filter_cipher_suites([ssl_cipher:suite_definition(S) || S <-ssl_cipher:des_suites(Version)], []). tuple_to_map({Kex, Cipher, Mac}) -> #{key_exchange => Kex, @@ -1313,6 +1304,32 @@ cipher_restriction(Config0) -> Config0 end. +openssl_dsa_support() -> + case os:cmd("openssl version") of + "LibreSSL 2.6.1" ++ _ -> + true; + "LibreSSL 2.6.2" ++ _ -> + true; + "LibreSSL 2.6" ++ _ -> + false; + "LibreSSL 2.4" ++ _ -> + true; + "LibreSSL 2.3" ++ _ -> + true; + "LibreSSL 2.2" ++ _ -> + true; + "LibreSSL 2.1" ++ _ -> + true; + "LibreSSL 2.0" ++ _ -> + true; + "LibreSSL" ++ _ -> + false; + "OpenSSL 1.0.1" ++ Rest -> + hd(Rest) >= s; + _ -> + true + end. + check_sane_openssl_version(Version) -> case supports_ssl_tls_version(Version) of true -> @@ -1391,7 +1408,9 @@ filter_suites(Ciphers0, AtomVersion) -> Supported0 = ssl_cipher:suites(Version) ++ ssl_cipher:anonymous_suites(Version) ++ ssl_cipher:psk_suites(Version) + ++ ssl_cipher:psk_suites_anon(Version) ++ ssl_cipher:srp_suites() + ++ ssl_cipher:srp_suites_anon() ++ ssl_cipher:rc4_suites(Version), Supported1 = ssl_cipher:filter_suites(Supported0), Supported2 = [ssl_cipher:erl_suite_definition(S) || S <- Supported1], diff --git a/lib/ssl/test/ssl_to_openssl_SUITE.erl b/lib/ssl/test/ssl_to_openssl_SUITE.erl index 33cdc325f4..dcdea6beb5 100644 --- a/lib/ssl/test/ssl_to_openssl_SUITE.erl +++ b/lib/ssl/test/ssl_to_openssl_SUITE.erl @@ -70,6 +70,9 @@ all_versions_tests() -> erlang_server_openssl_client, erlang_client_openssl_server_dsa_cert, erlang_server_openssl_client_dsa_cert, + erlang_client_openssl_server_anon, + erlang_server_openssl_client_anon, + erlang_server_openssl_client_anon_with_cert, erlang_server_openssl_client_reuse_session, erlang_client_openssl_server_renegotiate, erlang_client_openssl_server_nowrap_seqnum, @@ -89,6 +92,9 @@ dtls_all_versions_tests() -> erlang_server_openssl_client, erlang_client_openssl_server_dsa_cert, erlang_server_openssl_client_dsa_cert, + erlang_client_openssl_server_anon, + erlang_server_openssl_client_anon, + erlang_server_openssl_client_anon_with_cert, erlang_server_openssl_client_reuse_session, erlang_client_openssl_server_renegotiate, erlang_client_openssl_server_nowrap_seqnum, @@ -143,10 +149,15 @@ init_per_suite(Config0) -> try crypto:start() of ok -> ssl_test_lib:clean_start(), - - Config1 = ssl_test_lib:make_rsa_cert(Config0), - Config2 = ssl_test_lib:make_dsa_cert(Config1), - ssl_test_lib:cipher_restriction(Config2) + Config = + case ssl_test_lib:openssl_dsa_support() of + true -> + Config1 = ssl_test_lib:make_rsa_cert(Config0), + ssl_test_lib:make_dsa_cert(Config1); + false -> + ssl_test_lib:make_rsa_cert(Config0) + end, + ssl_test_lib:cipher_restriction(Config) catch _:_ -> {skip, "Crypto did not start"} end @@ -199,15 +210,27 @@ init_per_testcase(expired_session, Config) -> ssl:start(), Config; -init_per_testcase(TestCase, Config) when TestCase == ciphers_rsa_signed_certs; - TestCase == ciphers_dsa_signed_certs -> - ct:timetrap({seconds, 90}), - special_init(TestCase, Config); - +init_per_testcase(TestCase, Config) when + TestCase == ciphers_dsa_signed_certs; + TestCase == erlang_client_openssl_server_dsa_cert; + TestCase == erlang_server_openssl_client_dsa_cert; + TestCase == erlang_client_openssl_server_dsa_cert; + TestCase == erlang_server_openssl_client_dsa_cert -> + case ssl_test_lib:openssl_dsa_support() of + true -> + special_init(TestCase, Config); + false -> + {skip, "DSA not supported by OpenSSL"} + end; init_per_testcase(TestCase, Config) -> ct:timetrap({seconds, 35}), special_init(TestCase, Config). +special_init(TestCase, Config) when + TestCase == ciphers_rsa_signed_certs; + TestCase == ciphers_dsa_signed_certs-> + ct:timetrap({seconds, 90}), + Config; special_init(TestCase, Config) when TestCase == erlang_client_openssl_server_renegotiate; TestCase == erlang_client_openssl_server_nowrap_seqnum; @@ -533,7 +556,121 @@ erlang_server_openssl_client_dsa_cert(Config) when is_list(Config) -> ssl_test_lib:close_port(OpenSslPort), process_flag(trap_exit, false). -%%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +erlang_client_openssl_server_anon() -> + [{doc,"Test erlang client with openssl server, anonymous"}]. +erlang_client_openssl_server_anon(Config) when is_list(Config) -> + process_flag(trap_exit, true), + %% OpenSSL expects a certificate and key, even if the cipher spec + %% is restructed to aNULL, so we use 'server_rsa_opts' here + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_anon_opts, Config), + VersionTuple = ssl_test_lib:protocol_version(Config, tuple), + Ciphers = ssl_test_lib:anonymous_suites(VersionTuple), + + {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), + + Data = "From openssl to erlang", + + Port = ssl_test_lib:inet_port(node()), + CertFile = proplists:get_value(certfile, ServerOpts), + KeyFile = proplists:get_value(keyfile, ServerOpts), + Version = ssl_test_lib:protocol_version(Config), + Exe = "openssl", + Args = ["s_server", "-accept", integer_to_list(Port), + ssl_test_lib:version_flag(Version), + "-cert", CertFile, "-key", KeyFile, + "-cipher", "aNULL", "-msg"], + + OpensslPort = ssl_test_lib:portable_open_port(Exe, Args), + + ssl_test_lib:wait_for_openssl_server(Port, proplists:get_value(protocol, Config)), + + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, + erlang_ssl_receive, [Data]}}, + {options, [{ciphers, Ciphers} | ClientOpts]}]), + + true = port_command(OpensslPort, Data), + + ssl_test_lib:check_result(Client, ok), + + %% Clean close down! Server needs to be closed first !! + ssl_test_lib:close_port(OpensslPort), + ssl_test_lib:close(Client), + process_flag(trap_exit, false), + ok. +%%-------------------------------------------------------------------- +erlang_server_openssl_client_anon() -> + [{doc,"Test erlang server with openssl client, anonymous"}]. +erlang_server_openssl_client_anon(Config) when is_list(Config) -> + process_flag(trap_exit, true), + ServerOpts = ssl_test_lib:ssl_options(server_anon_opts, Config), + VersionTuple = ssl_test_lib:protocol_version(Config, tuple), + Ciphers = ssl_test_lib:anonymous_suites(VersionTuple), + + {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Data = "From openssl to erlang", + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, erlang_ssl_receive, [Data]}}, + {options, [{ciphers, Ciphers} | ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + Version = ssl_test_lib:protocol_version(Config), + Exe = "openssl", + Args = ["s_client", "-connect", hostname_format(Hostname) ++ ":" ++ integer_to_list(Port), + ssl_test_lib:version_flag(Version), + "-cipher", "aNULL", "-msg"], + + OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), + true = port_command(OpenSslPort, Data), + + ssl_test_lib:check_result(Server, ok), + + %% Clean close down! Server needs to be closed first !! + ssl_test_lib:close(Server), + ssl_test_lib:close_port(OpenSslPort), + process_flag(trap_exit, false). + + %%-------------------------------------------------------------------- + erlang_server_openssl_client_anon_with_cert() -> + [{doc,"Test erlang server with openssl client, anonymous (with cert)"}]. + erlang_server_openssl_client_anon_with_cert(Config) when is_list(Config) -> + process_flag(trap_exit, true), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), + VersionTuple = ssl_test_lib:protocol_version(Config, tuple), + Ciphers = ssl_test_lib:anonymous_suites(VersionTuple), + + {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Data = "From openssl to erlang", + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, erlang_ssl_receive, [Data]}}, + {options, [{ciphers, Ciphers} | ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + Version = ssl_test_lib:protocol_version(Config), + Exe = "openssl", + Args = ["s_client", "-connect", hostname_format(Hostname) ++ ":" ++ integer_to_list(Port), + ssl_test_lib:version_flag(Version), + "-cipher", "aNULL", "-msg"], + + OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), + true = port_command(OpenSslPort, Data), + + ssl_test_lib:check_result(Server, ok), + + %% Clean close down! Server needs to be closed first !! + ssl_test_lib:close(Server), + ssl_test_lib:close_port(OpenSslPort), + process_flag(trap_exit, false). + +%%-------------------------------------------------------------------- erlang_server_openssl_client_reuse_session() -> [{doc, "Test erlang server with openssl client that reconnects with the" diff --git a/lib/stdlib/doc/src/assert_hrl.xml b/lib/stdlib/doc/src/assert_hrl.xml index ea23cca2ee..33f29f38da 100644 --- a/lib/stdlib/doc/src/assert_hrl.xml +++ b/lib/stdlib/doc/src/assert_hrl.xml @@ -93,7 +93,7 @@ erlc -DNOASSERT=true *.erl</code> <taglist> <tag><c>assert(BoolExpr)</c></tag> <item></item> - <tag><c>URKAassert(BoolExpr, Comment)</c></tag> + <tag><c>assert(BoolExpr, Comment)</c></tag> <item> <p>Tests that <c>BoolExpr</c> completes normally returning <c>true</c>.</p> diff --git a/lib/stdlib/doc/src/erl_tar.xml b/lib/stdlib/doc/src/erl_tar.xml index 337028568a..14c543ee2b 100644 --- a/lib/stdlib/doc/src/erl_tar.xml +++ b/lib/stdlib/doc/src/erl_tar.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2003</year><year>2017</year> + <year>2003</year><year>2018</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -136,6 +136,9 @@ <v>Filename = filename()|{NameInArchive,FilenameOrBin}</v> <v>Options = [Option]</v> <v>Option = dereference|verbose|{chunks,ChunkSize}</v> + <v>|{atime,non_neg_integer()}|{mtime,non_neg_integer()}</v> + <v>|{ctime,non_neg_integer()}|{uid,non_neg_integer()}</v> + <v>|{gid,non_neg_integer()}</v> <v>ChunkSize = positive_integer()</v> <v>RetValue = ok|{error,{Filename,Reason}}</v> <v>Reason = term()</v> @@ -167,6 +170,42 @@ <seealso marker="ssh:ssh_sftp#open_tar/3"> <c>ssh_sftp:open_tar/3</c></seealso>.</p> </item> + <tag><c>{atime,non_neg_integer()}</c></tag> + <item> + <p>Sets the last time, as + <seealso marker="erts:time_correction#POSIX_Time"> + POSIX time</seealso>, when the file was read. See also + <seealso marker="kernel:file#read_file_info/1"> + <c>file:read_file_info/1</c></seealso>.</p> + </item> + <tag><c>{mtime,non_neg_integer()}</c></tag> + <item> + <p>Sets the last time, as + <seealso marker="erts:time_correction#POSIX_Time"> + POSIX time</seealso>, when the file was written. See also + <seealso marker="kernel:file#read_file_info/1"> + <c>file:read_file_info/1</c></seealso>.</p> + </item> + <tag><c>{ctime,non_neg_integer()}</c></tag> + <item> + <p>Sets the time, as + <seealso marker="erts:time_correction#POSIX_Time"> + POSIX time</seealso>, when the file was created. See also + <seealso marker="kernel:file#read_file_info/1"> + <c>file:read_file_info/1</c></seealso>.</p> + </item> + <tag><c>{uid,non_neg_integer()}</c></tag> + <item> + <p>Sets the file owner. + <seealso marker="kernel:file#read_file_info/1"> + <c>file:read_file_info/1</c></seealso>.</p> + </item> + <tag><c>{gid,non_neg_integer()}</c></tag> + <item> + <p>Sets the group that the file owner belongs to. + <seealso marker="kernel:file#read_file_info/1"> + <c>file:read_file_info/1</c></seealso>.</p> + </item> </taglist> </desc> </func> @@ -378,7 +417,7 @@ <v>Reason = term()</v> </type> <desc> - <p>Cconverts an error reason term to a human-readable error message + <p>Converts an error reason term to a human-readable error message string.</p> </desc> </func> diff --git a/lib/stdlib/doc/src/ets.xml b/lib/stdlib/doc/src/ets.xml index a0ec22c515..305376a425 100644 --- a/lib/stdlib/doc/src/ets.xml +++ b/lib/stdlib/doc/src/ets.xml @@ -487,6 +487,11 @@ Error: fun containing local Erlang function calls <p>The pid of the heir of the table, or <c>none</c> if no heir is set.</p> </item> + <tag><c>{id,</c><seealso marker="#type-tid"> + <c>tid()</c></seealso><c>}</c></tag> + <item> + <p>The table identifier.</p> + </item> <tag><c>{keypos, integer() >= 1}</c></tag> <item> <p>The key position.</p> @@ -963,11 +968,11 @@ ets:is_compiled_ms(Broken).</code> <func> <name name="match_spec_run" arity="2"/> <fsummary>Perform matching, using a compiled match specification on a - list of tuples.</fsummary> + list of terms.</fsummary> <desc> <p>Executes the matching specified in a compiled <seealso marker="#match_spec">match specification</seealso> on a list - of tuples. Term <c><anno>CompiledMatchSpec</anno></c> is to be + of terms. Term <c><anno>CompiledMatchSpec</anno></c> is to be the result of a call to <seealso marker="#match_spec_compile/1"> <c>match_spec_compile/1</c></seealso> and is hence the internal representation of the match specification one wants to use.</p> @@ -985,7 +990,7 @@ Table = ets:new... MatchSpec = ... % The following call... ets:match_spec_run(ets:tab2list(Table), -ets:match_spec_compile(MatchSpec)), + ets:match_spec_compile(MatchSpec)), % ...gives the same result as the more common (and more efficient) ets:select(Table, MatchSpec),</code> <note> @@ -1074,10 +1079,13 @@ ets:select(Table, MatchSpec),</code> </item> <tag><c>named_table</c></tag> <item> - <p>If this option is present, name <c><anno>Name</anno></c> is - associated with the table identifier. The name can then - be used instead of the table identifier in subsequent - operations.</p> + <p>If this option is present, the table is registered under its + <c><anno>Name</anno></c> which can then be used instead of the + table identifier in subsequent operations.</p> + <p>The function will also return the <c><anno>Name</anno></c> + instead of the table identifier. To get the table identifier of a + named table, use + <seealso marker="#whereis/1"><c>whereis/1</c></seealso>.</p> </item> <tag><c>{keypos,<anno>Pos</anno>}</c></tag> <item> @@ -2037,6 +2045,21 @@ true</pre> </list> </desc> </func> + + <func> + <name name="whereis" arity="1"/> + <fsummary>Retrieves the tid() of a named table.</fsummary> + <desc> + <p>This function returns the + <seealso marker="#type-tid"><c>tid()</c></seealso> of the named table + identified by <c><anno>TableName</anno></c>, or <c>undefined</c> if + no such table exists. The <c>tid()</c> can be used in place of the + table name in all operations, which is slightly faster since the name + does not have to be resolved on each call.</p> + <p>If the table is deleted, the <c>tid()</c> will be invalid even if + another named table is created with the same name.</p> + </desc> + </func> </funcs> </erlref> diff --git a/lib/stdlib/doc/src/timer.xml b/lib/stdlib/doc/src/timer.xml index fcaccdb2cb..350847bf7d 100644 --- a/lib/stdlib/doc/src/timer.xml +++ b/lib/stdlib/doc/src/timer.xml @@ -270,7 +270,7 @@ <item> <p>Evaluates <c>apply(<anno>Module</anno>, <anno>Function</anno>, <anno>Arguments</anno>)</c> and measures the elapsed real time as - reported by <seealso marker="os:timestamp/0"> + reported by <seealso marker="kernel:os#timestamp/0"> <c>os:timestamp/0</c></seealso>.</p> <p>Returns <c>{<anno>Time</anno>, <anno>Value</anno>}</c>, where <c><anno>Time</anno></c> is the elapsed real time in diff --git a/lib/stdlib/doc/src/uri_string.xml b/lib/stdlib/doc/src/uri_string.xml index 21f470e763..88d4600611 100644 --- a/lib/stdlib/doc/src/uri_string.xml +++ b/lib/stdlib/doc/src/uri_string.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2017</year><year>2017</year> + <year>2017</year><year>2018</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -24,7 +24,7 @@ <title>uri_string</title> <prepared>Péter Dimitrov</prepared> <docno>1</docno> - <date>2017-10-24</date> + <date>2018-02-07</date> <rev>A</rev> </header> <module>uri_string</module> @@ -32,7 +32,11 @@ <description> <p>This module contains functions for parsing and handling URIs (<url href="https://www.ietf.org/rfc/rfc3986.txt">RFC 3986</url>) and - form-urlencoded query strings (<url href="https://www.w3.org/TR/html5/forms.html">HTML5</url>). + form-urlencoded query strings (<url href="https://www.w3.org/TR/html52/">HTML 5.2</url>). + </p> + <p> + Parsing and serializing non-UTF-8 form-urlencoded query strings are also supported + (<url href="https://www.w3.org/TR/html50/">HTML 5.0</url>). </p> <p>A URI is an identifier consisting of a sequence of characters matching the syntax rule named <em>URI</em> in <url href="https://www.ietf.org/rfc/rfc3986.txt">RFC 3986</url>. @@ -70,7 +74,8 @@ <seealso marker="#transcode/2"><c>transcode/2</c></seealso> </item> <item>Transforming URIs into a normalized form<br></br> - <seealso marker="#normalize/1"><c>normalize/1</c></seealso> + <seealso marker="#normalize/1"><c>normalize/1</c></seealso><br></br> + <seealso marker="#normalize/2"><c>normalize/2</c></seealso> </item> <item>Composing form-urlencoded query strings from a list of key-value pairs<br></br> <seealso marker="#compose_query/1"><c>compose_query/1</c></seealso><br></br> @@ -151,8 +156,10 @@ <p>Composes a form-urlencoded <c><anno>QueryString</anno></c> based on a <c><anno>QueryList</anno></c>, a list of non-percent-encoded key-value pairs. Form-urlencoding is defined in section - 4.10.22.6 of the <url href="https://www.w3.org/TR/html5/forms.html">HTML5</url> - specification. + 4.10.21.6 of the <url href="https://www.w3.org/TR/html52/">HTML 5.2</url> + specification and in section 4.10.22.6 of the + <url href="https://www.w3.org/TR/html50/">HTML 5.0</url> specification for + non-UTF-8 encodings. </p> <p>See also the opposite operation <seealso marker="#dissect_query/1"> <c>dissect_query/1</c></seealso>. @@ -209,12 +216,11 @@ <p>Dissects an urlencoded <c><anno>QueryString</anno></c> and returns a <c><anno>QueryList</anno></c>, a list of non-percent-encoded key-value pairs. Form-urlencoding is defined in section - 4.10.22.6 of the <url href="https://www.w3.org/TR/html5/forms.html">HTML5</url> - specification. + 4.10.21.6 of the <url href="https://www.w3.org/TR/html52/">HTML 5.2</url> + specification and in section 4.10.22.6 of the + <url href="https://www.w3.org/TR/html50/">HTML 5.0</url> specification for + non-UTF-8 encodings. </p> - <p>It is not as strict for its input as the decoding algorithm defined by - <url href="https://www.w3.org/TR/html5/forms.html">HTML5</url> - and accepts all unicode characters.</p> <p>See also the opposite operation <seealso marker="#compose_query/1"> <c>compose_query/1</c></seealso>. </p> @@ -233,7 +239,7 @@ <name name="normalize" arity="1"/> <fsummary>Syntax-based normalization.</fsummary> <desc> - <p>Transforms <c><anno>URIString</anno></c> into a normalized form + <p>Transforms an <c><anno>URI</anno></c> into a normalized form using Syntax-Based Normalization as defined by <url href="https://www.ietf.org/rfc/rfc3986.txt">RFC 3986</url>.</p> <p>This function implements case normalization, percent-encoding @@ -247,6 +253,33 @@ <![CDATA[<<"mid/6">>]]> 3> uri_string:normalize("http://localhost:80"). "https://localhost/" +4> <input>uri_string:normalize(#{scheme => "http",port => 80,path => "/a/b/c/./../../g",</input> +4> host => "localhost-örebro"}). +"http://localhost-%C3%B6rebro/a/g" + </pre> + </desc> + </func> + + <func> + <name name="normalize" arity="2"/> + <fsummary>Syntax-based normalization.</fsummary> + <desc> + <p>Same as <c>normalize/1</c> but with an additional + <c><anno>Options</anno></c> parameter, that controls if the normalized URI + shall be returned as an uri_map(). + There is one supported option: <c>return_map</c>. + </p> + <p><em>Example:</em></p> + <pre> +1> <input>uri_string:normalize("/a/b/c/./../../g", [return_map]).</input> +#{path => "/a/g"} +2> <![CDATA[uri_string:normalize(<<"mid/content=5/../6">>, [return_map]).]]> +<![CDATA[#{path => <<"mid/6">>}]]> +3> uri_string:normalize("http://localhost:80", [return_map]). +#{scheme => "http",path => "/",host => "localhost"} +4> <input>uri_string:normalize(#{scheme => "http",port => 80,path => "/a/b/c/./../../g",</input> +4> host => "localhost-örebro"}, [return_map]). +#{scheme => "http",path => "/a/g",host => "localhost-örebro"} </pre> </desc> </func> diff --git a/lib/stdlib/include/assert.hrl b/lib/stdlib/include/assert.hrl index 2fbaeba0b2..2ec89e7d8a 100644 --- a/lib/stdlib/include/assert.hrl +++ b/lib/stdlib/include/assert.hrl @@ -309,7 +309,7 @@ {unexpected_success, __V}]}) catch Class:Term -> ok; - __C:__T -> + __C:__T:__S -> erlang:error({assertException, [{module, ?MODULE}, {line, ?LINE}, @@ -318,8 +318,7 @@ "{ "++(??Class)++" , "++(??Term) ++" , [...] }"}, {unexpected_exception, - {__C, __T, - erlang:get_stacktrace()}}]}) + {__C, __T, __S}}]}) end end)()) end). @@ -338,7 +337,7 @@ {unexpected_success, __V}]}) catch Class:Term -> ok; - __C:__T -> + __C:__T:__S -> erlang:error({assertException, [{module, ?MODULE}, {line, ?LINE}, @@ -348,8 +347,7 @@ "{ "++(??Class)++" , "++(??Term) ++" , [...] }"}, {unexpected_exception, - {__C, __T, - erlang:get_stacktrace()}}]}) + {__C, __T, __S}}]}) end end)()) end). @@ -378,7 +376,7 @@ try (Expr) of _ -> ok catch - __C:__T -> + __C:__T:__S -> case __C of Class -> case __T of @@ -391,9 +389,7 @@ "{ "++(??Class)++" , " ++(??Term)++" , [...] }"}, {unexpected_exception, - {__C, __T, - erlang:get_stacktrace() - }}]}); + {__C, __T, __S}}]}); _ -> ok end; _ -> ok @@ -407,7 +403,7 @@ try (Expr) of _ -> ok catch - __C:__T -> + __C:__T:__S -> case __C of Class -> case __T of @@ -421,9 +417,7 @@ "{ "++(??Class)++" , " ++(??Term)++" , [...] }"}, {unexpected_exception, - {__C, __T, - erlang:get_stacktrace() - }}]}); + {__C, __T, __S}}]}); _ -> ok end; _ -> ok diff --git a/lib/stdlib/src/beam_lib.erl b/lib/stdlib/src/beam_lib.erl index 06c15fceda..24349c74e8 100644 --- a/lib/stdlib/src/beam_lib.erl +++ b/lib/stdlib/src/beam_lib.erl @@ -148,7 +148,8 @@ chunks(File, Chunks, Options) -> try read_chunk_data(File, Chunks, Options) catch Error -> Error end. --spec all_chunks(beam()) -> {'ok', 'beam_lib', [{chunkid(), dataB()}]}. +-spec all_chunks(beam()) -> + {'ok', 'beam_lib', [{chunkid(), dataB()}]} | {'error', 'beam_lib', info_rsn()}. all_chunks(File) -> read_all_chunks(File). diff --git a/lib/stdlib/src/binary.erl b/lib/stdlib/src/binary.erl index 6a64133b45..7d0e42489e 100644 --- a/lib/stdlib/src/binary.erl +++ b/lib/stdlib/src/binary.erl @@ -47,23 +47,39 @@ at(_, _) -> -spec bin_to_list(Subject) -> [byte()] when Subject :: binary(). -bin_to_list(_) -> - erlang:nif_error(undef). +bin_to_list(Subject) -> + binary_to_list(Subject). -spec bin_to_list(Subject, PosLen) -> [byte()] when Subject :: binary(), PosLen :: part(). -bin_to_list(_, _) -> - erlang:nif_error(undef). +bin_to_list(Subject, {Pos, Len}) -> + bin_to_list(Subject, Pos, Len); +bin_to_list(_Subject, _BadArg) -> + erlang:error(badarg). -spec bin_to_list(Subject, Pos, Len) -> [byte()] when Subject :: binary(), Pos :: non_neg_integer(), Len :: integer(). -bin_to_list(_, _, _) -> - erlang:nif_error(undef). +bin_to_list(Subject, Pos, Len) when not is_binary(Subject); + not is_integer(Pos); + not is_integer(Len) -> + %% binary_to_list/3 allows bitstrings as long as the slice fits, and we + %% want to badarg when Pos/Len aren't integers instead of raising badarith + %% when adjusting args for binary_to_list/3. + erlang:error(badarg); +bin_to_list(Subject, Pos, 0) when Pos >= 0, Pos =< byte_size(Subject) -> + %% binary_to_list/3 doesn't handle this case. + []; +bin_to_list(_Subject, _Pos, 0) -> + erlang:error(badarg); +bin_to_list(Subject, Pos, Len) when Len < 0 -> + bin_to_list(Subject, Pos + Len, -Len); +bin_to_list(Subject, Pos, Len) when Len > 0 -> + binary_to_list(Subject, Pos + 1, Pos + Len). -spec compile_pattern(Pattern) -> cp() when Pattern :: binary() | [binary()]. diff --git a/lib/stdlib/src/c.erl b/lib/stdlib/src/c.erl index 9a447af5b7..3597e61c26 100644 --- a/lib/stdlib/src/c.erl +++ b/lib/stdlib/src/c.erl @@ -1034,8 +1034,8 @@ appcall(App, M, F, Args) -> try apply(M, F, Args) catch - error:undef -> - case erlang:get_stacktrace() of + error:undef:S -> + case S of [{M,F,Args,_}|_] -> Arity = length(Args), io:format("Call to ~w:~w/~w in application ~w failed.\n", diff --git a/lib/stdlib/src/dets.erl b/lib/stdlib/src/dets.erl index 4e3fe0e5c1..e1a36abc70 100644 --- a/lib/stdlib/src/dets.erl +++ b/lib/stdlib/src/dets.erl @@ -1288,8 +1288,8 @@ init(Parent, Server) -> catch exit:normal -> exit(normal); - _:Bad -> - bug_found(no_name, Op, Bad, From), + _:Bad:Stacktrace -> + bug_found(no_name, Op, Bad, Stacktrace, From), exit(Bad) % give up end end. @@ -1371,8 +1371,8 @@ do_apply_op(Op, From, Head, N) -> catch exit:normal -> exit(normal); - _:Bad -> - bug_found(Head#head.name, Op, Bad, From), + _:Bad:Stacktrace -> + bug_found(Head#head.name, Op, Bad, Stacktrace, From), open_file_loop(Head, N) end. @@ -1581,7 +1581,7 @@ apply_op(Op, From, Head, N) -> ok end. -bug_found(Name, Op, Bad, From) -> +bug_found(Name, Op, Bad, Stacktrace, From) -> case dets_utils:debug_mode() of true -> %% If stream_op/5 found more requests, this is not @@ -1590,7 +1590,7 @@ bug_found(Name, Op, Bad, From) -> ("** dets: Bug was found when accessing table ~tw,~n" "** dets: operation was ~tp and reply was ~tw.~n" "** dets: Stacktrace: ~tw~n", - [Name, Op, Bad, erlang:get_stacktrace()]); + [Name, Op, Bad, Stacktrace]); false -> error_logger:format ("** dets: Bug was found when accessing table ~tw~n", diff --git a/lib/stdlib/src/dets_utils.erl b/lib/stdlib/src/dets_utils.erl index 17f55ebdc2..4c8ea9e82b 100644 --- a/lib/stdlib/src/dets_utils.erl +++ b/lib/stdlib/src/dets_utils.erl @@ -377,7 +377,8 @@ corrupt_reason(Head, Reason0) -> no_disk_map -> Reason0; DM -> - ST = erlang:get_stacktrace(), + {current_stacktrace, ST} = + erlang:process_info(self(), current_stacktrace), PD = get(), {Reason0, ST, PD, DM} end, diff --git a/lib/stdlib/src/epp.erl b/lib/stdlib/src/epp.erl index 00e6a10d8a..77cc88eb08 100644 --- a/lib/stdlib/src/epp.erl +++ b/lib/stdlib/src/epp.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2017. All Rights Reserved. +%% Copyright Ericsson AB 1996-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1197,21 +1197,21 @@ skip_else(_Else, From, St, Sis) -> %% macro_expansion(Tokens, Anno) %% Extract the macro parameters and the expansion from a macro definition. -macro_pars([{')',_Lp}, {',',Ld}|Ex], Args) -> - {ok, {lists:reverse(Args), macro_expansion(Ex, Ld)}}; -macro_pars([{var,_,Name}, {')',_Lp}, {',',Ld}|Ex], Args) -> +macro_pars([{')',_Lp}, {',',_Ld}=Comma|Ex], Args) -> + {ok, {lists:reverse(Args), macro_expansion(Ex, Comma)}}; +macro_pars([{var,_,Name}, {')',_Lp}, {',',_Ld}=Comma|Ex], Args) -> false = lists:member(Name, Args), %Prolog is nice - {ok, {lists:reverse([Name|Args]), macro_expansion(Ex, Ld)}}; + {ok, {lists:reverse([Name|Args]), macro_expansion(Ex, Comma)}}; macro_pars([{var,_L,Name}, {',',_}|Ts], Args) -> false = lists:member(Name, Args), macro_pars(Ts, [Name|Args]). -macro_expansion([{')',_Lp},{dot,_Ld}], _Anno0) -> []; -macro_expansion([{dot,_}=Dot], _Anno0) -> +macro_expansion([{')',_Lp},{dot,_Ld}], _T0) -> []; +macro_expansion([{dot,_}=Dot], _T0) -> throw({error,loc(Dot),missing_parenthesis}); -macro_expansion([T|Ts], _Anno0) -> +macro_expansion([T|Ts], _T0) -> [T|macro_expansion(Ts, T)]; -macro_expansion([], Anno0) -> throw({error,loc(Anno0),premature_end}). +macro_expansion([], T0) -> throw({error,loc(T0),premature_end}). %% expand_macros(Tokens, St) %% expand_macro(Tokens, MacroToken, RestTokens) diff --git a/lib/stdlib/src/erl_tar.erl b/lib/stdlib/src/erl_tar.erl index 5ee584d612..d8b8f466b1 100644 --- a/lib/stdlib/src/erl_tar.erl +++ b/lib/stdlib/src/erl_tar.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2017. All Rights Reserved. +%% Copyright Ericsson AB 1997-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -457,26 +457,61 @@ add(Reader, NameOrBin, NameInArchive, Options) do_add(#reader{access=write}=Reader, Name, NameInArchive, Options) when is_list(NameInArchive), is_list(Options) -> - RF = fun(F) -> file:read_link_info(F, [{time, posix}]) end, + RF = apply_file_info_opts_fun(Options, read_link_info), Opts = #add_opts{read_info=RF}, - add1(Reader, Name, NameInArchive, add_opts(Options, Opts)); + add1(Reader, Name, NameInArchive, add_opts(Options, Options, Opts)); do_add(#reader{access=read},_,_,_) -> {error, eacces}; do_add(Reader,_,_,_) -> {error, {badarg, Reader}}. -add_opts([dereference|T], Opts) -> - RF = fun(F) -> file:read_file_info(F, [{time, posix}]) end, - add_opts(T, Opts#add_opts{read_info=RF}); -add_opts([verbose|T], Opts) -> - add_opts(T, Opts#add_opts{verbose=true}); -add_opts([{chunks,N}|T], Opts) -> - add_opts(T, Opts#add_opts{chunk_size=N}); -add_opts([_|T], Opts) -> - add_opts(T, Opts); -add_opts([], Opts) -> +add_opts([dereference|T], AllOptions, Opts) -> + RF = apply_file_info_opts_fun(AllOptions, read_file_info), + add_opts(T, AllOptions, Opts#add_opts{read_info=RF}); +add_opts([verbose|T], AllOptions, Opts) -> + add_opts(T, AllOptions, Opts#add_opts{verbose=true}); +add_opts([{chunks,N}|T], AllOptions, Opts) -> + add_opts(T, AllOptions, Opts#add_opts{chunk_size=N}); +add_opts([{atime,Value}|T], AllOptions, Opts) -> + add_opts(T, AllOptions, Opts#add_opts{atime=Value}); +add_opts([{mtime,Value}|T], AllOptions, Opts) -> + add_opts(T, AllOptions, Opts#add_opts{mtime=Value}); +add_opts([{ctime,Value}|T], AllOptions, Opts) -> + add_opts(T, AllOptions, Opts#add_opts{ctime=Value}); +add_opts([{uid,Value}|T], AllOptions, Opts) -> + add_opts(T, AllOptions, Opts#add_opts{uid=Value}); +add_opts([{gid,Value}|T], AllOptions, Opts) -> + add_opts(T, AllOptions, Opts#add_opts{gid=Value}); +add_opts([_|T], AllOptions, Opts) -> + add_opts(T, AllOptions, Opts); +add_opts([], _AllOptions, Opts) -> Opts. +apply_file_info_opts(Opts, {ok, FileInfo}) -> + {ok, do_apply_file_info_opts(Opts, FileInfo)}; +apply_file_info_opts(_Opts, Other) -> + Other. + +do_apply_file_info_opts([{atime,Value}|T], FileInfo) -> + do_apply_file_info_opts(T, FileInfo#file_info{atime=Value}); +do_apply_file_info_opts([{mtime,Value}|T], FileInfo) -> + do_apply_file_info_opts(T, FileInfo#file_info{mtime=Value}); +do_apply_file_info_opts([{ctime,Value}|T], FileInfo) -> + do_apply_file_info_opts(T, FileInfo#file_info{ctime=Value}); +do_apply_file_info_opts([{uid,Value}|T], FileInfo) -> + do_apply_file_info_opts(T, FileInfo#file_info{uid=Value}); +do_apply_file_info_opts([{gid,Value}|T], FileInfo) -> + do_apply_file_info_opts(T, FileInfo#file_info{gid=Value}); +do_apply_file_info_opts([_|T], FileInfo) -> + do_apply_file_info_opts(T, FileInfo); +do_apply_file_info_opts([], FileInfo) -> + FileInfo. + +apply_file_info_opts_fun(Options, InfoFunction) -> + fun(F) -> + apply_file_info_opts(Options, file:InfoFunction(F, [{time, posix}])) + end. + add1(#reader{}=Reader, Name, NameInArchive, #add_opts{read_info=ReadInfo}=Opts) when is_list(Name) -> Res = case ReadInfo(Name) of @@ -515,9 +550,11 @@ add1(Reader, Bin, NameInArchive, Opts) when is_binary(Bin) -> name = NameInArchive, size = byte_size(Bin), typeflag = ?TYPE_REGULAR, - atime = Now, - mtime = Now, - ctime = Now, + atime = add_opts_time(Opts#add_opts.atime, Now), + mtime = add_opts_time(Opts#add_opts.mtime, Now), + ctime = add_opts_time(Opts#add_opts.ctime, Now), + uid = Opts#add_opts.uid, + gid = Opts#add_opts.gid, mode = 8#100644}, {ok, Reader2} = add_header(Reader, Header, Opts), Padding = skip_padding(byte_size(Bin)), @@ -527,6 +564,9 @@ add1(Reader, Bin, NameInArchive, Opts) when is_binary(Bin) -> {error, Reason} -> {error, {NameInArchive, Reason}} end. +add_opts_time(undefined, Now) -> Now; +add_opts_time(Time, _Now) -> Time. + add_directory(Reader, DirName, NameInArchive, Info, Opts) -> case file:list_dir(DirName) of {ok, []} -> @@ -1650,8 +1690,12 @@ write_file(Name, Bin) -> case file:write_file(Name, Bin) of ok -> ok; {error,enoent} -> - ok = make_dirs(Name, file), - write_file(Name, Bin); + case make_dirs(Name, file) of + ok -> + write_file(Name, Bin); + {error,Reason} -> + throw({error, Reason}) + end; {error,Reason} -> throw({error, Reason}) end. diff --git a/lib/stdlib/src/erl_tar.hrl b/lib/stdlib/src/erl_tar.hrl index cff0c2f500..5d6cecbb66 100644 --- a/lib/stdlib/src/erl_tar.hrl +++ b/lib/stdlib/src/erl_tar.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2017. All Rights Reserved. +%% Copyright Ericsson AB 1997-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -21,7 +21,12 @@ -record(add_opts, { read_info, %% Fun to use for read file/link info. chunk_size = 0, %% For file reading when sending to sftp. 0=do not chunk - verbose = false}). %% Verbose on/off. + verbose = false, %% Verbose on/off. + atime = undefined, + mtime = undefined, + ctime = undefined, + uid = 0, + gid = 0}). -type add_opts() :: #add_opts{}. %% Options used when reading a tar archive. @@ -36,7 +41,12 @@ -type add_opt() :: dereference | verbose | - {chunks, pos_integer()}. + {chunks, pos_integer()} | + {atime, non_neg_integer()} | + {mtime, non_neg_integer()} | + {ctime, non_neg_integer()} | + {uid, non_neg_integer()} | + {gid, non_neg_integer()}. -type extract_opt() :: {cwd, string()} | {files, [string()]} | diff --git a/lib/stdlib/src/escript.erl b/lib/stdlib/src/escript.erl index 132f8efbbe..beea9927d2 100644 --- a/lib/stdlib/src/escript.erl +++ b/lib/stdlib/src/escript.erl @@ -283,8 +283,7 @@ start(EscriptOptions) -> throw:Str -> io:format("escript: ~ts\n", [Str]), my_halt(127); - _:Reason -> - Stk = erlang:get_stacktrace(), + _:Reason:Stk -> io:format("escript: Internal error: ~tp\n", [Reason]), io:format("~tp\n", [Stk]), my_halt(127) @@ -759,8 +758,8 @@ run(Module, Args) -> Module:main(Args), my_halt(0) catch - Class:Reason -> - fatal(format_exception(Class, Reason)) + Class:Reason:StackTrace -> + fatal(format_exception(Class, Reason, StackTrace)) end. -spec interpret(_, _, _, _) -> no_return(). @@ -793,8 +792,8 @@ interpret(Forms, HasRecs, File, Args) -> end}), my_halt(0) catch - Class:Reason -> - fatal(format_exception(Class, Reason)) + Class:Reason:StackTrace -> + fatal(format_exception(Class, Reason, StackTrace)) end. report_errors(Errors) -> @@ -873,7 +872,7 @@ eval_exprs([E|Es], Bs0, Lf, Ef, RBs) -> {value,_V,Bs} = erl_eval:expr(E, Bs0, Lf, Ef, RBs1), eval_exprs(Es, Bs, Lf, Ef, RBs). -format_exception(Class, Reason) -> +format_exception(Class, Reason, StackTrace) -> Enc = encoding(), P = case Enc of latin1 -> "P"; @@ -882,7 +881,6 @@ format_exception(Class, Reason) -> PF = fun(Term, I) -> io_lib:format("~." ++ integer_to_list(I) ++ P, [Term, 50]) end, - StackTrace = erlang:get_stacktrace(), StackFun = fun(M, _F, _A) -> (M =:= erl_eval) or (M =:= ?MODULE) end, lib:format_exception(1, Class, Reason, StackTrace, StackFun, PF, Enc). @@ -916,8 +914,8 @@ hidden_apply(App, M, F, Args) -> try apply(fun() -> M end(), F, Args) catch - error:undef -> - case erlang:get_stacktrace() of + error:undef:StackTrace -> + case StackTrace of [{M,F,Args,_} | _] -> Arity = length(Args), Text = io_lib:format("Call to ~w:~w/~w in application ~w failed.\n", diff --git a/lib/stdlib/src/ets.erl b/lib/stdlib/src/ets.erl index b6548626f3..6a559f0be5 100644 --- a/lib/stdlib/src/ets.erl +++ b/lib/stdlib/src/ets.erl @@ -73,7 +73,8 @@ select_count/2, select_delete/2, select_replace/2, select_reverse/1, select_reverse/2, select_reverse/3, setopts/2, slot/2, take/2, - update_counter/3, update_counter/4, update_element/3]). + update_counter/3, update_counter/4, update_element/3, + whereis/1]). %% internal exports -export([internal_request_all/0]). @@ -145,6 +146,7 @@ give_away(_, _, _) -> InfoList :: [InfoTuple], InfoTuple :: {compressed, boolean()} | {heir, pid() | none} + | {id, tid()} | {keypos, pos_integer()} | {memory, non_neg_integer()} | {name, atom()} @@ -162,7 +164,7 @@ info(_) -> -spec info(Tab, Item) -> Value | undefined when Tab :: tab(), - Item :: compressed | fixed | heir | keypos | memory + Item :: compressed | fixed | heir | id | keypos | memory | name | named_table | node | owner | protection | safe_fixed | safe_fixed_monotonic_time | size | stats | type | write_concurrency | read_concurrency, @@ -277,7 +279,7 @@ match_spec_compile(_) -> erlang:nif_error(undef). -spec match_spec_run_r(List, CompiledMatchSpec, list()) -> list() when - List :: [tuple()], + List :: [term()], CompiledMatchSpec :: comp_match_spec(). match_spec_run_r(_, _, _) -> @@ -512,12 +514,17 @@ update_counter(_, _, _, _) -> update_element(_, _, _) -> erlang:nif_error(undef). +-spec whereis(TableName) -> tid() | undefined when + TableName :: atom(). +whereis(_) -> + erlang:nif_error(undef). + %%% End of BIFs -opaque comp_match_spec() :: reference(). -spec match_spec_run(List, CompiledMatchSpec) -> list() when - List :: [tuple()], + List :: [term()], CompiledMatchSpec :: comp_match_spec(). match_spec_run(List, CompiledMS) -> @@ -882,10 +889,10 @@ tab2file(Tab, File, Options) -> _ = disk_log:close(Name), _ = file:delete(File), exit(ExReason); - error:ErReason -> + error:ErReason:StackTrace -> _ = disk_log:close(Name), _ = file:delete(File), - erlang:raise(error,ErReason,erlang:get_stacktrace()) + erlang:raise(error,ErReason,StackTrace) end catch throw:TReason2 -> @@ -1060,9 +1067,9 @@ file2tab(File, Opts) -> exit:ExReason -> ets:delete(Tab), exit(ExReason); - error:ErReason -> + error:ErReason:StackTrace -> ets:delete(Tab), - erlang:raise(error,ErReason,erlang:get_stacktrace()) + erlang:raise(error,ErReason,StackTrace) end after _ = disk_log:close(Name) diff --git a/lib/stdlib/src/file_sorter.erl b/lib/stdlib/src/file_sorter.erl index 3aeaff8dc4..7f74e71136 100644 --- a/lib/stdlib/src/file_sorter.erl +++ b/lib/stdlib/src/file_sorter.erl @@ -1314,9 +1314,9 @@ infun(W) -> {cont, W#w{in = NFun}, Objs}; Error -> error(Error, W1) - catch Class:Reason -> + catch Class:Reason:Stacktrace -> cleanup(W1), - erlang:raise(Class, Reason, erlang:get_stacktrace()) + erlang:raise(Class, Reason, Stacktrace) end. outfun(A, #w{inout_value = Val} = W) when Val =/= no_value -> @@ -1336,9 +1336,9 @@ outfun(A, W) -> W#w{out = NF}; Error -> error(Error, W1) - catch Class:Reason -> + catch Class:Reason:Stacktrace -> cleanup(W1), - erlang:raise(Class, Reason, erlang:get_stacktrace()) + erlang:raise(Class, Reason, Stacktrace) end. is_keypos(Keypos) when is_integer(Keypos), Keypos > 0 -> diff --git a/lib/stdlib/src/gen_server.erl b/lib/stdlib/src/gen_server.erl index ac172325b5..77a46419f6 100644 --- a/lib/stdlib/src/gen_server.erl +++ b/lib/stdlib/src/gen_server.erl @@ -109,7 +109,7 @@ -define( STACKTRACE(), - try throw(ok) catch _ -> erlang:get_stacktrace() end). + element(2, erlang:process_info(self(), current_stacktrace))). %%%========================================================================= %%% API @@ -369,7 +369,7 @@ init_it(Mod, Args) -> {ok, Mod:init(Args)} catch throw:R -> {ok, R}; - Class:R -> {'EXIT', Class, R, erlang:get_stacktrace()} + Class:R:S -> {'EXIT', Class, R, S} end. %%%======================================================================== @@ -634,7 +634,7 @@ try_dispatch(Mod, Func, Msg, State) -> catch throw:R -> {ok, R}; - error:undef = R when Func == handle_info -> + error:undef = R:Stacktrace when Func == handle_info -> case erlang:function_exported(Mod, handle_info, 2) of false -> error_logger:warning_msg("** Undefined handle_info in ~p~n" @@ -642,10 +642,10 @@ try_dispatch(Mod, Func, Msg, State) -> [Mod, Msg]), {ok, {noreply, State}}; true -> - {'EXIT', error, R, erlang:get_stacktrace()} + {'EXIT', error, R, Stacktrace} end; - Class:R -> - {'EXIT', Class, R, erlang:get_stacktrace()} + Class:R:Stacktrace -> + {'EXIT', Class, R, Stacktrace} end. try_handle_call(Mod, Msg, From, State) -> @@ -654,8 +654,8 @@ try_handle_call(Mod, Msg, From, State) -> catch throw:R -> {ok, R}; - Class:R -> - {'EXIT', Class, R, erlang:get_stacktrace()} + Class:R:Stacktrace -> + {'EXIT', Class, R, Stacktrace} end. try_terminate(Mod, Reason, State) -> @@ -666,8 +666,8 @@ try_terminate(Mod, Reason, State) -> catch throw:R -> {ok, R}; - Class:R -> - {'EXIT', Class, R, erlang:get_stacktrace()} + Class:R:Stacktrace -> + {'EXIT', Class, R, Stacktrace} end; false -> {ok, ok} diff --git a/lib/stdlib/src/gen_statem.erl b/lib/stdlib/src/gen_statem.erl index 1a7736fc7e..f95b2ea9cd 100644 --- a/lib/stdlib/src/gen_statem.erl +++ b/lib/stdlib/src/gen_statem.erl @@ -365,7 +365,7 @@ event_type(Type) -> -define( STACKTRACE(), - try throw(ok) catch _ -> erlang:get_stacktrace() end). + element(2, erlang:process_info(self(), current_stacktrace))). -define(not_sys_debug, []). %% @@ -590,11 +590,11 @@ call_dirty(ServerRef, Request, Timeout, T) -> {ok,Reply} -> Reply catch - Class:Reason -> + Class:Reason:Stacktrace -> erlang:raise( Class, {Reason,{?MODULE,call,[ServerRef,Request,Timeout]}}, - erlang:get_stacktrace()) + Stacktrace) end. call_clean(ServerRef, Request, Timeout, T) -> @@ -608,9 +608,8 @@ call_clean(ServerRef, Request, Timeout, T) -> ServerRef, '$gen_call', Request, T) of Result -> {Ref,Result} - catch Class:Reason -> - {Ref,Class,Reason, - erlang:get_stacktrace()} + catch Class:Reason:Stacktrace -> + {Ref,Class,Reason,Stacktrace} end end), Mref = monitor(process, Pid), @@ -697,8 +696,7 @@ init_it(Starter, Parent, ServerRef, Module, Args, Opts) -> catch Result -> init_result(Starter, Parent, ServerRef, Module, Result, Opts); - Class:Reason -> - Stacktrace = erlang:get_stacktrace(), + Class:Reason:Stacktrace -> Name = gen:get_proc_name(ServerRef), gen:unregister_name(ServerRef), proc_lib:init_ack(Starter, {error,Reason}), @@ -1584,8 +1582,8 @@ call_callback_mode(#state{module = Module} = S) -> catch CallbackMode -> callback_mode_result(S, CallbackMode); - Class:Reason -> - [Class,Reason,erlang:get_stacktrace()] + Class:Reason:Stacktrace -> + [Class,Reason,Stacktrace] end. callback_mode_result(S, CallbackMode) -> @@ -1638,8 +1636,8 @@ call_state_function( catch Result -> {Result,S}; - Class:Reason -> - [Class,Reason,erlang:get_stacktrace()] + Class:Reason:Stacktrace -> + [Class,Reason,Stacktrace] end. @@ -1827,8 +1825,7 @@ terminate( _ -> ok catch _ -> ok; - C:R -> - ST = erlang:get_stacktrace(), + C:R:ST -> error_info( C, R, ST, S, Q, format_status(terminate, get(), S)), diff --git a/lib/stdlib/src/proc_lib.erl b/lib/stdlib/src/proc_lib.erl index 8e10cbe93b..1991585c13 100644 --- a/lib/stdlib/src/proc_lib.erl +++ b/lib/stdlib/src/proc_lib.erl @@ -231,8 +231,8 @@ init_p(Parent, Ancestors, Fun) when is_function(Fun) -> try Fun() catch - Class:Reason -> - exit_p(Class, Reason, erlang:get_stacktrace()) + Class:Reason:Stacktrace -> + exit_p(Class, Reason, Stacktrace) end. -spec init_p(pid(), [pid()], atom(), atom(), [term()]) -> term(). @@ -246,8 +246,8 @@ init_p_do_apply(M, F, A) -> try apply(M, F, A) catch - Class:Reason -> - exit_p(Class, Reason, erlang:get_stacktrace()) + Class:Reason:Stacktrace -> + exit_p(Class, Reason, Stacktrace) end. -spec wake_up(atom(), atom(), [term()]) -> term(). @@ -256,8 +256,8 @@ wake_up(M, F, A) when is_atom(M), is_atom(F), is_list(A) -> try apply(M, F, A) catch - Class:Reason -> - exit_p(Class, Reason, erlang:get_stacktrace()) + Class:Reason:Stacktrace -> + exit_p(Class, Reason, Stacktrace) end. exit_p(Class, Reason, Stacktrace) -> diff --git a/lib/stdlib/src/qlc.erl b/lib/stdlib/src/qlc.erl index f11f9d0a0b..3a66f6930b 100644 --- a/lib/stdlib/src/qlc.erl +++ b/lib/stdlib/src/qlc.erl @@ -301,11 +301,11 @@ eval(QH, Options) -> post_funs(Post) end end - catch Term -> - case erlang:get_stacktrace() of + catch throw:Term:Stacktrace -> + case Stacktrace of [?THROWN_ERROR | _] -> Term; - Stacktrace -> + _ -> erlang:raise(throw, Term, Stacktrace) end end @@ -359,11 +359,11 @@ fold(Fun, Acc0, QH, Options) -> post_funs(Post) end end - catch Term -> - case erlang:get_stacktrace() of + catch throw:Term:Stacktrace -> + case Stacktrace of [?THROWN_ERROR | _] -> Term; - Stacktrace -> + _ -> erlang:raise(throw, Term, Stacktrace) end end @@ -457,11 +457,11 @@ info(QH, Options) -> debug -> % Not documented. Intended for testing only. Info end - catch Term -> - case erlang:get_stacktrace() of + catch throw:Term:Stacktrace -> + case Stacktrace of [?THROWN_ERROR | _] -> Term; - Stacktrace -> + _ -> erlang:raise(throw, Term, Stacktrace) end end @@ -1056,9 +1056,9 @@ cursor_process(H, GUnique, GCache, TmpDir, SpawnOptions, MaxList, TmpUsage) -> Prep = prepare_qlc(H, not_a_list, GUnique, GCache, TmpDir, MaxList, TmpUsage), setup_qlc(Prep, Setup) - catch Class:Reason -> - Parent ! {self(), {caught, Class, Reason, - erlang:get_stacktrace()}}, + catch Class:Reason:Stacktrace -> + Parent ! {self(), + {caught, Class, Reason, Stacktrace}}, exit(normal) end, Parent ! {self(), ok}, @@ -1075,8 +1075,8 @@ parent_fun(Pid, Parent) -> {TPid, {parent_fun, Fun}} -> V = try {value, Fun()} - catch Class:Reason -> - {parent_fun_caught, Class, Reason, erlang:get_stacktrace()} + catch Class:Reason:Stacktrace -> + {parent_fun_caught, Class, Reason, Stacktrace} end, TPid ! {Parent, V}, parent_fun(Pid, Parent); @@ -1101,9 +1101,9 @@ reply(Parent, MonRef, Post, Cont) -> throw_error(Cont) end catch - Class:Reason -> + Class:Reason:Stacktrace -> post_funs(Post), - Message = {caught, Class, Reason, erlang:get_stacktrace()}, + Message = {caught, Class, Reason, Stacktrace}, Parent ! {self(), Message}, exit(normal) end, @@ -1392,9 +1392,8 @@ next_loop(Pid, L, N) when N =/= 0 -> {caught, throw, Error, [?THROWN_ERROR | _]} -> Error; {caught, Class, Reason, Stacktrace} -> - CurrentStacktrace = try erlang:error(foo) - catch error:_ -> erlang:get_stacktrace() - end, + {current_stacktrace, CurrentStacktrace} = + erlang:process_info(self(), current_stacktrace), erlang:raise(Class, Reason, Stacktrace ++ CurrentStacktrace); error -> erlang:error({qlc_cursor_pid_no_longer_exists, Pid}) @@ -2627,9 +2626,9 @@ table_handle(#qlc_table{trav_fun = TraverseFun, trav_MS = TravMS, Parent =:= self() -> try ParentFun() - catch Class:Reason -> + catch Class:Reason:Stacktrace -> post_funs(Post), - erlang:raise(Class, Reason, erlang:get_stacktrace()) + erlang:raise(Class, Reason, Stacktrace) end; true -> case monitor_request(Parent, {parent_fun, ParentFun}) of @@ -3033,9 +3032,9 @@ file_sort_handle(H, Kp, SortOptions, TmpDir, Compressed, Post, LocalPost) -> {terms, BTerms} -> try {[binary_to_term(B) || B <- BTerms], Post, LocalPost} - catch Class:Reason -> + catch Class:Reason:Stacktrace -> post_funs(Post), - erlang:raise(Class, Reason, erlang:get_stacktrace()) + erlang:raise(Class, Reason, Stacktrace) end end. @@ -3045,9 +3044,9 @@ do_sort(In, Out, Sort, SortOptions, Post) -> {error, Reason} -> throw_reason(Reason); Reply -> Reply end - catch Class:Term -> + catch Class:Term:Stacktrace -> post_funs(Post), - erlang:raise(Class, Term, erlang:get_stacktrace()) + erlang:raise(Class, Term, Stacktrace) end. do_sort(In, Out, sort, SortOptions) -> @@ -3797,9 +3796,9 @@ call(undefined, _Arg, Default, _Post) -> call(Fun, Arg, _Default, Post) -> try Fun(Arg) - catch Class:Reason -> + catch Class:Reason:Stacktrace -> post_funs(Post), - erlang:raise(Class, Reason, erlang:get_stacktrace()) + erlang:raise(Class, Reason, Stacktrace) end. grd(undefined, _Arg) -> diff --git a/lib/stdlib/src/shell.erl b/lib/stdlib/src/shell.erl index ad4984b64c..e4153e7899 100644 --- a/lib/stdlib/src/shell.erl +++ b/lib/stdlib/src/shell.erl @@ -645,8 +645,7 @@ eval_exprs(Es, Shell, Bs0, RT, Lf, Ef, W) -> catch exit:normal -> exit(normal); - Class:Reason -> - Stacktrace = erlang:get_stacktrace(), + Class:Reason:Stacktrace -> M = {self(),Class,{Reason,Stacktrace}}, case do_catch(Class, Reason) of true -> @@ -807,8 +806,8 @@ restrict_handlers(RShMod, Shell, RT) -> -define(BAD_RETURN(M, F, V), try erlang:error(reason) - catch _:_ -> erlang:raise(exit, {restricted_shell_bad_return,V}, - [{M,F,3} | erlang:get_stacktrace()]) + catch _:_:S -> erlang:raise(exit, {restricted_shell_bad_return,V}, + [{M,F,3} | S]) end). local_allowed(F, As, RShMod, Bs, Shell, RT) when is_atom(F) -> diff --git a/lib/stdlib/src/string.erl b/lib/stdlib/src/string.erl index e01bb7d85e..4e89819e41 100644 --- a/lib/stdlib/src/string.erl +++ b/lib/stdlib/src/string.erl @@ -420,10 +420,12 @@ to_number(_, Number, Rest, _, Tail) -> %% Return the remaining string with prefix removed or else nomatch -spec prefix(String::unicode:chardata(), Prefix::unicode:chardata()) -> 'nomatch' | unicode:chardata(). -prefix(Str, []) -> Str; prefix(Str, Prefix0) -> - Prefix = unicode:characters_to_list(Prefix0), - case prefix_1(Str, Prefix) of + Result = case unicode:characters_to_list(Prefix0) of + [] -> Str; + Prefix -> prefix_1(Str, Prefix) + end, + case Result of [] when is_binary(Str) -> <<>>; Res -> Res end. diff --git a/lib/stdlib/src/uri_string.erl b/lib/stdlib/src/uri_string.erl index a84679c595..28d36ea229 100644 --- a/lib/stdlib/src/uri_string.erl +++ b/lib/stdlib/src/uri_string.erl @@ -227,7 +227,7 @@ %% External API %%------------------------------------------------------------------------- -export([compose_query/1, compose_query/2, - dissect_query/1, normalize/1, parse/1, + dissect_query/1, normalize/1, normalize/2, parse/1, recompose/1, transcode/2]). -export_type([error/0, uri_map/0, uri_string/0]). @@ -292,18 +292,36 @@ %%------------------------------------------------------------------------- %% Normalize URIs %%------------------------------------------------------------------------- --spec normalize(URIString) -> NormalizedURI when - URIString :: uri_string(), - NormalizedURI :: uri_string(). -normalize(URIString) -> - %% Percent-encoding normalization and case normalization for - %% percent-encoded triplets are achieved by running parse and - %% recompose on the input URI string. - recompose( - normalize_path_segment( - normalize_scheme_based( - normalize_case( - parse(URIString))))). +-spec normalize(URI) -> NormalizedURI when + URI :: uri_string() | uri_map(), + NormalizedURI :: uri_string() + | error(). +normalize(URIMap) -> + normalize(URIMap, []). + + +-spec normalize(URI, Options) -> NormalizedURI when + URI :: uri_string() | uri_map(), + Options :: [return_map], + NormalizedURI :: uri_string() | uri_map(). +normalize(URIMap, []) when is_map(URIMap) -> + recompose(normalize_map(URIMap)); +normalize(URIMap, [return_map]) when is_map(URIMap) -> + normalize_map(URIMap); +normalize(URIString, []) -> + case parse(URIString) of + Value when is_map(Value) -> + recompose(normalize_map(Value)); + Error -> + Error + end; +normalize(URIString, [return_map]) -> + case parse(URIString) of + Value when is_map(Value) -> + normalize_map(Value); + Error -> + Error + end. %%------------------------------------------------------------------------- @@ -385,7 +403,8 @@ transcode(URIString, Options) when is_list(URIString) -> %%------------------------------------------------------------------------- %% Functions for working with the query part of a URI as a list %% of key/value pairs. -%% HTML5 - 4.10.22.6 URL-encoded form data +%% HTML 5.2 - 4.10.21.6 URL-encoded form data - WHATWG URL (10 Jan 2018) - UTF-8 +%% HTML 5.0 - 4.10.22.6 URL-encoded form data - non UTF-8 %%------------------------------------------------------------------------- %%------------------------------------------------------------------------- @@ -393,7 +412,7 @@ transcode(URIString, Options) when is_list(URIString) -> %% (application/x-www-form-urlencoded encoding algorithm) %%------------------------------------------------------------------------- -spec compose_query(QueryList) -> QueryString when - QueryList :: [{uri_string(), uri_string()}], + QueryList :: [{unicode:chardata(), unicode:chardata()}], QueryString :: uri_string() | error(). compose_query(List) -> @@ -401,7 +420,7 @@ compose_query(List) -> -spec compose_query(QueryList, Options) -> QueryString when - QueryList :: [{uri_string(), uri_string()}], + QueryList :: [{unicode:chardata(), unicode:chardata()}], Options :: [{encoding, atom()}], QueryString :: uri_string() | error(). @@ -432,7 +451,7 @@ compose_query([], _Options, IsList, Acc) -> %%------------------------------------------------------------------------- -spec dissect_query(QueryString) -> QueryList when QueryString :: uri_string(), - QueryList :: [{uri_string(), uri_string()}] + QueryList :: [{unicode:chardata(), unicode:chardata()}] | error(). dissect_query(<<>>) -> []; @@ -1755,7 +1774,8 @@ get_separator(_L) -> <<"&">>. -%% HTML5 - 4.10.22.6 URL-encoded form data - encoding +%% HTML 5.2 - 4.10.21.6 URL-encoded form data - WHATWG URL (10 Jan 2018) - UTF-8 +%% HTML 5.0 - 4.10.22.6 URL-encoded form data - encoding (non UTF-8) form_urlencode(Cs, [{encoding, latin1}]) when is_list(Cs) -> B = convert_to_binary(Cs, utf8, utf8), html5_byte_encode(base10_encode(B)); @@ -1850,7 +1870,8 @@ dissect_query_value(<<>>, IsList, Acc, Key, Value) -> lists:reverse([{K,V}|Acc]). -%% Form-urldecode input based on RFC 1866 [8.2.1] +%% HTML 5.2 - 4.10.21.6 URL-encoded form data - WHATWG URL (10 Jan 2018) - UTF-8 +%% HTML 5.0 - 4.10.22.6 URL-encoded form data - decoding (non UTF-8) form_urldecode(true, B) -> Result = base10_decode(form_urldecode(B, <<>>)), convert_to_list(Result, utf8); @@ -1903,6 +1924,12 @@ base10_decode_unicode(<<H,_/binary>>, _, _) -> %% Helper functions for normalize %%------------------------------------------------------------------------- +normalize_map(URIMap) -> + normalize_path_segment( + normalize_scheme_based( + normalize_case(URIMap))). + + %% 6.2.2.1. Case Normalization normalize_case(#{scheme := Scheme, host := Host} = Map) -> Map#{scheme => to_lower(Scheme), diff --git a/lib/stdlib/src/zip.erl b/lib/stdlib/src/zip.erl index 81f927f399..39be2abff6 100644 --- a/lib/stdlib/src/zip.erl +++ b/lib/stdlib/src/zip.erl @@ -457,8 +457,7 @@ do_zip(F, Files, Options) -> Out3 = Output({close, F}, Out2), {ok, Out3} catch - C:R -> - Stk = erlang:get_stacktrace(), + C:R:Stk -> zlib:close(Z), Output({close, F}, Out0), erlang:raise(C, R, Stk) diff --git a/lib/stdlib/test/array_SUITE.erl b/lib/stdlib/test/array_SUITE.erl index 5836f275ba..956582c4fd 100644 --- a/lib/stdlib/test/array_SUITE.erl +++ b/lib/stdlib/test/array_SUITE.erl @@ -141,10 +141,10 @@ t(What) -> io:format("Test ~p ~n",[T]), try ?MODULE:T([]) - catch _E:_R -> + catch _E:_R:_S -> Line = get(test_server_loc), io:format("Failed ~p:~p ~p ~p~n ~p~n", - [T,Line,_E,_R, erlang:get_stacktrace()]) + [T,Line,_E,_R,_S]) end end, What). @@ -161,8 +161,8 @@ extract_tests() -> end, [Call(Test) || Test <- Tests], io:format("Tests ~p~n", [Tests]) - catch _:Err -> - io:format("Error: ~p ~p~n", [Err, erlang:get_stacktrace()]) + catch _:Err:Stacktrace -> + io:format("Error: ~p ~p~n", [Err, Stacktrace]) end, file:close(In), file:close(Out). diff --git a/lib/stdlib/test/error_logger_h_SUITE.erl b/lib/stdlib/test/error_logger_h_SUITE.erl index 1f2a9fda0b..9dc04f27a1 100644 --- a/lib/stdlib/test/error_logger_h_SUITE.erl +++ b/lib/stdlib/test/error_logger_h_SUITE.erl @@ -257,8 +257,7 @@ match_output([Item|T], Lines0, AtNode, Depth) -> Lines -> match_output(T, Lines, AtNode, Depth) catch - C:E -> - Stk = erlang:get_stacktrace(), + C:E:Stk -> io:format("ITEM: ~p", [Item]), io:format("LINES: ~p", [Lines0]), erlang:raise(C, E, Stk) diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 07c8b60cbd..8b651f4b43 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -78,6 +78,7 @@ -export([ets_all/1]). -export([massive_ets_all/1]). -export([take/1]). +-export([whereis_table/1]). -export([init_per_testcase/2, end_per_testcase/2]). %% Convenience for manual testing @@ -137,7 +138,8 @@ all() -> otp_9423, ets_all, massive_ets_all, - take]. + take, + whereis_table]. groups() -> [{new, [], @@ -4099,6 +4101,7 @@ info_do(Opts) -> {value, {keypos, 2}} = lists:keysearch(keypos, 1, Res), {value, {protection, protected}} = lists:keysearch(protection, 1, Res), + {value, {id, Tab}} = lists:keysearch(id, 1, Res), true = ets:delete(Tab), undefined = ets:info(non_existing_table_xxyy), undefined = ets:info(non_existing_table_xxyy,type), @@ -5892,6 +5895,36 @@ take(Config) when is_list(Config) -> ets:delete(T3), ok. +whereis_table(Config) when is_list(Config) -> + %% Do we return 'undefined' when the named table doesn't exist? + undefined = ets:whereis(whereis_test), + + %% Does the tid() refer to the same table as the name? + whereis_test = ets:new(whereis_test, [named_table]), + Tid = ets:whereis(whereis_test), + + ets:insert(whereis_test, [{hello}, {there}]), + + [[{hello}],[{there}]] = ets:match(whereis_test, '$1'), + [[{hello}],[{there}]] = ets:match(Tid, '$1'), + + true = ets:delete_all_objects(Tid), + + [] = ets:match(whereis_test, '$1'), + [] = ets:match(Tid, '$1'), + + %% Does the name disappear when deleted through the tid()? + true = ets:delete(Tid), + undefined = ets:info(whereis_test), + {'EXIT',{badarg, _}} = (catch ets:match(whereis_test, '$1')), + + %% Is the old tid() broken when the table is re-created with the same + %% name? + whereis_test = ets:new(whereis_test, [named_table]), + [] = ets:match(whereis_test, '$1'), + {'EXIT',{badarg, _}} = (catch ets:match(Tid, '$1')), + + ok. %% %% Utility functions: @@ -6023,17 +6056,23 @@ etsmem() -> end}, {Mem,AllTabs}. -verify_etsmem({MemInfo,AllTabs}) -> + +verify_etsmem(MI) -> wait_for_test_procs(), + verify_etsmem(MI, 1). + +verify_etsmem({MemInfo,AllTabs}, Try) -> case etsmem() of {MemInfo,_} -> io:format("Ets mem info: ~p", [MemInfo]), - case MemInfo of - {ErlMem,EtsAlloc} when ErlMem == notsup; EtsAlloc == undefined -> + case {MemInfo, Try} of + {{ErlMem,EtsAlloc},_} when ErlMem == notsup; EtsAlloc == undefined -> %% Use 'erl +Mea max' to do more complete memory leak testing. {comment,"Incomplete or no mem leak testing"}; - _ -> - ok + {_, 1} -> + ok; + _ -> + {comment, "Transient memory discrepancy"} end; {MemInfo2, AllTabs2} -> @@ -6041,7 +6080,15 @@ verify_etsmem({MemInfo,AllTabs}) -> io:format("Actual: ~p", [MemInfo2]), io:format("Changed tables before: ~p\n",[AllTabs -- AllTabs2]), io:format("Changed tables after: ~p\n", [AllTabs2 -- AllTabs]), - ct:fail("Failed memory check") + case Try < 2 of + true -> + io:format("\nThis discrepancy could be caused by an " + "inconsistent memory \"snapshot\"" + "\nTry again...\n", []), + verify_etsmem({MemInfo, AllTabs}, Try+1); + false -> + ct:fail("Failed memory check") + end end. diff --git a/lib/stdlib/test/gen_statem_SUITE.erl b/lib/stdlib/test/gen_statem_SUITE.erl index c747db475a..7c8a386116 100644 --- a/lib/stdlib/test/gen_statem_SUITE.erl +++ b/lib/stdlib/test/gen_statem_SUITE.erl @@ -2040,9 +2040,9 @@ handle_event(Type, Event, State, Data) -> Result -> wrap_result(Result) catch - throw:Result -> + throw:Result:Stacktrace -> erlang:raise( - throw, wrap_result(Result), erlang:get_stacktrace()) + throw, wrap_result(Result), Stacktrace) end. unwrap_state([State]) -> diff --git a/lib/stdlib/test/proc_lib_SUITE.erl b/lib/stdlib/test/proc_lib_SUITE.erl index 7686889360..fbdcb518b2 100644 --- a/lib/stdlib/test/proc_lib_SUITE.erl +++ b/lib/stdlib/test/proc_lib_SUITE.erl @@ -446,8 +446,8 @@ init_dont_hang(Config) when is_list(Config) -> StartLinkRes = proc_lib:start(?MODULE, init_dont_hang_init, [self()], 1000), StartLinkRes = proc_lib:start(?MODULE, init_dont_hang_init, [self()], 1000, []), ok - catch _:Error -> - io:format("Error ~p /= ~p ~n",[erlang:get_stacktrace(), StartLinkRes]), + catch _:Error:Stacktrace -> + io:format("Error ~p /= ~p ~n",[Stacktrace, StartLinkRes]), exit(Error) end. diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl index 3d3241b33d..d753d929f5 100644 --- a/lib/stdlib/test/rand_SUITE.erl +++ b/lib/stdlib/test/rand_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2017. All Rights Reserved. +%% Copyright Ericsson AB 2000-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -52,7 +52,9 @@ all() -> [seed, interval_int, interval_float, api_eq, reference, - {group, basic_stats}, uniform_real_conv, + {group, basic_stats}, + {group, distr_stats}, + uniform_real_conv, plugin, measure, {group, reference_jump} ]. @@ -60,8 +62,9 @@ all() -> groups() -> [{basic_stats, [parallel], [basic_stats_uniform_1, basic_stats_uniform_2, - basic_stats_standard_normal, - stats_standard_normal_box_muller, + basic_stats_standard_normal]}, + {distr_stats, [parallel], + [stats_standard_normal_box_muller, stats_standard_normal_box_muller_2, stats_standard_normal]}, {reference_jump, [parallel], @@ -70,6 +73,9 @@ groups() -> group(basic_stats) -> %% valgrind needs a lot of time [{timetrap,{minutes,10}}]; +group(distr_stats) -> + %% valgrind needs a lot of time + [{timetrap,{minutes,10}}]; group(reference_jump) -> %% valgrind needs a lot of time [{timetrap,{minutes,10}}]. @@ -82,9 +88,9 @@ test() -> try ok = ?MODULE:Test([]), io:format("~p: ok~n", [Test]) - catch _:Reason -> + catch _:Reason:Stacktrace -> io:format("Failed: ~p: ~p ~p~n", - [Test, Reason, erlang:get_stacktrace()]) + [Test, Reason, Stacktrace]) end end, Tests). @@ -98,8 +104,8 @@ seed(Config) when is_list(Config) -> Algs = algs(), Test = fun(Alg) -> try seed_1(Alg) - catch _:Reason -> - ct:fail({Alg, Reason, erlang:get_stacktrace()}) + catch _:Reason:Stacktrace -> + ct:fail({Alg, Reason, Stacktrace}) end end, [Test(Alg) || Alg <- Algs], @@ -437,7 +443,7 @@ stats_standard_normal_box_muller(Config) when is_list(Config) -> {Z, [S]} end, State = [rand:seed(exrop)], - stats_standard_normal(NormalS, State) + stats_standard_normal(NormalS, State, 3) catch error:_ -> {skip, "math:erfc/1 not supported"} end. @@ -462,7 +468,7 @@ stats_standard_normal_box_muller_2(Config) when is_list(Config) -> {Z, [S]} end, State = [rand:seed(exrop)], - stats_standard_normal(NormalS, State) + stats_standard_normal(NormalS, State, 3) catch error:_ -> {skip, "math:erfc/1 not supported"} end. @@ -472,21 +478,21 @@ stats_standard_normal(Config) when is_list(Config) -> try math:erfc(1.0) of _ -> stats_standard_normal( - fun rand:normal_s/1, rand:seed_s(exrop)) + fun rand:normal_s/1, rand:seed_s(exrop), 3) catch error:_ -> {skip, "math:erfc/1 not supported"} end. %% -stats_standard_normal(Fun, S) -> +stats_standard_normal(Fun, S, Retries) -> %%% %%% ct config: -%%% {rand_SUITE, [{stats_standard_normal,[{seconds, 8}, {std_devs, 4.2}]}]}. +%%% {rand_SUITE, [{stats_standard_normal,[{seconds, 8}, {std_devs, 4.0}]}]}. %%% Seconds = ct:get_config({?MODULE, ?FUNCTION_NAME, seconds}, 8), StdDevs = ct:get_config( {?MODULE, ?FUNCTION_NAME, std_devs}, - 4.2), % probability erfc(4.2/sqrt(2)) (1/37465) to fail a bucket + 4.0), % probability erfc(4.0/sqrt(2)) (1/15787) to fail a bucket %%% ct:timetrap({seconds, Seconds + 120}), %% Buckets is chosen to get a range where the the probability to land @@ -505,11 +511,11 @@ stats_standard_normal(Fun, S) -> P0 = math:erf(1 / W), Rounds = TargetHits * ceil(1.0 / P0), Histogram = array:new({default, 0}), - StopTime = erlang:monotonic_time(second) + Seconds, ct:pal( "Running standard normal test against ~w std devs for ~w seconds...", [StdDevs, Seconds]), - {PositiveHistogram, NegativeHistogram, Outlier, TotalRounds} = + StopTime = erlang:monotonic_time(second) + Seconds, + {PositiveHistogram, NegativeHistogram, Outlier, TotalRounds, NewS} = stats_standard_normal( InvDelta, Buckets, Histogram, Histogram, 0.0, Fun, S, Rounds, StopTime, Rounds, 0), @@ -522,16 +528,33 @@ stats_standard_normal(Fun, S) -> "Total rounds: ~w, tolerance: 1/~.2f..1/~.2f, " "outlier: ~.2f, probability 1/~.2f.", [TotalRounds, Precision, TopPrecision, Outlier, InvOP]), - {TotalRounds, [], []} = - {TotalRounds, + case + {bucket_error, TotalRounds, check_histogram( W, TotalRounds, StdDevs, PositiveHistogram, Buckets), check_histogram( - W, TotalRounds, StdDevs, NegativeHistogram, Buckets)}, - %% If the probability for getting this Outlier is lower than 1/50, - %% then this is fishy! - true = (1/50 =< OutlierProbability), - {comment, {tp, TopPrecision, op, InvOP}}. + W, TotalRounds, StdDevs, NegativeHistogram, Buckets)} + of + {_, _, [], []} when InvOP < 100 -> + {comment, {tp, TopPrecision, op, InvOP}}; + {_, _, [], []} -> + %% If the probability for getting this Outlier is lower than + %% 1/100, then this is fishy! + stats_standard_normal( + Fun, NewS, Retries, {outlier_fishy, InvOP}); + BucketErrors -> + stats_standard_normal( + Fun, NewS, Retries, BucketErrors) + end. +%% +stats_standard_normal(Fun, S, Retries, Failure) -> + case Retries - 1 of + 0 -> + ct:fail(Failure); + NewRetries -> + ct:pal("Retry due to TC glitch: ~p", [Failure]), + stats_standard_normal(Fun, S, NewRetries) + end. %% stats_standard_normal( InvDelta, Buckets, PositiveHistogram, NegativeHistogram, Outlier, @@ -544,7 +567,7 @@ stats_standard_normal( Fun, S, Rounds, StopTime, Rounds, TotalRounds + Rounds); _ -> {PositiveHistogram, NegativeHistogram, - Outlier, TotalRounds + Rounds} + Outlier, TotalRounds + Rounds, S} end; stats_standard_normal( InvDelta, Buckets, PositiveHistogram, NegativeHistogram, Outlier, @@ -571,9 +594,6 @@ increment_bucket(Bucket, Array) -> array:set(Bucket, array:get(Bucket, Array) + 1, Array). check_histogram(W, Rounds, StdDevs, Histogram, Buckets) -> - %%PrevBucket = 512, - %%Bucket = PrevBucket - 1, - %%P = 0.5 * math:erfc(PrevBucket / W), TargetP = 0.5 * math:erfc(Buckets / W), P = 0.0, N = 0, @@ -592,7 +612,7 @@ check_histogram( P = 0.5 * math:erfc(Bucket / W), BucketP = P - PrevP, if - TargetP =< BucketP -> + BucketP < TargetP -> check_histogram( W, Rounds, StdDevs, Histogram, TargetP, Bucket - 1, PrevBucket, PrevP, N); @@ -604,7 +624,7 @@ check_histogram( UpperLimit = ceil(Exp + Threshold), if N < LowerLimit; UpperLimit < N -> - [#{bucket => {Bucket, PrevBucket}, n => N, exp => Exp, + [#{bucket => {Bucket, PrevBucket}, n => N, lower => LowerLimit, upper => UpperLimit} | check_histogram( W, Rounds, StdDevs, Histogram, TargetP, @@ -722,12 +742,12 @@ uniform_real_conv_check(M, E, Gen) -> [["16#",integer_to_list(G,16),$\s]||G<-Gen]]), ct:fail({neq, FF, F}) catch - Error:Reason -> + Error:Reason:Stacktrace -> ct:pal( "~w:~p ~s: ~s~n", [Error, Reason, rand:float2str(F), [["16#",integer_to_list(G,16),$\s]||G<-Gen]]), - ct:fail({Error, Reason, F, erlang:get_stacktrace()}) + ct:fail({Error, Reason, F, Stacktrace}) end. diff --git a/lib/stdlib/test/re_SUITE.erl b/lib/stdlib/test/re_SUITE.erl index 71f86e32e5..7b82647416 100644 --- a/lib/stdlib/test/re_SUITE.erl +++ b/lib/stdlib/test/re_SUITE.erl @@ -894,10 +894,13 @@ match_limit(Config) when is_list(Config) -> %% Test that we get sub-binaries if subject is a binary and we capture %% binaries. sub_binaries(Config) when is_list(Config) -> - Bin = list_to_binary(lists:seq(1,255)), - {match,[B,C]}=re:run(Bin,"(a)",[{capture,all,binary}]), - 255 = binary:referenced_byte_size(B), - 255 = binary:referenced_byte_size(C), - {match,[D]}=re:run(Bin,"(a)",[{capture,[1],binary}]), - 255 = binary:referenced_byte_size(D), + %% The GC can auto-convert tiny sub-binaries to heap binaries, so we + %% extract large sequences to make the test more stable. + Bin = << <<I>> || I <- lists:seq(1, 4096) >>, + {match,[B,C]}=re:run(Bin,"a(.+)$",[{capture,all,binary}]), + true = byte_size(B) =/= byte_size(C), + 4096 = binary:referenced_byte_size(B), + 4096 = binary:referenced_byte_size(C), + {match,[D]}=re:run(Bin,"a(.+)$",[{capture,[1],binary}]), + 4096 = binary:referenced_byte_size(D), ok. diff --git a/lib/stdlib/test/string_SUITE.erl b/lib/stdlib/test/string_SUITE.erl index d02a6eac0a..fdff2d24b8 100644 --- a/lib/stdlib/test/string_SUITE.erl +++ b/lib/stdlib/test/string_SUITE.erl @@ -486,6 +486,10 @@ to_float(_) -> prefix(_) -> ?TEST("", ["a"], nomatch), ?TEST("a", [""], "a"), + ?TEST("a", [[[]]], "a"), + ?TEST("a", [<<>>], "a"), + ?TEST("a", [[<<>>]], "a"), + ?TEST("a", [[[<<>>]]], "a"), ?TEST("b", ["a"], nomatch), ?TEST("a", ["a"], ""), ?TEST("å", ["a"], nomatch), @@ -877,9 +881,9 @@ test_1(Line, Func, Str, Args, Exp) -> catch error:Exp -> ok; - error:Reason -> + error:Reason:Stacktrace -> io:format("~p:~p: Crash ~p ~p~n", - [?MODULE,Line, Reason, erlang:get_stacktrace()]), + [?MODULE,Line, Reason, Stacktrace]), exit({error, Func}) end. @@ -944,10 +948,10 @@ check_types(Line, Func, [Str|_], Res) -> io:format("Failed: ~p ~p: ~p ~p~n",[Line, Func, T1, T2]), io:format(" ~p => ~p~n", [Str, Res]), error; - _:Reason -> - io:format("Crash: ~p in~n ~p~n",[Reason, erlang:get_stacktrace()]), + _:Reason:Stacktrace -> + io:format("Crash: ~p in~n ~p~n",[Reason, Stacktrace]), io:format("Failed: ~p ~p: ~p => ~p~n", [Line, Func, Str, Res]), - exit({Reason, erlang:get_stacktrace()}) + exit({Reason, Stacktrace}) end. check_types_1(T, T) -> diff --git a/lib/stdlib/test/tar_SUITE.erl b/lib/stdlib/test/tar_SUITE.erl index 4061008812..32a33283d1 100644 --- a/lib/stdlib/test/tar_SUITE.erl +++ b/lib/stdlib/test/tar_SUITE.erl @@ -28,7 +28,7 @@ extract_from_open_file/1, symlinks/1, open_add_close/1, cooked_compressed/1, memory/1,unicode/1,read_other_implementations/1, sparse/1, init/1, leading_slash/1, dotdot/1, - roundtrip_metadata/1]). + roundtrip_metadata/1, apply_file_info_opts/1]). -include_lib("common_test/include/ct.hrl"). -include_lib("kernel/include/file.hrl"). @@ -42,7 +42,8 @@ all() -> extract_filtered, symlinks, open_add_close, cooked_compressed, memory, unicode, read_other_implementations, - sparse,init,leading_slash,dotdot,roundtrip_metadata]. + sparse,init,leading_slash,dotdot,roundtrip_metadata, + apply_file_info_opts]. groups() -> []. @@ -989,6 +990,31 @@ do_roundtrip_metadata(Dir, File) -> ok end. +apply_file_info_opts(Config) when is_list(Config) -> + ok = file:set_cwd(proplists:get_value(priv_dir, Config)), + + ok = file:make_dir("empty_directory"), + ok = file:write_file("file", "contents"), + + Opts = [{atime, 0}, {mtime, 0}, {ctime, 0}, {uid, 0}, {gid, 0}], + TarFile = "reproducible.tar", + {ok, Tar} = erl_tar:open(TarFile, [write]), + ok = erl_tar:add(Tar, "file", Opts), + ok = erl_tar:add(Tar, "empty_directory", Opts), + ok = erl_tar:add(Tar, <<"contents">>, "memory_file", Opts), + erl_tar:close(Tar), + + ok = file:make_dir("extracted"), + erl_tar:extract(TarFile, [{cwd, "extracted"}]), + + {ok, #file_info{mtime=0}} = + file:read_file_info("extracted/empty_directory", [{time, posix}]), + {ok, #file_info{mtime=0}} = + file:read_file_info("extracted/file", [{time, posix}]), + {ok, #file_info{mtime=0}} = + file:read_file_info("extracted/memory_file", [{time, posix}]), + + ok. %% Delete the given list of files. delete_files([]) -> ok; diff --git a/lib/stdlib/test/unicode_util_SUITE.erl b/lib/stdlib/test/unicode_util_SUITE.erl index 632d9ae6e6..40b1c260a5 100644 --- a/lib/stdlib/test/unicode_util_SUITE.erl +++ b/lib/stdlib/test/unicode_util_SUITE.erl @@ -136,10 +136,10 @@ verify_gc(Line0, N, Acc) -> io:format("Expected: ~p~n", [Res]), io:format("Got: ~w~n", [Other]), Acc+1; - Cl:R -> + Cl:R:Stacktrace -> io:format("~p: ~ts => |~tp|~n",[N, Line, Str]), io:format("Expected: ~p~n", [Res]), - erlang:raise(Cl,R,erlang:get_stacktrace()) + erlang:raise(Cl,R,Stacktrace) end. gc_test_data([[247]|Rest], Str, [First|GCs]) -> @@ -175,29 +175,29 @@ verify_nfd(Data0, LineNo, _Acc) -> C3GC = fetch(C1, fun unicode_util:nfd/1), C3GC = fetch(C2, fun unicode_util:nfd/1), C3GC = fetch(C3, fun unicode_util:nfd/1) - catch _Cl:{badmatch, Other} = _R-> + catch _Cl:{badmatch, Other} = _R: Stacktrace -> io:format("Failed: ~p~nInput: ~ts~n\t=> ~w |~ts|~n",[LineNo, Data1, C1, C1]), io:format("Expected: ~ts ~w~n", [C3GC, C3GC]), io:format("Got: ~ts ~w~n", [Other, Other]), - erlang:raise(_Cl,_R,erlang:get_stacktrace()); - Cl:R -> + erlang:raise(_Cl,_R,Stacktrace); + Cl:R:Stacktrace -> io:format("~p: ~ts => |~tp|~n",[LineNo, Data1, C1]), io:format("Expected: ~p~n", [C3]), - erlang:raise(Cl,R,erlang:get_stacktrace()) + erlang:raise(Cl,R,Stacktrace) end, C5GC = fetch(C5, fun unicode_util:gc/1), try C5GC = fetch(C4, fun unicode_util:nfd/1), C5GC = fetch(C5, fun unicode_util:nfd/1) - catch _Cl2:{badmatch, Other2} = _R2-> + catch _Cl2:{badmatch, Other2} = _R2:Stacktrace2 -> io:format("Failed: ~p~nInput: ~ts~n\t=> ~w |~ts|~n",[LineNo, Data1, C1, C1]), io:format("Expected: ~ts ~w~n", [C5GC, C5GC]), io:format("Got: ~ts ~w~n", [Other2, Other2]), - erlang:raise(_Cl2,_R2,erlang:get_stacktrace()); - Cl2:R2 -> + erlang:raise(_Cl2,_R2,Stacktrace2); + Cl2:R2:Stacktrace2 -> io:format("~p: ~ts => |~tp|~n",[LineNo, Data1, C1]), io:format("Expected: ~p~n", [C5]), - erlang:raise(Cl2,R2,erlang:get_stacktrace()) + erlang:raise(Cl2,R2,Stacktrace2) end, ok. @@ -218,29 +218,29 @@ verify_nfc(Data0, LineNo, _Acc) -> C2GC = fetch(C1, fun unicode_util:nfc/1), C2GC = fetch(C2, fun unicode_util:nfc/1), C2GC = fetch(C3, fun unicode_util:nfc/1) - catch _Cl:{badmatch, Other} = _R-> + catch _Cl:{badmatch, Other} = _R:Stacktrace -> io:format("Failed: ~p~nInput: ~ts~n\t=> ~w |~ts|~n",[LineNo, Data1, C1, C1]), io:format("Expected: ~ts ~w~n", [C2GC, C2GC]), io:format("Got: ~ts ~w~n", [Other, Other]), - erlang:raise(_Cl,_R,erlang:get_stacktrace()); - Cl:R -> + erlang:raise(_Cl,_R,Stacktrace); + Cl:R:Stacktrace -> io:format("~p: ~ts => |~tp|~n",[LineNo, Data1, C1]), io:format("Expected: ~p~n", [C3]), - erlang:raise(Cl,R,erlang:get_stacktrace()) + erlang:raise(Cl,R,Stacktrace) end, C4GC = fetch(C4, fun unicode_util:gc/1), try C4GC = fetch(C4, fun unicode_util:nfc/1), C4GC = fetch(C5, fun unicode_util:nfc/1) - catch _Cl2:{badmatch, Other2} = _R2-> + catch _Cl2:{badmatch, Other2} = _R2:Stacktrace2 -> io:format("Failed: ~p~nInput: ~ts~n\t=> ~w |~ts|~n",[LineNo, Data1, C1, C1]), io:format("Expected: ~ts ~w~n", [C4GC, C4GC]), io:format("Got: ~ts ~w~n", [Other2, Other2]), - erlang:raise(_Cl2,_R2,erlang:get_stacktrace()); - Cl2:R2 -> + erlang:raise(_Cl2,_R2,Stacktrace2); + Cl2:R2:Stacktrace2 -> io:format("~p: ~ts => |~tp|~n",[LineNo, Data1, C1]), io:format("Expected: ~p~n", [C5]), - erlang:raise(Cl2,R2,erlang:get_stacktrace()) + erlang:raise(Cl2,R2,Stacktrace2) end, ok. @@ -263,15 +263,15 @@ verify_nfkd(Data0, LineNo, _Acc) -> C5GC = lists:flatten(fetch(C3, fun unicode_util:nfkd/1)), C5GC = lists:flatten(fetch(C4, fun unicode_util:nfkd/1)), C5GC = lists:flatten(fetch(C5, fun unicode_util:nfkd/1)) - catch _Cl:{badmatch, Other} = _R-> + catch _Cl:{badmatch, Other} = _R:Stacktrace -> io:format("Failed: ~p~nInput: ~ts~n\t=> ~w |~ts|~n",[LineNo, Data1, C5, C5]), io:format("Expected: ~ts ~w~n", [C5GC, C5GC]), io:format("Got: ~ts ~w~n", [Other, Other]), - erlang:raise(_Cl,_R,erlang:get_stacktrace()); - Cl:R -> + erlang:raise(_Cl,_R,Stacktrace); + Cl:R:Stacktrace -> io:format("~p: ~ts => |~tp|~n",[LineNo, Data1, C1]), io:format("Expected: ~p~n", [C3]), - erlang:raise(Cl,R,erlang:get_stacktrace()) + erlang:raise(Cl,R,Stacktrace) end, ok. @@ -296,15 +296,15 @@ verify_nfkc(Data0, LineNo, _Acc) -> C4GC = lists:flatten(fetch(C4, fun unicode_util:nfkc/1)), C4GC = lists:flatten(fetch(C5, fun unicode_util:nfkc/1)) - catch _Cl:{badmatch, Other} = _R-> + catch _Cl:{badmatch, Other} = _R:Stacktrace -> io:format("Failed: ~p~nInput: ~ts~n\t=> ~w |~ts|~n",[LineNo, Data1, C4, C4]), io:format("Expected: ~ts ~w~n", [C4GC, C4GC]), io:format("Got: ~ts ~w~n", [Other, Other]), - erlang:raise(_Cl,_R,erlang:get_stacktrace()); - Cl:R -> + erlang:raise(_Cl,_R,Stacktrace); + Cl:R:Stacktrace -> io:format("~p: ~ts => |~tp|~n",[LineNo, Data1, C1]), io:format("Expected: ~p~n", [C3]), - erlang:raise(Cl,R,erlang:get_stacktrace()) + erlang:raise(Cl,R,Stacktrace) end, ok. diff --git a/lib/stdlib/test/uri_string_SUITE.erl b/lib/stdlib/test/uri_string_SUITE.erl index fef356355c..92f8bb3292 100644 --- a/lib/stdlib/test/uri_string_SUITE.erl +++ b/lib/stdlib/test/uri_string_SUITE.erl @@ -22,7 +22,7 @@ -include_lib("common_test/include/ct.hrl"). -export([all/0, suite/0,groups/0, - normalize/1, + normalize/1, normalize_map/1, normalize_return_map/1, normalize_negative/1, parse_binary_fragment/1, parse_binary_host/1, parse_binary_host_ipv4/1, parse_binary_host_ipv6/1, parse_binary_path/1, parse_binary_pct_encoded_fragment/1, parse_binary_pct_encoded_query/1, @@ -68,6 +68,9 @@ suite() -> all() -> [ normalize, + normalize_map, + normalize_return_map, + normalize_negative, parse_binary_scheme, parse_binary_userinfo, parse_binary_pct_encoded_userinfo, @@ -912,6 +915,56 @@ normalize(_Config) -> <<"tftp://localhost">> = uri_string:normalize(<<"tftp://localhost:69">>). +normalize_map(_Config) -> + "/a/g" = uri_string:normalize(#{path => "/a/b/c/./../../g"}), + <<"mid/6">> = uri_string:normalize(#{path => <<"mid/content=5/../6">>}), + "http://localhost-%C3%B6rebro/a/g" = + uri_string:normalize(#{scheme => "http",port => 80,path => "/a/b/c/./../../g", + host => "localhost-örebro"}), + <<"http://localhost-%C3%B6rebro/a/g">> = + uri_string:normalize(#{scheme => <<"http">>,port => 80, + path => <<"/a/b/c/./../../g">>, + host => <<"localhost-örebro"/utf8>>}), + <<"https://localhost/">> = + uri_string:normalize(#{scheme => <<"https">>,port => 443,path => <<>>, + host => <<"localhost">>}), + <<"https://localhost:445/">> = + uri_string:normalize(#{scheme => <<"https">>,port => 445,path => <<>>, + host => <<"localhost">>}), + <<"ftp://localhost">> = + uri_string:normalize(#{scheme => <<"ftp">>,port => 21,path => <<>>, + host => <<"localhost">>}), + <<"ssh://localhost">> = + uri_string:normalize(#{scheme => <<"ssh">>,port => 22,path => <<>>, + host => <<"localhost">>}), + <<"sftp://localhost">> = + uri_string:normalize(#{scheme => <<"sftp">>,port => 22,path => <<>>, + host => <<"localhost">>}), + <<"tftp://localhost">> = + uri_string:normalize(#{scheme => <<"tftp">>,port => 69,path => <<>>, + host => <<"localhost">>}). + +normalize_return_map(_Config) -> + #{scheme := "http",path := "/a/g",host := "localhost-örebro"} = + uri_string:normalize("http://localhos%74-%c3%b6rebro:80/a/b/c/./../../g", + [return_map]), + #{scheme := <<"http">>,path := <<"/a/g">>, host := <<"localhost-örebro"/utf8>>} = + uri_string:normalize(<<"http://localhos%74-%c3%b6rebro:80/a/b/c/./../../g">>, + [return_map]), + #{scheme := <<"https">>,path := <<"/">>, host := <<"localhost">>} = + uri_string:normalize(#{scheme => <<"https">>,port => 443,path => <<>>, + host => <<"localhost">>}, [return_map]). + +normalize_negative(_Config) -> + {error,invalid_uri,":"} = + uri_string:normalize("http://local>host"), + {error,invalid_uri,":"} = + uri_string:normalize(<<"http://local>host">>), + {error,invalid_uri,":"} = + uri_string:normalize("http://[192.168.0.1]", [return_map]), + {error,invalid_uri,":"} = + uri_string:normalize(<<"http://[192.168.0.1]">>, [return_map]). + interop_query_utf8(_Config) -> Q = uri_string:compose_query([{"foo bar","1"}, {"合", "2"}]), Uri = uri_string:recompose(#{path => "/", query => Q}), diff --git a/lib/stdlib/test/zip_SUITE.erl b/lib/stdlib/test/zip_SUITE.erl index 1dfcda4ed0..e5ba629c55 100644 --- a/lib/stdlib/test/zip_SUITE.erl +++ b/lib/stdlib/test/zip_SUITE.erl @@ -909,8 +909,7 @@ do_fd_leak(Bad, N) -> ok -> do_fd_leak(Bad, N + 1) catch - C:R -> - Stk = erlang:get_stacktrace(), + C:R:Stk -> io:format("Bad error after ~p attempts\n", [N]), erlang:raise(C, R, Stk) end. diff --git a/lib/tools/doc/src/lcnt.xml b/lib/tools/doc/src/lcnt.xml index 5bdfc60448..0c24375b91 100644 --- a/lib/tools/doc/src/lcnt.xml +++ b/lib/tools/doc/src/lcnt.xml @@ -371,7 +371,7 @@ <v>Serial = integer()</v> </type> <desc> - <p>Creates a process id with creation 0. Example:</p> + <p>Creates a process id with creation 0.</p> </desc> </func> diff --git a/lib/tools/emacs/Makefile b/lib/tools/emacs/Makefile index 35c93ba4ed..ea4d6cb723 100644 --- a/lib/tools/emacs/Makefile +++ b/lib/tools/emacs/Makefile @@ -54,8 +54,6 @@ EL_FILES = $(EMACS_FILES:%=%.el) ELC_FILES = $(EMACS_FILES:%=%.elc) -TEST_FILES = test.erl.indented test.erl.orig - # ---------------------------------------------------- # Targets # ---------------------------------------------------- @@ -75,7 +73,7 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_spec: opt $(INSTALL_DIR) "$(RELSYSDIR)/emacs" - $(INSTALL_DATA) $(EL_FILES) $(README_FILES) $(TEST_FILES) \ + $(INSTALL_DATA) $(EL_FILES) $(README_FILES) \ "$(RELSYSDIR)/emacs" ifeq ($(DOCTYPE),pdf) @@ -89,19 +87,3 @@ release_docs_spec: docs $(INSTALL_DATA) $(MAN_FILES) "$(RELEASE_PATH)/man/man3" endif endif - -EMACS ?= emacs - -test_indentation: - @rm -f test.erl - @rm -f test_indent.el - @echo '(load "erlang-start")' >> test_indent.el - @echo '(find-file "test.erl.orig")' >> test_indent.el - @echo "(require 'cl) ; required with Emacs < 23 for ignore-errors" >> test_indent.el - @echo '(erlang-mode)' >> test_indent.el - @echo '(toggle-debug-on-error)' >> test_indent.el - @echo '(erlang-indent-current-buffer)' >> test_indent.el - @echo '(write-file "test.erl")' >> test_indent.el - $(EMACS) --batch -Q -L . -l test_indent.el - diff -u test.erl.indented test.erl - @echo "No differences between expected and actual indentation" diff --git a/lib/tools/emacs/erlang-skels.el b/lib/tools/emacs/erlang-skels.el index bdb3d9ad4a..534f50ab33 100644 --- a/lib/tools/emacs/erlang-skels.el +++ b/lib/tools/emacs/erlang-skels.el @@ -279,7 +279,8 @@ Please see the function `tempo-define-template'.") '((erlang-skel-include erlang-skel-large-header) "-behaviour(application)." n n "%% Application callbacks" n - "-export([start/2, stop/1])." n n + "-export([start/2, start_phase/3, stop/1, prep_stop/1," n> + "config_change/3])." n n (erlang-skel-double-separator-start 3) "%%% Application callbacks" n (erlang-skel-double-separator-end 3) n @@ -291,13 +292,14 @@ Please see the function `tempo-define-template'.") "%% application. If the application is structured according to the OTP" n "%% design principles as a supervision tree, this means starting the" n "%% top supervisor of the tree." n - "%%" n - "%% @spec start(StartType, StartArgs) -> {ok, Pid} |" n - "%% {ok, Pid, State} |" n - "%% {error, Reason}" n - "%% StartType = normal | {takeover, Node} | {failover, Node}" n - "%% StartArgs = term()" n (erlang-skel-separator-end 2) + "-spec start(StartType :: normal |" n> + "{takeover, Node :: node()} |" n> + "{failover, Node :: node()}," n> + "StartArgs :: term()) ->" n> + "{ok, Pid :: pid()} |" n> + "{ok, Pid :: pid(), State :: term()} |" n> + "{error, Reason :: term()}." n "start(_StartType, _StartArgs) ->" n> "case 'TopSupervisor':start_link() of" n> "{ok, Pid} ->" n> @@ -309,15 +311,52 @@ Please see the function `tempo-define-template'.") (erlang-skel-separator-start 2) "%% @private" n "%% @doc" n + "%% top supervisor of the tree." n + "%% Starts an application with included applications, when" n + "%% synchronization is needed between processes in the different" n + "%% applications during startup." + (erlang-skel-separator-end 2) + "-spec start_phase(Phase :: atom()," n> + "StartType :: normal |" n> + "{takeover, Node :: node()} |" n> + "{failover, Node :: node()}," n> + "PhaseArgs :: term()) -> ok | {error, Reason :: term()}." n + "start_phase(_Phase, _StartType, _PhaseArgs) ->" n> + "ok."n + n + (erlang-skel-separator-start 2) + "%% @private" n + "%% @doc" n "%% This function is called whenever an application has stopped. It" n "%% is intended to be the opposite of Module:start/2 and should do" n "%% any necessary cleaning up. The return value is ignored." n - "%%" n - "%% @spec stop(State) -> void()" n (erlang-skel-separator-end 2) + "-spec stop(State :: term()) -> any()." n "stop(_State) ->" n> "ok." n n + (erlang-skel-separator-start 2) + "%% @private" n + "%% @doc" n + "%% This function is called when an application is about to be stopped," n + "%% before shutting down the processes of the application." n + (erlang-skel-separator-end 2) + "-spec prep_stop(State :: term()) -> NewState :: term()." n + "prep_stop(State) ->" n> + "State." n + n + (erlang-skel-separator-start 2) + "%% @private" n + "%% @doc" n + "%% This function is called by an application after a code replacement," n + "%% if the configuration parameters have changed." n + (erlang-skel-separator-end 2) + "-spec config_change(Changed :: [{Par :: atom(), Val :: term()}]," n> + "New :: [{Par :: atom(), Val :: term()}]," n> + "Removed :: [Par :: atom()]) -> ok." n + "config_change(_Changed, _New, _Removed) ->" n> + "ok." n + n (erlang-skel-double-separator-start 3) "%%% Internal functions" n (erlang-skel-double-separator-end 3) @@ -343,9 +382,12 @@ Please see the function `tempo-define-template'.") (erlang-skel-separator-start 2) "%% @doc" n "%% Starts the supervisor" n - "%%" n - "%% @spec start_link() -> {ok, Pid} | ignore | {error, Error}" n (erlang-skel-separator-end 2) + "-spec start_link() -> {ok, Pid :: pid()} |" n> + "{error, {already_started, Pid :: pid()}} |" n> + "{error, {shutdown, term()}} |" n> + "{error, term()} |" n> + "ignore." n "start_link() ->" n> "supervisor:start_link({local, ?SERVER}, ?MODULE, [])." n n @@ -359,11 +401,11 @@ Please see the function `tempo-define-template'.") "%% this function is called by the new process to find out about" n "%% restart strategy, maximum restart intensity, and child" n "%% specifications." n - "%%" n - "%% @spec init(Args) -> {ok, {SupFlags, [ChildSpec]}} |" n - "%% ignore |" n - "%% {error, Reason}" n (erlang-skel-separator-end 2) + "-spec init(Args :: term()) ->" n> + "{ok, {SupFlags :: supervisor:sup_flags()," n> + "[ChildSpec :: supervisor:child_spec()]}} |" n> + "ignore." n "init([]) ->" n "" n> "SupFlags = #{strategy => one_for_one," n> @@ -406,9 +448,11 @@ Please see the function `tempo-define-template'.") (erlang-skel-separator-start 2) "%% @doc" n "%% Starts the supervisor bridge" n - "%%" n - "%% @spec start_link() -> {ok, Pid} | ignore | {error, Error}" n (erlang-skel-separator-end 2) + "-spec start_link() -> {ok, Pid :: pid()} |" n> + "{error, {already_started, Pid :: pid()}} |" n> + "{error, term()} |" n> + "ignore." n "start_link() ->" n> "supervisor_bridge:start_link({local, ?SERVER}, ?MODULE, [])." n n @@ -422,11 +466,10 @@ Please see the function `tempo-define-template'.") "%% which calls Module:init/1 to start the subsystem. To ensure a" n "%% synchronized start-up procedure, this function does not return" n "%% until Module:init/1 has returned." n - "%%" n - "%% @spec init(Args) -> {ok, Pid, State} |" n - "%% ignore |" n - "%% {error, Reason}" n (erlang-skel-separator-end 2) + "-spec init(Args :: term()) -> {ok, Pid :: pid(), State :: term()} |" n> + "{error, Error :: term()} |" n> + "ignore." n "init([]) ->" n> "case 'AModule':start_link() of" n> "{ok, Pid} ->" n> @@ -442,10 +485,9 @@ Please see the function `tempo-define-template'.") "%% to terminate. It should be the opposite of Module:init/1 and stop" n "%% the subsystem and do any necessary cleaning up.The return value is" n "%% ignored." n - "%%" n - "%% @spec terminate(Reason, State) -> void()" n (erlang-skel-separator-end 2) - "terminate(Reason, State) ->" n> + "-spec terminate(Reason :: shutdown | term(), State :: term()) -> any()." n + "terminate(_Reason, _State) ->" n> "'AModule':stop()," n> "ok." n n @@ -464,9 +506,8 @@ Please see the function `tempo-define-template'.") "-export([start_link/0])." n n "%% gen_server callbacks" n - "-export([init/1, handle_call/3, handle_cast/2, " - "handle_info/2," n> - "terminate/2, code_change/3])." n n + "-export([init/1, handle_call/3, handle_cast/2, handle_info/2," n> + "terminate/2, code_change/3, format_status/2])." n n "-define(SERVER, ?MODULE)." n n @@ -478,9 +519,11 @@ Please see the function `tempo-define-template'.") (erlang-skel-separator-start 2) "%% @doc" n "%% Starts the server" n - "%%" n - "%% @spec start_link() -> {ok, Pid} | ignore | {error, Error}" n (erlang-skel-separator-end 2) + "-spec start_link() -> {ok, Pid :: pid()} |" n> + "{error, Error :: {already_started, pid()}} |" n> + "{error, Error :: term()} |" n> + "ignore." n "start_link() ->" n> "gen_server:start_link({local, ?SERVER}, ?MODULE, [], [])." n n @@ -492,12 +535,12 @@ Please see the function `tempo-define-template'.") "%% @private" n "%% @doc" n "%% Initializes the server" n - "%%" n - "%% @spec init(Args) -> {ok, State} |" n - "%% {ok, State, Timeout} |" n - "%% ignore |" n - "%% {stop, Reason}" n (erlang-skel-separator-end 2) + "-spec init(Args :: term()) -> {ok, State :: term()} |" n> + "{ok, State :: term(), Timeout :: timeout()} |" n> + "{ok, State :: term(), hibernate} |" n> + "{stop, Reason :: term()} |" n> + "ignore." n "init([]) ->" n> "process_flag(trap_exit, true)," n> "{ok, #state{}}." n @@ -506,15 +549,16 @@ Please see the function `tempo-define-template'.") "%% @private" n "%% @doc" n "%% Handling call messages" n - "%%" n - "%% @spec handle_call(Request, From, State) ->" n - "%% {reply, Reply, State} |" n - "%% {reply, Reply, State, Timeout} |" n - "%% {noreply, State} |" n - "%% {noreply, State, Timeout} |" n - "%% {stop, Reason, Reply, State} |" n - "%% {stop, Reason, State}" n (erlang-skel-separator-end 2) + "-spec handle_call(Request :: term(), From :: {pid(), term()}, State :: term()) ->" n> + "{reply, Reply :: term(), NewState :: term()} |" n> + "{reply, Reply :: term(), NewState :: term(), Timeout :: timeout()} |" n> + "{reply, Reply :: term(), NewState :: term(), hibernate} |" n> + "{noreply, NewState :: term()} |" n> + "{noreply, NewState :: term(), Timeout :: timeout()} |" n> + "{noreply, NewState :: term(), hibernate} |" n> + "{stop, Reason :: term(), Reply :: term(), NewState :: term()} |" n> + "{stop, Reason :: term(), NewState :: term()}." n "handle_call(_Request, _From, State) ->" n> "Reply = ok," n> "{reply, Reply, State}." n @@ -523,23 +567,25 @@ Please see the function `tempo-define-template'.") "%% @private" n "%% @doc" n "%% Handling cast messages" n - "%%" n - "%% @spec handle_cast(Msg, State) -> {noreply, State} |" n - "%% {noreply, State, Timeout} |" n - "%% {stop, Reason, State}" n (erlang-skel-separator-end 2) - "handle_cast(_Msg, State) ->" n> + "-spec handle_cast(Request :: term(), State :: term()) ->" n> + "{noreply, NewState :: term()} |" n> + "{noreply, NewState :: term(), Timeout :: timeout()} |" n> + "{noreply, NewState :: term(), hibernate} |" n> + "{stop, Reason :: term(), NewState :: term()}." n + "handle_cast(_Request, State) ->" n> "{noreply, State}." n n (erlang-skel-separator-start 2) "%% @private" n "%% @doc" n "%% Handling all non call/cast messages" n - "%%" n - "%% @spec handle_info(Info, State) -> {noreply, State} |" n - "%% {noreply, State, Timeout} |" n - "%% {stop, Reason, State}" n (erlang-skel-separator-end 2) + "-spec handle_info(Info :: timeout() | term(), State :: term()) ->" n> + "{noreply, NewState :: term()} |" n> + "{noreply, NewState :: term(), Timeout :: timeout()} |" n> + "{noreply, NewState :: term(), hibernate} |" n> + "{stop, Reason :: normal | term(), NewState :: term()}." n "handle_info(_Info, State) ->" n> "{noreply, State}." n n @@ -550,9 +596,9 @@ Please see the function `tempo-define-template'.") "%% terminate. It should be the opposite of Module:init/1 and do any" n "%% necessary cleaning up. When it returns, the gen_server terminates" n "%% with Reason. The return value is ignored." n - "%%" n - "%% @spec terminate(Reason, State) -> void()" n (erlang-skel-separator-end 2) + "-spec terminate(Reason :: normal | shutdown | {shutdown, term()} | term()," n> + "State :: term()) -> any()." n "terminate(_Reason, _State) ->" n> "ok." n n @@ -560,12 +606,26 @@ Please see the function `tempo-define-template'.") "%% @private" n "%% @doc" n "%% Convert process state when code is changed" n - "%%" n - "%% @spec code_change(OldVsn, State, Extra) -> {ok, NewState}" n (erlang-skel-separator-end 2) + "-spec code_change(OldVsn :: term() | {down, term()}," n> + "State :: term()," n> + "Extra :: term()) -> {ok, NewState :: term()} |" n> + "{error, Reason :: term()}." n "code_change(_OldVsn, State, _Extra) ->" n> "{ok, State}." n n + (erlang-skel-separator-start 2) + "%% @private" n + "%% @doc" n + "%% This function is called for changing the form and appearance" n + "%% of gen_server status when it is returned from sys:get_status/1,2" n + "%% or when it appears in termination error logs." n + (erlang-skel-separator-end 2) + "-spec format_status(Opt :: normal | terminate," n> + "Status :: list()) -> Status :: term()." n + "format_status(_Opt, Status) ->" n> + "Status." n + n (erlang-skel-double-separator-start 3) "%%% Internal functions" n (erlang-skel-double-separator-end 3) @@ -581,8 +641,8 @@ Please see the function `tempo-define-template'.") "-export([start_link/0, add_handler/0])." n n "%% gen_event callbacks" n - "-export([init/1, handle_event/2, handle_call/2, " n> - "handle_info/2, terminate/2, code_change/3])." n n + "-export([init/1, handle_event/2, handle_call/2, handle_info/2," n> + "terminate/2, code_change/3, format_status/2])." n n "-define(SERVER, ?MODULE)." n n @@ -594,18 +654,17 @@ Please see the function `tempo-define-template'.") (erlang-skel-separator-start 2) "%% @doc" n "%% Creates an event manager" n - "%%" n - "%% @spec start_link() -> {ok, Pid} | {error, Error}" n (erlang-skel-separator-end 2) + "-spec start_link() -> {ok, Pid :: pid()} |" n> + "{error, Error :: {already_started, pid()} | term()}." n "start_link() ->" n> "gen_event:start_link({local, ?SERVER})." n n (erlang-skel-separator-start 2) "%% @doc" n "%% Adds an event handler" n - "%%" n - "%% @spec add_handler() -> ok | {'EXIT', Reason} | term()" n (erlang-skel-separator-end 2) + "-spec add_handler() -> ok | {'EXIT', Reason :: term()} | term()." n "add_handler() ->" n> "gen_event:add_handler(?SERVER, ?MODULE, [])." n n @@ -617,9 +676,11 @@ Please see the function `tempo-define-template'.") "%% @doc" n "%% Whenever a new event handler is added to an event manager," n "%% this function is called to initialize the event handler." n - "%%" n - "%% @spec init(Args) -> {ok, State}" n (erlang-skel-separator-end 2) + "-spec init(Args :: term() | {Args :: term(), Term :: term()}) ->" n> + "{ok, State :: term()} |" n> + "{ok, State :: term(), hibernate} |" n> + "{error, Reason :: term()}." n "init([]) ->" n> "{ok, #state{}}." n n @@ -629,12 +690,13 @@ Please see the function `tempo-define-template'.") "%% Whenever an event manager receives an event sent using" n "%% gen_event:notify/2 or gen_event:sync_notify/2, this function is" n "%% called for each installed event handler to handle the event." n - "%%" n - "%% @spec handle_event(Event, State) ->" n - "%% {ok, State} |" n - "%% {swap_handler, Args1, State1, Mod2, Args2} |"n - "%% remove_handler" n (erlang-skel-separator-end 2) + "-spec handle_event(Event :: term(), State :: term()) ->" n> + "{ok, NewState :: term()} |" n> + "{ok, NewState :: term(), hibernate} |" n> + "remove_handler |" n> + "{swap_handler, Args1 :: term(), NewState :: term()," n> + "Handler2 :: atom() | {atom(), term()} , Args2 :: term()}." n> "handle_event(_Event, State) ->" n> "{ok, State}." n n @@ -644,12 +706,13 @@ Please see the function `tempo-define-template'.") "%% Whenever an event manager receives a request sent using" n "%% gen_event:call/3,4, this function is called for the specified" n "%% event handler to handle the request." n - "%%" n - "%% @spec handle_call(Request, State) ->" n - "%% {ok, Reply, State} |" n - "%% {swap_handler, Reply, Args1, State1, Mod2, Args2} |" n - "%% {remove_handler, Reply}" n (erlang-skel-separator-end 2) + "-spec handle_call(Request :: term(), State :: term()) ->" n> + "{ok, Reply :: term(), NewState :: term()} |" n> + "{ok, Reply :: term(), NewState :: term(), hibernate} |" n> + "{remove_handler, Reply :: term()} |" n> + "{swap_handler, Reply :: term(), Args1 :: term(), NewState :: term()," n> + "Handler2 :: atom() | {atom(), term()}, Args2 :: term()}." n "handle_call(_Request, State) ->" n> "Reply = ok," n> "{ok, Reply, State}." n @@ -660,12 +723,13 @@ Please see the function `tempo-define-template'.") "%% This function is called for each installed event handler when" n "%% an event manager receives any other message than an event or a" n "%% synchronous request (or a system message)." n - "%%" n - "%% @spec handle_info(Info, State) ->" n - "%% {ok, State} |" n - "%% {swap_handler, Args1, State1, Mod2, Args2} |" n - "%% remove_handler" n (erlang-skel-separator-end 2) + "-spec handle_info(Info :: term(), State :: term()) ->" n> + "{ok, NewState :: term()} |" n> + "{ok, NewState :: term(), hibernate} |" n> + "remove_handler |" n> + "{swap_handler, Args1 :: term(), NewState :: term()," n> + "Handler2 :: atom() | {atom(), term()}, Args2 :: term()}." n "handle_info(_Info, State) ->" n> "{ok, State}." n n @@ -675,22 +739,40 @@ Please see the function `tempo-define-template'.") "%% Whenever an event handler is deleted from an event manager, this" n "%% function is called. It should be the opposite of Module:init/1 and" n "%% do any necessary cleaning up." n - "%%" n - "%% @spec terminate(Reason, State) -> void()" n (erlang-skel-separator-end 2) - "terminate(_Reason, _State) ->" n> + "-spec terminate(Arg :: {stop, Reason :: term()} |" n> + "stop |" n> + "remove_handler |" n> + "{error, {'EXIT', Reason :: term()}} |" n> + "{error, Term :: term()} |" n> + "term()," n> + "State :: term()) -> any()." n + "terminate(_Arg, _State) ->" n> "ok." n n (erlang-skel-separator-start 2) "%% @private" n "%% @doc" n "%% Convert process state when code is changed" n - "%%" n - "%% @spec code_change(OldVsn, State, Extra) -> {ok, NewState}" n (erlang-skel-separator-end 2) + "-spec code_change(OldVsn :: term() | {down, term()}," n> + "State :: term()," n> + "Extra :: term()) -> {ok, NewState :: term()}." n "code_change(_OldVsn, State, _Extra) ->" n> "{ok, State}." n n + (erlang-skel-separator-start 2) + "%% @private" n + "%% @doc" n + "%% This function is called for changing the form and appearance" n + "%% of gen_event status when it is returned from sys:get_status/1,2" n + "%% or when it appears in termination error logs." n + (erlang-skel-separator-end 2) + "-spec format_status(Opt :: normal | terminate," n> + "Status :: list()) -> Status :: term()." n + "format_status(_Opt, Status) ->" n> + "Status." n + n (erlang-skel-double-separator-start 3) "%%% Internal functions" n (erlang-skel-double-separator-end 3) diff --git a/lib/tools/emacs/erlang.el b/lib/tools/emacs/erlang.el index 6b93d63182..b3411c3ce7 100644 --- a/lib/tools/emacs/erlang.el +++ b/lib/tools/emacs/erlang.el @@ -4,7 +4,7 @@ ;; Author: Anders Lindgren ;; Keywords: erlang, languages, processes ;; Date: 2011-12-11 -;; Version: 2.8.0 +;; Version: 2.8.1 ;; Package-Requires: ((emacs "24.1")) ;; %CopyrightBegin% @@ -84,7 +84,7 @@ "The Erlang programming language." :group 'languages) -(defconst erlang-version "2.8.0" +(defconst erlang-version "2.8.1" "The version number of Erlang mode.") (defcustom erlang-root-dir nil @@ -2742,7 +2742,7 @@ Return nil if inside string, t if in a comment." (1+ (nth 2 stack-top))) ((= (char-syntax (following-char)) ?\)) (goto-char (nth 1 stack-top)) - (cond ((looking-at "[({]\\s *\\($\\|%\\)") + (cond ((erlang-record-or-function-args-p) ;; Line ends with parenthesis. (let ((previous (erlang-indent-find-preceding-expr)) (stack-pos (nth 2 stack-top))) @@ -2752,19 +2752,10 @@ Return nil if inside string, t if in a comment." (nth 2 stack-top)))) ((= (following-char) ?,) ;; a comma at the start of the line: line up with opening parenthesis. - (nth 2 stack-top)) + (min (nth 2 stack-top) + (erlang-indent-element stack-top indent-point token))) (t - (goto-char (nth 1 stack-top)) - (let ((base (cond ((looking-at "[({]\\s *\\($\\|%\\)") - ;; Line ends with parenthesis. - (erlang-indent-parenthesis (nth 2 stack-top))) - (t - ;; Indent to the same column as the first - ;; argument. - (goto-char (1+ (nth 1 stack-top))) - (skip-chars-forward " \t") - (current-column))))) - (erlang-indent-standard indent-point token base 't))))) + (erlang-indent-element stack-top indent-point token)))) ;; ((eq (car stack-top) '<<) ;; Element of binary (possible comprehension) expression, @@ -2773,13 +2764,11 @@ Return nil if inside string, t if in a comment." (+ 2 (nth 2 stack-top))) ((looking-at "\\(>>\\)[^_a-zA-Z0-9]") (nth 2 stack-top)) + ((= (following-char) ?,) + (min (+ (nth 2 stack-top) 1) + (- (erlang-indent-to-first-element stack-top 2) 1))) (t - (goto-char (nth 1 stack-top)) - ;; Indent to the same column as the first - ;; argument. - (goto-char (+ 2 (nth 1 stack-top))) - (skip-chars-forward " \t") - (current-column)))) + (erlang-indent-to-first-element stack-top 2)))) ((memq (car stack-top) '(icr fun spec)) ;; The default indentation is the column of the option @@ -2835,12 +2824,13 @@ Return nil if inside string, t if in a comment." (let ((base (erlang-indent-find-base stack indent-point off skip))) ;; Special cases (goto-char indent-point) - (cond ((looking-at "\\(end\\|after\\)\\($\\|[^_a-zA-Z0-9]\\)") + (cond ((looking-at "\\(;\\|end\\|after\\)\\($\\|[^_a-zA-Z0-9]\\)") (if (eq (car stack-top) '->) (erlang-pop stack)) - (if stack - (erlang-caddr (car stack)) - 0)) + (cond ((and stack (looking-at ";")) + (+ (erlang-caddr (car stack)) (- erlang-indent-level 2))) + (stack (erlang-caddr (car stack))) + (t off))) ((looking-at "catch\\b\\($\\|[^_a-zA-Z0-9]\\)") ;; Are we in a try (let ((start (if (eq (car stack-top) '->) @@ -2914,6 +2904,22 @@ Return nil if inside string, t if in a comment." (current-column))) start-alternativ)))))) ))) +(defun erlang-indent-to-first-element (stack-top extra) + ;; Indent to the same column as the first + ;; argument. extra should be 1 for lists tuples or 2 for binaries + (goto-char (+ (nth 1 stack-top) extra)) + (skip-chars-forward " \t") + (current-column)) + +(defun erlang-indent-element (stack-top indent-point token) + (goto-char (nth 1 stack-top)) + (let ((base (cond ((erlang-record-or-function-args-p) + ;; Line ends with parenthesis. + (erlang-indent-parenthesis (nth 2 stack-top))) + (t + (erlang-indent-to-first-element stack-top 1))))) + (erlang-indent-standard indent-point token base 't))) + (defun erlang-indent-standard (indent-point token base inside-parenthesis) "Standard indent when in blocks or tuple or arguments. Look at last thing to see in what state we are, move relative to the base." @@ -2939,6 +2945,9 @@ Return nil if inside string, t if in a comment." ;; Avoid treating comments a continued line. ((= (following-char) ?%) base) + ((and (= (following-char) ?,) inside-parenthesis) + ;; a comma at the start of the line line up with parenthesis + (- base 1)) ;; Continued line (e.g. line beginning ;; with an operator.) (t @@ -3028,11 +3037,21 @@ This assumes that the preceding expression is either simple (t col))) col)))) +(defun erlang-record-or-function-args-p () + (and (looking-at "[({]\\s *\\($\\|%\\)") + (or (eq (following-char) ?\( ) + (save-excursion + (ignore-errors (forward-sexp (- 1))) + (eq (preceding-char) ?#))))) + (defun erlang-indent-parenthesis (stack-position) (let ((previous (erlang-indent-find-preceding-expr))) - (if (> previous stack-position) - (+ stack-position erlang-argument-indent) - (+ previous erlang-argument-indent)))) + (cond ((eq previous stack-position) ;; tuple or map not a record + (1+ stack-position)) + ((> previous stack-position) + (+ stack-position erlang-argument-indent)) + (t + (+ previous erlang-argument-indent))))) (defun erlang-skip-blank (&optional lim) "Skip over whitespace and comments until limit reached." @@ -5166,7 +5185,6 @@ future, a new shell on an already running host will be started." ;; e.g. it does not assume that we are running an inferior ;; Erlang, there exists a lot of other possibilities. - (defvar erlang-shell-buffer-name "*erlang*" "The name of the Erlang link shell buffer.") @@ -5177,46 +5195,28 @@ Also see the description of `ielm-prompt-read-only'." :type 'boolean :package-version '(erlang . "2.8.0")) -(defvar erlang-shell-mode-map nil - "Keymap used by Erlang shells.") - - -(defvar erlang-shell-mode-hook nil - "User functions to run when an Erlang shell is started. - -This hook is used to change the behaviour of Erlang mode. It is -normally used by the user to personalise the programming environment. -When used in a site init file, it could be used to customise Erlang -mode for all users on the system. - -The function added to this hook is run every time a new Erlang -shell is started. +(defvar erlang-shell-mode-map + (let ((map (make-sparse-keymap))) + (define-key map "\M-\t" 'erlang-complete-tag) -See also `erlang-load-hook', a hook which is run once, when Erlang -mode is loaded, and `erlang-mode-hook' which is run every time a new -Erlang source file is loaded into Emacs.") + ;; Normally the other way around. + (define-key map "\C-a" 'comint-bol) + (define-key map "\C-c\C-a" 'beginning-of-line) + (define-key map "\C-d" nil) ; Was `comint-delchar-or-maybe-eof' + (define-key map "\M-\C-m" 'compile-goto-error) + map) + "Keymap used by Erlang shells.") (defvar erlang-input-ring-file-name "~/.erlang_history" "When non-nil, file name used to store Erlang shell history information.") - -(defun erlang-shell-mode () +(define-derived-mode erlang-shell-mode comint-mode "Erlang Shell" "Major mode for interacting with an Erlang shell. -We assume that we already are in Comint mode. - The following special commands are available: \\{erlang-shell-mode-map}" - (interactive) - (setq major-mode 'erlang-shell-mode) - (setq mode-name "Erlang Shell") (erlang-mode-variables) - (if erlang-shell-mode-map - nil - (setq erlang-shell-mode-map (copy-keymap comint-mode-map)) - (erlang-shell-mode-commands erlang-shell-mode-map)) - (use-local-map erlang-shell-mode-map) ;; Needed when compiling directly from the Erlang shell. (setq compilation-last-buffer (current-buffer)) (setq comint-prompt-regexp "^[^>=]*> *") @@ -5230,7 +5230,6 @@ The following special commands are available: 'inferior-erlang-strip-delete nil t) (add-hook 'comint-output-filter-functions 'inferior-erlang-strip-ctrl-m nil t) - (setq comint-input-ring-file-name erlang-input-ring-file-name) (comint-read-input-ring t) (make-local-variable 'kill-buffer-hook) @@ -5249,8 +5248,7 @@ The following special commands are available: (define-key map [menu-bar compilation] (cons "Errors" compilation-menu-map))) map)))) - (erlang-tags-init) - (run-hooks 'erlang-shell-mode-hook)) + (erlang-tags-init)) (defun erlang-mouse-2-command (event) @@ -5272,13 +5270,6 @@ Selects Comint or Compilation mode command as appropriate." (call-interactively (lookup-key compilation-mode-map "\C-m")) (call-interactively (lookup-key comint-mode-map "\C-m")))) -(defun erlang-shell-mode-commands (map) - (define-key map "\M-\t" 'erlang-complete-tag) - (define-key map "\C-a" 'comint-bol) ; Normally the other way around. - (define-key map "\C-c\C-a" 'beginning-of-line) - (define-key map "\C-d" nil) ; Was `comint-delchar-or-maybe-eof' - (define-key map "\M-\C-m" 'compile-goto-error)) - ;;; ;;; Inferior Erlang -- Run an Erlang shell as a subprocess. ;;; diff --git a/lib/tools/emacs/test.erl.indented b/lib/tools/emacs/test.erl.indented deleted file mode 100644 index 14a4eca7c3..0000000000 --- a/lib/tools/emacs/test.erl.indented +++ /dev/null @@ -1,784 +0,0 @@ -%% -*- Mode: erlang; indent-tabs-mode: nil -*- -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2009-2016. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% - -%%%------------------------------------------------------------------- -%%% File : test.erl -%%% Author : Dan Gudmundsson <[email protected]> -%%% Description : Test emacs mode indention and font-locking -%%% this file is intentionally not indented. -%%% Copy the file and indent it and you should end up with test.erl.indented -%%% Created : 6 Oct 2009 by Dan Gudmundsson <[email protected]> -%%%------------------------------------------------------------------- - -%% Start off with syntax highlighting you have to verify this by looking here -%% and see that the code looks alright - --module(test). --compile(export_all). - -%% Used to cause an "Unbalanced parentheses" error. -foo(M) -> - M#{a :=<<"a">> - ,b:=1}. -foo() -> - #{a =><<"a">> - ,b=>1}. - -%% Module attributes should be highlighted - --export([t/1]). --record(record1, {a, - b, - c - }). --record(record2, { - a, - b - }). - --record(record3, {a = 8#42423 bor - 8#4234, - b = 8#5432 - bor 2#1010101 - c = 123 + - 234, - d}). - --record(record4, { - a = 8#42423 bor - 8#4234, - b = 8#5432 - bor 2#1010101 - c = 123 + - 234, - d}). - --record(record5, { a = 1 :: integer() - , b = foobar :: atom() - }). - --define(MACRO_1, macro). --define(MACRO_2(_), macro). - --spec t(integer()) -> any(). - --type ann() :: Var :: integer(). --type ann2() :: Var :: - 'return' - | 'return_white_spaces' - | 'return_comments' - | 'text' | ann(). --type paren() :: - (ann2()). --type t1() :: atom(). --type t2() :: [t1()]. --type t3(Atom) :: integer(Atom). --type t4() :: t3(foobar). --type t5() :: {t1(), t3(foo)}. --type t6() :: 1 | 2 | 3 | - 'foo' | 'bar'. --type t7() :: []. --type t71() :: [_]. --type t8() :: {any(),none(),pid(),port(), - reference(),float()}. --type t9() :: [1|2|3|foo|bar] | - list(a | b | c) | t71(). --type t10() :: {1|2|3|foo|t9()} | {}. --type t11() :: 1..2. --type t13() :: maybe_improper_list(integer(), t11()). --type t14() :: [erl_scan:foo() | - %% Should be highlighted - term() | - bool() | - byte() | - char() | - non_neg_integer() | nonempty_list() | - pos_integer() | - neg_integer() | - number() | - list() | - nonempty_improper_list() | nonempty_maybe_improper_list() | - maybe_improper_list() | string() | iolist() | byte() | - module() | - mfa() | - node() | - timeout() | - no_return() | - %% Should not be highlighted - nonempty_() | nonlist() | - erl_scan:bar(34, 92) | t13() | m:f(integer() | <<_:_*16>>)]. - - --type t15() :: {binary(),<<>>,<<_:34>>,<<_:_*42>>, - <<_:3,_:_*14>>,<<>>} | [<<>>|<<_:34>>|<<_:16>>| - <<_:3,_:_*1472>>|<<_:19,_:_*14>>| <<_:34>>| - <<_:34>>|<<_:34>>|<<_:34>>]. --type t16() :: fun(). --type t17() :: fun((...) -> paren()). --type t18() :: fun(() -> t17() | t16()). --type t19() :: fun((t18()) -> t16()) | - fun((nonempty_maybe_improper_list('integer', any())| - 1|2|3|a|b|<<_:3,_:_*14>>|integer()) -> - nonempty_maybe_improper_list('integer', any())| - 1|2|3|a|b|<<_:3,_:_*14>>|integer()). --type t20() :: [t19(), ...]. --type t21() :: tuple(). --type t21(A) :: A. --type t22() :: t21(integer()). --type t23() :: #rec1{}. --type t24() :: #rec2{a :: t23(), b :: [atom()]}. --type t25() :: #rec3{f123 :: [t24() | - 1|2|3|4|a|b|c|d| - nonempty_maybe_improper_list(integer, any())]}. --type t26() :: #rec4{ a :: integer() - , b :: any() - }. --type t27() :: { integer() - , atom() - }. --type t99() :: - {t2(),t4(),t5(),t6(),t7(),t8(),t10(),t14(), - t15(),t20(),t21(), t22(),t25()}. --spec t1(FooBar :: t99()) -> t99(); - (t2()) -> t2(); - (t4()) -> t4() when is_subtype(t4(), t24); - (t23()) -> t23() when is_subtype(t23(), atom()), - is_subtype(t23(), t14()); - (t24()) -> t24() when is_subtype(t24(), atom()), - is_subtype(t24(), t14()), - is_subtype(t24(), t4()). - --spec over(I :: integer()) -> R1 :: foo:typen(); - (A :: atom()) -> R2 :: foo:atomen(); - (T :: tuple()) -> R3 :: bar:typen(). - --spec mod:t2() -> any(). - --spec handle_cast(Cast :: {'exchange', node(), [[name(),...]]} - | {'del_member', name(), pid()}, - #state{}) -> {'noreply', #state{}}. - --spec handle_cast(Cast :: - {'exchange', node(), [[name(),...]]} - | {'del_member', name(), pid()}, - #state{}) -> {'noreply', #state{}}. - --spec all(fun((T) -> boolean()), List :: [T]) -> - boolean() when is_subtype(T, term()). % (*) - --spec get_closest_pid(term()) -> - Return :: pid() - | {'error', {'no_process', term()} - | {'no_such_group', term()}}. - --spec add( X :: integer() - , Y :: integer() - ) -> integer(). - --opaque attributes_data() :: - [{'column', column()} | {'line', info_line()} | - {'text', string()}] | {line(),column()}. --record(r,{ - f1 :: attributes_data(), - f222 = foo:bar(34, #rec3{}, 234234234423, - aassdsfsdfsdf, 2234242323) :: - [t24() | 1|2|3|4|a|b|c|d| - nonempty_maybe_improper_list(integer, any())], - f333 :: [t24() | 1|2|3|4|a|b|c|d| - nonempty_maybe_improper_list(integer, any())], - f3 = x:y(), - f4 = x:z() :: t99(), - f17 :: 'undefined', - f18 :: 1 | 2 | 'undefined', - f19 = 3 :: integer()|undefined, - f5 = 3 :: undefined|integer()}). - --record(state, { - sequence_number = 1 :: integer() - }). - - -highlighting(X) % Function definitions should be highlighted - when is_integer(X) -> % and so should `when' and `is_integer' be - %% Highlighting - %% Various characters (we keep an `atom' after to see that highlighting ends) - $a,atom, % Characters should be marked - "string",atom, % and strings - 'asdasd',atom, % quote should be atoms?? - 'VaV',atom, - 'aVa',atom, - '\'atom',atom, - 'atom\'',atom, - 'at\'om',atom, - '#1',atom, - - $", atom, % atom should be ok - $', atom, - - "string$", atom, "string$", atom, % currently buggy I know... - "string\$", atom, % workaround for bug above - - "char $in string", atom, - - 'atom$', atom, 'atom$', atom, - 'atom\$', atom, - - 'char $in atom', atom, - - $[, ${, $\\, atom, - ?MACRO_1, - ?MACRO_2(foo), - - %% Numerical constants - 16#DD, % AD Should not be highlighted - 32#dd, % AD Should not be highlighted - 32#ddAB, % AD Should not be highlighted - 32#101, % AD Should not be highlighted - 32#ABTR, % AD Should not be highlighted - - %% Variables - Variables = lists:foo(), - _Variables = lists:foo(), % AD - AppSpec = Xyz/2, - Module42 = Xyz(foo, bar), - Module:foo(), - _Module:foo(), % AD - FooÅÅ = lists:reverse([tl,hd,tl,hd]), % AD Should highlight FooÅÅ - _FooÅÅ = 42, % AD Should highlight _FooÅÅ - - %% Bifs - erlang:registered(), - registered(), - hd(tl(tl(hd([a,b,c])))), - erlang:anything(lists), - %% Guards - is_atom(foo), is_float(2.3), is_integer(32), is_number(4323.3), - is_function(Fun), is_pid(self()), - not_a_guard:is_list([]), - %% Other Types - - atom, % not (currently) hightlighted - 234234, - 234.43, - - [list, are, not, higlighted], - {nor, is, tuple}, - ok. - -%%% -%%% Indentation -%%% - -%%% Left - -%% Indented - - % Right - - -indent_basics(X, Y, Z) - when X > 42, - Z < 13; - Y =:= 4711 -> - %% comments - % right comments - case lists:filter(fun(_, AlongName, - B, - C) -> - true - end, - [a,v,b]) - of - [] -> - Y = 5 * 43, - ok; - [_|_] -> - Y = 5 * 43, - ok - end, - Y, - %% List, tuples and binaries - [a, - b, c - ], - [ a, - b, c - ], - - [ - a, - b - ], - {a, - b,c - }, - { a, - b,c - }, - - { - a, - b - }, - - <<1:8, - 2:8 - >>, - << - 1:8, - 2:8 - >>, - << 1:8, - 2:8 - >>, - - (a, - b, - c - ), - - ( a, - b, - c - ), - - - ( - a, - b, - c - ), - - call(2#42423 bor - #4234, - 2#5432, - other_arg), - ok; -indent_basics(Xlongname, - #struct{a=Foo, - b=Bar}, - [X| - Y]) -> - testing_next_clause, - ok; -indent_basics( % AD added clause - X, % not sure how this should look - Y, - Z) - when - X < 42, Z > 13; - Y =:= 4711 -> - foo; -indent_basics(X, Y, Z) when % AD added clause - X < 42, Z > 13; % testing when indentation - Y =:= 4711 -> - foo; -indent_basics(X, Y, Z) % AD added clause - when % testing when indentation - X < 42, Z > 13; % unsure about this one - Y =:= 4711 -> - foo. - - - -indent_nested() -> - [ - {foo, 2, "string"}, - {bar, 3, "another string"} - ]. - - -indent_icr(Z) -> % icr = if case receive - %% If - if Z >= 0 -> - X = 43 div 4, - foo(X); - Z =< 10 -> - X = 43 div 4, - foo(X); - Z == 5 orelse - Z == 7 -> - X = 43 div 4, - foo(X); - true -> - if_works - end, - %% Case - case {Z, foo, bar} of - {Z,_,_} -> - X = 43 div 4, - foo(X); - {Z,_,_} when - Z =:= 42 -> % AD line should be indented as a when - X = 43 div 4, - foo(X); - {Z,_,_} - when Z < 10 -> % AD when should be indented - X = 43 div 4, - foo(X); - {Z,_,_} - when % AD when should be indented - Z < 10 % and the guards should follow when - andalso % unsure about how though - true -> - X = 43 div 4, - foo(X) - end, - %% begin - begin - sune, - X = 74234 + foo(8456) + - 345 div 43, - ok - end, - - - %% receive - receive - {Z,_,_} -> - X = 43 div 4, - foo(X); - Z -> - X = 43 div 4, - foo(X) - end, - receive - {Z,_,_} -> - X = 43 div 4, - foo(X); - Z % AD added clause - when Z =:= 1 -> % This line should be indented by 2 - X = 43 div 4, - foo(X); - Z when % AD added clause - Z =:= 2 -> % This line should be indented by 2 - X = 43 div 4, - foo(X); - Z -> - X = 43 div 4, - foo(X) - after infinity -> - foo(X), - asd(X), - 5*43 - end, - receive - after 10 -> - foo(X), - asd(X), - 5*43 - end, - ok. - -indent_fun() -> - %% Changed fun to one indention level - Var = spawn(fun(X) - when X == 2; - X > 10 -> - hello, - case Hello() of - true when is_atom(X) -> - foo; - false -> - bar - end; - (Foo) when is_atom(Foo), - is_integer(X) -> - X = 6* 45, - Y = true andalso - kalle - end), - %% check EEP37 named funs - Fn1 = fun Fact(N) when N > 0 -> - F = Fact(N-1), - N * F; - Fact(0) -> - 1 - end, - %% check anonymous funs too - Fn2 = fun(0) -> - 1; - (N) -> - N - end, - ok. - -indent_try_catch() -> - try - io:format(stdout, "Parsing file ~s, ", - [St0#leex.xfile]), - {ok,Line3,REAs,Actions,St3} = - parse_rules(Xfile, Line2, Macs, St2) - catch - exit:{badarg,R} -> - foo(R), - io:format(stdout, - "ERROR reason ~p~n", - R); - error:R % AD added clause - when R =:= 42 -> % when should be indented - foo(R); - error:R % AD added clause - when % when should be indented - R =:= 42 -> % but unsure about this (maybe 2 more) - foo(R); - error:R when % AD added clause - R =:= foo -> % line should be 2 indented (works) - foo(R); - error:R -> - foo(R), - io:format(stdout, - "ERROR reason ~p~n", - R) - after - foo('after'), - file:close(Xfile) - end; -indent_try_catch() -> - try - foo(bar) - of - X when true andalso - kalle -> - io:format(stdout, "Parsing file ~s, ", - [St0#leex.xfile]), - {ok,Line3,REAs,Actions,St3} = - parse_rules(Xfile, Line2, Macs, St2); - X % AD added clause - when false andalso % when should be 2 indented - bengt -> - gurka(); - X when % AD added clause - false andalso % line should be 2 indented - not bengt -> - gurka(); - X -> - io:format(stdout, "Parsing file ~s, ", - [St0#leex.xfile]), - {ok,Line3,REAs,Actions,St3} = - parse_rules(Xfile, Line2, Macs, St2) - catch - exit:{badarg,R} -> - foo(R), - io:format(stdout, - "ERROR reason ~p~n", - R); - error:R -> - foo(R), - io:format(stdout, - "ERROR reason ~p~n", - R) - after - foo('after'), - file:close(Xfile), - bar(with_long_arg, - with_second_arg) - end; -indent_try_catch() -> - try foo() - after - foo(), - bar(with_long_arg, - with_second_arg) - end. - -indent_catch() -> - D = B + - float(43.1), - - B = catch oskar(X), - - A = catch (baz + - bax), - catch foo(), - - C = catch B + - float(43.1), - - case catch foo(X) of - A -> - B - end, - - case - catch foo(X) - of - A -> - B - end, - - case - foo(X) - of - A -> - catch B, - X - end, - - try sune of - _ -> foo - catch _:_ -> baf - end, - - try - sune - of - _ -> - X = 5, - (catch foo(X)), - X + 10 - catch _:_ -> baf - end, - - try - (catch sune) - of - _ -> - catch foo() %% BUGBUG can't handle catch inside try without parentheses - catch _:_ -> - baf - end, - - try - (catch exit()) - catch - _ -> - catch baf() - end, - ok. - -indent_binary() -> - X = lists:foldr(fun(M) -> - <<Ma/binary, " ">> - end, [], A), - A = <<X/binary, 0:8>>, - B. - - -indent_comprehensions() -> - %% I don't have a good idea how we want to handle this - %% but they are here to show how they are indented today. - Result1 = [X || - #record{a=X} <- lists:seq(1, 10), - true = (X rem 2) - ], - Result2 = [X || <<X:32,_:32>> <= <<0:512>>, - true = (X rem 2) - ], - - Binary1 = << <<X:8>> || - #record{a=X} <- lists:seq(1, 10), - true = (X rem 2) - >>, - - Binary2 = << <<X:8>> || <<X:32,_:32>> <= <<0:512>>, - true = (X rem 2) - >>, - ok. - -%% This causes an error in earlier erlang-mode versions. -foo() -> - [#foo{ - foo = foo}]. - -%% Record indentation -some_function_with_a_very_long_name() -> - #'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ - field1=a, - field2=b}, - case dummy_function_with_a_very_very_long_name(x) of - #'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ - field1=a, - field2=b} -> - ok; - Var = #'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ - field1=a, - field2=b} -> - Var#'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ - field1=a, - field2=b}; - #xyz{ - a=1, - b=2} -> - ok - end. - -another_function_with_a_very_very_long_name() -> - #rec{ - field1=1, - field2=1}. - -some_function_name_xyz(xyzzy, #some_record{ - field1=Field1, - field2=Field2}) -> - SomeVariable = f(#'Some-long-record-name'{ - field_a = 1, - 'inter-xyz-parameters' = - #'Some-other-very-long-record-name'{ - field2 = Field1, - field2 = Field2}}), - {ok, SomeVariable}. - -commas_first() -> - {abc, [ {some_var, 1} - , {some_other_var, 2} - , {erlang_ftw, 9} - , {erlang_cookie, 'cookie'} - , {cmds, - [ {one, "sudo ls"} - , {one, "sudo ls"} - , {two, "sudo ls"} - , {three, "sudo ls"} - , {four, "sudo ls"} - , {three, "sudo ls"} - ] } - , {ssh_username, "yow"} - , {cluster, - [ {aaaa, [ {"10.198.55.12" , "" } - , {"10.198.55.13" , "" } - ] } - , {bbbb, [ {"10.198.55.151", "" } - , {"10.198.55.123", "" } - , {"10.198.55.34" , "" } - , {"10.198.55.85" , "" } - , {"10.198.55.67" , "" } - ] } - , {cccc, [ {"10.198.55.68" , "" } - , {"10.198.55.69" , "" } - ] } - ] } - ] - }. - - -%% this used to result in a scan-sexp error -[{ - }]. - -%% this used to result in 2x the correct indentation within the function -%% body, due to the function name being mistaken for a keyword -catcher(N) -> - try generate_exception(N) of - Val -> {N, normal, Val} - catch - throw:X -> {N, caught, thrown, X}; - exit:X -> {N, caught, exited, X}; - error:X -> {N, caught, error, X} - end. diff --git a/lib/tools/emacs/test.erl.orig b/lib/tools/emacs/test.erl.orig deleted file mode 100644 index c0cf1749b6..0000000000 --- a/lib/tools/emacs/test.erl.orig +++ /dev/null @@ -1,784 +0,0 @@ -%% -*- Mode: erlang; indent-tabs-mode: nil -*- -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2009-2016. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% - -%%%------------------------------------------------------------------- -%%% File : test.erl -%%% Author : Dan Gudmundsson <[email protected]> -%%% Description : Test emacs mode indention and font-locking -%%% this file is intentionally not indented. -%%% Copy the file and indent it and you should end up with test.erl.indented -%%% Created : 6 Oct 2009 by Dan Gudmundsson <[email protected]> -%%%------------------------------------------------------------------- - -%% Start off with syntax highlighting you have to verify this by looking here -%% and see that the code looks alright - --module(test). --compile(export_all). - -%% Used to cause an "Unbalanced parentheses" error. -foo(M) -> -M#{a :=<<"a">> -,b:=1}. -foo() -> -#{a =><<"a">> -,b=>1}. - -%% Module attributes should be highlighted - --export([t/1]). --record(record1, {a, - b, - c -}). --record(record2, { - a, - b - }). - --record(record3, {a = 8#42423 bor - 8#4234, - b = 8#5432 - bor 2#1010101 - c = 123 + -234, - d}). - --record(record4, { - a = 8#42423 bor - 8#4234, - b = 8#5432 - bor 2#1010101 - c = 123 + - 234, - d}). - --record(record5, { a = 1 :: integer() -, b = foobar :: atom() -}). - --define(MACRO_1, macro). --define(MACRO_2(_), macro). - --spec t(integer()) -> any(). - --type ann() :: Var :: integer(). --type ann2() :: Var :: - 'return' - | 'return_white_spaces' - | 'return_comments' - | 'text' | ann(). --type paren() :: - (ann2()). --type t1() :: atom(). --type t2() :: [t1()]. --type t3(Atom) :: integer(Atom). --type t4() :: t3(foobar). --type t5() :: {t1(), t3(foo)}. --type t6() :: 1 | 2 | 3 | - 'foo' | 'bar'. --type t7() :: []. --type t71() :: [_]. --type t8() :: {any(),none(),pid(),port(), - reference(),float()}. --type t9() :: [1|2|3|foo|bar] | - list(a | b | c) | t71(). --type t10() :: {1|2|3|foo|t9()} | {}. --type t11() :: 1..2. --type t13() :: maybe_improper_list(integer(), t11()). --type t14() :: [erl_scan:foo() | - %% Should be highlighted - term() | - bool() | - byte() | - char() | - non_neg_integer() | nonempty_list() | - pos_integer() | - neg_integer() | - number() | - list() | - nonempty_improper_list() | nonempty_maybe_improper_list() | - maybe_improper_list() | string() | iolist() | byte() | - module() | - mfa() | - node() | - timeout() | - no_return() | - %% Should not be highlighted - nonempty_() | nonlist() | - erl_scan:bar(34, 92) | t13() | m:f(integer() | <<_:_*16>>)]. - - --type t15() :: {binary(),<<>>,<<_:34>>,<<_:_*42>>, - <<_:3,_:_*14>>,<<>>} | [<<>>|<<_:34>>|<<_:16>>| -<<_:3,_:_*1472>>|<<_:19,_:_*14>>| <<_:34>>| -<<_:34>>|<<_:34>>|<<_:34>>]. --type t16() :: fun(). --type t17() :: fun((...) -> paren()). --type t18() :: fun(() -> t17() | t16()). --type t19() :: fun((t18()) -> t16()) | - fun((nonempty_maybe_improper_list('integer', any())| - 1|2|3|a|b|<<_:3,_:_*14>>|integer()) -> -nonempty_maybe_improper_list('integer', any())| -1|2|3|a|b|<<_:3,_:_*14>>|integer()). --type t20() :: [t19(), ...]. --type t21() :: tuple(). --type t21(A) :: A. --type t22() :: t21(integer()). --type t23() :: #rec1{}. --type t24() :: #rec2{a :: t23(), b :: [atom()]}. --type t25() :: #rec3{f123 :: [t24() | -1|2|3|4|a|b|c|d| -nonempty_maybe_improper_list(integer, any())]}. --type t26() :: #rec4{ a :: integer() -, b :: any() -}. --type t27() :: { integer() -, atom() -}. --type t99() :: -{t2(),t4(),t5(),t6(),t7(),t8(),t10(),t14(), -t15(),t20(),t21(), t22(),t25()}. --spec t1(FooBar :: t99()) -> t99(); -(t2()) -> t2(); - (t4()) -> t4() when is_subtype(t4(), t24); -(t23()) -> t23() when is_subtype(t23(), atom()), - is_subtype(t23(), t14()); -(t24()) -> t24() when is_subtype(t24(), atom()), - is_subtype(t24(), t14()), - is_subtype(t24(), t4()). - --spec over(I :: integer()) -> R1 :: foo:typen(); - (A :: atom()) -> R2 :: foo:atomen(); - (T :: tuple()) -> R3 :: bar:typen(). - --spec mod:t2() -> any(). - --spec handle_cast(Cast :: {'exchange', node(), [[name(),...]]} - | {'del_member', name(), pid()}, - #state{}) -> {'noreply', #state{}}. - --spec handle_cast(Cast :: - {'exchange', node(), [[name(),...]]} - | {'del_member', name(), pid()}, - #state{}) -> {'noreply', #state{}}. - --spec all(fun((T) -> boolean()), List :: [T]) -> - boolean() when is_subtype(T, term()). % (*) - --spec get_closest_pid(term()) -> - Return :: pid() - | {'error', {'no_process', term()} - | {'no_such_group', term()}}. - --spec add( X :: integer() -, Y :: integer() -) -> integer(). - --opaque attributes_data() :: -[{'column', column()} | {'line', info_line()} | - {'text', string()}] | {line(),column()}. --record(r,{ - f1 :: attributes_data(), -f222 = foo:bar(34, #rec3{}, 234234234423, - aassdsfsdfsdf, 2234242323) :: -[t24() | 1|2|3|4|a|b|c|d| - nonempty_maybe_improper_list(integer, any())], -f333 :: [t24() | 1|2|3|4|a|b|c|d| - nonempty_maybe_improper_list(integer, any())], -f3 = x:y(), -f4 = x:z() :: t99(), -f17 :: 'undefined', -f18 :: 1 | 2 | 'undefined', -f19 = 3 :: integer()|undefined, -f5 = 3 :: undefined|integer()}). - --record(state, { - sequence_number = 1 :: integer() - }). - - -highlighting(X) % Function definitions should be highlighted - when is_integer(X) -> % and so should `when' and `is_integer' be - %% Highlighting - %% Various characters (we keep an `atom' after to see that highlighting ends) - $a,atom, % Characters should be marked - "string",atom, % and strings - 'asdasd',atom, % quote should be atoms?? - 'VaV',atom, - 'aVa',atom, - '\'atom',atom, - 'atom\'',atom, - 'at\'om',atom, - '#1',atom, - - $", atom, % atom should be ok - $', atom, - - "string$", atom, "string$", atom, % currently buggy I know... - "string\$", atom, % workaround for bug above - - "char $in string", atom, - - 'atom$', atom, 'atom$', atom, - 'atom\$', atom, - - 'char $in atom', atom, - - $[, ${, $\\, atom, - ?MACRO_1, - ?MACRO_2(foo), - - %% Numerical constants - 16#DD, % AD Should not be highlighted - 32#dd, % AD Should not be highlighted - 32#ddAB, % AD Should not be highlighted - 32#101, % AD Should not be highlighted - 32#ABTR, % AD Should not be highlighted - - %% Variables - Variables = lists:foo(), - _Variables = lists:foo(), % AD - AppSpec = Xyz/2, - Module42 = Xyz(foo, bar), - Module:foo(), - _Module:foo(), % AD - FooÅÅ = lists:reverse([tl,hd,tl,hd]), % AD Should highlight FooÅÅ - _FooÅÅ = 42, % AD Should highlight _FooÅÅ - - %% Bifs - erlang:registered(), - registered(), - hd(tl(tl(hd([a,b,c])))), - erlang:anything(lists), - %% Guards - is_atom(foo), is_float(2.3), is_integer(32), is_number(4323.3), - is_function(Fun), is_pid(self()), - not_a_guard:is_list([]), - %% Other Types - - atom, % not (currently) hightlighted - 234234, - 234.43, - - [list, are, not, higlighted], - {nor, is, tuple}, - ok. - -%%% -%%% Indentation -%%% - -%%% Left - -%% Indented - -% Right - - -indent_basics(X, Y, Z) - when X > 42, -Z < 13; -Y =:= 4711 -> - %% comments - % right comments - case lists:filter(fun(_, AlongName, - B, - C) -> - true - end, - [a,v,b]) - of - [] -> - Y = 5 * 43, - ok; - [_|_] -> - Y = 5 * 43, - ok - end, - Y, - %% List, tuples and binaries - [a, - b, c - ], - [ a, - b, c - ], - - [ - a, - b -], - {a, - b,c - }, - { a, - b,c - }, - - { - a, - b - }, - -<<1:8, - 2:8 - >>, - << - 1:8, - 2:8 - >>, - << 1:8, - 2:8 - >>, - - (a, - b, - c - ), - - ( a, - b, - c - ), - - - ( - a, - b, - c - ), - - call(2#42423 bor - #4234, - 2#5432, - other_arg), - ok; -indent_basics(Xlongname, - #struct{a=Foo, - b=Bar}, - [X| - Y]) -> - testing_next_clause, - ok; -indent_basics( % AD added clause - X, % not sure how this should look - Y, - Z) - when - X < 42, Z > 13; - Y =:= 4711 -> - foo; -indent_basics(X, Y, Z) when % AD added clause - X < 42, Z > 13; % testing when indentation - Y =:= 4711 -> - foo; -indent_basics(X, Y, Z) % AD added clause - when % testing when indentation - X < 42, Z > 13; % unsure about this one - Y =:= 4711 -> - foo. - - - -indent_nested() -> - [ - {foo, 2, "string"}, - {bar, 3, "another string"} - ]. - - -indent_icr(Z) -> % icr = if case receive - %% If - if Z >= 0 -> - X = 43 div 4, - foo(X); - Z =< 10 -> - X = 43 div 4, - foo(X); - Z == 5 orelse - Z == 7 -> - X = 43 div 4, - foo(X); - true -> - if_works - end, - %% Case - case {Z, foo, bar} of - {Z,_,_} -> - X = 43 div 4, - foo(X); - {Z,_,_} when - Z =:= 42 -> % AD line should be indented as a when - X = 43 div 4, - foo(X); - {Z,_,_} - when Z < 10 -> % AD when should be indented - X = 43 div 4, - foo(X); - {Z,_,_} - when % AD when should be indented - Z < 10 % and the guards should follow when - andalso % unsure about how though - true -> - X = 43 div 4, - foo(X) - end, - %% begin - begin - sune, - X = 74234 + foo(8456) + - 345 div 43, - ok - end, - - - %% receive - receive - {Z,_,_} -> - X = 43 div 4, - foo(X); - Z -> - X = 43 div 4, - foo(X) - end, - receive - {Z,_,_} -> - X = 43 div 4, - foo(X); - Z % AD added clause - when Z =:= 1 -> % This line should be indented by 2 - X = 43 div 4, - foo(X); - Z when % AD added clause - Z =:= 2 -> % This line should be indented by 2 - X = 43 div 4, - foo(X); - Z -> - X = 43 div 4, - foo(X) - after infinity -> - foo(X), - asd(X), - 5*43 - end, - receive - after 10 -> - foo(X), - asd(X), - 5*43 - end, - ok. - -indent_fun() -> - %% Changed fun to one indention level -Var = spawn(fun(X) - when X == 2; - X > 10 -> - hello, - case Hello() of - true when is_atom(X) -> - foo; - false -> - bar - end; - (Foo) when is_atom(Foo), - is_integer(X) -> - X = 6* 45, - Y = true andalso - kalle - end), -%% check EEP37 named funs -Fn1 = fun Fact(N) when N > 0 -> - F = Fact(N-1), - N * F; -Fact(0) -> - 1 - end, -%% check anonymous funs too - Fn2 = fun(0) -> -1; - (N) -> - N - end, - ok. - -indent_try_catch() -> - try - io:format(stdout, "Parsing file ~s, ", - [St0#leex.xfile]), - {ok,Line3,REAs,Actions,St3} = - parse_rules(Xfile, Line2, Macs, St2) - catch - exit:{badarg,R} -> - foo(R), - io:format(stdout, - "ERROR reason ~p~n", - R); - error:R % AD added clause - when R =:= 42 -> % when should be indented - foo(R); - error:R % AD added clause - when % when should be indented - R =:= 42 -> % but unsure about this (maybe 2 more) - foo(R); - error:R when % AD added clause - R =:= foo -> % line should be 2 indented (works) - foo(R); - error:R -> - foo(R), - io:format(stdout, - "ERROR reason ~p~n", - R) - after - foo('after'), - file:close(Xfile) - end; -indent_try_catch() -> - try - foo(bar) - of - X when true andalso - kalle -> - io:format(stdout, "Parsing file ~s, ", - [St0#leex.xfile]), - {ok,Line3,REAs,Actions,St3} = - parse_rules(Xfile, Line2, Macs, St2); - X % AD added clause - when false andalso % when should be 2 indented - bengt -> - gurka(); - X when % AD added clause - false andalso % line should be 2 indented - not bengt -> - gurka(); - X -> - io:format(stdout, "Parsing file ~s, ", - [St0#leex.xfile]), - {ok,Line3,REAs,Actions,St3} = - parse_rules(Xfile, Line2, Macs, St2) - catch - exit:{badarg,R} -> - foo(R), - io:format(stdout, - "ERROR reason ~p~n", - R); - error:R -> - foo(R), - io:format(stdout, - "ERROR reason ~p~n", - R) - after - foo('after'), - file:close(Xfile), - bar(with_long_arg, - with_second_arg) - end; - indent_try_catch() -> - try foo() - after - foo(), - bar(with_long_arg, - with_second_arg) - end. - -indent_catch() -> - D = B + - float(43.1), - - B = catch oskar(X), - - A = catch (baz + - bax), - catch foo(), - - C = catch B + - float(43.1), - - case catch foo(X) of - A -> - B - end, - - case - catch foo(X) - of - A -> - B - end, - - case - foo(X) - of - A -> - catch B, - X - end, - - try sune of - _ -> foo - catch _:_ -> baf - end, - - try -sune - of - _ -> - X = 5, - (catch foo(X)), - X + 10 - catch _:_ -> baf - end, - - try - (catch sune) - of - _ -> - catch foo() %% BUGBUG can't handle catch inside try without parentheses - catch _:_ -> - baf - end, - - try -(catch exit()) - catch -_ -> - catch baf() - end, - ok. - -indent_binary() -> - X = lists:foldr(fun(M) -> - <<Ma/binary, " ">> - end, [], A), - A = <<X/binary, 0:8>>, - B. - - -indent_comprehensions() -> -%% I don't have a good idea how we want to handle this -%% but they are here to show how they are indented today. -Result1 = [X || - #record{a=X} <- lists:seq(1, 10), - true = (X rem 2) - ], -Result2 = [X || <<X:32,_:32>> <= <<0:512>>, - true = (X rem 2) - ], - -Binary1 = << <<X:8>> || - #record{a=X} <- lists:seq(1, 10), - true = (X rem 2) - >>, - -Binary2 = << <<X:8>> || <<X:32,_:32>> <= <<0:512>>, - true = (X rem 2) - >>, -ok. - -%% This causes an error in earlier erlang-mode versions. -foo() -> -[#foo{ -foo = foo}]. - -%% Record indentation -some_function_with_a_very_long_name() -> - #'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ - field1=a, - field2=b}, - case dummy_function_with_a_very_very_long_name(x) of - #'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ - field1=a, - field2=b} -> - ok; - Var = #'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ - field1=a, - field2=b} -> - Var#'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ - field1=a, - field2=b}; - #xyz{ - a=1, - b=2} -> - ok - end. - -another_function_with_a_very_very_long_name() -> - #rec{ - field1=1, - field2=1}. - -some_function_name_xyz(xyzzy, #some_record{ - field1=Field1, - field2=Field2}) -> - SomeVariable = f(#'Some-long-record-name'{ - field_a = 1, - 'inter-xyz-parameters' = - #'Some-other-very-long-record-name'{ - field2 = Field1, - field2 = Field2}}), - {ok, SomeVariable}. - -commas_first() -> - {abc, [ {some_var, 1} - , {some_other_var, 2} - , {erlang_ftw, 9} - , {erlang_cookie, 'cookie'} - , {cmds, - [ {one, "sudo ls"} - , {one, "sudo ls"} - , {two, "sudo ls"} - , {three, "sudo ls"} - , {four, "sudo ls"} - , {three, "sudo ls"} - ] } - , {ssh_username, "yow"} - , {cluster, - [ {aaaa, [ {"10.198.55.12" , "" } - , {"10.198.55.13" , "" } - ] } - , {bbbb, [ {"10.198.55.151", "" } - , {"10.198.55.123", "" } - , {"10.198.55.34" , "" } - , {"10.198.55.85" , "" } - , {"10.198.55.67" , "" } - ] } - , {cccc, [ {"10.198.55.68" , "" } - , {"10.198.55.69" , "" } - ] } - ] } -] -}. - - -%% this used to result in a scan-sexp error -[{ -}]. - -%% this used to result in 2x the correct indentation within the function -%% body, due to the function name being mistaken for a keyword -catcher(N) -> -try generate_exception(N) of -Val -> {N, normal, Val} -catch -throw:X -> {N, caught, thrown, X}; -exit:X -> {N, caught, exited, X}; -error:X -> {N, caught, error, X} -end. diff --git a/lib/tools/src/fprof.erl b/lib/tools/src/fprof.erl index fb657c2928..a3b4bfdddf 100644 --- a/lib/tools/src/fprof.erl +++ b/lib/tools/src/fprof.erl @@ -1242,8 +1242,7 @@ spawn_3step(Spawn, FunPrelude, FunAck, FunBody) catch Child ! {Parent, Ref, Go}, Result catch - Class:Reason -> - Stacktrace = erlang:get_stacktrace(), + Class:Reason:Stacktrace -> catch exit(Child, kill), erlang:raise(Class, Reason, Stacktrace) end; diff --git a/lib/tools/src/lcnt.erl b/lib/tools/src/lcnt.erl index 139b3d8a4a..d0152a4915 100644 --- a/lib/tools/src/lcnt.erl +++ b/lib/tools/src/lcnt.erl @@ -218,9 +218,11 @@ raw() -> call(raw). set(Option, Value) -> call({set, Option, Value}). set({Option, Value}) -> call({set, Option, Value}). save(Filename) -> call({save, Filename}). -load(Filename) -> ok = start_internal(), call({load, Filename}). +load(Filename) -> call({load, Filename}). -call(Msg) -> gen_server:call(?MODULE, Msg, infinity). +call(Msg) -> + ok = start_internal(), + gen_server:call(?MODULE, Msg, infinity). %% -------------------------------------------------------------------- %% %% @@ -237,7 +239,6 @@ apply(Fun) when is_function(Fun) -> lcnt:apply(Fun, []). apply(Fun, As) when is_function(Fun) -> - ok = start_internal(), Opt = lcnt:rt_opt({copy_save, true}), lcnt:clear(), Res = erlang:apply(Fun, As), @@ -943,7 +944,7 @@ print_state_information(#state{locks = Locks} = State) -> print(kv("#tries", s(Stats#stats.tries))), print(kv("#colls", s(Stats#stats.colls))), print(kv("wait time", s(Stats#stats.time) ++ " us" ++ " ( " ++ s(Stats#stats.time/1000000) ++ " s)")), - print(kv("percent of duration", s(Stats#stats.time/State#state.duration*100) ++ " %")), + print(kv("percent of duration", s(percent(Stats#stats.time, State#state.duration)) ++ " %")), ok. diff --git a/lib/tools/test/emacs_SUITE.erl b/lib/tools/test/emacs_SUITE.erl index 77a8813db5..f4e78da667 100644 --- a/lib/tools/test/emacs_SUITE.erl +++ b/lib/tools/test/emacs_SUITE.erl @@ -23,10 +23,10 @@ -export([all/0, init_per_testcase/2, end_per_testcase/2]). --export([bif_highlight/1]). +-export([bif_highlight/1, indent/1]). -all() -> - [bif_highlight]. +all() -> + [bif_highlight, indent]. init_per_testcase(_Case, Config) -> ErlangEl = filename:join([code:lib_dir(tools),"emacs","erlang.el"]), @@ -74,4 +74,69 @@ check_bif_highlight(Bin, Tag, Compare) -> [] = Compare -- EmacsIntBifs, [] = EmacsIntBifs -- Compare. - +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +indent(Config) -> + case emacs_version_ok() of + false -> {skip, "Old or no emacs found"}; + true -> + Def = filename:dirname(code:which(?MODULE)) ++ "/" ++ ?MODULE_STRING ++ "_data", + Dir = proplists:get_value(data_dir, Config, Def), + OrigFs = filelib:wildcard(Dir ++ "/*"), + io:format("Dir: ~s~nFs: ~p~n", [Dir, OrigFs]), + Fs = [{File, unindent(File)} || File <- OrigFs, + filename:extension(File) =:= ""], + Indent = fun emacs/1, + [Indent(File) || {_, File} <- Fs], + Res = [diff(Orig, File) || {Orig, File} <- Fs], + [file:delete(File) || {ok, File} <- Res], %% Cleanup + [] = [Fail || {fail, Fail} <- Res], + ok + end. + +unindent(Input) -> + Output = Input ++ ".erl", + {ok, Bin} = file:read_file(Input), + Lines0 = string:split(Bin, "\n", all), + Lines = [string:trim(Line, leading, [$\s,$\t]) || Line <- Lines0], + %% io:format("File: ~s lines: ~w~n", [Input, length(Lines0)]), + %% [io:format("~s~n", [L]) || L <- Lines], + ok = file:write_file(Output, lists:join("\n", Lines)), + Output. + +diff(Orig, File) -> + case os:cmd(["diff ", Orig, " ", File]) of + "" -> {ok, File}; + Diff -> + io:format("Fail: ~s vs ~s~n~s~n~n",[Orig, File, Diff]), + {fail, File} + end. + +emacs_version_ok() -> + case os:cmd("emacs --version | head -1") of + "GNU Emacs " ++ Ver -> + case string:to_float(Ver) of + {Vsn, _} when Vsn >= 24.1 -> + true; + _ -> + io:format("Emacs version fail~n~s~n~n",[Ver]), + false + end; + Res -> + io:format("Emacs version fail~n~s~n~n",[Res]), + false + end. + +emacs(File) -> + EmacsErlDir = filename:join([code:lib_dir(tools), "emacs"]), + Cmd = ["emacs ", + "--batch --quick ", + "--directory ", EmacsErlDir, " ", + "--eval \"(require 'erlang-start)\" ", + File, " ", + "--eval '(indent-region (point-min) (point-max) nil)' ", + "--eval '(save-buffer 0)'" + ], + _Res = os:cmd(Cmd), + % io:format("cmd ~s:~n=> ~s~n", [Cmd, _Res]), + ok. diff --git a/lib/tools/test/emacs_SUITE_data/comments b/lib/tools/test/emacs_SUITE_data/comments new file mode 100644 index 0000000000..ff974ca295 --- /dev/null +++ b/lib/tools/test/emacs_SUITE_data/comments @@ -0,0 +1,25 @@ +%% -*- Mode: erlang; indent-tabs-mode: nil -*- +%% Copyright Ericsson AB 2017. All Rights Reserved. + +%%% 3 comment chars: always left indented +%%% 2 comment chars: Context indented +%%% 1 comment char: Rigth indented + +%%% left +%% context dependent + % rigth + +func() -> +%%% left + %% context dependent + % right indented + case get(foo) of + undefined -> + %% Testing indention + ok; + %% Catch all + Other -> + Other + end, + ok. + diff --git a/lib/tools/test/emacs_SUITE_data/comprehensions b/lib/tools/test/emacs_SUITE_data/comprehensions new file mode 100644 index 0000000000..45279850a5 --- /dev/null +++ b/lib/tools/test/emacs_SUITE_data/comprehensions @@ -0,0 +1,47 @@ +%% -*- Mode: erlang; indent-tabs-mode: nil -*- +%% Copyright Ericsson AB 2017. All Rights Reserved. + +%%% indentation of comprehensions + +%%% Not everything in these test are set in stone +%%% better indentation rules can be added but by having +%%% these tests we can see what changes in new implementations +%%% and notice when doing unintentional changes + +list() -> + %% I don't have a good idea how we want to handle this + %% but they are here to show how they are indented today. + Result1 = [X || + #record{a=X} <- lists:seq(1, 10), + true = (X rem 2) + ], + Result2 = [X || <<X:32,_:32>> <= <<0:512>>, + true = (X rem 2) + ], + Res = [ func(X, + arg2) + || + #record{a=X} <- lists:seq(1, 10), + true = (X rem 2) + ], + Result1. + +binary(B) -> + Binary1 = << <<X:8>> || + #record{a=X} <- lists:seq(1, 10), + true = (X rem 2) + >>, + + Binary2 = << <<X:8>> || <<X:32,_:32>> <= <<0:512>>, + true = (X rem 2) + >>, + + Bin3 = << + << + X:8, + 34:8 + >> + || <<X:32,_:32>> <= <<0:512>>, + true = (X rem 2) + >>, + ok. diff --git a/lib/tools/test/emacs_SUITE_data/funcs b/lib/tools/test/emacs_SUITE_data/funcs new file mode 100644 index 0000000000..877f982005 --- /dev/null +++ b/lib/tools/test/emacs_SUITE_data/funcs @@ -0,0 +1,174 @@ +%% -*- Mode: erlang; indent-tabs-mode: nil -*- +%% Copyright Ericsson AB 2017. All Rights Reserved. + +%%% Function (and funs) indentation + +%%% Not everything in these test are set in stone +%%% better indentation rules can be added but by having +%%% these tests we can see what changes in new implementations +%%% and notice when doing unintentional changes + +-export([ + func1/0, + func2/0, + a_function_with_a_very_very_long_name/0, + when1/2 + ]). + +-compile([nowarn_unused_functions, + {inline, [ + func2/2, + func3/2 + ] + } + ]). + +func1() -> + basic. + +func2(A1, + A2) -> + ok. + +func3( + A1, + A2 + ) -> + ok. + +%% Okeefe style +func4(A1 + ,A2 + ,A3 + ) -> + ok. + +func5( + A41 + ,A42) -> + ok. + +a_function_with_a_very_very_long_name() -> + A00 = #record{ + field1=1, + field2=1 + }, + A00. + +when1(W1, W2) + when is_number(W1), + is_number(W2) -> + ok. + +when2(W1,W2,W3) when + W1 > W2, + W2 > W3 -> + ok. + +when3(W1,W2,W3) when + W1 > W2, + W2 > W3 + -> + ok. + +when4(W1,W2,W3) + when + W1 > W2, + W2 > W3 -> + ok. + +match1({[H|T], + Other}, + M1A2) -> + ok. + +match2( + { + [H|T], + Other + }, + M2A2 + ) -> + ok. + +match3({ + M3A1, + [ + H | + T + ], + Other + }, + M3A2 + ) -> + ok. + +match4(<< + M4A:8, + M4B:16/unsigned-integer, + _/binary + >>, + M4C) -> + ok. + +match5(M5A, + #record{ + b=M5B, + c=M5C + } + ) -> + ok. + +match6(M6A, + #{key6a := a6, + key6b := b6 + }) -> + ok. + +funs(1) + when + X -> + %% Changed fun to one indention level + %% 'when' and several clause forces a depth of '4' + Var = spawn(fun(X, _) + when X == 2; + X > 10 -> + hello, + case Hello() of + true when is_atom(X) -> + foo; + false -> + bar + end; + (Foo) when is_atom(Foo), + is_integer(X) -> + X = 6 * 45, + Y = true andalso + kalle + end), + Var; +funs(2) -> + %% check EEP37 named funs + Fn1 = fun + Factory(N) when + N > 0 -> + F = Fact(N-1), + N * F; + Factory(0) -> + 1 + end, + Fn1; +funs(3) -> + %% check anonymous funs too + Fn2 = fun(0) -> + 1; + (N) -> + N + end, + ok; +funs(4) -> + X = lists:foldr(fun(M) -> + <<M/binary, " ">> + end, [], Z), + A = <<X/binary, 0:8>>, + A. diff --git a/lib/tools/test/emacs_SUITE_data/highlight b/lib/tools/test/emacs_SUITE_data/highlight new file mode 100644 index 0000000000..0719f6516a --- /dev/null +++ b/lib/tools/test/emacs_SUITE_data/highlight @@ -0,0 +1,78 @@ +%% -*- Mode: erlang; indent-tabs-mode: nil -*- +%% Copyright Ericsson AB 2017. All Rights Reserved. + +%%% Open this file in your editor and manually check the colors of +%%% different types and calls and builtin words + +%%% Not everything in these test are set in stone +%%% better indentation rules can be added but by having +%%% these tests we can see what changes in new implementations +%%% and notice when doing unintentional changes + + +highlighting(X) % Function definitions should be highlighted + when is_integer(X) -> % and so should `when' and `is_integer' be + %% Highlighting + %% Various characters (we keep an `atom' after to see that highlighting ends) + $a,atom, % Characters should be marked + "string",atom, % and strings + 'asdasd',atom, % quote should be atoms?? + 'VaV',atom, + 'aVa',atom, + '\'atom',atom, + 'atom\'',atom, + 'at\'om',atom, + '#1',atom, + + $", atom, % atom should be ok + $', atom, + + "string$", atom, "string$", atom, % currently buggy I know... + "string\$", atom, % workaround for bug above + + "char $in string", atom, + + 'atom$', atom, 'atom$', atom, + 'atom\$', atom, + + 'char $in atom', atom, + + $[, ${, $\\, atom, + ?MACRO_1, + ?MACRO_2(foo), + + %% Numerical constants + 16#DD, % Should not be highlighted + 32#dd, % Should not be highlighted + 32#ddAB, % Should not be highlighted + 32#101, % Should not be highlighted + 32#ABTR, % Should not be highlighted + + %% Variables + Variables = lists:foo(), + _Variables = lists:foo(), + AppSpec = Xyz/2, + Module42 = Xyz(foo, bar), + Module:foo(), + _Module:foo(), % + FooÅÅ = lists:reverse([tl,hd,tl,hd]), % Should highlight FooÅÅ + _FooÅÅ = 42, % Should highlight _FooÅÅ + + %% Bifs + erlang:registered(), + registered(), + hd(tl(tl(hd([a,b,c])))), + erlang:anything(lists), + %% Guards + is_atom(foo), is_float(2.3), is_integer(32), is_number(4323.3), + is_function(Fun), is_pid(self()), + not_a_guard:is_list([]), + %% Other Types + + atom, % not (currently) hightlighted + 234234, + 234.43, + + [list, are, not, higlighted], + {nor, is, tuple}, + ok. diff --git a/lib/tools/test/emacs_SUITE_data/icr b/lib/tools/test/emacs_SUITE_data/icr new file mode 100644 index 0000000000..8445c1a74d --- /dev/null +++ b/lib/tools/test/emacs_SUITE_data/icr @@ -0,0 +1,157 @@ +%% -*- Mode: erlang; indent-tabs-mode: nil -*- +%% Copyright Ericsson AB 2017. All Rights Reserved. + +%%% indentation of if case receive statements + +%%% Not everything in these test are set in stone +%%% better indentation rules can be added but by having +%%% these tests we can see what changes in new implementations +%%% and notice when doing unintentional changes + +indent_if(1, Z) -> + %% If + if Z >= 0 -> + X = 43 div Z, + X; + Z =< 10 -> + X = 43 div Z, + X; + Z == 5 orelse + Z == 7 -> + X = 43 div Z, + X; + is_number(Z), + Z < 32 -> + Z; + is_number(Z); + Z < 32 -> + Z * 32; + true -> + if_works + end; +indent_if(2, Z) -> + %% If + if + Z >= 0 -> + X = 43 div Z, + X + ; Z =< 10 -> + 43 div Z + ; Z == 5 orelse + Z == 7 -> + X = 43 div Z, + X + ; is_number(Z), + Z < 32 -> + Z + ; true -> + if_works + end. + +indent_case(1, Z) -> + %% Case + case {Z, foo, bar} of + {Z,_,_} -> + X = 43 div 4, + foo(X); + {Z,_,_} when + Z =:= 42 -> % line should be indented as a when + X = 43 div 4, + foo(X); + {Z,_,_} + when Z < 10 orelse + Z =:= foo -> % Binary op alignment here !!! + X = 43 div 4, + Bool = Z < 5 orelse % Binary op args align differently after when + Z =:= foo, % and elsewhere ??? + foo(X); + {Z,_,_} + when % when should be indented + Z < 10 % and the guards should follow when + andalso % unsure about how though + true -> + X = 43 div 4, + foo(X) + end; +indent_case(2, Z) -> + %% Case + case {Z, foo, bar} of + {Z,_,_} -> + X = 43 div 4, + foo(X) + ; {Z,_,_} when + Z =:= 42 -> % line should be indented as a when + X = 43 div 4, + foo(X) + ; {Z,_,_} + when Z < 10 -> % when should be indented + X = 43 div 4, + foo(X) + ; {Z,_,_} + when % when should be indented + Z < 10 % and the guards should follow when + andalso % unsure about how though + true -> + X = 43 div 4, + foo(X) + end. + +indent_begin(Z) -> + %% Begin + begin + sune, + Z = 74234 + + foo(8456) + + 345 div 43, + Foo = begin + ok, + foo(234), + begin + io:format("Down here\n") + end + end, + {Foo, + bar} + end. + +indent_receive(1) -> + %% receive + receive + {Z,_,_} -> + X = 43 div 4, + foo(X) + ; Z -> + X = 43 div 4, + foo(X) + end, + ok; +indent_receive(2) -> + receive + {Z,_,_} -> + X = 43 div 4, + foo(X); + Z % added clause + when Z =:= 1 -> % This line should be indented by 2 + X = 43 div 4, + foo(X); + Z when % added clause + Z =:= 2 -> % This line should be indented by 2 + X = 43 div 4, + foo(X); + Z -> + X = 43 div 4, + foo(X) + after infinity -> + foo(X), + asd(X), + 5*43 + end, + ok; +indent_receive() -> + receive + after 10 -> + foo(X), + asd(X), + 5*43 + end, + ok. diff --git a/lib/tools/test/emacs_SUITE_data/macros b/lib/tools/test/emacs_SUITE_data/macros new file mode 100644 index 0000000000..6c874e9187 --- /dev/null +++ b/lib/tools/test/emacs_SUITE_data/macros @@ -0,0 +1,31 @@ +%% -*- Mode: erlang; indent-tabs-mode: nil -*- +%% Copyright Ericsson AB 2017. All Rights Reserved. + +%%% Macros should be indented as code + +-define(M0, ok). + +-define(M1, + case X of + undefined -> error; + _ -> ok + end). + +-define(M2(M2A1, + M2A2), + func(M2A1, + M2A2) + ). + +-define( + M3, + undefined + ). + +-ifdef(DEBUG). +-define(LOG, + logger:log(?MODULE,?LINE) + ). +-else(). +-define(LOG, ok). +-endif(). diff --git a/lib/tools/test/emacs_SUITE_data/records b/lib/tools/test/emacs_SUITE_data/records new file mode 100644 index 0000000000..241582718c --- /dev/null +++ b/lib/tools/test/emacs_SUITE_data/records @@ -0,0 +1,35 @@ +%% -*- Mode: erlang; indent-tabs-mode: nil -*- +%% Copyright Ericsson AB 2017. All Rights Reserved. + +%% Test that records are indented correctly + +-record(record0, + { + r0a, + r0b, + r0c + }). + +-record(record1, {r1a, + r1b, + r1c + }). + +-record(record2, { + r2a, + r2b + }). + +-record(record3, {r3a = 8#42423 bor + 8#4234, + r3b = 8#5432 + bor 2#1010101, + r3c = 123 + + 234, + r3d}). + +-record(record5, + { r5a = 1 :: integer() + , r5b = foobar :: atom() + }). + diff --git a/lib/tools/test/emacs_SUITE_data/terms b/lib/tools/test/emacs_SUITE_data/terms new file mode 100644 index 0000000000..352364a73c --- /dev/null +++ b/lib/tools/test/emacs_SUITE_data/terms @@ -0,0 +1,174 @@ +%% -*- Mode: erlang; indent-tabs-mode: nil -*- +%% Copyright Ericsson AB 2017. All Rights Reserved. + +%%% indentation of terms contain builtin types + +%%% Not everything in these test are set in stone +%%% better indentation rules can be added but by having +%%% these tests we can see what changes in new implementations +%%% and notice when doing unintentional changes + + +list(1) -> + [a, + b, + c + ]; +list(2) -> + [ a, + b, c + ]; +list(3) -> + [ + a, + b, c + ]; +list(4) -> + [ a + , b + , c + ]. + +tuple(1) -> + {a, + b,c + }; +tuple(2) -> + { a, + b,c + }; +tuple(3) -> + { + a, + b,c + }; +tuple(4) -> + { a + , b + ,c + }. + +binary(1) -> + <<1:8, + 2:8 + >>; +binary(2) -> + << + 1:8, + 2:8 + >>; +binary(3) -> + << 1:8, + 2:8 + >>; +binary(4) -> + << + 1:8 + ,2:8 + >>; +binary(5) -> + << 1:8 + , 2:8 + >>. + +record(1) -> + #record{a=1, + b=2 + }; +record(2) -> + #record{ a=1, + b=2 + }; +record(3) -> + #record{ + a=1, + b=2 + }; +record(4) -> + #record{ + a=1 + ,b=2 + }; +record(Record) -> + Record#record{ + a=1 + ,b=2 + }. + +map(1) -> + #{a=>1, + b=>2 + }; +map(2) -> + #{ a=>1, + b=>2 + }; +map(3) -> + #{ + a=>1, + b=>2 + }; +map(4) -> + #{ + a => <<"a">> + ,b => 2 + }; +map(MapVar) -> + MapVar = #{a :=<<"a">> + ,b:=1}. + +deep(Rec) -> + Rec#rec{ atom = 'atom', + map = #{ k1 => {v, + 1}, + k2 => [ + 1, + 2, + 3 + ], + {key, + 3} + => + << + 123:8, + 255:8 + >> + } + }. + +%% Record indentation +some_function_with_a_very_long_name() -> + #'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ + field1=a, + field2=b}, + case dummy_function_with_a_very_very_long_name(x) of + #'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ + field1=a, + field2=b} -> + ok; + Var = #'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ + field1=a, + field2=b} -> + Var#'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ + field1=a, + field2=b}; + #xyz{ + a=1, + b=2} -> + ok + end. + +some_function_name_xyz(xyzzy, #some_record{ + field1=Field1, + field2=Field2}) -> + SomeVariable = f(#'Some-long-record-name'{ + field_a = 1, + 'inter-xyz-parameters' = + #'Some-other-very-long-record-name'{ + field2 = Field1, + field2 = Field2}}), + {ok, SomeVariable}. + +foo() -> + [#foo{ + foo = foo}]. diff --git a/lib/tools/test/emacs_SUITE_data/try_catch b/lib/tools/test/emacs_SUITE_data/try_catch new file mode 100644 index 0000000000..0005b2003a --- /dev/null +++ b/lib/tools/test/emacs_SUITE_data/try_catch @@ -0,0 +1,166 @@ +%% -*- Mode: erlang; indent-tabs-mode: nil -*- +%% Copyright Ericsson AB 2017. All Rights Reserved. + +%%% Try and catch indentation is hard + +%%% Not everything in these test are set in stone +%%% better indentation rules can be added but by having +%%% these tests we can see what changes in new implementations +%%% and notice when doing unintentional changes + +try_catch() -> + try + io:format(stdout, "Parsing file ~s, ", + [St0#leex.xfile]), + {ok,Line3,REAs,Actions,St3} = + parse_rules(Xfile, Line2, Macs, St2) + catch + exit:{badarg,R} -> + foo(R), + io:format(stdout, + "ERROR reason ~p~n", + R); + error:R + when R =:= 42 -> % when should be indented + foo(R); + error:R + when % when should be indented + R =:= 42 -> % but unsure about this (maybe 2 more) + foo(R); + error:R when + R =:= foo -> % line should be 2 indented (works) + foo(R); + error:R -> + foo(R), + io:format(stdout, + "ERROR reason ~p~n", + R) + after + foo('after'), + file:close(Xfile) + end; +try_catch() -> + try + foo(bar) + of + X when true andalso + kalle -> + io:format(stdout, "Parsing file ~s, ", + [St0#leex.xfile]), + {ok,Line3,REAs,Actions,St3} = + parse_rules(Xfile, Line2, Macs, St2); + X + when false andalso % when should be 2 indented + bengt -> + gurka(); + X when + false andalso % line should be 2 indented + not bengt -> + gurka(); + X -> + io:format(stdout, "Parsing file ~s, ", + [St0#leex.xfile]), + {ok,Line3,REAs,Actions,St3} = + parse_rules(Xfile, Line2, Macs, St2) + catch + exit:{badarg,R} -> + foo(R), + io:format(stdout, + "ERROR reason ~p~n", + R); + error:R -> + foo(R), + io:format(stdout, + "ERROR reason ~p~n", + R) + after + foo('after'), + file:close(Xfile), + bar(with_long_arg, + with_second_arg) + end; +try_catch() -> + try foo() + after + foo(), + bar(with_long_arg, + with_second_arg) + end. + +indent_catch() -> + D = B + + float(43.1), + + B = catch oskar(X), + + A = catch (baz + + bax), + catch foo(), + + C = catch B + + float(43.1), + + case catch foo(X) of + A -> + B + end, + + case + catch foo(X) + of + A -> + B + end, + + case + foo(X) + of + A -> + catch B, + X + end, + + try sune of + _ -> foo + catch _:_ -> baf + end, + + Variable = try + sune + of + _ -> + X = 5, + (catch foo(X)), + X + 10 + catch _:_ -> baf + after cleanup() + end, + + try + (catch sune) + of + _ -> + foo1(), + catch foo() %% BUGBUG can't handle catch inside try without parentheses + catch _:_ -> + baf + end, + + try + (catch exit()) + catch + _ -> + catch baf() + end, + ok. + +%% this used to result in 2x the correct indentation within the function +%% body, due to the function name being mistaken for a keyword +catcher(N) -> + try generate_exception(N) of + Val -> {N, normal, Val} + catch + throw:X -> {N, caught, thrown, X}; + exit:X -> {N, caught, exited, X}; + error:X -> {N, caught, error, X} + end. diff --git a/lib/tools/test/emacs_SUITE_data/type_specs b/lib/tools/test/emacs_SUITE_data/type_specs new file mode 100644 index 0000000000..e71841cc7a --- /dev/null +++ b/lib/tools/test/emacs_SUITE_data/type_specs @@ -0,0 +1,110 @@ +%% -*- Mode: erlang; indent-tabs-mode: nil -*- +%% Copyright Ericsson AB 2017. All Rights Reserved. + +%% Tests how types and specs are indented (also that the editor can parse them) +%% May need improvements + + +-type ann() :: Var :: integer(). +-type ann2() :: + 'return' + | 'return_white_spaces' + | 'return_comments' + | 'text' | ann(). +-type paren() :: + (ann2()). + +-type t6() :: + 1 | 2 | 3 | + 'foo' + | 'bar'. + +-type t8() :: {any(),none(),pid(),port(), + reference(),float()}. + +-type t14() :: [erl_scan:foo() | + %% Should be highlighted + term() | + boolean() | + byte() | + char() | + non_neg_integer() | nonempty_list() | + pos_integer() | + neg_integer() | + number() | + list() | + nonempty_improper_list() | nonempty_maybe_improper_list() | + maybe_improper_list() | string() | iolist() | byte() | + module() | + mfa() | + node() | + timeout() | + no_return() | + %% Should not be highlighted + nonempty_() | nonlist() | + erl_scan:bar(34, 92) | t13() | m:f(integer() | <<_:_*16>>)]. + +-type t15() :: {binary(),<<>>,<<_:34>>,<<_:_*42>>, + <<_:3,_:_*14>>,<<>>} | [<<>>|<<_:34>>|<<_:16>>| + <<_:3,_:_*1472>>|<<_:19,_:_*14>>| <<_:34>>| + <<_:34>>|<<_:34>>|<<_:34>>]. + +-type t18() :: + fun(() -> t17() | t16()). +-type t19() :: + fun((t18()) -> t16()) | + fun((nonempty_maybe_improper_list('integer', any())| + 1|2|3|a|b|<<_:3,_:_*14>>|integer()) + -> + nonempty_maybe_improper_list('integer', any())| %% left to col 16? + 1|2|3|a|b|<<_:3,_:_*14>>|integer()). %% left to col 16? +-type t20() :: [t19(), ...]. +-type t25() :: #rec3{f123 :: [t24() | + 1|2|3|4|a|b|c|d| + nonempty_maybe_improper_list(integer, any())]}. +-type t26() :: #rec4{ a :: integer() + , b :: any() + }. + +%% Spec + +-spec t1(FooBar :: t99()) -> t99(); + (t2()) -> t2(); + (t4()) -> t4() when is_subtype(t4(), t24); + (t23()) -> t23() when is_subtype(t23(), atom()), + is_subtype(t23(), t14()); + (t24()) -> t24() when is_subtype(t24(), atom()), + is_subtype(t24(), t14()), + is_subtype(t24(), t4()). + +-spec over(I :: integer()) -> R1 :: foo:typen(); + (A :: atom()) -> R2 :: foo:atomen(); + (T :: tuple()) -> R3 :: bar:typen(). + +-spec mod:t2() -> any(). + +-spec handle_cast(Cast :: {'exchange', node(), [[name(),...]]} + | {'del_member', name(), pid()}, + #state{}) -> {'noreply', #state{}}. + +-spec handle_cast(Cast :: + {'exchange', node(), [[name(),...]]} + | {'del_member', name(), pid()}, + #state{}) -> + {'noreply', #state{}}. %% left to col 10? + +-spec all(fun((T) -> boolean()), List :: [T]) -> + boolean() when is_subtype(T, term()). % (*) + +-spec get_closest_pid(term()) -> + Return :: pid() %% left to col 10? + | {'error', {'no_process', term()}} %% left to col 10? + | {'no_such_group', term()}. %% left to col 10? + +-spec add( X :: integer() + , Y :: integer() + ) -> integer(). + +-opaque attributes_data() :: + [{'column', column()} | {'line', info_line()} | + {'text', string()}] | {line(),column()}. diff --git a/lib/tools/test/lcnt_SUITE.erl b/lib/tools/test/lcnt_SUITE.erl index 146c915087..a79572a742 100644 --- a/lib/tools/test/lcnt_SUITE.erl +++ b/lib/tools/test/lcnt_SUITE.erl @@ -30,6 +30,8 @@ t_conflicts/1, t_locations/1, t_swap_keys/1, + t_implicit_start/1, + t_crash_before_collect/1, smoke_lcnt/1]). init_per_testcase(_Case, Config) -> @@ -44,8 +46,8 @@ suite() -> {timetrap,{minutes,4}}]. all() -> - [t_load, t_conflicts, t_locations, t_swap_keys, - smoke_lcnt]. + [t_load, t_conflicts, t_locations, t_swap_keys, t_implicit_start, + t_crash_before_collect, smoke_lcnt]. %%---------------------------------------------------------------------- %% Tests @@ -149,6 +151,15 @@ t_swap_keys_file([File|Files]) -> ok = lcnt:stop(), t_swap_keys_file(Files). +%% Prior to OTP-14913 this would crash with 'noproc' as the lcnt server hadn't +%% been started yet. +t_implicit_start(Config) when is_list(Config) -> + ok = lcnt:conflicts(). + +t_crash_before_collect(Config) when is_list(Config) -> + {ok, _} = lcnt:start(), + ok = lcnt:information(). + %% Simple smoke test of actual lock-counting, if running on %% a run-time with lock-counting enabled. smoke_lcnt(Config) -> diff --git a/lib/wx/api_gen/gen_util.erl b/lib/wx/api_gen/gen_util.erl index 5e2f405498..3068a2f4ea 100644 --- a/lib/wx/api_gen/gen_util.erl +++ b/lib/wx/api_gen/gen_util.erl @@ -106,8 +106,8 @@ check_diff(Diff) -> throw:_ -> diff; error:{badmatch,_} -> diff; - _:What -> - io:format("~p:~p: ~p ~p~n", [?MODULE,?LINE, What, erlang:get_stacktrace()]), + _:What:Stacktrace -> + io:format("~p:~p: ~p ~p~n", [?MODULE,?LINE, What, Stacktrace]), diff end. diff --git a/lib/wx/api_gen/gl_gen.erl b/lib/wx/api_gen/gl_gen.erl index 7e3766a43b..6f68517c16 100644 --- a/lib/wx/api_gen/gl_gen.erl +++ b/lib/wx/api_gen/gl_gen.erl @@ -47,9 +47,9 @@ safe(What, QuitOnErr) -> What(), io:format("Completed successfully~n~n", []), QuitOnErr andalso gen_util:halt(0) - catch Err:Reason -> + catch Err:Reason:Stacktrace -> io:format("Error ~p: ~p:~p~n ~p~n", - [get(current_func),Err,Reason,erlang:get_stacktrace()]), + [get(current_func),Err,Reason,Stacktrace]), (catch gen_util:close()), timer:sleep(1999), QuitOnErr andalso gen_util:halt(1) diff --git a/lib/wx/api_gen/gl_gen_erl.erl b/lib/wx/api_gen/gl_gen_erl.erl index 45f5fd8f4c..7e673c2987 100644 --- a/lib/wx/api_gen/gl_gen_erl.erl +++ b/lib/wx/api_gen/gl_gen_erl.erl @@ -246,8 +246,8 @@ gen_types(Where) -> gen_export(F) -> try gen_export_1(F) - catch E:R -> - io:format("Crash ~p:~p in ~p ~n",[E,R, erlang:get_stacktrace()]), + catch E:R:S -> + io:format("Crash ~p:~p in ~p ~n",[E,R,S]), io:format("Func = ~p~n ~p", [F, get(F)]) end. @@ -489,8 +489,8 @@ doc_return_types2(T, Ps) -> doc_arg_type(#arg{name=Name,type=T}) -> try erl_arg_name(Name) ++ " :: " ++ doc_arg_type2(T) - catch _:Error -> - io:format("Error spec: ~p ~p~n~p~n",[Name, Error, erlang:get_stacktrace()]), + catch _:Error:Stacktrace -> + io:format("Error spec: ~p ~p~n~p~n",[Name, Error, Stacktrace]), exit(error) end. diff --git a/lib/wx/api_gen/wx_gen.erl b/lib/wx/api_gen/wx_gen.erl index aadfe4b111..ab70a588ab 100644 --- a/lib/wx/api_gen/wx_gen.erl +++ b/lib/wx/api_gen/wx_gen.erl @@ -47,9 +47,9 @@ safe(What, QuitOnErr) -> What(), io:format("Completed successfully~n~n", []), QuitOnErr andalso gen_util:halt(0) - catch Err:Reason -> + catch Err:Reason:Stacktrace -> io:format("Error in ~p ~p~n", [get(current_class),get(current_func)]), - erlang:display({Err,Reason, erlang:get_stacktrace()}), + erlang:display({Err,Reason,Stacktrace}), catch gen_util:close(), QuitOnErr andalso gen_util:halt(1) end. diff --git a/lib/wx/examples/demo/ex_aui.erl b/lib/wx/examples/demo/ex_aui.erl index d8fc0021f1..97805621ed 100644 --- a/lib/wx/examples/demo/ex_aui.erl +++ b/lib/wx/examples/demo/ex_aui.erl @@ -87,8 +87,7 @@ do_init(Config) -> wxAuiManager:update(Manager), process_flag(trap_exit, true), {Panel, #state{parent=Panel, config=Config, aui=Manager}} - catch Class:Reason -> - ST = erlang:get_stacktrace(), + catch Class:Reason:ST -> io:format("AUI Crashed ~p ~p~n",[Reason, ST]), wxAuiManager:unInit(Manager), wxAuiManager:destroy(Manager), diff --git a/lib/wx/src/wx.erl b/lib/wx/src/wx.erl index 34bf06cf46..6dd3c148db 100644 --- a/lib/wx/src/wx.erl +++ b/lib/wx/src/wx.erl @@ -183,7 +183,7 @@ batch(Fun) -> ok = wxe_util:cast(?BATCH_BEGIN, <<>>), try Fun() catch - error:W -> erlang:exit({W, erlang:get_stacktrace()}); + error:W:S -> erlang:exit({W, S}); throw:W -> erlang:throw(W); exit:W -> erlang:exit(W) after @@ -196,7 +196,7 @@ foreach(Fun, List) -> ok = wxe_util:cast(?BATCH_BEGIN, <<>>), try lists:foreach(Fun, List) catch - error:W -> erlang:exit({W, erlang:get_stacktrace()}); + error:W:S -> erlang:exit({W, S}); throw:W -> erlang:throw(W); exit:W -> erlang:exit(W) after @@ -209,7 +209,7 @@ map(Fun, List) -> ok = wxe_util:cast(?BATCH_BEGIN, <<>>), try lists:map(Fun, List) catch - error:W -> erlang:exit({W, erlang:get_stacktrace()}); + error:W:S -> erlang:exit({W, S}); throw:W -> erlang:throw(W); exit:W -> erlang:exit(W) after @@ -222,7 +222,7 @@ foldl(Fun, Acc, List) -> ok = wxe_util:cast(?BATCH_BEGIN, <<>>), try lists:foldl(Fun, Acc, List) catch - error:W -> erlang:exit({W, erlang:get_stacktrace()}); + error:W:S -> erlang:exit({W, S}); throw:W -> erlang:throw(W); exit:W -> erlang:exit(W) after @@ -235,7 +235,7 @@ foldr(Fun, Acc, List) -> ok = wxe_util:cast(?BATCH_BEGIN, <<>>), try lists:foldr(Fun, Acc, List) catch - error:W -> erlang:exit({W, erlang:get_stacktrace()}); + error:W:S -> erlang:exit({W, S}); throw:W -> erlang:throw(W); exit:W -> erlang:exit(W) after diff --git a/lib/wx/src/wxe_server.erl b/lib/wx/src/wxe_server.erl index 58fcaf8f23..d6d9dbd629 100644 --- a/lib/wx/src/wxe_server.erl +++ b/lib/wx/src/wxe_server.erl @@ -283,10 +283,10 @@ invoke_callback(Pid, Ev, Ref) -> Return -> exit({bad_return, Return}) end end - catch _:Reason -> + catch _:Reason:Stacktrace -> wxEvent:skip(Ref), ?log("Callback fun crashed with {'EXIT, ~p, ~p}~n", - [Reason, erlang:get_stacktrace()]) + [Reason, Stacktrace]) end, wxe_util:cast(?WXE_CB_RETURN, <<>>) end, @@ -299,9 +299,9 @@ invoke_callback_fun(Fun) -> Return = Fun(), true = is_binary(Return), Return - catch _:Reason -> + catch _:Reason:Stacktrace -> ?log("Callback fun crashed with {'EXIT, ~p, ~p}~n", - [Reason, erlang:get_stacktrace()]), + [Reason, Stacktrace]), <<>> end, wxe_util:cast(?WXE_CB_RETURN, Res). |