diff options
Diffstat (limited to 'lib')
452 files changed, 14807 insertions, 7956 deletions
diff --git a/lib/Makefile b/lib/Makefile index 4740e6eb59..ae466ed518 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -35,7 +35,7 @@ ALL_ERLANG_APPLICATIONS = xmerl edoc erl_docgen snmp otp_mibs erl_interface \ public_key ssl observer odbc diameter \ cosTransactions cosEvent cosTime cosNotification \ cosProperty cosFileTransfer cosEventDomain et megaco \ - eunit ssh typer eldap dialyzer hipe + eunit ssh eldap dialyzer hipe ifdef BUILD_ALL ERLANG_APPLICATIONS += $(ALL_ERLANG_APPLICATIONS) diff --git a/lib/asn1/c_src/asn1_erl_nif.c b/lib/asn1/c_src/asn1_erl_nif.c index b29c9a7ed3..7b7e11b02d 100644 --- a/lib/asn1/c_src/asn1_erl_nif.c +++ b/lib/asn1/c_src/asn1_erl_nif.c @@ -901,31 +901,35 @@ static int ber_decode_tag(ErlNifEnv* env, ERL_NIF_TERM *tag, unsigned char *in_b /* then get the tag number */ if ((tmp_tag = (int) INVMASK(in_buf[*ib_index],ASN1_CLASSFORM)) < 31) { - *tag = enif_make_uint(env, tag_no + tmp_tag); + *tag = enif_make_uint(env, tag_no | tmp_tag); (*ib_index)++; } else { - int n = 0; /* n is used to check that the 64K limit is not - exceeded*/ - /* should check that at least three bytes are left in in-buffer,at least two tag byte and at least one length byte */ if ((*ib_index + 3) > in_buf_len) return ASN1_VALUE_ERROR; (*ib_index)++; - /* The tag is in the following bytes in in_buf as - 1ttttttt 1ttttttt ... 0ttttttt, where the t-bits - is the tag number*/ - /* In practice is the tag size limited to 64K, i.e. 16 bits. If - the tag is greater then 64K return an error */ - while (((tmp_tag = (int) in_buf[*ib_index]) >= 128) && n < 2) { - /* m.s.b. = 1 */ - tag_no = tag_no + (MASK(tmp_tag,ASN1_LONG_TAG) << 7); + /* + * The tag is in the following bytes in in_buf as: + * + * 1ttttttt 0ttttttt + * + * or + * + * 0ttttttt + * + * where the t-bits is the tag number. If the tag does not + * fit in two tag bytes (16K), return an error. + */ + if ((tmp_tag = (int) in_buf[*ib_index]) >= 128) { + tag_no = tag_no | (MASK(tmp_tag,ASN1_LONG_TAG) << 7); (*ib_index)++; - n++; - }; - if ((n == 2) && in_buf[*ib_index] > 3) - return ASN1_TAG_ERROR; /* tag number > 64K */ - tag_no = tag_no + in_buf[*ib_index]; + } + tmp_tag = (int) in_buf[*ib_index]; + if (tmp_tag >= 128) { + return ASN1_TAG_ERROR; /* tag number > 16K */ + } + tag_no = tag_no | tmp_tag; (*ib_index)++; *tag = enif_make_uint(env, tag_no); } diff --git a/lib/asn1/doc/src/asn1_getting_started.xml b/lib/asn1/doc/src/asn1_getting_started.xml index d2b73d63c3..c036d289fc 100644 --- a/lib/asn1/doc/src/asn1_getting_started.xml +++ b/lib/asn1/doc/src/asn1_getting_started.xml @@ -266,6 +266,10 @@ asn1ct:compile("H323-MESSAGES.asn1",[per]). </pre> <c>{error, {asn1, Description}}</c> where <c>Description</c> is an Erlang term describing the error.</p> + <p>Currently, <c>Description</c> looks like this: + <c>{ErrorDescription, StackTrace}</c>. Applications should + not depend on the exact contents of <c>Description</c> as it + could change in the future.</p> </section> </section> diff --git a/lib/asn1/src/asn1ct.erl b/lib/asn1/src/asn1ct.erl index 9f77a557e5..58cbc89db5 100644 --- a/lib/asn1/src/asn1ct.erl +++ b/lib/asn1/src/asn1ct.erl @@ -23,10 +23,10 @@ %% Compile Time functions for ASN.1 (e.g ASN.1 compiler). -%%-compile(export_all). %% Public exports -export([compile/1, compile/2]). -export([test/1, test/2, test/3, value/2, value/3]). + %% Application internal exports -export([compile_asn/3,compile_asn1/3,compile_py/3,compile/3, vsn/0, @@ -75,12 +75,9 @@ -define(ALTERNATIVE,alt). -define(ALTERNATIVE_UNDECODED,alt_undec). -define(ALTERNATIVE_PARTS,alt_parts). -%-define(BINARY,bin). %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% This is the interface to the compiler -%% -%% compile(File) -> compile(File,[]). @@ -751,7 +748,6 @@ remove_import_doubles([]) -> remove_import_doubles(ImportList) -> MergedImportList = merge_symbols_from_module(ImportList,[]), -%% io:format("MergedImportList: ~p~n",[MergedImportList]), delete_double_of_symbol(MergedImportList,[]). merge_symbols_from_module([Imp|Imps],Acc) -> @@ -769,7 +765,6 @@ merge_symbols_from_module([Imp|Imps],Acc) -> end, Imps), NewImps = lists:subtract(Imps,IfromModName), -%% io:format("Imp: ~p~nIfromModName: ~p~n",[Imp,IfromModName]), NewImp = Imp#'SymbolsFromModule'{ symbols = lists:append( @@ -835,7 +830,6 @@ generate({M,CodeTuple}, OutFile, EncodingRule, Options) -> Code = #abst{name=M#module.name, types=Types,values=Values,ptypes=Ptypes, classes=Classes,objects=Objects,objsets=ObjectSets}, - debug_on(Options), setup_bit_string_format(Options), setup_legacy_erlang_types(Options), asn1ct_table:new(check_functions), @@ -855,7 +849,6 @@ generate({M,CodeTuple}, OutFile, EncodingRule, Options) -> end, asn1ct_gen:pgen(OutFile, Gen, Code), - debug_off(Options), cleanup_bit_string_format(), erase(tlv_format), % used in ber erase(class_default_type),% used in ber @@ -990,12 +983,8 @@ get_input_file(Module,[]) -> get_input_file(Module,[I|Includes]) -> case (catch input_file_type(filename:join([I,Module]))) of {single_file,FileName} -> -%% case file:read_file_info(FileName) of -%% {ok,_} -> {file,FileName}; -%% _ -> get_input_file(Module,Includes) -%% end; - _ -> + _ -> get_input_file(Module,Includes) end. @@ -1151,20 +1140,8 @@ is_asn1_flag(verbose) -> true; %% 'warnings_as_errors' is intentionally passed through to the compiler. is_asn1_flag(_) -> false. -debug_on(Options) -> - case lists:member(debug,Options) of - true -> - put(asndebug,true); - _ -> - true - end. - -debug_off(_Options) -> - erase(asndebug). - outfile(Base, Ext, Opts) -> -% io:format("Opts. ~p~n",[Opts]), Obase = case lists:keysearch(outdir, 1, Opts) of {value, {outdir, Odir}} -> filename:join(Odir, Base); _NotFound -> Base % Not found or bad format @@ -1215,9 +1192,6 @@ compile_py(File,OutFile,Options) -> compile(File, _OutFile, Options) -> case compile(File, make_erl_options(Options)) of {error,_Reason} -> - %% case occurs due to error in asn1ct_parser2,asn1ct_check -%% io:format("~p~n",[_Reason]), -%% io:format("~p~n~s~n",[_Reason,"error"]), error; ok -> ok; @@ -1512,7 +1486,8 @@ create_pdec_inc_command(_ModName,_,[],Acc) -> create_pdec_inc_command(ModName,{Comps1,Comps2},TNL,Acc) when is_list(Comps1),is_list(Comps2) -> create_pdec_inc_command(ModName,Comps1 ++ Comps2,TNL,Acc); -%% The following two functionclauses matches on the type after the top type. This one if the top type had no tag, i.e. a CHOICE +%% The following two clauses match on the type after the top +%% type. This one if the top type had no tag, i.e. a CHOICE. create_pdec_inc_command(ModN,Clist,[CL|_Rest],[[]]) when is_list(CL) -> create_pdec_inc_command(ModN,Clist,CL,[]); create_pdec_inc_command(ModN,Clist,[CL|_Rest],Acc) when is_list(CL) -> @@ -1523,17 +1498,14 @@ create_pdec_inc_command(ModName, prop=Prop}|Comps], TNL=[C1|Cs],Acc) -> case C1 of -% Name -> -% %% In this case C1 is an atom -% TagCommand = get_tag_command(TS,?MANDATORY,Prop), -% create_pdec_inc_command(ModName,get_components(TS#type.def),Cs,[TagCommand|Acc]); {Name,undecoded} -> TagCommand = get_tag_command(TS,?UNDECODED,Prop), create_pdec_inc_command(ModName,Comps,Cs,concat_sequential(TagCommand,Acc)); {Name,parts} -> TagCommand = get_tag_command(TS,?PARTS,Prop), create_pdec_inc_command(ModName,Comps,Cs,concat_sequential(TagCommand,Acc)); - L when is_list(L) -> % I guess this never happens due to previous function clause + L when is_list(L) -> + %% I guess this never happens due to previous clause. %% This case is only possible as the first element after %% the top type element, when top type is SEGUENCE or SET. %% Follow each element in L. Must note every tag on the @@ -1555,8 +1527,6 @@ create_pdec_inc_command(ModName, RestPartsList,[]), create_pdec_inc_command(ModName,Comps,Cs, [[?MANDATORY,InnerDirectives]|Acc]); -% create_pdec_inc_command(ModName,Comps,Cs, -% [InnerDirectives,?MANDATORY|Acc]); [Opt,EncTag] -> InnerDirectives = create_pdec_inc_command(ModName,TS#type.def, @@ -1564,9 +1534,8 @@ create_pdec_inc_command(ModName, create_pdec_inc_command(ModName,Comps,Cs, [[Opt,EncTag,InnerDirectives]|Acc]) end; -% create_pdec_inc_command(ModName,CList,RestPartsList,Acc); -%% create_pdec_inc_command(ModName,TS#type.def,RestPartsList,Acc); - _ -> %% this component may not be in the config list + _ -> + %% this component may not be in the config list TagCommand = get_tag_command(TS,?MANDATORY,Prop), create_pdec_inc_command(ModName,Comps,TNL,concat_sequential(TagCommand,Acc)) end; @@ -1577,7 +1546,6 @@ create_pdec_inc_command(ModName, [{C1,Directive}|Rest],Acc) -> case Directive of List when is_list(List) -> -% [Command,Tag] = get_tag_command(TS,?ALTERNATIVE,Prop), TagCommand = get_tag_command(TS,?ALTERNATIVE,Prop), CompAcc = create_pdec_inc_command(ModName, @@ -1586,9 +1554,6 @@ create_pdec_inc_command(ModName, [Command,Tag] when is_atom(Command) -> [[Command,Tag,CompAcc]|Acc]; [L1,_L2|Rest] when is_list(L1) -> -% [LastComm|Comms] = lists:reverse(TagCommand), -% [concat_sequential(lists:reverse(Comms), -% [LastComm,CompAcc])|Acc] case lists:reverse(TagCommand) of [Atom|Comms] when is_atom(Atom) -> [concat_sequential(lists:reverse(Comms), @@ -1597,12 +1562,8 @@ create_pdec_inc_command(ModName, [concat_sequential(lists:reverse(Comms), [[Command2,Tag2,CompAcc]])|Acc] end -% [concat_sequential(lists:reverse(Comms), -% InnerCommand)|Acc] - end, create_pdec_inc_command(ModName,{'CHOICE',Comps},Rest, -% [[Command,Tag,CompAcc]|Acc]); NewAcc); undecoded -> TagCommand = get_tag_command(TS,?ALTERNATIVE_UNDECODED,Prop), @@ -1658,7 +1619,6 @@ create_partial_decode_gen_info(_M1,{M2,_}) -> throw({error,{"wrong module name in asn1 config file", M2}}). -%create_partial_decode_gen_info1(ModName,{ModName,TypeList}) -> create_partial_decode_gen_info1(ModName,{FuncName,TypeList}) -> case TypeList of [TopType|Rest] -> @@ -1678,11 +1638,6 @@ create_partial_decode_gen_info1(ModName,{FuncName,TypeList}) -> end; create_partial_decode_gen_info1(_,_) -> ok. -% create_partial_decode_gen_info1(_,[]) -> -% []; -% create_partial_decode_gen_info1(_M1,{M2,_}) -> -% throw({error,{"wrong module name in asn1 config file", -% M2}}). %% create_pdec_command/4 for each name (type or component) in the %% third argument, TypeNameList, a command is created. The command has @@ -1698,7 +1653,6 @@ create_pdec_command(_ModName,_,[],Acc) -> Fun(L,[H|Res],Fun) end, Remove_empty_lists(Acc,[],Remove_empty_lists); -% lists:reverse(Acc); create_pdec_command(ModName,[#'ComponentType'{name=C1,typespec=TS}|_Comps], [C1|Cs],Acc) -> %% this component is a constructed type or the last in the @@ -1747,9 +1701,7 @@ create_pdec_command(ModName,TS=#type{def=Def},[C1|Cs],Acc) -> create_pdec_command(_,_,TNL,_) -> throw({error,{"unexpected error when creating partial " "decode command",TNL}}). - -% get_components({'CHOICE',Components}) -> -% Components; + get_components(#'SEQUENCE'{components={C1,C2}}) when is_list(C1),is_list(C2) -> C1++C2; get_components(#'SEQUENCE'{components=Components}) -> @@ -1820,8 +1772,6 @@ get_tag_command(#type{tag=[Tag]},Command) -> [Command,encode_tag_val(decode_class(Tag#tag.class),Tag#tag.form, Tag#tag.number)]; get_tag_command(T=#type{tag=[Tag|Tags]},Command) -> -% [get_tag_command(T#type{tag=[Tag]},Command)| -% [get_tag_command(T#type{tag=Tags},Command)]]. TC = get_tag_command(T#type{tag=[Tag]},Command), TCs = get_tag_command(T#type{tag=Tags},Command), case many_tags(TCs) of @@ -1849,7 +1799,6 @@ get_tag_command(#type{tag=Tag},Command,Prop) when is_record(Tag,tag) -> get_tag_command(#type{tag=[Tag]},Command,Prop); get_tag_command(T=#type{tag=[Tag|Tags]},Command,Prop) -> [get_tag_command(T#type{tag=[Tag]},Command,Prop)|[ -% get_tag_command(T#type{tag=Tags},?MANDATORY,Prop)]]. get_tag_command(T#type{tag=Tags},Command,Prop)]]. anonymous_dec_command(?UNDECODED,'OPTIONAL') -> @@ -1964,8 +1913,8 @@ read_config_data(Key) -> true -> case asn1ct_table:lookup(asn1_general,{asn1_config,Key}) of [{_,Data}] -> Data; - Err -> % Err is [] when nothing was saved in the ets table -%% io:format("strange data from config file ~w~n",[Err]), + Err -> + %% Err is [] when nothing was saved in the ets table Err end end. @@ -1978,7 +1927,6 @@ read_config_data(Key) -> %% saves input data in a new gen_state record save_gen_state(exclusive_decode,{_,ConfList},PartIncTlvTagList) -> - %ConfList=[{FunctionName,PatternList}|Rest] State = case get_gen_state() of S when is_record(S,gen_state) -> S; @@ -1988,14 +1936,12 @@ save_gen_state(exclusive_decode,{_,ConfList},PartIncTlvTagList) -> inc_type_pattern=ConfList}, save_config(gen_state,StateRec); save_gen_state(_,_,_) -> -%% ok. case get_gen_state() of S when is_record(S,gen_state) -> ok; _ -> save_config(gen_state,#gen_state{}) end. save_gen_state(selective_decode,{_,Type_component_name_list}) -> -%% io:format("Selective_decode: ~p~n",[Type_component_name_list]), State = case get_gen_state() of S when is_record(S,gen_state) -> S; @@ -2077,11 +2023,6 @@ update_gen_state(type_pattern,State,Data) -> update_gen_state(func_name,State,Data) -> save_gen_state(State#gen_state{func_name=Data}); update_gen_state(namelist,State,Data) -> -% SData = -% case Data of -% [D] when is_list(D) -> D; -% _ -> Data -% end, save_gen_state(State#gen_state{namelist=Data}); update_gen_state(tobe_refed_funcs,State,Data) -> save_gen_state(State#gen_state{tobe_refed_funcs=Data}); @@ -2136,7 +2077,6 @@ get_tobe_refed_func(Name) -> %% tuple. Do not save if it exists in generated_functions, because %% then it will be or already is generated. add_tobe_refed_func(Data) -> - %% {Name,SI,Pattern} = fun({N,Si,P,_}) -> {N,Si,P}; (D) -> D end (Data), @@ -2144,8 +2084,6 @@ add_tobe_refed_func(Data) -> case SI of I when is_integer(I) -> fun(D) -> D end(Data); -% fun({N,Ix,P}) -> {N,Ix+1,P}; -% ({N,Ix,P,T}) -> {N,Ix+1,P,T} end (Data); _ -> fun({N,_,P}) -> {N,0,P}; ({N,_,P,T}) -> {N,0,P,T} end (Data) @@ -2153,12 +2091,13 @@ add_tobe_refed_func(Data) -> L = get_gen_state_field(generated_functions), case generated_functions_member(get(currmod),Name,L,Pattern) of - true -> % it exists in generated_functions, it has already - % been generated or saved in tobe_refed_func + true -> + %% it exists in generated_functions, it has already + %% been generated or saved in tobe_refed_func ok; _ -> add_once_tobe_refed_func(NewData), - %%only to get it saved in generated_functions + %% only to get it saved in generated_functions maybe_rename_function(tobe_refed,Name,Pattern) end. @@ -2173,16 +2112,13 @@ add_once_tobe_refed_func(Data) -> ({N,I,_,_}) when N==Name,I==Index -> true; (_) -> false end,TRFL) of [] -> -%% case lists:keysearch(element(1,Data),1,TRFL) of -%% false -> update_gen_state(tobe_refed_funcs,[Data|TRFL]); _ -> ok end. - -%% moves Name from the to be list to the generated list. +%% Moves Name from the to be list to the generated list. generated_refed_func(Name) -> L = get_gen_state_field(tobe_refed_funcs), NewL = lists:keydelete(Name,1,L), @@ -2190,7 +2126,7 @@ generated_refed_func(Name) -> L2 = get_gen_state_field(gen_refed_funcs), update_gen_state(gen_refed_funcs,[Name|L2]). -%% adds Data to gen_refed_funcs field in gen_state. +%% Adds Data to gen_refed_funcs field in gen_state. add_generated_refed_func(Data) -> case is_function_generated(Data) of true -> @@ -2212,7 +2148,7 @@ next_refed_func() -> reset_gen_state() -> save_gen_state(#gen_state{}). -%% adds Data to generated_functions field in gen_state. +%% Adds Data to generated_functions field in gen_state. add_generated_function(Data) -> L = get_gen_state_field(generated_functions), update_gen_state(generated_functions,[Data|L]). @@ -2231,16 +2167,18 @@ maybe_rename_function(Mode,Name,Pattern) -> {_,true} -> L2 = generated_functions_filter(get(currmod),Name,L), case lists:keysearch(Pattern,3,L2) of - false -> %name existed, but not pattern + false -> + %% name existed, but not pattern NextIndex = length(L2), - %%rename function + %% rename function Suffix = lists:concat(["_",NextIndex]), NewName = maybe_rename_function2(type_check(Name),Name, Suffix), add_generated_function({Name,NextIndex,Pattern}), NewName; - Value -> % name and pattern existed + Value -> + %% name and pattern existed %% do not save any new index Suffix = make_suffix(Value), Name2 = @@ -2250,9 +2188,9 @@ maybe_rename_function(Mode,Name,Pattern) -> end, lists:concat([Name2,Suffix]) end; - {inc_disp,_} -> %% this is when - %% decode_partial_inc_disp/2 is - %% generated + {inc_disp,_} -> + %% this is when decode_partial_inc_disp/2 is + %% generated add_generated_function({Name,0,Pattern}), Name; _ -> % this if call from add_tobe_refed_func @@ -2298,23 +2236,12 @@ generated_functions_member(M,Name,[_|T]) -> generated_functions_member(_,_,[]) -> false. -% generated_functions_member(M,Name,L) -> -% case lists:keymember(Name,1,L) of -% true -> -% true; -% _ -> -% generated_functions_member1(M,Name,L) -% end. -% generated_functions_member1(M,#'Externaltypereference'{module=M,type=Name},L) -> -% lists:keymember(Name,1,L); -% generated_functions_member1(_,_,_) -> false. - generated_functions_filter(_,Name,L) when is_atom(Name);is_list(Name) -> lists:filter(fun({N,_,_}) when N==Name -> true; (_) -> false end, L); generated_functions_filter(M,#'Externaltypereference'{module=M,type=Name},L)-> - % remove toptypename from patterns + %% remove top typename from patterns RemoveTType = fun({N,I,[N,P]}) when N == Name -> {N,I,P}; @@ -2351,8 +2278,6 @@ set_current_sindex(Index) -> type_check(A) when is_atom(A) -> atom; -%% type_check(I) when is_integer(I) -> -%% integer; type_check(L) when is_list(L) -> Pred = fun(X) when X=<255 -> false; diff --git a/lib/asn1/src/asn1ct_check.erl b/lib/asn1/src/asn1ct_check.erl index 321f4147f5..e867b9606a 100644 --- a/lib/asn1/src/asn1ct_check.erl +++ b/lib/asn1/src/asn1ct_check.erl @@ -23,10 +23,9 @@ %% Main Module for ASN.1 compile time functions -%-compile(export_all). -export([check/2,storeindb/2,format_error/1]). -%-define(debug,1). -include("asn1_records.hrl"). + %%% The tag-number for universal types -define(N_BOOLEAN, 1). -define(N_INTEGER, 2). @@ -63,7 +62,8 @@ -define(TAG_CONSTRUCTED(Num), #tag{class='UNIVERSAL',number=Num,type='IMPLICIT',form=32}). --record(newt,{type=unchanged,tag=unchanged,constraint=unchanged,inlined=no}). % used in check_type to update type and tag +%% used in check_type to update type and tag +-record(newt,{type=unchanged,tag=unchanged,constraint=unchanged,inlined=no}). check(S,{Types,Values,ParameterizedTypes,Classes,Objects,ObjectSets}) -> %%Predicates used to filter errors @@ -561,7 +561,6 @@ check_class_fields(S,[F|Fields],Acc) -> D; {undefined,user} -> %% neither of {primitive,bif} or {constructed,bif} - {_,D} = get_referenced_type(S,#'Externaltypereference'{module=S#state.mname,type=Type#type.def}), D; _ -> @@ -623,7 +622,6 @@ if_current_checked_type(S,#type{def=Def}) -> CurrentModule = S#state.mname, CurrentCheckedName = S#state.tname, MergedModules = S#state.inputmodules, - % CurrentCheckedModule = S#state.mname, case Def of #'Externaltypereference'{module=CurrentModule, type=CurrentCheckedName} -> @@ -656,7 +654,6 @@ check_pobjectset(S,PObjSet) -> ClassName = #'Externaltypereference'{module=Mod, type=get_datastr_name(Def)}, {valueset,Set} = ValueSet, -% ObjectSet = #'ObjectSet'{class={objectclassname,ClassName}, ObjectSet = #'ObjectSet'{class=ClassName, set=Set}, #pobjectsetdef{pos=Pos,name=Name,args=Args,class=Type#type.def, @@ -1696,7 +1693,7 @@ check_value(OldS,V) when is_record(V,typedef) -> %% reference to class check_value(OldS,V#typedef{typespec=TS#'ObjectSet'{class=Eref}}); #typedef{typespec=HostType} -> - % an ordinary value set with a type in #typedef.typespec + %% an ordinary value set with a type in #typedef.typespec ValueSet0 = TS#'ObjectSet'.set, Constr = check_constraints(OldS, HostType, [ValueSet0]), Type = check_type(OldS,TSDef,TSDef#typedef.typespec), @@ -2381,15 +2378,6 @@ normalize_s_of(SorS,S,Value,Type,NameList) %% normalize_restrictedstring handles all format of restricted strings. -%% tuple case -% normalize_restrictedstring(_S,[Int1,Int2],_) when is_integer(Int1),is_integer(Int2) -> -% {Int1,Int2}; -% %% quadruple case -% normalize_restrictedstring(_S,[Int1,Int2,Int3,Int4],_) when is_integer(Int1), -% is_integer(Int2), -% is_integer(Int3), -% is_integer(Int4) -> -% {Int1,Int2,Int3,Int4}; %% character string list case normalize_restrictedstring(S,[H|T],CType) when is_list(H);is_tuple(H) -> [normalize_restrictedstring(S,H,CType)|normalize_restrictedstring(S,T,CType)]; @@ -2491,7 +2479,7 @@ check_ptype(S,Type,Ts) when is_record(Ts,type) -> Ts#type{def=TDef} end, Ts2; -%parameterized class +%% parameterized class check_ptype(_S,_PTDef,Ts) when is_record(Ts,objectclass) -> throw({asn1_param_class,Ts}). @@ -2506,8 +2494,6 @@ check_formal_parameter(_, #'Externaltypereference'{}) -> check_formal_parameter(S, #'Externalvaluereference'{value=Name}) -> asn1_error(S, {illegal_typereference,Name}). -% check_type(S,Type,ObjSpec={{objectclassname,_},_}) -> - % check_class(S,ObjSpec); check_type(_S,Type,Ts) when is_record(Type,typedef), (Type#typedef.checked==true) -> Ts; @@ -2606,7 +2592,6 @@ check_type(S=#state{recordtopname=TopName},Type,Ts) when is_record(Ts,type) -> constraint = NewC}; _ -> %% Here we only expand the tags and keep the ext ref. - NewExt = ExtRef#'Externaltypereference'{module=merged_mod(S,RefMod,Ext)}, TempNewDef#newt{ type = check_externaltypereference(S,NewExt), @@ -2749,7 +2734,6 @@ check_type(S=#state{recordtopname=TopName},Type,Ts) when is_record(Ts,type) -> case TopName of [] -> [get_datastr_name(Type)]; -% [Type#typedef.name]; _ -> TopName end, @@ -2773,7 +2757,6 @@ check_type(S=#state{recordtopname=TopName},Type,Ts) when is_record(Ts,type) -> case TopName of [] -> [get_datastr_name(Type)]; -% [Type#typedef.name]; _ -> TopName end, @@ -2898,8 +2881,6 @@ tablecinf_choose(#'SEQUENCE'{tablecinf=TCI}) -> get_innertag(_S,#'ObjectClassFieldType'{type=Type}) -> case Type of -% #type{tag=Tag} -> Tag; -% {fixedtypevaluefield,_,#type{tag=[]}=T} -> get_taglist(S,T); {fixedtypevaluefield,_,#type{tag=Tag}} -> Tag; {TypeFieldName,_} when is_atom(TypeFieldName) -> []; _ -> [] @@ -3754,14 +3735,8 @@ check_reference(S,#'Externaltypereference'{pos=Pos,module=Emod,type=Name}) -> {ok,Imodule} -> check_imported(S,Imodule,Name), #'Externaltypereference'{module=Imodule,type=Name}; -%% case check_imported(S,Imodule,Name) of -%% ok -> -%% #'Externaltypereference'{module=Imodule,type=Name}; -%% Err -> -%% Err -%% end; _ -> - %may be a renamed type in multi file compiling! + %% may be a renamed type in multi file compiling! {M,T}=get_renamed_reference(S,Name,Emod), NewName = asn1ct:get_name_of_def(T), NewPos = asn1ct:get_pos_of_def(T), @@ -4170,7 +4145,6 @@ iof_associated_type(S,[]) -> def=AssociateSeq}}, asn1_db:dbput(S#state.mname,'INSTANCE OF',TypeDef), instance_of_decl(S#state.mname); -%% put(instance_of,{generate,S#state.mname}); _ -> instance_of_decl(S#state.mname), ok @@ -4199,14 +4173,12 @@ iof_associated_type1(S,C) -> ObjectIdentifier = #'ObjectClassFieldType'{classname=TypeIdentifierRef, class=[], -%% fieldname=[{valuefieldreference,id}], fieldname={id,[]}, type={fixedtypevaluefield,id, #type{def='OBJECT IDENTIFIER'}}}, Typefield = #'ObjectClassFieldType'{classname=TypeIdentifierRef, class=[], -%% fieldname=[{typefieldreference,'Type'}], fieldname={'Type',[]}, type=Typefield_type}, IOFComponents0 = @@ -4360,11 +4332,11 @@ check_boolean(_S,_Constr) -> check_octetstring(_S,_Constr) -> ok. -% check all aspects of a SEQUENCE -% - that all component names are unique -% - that all TAGS are ok (when TAG default is applied) -% - that each component is of a valid type -% - that the extension marks are valid +%% check all aspects of a SEQUENCE +%% - that all component names are unique +%% - that all TAGS are ok (when TAG default is applied) +%% - that each component is of a valid type +%% - that the extension marks are valid check_sequence(S,Type,Comps) -> Components = expand_components(S,Comps), @@ -4705,11 +4677,11 @@ check_objectidentifier(_S,_Constr) -> check_relative_oid(_S,_Constr) -> ok. -% check all aspects of a CHOICE -% - that all alternative names are unique -% - that all TAGS are ok (when TAG default is applied) -% - that each alternative is of a valid type -% - that the extension marks are valid +%% check all aspects of a CHOICE +%% - that all alternative names are unique +%% - that all TAGS are ok (when TAG default is applied) +%% - that each alternative is of a valid type +%% - that the extension marks are valid check_choice(S,Type,Components) when is_list(Components) -> Components1 = [C||C = #'ComponentType'{} <- Components], case check_unique(Components1,#'ComponentType'.name) of @@ -4948,7 +4920,7 @@ componentrelation_leadingattr(S,CompList) -> %%FIXME expand_ExtAddGroups([C#'ExtensionAdditionGroup'{components=ExtAdds}|T], %% CurrPos,PosAcc,CompAcc) -> -%% expand_ExtAddGroups(T,CurrPos+ L = lenght(ExtAdds),[{CurrPos,L}|PosAcc],ExtAdds++CompAcc); +%% expand_ExtAddGroups(T,CurrPos+ L = length(ExtAdds),[{CurrPos,L}|PosAcc],ExtAdds++CompAcc); %% expand_ExtAddGroups([C|T],CurrPos,PosAcc,CompAcc) -> %% expand_ExtAddGroups(T,CurrPos+ 1,PosAcc,[C|CompAcc]); %% expand_ExtAddGroups([],_CurrPos,PosAcc,CompAcc) -> @@ -5063,12 +5035,12 @@ remove_doubles1(El,L) -> %% referred to in the ObjectClassFieldType, and the name of the unique %% field of the class of the ObjectClassFieldType. %% -% %% The level information outermost/innermost must be kept. There are -% %% at least two possibilities to cover here for an outermost case: 1) -% %% Both the simple table and the component relation have a common path -% %% at least one step below the outermost level, i.e. the leading -% %% information shall be on a sub level. 2) They don't have any common -% %% path. +%% The level information outermost/innermost must be kept. There are +%% at least two possibilities to cover here for an outermost case: 1) +%% Both the simple table and the component relation have a common path +%% at least one step below the outermost level, i.e. the leading +%% information shall be on a sub level. 2) They don't have any common +%% path. get_simple_table_info(S, Cs, AtLists) -> [get_simple_table_info1(S, Cs, AtList, []) || AtList <- AtLists]. @@ -5109,10 +5081,10 @@ simple_table_info(S,#'ObjectClassFieldType'{classname=ClRef, {_FirstFieldName,FieldNames} -> lists:last(FieldNames) end, - %%ObjectClassFieldName is the last element in the dotted - %%list of the ObjectClassFieldType. The last element may - %%be of another class, that is referenced from the class - %%of the ObjectClassFieldType + %% ObjectClassFieldName is the last element in the dotted list of + %% the ObjectClassFieldType. The last element may be of another + %% class, that is referenced from the class of the + %% ObjectClassFieldType ClassDef = case ObjectClass of [] -> @@ -5128,7 +5100,7 @@ simple_table_info(S,#'ObjectClassFieldType'{classname=ClRef, %% the "name path" in the at-list to the component relation constraint %% that must refer to a simple table constraint. The list is empty if %% no component relation constraints were found. -%% +%% %% NamePath has the names of all components that are followed from the %% beginning of the search. CNames holds the names of all components %% of the start level, this info is used if an outermost at-notation @@ -5141,6 +5113,7 @@ any_component_relation(S,[#'ComponentType'{name=CName,typespec=Type}|Cs],CNames, %% whether this constraint is relevant for the level %% where the search started AtNot = extract_at_notation(AtNotation), + %% evaluate_atpath returns the relative path to the %% simple table constraint from where the component %% relation is found. @@ -5246,12 +5219,10 @@ get_components(_,#'SET'{components=Cs}) -> tuple2complist(Cs); get_components(_,{'CHOICE',Cs}) -> tuple2complist(Cs); -%do not step in inlined structures +%%do not step in inlined structures get_components(any,{'SEQUENCE OF',T = #type{def=_Def,inlined=no}}) -> -% get_components(any,Def); T; get_components(any,{'SET OF',T = #type{def=_Def,inlined=no}}) -> -% get_components(any,Def); T; get_components(_,_) -> []. @@ -5281,15 +5252,12 @@ extract_at_notation([{Level,ValueRefs}]) -> componentrelation1(S,C = #type{def=Def,constraint=Constraint,tablecinf=TCI}, Path) -> Ret = -% case Constraint of -% [{componentrelation,{_,_,ObjectSet},AtList}|_Rest] -> case lists:keyfind(componentrelation, 1, Constraint) of {_,{_,_,ObjectSet},AtList} -> [{_,AL=[#'Externalvaluereference'{}|_R1]}|_R2] = AtList, %% Note: if Path is longer than one,i.e. it is within %% an inner type of the actual level, then the only %% relevant at-list is of "outermost" type. -%% #'ObjectClassFieldType'{class=ClassDef} = Def, ClassDef = get_ObjectClassFieldType_classdef(S,Def), AtPath = lists:map(fun(#'Externalvaluereference'{value=V})->V end, @@ -5375,7 +5343,6 @@ innertype_comprel1(S,T = #type{def=Def,constraint=Cons,tablecinf=TCI},Path) -> %% relevent here. [{_,AL=[#'Externalvaluereference'{value=_Attr}|_R1]}|_R2] = AtList, -%% #'ObjectClassFieldType'{class=ClassDef} = Def, ClassDef = get_ObjectClassFieldType_classdef(S,Def), AtPath = lists:map(fun(#'Externalvaluereference'{value=V})->V end, @@ -5444,7 +5411,7 @@ leading_attr_index1(S,[C|Cs],Arg={ObjectSet,_,CDef,P}, value_match(S,C,Name,SubAttr) -> value_match(S,C,Name,SubAttr,[]). % C has name Name value_match(_S,#'ComponentType'{},_Name,[],Acc) -> - Acc;% do not reverse, indexes in reverse order + Acc; % do not reverse, indexes in reverse order value_match(S,#'ComponentType'{typespec=Type},Name,[At|Ats],Acc) -> InnerType = asn1ct_gen:get_inner(Type#type.def), Components = @@ -5514,8 +5481,6 @@ get_tableconstraint_info(S,Type,[C=#'ComponentType'{typespec=CheckedTs}|Cs],Acc) CheckedTs#type{ def=NewOCFT }}; -% constraint=[{tableconstraint_info, -% FieldRef}]}}; {'SEQUENCE OF',SOType} when is_record(SOType,type), (element(1,SOType#type.def)=='CHOICE') -> CTypeList = element(2,SOType#type.def), @@ -5618,51 +5583,6 @@ get_taglist1(S,[_H|Rest]) -> % skip EXTENSIONMARK get_taglist1(_S,[]) -> []. -%% def_to_tag(S,Def) -> -%% case asn1ct_gen:def_to_tag(Def) of -%% {'UNIVERSAL',T} -> -%% case asn1ct_gen:prim_bif(T) of -%% true -> -%% ?TAG_PRIMITIVE(tag_number(T)); -%% _ -> -%% ?TAG_CONSTRUCTED(tag_number(T)) -%% end; -%% _ -> [] -%% end. -%% tag_number('BOOLEAN') -> 1; -%% tag_number('INTEGER') -> 2; -%% tag_number('BIT STRING') -> 3; -%% tag_number('OCTET STRING') -> 4; -%% tag_number('NULL') -> 5; -%% tag_number('OBJECT IDENTIFIER') -> 6; -%% tag_number('ObjectDescriptor') -> 7; -%% tag_number('EXTERNAL') -> 8; -%% tag_number('INSTANCE OF') -> 8; -%% tag_number('REAL') -> 9; -%% tag_number('ENUMERATED') -> 10; -%% tag_number('EMBEDDED PDV') -> 11; -%% tag_number('UTF8String') -> 12; -%% %%tag_number('RELATIVE-OID') -> 13; -%% tag_number('SEQUENCE') -> 16; -%% tag_number('SEQUENCE OF') -> 16; -%% tag_number('SET') -> 17; -%% tag_number('SET OF') -> 17; -%% tag_number('NumericString') -> 18; -%% tag_number('PrintableString') -> 19; -%% tag_number('TeletexString') -> 20; -%% %%tag_number('T61String') -> 20; -%% tag_number('VideotexString') -> 21; -%% tag_number('IA5String') -> 22; -%% tag_number('UTCTime') -> 23; -%% tag_number('GeneralizedTime') -> 24; -%% tag_number('GraphicString') -> 25; -%% tag_number('VisibleString') -> 26; -%% %%tag_number('ISO646String') -> 26; -%% tag_number('GeneralString') -> 27; -%% tag_number('UniversalString') -> 28; -%% tag_number('CHARACTER STRING') -> 29; -%% tag_number('BMPString') -> 30. - merge_tags(T1, T2) when is_list(T2) -> merge_tags2(T1 ++ T2, []); merge_tags(T1, T2) -> diff --git a/lib/asn1/src/asn1ct_constructed_ber_bin_v2.erl b/lib/asn1/src/asn1ct_constructed_ber_bin_v2.erl index 16af09bca9..bfb69a09b3 100644 --- a/lib/asn1/src/asn1ct_constructed_ber_bin_v2.erl +++ b/lib/asn1/src/asn1ct_constructed_ber_bin_v2.erl @@ -32,17 +32,17 @@ -include("asn1_records.hrl"). --import(asn1ct_gen, [emit/1,demit/1,get_record_name_prefix/1]). +-import(asn1ct_gen, [emit/1,get_record_name_prefix/1]). -define(ASN1CT_GEN_BER,asn1ct_gen_ber_bin_v2). -% the encoding of class of tag bits 8 and 7 +%% the encoding of class of tag bits 8 and 7 -define(UNIVERSAL, 0). -define(APPLICATION, 16#40). -define(CONTEXT, 16#80). -define(PRIVATE, 16#C0). -% primitive or constructed encoding % bit 6 +%% primitive or constructed encoding % bit 6 -define(PRIMITIVE, 0). -define(CONSTRUCTED, 2#00100000). @@ -103,7 +103,6 @@ gen_encode_sequence(Gen, Typename, #type{}=D) -> uniqueclassfield=Unique} when Used /= Unique -> false; %% ObjectSet, name of the object set in constraints - %% #simpletableattributes{objectsetname=ObjectSetRef, c_name=AttrN, c_index=N, @@ -230,7 +229,6 @@ gen_decode_sequence(Gen, Typename, #type{}=D) -> usedclassfield=UniqueFieldName, uniqueclassfield=UniqueFieldName, valueindex=ValIndex} -> -% {ObjectSetRef,AttrN,_N,UniqueFieldName} ->%% N is index of attribute that determines constraint F = fun(#'ComponentType'{typespec=CT})-> case {asn1ct_gen:get_constraint(CT#type.constraint,componentrelation),CT#type.tablecinf} of {no,[{objfun,_}|_]} -> true; @@ -279,12 +277,12 @@ gen_decode_sequence(Gen, Typename, #type{}=D) -> ValueMatch,"),",nl]), gen_dec_postponed_decs(DecObj,PostponedDecArgs) end, - demit(["Result = "]), %dbg %% return value as record case Ext of {ext,_,_} -> emit(["case ",{prev,tlv}," of [] -> true; _ -> true end, % ... extra fields skipped",nl]); - _ -> % noext | extensible + _ -> + %% noext | extensible emit(["case ",{prev,tlv}," of",nl, "[] -> true;", "_ -> exit({error,{asn1, {unexpected,",{prev,tlv}, @@ -431,7 +429,6 @@ gen_decode_set(Gen, Typename, #type{}=D) -> {DecObjInf,ValueIndex} = case TableConsInfo of -%% {ObjectSetRef,AttrN,_N,UniqueFieldName} ->%% N is index of attribute that determines constraint #simpletableattributes{objectsetname=ObjectSetRef, c_name=AttrN, usedclassfield=UniqueFieldName, @@ -446,7 +443,8 @@ gen_decode_set(Gen, Typename, #type{}=D) -> end end, case lists:any(F,CompList) of - true -> % when component relation constraint establish + true -> + %% when component relation constraint establish %% relation from a component to another components %% subtype component {{AttrN,{deep,ObjectSetRef,UniqueFieldName,ValIndex}}, @@ -503,7 +501,6 @@ gen_decode_set(Gen, Typename, #type{}=D) -> ValueMatch,"),",nl]), gen_dec_postponed_decs(DecObj,PostponedDecArgs) end, - demit(["Result = "]), %dbg %% return value as record case Ext of Extnsn when Extnsn =/= noext -> @@ -722,7 +719,7 @@ gen_dec_sequence_call2(Erules,TopType,{Root1,EList,Root2},_Ext,DecObjInf) -> length(Root1)+length(EList),noext, DecObjInf,LA,ArgsAcc). -%% returns a list of tags of the elements in the component (second +%% Returns a list of tags of the elements in the component (second %% root) list up to and including the first mandatory tag. See 24.6 in %% X.680 (7/2002) get_root2_taglist([],Acc) -> @@ -811,8 +808,6 @@ gen_dec_set_cases(Erules,TopType,[Comp|RestComps],Pos) -> [FirstTag|_] -> [(?ASN1CT_GEN_BER:decode_class(FirstTag#tag.class) bsl 10) + FirstTag#tag.number] end, -% emit([indent(6),"%Tags: ",Tags,nl]), -% emit([indent(6),"%Type#type.tag: ",Type#type.tag,nl]), CaseFun = fun(TagList=[H|T],Fun,N) -> Semicolon = case TagList of [_Tag1,_|_] -> [";",nl]; @@ -827,7 +822,6 @@ gen_dec_set_cases(Erules,TopType,[Comp|RestComps],Pos) -> emit([";",nl]) end, CaseFun(Tags,CaseFun,0), -%% emit([";",nl]), gen_dec_set_cases(Erules,TopType,RestComps,Pos+1). @@ -1007,14 +1001,6 @@ gen_enc_line(Erules,TopType,Cname, ["{",{curr,encBytes},",",{curr,encLen},"} = "], EncObj) end; -% gen_enc_line(Erules,TopType,Cname, -% Type=#type{constraint=[{componentrelation,_,_}], -% def=#'ObjectClassFieldType'{type={typefield,_}}}, -% Element,Indent,OptOrMand=mandatory,EncObj) -% when is_list(Element) -> -% asn1ct_name:new(tmpBytes), -% gen_enc_line(Erules,TopType,Cname,Type,Element,Indent,OptOrMand, -% ["{",{curr,tmpBytes},",_} = "],EncObj); gen_enc_line(Erules,TopType,Cname,Type,Element,Indent,OptOrMand,EncObj) when is_list(Element) -> gen_enc_line(Erules,TopType,Cname,Type,Element,Indent,OptOrMand, @@ -1035,37 +1021,30 @@ gen_enc_line(Erules,TopType,Cname,Type,Element,Indent,OptOrMand,Assign,EncObj) gen_optormand_case(OptOrMand, Erules, TopType, Cname, Type, Element), case {Type,asn1ct_gen:get_constraint(Type#type.constraint, componentrelation)} of -% #type{constraint=[{tableconstraint_info,RefedFieldName}], -% def={typefield,_}} -> {#type{def=#'ObjectClassFieldType'{type={typefield,_}, fieldname=RefedFieldName}}, {componentrelation,_,_}} -> {_LeadingAttrName,Fun} = EncObj, - case RefedFieldName of - {Name,RestFieldNames} when is_atom(Name) -> - case OptOrMand of - mandatory -> ok; - _ -> -% emit(["{",{curr,tmpBytes},",",{curr,tmpLen}, - emit(["{",{curr,tmpBytes},",_ } = "]) -% "} = "]) - end, - emit([Fun,"(",{asis,Name},", ",Element,", ", - {asis,RestFieldNames},"),",nl]), - emit(IndDeep), - case OptOrMand of - mandatory -> - emit(["{",{curr,encBytes},",",{curr,encLen}, - "} = ", - {call,ber,encode_open_type, - [{curr,tmpBytes},{asis,Tag}]},nl]); - _ -> - emit([{call,ber,encode_open_type, - [{curr,tmpBytes},{asis,Tag}]}]) - end; - Err -> - throw({asn1,{'internal error',Err}}) - end; + {Name,RestFieldNames} = RefedFieldName, + true = is_atom(Name), %Assertion. + case OptOrMand of + mandatory -> ok; + _ -> + emit(["{",{curr,tmpBytes},",_ } = "]) + end, + emit([Fun,"(",{asis,Name},", ",Element,", ", + {asis,RestFieldNames},"),",nl]), + emit(IndDeep), + case OptOrMand of + mandatory -> + emit(["{",{curr,encBytes},",",{curr,encLen}, + "} = ", + {call,ber,encode_open_type, + [{curr,tmpBytes},{asis,Tag}]},nl]); + _ -> + emit([{call,ber,encode_open_type, + [{curr,tmpBytes},{asis,Tag}]}]) + end; _ -> case WhatKind of {primitive,bif} -> @@ -1166,7 +1145,9 @@ gen_dec_line(Erules,TopType,Cname,CTags,Type,OptOrMand,DecObjInf) -> gen_dec_call(InnerType,Erules,TopType,Cname,Type, BytesVar,Tag, mandatory,", mandatory, ",DecObjInf,OptOrMand); - _ -> %optional or default or a mandatory component after an extensionmark + _ -> + %% optional or default, or a mandatory component after + %% an extension marker {FirstTag,RestTag} = case Tag of [] -> @@ -1241,9 +1222,9 @@ gen_dec_line(Erules,TopType,Cname,CTags,Type,OptOrMand,DecObjInf) -> PostponedDec end, case DecObjInf of - {Cname,ObjSet} -> % this must be the component were an object is - %% choosen from the object set according to the table - %% constraint. + {Cname,ObjSet} -> + %% This must be the component were an object is chosen + %% from the object set according to the table constraint. ObjSetName = case ObjSet of {deep,OSName,_,_} -> OSName; @@ -1280,10 +1261,7 @@ gen_dec_call({typefield,_},_,_,_Cname,Type,BytesVar,Tag,_,_,false,_) -> []; gen_dec_call({typefield,_},_,_,Cname,Type,BytesVar,Tag,_,_,_DecObjInf,OptOrMandComp) -> call(decode_open_type, [BytesVar,{asis,Tag}]), - RefedFieldName = -% asn1ct_gen:get_constraint(Type#type.constraint, -% tableconstraint_info), - (Type#type.def)#'ObjectClassFieldType'.fieldname, + RefedFieldName = (Type#type.def)#'ObjectClassFieldType'.fieldname, [{Cname,RefedFieldName,asn1ct_gen:mk_var(asn1ct_name:curr(term)), asn1ct_gen:mk_var(asn1ct_name:curr(tmpterm)),Tag,OptOrMandComp}]; gen_dec_call(InnerType, Gen, TopType, Cname, Type, BytesVar, @@ -1339,8 +1317,6 @@ gen_dec_call1(WhatKind, _, TopType, Cname, Type, BytesVar, Tag) -> emit(["{'",asn1ct_gen:list2name([Cname|TopType]),"',", BytesVar,"}"]); _ -> -% {DecFunName, _DecMod, _DecFun} = -% case {asn1ct:get_gen_state_field(namelist),WhatKind} of EmitDecFunCall = fun(FuncName) -> case {WhatKind,Type#type.tablecinf} of @@ -1356,14 +1332,11 @@ gen_dec_call1(WhatKind, _, TopType, Cname, Type, BytesVar, Tag) -> Sindex = case WhatKind of #'Externaltypereference'{} -> -% asn1ct:maybe_rename_function(WhatKind,List), SI = asn1ct:maybe_saved_sindex(WhatKind,List), Saves = {WhatKind,SI,List}, asn1ct:add_tobe_refed_func(Saves), SI; _ -> -% asn1ct:maybe_rename_function([Cname|TopType], -% List), SI = asn1ct:maybe_saved_sindex([Cname|TopType],List), Saves = {[Cname|TopType],SI,List,Type}, asn1ct:add_tobe_refed_func(Saves), @@ -1371,8 +1344,6 @@ gen_dec_call1(WhatKind, _, TopType, Cname, Type, BytesVar, Tag) -> end, asn1ct:update_gen_state(namelist,Rest), Prefix=asn1ct:get_gen_state_field(prefix), -% Suffix = -% lists:concat(["_",asn1ct:latest_sindex()]), Suffix = case Sindex of I when is_integer(I),I>0 -> lists:concat(["_",I]); @@ -1380,8 +1351,6 @@ gen_dec_call1(WhatKind, _, TopType, Cname, Type, BytesVar, Tag) -> end, {DecFunName,_,_}= mkfuncname(TopType,Cname,WhatKind,Prefix,Suffix), -% SuffixedName = -% lists:concat([DecFunName,asn1ct:latest_sindex()]), EmitDecFunCall(DecFunName); [{Cname,parts}|Rest] -> asn1ct:update_gen_state(namelist,Rest), @@ -1401,13 +1370,6 @@ gen_dec_call1(WhatKind, _, TopType, Cname, Type, BytesVar, Tag) -> mkfuncname(TopType,Cname,WhatKind,"dec_",""), EmitDecFunCall(DecFunName) end -% case {WhatKind,Type#type.tablecinf} of -% {{constructed,bif},[{objfun,_}|_Rest]} -> -% emit([DecFunName,"(",BytesVar,", ",{asis,Tag}, -% ", ObjFun)"]); -% _ -> -% emit([DecFunName,"(",BytesVar,", ",{asis,Tag},")"]) -% end end. @@ -1464,6 +1426,9 @@ print_attribute_comment(InnerType,Pos,Cname,Prop) -> case InnerType of #'Externaltypereference'{module=XModule,type=Name} -> emit([nl,"%% attribute ",Cname,"(",Pos,") External ",XModule,":",Name]); + _ when is_tuple(InnerType) -> + emit([nl,"%% attribute ",Cname,"(",Pos,") with type "| + tuple_to_list(InnerType)]); _ -> emit([nl,"%% attribute ",Cname,"(",Pos,") with type ",InnerType]) end, diff --git a/lib/asn1/src/asn1ct_constructed_per.erl b/lib/asn1/src/asn1ct_constructed_per.erl index b7579c8065..986d88b677 100644 --- a/lib/asn1/src/asn1ct_constructed_per.erl +++ b/lib/asn1/src/asn1ct_constructed_per.erl @@ -30,9 +30,8 @@ -export([gen_decode_choice/3]). -include("asn1_records.hrl"). -%-compile(export_all). --import(asn1ct_gen, [emit/1,demit/1,get_record_name_prefix/1]). +-import(asn1ct_gen, [emit/1,get_record_name_prefix/1]). -type type_name() :: any(). @@ -357,7 +356,6 @@ gen_dec_constructed_imm(Erule, Typename, #type{}=D) -> #'SEQUENCE'{tablecinf=TCI,components=CL} -> {add_textual_order(CL),TCI}; #'SET'{tablecinf=TCI,components=CL} -> -%% {add_textual_order(CL),TCI} {CL,TCI} % the textual order is already taken care of end, Ext = extensible_dec(CompList), @@ -375,13 +373,11 @@ gen_dec_constructed_imm(Erule, Typename, #type{}=D) -> end, ObjSetInfo = case TableConsInfo of -%% {ObjectSet,AttrN,N,UniqueFieldName} ->%% N is index of attribute that determines constraint #simpletableattributes{objectsetname=ObjectSet, c_name=AttrN, usedclassfield=UniqueFieldName, uniqueclassfield=UniqueFieldName, valueindex=ValIndex} -> -%% {AttrN,ObjectSet}; F = fun(#'ComponentType'{typespec=CT})-> case {asn1ct_gen:get_constraint(CT#type.constraint,componentrelation),CT#type.tablecinf} of {no,[{objfun,_}|_R]} -> true; @@ -686,10 +682,10 @@ gen_decode_choice(Erules,Typename,D) when is_record(D,type) -> {'CHOICE',CompList} = D#type.def, Ext = extensible_enc(CompList), gen_dec_choice(Erules,Typename,CompList,Ext), - emit({".",nl}). + emit([".",nl]). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% Encode generator for SEQUENCE OF type +%% Encode generator for SEQUENCE OF type gen_encode_sof(Erule, Typename, SeqOrSetOf, D) -> asn1ct_name:start(), @@ -781,20 +777,20 @@ gen_decode_sof_components(Erule, Name, Typename, SeqOrSetOf, Cont) -> case asn1ct_gen:type(Conttype) of {primitive,bif} -> asn1ct_gen_per:gen_dec_prim(Erule, Cont, "Bytes"), - emit({com,nl}); + emit([com,nl]); {constructed,bif} -> NewTypename = [Constructed_Suffix|Typename], - emit({"'dec_",asn1ct_gen:list2name(NewTypename), - "'(Bytes",ObjFun,"),",nl}); + emit([{asis,dec_func(asn1ct_gen:list2name(NewTypename))}, + "(Bytes",ObjFun,"),",nl]); #'Externaltypereference'{}=Etype -> asn1ct_gen_per:gen_dec_external(Etype, "Bytes"), emit([com,nl]); 'ASN1_OPEN_TYPE' -> asn1ct_gen_per:gen_dec_prim(Erule, #type{def='ASN1_OPEN_TYPE'}, "Bytes"), - emit({com,nl}); + emit([com,nl]); _ -> - emit({"'dec_",Conttype,"'(Bytes),",nl}) + emit([{asis,dec_func(Conttype)},"(Bytes),",nl]) end, emit([{asis,Name},"(Num-1, Remain",ObjFun,", [Term|Acc]).",nl]). @@ -934,9 +930,7 @@ add_textual_order({R1,Ext,R2}) -> {NewExt,Num2} = add_textual_order1(Ext,Num1), {NewR2,_} = add_textual_order1(R2,Num2), {NewR1,NewExt,NewR2}. -%%add_textual_order1(Cs=[#'ComponentType'{textual_order=Int}|_],I) -%% when is_integer(Int) -> -%% {Cs,I}; + add_textual_order1(Cs,NumIn) -> lists:mapfoldl(fun(C=#'ComponentType'{},Num) -> {C#'ComponentType'{textual_order=Num}, @@ -979,6 +973,10 @@ mark_optional(Other) -> gen_enc_components_call1(Gen, TopType, [C|Rest], DynamicEnc, Ext) -> #'ComponentType'{name=Cname,typespec=Type, prop=Prop,textual_order=Num} = C, + InnerType = asn1ct_gen:get_inner(Type#type.def), + CommentString = attribute_comment(InnerType, Num, Cname), + ImmComment = asn1ct_imm:enc_comment(CommentString), + {Imm0,Element} = enc_fetch_field(Gen, Num, Prop), Imm1 = gen_enc_line_imm(Gen, TopType, Cname, Type, Element, DynamicEnc, Ext), @@ -993,7 +991,7 @@ gen_enc_components_call1(Gen, TopType, [C|Rest], DynamicEnc, Ext) -> end, Imm = case Imm2 of [] -> []; - _ -> Imm0 ++ Imm2 + _ -> [ImmComment|Imm0 ++ Imm2] end, [Imm|gen_enc_components_call1(Gen, TopType, Rest, DynamicEnc, Ext)]; gen_enc_components_call1(_Gen, _TopType, [], _, _) -> @@ -1328,27 +1326,17 @@ gen_dec_comp_calls([], _, _, _, _, _, _, Tpos, Acc) -> gen_dec_comp_call(Comp, Gen, TopType, Tpos, OptTable, DecInfObj, Ext, NumberOfOptionals) -> - #'ComponentType'{typespec=Type,prop=Prop,textual_order=TextPos} = Comp, + #'ComponentType'{name=Cname,typespec=Type, + prop=Prop,textual_order=TextPos} = Comp, Pos = case Ext of noext -> Tpos; {ext,Epos,_Enum} -> Tpos - Epos + 1 end, - InnerType = - case Type#type.def of - #'ObjectClassFieldType'{type=InType} -> - InType; - Def -> - asn1ct_gen:get_inner(Def) - end, + InnerType = asn1ct_gen:get_inner(Type#type.def), - DispType = case InnerType of - #'Externaltypereference'{type=T} -> T; - IT when is_tuple(IT) -> element(2,IT); - _ -> InnerType - end, + CommentString = attribute_comment(InnerType, TextPos, Cname), Comment = fun(St) -> - emit([nl,"%% attribute number ",TextPos, - " with type ",DispType,nl]), + emit([nl,"%% ",CommentString,nl]), St end, @@ -1500,9 +1488,9 @@ gen_dec_component_no_val(_, Type, {'DEFAULT',DefVal0}) -> DefVal = asn1ct_gen:conform_value(Type, DefVal0), emit([{asis,DefVal}]); gen_dec_component_no_val(_, _, 'OPTIONAL') -> - emit({"asn1_NOVALUE"}); + emit(["asn1_NOVALUE"]); gen_dec_component_no_val({ext,_,_}, _, mandatory) -> - emit({"asn1_NOVALUE"}). + emit(["asn1_NOVALUE"]). dec_map_extaddgroup_no_val(Ext, Type, Comp) -> L0 = [dec_map_extaddgroup_no_val_1(N, P, Ext, Type) || @@ -1699,16 +1687,15 @@ gen_dec_line_other(Erule, Atype, TopType, Comp) -> end; {constructed,bif} -> NewTypename = [Cname|TopType], + DecFunc = dec_func(asn1ct_gen:list2name(NewTypename)), case Type#type.tablecinf of [{objfun,_}|_R] -> fun(BytesVar) -> - emit({"'dec_",asn1ct_gen:list2name(NewTypename), - "'(",BytesVar,", ObjFun)"}) + emit([{asis,DecFunc},"(",BytesVar,", ObjFun)"]) end; _ -> fun(BytesVar) -> - emit({"'dec_",asn1ct_gen:list2name(NewTypename), - "'(",BytesVar,")"}) + emit([{asis,DecFunc},"(",BytesVar,")"]) end end end. @@ -1914,7 +1901,7 @@ emit_extaddgroupTerms(VarSeries,[_]) -> ok; emit_extaddgroupTerms(VarSeries,[_|Rest]) -> asn1ct_name:new(VarSeries), - emit({{curr,VarSeries},","}), + emit([{curr,VarSeries},","]), emit_extaddgroupTerms(VarSeries,Rest); emit_extaddgroupTerms(_,[]) -> ok. @@ -1987,3 +1974,15 @@ enc_dig_out_value(#gen{pack=map}=Gen, [{_,Name}|T], Value) -> make_var(Base) -> {var,atom_to_list(asn1ct_gen:mk_var(asn1ct_name:curr(Base)))}. + +attribute_comment(InnerType, TextPos, Cname) -> + DispType = case InnerType of + #'Externaltypereference'{type=T} -> T; + IT when is_tuple(IT) -> element(2,IT); + _ -> InnerType + end, + Comment = ["attribute ",Cname,"(",TextPos,") with type ",DispType], + lists:concat(Comment). + +dec_func(Tname) -> + list_to_atom(lists:concat(["dec_",Tname])). diff --git a/lib/asn1/src/asn1ct_func.erl b/lib/asn1/src/asn1ct_func.erl index 0cd72acf9d..016161fcaf 100644 --- a/lib/asn1/src/asn1ct_func.erl +++ b/lib/asn1/src/asn1ct_func.erl @@ -65,7 +65,7 @@ generate(Fd) -> Funcs = sofs:to_external(Funcs0), ok = file:write(Fd, Funcs). -is_used({_,_,_}=MFA) -> +is_used({M,F,A}=MFA) when is_atom(M), is_atom(F), is_integer(A) -> req({is_used,MFA}). diff --git a/lib/asn1/src/asn1ct_gen.erl b/lib/asn1/src/asn1ct_gen.erl index 9943bd056a..fa312ed052 100644 --- a/lib/asn1/src/asn1ct_gen.erl +++ b/lib/asn1/src/asn1ct_gen.erl @@ -22,8 +22,7 @@ -include("asn1_records.hrl"). --export([demit/1, - emit/1, +-export([emit/1, open_output_file/1,close_output_file/0, get_inner/1,type/1,def_to_tag/1,prim_bif/1, list2name/1, @@ -191,13 +190,9 @@ pgen_partial_decode(_, _, _) -> ok. pgen_partial_inc_dec(Rtmod,Erules,Module) -> -% io:format("Start partial incomplete decode gen?~n"), case asn1ct:get_gen_state_field(inc_type_pattern) of undefined -> -% io:format("Partial incomplete decode gen not started: ~w~n",[asn1ct:get_gen_state_field(active)]), ok; -% [] -> -% ok; ConfList -> PatternLists=lists:map(fun({_,P}) -> P end,ConfList), pgen_partial_inc_dec1(Rtmod,Erules,Module,PatternLists), @@ -215,11 +210,9 @@ pgen_partial_inc_dec1(Rtmod,Erules,Module,[P|Ps]) -> asn1ct:update_gen_state(prefix,"dec-inc-"), case asn1ct:maybe_saved_sindex(TopTypeName,P) of I when is_integer(I),I > 0 -> -% io:format("Index:~p~n",[I]), asn1ct:set_current_sindex(I); _I -> asn1ct:set_current_sindex(0), -% io:format("Index=~p~n",[_I]), ok end, Rtmod:gen_decode(Erules,TypeDef), @@ -250,8 +243,8 @@ gen_partial_inc_dec_refed_funcs(Rtmod, #gen{erule=ber}=Gen) -> pgen_partial_dec(_Rtmod,Erules,_Module) -> Type_pattern = asn1ct:get_gen_state_field(type_pattern), -% io:format("Type_pattern: ~w~n",[Type_pattern]), - %% Get the typedef of the top type and follow into the choosen components until the last type/component. + %% Get the typedef of the top type and follow into the choosen + %% components until the last type/component. pgen_partial_types(Erules,Type_pattern), ok. @@ -266,7 +259,6 @@ pgen_partial_types(#gen{options=Options}=Gen, TypePattern) -> pgen_partial_types1(Erules,[{FuncName,[TopType|RestTypes]}|Rest]) -> -% emit([FuncName,"(Bytes) ->",nl]), CurrMod = get(currmod), TypeDef = asn1_db:dbget(CurrMod,TopType), traverse_type_structure(Erules,TypeDef,RestTypes,FuncName, @@ -291,8 +283,9 @@ traverse_type_structure(Erules,Type,[],FuncName,TopTypeName) -> end, Ctmod:gen_decode_selected(Erules,TypeDef,FuncName); % what if Type is #type{} traverse_type_structure(Erules,#type{def=Def},[[N]],FuncName,TopTypeName) - when is_integer(N) -> % this case a decode of one of the elements in - % the SEQUENCE OF is required. + when is_integer(N) -> + %% In this case a decode of one of the elements in the SEQUENCE OF is + %% required. InnerType = asn1ct_gen:get_inner(Def), case InnerType of 'SEQUENCE OF' -> @@ -368,8 +361,9 @@ traverse_type_structure(Erules,#typedef{typespec=Def},[T|Ts],FuncName, TypeDef = asn1_db:dbget(M,TName), traverse_type_structure(Erules,TypeDef,[T|Ts],FuncName, [TypeDef#typedef.name]); - _ -> %this may be a referenced type that shall be traversed or - %the selected type + _ -> + %% This may be a referenced type that shall be traversed + %% or the selected type traverse_type_structure(Erules,Def,Ts,FuncName,[T|TopTypeName]) end. @@ -384,9 +378,7 @@ get_component(Name,{C1,C2}) when is_list(C1),is_list(C2) -> get_component(Name,[C=#'ComponentType'{name=Name}|_Cs]) -> C; get_component(Name,[_C|Cs]) -> - get_component(Name,Cs); -get_component(Name,_) -> - throw({error,{asn1,{internal_error,Name}}}). + get_component(Name,Cs). %% generate code for all inner types that are called from the top type %% of the partial incomplete decode and are defined within the top @@ -451,7 +443,6 @@ pgen_partial_incomplete_decode1(#gen{erule=ber}) -> lists:foreach(fun emit_partial_incomplete_decode/1,Data) end, GeneratedFs= asn1ct:get_gen_state_field(gen_refed_funcs), -% io:format("GeneratedFs :~n~p~n",[GeneratedFs]), gen_part_decode_funcs(GeneratedFs,0); pgen_partial_incomplete_decode1(#gen{}) -> ok. @@ -604,9 +595,7 @@ gen_encode_constructed(Erules,Typename,InnerType,D) when is_record(D,type) -> Rtmod:gen_encode_sof(Erules,Typename,InnerType,D), {_,Type} = D#type.def, NameSuffix = asn1ct_gen:constructed_suffix(InnerType,Type#type.def), - gen_types(Erules, [NameSuffix|Typename], Type, gen_encode); - _ -> - exit({nyi,InnerType}) + gen_types(Erules, [NameSuffix|Typename], Type, gen_encode) end; gen_encode_constructed(Erules,Typename,InnerType,D) when is_record(D,typedef) -> @@ -802,11 +791,12 @@ result_line_1(Items) -> try_catch() -> [" catch",nl, " Class:Exception when Class =:= error; Class =:= exit ->",nl, + " Stk = erlang:get_stacktrace(),",nl, " case Exception of",nl, - " {error,Reason}=Error ->",nl, - " Error;",nl, + " {error,{asn1,Reason}} ->",nl, + " {error,{asn1,{Reason,Stk}}};",nl, " Reason ->",nl, - " {error,{asn1,Reason}}",nl, + " {error,{asn1,{Reason,Stk}}}",nl, " end",nl, "end."]. @@ -878,7 +868,6 @@ gen_partial_inc_dispatcher(#gen{erule=ber}) -> {_,undefined} -> ok; {Data1,Data2} -> -% io:format("partial_incomplete_decode: ~p~ninc_type_pattern: ~p~n",[Data,Data2]), gen_partial_inc_dispatcher(Data1, Data2, "") end; gen_partial_inc_dispatcher(#gen{}) -> @@ -953,71 +942,39 @@ hrl_protector(OutFile) -> end || C <- P]. -%% EMIT functions ************************ -%% *************************************** - - % debug generation -demit(Term) -> - case get(asndebug) of - true -> emit(Term); - _ ->true - end. - - % always generation emit(Term) -> ok = file:write(get(gen_file_out), do_emit(Term)). -do_emit({external,_M,T}) -> - do_emit(T); - do_emit({prev,Variable}) when is_atom(Variable) -> do_emit({var,asn1ct_name:prev(Variable)}); - do_emit({next,Variable}) when is_atom(Variable) -> do_emit({var,asn1ct_name:next(Variable)}); - do_emit({curr,Variable}) when is_atom(Variable) -> do_emit({var,asn1ct_name:curr(Variable)}); - do_emit({var,Variable}) when is_atom(Variable) -> [Head|V] = atom_to_list(Variable), [Head-32|V]; - -do_emit({var,Variable}) -> - [Head|V] = Variable, - [Head-32|V]; - do_emit({asis,What}) -> io_lib:format("~w", [What]); - do_emit({call,M,F,A}) -> MFA = {M,F,length(A)}, asn1ct_func:need(MFA), [atom_to_list(F),"(",call_args(A, "")|")"]; - do_emit(nl) -> "\n"; - do_emit(com) -> ","; - -do_emit(tab) -> - " "; - +do_emit([C|_]=Str) when is_integer(C) -> + Str; +do_emit([_|_]=L) -> + [do_emit(E) || E <- L]; +do_emit([]) -> + []; do_emit(What) when is_integer(What) -> integer_to_list(What); - -do_emit(What) when is_list(What), is_integer(hd(What)) -> - What; - do_emit(What) when is_atom(What) -> - atom_to_list(What); + atom_to_list(What). -do_emit(What) when is_tuple(What) -> - [do_emit(E) || E <- tuple_to_list(What)]; - -do_emit(What) when is_list(What) -> - [do_emit(E) || E <- What]. call_args([A|As], Sep) -> [Sep,do_emit(A)|call_args(As, ", ")]; @@ -1123,8 +1080,6 @@ gen_record(Gen, TorPtype, Name, #type{}=Type, Num) -> case Seq#'SEQUENCE'.pname of false -> {record,Seq#'SEQUENCE'.components}; -%% _Pname when TorPtype == type -> -%% false; _ -> {record,Seq#'SEQUENCE'.components} end; @@ -1137,8 +1092,6 @@ gen_record(Gen, TorPtype, Name, #type{}=Type, Num) -> _ -> {record,to_textual_order(Set#'SET'.components)} end; -% {'SET',{_,_CompList}} -> -% {record,_CompList}; {'CHOICE',_CompList} -> {inner,Def}; {'SEQUENCE OF',_CompList} -> {['SEQOF'|Name],Def}; {'SET OF',_CompList} -> {['SETOF'|Name],Def}; @@ -1344,7 +1297,6 @@ get_inner({fixedtypevaluefield,_,Type}) -> get_inner({typefield,TypeName}) -> TypeName; get_inner(#'ObjectClassFieldType'{type=Type}) -> -% get_inner(Type); Type; get_inner(T) when is_tuple(T) -> case element(1,T) of @@ -1353,9 +1305,7 @@ get_inner(T) when is_tuple(T) -> {valuefieldreference,FieldName} -> get_fieldtype(element(2,Tuple),FieldName); {typefieldreference,FieldName} -> - get_fieldtype(element(2,Tuple),FieldName); - {'EXIT',Reason} -> - throw({asn1,{'internal error in get_inner/1',Reason}}) + get_fieldtype(element(2,Tuple),FieldName) end; _ -> element(1,T) end. diff --git a/lib/asn1/src/asn1ct_gen_ber_bin_v2.erl b/lib/asn1/src/asn1ct_gen_ber_bin_v2.erl index 6c6d4193f3..948566a6fc 100644 --- a/lib/asn1/src/asn1ct_gen_ber_bin_v2.erl +++ b/lib/asn1/src/asn1ct_gen_ber_bin_v2.erl @@ -35,21 +35,21 @@ -export([extaddgroup2sequence/1]). -export([dialyzer_suppressions/1]). --import(asn1ct_gen, [emit/1,demit/1]). +-import(asn1ct_gen, [emit/1]). - % the encoding of class of tag bits 8 and 7 +%% The encoding of class of tag bits 8 and 7 -define(UNIVERSAL, 0). -define(APPLICATION, 16#40). -define(CONTEXT, 16#80). -define(PRIVATE, 16#C0). - % primitive or constructed encoding % bit 6 +%% Primitive or constructed encoding % bit 6 -define(PRIMITIVE, 0). -define(CONSTRUCTED, 2#00100000). -define(T_ObjectDescriptor, ?UNIVERSAL bor ?PRIMITIVE bor 7). - % restricted character string types +%% Restricted character string types -define(T_NumericString, ?UNIVERSAL bor ?PRIMITIVE bor 18). %can be constructed -define(T_PrintableString, ?UNIVERSAL bor ?PRIMITIVE bor 19). %can be constructed -define(T_TeletexString, ?UNIVERSAL bor ?PRIMITIVE bor 20). %can be constructed @@ -107,20 +107,12 @@ gen_encode(Erules,Typename,Type) when is_record(Type,type) -> case asn1ct_gen:type(InnerType) of {constructed,bif} -> - emit([nl,nl,nl,"%%================================"]), - emit([nl,"%% ",asn1ct_gen:list2name(Typename)]), - emit([nl,"%%================================",nl]), - case length(Typename) of - 1 -> % top level type - emit(["'enc_",asn1ct_gen:list2name(Typename), - "'(Val",ObjFun,") ->",nl]), - emit([" 'enc_",asn1ct_gen:list2name(Typename), - "'(Val, ", {asis,lists:reverse(Type#type.tag)},ObjFun,").",nl,nl]); - _ -> % embedded type with constructed name - true - end, - emit(["'enc_",asn1ct_gen:list2name(Typename), - "'(Val, TagIn",ObjFun,") ->",nl," "]), + Func = {asis,enc_func(asn1ct_gen:list2name(Typename))}, + emit([nl,nl,nl,"%%================================",nl, + "%% ",asn1ct_gen:list2name(Typename),nl, + "%%================================",nl, + Func,"(Val, TagIn",ObjFun,") ->",nl, + " "]), asn1ct_gen:gen_encode_constructed(Erules,Typename,InnerType,Type); _ -> true @@ -146,7 +138,7 @@ gen_encode_user(Erules, #typedef{}=D, Wrapper) -> emit([nl,nl,"%%================================"]), emit([nl,"%% ",Typename]), emit([nl,"%%================================",nl]), - FuncName = "'enc_" ++ asn1ct_gen:list2name(Typename) ++ "'", + FuncName = {asis,enc_func(asn1ct_gen:list2name(Typename))}, case Wrapper of true -> %% This is a top-level type. Generate an 'enc_Type'/1 @@ -169,9 +161,10 @@ gen_encode_user(Erules, #typedef{}=D, Wrapper) -> gen_encode_prim(ber,Type,"TagIn","Val"), emit([".",nl]); #'Externaltypereference'{module=CurrentMod,type=Etype} -> - emit([" 'enc_",Etype,"'(Val, TagIn).",nl]); + emit([" ",{asis,enc_func(Etype)},"(Val, TagIn).",nl]); #'Externaltypereference'{module=Emod,type=Etype} -> - emit([" '",Emod,"':'enc_",Etype,"'(Val, TagIn).",nl]); + emit([" ",{asis,Emod},":",{asis,enc_func(Etype)}, + "(Val, TagIn).",nl]); 'ASN1_OPEN_TYPE' -> emit(["%% OPEN TYPE",nl]), gen_encode_prim(ber, @@ -326,40 +319,39 @@ gen_decode(Erules,Type) when is_record(Type,typedef) -> Tag = [(decode_class(X#tag.class) bsl 10) + X#tag.number || X <- InnerTag], - FunctionName = + FuncName0 = case {asn1ct:get_gen_state_field(active), asn1ct:get_gen_state_field(prefix)} of {true,Pref} -> %% prevent duplicated function definitions -% Pattern = asn1ct:get_gen_state_field(namelist), -% FuncName=asn1ct:maybe_rename_function(Type#typedef.name, -% Pattern), case asn1ct:current_sindex() of - I when is_integer(I),I>0 -> - lists:concat([Pref,Type#typedef.name,"_",I]); + I when is_integer(I), I > 0 -> + [Pref,Type#typedef.name,"_",I]; _-> - lists:concat([Pref,Type#typedef.name]) - end; % maybe the current_sindex must be reset - _ -> lists:concat(["dec_",Type#typedef.name]) + [Pref,Type#typedef.name] + end; + {_,_} -> + ["dec_",Type#typedef.name] end, - emit({nl,nl}), - emit(["'",FunctionName,"'(Tlv) ->",nl]), - emit([" '",FunctionName,"'(Tlv, ",{asis,Tag},").",nl,nl]), - emit(["'",FunctionName,"'(Tlv, TagIn) ->",nl]), - dbdec(Type#typedef.name,"Tlv"), + FuncName = {asis,list_to_atom(lists:concat(FuncName0))}, + emit([nl,nl, + FuncName,"(Tlv) ->",nl, + " ",FuncName,"(Tlv, ",{asis,Tag},").",nl,nl, + FuncName,"(Tlv, TagIn) ->",nl]), gen_decode_user(Erules,Type). gen_inc_decode(Erules,Type) when is_record(Type,typedef) -> Prefix = asn1ct:get_gen_state_field(prefix), Suffix = asn1ct_gen:index2suffix(asn1ct:current_sindex()), - emit({nl,nl}), - emit(["'",Prefix,Type#typedef.name,Suffix,"'(Tlv, TagIn) ->",nl]), + FuncName0 = [Prefix,Type#typedef.name,Suffix], + FuncName = {asis,list_to_atom(lists:concat(FuncName0))}, + emit([nl,nl, + FuncName,"(Tlv, TagIn) ->",nl]), gen_decode_user(Erules,Type). %% gen_decode_selected exported function for selected decode gen_decode_selected(Erules,Type,FuncName) -> emit([FuncName,"(Bin) ->",nl]), -% Pattern = asn1ct:get_gen_state_field(tag_pattern), Patterns = asn1ct:read_config_data(partial_decode), Pattern = case lists:keysearch(FuncName,1,Patterns) of @@ -398,12 +390,10 @@ gen_decode_selected_type(_Erules,TypeDef) -> asn1ct_gen:list2name(TopType),"'"]), emit([DecFunName,"(",BytesVar, ", ",{asis,Tag},")"]); -% emit([";",nl]); TheType -> DecFunName = mkfuncname(TheType,dec), emit([DecFunName,"(",BytesVar, ", ",{asis,Tag},")"]) -% emit([";",nl]) end. %%=============================================================================== @@ -418,7 +408,6 @@ gen_decode(Erules,Typename,Type) when is_record(Type,type) -> FunctionName = case asn1ct:get_gen_state_field(active) of true -> -% Suffix = asn1ct_gen:index2suffix(SIndex), Pattern = asn1ct:get_gen_state_field(namelist), Suffix = case asn1ct:maybe_saved_sindex(Typename,Pattern) of @@ -431,8 +420,6 @@ gen_decode(Erules,Typename,Type) when is_record(Type,type) -> _ -> lists:concat(["'dec_",asn1ct_gen:list2name(Typename)]) end, -% io:format("Typename: ~p,~n",[Typename]), -% io:format("FunctionName: ~p~n",[FunctionName]), case asn1ct_gen:type(InnerType) of {constructed,bif} -> ObjFun = @@ -442,9 +429,7 @@ gen_decode(Erules,Typename,Type) when is_record(Type,type) -> _ -> "" end, -% emit([Prefix,asn1ct_gen:list2name(Typename),"'(Tlv, TagIn",ObjFun,") ->",nl]), emit([FunctionName,"'(Tlv, TagIn",ObjFun,") ->",nl]), - dbdec(Typename,"Tlv"), asn1ct_gen:gen_decode_constructed(Erules,Typename,InnerType,Type); Rec when is_record(Rec,'Externaltypereference') -> case {Typename,asn1ct:get_gen_state_field(namelist)} of @@ -476,10 +461,10 @@ gen_decode(Erules,Typename,Type) when is_record(Type,type) -> gen_decode(Erules,Tname,#'ComponentType'{name=Cname,typespec=Type}) -> NewTname = [Cname|Tname], - %% The tag is set to [] to avoid that it is - %% taken into account twice, both as a component/alternative (passed as - %% argument to the encode decode function and within the encode decode - %% function it self. + %% The tag is set to [] to avoid that it is taken into account + %% twice, both as a component/alternative (passed as argument to + %% the encode/decode function), and within the encode decode + %% function itself. NewType = Type#type{tag=[]}, case {asn1ct:get_gen_state_field(active), asn1ct:get_tobe_refed_func(NewTname)} of @@ -504,7 +489,7 @@ gen_decode_user(Erules,D) when is_record(D,typedef) -> asn1ct_name:new(len), gen_dec_prim(Def#type{def='ASN1_OPEN_TYPE'}, BytesVar, {string,"TagIn"}), - emit({".",nl,nl}); + emit([".",nl,nl]); {primitive,bif} -> asn1ct_name:new(len), gen_dec_prim(Def, BytesVar, {string,"TagIn"}), @@ -515,8 +500,7 @@ gen_decode_user(Erules,D) when is_record(D,typedef) -> TheType -> DecFunName = mkfuncname(TheType,dec), emit([DecFunName,"(",BytesVar, - ", TagIn)"]), - emit([".",nl,nl]) + ", TagIn).",nl,nl]) end. @@ -746,9 +730,10 @@ gen_obj_code(Erules,_Module,Obj) when is_record(Obj,typedef) -> #'Externaltypereference'{module=M,type=ClName} = Def#'Object'.classname, Class = asn1_db:dbget(M,ClName), {object,_,Fields} = Def#'Object'.def, - emit({nl,nl,nl,"%%================================"}), - emit({nl,"%% ",ObjName}), - emit({nl,"%%================================",nl}), + emit([nl,nl,nl, + "%%================================",nl, + "%% ",ObjName,nl, + "%%================================",nl]), EncConstructed = gen_encode_objectfields(ClName,get_class_fields(Class), ObjName,Fields,[]), @@ -766,11 +751,9 @@ gen_encode_objectfields(ClassName,[{typefield,Name,OptOrMand}|Rest], ObjName,ObjectFields,ConstrAcc) -> EmitFuncClause = fun(Arg) -> - emit(["'enc_",ObjName,"'(",{asis,Name}, - ", ",Arg,", _RestPrimFieldName) ->",nl]) + emit([{asis,enc_func(ObjName)},"(",{asis,Name}, + ", ",Arg,", _RestPrimFieldName) ->",nl]) end, -% emit(["'enc_",ObjName,"'(",{asis,Name}, -% ", Val, RestPrimFieldName) ->",nl]), MaybeConstr= case {get_object_field(Name,ObjectFields),OptOrMand} of {false,'OPTIONAL'} -> @@ -799,11 +782,9 @@ gen_encode_objectfields(ClassName,[{objectfield,Name,_,_,OptOrMand}|Rest], CurrentMod = get(currmod), EmitFuncClause = fun(Args) -> - emit(["'enc_",ObjName,"'(",{asis,Name}, + emit([{asis,enc_func(ObjName)},"(",{asis,Name}, ", ",Args,") ->",nl]) end, -% emit(["'enc_",ObjName,"'(",{asis,Name}, -% ", Val,[H|T]) ->",nl]), case {get_object_field(Name,ObjectFields),OptOrMand} of {false,'OPTIONAL'} -> EmitFuncClause("_,_"), @@ -814,19 +795,14 @@ gen_encode_objectfields(ClassName,[{objectfield,Name,_,_,OptOrMand}|Rest], {{Name,#'Externalvaluereference'{module=CurrentMod, value=TypeName}},_} -> EmitFuncClause(" Val, [H|T]"), - emit({indent(3),"'enc_",TypeName,"'(H, Val, T)"}); + emit([indent(3),{asis,enc_func(TypeName)},"(H, Val, T)"]); {{Name,#'Externalvaluereference'{module=M,value=TypeName}},_} -> EmitFuncClause(" Val, [H|T]"), - emit({indent(3),"'",M,"':'enc_",TypeName,"'(H, Val, T)"}); - {{Name,TypeSpec},_} -> + emit([indent(3),{asis,M},":",{asis,enc_func(TypeName)}, + "(H, Val, T)"]); + {{Name,#typedef{name=TypeName}},_} when is_atom(TypeName) -> EmitFuncClause(" Val, [H|T]"), - case TypeSpec#typedef.name of - {ExtMod,TypeName} -> - emit({indent(3),"'",ExtMod,"':'enc_",TypeName, - "'(H, Val, T)"}); - TypeName -> - emit({indent(3),"'enc_",TypeName,"'(H, Val, T)"}) - end + emit([indent(3),{asis,enc_func(TypeName)},"(H, Val, T)"]) end, case more_genfields(Rest) of true -> @@ -862,10 +838,11 @@ gen_encode_field_call(_ObjName,_FieldName, X <- OTag], if M == CurrentMod -> - emit({" 'enc_",T,"'(Val, ",{asis,Tag},")"}), + emit([" ",{asis,enc_func(T)},"(Val, ",{asis,Tag},")"]), []; true -> - emit({" '",M,"':'enc_",T,"'(Val, ",{asis,Tag},")"}), + emit([" ",{asis,M},":",{asis,enc_func(T)}, + "(Val, ",{asis,Tag},")"]), [] end; gen_encode_field_call(ObjName,FieldName,Type) -> @@ -875,24 +852,21 @@ gen_encode_field_call(ObjName,FieldName,Type) -> X#tag.form,X#tag.number)|| X <- OTag], case Type#typedef.name of - {primitive,bif} -> %%tag should be the primitive tag -% OTag = Def#type.tag, -% Tag = [encode_tag_val(decode_class(X#tag.class), -% X#tag.form,X#tag.number)|| -% X <- OTag], + {primitive,bif} -> %tag should be the primitive tag gen_encode_prim(ber,Def,{asis,lists:reverse(Tag)}, "Val"), []; {constructed,bif} -> - emit({" 'enc_",ObjName,'_',FieldName, - "'(Val,",{asis,Tag},")"}), - [Type#typedef{name=list_to_atom(lists:concat([ObjName,'_',FieldName]))}]; + Name = lists:concat([ObjName,'_',FieldName]), + emit([" ",{asis,enc_func(Name)},"(Val,",{asis,Tag},")"]), + [Type#typedef{name=list_to_atom(Name)}]; {ExtMod,TypeName} -> - emit({" '",ExtMod,"':'enc_",TypeName, - "'(Val,",{asis,Tag},")"}), + emit([" ",{asis,ExtMod},":",{asis,enc_func(TypeName)}, + "(Val,",{asis,Tag},")"]), []; TypeName -> - emit({" 'enc_",TypeName,"'(Val,",{asis,Tag},")"}), + emit([" ",{asis,enc_func(TypeName)}, + "(Val,",{asis,Tag},")"]), [] end. @@ -903,10 +877,10 @@ gen_encode_default_call(ClassName,FieldName,Type) -> Tag = [encode_tag_val(decode_class(X#tag.class),X#tag.form,X#tag.number)|| X <- OTag], case asn1ct_gen:type(InnerType) of {constructed,bif} -> - emit([" 'enc_",ClassName,'_',FieldName,"'", + Name = lists:concat([ClassName,'_',FieldName]), + emit([" ",{asis,enc_func(Name)}, "(Val, ",{asis,Tag},")"]), - [#typedef{name=list_to_atom(lists:concat([ClassName,'_',FieldName])), - typespec=Type}]; + [#typedef{name=list_to_atom(Name),typespec=Type}]; {primitive,bif} -> gen_encode_prim(ber,Type,{asis,lists:reverse(Tag)},"Val"), []; @@ -916,12 +890,6 @@ gen_encode_default_call(ClassName,FieldName,Type) -> #'Externaltypereference'{module=Emod,type=Etype} -> emit([" '",Emod,"':'enc_",Etype,"'(Val, ",{asis,Tag},")",nl]), [] -% 'ASN1_OPEN_TYPE' -> -% emit(["%% OPEN TYPE",nl]), -% gen_encode_prim(ber, -% Type#type{def='ASN1_OPEN_TYPE'}, -% "TagIn","Val"), -% emit([".",nl]) end. %%%%%%%%%%%%%%%% @@ -930,11 +898,9 @@ gen_decode_objectfields(ClassName,[{typefield,Name,OptOrMand}|Rest], ObjName,ObjectFields,ConstrAcc) -> EmitFuncClause = fun(Arg) -> - emit(["'dec_",ObjName,"'(",{asis,Name}, + emit([{asis,dec_func(ObjName)},"(",{asis,Name}, ", ",Arg,",_) ->",nl]) end, -% emit(["'dec_",ObjName,"'(",{asis,Name}, -% ", Bytes, RestPrimFieldName) ->",nl]), MaybeConstr= case {get_object_field(Name,ObjectFields),OptOrMand} of {false,'OPTIONAL'} -> @@ -964,12 +930,9 @@ gen_decode_objectfields(ClassName,[{objectfield,Name,_,_,OptOrMand}|Rest], CurrentMod = get(currmod), EmitFuncClause = fun(Args) -> - emit(["'dec_",ObjName,"'(",{asis,Name}, + emit([{asis,dec_func(ObjName)},"(",{asis,Name}, ", ",Args,") ->",nl]) end, -% emit(["'dec_",ObjName,"'(",{asis,Name}, -% ", Bytes,[H|T]) ->",nl]), -% emit_tlv_format("Bytes"), case {get_object_field(Name,ObjectFields),OptOrMand} of {false,'OPTIONAL'} -> EmitFuncClause("_,_"), @@ -980,21 +943,14 @@ gen_decode_objectfields(ClassName,[{objectfield,Name,_,_,OptOrMand}|Rest], {{Name,#'Externalvaluereference'{module=CurrentMod, value=TypeName}},_} -> EmitFuncClause("Bytes,[H|T]"), - emit({indent(3),"'dec_",TypeName,"'(H, Bytes, T)"}); + emit([indent(3),{asis,dec_func(TypeName)},"(H, Bytes, T)"]); {{Name,#'Externalvaluereference'{module=M,value=TypeName}},_} -> EmitFuncClause("Bytes,[H|T]"), - emit({indent(3),"'",M,"':'dec_",TypeName, - "'(H, Bytes, T)"}); - {{Name,TypeSpec},_} -> - EmitFuncClause("Bytes,[H|T]"), -% emit_tlv_format("Bytes"), - case TypeSpec#typedef.name of - {ExtMod,TypeName} -> - emit({indent(3),"'",ExtMod,"':'dec_",TypeName, - "'(H, Bytes, T)"}); - TypeName -> - emit({indent(3),"'dec_",TypeName,"'(H, Bytes, T)"}) - end + emit([indent(3),{asis,M},":",{asis,dec_func(TypeName)}, + "(H, Bytes, T)"]); + {{Name,#typedef{name=TypeName}},_} when is_atom(TypeName) -> + EmitFuncClause("Bytes,[H|T]"), + emit([indent(3),{asis,dec_func(TypeName)},"(H, Bytes, T)"]) end, case more_genfields(Rest) of true -> @@ -1014,24 +970,20 @@ emit_tlv_format(Bytes) -> notice_tlv_format_gen() -> Module = get(currmod), -% io:format("Noticed: ~p~n",[Module]), case get(tlv_format) of {done,Module} -> ok; - _ -> % true or undefined + _ -> % true or undefined put(tlv_format,true) end. emit_tlv_format_function() -> Module = get(currmod), -% io:format("Tlv formated: ~p",[Module]), case get(tlv_format) of true -> -% io:format(" YES!~n"), emit_tlv_format_function1(), put(tlv_format,{done,Module}); _ -> -% io:format(" NO!~n"), ok end. emit_tlv_format_function1() -> @@ -1066,12 +1018,12 @@ gen_decode_field_call(_ObjName,_FieldName,Bytes, X <- OTag], if M == CurrentMod -> - emit({" 'dec_",T,"'(",Bytes, - ", ",{asis,Tag},")"}), + emit([" ",{asis,dec_func(T)},"(",Bytes, + ", ",{asis,Tag},")"]), []; true -> - emit({" '",M,"':'dec_",T, - "'(",Bytes,", ",{asis,Tag},")"}), + emit([" ",{asis,M},":",{asis,dec_func(T)}, + "(",Bytes,", ",{asis,Tag},")"]), [] end; gen_decode_field_call(ObjName,FieldName,Bytes,Type) -> @@ -1084,15 +1036,17 @@ gen_decode_field_call(ObjName,FieldName,Bytes,Type) -> gen_dec_prim(Def, Bytes, Tag), []; {constructed,bif} -> - emit({" 'dec_",ObjName,'_',FieldName, - "'(",Bytes,",",{asis,Tag},")"}), - [Type#typedef{name=list_to_atom(lists:concat([ObjName,'_',FieldName]))}]; + Name = lists:concat([ObjName,"_",FieldName]), + emit([" ",{asis,dec_func(Name)}, + "(",Bytes,",",{asis,Tag},")"]), + [Type#typedef{name=list_to_atom(Name)}]; {ExtMod,TypeName} -> - emit({" '",ExtMod,"':'dec_",TypeName, - "'(",Bytes,",",{asis,Tag},")"}), + emit([" ",{asis,ExtMod},":",{asis,dec_func(TypeName)}, + "(",Bytes,",",{asis,Tag},")"]), []; TypeName -> - emit({" 'dec_",TypeName,"'(",Bytes,",",{asis,Tag},")"}), + emit([" ",{asis,dec_func(TypeName)}, + "(",Bytes,",",{asis,Tag},")"]), [] end. @@ -1118,12 +1072,6 @@ gen_decode_default_call(ClassName,FieldName,Bytes,Type) -> emit([" '",Emod,"':'dec_",Etype,"'(",Bytes,", ", {asis,Tag},")",nl]), [] -% 'ASN1_OPEN_TYPE' -> -% emit(["%% OPEN TYPE",nl]), -% gen_encode_prim(ber, -% Type#type{def='ASN1_OPEN_TYPE'}, -% "TagIn","Val"), -% emit([".",nl]) end. %%%%%%%%%%% @@ -1162,15 +1110,15 @@ more_genfields([Field|Fields]) -> gen_objectset_code(Erules,ObjSet) -> ObjSetName = ObjSet#typedef.name, Def = ObjSet#typedef.typespec, -% {ClassName,ClassDef} = Def#'ObjectSet'.class, #'Externaltypereference'{module=ClassModule, type=ClassName} = Def#'ObjectSet'.class, ClassDef = asn1_db:dbget(ClassModule,ClassName), UniqueFName = Def#'ObjectSet'.uniquefname, Set = Def#'ObjectSet'.set, - emit({nl,nl,nl,"%%================================"}), - emit({nl,"%% ",ObjSetName}), - emit({nl,"%%================================",nl}), + emit([nl,nl,nl, + "%%================================",nl, + "%% ",ObjSetName,nl, + "%%================================",nl]), case ClassName of {_Module,ExtClassName} -> gen_objset_code(Erules,ObjSetName,UniqueFName,Set,ExtClassName,ClassDef); @@ -1200,19 +1148,20 @@ gen_objset_enc(Erules, ObjSetName, UniqueName, {no_mod,no_name} -> gen_inlined_enc_funs(Fields, ClFields, ObjSetName, Val, NthObj); {CurrMod,Name} -> - emit(["'getenc_",ObjSetName,"'(Id) when Id =:= ", - {asis,Val}," ->",nl, - " fun 'enc_",Name,"'/3;",nl]), + emit([asis_atom(["getenc_",ObjSetName]), + "(Id) when Id =:= ",{asis,Val}," ->",nl, + " fun ",asis_atom(["enc_",Name]),"/3;",nl]), {[],NthObj}; {ModuleName,Name} -> - emit(["'getenc_",ObjSetName,"'(Id) when Id =:= ", - {asis,Val}," ->",nl]), + emit([asis_atom(["getenc_",ObjSetName]), + "(Id) when Id =:= ",{asis,Val}," ->",nl]), emit_ext_fun(enc,ModuleName,Name), emit([";",nl]), {[],NthObj}; _ -> - emit(["'getenc_",ObjSetName,"'(",{asis,Val},") ->",nl, - " fun 'enc_",ObjName,"'/3;",nl]), + emit([asis_atom(["getenc_",ObjSetName]), + "(",{asis,Val},") ->",nl, + " fun ",asis_atom(["enc_",ObjName]),"/3;",nl]), {[],NthObj} end, gen_objset_enc(Erules, ObjSetName, UniqueName, T, ClName, ClFields, @@ -1220,7 +1169,7 @@ gen_objset_enc(Erules, ObjSetName, UniqueName, %% See X.681 Annex E for the following case gen_objset_enc(_,ObjSetName,_UniqueName,['EXTENSIONMARK'],_ClName, _ClFields,_NthObj,Acc) -> - emit(["'getenc_",ObjSetName,"'(_) ->",nl, + emit([asis_atom(["getenc_",ObjSetName]),"(_) ->",nl, indent(2),"fun(_, Val, _RestPrimFieldName) ->",nl]), emit_enc_open_type(4), emit([nl, @@ -1228,7 +1177,7 @@ gen_objset_enc(_,ObjSetName,_UniqueName,['EXTENSIONMARK'],_ClName, Acc; gen_objset_enc(_, ObjSetName, UniqueName, [], _, _, _, Acc) -> emit_default_getenc(ObjSetName, UniqueName), - emit({".",nl,nl}), + emit([".",nl,nl]), Acc. emit_ext_fun(EncDec,ModuleName,Name) -> @@ -1236,14 +1185,15 @@ emit_ext_fun(EncDec,ModuleName,Name) -> Name,"'(T,V,O) end"]). emit_default_getenc(ObjSetName,UniqueName) -> - emit(["'getenc_",ObjSetName,"'(ErrV) ->",nl]), - emit([indent(3),"fun(C,V,_) -> exit({'Type not compatible with table constraint',{component,C},{value,V}, {unique_name_and_value,",{asis,UniqueName},", ErrV}}) end"]). + emit([asis_atom(["getenc_",ObjSetName]),"(ErrV) ->",nl, + indent(3),"fun(C,V,_) ->",nl, + "exit({'Type not compatible with table constraint',{component,C},{value,V}, {unique_name_and_value,",{asis,UniqueName},", ErrV}}) end"]). %% gen_inlined_enc_funs for each object iterates over all fields of a %% class, and for each typefield it checks if the object has that %% field and emits the proper code. gen_inlined_enc_funs(Fields, [{typefield,_,_}|_]=T, ObjSetName, Val, NthObj) -> - emit(["'getenc_",ObjSetName,"'(",{asis,Val},") ->",nl, + emit([asis_atom(["getenc_",ObjSetName]),"(",{asis,Val},") ->",nl, indent(3),"fun(Type, Val, _RestPrimFieldName) ->",nl, indent(6),"case Type of",nl]), gen_inlined_enc_funs1(Fields, T, ObjSetName, [], NthObj, []); @@ -1283,8 +1233,8 @@ gen_inlined_enc_funs1(Fields, [{typefield,Name,_}|Rest], ObjSetName, end, {Acc0,0}; false -> - %% This field was not present in the object thus there - %% were no type in the table and we therefore generate + %% This field was not present in the object; thus, there + %% was no type in the table and we therefore generate %% code that returns the input for application %% treatment. emit([indent(9),{asis,Name}," ->",nl]), @@ -1322,7 +1272,6 @@ emit_inner_of_fun(TDef=#typedef{name={ExtMod,Name},typespec=Type}, InternalDefFunName) -> OTag = Type#type.tag, Tag = [encode_tag_val(decode_class(X#tag.class),X#tag.form,X#tag.number)|| X <- OTag], -% remove Tag = [X#tag{class=decode_class(X#tag.class)}|| X <- OTag], case {ExtMod,Name} of {primitive,bif} -> emit(indent(12)), @@ -1333,20 +1282,14 @@ emit_inner_of_fun(TDef=#typedef{name={ExtMod,Name},typespec=Type}, InternalDefFunName,"'(Val, ",{asis,Tag},")"]), {[TDef#typedef{name=InternalDefFunName}],1}; _ -> - emit({indent(12),"'",ExtMod,"':'enc_",Name,"'(Val",{asis,Tag},")"}), + emit([indent(12),"'",ExtMod,"':'enc_",Name,"'(Val",{asis,Tag},")"]), {[],0} end; emit_inner_of_fun(#typedef{name=Name},_) -> -% OTag = Type#type.tag, -% remove Tag = [X#tag{class=decode_class(X#tag.class)}|| X <- OTag], -% Tag = [encode_tag_val(decode_class(X#tag.class),X#tag.form,X#tag.number)|| X <- OTag], - emit({indent(12),"'enc_",Name,"'(Val)"}), + emit([indent(12),"'enc_",Name,"'(Val)"]), {[],0}; emit_inner_of_fun(Type,_) when is_record(Type,type) -> CurrMod = get(currmod), -% OTag = Type#type.tag, -% remove Tag = [X#tag{class=decode_class(X#tag.class)}|| X <- OTag], -% Tag = [encode_tag_val(decode_class(X#tag.class),X#tag.form,X#tag.number)|| X <- OTag], case Type#type.def of Def when is_atom(Def) -> OTag = Type#type.tag, @@ -1384,18 +1327,19 @@ gen_objset_dec(Erules, ObjSName, UniqueName, [{ObjName,Val,Fields}|T], {no_mod,no_name} -> gen_inlined_dec_funs(Fields,ClFields,ObjSName,Val,NthObj); {CurrMod,Name} -> - emit(["'getdec_",ObjSName,"'(Id) when Id =:= ", - {asis,Val}," ->",nl, + emit([asis_atom(["getdec_",ObjSName]), + "(Id) when Id =:= ",{asis,Val}," ->",nl, " fun 'dec_",Name,"'/3;", nl]), NthObj; {ModuleName,Name} -> - emit(["'getdec_",ObjSName,"'(Id) when Id =:= ", - {asis,Val}," ->",nl]), + emit([asis_atom(["getdec_",ObjSName]), + "(Id) when Id =:= ",{asis,Val}," ->",nl]), emit_ext_fun(dec,ModuleName,Name), emit([";",nl]), NthObj; _ -> - emit(["'getdec_",ObjSName,"'(",{asis,Val},") ->",nl, + emit([asis_atom(["getdec_",ObjSName]), + "(",{asis,Val},") ->",nl, " fun 'dec_",ObjName,"'/3;", nl]), NthObj end, @@ -1403,8 +1347,8 @@ gen_objset_dec(Erules, ObjSName, UniqueName, [{ObjName,Val,Fields}|T], ClFields, NewNthObj); gen_objset_dec(_,ObjSetName,_UniqueName,['EXTENSIONMARK'],_ClName, _ClFields,_NthObj) -> - emit(["'getdec_",ObjSetName,"'(_) ->",nl]), - emit([indent(2),"fun(_,Bytes, _RestPrimFieldName) ->",nl]), + emit([asis_atom(["getdec_",ObjSetName]),"(_) ->",nl, + indent(2),"fun(_,Bytes, _RestPrimFieldName) ->",nl]), emit_dec_open_type(4), emit([nl, indent(2),"end.",nl,nl]), @@ -1495,7 +1439,6 @@ emit_dec_open_type(I) -> emit_inner_of_decfun(#typedef{name={ExtName,Name},typespec=Type}, _Prop, InternalDefFunName) -> OTag = Type#type.tag, -%% Tag = [X#tag{class=decode_class(X#tag.class)}|| X <- OTag], Tag = [(decode_class(X#tag.class) bsl 10) + X#tag.number || X <- OTag], case {ExtName,Name} of {primitive,bif} -> @@ -1504,8 +1447,6 @@ emit_inner_of_decfun(#typedef{name={ExtName,Name},typespec=Type}, _Prop, 0; {constructed,bif} -> emit([indent(12),"'dec_", -% asn1ct_gen:list2name(InternalDefFunName),"'(Bytes, ",Prop, -% ", ",{asis,Tag},")"]), asn1ct_gen:list2name(InternalDefFunName),"'(Bytes, ", {asis,Tag},")"]), 1; @@ -1519,7 +1460,6 @@ emit_inner_of_decfun(#typedef{name=Name},_Prop,_) -> 0; emit_inner_of_decfun(#type{}=Type, _Prop, _) -> OTag = Type#type.tag, -%% Tag = [X#tag{class=decode_class(X#tag.class)}|| X <- OTag], Tag = [(decode_class(X#tag.class) bsl 10) + X#tag.number || X <- OTag], CurrMod = get(currmod), Def = Type#type.def, @@ -1531,11 +1471,9 @@ emit_inner_of_decfun(#type{}=Type, _Prop, _) -> gen_dec_prim(Type, "Bytes", Tag); #'Externaltypereference'{module=CurrMod,type=T} -> emit([indent(9),T," ->",nl,indent(12),"'dec_",T, -% "'(Bytes, ",Prop,")"]); "'(Bytes)"]); #'Externaltypereference'{module=ExtMod,type=T} -> emit([indent(9),T," ->",nl,indent(12),ExtMod,":'dec_", -% T,"'(Bytes, ",Prop,")"]) T,"'(Bytes, ",{asis,Tag},")"]) end, 0. @@ -1550,10 +1488,6 @@ gen_internal_funcs(Erules,[TypeDef|Rest]) -> gen_internal_funcs(Erules,Rest). -dbdec(Type,Arg) -> - demit({"io:format(\"decoding: ",{asis,Type},"~w~n\",[",Arg,"]),",nl}). - - decode_class('UNIVERSAL') -> ?UNIVERSAL; decode_class('APPLICATION') -> @@ -1605,7 +1539,7 @@ encode_tag_val(Class, Form, TagNo) -> %%%%%%%%%%% %% mk_object_val(Value) -> {OctetList, Len} -%% returns a Val as a list of octets, the 8 bit is allways set to one except +%% returns a Val as a list of octets, the 8 bit is always set to one except %% for the last octet, where its 0 %% @@ -1619,8 +1553,9 @@ mk_object_val(0, Ack, Len) -> mk_object_val(Val, Ack, Len) -> mk_object_val(Val bsr 7, [((Val band 127) bor 128) | Ack], Len + 1). -%% For BER the ExtensionAdditionGroup notation has no impact on the encoding/decoding -%% and therefore we only filter away the ExtensionAdditionGroup start and end markers +%% For BER the ExtensionAdditionGroup notation has no impact on the +%% encoding/decoding. Therefore we can filter away the +%% ExtensionAdditionGroup start and end markers. extaddgroup2sequence(ExtList) when is_list(ExtList) -> lists:filter(fun(#'ExtensionAdditionGroup'{}) -> false; @@ -1632,3 +1567,12 @@ extaddgroup2sequence(ExtList) when is_list(ExtList) -> call(F, Args) -> asn1ct_func:call(ber, F, Args). + +enc_func(Tname) -> + list_to_atom(lists:concat(["enc_",Tname])). + +dec_func(Tname) -> + list_to_atom(lists:concat(["dec_",Tname])). + +asis_atom(List) -> + {asis,list_to_atom(lists:concat(List))}. diff --git a/lib/asn1/src/asn1ct_gen_per.erl b/lib/asn1/src/asn1ct_gen_per.erl index 9671a566bf..22719bba74 100644 --- a/lib/asn1/src/asn1ct_gen_per.erl +++ b/lib/asn1/src/asn1ct_gen_per.erl @@ -24,7 +24,6 @@ %% all types in an ASN.1 module -include("asn1_records.hrl"). -%-compile(export_all). -export([gen_dec_imm/2]). -export([gen_dec_prim/3,gen_encode_prim_imm/3]). @@ -35,15 +34,20 @@ -export([extaddgroup2sequence/1]). -export([dialyzer_suppressions/1]). --import(asn1ct_gen, [emit/1,demit/1]). +-import(asn1ct_gen, [emit/1]). -import(asn1ct_func, [call/3]). -%% Generate ENCODING ****************************** -%%****************************************x +%%**************************************** +%% Generate ENCODING +%%**************************************** -dialyzer_suppressions(Erules) -> - case asn1ct_func:is_used({Erules,complete,1}) of +dialyzer_suppressions(#gen{erule=per,aligned=Aligned}) -> + Mod = case Aligned of + false -> uper; + true -> per + end, + case asn1ct_func:is_used({Mod,complete,1}) of false -> ok; true -> @@ -54,14 +58,6 @@ dialyzer_suppressions(Erules) -> gen_encode(Erules,Type) when is_record(Type,typedef) -> gen_encode_user(Erules,Type). -%% case Type#typedef.typespec of -%% Def when is_record(Def,type) -> -%% gen_encode_user(Erules,Type); -%% Def when is_tuple(Def),(element(1,Def) == 'Object') -> -%% gen_encode_object(Erules,Type); -%% Other -> -%% exit({error,{asn1,{unknown,Other}}}) -%% end. gen_encode(Erules,Typename,#'ComponentType'{name=Cname,typespec=Type}) -> NewTypename = [Cname|Typename], @@ -72,15 +68,14 @@ gen_encode(Erules,Typename,Type) when is_record(Type,type) -> ObjFun = case lists:keysearch(objfun,1,Type#type.tablecinf) of {value,{_,_Name}} -> -%% lists:concat([", ObjFun",Name]); ", ObjFun"; false -> "" end, case asn1ct_gen:type(InnerType) of {constructed,bif} -> - emit({"'enc_",asn1ct_gen:list2name(Typename),"'(Val",ObjFun, - ") ->",nl}), + Func = enc_func(asn1ct_gen:list2name(Typename)), + emit([{asis,Func},"(Val",ObjFun,") ->",nl]), asn1ct_gen:gen_encode_constructed(Erules,Typename,InnerType,Type); _ -> true @@ -92,20 +87,21 @@ gen_encode_user(Erules,D) when is_record(D,typedef) -> Typename = [D#typedef.name], Def = D#typedef.typespec, InnerType = asn1ct_gen:get_inner(Def#type.def), - emit({"'enc_",asn1ct_gen:list2name(Typename),"'(Val) ->",nl}), + Func = enc_func(asn1ct_gen:list2name(Typename)), + emit([{asis,Func},"(Val) ->",nl]), case asn1ct_gen:type(InnerType) of {primitive,bif} -> gen_encode_prim(Erules, Def), - emit({".",nl}); + emit([".",nl]); 'ASN1_OPEN_TYPE' -> gen_encode_prim(Erules, Def#type{def='ASN1_OPEN_TYPE'}), - emit({".",nl}); + emit([".",nl]); {constructed,bif} -> asn1ct_gen:gen_encode_constructed(Erules,Typename,InnerType,D); #'Externaltypereference'{module=CurrMod,type=Etype} -> - emit({"'enc_",Etype,"'(Val).",nl,nl}); + emit([{asis,enc_func(Etype)},"(Val).",nl]); #'Externaltypereference'{module=Emod,type=Etype} -> - emit({"'",Emod,"':'enc_",Etype,"'(Val).",nl,nl}) + emit([{asis,Emod},":",enc_func(Etype),"(Val).",nl]) end. @@ -220,7 +216,6 @@ gen_objectset_code(_Erules, _ObjSet) -> gen_decode(Erules, #typedef{}=Type) -> DecFunc = dec_func(Type#typedef.name), emit([nl,nl,{asis,DecFunc},"(Bytes) ->",nl]), - dbdec(Type#typedef.name), gen_decode_user(Erules, Type). gen_decode(Erules,Tname,#'ComponentType'{name=Cname,typespec=Type}) -> @@ -241,17 +236,11 @@ gen_decode(Erules,Typename,Type) when is_record(Type,type) -> emit([nl, {asis,dec_func(asn1ct_gen:list2name(Typename))}, "(Bytes",ObjFun,") ->",nl]), - dbdec(Typename), asn1ct_gen:gen_decode_constructed(Erules,Typename,InnerType,Type); _ -> true end. -dbdec(Type) when is_list(Type)-> - demit({"io:format(\"decoding: ",asn1ct_gen:list2name(Type),"~w~n\",[Bytes]),",nl}); -dbdec(Type) -> - demit({"io:format(\"decoding: ",{asis,Type},"~w~n\",[Bytes]),",nl}). - gen_decode_user(Erules,D) when is_record(D,typedef) -> Typename = [D#typedef.name], Def = D#typedef.typespec, @@ -259,17 +248,15 @@ gen_decode_user(Erules,D) when is_record(D,typedef) -> case asn1ct_gen:type(InnerType) of {primitive,bif} -> gen_dec_prim(Erules,Def,"Bytes"), - emit({".",nl,nl}); + emit([".",nl,nl]); 'ASN1_OPEN_TYPE' -> gen_dec_prim(Erules,Def#type{def='ASN1_OPEN_TYPE'},"Bytes"), - emit({".",nl,nl}); + emit([".",nl,nl]); {constructed,bif} -> asn1ct_gen:gen_decode_constructed(Erules,Typename,InnerType,D); #'Externaltypereference'{}=Etype -> gen_dec_external(Etype, "Bytes"), - emit([".",nl,nl]); - Other -> - exit({error,{asn1,{unknown,Other}}}) + emit([".",nl,nl]) end. gen_dec_external(Ext, BytesVar) -> @@ -398,10 +385,11 @@ gen_dec_prim(Erule, Type, BytesVar) -> asn1ct_imm:dec_code_gen(Imm, BytesVar). -%% For PER the ExtensionAdditionGroup notation has significance for the encoding and decoding -%% the components within the ExtensionAdditionGroup is treated in a similar way as if they -%% have been specified within a SEQUENCE, therefore we construct a fake sequence type here -%% so that we can generate code for it +%% For PER the ExtensionAdditionGroup notation has significance for +%% the encoding and decoding. The components within the +%% ExtensionAdditionGroup is treated in a similar way as if they have +%% been specified within a SEQUENCE. Therefore we construct a fake +%% sequence type here so that we can generate code for it. extaddgroup2sequence(ExtList) -> extaddgroup2sequence(ExtList,0,[]). diff --git a/lib/asn1/src/asn1ct_imm.erl b/lib/asn1/src/asn1ct_imm.erl index 2ab848652e..130f68c21d 100644 --- a/lib/asn1/src/asn1ct_imm.erl +++ b/lib/asn1/src/asn1ct_imm.erl @@ -41,7 +41,8 @@ per_enc_extensions_map/4, per_enc_optional/2]). -export([per_enc_sof/5]). --export([enc_absent/3,enc_append/1,enc_element/2,enc_maps_get/2]). +-export([enc_absent/3,enc_append/1,enc_element/2,enc_maps_get/2, + enc_comment/1]). -export([enc_cg/2]). -export([optimize_alignment/1,optimize_alignment/2, dec_slim_cg/2,dec_code_gen/2]). @@ -216,7 +217,8 @@ per_enc_legacy_bit_string(Val0, NNL0, Constraint0, Aligned) -> per_enc_boolean(Val0, _Aligned) -> {B,[Val]} = mk_vars(Val0, []), B++build_cond([[{eq,Val,false},{put_bits,0,1,[1]}], - [{eq,Val,true},{put_bits,1,1,[1]}]]). + [{eq,Val,true},{put_bits,1,1,[1]}], + ['_',{error,{illegal_boolean,Val}}]]). per_enc_choice(Val0, Cs0, _Aligned) -> {B,[Val]} = mk_vars(Val0, []), @@ -237,7 +239,7 @@ per_enc_enumerated(Val0, Root, Aligned) -> B++[{'cond',Cs++enumerated_error(Val)}]. enumerated_error(Val) -> - [['_',{error,Val}]]. + [['_',{error,{illegal_enumerated,Val}}]]. per_enc_integer(Val0, Constraint0, Aligned) -> {B,[Val]} = mk_vars(Val0, []), @@ -437,6 +439,9 @@ enc_maps_get(N, Val0) -> {var,SrcVar} = Val, {[{assign,DstExpr,SrcVar}],Dst0}. +enc_comment(Comment) -> + {comment,Comment}. + enc_cg(Imm0, false) -> Imm1 = enc_cse(Imm0), Imm2 = enc_pre_cg(Imm1), @@ -874,10 +879,8 @@ flatten_map_cs_1([integer_default], {Int,_}) -> [{'_',Int}]; flatten_map_cs_1([enum_default], {Int,_}) -> [{'_',["{asn1_enum,",Int,"}"]}]; -flatten_map_cs_1([enum_error], {Var,Cs}) -> - Vs = [V || {_,V} <- Cs], - [{'_',["exit({error,{asn1,{decode_enumerated,{",Var,",", - {asis,Vs},"}}}})"]}]; +flatten_map_cs_1([enum_error], {Var,_}) -> + [{'_',["exit({error,{asn1,{decode_enumerated,",Var,"}}})"]}]; flatten_map_cs_1([], _) -> []. flatten_hoist_align([[{align_bits,_,_}=Ab|T]|Cs]) -> @@ -1051,6 +1054,7 @@ split_off_nonbuilding(Imm) -> is_nonbuilding({assign,_,_}) -> true; is_nonbuilding({call,_,_,_,_}) -> true; +is_nonbuilding({comment,_}) -> true; is_nonbuilding({lc,_,_,_,_}) -> true; is_nonbuilding({set,_,_}) -> true; is_nonbuilding({list,_,_}) -> true; @@ -1107,7 +1111,7 @@ per_enc_integer_1(Val0, [{{_,_}=Constr,[]}], Aligned) -> per_enc_integer_1(Val0, [Constr], Aligned) -> {Prefix,Check,Action} = per_enc_integer_2(Val0, Constr, Aligned), Prefix++build_cond([[Check|Action], - ['_',{error,Val0}]]). + ['_',{error,{illegal_integer,Val0}}]]). per_enc_integer_2(Val, {'SingleValue',Sv}, Aligned) when is_integer(Sv) -> per_enc_constrained(Val, Sv, Sv, Aligned); @@ -1931,6 +1935,8 @@ enc_opt({'cond',Cs0}, St0) -> {Cs,Type} = enc_opt_cond_1(Cs1, Type0, [{Cond,Imm}]), {{'cond',Cs},St0#ost{t=Type}} end; +enc_opt({comment,_}=Imm, St) -> + {Imm,St#ost{t=undefined}}; enc_opt({cons,H0,T0}, St0) -> {H,#ost{t=TypeH}=St1} = enc_opt(H0, St0), {T,#ost{t=TypeT}=St} = enc_opt(T0, St1), @@ -2320,6 +2326,9 @@ enc_cg({block,Imm}) -> enc_cg(Imm), emit([nl, "end"]); +enc_cg({seq,{comment,Comment},Then}) -> + emit(["%% ",Comment,nl]), + enc_cg(Then); enc_cg({seq,First,Then}) -> enc_cg(First), emit([com,nl]), @@ -2353,9 +2362,9 @@ enc_cg({'cond',Cs}) -> enc_cg_cond(Cs); enc_cg({error,Error}) when is_function(Error, 0) -> Error(); -enc_cg({error,Var0}) -> +enc_cg({error,{Tag,Var0}}) -> Var = mk_val(Var0), - emit(["exit({error,{asn1,{illegal_value,",Var,"}}})"]); + emit(["exit({error,{asn1,{",Tag,",",Var,"}}})"]); enc_cg({integer,Int}) -> emit(mk_val(Int)); enc_cg({lc,Body,Var,List}) -> @@ -2618,6 +2627,8 @@ enc_opt_al({call,per_common,encode_unconstrained_number,[_]}=Call, _) -> {[Call],0}; enc_opt_al({call,_,_,_,_}=Call, Al) -> {[Call],Al}; +enc_opt_al({comment,_}=Imm, Al) -> + {[Imm],Al}; enc_opt_al({'cond',Cs0}, Al0) -> {Cs,Al} = enc_opt_al_cond(Cs0, Al0), {[{'cond',Cs}],Al}; @@ -2714,6 +2725,8 @@ per_fixup([{block,Block}|T]) -> [{block,per_fixup(Block)}|per_fixup(T)]; per_fixup([{'assign',_,_}=H|T]) -> [H|per_fixup(T)]; +per_fixup([{comment,_}=H|T]) -> + [H|per_fixup(T)]; per_fixup([{'cond',Cs0}|T]) -> Cs = [[C|per_fixup(Act)] || [C|Act] <- Cs0], [{'cond',Cs}|per_fixup(T)]; diff --git a/lib/asn1/src/asn1ct_name.erl b/lib/asn1/src/asn1ct_name.erl index 72d541cbbc..06f6604a26 100644 --- a/lib/asn1/src/asn1ct_name.erl +++ b/lib/asn1/src/asn1ct_name.erl @@ -20,7 +20,6 @@ %% -module(asn1ct_name). -%%-compile(export_all). -export([start/0, curr/1, clear/0, @@ -44,7 +43,6 @@ start() -> end. name_server_loop({Ref, Parent} = Monitor,Vars) -> -%% io:format("name -- ~w~n",[Vars]), receive {_From,clear} -> name_server_loop(Monitor, []); diff --git a/lib/asn1/src/asn1ct_parser2.erl b/lib/asn1/src/asn1ct_parser2.erl index 2de9b0e2f0..3f1819b660 100644 --- a/lib/asn1/src/asn1ct_parser2.erl +++ b/lib/asn1/src/asn1ct_parser2.erl @@ -1496,7 +1496,7 @@ parse_ContentsConstraint([{'ENCODED',_},{'BY',_}|Rest]) -> parse_ContentsConstraint(Tokens) -> parse_error(Tokens). -% X.683 Parameterization of ASN.1 specifications +%% X.683 Parameterization of ASN.1 specifications parse_Governor(Tokens) -> Flist = [fun parse_Type/1, diff --git a/lib/asn1/src/asn1ct_value.erl b/lib/asn1/src/asn1ct_value.erl index 8bd99d995b..f7d986aa91 100644 --- a/lib/asn1/src/asn1ct_value.erl +++ b/lib/asn1/src/asn1ct_value.erl @@ -24,12 +24,12 @@ %% The value is randomized within it's constraints -include("asn1_records.hrl"). -%-compile(export_all). -export([from_type/2]). -%% Generate examples of values ****************************** -%%****************************************x +%%**************************************** +%% Generate examples of values +%%**************************************** from_type(M,Typename) -> @@ -92,9 +92,6 @@ get_inner(T) when is_tuple(T) -> Other -> Other end. -%%get_inner(T) when is_tuple(T) -> element(1,T). - - from_type_constructed(M,Typename,InnerType,D) when is_record(D,type) -> case InnerType of @@ -111,9 +108,7 @@ from_type_constructed(M,Typename,InnerType,D) when is_record(D,type) -> 'SET OF' -> {_,Type} = D#type.def, NameSuffix = asn1ct_gen:constructed_suffix(InnerType,Type#type.def), - get_sequence_of(M,Typename,D,NameSuffix); - _ -> - exit({nyi,InnerType}) + get_sequence_of(M,Typename,D,NameSuffix) end. get_sequence(M,Typename,Type) -> @@ -147,7 +142,8 @@ get_choice(M,Typename,Type) -> case TCompList of [] -> {asn1_EMPTY,asn1_EMPTY}; - {CompList,ExtList} -> % Should be enhanced to handle extensions too + {CompList,ExtList} -> + %% should be enhanced to handle extensions too. CList = CompList ++ ExtList, C = lists:nth(random(length(CList)),CList), {C#'ComponentType'.name,from_type(M,Typename,C)}; @@ -247,14 +243,6 @@ from_type_prim(M, D) -> _ -> {2#11111111,2,2} end; -%% Sign1 = random_sign(integer), -%% Sign2 = random_sign(integer), -%% {Sign1*random(10000),2,Sign2*random(1028)}; -%% 2 -> -%% %% base 10 tuple format -%% Sign1 = random_sign(integer), -%% Sign2 = random_sign(integer), -%% {Sign1*random(10000),10,Sign2*random(1028)}; _ -> %% base 10 string format, NR3 format case random(2) of @@ -302,9 +290,7 @@ from_type_prim(M, D) -> 16#ffff,16#ffee,16#10ffff,16#ffff,16#fff]), unicode:characters_to_binary(L); 'UniversalString' -> - adjust_list(size_random(C),c_string(C,"UniversalString")); - XX -> - exit({asn1_error,nyi,XX}) + adjust_list(size_random(C),c_string(C,"UniversalString")) end. c_string(C,Default) -> @@ -343,22 +329,6 @@ random_unnamed_bit_string(M, C) -> {PadLen,<<BitString/bitstring,0:PadLen>>} end. -%% FIXME: -%% random_sign(integer) -> -%% case random(2) of -%% 2 -> -%% -1; -%% _ -> -%% 1 -%% end; -%% random_sign(string) -> -%% case random(2) of -%% 2 -> -%% "-"; -%% _ -> -%% "" -%% end. - random(Upper) -> rand:uniform(Upper). @@ -409,13 +379,6 @@ c_random(VRange,Single) -> S; {_,S} when is_list(S) -> lists:nth(random(length(S)),S) -%% {S1,S2} -> -%% io:format("asn1ct_value: hejsan hoppsan~n"); -%% _ -> -%% io:format("asn1ct_value: hejsan hoppsan 2~n") -%% io:format("asn1ct_value: c_random/2: S1 = ~w~n" -%% "S2 = ~w,~n",[S1,S2]) -%% exit(self(),goodbye) end. adjust_list(Len,Orig) -> diff --git a/lib/asn1/src/asn1rt_nif.erl b/lib/asn1/src/asn1rt_nif.erl index ff464885f6..e540b9f50d 100644 --- a/lib/asn1/src/asn1rt_nif.erl +++ b/lib/asn1/src/asn1rt_nif.erl @@ -26,6 +26,7 @@ decode_ber_tlv/1, encode_ber_tlv/1]). +-compile(no_native). -on_load(load_nif/0). -define(ASN1_NIF_VSN,1). diff --git a/lib/asn1/src/asn1rtt_ber.erl b/lib/asn1/src/asn1rtt_ber.erl index fdb9b9061f..882a25c332 100644 --- a/lib/asn1/src/asn1rtt_ber.erl +++ b/lib/asn1/src/asn1rtt_ber.erl @@ -92,7 +92,7 @@ -define(N_BMPString, 30). -% the complete tag-word of built-in types +%% The complete tag-word of built-in types -define(T_BOOLEAN, ?UNIVERSAL bor ?PRIMITIVE bor 1). -define(T_INTEGER, ?UNIVERSAL bor ?PRIMITIVE bor 2). -define(T_BIT_STRING, ?UNIVERSAL bor ?PRIMITIVE bor 3). % can be CONSTRUCTED @@ -137,11 +137,11 @@ ber_decode_erlang(Tlv) -> decode_primitive(Bin) -> {Form,TagNo,V,Rest} = decode_tag_and_length(Bin), case Form of - 1 -> % constructed + 1 -> % constructed {{TagNo,decode_constructed(V)},Rest}; - 0 -> % primitive + 0 -> % primitive {{TagNo,V},Rest}; - 2 -> % constructed indefinite + 2 -> % constructed indefinite {Vlist,Rest2} = decode_constructed_indefinite(V,[]), {{TagNo,Vlist},Rest2} end. @@ -165,31 +165,30 @@ decode_primitive_incomplete([[default,TagNo]],Bin) -> %default {Form,TagNo,V,Rest} -> decode_incomplete2(Form,TagNo,V,[],Rest); _ -> - %{asn1_DEFAULT,Bin} asn1_NOVALUE end; -decode_primitive_incomplete([[default,TagNo,Directives]],Bin) -> %default, constructed type, Directives points into this type +decode_primitive_incomplete([[default,TagNo,Directives]],Bin) -> + %% default, constructed type, Directives points into this type case decode_tag_and_length(Bin) of {Form,TagNo,V,Rest} -> decode_incomplete2(Form,TagNo,V,Directives,Rest); _ -> - %{asn1_DEFAULT,Bin} asn1_NOVALUE end; -decode_primitive_incomplete([[opt,TagNo]],Bin) -> %optional +decode_primitive_incomplete([[opt,TagNo]],Bin) -> + %% optional case decode_tag_and_length(Bin) of {Form,TagNo,V,Rest} -> decode_incomplete2(Form,TagNo,V,[],Rest); _ -> - %{{TagNo,asn1_NOVALUE},Bin} asn1_NOVALUE end; -decode_primitive_incomplete([[opt,TagNo,Directives]],Bin) -> %optional +decode_primitive_incomplete([[opt,TagNo,Directives]],Bin) -> + %% optional case decode_tag_and_length(Bin) of {Form,TagNo,V,Rest} -> decode_incomplete2(Form,TagNo,V,Directives,Rest); _ -> - %{{TagNo,asn1_NOVALUE},Bin} asn1_NOVALUE end; %% An optional that shall be undecoded @@ -236,7 +235,8 @@ decode_primitive_incomplete([[alt_parts,TagNo]|RestAlts],Bin) -> _ -> decode_primitive_incomplete(RestAlts,Bin) end; -decode_primitive_incomplete([[undec,_TagNo]|_RestTag],Bin) -> %incomlete decode +decode_primitive_incomplete([[undec,_TagNo]|_RestTag],Bin) -> + %% incomlete decode decode_incomplete_bin(Bin); decode_primitive_incomplete([[parts,TagNo]|_RestTag],Bin) -> case decode_tag_and_length(Bin) of @@ -301,7 +301,8 @@ decode_constructed_incomplete(Directives=[[Alt,_]|_],Bin) {TagNo,Tlv}; {alt_parts,_} -> [{TagNo,decode_parts_incomplete(V)}]; - no_match -> %% if a choice alternative was encoded that + no_match -> + %% if a choice alternative was encoded that %% was not specified in the config file, %% thus decode component anonomous. {Tlv,_}=decode_primitive(Bin), @@ -546,7 +547,7 @@ decode_tag_and_length(<<Class:2, Form:1, 31:5, Buffer/binary>>) -> decode_tag(<<0:1,PartialTag:7, Buffer/binary>>, TagAck) -> TagNo = (TagAck bsl 7) bor PartialTag, {TagNo, Buffer}; -% more tags +%% more tags decode_tag(<<_:1,PartialTag:7, Buffer/binary>>, TagAck) -> TagAck1 = (TagAck bsl 7) bor PartialTag, decode_tag(Buffer, TagAck1). @@ -941,12 +942,12 @@ encode_bit_string_bits(C, BitListVal, _NamedBitList, TagIn) when is_list(BitList case length(BitListVal) of BitSize when BitSize == Size -> {Len, Unused, OctetList} = encode_bitstring(BitListVal), - %%add unused byte to the Len + %% add unused byte to the Len encode_tags(TagIn, [Unused | OctetList], Len+1); BitSize when BitSize < Size -> PaddedList = pad_bit_list(Size-BitSize,BitListVal), {Len, Unused, OctetList} = encode_bitstring(PaddedList), - %%add unused byte to the Len + %% add unused byte to the Len encode_tags(TagIn, [Unused | OctetList], Len+1); BitSize -> exit({error,{asn1, diff --git a/lib/asn1/src/asn1rtt_per_common.erl b/lib/asn1/src/asn1rtt_per_common.erl index 3896cb7fa5..e7edfb1ee0 100644 --- a/lib/asn1/src/asn1rtt_per_common.erl +++ b/lib/asn1/src/asn1rtt_per_common.erl @@ -140,6 +140,8 @@ encode_relative_oid(Val) when is_tuple(Val) -> encode_relative_oid(Val) when is_list(Val) -> list_to_binary([e_object_element(X)||X <- Val]). +encode_unconstrained_number(Val) when not is_integer(Val) -> + exit({error,{asn1,{illegal_integer,Val}}}); encode_unconstrained_number(Val) when Val >= 0 -> if Val < 16#80 -> diff --git a/lib/asn1/test/asn1_SUITE.erl b/lib/asn1/test/asn1_SUITE.erl index 580c919b9d..d99190b6b0 100644 --- a/lib/asn1/test/asn1_SUITE.erl +++ b/lib/asn1/test/asn1_SUITE.erl @@ -1108,6 +1108,7 @@ test_modules() -> "From", "H235-SECURITY-MESSAGES", "H323-MESSAGES", + "HighTagNumbers", "Import", "Int", "MAP-commonDataTypes", diff --git a/lib/asn1/test/asn1_SUITE_data/HighTagNumbers.asn1 b/lib/asn1/test/asn1_SUITE_data/HighTagNumbers.asn1 new file mode 100644 index 0000000000..b681063965 --- /dev/null +++ b/lib/asn1/test/asn1_SUITE_data/HighTagNumbers.asn1 @@ -0,0 +1,17 @@ +HighTagNumbers DEFINITIONS ::= +BEGIN + +S ::= SEQUENCE { + a [127] INTEGER, + b [128] INTEGER, + c [150] INTEGER, + d [207] INTEGER, + e [255] INTEGER, + f [256] INTEGER, + g [7777] INTEGER, + h [9999] INTEGER, + i [16382] INTEGER, + j [16383] INTEGER +} + +END diff --git a/lib/asn1/test/asn1_SUITE_data/Prim.asn1 b/lib/asn1/test/asn1_SUITE_data/Prim.asn1 index 4fe0901683..91c8696e61 100644 --- a/lib/asn1/test/asn1_SUITE_data/Prim.asn1 +++ b/lib/asn1/test/asn1_SUITE_data/Prim.asn1 @@ -18,6 +18,8 @@ BEGIN IntExpPri ::= [PRIVATE 51] EXPLICIT INTEGER IntExpApp ::= [APPLICATION 52] EXPLICIT INTEGER + IntConstrained ::= INTEGER (0..255) + IntEnum ::= INTEGER {first(1),last(31)} Enum ::= ENUMERATED {monday(1),tuesday(2),wednesday(3),thursday(4), diff --git a/lib/asn1/test/asn1_SUITE_data/testobj.erl b/lib/asn1/test/asn1_SUITE_data/testobj.erl index e547ea4572..66f4a92188 100644 --- a/lib/asn1/test/asn1_SUITE_data/testobj.erl +++ b/lib/asn1/test/asn1_SUITE_data/testobj.erl @@ -967,7 +967,7 @@ pdu_pdp() -> 116,101,115,116, % lable1 = test 4, % length lable2 116,101,115,116, % lable2 = test - 4, % lenght lable3 + 4, % length lable3 116,101,115,116, % lable3 = test 4, % length lable3 116,101,115,116, % lable4 = test diff --git a/lib/asn1/test/ber_decode_error.erl b/lib/asn1/test/ber_decode_error.erl index c0840e02d7..c45d130ff4 100644 --- a/lib/asn1/test/ber_decode_error.erl +++ b/lib/asn1/test/ber_decode_error.erl @@ -26,48 +26,41 @@ run([]) -> {ok,B} = 'Constructed':encode('S3', {'S3',17}), [T,L|V] = binary_to_list(B), Bytes = list_to_binary([T,L+3|V] ++ [2,1,3]), - case 'Constructed':decode('S3', Bytes) of - {error,{asn1,{unexpected,_}}} -> ok - end, + {unexpected,_} = dec_error('S3', Bytes), %% Unexpected bytes must be accepted if there is an extensionmark {ok,{'S3ext',17}} = 'Constructed':decode('S3ext', Bytes), %% Truncated tag. - {error,{asn1,{invalid_tag,_}}} = - (catch 'Constructed':decode('I', <<31,255,255>>)), + {invalid_tag,_} = dec_error('I', <<31,255,255>>), %% Overlong tag. - {error,{asn1,{invalid_tag,_}}} = - (catch 'Constructed':decode('I', <<31,255,255,255,127>>)), + {invalid_tag,_} = dec_error('I', <<31,255,255,255,127>>), %% Invalid length. - {error,{asn1,{invalid_length,_}}} = - (catch 'Constructed':decode('I', <<8,255>>)), + {invalid_length,_} = dec_error('I', <<8,255>>), %% Other errors. - {error,{asn1,{invalid_value,_}}} = - (catch 'Constructed':decode('I', <<>>)), + {invalid_value,_} = dec_error('I', <<>>), - {error,{asn1,{invalid_value,_}}} = - (catch 'Constructed':decode('I', <<8,7>>)), + {invalid_value,_} = dec_error('I', <<8,7>>), %% Short indefinite length. Make sure that the decoder doesn't look %% beyond the end of binary when looking for a 0,0 terminator. - {error,{asn1,{invalid_length,_}}} = - (catch 'Constructed':decode('S', sub(<<8,16#80,0,0>>, 3))), - {error,{asn1,{invalid_length,_}}} = - (catch 'Constructed':decode('S', sub(<<8,16#80,0,0>>, 2))), - {error,{asn1,{invalid_length,_}}} = - (catch 'Constructed':decode('S', sub(<<40,16#80,1,1,255,0,0>>, 6))), - {error,{asn1,{invalid_length,_}}} = - (catch 'Constructed':decode('S', sub(<<40,16#80,1,1,255,0,0>>, 5))), + {invalid_length,_} = dec_error('S', sub(<<8,16#80,0,0>>, 3)), + {invalid_length,_} = dec_error('S', sub(<<8,16#80,0,0>>, 2)), + {invalid_length,_} = dec_error('S', sub(<<40,16#80,1,1,255,0,0>>, 6)), + {invalid_length,_} = dec_error('S', sub(<<40,16#80,1,1,255,0,0>>, 5)), %% A primitive must not be encoded with an indefinite length. - {error,{asn1,{invalid_length,_}}} = - (catch 'Constructed':decode('OS', <<4,128,4,3,97,98,99,0,0>>)), + {invalid_length,_} = dec_error('OS', <<4,128,4,3,97,98,99,0,0>>), ok. +dec_error(T, Bin) -> + {error,{asn1,{Reason,Stk}}} = 'Constructed':decode(T, Bin), + [{_,_,_,_}|_] = Stk, + Reason. + sub(Bin, Bytes) -> <<B:Bytes/binary,_/binary>> = Bin, B. diff --git a/lib/asn1/test/testChoPrim.erl b/lib/asn1/test/testChoPrim.erl index 573c482f2b..61b6ab2d05 100644 --- a/lib/asn1/test/testChoPrim.erl +++ b/lib/asn1/test/testChoPrim.erl @@ -31,10 +31,10 @@ bool(Rules) -> roundtrip('ChoCon', {int2,233}), case Rules of ber -> - {error,{asn1,{invalid_choice_type,wrong}}} = - (catch 'ChoPrim':encode('ChoCon', {wrong,233})), - {error,{asn1,{invalid_choice_tag,_WrongTag}}} = - (catch 'ChoPrim':decode('ChoCon', <<131,2,0,233>>)); + {error,{asn1,{{invalid_choice_type,wrong},[_|_]}}} = + (catch 'ChoPrim':encode('ChoCon', {wrong,233})), + {error,{asn1,{{invalid_choice_tag,_WrongTag},[_|_]}}} = + (catch 'ChoPrim':decode('ChoCon', <<131,2,0,233>>)); per -> ok; uper -> diff --git a/lib/asn1/test/testInfObjectClass.erl b/lib/asn1/test/testInfObjectClass.erl index 560986fac9..540407fa51 100644 --- a/lib/asn1/test/testInfObjectClass.erl +++ b/lib/asn1/test/testInfObjectClass.erl @@ -33,19 +33,29 @@ main(Rule) -> roundtrip('Seq', Val), %% OTP-5783 - {error,{asn1,{'Type not compatible with table constraint', - {component,'ArgumentType'}, - {value,_},_}}} = 'InfClass':encode('Seq', {'Seq',12,13,1}), + {'Type not compatible with table constraint', + {component,'ArgumentType'}, + {value,_},_} = enc_error('Seq', {'Seq',12,13,1}), Bytes2 = case Rule of ber -> <<48,9,2,1,12,2,1,11,2,1,1>>; _ -> <<1,12,1,11,1,1>> end, - {error,{asn1,{'Type not compatible with table constraint', - {{component,_}, - {value,_B},_}}}} = 'InfClass':decode('Seq', Bytes2), + {'Type not compatible with table constraint', + {{component,_}, + {value,_B},_}} = dec_error('Seq', Bytes2), ok. roundtrip(T, V) -> asn1_test_lib:roundtrip('InfClass', T, V). + +enc_error(T, V) -> + {error,{asn1,{Reason,Stk}}} = 'InfClass':encode(T, V), + [{_,_,_,_}|_] = Stk, + Reason. + +dec_error(T, Bin) -> + {error,{asn1,{Reason,Stk}}} = 'InfClass':decode(T, Bin), + [{_,_,_,_}|_] = Stk, + Reason. diff --git a/lib/asn1/test/testPrim.erl b/lib/asn1/test/testPrim.erl index 96a2dd6c79..b2933dfabc 100644 --- a/lib/asn1/test/testPrim.erl +++ b/lib/asn1/test/testPrim.erl @@ -34,15 +34,12 @@ bool(Rules) -> Types = ['Bool','BoolCon','BoolPri','BoolApp', 'BoolExpCon','BoolExpPri','BoolExpApp'], [roundtrip(T, V) || T <- Types, V <- [true,false]], - case Rules of - ber -> - [begin - {error,{asn1,{encode_boolean,517}}} = enc_error(T, 517) - end || T <- Types], - ok; - _ -> - ok - end. + Tag = case Rules of + ber -> encode_boolean; + _ -> illegal_boolean + end, + [{Tag,517} = enc_error(T, 517) || T <- Types], + ok. int(Rules) -> @@ -60,10 +57,22 @@ int(Rules) -> 123456789,12345678901234567890, -1,-2,-3,-4,-100,-127,-255,-256,-257, -1234567890,-2147483648], - [roundtrip(T, V) || - T <- ['Int','IntCon','IntPri','IntApp', - 'IntExpCon','IntExpPri','IntExpApp'], - V <- [1|Values]], + Types = ['Int','IntCon','IntPri','IntApp', + 'IntExpCon','IntExpPri','IntExpApp'], + _ = [roundtrip(T, V) || T <- Types, V <- [1|Values]], + Tag = case Rules of + ber -> encode_integer; + _ -> illegal_integer + end, + _ = [{Tag,V} = enc_error(T, V) || + T <- Types, V <- [atom,42.0,{a,b,c}]], + case Rules of + ber -> + ok; + _ -> + _ = [{Tag,V} = enc_error('IntConstrained', V) || + V <- [atom,-1,256,42.0]] + end, %%========================================================== %% IntEnum ::= INTEGER {first(1),last(31)} @@ -119,7 +128,11 @@ enum(Rules) -> roundtrip('Enum', monday), roundtrip('Enum', thursday), - {error,{asn1,{_,4}}} = enc_error('Enum', 4), + Tag = case Rules of + ber -> enumerated_not_in_range; + _ -> illegal_enumerated + end, + {Tag,4} = enc_error('Enum', 4), case Rules of Per when Per =:= per; Per =:= uper -> @@ -182,13 +195,15 @@ roundtrip(Type, Value, ExpectedValue) -> enc_error(T, V) -> case get(no_ok_wrapper) of false -> - 'Prim':encode(T, V); + {error,{asn1,{Reason,Stk}}} = 'Prim':encode(T, V), + [{_,_,_,_}|_] = Stk, + Reason; true -> try 'Prim':encode(T, V) of _ -> ?t:fail() catch - _:Reason -> + _:{error,{asn1,Reason}} -> Reason end end. diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index 48ffe653e4..d407a0a53f 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -224,7 +224,9 @@ </type> <desc> - <p>OPTIONAL</p> + <p>OPTIONAL; if this function is defined, then <seealso + marker="#Module:end_per_suite-1"><c>end_per_suite/1</c></seealso> + must also be defined.</p> <p>This configuration function is called as the first function in the suite. It typically contains initializations that are common for @@ -256,7 +258,9 @@ </type> <desc> - <p>OPTIONAL</p> + <p>OPTIONAL; if this function is defined, then <seealso + marker="#Module:init_per_suite-1"><c>init_per_suite/1</c></seealso> + must also be defined.</p> <p>This function is called as the last test case in the suite. It is meant to be used for cleaning up after @@ -360,7 +364,9 @@ </type> <desc> - <p>OPTIONAL</p> + <p>OPTIONAL; if this function is defined, then <seealso + marker="#Module:end_per_group-2"><c>end_per_group/2</c></seealso> + must also be defined.</p> <p>This configuration function is called before execution of a test case group. It typically contains initializations that are @@ -396,7 +402,9 @@ </type> <desc> - <p>OPTIONAL</p> + <p>OPTIONAL; if this function is defined, then <seealso + marker="#Module:init_per_group-2"><c>init_per_group/2</c></seealso> + must also be defined.</p> <p>This function is called after the execution of a test case group is finished. It is meant to be used for cleaning up after @@ -427,7 +435,10 @@ </type> <desc> - <p>OPTIONAL</p> + <p>OPTIONAL; if this function is defined, + then <seealso marker="#Module:end_per_testcase-2"> + <c>end_per_testcase/2</c></seealso> must also be + defined.</p> <p>This function is called before each test case. Argument <c>TestCase</c> is the test case name, and @@ -454,7 +465,10 @@ </type> <desc> - <p>OPTIONAL</p> + <p>OPTIONAL; if this function is defined, + then <seealso marker="#Module:init_per_testcase-2"> + <c>init_per_testcase/2</c></seealso> must also be + defined.</p> <p>This function is called after each test case, and can be used to clean up after diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml index c2cf29c530..a085f30262 100644 --- a/lib/common_test/doc/src/ct_hooks.xml +++ b/lib/common_test/doc/src/ct_hooks.xml @@ -208,9 +208,10 @@ </func> <func> - <name>Module:pre_init_per_group(GroupName, InitData, CTHState) -> Result</name> + <name>Module:pre_init_per_group(SuiteName, GroupName, InitData, CTHState) -> Result</name> <fsummary>Called before init_per_group.</fsummary> <type> + <v>SuiteName = atom()</v> <v>GroupName = atom()</v> <v>InitData = Config | SkipOrFail</v> <v>Config = NewConfig = [{Key,Value}]</v> @@ -231,13 +232,19 @@ but for function <seealso marker="common_test#Module:init_per_group-2"><c>init_per_group</c></seealso> instead.</p> + + <p>If <c>Module:pre_init_per_group/4</c> is not exported, common_test + will attempt to call <c>Module:pre_init_per_group(GroupName, + InitData, CTHState)</c> instead. This is for backwards + compatibility.</p> </desc> </func> <func> - <name>Module:post_init_per_group(GroupName, Config, Return, CTHState) -> Result</name> + <name>Module:post_init_per_group(SuiteName, GroupName, Config, Return, CTHState) -> Result</name> <fsummary>Called after init_per_group.</fsummary> <type> + <v>SuiteName = atom()</v> <v>GroupName = atom()</v> <v>Config = [{Key,Value}]</v> <v>Return = NewReturn = Config | SkipOrFail | term()</v> @@ -258,13 +265,19 @@ but for function <seealso marker="common_test#Module:init_per_group-2"><c>init_per_group</c></seealso> instead.</p> + + <p>If <c>Module:post_init_per_group/5</c> is not exported, common_test + will attempt to call <c>Module:post_init_per_group(GroupName, + Config, Return, CTHState)</c> instead. This is for backwards + compatibility.</p> </desc> </func> <func> - <name>Module:pre_init_per_testcase(TestcaseName, InitData, CTHState) -> Result</name> + <name>Module:pre_init_per_testcase(SuiteName, TestcaseName, InitData, CTHState) -> Result</name> <fsummary>Called before init_per_testcase.</fsummary> <type> + <v>SuiteName = atom()</v> <v>TestcaseName = atom()</v> <v>InitData = Config | SkipOrFail</v> <v>Config = NewConfig = [{Key,Value}]</v> @@ -286,6 +299,11 @@ <seealso marker="common_test#Module:init_per_testcase-2"><c>init_per_testcase</c></seealso> instead.</p> + <p>If <c>Module:pre_init_per_testcase/4</c> is not exported, common_test + will attempt to call <c>Module:pre_init_per_testcase(TestcaseName, + InitData, CTHState)</c> instead. This is for backwards + compatibility.</p> + <p>CTHs cannot be added here right now. That feature may be added in a later release, but it would right now break backwards compatibility.</p> @@ -293,9 +311,10 @@ </func> <func> - <name>Module:post_init_per_testcase(TestcaseName, Config, Return, CTHState) -> Result</name> + <name>Module:post_init_per_testcase(SuiteName, TestcaseName, Config, Return, CTHState) -> Result</name> <fsummary>Called after init_per_testcase.</fsummary> <type> + <v>SuiteName = atom()</v> <v>TestcaseName = atom()</v> <v>Config = [{Key,Value}]</v> <v>Return = NewReturn = Config | SkipOrFail | term()</v> @@ -316,15 +335,21 @@ but for function <seealso marker="common_test#Module:init_per_testcase-2"><c>init_per_testcase</c></seealso> instead.</p> + + <p>If <c>Module:post_init_per_testcase/5</c> is not exported, common_test + will attempt to call <c>Module:post_init_per_testcase(TestcaseName, + Config, Return, CTHState)</c> instead. This is for backwards + compatibility.</p> </desc> </func> <func> - <name>Module:pre_end_per_testcase(TestcaseName, InitData, CTHState) -> Result</name> + <name>Module:pre_end_per_testcase(SuiteName, TestcaseName, EndData, CTHState) -> Result</name> <fsummary>Called before end_per_testcase.</fsummary> <type> + <v>SuiteName = atom()</v> <v>TestcaseName = atom()</v> - <v>InitData = Config</v> + <v>EndData = Config</v> <v>Config = NewConfig = [{Key,Value}]</v> <v>CTHState = NewCTHState = term()</v> <v>Result = {NewConfig, NewCTHState}</v> @@ -345,14 +370,20 @@ <p>This function can not change the result of the test case by returning skip or fail tuples, but it may insert items in <c>Config</c> that can be read in - <c>end_per_testcase/2</c> or in <c>post_end_per_testcase/4</c>.</p> + <c>end_per_testcase/2</c> or in <c>post_end_per_testcase/5</c>.</p> + + <p>If <c>Module:pre_end_per_testcase/4</c> is not exported, common_test + will attempt to call <c>Module:pre_end_per_testcase(TestcaseName, + EndData, CTHState)</c> instead. This is for backwards + compatibility.</p> </desc> </func> <func> - <name>Module:post_end_per_testcase(TestcaseName, Config, Return, CTHState) -> Result</name> + <name>Module:post_end_per_testcase(SuiteName, TestcaseName, Config, Return, CTHState) -> Result</name> <fsummary>Called after end_per_testcase.</fsummary> <type> + <v>SuiteName = atom()</v> <v>TestcaseName = atom()</v> <v>Config = [{Key,Value}]</v> <v>Return = NewReturn = Config | SkipOrFail | term()</v> @@ -373,13 +404,19 @@ but for function <seealso marker="common_test#Module:end_per_testcase-2"><c>end_per_testcase</c></seealso> instead.</p> + + <p>If <c>Module:post_end_per_testcase/5</c> is not exported, common_test + will attempt to call <c>Module:post_end_per_testcase(TestcaseName, + Config, Return, CTHState)</c> instead. This is for backwards + compatibility.</p> </desc> </func> <func> - <name>Module:pre_end_per_group(GroupName, EndData, CTHState) -> Result</name> + <name>Module:pre_end_per_group(SuiteName, GroupName, EndData, CTHState) -> Result</name> <fsummary>Called before end_per_group.</fsummary> <type> + <v>SuiteName = atom()</v> <v>GroupName = atom()</v> <v>EndData = Config | SkipOrFail</v> <v>Config = NewConfig = [{Key,Value}]</v> @@ -400,13 +437,19 @@ but for function <seealso marker="common_test#Module:end_per_group-2"><c>end_per_group</c></seealso> instead.</p> + + <p>If <c>Module:pre_end_per_group/4</c> is not exported, common_test + will attempt to call <c>Module:pre_end_per_group(GroupName, + EndData, CTHState)</c> instead. This is for backwards + compatibility.</p> </desc> </func> <func> - <name>Module:post_end_per_group(GroupName, Config, Return, CTHState) -> Result</name> + <name>Module:post_end_per_group(SuiteName, GroupName, Config, Return, CTHState) -> Result</name> <fsummary>Called after end_per_group.</fsummary> <type> + <v>SuiteName = atom()</v> <v>GroupName = atom()</v> <v>Config = [{Key,Value}]</v> <v>Return = NewReturn = Config | SkipOrFail | term()</v> @@ -427,6 +470,11 @@ but for function <seealso marker="common_test#Module:end_per_group-2">end_per_group</seealso> instead.</p> + + <p>If <c>Module:post_end_per_group/5</c> is not exported, common_test + will attempt to call <c>Module:post_end_per_group(GroupName, + Config, Return, CTHState)</c> instead. This is for backwards + compatibility.</p> </desc> </func> @@ -485,9 +533,10 @@ </func> <func> - <name>Module:on_tc_fail(TestName, Reason, CTHState) -> NewCTHState</name> + <name>Module:on_tc_fail(SuiteName, TestName, Reason, CTHState) -> NewCTHState</name> <fsummary>Called after the CTH scope ends.</fsummary> <type> + <v>SuiteName = atom()</v> <v>TestName = init_per_suite | end_per_suite | {init_per_group,GroupName} | {end_per_group,GroupName} | {FuncName,GroupName} | FuncName</v> <v>FuncName = atom()</v> <v>GroupName = atom()</v> @@ -505,7 +554,7 @@ <item><p>If <c>init_per_suite</c> fails, this function is called after <seealso marker="#Module:post_init_per_suite-4"><c>post_init_per_suite</c></seealso>.</p></item> <item><p>If a test case fails, this funcion is called after - <seealso marker="#Module:post_end_per_testcase-4"><c>post_end_per_testcase</c></seealso>.</p></item> + <seealso marker="#Module:post_end_per_testcase-5"><c>post_end_per_testcase</c></seealso>.</p></item> </list> <p>If the failed test case belongs to a test case group, the first @@ -519,13 +568,19 @@ For details, see section <seealso marker="event_handler_chapter#events">Event Handling</seealso> in the User's Guide.</p> + + <p>If <c>Module:on_tc_fail/4</c> is not exported, common_test + will attempt to call <c>Module:on_tc_fail(TestName, Reason, + CTHState)</c> instead. This is for backwards + compatibility.</p> </desc> </func> <func> - <name>Module:on_tc_skip(TestName, Reason, CTHState) -> NewCTHState</name> + <name>Module:on_tc_skip(SuiteName, TestName, Reason, CTHState) -> NewCTHState</name> <fsummary>Called after the CTH scope ends.</fsummary> <type> + <v>SuiteName = atom()</v> <v>TestName = init_per_suite | end_per_suite | {init_per_group,GroupName} | {end_per_group,GroupName} | {FuncName,GroupName} | FuncName</v> <v>FuncName = atom()</v> <v>GroupName = atom()</v> @@ -542,9 +597,9 @@ <list type="bulleted"> <item><p>If <c>init_per_group</c> is skipped, this function is called after - <seealso marker="#Module:post_init_per_group-4"><c>post_init_per_group</c></seealso>.</p></item> + <seealso marker="#Module:post_init_per_group-5"><c>post_init_per_group</c></seealso>.</p></item> <item><p>If a test case is skipped, this function is called after - <seealso marker="#Module:post_end_per_testcase-4"><c>post_end_per_testcase</c></seealso>.</p></item> + <seealso marker="#Module:post_end_per_testcase-5"><c>post_end_per_testcase</c></seealso>.</p></item> </list> <p>If the skipped test case belongs to a test case group, the first @@ -559,6 +614,11 @@ For details, see section <seealso marker="event_handler_chapter#events">Event Handling</seealso> in the User's Guide.</p> + + <p>If <c>Module:on_tc_skip/4</c> is not exported, common_test + will attempt to call <c>Module:on_tc_skip(TestName, Reason, + CTHState)</c> instead. This is for backwards + compatibility.</p> </desc> </func> diff --git a/lib/common_test/doc/src/ct_hooks_chapter.xml b/lib/common_test/doc/src/ct_hooks_chapter.xml index 0e4c35e11f..bfad96e489 100644 --- a/lib/common_test/doc/src/ct_hooks_chapter.xml +++ b/lib/common_test/doc/src/ct_hooks_chapter.xml @@ -38,7 +38,7 @@ extensions of the default behavior of <c>Common Test</c> using hooks before and after all test suite calls. CTHs allow advanced <c>Common Test</c> users to abstract out behavior that is common to multiple test suites - without littering all test suites with library calls. this can be used + without littering all test suites with library calls. This can be used for logging, starting, and monitoring external systems, building C files needed by the tests, and so on.</p> @@ -175,10 +175,10 @@ <row> <cell><seealso marker="common_test#Module:init_per_group-2"> init_per_group/2</seealso></cell> - <cell><seealso marker="ct_hooks#Module:post_init_per_group-4"> - post_init_per_group/4</seealso> is called</cell> - <cell><seealso marker="ct_hooks#Module:post_end_per_suite-4"> - post_end_per_group/4</seealso> has been called for that group</cell> + <cell><seealso marker="ct_hooks#Module:post_init_per_group-5"> + post_init_per_group/5</seealso> is called</cell> + <cell><seealso marker="ct_hooks#Module:post_end_per_group-5"> + post_end_per_group/5</seealso> has been called for that group</cell> </row> <tcaption>Scope of a CTH</tcaption> </table> @@ -245,16 +245,18 @@ </list> <p> - This is done in the CTH functions called pre_<name of function>. - These functions take the same three arguments, <c>Name</c>, + This is done in the CTH functions called <c>pre_<name of function></c>. + These functions take the arguments <c>SuiteName</c>, <c>Name</c> (group or test case name, if applicable), <c>Config</c>, and <c>CTHState</c>. The return value of the CTH function is always a combination of a result for the suite/group/test and an updated <c>CTHState</c>.</p> <p>To let the test suite continue on executing, return the configuration - list that you want the test to use as the result. To skip or - fail the test, return a tuple with <c>skip</c> or <c>fail</c>, and a reason - as the result.</p> + list that you want the test to use as the result.</p> + + <p>All pre hooks, except <c>pre_end_per_testcase/4</c>, can + skip or fail the test by returning a tuple with <c>skip</c> or + <c>fail</c>, and a reason as the result.</p> <p><em>Example:</em></p> <code> @@ -290,7 +292,7 @@ <p> This is done in the CTH functions called <c>post_<name of function></c>. - These functions take the same four arguments, <c>Name</c>, + These functions take the arguments <c>SuiteName</c>, <c>Name</c> (group or test case name, if applicable), <c>Config</c>, <c>Return</c>, and <c>CTHState</c>. <c>Config</c> in this case is the same <c>Config</c> as the testcase is called with. <c>Return</c> is the value returned by the testcase. If the testcase @@ -308,7 +310,7 @@ <p><em>Example:</em></p> <code> - post_end_per_testcase(_TC, Config, {'EXIT',{_,_}}, CTHState) -> + post_end_per_testcase(_Suite, _TC, Config, {'EXIT',{_,_}}, CTHState) -> case db:check_consistency() of true -> %% DB is good, pass the test. @@ -317,7 +319,7 @@ %% DB is not good, mark as skipped instead of failing {{skip, "DB is inconsisten!"}, CTHState} end; - post_end_per_testcase(_TC, Config, Return, CTHState) -> + post_end_per_testcase(_Suite, _TC, Config, Return, CTHState) -> %% Do nothing if tc does not crash. {Return, CTHState}.</code> @@ -331,8 +333,8 @@ <title>Skip and Fail Hooks</title> <p> After any post hook has been executed for all installed CTHs, - <seealso marker="ct_hooks#Module:on_tc_fail-3">on_tc_fail</seealso> - or <seealso marker="ct_hooks#Module:on_tc_skip-3">on_tc_skip</seealso> + <seealso marker="ct_hooks#Module:on_tc_fail-4">on_tc_fail</seealso> + or <seealso marker="ct_hooks#Module:on_tc_skip-4">on_tc_skip</seealso> is called if the testcase failed or was skipped, respectively. You cannot affect the outcome of the tests any further at this point. </p> @@ -389,18 +391,18 @@ -export([pre_end_per_suite/3]). -export([post_end_per_suite/4]). - -export([pre_init_per_group/3]). - -export([post_init_per_group/4]). - -export([pre_end_per_group/3]). - -export([post_end_per_group/4]). + -export([pre_init_per_group/4]). + -export([post_init_per_group/5]). + -export([pre_end_per_group/4]). + -export([post_end_per_group/5]). - -export([pre_init_per_testcase/3]). - -export([post_init_per_testcase/4]). - -export([pre_end_per_testcase/3]). - -export([post_end_per_testcase/4]). + -export([pre_init_per_testcase/4]). + -export([post_init_per_testcase/5]). + -export([pre_end_per_testcase/4]). + -export([post_end_per_testcase/5]). - -export([on_tc_fail/3]). - -export([on_tc_skip/3]). + -export([on_tc_fail/4]). + -export([on_tc_skip/4]). -export([terminate/1]). @@ -435,46 +437,46 @@ total = State#state.total + State#state.suite_total } }. %% @doc Called before each init_per_group. - pre_init_per_group(Group,Config,State) -> + pre_init_per_group(Suite,Group,Config,State) -> {Config, State}. %% @doc Called after each init_per_group. - post_init_per_group(Group,Config,Return,State) -> + post_init_per_group(Suite,Group,Config,Return,State) -> {Return, State}. %% @doc Called before each end_per_group. - pre_end_per_group(Group,Config,State) -> + pre_end_per_group(Suite,Group,Config,State) -> {Config, State}. %% @doc Called after each end_per_group. - post_end_per_group(Group,Config,Return,State) -> + post_end_per_group(Suite,Group,Config,Return,State) -> {Return, State}. %% @doc Called before each init_per_testcase. - pre_init_per_testcase(TC,Config,State) -> + pre_init_per_testcase(Suite,TC,Config,State) -> {Config, State#state{ ts = now(), total = State#state.suite_total + 1 } }. %% Called after each init_per_testcase (immediately before the test case). - post_init_per_testcase(TC,Config,Return,State) -> + post_init_per_testcase(Suite,TC,Config,Return,State) -> {Return, State} %% @doc Called before each end_per_testcase (immediately after the test case). - pre_end_per_testcase(TC,Config,State) -> + pre_end_per_testcase(Suite,TC,Config,State) -> {Config, State}. %% @doc Called after each end_per_testcase. - post_end_per_testcase(TC,Config,Return,State) -> - TCInfo = {testcase, TC, Return, timer:now_diff(now(), State#state.ts)}, + post_end_per_testcase(Suite,TC,Config,Return,State) -> + TCInfo = {testcase, Suite, TC, Return, timer:now_diff(now(), State#state.ts)}, {Return, State#state{ ts = undefined, tcs = [TCInfo | State#state.tcs] } }. %% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, %% post_end_per_group and post_end_per_testcase if the suite, group or test case failed. - on_tc_fail(TC, Reason, State) -> + on_tc_fail(Suite, TC, Reason, State) -> State. %% @doc Called when a test case is skipped by either user action %% or due to an init function failing. - on_tc_skip(TC, Reason, State) -> + on_tc_skip(Suite, TC, Reason, State) -> State. %% @doc Called when the scope of the CTH is done diff --git a/lib/common_test/doc/src/notes.xml b/lib/common_test/doc/src/notes.xml index 83e6511c04..efeacd4a72 100644 --- a/lib/common_test/doc/src/notes.xml +++ b/lib/common_test/doc/src/notes.xml @@ -33,6 +33,123 @@ <file>notes.xml</file> </header> +<section><title>Common_Test 1.14</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>The following corrections and improvements are done in + the common_test hook handling:</p> <list> <item> <p>An + extra argument, <c>Suite</c>, is added as the first + argument to each of the following hook callback + functions:</p> <list> + <item><c>pre_init_per_group</c></item> + <item><c>post_init_per_group</c></item> + <item><c>pre_end_per_group</c></item> + <item><c>post_end_per_group</c></item> + <item><c>pre_init_per_testcase</c></item> + <item><c>post_init_per_testcase</c></item> + <item><c>pre_end_per_testcase</c></item> + <item><c>post_end_per_testcase</c></item> + <item><c>on_tc_fail</c></item> + <item><c>on_tc_skip</c></item> </list> <p>For backwards + compatibility, if the new function is not exported from a + hook callback module, <c>common_test</c> will fall back + to the old interface and call the function without the + <c>Suite</c> argument.</p> </item> <item> <p>If either + <c>init_per_suite</c> or <c>end_per_suite</c> exists, but + not the other, then the non-existing function will be + reported as failed with reason <c>undef</c> in the test + log. The same goes for <c>init/end_per_group</c>. This + has always been a requirement according to the user's + guide, but now <c>common_test</c> is more explicit in the + report.</p> </item> <item> <p>If <c>init_per_suite</c> + was exported from a test suite, but not + <c>end_per_suite</c>, then <c>pre/post_end_per_suite</c> + was called with <c>Suite=ct_framework</c> instead of the + correct suite name. This is now corrected.</p> </item> + <item> <p>If <c>end_per_group</c> was exported from a + suite, but not <c>init_per_group</c>, then + <c>end_per_group</c> was never called. This is now + corrected.</p> </item> <item> <p>Tests that were skipped + before calling <c>pre_init_per_*</c> got faulty calls to + the corresponding <c>post_init_per_*</c>. E.g. if a test + was skipped because <c>suite/0</c> failed, then + <c>post_init_per_suite</c> would be called even though + <c>pre_init_per_suite</c> and <c>init_per_suite</c> were + not called. This is now corrected so a <c>post_*</c> + callback will never be called unless the corresponding + <c>pre_*</c> callback has been called first.</p> </item> + <item> <p>Tests that were skipped before or in + <c>init_per_testcase</c> got faulty calls to + <c>pre_end_per_testcase</c> and + <c>post_end_per_testcase</c>. This is now corrected so + <c>pre/post_end_per_testcase</c> are not called when + <c>end_per_testcase</c> is not called.</p> </item> <item> + <p>If an exit signal causes the test case process to die + while running <c>init_per_testcase</c>, the case was + earlier reported as failed with reason <c>{skip,...}</c>. + This is now corrected so the case will be marked as + skipped.</p> </item> <item> <p>If an exist signal causes + the test case process to die while running + <c>end_per_testcase</c>, the case was earlier marked as + failed. This is now corrected so the status of the test + case is not changed - there is only a warning added to + the comment field.</p> </item> <item> <p>If a test case + was skipped because of option + <c>{force_stop,skip_rest}</c> or because of a failed + sequence, then no <c>tc_start</c> event would be sent, + only <c>tc_done</c>. This is now corrected so both events + are sent.</p> </item> <item> <p>When skipping or failing + in a configuration function, the configuration function + itself would get <c>{auto_skipped,Reason}</c>, + <c>{skipped,Reason}</c> or <c>{failed,Reason}</c> in the + hook callbacks <c>on_tc_skip</c> or <c>on_tc_fail</c>. + The other test cases that were skipped as a result of + this would only get <c>Reason</c> in <c>on_tc_skip</c>. + This is now corrected so even the configuration function + that caused the skip/fail will only get <c>Reason</c> in + the hook callback.</p> </item> </list> + <p> + Own Id: OTP-10599 Aux Id: kunagi-344 [255] </p> + </item> + <item> + <p> + When a test case was skipped by a <c>skip_cases</c> + statement in a test spec, then <c>cth_surefire</c> would + erroneously mark the previous test case as skipped in the + xml report. The actually skipped test case would not be + present in the xml report at all. This is now corrected.</p> + <p> + Own Id: OTP-14129 Aux Id: seq13244 </p> + </item> + <item> + <p>The <c>multiply_timetraps</c> and + <c>scale_timetraps</c> options did not work with test + specifications, which has been corrected.</p> + <p> + Own Id: OTP-14210</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + ct_testspec:get_tests/1 is added. This is used by rebar3 + to get all directories that must be compiled when running + tests from testspec - instead of implementing testspec + parsing in rebar3.</p> + <p> + Own Id: OTP-14132</p> + </item> + </list> + </section> + +</section> + <section><title>Common_Test 1.13</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index f70bdb16c5..c230148b29 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -566,7 +566,7 @@ for the test cases in the group. After execution of the group is finished, function <seealso marker="common_test#Module:end_per_group-2"><c>end_per_group(GroupName, Config)</c></seealso> is called. This function is meant to be used for cleaning up after - <c>init_per_group/2</c>.</p> + <c>init_per_group/2</c>. If the init function is defined, so must the end function be.</p> <p>Whenever a group is executed, if <c>init_per_group</c> and <c>end_per_group</c> do not exist in the suite, <c>Common Test</c> calls @@ -1036,7 +1036,7 @@ Importance >= (100-VerbosityLevel)</pre> <p>Note that the category argument is not required in order to only specify the importance of a printout. Example:</p> <pre> -<c>ct:pal(?LOW_IMPORTANCE, "Info report: ~p", [Info])</c></pre> +ct:pal(?LOW_IMPORTANCE, "Info report: ~p", [Info])</pre> <p>Or perhaps in combination with constants:</p> <pre> -define(INFO, ?LOW_IMPORTANCE). diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 291a4d716c..43f1c9de0f 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -52,6 +52,10 @@ %%% %%% @doc Test server framework callback, called by the test_server %%% when a new test case is started. +init_tc(_,{end_per_testcase_not_run,_},[Config]) -> + %% Testcase is completed (skipped or failed), but end_per_testcase + %% is not run - don't call pre-hook. + {ok,[Config]}; init_tc(Mod,EPTC={end_per_testcase,_},[Config]) -> %% in case Mod == ct_framework, lookup the suite name Suite = get_suite_name(Mod, Config), @@ -62,7 +66,7 @@ init_tc(Mod,EPTC={end_per_testcase,_},[Config]) -> Other end; -init_tc(Mod,Func0,Args) -> +init_tc(Mod,Func0,Args) -> %% in case Mod == ct_framework, lookup the suite name Suite = get_suite_name(Mod, Args), {Func,HookFunc} = case Func0 of @@ -84,12 +88,15 @@ init_tc(Mod,Func0,Args) -> andalso Func=/=end_per_group andalso ct_util:get_testdata(skip_rest) of true -> + initialize(false,Mod,Func,Args), {auto_skip,"Repeated test stopped by force_stop option"}; _ -> case ct_util:get_testdata(curr_tc) of {Suite,{suite0_failed,{require,Reason}}} -> + initialize(false,Mod,Func,Args), {auto_skip,{require_failed_in_suite0,Reason}}; {Suite,{suite0_failed,_}=Failure} -> + initialize(false,Mod,Func,Args), {fail,Failure}; _ -> ct_util:update_testdata(curr_tc, @@ -118,16 +125,14 @@ init_tc(Mod,Func0,Args) -> end, init_tc1(Mod,Suite,Func,HookFunc,Args); {failed,Seq,BadFunc} -> - {auto_skip,{sequence_failed,Seq,BadFunc}} + initialize(false,Mod,Func,Args), + {auto_skip,{sequence_failed,Seq,BadFunc}} end end end. init_tc1(?MODULE,_,error_in_suite,_,[Config0]) when is_list(Config0) -> - ct_logs:init_tc(false), - ct_event:notify(#event{name=tc_start, - node=node(), - data={?MODULE,error_in_suite}}), + initialize(false,?MODULE,error_in_suite), _ = ct_suite_init(?MODULE,error_in_suite,[],Config0), case ?val(error,Config0) of undefined -> @@ -177,27 +182,21 @@ init_tc1(Mod,Suite,Func,HookFunc,[Config0]) when is_list(Config0) -> ct_config:delete_default_config(testcase), HookFunc end, - Initialize = fun() -> - ct_logs:init_tc(false), - ct_event:notify(#event{name=tc_start, - node=node(), - data={Mod,FuncSpec}}) - end, case add_defaults(Mod,Func,AllGroups) of Error = {suite0_failed,_} -> - Initialize(), + initialize(false,Mod,FuncSpec), ct_util:set_testdata({curr_tc,{Suite,Error}}), {error,Error}; Error = {group0_failed,_} -> - Initialize(), + initialize(false,Mod,FuncSpec), {auto_skip,Error}; Error = {testcase0_failed,_} -> - Initialize(), + initialize(false,Mod,FuncSpec), {auto_skip,Error}; {SuiteInfo,MergeResult} -> case MergeResult of {error,Reason} -> - Initialize(), + initialize(false,Mod,FuncSpec), {fail,Reason}; _ -> init_tc2(Mod,Suite,Func,HookFunc1, @@ -236,11 +235,8 @@ init_tc2(Mod,Suite,Func,HookFunc,SuiteInfo,MergeResult,Config) -> Conns -> ct_util:silence_connections(Conns) end, - ct_logs:init_tc(Func == init_per_suite), FuncSpec = group_or_func(Func,Config), - ct_event:notify(#event{name=tc_start, - node=node(), - data={Mod,FuncSpec}}), + initialize((Func==init_per_suite),Mod,FuncSpec), case catch configure(MergedInfo,MergedInfo,SuiteInfo, FuncSpec,[],Config) of @@ -268,6 +264,18 @@ init_tc2(Mod,Suite,Func,HookFunc,SuiteInfo,MergeResult,Config) -> end end. +initialize(RefreshLogs,Mod,Func,[Config]) when is_list(Config) -> + initialize(RefreshLogs,Mod,group_or_func(Func,Config)); +initialize(RefreshLogs,Mod,Func,_) -> + initialize(RefreshLogs,Mod,Func). + +initialize(RefreshLogs,Mod,FuncSpec) -> + ct_logs:init_tc(RefreshLogs), + ct_event:notify(#event{name=tc_start, + node=node(), + data={Mod,FuncSpec}}). + + ct_suite_init(Suite,HookFunc,PostInitHook,Config) when is_list(Config) -> case ct_hooks:init_tc(Suite,HookFunc,Config) of NewConfig when is_list(NewConfig) -> @@ -675,22 +683,35 @@ end_tc(Mod,Func,{Result,[Args]}, Return) -> end_tc(Mod,Func,self(),Result,Args,Return). end_tc(Mod,IPTC={init_per_testcase,_Func},_TCPid,Result,Args,Return) -> - %% in case Mod == ct_framework, lookup the suite name - Suite = get_suite_name(Mod, Args), - case ct_hooks:end_tc(Suite,IPTC,Args,Result,Return) of - '$ct_no_change' -> - ok; - HookResult -> - HookResult + case end_hook_func(IPTC,Return,IPTC) of + undefined -> ok; + _ -> + %% in case Mod == ct_framework, lookup the suite name + Suite = get_suite_name(Mod, Args), + case ct_hooks:end_tc(Suite,IPTC,Args,Result,Return) of + '$ct_no_change' -> + ok; + HookResult -> + HookResult + end end; end_tc(Mod,Func0,TCPid,Result,Args,Return) -> %% in case Mod == ct_framework, lookup the suite name Suite = get_suite_name(Mod, Args), - {EPTC,Func} = case Func0 of - {end_per_testcase,F} -> {true,F}; - _ -> {false,Func0} - end, + {Func,FuncSpec,HookFunc} = + case Func0 of + {end_per_testcase_not_run,F} -> + %% Testcase is completed (skipped or failed), but + %% end_per_testcase is not run - don't call post-hook. + {F,F,undefined}; + {end_per_testcase,F} -> + {F,F,Func0}; + _ -> + FS = group_or_func(Func0,Args), + HF = end_hook_func(Func0,Return,FS), + {Func0,FS,HF} + end, test_server:timetrap_cancel(), @@ -717,20 +738,18 @@ end_tc(Mod,Func0,TCPid,Result,Args,Return) -> end, ct_util:delete_suite_data(last_saved_config), - {FuncSpec,HookFunc} = - if not EPTC -> - FS = group_or_func(Func,Args), - {FS,FS}; - true -> - {Func,Func0} - end, {Result1,FinalNotify} = - case ct_hooks:end_tc(Suite,HookFunc,Args,Result,Return) of - '$ct_no_change' -> - {ok,Result}; - HookResult -> - {HookResult,HookResult} - end, + case HookFunc of + undefined -> + {ok,Result}; + _ -> + case ct_hooks:end_tc(Suite,HookFunc,Args,Result,Return) of + '$ct_no_change' -> + {ok,Result}; + HookResult -> + {HookResult,HookResult} + end + end, FinalResult = case get('$test_server_framework_test') of undefined -> @@ -821,6 +840,34 @@ end_tc(Mod,Func0,TCPid,Result,Args,Return) -> end, FinalResult. +%% This is to make sure that no post_init_per_* is ever called if the +%% corresponding pre_init_per_* was not called. +%% The skip or fail reasons are those that can be returned from +%% init_tc above in situations where we never came to call +%% ct_hooks:init_tc/3, e.g. if suite/0 fails, then we never call +%% ct_hooks:init_tc for init_per_suite, and thus we must not call +%% ct_hooks:end_tc for init_per_suite either. +end_hook_func({init_per_testcase,_},{auto_skip,{sequence_failed,_,_}},_) -> + undefined; +end_hook_func({init_per_testcase,_},{auto_skip,"Repeated test stopped by force_stop option"},_) -> + undefined; +end_hook_func({init_per_testcase,_},{fail,{config_name_already_in_use,_}},_) -> + undefined; +end_hook_func({init_per_testcase,_},{auto_skip,{InfoFuncError,_}},_) + when InfoFuncError==testcase0_failed; + InfoFuncError==require_failed -> + undefined; +end_hook_func(init_per_group,{auto_skip,{InfoFuncError,_}},_) + when InfoFuncError==group0_failed; + InfoFuncError==require_failed -> + undefined; +end_hook_func(init_per_suite,{auto_skip,{require_failed_in_suite0,_}},_) -> + undefined; +end_hook_func(init_per_suite,{auto_skip,{failed,{error,{suite0_failed,_}}}},_) -> + undefined; +end_hook_func(_,_,Default) -> + Default. + %% {error,Reason} | {skip,Reason} | {timetrap_timeout,TVal} | %% {testcase_aborted,Reason} | testcase_aborted_or_killed | %% {'EXIT',Reason} | {fail,Reason} | {failed,Reason} | @@ -1339,25 +1386,25 @@ report(What,Data) -> ok; tc_done -> {Suite,{Func,GrName},Result} = Data, - Data1 = if GrName == undefined -> {Suite,Func,Result}; - true -> Data - end, + FuncSpec = if GrName == undefined -> Func; + true -> {Func,GrName} + end, %% Register the group leader for the process calling the report %% function, making it possible for a hook function to print %% in the test case log file ReportingPid = self(), ct_logs:register_groupleader(ReportingPid, group_leader()), case Result of - {failed, _} -> - ct_hooks:on_tc_fail(What, Data1); - {skipped,{failed,{_,init_per_testcase,_}}} -> - ct_hooks:on_tc_skip(tc_auto_skip, Data1); - {skipped,{require_failed,_}} -> - ct_hooks:on_tc_skip(tc_auto_skip, Data1); - {skipped,_} -> - ct_hooks:on_tc_skip(tc_user_skip, Data1); - {auto_skipped,_} -> - ct_hooks:on_tc_skip(tc_auto_skip, Data1); + {failed, Reason} -> + ct_hooks:on_tc_fail(What, {Suite,FuncSpec,Reason}); + {skipped,{failed,{_,init_per_testcase,_}}=Reason} -> + ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason}); + {skipped,{require_failed,_}=Reason} -> + ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason}); + {skipped,Reason} -> + ct_hooks:on_tc_skip(tc_user_skip, {Suite,FuncSpec,Reason}); + {auto_skipped,Reason} -> + ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason}); _Else -> ok end, diff --git a/lib/common_test/src/ct_groups.erl b/lib/common_test/src/ct_groups.erl index 1375e7dcc7..1c9faf6a70 100644 --- a/lib/common_test/src/ct_groups.erl +++ b/lib/common_test/src/ct_groups.erl @@ -442,17 +442,21 @@ make_conf(Mod, Name, Props, TestSpec) -> ok end, {InitConf,EndConf,ExtraProps} = - case erlang:function_exported(Mod,init_per_group,2) of - true -> - {{Mod,init_per_group},{Mod,end_per_group},[]}; - false -> + case {erlang:function_exported(Mod,init_per_group,2), + erlang:function_exported(Mod,end_per_group,2)} of + {false,false} -> ct_logs:log("TEST INFO", "init_per_group/2 and " "end_per_group/2 missing for group " "~w in ~w, using default.", [Name,Mod]), {{ct_framework,init_per_group}, {ct_framework,end_per_group}, - [{suite,Mod}]} + [{suite,Mod}]}; + _ -> + %% If any of these exist, the other should too + %% (required and documented). If it isn't, it will fail + %% with reason 'undef'. + {{Mod,init_per_group},{Mod,end_per_group},[]} end, {conf,[{name,Name}|Props++ExtraProps],InitConf,TestSpec,EndConf}. diff --git a/lib/common_test/src/ct_hooks.erl b/lib/common_test/src/ct_hooks.erl index c9a4abb5ee..60d1ea2b1c 100644 --- a/lib/common_test/src/ct_hooks.erl +++ b/lib/common_test/src/ct_hooks.erl @@ -92,15 +92,17 @@ init_tc(Mod, end_per_suite, Config) -> call(fun call_generic/3, Config, [pre_end_per_suite, Mod]); init_tc(Mod, {init_per_group, GroupName, Properties}, Config) -> maybe_start_locker(Mod, GroupName, Properties), - call(fun call_generic/3, Config, [pre_init_per_group, GroupName]); -init_tc(_Mod, {end_per_group, GroupName, _}, Config) -> - call(fun call_generic/3, Config, [pre_end_per_group, GroupName]); -init_tc(_Mod, {init_per_testcase,TC}, Config) -> - call(fun call_generic/3, Config, [pre_init_per_testcase, TC]); -init_tc(_Mod, {end_per_testcase,TC}, Config) -> - call(fun call_generic/3, Config, [pre_end_per_testcase, TC]); -init_tc(_Mod, TC = error_in_suite, Config) -> - call(fun call_generic/3, Config, [pre_init_per_testcase, TC]). + call(fun call_generic_fallback/3, Config, + [pre_init_per_group, Mod, GroupName]); +init_tc(Mod, {end_per_group, GroupName, _}, Config) -> + call(fun call_generic_fallback/3, Config, + [pre_end_per_group, Mod, GroupName]); +init_tc(Mod, {init_per_testcase,TC}, Config) -> + call(fun call_generic_fallback/3, Config, [pre_init_per_testcase, Mod, TC]); +init_tc(Mod, {end_per_testcase,TC}, Config) -> + call(fun call_generic_fallback/3, Config, [pre_end_per_testcase, Mod, TC]); +init_tc(Mod, TC = error_in_suite, Config) -> + call(fun call_generic_fallback/3, Config, [pre_init_per_testcase, Mod, TC]). %% @doc Called as each test case is completed. This includes all configuration %% tests. @@ -126,23 +128,23 @@ end_tc(Mod, init_per_suite, Config, _Result, Return) -> end_tc(Mod, end_per_suite, Config, Result, _Return) -> call(fun call_generic/3, Result, [post_end_per_suite, Mod, Config], '$ct_no_change'); -end_tc(_Mod, {init_per_group, GroupName, _}, Config, _Result, Return) -> - call(fun call_generic/3, Return, [post_init_per_group, GroupName, Config], - '$ct_no_change'); +end_tc(Mod, {init_per_group, GroupName, _}, Config, _Result, Return) -> + call(fun call_generic_fallback/3, Return, + [post_init_per_group, Mod, GroupName, Config], '$ct_no_change'); end_tc(Mod, {end_per_group, GroupName, Properties}, Config, Result, _Return) -> - Res = call(fun call_generic/3, Result, - [post_end_per_group, GroupName, Config], '$ct_no_change'), + Res = call(fun call_generic_fallback/3, Result, + [post_end_per_group, Mod, GroupName, Config], '$ct_no_change'), maybe_stop_locker(Mod, GroupName, Properties), Res; -end_tc(_Mod, {init_per_testcase,TC}, Config, Result, _Return) -> - call(fun call_generic/3, Result, [post_init_per_testcase, TC, Config], - '$ct_no_change'); -end_tc(_Mod, {end_per_testcase,TC}, Config, Result, _Return) -> - call(fun call_generic/3, Result, [post_end_per_testcase, TC, Config], - '$ct_no_change'); -end_tc(_Mod, TC = error_in_suite, Config, Result, _Return) -> - call(fun call_generic/3, Result, [post_end_per_testcase, TC, Config], - '$ct_no_change'). +end_tc(Mod, {init_per_testcase,TC}, Config, Result, _Return) -> + call(fun call_generic_fallback/3, Result, + [post_init_per_testcase, Mod, TC, Config], '$ct_no_change'); +end_tc(Mod, {end_per_testcase,TC}, Config, Result, _Return) -> + call(fun call_generic_fallback/3, Result, + [post_end_per_testcase, Mod, TC, Config], '$ct_no_change'); +end_tc(Mod, TC = error_in_suite, Config, Result, _Return) -> + call(fun call_generic_fallback/3, Result, + [post_end_per_testcase, Mod, TC, Config], '$ct_no_change'). %% Case = TestCase | {TestCase,GroupName} @@ -181,15 +183,21 @@ call_terminate(#ct_hook_config{ module = Mod, state = State} = Hook, _, _) -> {[],Hook}. call_cleanup(#ct_hook_config{ module = Mod, state = State} = Hook, - Reason, [Function, _Suite | Args]) -> + Reason, [Function | Args]) -> NewState = catch_apply(Mod,Function, Args ++ [Reason, State], - State), + State, true), {Reason, Hook#ct_hook_config{ state = NewState } }. -call_generic(#ct_hook_config{ module = Mod, state = State} = Hook, - Value, [Function | Args]) -> +call_generic(Hook, Value, Meta) -> + do_call_generic(Hook, Value, Meta, false). + +call_generic_fallback(Hook, Value, Meta) -> + do_call_generic(Hook, Value, Meta, true). + +do_call_generic(#ct_hook_config{ module = Mod, state = State} = Hook, + Value, [Function | Args], Fallback) -> {NewValue, NewState} = catch_apply(Mod, Function, Args ++ [Value, State], - {Value,State}), + {Value,State}, Fallback), {NewValue, Hook#ct_hook_config{ state = NewState } }. %% Generic call function @@ -257,15 +265,15 @@ remove(Key,List) when is_list(List) -> remove(_, Else) -> Else. -%% Translate scopes, i.e. init_per_group,group1 -> end_per_group,group1 etc -scope([pre_init_per_testcase, TC|_]) -> - [post_init_per_testcase, TC]; -scope([pre_end_per_testcase, TC|_]) -> - [post_end_per_testcase, TC]; -scope([pre_init_per_group, GroupName|_]) -> - [post_end_per_group, GroupName]; -scope([post_init_per_group, GroupName|_]) -> - [post_end_per_group, GroupName]; +%% Translate scopes, i.e. is_tuplenit_per_group,group1 -> end_per_group,group1 etc +scope([pre_init_per_testcase, SuiteName, TC|_]) -> + [post_init_per_testcase, SuiteName, TC]; +scope([pre_end_per_testcase, SuiteName, TC|_]) -> + [post_end_per_testcase, SuiteName, TC]; +scope([pre_init_per_group, SuiteName, GroupName|_]) -> + [post_end_per_group, SuiteName, GroupName]; +scope([post_init_per_group, SuiteName, GroupName|_]) -> + [post_end_per_group, SuiteName, GroupName]; scope([pre_init_per_suite, SuiteName|_]) -> [post_end_per_suite, SuiteName]; scope([post_init_per_suite, SuiteName|_]) -> @@ -273,14 +281,29 @@ scope([post_init_per_suite, SuiteName|_]) -> scope(init) -> none. -terminate_if_scope_ends(HookId, [on_tc_skip,_Suite,{end_per_group,Name}], +strip_config([post_init_per_testcase, SuiteName, TC|_]) -> + [post_init_per_testcase, SuiteName, TC]; +strip_config([post_end_per_testcase, SuiteName, TC|_]) -> + [post_end_per_testcase, SuiteName, TC]; +strip_config([post_init_per_group, SuiteName, GroupName|_]) -> + [post_init_per_group, SuiteName, GroupName]; +strip_config([post_end_per_group, SuiteName, GroupName|_]) -> + [post_end_per_group, SuiteName, GroupName]; +strip_config([post_init_per_suite, SuiteName|_]) -> + [post_init_per_suite, SuiteName]; +strip_config([post_end_per_suite, SuiteName|_]) -> + [post_end_per_suite, SuiteName]; +strip_config(Other) -> + Other. + + +terminate_if_scope_ends(HookId, [on_tc_skip,Suite,{end_per_group,Name}], Hooks) -> - terminate_if_scope_ends(HookId, [post_end_per_group, Name], Hooks); + terminate_if_scope_ends(HookId, [post_end_per_group, Suite, Name], Hooks); terminate_if_scope_ends(HookId, [on_tc_skip,Suite,end_per_suite], Hooks) -> terminate_if_scope_ends(HookId, [post_end_per_suite, Suite], Hooks); -terminate_if_scope_ends(HookId, [Function,Tag|T], Hooks) when T =/= [] -> - terminate_if_scope_ends(HookId,[Function,Tag],Hooks); -terminate_if_scope_ends(HookId, Function, Hooks) -> +terminate_if_scope_ends(HookId, Function0, Hooks) -> + Function = strip_config(Function0), case lists:keyfind(HookId, #ct_hook_config.id, Hooks) of #ct_hook_config{ id = HookId, scope = Function} = Hook -> terminate([Hook]), @@ -384,21 +407,29 @@ pos(Id,[_|Rest],Num) -> catch_apply(M,F,A, Default) -> + catch_apply(M,F,A,Default,false). +catch_apply(M,F,A, Default, Fallback) -> + not erlang:module_loaded(M) andalso (catch M:module_info()), + case erlang:function_exported(M,F,length(A)) of + false when Fallback -> + catch_apply(M,F,tl(A),Default,false); + false -> + Default; + true -> + catch_apply(M,F,A) + end. + +catch_apply(M,F,A) -> try - erlang:apply(M,F,A) + erlang:apply(M,F,A) catch _:Reason -> - case erlang:get_stacktrace() of - %% Return the default if it was the CTH module which did not have the function. - [{M,F,A,_}|_] when Reason == undef -> - Default; - Trace -> - ct_logs:log("Suite Hook","Call to CTH failed: ~w:~p", - [error,{Reason,Trace}]), - throw({error_in_cth_call, - lists:flatten( - io_lib:format("~w:~w/~w CTH call failed", - [M,F,length(A)]))}) - end + Trace = erlang:get_stacktrace(), + ct_logs:log("Suite Hook","Call to CTH failed: ~w:~p", + [error,{Reason,Trace}]), + throw({error_in_cth_call, + lists:flatten( + io_lib:format("~w:~w/~w CTH call failed", + [M,F,length(A)]))}) end. diff --git a/lib/common_test/src/ct_release_test.erl b/lib/common_test/src/ct_release_test.erl index d783f8d04e..c53e72ee88 100644 --- a/lib/common_test/src/ct_release_test.erl +++ b/lib/common_test/src/ct_release_test.erl @@ -132,7 +132,7 @@ %%----------------------------------------------------------------- -define(testnode, 'ct_release_test-upgrade'). --define(exclude_apps, [hipe, typer, dialyzer]). % never include these apps +-define(exclude_apps, [hipe, dialyzer]). % never include these apps %%----------------------------------------------------------------- -record(ct_data, {from,to}). diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index a049ef5695..cac176de3a 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -76,8 +76,8 @@ abort_if_missing_suites, silent_connections = [], stylesheet, - multiply_timetraps = 1, - scale_timetraps = false, + multiply_timetraps, + scale_timetraps, create_priv_dir, testspec_files = [], current_testspec, @@ -264,11 +264,11 @@ script_start1(Parent, Args) -> [], Args), Verbosity = verbosity_args2opts(Args), MultTT = get_start_opt(multiply_timetraps, - fun([MT]) -> list_to_integer(MT) end, 1, Args), + fun([MT]) -> list_to_integer(MT) end, Args), ScaleTT = get_start_opt(scale_timetraps, fun([CT]) -> list_to_atom(CT); ([]) -> true - end, false, Args), + end, Args), CreatePrivDir = get_start_opt(create_priv_dir, fun([PD]) -> list_to_atom(PD); ([]) -> auto_per_tc @@ -1055,8 +1055,8 @@ run_test2(StartOpts) -> CoverStop = get_start_opt(cover_stop, value, StartOpts), %% timetrap manipulation - MultiplyTT = get_start_opt(multiply_timetraps, value, 1, StartOpts), - ScaleTT = get_start_opt(scale_timetraps, value, false, StartOpts), + MultiplyTT = get_start_opt(multiply_timetraps, value, StartOpts), + ScaleTT = get_start_opt(scale_timetraps, value, StartOpts), %% create unique priv dir names CreatePrivDir = get_start_opt(create_priv_dir, value, StartOpts), @@ -2280,8 +2280,19 @@ do_run_test(Tests, Skip, Opts0) -> _Lower -> ok end, - test_server_ctrl:multiply_timetraps(Opts0#opts.multiply_timetraps), - test_server_ctrl:scale_timetraps(Opts0#opts.scale_timetraps), + + case Opts0#opts.multiply_timetraps of + undefined -> MultTT = 1; + MultTT -> MultTT + end, + case Opts0#opts.scale_timetraps of + undefined -> ScaleTT = false; + ScaleTT -> ScaleTT + end, + ct_logs:log("TEST INFO","Timetrap time multiplier = ~w~n" + "Timetrap scaling enabled = ~w", [MultTT,ScaleTT]), + test_server_ctrl:multiply_timetraps(MultTT), + test_server_ctrl:scale_timetraps(ScaleTT), test_server_ctrl:create_priv_dir(choose_val( Opts0#opts.create_priv_dir, diff --git a/lib/common_test/src/ct_slave.erl b/lib/common_test/src/ct_slave.erl index 571958ca03..4188bd7c3b 100644 --- a/lib/common_test/src/ct_slave.erl +++ b/lib/common_test/src/ct_slave.erl @@ -309,7 +309,12 @@ is_started(ENode) -> % make a Erlang node name from name and hostname enodename(Host, Node) -> - list_to_atom(atom_to_list(Node)++"@"++atom_to_list(Host)). + case lists:member($@, atom_to_list(Node)) of + true -> + Node; + false -> + list_to_atom(atom_to_list(Node)++"@"++atom_to_list(Host)) + end. % performs actual start of the "slave" node do_start(Host, Node, Options) -> diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index 16001ce4c8..466a2c7658 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -1158,6 +1158,11 @@ handle_data(verbosity,Node,VLvls,_Spec) when is_list(VLvls) -> VLvls1 = lists:map(fun(VLvl = {_Cat,_Lvl}) -> VLvl; (Lvl) -> {'$unspecified',Lvl} end, VLvls), [{Node,VLvls1}]; +handle_data(multiply_timetraps,Node,Mult,_Spec) when is_integer(Mult) -> + [{Node,Mult}]; +handle_data(scale_timetraps,Node,Scale,_Spec) when Scale == true; + Scale == false -> + [{Node,Scale}]; handle_data(silent_connections,Node,all,_Spec) -> [{Node,[all]}]; handle_data(silent_connections,Node,Conn,_Spec) when is_atom(Conn) -> @@ -1176,6 +1181,8 @@ should_be_added(Tag,Node,_Data,Spec) -> Tag == label; Tag == auto_compile; Tag == abort_if_missing_suites; Tag == stylesheet; Tag == verbosity; + Tag == multiply_timetraps; + Tag == scale_timetraps; Tag == silent_connections -> lists:keymember(ref2node(Node,Spec#testspec.nodes),1, read_field(Spec,Tag)) == false; diff --git a/lib/common_test/src/cth_conn_log.erl b/lib/common_test/src/cth_conn_log.erl index 883da0da0a..ce8852b3ea 100644 --- a/lib/common_test/src/cth_conn_log.erl +++ b/lib/common_test/src/cth_conn_log.erl @@ -54,8 +54,8 @@ -include_lib("common_test/include/ct.hrl"). -export([init/2, - pre_init_per_testcase/3, - post_end_per_testcase/4]). + pre_init_per_testcase/4, + post_end_per_testcase/5]). %%---------------------------------------------------------------------- %% Exported types @@ -104,7 +104,7 @@ get_log_opts(Mod,Opts) -> Hosts = proplists:get_value(hosts,Opts,[]), {LogType,Hosts}. -pre_init_per_testcase(TestCase,Config,CthState) -> +pre_init_per_testcase(_Suite,TestCase,Config,CthState) -> Logs = lists:map( fun({ConnMod,{LogType,Hosts}}) -> @@ -158,7 +158,7 @@ pre_init_per_testcase(TestCase,Config,CthState) -> ct_util:update_testdata(?MODULE, Update, [create]), {Config,CthState}. -post_end_per_testcase(TestCase,_Config,Return,CthState) -> +post_end_per_testcase(_Suite,TestCase,_Config,Return,CthState) -> Update = fun(PrevUsers) -> case lists:delete(TestCase, PrevUsers) of diff --git a/lib/common_test/src/cth_log_redirect.erl b/lib/common_test/src/cth_log_redirect.erl index 6d77d7ee9e..eda090d4f5 100644 --- a/lib/common_test/src/cth_log_redirect.erl +++ b/lib/common_test/src/cth_log_redirect.erl @@ -28,10 +28,10 @@ %% CTH Callbacks -export([id/1, init/2, pre_init_per_suite/3, pre_end_per_suite/3, post_end_per_suite/4, - pre_init_per_group/3, post_init_per_group/4, - pre_end_per_group/3, post_end_per_group/4, - pre_init_per_testcase/3, post_init_per_testcase/4, - pre_end_per_testcase/3, post_end_per_testcase/4]). + pre_init_per_group/4, post_init_per_group/5, + pre_end_per_group/4, post_end_per_group/5, + pre_init_per_testcase/4, post_init_per_testcase/5, + pre_end_per_testcase/4, post_end_per_testcase/5]). %% Event handler Callbacks -export([init/1, @@ -71,11 +71,11 @@ post_end_per_suite(_Suite, Config, Return, State) -> set_curr_func(undefined, Config), {Return, State}. -pre_init_per_group(Group, Config, State) -> +pre_init_per_group(_Suite, Group, Config, State) -> set_curr_func({group,Group,init_per_group}, Config), {Config, State}. -post_init_per_group(Group, Config, Result, tc_log_async) when is_list(Config) -> +post_init_per_group(_Suite, Group, Config, Result, tc_log_async) when is_list(Config) -> case lists:member(parallel,proplists:get_value( tc_group_properties,Config,[])) of true -> @@ -83,33 +83,33 @@ post_init_per_group(Group, Config, Result, tc_log_async) when is_list(Config) -> false -> {Result, tc_log_async} end; -post_init_per_group(_Group, _Config, Result, State) -> +post_init_per_group(_Suite, _Group, _Config, Result, State) -> {Result, State}. -pre_init_per_testcase(TC, Config, State) -> +pre_init_per_testcase(_Suite, TC, Config, State) -> set_curr_func(TC, Config), {Config, State}. -post_init_per_testcase(_TC, _Config, Return, State) -> +post_init_per_testcase(_Suite, _TC, _Config, Return, State) -> {Return, State}. -pre_end_per_testcase(_TC, Config, State) -> +pre_end_per_testcase(_Suite, _TC, Config, State) -> {Config, State}. -post_end_per_testcase(_TC, _Config, Result, State) -> +post_end_per_testcase(_Suite, _TC, _Config, Result, State) -> %% Make sure that the event queue is flushed %% before ending this test case. gen_event:call(error_logger, ?MODULE, flush, 300000), {Result, State}. -pre_end_per_group(Group, Config, {tc_log, Group}) -> +pre_end_per_group(_Suite, Group, Config, {tc_log, Group}) -> set_curr_func({group,Group,end_per_group}, Config), {Config, set_log_func(tc_log_async)}; -pre_end_per_group(Group, Config, State) -> +pre_end_per_group(_Suite, Group, Config, State) -> set_curr_func({group,Group,end_per_group}, Config), {Config, State}. -post_end_per_group(_Group, Config, Return, State) -> +post_end_per_group(_Suite, _Group, Config, Return, State) -> set_curr_func({group,undefined}, Config), {Return, State}. diff --git a/lib/common_test/src/cth_surefire.erl b/lib/common_test/src/cth_surefire.erl index 59b916851e..c4941948cc 100644 --- a/lib/common_test/src/cth_surefire.erl +++ b/lib/common_test/src/cth_surefire.erl @@ -33,16 +33,16 @@ -export([pre_end_per_suite/3]). -export([post_end_per_suite/4]). --export([pre_init_per_group/3]). --export([post_init_per_group/4]). --export([pre_end_per_group/3]). --export([post_end_per_group/4]). +-export([pre_init_per_group/4]). +-export([post_init_per_group/5]). +-export([pre_end_per_group/4]). +-export([post_end_per_group/5]). --export([pre_init_per_testcase/3]). --export([post_end_per_testcase/4]). +-export([pre_init_per_testcase/4]). +-export([post_end_per_testcase/5]). --export([on_tc_fail/3]). --export([on_tc_skip/3]). +-export([on_tc_fail/4]). +-export([on_tc_skip/4]). -export([terminate/1]). @@ -116,29 +116,29 @@ pre_end_per_suite(_Suite,Config,State) -> post_end_per_suite(_Suite,Config,Result,State) -> {Result, end_tc(end_per_suite,Config,Result,State)}. -pre_init_per_group(Group,Config,State) -> +pre_init_per_group(_Suite,Group,Config,State) -> {Config, init_tc(State#state{ curr_group = [Group|State#state.curr_group]}, Config)}. -post_init_per_group(_Group,Config,Result,State) -> +post_init_per_group(_Suite,_Group,Config,Result,State) -> {Result, end_tc(init_per_group,Config,Result,State)}. -pre_end_per_group(_Group,Config,State) -> +pre_end_per_group(_Suite,_Group,Config,State) -> {Config, init_tc(State, Config)}. -post_end_per_group(_Group,Config,Result,State) -> +post_end_per_group(_Suite,_Group,Config,Result,State) -> NewState = end_tc(end_per_group, Config, Result, State), {Result, NewState#state{ curr_group = tl(NewState#state.curr_group)}}. -pre_init_per_testcase(_TC,Config,State) -> +pre_init_per_testcase(_Suite,_TC,Config,State) -> {Config, init_tc(State, Config)}. -post_end_per_testcase(TC,Config,Result,State) -> +post_end_per_testcase(_Suite,TC,Config,Result,State) -> {Result, end_tc(TC,Config, Result,State)}. -on_tc_fail(_TC, _Res, State = #state{test_cases = []}) -> +on_tc_fail(_Suite,_TC, _Res, State = #state{test_cases = []}) -> State; -on_tc_fail(_TC, Res, State) -> +on_tc_fail(_Suite,_TC, Res, State) -> TCs = State#state.test_cases, TC = hd(TCs), NewTC = TC#testcase{ @@ -146,10 +146,9 @@ on_tc_fail(_TC, Res, State) -> {fail,lists:flatten(io_lib:format("~p",[Res]))} }, State#state{ test_cases = [NewTC | tl(TCs)]}. -on_tc_skip({ConfigFunc,_GrName},{Type,_Reason} = Res, State0) - when Type == tc_auto_skip; Type == tc_user_skip -> - on_tc_skip(ConfigFunc, Res, State0); -on_tc_skip(Tc,{Type,_Reason} = Res, State0) when Type == tc_auto_skip -> +on_tc_skip(Suite,{ConfigFunc,_GrName}, Res, State) -> + on_tc_skip(Suite,ConfigFunc, Res, State); +on_tc_skip(Suite,Tc, Res, State0) -> TcStr = atom_to_list(Tc), State = case State0#state.test_cases of @@ -158,11 +157,7 @@ on_tc_skip(Tc,{Type,_Reason} = Res, State0) when Type == tc_auto_skip -> _ -> State0 end, - do_tc_skip(Res, end_tc(Tc,[],Res,init_tc(State,[]))); -on_tc_skip(_Tc, _Res, State = #state{test_cases = []}) -> - State; -on_tc_skip(_Tc, Res, State) -> - do_tc_skip(Res, State). + do_tc_skip(Res, end_tc(Tc,[],Res,init_tc(set_suite(Suite,State),[]))). do_tc_skip(Res, State) -> TCs = State#state.test_cases, @@ -209,6 +204,12 @@ end_tc(Name, _Config, _Res, State = #state{ curr_suite = Suite, result = passed }| State#state.test_cases], tc_log = ""}. % so old tc_log is not set if next is on_tc_skip + +set_suite(Suite,#state{curr_suite=undefined}=State) -> + State#state{curr_suite=Suite, curr_suite_ts=?now}; +set_suite(_,State) -> + State. + close_suite(#state{ test_cases = [] } = State) -> State; close_suite(#state{ test_cases = TCs, url_base = UrlBase } = State) -> @@ -228,7 +229,8 @@ close_suite(#state{ test_cases = TCs, url_base = UrlBase } = State) -> testcases = lists:reverse(TCs), log = SuiteLog, url = SuiteUrl}, - State#state{ test_cases = [], + State#state{ curr_suite = undefined, + test_cases = [], test_suites = [Suite | State#state.test_suites]}. terminate(State = #state{ test_cases = [] }) -> diff --git a/lib/common_test/src/test_server.erl b/lib/common_test/src/test_server.erl index 924086f2bd..be49191f2e 100644 --- a/lib/common_test/src/test_server.erl +++ b/lib/common_test/src/test_server.erl @@ -778,9 +778,9 @@ spawn_fw_call(Mod,IPTC={init_per_testcase,Func},CurrConf,Pid, %% if init_per_testcase fails, the test case %% should be skipped try begin do_end_tc_call(Mod,IPTC, {Pid,Skip,[CurrConf]}, Why), - do_init_tc_call(Mod,{end_per_testcase,Func}, + do_init_tc_call(Mod,{end_per_testcase_not_run,Func}, [CurrConf],{ok,[CurrConf]}), - do_end_tc_call(Mod,{end_per_testcase,Func}, + do_end_tc_call(Mod,{end_per_testcase_not_run,Func}, {Pid,Skip,[CurrConf]}, Why) end of _ -> ok catch @@ -1151,14 +1151,14 @@ do_end_tc_call(Mod, IPTC={init_per_testcase,Func}, Res, Return) -> Args end, EPTCInitRes = - case do_init_tc_call(Mod,{end_per_testcase,Func}, + case do_init_tc_call(Mod,{end_per_testcase_not_run,Func}, IPTCEndRes,Return) of {ok,EPTCInitConfig} when is_list(EPTCInitConfig) -> {Return,EPTCInitConfig}; _ -> - Return + {Return,IPTCEndRes} end, - do_end_tc_call1(Mod, {end_per_testcase,Func}, + do_end_tc_call1(Mod, {end_per_testcase_not_run,Func}, EPTCInitRes, Return); _Ok -> do_end_tc_call1(Mod, IPTC, Res, Return) diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl index b52e4bef9b..39c523f8b3 100644 --- a/lib/common_test/src/test_server_ctrl.erl +++ b/lib/common_test/src/test_server_ctrl.erl @@ -2051,17 +2051,21 @@ add_init_and_end_per_suite([], _LastMod, skipped_suite, _FwMod) -> add_init_and_end_per_suite([], LastMod, LastRef, FwMod) -> %% we'll add end_per_suite here even if it's not exported %% (and simply let the call fail if it's missing) - case erlang:function_exported(LastMod, end_per_suite, 1) of - true -> - [{conf,LastRef,[],{LastMod,end_per_suite}}]; - false -> + case {erlang:function_exported(LastMod, end_per_suite, 1), + erlang:function_exported(LastMod, init_per_suite, 1)} of + {false,false} -> %% let's call a "fake" end_per_suite if it exists case erlang:function_exported(FwMod, end_per_suite, 1) of true -> [{conf,LastRef,[{suite,LastMod}],{FwMod,end_per_suite}}]; false -> [{conf,LastRef,[],{LastMod,end_per_suite}}] - end + end; + _ -> + %% If any of these exist, the other should too + %% (required and documented). If it isn't, it will fail + %% with reason 'undef'. + [{conf,LastRef,[],{LastMod,end_per_suite}}] end. do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) -> @@ -2070,11 +2074,9 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) -> _ -> ok end, {Init,NextMod,NextRef} = - case erlang:function_exported(Mod, init_per_suite, 1) of - true -> - Ref = make_ref(), - {[{conf,Ref,[],{Mod,init_per_suite}}],Mod,Ref}; - false -> + case {erlang:function_exported(Mod, init_per_suite, 1), + erlang:function_exported(Mod, end_per_suite, 1)} of + {false,false} -> %% let's call a "fake" init_per_suite if it exists case erlang:function_exported(FwMod, init_per_suite, 1) of true -> @@ -2083,8 +2085,13 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) -> {FwMod,init_per_suite}}],Mod,Ref}; false -> {[],Mod,undefined} - end - + end; + _ -> + %% If any of these exist, the other should too + %% (required and documented). If it isn't, it will fail + %% with reason 'undef'. + Ref = make_ref(), + {[{conf,Ref,[],{Mod,init_per_suite}}],Mod,Ref} end, Cases = if LastRef==undefined -> @@ -2094,10 +2101,9 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) -> true -> %% we'll add end_per_suite here even if it's not exported %% (and simply let the call fail if it's missing) - case erlang:function_exported(LastMod, end_per_suite, 1) of - true -> - [{conf,LastRef,[],{LastMod,end_per_suite}}|Init]; - false -> + case {erlang:function_exported(LastMod, end_per_suite, 1), + erlang:function_exported(LastMod, init_per_suite, 1)} of + {false,false} -> %% let's call a "fake" end_per_suite if it exists case erlang:function_exported(FwMod, end_per_suite, 1) of true -> @@ -2105,8 +2111,13 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) -> {FwMod,end_per_suite}}|Init]; false -> [{conf,LastRef,[],{LastMod,end_per_suite}}|Init] - end - end + end; + _ -> + %% If any of these exist, the other should too + %% (required and documented). If it isn't, it will fail + %% with reason 'undef'. + [{conf,LastRef,[],{LastMod,end_per_suite}}|Init] + end end, {Cases,NextMod,NextRef}. @@ -2115,11 +2126,9 @@ do_add_end_per_suite_and_skip(LastMod, LastRef, Mod, FwMod) -> No when No==undefined ; No==skipped_suite -> {[],Mod,skipped_suite}; _Ref -> - case erlang:function_exported(LastMod, end_per_suite, 1) of - true -> - {[{conf,LastRef,[],{LastMod,end_per_suite}}], - Mod,skipped_suite}; - false -> + case {erlang:function_exported(LastMod, end_per_suite, 1), + erlang:function_exported(LastMod, init_per_suite, 1)} of + {false,false} -> case erlang:function_exported(FwMod, end_per_suite, 1) of true -> %% let's call "fake" end_per_suite if it exists @@ -2128,7 +2137,13 @@ do_add_end_per_suite_and_skip(LastMod, LastRef, Mod, FwMod) -> false -> {[{conf,LastRef,[],{LastMod,end_per_suite}}], Mod,skipped_suite} - end + end; + _ -> + %% If any of these exist, the other should too + %% (required and documented). If it isn't, it will fail + %% with reason 'undef'. + {[{conf,LastRef,[],{LastMod,end_per_suite}}], + Mod,skipped_suite} end end. @@ -2924,22 +2939,21 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status) exit(framework_error); %% sequential execution of test case finished {Time,RetVal,_} -> + RetTag = + if is_tuple(RetVal) -> element(1,RetVal); + true -> undefined + end, {Failed,Status1} = - case Time of - died -> - {true,update_status(failed, Mod, Func, Status)}; - _ when is_tuple(RetVal) -> - case element(1, RetVal) of - R when R=='EXIT'; R==failed -> - {true,update_status(failed, Mod, Func, Status)}; - R when R==skip; R==skipped -> - {false,update_status(skipped, Mod, Func, Status)}; - _ -> - {false,update_status(ok, Mod, Func, Status)} - end; - _ -> - {false,update_status(ok, Mod, Func, Status)} - end, + case RetTag of + Skip when Skip==skip; Skip==skipped -> + {false,update_status(skipped, Mod, Func, Status)}; + Fail when Fail=='EXIT'; Fail==failed -> + {true,update_status(failed, Mod, Func, Status)}; + _ when Time==died, RetVal=/=ok -> + {true,update_status(failed, Mod, Func, Status)}; + _ -> + {false,update_status(ok, Mod, Func, Status)} + end, case check_prop(sequence, Mode) of false -> stop_minor_log_file(), @@ -3794,7 +3808,15 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, {died,{timetrap_timeout,TimetrapTimeout}} -> progress(failed, Num, Mod, Func, GrName, Loc, timetrap_timeout, TimetrapTimeout, Comment, Style); - {died,Reason} -> + {died,{Skip,Reason}} when Skip==skip; Skip==skipped -> + %% died in init_per_testcase + progress(skip, Num, Mod, Func, GrName, Loc, Reason, + Time, Comment, Style); + {died,Reason} when Reason=/=ok -> + %% (If Reason==ok it means that process died in + %% end_per_testcase after successfully completing the + %% test case itself - then we shall not fail, but a + %% warning will be issued in the comment field.) progress(failed, Num, Mod, Func, GrName, Loc, Reason, Time, Comment, Style); {_,{'EXIT',{Skip,Reason}}} when Skip==skip; Skip==skipped; @@ -3943,6 +3965,9 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time, [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName}, {ReportTag,Reason1}}]), + TimeStr = io_lib:format(if is_float(Time) -> "~.3fs"; + true -> "~w" + end, [Time]), ReasonStr = escape_chars(reason_to_string(Reason1)), ReasonStr1 = lists:flatten([string:strip(S,left) || S <- string:tokens(ReasonStr,[$\n])]), @@ -3957,10 +3982,10 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time, _ -> xhtml("<br>(","<br />(") ++ to_string(Comment) ++ ")" end, print(html, - "<td>" ++ St0 ++ "~.3fs" ++ St1 ++ "</td>" + "<td>" ++ St0 ++ "~ts" ++ St1 ++ "</td>" "<td><font color=\"~ts\">SKIPPED</font></td>" "<td>~ts~ts</td></tr>\n", - [Time,Color,ReasonStr2,Comment1]), + [TimeStr,Color,ReasonStr2,Comment1]), FormatLoc = test_server_sup:format_loc(Loc), print(minor, "=== Location: ~ts", [FormatLoc]), print(minor, "=== Reason: ~ts", [ReasonStr1]), @@ -4098,6 +4123,9 @@ progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time, Comment0, {St0,St1}) -> print(minor, "successfully completed test case", []), test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName},ok}]), + TimeStr = io_lib:format(if is_float(Time) -> "~.3fs"; + true -> "~w" + end, [Time]), Comment = case RetVal of {comment,RetComment} -> @@ -4116,10 +4144,10 @@ progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time, end, print(major, "=elapsed ~p", [Time]), print(html, - "<td>" ++ St0 ++ "~.3fs" ++ St1 ++ "</td>" + "<td>" ++ St0 ++ "~ts" ++ St1 ++ "</td>" "<td><font color=\"green\">Ok</font></td>" "~ts</tr>\n", - [Time,Comment]), + [TimeStr,Comment]), print(minor, escape_chars(io_lib:format("=== Returned value: ~tp", [RetVal])), []), diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index fae23484e6..621f3b6d2d 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -1531,17 +1531,17 @@ test_events(config_func_errors) -> {?eh,tc_start,{config_func_error_1_SUITE,exit_in_iptc}}, {?eh,tc_done,{config_func_error_1_SUITE,exit_in_iptc,'_'}}, - {?eh,test_stats,{0,1,{0,0}}}, + {?eh,test_stats,{0,0,{0,1}}}, {?eh,tc_start,{config_func_error_1_SUITE,exit_in_eptc}}, {?eh,tc_done,{config_func_error_1_SUITE,exit_in_eptc,'_'}}, - {?eh,test_stats,{0,2,{0,0}}}, + {?eh,test_stats,{1,0,{0,1}}}, [{?eh,tc_start,{config_func_error_1_SUITE,{init_per_group,g1,[]}}}, {?eh,tc_done,{config_func_error_1_SUITE,{init_per_group,g1,[]},ok}}, {?eh,tc_start,{config_func_error_1_SUITE,exit_in_iptc}}, {?eh,tc_done,{config_func_error_1_SUITE,exit_in_iptc,'_'}}, - {?eh,test_stats,{0,3,{0,0}}}, + {?eh,test_stats,{1,0,{0,2}}}, {?eh,tc_start,{config_func_error_1_SUITE,{end_per_group,g1,[]}}}, {?eh,tc_done,{config_func_error_1_SUITE,{end_per_group,g1,[]},ok}}], @@ -1549,7 +1549,7 @@ test_events(config_func_errors) -> {?eh,tc_done,{config_func_error_1_SUITE,{init_per_group,g2,[]},ok}}, {?eh,tc_start,{config_func_error_1_SUITE,exit_in_eptc}}, {?eh,tc_done,{config_func_error_1_SUITE,exit_in_eptc,'_'}}, - {?eh,test_stats,{0,4,{0,0}}}, + {?eh,test_stats,{2,0,{0,2}}}, {?eh,tc_start,{config_func_error_1_SUITE,{end_per_group,g2,[]}}}, {?eh,tc_done,{config_func_error_1_SUITE,{end_per_group,g2,[]},ok}}], diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl index bc716fb5e3..93bcb8fe52 100644 --- a/lib/common_test/test/ct_hooks_SUITE.erl +++ b/lib/common_test/test/ct_hooks_SUITE.erl @@ -82,10 +82,13 @@ all(suite) -> scope_suite_state_cth, fail_pre_suite_cth, double_fail_pre_suite_cth, fail_post_suite_cth, skip_pre_suite_cth, skip_pre_end_cth, + skip_pre_init_tc_cth, skip_post_suite_cth, recover_post_suite_cth, update_config_cth, state_update_cth, options_cth, same_id_cth, fail_n_skip_with_minimal_cth, prio_cth, no_config, - data_dir, cth_log + no_init_suite_config, no_init_config, no_end_config, + failed_sequence, repeat_force_stop, config_clash, + callbacks_on_skip, fallback, data_dir, cth_log ] ). @@ -190,6 +193,10 @@ skip_post_suite_cth(Config) when is_list(Config) -> do_test(skip_post_suite_cth, "ct_cth_empty_SUITE.erl", [skip_post_suite_cth],Config). +skip_pre_init_tc_cth(Config) -> + do_test(skip_pre_init_tc_cth, "ct_cth_empty_SUITE.erl", + [skip_pre_init_tc_cth],Config). + recover_post_suite_cth(Config) when is_list(Config) -> do_test(recover_post_suite_cth, "ct_cth_fail_per_suite_SUITE.erl", [recover_post_suite_cth],Config). @@ -223,6 +230,16 @@ no_config(Config) when is_list(Config) -> do_test(no_config, "ct_no_config_SUITE.erl", [verify_config_cth],Config). +no_init_suite_config(Config) when is_list(Config) -> + do_test(no_init_suite_config, "ct_no_init_suite_config_SUITE.erl", + [empty_cth],Config). + +no_init_config(Config) when is_list(Config) -> + do_test(no_init_config, "ct_no_init_config_SUITE.erl",[empty_cth],Config). + +no_end_config(Config) when is_list(Config) -> + do_test(no_end_config, "ct_no_end_config_SUITE.erl",[empty_cth],Config). + data_dir(Config) when is_list(Config) -> do_test(data_dir, "ct_data_dir_SUITE.erl", [verify_data_dir_cth],Config). @@ -254,24 +271,53 @@ cth_log(Config) when is_list(Config) -> end, UnexpIoLogs), ok. +%% OTP-10599 adds the Suite argument as first argument to all hook +%% callbacks that did not have a Suite argument from before. This test +%% checks that ct_hooks will fall back to old versions of callbacks if +%% new versions are not exported. +fallback(Config) -> + do_test(fallback, "all_hook_callbacks_SUITE.erl",[fallback_cth], Config). + +%% Test that expected callbacks, and only those, are called when tests +%% are skipped in different ways +callbacks_on_skip(Config) -> + do_test(callbacks_on_skip, {spec,"skip.spec"},[skip_cth], Config). + +%% Test that expected callbacks, and only those, are called when tests +%% are skipped due to failed sequence +failed_sequence(Config) -> + do_test(failed_sequence, "seq_SUITE.erl", [skip_cth], Config). + +%% Test that expected callbacks, and only those, are called when tests +%% are skipped due to {force_stop,skip_rest} option +repeat_force_stop(Config) -> + do_test(repeat_force_stop, "repeat_SUITE.erl", [skip_cth], Config, ok, 2, + [{force_stop,skip_rest},{duration,"000009"}]). + +%% Test that expected callbacks, and only those, are called when a test +%% are fails due to clash in config alias names +config_clash(Config) -> + do_test(config_clash, "config_clash_SUITE.erl", [skip_cth], Config). %%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- -do_test(Tag, SWC, CTHs, Config) -> - do_test(Tag, SWC, CTHs, Config, ok). -do_test(Tag, SWC, CTHs, Config, {error,_} = Res) -> - do_test(Tag, SWC, CTHs, Config, Res, 1); -do_test(Tag, SWC, CTHs, Config, Res) -> - do_test(Tag, SWC, CTHs, Config, Res, 2). +do_test(Tag, WTT, CTHs, Config) -> + do_test(Tag, WTT, CTHs, Config, ok). +do_test(Tag, WTT, CTHs, Config, {error,_} = Res) -> + do_test(Tag, WTT, CTHs, Config, Res, 1,[]); +do_test(Tag, WTT, CTHs, Config, Res) -> + do_test(Tag, WTT, CTHs, Config, Res, 2,[]). -do_test(Tag, SuiteWildCard, CTHs, Config, Res, EC) -> +do_test(Tag, WhatToTest, CTHs, Config, Res, EC, ExtraOpts) when is_list(WhatToTest) -> + do_test(Tag, {suite,WhatToTest}, CTHs, Config, Res, EC, ExtraOpts); +do_test(Tag, {WhatTag,Wildcard}, CTHs, Config, Res, EC, ExtraOpts) -> DataDir = ?config(data_dir, Config), - Suites = filelib:wildcard( - filename:join([DataDir,"cth/tests",SuiteWildCard])), - {Opts,ERPid} = setup([{suite,Suites}, - {ct_hooks,CTHs},{label,Tag}], Config), + Files = filelib:wildcard( + filename:join([DataDir,"cth/tests",Wildcard])), + {Opts,ERPid} = + setup([{WhatTag,Files},{ct_hooks,CTHs},{label,Tag}|ExtraOpts], Config), Res = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), @@ -323,10 +369,10 @@ test_events(one_empty_cth) -> {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, - {?eh,cth,{empty_cth,pre_init_per_testcase,[test_case,'$proplist',[]]}}, - {?eh,cth,{empty_cth,post_init_per_testcase,[test_case,'$proplist','_',[]]}}, - {?eh,cth,{empty_cth,pre_end_per_testcase,[test_case,'$proplist',[]]}}, - {?eh,cth,{empty_cth,post_end_per_testcase,[test_case,'$proplist','_',[]]}}, + {?eh,cth,{empty_cth,pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist','_',[]]}}, + {?eh,cth,{empty_cth,pre_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist','_',[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, @@ -355,10 +401,10 @@ test_events(two_empty_cth) -> {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',ok,[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, @@ -423,8 +469,8 @@ test_events(minimal_and_maximal_cth) -> {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, @@ -512,8 +558,8 @@ test_events(scope_per_suite_cth) -> {?eh,tc_done,{ct_scope_per_suite_cth_SUITE,init_per_suite,ok}}, {?eh,tc_start,{ct_scope_per_suite_cth_SUITE,test_case}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_per_suite_cth_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_scope_per_suite_cth_SUITE,test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_scope_per_suite_cth_SUITE,test_case,ok}}, {?eh,tc_start,{ct_scope_per_suite_cth_SUITE,end_per_suite}}, @@ -538,8 +584,8 @@ test_events(scope_suite_cth) -> {?eh,tc_done,{ct_scope_suite_cth_SUITE,init_per_suite,ok}}, {?eh,tc_start,{ct_scope_suite_cth_SUITE,test_case}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_suite_cth_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_scope_suite_cth_SUITE,test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_scope_suite_cth_SUITE,test_case,ok}}, {?eh,tc_start,{ct_scope_suite_cth_SUITE,end_per_suite}}, @@ -561,17 +607,17 @@ test_events(scope_per_group_cth) -> [{?eh,tc_start,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]}}}, {?eh,cth,{'_',id,[[]]}}, {?eh,cth,{'_',init,['_',[]]}}, - {?eh,cth,{'_',post_init_per_group,[group1,'$proplist','$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_group,[ct_scope_per_group_cth_SUITE,group1, '$proplist','$proplist',[]]}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]},ok}}, {?eh,tc_start,{ct_scope_per_group_cth_SUITE,test_case}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_per_group_cth_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_scope_per_group_cth_SUITE,test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,test_case,ok}}, {?eh,tc_start,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]}}}, - {?eh,cth,{'_',pre_end_per_group,[group1,'$proplist',[]]}}, - {?eh,cth,{'_',post_end_per_group,[group1,'$proplist','_',[]]}}, + {?eh,cth,{'_',pre_end_per_group,[ct_scope_per_group_cth_SUITE,group1,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_group,[ct_scope_per_group_cth_SUITE,group1,'$proplist','_',[]]}}, {?eh,cth,{'_',terminate,[[]]}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]},ok}}], @@ -592,8 +638,8 @@ test_events(scope_per_suite_state_cth) -> {?eh,tc_done,{ct_scope_per_suite_state_cth_SUITE,init_per_suite,ok}}, {?eh,tc_start,{ct_scope_per_suite_state_cth_SUITE,test_case}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_per_suite_state_cth_SUITE,test_case,'$proplist',[test]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_scope_per_suite_state_cth_SUITE,test_case,'$proplist',ok,[test]]}}, {?eh,tc_done,{ct_scope_per_suite_state_cth_SUITE,test_case,ok}}, {?eh,tc_start,{ct_scope_per_suite_state_cth_SUITE,end_per_suite}}, @@ -618,8 +664,8 @@ test_events(scope_suite_state_cth) -> {?eh,tc_done,{ct_scope_suite_state_cth_SUITE,init_per_suite,ok}}, {?eh,tc_start,{ct_scope_suite_state_cth_SUITE,test_case}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_suite_state_cth_SUITE,test_case,'$proplist',[test]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_scope_suite_state_cth_SUITE,test_case,'$proplist',ok,[test]]}}, {?eh,tc_done,{ct_scope_suite_state_cth_SUITE,test_case,ok}}, {?eh,tc_start,{ct_scope_suite_state_cth_SUITE,end_per_suite}}, @@ -641,17 +687,17 @@ test_events(scope_per_group_state_cth) -> [{?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,{init_per_group,group1,[]}}}, {?eh,cth,{'_',id,[[test]]}}, {?eh,cth,{'_',init,['_',[test]]}}, - {?eh,cth,{'_',post_init_per_group,[group1,'$proplist','$proplist',[test]]}}, + {?eh,cth,{'_',post_init_per_group,[ct_scope_per_group_state_cth_SUITE,group1,'$proplist','$proplist',[test]]}}, {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,{init_per_group,group1,[]},ok}}, {?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,test_case}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_per_group_state_cth_SUITE,test_case,'$proplist',[test]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_scope_per_group_state_cth_SUITE,test_case,'$proplist',ok,[test]]}}, {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,test_case,ok}}, {?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,{end_per_group,group1,[]}}}, - {?eh,cth,{'_',pre_end_per_group,[group1,'$proplist',[test]]}}, - {?eh,cth,{'_',post_end_per_group,[group1,'$proplist','_',[test]]}}, + {?eh,cth,{'_',pre_end_per_group,[ct_scope_per_group_state_cth_SUITE,group1,'$proplist',[test]]}}, + {?eh,cth,{'_',post_end_per_group,[ct_scope_per_group_state_cth_SUITE,group1,'$proplist','_',[test]]}}, {?eh,cth,{'_',terminate,[[test]]}}, {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,{end_per_group,group1,[]},ok}}], @@ -674,14 +720,14 @@ test_events(fail_pre_suite_cth) -> {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite, {failed, {error,"Test failure"}}}}, {?eh,cth,{'_',on_tc_fail, - [init_per_suite,{failed,"Test failure"},[]]}}, + [ct_cth_empty_SUITE,init_per_suite,"Test failure",[]]}}, {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case, {failed,{ct_cth_empty_SUITE,init_per_suite, {failed,"Test failure"}}}}}, {?eh,cth,{'_',on_tc_skip, - [test_case, {tc_auto_skip, + [ct_cth_empty_SUITE,test_case, {tc_auto_skip, {failed, {ct_cth_empty_SUITE, init_per_suite, {failed, "Test failure"}}}},[]]}}, @@ -690,7 +736,7 @@ test_events(fail_pre_suite_cth) -> {failed, {ct_cth_empty_SUITE, init_per_suite, {failed, "Test failure"}}}}}, {?eh,cth,{'_',on_tc_skip, - [end_per_suite, {tc_auto_skip, + [ct_cth_empty_SUITE,end_per_suite, {tc_auto_skip, {failed, {ct_cth_empty_SUITE, init_per_suite, {failed, "Test failure"}}}},[]]}}, @@ -727,17 +773,17 @@ test_events(fail_post_suite_cth) -> {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite, {failed,{error,"Test failure"}}}}, - {?eh,cth,{'_',on_tc_fail,[init_per_suite, {failed,"Test failure"}, []]}}, + {?eh,cth,{'_',on_tc_fail,[ct_cth_empty_SUITE,init_per_suite, "Test failure", []]}}, {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case, {failed,{ct_cth_empty_SUITE,init_per_suite, {failed,"Test failure"}}}}}, - {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,'_'},[]]}}, + {?eh,cth,{'_',on_tc_skip,[ct_cth_empty_SUITE,test_case,{tc_auto_skip,'_'},[]]}}, {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite, {failed, {ct_cth_empty_SUITE, init_per_suite, {failed, "Test failure"}}}}}, - {?eh,cth,{'_',on_tc_skip,[end_per_suite,{tc_auto_skip,'_'},[]]}}, + {?eh,cth,{'_',on_tc_skip,[ct_cth_empty_SUITE,end_per_suite,{tc_auto_skip,'_'},[]]}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,cth, {'_',terminate,[[]]}}, @@ -754,10 +800,10 @@ test_events(skip_pre_suite_cth) -> {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist',{skip,"Test skip"},[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,{skipped,"Test skip"}}}, {?eh,cth,{'_',on_tc_skip, - [init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}}, + [ct_cth_empty_SUITE,init_per_suite,{tc_user_skip,"Test skip"},[]]}}, {?eh,tc_user_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, - {?eh,cth,{'_',on_tc_skip,[test_case,{tc_user_skip,"Test skip"},[]]}}, + {?eh,cth,{'_',on_tc_skip,[ct_cth_empty_SUITE,test_case,{tc_user_skip,"Test skip"},[]]}}, {?eh,tc_user_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, @@ -776,27 +822,29 @@ test_events(skip_pre_end_cth) -> [{?eh,tc_start,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]}}}, {?eh,cth,{'_',id,[[]]}}, {?eh,cth,{'_',init,['_',[]]}}, - {?eh,cth,{'_',post_init_per_group,[group1,'$proplist','$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_group,[ct_scope_per_group_cth_SUITE,group1,'$proplist','$proplist',[]]}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]},ok}}, {?eh,tc_start,{ct_scope_per_group_cth_SUITE,test_case}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_per_group_cth_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_scope_per_group_cth_SUITE,test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,test_case,ok}}, {?eh,tc_start,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]}}}, - {?eh,cth,{'_',pre_end_per_group,[group1,'$proplist',[]]}}, - {?eh,cth,{'_',post_end_per_group,[group1,'$proplist','_',[]]}}, + {?eh,cth,{'_',pre_end_per_group,[ct_scope_per_group_cth_SUITE,group1,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_group,[ct_scope_per_group_cth_SUITE,group1,'$proplist','_',[]]}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]}, {skipped,"Test skip"}}}], - {?eh,cth,{'_',on_tc_skip,[{end_per_group,group1}, - {tc_user_skip,{skipped,"Test skip"}}, + {?eh,cth,{'_',on_tc_skip,[ct_scope_per_group_cth_SUITE, + {end_per_group,group1}, + {tc_user_skip,"Test skip"}, []]}}, {?eh,tc_start,{ct_scope_per_group_cth_SUITE,end_per_suite}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,end_per_suite, {skipped,"Test skip"}}}, - {?eh,cth,{'_',on_tc_skip,[end_per_suite, - {tc_user_skip,{skipped,"Test skip"}}, + {?eh,cth,{'_',on_tc_skip,[ct_scope_per_group_cth_SUITE, + end_per_suite, + {tc_user_skip,"Test skip"}, []]}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,cth,{'_',terminate,[[]]}}, @@ -814,10 +862,10 @@ test_events(skip_post_suite_cth) -> {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,{skipped,"Test skip"}}}, {?eh,cth,{'_',on_tc_skip, - [init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}}, + [ct_cth_empty_SUITE,init_per_suite,{tc_user_skip,"Test skip"},[]]}}, {?eh,tc_user_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, - {?eh,cth,{'_',on_tc_skip,[test_case,{tc_user_skip,"Test skip"},[]]}}, + {?eh,cth,{'_',on_tc_skip,[ct_cth_empty_SUITE,test_case,{tc_user_skip,"Test skip"},[]]}}, {?eh,tc_user_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, @@ -826,6 +874,41 @@ test_events(skip_post_suite_cth) -> {?eh,stop_logging,[]} ]; +test_events(skip_pre_init_tc_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,init,['_',[]]}}, + {?eh,start_info,{1,1,1}}, + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [ct_cth_empty_SUITE,test_case,'$proplist', + {skip,"Skipped in pre_init_per_testcase"}, + []]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,test_case, + {skipped,"Skipped in pre_init_per_testcase"}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [ct_cth_empty_SUITE,test_case, + {tc_user_skip,"Skipped in pre_init_per_testcase"}, + []]}}, + {?eh,test_stats,{0,0,{1,0}}}, + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_suite, + [ct_cth_empty_SUITE,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + test_events(recover_post_suite_cth) -> Suite = ct_cth_fail_per_suite_SUITE, [ @@ -840,9 +923,9 @@ test_events(recover_post_suite_cth) -> {?eh,tc_start,{Suite,test_case}}, {?eh,cth,{'_',pre_init_per_testcase, - [test_case, not_contains([tc_status]),[]]}}, + [Suite,test_case, not_contains([tc_status]),[]]}}, {?eh,cth,{'_',post_end_per_testcase, - [test_case, contains([tc_status]),'_',[]]}}, + [Suite,test_case, contains([tc_status]),'_',[]]}}, {?eh,tc_done,{Suite,test_case,ok}}, {?eh,tc_start,{Suite,end_per_suite}}, @@ -876,13 +959,15 @@ test_events(update_config_cth) -> {?eh,tc_start,{ct_update_config_SUITE, {init_per_group,group1,[]}}}, {?eh,cth,{'_',pre_init_per_group, - [group1,contains( + [ct_update_config_SUITE, + group1,contains( [post_init_per_suite, init_per_suite, pre_init_per_suite]), []]}}, {?eh,cth,{'_',post_init_per_group, - [group1, + [ct_update_config_SUITE, + group1, contains( [post_init_per_suite, init_per_suite, @@ -898,7 +983,8 @@ test_events(update_config_cth) -> {?eh,tc_start,{ct_update_config_SUITE,test_case}}, {?eh,cth,{'_',pre_init_per_testcase, - [test_case,contains( + [ct_update_config_SUITE, + test_case,contains( [post_init_per_group, init_per_group, pre_init_per_group, @@ -907,7 +993,8 @@ test_events(update_config_cth) -> pre_init_per_suite]), []]}}, {?eh,cth,{'_',post_end_per_testcase, - [test_case,contains( + [ct_update_config_SUITE, + test_case,contains( [init_per_testcase, pre_init_per_testcase, post_init_per_group, @@ -921,7 +1008,8 @@ test_events(update_config_cth) -> {?eh,tc_start,{ct_update_config_SUITE, {end_per_group,group1,[]}}}, {?eh,cth,{'_',pre_end_per_group, - [group1,contains( + [ct_update_config_SUITE, + group1,contains( [post_init_per_group, init_per_group, pre_init_per_group, @@ -930,7 +1018,8 @@ test_events(update_config_cth) -> pre_init_per_suite]), []]}}, {?eh,cth,{'_',post_end_per_group, - [group1, + [ct_update_config_SUITE, + group1, contains( [pre_end_per_group, post_init_per_group, @@ -1018,8 +1107,8 @@ test_events(options_cth) -> {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, - {?eh,cth,{empty_cth,pre_init_per_testcase,[test_case,'$proplist',[test]]}}, - {?eh,cth,{empty_cth,post_end_per_testcase,[test_case,'$proplist','_',[test]]}}, + {?eh,cth,{empty_cth,pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[test]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist','_',[test]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, @@ -1051,12 +1140,12 @@ test_events(same_id_cth) -> {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}}, {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, {negative, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',ok,[]]}}}, {negative, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}}, {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, @@ -1094,11 +1183,13 @@ test_events(fail_n_skip_with_minimal_cth) -> {?eh,tc_done,{ct_cth_fail_one_skip_one_SUITE,test_case2,{skipped,"skip it"}}}, {?eh,tc_start,{ct_cth_fail_one_skip_one_SUITE,test_case3}}, {?eh,tc_done,{ct_cth_fail_one_skip_one_SUITE,test_case3,{skipped,"skip it"}}}, - {?eh,cth,{empty_cth,on_tc_skip,[{test_case2,group2}, - {tc_user_skip,{skipped,"skip it"}}, + {?eh,cth,{empty_cth,on_tc_skip,[ct_cth_fail_one_skip_one_SUITE, + {test_case2,group2}, + {tc_user_skip,"skip it"}, []]}}, - {?eh,cth,{empty_cth,on_tc_skip,[{test_case3,group2}, - {tc_user_skip,{skipped,"skip it"}}, + {?eh,cth,{empty_cth,on_tc_skip,[ct_cth_fail_one_skip_one_SUITE, + {test_case3,group2}, + {tc_user_skip,"skip it"}, []]}}, {?eh,tc_start,{ct_cth_fail_one_skip_one_SUITE,{end_per_group, group2,[parallel]}}}, @@ -1115,13 +1206,24 @@ test_events(fail_n_skip_with_minimal_cth) -> ]; test_events(prio_cth) -> - GenPre = fun(Func,States) -> - [{?eh,cth,{'_',Func,['_','_',State]}} || State <- States] + GenPre = fun(Func,States) when Func==pre_init_per_suite; + Func==pre_end_per_suite -> + [{?eh,cth,{'_',Func,['_','_',State]}} || + State <- States]; + (Func,States) -> + [{?eh,cth,{'_',Func,['_','_','_',State]}} || + State <- States] end, - GenPost = fun(Func,States) -> - [{?eh,cth,{'_',Func,['_','_','_',State]}} || State <- States] - end, + GenPost = fun(Func,States) when Func==post_init_per_suite; + Func==post_end_per_suite -> + [{?eh,cth,{'_',Func,['_','_','_',State]}} || + State <- States]; + (Func,States) -> + [{?eh,cth,{'_',Func,['_','_','_','_',State]}} || + State <- States] + + end, [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}] ++ @@ -1197,30 +1299,30 @@ test_events(no_config) -> {?eh,tc_done,{ct_framework,init_per_suite,ok}}, {?eh,tc_start,{ct_no_config_SUITE,test_case_1}}, {?eh,cth,{empty_cth,pre_init_per_testcase, - [test_case_1,'$proplist',[]]}}, + [ct_no_config_SUITE,test_case_1,'$proplist',[]]}}, {?eh,cth,{empty_cth,post_end_per_testcase, - [test_case_1,'$proplist',ok,[]]}}, + [ct_no_config_SUITE,test_case_1,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_no_config_SUITE,test_case_1,ok}}, {?eh,test_stats,{1,0,{0,0}}}, [{?eh,tc_start,{ct_framework,{init_per_group,test_group,'$proplist'}}}, {?eh,cth,{empty_cth,pre_init_per_group, - [test_group,'$proplist',[]]}}, + [ct_no_config_SUITE,test_group,'$proplist',[]]}}, {?eh,cth,{empty_cth,post_init_per_group, - [test_group,'$proplist','$proplist',[]]}}, + [ct_no_config_SUITE,test_group,'$proplist','$proplist',[]]}}, {?eh,tc_done,{ct_framework, {init_per_group,test_group,'$proplist'},ok}}, {?eh,tc_start,{ct_no_config_SUITE,test_case_2}}, {?eh,cth,{empty_cth,pre_init_per_testcase, - [test_case_2,'$proplist',[]]}}, + [ct_no_config_SUITE,test_case_2,'$proplist',[]]}}, {?eh,cth,{empty_cth,post_end_per_testcase, - [test_case_2,'$proplist',ok,[]]}}, + [ct_no_config_SUITE,test_case_2,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_no_config_SUITE,test_case_2,ok}}, {?eh,test_stats,{2,0,{0,0}}}, {?eh,tc_start,{ct_framework,{end_per_group,test_group,'$proplist'}}}, {?eh,cth,{empty_cth,pre_end_per_group, - [test_group,'$proplist',[]]}}, + [ct_no_config_SUITE,test_group,'$proplist',[]]}}, {?eh,cth,{empty_cth,post_end_per_group, - [test_group,'$proplist',ok,[]]}}, + [ct_no_config_SUITE,test_group,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_framework,{end_per_group,test_group,'$proplist'},ok}}], {?eh,tc_start,{ct_framework,end_per_suite}}, {?eh,cth,{empty_cth,pre_end_per_suite, @@ -1233,6 +1335,166 @@ test_events(no_config) -> {?eh,stop_logging,[]} ]; +test_events(no_init_suite_config) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,start_info,{1,1,1}}, + {?eh,tc_start,{ct_no_init_suite_config_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite, + [ct_no_init_suite_config_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [ct_no_init_suite_config_SUITE,'$proplist','_',[]]}}, + {?eh,tc_done,{ct_no_init_suite_config_SUITE,init_per_suite, + {failed,{error,{undef,'_'}}}}}, + {?eh,cth,{empty_cth,on_tc_fail,[ct_no_init_suite_config_SUITE, + init_per_suite, + {undef,'_'},[]]}}, + {?eh,tc_auto_skip,{ct_no_init_suite_config_SUITE,test_case, + {failed,{ct_no_init_suite_config_SUITE,init_per_suite, + {'EXIT',{undef,'_'}}}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [ct_no_init_suite_config_SUITE, + test_case, + {tc_auto_skip, + {failed,{ct_no_init_suite_config_SUITE,init_per_suite, + {'EXIT',{undef,'_'}}}}}, + []]}}, + {?eh,test_stats,{0,0,{0,1}}}, + {?eh,tc_auto_skip,{ct_no_init_suite_config_SUITE,end_per_suite, + {failed,{ct_no_init_suite_config_SUITE,init_per_suite, + {'EXIT',{undef,'_'}}}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [ct_no_init_suite_config_SUITE, + end_per_suite, + {tc_auto_skip, + {failed,{ct_no_init_suite_config_SUITE,init_per_suite, + {'EXIT',{undef,'_'}}}}}, + []]}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(no_init_config) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,start_info,{1,1,2}}, + {?eh,tc_start,{ct_no_init_config_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite, + [ct_no_init_config_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [ct_no_init_config_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_no_init_config_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{ct_no_init_config_SUITE,test_case_1}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [ct_no_init_config_SUITE,test_case_1,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [ct_no_init_config_SUITE,test_case_1,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_no_init_config_SUITE,test_case_1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + [{?eh,tc_start,{ct_no_init_config_SUITE,{init_per_group,test_group,[]}}}, + {?eh,cth,{empty_cth,pre_init_per_group, + [ct_no_init_config_SUITE,test_group,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_group, + [ct_no_init_config_SUITE,test_group,'$proplist','_',[]]}}, + {?eh,tc_done,{ct_no_init_config_SUITE,{init_per_group,test_group,[]}, + {failed,{error,{undef,'_'}}}}}, + {?eh,cth,{empty_cth,on_tc_fail,[ct_no_init_config_SUITE, + {init_per_group,test_group}, + {undef,'_'},[]]}}, + {?eh,tc_auto_skip,{ct_no_init_config_SUITE,{test_case_2,test_group}, + {failed,{ct_no_init_config_SUITE,init_per_group, + {'EXIT',{undef,'_'}}}}}}, + {?eh,cth,{empty_cth,on_tc_skip,[ct_no_init_config_SUITE, + {test_case_2,test_group}, + {tc_auto_skip, + {failed, + {ct_no_init_config_SUITE,init_per_group, + {'EXIT',{undef,'_'}}}}}, + []]}}, + {?eh,test_stats,{1,0,{0,1}}}, + {?eh,tc_auto_skip,{ct_no_init_config_SUITE,{end_per_group,test_group}, + {failed,{ct_no_init_config_SUITE,init_per_group, + {'EXIT',{undef,'_'}}}}}}, + {?eh,cth,{empty_cth,on_tc_skip,[ct_no_init_config_SUITE, + {end_per_group,test_group}, + {tc_auto_skip, + {failed, + {ct_no_init_config_SUITE,init_per_group, + {'EXIT',{undef,'_'}}}}}, + []]}}], + {?eh,tc_start,{ct_no_init_config_SUITE,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite, + [ct_no_init_config_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_suite, + [ct_no_init_config_SUITE,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_no_init_config_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(no_end_config) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,start_info,{1,1,2}}, + {?eh,tc_start,{ct_no_end_config_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite, + [ct_no_end_config_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [ct_no_end_config_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_no_end_config_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{ct_no_end_config_SUITE,test_case_1}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [ct_no_end_config_SUITE,test_case_1,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [ct_no_end_config_SUITE,test_case_1,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_no_end_config_SUITE,test_case_1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + [{?eh,tc_start,{ct_no_end_config_SUITE, + {init_per_group,test_group,'$proplist'}}}, + {?eh,cth,{empty_cth,pre_init_per_group, + [ct_no_end_config_SUITE,test_group,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_group, + [ct_no_end_config_SUITE,test_group,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_no_end_config_SUITE, + {init_per_group,test_group,'$proplist'},ok}}, + {?eh,tc_start,{ct_no_end_config_SUITE,test_case_2}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [ct_no_end_config_SUITE,test_case_2,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [ct_no_end_config_SUITE,test_case_2,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_no_end_config_SUITE,test_case_2,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_start,{ct_no_end_config_SUITE, + {end_per_group,test_group,'$proplist'}}}, + {?eh,cth,{empty_cth,pre_end_per_group, + [ct_no_end_config_SUITE,test_group,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_group, + [ct_no_end_config_SUITE,test_group,'$proplist','_',[]]}}, + {?eh,tc_done,{ct_no_end_config_SUITE,{end_per_group,test_group,[]}, + {failed,{error,{undef,'_'}}}}}, + {?eh,cth,{empty_cth,on_tc_fail,[ct_no_end_config_SUITE, + {end_per_group,test_group}, + {undef,'_'},[]]}}], + {?eh,tc_start,{ct_no_end_config_SUITE,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite, + [ct_no_end_config_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_suite, + [ct_no_end_config_SUITE,'$proplist','_',[]]}}, + {?eh,tc_done,{ct_no_end_config_SUITE,end_per_suite, + {failed,{error,{undef,'_'}}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + test_events(data_dir) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, @@ -1247,30 +1509,30 @@ test_events(data_dir) -> {?eh,tc_done,{ct_framework,init_per_suite,ok}}, {?eh,tc_start,{ct_data_dir_SUITE,test_case_1}}, {?eh,cth,{empty_cth,pre_init_per_testcase, - [test_case_1,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, + [ct_data_dir_SUITE,test_case_1,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, {?eh,cth,{empty_cth,post_end_per_testcase, - [test_case_1,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, + [ct_data_dir_SUITE,test_case_1,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, {?eh,tc_done,{ct_data_dir_SUITE,test_case_1,ok}}, {?eh,test_stats,{1,0,{0,0}}}, [{?eh,tc_start,{ct_framework,{init_per_group,test_group,'$proplist'}}}, {?eh,cth,{empty_cth,pre_init_per_group, - [test_group,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, + [ct_data_dir_SUITE,test_group,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, {?eh,cth,{empty_cth,post_init_per_group, - [test_group,'$proplist','$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, + [ct_data_dir_SUITE,test_group,'$proplist','$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, {?eh,tc_done,{ct_framework, {init_per_group,test_group,'$proplist'},ok}}, {?eh,tc_start,{ct_data_dir_SUITE,test_case_2}}, {?eh,cth,{empty_cth,pre_init_per_testcase, - [test_case_2,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, + [ct_data_dir_SUITE,test_case_2,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, {?eh,cth,{empty_cth,post_end_per_testcase, - [test_case_2,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, + [ct_data_dir_SUITE,test_case_2,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, {?eh,tc_done,{ct_data_dir_SUITE,test_case_2,ok}}, {?eh,test_stats,{2,0,{0,0}}}, {?eh,tc_start,{ct_framework,{end_per_group,test_group,'$proplist'}}}, {?eh,cth,{empty_cth,pre_end_per_group, - [test_group,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, + [ct_data_dir_SUITE,test_group,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, {?eh,cth,{empty_cth,post_end_per_group, - [test_group,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, + [ct_data_dir_SUITE,test_group,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, {?eh,tc_done,{ct_framework,{end_per_group,test_group,'$proplist'},ok}}], {?eh,tc_start,{ct_framework,end_per_suite}}, {?eh,cth,{empty_cth,pre_end_per_suite, @@ -1303,6 +1565,645 @@ test_events(cth_log) -> {?eh,stop_logging,[]} ]; +test_events(fallback) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,tc_start,{all_hook_callbacks_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite, + [all_hook_callbacks_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [all_hook_callbacks_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{all_hook_callbacks_SUITE,init_per_suite,ok}}, + + [{?eh,tc_start,{ct_framework,{init_per_group,test_group,'$proplist'}}}, + {?eh,cth,{empty_cth,pre_init_per_group, + [fallback_nosuite,test_group,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_group, + [fallback_nosuite,test_group,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_framework, + {init_per_group,test_group,'$proplist'},ok}}, + {?eh,tc_start,{all_hook_callbacks_SUITE,test_case}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [fallback_nosuite,test_case,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [fallback_nosuite,test_case,'$proplist',ok,[]]}}, + {?eh,tc_done,{all_hook_callbacks_SUITE,test_case,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{ct_framework,{end_per_group,test_group,'$proplist'}}}, + {?eh,cth,{empty_cth,pre_end_per_group, + [fallback_nosuite,test_group,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_group, + [fallback_nosuite,test_group,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_framework,{end_per_group,test_group,'$proplist'},ok}}], + {?eh,tc_start,{all_hook_callbacks_SUITE,test_case}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [fallback_nosuite,test_case,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [fallback_nosuite,test_case,'$proplist','_',[]]}}, + {?eh,cth,{empty_cth,pre_end_per_testcase, + [fallback_nosuite,test_case,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [fallback_nosuite,test_case,'$proplist','_',[]]}}, + {?eh,tc_done,{all_hook_callbacks_SUITE,test_case,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_start,{all_hook_callbacks_SUITE,skip_case}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [fallback_nosuite,skip_case,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [fallback_nosuite,skip_case,'$proplist', + {skip,"Skipped in init_per_testcase/2"},[]]}}, + {?eh,tc_done,{all_hook_callbacks_SUITE,skip_case, + {skipped,"Skipped in init_per_testcase/2"}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [fallback_nosuite,skip_case, + {tc_user_skip,"Skipped in init_per_testcase/2"}, + []]}}, + {?eh,test_stats,{2,0,{1,0}}}, + {?eh,tc_start,{all_hook_callbacks_SUITE,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite, + [all_hook_callbacks_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_suite, + [all_hook_callbacks_SUITE,'$proplist','_',[]]}}, + {?eh,tc_done,{all_hook_callbacks_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(callbacks_on_skip) -> + %% skip_cth.erl will send a 'cth_error' event if a hook is + %% erroneously called. Therefore, all Events are changed to + %% {negative,{?eh,cth_error,'_'},Event} + %% at the end of this function. + Events = + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,start_info,{6,6,15}}, + + %% all_hook_callbacks_SUITE is skipped in spec + %% Only the on_tc_skip callback shall be called + {?eh,tc_user_skip,{all_hook_callbacks_SUITE,all,"Skipped in spec"}}, + {?eh,cth,{empty_cth,on_tc_skip, + [all_hook_callbacks_SUITE,all, + {tc_user_skip,"Skipped in spec"}, + []]}}, + {?eh,test_stats,{0,0,{1,0}}}, + + %% skip_init_SUITE is skipped in its init_per_suite function + %% No group- or testcase-functions shall be called. + {?eh,tc_start,{skip_init_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite, + [skip_init_SUITE, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [skip_init_SUITE, + '$proplist', + {skip,"Skipped in init_per_suite/1"}, + []]}}, + {?eh,tc_done,{skip_init_SUITE,init_per_suite, + {skipped,"Skipped in init_per_suite/1"}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_init_SUITE,init_per_suite, + {tc_user_skip,"Skipped in init_per_suite/1"}, + []]}}, + {?eh,tc_user_skip,{skip_init_SUITE,test_case,"Skipped in init_per_suite/1"}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_init_SUITE,test_case, + {tc_user_skip,"Skipped in init_per_suite/1"}, + []]}}, + {?eh,test_stats,{0,0,{2,0}}}, + {?eh,tc_user_skip,{skip_init_SUITE,end_per_suite, + "Skipped in init_per_suite/1"}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_init_SUITE,end_per_suite, + {tc_user_skip,"Skipped in init_per_suite/1"}, + []]}}, + + %% skip_req_SUITE is auto-skipped since a 'require' statement + %% returned by suite/0 is not fulfilled. + %% No group- or testcase-functions shall be called. + {?eh,tc_start,{skip_req_SUITE,init_per_suite}}, + {?eh,tc_done,{skip_req_SUITE,init_per_suite, + {auto_skipped,{require_failed_in_suite0, + {not_available,whatever}}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_req_SUITE,init_per_suite, + {tc_auto_skip,{require_failed_in_suite0, + {not_available,whatever}}}, + []]}}, + {?eh,tc_auto_skip,{skip_req_SUITE,test_case,{require_failed_in_suite0, + {not_available,whatever}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_req_SUITE,test_case, + {tc_auto_skip,{require_failed_in_suite0, + {not_available,whatever}}}, + []]}}, + {?eh,test_stats,{0,0,{2,1}}}, + {?eh,tc_auto_skip,{skip_req_SUITE,end_per_suite, + {require_failed_in_suite0, + {not_available,whatever}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_req_SUITE,end_per_suite, + {tc_auto_skip,{require_failed_in_suite0, + {not_available,whatever}}}, + []]}}, + + %% skip_fail_SUITE is auto-skipped since the suite/0 function + %% retuns a faluty format. + %% No group- or testcase-functions shall be called. + {?eh,tc_start,{skip_fail_SUITE,init_per_suite}}, + {?eh,tc_done,{skip_fail_SUITE,init_per_suite, + {failed,{error,{suite0_failed,bad_return_value}}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_fail_SUITE,init_per_suite, + {tc_auto_skip, + {failed,{error,{suite0_failed,bad_return_value}}}}, + []]}}, + {?eh,tc_auto_skip,{skip_fail_SUITE,test_case, + {failed,{error,{suite0_failed,bad_return_value}}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_fail_SUITE,test_case, + {tc_auto_skip, + {failed,{error,{suite0_failed,bad_return_value}}}}, + []]}}, + {?eh,test_stats,{0,0,{2,2}}}, + {?eh,tc_auto_skip,{skip_fail_SUITE,end_per_suite, + {failed,{error,{suite0_failed,bad_return_value}}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_fail_SUITE,end_per_suite, + {tc_auto_skip, + {failed,{error,{suite0_failed,bad_return_value}}}}, + []]}}, + + %% skip_group_SUITE + {?eh,tc_start,{skip_group_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite, + [skip_group_SUITE, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [skip_group_SUITE, + '$proplist', + '_', + []]}}, + {?eh,tc_done,{skip_group_SUITE,init_per_suite,ok}}, + + %% test_group_1 - auto_skip due to require failed + [{?eh,tc_start,{skip_group_SUITE,{init_per_group,test_group_1,[]}}}, + {?eh,tc_done, + {skip_group_SUITE,{init_per_group,test_group_1,[]}, + {auto_skipped,{require_failed,{not_available,whatever}}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_group_SUITE, + {init_per_group,test_group_1}, + {tc_auto_skip,{require_failed,{not_available,whatever}}}, + []]}}, + {?eh,tc_auto_skip,{skip_group_SUITE,{test_case,test_group_1}, + {require_failed,{not_available,whatever}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_group_SUITE, + {test_case,test_group_1}, + {tc_auto_skip,{require_failed,{not_available,whatever}}}, + []]}}, + {?eh,test_stats,{0,0,{2,3}}}, + {?eh,tc_auto_skip,{skip_group_SUITE,{end_per_group,test_group_1}, + {require_failed,{not_available,whatever}}}}], + %% The following appears to be outside of the group, but + %% that's only an implementation detail in + %% ct_test_support.erl - it does not know about events from + %% test suite specific hooks and regards the group ended with + %% the above tc_auto_skip-event for end_per_group. + {?eh,cth,{empty_cth,on_tc_skip, + [skip_group_SUITE, + {end_per_group,test_group_1}, + {tc_auto_skip,{require_failed,{not_available,whatever}}}, + []]}}, + + %% test_group_2 - auto_skip due to failed return from group/1 + [{?eh,tc_start,{skip_group_SUITE,{init_per_group,test_group_2,[]}}}, + {?eh,tc_done, + {skip_group_SUITE,{init_per_group,test_group_2,[]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_group_SUITE, + {init_per_group,test_group_2}, + {tc_auto_skip,{group0_failed,bad_return_value}}, + []]}}, + {?eh,tc_auto_skip,{skip_group_SUITE,{test_case,test_group_2}, + {group0_failed,bad_return_value}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_group_SUITE, + {test_case,test_group_2}, + {tc_auto_skip,{group0_failed,bad_return_value}}, + []]}}, + {?eh,test_stats,{0,0,{2,4}}}, + {?eh,tc_auto_skip,{skip_group_SUITE,{end_per_group,test_group_2}, + {group0_failed,bad_return_value}}}], + {?eh,cth,{empty_cth,on_tc_skip, + [skip_group_SUITE, + {end_per_group,test_group_2}, + {tc_auto_skip,{group0_failed,bad_return_value}}, + []]}}, + %% test_group_3 - user_skip in init_per_group/2 + [{?eh,tc_start, + {skip_group_SUITE,{init_per_group,test_group_3,[]}}}, + {?eh,cth,{empty_cth,pre_init_per_group, + [skip_group_SUITE,test_group_3,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_group, + [skip_group_SUITE,test_group_3,'$proplist', + {skip,"Skipped in init_per_group/2"}, + []]}}, + {?eh,tc_done,{skip_group_SUITE, + {init_per_group,test_group_3,[]}, + {skipped,"Skipped in init_per_group/2"}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_group_SUITE, + {init_per_group,test_group_3}, + {tc_user_skip,"Skipped in init_per_group/2"}, + []]}}, + {?eh,tc_user_skip,{skip_group_SUITE, + {test_case,test_group_3}, + "Skipped in init_per_group/2"}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_group_SUITE, + {test_case,test_group_3}, + {tc_user_skip,"Skipped in init_per_group/2"}, + []]}}, + {?eh,test_stats,{0,0,{3,4}}}, + {?eh,tc_user_skip,{skip_group_SUITE, + {end_per_group,test_group_3}, + "Skipped in init_per_group/2"}}], + {?eh,cth,{empty_cth,on_tc_skip, + [skip_group_SUITE, + {end_per_group,test_group_3}, + {tc_user_skip,"Skipped in init_per_group/2"}, + []]}}, + + {?eh,tc_start,{skip_group_SUITE,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite, + [skip_group_SUITE, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_end_per_suite, + [skip_group_SUITE, + '$proplist', + ok,[]]}}, + {?eh,tc_done,{skip_group_SUITE,end_per_suite,ok}}, + + + %% skip_case_SUITE has 4 test cases which are all skipped in + %% different ways + {?eh,tc_start,{skip_case_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite, + [skip_case_SUITE, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [skip_case_SUITE, + '$proplist', + '_', + []]}}, + {?eh,tc_done,{skip_case_SUITE,init_per_suite,ok}}, + + %% Skip in spec -> only on_tc_skip shall be called + {?eh,tc_user_skip,{skip_case_SUITE,skip_in_spec,"Skipped in spec"}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_case_SUITE,skip_in_spec, + {tc_user_skip,"Skipped in spec"}, + []]}}, + {?eh,test_stats,{0,0,{4,4}}}, + + %% Skip in init_per_testcase -> pre/post_end_per_testcase + %% shall not be called + {?eh,tc_start,{skip_case_SUITE,skip_in_init}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [skip_case_SUITE,skip_in_init, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [skip_case_SUITE,skip_in_init, + '$proplist', + {skip,"Skipped in init_per_testcase/2"}, + []]}}, + {?eh,tc_done,{skip_case_SUITE,skip_in_init, + {skipped,"Skipped in init_per_testcase/2"}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_case_SUITE,skip_in_init, + {tc_user_skip,"Skipped in init_per_testcase/2"}, + []]}}, + {?eh,test_stats,{0,0,{5,4}}}, + + %% Fail in init_per_testcase -> pre/post_end_per_testcase + %% shall not be called + {?eh,tc_start,{skip_case_SUITE,fail_in_init}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [skip_case_SUITE,fail_in_init, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [skip_case_SUITE,fail_in_init, + '$proplist', + {skip,{failed,'_'}}, + []]}}, + {?eh,tc_done,{skip_case_SUITE,fail_in_init, + {auto_skipped,{failed,'_'}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_case_SUITE,fail_in_init, + {tc_auto_skip,{failed,'_'}}, + []]}}, + {?eh,test_stats,{0,0,{5,5}}}, + + %% Exit in init_per_testcase -> pre/post_end_per_testcase + %% shall not be called + {?eh,tc_start,{skip_case_SUITE,exit_in_init}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [skip_case_SUITE,exit_in_init, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [skip_case_SUITE,exit_in_init, + '$proplist', + {skip,{failed,'_'}}, + []]}}, + {?eh,tc_done,{skip_case_SUITE,exit_in_init, + {auto_skipped,{failed,'_'}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_case_SUITE,exit_in_init, + {tc_auto_skip,{failed,'_'}}, + []]}}, + {?eh,test_stats,{0,0,{5,6}}}, + + %% Fail in end_per_testcase -> all hooks shall be called and + %% test shall succeed. + {?eh,tc_start,{skip_case_SUITE,fail_in_end}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [skip_case_SUITE,fail_in_end, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [skip_case_SUITE,fail_in_end, + '$proplist', + ok, + []]}}, + {?eh,cth,{empty_cth,pre_end_per_testcase, + [skip_case_SUITE,fail_in_end, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [skip_case_SUITE,fail_in_end, + '$proplist', + {failed, + {skip_case_SUITE,end_per_testcase, + {'EXIT', + {test_case_failed,"Failed in end_per_testcase/2"}}}}, + []]}}, + {?eh,tc_done,{skip_case_SUITE,fail_in_end, + {failed, + {skip_case_SUITE,end_per_testcase, + {'EXIT', + {test_case_failed,"Failed in end_per_testcase/2"}}}}}}, + {?eh,test_stats,{1,0,{5,6}}}, + + %% Exit in end_per_testcase -> all hooks shall be called and + %% test shall succeed. + {?eh,tc_start,{skip_case_SUITE,exit_in_end}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [skip_case_SUITE,exit_in_end, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [skip_case_SUITE,exit_in_end, + '$proplist', + ok, + []]}}, + {?eh,cth,{empty_cth,pre_end_per_testcase, + [skip_case_SUITE,exit_in_end, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [skip_case_SUITE,exit_in_end, + '$proplist', + {failed, + {skip_case_SUITE,end_per_testcase, + {'EXIT',"Exit in end_per_testcase/2"}}}, + []]}}, + {?eh,tc_done,{skip_case_SUITE,exit_in_end, + {failed, + {skip_case_SUITE,end_per_testcase, + {'EXIT',"Exit in end_per_testcase/2"}}}}}, + {?eh,test_stats,{2,0,{5,6}}}, + + %% Skip in testcase function -> all callbacks shall be called + {?eh,tc_start,{skip_case_SUITE,skip_in_case}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [skip_case_SUITE,skip_in_case, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [skip_case_SUITE,skip_in_case, + '$proplist', + ok,[]]}}, + {?eh,cth,{empty_cth,pre_end_per_testcase, + [skip_case_SUITE,skip_in_case, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [skip_case_SUITE,skip_in_case, + '$proplist', + {skip,"Skipped in test case function"}, + []]}}, + {?eh,tc_done,{skip_case_SUITE,skip_in_case, + {skipped,"Skipped in test case function"}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_case_SUITE,skip_in_case, + {tc_user_skip,"Skipped in test case function"}, + []]}}, + {?eh,test_stats,{2,0,{6,6}}}, + + %% Auto skip due to failed 'require' -> only the on_tc_skip + %% callback shall be called + {?eh,tc_start,{skip_case_SUITE,req_auto_skip}}, + {?eh,tc_done,{skip_case_SUITE,req_auto_skip, + {auto_skipped,{require_failed,{not_available,whatever}}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_case_SUITE,req_auto_skip, + {tc_auto_skip,{require_failed,{not_available,whatever}}}, + []]}}, + {?eh,test_stats,{2,0,{6,7}}}, + + %% Auto skip due to failed testcase/0 function -> only the + %% on_tc_skip callback shall be called + {?eh,tc_start,{skip_case_SUITE,fail_auto_skip}}, + {?eh,tc_done,{skip_case_SUITE,fail_auto_skip, + {auto_skipped,{testcase0_failed,bad_return_value}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_case_SUITE,fail_auto_skip, + {tc_auto_skip,{testcase0_failed,bad_return_value}}, + []]}}, + {?eh,test_stats,{2,0,{6,8}}}, + + {?eh,tc_start,{skip_case_SUITE,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite, + [skip_case_SUITE, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_end_per_suite, + [skip_case_SUITE, + '$proplist', + ok,[]]}}, + {?eh,tc_done,{skip_case_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ], + %% Make sure no 'cth_error' events are received! + [{negative,{?eh,cth_error,'_'},E} || E <- Events]; + +test_events(failed_sequence) -> + %% skip_cth.erl will send a 'cth_error' event if a hook is + %% erroneously called. Therefore, all Events are changed to + %% {negative,{?eh,cth_error,'_'},Event} + %% at the end of this function. + Events = + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,start_info,{1,1,2}}, + {?eh,tc_start,{ct_framework,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite,[seq_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [seq_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_framework,init_per_suite,ok}}, + {?eh,tc_start,{seq_SUITE,test_case_1}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [seq_SUITE,test_case_1,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [seq_SUITE,test_case_1,'$proplist',ok,[]]}}, + {?eh,cth,{empty_cth,pre_end_per_testcase, + [seq_SUITE,test_case_1,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [seq_SUITE,test_case_1,'$proplist', + {error,failed_on_purpose},[]]}}, + {?eh,tc_done,{seq_SUITE,test_case_1,{failed,{error,failed_on_purpose}}}}, + {?eh,cth,{empty_cth,on_tc_fail, + [seq_SUITE,test_case_1,failed_on_purpose,[]]}}, + {?eh,test_stats,{0,1,{0,0}}}, + {?eh,tc_start,{seq_SUITE,test_case_2}}, + {?eh,tc_done,{seq_SUITE,test_case_2, + {auto_skipped,{sequence_failed,seq1,test_case_1}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [seq_SUITE,test_case_2, + {tc_auto_skip,{sequence_failed,seq1,test_case_1}}, + []]}}, + {?eh,test_stats,{0,1,{0,1}}}, + {?eh,tc_start,{ct_framework,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite,[seq_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_suite,[seq_SUITE,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_framework,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ], + %% Make sure no 'cth_error' events are received! + [{negative,{?eh,cth_error,'_'},E} || E <- Events]; + +test_events(repeat_force_stop) -> + %% skip_cth.erl will send a 'cth_error' event if a hook is + %% erroneously called. Therefore, all Events are changed to + %% {negative,{?eh,cth_error,'_'},Event} + %% at the end of this function. + Events= + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,start_info,{1,1,2}}, + {?eh,tc_start,{ct_framework,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite,[repeat_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [repeat_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_framework,init_per_suite,ok}}, + {?eh,tc_start,{repeat_SUITE,test_case_1}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [repeat_SUITE,test_case_1,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [repeat_SUITE,test_case_1,'$proplist',ok,[]]}}, + {?eh,cth,{empty_cth,pre_end_per_testcase, + [repeat_SUITE,test_case_1,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [repeat_SUITE,test_case_1,'$proplist',ok,[]]}}, + {?eh,tc_done,{repeat_SUITE,test_case_1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{repeat_SUITE,test_case_2}}, + {?eh,tc_done,{repeat_SUITE,test_case_2, + {auto_skipped, + "Repeated test stopped by force_stop option"}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [repeat_SUITE,test_case_2, + {tc_auto_skip,"Repeated test stopped by force_stop option"}, + []]}}, + {?eh,test_stats,{1,0,{0,1}}}, + {?eh,tc_start,{ct_framework,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite,[repeat_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_suite, + [repeat_SUITE,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_framework,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ], + %% Make sure no 'cth_error' events are received! + [{negative,{?eh,cth_error,'_'},E} || E <- Events]; + +test_events(config_clash) -> + %% skip_cth.erl will send a 'cth_error' event if a hook is + %% erroneously called. Therefore, all Events are changed to + %% {negative,{?eh,cth_error,'_'},Event} + %% at the end of this function. + Events = + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,start_info,{1,1,1}}, + {?eh,tc_start,{ct_framework,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite, + [config_clash_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [config_clash_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_framework,init_per_suite,ok}}, + {?eh,tc_start,{config_clash_SUITE,test_case_1}}, + {?eh,tc_done,{config_clash_SUITE,test_case_1, + {failed,{error,{config_name_already_in_use,[aa]}}}}}, + {?eh,cth,{empty_cth,on_tc_fail, + [config_clash_SUITE,test_case_1, + {config_name_already_in_use,[aa]}, + []]}}, + {?eh,test_stats,{0,1,{0,0}}}, + {?eh,tc_start,{ct_framework,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite, + [config_clash_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_suite, + [config_clash_SUITE,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_framework,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ], + %% Make sure no 'cth_error' events are received! + [{negative,{?eh,cth_error,'_'},E} || E <- Events]; + test_events(ok) -> ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_hook_callbacks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_hook_callbacks_SUITE.erl new file mode 100644 index 0000000000..5b50548694 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_hook_callbacks_SUITE.erl @@ -0,0 +1,62 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(all_hook_callbacks_SUITE). + +-suite_defaults([{timetrap, {minutes, 10}}]). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include("ct.hrl"). + +%% Test server callback functions +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(Config) -> + Config. + +end_per_group(_Config) -> + ok. + +init_per_testcase(skip_case, Config) -> + {skip,"Skipped in init_per_testcase/2"}; +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +all() -> + [{group,test_group},test_case,skip_case]. + +groups() -> + [{test_group,[test_case]}]. + +%% Test cases starts here. +test_case(Config) -> + ok. + +skip_case(Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/config_clash_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/config_clash_SUITE.erl new file mode 100644 index 0000000000..f74c757cc1 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/config_clash_SUITE.erl @@ -0,0 +1,43 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(config_clash_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> + [{require,aa,yy},{default_config,yy,"this is a default value"}]. + +init_per_testcase(_,Config) -> + Config. + +end_per_testcase(_,_) -> + ok. + +all() -> + [test_case_1]. + +%% Test cases starts here. +test_case_1() -> + [{require,aa,xx}]. +test_case_1(_Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_end_config_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_end_config_SUITE.erl new file mode 100644 index 0000000000..7cdaf2024b --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_end_config_SUITE.erl @@ -0,0 +1,51 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ct_no_end_config_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +%%% This suite is used to verify that all pre/post_end_per_* callbacks +%%% are called with correct SuiteName even if no end_per_* config +%%% function exist in the suite, and that the non-exported config +%%% functions fail with 'undef'. + +init_per_suite(Config) -> + Config. + +init_per_group(_Group,Config) -> + Config. + +init_per_testcase(_TC,Config) -> + Config. + +all() -> + [test_case_1, {group,test_group}]. + +groups() -> + [{test_group,[],[test_case_2]}]. + +test_case_1(Config) -> + ok. + +test_case_2(Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_config_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_config_SUITE.erl new file mode 100644 index 0000000000..43c062d66f --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_config_SUITE.erl @@ -0,0 +1,54 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ct_no_init_config_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +%%% This suite is used to verify that all +%%% pre/post_init_per_group/testcase callbacks are called with correct +%%% SuiteName even if no init_per_group/testcase function exist in the +%%% suite, and that the non-exported config functions fail with 'undef'. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + Config. + +end_per_group(_Group,Config) -> + Config. + +end_per_testcase(_TC,Config) -> + Config. + +all() -> + [test_case_1, {group,test_group}]. + +groups() -> + [{test_group,[],[test_case_2]}]. + +test_case_1(Config) -> + ok. + +test_case_2(Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_suite_config_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_suite_config_SUITE.erl new file mode 100644 index 0000000000..85dfe8ca4b --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_suite_config_SUITE.erl @@ -0,0 +1,39 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ct_no_init_suite_config_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +%%% This suite is used to verify that pre/post_init_per_suite +%%% callbacks are called with correct SuiteName even if no +%%% init_per_suite function exist in the suite, and that the +%%% non-exported config function fails with 'undef'. + +end_per_suite(Config) -> + Config. + +all() -> + [test_case]. + +test_case(Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl index c00eb5cf93..37742f0d20 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl @@ -44,18 +44,18 @@ -export([pre_end_per_suite/3]). -export([post_end_per_suite/4]). --export([pre_init_per_group/3]). --export([post_init_per_group/4]). --export([pre_end_per_group/3]). --export([post_end_per_group/4]). +-export([pre_init_per_group/4]). +-export([post_init_per_group/5]). +-export([pre_end_per_group/4]). +-export([post_end_per_group/5]). --export([pre_init_per_testcase/3]). --export([post_init_per_testcase/4]). --export([pre_end_per_testcase/3]). --export([post_end_per_testcase/4]). +-export([pre_init_per_testcase/4]). +-export([post_init_per_testcase/5]). +-export([pre_end_per_testcase/4]). +-export([post_end_per_testcase/5]). --export([on_tc_fail/3]). --export([on_tc_skip/3]). +-export([on_tc_fail/4]). +-export([on_tc_skip/4]). -export([terminate/1]). @@ -154,150 +154,160 @@ post_end_per_suite(Suite,Config,Return,State) -> %% @doc Called before each init_per_group. %% You can change the config in this function. --spec pre_init_per_group(Group :: atom(), - Config :: config(), - State :: #state{}) -> +-spec pre_init_per_group(Suite :: atom(), + Group :: atom(), + Config :: config(), + State :: #state{}) -> {config() | skip_or_fail(), NewState :: #state{}}. -pre_init_per_group(Group,Config,State) -> +pre_init_per_group(Suite,Group,Config,State) -> gen_event:notify( ?CT_EVMGR_REF, #event{ name = cth, node = node(), data = {?MODULE, pre_init_per_group, - [Group,Config,State]}}), - ct:log("~w:pre_init_per_group(~w) called", [?MODULE,Group]), + [Suite,Group,Config,State]}}), + ct:log("~w:pre_init_per_group(~w,~w) called", [?MODULE,Suite,Group]), {Config, State}. %% @doc Called after each init_per_group. %% You can change the return value in this function. --spec post_init_per_group(Group :: atom(), +-spec post_init_per_group(Suite :: atom(), + Group :: atom(), Config :: config(), Return :: config() | skip_or_fail(), State :: #state{}) -> {config() | skip_or_fail(), NewState :: #state{}}. -post_init_per_group(Group,Config,Return,State) -> +post_init_per_group(Suite,Group,Config,Return,State) -> gen_event:notify( ?CT_EVMGR_REF, #event{ name = cth, node = node(), data = {?MODULE, post_init_per_group, - [Group,Config,Return,State]}}), - ct:log("~w:post_init_per_group(~w) called", [?MODULE,Group]), + [Suite,Group,Config,Return,State]}}), + ct:log("~w:post_init_per_group(~w,~w) called", [?MODULE,Suite,Group]), {Return, State}. %% @doc Called after each end_per_group. The config/state can be changed here, %% though it will only affect the *end_per_group functions. --spec pre_end_per_group(Group :: atom(), +-spec pre_end_per_group(Suite :: atom(), + Group :: atom(), Config :: config() | skip_or_fail(), State :: #state{}) -> {ok | skip_or_fail(), NewState :: #state{}}. -pre_end_per_group(Group,Config,State) -> +pre_end_per_group(Suite,Group,Config,State) -> gen_event:notify( ?CT_EVMGR_REF, #event{ name = cth, node = node(), data = {?MODULE, pre_end_per_group, - [Group,Config,State]}}), - ct:log("~w:pre_end_per_group(~w) called", [?MODULE,Group]), + [Suite,Group,Config,State]}}), + ct:log("~w:pre_end_per_group(~w~w) called", [?MODULE,Suite,Group]), {Config, State}. %% @doc Called after each end_per_group. Note that the config cannot be %% changed here, only the status of the group. --spec post_end_per_group(Group :: atom(), +-spec post_end_per_group(Suite :: atom(), + Group :: atom(), Config :: config(), Return :: term(), State :: #state{}) -> {ok | skip_or_fail(), NewState :: #state{}}. -post_end_per_group(Group,Config,Return,State) -> +post_end_per_group(Suite,Group,Config,Return,State) -> gen_event:notify( ?CT_EVMGR_REF, #event{ name = cth, node = node(), data = {?MODULE, post_end_per_group, - [Group,Config,Return,State]}}), - ct:log("~w:post_end_per_group(~w) called", [?MODULE,Group]), + [Suite,Group,Config,Return,State]}}), + ct:log("~w:post_end_per_group(~w,~w) called", [?MODULE,Suite,Group]), {Return, State}. %% @doc Called before init_per_testcase/2 for each test case. %% You can change the config in this function. --spec pre_init_per_testcase(TC :: atom(), - Config :: config(), - State :: #state{}) -> +-spec pre_init_per_testcase(Suite :: atom(), + TC :: atom(), + Config :: config(), + State :: #state{}) -> {config() | skip_or_fail(), NewState :: #state{}}. -pre_init_per_testcase(TC,Config,State) -> +pre_init_per_testcase(Suite,TC,Config,State) -> gen_event:notify( ?CT_EVMGR_REF, #event{ name = cth, node = node(), data = {?MODULE, pre_init_per_testcase, - [TC,Config,State]}}), - ct:log("~w:pre_init_per_testcase(~w) called", [?MODULE,TC]), + [Suite,TC,Config,State]}}), + ct:log("~w:pre_init_per_testcase(~w,~w) called", [?MODULE,Suite,TC]), {Config, State}. %% @doc Called after init_per_testcase/2, and before the test case. --spec post_init_per_testcase(TC :: atom(), +-spec post_init_per_testcase(Suite :: atom(), + TC :: atom(), Config :: config(), Return :: config() | skip_or_fail(), State :: #state{}) -> {config() | skip_or_fail(), NewState :: #state{}}. -post_init_per_testcase(TC,Config,Return,State) -> +post_init_per_testcase(Suite,TC,Config,Return,State) -> gen_event:notify( ?CT_EVMGR_REF, #event{ name = cth, node = node(), data = {?MODULE, post_init_per_testcase, - [TC,Config,Return,State]}}), - ct:log("~w:post_init_per_testcase(~w) called", [?MODULE,TC]), + [Suite,TC,Config,Return,State]}}), + ct:log("~w:post_init_per_testcase(~w,~w) called", [?MODULE,Suite,TC]), {Return, State}. %% @doc Called before end_per_testacse/2. No skip or fail allowed here, %% only config additions. --spec pre_end_per_testcase(TC :: atom(), - Config :: config(), - State :: #state{}) -> +-spec pre_end_per_testcase(Suite :: atom(), + TC :: atom(), + Config :: config(), + State :: #state{}) -> {config(), NewState :: #state{}}. -pre_end_per_testcase(TC,Config,State) -> +pre_end_per_testcase(Suite,TC,Config,State) -> gen_event:notify( ?CT_EVMGR_REF, #event{ name = cth, node = node(), data = {?MODULE, pre_end_per_testcase, - [TC,Config,State]}}), - ct:log("~w:pre_end_per_testcase(~w) called", [?MODULE,TC]), + [Suite,TC,Config,State]}}), + ct:log("~w:pre_end_per_testcase(~w,~w) called", [?MODULE,Suite,TC]), {Config, State}. %% @doc Called after end_per_testcase/2 for each test case. Note that %% the config cannot be changed here, only the status of the test case. --spec post_end_per_testcase(TC :: atom(), +-spec post_end_per_testcase(Suite :: atom(), + TC :: atom(), Config :: config(), Return :: term(), State :: #state{}) -> {ok | skip_or_fail(), NewState :: #state{}}. -post_end_per_testcase(TC,Config,Return,State) -> +post_end_per_testcase(Suite,TC,Config,Return,State) -> gen_event:notify( ?CT_EVMGR_REF, #event{ name = cth, node = node(), data = {?MODULE, post_end_per_testcase, - [TC,Config,Return,State]}}), - ct:log("~w:post_end_per_testcase(~w) called", [?MODULE,TC]), + [Suite,TC,Config,Return,State]}}), + ct:log("~w:post_end_per_testcase(~w,~w) called", [?MODULE,Suite,TC]), {Return, State}. %% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, %% post_end_per_group and post_end_per_tc if the suite, group or test case failed. %% This function should be used for extra cleanup which might be needed. %% It is not possible to modify the config or the status of the test run. --spec on_tc_fail(TC :: init_per_suite | end_per_suite | +-spec on_tc_fail(Suite :: atom(), + TC :: init_per_suite | end_per_suite | init_per_group | end_per_group | atom() | {Function :: atom(), GroupName :: atom()}, Reason :: term(), State :: #state{}) -> NewState :: #state{}. -on_tc_fail(TC, Reason, State) -> +on_tc_fail(Suite, TC, Reason, State) -> gen_event:notify( ?CT_EVMGR_REF, #event{ name = cth, node = node(), data = {?MODULE, on_tc_fail, - [TC,Reason,State]}}), - ct:log("~w:on_tc_fail(~w) called", [?MODULE,TC]), + [Suite,TC,Reason,State]}}), + ct:log("~w:on_tc_fail(~w,~w) called", [?MODULE,Suite,TC]), State. %% @doc Called when a test case is skipped by either user action %% or due to an init function failing. Test case can be %% end_per_suite, init_per_group, end_per_group and the actual test cases. --spec on_tc_skip(TC :: end_per_suite | +-spec on_tc_skip(Suite :: atom(), + TC :: end_per_suite | init_per_group | end_per_group | atom() | {Function :: atom(), GroupName :: atom()}, {tc_auto_skip, {failed, {Mod :: atom(), Function :: atom(), Reason :: term()}}} | {tc_user_skip, {skipped, Reason :: term()}}, State :: #state{}) -> NewState :: #state{}. -on_tc_skip(TC, Reason, State) -> +on_tc_skip(Suite, TC, Reason, State) -> gen_event:notify( ?CT_EVMGR_REF, #event{ name = cth, node = node(), data = {?MODULE, on_tc_skip, - [TC,Reason,State]}}), - ct:log("~w:on_tc_skip(~w) called", [?MODULE,TC]), + [Suite,TC,Reason,State]}}), + ct:log("~w:on_tc_skip(~w,~w) called", [?MODULE,Suite,TC]), State. %% @doc Called when the scope of the CTH is done, this depends on diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl index 559b22bc9f..141b933697 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl @@ -45,29 +45,29 @@ pre_end_per_suite(Suite,Config,State) -> post_end_per_suite(Suite,Config,Return,State) ->
empty_cth:post_end_per_suite(Suite,Config,Return,State).
-pre_init_per_group(Group,Config,State) ->
- empty_cth:pre_init_per_group(Group,Config,State).
+pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) ->
- empty_cth:post_init_per_group(Group,Config,Return,State).
+post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) ->
- empty_cth:pre_end_per_group(Group,Config,State).
+pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) ->
- empty_cth:post_end_per_group(Group,Config,Return,State).
+post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) ->
- empty_cth:pre_init_per_testcase(TC,Config,State).
+pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_end_per_testcase(TC,Config,Return,State).
+post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) ->
- empty_cth:on_tc_fail(TC,Reason,State).
+on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) ->
- empty_cth:on_tc_skip(TC,Reason,State).
+on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) ->
empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl index 51202443bf..07d7c84ed5 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl @@ -45,35 +45,35 @@ pre_end_per_suite(Suite,Config,State) -> post_end_per_suite(Suite,Config,Return,State) -> empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> - empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> - empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> - empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> - empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> - empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> - empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fallback_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fallback_cth.erl new file mode 100644 index 0000000000..59a3d5cbf9 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fallback_cth.erl @@ -0,0 +1,81 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(fallback_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + + +%% CT Hooks +-compile(export_all). + +id(Opts) -> + empty_cth:id(Opts). + +init(Id, Opts) -> + empty_cth:init(Id, Opts). + +pre_init_per_suite(Suite, Config, State) -> + empty_cth:pre_init_per_suite(Suite,Config,State). + +post_init_per_suite(Suite,Config,Return,State) -> + empty_cth:post_init_per_suite(Suite,Config,Return,State). + +pre_end_per_suite(Suite,Config,State) -> + empty_cth:pre_end_per_suite(Suite,Config,State). + +post_end_per_suite(Suite,Config,Return,State) -> + empty_cth:post_end_per_suite(Suite,Config,Return,State). + +pre_init_per_group(Group,Config,State) -> + empty_cth:pre_init_per_group(fallback_nosuite,Group,Config,State). + +post_init_per_group(Group,Config,Return,State) -> + empty_cth:post_init_per_group(fallback_nosuite,Group,Config,Return,State). + +pre_end_per_group(Group,Config,State) -> + empty_cth:pre_end_per_group(fallback_nosuite,Group,Config,State). + +post_end_per_group(Group,Config,Return,State) -> + empty_cth:post_end_per_group(fallback_nosuite,Group,Config,Return,State). + +pre_init_per_testcase(TC,Config,State) -> + empty_cth:pre_init_per_testcase(fallback_nosuite,TC,Config,State). + +post_init_per_testcase(TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(fallback_nosuite,TC,Config,Return,State). + +pre_end_per_testcase(TC,Config,State) -> + empty_cth:pre_end_per_testcase(fallback_nosuite,TC,Config,State). + +post_end_per_testcase(TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(fallback_nosuite,TC,Config,Return,State). + +on_tc_fail(TC, Reason, State) -> + empty_cth:on_tc_fail(fallback_nosuite,TC,Reason,State). + +on_tc_skip(TC, Reason, State) -> + empty_cth:on_tc_skip(fallback_nosuite,TC,Reason,State). + +terminate(State) -> + empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl index b49cbe7fb4..679f076f3a 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl @@ -29,13 +29,13 @@ %% CT Hooks -export([init/2]). -export([terminate/1]). --export([on_tc_skip/3]). +-export([on_tc_skip/4]). init(Id, Opts) -> empty_cth:init(Id, Opts). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite, TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/prio_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/prio_cth.erl index a687743641..95bb76b4c1 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/prio_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/prio_cth.erl @@ -47,35 +47,35 @@ pre_end_per_suite(Suite,Config,State) -> post_end_per_suite(Suite,Config,Return,State) -> empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> - empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> - empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> - empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> - empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> - empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> - empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl index 4d9c60f1ca..3562d39967 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl @@ -47,35 +47,35 @@ pre_end_per_suite(Suite,Config,State) -> post_end_per_suite(Suite,Config,Return,State) -> empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> - empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> - empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> - empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> - empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> - empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> - empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/typer/src/typer.appup.src b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/repeat_SUITE.erl index 3b7464a97c..fded4c02ab 100644 --- a/lib/typer/src/typer.appup.src +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/repeat_SUITE.erl @@ -1,7 +1,7 @@ -%% -*- erlang -*- +%% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2014-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -16,7 +16,27 @@ %% limitations under the License. %% %% %CopyrightEnd% -{"%VSN%", - [{<<".*">>,[{restart_application, typer}]}], - [{<<".*">>,[{restart_application, typer}]}] -}. +%% + +-module(repeat_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +init_per_testcase(_,Config) -> + Config. + +end_per_testcase(_,_) -> + ok. + +all() -> + [test_case_1, test_case_2]. + +%% Test cases starts here. +test_case_1(_Config) -> + timer:sleep(10000), + ok. + +test_case_2(_Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl index 494f398fc1..b9d9d4cec1 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl @@ -48,35 +48,35 @@ pre_end_per_suite(Suite,Config,State) -> post_end_per_suite(Suite,Config,Return,State) -> empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> - empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> - empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> - empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> - empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> - empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> - empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/seq_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/seq_SUITE.erl new file mode 100644 index 0000000000..6d1302fd35 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/seq_SUITE.erl @@ -0,0 +1,45 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(seq_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +init_per_testcase(_,Config) -> + Config. + +end_per_testcase(_,_) -> + ok. + +all() -> + [{sequence,seq1}]. + +sequences() -> + [{seq1,[test_case_1,test_case_2]}]. + +%% Test cases starts here. +test_case_1(_Config) -> + exit(failed_on_purpose). + +test_case_2(_Config) -> + ct:fail("This test shall never be run since test_case_1 fails " + "and they are run in sequence"). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip.spec b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip.spec new file mode 100644 index 0000000000..a271c5e8b2 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip.spec @@ -0,0 +1,8 @@ +{suites,".",[all_hook_callbacks_SUITE, + skip_init_SUITE, + skip_req_SUITE, + skip_fail_SUITE, + skip_group_SUITE, + skip_case_SUITE]}. +{skip_suites,".",all_hook_callbacks_SUITE,"Skipped in spec"}. +{skip_cases,".",skip_case_SUITE,skip_in_spec,"Skipped in spec"}. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl new file mode 100644 index 0000000000..dad80ae914 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl @@ -0,0 +1,106 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(skip_case_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + ok. + +init_per_group(_,Config) -> + Config. + +end_per_group(_,_) -> + ok. + +init_per_testcase(skip_in_init,Config) -> + {skip,"Skipped in init_per_testcase/2"}; +init_per_testcase(fail_in_init,Config) -> + ct:fail("Failed in init_per_testcase/2"); +init_per_testcase(exit_in_init,Config) -> + exit(self(),"Exit in init_per_testcase/2"); +init_per_testcase(_,Config) -> + Config. + +end_per_testcase(fail_in_end,_) -> + ct:fail("Failed in end_per_testcase/2"); +end_per_testcase(exit_in_end,_) -> + exit(self(),"Exit in end_per_testcase/2"); +end_per_testcase(_,_) -> + ok. + +all() -> + [skip_in_spec, + skip_in_init, + fail_in_init, + exit_in_init, + fail_in_end, + exit_in_end, + skip_in_case, + req_auto_skip, + fail_auto_skip + ]. + +%% Test cases starts here. +skip_in_spec(Config) -> + ct:fail("This test shall never be run. " + "It shall be skipped in the test spec."). + +skip_in_init(Config) -> + ct:fail("This test shall never be run. " + "It shall be skipped in init_per_testcase/2."). + +fail_in_init(Config) -> + ct:fail("This test shall never be run. " + "It shall fail in init_per_testcase/2."). + +exit_in_init(Config) -> + ct:fail("This test shall never be run. " + "It shall exit in init_per_testcase/2."). + +fail_in_end(Config) -> + ok. + +exit_in_end(Config) -> + ok. + +skip_in_case(Config) -> + {skip,"Skipped in test case function"}. + +req_auto_skip() -> + [{require,whatever}]. +req_auto_skip(Config) -> + ct:fail("This test shall never be run due to " + "failed require"). + +fail_auto_skip() -> + faulty_return_value. +fail_auto_skip(Config) -> + ct:fail("This test shall never be run due to " + "faulty return from info function"). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl new file mode 100644 index 0000000000..16f015fe7a --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl @@ -0,0 +1,182 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(skip_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +%% Send a cth_error event if a callback is called with unexpected arguments +-define(fail(Info), + gen_event:notify( + ?CT_EVMGR_REF, + #event{ name = cth_error, + node = node(), + data = {illegal_hook_callback,{?MODULE,?FUNCTION_NAME,Info}}})). + +%% CT Hooks +-compile(export_all). + +id(Opts) -> + empty_cth:id(Opts). + +init(Id, Opts) -> + empty_cth:init(Id, Opts). + +pre_init_per_suite(Suite, Config, State) -> + Suite==skip_init_SUITE + orelse Suite==skip_group_SUITE + orelse Suite==skip_case_SUITE + orelse Suite==seq_SUITE + orelse Suite==repeat_SUITE + orelse Suite==config_clash_SUITE + orelse ?fail(Suite), + empty_cth:pre_init_per_suite(Suite,Config,State). + +post_init_per_suite(Suite,Config,Return,State) -> + Suite==skip_init_SUITE + orelse Suite==skip_group_SUITE + orelse Suite==skip_case_SUITE + orelse Suite==seq_SUITE + orelse Suite==repeat_SUITE + orelse Suite==config_clash_SUITE + orelse ?fail(Suite), + empty_cth:post_init_per_suite(Suite,Config,Return,State). + +pre_end_per_suite(Suite,Config,State) -> + Suite==skip_case_SUITE + orelse Suite==skip_group_SUITE + orelse Suite==seq_SUITE + orelse Suite==repeat_SUITE + orelse Suite==config_clash_SUITE + orelse ?fail(Suite), + empty_cth:pre_end_per_suite(Suite,Config,State). + +post_end_per_suite(Suite,Config,Return,State) -> + Suite==skip_case_SUITE + orelse Suite==skip_group_SUITE + orelse Suite==seq_SUITE + orelse Suite==repeat_SUITE + orelse Suite==config_clash_SUITE + orelse ?fail(Suite), + empty_cth:post_end_per_suite(Suite,Config,Return,State). + +pre_init_per_group(Suite,Group,Config,State) -> + (Suite==skip_group_SUITE andalso Group==test_group_3) + orelse ?fail({Suite,Group}), + empty_cth:pre_init_per_group(Suite,Group,Config,State). + +post_init_per_group(Suite,Group,Config,Return,State) -> + (Suite==skip_group_SUITE andalso Group==test_group_3) + orelse ?fail({Suite,Group}), + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). + +pre_end_per_group(Suite,Group,Config,State) -> + ?fail({Suite,Group}), + empty_cth:pre_end_per_group(Suite,Group,Config,State). + +post_end_per_group(Suite,Group,Config,Return,State) -> + ?fail({Suite,Group}), + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). + +pre_init_per_testcase(Suite,TC,Config,State) -> + (Suite==skip_case_SUITE andalso (TC==skip_in_init + orelse TC==fail_in_init + orelse TC==exit_in_init + orelse TC==fail_in_end + orelse TC==exit_in_end + orelse TC==skip_in_case)) + orelse (Suite==seq_SUITE andalso TC==test_case_1) + orelse (Suite==repeat_SUITE andalso TC==test_case_1) + orelse ?fail({Suite,TC}), + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). + +post_init_per_testcase(Suite,TC,Config,Return,State) -> + (Suite==skip_case_SUITE andalso (TC==skip_in_init + orelse TC==fail_in_init + orelse TC==exit_in_init + orelse TC==fail_in_end + orelse TC==exit_in_end + orelse TC==skip_in_case)) + orelse (Suite==seq_SUITE andalso TC==test_case_1) + orelse (Suite==repeat_SUITE andalso TC==test_case_1) + orelse ?fail({Suite,TC}), + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). + +pre_end_per_testcase(Suite,TC,Config,State) -> + (Suite==skip_case_SUITE andalso (TC==skip_in_case + orelse TC==fail_in_end + orelse TC==exit_in_end)) + orelse (Suite==seq_SUITE andalso TC==test_case_1) + orelse (Suite==repeat_SUITE andalso TC==test_case_1) + orelse ?fail({Suite,TC}), + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). + +post_end_per_testcase(Suite,TC,Config,Return,State) -> + (Suite==skip_case_SUITE andalso (TC==skip_in_case + orelse TC==fail_in_end + orelse TC==exit_in_end)) + orelse (Suite==seq_SUITE andalso TC==test_case_1) + orelse (Suite==repeat_SUITE andalso TC==test_case_1) + orelse ?fail({Suite,TC}), + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). + +on_tc_fail(Suite,TC,Reason,State) -> + (Suite==seq_SUITE andalso TC==test_case_1) + orelse (Suite==config_clash_SUITE andalso TC==test_case_1) + orelse ?fail({Suite,TC}), + empty_cth:on_tc_fail(Suite,TC,Reason,State). + +on_tc_skip(all_hook_callbacks_SUITE=Suite,all=TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State); +on_tc_skip(Suite,TC,Reason,State) + when (Suite==skip_init_SUITE + orelse Suite==skip_req_SUITE + orelse Suite==skip_fail_SUITE) + andalso + (TC==init_per_suite + orelse TC==test_case + orelse TC==end_per_suite) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State); +on_tc_skip(skip_group_SUITE=Suite,TC={C,G},Reason,State) + when (C==init_per_group orelse C==test_case orelse C==end_per_group) andalso + (G==test_group_1 orelse G==test_group_2 orelse G==test_group_3) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State); +on_tc_skip(skip_case_SUITE=Suite,TC,Reason,State) + when TC==skip_in_spec; + TC==skip_in_init; + TC==fail_in_init; + TC==exit_in_init; + TC==skip_in_case; + TC==req_auto_skip; + TC==fail_auto_skip -> + empty_cth:on_tc_skip(Suite,TC,Reason,State); +on_tc_skip(Suite,TC,Reason,State) + when (Suite==seq_SUITE andalso TC==test_case_2) + orelse (Suite==repeat_SUITE andalso TC==test_case_2) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State); +on_tc_skip(Suite,TC,Reason,State) -> + ?fail({Suite,TC}), + empty_cth:on_tc_skip(Suite,TC,Reason,State). + +terminate(State) -> + empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_fail_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_fail_SUITE.erl new file mode 100644 index 0000000000..9f5dfee6b9 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_fail_SUITE.erl @@ -0,0 +1,53 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(skip_fail_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> + faulty_return_value. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + ok. + +init_per_group(_,Config) -> + Config. + +end_per_group(_,_) -> + ok. + +init_per_testcase(_,Config) -> + Config. + +end_per_testcase(_,_) -> + ok. + +all() -> + [test_case]. + +%% Test cases starts here. +test_case(Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_group_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_group_SUITE.erl new file mode 100644 index 0000000000..d3b848bfbd --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_group_SUITE.erl @@ -0,0 +1,64 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(skip_group_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + ok. + +group(test_group_1) -> + [{require,whatever}]; +group(test_group_2) -> + faulty_return_value; +group(_) -> + []. + +init_per_group(test_group_3,Config) -> + {skip,"Skipped in init_per_group/2"}; +init_per_group(_,Config) -> + ct:fail("This shall never be run due to auto_skip from group/1"). + +end_per_group(_,_) -> + ct:fail("This shall never be run"). + +all() -> + [{group,test_group_1}, + {group,test_group_2}, + {group,test_group_3}]. + +groups() -> + [{test_group_1,[test_case]}, + {test_group_2,[test_case]}, + {test_group_3,[test_case]}]. + +%% Test cases starts here. +test_case(_Config) -> + ct:fail("This test case shall never be run due to skip on group level"). + diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_init_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_init_SUITE.erl new file mode 100644 index 0000000000..70305421ac --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_init_SUITE.erl @@ -0,0 +1,53 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(skip_init_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> + []. + +init_per_suite(Config) -> + {skip,"Skipped in init_per_suite/1"}. + +end_per_suite(Config) -> + ok. + +init_per_group(_,Config) -> + Config. + +end_per_group(_,_) -> + ok. + +init_per_testcase(_,Config) -> + Config. + +end_per_testcase(_,_) -> + ok. + +all() -> + [test_case]. + +%% Test cases starts here. +test_case(Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl index d5b347e723..48a2d70e22 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl @@ -45,35 +45,35 @@ pre_end_per_suite(Suite,Config,State) -> post_end_per_suite(Suite,Config,Return,State) -> empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> - empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> - empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> - empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> - empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> - empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> - empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_end_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_end_cth.erl index 36abac0bf8..d638954d3c 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_end_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_end_cth.erl @@ -46,36 +46,36 @@ pre_end_per_suite(Suite,Config,State) -> post_end_per_suite(Suite,Config,Return,State) -> empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> - empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> - empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> - empty_cth:pre_end_per_group(Group,Config,State), +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State), {{skip, "Test skip"}, State}. -post_end_per_group(Group,Config,Return,State) -> - empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> - empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> - empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_init_tc_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_init_tc_cth.erl new file mode 100644 index 0000000000..e1d261d59a --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_init_tc_cth.erl @@ -0,0 +1,79 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(skip_pre_init_tc_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + + +%% CT Hooks +-compile(export_all). + +init(Id, Opts) -> + empty_cth:init(Id, Opts). + +pre_init_per_suite(Suite, Config, State) -> + empty_cth:pre_init_per_suite(Suite,Config,State). + +post_init_per_suite(Suite,Config,Return,State) -> + empty_cth:post_init_per_suite(Suite,Config,Return,State). + +pre_end_per_suite(Suite,Config,State) -> + empty_cth:pre_end_per_suite(Suite,Config,State). + +post_end_per_suite(Suite,Config,Return,State) -> + empty_cth:post_end_per_suite(Suite,Config,Return,State). + +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). + +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). + +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). + +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). + +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State), + {{skip, "Skipped in pre_init_per_testcase"}, State}. + +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). + +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). + +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). + +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). + +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). + +terminate(State) -> + empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl index fa510b2d54..d7b07ee33c 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl @@ -46,35 +46,35 @@ pre_end_per_suite(Suite,Config,State) -> post_end_per_suite(Suite,Config,Return,State) -> empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> - empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> - empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> - empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> - empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> - empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> - empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_req_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_req_SUITE.erl new file mode 100644 index 0000000000..bc69dd5ea4 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_req_SUITE.erl @@ -0,0 +1,53 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(skip_req_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> + [{require,whatever}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + ok. + +init_per_group(_,Config) -> + Config. + +end_per_group(_,_) -> + ok. + +init_per_testcase(_,Config) -> + Config. + +end_per_testcase(_,_) -> + ok. + +all() -> + [test_case]. + +%% Test cases starts here. +test_case(Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl index 7ec0d458b6..c6e0419c50 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl @@ -48,44 +48,44 @@ post_end_per_suite(Suite,Config,Return,State) -> empty_cth:post_end_per_suite(Suite,Config,Return,State), {Return, [post_end_per_suite|State]}. -pre_init_per_group(Group,Config,State) -> - empty_cth:pre_init_per_group(Group,Config,State), +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State), {Config, [pre_init_per_group|State]}. -post_init_per_group(Group,Config,Return,State) -> - empty_cth:post_init_per_group(Group,Config,Return,State), +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State), {Return, [post_init_per_group|State]}. -pre_end_per_group(Group,Config,State) -> - empty_cth:pre_end_per_group(Group,Config,State), +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State), {Config, [pre_end_per_group|State]}. -post_end_per_group(Group,Config,Return,State) -> - empty_cth:post_end_per_group(Group,Config,Return,State), +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State), {Return, [post_end_per_group|State]}. -pre_init_per_testcase(TC,Config,State) -> - empty_cth:pre_init_per_testcase(TC,Config,State), +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State), {Config, [pre_init_per_testcase|State]}. -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State), +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State), {Return, [post_init_per_testcase|State]}. -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State), +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State), {Config, [pre_end_per_testcase|State]}. -post_end_per_testcase(TC,Config,Return,State) -> - empty_cth:post_end_per_testcase(TC,Config,Return,State), +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State), {Return, [post_end_per_testcase|State]}. -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State), +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State), [on_tc_fail|State]. -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State), +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State), [on_tc_skip|State]. terminate(State) -> diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl index 2b9e726819..10a7047899 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl @@ -44,35 +44,35 @@ pre_end_per_suite(Suite,Config,State) -> post_end_per_suite(Suite,Config,Return,State) -> empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> - empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> - empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> - empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> - empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> - empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> - empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl index d48981f667..f933c7702e 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl @@ -50,43 +50,43 @@ post_end_per_suite(Suite,Config,Return,State) -> NewConfig = [{post_end_per_suite,?now}|Config], {NewConfig,NewConfig}. -pre_init_per_group(Group,Config,State) -> - empty_cth:pre_init_per_group(Group,Config,State), +pre_init_per_group(Suite, Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State), {[{pre_init_per_group,?now}|Config],State}. -post_init_per_group(Group,Config,Return,State) -> - empty_cth:post_init_per_group(Group,Config,Return,State), +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State), {[{post_init_per_group,?now}|Return],State}. -pre_end_per_group(Group,Config,State) -> - empty_cth:pre_end_per_group(Group,Config,State), +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State), {[{pre_end_per_group,?now}|Config],State}. -post_end_per_group(Group,Config,Return,State) -> - empty_cth:post_end_per_group(Group,Config,Return,State), +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State), {[{post_end_per_group,?now}|Config],State}. -pre_init_per_testcase(TC,Config,State) -> - empty_cth:pre_init_per_testcase(TC,Config,State), +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State), {[{pre_init_per_testcase,?now}|Config],State}. -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State), +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State), {[{post_init_per_testcase,?now}|Config],State}. -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State), +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State), {[{pre_end_per_testcase,?now}|Config],State}. -post_end_per_testcase(TC,Config,Return,State) -> - empty_cth:post_end_per_testcase(TC,Config,Return,State), +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State), {[{post_end_per_testcase,?now}|Config],State}. -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_config_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_config_cth.erl index 71d84781e0..b29256a77e 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_config_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_config_cth.erl @@ -60,37 +60,37 @@ post_end_per_suite(Suite,Config,Return,State) -> ct_no_config_SUITE = ct:get_config(suite_cfg), empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> +pre_init_per_group(Suite,Group,Config,State) -> true = ?val(post_init_per_suite, Config), ct_no_config_SUITE = ct:get_config(suite_cfg), test_group = ct:get_config(group_cfg), - empty_cth:pre_init_per_group(Group, + empty_cth:pre_init_per_group(Suite,Group, [{pre_init_per_group,true} | Config], State). -post_init_per_group(Group,Config,Return,State) -> +post_init_per_group(Suite,Group,Config,Return,State) -> true = ?val(pre_init_per_group, Return), test_group = ct:get_config(group_cfg), - empty_cth:post_init_per_group(Group, + empty_cth:post_init_per_group(Suite,Group, Config, [{post_init_per_group,true} | Return], State). -pre_end_per_group(Group,Config,State) -> +pre_end_per_group(Suite,Group,Config,State) -> true = ?val(post_init_per_group, Config), ct_no_config_SUITE = ct:get_config(suite_cfg), test_group = ct:get_config(group_cfg), - empty_cth:pre_end_per_group(Group, + empty_cth:pre_end_per_group(Suite,Group, [{pre_end_per_group,true} | Config], State). -post_end_per_group(Group,Config,Return,State) -> +post_end_per_group(Suite,Group,Config,Return,State) -> true = ?val(pre_end_per_group, Config), ct_no_config_SUITE = ct:get_config(suite_cfg), test_group = ct:get_config(group_cfg), - empty_cth:post_end_per_group(Group,Config,Return,State). + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> +pre_init_per_testcase(Suite,TC,Config,State) -> true = ?val(post_init_per_suite, Config), case ?val(name, ?val(tc_group_properties, Config)) of undefined -> @@ -102,19 +102,19 @@ pre_init_per_testcase(TC,Config,State) -> ct_no_config_SUITE = ct:get_config(suite_cfg), CfgKey = list_to_atom(atom_to_list(TC) ++ "_cfg"), TC = ct:get_config(CfgKey), - empty_cth:pre_init_per_testcase(TC, + empty_cth:pre_init_per_testcase(Suite,TC, [{pre_init_per_testcase,true} | Config], State). %%! TODO: Verify Config also in post_init and pre_end! -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> +post_end_per_testcase(Suite,TC,Config,Return,State) -> true = ?val(post_init_per_suite, Config), true = ?val(pre_init_per_testcase, Config), case ?val(name, ?val(tc_group_properties, Config)) of @@ -127,13 +127,13 @@ post_end_per_testcase(TC,Config,Return,State) -> ct_no_config_SUITE = ct:get_config(suite_cfg), CfgKey = list_to_atom(atom_to_list(TC) ++ "_cfg"), TC = ct:get_config(CfgKey), - empty_cth:post_end_per_testcase(TC,Config,Return,State). + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_data_dir_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_data_dir_cth.erl index 9abd2e5e83..42e086b96e 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_data_dir_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_data_dir_cth.erl @@ -62,43 +62,43 @@ post_end_per_suite(Suite,Config,Return,State) -> check_dirs(State,Config), empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> +pre_init_per_group(Suite,Group,Config,State) -> check_dirs(State,Config), - empty_cth:pre_init_per_group(Group,Config,State). + empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> +post_init_per_group(Suite,Group,Config,Return,State) -> check_dirs(State,Return), - empty_cth:post_init_per_group(Group,Config,Return,State). + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> +pre_end_per_group(Suite,Group,Config,State) -> check_dirs(State,Config), - empty_cth:pre_end_per_group(Group,Config,State). + empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> +post_end_per_group(Suite,Group,Config,Return,State) -> check_dirs(State,Config), - empty_cth:post_end_per_group(Group,Config,Return,State). + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> +pre_init_per_testcase(Suite,TC,Config,State) -> check_dirs(State,Config), - empty_cth:pre_init_per_testcase(TC,Config,State). + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> +post_init_per_testcase(Suite,TC,Config,Return,State) -> check_dirs(State,Config), - empty_cth:post_init_per_testcase(TC,Config,Return,State). + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> +pre_end_per_testcase(Suite,TC,Config,State) -> check_dirs(State,Config), - empty_cth:pre_end_per_testcase(TC,Config,State). + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> +post_end_per_testcase(Suite,TC,Config,Return,State) -> check_dirs(State,Config), - empty_cth:post_end_per_testcase(TC,Config,Return,State). + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_repeat_testrun_SUITE.erl b/lib/common_test/test/ct_repeat_testrun_SUITE.erl index f8b6a379f6..76611a2db3 100644 --- a/lib/common_test/test/ct_repeat_testrun_SUITE.erl +++ b/lib/common_test/test/ct_repeat_testrun_SUITE.erl @@ -363,14 +363,17 @@ skip_first_tc1(Suite) -> {?eh,tc_start,{Suite,tc1}}, {?eh,tc_done,{Suite,tc1,ok}}, {?eh,test_stats,{'_',0,{0,0}}}, + {?eh,tc_start,{Suite,tc2}}, {?eh,tc_done,{Suite,tc2,?skipped}}, {?eh,test_stats,{'_',0,{0,1}}}, + {?eh,tc_start,{Suite,{init_per_group,g,[]}}}, {?eh,tc_done,{Suite,{init_per_group,g,[]},?skipped}}, {?eh,tc_auto_skip,{Suite,{tc1,g},?skip_reason}}, {?eh,test_stats,{'_',0,{0,2}}}, {?eh,tc_auto_skip,{Suite,{tc2,g},?skip_reason}}, {?eh,test_stats,{'_',0,{0,3}}}, {?eh,tc_auto_skip,{Suite,{end_per_group,g},?skip_reason}}, + {?eh,tc_start,{Suite,tc2}}, {?eh,tc_done,{Suite,tc2,?skipped}}, {?eh,test_stats,{'_',0,{0,4}}}, {?eh,tc_start,{Suite,end_per_suite}}, @@ -390,10 +393,12 @@ skip_tc1_in_group(Suite) -> {?eh,tc_start,{Suite,tc1}}, {?eh,tc_done,{Suite,tc1,ok}}, {?eh,test_stats,{'_',0,{0,0}}}, + {?eh,tc_start,{Suite,tc2}}, {?eh,tc_done,{Suite,tc2,?skipped}}, {?eh,test_stats,{'_',0,{0,1}}}, {?eh,tc_start,{Suite,{end_per_group,g,[]}}}, {?eh,tc_done,{Suite,{end_per_group,g,[]},ok}}], + {?eh,tc_start,{Suite,tc2}}, {?eh,tc_done,{Suite,tc2,?skipped}}, {?eh,test_stats,{'_',0,{0,2}}}, {?eh,tc_start,{Suite,end_per_suite}}, diff --git a/lib/common_test/test/ct_surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE.erl index 42ec685c16..884217afc2 100644 --- a/lib/common_test/test/ct_surefire_SUITE.erl +++ b/lib/common_test/test/ct_surefire_SUITE.erl @@ -73,7 +73,9 @@ all() -> relative_path, url, logdir, - fail_pre_init_per_suite + fail_pre_init_per_suite, + skip_case_in_spec, + skip_suite_in_spec ]. %%-------------------------------------------------------------------- @@ -119,6 +121,18 @@ fail_pre_init_per_suite(Config) when is_list(Config) -> run(fail_pre_init_per_suite,[fail_pre_init_per_suite, {cth_surefire,[{path,Path}]}],Path,Config,[],Suites). +skip_case_in_spec(Config) -> + DataDir = ?config(data_dir,Config), + Spec = filename:join(DataDir,"skip_one_case.spec"), + Path = "skip_case_in_spec.xml", + run_spec(skip_case_in_spec,[{cth_surefire,[{path,Path}]}],Path,Config,Spec). + +skip_suite_in_spec(Config) -> + DataDir = ?config(data_dir,Config), + Spec = filename:join(DataDir,"skip_one_suite.spec"), + Path = "skip_suite_in_spec.xml", + run_spec(skip_suite_in_spec,[{cth_surefire,[{path,Path}]}],Path,Config,Spec). + %%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- @@ -129,8 +143,15 @@ run(Case,CTHs,Report,Config,ExtraOpts) -> Suite = filename:join(DataDir, "surefire_SUITE"), run(Case,CTHs,Report,Config,ExtraOpts,Suite). run(Case,CTHs,Report,Config,ExtraOpts,Suite) -> - {Opts,ERPid} = setup([{suite,Suite},{ct_hooks,CTHs},{label,Case}|ExtraOpts], - Config), + Test = [{suite,Suite},{ct_hooks,CTHs},{label,Case}|ExtraOpts], + do_run(Case, Report, Test, Config). + +run_spec(Case,CTHs,Report,Config,Spec) -> + Test = [{spec,Spec},{ct_hooks,CTHs},{label,Case}], + do_run(Case, Report, Test, Config). + +do_run(Case, Report, Test, Config) -> + {Opts,ERPid} = setup(Test, Config), ok = execute(Case, Opts, ERPid, Config), LogDir = case lists:keyfind(logdir,1,Opts) of @@ -201,7 +222,10 @@ test_suite_events(pass_SUITE) -> {?eh,test_stats,{1,0,{0,0}}}, {?eh,tc_start,{ct_framework,end_per_suite}}, {?eh,tc_done,{ct_framework,end_per_suite,ok}}]; -test_suite_events(_) -> +test_suite_events(skip_all_surefire_SUITE) -> + [{?eh,tc_user_skip,{skip_all_surefire_SUITE,all,"skipped in spec"}}, + {?eh,test_stats,{0,0,{1,0}}}]; +test_suite_events(Test) -> [{?eh,tc_start,{surefire_SUITE,init_per_suite}}, {?eh,tc_done,{surefire_SUITE,init_per_suite,ok}}, {?eh,tc_start,{surefire_SUITE,tc_ok}}, @@ -210,46 +234,55 @@ test_suite_events(_) -> {?eh,tc_start,{surefire_SUITE,tc_fail}}, {?eh,tc_done,{surefire_SUITE,tc_fail, {failed,{error,{test_case_failed,"this test should fail"}}}}}, - {?eh,test_stats,{1,1,{0,0}}}, - {?eh,tc_start,{surefire_SUITE,tc_skip}}, - {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}}, - {?eh,test_stats,{1,1,{1,0}}}, - {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, - {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, - {auto_skipped,{require_failed,'_'}}}}, - {?eh,test_stats,{1,1,{1,1}}}, - [{?eh,tc_start,{surefire_SUITE,{init_per_group,g,[]}}}, - {?eh,tc_done,{surefire_SUITE,{init_per_group,g,[]},ok}}, - {?eh,tc_start,{surefire_SUITE,tc_ok}}, - {?eh,tc_done,{surefire_SUITE,tc_ok,ok}}, - {?eh,test_stats,{2,1,{1,1}}}, - {?eh,tc_start,{surefire_SUITE,tc_fail}}, - {?eh,tc_done,{surefire_SUITE,tc_fail, - {failed,{error,{test_case_failed,"this test should fail"}}}}}, - {?eh,test_stats,{2,2,{1,1}}}, - {?eh,tc_start,{surefire_SUITE,tc_skip}}, - {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}}, - {?eh,test_stats,{2,2,{2,1}}}, - {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, - {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, - {auto_skipped,{require_failed,'_'}}}}, - {?eh,test_stats,{2,2,{2,2}}}, - {?eh,tc_start,{surefire_SUITE,{end_per_group,g,[]}}}, - {?eh,tc_done,{surefire_SUITE,{end_per_group,g,[]},ok}}], - [{?eh,tc_start,{surefire_SUITE,{init_per_group,g_fail,[]}}}, - {?eh,tc_done,{surefire_SUITE,{init_per_group,g_fail,[]}, - {failed,{error,all_cases_should_be_skipped}}}}, - {?eh,tc_auto_skip,{surefire_SUITE,{tc_ok,g_fail}, - {failed, - {surefire_SUITE,init_per_group, - {'EXIT',all_cases_should_be_skipped}}}}}, - {?eh,test_stats,{2,2,{2,3}}}, - {?eh,tc_auto_skip,{surefire_SUITE,{end_per_group,g_fail}, - {failed, - {surefire_SUITE,init_per_group, - {'EXIT',all_cases_should_be_skipped}}}}}], - {?eh,tc_start,{surefire_SUITE,end_per_suite}}, - {?eh,tc_done,{surefire_SUITE,end_per_suite,ok}}]. + {?eh,test_stats,{1,1,{0,0}}}] ++ + tc_skip_events(Test,undefined) ++ + [{?eh,test_stats,{1,1,{1,0}}}, + {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, + {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, + {auto_skipped,{require_failed,'_'}}}}, + {?eh,test_stats,{1,1,{1,1}}}, + [{?eh,tc_start,{surefire_SUITE,{init_per_group,g,[]}}}, + {?eh,tc_done,{surefire_SUITE,{init_per_group,g,[]},ok}}, + {?eh,tc_start,{surefire_SUITE,tc_ok}}, + {?eh,tc_done,{surefire_SUITE,tc_ok,ok}}, + {?eh,test_stats,{2,1,{1,1}}}, + {?eh,tc_start,{surefire_SUITE,tc_fail}}, + {?eh,tc_done,{surefire_SUITE,tc_fail, + {failed,{error,{test_case_failed,"this test should fail"}}}}}, + {?eh,test_stats,{2,2,{1,1}}}] ++ + tc_skip_events(Test,g) ++ + [{?eh,test_stats,{2,2,{2,1}}}, + {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, + {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, + {auto_skipped,{require_failed,'_'}}}}, + {?eh,test_stats,{2,2,{2,2}}}, + {?eh,tc_start,{surefire_SUITE,{end_per_group,g,[]}}}, + {?eh,tc_done,{surefire_SUITE,{end_per_group,g,[]},ok}}], + [{?eh,tc_start,{surefire_SUITE,{init_per_group,g_fail,[]}}}, + {?eh,tc_done,{surefire_SUITE,{init_per_group,g_fail,[]}, + {failed,{error,all_cases_should_be_skipped}}}}, + {?eh,tc_auto_skip,{surefire_SUITE,{tc_ok,g_fail}, + {failed, + {surefire_SUITE,init_per_group, + {'EXIT',all_cases_should_be_skipped}}}}}, + {?eh,test_stats,{2,2,{2,3}}}, + {?eh,tc_auto_skip,{surefire_SUITE,{end_per_group,g_fail}, + {failed, + {surefire_SUITE,init_per_group, + {'EXIT',all_cases_should_be_skipped}}}}}], + {?eh,tc_start,{surefire_SUITE,end_per_suite}}, + {?eh,tc_done,{surefire_SUITE,end_per_suite,ok}}]. + +tc_skip_events(skip_case_in_spec,Group) -> + [{?eh,tc_user_skip,{surefire_SUITE,tc_skip_name(Group),"skipped in spec"}}]; +tc_skip_events(_Test,_Group) -> + [{?eh,tc_start,{surefire_SUITE,tc_skip}}, + {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}}]. + +tc_skip_name(undefined) -> + tc_skip; +tc_skip_name(Group) -> + {tc_skip,Group}. test_events(fail_pre_init_per_suite) -> [{?eh,start_logging,{'DEF','RUNDIR'}}, @@ -257,6 +290,10 @@ test_events(fail_pre_init_per_suite) -> test_suite_events(pass_SUITE) ++ test_suite_events(fail_SUITE, {1,0,{0,1}}) ++ [{?eh,stop_logging,[]}]; +test_events(skip_suite_in_spec) -> + [{?eh,start_logging,'_'},{?eh,start_info,{1,1,0}}] ++ + test_suite_events(skip_all_surefire_SUITE) ++ + [{?eh,stop_logging,[]}]; test_events(Test) -> [{?eh,start_logging,'_'}, {?eh,start_info,{1,1,9}}] ++ test_suite_events(Test) ++ @@ -364,6 +401,8 @@ failed_or_skipped([]) -> events_to_result(E) -> events_to_result(E, []). +events_to_result([{?eh,tc_user_skip,{_Suite,all,_}}|E], Result) -> + events_to_result(E, [[[s]]|Result]); events_to_result([{?eh,tc_auto_skip,{_Suite,init_per_suite,_}}|E], Result) -> {Suite,Rest} = events_to_result1(E), events_to_result(Rest, [[[s]|Suite]|Result]); @@ -382,7 +421,7 @@ events_to_result1([{?eh,tc_done,{_Suite, end_per_suite,R}}|E]) -> events_to_result1([{?eh,tc_done,{_Suite,_Case,R}}|E]) -> {Suite,Rest} = events_to_result1(E), {[result(R)|Suite],Rest}; -events_to_result1([{?eh,tc_auto_skip,_}|E]) -> +events_to_result1([{?eh,Skip,_}|E]) when Skip==tc_auto_skip; Skip==tc_user_skip -> {Suite,Rest} = events_to_result1(E), {[[s]|Suite],Rest}; events_to_result1([_|E]) -> diff --git a/lib/common_test/test/ct_surefire_SUITE_data/skip_one_case.spec b/lib/common_test/test/ct_surefire_SUITE_data/skip_one_case.spec new file mode 100644 index 0000000000..42df8a7d1a --- /dev/null +++ b/lib/common_test/test/ct_surefire_SUITE_data/skip_one_case.spec @@ -0,0 +1,2 @@ +{suites,".",surefire_SUITE}. +{skip_cases,".",surefire_SUITE,tc_skip,"skipped in spec"}. diff --git a/lib/common_test/test/ct_surefire_SUITE_data/skip_one_suite.spec b/lib/common_test/test/ct_surefire_SUITE_data/skip_one_suite.spec new file mode 100644 index 0000000000..57966328ab --- /dev/null +++ b/lib/common_test/test/ct_surefire_SUITE_data/skip_one_suite.spec @@ -0,0 +1,2 @@ +{suites,".",[skip_all_surefire_SUITE]}. +{skip_suites,".",skip_all_surefire_SUITE,"skipped in spec"}. diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE.erl index 228d900545..ea8a1a5662 100644 --- a/lib/common_test/test/ct_test_server_if_1_SUITE.erl +++ b/lib/common_test/test/ct_test_server_if_1_SUITE.erl @@ -161,6 +161,7 @@ test_events(ts_if_1) -> {?eh,tc_start,{ts_if_1_SUITE,tc4}}, {?eh,tc_done,{ts_if_1_SUITE,tc4,{failed,{error,failed_on_purpose}}}}, {?eh,test_stats,{1,2,{0,1}}}, + {?eh,tc_start,{ts_if_1_SUITE,tc5}}, {?eh,tc_done,{ts_if_1_SUITE,tc5,{auto_skipped,{sequence_failed,seq1,tc4}}}}, {?eh,test_stats,{1,2,{0,2}}}, diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index e926abd885..05a452b99d 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -765,23 +765,23 @@ locate({parallel,TEvs}, Node, Evs, Config) -> {Done,RemEvs2,length(RemEvs2)} end; %% end_per_group auto- or user skipped - (TEv={TEH,AutoOrUserSkip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) + (TEv={TEH,AutoOrUserSkip,{M,{end_per_group,G},R}}, {Done,RemEvs,_RemSize}) when AutoOrUserSkip == tc_auto_skip; AutoOrUserSkip == tc_user_skip -> RemEvs1 = lists:dropwhile( fun({EH,#event{name=tc_auto_skip, node=EvNode, - data={Mod,end_per_group,Reason}}}) when - EH == TEH, EvNode == Node, Mod == M -> + data={Mod,{end_per_group,EvGroupName},Reason}}}) when + EH == TEH, EvNode == Node, Mod == M, EvGroupName == G -> case match_data(R, Reason) of match -> false; _ -> true end; ({EH,#event{name=tc_user_skip, node=EvNode, - data={Mod,end_per_group,Reason}}}) when - EH == TEH, EvNode == Node, Mod == M -> + data={Mod,{end_per_group,EvGroupName},Reason}}}) when + EH == TEH, EvNode == Node, Mod == M, EvGroupName == G -> case match_data(R, Reason) of match -> false; _ -> true @@ -1008,20 +1008,20 @@ locate({shuffle,TEvs}, Node, Evs, Config) -> {Done,RemEvs2,length(RemEvs2)} end; %% end_per_group auto-or user skipped - (TEv={TEH,AutoOrUserSkip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) + (TEv={TEH,AutoOrUserSkip,{M,{end_per_group,G},R}}, {Done,RemEvs,_RemSize}) when AutoOrUserSkip == tc_auto_skip; AutoOrUserSkip == tc_user_skip -> RemEvs1 = lists:dropwhile( fun({EH,#event{name=tc_auto_skip, node=EvNode, - data={Mod,end_per_group,Reason}}}) when - EH == TEH, EvNode == Node, Mod == M, Reason == R -> + data={Mod,{end_per_group,EvGroupName},Reason}}}) when + EH == TEH, EvNode == Node, Mod == M, EvGroupName == G, Reason == R -> false; ({EH,#event{name=tc_user_skip, node=EvNode, - data={Mod,end_per_group,Reason}}}) when - EH == TEH, EvNode == Node, Mod == M, Reason == R -> + data={Mod,{end_per_group,EvGroupName},Reason}}}) when + EH == TEH, EvNode == Node, Mod == M, EvGroupName == G, Reason == R -> false; ({EH,#event{name=stop_logging, node=EvNode,data=_}}) when @@ -1264,10 +1264,10 @@ log_events1([E={_EH,tc_done,{_M,{end_per_group,_GrName,Props},_R}} | Evs], Dev, io:format(Dev, "~s~p]},~n", [Ind,E]), log_events1(Evs, Dev, Ind--" ") end; -log_events1([E={_EH,tc_auto_skip,{_M,end_per_group,_Reason}} | Evs], Dev, Ind) -> +log_events1([E={_EH,tc_auto_skip,{_M,{end_per_group,_GrName},_Reason}} | Evs], Dev, Ind) -> io:format(Dev, "~s~p],~n", [Ind,E]), log_events1(Evs, Dev, Ind--" "); -log_events1([E={_EH,tc_user_skip,{_M,end_per_group,_Reason}} | Evs], Dev, Ind) -> +log_events1([E={_EH,tc_user_skip,{_M,{end_per_group,_GrName},_Reason}} | Evs], Dev, Ind) -> io:format(Dev, "~s~p],~n", [Ind,E]), log_events1(Evs, Dev, Ind--" "); log_events1([E], Dev, Ind) -> diff --git a/lib/common_test/vsn.mk b/lib/common_test/vsn.mk index 2fab4d3883..e6ae8b2e7a 100644 --- a/lib/common_test/vsn.mk +++ b/lib/common_test/vsn.mk @@ -1 +1 @@ -COMMON_TEST_VSN = 1.13 +COMMON_TEST_VSN = 1.14 diff --git a/lib/compiler/doc/src/notes.xml b/lib/compiler/doc/src/notes.xml index 2e58b68bf0..449453bf88 100644 --- a/lib/compiler/doc/src/notes.xml +++ b/lib/compiler/doc/src/notes.xml @@ -32,6 +32,22 @@ <p>This document describes the changes made to the Compiler application.</p> +<section><title>Compiler 7.0.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Minor internal changes. A typo in the documentation was + also fixed.</p> + <p> + Own Id: OTP-14240</p> + </item> + </list> + </section> + +</section> + <section><title>Compiler 7.0.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/compiler/src/Makefile b/lib/compiler/src/Makefile index cf60355a40..59b80ade5d 100644 --- a/lib/compiler/src/Makefile +++ b/lib/compiler/src/Makefile @@ -63,6 +63,7 @@ MODULES = \ beam_peep \ beam_receive \ beam_reorder \ + beam_record \ beam_split \ beam_trim \ beam_type \ diff --git a/lib/compiler/src/beam_disasm.erl b/lib/compiler/src/beam_disasm.erl index c699672db1..8fd0b36d05 100644 --- a/lib/compiler/src/beam_disasm.erl +++ b/lib/compiler/src/beam_disasm.erl @@ -815,6 +815,9 @@ resolve_inst({is_tuple=I,Args0},_,_,_) -> resolve_inst({test_arity=I,Args0},_,_,_) -> [L|Args] = resolve_args(Args0), {test,I,L,Args}; +resolve_inst({is_tagged_tuple=I,Args0},_,_,_) -> + [F|Args] = resolve_args(Args0), + {test,I,F,Args}; resolve_inst({select_val,Args},_,_,_) -> [Reg,FLbl,{{z,1},{u,_Len},List0}] = Args, List = resolve_args(List0), diff --git a/lib/compiler/src/beam_record.erl b/lib/compiler/src/beam_record.erl new file mode 100644 index 0000000000..419089b1bc --- /dev/null +++ b/lib/compiler/src/beam_record.erl @@ -0,0 +1,106 @@ +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2014-2017. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% File: beam_record.erl +%% Author: Björn-Egil Dahlberg +%% Created: 2014-09-03 +%% + +-module(beam_record). +-export([module/2]). + +%% Rewrite the instruction stream on tagged tuple tests. +%% Tagged tuples means a tuple of any arity with an atom as its first element. +%% Typically records, ok-tuples and error-tuples. +%% +%% from: +%% ... +%% {test,is_tuple,Fail,[Src]}. +%% {test,test_arity,Fail,[Src,Sz]}. +%% ... +%% {get_tuple_element,Src,0,Dst}. +%% ... +%% {test,is_eq_exact,Fail,[Dst,Atom]}. +%% ... +%% to: +%% ... +%% {test,is_tagged_tuple,Fail,[Src,Sz,Atom]}. +%% ... + + +-import(lists, [reverse/1]). + +-spec module(beam_utils:module_code(), [compile:option()]) -> + {'ok',beam_utils:module_code()}. + +module({Mod,Exp,Attr,Fs0,Lc}, _Opt) -> + Fs = [function(F) || F <- Fs0], + {ok,{Mod,Exp,Attr,Fs,Lc}}. + +function({function,Name,Arity,CLabel,Is}) -> + try + Idx = beam_utils:index_labels(Is), + {function,Name,Arity,CLabel,rewrite(Is,Idx)} + catch + Class:Error -> + Stack = erlang:get_stacktrace(), + io:fwrite("Function: ~w/~w\n", [Name,Arity]), + erlang:raise(Class, Error, Stack) + end. + +rewrite(Is,Idx) -> + rewrite(Is,Idx,[]). + +rewrite([{test,is_tuple,Fail,[Src]}=I1, + {test,test_arity,Fail,[Src,N]}=I2|Is],Idx,Acc) -> + case is_tagged_tuple(Is,Fail,Src,Idx) of + no -> + rewrite(Is,Idx,[I2,I1|Acc]); + {Atom,[{block,[]}|Is1]} -> + rewrite(Is1,Idx,[{test,is_tagged_tuple,Fail,[Src,N,Atom]}|Acc]); + {Atom,Is1} -> + rewrite(Is1,Idx,[{test,is_tagged_tuple,Fail,[Src,N,Atom]}|Acc]) + end; +rewrite([I|Is],Idx,Acc) -> + rewrite(Is,Idx,[I|Acc]); +rewrite([],_,Acc) -> reverse(Acc). + +is_tagged_tuple([{block,[{set,[Dst],[Src],{get_tuple_element,0}}=B|Bs]}, + {test,is_eq_exact,Fail,[Dst,{atom,_}=Atom]}|Is],Fail,Src,Idx) -> + + %% if Dst is killed in the instruction stream and at fail label, + %% we can safely remove get_tuple_element. + %% + %% if Dst is not killed in the stream, we cannot remove get_tuple_element + %% since it is referenced. + + case is_killed(Dst,Is,Fail,Idx) of + true -> {Atom,[{block,Bs}|Is]}; + false -> {Atom,[{block,[B|Bs]}|Is]} + end; +is_tagged_tuple([{block,[{set,_,_,_}=B|Bs]}, + {test,is_eq_exact,_,_}=I|Is],Fail,Src,Idx) -> + case is_tagged_tuple([{block,Bs},I|Is],Fail,Src,Idx) of + {Atom,[{block,Bsr}|Isr]} -> {Atom,[{block,[B|Bsr]}|Isr]}; + no -> no + end; +is_tagged_tuple(_Is,_Fail,_Src,_Idx) -> + no. + +is_killed(Dst,Is,{_,Lbl},Idx) -> + beam_utils:is_killed(Dst,Is,Idx) andalso + beam_utils:is_killed_at(Dst,Lbl,Idx). diff --git a/lib/compiler/src/beam_type.erl b/lib/compiler/src/beam_type.erl index 050c599d6b..2b5d558ee4 100644 --- a/lib/compiler/src/beam_type.erl +++ b/lib/compiler/src/beam_type.erl @@ -683,6 +683,9 @@ op_type('bsr') -> integer; op_type('div') -> integer; op_type(_) -> unknown. +flush(Rs, [{set,[_],[_,_,_],{bif,is_record,_}}|_]=Is0, Acc0) -> + Acc = flush_all(Rs, Is0, Acc0), + {[],Acc}; flush(Rs, [{set,[_],[],{put_tuple,_}}|_]=Is0, Acc0) -> Acc = flush_all(Rs, Is0, Acc0), {[],Acc}; diff --git a/lib/compiler/src/beam_validator.erl b/lib/compiler/src/beam_validator.erl index bf33ae0aeb..c26e5719aa 100644 --- a/lib/compiler/src/beam_validator.erl +++ b/lib/compiler/src/beam_validator.erl @@ -653,6 +653,9 @@ valfun_4({test,is_nonempty_list,{f,Lbl},[Cons]}, Vst) -> valfun_4({test,test_arity,{f,Lbl},[Tuple,Sz]}, Vst) when is_integer(Sz) -> assert_type(tuple, Tuple, Vst), set_type_reg({tuple,Sz}, Tuple, branch_state(Lbl, Vst)); +valfun_4({test,is_tagged_tuple,{f,Lbl},[Src,Sz,_Atom]}, Vst) -> + validate_src([Src], Vst), + set_type_reg({tuple, Sz}, Src, branch_state(Lbl, Vst)); valfun_4({test,has_map_fields,{f,Lbl},Src,{list,List}}, Vst) -> assert_type(map, Src, Vst), assert_unique_map_keys(List), diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index c849306c0d..03b52932d1 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -216,19 +216,19 @@ expand_opt(return, Os) -> expand_opt(r12, Os) -> [no_recv_opt,no_line_info,no_utf8_atoms|Os]; expand_opt(r13, Os) -> - [no_recv_opt,no_line_info,no_utf8_atoms|Os]; + [no_record_opt,no_recv_opt,no_line_info,no_utf8_atoms|Os]; expand_opt(r14, Os) -> - [no_line_info,no_utf8_atoms|Os]; + [no_record_opt,no_line_info,no_utf8_atoms|Os]; expand_opt(r15, Os) -> - [no_utf8_atoms|Os]; + [no_record_opt,no_utf8_atoms|Os]; expand_opt(r16, Os) -> - [no_utf8_atoms|Os]; + [no_record_opt,no_utf8_atoms|Os]; expand_opt(r17, Os) -> - [no_utf8_atoms|Os]; + [no_record_opt,no_utf8_atoms|Os]; expand_opt(r18, Os) -> - [no_utf8_atoms|Os]; + [no_record_opt,no_utf8_atoms|Os]; expand_opt(r19, Os) -> - [no_utf8_atoms|Os]; + [no_record_opt,no_utf8_atoms|Os]; expand_opt({debug_info_key,_}=O, Os) -> [encrypt_debug_info,O|Os]; expand_opt(no_float_opt, Os) -> @@ -755,6 +755,8 @@ asm_passes() -> {iff,dbsm,{listing,"bsm"}}, {unless,no_recv_opt,{pass,beam_receive}}, {iff,drecv,{listing,"recv"}}, + {unless,no_record_opt,{pass,beam_record}}, + {iff,drecord,{listing,"record"}}, {unless,no_stack_trimming,{pass,beam_trim}}, {iff,dtrim,{listing,"trim"}}, {pass,beam_flatten}]}, @@ -1849,6 +1851,7 @@ pre_load() -> beam_opcodes, beam_peep, beam_receive, + beam_record, beam_reorder, beam_split, beam_trim, diff --git a/lib/compiler/src/compiler.app.src b/lib/compiler/src/compiler.app.src index 3cb991687b..3961b2af86 100644 --- a/lib/compiler/src/compiler.app.src +++ b/lib/compiler/src/compiler.app.src @@ -38,6 +38,7 @@ beam_peep, beam_receive, beam_reorder, + beam_record, beam_split, beam_trim, beam_type, diff --git a/lib/compiler/src/genop.tab b/lib/compiler/src/genop.tab index dcbdeb32e6..5e0c2b3ebf 100755 --- a/lib/compiler/src/genop.tab +++ b/lib/compiler/src/genop.tab @@ -537,3 +537,9 @@ BEAM_FORMAT_NUMBER=0 156: is_map/2 157: has_map_fields/3 158: get_map_elements/3 + +## @spec is_tagged_tuple Lbl Reg N Atom +## @doc Test the type of Reg and jumps to Lbl if it is not a tuple. +## Test the arity of Reg and jumps to Lbl if it is not N. +## Test the first element of the tuple and jumps to Lbl if it is not Atom. +159: is_tagged_tuple/4 diff --git a/lib/compiler/test/Makefile b/lib/compiler/test/Makefile index e338dbb4e3..63763f31b2 100644 --- a/lib/compiler/test/Makefile +++ b/lib/compiler/test/Makefile @@ -185,6 +185,7 @@ release_tests_spec: make_emakefile echo "-module($$module). %% dummy .erl file" >$$file; \ done $(INSTALL_DATA) $(ERL_DUMMY_FILES) "$(RELSYSDIR)" + rm $(ERL_DUMMY_FILES) chmod -R u+w "$(RELSYSDIR)" @tar cf - *_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -) diff --git a/lib/compiler/test/beam_type_SUITE.erl b/lib/compiler/test/beam_type_SUITE.erl index 492067ef00..7ca544a537 100644 --- a/lib/compiler/test/beam_type_SUITE.erl +++ b/lib/compiler/test/beam_type_SUITE.erl @@ -22,7 +22,7 @@ -export([all/0,suite/0,groups/0,init_per_suite/1,end_per_suite/1, init_per_group/2,end_per_group/2, integers/1,coverage/1,booleans/1,setelement/1,cons/1, - tuple/1]). + tuple/1,record_float/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -37,7 +37,8 @@ groups() -> booleans, setelement, cons, - tuple + tuple, + record_float ]}]. init_per_suite(Config) -> @@ -126,5 +127,22 @@ tuple(_Config) -> do_tuple() -> {0, _} = {necessary}. +-record(x, {a}). + +record_float(_Config) -> + 17.0 = record_float(#x{a={0}}, 1700), + 23.0 = record_float(#x{a={0}}, 2300.0), + {'EXIT',{if_clause,_}} = (catch record_float(#x{a={1}}, 88)), + {'EXIT',{if_clause,_}} = (catch record_float(#x{a={}}, 88)), + {'EXIT',{if_clause,_}} = (catch record_float(#x{}, 88)), + ok. + +record_float(R, N0) -> + N = N0 / 100, + if element(1, R#x.a) =:= 0 -> + N + end. + + id(I) -> I. diff --git a/lib/compiler/test/guard_SUITE.erl b/lib/compiler/test/guard_SUITE.erl index a662d85272..ccb9b58225 100644 --- a/lib/compiler/test/guard_SUITE.erl +++ b/lib/compiler/test/guard_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2016. All Rights Reserved. +%% Copyright Ericsson AB 2001-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1538,7 +1538,7 @@ literal_type_tests_1(Config) -> Func = {function, Anno, test, 0, [{clause,Anno,[],[],Tests}]}, Form = [{attribute,Anno,module,Mod}, {attribute,Anno,compile,export_all}, - Func, {eof,Anno}], + Func, {eof,999}], %% Print generated code for inspection. lists:foreach(fun (F) -> io:put_chars([erl_pp:form(F),"\n"]) end, Form), diff --git a/lib/compiler/test/misc_SUITE.erl b/lib/compiler/test/misc_SUITE.erl index 621524114f..fa6d5ee957 100644 --- a/lib/compiler/test/misc_SUITE.erl +++ b/lib/compiler/test/misc_SUITE.erl @@ -280,6 +280,23 @@ silly_coverage(Config) when is_list(Config) -> {block,[a|b]}]}],0}, expect_error(fun() -> beam_receive:module(ReceiveInput, []) end), + %% beam_record. + RecordInput = {?MODULE,[{foo,0}],[], + [{function,foo,1,2, + [{label,1}, + {func_info,{atom,?MODULE},{atom,foo},1}, + {label,2}, + {test,is_tuple,{f,1},[{x,0}]}, + {test,test_arity,{f,1},[{x,0},3]}, + {block,[{set,[{x,1}],[{x,0}],{get_tuple_element,0}}]}, + {test,is_eq_exact,{f,1},[{x,1},{atom,bar}]}, + {block,[{set,[{x,2}],[{x,0}],{get_tuple_element,1}}|a]}, + {test,is_eq_exact,{f,1},[{x,2},{integer,1}]}, + {block,[{set,[{x,0}],[{atom,ok}],move}]}, + return]}],0}, + + expect_error(fun() -> beam_record:module(RecordInput, []) end), + BeamZInput = {?MODULE,[{foo,0}],[], [{function,foo,0,2, [{label,1}, diff --git a/lib/compiler/vsn.mk b/lib/compiler/vsn.mk index 9c3cf1f34b..5c87304a01 100644 --- a/lib/compiler/vsn.mk +++ b/lib/compiler/vsn.mk @@ -1 +1 @@ -COMPILER_VSN = 7.0.3 +COMPILER_VSN = 7.0.4 diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c index 44c3fc4f06..1f4ce9a3da 100644 --- a/lib/crypto/c_src/crypto.c +++ b/lib/crypto/c_src/crypto.c @@ -71,6 +71,46 @@ PACKED_OPENSSL_VERSION(MAJ,MIN,FIX,('a'-1)) +/* LibreSSL was cloned from OpenSSL 1.0.1g and claims to be API and BPI compatible + * with 1.0.1. + * + * LibreSSL has the same names on include files and symbols as OpenSSL, but defines + * the OPENSSL_VERSION_NUMBER to be >= 2.0.0 + * + * Therefor works tests like this as intendend: + * OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) + * (The test is for example "2.4.2" >= "1.0.0" although the test + * with the cloned OpenSSL test would be "1.0.1" >= "1.0.0") + * + * But tests like this gives wrong result: + * OPENSSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION_PLAIN(1,1,0) + * (The test is false since "2.4.2" < "1.1.0". It should have been + * true because the LibreSSL API version is "1.0.1") + * + */ + +#ifdef LIBRESSL_VERSION_NUMBER +/* A macro to test on in this file */ +#define HAS_LIBRESSL +#endif + +#ifdef HAS_LIBRESSL +/* LibreSSL dislikes FIPS */ +# ifdef FIPS_SUPPORT +# undef FIPS_SUPPORT +# endif + +/* LibreSSL wants the 1.0.1 API */ +# define NEED_EVP_COMPATIBILITY_FUNCTIONS +#endif + + +#if OPENSSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION_PLAIN(1,1,0) +# define NEED_EVP_COMPATIBILITY_FUNCTIONS +#endif + + + #if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) #include <openssl/modes.h> #endif @@ -120,7 +160,9 @@ #endif #if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,1,0) -# define HAVE_CHACHA20_POLY1305 +# ifndef HAS_LIBRESSL +# define HAVE_CHACHA20_POLY1305 +# endif #endif #if OPENSSL_VERSION_NUMBER <= PACKED_OPENSSL_VERSION(0,9,8,'l') @@ -205,8 +247,8 @@ do { \ } \ } while (0) -#if OPENSSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION_PLAIN(1,1,0) +#ifdef NEED_EVP_COMPATIBILITY_FUNCTIONS /* * In OpenSSL 1.1.0, most structs are opaque. That means that * the structs cannot be allocated as automatic variables on the @@ -237,9 +279,19 @@ static void HMAC_CTX_free(HMAC_CTX *ctx) #define EVP_MD_CTX_new() EVP_MD_CTX_create() #define EVP_MD_CTX_free(ctx) EVP_MD_CTX_destroy(ctx) +static INLINE void *BN_GENCB_get_arg(BN_GENCB *cb); + +static INLINE void *BN_GENCB_get_arg(BN_GENCB *cb) +{ + return cb->arg; +} + static INLINE int RSA_set0_key(RSA *r, BIGNUM *n, BIGNUM *e, BIGNUM *d); +static INLINE void RSA_get0_key(const RSA *r, const BIGNUM **n, const BIGNUM **e, const BIGNUM **d); static INLINE int RSA_set0_factors(RSA *r, BIGNUM *p, BIGNUM *q); +static INLINE void RSA_get0_factors(const RSA *r, const BIGNUM **p, const BIGNUM **q); static INLINE int RSA_set0_crt_params(RSA *r, BIGNUM *dmp1, BIGNUM *dmq1, BIGNUM *iqmp); +static INLINE void RSA_get0_crt_params(const RSA *r, const BIGNUM **dmp1, const BIGNUM **dmq1, const BIGNUM **iqmp); static INLINE int RSA_set0_key(RSA *r, BIGNUM *n, BIGNUM *e, BIGNUM *d) { @@ -249,6 +301,13 @@ static INLINE int RSA_set0_key(RSA *r, BIGNUM *n, BIGNUM *e, BIGNUM *d) return 1; } +static INLINE void RSA_get0_key(const RSA *r, const BIGNUM **n, const BIGNUM **e, const BIGNUM **d) +{ + *n = r->n; + *e = r->e; + *d = r->d; +} + static INLINE int RSA_set0_factors(RSA *r, BIGNUM *p, BIGNUM *q) { r->p = p; @@ -256,6 +315,12 @@ static INLINE int RSA_set0_factors(RSA *r, BIGNUM *p, BIGNUM *q) return 1; } +static INLINE void RSA_get0_factors(const RSA *r, const BIGNUM **p, const BIGNUM **q) +{ + *p = r->p; + *q = r->q; +} + static INLINE int RSA_set0_crt_params(RSA *r, BIGNUM *dmp1, BIGNUM *dmq1, BIGNUM *iqmp) { r->dmp1 = dmp1; @@ -264,6 +329,13 @@ static INLINE int RSA_set0_crt_params(RSA *r, BIGNUM *dmp1, BIGNUM *dmq1, BIGNUM return 1; } +static INLINE void RSA_get0_crt_params(const RSA *r, const BIGNUM **dmp1, const BIGNUM **dmq1, const BIGNUM **iqmp) +{ + *dmp1 = r->dmp1; + *dmq1 = r->dmq1; + *iqmp = r->iqmp; +} + static INLINE int DSA_set0_key(DSA *d, BIGNUM *pub_key, BIGNUM *priv_key); static INLINE int DSA_set0_pqg(DSA *d, BIGNUM *p, BIGNUM *q, BIGNUM *g); @@ -326,7 +398,11 @@ DH_get0_key(const DH *dh, const BIGNUM **pub_key, const BIGNUM **priv_key) *priv_key = dh->priv_key; } -#endif /* End of compatibility definitions. */ +#else /* End of compatibility definitions. */ + +#define HAVE_OPAQUE_BN_GENCB + +#endif /* NIF interface declarations */ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info); @@ -349,10 +425,12 @@ static ERL_NIF_TERM hmac_final_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM static ERL_NIF_TERM cmac_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM block_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM aes_cfb_8_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM aes_cfb_128_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM aes_ige_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM aes_ctr_stream_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM aes_ctr_stream_encrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM strong_rand_bytes_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM strong_rand_range_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM rand_uniform_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM mod_exp_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM dss_verify_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); @@ -364,6 +442,7 @@ static ERL_NIF_TERM rsa_sign_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar static ERL_NIF_TERM dss_sign_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM rsa_public_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM rsa_private_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM rsa_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM dh_generate_parameters_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM dh_check(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM dh_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); @@ -397,6 +476,7 @@ static EC_KEY* ec_key_new(ErlNifEnv* env, ERL_NIF_TERM curve_arg); static int term2point(ErlNifEnv* env, ERL_NIF_TERM term, EC_GROUP *group, EC_POINT **pptr); #endif +static ERL_NIF_TERM bin_from_bn(ErlNifEnv* env, const BIGNUM *bn); static int library_refc = 0; /* number of users of this dynamic library */ @@ -423,6 +503,7 @@ static ErlNifFunc nif_funcs[] = { {"aes_ctr_stream_encrypt", 2, aes_ctr_stream_encrypt}, {"aes_ctr_stream_decrypt", 2, aes_ctr_stream_encrypt}, {"strong_rand_bytes_nif", 1, strong_rand_bytes_nif}, + {"strong_rand_range_nif", 1, strong_rand_range_nif}, {"rand_uniform_nif", 2, rand_uniform_nif}, {"mod_exp_nif", 4, mod_exp_nif}, {"dss_verify_nif", 4, dss_verify_nif}, @@ -434,6 +515,7 @@ static ErlNifFunc nif_funcs[] = { {"dss_sign_nif", 3, dss_sign_nif}, {"rsa_public_crypt", 4, rsa_public_crypt}, {"rsa_private_crypt", 4, rsa_private_crypt}, + {"rsa_generate_key_nif", 2, rsa_generate_key_nif}, {"dh_generate_parameters_nif", 2, dh_generate_parameters_nif}, {"dh_check", 1, dh_check}, {"dh_generate_key_nif", 4, dh_generate_key_nif}, @@ -883,6 +965,7 @@ static int initialize(ErlNifEnv* env, ERL_NIF_TERM load_info) CRYPTO_set_dynlock_destroy_callback(ccb->dyn_destroy_function); } #endif /* OPENSSL_THREADS */ + return 0; } @@ -1656,17 +1739,20 @@ static ERL_NIF_TERM block_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM return enif_raise_exception(env, atom_notsup); } - if ((argv[0] == atom_aes_cfb8 || argv[0] == atom_aes_cfb128) - && (key.size == 24 || key.size == 32) -#ifdef FIPS_SUPPORT - && !FIPS_mode() -#endif - ) { + if (argv[0] == atom_aes_cfb8 + && (key.size == 24 || key.size == 32)) { /* Why do EVP_CIPHER_CTX_set_key_length() fail on these key sizes? * Fall back on low level API */ return aes_cfb_8_crypt(env, argc-1, argv+1); } + else if (argv[0] == atom_aes_cfb128 + && (key.size == 24 || key.size == 32)) { + /* Why do EVP_CIPHER_CTX_set_key_length() fail on these key sizes? + * Fall back on low level API + */ + return aes_cfb_128_crypt_nif(env, argc-1, argv+1); + } ivec_size = EVP_CIPHER_iv_length(cipher); @@ -1742,6 +1828,31 @@ static ERL_NIF_TERM aes_cfb_8_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM return ret; } +static ERL_NIF_TERM aes_cfb_128_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Key, IVec, Data, IsEncrypt) */ + ErlNifBinary key, ivec, text; + AES_KEY aes_key; + unsigned char ivec_clone[16]; /* writable copy */ + int new_ivlen = 0; + ERL_NIF_TERM ret; + + if (!enif_inspect_iolist_as_binary(env, argv[0], &key) + || !(key.size == 16 || key.size == 24 || key.size == 32) + || !enif_inspect_binary(env, argv[1], &ivec) || ivec.size != 16 + || !enif_inspect_iolist_as_binary(env, argv[2], &text)) { + return enif_make_badarg(env); + } + + memcpy(ivec_clone, ivec.data, 16); + AES_set_encrypt_key(key.data, key.size * 8, &aes_key); + AES_cfb128_encrypt((unsigned char *) text.data, + enif_make_new_binary(env, text.size, &ret), + text.size, &aes_key, ivec_clone, &new_ivlen, + (argv[3] != atom_true)); + CONSUME_REDS(env,text); + return ret; +} + static ERL_NIF_TERM aes_ige_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {/* (Key, IVec, Data, IsEncrypt) */ #ifdef HAVE_AES_IGE @@ -2237,6 +2348,41 @@ static int get_bn_from_bin(ErlNifEnv* env, ERL_NIF_TERM term, BIGNUM** bnp) return 1; } +static ERL_NIF_TERM bin_from_bn(ErlNifEnv* env, const BIGNUM *bn) +{ + int bn_len; + unsigned char *bin_ptr; + ERL_NIF_TERM term; + + /* Copy the bignum into an erlang binary. */ + bn_len = BN_num_bytes(bn); + bin_ptr = enif_make_new_binary(env, bn_len, &term); + BN_bn2bin(bn, bin_ptr); + + return term; +} + +static ERL_NIF_TERM strong_rand_range_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Range) */ + BIGNUM *bn_range, *bn_rand; + ERL_NIF_TERM ret; + + if(!get_bn_from_bin(env, argv[0], &bn_range)) { + return enif_make_badarg(env); + } + + bn_rand = BN_new(); + if (BN_rand_range(bn_rand, bn_range) != 1) { + ret = atom_false; + } + else { + ret = bin_from_bn(env, bn_rand); + } + BN_free(bn_rand); + BN_free(bn_range); + return ret; +} + static ERL_NIF_TERM rand_uniform_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {/* (Lo,Hi) */ BIGNUM *bn_from = NULL, *bn_to, *bn_rand; @@ -2808,6 +2954,119 @@ static ERL_NIF_TERM rsa_private_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TE } } +/* Creates a term which can be parsed by get_rsa_private_key(). This is a list of plain integer binaries (not mpints). */ +static ERL_NIF_TERM put_rsa_private_key(ErlNifEnv* env, const RSA *rsa) +{ + ERL_NIF_TERM result[8]; + const BIGNUM *n, *e, *d, *p, *q, *dmp1, *dmq1, *iqmp; + + /* Return at least [E,N,D] */ + n = NULL; e = NULL; d = NULL; + RSA_get0_key(rsa, &n, &e, &d); + + result[0] = bin_from_bn(env, e); // Exponent E + result[1] = bin_from_bn(env, n); // Modulus N = p*q + result[2] = bin_from_bn(env, d); // Exponent D + + /* Check whether the optional additional parameters are available */ + p = NULL; q = NULL; + RSA_get0_factors(rsa, &p, &q); + dmp1 = NULL; dmq1 = NULL; iqmp = NULL; + RSA_get0_crt_params(rsa, &dmp1, &dmq1, &iqmp); + + if (p && q && dmp1 && dmq1 && iqmp) { + result[3] = bin_from_bn(env, p); // Factor p + result[4] = bin_from_bn(env, q); // Factor q + result[5] = bin_from_bn(env, dmp1); // D mod (p-1) + result[6] = bin_from_bn(env, dmq1); // D mod (q-1) + result[7] = bin_from_bn(env, iqmp); // (1/q) mod p + + return enif_make_list_from_array(env, result, 8); + } else { + return enif_make_list_from_array(env, result, 3); + } +} + +static int check_erlang_interrupt(int maj, int min, BN_GENCB *ctxt) +{ + ErlNifEnv *env = BN_GENCB_get_arg(ctxt); + + if (!enif_is_current_process_alive(env)) { + return 0; + } else { + return 1; + } +} + +static ERL_NIF_TERM rsa_generate_key(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (ModulusSize, PublicExponent) */ + int modulus_bits; + BIGNUM *pub_exp, *three; + RSA *rsa; + int success; + ERL_NIF_TERM result; + BN_GENCB *intr_cb; +#ifndef HAVE_OPAQUE_BN_GENCB + BN_GENCB intr_cb_buf; +#endif + + if (!enif_get_int(env, argv[0], &modulus_bits) || modulus_bits < 256) { + return enif_make_badarg(env); + } + + if (!get_bn_from_bin(env, argv[1], &pub_exp)) { + return enif_make_badarg(env); + } + + /* Make sure the public exponent is large enough (at least 3). + * Without this, RSA_generate_key_ex() can run forever. */ + three = BN_new(); + BN_set_word(three, 3); + success = BN_cmp(pub_exp, three); + BN_free(three); + if (success < 0) { + BN_free(pub_exp); + return enif_make_badarg(env); + } + + /* For large keys, prime generation can take many seconds. Set up + * the callback which we use to test whether the process has been + * interrupted. */ +#ifdef HAVE_OPAQUE_BN_GENCB + intr_cb = BN_GENCB_new(); +#else + intr_cb = &intr_cb_buf; +#endif + BN_GENCB_set(intr_cb, check_erlang_interrupt, env); + + rsa = RSA_new(); + success = RSA_generate_key_ex(rsa, modulus_bits, pub_exp, intr_cb); + BN_free(pub_exp); + +#ifdef HAVE_OPAQUE_BN_GENCB + BN_GENCB_free(intr_cb); +#endif + + if (!success) { + RSA_free(rsa); + return atom_error; + } + + result = put_rsa_private_key(env, rsa); + RSA_free(rsa); + + return result; +} + +static ERL_NIF_TERM rsa_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + /* RSA key generation can take a long time (>1 sec for a large + * modulus), so schedule it as a CPU-bound operation. */ + return enif_schedule_nif(env, "rsa_generate_key", + ERL_NIF_DIRTY_JOB_CPU_BOUND, + rsa_generate_key, argc, argv); +} + static ERL_NIF_TERM dh_generate_parameters_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {/* (PrimeLen, Generator) */ int prime_len, generator; diff --git a/lib/crypto/c_src/crypto_callback.h b/lib/crypto/c_src/crypto_callback.h index 2641cc0c8b..489810116f 100644 --- a/lib/crypto/c_src/crypto_callback.h +++ b/lib/crypto/c_src/crypto_callback.h @@ -19,7 +19,7 @@ */ #include <openssl/crypto.h> -#if OPENSSL_VERSION_NUMBER < 0x10100000L +#ifdef NEED_EVP_COMPATIBILITY_FUNCTIONS # define CCB_FILE_LINE_ARGS #else # define CCB_FILE_LINE_ARGS , const char *file, int line diff --git a/lib/crypto/doc/src/crypto.xml b/lib/crypto/doc/src/crypto.xml index a4b34657ba..552d95d7dc 100644 --- a/lib/crypto/doc/src/crypto.xml +++ b/lib/crypto/doc/src/crypto.xml @@ -77,7 +77,7 @@ <code>rsa_private() = [key_value()] = [E, N, D] | [E, N, D, P1, P2, E1, E2, C] </code> <p>Where E is the public exponent, N is public modulus and D is - the private exponent.The longer key format contains redundant + the private exponent. The longer key format contains redundant information that will make the calculation faster. P1,P2 are first and second prime factors. E1,E2 are first and second exponents. C is the CRT coefficient. Terminology is taken from <url href="http://www.ietf.org/rfc/rfc3477.txt"> RFC 3447</url>.</p> @@ -298,22 +298,32 @@ <func> <name>generate_key(Type, Params) -> {PublicKey, PrivKeyOut} </name> <name>generate_key(Type, Params, PrivKeyIn) -> {PublicKey, PrivKeyOut} </name> - <fsummary>Generates a public keys of type <c>Type</c></fsummary> + <fsummary>Generates a public key of type <c>Type</c></fsummary> <type> - <v> Type = dh | ecdh | srp </v> - <v>Params = dh_params() | ecdh_params() | SrpUserParams | SrpHostParams </v> + <v> Type = dh | ecdh | rsa | srp </v> + <v>Params = dh_params() | ecdh_params() | RsaParams | SrpUserParams | SrpHostParams </v> + <v>RsaParams = {ModulusSizeInBits::integer(), PublicExponent::key_value()}</v> <v>SrpUserParams = {user, [Generator::binary(), Prime::binary(), Version::atom()]}</v> <v>SrpHostParams = {host, [Verifier::binary(), Generator::binary(), Prime::binary(), Version::atom()]}</v> - <v>PublicKey = dh_public() | ecdh_public() | srp_public() </v> + <v>PublicKey = dh_public() | ecdh_public() | rsa_public() | srp_public() </v> <v>PrivKeyIn = undefined | dh_private() | ecdh_private() | srp_private() </v> - <v>PrivKeyOut = dh_private() | ecdh_private() | srp_private() </v> + <v>PrivKeyOut = dh_private() | ecdh_private() | rsa_private() | srp_private() </v> </type> <desc> - <p>Generates public keys of type <c>Type</c>. - See also <seealso marker="public_key:public_key#generate_key-1">public_key:generate_key/1</seealso> - May throw exception <c>low_entropy</c> in case the random generator - failed due to lack of secure "randomness". - </p> + <p>Generates a public key of type <c>Type</c>. + See also <seealso marker="public_key:public_key#generate_key-1">public_key:generate_key/1</seealso>. + May throw exception an exception of class <c>error</c>: + </p> + <list type="bulleted"> + <item><c>badarg</c>: an argument is of wrong type or has an illegal value,</item> + <item><c>low_entropy</c>: the random generator failed due to lack of secure "randomness",</item> + <item><c>computation_failed</c>: the computation fails of another reason than <c>low_entropy</c>.</item> + </list> + <note> + <p>RSA key generation is only available if the runtime was + built with dirty scheduler support. Otherwise, attempting to + generate an RSA key will throw exception <c>error:notsup</c>.</p> + </note> </desc> </func> @@ -648,10 +658,11 @@ </type> <desc> <p>Set the seed for PRNG to the given binary. This calls the - RAND_seed function from openssl. Only use this if the system - you are running on does not have enough "randomness" built in. - Normally this is when <seealso marker="#strong_rand_bytes/1"> - strong_rand_bytes/1</seealso> returns <c>low_entropy</c></p> + RAND_seed function from openssl. Only use this if the system + you are running on does not have enough "randomness" built in. + Normally this is when + <seealso marker="#strong_rand_bytes/1">strong_rand_bytes/1</seealso> + throws <c>low_entropy</c></p> </desc> </func> @@ -718,6 +729,43 @@ failed due to lack of secure "randomness".</p> </desc> </func> + + <func> + <name>rand_seed() -> rand:state()</name> + <fsummary>Strong random number generation plugin state</fsummary> + <desc> + <p> + Creates state object for + <seealso marker="stdlib:rand">random number generation</seealso>, + in order to generate cryptographically strong random numbers + (based on OpenSSL's <c>BN_rand_range</c>), + and saves it on process dictionary before returning it as well. + See also + <seealso marker="stdlib:rand#seed-1">rand:seed/1</seealso>. + </p> + <p><em>Example</em></p> + <pre> +_ = crypto:rand_seed(), +_IntegerValue = rand:uniform(42), % [1; 42] +_FloatValue = rand:uniform(). % [0.0; 1.0[</pre> + </desc> + </func> + + <func> + <name>rand_seed_s() -> rand:state()</name> + <fsummary>Strong random number generation plugin state</fsummary> + <desc> + <p> + Creates state object for + <seealso marker="stdlib:rand">random number generation</seealso>, + in order to generate cryptographically strongly random numbers + (based on OpenSSL's <c>BN_rand_range</c>). + See also + <seealso marker="stdlib:rand#seed_s-1">rand:seed_s/1</seealso>. + </p> + </desc> + </func> + <func> <name>stream_init(Type, Key) -> State</name> <fsummary></fsummary> diff --git a/lib/crypto/doc/src/notes.xml b/lib/crypto/doc/src/notes.xml index 53ea6bb58b..887aeca680 100644 --- a/lib/crypto/doc/src/notes.xml +++ b/lib/crypto/doc/src/notes.xml @@ -31,6 +31,40 @@ </header> <p>This document describes the changes made to the Crypto application.</p> +<section><title>Crypto 3.7.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix a bug with AES CFB 128 for 192 and 256 bit keys. + Thanks to kellymclaughlin !</p> + <p> + Own Id: OTP-14313 Aux Id: PR-1393 </p> + </item> + </list> + </section> + +</section> + +<section><title>Crypto 3.7.3</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + The implementation of the key exchange algorithms + diffie-hellman-group-exchange-sha* are optimized, up to a + factor of 11 for the slowest ( = biggest and safest) + group size.</p> + <p> + Own Id: OTP-14169 Aux Id: seq-13261 </p> + </item> + </list> + </section> + +</section> + <section><title>Crypto 3.7.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/crypto/src/crypto.app.src b/lib/crypto/src/crypto.app.src index 460894c012..3bf4279ae1 100644 --- a/lib/crypto/src/crypto.app.src +++ b/lib/crypto/src/crypto.app.src @@ -25,6 +25,6 @@ {registered, []}, {applications, [kernel, stdlib]}, {env, [{fips_mode, false}]}, - {runtime_dependencies, ["erts-6.0","stdlib-2.0","kernel-3.0"]}]}. + {runtime_dependencies, ["erts-9.0","stdlib-3.4","kernel-5.3"]}]}. diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index 5a915d4233..1287ec6176 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -30,6 +30,12 @@ -export([hmac/3, hmac/4, hmac_init/2, hmac_update/2, hmac_final/1, hmac_final_n/2]). -export([cmac/3, cmac/4]). -export([exor/2, strong_rand_bytes/1, mod_pow/3]). +-export([rand_seed/0]). +-export([rand_seed_s/0]). +-export([rand_plugin_next/1]). +-export([rand_plugin_uniform/1]). +-export([rand_plugin_uniform/2]). +-export([rand_plugin_jump/1]). -export([rand_uniform/2]). -export([block_encrypt/3, block_decrypt/3, block_encrypt/4, block_decrypt/4]). -export([next_iv/2, next_iv/3]). @@ -40,9 +46,14 @@ -export([ec_curve/1, ec_curves/0]). -export([rand_seed/1]). +-deprecated({rand_uniform, 2, next_major_release}). + %% This should correspond to the similar macro in crypto.c -define(MAX_BYTES_TO_NIF, 20000). %% Current value is: erlang:system_info(context_reductions) * 10 +%% Used by strong_rand_float/0 +-define(HALF_DBL_EPSILON, 1.1102230246251565e-16). % math:pow(2, -53) + %%-type ecdsa_digest_type() :: 'md5' | 'sha' | 'sha256' | 'sha384' | 'sha512'. -type crypto_integer() :: binary() | integer(). %%-type ec_named_curve() :: atom(). @@ -54,6 +65,7 @@ %%-type ec_curve() :: ec_named_curve() | ec_curve_spec(). %%-type ec_key() :: {Curve :: ec_curve(), PrivKey :: binary() | undefined, PubKey :: ec_point() | undefined}. +-compile(no_native). -on_load(on_load/0). -define(CRYPTO_NIF_VSN,302). @@ -283,9 +295,11 @@ stream_decrypt(State, Data0) -> stream_crypt(fun do_stream_decrypt/2, State, Data, erlang:byte_size(Data), MaxByts, []). %% -%% RAND - pseudo random numbers using RN_ functions in crypto lib +%% RAND - pseudo random numbers using RN_ and BN_ functions in crypto lib %% -spec strong_rand_bytes(non_neg_integer()) -> binary(). +-spec rand_seed() -> rand:state(). +-spec rand_seed_s() -> rand:state(). -spec rand_uniform(crypto_integer(), crypto_integer()) -> crypto_integer(). @@ -297,6 +311,46 @@ strong_rand_bytes(Bytes) -> strong_rand_bytes_nif(_Bytes) -> ?nif_stub. +rand_seed() -> + rand:seed(rand_seed_s()). + +rand_seed_s() -> + {#{ type => ?MODULE, + max => infinity, + next => fun ?MODULE:rand_plugin_next/1, + uniform => fun ?MODULE:rand_plugin_uniform/1, + uniform_n => fun ?MODULE:rand_plugin_uniform/2, + jump => fun ?MODULE:rand_plugin_jump/1}, + no_seed}. + +rand_plugin_next(Seed) -> + {bytes_to_integer(strong_rand_range(1 bsl 64)), Seed}. + +rand_plugin_uniform(State) -> + {strong_rand_float(), State}. + +rand_plugin_uniform(Max, State) -> + {bytes_to_integer(strong_rand_range(Max)) + 1, State}. + +rand_plugin_jump(State) -> + State. + +strong_rand_range(Range) when is_integer(Range), Range > 0 -> + BinRange = int_to_bin(Range), + strong_rand_range(BinRange); +strong_rand_range(BinRange) when is_binary(BinRange) -> + case strong_rand_range_nif(BinRange) of + false -> + erlang:error(low_entropy); + <<BinResult/binary>> -> + BinResult + end. +strong_rand_range_nif(_BinRange) -> ?nif_stub. + +strong_rand_float() -> + WholeRange = strong_rand_range(1 bsl 53), + ?HALF_DBL_EPSILON * bytes_to_integer(WholeRange). + rand_uniform(From,To) when is_binary(From), is_binary(To) -> case rand_uniform_nif(From,To) of <<Len:32/integer, MSB, Rest/binary>> when MSB > 127 -> @@ -325,6 +379,7 @@ rand_uniform_pos(_,_) -> rand_uniform_nif(_From,_To) -> ?nif_stub. + -spec rand_seed(binary()) -> ok. rand_seed(Seed) -> rand_seed_nif(Seed). @@ -452,6 +507,15 @@ generate_key(srp, {user, [Generator, Prime, Version]}, PrivateArg) end, user_srp_gen_key(Private, Generator, Prime); +generate_key(rsa, {ModulusSize, PublicExponent}, undefined) -> + case rsa_generate_key_nif(ModulusSize, ensure_int_as_bin(PublicExponent)) of + error -> + erlang:error(computation_failed, + [rsa,{ModulusSize,PublicExponent}]); + Private -> + {lists:sublist(Private, 2), Private} + end; + generate_key(ecdh, Curve, PrivKey) -> ec_key_generate(nif_curve_params(Curve), ensure_int_as_bin(PrivKey)). @@ -787,6 +851,11 @@ rsa_verify_nif(_Type, _Digest, _Signature, _Key) -> ?nif_stub. ecdsa_verify_nif(_Type, _Digest, _Signature, _Curve, _Key) -> ?nif_stub. %% Public Keys -------------------------------------------------------------------- +%% RSA Rivest-Shamir-Adleman functions +%% + +rsa_generate_key_nif(_Bits, _Exp) -> ?nif_stub. + %% DH Diffie-Hellman functions %% diff --git a/lib/crypto/test/crypto_SUITE.erl b/lib/crypto/test/crypto_SUITE.erl index 31f4e89ffe..1b7456af18 100644 --- a/lib/crypto/test/crypto_SUITE.erl +++ b/lib/crypto/test/crypto_SUITE.erl @@ -36,7 +36,9 @@ all() -> {group, non_fips}, mod_pow, exor, - rand_uniform + rand_uniform, + rand_plugin, + rand_plugin_s ]. groups() -> @@ -119,7 +121,8 @@ groups() -> {sha384, [], [hash, hmac]}, {sha512, [], [hash, hmac]}, {rsa, [], [sign_verify, - public_encrypt + public_encrypt, + generate ]}, {dss, [], [sign_verify]}, {ecdsa, [], [sign_verify]}, @@ -247,6 +250,21 @@ init_per_testcase(cmac, Config) -> % The CMAC functionality was introduced in OpenSSL 1.0.1 {skip, "OpenSSL is too old"} end; +init_per_testcase(generate, Config) -> + case proplists:get_value(type, Config) of + rsa -> + % RSA key generation is a lengthy process, and is only available + % if dirty CPU scheduler support was enabled for this runtime. + case try erlang:system_info(dirty_cpu_schedulers) of + N -> N > 0 + catch + error:badarg -> false + end of + true -> Config; + false -> {skip, "RSA key generation requires dirty scheduler support."} + end; + _ -> Config + end; init_per_testcase(_Name,Config) -> Config. @@ -470,6 +488,17 @@ rand_uniform(Config) when is_list(Config) -> 10 = byte_size(crypto:strong_rand_bytes(10)). %%-------------------------------------------------------------------- +rand_plugin() -> + [{doc, "crypto rand plugin testing (implicit state / process dictionary)"}]. +rand_plugin(Config) when is_list(Config) -> + rand_plugin_aux(implicit_state). + +rand_plugin_s() -> + [{doc, "crypto rand plugin testing (explicit state)"}]. +rand_plugin_s(Config) when is_list(Config) -> + rand_plugin_aux(explicit_state). + +%%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- hash(_, [], []) -> @@ -756,7 +785,10 @@ do_generate({ecdh = Type, Curve, Priv, Pub}) -> ok; {Other, _} -> ct:fail({{crypto, generate_key, [Type, Priv, Curve]}, {expected, Pub}, {got, Other}}) - end. + end; +do_generate({rsa = Type, Mod, Exp}) -> + {Pub,Priv} = crypto:generate_key(Type, {Mod,Exp}), + do_sign_verify({rsa, sha256, Pub, Priv, rsa_plain()}). notsup(Fun, Args) -> Result = @@ -932,6 +964,101 @@ crypto_rand_uniform(L,H) -> ct:fail({"Not in interval", R1, L, H}) end. +foldallmap(_Fun, AccN, []) -> + {true, AccN}; +foldallmap(Fun, AccN, [H|T]) -> + case Fun(H, AccN) of + {true, AccM} -> foldallmap(Fun, AccM, T); + {{false, Result}, AccM} -> {Result, AccM} + end. + +allmap(_Fun, []) -> + true; +allmap(Fun, [H|T]) -> + case Fun(H) of + true -> allmap(Fun, T); + {false, Result} -> Result + end. + +rand_plugin_aux(StateType) -> + {Seeder, SeedExporter, FloatGenerator, IntegerGenerator} = rand_plugin_functions(StateType), + State0 = Seeder(), + {crypto, no_seed} = SeedExporter(State0), + {FloatTestResult, State1} = rand_plugin_aux_floats(State0, FloatGenerator), + case FloatTestResult of + true -> + {IntegerTestResult, _State2} = rand_plugin_aux_integers(State1, IntegerGenerator), + IntegerTestResult; + {false, _} -> + FloatTestResult + end. + +% returns {Seeder, SeedExporter, FloatGenerator, IntegerGenerator} with consistent signatures +rand_plugin_functions(implicit_state) -> + {fun () -> crypto:rand_seed(), implicit_state end, + fun (implicit_state) -> rand:export_seed() end, + fun (implicit_state) -> {rand:uniform(), implicit_state} end, + fun (N, implicit_state) -> {rand:uniform(N), implicit_state} end}; +rand_plugin_functions(explicit_state) -> + {fun crypto:rand_seed_s/0, + fun rand:export_seed_s/1, + fun rand:uniform_s/1, + fun rand:uniform_s/2}. + +rand_plugin_aux_floats(State0, FloatGenerator) -> + {FloatSamples, State1} = + lists:mapfoldl( + fun (_, StateAcc) -> + FloatGenerator(StateAcc) + end, + State0, + lists:seq(1, 10000)), + + {allmap( + fun (V) -> + (V >= 0.0 andalso V < 1.0) + orelse {false, ct:fail({"Float sample not in interval", V, 0.0, 1.0})} + end, + FloatSamples), + State1}. + +rand_plugin_aux_integers(State0, IntegerGenerator) -> + MaxIntegerCeiling = 1 bsl 32, + {IntegerCeilings, State1} = + lists:mapfoldl( + fun (_, StateAcc) -> + IntegerGenerator(MaxIntegerCeiling, StateAcc) + end, + State0, + lists:seq(1, 100)), + + foldallmap( + fun (Ceiling, StateAcc) -> + case Ceiling >= 1 andalso Ceiling =< MaxIntegerCeiling of + false -> + {{false, ct:fail({"Integer ceiling not in interval", + Ceiling, 1, MaxIntegerCeiling})}, + StateAcc}; + true -> + foldallmap( + fun (_, SubStateAcc) -> + {Sample, NewSubStateAcc} = IntegerGenerator(Ceiling, SubStateAcc), + case Sample >= 1 andalso Sample =< Ceiling of + false -> + {{false, ct:fail({"Integer sample not in interval", + Sample, 1, Ceiling})}, + NewSubStateAcc}; + true -> + {true, NewSubStateAcc} + end + end, + StateAcc, + lists:seq(1, 100)) + end + end, + State1, + IntegerCeilings). + %%-------------------------------------------------------------------- %% Test data ------------------------------------------------ %%-------------------------------------------------------------------- @@ -1008,7 +1135,8 @@ group_config(rsa = Type, Config) -> rsa_oaep(), no_padding() ], - [{sign_verify, SignVerify}, {pub_priv_encrypt, PubPrivEnc} | Config]; + Generate = [{rsa, 2048, 3}, {rsa, 3072, 65537}], + [{sign_verify, SignVerify}, {pub_priv_encrypt, PubPrivEnc}, {generate, Generate} | Config]; group_config(dss = Type, Config) -> Msg = dss_plain(), Public = dss_params() ++ [dss_public()], diff --git a/lib/crypto/vsn.mk b/lib/crypto/vsn.mk index 38e2db9033..f3e0623ac9 100644 --- a/lib/crypto/vsn.mk +++ b/lib/crypto/vsn.mk @@ -1 +1 @@ -CRYPTO_VSN = 3.7.2 +CRYPTO_VSN = 3.7.4 diff --git a/lib/debugger/src/dbg_ieval.erl b/lib/debugger/src/dbg_ieval.erl index f5e079ef7e..88c7caacb0 100644 --- a/lib/debugger/src/dbg_ieval.erl +++ b/lib/debugger/src/dbg_ieval.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2016. All Rights Reserved. +%% Copyright Ericsson AB 1998-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1486,7 +1486,6 @@ guard_expr({map,_,E0,Fs0}, Bs) -> Value = lists:foldl(fun ({map_assoc,K,V}, Mi) -> maps:put(K,V,Mi); ({map_exact,K,V}, Mi) -> maps:update(K,V,Mi) end, E, Fs), - io:format("~p~n", [{E,Value}]), {value,Value}; guard_expr({bin,_,Flds}, Bs) -> {value,V,_Bs} = diff --git a/lib/debugger/src/dbg_wx_trace.erl b/lib/debugger/src/dbg_wx_trace.erl index 29c8e8cefb..f4ee30618c 100644 --- a/lib/debugger/src/dbg_wx_trace.erl +++ b/lib/debugger/src/dbg_wx_trace.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2016. All Rights Reserved. +%% Copyright Ericsson AB 2008-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -524,7 +524,8 @@ gui_cmd({edit, {Var, Value}}, State) -> cancel -> State; {Var, Term} -> - Cmd = atom_to_list(Var)++"="++io_lib:format("~w", [Term]), + %% The space after "=" is needed for handling "B= <<1>>". + Cmd = atom_to_list(Var)++"= "++io_lib:format("~w", [Term]), gui_cmd({user_command, lists:flatten(Cmd)}, State) end. diff --git a/lib/debugger/src/dbg_wx_win.erl b/lib/debugger/src/dbg_wx_win.erl index d302423077..2c9d83ea74 100644 --- a/lib/debugger/src/dbg_wx_win.erl +++ b/lib/debugger/src/dbg_wx_win.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2016. All Rights Reserved. +%% Copyright Ericsson AB 2008-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -276,7 +276,7 @@ verify(Type, Str) -> case erl_scan:string(Str) of {ok, Tokens, _EndLine} when Type==term -> - case erl_parse:parse_term(Tokens++[{dot, 1}]) of + case erl_parse:parse_term(Tokens++[{dot, erl_anno:new(1)}]) of {ok, Value} -> {edit, Value}; _Error -> ignore diff --git a/lib/debugger/test/int_SUITE.erl b/lib/debugger/test/int_SUITE.erl index f697ace4e5..cb1fcb83f3 100644 --- a/lib/debugger/test/int_SUITE.erl +++ b/lib/debugger/test/int_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2016. All Rights Reserved. +%% Copyright Ericsson AB 1998-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -241,7 +241,8 @@ interpretable(Config) when is_list(Config) -> true = code:del_path(PrivDir), %% {error, no_src} - {ok, lists2, Binary} = compile:forms([{attribute,1,module,lists2}], []), + A1 = erl_anno:new(1), + {ok, lists2, Binary} = compile:forms([{attribute,A1,module,lists2}], []), code:load_binary(lists2, "unknown", Binary), {error, no_src} = int:interpretable(lists2), diff --git a/lib/dialyzer/doc/src/notes.xml b/lib/dialyzer/doc/src/notes.xml index 54abd09504..cd4ec4c068 100644 --- a/lib/dialyzer/doc/src/notes.xml +++ b/lib/dialyzer/doc/src/notes.xml @@ -32,6 +32,48 @@ <p>This document describes the changes made to the Dialyzer application.</p> +<section><title>Dialyzer 3.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Fix a bug concerning parameterized opaque types. </p> + <p> + Own Id: OTP-14130</p> + </item> + <item> + <p> Improve a few warnings. One of them could cause a + crash. </p> + <p> + Own Id: OTP-14177</p> + </item> + <item> + <p>The dialyzer and observer applications will now use a + portable way to find the home directory. That means that + there is no longer any need to manually set the HOME + environment variable on Windows.</p> + <p> + Own Id: OTP-14249 Aux Id: ERL-161 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> The peak memory consumption is reduced. </p><p> The + evaluation of huge SCCs in <c>dialyzer_typesig</c> is + optimized. </p><p> Analyzing modules with binary + construction with huge strings is now much faster. </p> + <p> + Own Id: OTP-14126 Aux Id: ERL-308 </p> + </item> + </list> + </section> + +</section> + <section><title>Dialyzer 3.0.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/dialyzer/src/dialyzer.app.src b/lib/dialyzer/src/dialyzer.app.src index 5b28f7ae86..f517c51ec1 100644 --- a/lib/dialyzer/src/dialyzer.app.src +++ b/lib/dialyzer/src/dialyzer.app.src @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2016. All Rights Reserved. +%% Copyright Ericsson AB 2006-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -48,5 +48,5 @@ {applications, [compiler, hipe, kernel, stdlib, wx]}, {env, []}, {runtime_dependencies, ["wx-1.2","syntax_tools-2.0","stdlib-3.0", - "kernel-5.0","hipe-3.15.1","erts-8.0", + "kernel-5.0","hipe-3.15.4","erts-8.0", "compiler-7.0"]}]}. diff --git a/lib/dialyzer/src/dialyzer_dataflow.erl b/lib/dialyzer/src/dialyzer_dataflow.erl index dc2238e63a..4c29b4f1eb 100644 --- a/lib/dialyzer/src/dialyzer_dataflow.erl +++ b/lib/dialyzer/src/dialyzer_dataflow.erl @@ -1344,8 +1344,6 @@ do_clause(C, Arg, ArgType0, OrigArgType, Map, State, Warns) -> {Msg, Force} = case t_is_none(ArgType0) of true -> - PatString = format_patterns(Pats), - PatTypes = [PatString, format_type(OrigArgType, State1)], %% See if this is covered by an earlier clause or if it %% simply cannot match OrigArgTypes = @@ -1353,14 +1351,24 @@ do_clause(C, Arg, ArgType0, OrigArgType, Map, State, Warns) -> true -> Any = t_any(), [Any || _ <- Pats]; false -> t_to_tlist(OrigArgType) end, + PatString = format_patterns(Pats), + ArgTypeString = format_type(OrigArgType, State1), + BindResOrig = + bind_pat_vars(Pats, OrigArgTypes, [], Map1, State1), Tag = - case bind_pat_vars(Pats, OrigArgTypes, [], Map1, State1) of + case BindResOrig of {error, bind, _, _, _} -> pattern_match; {error, record, _, _, _} -> record_match; {error, opaque, _, _, _} -> opaque_match; {_, _} -> pattern_match_cov end, - {{Tag, PatTypes}, false}; + PatTypes = case BindResOrig of + {error, opaque, _, _, OpaqueType} -> + [PatString, ArgTypeString, + format_type(OpaqueType, State1)]; + _ -> [PatString, ArgTypeString] + end, + {{Tag, PatTypes}, false}; false -> %% Try to find out if this is a default clause in a list %% comprehension and suppress this. A real Hack(tm) diff --git a/lib/dialyzer/src/dialyzer_plt.erl b/lib/dialyzer/src/dialyzer_plt.erl index eb63e9e695..bfd3f84fc5 100644 --- a/lib/dialyzer/src/dialyzer_plt.erl +++ b/lib/dialyzer/src/dialyzer_plt.erl @@ -233,12 +233,8 @@ contains_mfa(#plt{info = Info, contracts = Contracts}, MFA) -> get_default_plt() -> case os:getenv("DIALYZER_PLT") of false -> - case os:getenv("HOME") of - false -> - plt_error("The HOME environment variable needs to be set " ++ - "so that Dialyzer knows where to find the default PLT"); - HomeDir -> filename:join(HomeDir, ".dialyzer_plt") - end; + {ok,[[HomeDir]]} = init:get_argument(home), + filename:join(HomeDir, ".dialyzer_plt"); UserSpecPlt -> UserSpecPlt end. diff --git a/lib/dialyzer/src/dialyzer_typesig.erl b/lib/dialyzer/src/dialyzer_typesig.erl index c4f8adf7ee..c3ba44fde7 100644 --- a/lib/dialyzer/src/dialyzer_typesig.erl +++ b/lib/dialyzer/src/dialyzer_typesig.erl @@ -2080,8 +2080,6 @@ v2_solve_disjunct(Disj, Map, V2State0) -> var_occurs_everywhere(V, Masks, NotFailed) -> ordsets:is_subset(NotFailed, get_mask(V, Masks)). --dialyzer({no_improper_lists, [v2_solve_disj/10, v2_solve_conj/12]}). - v2_solve_disj([I|Is], [C|Cs], I, Map0, V2State0, UL, MapL, Eval, Uneval, Failed0) -> Id = C#constraint_list.id, @@ -2100,10 +2098,10 @@ v2_solve_disj([I|Is], [C|Cs], I, Map0, V2State0, UL, MapL, Eval, Uneval, end; v2_solve_disj([], [], _I, _Map, V2State, UL, MapL, Eval, Uneval, Failed) -> {ok, V2State, lists:reverse(Eval), UL, MapL, lists:reverse(Uneval), Failed}; -v2_solve_disj(every_i, Cs, I, Map, V2State, UL, MapL, Eval, Uneval, Failed) -> +v2_solve_disj([every_i], Cs, I, Map, V2State, UL, MapL, Eval, Uneval, Failed) -> NewIs = case Cs of [] -> []; - _ -> [I|every_i] + _ -> [I, every_i] end, v2_solve_disj(NewIs, Cs, I, Map, V2State, UL, MapL, Eval, Uneval, Failed); v2_solve_disj(Is, [C|Cs], I, Map, V2State, UL, MapL, Eval, Uneval0, Failed) -> @@ -2199,11 +2197,11 @@ v2_solve_conj([], _Cs, _I, Map, Conj, IsFlat, V2State, UL, NewFs, VarsUp, v2_solve_conj(NewFlags, Cs, 1, Map, Conj, IsFlat, V2State, [], [], [U|VarsUp], Map, NewFlags) end; -v2_solve_conj(every_i, Cs, I, Map, Conj, IsFlat, V2State, UL, NewFs, VarsUp, +v2_solve_conj([every_i], Cs, I, Map, Conj, IsFlat, V2State, UL, NewFs, VarsUp, LastMap, LastFlags) -> NewIs = case Cs of [] -> []; - _ -> [I|every_i] + _ -> [I, every_i] end, v2_solve_conj(NewIs, Cs, I, Map, Conj, IsFlat, V2State, UL, NewFs, VarsUp, LastMap, LastFlags); @@ -2225,8 +2223,8 @@ add_mask_to_flags(Flags, [Im|M], I, L) when I > Im -> add_mask_to_flags(Flags, [_|M], _I, L) -> {umerge_mask(Flags, M), lists:reverse(L)}. -umerge_mask(every_i, _F) -> - every_i; +umerge_mask([every_i]=Is, _F) -> + Is; umerge_mask(Is, F) -> lists:umerge(Is, F). @@ -2242,7 +2240,7 @@ get_flags(#v2_state{constr_data = ConData}=V2State0, C) -> error -> ?debug("get_flags Id=~w Flags=all ~w\n", [Id, length(Cs)]), V2State = V2State0#v2_state{constr_data = maps:put(Id, {[],[]}, ConData)}, - {V2State, every_i}; + {V2State, [every_i]}; {ok, failed} -> {V2State0, failed_list}; {ok, {Part,U}} when U =/= [] -> diff --git a/lib/dialyzer/test/abstract_SUITE.erl b/lib/dialyzer/test/abstract_SUITE.erl index 269db3e836..37fb39cf27 100644 --- a/lib/dialyzer/test/abstract_SUITE.erl +++ b/lib/dialyzer/test/abstract_SUITE.erl @@ -7,7 +7,7 @@ -include_lib("common_test/include/ct.hrl"). -include("dialyzer_test_constants.hrl"). --export([suite/0, all/0, init_per_suite/0, init_per_suite/1]). +-export([suite/0, all/0, init_per_suite/0, init_per_suite/1, end_per_suite/1]). -export([generated_case/1]). suite() -> @@ -24,6 +24,10 @@ init_per_suite(Config) -> ok -> [{dialyzer_options, []}|Config] end. +end_per_suite(_Config) -> + %% This function is required since init_per_suite/1 exists. + ok. + generated_case(Config) when is_list(Config) -> %% Equivalent to: %% @@ -79,7 +83,8 @@ generated_case(Config) when is_list(Config) -> Config, [], []), ok. -test(Prog, Config, COpts, DOpts) -> +test(Prog0, Config, COpts, DOpts) -> + Prog = erl_parse:anno_from_term(Prog0), {ok, BeamFile} = compile(Config, Prog, COpts), run_dialyzer(Config, succ_typings, [BeamFile], DOpts). diff --git a/lib/dialyzer/test/opaque_SUITE_data/results/weird b/lib/dialyzer/test/opaque_SUITE_data/results/weird new file mode 100644 index 0000000000..d7f57cd152 --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/results/weird @@ -0,0 +1,6 @@ + +weird_warning1.erl:15: Matching of pattern {'a', Dict} tagged with a record name violates the declared type of #b{q::queue:queue(_)} +weird_warning2.erl:13: Matching of pattern <{'b', Queue}, Key, Value> tagged with a record name violates the declared type of <#a{d::dict:dict(_,_)},'my_key','my_value'> +weird_warning3.erl:14: The call weird_warning3:add_element(#a{d::queue:queue(_)},'my_key','my_value') does not have a term of type #a{d::dict:dict(_,_)} | #b{q::queue:queue(_)} (with opaque subterms) as 1st argument +weird_warning3.erl:16: The attempt to match a term of type #a{d::queue:queue(_)} against the pattern {'a', Dict} breaks the opacity of queue:queue(_) +weird_warning3.erl:18: Matching of pattern {'b', Queue} tagged with a record name violates the declared type of #a{d::queue:queue(_)} diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/weird/weird_warning1.erl b/lib/dialyzer/test/opaque_SUITE_data/src/weird/weird_warning1.erl new file mode 100644 index 0000000000..094138e72b --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/weird/weird_warning1.erl @@ -0,0 +1,18 @@ +-module(weird_warning1). +-export([public_func/0]). + +-record(a, { + d = dict:new() :: dict:dict() + }). + +-record(b, { + q = queue:new() :: queue:queue() + }). + +public_func() -> + add_element(#b{}, my_key, my_value). + +add_element(#a{d = Dict}, Key, Value) -> + dict:store(Key, Value, Dict); +add_element(#b{q = Queue}, Key, Value) -> + queue:in({Key, Value}, Queue). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/weird/weird_warning2.erl b/lib/dialyzer/test/opaque_SUITE_data/src/weird/weird_warning2.erl new file mode 100644 index 0000000000..4e4512157b --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/weird/weird_warning2.erl @@ -0,0 +1,14 @@ +-module(weird_warning2). +-export([public_func/0]). + +-record(a, {d = dict:new() :: dict:dict()}). + +-record(b, {q = queue:new() :: queue:queue()}). + +public_func() -> + add_element(#a{}, my_key, my_value). + +add_element(#a{d = Dict}, Key, Value) -> + dict:store(Key, Value, Dict); +add_element(#b{q = Queue}, Key, Value) -> + queue:in({Key, Value}, Queue). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/weird/weird_warning3.erl b/lib/dialyzer/test/opaque_SUITE_data/src/weird/weird_warning3.erl new file mode 100644 index 0000000000..b70ca645cb --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/weird/weird_warning3.erl @@ -0,0 +1,19 @@ +-module(weird_warning3). +-export([public_func/0]). + +-record(a, { + d = dict:new() :: dict:dict() + }). + +-record(b, { + q = queue:new() :: queue:queue() + }). + +public_func() -> + %% Notice that t_to_string() will create "#a{d::queue:queue(_)}". + add_element({a, queue:new()}, my_key, my_value). + +add_element(#a{d = Dict}, Key, Value) -> + dict:store(Key, Value, Dict); +add_element(#b{q = Queue}, Key, Value) -> + queue:in({Key, Value}, Queue). diff --git a/lib/dialyzer/test/plt_SUITE.erl b/lib/dialyzer/test/plt_SUITE.erl index fbfa979e1b..ba153c1c27 100644 --- a/lib/dialyzer/test/plt_SUITE.erl +++ b/lib/dialyzer/test/plt_SUITE.erl @@ -260,6 +260,8 @@ remove_plt(Config) -> ok. bad_dialyzer_attr(Config) -> + PrivDir = ?config(priv_dir, Config), + Plt = filename:join(PrivDir, "plt_bad_dialyzer_attr.plt"), Prog1 = <<"-module(dial). -dialyzer({no_return, [undef/0]}).">>, {ok, Beam1} = compile(Config, Prog1, dial, []), @@ -267,7 +269,7 @@ bad_dialyzer_attr(Config) -> "Analysis failed with error:\n" "Could not scan the following file(s):\n" " Unknown function undef/0 in line " ++ _} = - (catch run_dialyzer(plt_build, [Beam1], [])), + (catch run_dialyzer(plt_build, [Beam1], [{output_plt, Plt}])), Prog2 = <<"-module(dial). -dialyzer({no_return, [{undef,1,2}]}).">>, @@ -276,7 +278,7 @@ bad_dialyzer_attr(Config) -> "Analysis failed with error:\n" "Could not scan the following file(s):\n" " Bad function {undef,1,2} in line " ++ _} = - (catch run_dialyzer(plt_build, [Beam2], [])), + (catch run_dialyzer(plt_build, [Beam2], [{output_plt, Plt}])), ok. diff --git a/lib/dialyzer/test/r9c_SUITE_data/src/inets/httpd_response.erl b/lib/dialyzer/test/r9c_SUITE_data/src/inets/httpd_response.erl index 47c7fc1b8d..50e0e42786 100644 --- a/lib/dialyzer/test/r9c_SUITE_data/src/inets/httpd_response.erl +++ b/lib/dialyzer/test/r9c_SUITE_data/src/inets/httpd_response.erl @@ -34,7 +34,7 @@ -define(PROCEED_RESPONSE(StatusCode, Info), {proceed, [{response,{already_sent, StatusCode, - httpd_util:key1search(Info#mod.data,content_lenght)}}]}). + httpd_util:key1search(Info#mod.data,content_length)}}]}). -include("httpd.hrl"). diff --git a/lib/dialyzer/vsn.mk b/lib/dialyzer/vsn.mk index 9830a36e60..0919fba834 100644 --- a/lib/dialyzer/vsn.mk +++ b/lib/dialyzer/vsn.mk @@ -1 +1 @@ -DIALYZER_VSN = 3.0.3 +DIALYZER_VSN = 3.1 diff --git a/lib/diameter/doc/src/notes.xml b/lib/diameter/doc/src/notes.xml index c2bbed2e5a..70e1880be5 100644 --- a/lib/diameter/doc/src/notes.xml +++ b/lib/diameter/doc/src/notes.xml @@ -43,6 +43,30 @@ first.</p> <!-- ===================================================================== --> +<section><title>diameter 1.12.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + An improvement in the handling of peer failover in + diameter 1.12.1 adversely affected performance when + sending requests. Further, the inefficient use of a + public table to route incoming answers has been removed.</p> + <p> + Own Id: OTP-14206</p> + </item> + <item> + <p> + Fixed xml issues in old release notes</p> + <p> + Own Id: OTP-14269</p> + </item> + </list> + </section> + +</section> + <section><title>diameter 1.12.1</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -255,8 +279,8 @@ first.</p> Fix decode of Grouped AVPs containing errors.</p> <p> RFC 6733 says this of Failed-AVP in 7.5:</p> - <p> - <taglist><item><p><c> In the case where the offending AVP + + <taglist><tag></tag><item><p><c> In the case where the offending AVP is embedded within a Grouped AVP, the Failed-AVP MAY contain the grouped AVP, which in turn contains the single offending AVP. The same method MAY be employed if @@ -265,11 +289,11 @@ first.</p> the grouped AVP hierarchy up to the single offending AVP. This enables the recipient to detect the location of the offending AVP when embedded in a - group.</c></p></item></taglist></p> + group.</c></p></item></taglist> <p> It says this of DIAMETER_INVALID_AVP_LENGTH in 7.1.5:</p> - <p> - <taglist><item><p><c> The request contained an AVP with + + <taglist><tag></tag><item><p><c> The request contained an AVP with an invalid length. A Diameter message indicating this error MUST include the offending AVPs within a Failed-AVP AVP. In cases where the erroneous AVP length value @@ -284,7 +308,8 @@ first.</p> the minimum AVP header length, it is sufficient to include an offending AVP header that is formulated by padding the incomplete AVP header with zero up to the - minimum AVP header length.</c></p></item></taglist></p> + minimum AVP header length.</c></p></item></taglist> + <p> The AVPs placed in the errors field of a diameter_packet record are intended to be appropriate for inclusion in a @@ -949,8 +974,8 @@ first.</p> Be lenient with the M-bit in Grouped AVPs.</p> <p> RFC 6733 says this, in 4.4:</p> - <p> - <taglist><item><p><c>Receivers of a Grouped AVP that does + + <taglist><tag></tag><item><p><c>Receivers of a Grouped AVP that does not have the 'M' (mandatory) bit set and one or more of the encapsulated AVPs within the group has the 'M' (mandatory) bit set MAY simply be ignored if the Grouped @@ -958,14 +983,14 @@ first.</p> encapsulated AVP with its 'M' (mandatory) bit set is further encapsulated within other sub-groups, i.e., other Grouped AVPs embedded within the Grouped - AVP.</c></p></item></taglist></p> + AVP.</c></p></item></taglist> <p> The first sentence is mangled but take it to mean this:</p> - <p> - <taglist><item><p><c>An unrecognized AVP of type Grouped + + <taglist><tag></tag><item><p><c>An unrecognized AVP of type Grouped that does not set the 'M' bit MAY be ignored even if one of its encapsulated AVPs sets the 'M' - bit.</c></p></item></taglist></p> + bit.</c></p></item></taglist> <p> This is a bit of a non-statement since if the AVP is unrecognized then its type is unknown. We therefore don't diff --git a/lib/diameter/src/base/diameter.erl b/lib/diameter/src/base/diameter.erl index e8f2f63f86..253f64133c 100644 --- a/lib/diameter/src/base/diameter.erl +++ b/lib/diameter/src/base/diameter.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -377,6 +377,7 @@ call(SvcName, App, Message) -> | {capabilities, [capability()]} | {capabilities_cb, evaluable()} | {capx_timeout, 'Unsigned32'()} + | {capx_strictness, boolean()} | {disconnect_cb, evaluable()} | {dpr_timeout, 'Unsigned32'()} | {dpa_timeout, 'Unsigned32'()} diff --git a/lib/diameter/src/base/diameter_config.erl b/lib/diameter/src/base/diameter_config.erl index 1b48c0431f..e10804c931 100644 --- a/lib/diameter/src/base/diameter_config.erl +++ b/lib/diameter/src/base/diameter_config.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -580,6 +580,9 @@ opt({K, Tmo}) K == dpa_timeout -> ?IS_UINT32(Tmo); +opt({capx_strictness, B}) -> + is_boolean(B); + opt({length_errors, T}) -> lists:member(T, [exit, handle, discard]); diff --git a/lib/diameter/src/base/diameter_peer_fsm.erl b/lib/diameter/src/base/diameter_peer_fsm.erl index 4b394a2dbe..46d231da74 100644 --- a/lib/diameter/src/base/diameter_peer_fsm.erl +++ b/lib/diameter/src/base/diameter_peer_fsm.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -128,6 +128,7 @@ %% outgoing DPR; boolean says whether or not %% the request was sent explicitly with %% diameter:call/4. + strict :: boolean(), length_errors :: exit | handle | discard, incoming_maxlen :: integer() | infinity}). @@ -233,6 +234,7 @@ i({Ack, WPid, {M, Ref} = T, Opts, {SvcOpts, Nodes, Dict0, Svc}}) -> proplists:get_value(dpa_timeout, Opts, ?DPA_TIMEOUT)}), Tmo = proplists:get_value(capx_timeout, Opts, ?CAPX_TIMEOUT), + Strictness = proplists:get_value(capx_strictness, Opts, true), OnLengthErr = proplists:get_value(length_errors, Opts, exit), {TPid, Addrs} = start_transport(T, Rest, Svc), @@ -246,6 +248,7 @@ i({Ack, WPid, {M, Ref} = T, Opts, {SvcOpts, Nodes, Dict0, Svc}}) -> mode = M, service = svc(Svc, Addrs), length_errors = OnLengthErr, + strict = Strictness, incoming_maxlen = Maxlen}. %% The transport returns its local ip addresses so that different %% transports on the same service can use different local addresses. @@ -454,6 +457,9 @@ transition({timeout, _}, _) -> %% Outgoing message. transition({send, Msg}, S) -> outgoing(Msg, S); +transition({send, Msg, Route}, S) -> + put_route(Route), + outgoing(Msg, S); %% Request for graceful shutdown at remove_transport, stop_service of %% application shutdown. @@ -483,8 +489,10 @@ transition({'DOWN', _, process, TPid, _}, = S) -> start_next(S); -%% Transport has died after connection timeout. -transition({'DOWN', _, process, _, _}, _) -> +%% Transport has died after connection timeout, or handler process has +%% died. +transition({'DOWN', _, process, Pid, _}, _) -> + erase_route(Pid), ok; %% State query. @@ -494,6 +502,40 @@ transition({state, Pid}, #state{state = S, transport = TPid}) -> %% Crash on anything unexpected. +%% put_route/1 +%% +%% Map identifiers in an outgoing request to be able to lookup the +%% handler process when the answer is received. + +put_route({Pid, Ref, Seqs}) -> + MRef = monitor(process, Pid), + put(Pid, Seqs), + put(Seqs, {Pid, Ref, MRef}). + +%% get_route/1 + +get_route(#diameter_packet{header = #diameter_header{is_request = false}} + = Pkt) -> + Seqs = diameter_codec:sequence_numbers(Pkt), + case erase(Seqs) of + {Pid, Ref, MRef} -> + demonitor(MRef), + erase(Pid), + {Pid, Ref, self()}; + undefined -> + false + end; + +get_route(_) -> + false. + +%% erase_route/1 + +erase_route(Pid) -> + erase(erase(Pid)). + +%% capx/1 + capx(recv_CER) -> 'CER'; capx({'Wait-CEA', _, _}) -> @@ -576,8 +618,7 @@ incoming({Msg, NPid}, S) -> T catch {?MODULE, Name, Pkt} -> - S#state.parent ! {recv, self(), Name, {Pkt, NPid}}, - rcv(Name, Pkt, S) + incoming(Name, Pkt, NPid, S) end; incoming(Msg, S) -> @@ -585,10 +626,15 @@ incoming(Msg, S) -> recv(Msg, S) catch {?MODULE, Name, Pkt} -> - S#state.parent ! {recv, self(), Name, Pkt}, - rcv(Name, Pkt, S) + incoming(Name, Pkt, false, S) end. +%% incoming/4 + +incoming(Name, Pkt, NPid, #state{parent = Pid} = S) -> + Pid ! {recv, self(), get_route(Pkt), Name, Pkt, NPid}, + rcv(Name, Pkt, S). + %% recv/2 recv(#diameter_packet{header = #diameter_header{} = Hdr} @@ -614,6 +660,17 @@ recv1(_, when M < size(Bin) -> invalid(false, incoming_maxlen_exceeded, {size(Bin), H}); +%% Ignore anything but an expected CER/CEA if so configured. This is +%% non-standard behaviour. +recv1(Name, _, #state{state = {'Wait-CEA', _, _}, + strict = false}) + when Name /= 'CEA' -> + ok; +recv1(Name, _, #state{state = recv_CER, + strict = false}) + when Name /= 'CER' -> + ok; + %% Incoming request after outgoing DPR: discard. Don't discard DPR, so %% both ends don't do so when sending simultaneously. recv1(Name, diff --git a/lib/diameter/src/base/diameter_service.erl b/lib/diameter/src/base/diameter_service.erl index ccf68f4d93..e4f77e3a24 100644 --- a/lib/diameter/src/base/diameter_service.erl +++ b/lib/diameter/src/base/diameter_service.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1858,13 +1858,6 @@ eq(Any, Id, PeerId) -> %% OctetString() can be specified as an iolist() so test for string %% rather then term equality. -%% transports/1 - -transports(#state{watchdogT = WatchdogT}) -> - ets:select(WatchdogT, [{#watchdog{peer = '$1', _ = '_'}, - [{'is_pid', '$1'}], - ['$1']}]). - %% --------------------------------------------------------------------------- %% # service_info/2 %% --------------------------------------------------------------------------- @@ -1887,7 +1880,6 @@ transports(#state{watchdogT = WatchdogT}) -> -define(ALL_INFO, [capabilities, applications, transport, - pending, options]). %% The rest. @@ -1981,7 +1973,6 @@ complete_info(Item, #state{service = Svc} = S) -> applications -> info_apps(S); transport -> info_transport(S); options -> info_options(S); - pending -> info_pending(S); keys -> ?ALL_INFO ++ ?CAP_INFO ++ ?OTHER_INFO; all -> service_info(?ALL_INFO, S); statistics -> info_stats(S); @@ -2189,13 +2180,6 @@ info_apps(#state{service = #diameter_service{applications = Apps}}) -> mk_app(#diameter_app{} = A) -> lists:zip(record_info(fields, diameter_app), tl(tuple_to_list(A))). -%% info_pending/1 -%% -%% One entry for each outgoing request whose answer is outstanding. - -info_pending(#state{} = S) -> - diameter_traffic:pending(transports(S)). - %% info_info/1 %% %% Extract process_info from connections info. diff --git a/lib/diameter/src/base/diameter_sup.erl b/lib/diameter/src/base/diameter_sup.erl index 482289cb9a..01c51f0856 100644 --- a/lib/diameter/src/base/diameter_sup.erl +++ b/lib/diameter/src/base/diameter_sup.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -42,7 +42,7 @@ -define(TABLES, [{diameter_sequence, [set]}, {diameter_service, [set, {keypos, 3}]}, - {diameter_request, [bag]}, + {diameter_request, [set]}, {diameter_config, [bag, {keypos, 2}]}]). %% start_link/0 diff --git a/lib/diameter/src/base/diameter_traffic.erl b/lib/diameter/src/base/diameter_traffic.erl index d93a3e71e3..bc1ccf4feb 100644 --- a/lib/diameter/src/base/diameter_traffic.erl +++ b/lib/diameter/src/base/diameter_traffic.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2016. All Rights Reserved. +%% Copyright Ericsson AB 2013-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -30,7 +30,7 @@ -export([send_request/4]). %% towards diameter_watchdog --export([receive_message/4]). +-export([receive_message/6]). %% towards diameter_peer_fsm and diameter_watchdog -export([incr/4, @@ -40,11 +40,11 @@ %% towards diameter_service -export([make_recvdata/1, peer_up/1, - peer_down/1, - pending/1]). + peer_down/1]). -%% towards ?MODULE --export([send/1]). %% send from remote node +%% internal +-export([send/1, %% send from remote node + init/1]). %% monitor process start -include_lib("diameter/include/diameter.hrl"). -include("diameter_internal.hrl"). @@ -57,14 +57,12 @@ -define(DEFAULT_TIMEOUT, 5000). %% for outgoing requests -define(DEFAULT_SPAWN_OPTS, []). -%% Table containing outgoing requests for which a reply has yet to be -%% received. +%% Table containing outgoing entries that live and die with +%% peer_up/down. The name is historic, since the table used to contain +%% information about outgoing requests for which an answer has yet to +%% be received. -define(REQUEST_TABLE, diameter_request). -%% Workaround for dialyzer's lack of understanding of match specs. --type match(T) - :: T | '_' | '$1' | '$2' | '$3' | '$4'. - %% Record diameter:call/4 options are parsed into. -record(options, {filter = none :: diameter:peer_filter(), @@ -72,7 +70,7 @@ timeout = ?DEFAULT_TIMEOUT :: 0..16#FFFFFFFF, detach = false :: boolean()}). -%% Term passed back to receive_message/4 with every incoming message. +%% Term passed back to receive_message/6 with every incoming message. -record(recvdata, {peerT :: ets:tid(), service_name :: diameter:service_name(), @@ -87,12 +85,12 @@ %% Record stored in diameter_request for each outgoing request. -record(request, - {ref :: match(reference()), %% used to receive answer - caller :: match(pid()), %% calling process - handler :: match(pid()), %% request process - transport :: match(pid()), %% peer process - caps :: match(#diameter_caps{}), %% of connection - packet :: match(#diameter_packet{})}). %% of request + {ref :: reference(), %% used to receive answer + caller :: pid() | undefined, %% calling process + handler :: pid(), %% request process + transport :: pid() | undefined, %% peer process + caps :: #diameter_caps{} | undefined, %% of connection + packet :: #diameter_packet{} | undefined}). %% of request %% --------------------------------------------------------------------------- %% # make_recvdata/1 @@ -113,26 +111,27 @@ make_recvdata([SvcName, PeerT, Apps, SvcOpts | _]) -> %% peer_up/1 %% --------------------------------------------------------------------------- -%% Insert an element that is used to detect whether or not there has -%% been a failover when inserting an outgoing request. +%% Start a process that dies with peer_down/1, on which request +%% processes can monitor. There is no other process that dies with +%% peer_down since failover doesn't imply the loss of transport in the +%% case of a watchdog transition into state SUSPECT. peer_up(TPid) -> - ets:insert(?REQUEST_TABLE, {TPid}). + proc_lib:start(?MODULE, init, [TPid]). + +init(TPid) -> + ets:insert(?REQUEST_TABLE, {TPid, self()}), + proc_lib:init_ack(self()), + proc_lib:hibernate(erlang, exit, [{shutdown, TPid}]). %% --------------------------------------------------------------------------- %% peer_down/1 %% --------------------------------------------------------------------------- peer_down(TPid) -> - ets:delete_object(?REQUEST_TABLE, {TPid}), - lists:foreach(fun failover/1, ets:lookup(?REQUEST_TABLE, TPid)). -%% Note that a request process can store its request after failover -%% notifications are sent here: insert_request/2 sends the notification -%% in that case. - -%% failover/1 - -failover({_TPid, {Pid, TRef}}) -> - Pid ! {failover, TRef}. + [{_, Pid}] = ets:lookup(?REQUEST_TABLE, TPid), + ets:delete(?REQUEST_TABLE, TPid), + Pid ! ok, %% make it die + Pid. %% --------------------------------------------------------------------------- %% incr/4 @@ -207,54 +206,25 @@ incr_rc(Dir, Pkt, TPid, Dict0) -> incr_rc(Dir, Pkt, TPid, {Dict0, Dict0, Dict0}). %% --------------------------------------------------------------------------- -%% pending/1 -%% --------------------------------------------------------------------------- - -pending(TPids) -> - MatchSpec = [{{'$1', - #request{caller = '$2', - handler = '$3', - transport = '$4', - _ = '_'}, - '_'}, - [?ORCOND([{'==', T, '$4'} || T <- TPids])], - [{{'$1', [{{caller, '$2'}}, - {{handler, '$3'}}, - {{transport, '$4'}}]}}]}], - - try - ets:select(?REQUEST_TABLE, MatchSpec) - catch - error: badarg -> [] %% service has gone down - end. - -%% --------------------------------------------------------------------------- -%% # receive_message/4 +%% # receive_message/6 %% %% Handle an incoming Diameter message. %% --------------------------------------------------------------------------- -%% Handle an incoming Diameter message in the watchdog process. This -%% used to come through the service process but this avoids that -%% becoming a bottleneck. +%% Handle an incoming Diameter message in the watchdog process. -receive_message(TPid, {Pkt, NPid}, Dict0, RecvData) -> - NPid ! {diameter, incoming(TPid, Pkt, Dict0, RecvData)}; +receive_message(TPid, Route, Pkt, false, Dict0, RecvData) -> + incoming(TPid, Route, Pkt, Dict0, RecvData); -receive_message(TPid, Pkt, Dict0, RecvData) -> - incoming(TPid, Pkt, Dict0, RecvData). +receive_message(TPid, Route, Pkt, NPid, Dict0, RecvData) -> + NPid ! {diameter, incoming(TPid, Route, Pkt, Dict0, RecvData)}. %% incoming/4 -incoming(TPid, Pkt, Dict0, RecvData) +incoming(TPid, Route, Pkt, Dict0, RecvData) when is_pid(TPid) -> #diameter_packet{header = #diameter_header{is_request = R}} = Pkt, - recv(R, - (not R) andalso lookup_request(Pkt, TPid), - TPid, - Pkt, - Dict0, - RecvData). + recv(R, Route, TPid, Pkt, Dict0, RecvData). %% recv/6 @@ -269,8 +239,8 @@ recv(true, false, TPid, Pkt, Dict0, T) -> end; %% ... answer to known request ... -recv(false, #request{ref = Ref, handler = Pid} = Req, _, Pkt, Dict0, _) -> - Pid ! {answer, Ref, Req, Dict0, Pkt}, +recv(false, {Pid, Ref, TPid}, _, Pkt, Dict0, _) -> + Pid ! {answer, Ref, TPid, Dict0, Pkt}, {answer, Pid}; %% Note that failover could have happened prior to this message being @@ -1503,32 +1473,39 @@ send_R(Pkt0, packet = Pkt0}, incr(send, Pkt, TPid, AppDict), - TRef = send_request(TPid, Pkt, Req, SvcName, Timeout), + {TRef, MRef} = zend_requezt(TPid, Pkt, Req, SvcName, Timeout), Pid ! Ref, %% tell caller a send has been attempted handle_answer(SvcName, App, - recv_A(Timeout, SvcName, App, Opts, {TRef, Req})). + recv_A(Timeout, SvcName, App, Opts, {TRef, MRef, Req})). %% recv_A/5 -recv_A(Timeout, SvcName, App, Opts, {TRef, #request{ref = Ref} = Req}) -> +recv_A(Timeout, SvcName, App, Opts, {TRef, MRef, #request{ref = Ref} = Req}) -> %% Matching on TRef below ensures we ignore messages that pertain %% to a previous transport prior to failover. The answer message - %% includes the #request{} since it's not necessarily Req; that - %% is, from the last peer to which we've transmitted. + %% includes the pid of the transport on which it was received, + %% which may not be the last peer to which we've transmitted. receive - {answer = A, Ref, Rq, Dict0, Pkt} -> %% Answer from peer - {A, Rq, Dict0, Pkt}; + {answer = A, Ref, TPid, Dict0, Pkt} -> %% Answer from peer + {A, #request{} = erase(TPid), Dict0, Pkt}; {timeout = Reason, TRef, _} -> %% No timely reply {error, Req, Reason}; - {failover, TRef} -> %% Service says peer has gone down - retransmit(pick_peer(SvcName, App, Req, Opts), - Req, - Opts, - SvcName, - Timeout) + {'DOWN', MRef, process, _, _} when false /= MRef -> %% local peer_down + failover(SvcName, App, Req, Opts, Timeout); + {failover, TRef} -> %% local or remote peer_down + failover(SvcName, App, Req, Opts, Timeout) end. +%% failover/5 + +failover(SvcName, App, Req, Opts, Timeout) -> + retransmit(pick_peer(SvcName, App, Req, Opts), + Req, + Opts, + SvcName, + Timeout). + %% handle_answer/3 handle_answer(SvcName, App, {error, Req, Reason}) -> @@ -1705,44 +1682,63 @@ encode(DictT, TPid, #diameter_packet{bin = undefined} = Pkt) -> encode(_, _, #diameter_packet{} = Pkt) -> Pkt. +%% zend_requezt/5 +%% +%% Strip potentially large record fields that aren't used by the +%% processes the records can be send to, possibly on a remote node. + +zend_requezt(TPid, Pkt, Req, SvcName, Timeout) -> + put(TPid, Req), + send_request(TPid, z(Pkt), Req, SvcName, Timeout). + %% send_request/5 send_request(TPid, #diameter_packet{bin = Bin} = Pkt, Req, _SvcName, Timeout) when node() == node(TPid) -> Seqs = diameter_codec:sequence_numbers(Bin), TRef = erlang:start_timer(Timeout, self(), TPid), - Entry = {Seqs, #request{handler = Pid} = Req, TRef}, - - %% Ensure that request table is cleaned even if the process is - %% killed. - spawn(fun() -> diameter_lib:wait([Pid]), delete_request(Entry) end), - - insert_request(Entry), - send(TPid, Pkt), - TRef; + send(TPid, Pkt, _Route = {self(), Req#request.ref, Seqs}), + {TRef, _MRef = peer_monitor(TPid, TRef)}; %% Send using a remote transport: spawn a process on the remote node %% to relay the answer. send_request(TPid, #diameter_packet{} = Pkt, Req, SvcName, Timeout) -> TRef = erlang:start_timer(Timeout, self(), TPid), - T = {TPid, Pkt, Req, SvcName, Timeout, TRef}, + T = {TPid, Pkt, z(Req), SvcName, Timeout, TRef}, spawn(node(TPid), ?MODULE, send, [T]), - TRef. + {TRef, false}. + +%% z/1 +%% +%% Avoid sending potentially large terms unnecessarily. The records +%% themselves are retained since they're sent between nodes in send/1 +%% and changing what's sent causes upgrade issues. + +z(#request{ref = Ref, handler = Pid}) -> + #request{ref = Ref, + handler = Pid}; + +z(#diameter_packet{header = H, bin = Bin, transport_data = T}) -> + #diameter_packet{header = H, + bin = Bin, + transport_data = T}. %% send/1 send({TPid, Pkt, #request{handler = Pid} = Req0, SvcName, Timeout, TRef}) -> Req = Req0#request{handler = self()}, - recv(TPid, Pid, TRef, send_request(TPid, Pkt, Req, SvcName, Timeout)). + recv(TPid, Pid, TRef, zend_requezt(TPid, Pkt, Req, SvcName, Timeout)). %% recv/4 %% %% Relay an answer from a remote node. -recv(TPid, Pid, TRef, LocalTRef) -> +recv(TPid, Pid, TRef, {LocalTRef, MRef}) -> receive {answer, _, _, _, _} = A -> Pid ! A; + {'DOWN', MRef, process, _, _} -> + Pid ! {failover, TRef}; {failover = T, LocalTRef} -> Pid ! {T, TRef}; T -> @@ -1751,14 +1747,13 @@ recv(TPid, Pid, TRef, LocalTRef) -> %% send/2 -send(Pid, Pkt) -> %% Strip potentially large message terms. - #diameter_packet{header = H, - bin = Bin, - transport_data = T} - = Pkt, - Pid ! {send, #diameter_packet{header = H, - bin = Bin, - transport_data = T}}. +send(Pid, Pkt) -> + Pid ! {send, Pkt}. + +%% send/3 + +send(Pid, Pkt, Route) -> + Pid ! {send, Pkt, Route}. %% retransmit/4 @@ -1768,8 +1763,8 @@ retransmit({TPid, Caps, App} = Req, SvcName, Timeout) -> - have_request(Pkt0, TPid) %% Don't failover to a peer we've - andalso ?THROW(timeout), %% already sent to. + undefined == get(TPid) %% Don't failover to a peer we've + orelse ?THROW(timeout), %% already sent to. Pkt = make_retransmit_packet(Pkt0), @@ -1822,56 +1817,20 @@ resend_request(Pkt0, ?LOG(retransmission, Pkt#diameter_packet.header), incr(TPid, {msg_id(Pkt, AppDict), send, retransmission}), - TRef = send_request(TPid, Pkt, Req, SvcName, Tmo), - {TRef, Req}. - -%% insert_request/1 - -insert_request({_Seqs, #request{transport = TPid}, TRef} = T) -> - ets:insert(?REQUEST_TABLE, [T, {TPid, {self(), TRef}}]), - is_peer_up(TPid) - orelse (self() ! {failover, TRef}). %% failover/1 may have missed - -%% is_peer_up/1 -%% -%% Is the entry written by peer_up/1 and deleted by peer_down/1 still -%% in the request table? + {TRef, MRef} = zend_requezt(TPid, Pkt, Req, SvcName, Tmo), + {TRef, MRef, Req}. -is_peer_up(TPid) -> - Spec = [{{TPid}, [], ['$_']}], - '$end_of_table' /= ets:select(?REQUEST_TABLE, Spec, 1). +%% peer_monitor/2 -%% lookup_request/2 -%% -%% Note the match on both the key and transport pid. The latter is -%% necessary since the same Hop-by-Hop and End-to-End identifiers are -%% reused in the case of retransmission. - -lookup_request(Msg, TPid) -> - Seqs = diameter_codec:sequence_numbers(Msg), - Spec = [{{Seqs, #request{transport = TPid, _ = '_'}, '_'}, - [], - ['$_']}], - case ets:select(?REQUEST_TABLE, Spec) of - [{_, Req, _}] -> - Req; - [] -> +peer_monitor(TPid, TRef) -> + case ets:lookup(?REQUEST_TABLE, TPid) of %% at peer_up/1 + [{_, MPid}] -> + monitor(process, MPid); + [] -> %% transport has gone down + self() ! {failover, TRef}, false end. -%% delete_request/1 - -delete_request({_Seqs, #request{handler = Pid, transport = TPid}, TRef} = T) -> - Spec = [{R, [], [true]} || R <- [T, {TPid, {Pid, TRef}}]], - ets:select_delete(?REQUEST_TABLE, Spec). - -%% have_request/2 - -have_request(Pkt, TPid) -> - Seqs = diameter_codec:sequence_numbers(Pkt), - Pat = {Seqs, #request{transport = TPid, _ = '_'}, '_'}, - '$end_of_table' /= ets:select(?REQUEST_TABLE, [{Pat, [], ['$_']}], 1). - %% get_destination/2 get_destination(Dict, Msg) -> diff --git a/lib/diameter/src/base/diameter_watchdog.erl b/lib/diameter/src/base/diameter_watchdog.erl index 2ba60a65fb..f28b8f2910 100644 --- a/lib/diameter/src/base/diameter_watchdog.erl +++ b/lib/diameter/src/base/diameter_watchdog.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -283,7 +283,7 @@ event(Msg, ?LOG(transition, {From, To}). data(Msg, TPid, reopen, okay) -> - {recv, TPid, 'DWA', _Pkt} = Msg, %% assert + {recv, TPid, false, 'DWA', _Pkt, _NPid} = Msg, %% assert {TPid, T} = eraser(open), [T]; @@ -447,12 +447,14 @@ transition({'DOWN', _, process, TPid, _Reason} = D, end; %% Incoming message. -transition({recv, TPid, Name, PktT}, #watchdog{transport = TPid} = S) -> +transition({recv, TPid, Route, Name, Pkt, NPid}, + #watchdog{transport = TPid} + = S) -> try - incoming(Name, PktT, S) + incoming(Name, Pkt, NPid, S) catch #watchdog{dictionary = Dict0, receive_data = T} = NS -> - diameter_traffic:receive_message(TPid, PktT, Dict0, T), + diameter_traffic:receive_message(TPid, Route, Pkt, NPid, Dict0, T), NS end; @@ -582,15 +584,17 @@ send_watchdog(#watchdog{pending = false, %% Don't count encode errors since we don't expect any on DWR/DWA. -%% incoming/3 +%% incoming/4 -incoming(Name, {Pkt, NPid}, S) -> - NS = recv(Name, Pkt, S), - NPid ! {diameter, discard}, - NS; +incoming(Name, Pkt, false, S) -> + recv(Name, Pkt, S); -incoming(Name, Pkt, S) -> - recv(Name, Pkt, S). +incoming(Name, Pkt, NPid, S) -> + try + recv(Name, Pkt, S) + after + NPid ! {diameter, discard} + end. %% recv/3 diff --git a/lib/diameter/src/compiler/diameter_codegen.erl b/lib/diameter/src/compiler/diameter_codegen.erl index 864d5f0691..928ae37e7f 100644 --- a/lib/diameter/src/compiler/diameter_codegen.erl +++ b/lib/diameter/src/compiler/diameter_codegen.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -184,7 +184,7 @@ erl_forms(Mod, ParseD) -> f_enumerated_avp(ParseD), f_empty_value(ParseD), f_dict(ParseD), - {eof, erl_anno:new(?LINE)}]], + {eof, ?LINE}]], lists:append(Forms). diff --git a/lib/diameter/src/diameter.appup.src b/lib/diameter/src/diameter.appup.src index b1b8e38d39..eb5a5a44f3 100644 --- a/lib/diameter/src/diameter.appup.src +++ b/lib/diameter/src/diameter.appup.src @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -50,10 +50,8 @@ {"1.11", [{restart_application, diameter}]}, %% 18.1 {"1.11.1", [{restart_application, diameter}]}, %% 18.2 {"1.11.2", [{restart_application, diameter}]}, %% 18.3 - {"1.12", [{load_module, diameter_lib}, %% 19.0 - {load_module, diameter_traffic}, - {load_module, diameter_tcp}, - {load_module, diameter_sctp}]} + {"1.12", [{restart_application, diameter}]}, %% 19.0 + {"1.12.1", [{restart_application, diameter}]} %% 19.1 ], [ {"0.9", [{restart_application, diameter}]}, @@ -85,9 +83,7 @@ {"1.11", [{restart_application, diameter}]}, {"1.11.1", [{restart_application, diameter}]}, {"1.11.2", [{restart_application, diameter}]}, - {"1.12", [{load_module, diameter_sctp}, - {load_module, diameter_tcp}, - {load_module, diameter_traffic}, - {load_module, diameter_lib}]} + {"1.12", [{restart_application, diameter}]}, + {"1.12.1", [{restart_application, diameter}]} ] }. diff --git a/lib/diameter/vsn.mk b/lib/diameter/vsn.mk index 23219950bb..94d9d72a48 100644 --- a/lib/diameter/vsn.mk +++ b/lib/diameter/vsn.mk @@ -1,6 +1,6 @@ # %CopyrightBegin% # -# Copyright Ericsson AB 2010-2016. All Rights Reserved. +# Copyright Ericsson AB 2010-2017. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,5 +17,5 @@ # %CopyrightEnd% APPLICATION = diameter -DIAMETER_VSN = 1.12.1 +DIAMETER_VSN = 1.12.2 APP_VSN = $(APPLICATION)-$(DIAMETER_VSN)$(PRE_VSN) diff --git a/lib/edoc/src/edoc_layout.erl b/lib/edoc/src/edoc_layout.erl index 9407ae1321..5ef210980c 100644 --- a/lib/edoc/src/edoc_layout.erl +++ b/lib/edoc/src/edoc_layout.erl @@ -214,7 +214,7 @@ layout_module(#xmlElement{name = module, content = Es}=E, Opts) -> ++ functions(SortedFs, Opts) ++ [hr, ?NL] ++ navigation("bottom") - ++ timestamp()), + ++ footer()), Encoding = get_attrval(encoding, E), xhtml(Title, stylesheet(Opts), Body, Encoding). @@ -228,12 +228,8 @@ module_params(Es) -> [element(1, First) | [ {[", ",A]} || {A, _D} <- Rest]] end. -timestamp() -> - [?NL, {p, [{i, [io_lib:fwrite("Generated by EDoc, ~s, ~s.", - [edoc_lib:datestr(date()), - edoc_lib:timestr(time())]) - ]}]}, - ?NL]. +footer() -> + [?NL, {p, [{i, ["Generated by EDoc"]}]}, ?NL]. stylesheet(Opts) -> case Opts#opts.stylesheet of @@ -1039,7 +1035,7 @@ overview(E=#xmlElement{name = overview, content = Es}, Options) -> ++ FullDesc ++ [?NL, hr] ++ navigation("bottom") - ++ timestamp()), + ++ footer()), Encoding = get_attrval(encoding, E), XML = xhtml(Title, stylesheet(Opts), Body, Encoding), xmerl:export_simple(XML, ?HTML_EXPORT, []). diff --git a/lib/edoc/test/edoc_SUITE.erl b/lib/edoc/test/edoc_SUITE.erl index 00d7550bed..4d846ad63d 100644 --- a/lib/edoc/test/edoc_SUITE.erl +++ b/lib/edoc/test/edoc_SUITE.erl @@ -69,7 +69,7 @@ build_std(Config) when is_list(Config) -> {def, {vsn,"TEST"}}, {dir, PrivDir}]), - ok = edoc:application(xmerl, [{dir, PrivDir}]), + ok = edoc:application(xmerl, [{preprocess,true},{dir, PrivDir}]), ok. build_map_module(Config) when is_list(Config) -> diff --git a/lib/erl_interface/doc/src/notes.xml b/lib/erl_interface/doc/src/notes.xml index 69ba3cddb8..b5d8def655 100644 --- a/lib/erl_interface/doc/src/notes.xml +++ b/lib/erl_interface/doc/src/notes.xml @@ -31,6 +31,21 @@ </header> <p>This document describes the changes made to the Erl_interface application.</p> +<section><title>Erl_Interface 3.9.3</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Minor documentation update</p> + <p> + Own Id: OTP-14233 Aux Id: PR-1343 </p> + </item> + </list> + </section> + +</section> + <section><title>Erl_Interface 3.9.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/erl_interface/src/connect/ei_connect.c b/lib/erl_interface/src/connect/ei_connect.c index c193fd804a..27b919c093 100644 --- a/lib/erl_interface/src/connect/ei_connect.c +++ b/lib/erl_interface/src/connect/ei_connect.c @@ -497,7 +497,8 @@ int ei_connect_init(ei_cnode* ec, const char* this_node_name, } #endif /* _REENTRANT */ - if (gethostname(thishostname, EI_MAXHOSTNAMELEN) == -1) { + /* gethostname requires len to be max(hostname) + 1 */ + if (gethostname(thishostname, EI_MAXHOSTNAMELEN+1) == -1) { #ifdef __WIN32__ EI_TRACE_ERR1("ei_connect_init","Failed to get host name: %d", WSAGetLastError()); @@ -613,7 +614,8 @@ int ei_connect_tmo(ei_cnode* ec, char *nodename, unsigned ms) hp = ei_gethostbyname_r(hostname,&host,buffer,1024,&ei_h_errno); if (hp == NULL) { char thishostname[EI_MAXHOSTNAMELEN+1]; - if (gethostname(thishostname,EI_MAXHOSTNAMELEN) < 0) { + /* gethostname requies len to be max(hostname) + 1*/ + if (gethostname(thishostname,EI_MAXHOSTNAMELEN+1) < 0) { EI_TRACE_ERR0("ei_connect_tmo", "Failed to get name of this host"); erl_errno = EHOSTUNREACH; @@ -636,7 +638,8 @@ int ei_connect_tmo(ei_cnode* ec, char *nodename, unsigned ms) #else /* __WIN32__ */ if ((hp = ei_gethostbyname(hostname)) == NULL) { char thishostname[EI_MAXHOSTNAMELEN+1]; - if (gethostname(thishostname,EI_MAXHOSTNAMELEN) < 0) { + /* gethostname requires len to be max(hostname) + 1 */ + if (gethostname(thishostname,EI_MAXHOSTNAMELEN+1) < 0) { EI_TRACE_ERR1("ei_connect_tmo", "Failed to get name of this host: %d", WSAGetLastError()); diff --git a/lib/erl_interface/src/prog/erl_call.c b/lib/erl_interface/src/prog/erl_call.c index d233ed26a2..0b09d412db 100644 --- a/lib/erl_interface/src/prog/erl_call.c +++ b/lib/erl_interface/src/prog/erl_call.c @@ -325,7 +325,8 @@ int erl_call(int argc, char **argv) initWinSock(); #endif - if (gethostname(h_hostname, EI_MAXHOSTNAMELEN) < 0) { + /* gethostname requires len to be max(hostname) + 1 */ + if (gethostname(h_hostname, EI_MAXHOSTNAMELEN+1) < 0) { fprintf(stderr,"erl_call: failed to get host name: %d\n", errno); exit(1); } diff --git a/lib/erl_interface/vsn.mk b/lib/erl_interface/vsn.mk index c7981ed3a5..563694a0c1 100644 --- a/lib/erl_interface/vsn.mk +++ b/lib/erl_interface/vsn.mk @@ -1,2 +1,2 @@ -EI_VSN = 3.9.2 +EI_VSN = 3.9.3 ERL_INTERFACE_VSN = $(EI_VSN) diff --git a/lib/eunit/doc/overview.edoc b/lib/eunit/doc/overview.edoc index 3a46e991cb..dc9f858812 100644 --- a/lib/eunit/doc/overview.edoc +++ b/lib/eunit/doc/overview.edoc @@ -578,7 +578,7 @@ results for equality, if testing is enabled. If the values are not equal, an informative exception will be generated; see the `assert' macro for further details. -`assertEqual' is more suitable than than `assertMatch' when the +`assertEqual' is more suitable than `assertMatch' when the left-hand side is a computed value rather than a simple pattern, and gives more details than `?assert(Expect =:= Expr)'. @@ -994,7 +994,7 @@ specified node. `local' means that the current process will handle both setup/teardown and running the tests - the drawback is that if a test times out so that the process is killed, the <em>cleanup will not be performed</em>; hence, avoid this for persistent fixtures such as file -operations. In general, 'local' should only be used when: +operations. In general, `local' should only be used when: <ul> <li>the setup/teardown needs to be executed by the process that will run the tests;</li> diff --git a/lib/eunit/src/eunit.erl b/lib/eunit/src/eunit.erl index 2c832a7f7a..1ace85ffde 100644 --- a/lib/eunit/src/eunit.erl +++ b/lib/eunit/src/eunit.erl @@ -256,7 +256,7 @@ all_options(Opts) -> false -> Opts; S -> {ok, Ts, _} = erl_scan:string(S), - {ok, V} = erl_parse:parse_term(Ts ++ [{dot,1}]), + {ok, V} = erl_parse:parse_term(Ts ++ [{dot,erl_anno:new(1)}]), if is_list(V) -> Opts ++ V; true -> Opts ++ [V] end diff --git a/lib/eunit/src/eunit_surefire.erl b/lib/eunit/src/eunit_surefire.erl index 6b306c51d3..2b9f82b075 100644 --- a/lib/eunit/src/eunit_surefire.erl +++ b/lib/eunit/src/eunit_surefire.erl @@ -424,6 +424,7 @@ escape_suitename(String) -> escape_suitename([], Acc) -> lists:reverse(Acc); escape_suitename([$ | Tail], Acc) -> escape_suitename(Tail, [$_ | Acc]); escape_suitename([$' | Tail], Acc) -> escape_suitename(Tail, Acc); +escape_suitename([$" | Tail], Acc) -> escape_suitename(Tail, Acc); escape_suitename([$/ | Tail], Acc) -> escape_suitename(Tail, [$: | Acc]); escape_suitename([$\\ | Tail], Acc) -> escape_suitename(Tail, [$: | Acc]); escape_suitename([Char | Tail], Acc) when Char < $! -> escape_suitename(Tail, Acc); diff --git a/lib/hipe/amd64/Makefile b/lib/hipe/amd64/Makefile index 617f6749ac..d0da8cdff6 100644 --- a/lib/hipe/amd64/Makefile +++ b/lib/hipe/amd64/Makefile @@ -128,6 +128,7 @@ $(EBIN)/hipe_amd64_ra_postconditions.beam: ../main/hipe.hrl ../x86/hipe_x86.hrl $(EBIN)/hipe_amd64_ra_sse2_postconditions.beam: ../main/hipe.hrl $(EBIN)/hipe_amd64_registers.beam: ../rtl/hipe_literals.hrl $(EBIN)/hipe_amd64_spill_restore.beam: ../main/hipe.hrl ../x86/hipe_x86.hrl ../flow/cfg.hrl ../x86/hipe_x86_spill_restore.erl +$(EBIN)/hipe_amd64_subst.beam: ../x86/hipe_x86_subst.erl $(EBIN)/hipe_amd64_x87.beam: ../x86/hipe_x86_x87.erl $(EBIN)/hipe_amd64_sse2.beam: ../main/hipe.hrl ../x86/hipe_x86.hrl $(EBIN)/hipe_rtl_to_amd64.beam: ../x86/hipe_rtl_to_x86.erl ../rtl/hipe_rtl.hrl diff --git a/lib/hipe/amd64/hipe_amd64_encode.erl b/lib/hipe/amd64/hipe_amd64_encode.erl index f8cc0c7d83..bda2824ffc 100644 --- a/lib/hipe/amd64/hipe_amd64_encode.erl +++ b/lib/hipe/amd64/hipe_amd64_encode.erl @@ -1316,6 +1316,7 @@ dotest1(OS) -> RM64 = {rm64,rm_reg(?EDX)}, RM32 = {rm32,rm_reg(?EDX)}, RM16 = {rm16,rm_reg(?EDX)}, + RM16REX = {rm16,rm_reg(?R13)}, RM8 = {rm8,rm_reg(?EDX)}, RM8REX = {rm8,rm_reg(?SIL)}, Rel32 = {rel32,Word32}, @@ -1479,6 +1480,7 @@ dotest1(OS) -> t(OS,'test',{RM8,Imm8}), t(OS,'test',{RM8REX,Imm8}), t(OS,'test',{RM16,Imm16}), + t(OS,'test',{RM16REX,Imm16}), t(OS,'test',{RM32,Imm32}), t(OS,'test',{RM64,Imm32}), t(OS,'test',{RM32,Reg32}), diff --git a/lib/hipe/amd64/hipe_amd64_ra_sse2_postconditions.erl b/lib/hipe/amd64/hipe_amd64_ra_sse2_postconditions.erl index 8a3ea92156..891c874a15 100644 --- a/lib/hipe/amd64/hipe_amd64_ra_sse2_postconditions.erl +++ b/lib/hipe/amd64/hipe_amd64_ra_sse2_postconditions.erl @@ -53,6 +53,8 @@ do_insn(I, TempMap, Strategy) -> % Insn -> {Insn list, DidSpill} do_fp_unop(I, TempMap, Strategy); #fp_binop{} -> do_fp_binop(I, TempMap, Strategy); + #pseudo_spill_fmove{} -> + do_pseudo_spill_fmove(I, TempMap, Strategy); _ -> %% All non sse2 ops {[I], false} @@ -95,8 +97,13 @@ do_fmove(I, TempMap, Strategy) -> of true -> Tmp = spill_temp(double, Strategy), - {[#fmove{src=Src, dst=Tmp},I#fmove{src=Tmp,dst=Dst}], - true}; + %% pseudo_spill_fmove allows spill slot move coalescing, but must not + %% contain memory operands (except for spilled temps) + Is = case is_float_temp(Src) andalso is_float_temp(Dst) of + true -> [#pseudo_spill_fmove{src=Src, temp=Tmp, dst=Dst}]; + false -> [#fmove{src=Src, dst=Tmp},I#fmove{src=Tmp,dst=Dst}] + end, + {Is, true}; false -> {[I], false} end. @@ -104,6 +111,12 @@ do_fmove(I, TempMap, Strategy) -> is_float_temp(#x86_temp{type=Type}) -> Type =:= double; is_float_temp(#x86_mem{}) -> false. +%%% Fix an pseudo_spill_fmove op. +do_pseudo_spill_fmove(I = #pseudo_spill_fmove{temp=Temp}, TempMap, _Strategy) -> + %% Temp is above the low water mark and must not have been spilled + false = is_mem_opnd(Temp, TempMap), + {[I], false}. % nothing to do + %%% Check if an operand denotes a memory cell (mem or pseudo). is_mem_opnd(Opnd, TempMap) -> diff --git a/lib/hipe/amd64/hipe_amd64_registers.erl b/lib/hipe/amd64/hipe_amd64_registers.erl index a4cb71a106..a5cecef5a1 100644 --- a/lib/hipe/amd64/hipe_amd64_registers.erl +++ b/lib/hipe/amd64/hipe_amd64_registers.erl @@ -207,19 +207,14 @@ allocatable_x87() -> nr_args() -> ?AMD64_NR_ARG_REGS. -arg(N) -> - if N < ?AMD64_NR_ARG_REGS -> - case N of - 0 -> ?ARG0; - 1 -> ?ARG1; - 2 -> ?ARG2; - 3 -> ?ARG3; - 4 -> ?ARG4; - 5 -> ?ARG5; - _ -> exit({?MODULE, arg, N}) - end; - true -> - exit({?MODULE, arg, N}) +arg(N) when N < ?AMD64_NR_ARG_REGS -> + case N of + 0 -> ?ARG0; + 1 -> ?ARG1; + 2 -> ?ARG2; + 3 -> ?ARG3; + 4 -> ?ARG4; + 5 -> ?ARG5 end. is_arg(R) -> @@ -240,11 +235,7 @@ args(Arity) when is_integer(Arity), Arity >= 0 -> args(I, Rest) when I < 0 -> Rest; args(I, Rest) -> args(I-1, [arg(I) | Rest]). -ret(N) -> - case N of - 0 -> ?RAX; - _ -> exit({?MODULE, ret, N}) - end. +ret(0) -> ?RAX. %% Note: the fact that (allocatable() UNION allocatable_x87() UNION %% allocatable_sse2()) is a subset of call_clobbered() is hard-coded in diff --git a/lib/hipe/arm/hipe_arm.erl b/lib/hipe/arm/hipe_arm.erl index e34a00f561..3b090b501a 100644 --- a/lib/hipe/arm/hipe_arm.erl +++ b/lib/hipe/arm/hipe_arm.erl @@ -79,6 +79,9 @@ pseudo_move_dst/1, pseudo_move_src/1, + mk_pseudo_spill_move/3, + is_pseudo_spill_move/1, + mk_pseudo_switch/3, mk_pseudo_tailcall/4, @@ -250,6 +253,10 @@ is_pseudo_move(I) -> case I of #pseudo_move{} -> true; _ -> false end. pseudo_move_dst(#pseudo_move{dst=Dst}) -> Dst. pseudo_move_src(#pseudo_move{src=Src}) -> Src. +mk_pseudo_spill_move(Dst, Temp, Src) -> + #pseudo_spill_move{dst=Dst, temp=Temp, src=Src}. +is_pseudo_spill_move(I) -> is_record(I, pseudo_spill_move). + mk_pseudo_switch(JTab, Index, Labels) -> #pseudo_switch{jtab=JTab, index=Index, labels=Labels}. diff --git a/lib/hipe/arm/hipe_arm.hrl b/lib/hipe/arm/hipe_arm.hrl index 67bc07634e..be06b1ebd7 100644 --- a/lib/hipe/arm/hipe_arm.hrl +++ b/lib/hipe/arm/hipe_arm.hrl @@ -101,6 +101,7 @@ -record(pseudo_call_prepare, {nrstkargs}). -record(pseudo_li, {dst, imm, label}). % pre-generated label for use by the assembler -record(pseudo_move, {dst, src}). +-record(pseudo_spill_move, {dst, temp, src}). -record(pseudo_switch, {jtab, index, labels}). -record(pseudo_tailcall, {funv, arity, stkargs, linkage}). -record(pseudo_tailcall_prepare, {}). diff --git a/lib/hipe/arm/hipe_arm_assemble.erl b/lib/hipe/arm/hipe_arm_assemble.erl index 713c148742..9aa730afa9 100644 --- a/lib/hipe/arm/hipe_arm_assemble.erl +++ b/lib/hipe/arm/hipe_arm_assemble.erl @@ -31,7 +31,7 @@ assemble(CompiledCode, Closures, Exports, Options) -> || {MFA, Defun} <- CompiledCode], %% {ConstAlign,ConstSize,ConstMap,RefsFromConsts} = - hipe_pack_constants:pack_constants(Code, 4), + hipe_pack_constants:pack_constants(Code), %% {CodeSize,CodeBinary,AccRefs,LabelMap,ExportMap} = encode(translate(Code, ConstMap), Options), diff --git a/lib/hipe/arm/hipe_arm_cfg.erl b/lib/hipe/arm/hipe_arm_cfg.erl index ea6da67317..0bc3df30b9 100644 --- a/lib/hipe/arm/hipe_arm_cfg.erl +++ b/lib/hipe/arm/hipe_arm_cfg.erl @@ -24,6 +24,7 @@ -export([params/1, reverse_postorder/1]). -export([arity/1]). % for linear scan %%-export([redirect_jmp/3]). +-export([branch_preds/1]). %%% these tell cfg.inc what to define (ugly as hell) -define(BREADTH_ORDER,true). % for linear scan @@ -75,6 +76,26 @@ branch_successors(Branch) -> #pseudo_tailcall{} -> [] end. +branch_preds(Branch) -> + case Branch of + #pseudo_bc{true_label=TrueLab,false_label=FalseLab,pred=Pred} -> + [{FalseLab, 1.0-Pred}, {TrueLab, Pred}]; + #pseudo_call{contlab=ContLab, sdesc=#arm_sdesc{exnlab=[]}} -> + %% A function can still cause an exception, even if we won't catch it + [{ContLab, 1.0-hipe_bb_weights:call_exn_pred()}]; + #pseudo_call{contlab=ContLab, sdesc=#arm_sdesc{exnlab=ExnLab}} -> + CallExnPred = hipe_bb_weights:call_exn_pred(), + [{ContLab, 1.0-CallExnPred}, {ExnLab, CallExnPred}]; + #pseudo_switch{labels=Labels} -> + Prob = 1.0/length(Labels), + [{L, Prob} || L <- Labels]; + _ -> + case branch_successors(Branch) of + [] -> []; + [Single] -> [{Single, 1.0}] + end + end. + -ifdef(REMOVE_TRIVIAL_BBS_NEEDED). fails_to(_Instr) -> []. -endif. diff --git a/lib/hipe/arm/hipe_arm_defuse.erl b/lib/hipe/arm/hipe_arm_defuse.erl index 0e62070c6c..652299a514 100644 --- a/lib/hipe/arm/hipe_arm_defuse.erl +++ b/lib/hipe/arm/hipe_arm_defuse.erl @@ -40,6 +40,7 @@ insn_def_gpr(I) -> #pseudo_call{} -> call_clobbered_gpr(); #pseudo_li{dst=Dst} -> [Dst]; #pseudo_move{dst=Dst} -> [Dst]; + #pseudo_spill_move{dst=Dst, temp=Temp} -> [Dst, Temp]; #pseudo_tailcall_prepare{} -> tailcall_clobbered_gpr(); #smull{dstlo=DstLo,dsthi=DstHi,src1=Src1} -> %% ARM requires DstLo, DstHi, and Src1 to be distinct. @@ -83,6 +84,7 @@ insn_use_gpr(I) -> #pseudo_call{funv=FunV,sdesc=#arm_sdesc{arity=Arity}} -> funv_use(FunV, arity_use_gpr(Arity)); #pseudo_move{src=Src} -> [Src]; + #pseudo_spill_move{src=Src} -> [Src]; #pseudo_switch{jtab=JTabR,index=IndexR} -> addtemp(JTabR, [IndexR]); #pseudo_tailcall{funv=FunV,arity=Arity,stkargs=StkArgs} -> addargs(StkArgs, addtemps(tailcall_clobbered_gpr(), funv_use(FunV, arity_use_gpr(Arity)))); diff --git a/lib/hipe/arm/hipe_arm_frame.erl b/lib/hipe/arm/hipe_arm_frame.erl index e323907e31..a1004fb609 100644 --- a/lib/hipe/arm/hipe_arm_frame.erl +++ b/lib/hipe/arm/hipe_arm_frame.erl @@ -69,6 +69,8 @@ do_insn(I, LiveOut, Context, FPoff) -> do_pseudo_call_prepare(I, FPoff); #pseudo_move{} -> {do_pseudo_move(I, Context, FPoff), FPoff}; + #pseudo_spill_move{} -> + {do_pseudo_spill_move(I, Context, FPoff), FPoff}; #pseudo_tailcall{} -> {do_pseudo_tailcall(I, Context), context_framesize(Context)}; _ -> @@ -100,6 +102,26 @@ pseudo_offset(Temp, FPoff, Context) -> FPoff + context_offset(Context, Temp). %%% +%%% Moves from one spill slot to another +%%% + +do_pseudo_spill_move(I, Context, FPoff) -> + #pseudo_spill_move{dst=Dst, temp=Temp, src=Src} = I, + case temp_is_pseudo(Src) andalso temp_is_pseudo(Dst) of + false -> % Register allocator changed its mind, turn back to move + do_pseudo_move(hipe_arm:mk_pseudo_move(Dst, Src), Context, FPoff); + true -> + SrcOffset = pseudo_offset(Src, FPoff, Context), + DstOffset = pseudo_offset(Dst, FPoff, Context), + case SrcOffset =:= DstOffset of + true -> []; % omit move-to-self + false -> + mk_load('ldr', Temp, SrcOffset, mk_sp(), + mk_store('str', Temp, DstOffset, mk_sp(), [])) + end + end. + +%%% %%% Return - deallocate frame and emit 'ret $N' insn. %%% diff --git a/lib/hipe/arm/hipe_arm_ra_finalise.erl b/lib/hipe/arm/hipe_arm_ra_finalise.erl index 9bfe0a9a83..80cd470708 100644 --- a/lib/hipe/arm/hipe_arm_ra_finalise.erl +++ b/lib/hipe/arm/hipe_arm_ra_finalise.erl @@ -25,11 +25,17 @@ ra_bb(BB, Map) -> hipe_bb:code_update(BB, ra_code(hipe_bb:code(BB), Map, [])). ra_code([I|Insns], Map, Accum) -> - ra_code(Insns, Map, [ra_insn(I, Map) | Accum]); + ra_code(Insns, Map, ra_insn(I, Map, Accum)); ra_code([], _Map, Accum) -> lists:reverse(Accum). -ra_insn(I, Map) -> +ra_insn(I, Map, Accum) -> + case I of + #pseudo_move{} -> ra_pseudo_move(I, Map, Accum); + _ -> [ra_insn_1(I, Map) | Accum] + end. + +ra_insn_1(I, Map) -> case I of #alu{} -> ra_alu(I, Map); #cmp{} -> ra_cmp(I, Map); @@ -38,7 +44,7 @@ ra_insn(I, Map) -> #move{} -> ra_move(I, Map); #pseudo_call{} -> ra_pseudo_call(I, Map); #pseudo_li{} -> ra_pseudo_li(I, Map); - #pseudo_move{} -> ra_pseudo_move(I, Map); + #pseudo_spill_move{} -> ra_pseudo_spill_move(I, Map); #pseudo_switch{} -> ra_pseudo_switch(I, Map); #pseudo_tailcall{} -> ra_pseudo_tailcall(I, Map); #smull{} -> ra_smull(I, Map); @@ -80,10 +86,19 @@ ra_pseudo_li(I=#pseudo_li{dst=Dst}, Map) -> NewDst = ra_temp(Dst, Map), I#pseudo_li{dst=NewDst}. -ra_pseudo_move(I=#pseudo_move{dst=Dst,src=Src}, Map) -> +ra_pseudo_move(I=#pseudo_move{dst=Dst,src=Src}, Map, Accum) -> + NewDst = ra_temp(Dst, Map), + NewSrc = ra_temp(Src, Map), + case NewSrc#arm_temp.reg =:= NewDst#arm_temp.reg of + true -> Accum; + false -> [I#pseudo_move{dst=NewDst,src=NewSrc} | Accum] + end. + +ra_pseudo_spill_move(I=#pseudo_spill_move{dst=Dst,temp=Temp,src=Src}, Map) -> NewDst = ra_temp(Dst, Map), + NewTemp = ra_temp(Temp, Map), NewSrc = ra_temp(Src, Map), - I#pseudo_move{dst=NewDst,src=NewSrc}. + I#pseudo_spill_move{dst=NewDst, temp=NewTemp, src=NewSrc}. ra_pseudo_switch(I=#pseudo_switch{jtab=JTab,index=Index}, Map) -> NewJTab = ra_temp(JTab, Map), diff --git a/lib/hipe/arm/hipe_arm_ra_postconditions.erl b/lib/hipe/arm/hipe_arm_ra_postconditions.erl index 8d1ee1cb94..23c305511f 100644 --- a/lib/hipe/arm/hipe_arm_ra_postconditions.erl +++ b/lib/hipe/arm/hipe_arm_ra_postconditions.erl @@ -56,6 +56,7 @@ do_insn(I, TempMap, Strategy) -> #pseudo_call{} -> do_pseudo_call(I, TempMap, Strategy); #pseudo_li{} -> do_pseudo_li(I, TempMap, Strategy); #pseudo_move{} -> do_pseudo_move(I, TempMap, Strategy); + #pseudo_spill_move{} -> do_pseudo_spill_move(I, TempMap, Strategy); #pseudo_switch{} -> do_pseudo_switch(I, TempMap, Strategy); #pseudo_tailcall{} -> do_pseudo_tailcall(I, TempMap, Strategy); #smull{} -> do_smull(I, TempMap, Strategy); @@ -108,18 +109,25 @@ do_pseudo_li(I=#pseudo_li{dst=Dst}, TempMap, Strategy) -> do_pseudo_move(I=#pseudo_move{dst=Dst,src=Src}, TempMap, Strategy) -> %% Either Dst or Src (but not both) may be a pseudo temp. - %% pseudo_move and pseudo_tailcall are special cases: in - %% all other instructions, all temps must be non-pseudos - %% after register allocation. - case temp_is_spilled(Dst, TempMap) of - true -> % Src must not be a pseudo - {FixSrc,NewSrc,DidSpill} = fix_src1(Src, TempMap, Strategy), - NewI = I#pseudo_move{src=NewSrc}, - {FixSrc ++ [NewI], DidSpill}; + %% pseudo_move, pseudo_spill_move, and pseudo_tailcall + %% are special cases: in all other instructions, all + %% temps must be non-pseudos after register allocation. + case temp_is_spilled(Dst, TempMap) + andalso temp_is_spilled(Dst, TempMap) + of + true -> % Turn into pseudo_spill_move + Temp = clone(Src, temp1(Strategy)), + NewI = #pseudo_spill_move{dst=Dst, temp=Temp, src=Src}, + {[NewI], true}; _ -> {[I], false} end. +do_pseudo_spill_move(I = #pseudo_spill_move{temp=Temp}, TempMap, _Strategy) -> + %% Temp is above the low water mark and must not have been spilled + false = temp_is_spilled(Temp, TempMap), + {[I], false}. % nothing to do + do_pseudo_switch(I=#pseudo_switch{jtab=JTab,index=Index}, TempMap, Strategy) -> {FixJTab,NewJTab,DidSpill1} = fix_src1(JTab, TempMap, Strategy), {FixIndex,NewIndex,DidSpill2} = fix_src2(Index, TempMap, Strategy), diff --git a/lib/hipe/arm/hipe_arm_subst.erl b/lib/hipe/arm/hipe_arm_subst.erl index 7510c197bd..4ff245f414 100644 --- a/lib/hipe/arm/hipe_arm_subst.erl +++ b/lib/hipe/arm/hipe_arm_subst.erl @@ -13,7 +13,7 @@ %% limitations under the License. -module(hipe_arm_subst). --export([insn_temps/2]). +-export([insn_temps/2, insn_lbls/2]). -include("hipe_arm.hrl"). %% These should be moved to hipe_arm and exported @@ -31,6 +31,7 @@ -type am3() :: #am3{}. -type arg() :: temp() | integer(). -type funv() :: #arm_mfa{} | #arm_prim{} | temp(). +-type label() :: non_neg_integer(). -type insn() :: tuple(). % for now -type subst_fun() :: fun((temp()) -> temp()). @@ -58,6 +59,8 @@ insn_temps(T, I) -> #pseudo_call{funv=F} -> I#pseudo_call{funv=funv_temps(T, F)}; #pseudo_call_prepare{} -> I; #pseudo_li{dst=D} -> I#pseudo_li{dst=T(D)}; + #pseudo_spill_move{dst=D,temp=U,src=S} -> + I#pseudo_spill_move{dst=T(D),temp=T(U),src=T(S)}; #pseudo_switch{jtab=J=#arm_temp{},index=Ix=#arm_temp{}} -> I#pseudo_switch{jtab=T(J),index=T(Ix)}; #pseudo_tailcall{funv=F,stkargs=Stk} -> @@ -103,3 +106,22 @@ funv_temps(SubstTemp, T=#arm_temp{}) -> SubstTemp(T). -spec arg_temps(subst_fun(), arg()) -> arg(). arg_temps(_SubstTemp, Imm) when is_integer(Imm) -> Imm; arg_temps(SubstTemp, T=#arm_temp{}) -> SubstTemp(T). + +-type lbl_subst_fun() :: fun((label()) -> label()). + +%% @doc Maps over the branch targets in an instruction +-spec insn_lbls(lbl_subst_fun(), insn()) -> insn(). +insn_lbls(SubstLbl, I) -> + case I of + #b_label{label=Label} -> + I#b_label{label=SubstLbl(Label)}; + #pseudo_bc{true_label=T, false_label=F} -> + I#pseudo_bc{true_label=SubstLbl(T), false_label=SubstLbl(F)}; + #pseudo_call{sdesc=Sdesc, contlab=Contlab} -> + I#pseudo_call{sdesc=sdesc_lbls(SubstLbl, Sdesc), + contlab=SubstLbl(Contlab)} + end. + +sdesc_lbls(_SubstLbl, Sdesc=#arm_sdesc{exnlab=[]}) -> Sdesc; +sdesc_lbls(SubstLbl, Sdesc=#arm_sdesc{exnlab=Exnlab}) -> + Sdesc#arm_sdesc{exnlab=SubstLbl(Exnlab)}. diff --git a/lib/hipe/cerl/erl_bif_types.erl b/lib/hipe/cerl/erl_bif_types.erl index 8c96e60229..9321750d44 100644 --- a/lib/hipe/cerl/erl_bif_types.erl +++ b/lib/hipe/cerl/erl_bif_types.erl @@ -2029,17 +2029,14 @@ arith_rem(Min1, Max1, Min2, Max2) -> Min1_geq_zero = infinity_geq(Min1, 0), Max1_leq_zero = infinity_geq(0, Max1), Max_range2 = infinity_max([infinity_abs(Min2), infinity_abs(Max2)]), - Max_range2_leq_zero = infinity_geq(0, Max_range2), - New_min = + New_min = if Min1_geq_zero -> 0; Max_range2 =:= 0 -> 0; - Max_range2_leq_zero -> infinity_add(Max_range2, 1); true -> infinity_add(infinity_inv(Max_range2), 1) end, New_max = if Max1_leq_zero -> 0; Max_range2 =:= 0 -> 0; - Max_range2_leq_zero -> infinity_add(infinity_inv(Max_range2), -1); true -> infinity_add(Max_range2, -1) end, {New_min, New_max}. diff --git a/lib/hipe/cerl/erl_types.erl b/lib/hipe/cerl/erl_types.erl index 9d46d4ac81..ea8cc1677d 100644 --- a/lib/hipe/cerl/erl_types.erl +++ b/lib/hipe/cerl/erl_types.erl @@ -518,7 +518,8 @@ list_contains_opaque(List, Opaques) -> lists:any(fun(E) -> t_contains_opaque(E, Opaques) end, List). %% t_find_opaque_mismatch/2 of two types should only be used if their -%% t_inf is t_none() due to some opaque type violation. +%% t_inf is t_none() due to some opaque type violation. However, +%% 'error' is returned if a structure mismatch is found. %% %% The first argument of the function is the pattern and its second %% argument the type we are matching against the pattern. @@ -527,22 +528,30 @@ list_contains_opaque(List, Opaques) -> 'error' | {'ok', erl_type(), erl_type()}. t_find_opaque_mismatch(T1, T2, Opaques) -> - t_find_opaque_mismatch(T1, T2, T2, Opaques). + catch t_find_opaque_mismatch(T1, T2, T2, Opaques). t_find_opaque_mismatch(?any, _Type, _TopType, _Opaques) -> error; -t_find_opaque_mismatch(?none, _Type, _TopType, _Opaques) -> error; +t_find_opaque_mismatch(?none, _Type, _TopType, _Opaques) -> throw(error); t_find_opaque_mismatch(?list(T1, Tl1, _), ?list(T2, Tl2, _), TopType, Opaques) -> t_find_opaque_mismatch_ordlists([T1, Tl1], [T2, Tl2], TopType, Opaques); t_find_opaque_mismatch(T1, ?opaque(_) = T2, TopType, Opaques) -> case is_opaque_type(T2, Opaques) of - false -> {ok, TopType, T2}; + false -> + case t_is_opaque(T1) andalso compatible_opaque_types(T1, T2) =/= [] of + true -> error; + false -> {ok, TopType, T2} + end; true -> t_find_opaque_mismatch(T1, t_opaque_structure(T2), TopType, Opaques) end; t_find_opaque_mismatch(?opaque(_) = T1, T2, TopType, Opaques) -> %% The generated message is somewhat misleading: case is_opaque_type(T1, Opaques) of - false -> {ok, TopType, T1}; + false -> + case t_is_opaque(T2) andalso compatible_opaque_types(T1, T2) =/= [] of + true -> error; + false -> {ok, TopType, T1} + end; true -> t_find_opaque_mismatch(t_opaque_structure(T1), T2, TopType, Opaques) end; @@ -558,7 +567,11 @@ t_find_opaque_mismatch(?tuple(_, _, _) = T1, ?tuple_set(_) = T2, t_find_opaque_mismatch_lists(Tuples1, Tuples2, TopType, Opaques); t_find_opaque_mismatch(T1, ?union(U2), TopType, Opaques) -> t_find_opaque_mismatch_lists([T1], U2, TopType, Opaques); -t_find_opaque_mismatch(_T1, _T2, _TopType, _Opaques) -> error. +t_find_opaque_mismatch(T1, T2, _TopType, Opaques) -> + case t_is_none(t_inf(T1, T2, Opaques)) of + false -> error; + true -> throw(error) + end. t_find_opaque_mismatch_ordlists(L1, L2, TopType, Opaques) -> List = lists:zipwith(fun(T1, T2) -> @@ -567,10 +580,11 @@ t_find_opaque_mismatch_ordlists(L1, L2, TopType, Opaques) -> t_find_opaque_mismatch_list(List). t_find_opaque_mismatch_lists(L1, L2, _TopType, Opaques) -> - List = [t_find_opaque_mismatch(T1, T2, T2, Opaques) || T1 <- L1, T2 <- L2], + List = [catch t_find_opaque_mismatch(T1, T2, T2, Opaques) || + T1 <- L1, T2 <- L2], t_find_opaque_mismatch_list(List). -t_find_opaque_mismatch_list([]) -> error; +t_find_opaque_mismatch_list([]) -> throw(error); t_find_opaque_mismatch_list([H|T]) -> case H of {ok, _T1, _T2} -> H; @@ -3047,6 +3061,9 @@ inf_opaque_types(IsOpaque1, T1, IsOpaque2, T2, Opaques) -> end end. +compatible_opaque_types(?opaque(Es1), ?opaque(Es2)) -> + [{O1, O2} || O1 <- Es1, O2 <- Es2, is_compat_opaque_names(O1, O2)]. + is_compat_opaque_names(Opaque1, Opaque2) -> #opaque{mod = Mod1, name = Name1, args = Args1} = Opaque1, #opaque{mod = Mod2, name = Name2, args = Args2} = Opaque2, diff --git a/lib/hipe/doc/src/notes.xml b/lib/hipe/doc/src/notes.xml index 314fd55ba3..58ca0b2138 100644 --- a/lib/hipe/doc/src/notes.xml +++ b/lib/hipe/doc/src/notes.xml @@ -31,6 +31,26 @@ </header> <p>This document describes the changes made to HiPE.</p> +<section><title>Hipe 3.15.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Fix a bug concerning parameterized opaque types. </p> + <p> + Own Id: OTP-14130</p> + </item> + <item> + <p> + Fixed xml issues in old release notes</p> + <p> + Own Id: OTP-14269</p> + </item> + </list> + </section> + +</section> + <section><title>Hipe 3.15.3</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -130,12 +150,12 @@ </item> <item> <p> - Various fixes and improvements to the HiPE LLVM backend. + Various fixes and improvements to the HiPE LLVM backend.</p> <list> <item>Add support for LLVM 3.7 and 3.8 in the HiPE/LLVM x86_64 backend</item> <item>Reinstate support for the LLVM backend on x86 (works OK for LLVM 3.5 to 3.7 -- LLVM 3.8 has a bug that prevents it from generating - correct native code on x86)</item> </list></p> + correct native code on x86)</item> </list> <p> Own Id: OTP-13626</p> </item> @@ -191,7 +211,7 @@ <item> <p> Fix various binary construction inconsistencies for hipe - compiled code. <list> <item>Passing bad field sizes to + compiled code.</p> <list> <item>Passing bad field sizes to binary constructions would throw <c>badarith</c> rather than <c>badarg</c>. Worse, in guards, when the unit size of the field was 1, the exception would leak rather than @@ -211,7 +231,7 @@ missing check for unit size match when inserting a binary. For example, a faulty expression like <c><<<<1:7>>/binary>></c> would - succeed.</item> </list></p> + succeed.</item> </list> <p> Own Id: OTP-13272</p> </item> diff --git a/lib/hipe/flow/ebb.inc b/lib/hipe/flow/ebb.inc index 58213e44d5..e4b7fd0efb 100644 --- a/lib/hipe/flow/ebb.inc +++ b/lib/hipe/flow/ebb.inc @@ -40,12 +40,14 @@ %% | {ebb_leaf, SuccesorLabel} %%-------------------------------------------------------------------- -%% XXX: Cheating big time! no recursive types --type ebb() :: {ebb_node, icode_lbl(), _} - | {ebb_leaf, icode_lbl()}. +-type ebb() :: ebb_node() + | ebb_leaf(). -record(ebb_node, {label :: icode_lbl(), successors :: [ebb()]}). +-type ebb_node() :: #ebb_node{}. + -record(ebb_leaf, {successor :: icode_lbl()}). +-type ebb_leaf() :: #ebb_leaf{}. %%-------------------------------------------------------------------- %% Returns a list of extended basic blocks. @@ -193,7 +195,7 @@ add_succ([Lbl|Lbls], Visited, Node, MkFun, EBBs, CFG) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% --spec mk_node(icode_lbl(), [ebb()]) -> #ebb_node{}. +-spec mk_node(icode_lbl(), [ebb()]) -> ebb_node(). mk_node(Label, Successors) -> #ebb_node{label=Label, successors=Successors}. -spec node_label(#ebb_node{}) -> icode_lbl(). @@ -202,11 +204,11 @@ node_label(#ebb_node{label=Label}) -> Label. -spec node_successors(#ebb_node{}) -> [ebb()]. node_successors(#ebb_node{successors=Successors}) -> Successors. --spec mk_leaf(icode_lbl()) -> #ebb_leaf{}. +-spec mk_leaf(icode_lbl()) -> ebb_leaf(). mk_leaf(NextEbb) -> #ebb_leaf{successor=NextEbb}. %% leaf_next(Leaf) -> Leaf#ebb_leaf.successor. --spec type(#ebb_node{}) -> 'node' ; (#ebb_leaf{}) -> 'leaf'. +-spec type(ebb_node()) -> 'node' ; (ebb_leaf()) -> 'leaf'. type(#ebb_node{}) -> node; type(#ebb_leaf{}) -> leaf. diff --git a/lib/hipe/icode/hipe_beam_to_icode.erl b/lib/hipe/icode/hipe_beam_to_icode.erl index 100bc0b0e2..2abecf7f18 100644 --- a/lib/hipe/icode/hipe_beam_to_icode.erl +++ b/lib/hipe/icode/hipe_beam_to_icode.erl @@ -148,7 +148,8 @@ trans_mfa_code(M,F,A, FunBeamCode, ClosureInfo) -> {Code3,_Env3} = mk_debug_calltrace(MFA, Env1, Code2), {Code3,_Env3} = {Code2,Env1}), %% For stack optimization - Leafness = leafness(Code3), + IsClosure = get_closure_info(MFA, ClosureInfo) =/= not_a_closure, + Leafness = leafness(Code3, IsClosure), IsLeaf = is_leaf_code(Leafness), Code4 = [FunLbl | @@ -156,7 +157,6 @@ trans_mfa_code(M,F,A, FunBeamCode, ClosureInfo) -> false -> Code3; true -> [mk_redtest()|Code3] end], - IsClosure = get_closure_info(MFA, ClosureInfo) =/= not_a_closure, Code5 = hipe_icode:mk_icode(MFA, FunArgs, IsClosure, IsLeaf, remove_dead_code(Code4), hipe_gensym:var_range(icode), @@ -173,12 +173,12 @@ trans_mfa_code(M,F,A, FunBeamCode, ClosureInfo) -> mk_redtest() -> hipe_icode:mk_primop([], redtest, []). -leafness(Is) -> % -> true, selfrec, or false - leafness(Is, true). +leafness(Is, IsClosure) -> % -> true, selfrec, closure, or false + leafness(Is, IsClosure, true). -leafness([], Leafness) -> +leafness([], _IsClosure, Leafness) -> Leafness; -leafness([I|Is], Leafness) -> +leafness([I|Is], IsClosure, Leafness) -> case I of #icode_comment{} -> %% BEAM self-tailcalls become gotos, but they leave @@ -191,7 +191,7 @@ leafness([I|Is], Leafness) -> 'self_tail_recursive' -> selfrec; % call_only to selfrec _ -> Leafness end, - leafness(Is, NewLeafness); + leafness(Is, IsClosure, NewLeafness); #icode_call{} -> case hipe_icode:call_type(I) of 'primop' -> @@ -199,12 +199,12 @@ leafness([I|Is], Leafness) -> call_fun -> false; % Calls closure enter_fun -> false; % Calls closure #apply_N{} -> false; - _ -> leafness(Is, Leafness) % Other primop calls are ok + _ -> leafness(Is, IsClosure, Leafness) % Other primop calls are ok end; T when T =:= 'local' orelse T =:= 'remote' -> {M,F,A} = hipe_icode:call_fun(I), case erlang:is_builtin(M, F, A) of - true -> leafness(Is, Leafness); + true -> leafness(Is, IsClosure, Leafness); false -> false end end; @@ -223,11 +223,12 @@ leafness([I|Is], Leafness) -> T when T =:= 'local' orelse T =:= 'remote' -> {M,F,A} = hipe_icode:enter_fun(I), case erlang:is_builtin(M, F, A) of - true -> leafness(Is, Leafness); + true -> leafness(Is, IsClosure, Leafness); + _ when IsClosure -> leafness(Is, IsClosure, closure); _ -> false end end; - _ -> leafness(Is, Leafness) + _ -> leafness(Is, IsClosure, Leafness) end. %% XXX: this old stuff is passed around but essentially unused @@ -235,12 +236,20 @@ is_leaf_code(Leafness) -> case Leafness of true -> true; selfrec -> true; + closure -> false; false -> false end. needs_redtest(Leafness) -> case Leafness of true -> false; + %% A "leaf" closure may contain tailcalls to non-closures in addition to + %% what other leaves may contain. Omitting the redtest is useful to generate + %% shorter code for closures generated by (fun F/A), and is safe since + %% control flow cannot return to a "leaf" closure again without a reduction + %% being consumed. This is true since no function that can call a closure + %% will ever have its redtest omitted. + closure -> false; selfrec -> true; false -> true end. @@ -504,6 +513,19 @@ trans_fun([{test,test_arity,{f,Lbl},[Reg,N]}|Instructions], Env) -> I = hipe_icode:mk_type([trans_arg(Reg)],{tuple,N}, hipe_icode:label_name(True),map_label(Lbl)), [I,True | trans_fun(Instructions,Env)]; +%%--- test_is_tagged_tuple --- +trans_fun([{test,is_tagged_tuple,{f,Lbl},[Reg,N,Atom]}|Instructions], Env) -> + TrueArity = mk_label(new), + IArity = hipe_icode:mk_type([trans_arg(Reg)],{tuple,N}, + hipe_icode:label_name(TrueArity),map_label(Lbl)), + Var = hipe_icode:mk_new_var(), + IGet = hipe_icode:mk_primop([Var], + #unsafe_element{index=1}, + [trans_arg(Reg)]), + TrueAtom = mk_label(new), + IEQ = hipe_icode:mk_type([Var], Atom, hipe_icode:label_name(TrueAtom), + map_label(Lbl)), + [IArity,TrueArity,IGet,IEQ,TrueAtom | trans_fun(Instructions,Env)]; %%--- is_map --- trans_fun([{test,is_map,{f,Lbl},[Arg]}|Instructions], Env) -> {Code,Env1} = trans_type_test(map,Lbl,Arg,Env), diff --git a/lib/hipe/icode/hipe_icode_range.erl b/lib/hipe/icode/hipe_icode_range.erl index b884132327..287b1c80fe 100644 --- a/lib/hipe/icode/hipe_icode_range.erl +++ b/lib/hipe/icode/hipe_icode_range.erl @@ -392,14 +392,17 @@ widen(#range{range=Old}, #range{range=New}, T = #range{range=Wide}) -> -spec analyse_call(#icode_call{}, call_fun()) -> #icode_call{}. analyse_call(Call, LookupFun) -> + Args = hipe_icode:args(Call), + Fun = hipe_icode:call_fun(Call), + Type = hipe_icode:call_type(Call), + %% This call has side-effects (it might call LookupFun which sends messages to + %% hipe_icode_coordinator to update the argument ranges of Fun), and must thus + %% not be moved into the case statement. + DstRanges = analyse_call_or_enter_fun(Fun, Args, Type, LookupFun), case hipe_icode:call_dstlist(Call) of [] -> Call; Dsts -> - Args = hipe_icode:args(Call), - Fun = hipe_icode:call_fun(Call), - Type = hipe_icode:call_type(Call), - DstRanges = analyse_call_or_enter_fun(Fun, Args, Type, LookupFun), NewDefs = [update_info(Var, R) || {Var,R} <- lists:zip(Dsts, DstRanges)], hipe_icode:subst_defines(lists:zip(Dsts, NewDefs), Call) end. @@ -1306,16 +1309,15 @@ range_rem(Range1, Range2) -> Min1_geq_zero = inf_geq(Min1, 0), Max1_leq_zero = inf_geq(0, Max1), Max_range2 = inf_max([inf_abs(Min2), inf_abs(Max2)]), - Max_range2_leq_zero = inf_geq(0, Max_range2), New_min = if Min1_geq_zero -> 0; - Max_range2_leq_zero -> Max_range2; - true -> inf_inv(Max_range2) + Max_range2 =:= 0 -> 0; + true -> inf_add(inf_inv(Max_range2), 1) end, New_max = if Max1_leq_zero -> 0; - Max_range2_leq_zero -> inf_inv(Max_range2); - true -> Max_range2 + Max_range2 =:= 0 -> 0; + true -> inf_add(Max_range2, -1) end, range_init({New_min, New_max}, false). diff --git a/lib/hipe/icode/hipe_icode_type.erl b/lib/hipe/icode/hipe_icode_type.erl index 815d1e57a8..aafaeb5a0a 100644 --- a/lib/hipe/icode/hipe_icode_type.erl +++ b/lib/hipe/icode/hipe_icode_type.erl @@ -1410,9 +1410,10 @@ transform_element2(I) -> NewIndex = case test_type(integer, IndexType) of true -> - case t_number_vals(IndexType) of - unknown -> unknown; - [_|_] = Vals -> {number, Vals} + case {number_min(IndexType), number_max(IndexType)} of + {Lb0, Ub0} when is_integer(Lb0), is_integer(Ub0) -> + {number, Lb0, Ub0}; + {_, _} -> unknown end; _ -> unknown end, @@ -1427,19 +1428,19 @@ transform_element2(I) -> _ -> unknown end, case {NewIndex, MinSize} of - {{number, [_|_] = Ns}, {tuple, A}} when is_integer(A) -> - case lists:all(fun(X) -> 0 < X andalso X =< A end, Ns) of + {{number, Lb, Ub}, {tuple, A}} when is_integer(A) -> + case 0 < Lb andalso Ub =< A of true -> - case Ns of - [Idx] -> + case {Lb, Ub} of + {Idx, Idx} -> [_, Tuple] = hipe_icode:args(I), update_call_or_enter(I, #unsafe_element{index = Idx}, [Tuple]); - [_|_] -> + {_, _} -> NewFun = {element, [MinSize, valid]}, update_call_or_enter(I, NewFun) end; false -> - case lists:all(fun(X) -> hipe_tagscheme:is_fixnum(X) end, Ns) of + case lists:all(fun(X) -> hipe_tagscheme:is_fixnum(X) end, [Lb, Ub]) of true -> NewFun = {element, [MinSize, fixnums]}, update_call_or_enter(I, NewFun); @@ -1454,7 +1455,7 @@ transform_element2(I) -> NewFun = {element, [MinSize, fixnums]}, update_call_or_enter(I, NewFun); false -> - NewFun = {element, [MinSize, NewIndex]}, + NewFun = {element, [MinSize, NewIndex]}, update_call_or_enter(I, NewFun) end end. diff --git a/lib/hipe/llvm/hipe_llvm_merge.erl b/lib/hipe/llvm/hipe_llvm_merge.erl index 6e891ac3b0..58d862fbb2 100644 --- a/lib/hipe/llvm/hipe_llvm_merge.erl +++ b/lib/hipe/llvm/hipe_llvm_merge.erl @@ -13,7 +13,7 @@ finalize(CompiledCode, Closures, Exports) -> Code = [{MFA, [], ConstTab} || {MFA, _, _ , ConstTab, _, _} <- CompiledCode1], {ConstAlign, ConstSize, ConstMap, RefsFromConsts} = - hipe_pack_constants:pack_constants(Code, ?ARCH_REGISTERS:alignment()), + hipe_pack_constants:pack_constants(Code), %% Compute total code size separately as a sanity check for alignment CodeSize = compute_code_size(CompiledCode1, 0), %% io:format("Code Size (pre-computed): ~w~n", [CodeSize]), diff --git a/lib/hipe/llvm/hipe_rtl_to_llvm.erl b/lib/hipe/llvm/hipe_rtl_to_llvm.erl index 208d86841f..79e1bfd381 100644 --- a/lib/hipe/llvm/hipe_rtl_to_llvm.erl +++ b/lib/hipe/llvm/hipe_rtl_to_llvm.erl @@ -1364,7 +1364,7 @@ create_function_definition(Fun, Params, Code, LocalVars) -> EntryBlock = lists:flatten([EntryLabel, ExceptionSync, I2, LocalVars, StoredParams, I3]), Final_Code = EntryBlock ++ Code, - FunctionOptions = [nounwind, noredzone, list_to_atom("gc \"erlang\"")], + FunctionOptions = [nounwind, noredzone, 'gc "erlang"'], WordTy = hipe_llvm:mk_int(?BITS_IN_WORD), FunRetTy = hipe_llvm:mk_struct(lists:duplicate(?NR_PINNED_REGS + 1, WordTy)), hipe_llvm:mk_fun_def([], [], "cc 11", [], FunRetTy, FunctionName, Args, diff --git a/lib/hipe/main/hipe.app.src b/lib/hipe/main/hipe.app.src index af2c02006d..de0b255c01 100644 --- a/lib/hipe/main/hipe.app.src +++ b/lib/hipe/main/hipe.app.src @@ -76,6 +76,7 @@ hipe_arm_specific, hipe_arm_subst, hipe_bb, + hipe_bb_weights, hipe_beam_to_icode, hipe_coalescing_regalloc, hipe_consttab, @@ -83,6 +84,7 @@ hipe_digraph, hipe_dominators, hipe_dot, + hipe_dsets, hipe_gen_cfg, hipe_gensym, hipe_graph_coloring_regalloc, @@ -146,9 +148,11 @@ hipe_ppc_specific_fp, hipe_ppc_subst, hipe_profile, + hipe_range_split, hipe_reg_worklists, hipe_regalloc_loop, hipe_regalloc_prepass, + hipe_restore_reuse, hipe_rtl, hipe_rtl_arch, hipe_rtl_arith_32, diff --git a/lib/hipe/main/hipe.erl b/lib/hipe/main/hipe.erl index fff397b060..19b4e8bfe2 100644 --- a/lib/hipe/main/hipe.erl +++ b/lib/hipe/main/hipe.erl @@ -1230,6 +1230,18 @@ option_text(regalloc) -> " optimistic - another variant of a coalescing allocator"; option_text(remove_comments) -> "Strip comments from intermediate code"; +option_text(ra_range_split) -> + "Split live ranges of temporaries live over call instructions\n" + "before performing register allocation.\n" + "Heuristically tries to move stack accesses to the cold path of function.\n" + "This range splitter is more sophisticated than 'ra_restore_reuse', but has\n" + "a significantly larger impact on compile time.\n" + "Should only be used with move coalescing register allocators."; +option_text(ra_restore_reuse) -> + "Split live ranges of temporaries such that straight-line\n" + "code will not need to contain multiple restores from the same stack\n" + "location.\n" + "Should only be used with move coalescing register allocators."; option_text(rtl_ssa) -> "Perform SSA conversion on the RTL level -- default starting at O2"; option_text(rtl_ssa_const_prop) -> @@ -1371,6 +1383,12 @@ opt_keys() -> pp_rtl_linear, ra_partitioned, ra_prespill, + ra_range_split, + ra_restore_reuse, + range_split_min_gain, + range_split_mode1_fudge, + range_split_weight_power, + range_split_weights, regalloc, remove_comments, rtl_ssa, @@ -1409,7 +1427,8 @@ o1_opts(TargetArch) -> icode_ssa_const_prop, icode_ssa_copy_prop, icode_inline_bifs, rtl_ssa, rtl_ssa_const_prop, rtl_ssapre, spillmin_color, use_indexing, remove_comments, - binary_opt, {regalloc,coalescing} | o0_opts(TargetArch)], + binary_opt, {regalloc,coalescing}, ra_restore_reuse + | o0_opts(TargetArch)], case TargetArch of ultrasparc -> Common; @@ -1429,7 +1448,8 @@ o1_opts(TargetArch) -> o2_opts(TargetArch) -> Common = [icode_type, icode_call_elim, % icode_ssa_struct_reuse, - rtl_lcm | (o1_opts(TargetArch) -- [rtl_ssapre])], + ra_range_split, range_split_weights, % XXX: Having defaults here is ugly + rtl_lcm | (o1_opts(TargetArch) -- [rtl_ssapre, ra_restore_reuse])], case TargetArch of T when T =:= amd64 orelse T =:= ppc64 -> % 64-bit targets [icode_range | Common]; @@ -1477,6 +1497,9 @@ opt_negations() -> {no_pp_rtl_ssapre, pp_rtl_ssapre}, {no_ra_partitioned, ra_partitioned}, {no_ra_prespill, ra_prespill}, + {no_ra_range_split, ra_range_split}, + {no_ra_restore_reuse, ra_restore_reuse}, + {no_range_split_weights, range_split_weights}, {no_remove_comments, remove_comments}, {no_rtl_ssa, rtl_ssa}, {no_rtl_ssa_const_prop, rtl_ssa_const_prop}, diff --git a/lib/hipe/misc/hipe_consttab.erl b/lib/hipe/misc/hipe_consttab.erl index 64e3d3ccaa..741bdb2094 100644 --- a/lib/hipe/misc/hipe_consttab.erl +++ b/lib/hipe/misc/hipe_consttab.erl @@ -63,9 +63,7 @@ %% A hipe_consttab is a tuple {Data, ReferedLabels, NextConstLabel} %% @type hipe_constlbl(). %% An abstract datatype for referring to data. -%% @type element_type() = byte | word | ctab_array() -%% @type ctab_array() = {ctab_array, Type::element_type(), -%% NoElements::pos_integer()} +%% @type element_type() = byte | word %% @type block() = [integer() | label_ref()] %% @type label_ref() = {label, Label::code_label()} %% @type code_label() = hipe_sparc:label_name() | hipe_x86:label_name() @@ -110,8 +108,7 @@ -type label_ref() :: {'label', code_label()}. -type block() :: [hipe_constlbl() | label_ref()]. --type ctab_array() :: {'ctab_array', 'byte' | 'word', pos_integer()}. --type element_type() :: 'byte' | 'word' | ctab_array(). +-type element_type() :: 'byte' | 'word'. -type sort_order() :: term(). % XXX: FIXME @@ -187,7 +184,7 @@ insert_block({ConstTab, RefToLabels, NextLabel}, ElementType, InitList) -> ReferredLabels = get_labels(InitList, []), NewRefTo = ReferredLabels ++ RefToLabels, {NewTa, Id} = insert_const({ConstTab, NewRefTo, NextLabel}, - block, word_size(), false, + block, size_of(ElementType), false, {ElementType,InitList}), {insert_backrefs(NewTa, Id, ReferredLabels), Id}. @@ -256,13 +253,9 @@ get_labels([], Acc) -> %% @spec size_of(element_type()) -> pos_integer() %% @doc Returns the size in bytes of an element_type. -%% The is_atom/1 guard in the clause handling arrays -%% constraints the argument to 'byte' | 'word' -spec size_of(element_type()) -> pos_integer(). size_of(byte) -> 1; -size_of(word) -> word_size(); -size_of({ctab_array,S,N}) when is_atom(S), is_integer(N), N > 0 -> - N * size_of(S). +size_of(word) -> word_size(). %% @spec decompose({element_type(), block()}) -> [byte()] %% @doc Turns a block into a list of bytes. diff --git a/lib/hipe/misc/hipe_pack_constants.erl b/lib/hipe/misc/hipe_pack_constants.erl index 9dd18bce0f..6736d1f503 100644 --- a/lib/hipe/misc/hipe_pack_constants.erl +++ b/lib/hipe/misc/hipe_pack_constants.erl @@ -13,7 +13,7 @@ %% limitations under the License. -module(hipe_pack_constants). --export([pack_constants/2, slim_refs/1, slim_constmap/1, +-export([pack_constants/1, slim_refs/1, slim_constmap/1, find_const/2, mk_data_relocs/2, slim_sorted_exportmap/3]). -include("hipe_consttab.hrl"). @@ -37,8 +37,8 @@ -record(pcm_entry, {mfa :: mfa(), label :: hipe_constlbl(), - const_num :: const_num(), - start :: addr(), + const_num :: const_num(), + start :: addr(), type :: 0 | 1 | 2, raw_data :: raw_data()}). -type pcm_entry() :: #pcm_entry{}. @@ -53,11 +53,11 @@ %%----------------------------------------------------------------------------- --spec pack_constants([{mfa(),[_],hipe_consttab()}], ct_alignment()) -> +-spec pack_constants([{mfa(),[_],hipe_consttab()}]) -> {ct_alignment(), non_neg_integer(), packed_const_map(), mfa_refs_map()}. -pack_constants(Data, Align) -> - pack_constants(Data, 0, Align, 0, [], []). +pack_constants(Data) -> + pack_constants(Data, 0, 1, 0, [], []). % 1 = byte alignment pack_constants([{MFA,_,ConstTab}|Rest], Size, Align, ConstNo, Acc, Refs) -> Labels = hipe_consttab:labels(ConstTab), diff --git a/lib/hipe/opt/Makefile b/lib/hipe/opt/Makefile index 684d6f45b4..5a729d04ae 100644 --- a/lib/hipe/opt/Makefile +++ b/lib/hipe/opt/Makefile @@ -43,7 +43,8 @@ RELSYSDIR = $(RELEASE_PATH)/lib/hipe-$(VSN) # ---------------------------------------------------- # Target Specs # ---------------------------------------------------- -MODULES = hipe_spillmin hipe_spillmin_color hipe_spillmin_scan +MODULES = hipe_spillmin hipe_spillmin_color hipe_spillmin_scan \ + hipe_bb_weights HRL_FILES= ERL_FILES= $(MODULES:%=%.erl) diff --git a/lib/hipe/opt/hipe_bb_weights.erl b/lib/hipe/opt/hipe_bb_weights.erl new file mode 100644 index 0000000000..8ef113b94c --- /dev/null +++ b/lib/hipe/opt/hipe_bb_weights.erl @@ -0,0 +1,449 @@ +%% -*- erlang-indent-level: 2 -*- +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%@doc +%% BASIC BLOCK WEIGHTING +%% +%% Computes basic block weights by using branch probabilities as weights in a +%% linear equation system, that is then solved using Gauss-Jordan Elimination. +%% +%% The equation system representation is intentionally sparse, since most blocks +%% have at most two successors. +-module(hipe_bb_weights). +-export([compute/3, compute_fast/3, weight/2, call_exn_pred/0]). +-export_type([bb_weights/0]). + +-compile(inline). + +%%-define(DO_ASSERT,1). +%%-define(DEBUG,1). +-include("../main/hipe.hrl"). + +%% If the equation system is large, it might take too long to solve it exactly. +%% Thus, if there are more than ?HEUR_MAX_SOLVE labels, we use the iterative +%% approximation. +-define(HEUR_MAX_SOLVE, 10000). + +-opaque bb_weights() :: #{label() => float()}. + +-type cfg() :: any(). +-type target_module() :: module(). +-type target_context() :: any(). +-type target() :: {target_module(), target_context()}. + +-type label() :: integer(). +-type var() :: label(). +-type assignment() :: {var(), float()}. +-type eq_assoc() :: [{var(), key()}]. +-type solution() :: [assignment()]. + +%% Constant. Predicted probability of a call resulting in an exception. +-spec call_exn_pred() -> float(). +call_exn_pred() -> 0.01. + +-spec compute(cfg(), target_module(), target_context()) -> bb_weights(). +compute(CFG, TgtMod, TgtCtx) -> + Target = {TgtMod, TgtCtx}, + Labels = labels(CFG, Target), + if length(Labels) > ?HEUR_MAX_SOLVE -> + ?debug_msg("~w: Too many labels (~w), approximating.~n", + [?MODULE, length(Labels)]), + compute_fast(CFG, TgtMod, TgtCtx); + true -> + {EqSys, EqAssoc} = build_eq_system(CFG, Labels, Target), + case solve(EqSys, EqAssoc) of + {ok, Solution} -> + maps:from_list(Solution) + end + end. + +-spec build_eq_system(cfg(), [label()], target()) -> {eq_system(), eq_assoc()}. +build_eq_system(CFG, Labels, Target) -> + StartLb = hipe_gen_cfg:start_label(CFG), + EQS0 = eqs_new(), + {EQS1, Assoc} = build_eq_system(Labels, CFG, Target, [], EQS0), + {StartLb, StartKey} = lists:keyfind(StartLb, 1, Assoc), + StartRow0 = eqs_get(StartKey, EQS1), + StartRow = row_set_const(-1.0, StartRow0), % -1.0 since StartLb coef is -1.0 + EQS = eqs_put(StartKey, StartRow, EQS1), + {EQS, Assoc}. + +build_eq_system([], _CFG, _Target, Map, EQS) -> {EQS, lists:reverse(Map)}; +build_eq_system([L|Ls], CFG, Target, Map, EQS0) -> + PredProb = pred_prob(L, CFG, Target), + {Key, EQS} = eqs_insert(row_new([{L, -1.0}|PredProb], 0.0), EQS0), + build_eq_system(Ls, CFG, Target, [{L, Key}|Map], EQS). + +pred_prob(L, CFG, Target) -> + [begin + BB = bb(CFG, Pred, Target), + Ps = branch_preds(hipe_bb:last(BB), Target), + ?ASSERT(length(lists:ukeysort(1, Ps)) + =:= length(hipe_gen_cfg:succ(CFG, Pred))), + case lists:keyfind(L, 1, Ps) of + {L, Prob} when is_float(Prob) -> {Pred, Prob} + end + end || Pred <- hipe_gen_cfg:pred(CFG, L)]. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +-spec triangelise(eq_system(), eq_assoc()) -> {eq_system(), eq_assoc()}. +triangelise(EQS, VKs) -> + triangelise_1(mk_triix(EQS, VKs), []). + +triangelise_1(TIX0, Acc) -> + case triix_is_empty(TIX0) of + true -> {triix_eqs(TIX0), lists:reverse(Acc)}; + false -> + {V,Key,TIX1} = triix_pop_smallest(TIX0), + Row0 = triix_get(Key, TIX1), + case row_get(V, Row0) of + Coef when Coef > -0.0001, Coef < 0.0001 -> + throw(error); + _ -> + Row = row_normalise(V, Row0), + TIX2 = triix_put(Key, Row, TIX1), + TIX = eliminate_triix(V, Key, Row, TIX2), + triangelise_1(TIX, [{V,Key}|Acc]) + end + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Triangelisation maintains its own index, outside of eqs. This index is +%% essentially a BST (used as a heap) of all equations by size, with {Key,Var} +%% as the values and only containing a subset of all the keys in the whole +%% equation system. The key operation is triix_pop_smallest/1, which pops a +%% {Key,Var} from the heap corresponding to one of the smallest equations. This +%% is critical in order to prevent the equations from growing during +%% triangelisation, which would make the algorithm O(n^2) in the common case. +-type tri_eq_system() :: {eq_system(), + gb_trees:tree(non_neg_integer(), + gb_trees:tree(key(), var()))}. + +triix_eqs({EQS, _}) -> EQS. +triix_get(Key, {EQS, _}) -> eqs_get(Key, EQS). +triix_is_empty({_, Tree}) -> gb_trees:is_empty(Tree). +triix_lookup(V, {EQS, _}) -> eqs_lookup(V, EQS). + +mk_triix(EQS, VKs) -> + {EQS, + lists:foldl(fun({V,Key}, Tree) -> + Size = row_size(eqs_get(Key, EQS)), + sitree_insert(Size, Key, V, Tree) + end, gb_trees:empty(), VKs)}. + +sitree_insert(Size, Key, V, SiTree) -> + SubTree1 = + case gb_trees:lookup(Size, SiTree) of + none -> gb_trees:empty(); + {value, SubTree0} -> SubTree0 + end, + SubTree = gb_trees:insert(Key, V, SubTree1), + gb_trees:enter(Size, SubTree, SiTree). + +sitree_update_subtree(Size, SubTree, SiTree) -> + case gb_trees:is_empty(SubTree) of + true -> gb_trees:delete(Size, SiTree); + false -> gb_trees:update(Size, SubTree, SiTree) + end. + +triix_put(Key, Row, {EQS, Tree0}) -> + OldSize = row_size(eqs_get(Key, EQS)), + case row_size(Row) of + OldSize -> {eqs_put(Key, Row, EQS), Tree0}; + Size -> + Tree = + case gb_trees:lookup(OldSize, Tree0) of + none -> Tree0; + {value, SubTree0} -> + case gb_trees:lookup(Key, SubTree0) of + none -> Tree0; + {value, V} -> + SubTree = gb_trees:delete(Key, SubTree0), + Tree1 = sitree_update_subtree(OldSize, SubTree, Tree0), + sitree_insert(Size, Key, V, Tree1) + end + end, + {eqs_put(Key, Row, EQS), Tree} + end. + +triix_pop_smallest({EQS, Tree}) -> + {Size, SubTree0} = gb_trees:smallest(Tree), + {Key, V, SubTree} = gb_trees:take_smallest(SubTree0), + {V, Key, {EQS, sitree_update_subtree(Size, SubTree, Tree)}}. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +row_normalise(Var, Row) -> + %% Normalise v's coef to 1.0 + %% row_set_coef ensures the coef is exactly 1.0 (no rounding errors) + row_set_coef(Var, 1.0, row_scale(Row, 1.0/row_get(Var, Row))). + +%% Precondition: Row must be normalised; i.e. Vars coef must be 1.0 (mod +%% rounding errors) +-spec eliminate(var(), key(), row(), eq_system()) -> eq_system(). +eliminate(Var, Key, Row, TIX0) -> + eliminate_abstr(Var, Key, Row, TIX0, + fun eqs_get/2, fun eqs_lookup/2, fun eqs_put/3). + +-spec eliminate_triix(var(), key(), row(), tri_eq_system()) -> tri_eq_system(). +eliminate_triix(Var, Key, Row, TIX0) -> + eliminate_abstr(Var, Key, Row, TIX0, + fun triix_get/2, fun triix_lookup/2, fun triix_put/3). + +%% The same function implemented for two data types, eqs and triix. +-compile({inline, eliminate_abstr/7}). +-spec eliminate_abstr(var(), key(), row(), ADT, fun((key(), ADT) -> row()), + fun((var(), ADT) -> [key()]), + fun((key(), row(), ADT) -> ADT)) -> ADT. +eliminate_abstr(Var, Key, Row, ADT0, GetFun, LookupFun, PutFun) -> + ?ASSERT(1.0 =:= row_get(Var, Row)), + ADT = + lists:foldl(fun(RK, ADT1) when RK =:= Key -> ADT1; + (RK, ADT1) -> + R = GetFun(RK, ADT1), + PutFun(RK, row_addmul(R, Row, -row_get(Var, R)), ADT1) + end, ADT0, LookupFun(Var, ADT0)), + [Key] = LookupFun(Var, ADT), + ADT. + +-spec solve(eq_system(), eq_assoc()) -> error | {ok, solution()}. +solve(EQS0, EqAssoc0) -> + try triangelise(EQS0, EqAssoc0) + of {EQS1, EqAssoc} -> + {ok, solve_1(EqAssoc, maps:from_list(EqAssoc), EQS1, [])} + catch error -> error + end. + +solve_1([], _VarEqs, _EQS, Acc) -> Acc; +solve_1([{V,K}|Ps], VarEqs, EQS0, Acc0) -> + Row0 = eqs_get(K, EQS0), + VarsToKill = [Var || {Var, _} <- row_coefs(Row0), Var =/= V], + Row1 = kill_vars(VarsToKill, VarEqs, EQS0, Row0), + [{V,_}] = row_coefs(Row1), % assertion + Row = row_normalise(V, Row1), + [{V,1.0}] = row_coefs(Row), % assertion + EQS = eliminate(V, K, Row, EQS0), + [K] = eqs_lookup(V, EQS), + solve_1(Ps, VarEqs, eqs_remove(K, EQS), [{V, row_const(Row)}|Acc0]). + +kill_vars([], _VarEqs, _EQS, Row) -> Row; +kill_vars([V|Vs], VarEqs, EQS, Row0) -> + VRow0 = eqs_get(maps:get(V, VarEqs), EQS), + VRow = row_normalise(V, VRow0), + ?ASSERT(1.0 =:= row_get(V, VRow)), + Row = row_addmul(Row0, VRow, -row_get(V, Row0)), + ?ASSERT(0.0 =:= row_get(V, Row)), % V has been killed + kill_vars(Vs, VarEqs, EQS, Row). + +-spec weight(label(), bb_weights()) -> float(). +weight(Lbl, Weights) -> + maps:get(Lbl, Weights). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Row datatype +%% Invariant: No 0.0 coefficiets! +-spec row_empty() -> row(). +row_empty() -> {orddict:new(), 0.0}. + +-spec row_new([{var(), float()}], float()) -> row(). +row_new(Coefs, Const) when is_float(Const) -> + row_ensure_invar({row_squash_multiples(lists:keysort(1, Coefs)), Const}). + +row_squash_multiples([{K, C1},{K, C2}|Ps]) -> + row_squash_multiples([{K,C1+C2}|Ps]); +row_squash_multiples([P|Ps]) -> [P|row_squash_multiples(Ps)]; +row_squash_multiples([]) -> []. + +row_ensure_invar({Coef, Const}) -> + {orddict:filter(fun(_, 0.0) -> false; (_, F) when is_float(F) -> true end, + Coef), Const}. + +row_const({_, Const}) -> Const. +row_coefs({Coefs, _}) -> orddict:to_list(Coefs). +row_size({Coefs, _}) -> orddict:size(Coefs). + +row_get(Var, {Coefs, _}) -> + case lists:keyfind(Var, 1, Coefs) of + false -> 0.0; + {_, Coef} -> Coef + end. + +row_set_coef(Var, 0.0, {Coefs, Const}) -> + {orddict:erase(Var, Coefs), Const}; +row_set_coef(Var, Coef, {Coefs, Const}) -> + {orddict:store(Var, Coef, Coefs), Const}. + +row_set_const(Const, {Coefs, _}) -> {Coefs, Const}. + +%% Lhs + Rhs*Factor +-spec row_addmul(row(), row(), float()) -> row(). +row_addmul({LhsCoefs, LhsConst}, {RhsCoefs, RhsConst}, Factor) + when is_float(Factor) -> + Coefs = row_addmul_coefs(LhsCoefs, RhsCoefs, Factor), + Const = LhsConst + RhsConst * Factor, + {Coefs, Const}. + +row_addmul_coefs(Ls, [], Factor) when is_float(Factor) -> Ls; +row_addmul_coefs([], Rs, Factor) when is_float(Factor) -> + row_scale_coefs(Rs, Factor); +row_addmul_coefs([L={LV, _}|Ls], Rs=[{RV,_}|_], Factor) + when LV < RV, is_float(Factor) -> + [L|row_addmul_coefs(Ls, Rs, Factor)]; +row_addmul_coefs(Ls=[{LV, _}|_], [{RV, RC}|Rs], Factor) + when LV > RV, is_float(RC), is_float(Factor) -> + [{RV, RC*Factor}|row_addmul_coefs(Ls, Rs, Factor)]; +row_addmul_coefs([{V, LC}|Ls], [{V, RC}|Rs], Factor) + when is_float(LC), is_float(RC), is_float(Factor) -> + case LC + RC * Factor of + 0.0 -> row_addmul_coefs(Ls, Rs, Factor); + C -> [{V,C}|row_addmul_coefs(Ls, Rs, Factor)] + end. + +row_scale(_, 0.0) -> row_empty(); +row_scale({RowCoefs, RowConst}, Factor) when is_float(Factor) -> + {row_scale_coefs(RowCoefs, Factor), RowConst * Factor}. + +row_scale_coefs([{V,C}|Cs], Factor) when is_float(Factor), is_float(C) -> + [{V,C*Factor}|row_scale_coefs(Cs, Factor)]; +row_scale_coefs([], Factor) when is_float(Factor) -> + []. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Equation system ADT +%% +%% Stores a linear equation system, allowing for efficient updates and efficient +%% queries for all equations mentioning a variable. +%% +%% It is sort of like a "database" table of {Primary, Terms, Const} indexed both +%% on Primary as well as the vars (map keys) in Terms. +-type row() :: {Terms :: orddict:orddict(var(), float()), + Const :: float()}. +-type key() :: non_neg_integer(). +-type rev_index() :: #{var() => ordsets:ordset(key())}. +-record(eq_system, { + rows = #{} :: #{key() => row()}, + revidx = revidx_empty() :: rev_index(), + next_key = 0 :: key() + }). +-type eq_system() :: #eq_system{}. + +eqs_new() -> #eq_system{}. + +-spec eqs_insert(row(), eq_system()) -> {key(), eq_system()}. +eqs_insert(Row, EQS=#eq_system{next_key=NextKey0}) -> + Key = NextKey0, + NextKey = NextKey0 + 1, + {Key, eqs_insert(Key, Row, EQS#eq_system{next_key=NextKey})}. + +eqs_insert(Key, Row, EQS=#eq_system{rows=Rows, revidx=RevIdx0}) -> + RevIdx = revidx_add(Key, Row, RevIdx0), + EQS#eq_system{rows=Rows#{Key => Row}, revidx=RevIdx}. + +eqs_put(Key, Row, EQS0) -> + eqs_insert(Key, Row, eqs_remove(Key, EQS0)). + +eqs_remove(Key, EQS=#eq_system{rows=Rows, revidx=RevIdx0}) -> + OldRow = maps:get(Key, Rows), + RevIdx = revidx_remove(Key, OldRow, RevIdx0), + EQS#eq_system{rows = maps:remove(Key, Rows), revidx=RevIdx}. + +-spec eqs_get(key(), eq_system()) -> row(). +eqs_get(Key, #eq_system{rows=Rows}) -> maps:get(Key, Rows). + +%% Keys of all equations containing a nonzero coefficient for Var +-spec eqs_lookup(var(), eq_system()) -> ordsets:ordset(key()). +eqs_lookup(Var, #eq_system{revidx=RevIdx}) -> maps:get(Var, RevIdx). + +%% eqs_rows(#eq_system{rows=Rows}) -> maps:to_list(Rows). + +%% eqs_print(EQS) -> +%% lists:foreach(fun({_, Row}) -> +%% row_print(Row) +%% end, lists:sort(eqs_rows(EQS))). + +%% row_print(Row) -> +%% CoefStrs = [io_lib:format("~wl~w", [Coef, Var]) +%% || {Var, Coef} <- row_coefs(Row)], +%% CoefStr = lists:join(" + ", CoefStrs), +%% io:format("~w = ~s~n", [row_const(Row), CoefStr]). + +revidx_empty() -> #{}. + +-spec revidx_add(key(), row(), rev_index()) -> rev_index(). +revidx_add(Key, Row, RevIdx0) -> + orddict:fold(fun(Var, _Coef, RevIdx1) -> + ?ASSERT(_Coef /= 0.0), + RevIdx1#{Var => ordsets:add_element( + Key, maps:get(Var, RevIdx1, ordsets:new()))} + end, RevIdx0, row_coefs(Row)). + +-spec revidx_remove(key(), row(), rev_index()) -> rev_index(). +revidx_remove(Key, {Coefs, _}, RevIdx0) -> + orddict:fold(fun(Var, _Coef, RevIdx1) -> + case RevIdx1 of + #{Var := Keys0} -> + case ordsets:del_element(Key, Keys0) of + [] -> maps:remove(Var, RevIdx1); + Keys -> RevIdx1#{Var := Keys} + end + end + end, RevIdx0, Coefs). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +-define(FAST_ITERATIONS, 5). + +%% @doc Computes a rough approximation of BB weights. The approximation is +%% particularly poor (converges slowly) for recursive functions and loops. +-spec compute_fast(cfg(), target_module(), target_context()) -> bb_weights(). +compute_fast(CFG, TgtMod, TgtCtx) -> + Target = {TgtMod, TgtCtx}, + StartLb = hipe_gen_cfg:start_label(CFG), + RPO = reverse_postorder(CFG, Target), + PredProbs = [{L, pred_prob(L, CFG, Target)} || L <- RPO, L =/= StartLb], + Probs0 = (maps:from_list([{L, 0.0} || L <- RPO]))#{StartLb := 1.0}, + fast_iterate(?FAST_ITERATIONS, PredProbs, Probs0). + +fast_iterate(0, _Pred, Probs) -> Probs; +fast_iterate(Iters, Pred, Probs0) -> + fast_iterate(Iters-1, Pred, + fast_one(Pred, Probs0)). + +fast_one([{L, Pred}|Ls], Probs0) -> + Weight = fast_sum(Pred, Probs0, 0.0), + Probs = Probs0#{L => Weight}, + fast_one(Ls, Probs); +fast_one([], Probs) -> + Probs. + +fast_sum([{P,EWt}|Pred], Probs, Acc) when is_float(EWt), is_float(Acc) -> + case Probs of + #{P := PWt} when is_float(PWt) -> + fast_sum(Pred, Probs, Acc + PWt * EWt) + end; +fast_sum([], _Probs, Acc) when is_float(Acc) -> + Acc. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Target module interface functions +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +-define(TGT_IFACE_0(N), N( {M,C}) -> M:N( C)). +-define(TGT_IFACE_1(N), N(A1, {M,C}) -> M:N(A1, C)). +-define(TGT_IFACE_2(N), N(A1,A2, {M,C}) -> M:N(A1,A2, C)). +-define(TGT_IFACE_3(N), N(A1,A2,A3,{M,C}) -> M:N(A1,A2,A3,C)). + +?TGT_IFACE_2(bb). +?TGT_IFACE_1(branch_preds). +?TGT_IFACE_1(labels). +?TGT_IFACE_1(reverse_postorder). diff --git a/lib/hipe/opt/hipe_spillmin_color.erl b/lib/hipe/opt/hipe_spillmin_color.erl index 41f1972df7..f87d9a5b61 100644 --- a/lib/hipe/opt/hipe_spillmin_color.erl +++ b/lib/hipe/opt/hipe_spillmin_color.erl @@ -166,9 +166,13 @@ remap_temp_map0(Cols, [_Y|Ys], SpillIndex) -> %% build_ig(CFG, Live, Target, TempMap) -> - try build_ig0(CFG, Live, Target, TempMap) - catch error:Rsn -> exit({regalloc, build_ig, Rsn}) - end. + TempMapping = map_spilled_temporaries(TempMap), + TempMappingTable = setup_ets(TempMapping), + NumSpilled = length(TempMapping), + IG = build_ig_bbs(labels(CFG, Target), CFG, Live, empty_ig(NumSpilled), + Target, TempMap, TempMappingTable), + ets:delete(TempMappingTable), + {normalize_ig(IG), NumSpilled}. %% Creates an ETS table consisting of the keys given in List, with the values %% being an integer which is the position of the key in List. @@ -183,15 +187,6 @@ setup_ets0([X|Xs], Table, N) -> ets:insert(Table, {X, N}), setup_ets0(Xs, Table, N+1). -build_ig0(CFG, Live, Target, TempMap) -> - TempMapping = map_spilled_temporaries(TempMap), - TempMappingTable = setup_ets(TempMapping), - NumSpilled = length(TempMapping), - IG = build_ig_bbs(labels(CFG, Target), CFG, Live, empty_ig(NumSpilled), - Target, TempMap, TempMappingTable), - ets:delete(TempMappingTable), - {normalize_ig(IG), NumSpilled}. - build_ig_bbs([], _CFG, _Live, IG, _Target, _TempMap, _TempMapping) -> IG; build_ig_bbs([L|Ls], CFG, Live, IG, Target, TempMap, TempMapping) -> @@ -212,16 +207,26 @@ build_ig_bb([X|Xs], LiveOut, IG, Target, TempMap, TempMapping) -> build_ig_bb(Xs, LiveOut, IG, Target, TempMap, TempMapping), build_ig_instr(X, Live, NewIG, Target, TempMap, TempMapping). -build_ig_instr(X, Live, IG, Target, TempMap, TempMapping) -> +build_ig_instr(X, Live0, IG0, Target, TempMap, TempMapping) -> {Def, Use} = def_use(X, Target, TempMap), - ?report3("Live ~w\n~w : Def: ~w Use ~w\n",[Live, X, Def,Use]), + ?report3("Live ~w\n~w : Def: ~w Use ~w\n",[Live0, X, Def,Use]), DefListMapped = list_map(Def, TempMapping, []), UseListMapped = list_map(Use, TempMapping, []), DefSetMapped = ordsets:from_list(DefListMapped), UseSetMapped = ordsets:from_list(UseListMapped), - NewIG = interference_arcs(DefListMapped, ordsets:to_list(Live), IG), - NewLive = ordsets:union(UseSetMapped, ordsets:subtract(Live, DefSetMapped)), - {NewLive, NewIG}. + {Live1, IG1} = + analyze_move(X, Live0, IG0, Target, DefSetMapped, UseSetMapped), + IG = interference_arcs(DefListMapped, ordsets:to_list(Live1), IG1), + Live = ordsets:union(UseSetMapped, ordsets:subtract(Live1, DefSetMapped)), + {Live, IG}. + +analyze_move(X, Live0, IG0, Target, DefSetMapped, UseSetMapped) -> + case {is_spill_move(X, Target), DefSetMapped, UseSetMapped} of + {true, [Dst], [Src]} -> + {ordsets:del_element(Src, Live0), add_move(Src, Dst, IG0)}; + {_, _, _} -> + {Live0, IG0} + end. %% Given a list of Keys and an ets-table returns a list of the elements %% in Mapping corresponding to the Keys and appends Acc to this list. @@ -271,15 +276,6 @@ i_arcs(X, [Y|Ys], IG) -> %% throw an exception (the caller should retry with more stack slots) color(IG, StackSlots, NumNodes, Target) -> - try - color_0(IG, StackSlots, NumNodes, Target) - catch - error:Rsn -> - ?error_msg("Coloring failed with ~p~n", [Rsn]), - ?EXIT(Rsn) - end. - -color_0(IG, StackSlots, NumNodes, Target) -> ?report("simplification of IG~n", []), K = ordsets:size(StackSlots), Nodes = list_ig(IG), @@ -382,7 +378,8 @@ select_colors([{X,colorable}|Xs], IG, Cols, PhysRegs) -> select_color(X, IG, Cols, PhysRegs) -> UsedColors = get_colors(neighbors(X, IG), Cols), - Reg = select_unused_color(UsedColors, PhysRegs), + Preferences = get_colors(move_connected(X, IG), Cols), + Reg = select_unused_color(UsedColors, Preferences, PhysRegs), {Reg, set_color(X, Reg, Cols)}. %%%%%%%%%%%%%%%%%%%% @@ -396,10 +393,14 @@ get_colors([X|Xs], Cols) -> [R|get_colors(Xs, Cols)] end. -select_unused_color(UsedColors, PhysRegs) -> +select_unused_color(UsedColors, Preferences, PhysRegs) -> Summary = ordsets:from_list(UsedColors), - AvailRegs = ordsets:to_list(ordsets:subtract(PhysRegs, Summary)), - hd(AvailRegs). + case ordsets:subtract(ordsets:from_list(Preferences), Summary) of + [PreferredColor|_] -> PreferredColor; + _ -> + AvailRegs = ordsets:to_list(ordsets:subtract(PhysRegs, Summary)), + hd(AvailRegs) + end. push_colored(X, Stk) -> [{X, colorable} | Stk]. @@ -456,7 +457,11 @@ init_stackslots(NumSlots, Acc) -> %% %% Note: later on, we may wish to add 'move-related' support. --record(ig_info, {neighbors = [] :: [_], degree = 0 :: non_neg_integer()}). +-record(ig_info, { + neighbors = [] :: [_], + degree = 0 :: non_neg_integer(), + move_connected = [] :: [_] + }). empty_ig(NumNodes) -> hipe_vectors:new(NumNodes, #ig_info{}). @@ -467,16 +472,29 @@ degree(Info) -> neighbors(Info) -> Info#ig_info.neighbors. +move_connected(Info) -> + Info#ig_info.move_connected. + add_edge(X, X, IG) -> IG; add_edge(X, Y, IG) -> add_arc(X, Y, add_arc(Y, X, IG)). +add_move(X, X, IG) -> IG; +add_move(X, Y, IG) -> + add_move_arc(X, Y, add_move_arc(Y, X, IG)). + add_arc(X, Y, IG) -> Info = hipe_vectors:get(IG, X), Old = neighbors(Info), New = Info#ig_info{neighbors = [Y|Old]}, hipe_vectors:set(IG,X,New). +add_move_arc(X, Y, IG) -> + Info = hipe_vectors:get(IG, X), + Old = move_connected(Info), + New = Info#ig_info{move_connected = [Y|Old]}, + hipe_vectors:set(IG,X,New). + normalize_ig(IG) -> Size = hipe_vectors:size(IG), normalize_ig(Size-1, IG). @@ -486,7 +504,8 @@ normalize_ig(-1, IG) -> normalize_ig(I, IG) -> Info = hipe_vectors:get(IG, I), N = ordsets:from_list(neighbors(Info)), - NewInfo = Info#ig_info{neighbors = N, degree = length(N)}, + M = ordsets:subtract(ordsets:from_list(move_connected(Info)), N), + NewInfo = Info#ig_info{neighbors = N, degree = length(N), move_connected = M}, NewIG = hipe_vectors:set(IG, I, NewInfo), normalize_ig(I-1, NewIG). @@ -494,6 +513,10 @@ neighbors(X, IG) -> Info = hipe_vectors:get(IG, X), Info#ig_info.neighbors. +move_connected(X, IG) -> + Info = hipe_vectors:get(IG, X), + Info#ig_info.move_connected. + decrement_degree(X, IG) -> Info = hipe_vectors:get(IG, X), Degree = degree(Info), @@ -555,3 +578,6 @@ def_use(X, Target={TgtMod,TgtCtx}, TempMap) -> reg_names(Regs, {TgtMod,TgtCtx}) -> [TgtMod:reg_nr(X,TgtCtx) || X <- Regs]. + +is_spill_move(Instr, {TgtMod,TgtCtx}) -> + TgtMod:is_spill_move(Instr, TgtCtx). diff --git a/lib/hipe/ppc/hipe_ppc.erl b/lib/hipe/ppc/hipe_ppc.erl index df9f193fa3..63ecd0a0b8 100644 --- a/lib/hipe/ppc/hipe_ppc.erl +++ b/lib/hipe/ppc/hipe_ppc.erl @@ -98,6 +98,9 @@ pseudo_move_dst/1, pseudo_move_src/1, + mk_pseudo_spill_move/3, + is_pseudo_spill_move/1, + mk_pseudo_tailcall/4, pseudo_tailcall_func/1, pseudo_tailcall_stkargs/1, @@ -131,6 +134,9 @@ pseudo_fmove_dst/1, pseudo_fmove_src/1, + mk_pseudo_spill_fmove/3, + is_pseudo_spill_fmove/1, + mk_defun/8, defun_mfa/1, defun_formals/1, @@ -412,6 +418,10 @@ is_pseudo_move(I) -> case I of #pseudo_move{} -> true; _ -> false end. pseudo_move_dst(#pseudo_move{dst=Dst}) -> Dst. pseudo_move_src(#pseudo_move{src=Src}) -> Src. +mk_pseudo_spill_move(Dst, Temp, Src) -> + #pseudo_spill_move{dst=Dst, temp=Temp, src=Src}. +is_pseudo_spill_move(I) -> is_record(I, pseudo_spill_move). + mk_pseudo_tailcall(FunC, Arity, StkArgs, Linkage) -> #pseudo_tailcall{func=FunC, arity=Arity, stkargs=StkArgs, linkage=Linkage}. pseudo_tailcall_func(#pseudo_tailcall{func=FunC}) -> FunC. @@ -495,6 +505,10 @@ is_pseudo_fmove(I) -> case I of #pseudo_fmove{} -> true; _ -> false end. pseudo_fmove_dst(#pseudo_fmove{dst=Dst}) -> Dst. pseudo_fmove_src(#pseudo_fmove{src=Src}) -> Src. +mk_pseudo_spill_fmove(Dst, Temp, Src) -> + #pseudo_spill_fmove{dst=Dst, temp=Temp, src=Src}. +is_pseudo_spill_fmove(I) -> is_record(I, pseudo_spill_fmove). + mk_defun(MFA, Formals, IsClosure, IsLeaf, Code, Data, VarRange, LabelRange) -> #defun{mfa=MFA, formals=Formals, code=Code, data=Data, isclosure=IsClosure, isleaf=IsLeaf, diff --git a/lib/hipe/ppc/hipe_ppc.hrl b/lib/hipe/ppc/hipe_ppc.hrl index a96692c52e..3eef8be487 100644 --- a/lib/hipe/ppc/hipe_ppc.hrl +++ b/lib/hipe/ppc/hipe_ppc.hrl @@ -87,6 +87,7 @@ -record(pseudo_call_prepare, {nrstkargs}). -record(pseudo_li, {dst, imm}). -record(pseudo_move, {dst, src}). +-record(pseudo_spill_move, {dst, temp, src}). -record(pseudo_tailcall, {func, arity, stkargs, linkage}). -record(pseudo_tailcall_prepare, {}). -record(store, {stop, src, disp, base}). % non-indexed, non-update form @@ -99,6 +100,7 @@ -record(fp_binary, {fp_binop, dst, src1, src2}). -record(fp_unary, {fp_unop, dst, src}). -record(pseudo_fmove, {dst, src}). +-record(pseudo_spill_fmove, {dst, temp, src}). %%% Function definitions. diff --git a/lib/hipe/ppc/hipe_ppc_assemble.erl b/lib/hipe/ppc/hipe_ppc_assemble.erl index 66817837df..b0f57e5582 100644 --- a/lib/hipe/ppc/hipe_ppc_assemble.erl +++ b/lib/hipe/ppc/hipe_ppc_assemble.erl @@ -32,7 +32,7 @@ assemble(CompiledCode, Closures, Exports, Options) -> || {MFA, Defun} <- CompiledCode], %% {ConstAlign,ConstSize,ConstMap,RefsFromConsts} = - hipe_pack_constants:pack_constants(Code, hipe_rtl_arch:word_size()), + hipe_pack_constants:pack_constants(Code), %% {CodeSize,CodeBinary,AccRefs,LabelMap,ExportMap} = encode(translate(Code, ConstMap), Options), diff --git a/lib/hipe/ppc/hipe_ppc_cfg.erl b/lib/hipe/ppc/hipe_ppc_cfg.erl index f17c0ac503..d44d38f38d 100644 --- a/lib/hipe/ppc/hipe_ppc_cfg.erl +++ b/lib/hipe/ppc/hipe_ppc_cfg.erl @@ -21,8 +21,8 @@ bb/2, bb_add/3]). -export([postorder/1]). -export([linearise/1, params/1, reverse_postorder/1]). --export([arity/1]). -%%%-export([redirect_jmp/3, arity/1]). +-export([redirect_jmp/3, arity/1]). +-export([branch_preds/1]). %%% these tell cfg.inc what to define (ugly as hell) -define(BREADTH_ORDER,true). @@ -75,11 +75,30 @@ branch_successors(Branch) -> #pseudo_tailcall{} -> [] end. +branch_preds(Branch) -> + case Branch of + #bctr{labels=Labels} -> + Prob = 1.0/length(Labels), + [{L, Prob} || L <- Labels]; + #pseudo_bc{true_label=TrueLab,false_label=FalseLab,pred=Pred} -> + [{FalseLab, 1.0-Pred}, {TrueLab, Pred}]; + #pseudo_call{contlab=ContLab, sdesc=#ppc_sdesc{exnlab=[]}} -> + %% A function can still cause an exception, even if we won't catch it + [{ContLab, 1.0-hipe_bb_weights:call_exn_pred()}]; + #pseudo_call{contlab=ContLab, sdesc=#ppc_sdesc{exnlab=ExnLab}} -> + CallExnPred = hipe_bb_weights:call_exn_pred(), + [{ContLab, 1.0-CallExnPred}, {ExnLab, CallExnPred}]; + _ -> + case branch_successors(Branch) of + [] -> []; + [Single] -> [{Single, 1.0}] + end + end. + -ifdef(REMOVE_TRIVIAL_BBS_NEEDED). fails_to(_Instr) -> []. -endif. --ifdef(notdef). redirect_jmp(I, Old, New) -> case I of #b_label{label=Label} -> @@ -93,10 +112,16 @@ redirect_jmp(I, Old, New) -> if Old =:= FalseLab -> I1#pseudo_bc{false_label=New}; true -> I1 end; - %% handle pseudo_call too? - _ -> I + #pseudo_call{sdesc=SDesc0, contlab=ContLab0} -> + SDesc = case SDesc0 of + #ppc_sdesc{exnlab=Old} -> SDesc0#ppc_sdesc{exnlab=New}; + #ppc_sdesc{exnlab=_} -> SDesc0 + end, + ContLab = if Old =:= ContLab0 -> New; + true -> ContLab0 + end, + I#pseudo_call{sdesc=SDesc, contlab=ContLab} end. --endif. mk_goto(Label) -> hipe_ppc:mk_b_label(Label). diff --git a/lib/hipe/ppc/hipe_ppc_defuse.erl b/lib/hipe/ppc/hipe_ppc_defuse.erl index 9a99611493..d8a864f7d5 100644 --- a/lib/hipe/ppc/hipe_ppc_defuse.erl +++ b/lib/hipe/ppc/hipe_ppc_defuse.erl @@ -41,6 +41,7 @@ insn_def_gpr(I) -> #pseudo_call{} -> call_clobbered_gpr(); #pseudo_li{dst=Dst} -> [Dst]; #pseudo_move{dst=Dst} -> [Dst]; + #pseudo_spill_move{dst=Dst,temp=Temp} -> [Dst, Temp]; #pseudo_tailcall_prepare{} -> tailcall_clobbered_gpr(); #unary{dst=Dst} -> [Dst]; _ -> [] @@ -71,6 +72,7 @@ insn_use_gpr(I) -> #mtspr{src=Src} -> [Src]; #pseudo_call{sdesc=#ppc_sdesc{arity=Arity}} -> arity_use_gpr(Arity); #pseudo_move{src=Src} -> [Src]; + #pseudo_spill_move{src=Src} -> [Src]; #pseudo_tailcall{arity=Arity,stkargs=StkArgs} -> addsrcs(StkArgs, addtemps(tailcall_clobbered_gpr(), arity_use_gpr(Arity))); #store{src=Src,base=Base} -> addtemp(Src, [Base]); @@ -110,6 +112,7 @@ insn_def_fpr(I) -> #fp_binary{dst=Dst} -> [Dst]; #fp_unary{dst=Dst} -> [Dst]; #pseudo_fmove{dst=Dst} -> [Dst]; + #pseudo_spill_fmove{dst=Dst,temp=Temp} -> [Dst, Temp]; _ -> [] end. @@ -126,6 +129,7 @@ insn_use_fpr(I) -> #fp_binary{src1=Src1,src2=Src2} -> addtemp(Src1, [Src2]); #fp_unary{src=Src} -> [Src]; #pseudo_fmove{src=Src} -> [Src]; + #pseudo_spill_fmove{src=Src} -> [Src]; _ -> [] end. diff --git a/lib/hipe/ppc/hipe_ppc_frame.erl b/lib/hipe/ppc/hipe_ppc_frame.erl index a91cb18cc2..b88b75a5bd 100644 --- a/lib/hipe/ppc/hipe_ppc_frame.erl +++ b/lib/hipe/ppc/hipe_ppc_frame.erl @@ -66,10 +66,14 @@ do_insn(I, LiveOut, Context, FPoff) -> do_pseudo_call_prepare(I, FPoff); #pseudo_move{} -> {do_pseudo_move(I, Context, FPoff), FPoff}; + #pseudo_spill_move{} -> + {do_pseudo_spill_move(I, Context, FPoff), FPoff}; #pseudo_tailcall{} -> {do_pseudo_tailcall(I, Context), context_framesize(Context)}; #pseudo_fmove{} -> {do_pseudo_fmove(I, Context, FPoff), FPoff}; + #pseudo_spill_fmove{} -> + {do_pseudo_spill_fmove(I, Context, FPoff), FPoff}; _ -> {[I], FPoff} end. @@ -98,6 +102,22 @@ do_pseudo_move(I, Context, FPoff) -> end end. +do_pseudo_spill_move(I, Context, FPoff) -> + #pseudo_spill_move{dst=Dst,temp=Temp,src=Src} = I, + case temp_is_pseudo(Src) andalso temp_is_pseudo(Dst) of + false -> % Register allocator changed its mind, turn back to move + do_pseudo_move(hipe_ppc:mk_pseudo_move(Dst, Src), Context, FPoff); + true -> + SrcOffset = pseudo_offset(Src, FPoff, Context), + DstOffset = pseudo_offset(Dst, FPoff, Context), + case SrcOffset =:= DstOffset of + true -> []; % omit move-to-self + false -> + mk_load(hipe_ppc:ldop_word(), Temp, SrcOffset, mk_sp(), + mk_store(hipe_ppc:stop_word(), Temp, DstOffset, mk_sp(), [])) + end + end. + do_pseudo_fmove(I, Context, FPoff) -> Dst = hipe_ppc:pseudo_fmove_dst(I), Src = hipe_ppc:pseudo_fmove_src(I), @@ -115,6 +135,22 @@ do_pseudo_fmove(I, Context, FPoff) -> end end. +do_pseudo_spill_fmove(I, Context, FPoff) -> + #pseudo_spill_fmove{dst=Dst,temp=Temp,src=Src} = I, + case temp_is_pseudo(Src) andalso temp_is_pseudo(Dst) of + false -> % Register allocator changed its mind, turn back to move + do_pseudo_fmove(hipe_ppc:mk_pseudo_fmove(Dst, Src), Context, FPoff); + true -> + SrcOffset = pseudo_offset(Src, FPoff, Context), + DstOffset = pseudo_offset(Dst, FPoff, Context), + case SrcOffset =:= DstOffset of + true -> []; % omit move-to-self + false -> + hipe_ppc:mk_fload(Temp, SrcOffset, mk_sp(), 0) + ++ hipe_ppc:mk_fstore(Temp, DstOffset, mk_sp(), 0) + end + end. + pseudo_offset(Temp, FPoff, Context) -> FPoff + context_offset(Context, Temp). diff --git a/lib/hipe/ppc/hipe_ppc_ra_finalise.erl b/lib/hipe/ppc/hipe_ppc_ra_finalise.erl index 74ef7475eb..bca504d754 100644 --- a/lib/hipe/ppc/hipe_ppc_ra_finalise.erl +++ b/lib/hipe/ppc/hipe_ppc_ra_finalise.erl @@ -41,6 +41,7 @@ ra_insn(I, Map, FPMap) -> #mtspr{} -> ra_mtspr(I, Map); #pseudo_li{} -> ra_pseudo_li(I, Map); #pseudo_move{} -> ra_pseudo_move(I, Map); + #pseudo_spill_move{} -> ra_pseudo_spill_move(I, Map); #pseudo_tailcall{} -> ra_pseudo_tailcall(I, Map); #store{} -> ra_store(I, Map); #storex{} -> ra_storex(I, Map); @@ -52,6 +53,7 @@ ra_insn(I, Map, FPMap) -> #fp_binary{} -> ra_fp_binary(I, FPMap); #fp_unary{} -> ra_fp_unary(I, FPMap); #pseudo_fmove{} -> ra_pseudo_fmove(I, FPMap); + #pseudo_spill_fmove{} -> ra_pseudo_spill_fmove(I, FPMap); _ -> I end. @@ -98,6 +100,12 @@ ra_pseudo_move(I=#pseudo_move{dst=Dst,src=Src}, Map) -> NewSrc = ra_temp(Src, Map), I#pseudo_move{dst=NewDst,src=NewSrc}. +ra_pseudo_spill_move(I=#pseudo_spill_move{dst=Dst,temp=Temp,src=Src}, Map) -> + NewDst = ra_temp(Dst, Map), + NewTemp = ra_temp(Temp, Map), + NewSrc = ra_temp(Src, Map), + I#pseudo_spill_move{dst=NewDst,temp=NewTemp,src=NewSrc}. + ra_pseudo_tailcall(I=#pseudo_tailcall{stkargs=StkArgs}, Map) -> NewStkArgs = ra_args(StkArgs, Map), I#pseudo_tailcall{stkargs=NewStkArgs}. @@ -156,6 +164,13 @@ ra_pseudo_fmove(I=#pseudo_fmove{dst=Dst,src=Src}, FPMap) -> NewSrc = ra_temp_fp(Src, FPMap), I#pseudo_fmove{dst=NewDst,src=NewSrc}. +ra_pseudo_spill_fmove(I=#pseudo_spill_fmove{dst=Dst,temp=Temp,src=Src}, + FPMap) -> + NewDst = ra_temp_fp(Dst, FPMap), + NewTemp = ra_temp_fp(Temp, FPMap), + NewSrc = ra_temp_fp(Src, FPMap), + I#pseudo_spill_fmove{dst=NewDst,temp=NewTemp,src=NewSrc}. + ra_args([Arg|Args], Map) -> [ra_temp_or_imm(Arg, Map) | ra_args(Args, Map)]; ra_args([], _) -> diff --git a/lib/hipe/ppc/hipe_ppc_ra_postconditions.erl b/lib/hipe/ppc/hipe_ppc_ra_postconditions.erl index 95aa294fe5..0a97129666 100644 --- a/lib/hipe/ppc/hipe_ppc_ra_postconditions.erl +++ b/lib/hipe/ppc/hipe_ppc_ra_postconditions.erl @@ -57,6 +57,7 @@ do_insn(I, TempMap, Strategy) -> #mtspr{} -> do_mtspr(I, TempMap, Strategy); #pseudo_li{} -> do_pseudo_li(I, TempMap, Strategy); #pseudo_move{} -> do_pseudo_move(I, TempMap, Strategy); + #pseudo_spill_move{} -> do_pseudo_spill_move(I, TempMap, Strategy); #store{} -> do_store(I, TempMap, Strategy); #storex{} -> do_storex(I, TempMap, Strategy); #unary{} -> do_unary(I, TempMap, Strategy); @@ -117,18 +118,25 @@ do_pseudo_li(I=#pseudo_li{dst=Dst}, TempMap, Strategy) -> do_pseudo_move(I=#pseudo_move{dst=Dst,src=Src}, TempMap, Strategy) -> %% Either Dst or Src (but not both) may be a pseudo temp. - %% pseudo_move and pseudo_tailcall are special cases: in - %% all other instructions, all temps must be non-pseudos - %% after register allocation. - case temp_is_spilled(Dst, TempMap) of - true -> % Src must not be a pseudo - {FixSrc,NewSrc,DidSpill} = fix_src1(Src, TempMap, Strategy), - NewI = I#pseudo_move{src=NewSrc}, - {FixSrc ++ [NewI], DidSpill}; + %% pseudo_move, pseudo_spill_move, and pseudo_tailcall are + %% special cases: in all other instructions, all temps + %% must be non-pseudos after register allocation. + case temp_is_spilled(Src, TempMap) + andalso temp_is_spilled(Dst, TempMap) + of + true -> % Turn into pseudo_spill_move + Temp = clone(Src, temp1(Strategy)), + NewI = #pseudo_spill_move{dst=Dst,temp=Temp,src=Src}, + {[NewI], true}; _ -> {[I], false} end. +do_pseudo_spill_move(I=#pseudo_spill_move{temp=Temp}, TempMap, _Strategy) -> + %% Temp is above the low water mark and must not have been spilled + false = temp_is_spilled(Temp, TempMap), + {[I], false}. + do_store(I=#store{src=Src,base=Base}, TempMap, Strategy) -> {FixSrc,NewSrc,DidSpill1} = fix_src1(Src, TempMap, Strategy), {FixBase,NewBase,DidSpill2} = fix_src2(Base, TempMap, Strategy), diff --git a/lib/hipe/ppc/hipe_ppc_ra_postconditions_fp.erl b/lib/hipe/ppc/hipe_ppc_ra_postconditions_fp.erl index 5ec5f29577..7342053620 100644 --- a/lib/hipe/ppc/hipe_ppc_ra_postconditions_fp.erl +++ b/lib/hipe/ppc/hipe_ppc_ra_postconditions_fp.erl @@ -42,6 +42,7 @@ do_insn(I, TempMap) -> #fp_binary{} -> do_fp_binary(I, TempMap); #fp_unary{} -> do_fp_unary(I, TempMap); #pseudo_fmove{} -> do_pseudo_fmove(I, TempMap); + #pseudo_spill_fmove{} -> do_pseudo_spill_fmove(I, TempMap); _ -> {[I], false} end. @@ -81,15 +82,22 @@ do_fp_unary(I=#fp_unary{dst=Dst,src=Src}, TempMap) -> {FixSrc ++ [NewI | FixDst], DidSpill1 or DidSpill2}. do_pseudo_fmove(I=#pseudo_fmove{dst=Dst,src=Src}, TempMap) -> - case temp_is_spilled(Dst, TempMap) of - true -> - {FixSrc,NewSrc,DidSpill} = fix_src(Src, TempMap), - NewI = I#pseudo_fmove{src=NewSrc}, - {FixSrc ++ [NewI], DidSpill}; + case temp_is_spilled(Src, TempMap) + andalso temp_is_spilled(Dst, TempMap) + of + true -> % Turn into pseudo_spill_fmove + Temp = clone(Src), + NewI = #pseudo_spill_fmove{dst=Dst,temp=Temp,src=Src}, + {[NewI], true}; _ -> {[I], false} end. +do_pseudo_spill_fmove(I=#pseudo_spill_fmove{temp=Temp}, TempMap) -> + %% Temp is above the low water mark and must not have been spilled + false = temp_is_spilled(Temp, TempMap), + {[I], false}. + %%% Fix Dst and Src operands. fix_src(Src, TempMap) -> diff --git a/lib/hipe/ppc/hipe_ppc_subst.erl b/lib/hipe/ppc/hipe_ppc_subst.erl index 1cd18b5c01..e282b22774 100644 --- a/lib/hipe/ppc/hipe_ppc_subst.erl +++ b/lib/hipe/ppc/hipe_ppc_subst.erl @@ -48,6 +48,8 @@ insn_temps(T, I) -> #pseudo_call_prepare{} -> I; #pseudo_li{dst=D} -> I#pseudo_li{dst=T(D)}; #pseudo_move{dst=D,src=S} -> I#pseudo_move{dst=T(D),src=T(S)}; + #pseudo_spill_move{dst=D,temp=U,src=S} -> + I#pseudo_spill_move{dst=T(D),temp=T(U),src=T(S)}; #pseudo_tailcall{func=F,stkargs=Stk} when not is_record(F, ppc_temp) -> I#pseudo_tailcall{stkargs=lists:map(A,Stk)}; #pseudo_tailcall_prepare{} -> I; @@ -62,7 +64,9 @@ insn_temps(T, I) -> #fp_binary{dst=D,src1=L,src2=R} -> I#fp_binary{dst=T(D),src1=T(L),src2=T(R)}; #fp_unary{dst=D,src=S} -> I#fp_unary{dst=T(D),src=T(S)}; - #pseudo_fmove{dst=D,src=S} -> I#pseudo_fmove{dst=T(D),src=T(S)} + #pseudo_fmove{dst=D,src=S} -> I#pseudo_fmove{dst=T(D),src=T(S)}; + #pseudo_spill_fmove{dst=D,temp=U,src=S} -> + I#pseudo_spill_fmove{dst=T(D),temp=T(U),src=T(S)} end. -spec oper_temps(subst_fun(), oper()) -> oper(). diff --git a/lib/hipe/regalloc/Makefile b/lib/hipe/regalloc/Makefile index 209f230a9b..81a92e5d35 100644 --- a/lib/hipe/regalloc/Makefile +++ b/lib/hipe/regalloc/Makefile @@ -50,8 +50,10 @@ MODULES = hipe_ig hipe_ig_moves hipe_moves \ hipe_optimistic_regalloc \ hipe_coalescing_regalloc \ hipe_graph_coloring_regalloc \ + hipe_range_split \ hipe_regalloc_loop \ hipe_regalloc_prepass \ + hipe_restore_reuse \ hipe_ls_regalloc \ hipe_ppc_specific hipe_ppc_specific_fp \ hipe_sparc_specific hipe_sparc_specific_fp \ diff --git a/lib/hipe/regalloc/hipe_amd64_specific_sse2.erl b/lib/hipe/regalloc/hipe_amd64_specific_sse2.erl index 9c94539bc6..d592ba391c 100644 --- a/lib/hipe/regalloc/hipe_amd64_specific_sse2.erl +++ b/lib/hipe/regalloc/hipe_amd64_specific_sse2.erl @@ -30,6 +30,7 @@ def_use/2, is_arg/2, %% used by hipe_ls_regalloc is_move/2, + is_spill_move/2, is_fixed/2, %% used by hipe_graph_coloring_regalloc is_global/2, is_precoloured/2, @@ -50,12 +51,19 @@ -export([check_and_rewrite/3, check_and_rewrite/4]). -%% callbacks for hipe_regalloc_prepass --export([new_reg_nr/1, +%% callbacks for hipe_regalloc_prepass, hipe_range_split +-export([mk_move/3, + mk_goto/2, + redirect_jmp/4, + new_label/1, + new_reg_nr/1, update_reg_nr/3, update_bb/4, subst_temps/3]). +%% callbacks for hipe_bb_weights +-export([branch_preds/2]). + %%---------------------------------------------------------------------------- -include("../flow/cfg.hrl"). @@ -126,8 +134,8 @@ temp0(_) -> all_precoloured(Ctx) -> allocatable(Ctx). -is_precoloured(Reg, Ctx) -> - lists:member(Reg,all_precoloured(Ctx)). +is_precoloured(Reg, _) -> + hipe_amd64_registers:is_precoloured_sse2(Reg). physical_name(Reg, _) -> Reg. @@ -152,6 +160,9 @@ bb(CFG, L, _) -> update_bb(CFG,L,BB,_) -> hipe_x86_cfg:bb_add(CFG,L,BB). +branch_preds(Instr,_) -> + hipe_x86_cfg:branch_preds(Instr). + %% AMD64 stuff def_use(Instruction, _) -> @@ -184,10 +195,34 @@ is_move(Instruction, _) -> andalso hipe_x86:is_temp(Dst) andalso hipe_x86:temp_is_allocatable(Dst); false -> false end. + +is_spill_move(Instruction,_) -> + hipe_x86:is_pseudo_spill_fmove(Instruction). reg_nr(Reg, _) -> hipe_x86:temp_reg(Reg). +mk_move(Src, Dst, _) -> + hipe_x86:mk_fmove(Src, Dst). + +mk_goto(Label, _) -> + hipe_x86:mk_jmp_label(Label). + +redirect_jmp(Jmp, ToOld, ToNew, _) when is_integer(ToOld), is_integer(ToNew) -> + Ref = make_ref(), + put(Ref, false), + I = hipe_x86_subst:insn_lbls( + fun(Tgt) -> + if Tgt =:= ToOld -> put(Ref, true), ToNew; + is_integer(Tgt) -> Tgt + end + end, Jmp), + true = erase(Ref), % Assert that something was rewritten + I. + +new_label(_) -> + hipe_gensym:get_next_label(x86). + new_reg_nr(_) -> hipe_gensym:get_next_var(x86). diff --git a/lib/hipe/regalloc/hipe_arm_specific.erl b/lib/hipe/regalloc/hipe_arm_specific.erl index cef22e5af9..7ebc6aa336 100644 --- a/lib/hipe/regalloc/hipe_arm_specific.erl +++ b/lib/hipe/regalloc/hipe_arm_specific.erl @@ -24,6 +24,7 @@ ,reg_nr/2 ,def_use/2 ,is_move/2 + ,is_spill_move/2 ,is_precoloured/2 ,var_range/2 ,allocatable/1 @@ -46,12 +47,19 @@ %% callbacks for hipe_regalloc_loop -export([check_and_rewrite/3]). -%% callbacks for hipe_regalloc_prepass --export([new_reg_nr/1, +%% callbacks for hipe_regalloc_prepass, hipe_range_split +-export([mk_move/3, + mk_goto/2, + redirect_jmp/4, + new_label/1, + new_reg_nr/1, update_reg_nr/3, update_bb/4, subst_temps/3]). +%% callbacks for hipe_bb_weights, hipe_range_split +-export([branch_preds/2]). + check_and_rewrite(CFG, Coloring, no_context) -> hipe_arm_ra_postconditions:check_and_rewrite(CFG, Coloring, 'normal'). @@ -115,6 +123,9 @@ bb(CFG,L,_) -> update_bb(CFG,L,BB,_) -> hipe_arm_cfg:bb_add(CFG,L,BB). +branch_preds(Branch,_) -> + hipe_arm_cfg:branch_preds(Branch). + %% ARM stuff def_use(Instruction, Ctx) -> @@ -144,9 +155,33 @@ is_move(Instruction, _) -> false -> false end. +is_spill_move(Instruction, _) -> + hipe_arm:is_pseudo_spill_move(Instruction). + reg_nr(Reg, _) -> hipe_arm:temp_reg(Reg). +mk_move(Src, Dst, _) -> + hipe_arm:mk_pseudo_move(Dst, Src). + +mk_goto(Label, _) -> + hipe_arm:mk_b_label(Label). + +redirect_jmp(Jmp, ToOld, ToNew, _) when is_integer(ToOld), is_integer(ToNew) -> + Ref = make_ref(), + put(Ref, false), + I = hipe_arm_subst:insn_lbls( + fun(Tgt) -> + if Tgt =:= ToOld -> put(Ref, true), ToNew; + is_integer(Tgt) -> Tgt + end + end, Jmp), + true = erase(Ref), % Assert that something was rewritten + I. + +new_label(_) -> + hipe_gensym:get_next_label(arm). + new_reg_nr(_) -> hipe_gensym:get_next_var(arm). diff --git a/lib/hipe/regalloc/hipe_coalescing_regalloc.erl b/lib/hipe/regalloc/hipe_coalescing_regalloc.erl index e8ccbec9f1..b8f0a1974c 100644 --- a/lib/hipe/regalloc/hipe_coalescing_regalloc.erl +++ b/lib/hipe/regalloc/hipe_coalescing_regalloc.erl @@ -914,7 +914,7 @@ findCheapest([Node|Nodes], IG, Cost, Cheapest, SpillLimit) -> %% limit are extremely expensive. getCost(Node, IG, SpillLimit) -> - case Node > SpillLimit of + case Node >= SpillLimit of true -> inf; false -> hipe_ig:node_spill_cost(Node, IG) end. diff --git a/lib/hipe/regalloc/hipe_graph_coloring_regalloc.erl b/lib/hipe/regalloc/hipe_graph_coloring_regalloc.erl index 07aa812f4a..f82d3a2cbc 100644 --- a/lib/hipe/regalloc/hipe_graph_coloring_regalloc.erl +++ b/lib/hipe/regalloc/hipe_graph_coloring_regalloc.erl @@ -209,8 +209,8 @@ color(IG, Spill, PhysRegs, SpillIx, SpillLimit, NumNodes, Target, %% Any nodes above the spillimit must be colored first... MustNotSpill = - if NumNodes > SpillLimit+1 -> - sort_on_degree(lists:seq(SpillLimit+1,NumNodes-1) -- Low,IG); + if NumNodes > SpillLimit -> + sort_on_degree(lists:seq(SpillLimit,NumNodes-1) -- Low,IG); true -> [] end, @@ -401,7 +401,7 @@ spill_costs([{N,Info}|Ns], IG, Vis, Spill, SpillLimit, Target) -> true -> spill_costs(Ns, IG, Vis, Spill, SpillLimit, Target); false -> - if N > SpillLimit -> + if N >= SpillLimit -> spill_costs(Ns, IG, Vis, Spill, SpillLimit, Target); true -> [{spill_cost_of(N,Spill)/Deg,N} | diff --git a/lib/hipe/regalloc/hipe_optimistic_regalloc.erl b/lib/hipe/regalloc/hipe_optimistic_regalloc.erl index b96920cbcf..a019c46b90 100644 --- a/lib/hipe/regalloc/hipe_optimistic_regalloc.erl +++ b/lib/hipe/regalloc/hipe_optimistic_regalloc.erl @@ -1933,7 +1933,7 @@ findCheapest([Node|Nodes], IG, Cost, Cheapest, SpillLimit) -> %% limit are extremely expensive. getCost(Node, IG, SpillLimit) -> - case Node > SpillLimit of + case Node >= SpillLimit of true -> inf; false -> SpillCost = hipe_ig:node_spill_cost(Node, IG), diff --git a/lib/hipe/regalloc/hipe_ppc_specific.erl b/lib/hipe/regalloc/hipe_ppc_specific.erl index a6450b4d96..81bb551bd2 100644 --- a/lib/hipe/regalloc/hipe_ppc_specific.erl +++ b/lib/hipe/regalloc/hipe_ppc_specific.erl @@ -24,6 +24,7 @@ ,reg_nr/2 ,def_use/2 ,is_move/2 + ,is_spill_move/2 ,is_precoloured/2 ,var_range/2 ,allocatable/1 @@ -46,12 +47,19 @@ %% callbacks for hipe_regalloc_loop -export([check_and_rewrite/3]). -%% callbacks for hipe_regalloc_prepass --export([new_reg_nr/1, +%% callbacks for hipe_regalloc_prepass, hipe_range_split +-export([mk_move/3, + mk_goto/2, + redirect_jmp/4, + new_label/1, + new_reg_nr/1, update_reg_nr/3, update_bb/4, subst_temps/3]). +%% callbacks for hipe_bb_weights +-export([branch_preds/2]). + check_and_rewrite(CFG, Coloring, _) -> hipe_ppc_ra_postconditions:check_and_rewrite(CFG, Coloring, 'normal'). @@ -115,6 +123,9 @@ bb(CFG,L,_) -> update_bb(CFG,L,BB,_) -> hipe_ppc_cfg:bb_add(CFG,L,BB). +branch_preds(Instr,_) -> + hipe_ppc_cfg:branch_preds(Instr). + %% PowerPC stuff def_use(Instruction, Ctx) -> @@ -144,9 +155,24 @@ is_move(Instruction, _) -> false -> false end. +is_spill_move(Instruction, _) -> + hipe_ppc:is_pseudo_spill_move(Instruction). + reg_nr(Reg, _) -> hipe_ppc:temp_reg(Reg). +mk_move(Src, Dst, _) -> + hipe_ppc:mk_pseudo_move(Dst, Src). + +mk_goto(Label, _) -> + hipe_ppc:mk_b_label(Label). + +redirect_jmp(Jmp, ToOld, ToNew, _) when is_integer(ToOld), is_integer(ToNew) -> + hipe_ppc_cfg:redirect_jmp(Jmp, ToOld, ToNew). + +new_label(_) -> + hipe_gensym:get_next_label(ppc). + new_reg_nr(_) -> hipe_gensym:get_next_var(ppc). diff --git a/lib/hipe/regalloc/hipe_ppc_specific_fp.erl b/lib/hipe/regalloc/hipe_ppc_specific_fp.erl index 23cb6c0318..dcfdf6592c 100644 --- a/lib/hipe/regalloc/hipe_ppc_specific_fp.erl +++ b/lib/hipe/regalloc/hipe_ppc_specific_fp.erl @@ -24,6 +24,7 @@ ,reg_nr/2 ,def_use/2 ,is_move/2 + ,is_spill_move/2 ,is_precoloured/2 ,var_range/2 ,allocatable/1 @@ -46,12 +47,19 @@ %% callbacks for hipe_regalloc_loop -export([check_and_rewrite/3]). -%% callbacks for hipe_regalloc_prepass --export([new_reg_nr/1, +%% callbacks for hipe_regalloc_prepass, hipe_range_split +-export([mk_move/3, + mk_goto/2, + redirect_jmp/4, + new_label/1, + new_reg_nr/1, update_reg_nr/3, update_bb/4, subst_temps/3]). +%% callbacks for hipe_bb_weights +-export([branch_preds/2]). + check_and_rewrite(CFG, Coloring, _) -> hipe_ppc_ra_postconditions_fp:check_and_rewrite(CFG, Coloring). @@ -108,6 +116,9 @@ bb(CFG, L, _) -> update_bb(CFG,L,BB,_) -> hipe_ppc_cfg:bb_add(CFG,L,BB). +branch_preds(Instr,_) -> + hipe_ppc_cfg:branch_preds(Instr). + %% PowerPC stuff def_use(I, Ctx) -> @@ -125,9 +136,24 @@ defines_all_alloc(I, _) -> is_move(I, _) -> hipe_ppc:is_pseudo_fmove(I). +is_spill_move(I, _) -> + hipe_ppc:is_pseudo_spill_fmove(I). + reg_nr(Reg, _) -> hipe_ppc:temp_reg(Reg). +mk_move(Src, Dst, _) -> + hipe_ppc:mk_pseudo_fmove(Dst, Src). + +mk_goto(Label, _) -> + hipe_ppc:mk_b_label(Label). + +redirect_jmp(Jmp, ToOld, ToNew, _) when is_integer(ToOld), is_integer(ToNew) -> + hipe_ppc_cfg:redirect_jmp(Jmp, ToOld, ToNew). + +new_label(_) -> + hipe_gensym:get_next_label(ppc). + new_reg_nr(_) -> hipe_gensym:get_next_var(ppc). diff --git a/lib/hipe/regalloc/hipe_range_split.erl b/lib/hipe/regalloc/hipe_range_split.erl new file mode 100644 index 0000000000..39b086d9f7 --- /dev/null +++ b/lib/hipe/regalloc/hipe_range_split.erl @@ -0,0 +1,1187 @@ +%% -*- erlang-indent-level: 2 -*- +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%@doc +%% TEMPORARY LIVE RANGE SPLITTING PASS +%% +%% Live range splitting is useful to allow a register allocator to allocate a +%% temporary to register for a part of its lifetime, even if it cannot be for +%% the entirety. This improves register allocation quality, at the cost of +%% making the allocation problem more time and memory intensive to solve. +%% +%% Optimal allocation can be achieved if all temporaries are split at every +%% program point (between all instructions), but this makes register allocation +%% infeasably slow in practice. Instead, this module uses heuristics to choose +%% which temporaries should have their live ranges split, and at which points. +%% +%% The range splitter only considers temps which are live during a call +%% instruction, since they're known to be spilled. The control-flow graph is +%% partitioned at call instructions and splitting decisions are made separately +%% for each partition. The register copy of a temp (if any) gets a separate name +%% in each partition. +%% +%% There are three different ways the range splitter may choose to split a +%% temporary in a program partition: +%% +%% * Mode1: Spill the temp before calls, and restore it after them +%% * Mode2: Spill the temp after definitions, restore it after calls +%% * Mode3: Spill the temp after definitions, restore it before uses +%% +%% To pick which of these should be used for each temp×partiton pair, the range +%% splitter uses a cost function. The cost is simply the sum of the cost of all +%% expected stack accesses, and the cost for an individual stack access is based +%% on the probability weight of the basic block that it resides in. This biases +%% the range splitter so that it attempts moving stack accesses from a functions +%% hot path to the cold path. +%% +%% The heuristic has a couple of tuning knobs, adjusting its preference for +%% different spilling modes, aggressiveness, and how much influence the basic +%% block probability weights have. +%% +%% Edge case not handled: Call instructions directly defining a pseudo. In that +%% case, if that pseudo has been selected for mode2 spills, no spill is inserted +%% after the call. +-module(hipe_range_split). + +-export([split/5]). + +-compile(inline). + +%% -define(DO_ASSERT, 1). +%% -define(DEBUG, 1). +-include("../main/hipe.hrl"). + +%% Heuristic tuning constants +-define(DEFAULT_MIN_GAIN, 1.1). % option: range_split_min_gain +-define(DEFAULT_MODE1_FUDGE, 1.1). % option: range_split_mode1_fudge +-define(DEFAULT_WEIGHT_POWER, 2). % option: range_split_weight_power +-define(WEIGHT_CONST_FUN(Power), math:log(Power)/math:log(100)). +-define(WEIGHT_FUN(Wt, Const), math:pow(Wt, Const)). +-define(HEUR_MAX_TEMPS, 20000). + +-type target_cfg() :: any(). +-type target_instr() :: any(). +-type target_temp() :: any(). +-type liveness() :: any(). +-type target_module() :: module(). +-type target_context() :: any(). +-type target() :: {target_module(), target_context()}. +-type liveset() :: ordsets:ordset(temp()). +-type temp() :: non_neg_integer(). +-type label() :: non_neg_integer(). + +-spec split(target_cfg(), liveness(), target_module(), target_context(), + comp_options()) + -> target_cfg(). +split(TCFG0, Liveness, TargetMod, TargetContext, Options) -> + Target = {TargetMod, TargetContext}, + NoTemps = number_of_temporaries(TCFG0, Target), + if NoTemps > ?HEUR_MAX_TEMPS -> + ?debug_msg("~w: Too many temps (~w), falling back on restore_reuse.~n", + [?MODULE, NoTemps]), + hipe_restore_reuse:split(TCFG0, Liveness, TargetMod, TargetContext); + true -> + Wts = compute_weights(TCFG0, TargetMod, TargetContext, Options), + {CFG0, Temps} = convert(TCFG0, Target), + Avail = avail_analyse(TCFG0, Liveness, Target), + Defs = def_analyse(CFG0, TCFG0), + RDefs = rdef_analyse(CFG0), + PLive = plive_analyse(CFG0), + {CFG, DUCounts, Costs, DSets0} = + scan(CFG0, Liveness, PLive, Wts, Defs, RDefs, Avail, Target), + {DSets, _} = hipe_dsets:to_map(DSets0), + Renames = decide(DUCounts, Costs, Target, Options), + rewrite(CFG, TCFG0, Target, Liveness, PLive, Defs, Avail, DSets, Renames, + Temps) + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Internal program representation +%% +%% Second pass: Convert cfg to internal representation + +-record(cfg, { + rpo_labels :: [label()], + bbs :: #{label() => bb()} + }). +-type cfg() :: #cfg{}. + +cfg_bb(L, #cfg{bbs=BBS}) -> maps:get(L, BBS). + +cfg_postorder(#cfg{rpo_labels=RPO}) -> lists:reverse(RPO). + +-record(bb, { + code :: [code_elem()], + %% If the last instruction of code defines all allocatable registers + has_call :: boolean(), + succ :: [label()] + }). +-type bb() :: #bb{}. +-type code_elem() :: instr() | mode2_spills() | mode3_restores(). + +bb_code(#bb{code=Code}) -> Code. +bb_has_call(#bb{has_call=HasCall}) -> HasCall. +bb_succ(#bb{succ=Succ}) -> Succ. + +bb_butlast(#bb{code=Code}) -> + bb_butlast_1(Code). + +bb_butlast_1([_Last]) -> []; +bb_butlast_1([I|Is]) -> [I|bb_butlast_1(Is)]. + +bb_last(#bb{code=Code}) -> lists:last(Code). + +-record(instr, { + i :: target_instr(), + def :: ordsets:ordset(temp()), + use :: ordsets:ordset(temp()) + }). +-type instr() :: #instr{}. + +-record(mode2_spills, { + temps :: ordsets:ordset(temp()) + }). +-type mode2_spills() :: #mode2_spills{}. + +-record(mode3_restores, { + temps :: ordsets:ordset(temp()) + }). +-type mode3_restores() :: #mode3_restores{}. + +-spec convert(target_cfg(), target()) -> {cfg(), temps()}. +convert(CFG, Target) -> + RPO = reverse_postorder(CFG, Target), + {BBsList, Temps} = convert_bbs(RPO, CFG, Target, #{}, []), + {#cfg{rpo_labels = RPO, + bbs = maps:from_list(BBsList)}, + Temps}. + +convert_bbs([], _CFG, _Target, Temps, Acc) -> {Acc, Temps}; +convert_bbs([L|Ls], CFG, Target, Temps0, Acc) -> + Succs = hipe_gen_cfg:succ(CFG, L), + TBB = bb(CFG, L, Target), + TCode = hipe_bb:code(TBB), + {Code, Last, Temps} = convert_code(TCode, Target, Temps0, []), + HasCall = defines_all_alloc(Last#instr.i, Target), + BB = #bb{code = Code, + has_call = HasCall, + succ = Succs}, + convert_bbs(Ls, CFG, Target, Temps, [{L,BB}|Acc]). + +convert_code([], _Target, Temps, [Last|_]=Acc) -> + {lists:reverse(Acc), Last, Temps}; +convert_code([TI|TIs], Target, Temps0, Acc) -> + {TDef, TUse} = def_use(TI, Target), + I = #instr{i = TI, + def = ordsets:from_list(reg_names(TDef, Target)), + use = ordsets:from_list(reg_names(TUse, Target))}, + Temps = add_temps(TUse, Target, add_temps(TDef, Target, Temps0)), + convert_code(TIs, Target, Temps, [I|Acc]). + +-type temps() :: #{temp() => target_temp()}. +add_temps([], _Target, Temps) -> Temps; +add_temps([T|Ts], Target, Temps) -> + add_temps(Ts, Target, Temps#{reg_nr(T, Target) => T}). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Fourth pass: P({DEF}) lattice fwd dataflow (for eliding stores at SPILL +%% splits) +-type defsi() :: #{label() => defseti() | {call, defseti(), defseti()}}. +-type defs() :: #{label() => defsetf()}. + +-spec def_analyse(cfg(), target_cfg()) -> defs(). +def_analyse(CFG = #cfg{rpo_labels = RPO}, TCFG) -> + Defs0 = def_init(CFG), + def_dataf(RPO, TCFG, Defs0). + +-spec def_init(cfg()) -> defsi(). +def_init(#cfg{bbs = BBs}) -> + maps:from_list( + [begin + {L, case HasCall of + false -> def_init_scan(bb_code(BB), defseti_new()); + true -> + {call, def_init_scan(bb_butlast(BB), defseti_new()), + defseti_from_ordset((bb_last(BB))#instr.def)} + end} + end || {L, BB = #bb{has_call=HasCall}} <- maps:to_list(BBs)]). + +def_init_scan([], Defset) -> Defset; +def_init_scan([#instr{def=Def}|Is], Defset0) -> + Defset = defseti_add_ordset(Def, Defset0), + def_init_scan(Is, Defset). + +-spec def_dataf([label()], target_cfg(), defsi()) -> defs(). +def_dataf(Labels, TCFG, Defs0) -> + case def_dataf_once(Labels, TCFG, Defs0, 0) of + {Defs, 0} -> + def_finalise(Defs); + {Defs, _Changed} -> + def_dataf(Labels, TCFG, Defs) + end. + +-spec def_finalise(defsi()) -> defs(). +def_finalise(Defs) -> + maps:from_list([{K, defseti_finalise(BL)} + || {K, {call, BL, _}} <- maps:to_list(Defs)]). + +-spec def_dataf_once([label()], target_cfg(), defsi(), non_neg_integer()) + -> {defsi(), non_neg_integer()}. +def_dataf_once([], _TCFG, Defs, Changed) -> {Defs, Changed}; +def_dataf_once([L|Ls], TCFG, Defs0, Changed0) -> + AddPreds = + fun(Defset1) -> + lists:foldl(fun(P, Defset2) -> + defseti_union(defout(P, Defs0), Defset2) + end, Defset1, hipe_gen_cfg:pred(TCFG, L)) + end, + Defset = + case Defset0 = maps:get(L, Defs0) of + {call, Butlast, Defout} -> {call, AddPreds(Butlast), Defout}; + _ -> AddPreds(Defset0) + end, + Changed = case Defset =:= Defset0 of + true -> Changed0; + false -> Changed0+1 + end, + def_dataf_once(Ls, TCFG, Defs0#{L := Defset}, Changed). + +-spec defout(label(), defsi()) -> defseti(). +defout(L, Defs) -> + case maps:get(L, Defs) of + {call, _DefButLast, Defout} -> Defout; + Defout -> Defout + end. + +-spec defbutlast(label(), defs()) -> defsetf(). +defbutlast(L, Defs) -> maps:get(L, Defs). + +-spec defseti_new() -> defseti(). +-spec defseti_union(defseti(), defseti()) -> defseti(). +-spec defseti_add_ordset(ordset:ordset(temp()), defseti()) -> defseti(). +-spec defseti_from_ordset(ordset:ordset(temp())) -> defseti(). +-spec defseti_finalise(defseti()) -> defsetf(). +-spec defsetf_member(temp(), defsetf()) -> boolean(). +-spec defsetf_intersect_ordset(ordsets:ordset(temp()), defsetf()) + -> ordsets:ordset(temp()). + +-type defseti() :: bitord(). +defseti_new() -> bitord_new(). +defseti_union(A, B) -> bitord_union(A, B). +defseti_add_ordset(OS, D) -> defseti_union(defseti_from_ordset(OS), D). +defseti_from_ordset(OS) -> bitord_from_ordset(OS). +defseti_finalise(D) -> bitarr_from_bitord(D). + +-type defsetf() :: bitarr(). +defsetf_member(E, D) -> bitarr_get(E, D). + +defsetf_intersect_ordset([], _D) -> []; +defsetf_intersect_ordset([E|Es], D) -> + case bitarr_get(E, D) of + true -> [E|defsetf_intersect_ordset(Es,D)]; + false -> defsetf_intersect_ordset(Es,D) + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Fifth pass: P({DEF}) lattice reverse dataflow (for eliding stores at defines +%% in mode2) +-type rdefsi() :: #{label() => + {call, rdefseti(), [label()]} + | {nocall, rdefseti(), rdefseti(), [label()]}}. +-type rdefs() :: #{label() => {final, rdefsetf(), [label()]}}. + +-spec rdef_analyse(cfg()) -> rdefs(). +rdef_analyse(CFG = #cfg{rpo_labels=RPO}) -> + Defs0 = rdef_init(CFG), + PO = rdef_postorder(RPO, CFG, []), + rdef_dataf(PO, Defs0). + +%% Filter out 'call' labels, since they don't change +-spec rdef_postorder([label()], cfg(), [label()]) -> [label()]. +rdef_postorder([], _CFG, Acc) -> Acc; +rdef_postorder([L|Ls], CFG, Acc) -> + case bb_has_call(cfg_bb(L, CFG)) of + true -> rdef_postorder(Ls, CFG, Acc); + false -> rdef_postorder(Ls, CFG, [L|Acc]) + end. + +-spec rdef_init(cfg()) -> rdefsi(). +rdef_init(#cfg{bbs = BBs}) -> + maps:from_list( + [{L, case HasCall of + true -> + Defin = rdef_init_scan(bb_butlast(BB), rdefseti_empty()), + {call, Defin, Succs}; + false -> + Gen = rdef_init_scan(bb_code(BB), rdefseti_empty()), + {nocall, Gen, rdefseti_top(), Succs} + end} + || {L, BB = #bb{has_call=HasCall, succ=Succs}} <- maps:to_list(BBs)]). + +-spec rdef_init_scan([instr()], rdefseti()) -> rdefseti(). +rdef_init_scan([], Defset) -> Defset; +rdef_init_scan([#instr{def=Def}|Is], Defset0) -> + Defset = rdefseti_add_ordset(Def, Defset0), + rdef_init_scan(Is, Defset). + +-spec rdef_dataf([label()], rdefsi()) -> rdefs(). +rdef_dataf(Labels, Defs0) -> + case rdef_dataf_once(Labels, Defs0, 0) of + {Defs, 0} -> + rdef_finalise(Defs); + {Defs, _Changed} -> + rdef_dataf(Labels, Defs) + end. + +-spec rdef_finalise(rdefsi()) -> rdefs(). +rdef_finalise(Defs) -> + maps:map(fun(L, V) -> + Succs = rsuccs_val(V), + Defout0 = rdefout_intersect(L, Defs, rdefseti_top()), + {final, rdefset_finalise(Defout0), Succs} + end, Defs). + +-spec rdef_dataf_once([label()], rdefsi(), non_neg_integer()) + -> {rdefsi(), non_neg_integer()}. +rdef_dataf_once([], Defs, Changed) -> {Defs, Changed}; +rdef_dataf_once([L|Ls], Defs0, Changed0) -> + #{L := {nocall, Gen, Defin0, Succs}} = Defs0, + Defin = rdefseti_union(Gen, rdefout_intersect(L, Defs0, Defin0)), + Defset = {nocall, Gen, Defin, Succs}, + Changed = case Defin =:= Defin0 of + true -> Changed0; + false -> Changed0+1 + end, + rdef_dataf_once(Ls, Defs0#{L := Defset}, Changed). + +-spec rdefin(label(), rdefsi()) -> rdefseti(). +rdefin(L, Defs) -> rdefin_val(maps:get(L, Defs)). +rdefin_val({nocall, _Gen, Defin, _Succs}) -> Defin; +rdefin_val({call, Defin, _Succs}) -> Defin. + +-spec rsuccs(label(), rdefsi()) -> [label()]. +rsuccs(L, Defs) -> rsuccs_val(maps:get(L, Defs)). +rsuccs_val({nocall, _Gen, _Defin, Succs}) -> Succs; +rsuccs_val({call, _Defin, Succs}) -> Succs. + +-spec rdefout(label(), rdefs()) -> rdefsetf(). +rdefout(L, Defs) -> + #{L := {final, Defout, _Succs}} = Defs, + Defout. + +-spec rdefout_intersect(label(), rdefsi(), rdefseti()) -> rdefseti(). +rdefout_intersect(L, Defs, Init) -> + lists:foldl(fun(S, Acc) -> + rdefseti_intersect(rdefin(S, Defs), Acc) + end, Init, rsuccs(L, Defs)). + +-type rdefseti() :: bitord() | top. +rdefseti_top() -> top. +rdefseti_empty() -> bitord_new(). +-spec rdefseti_from_ordset(ordsets:ordset(temp())) -> rdefseti(). +rdefseti_from_ordset(OS) -> bitord_from_ordset(OS). + +-spec rdefseti_add_ordset(ordsets:ordset(temp()), rdefseti()) -> rdefseti(). +rdefseti_add_ordset(_, top) -> top; % Should never happen in rdef_dataf +rdefseti_add_ordset(OS, D) -> rdefseti_union(rdefseti_from_ordset(OS), D). + +-spec rdefseti_union(rdefseti(), rdefseti()) -> rdefseti(). +rdefseti_union(top, _) -> top; +rdefseti_union(_, top) -> top; +rdefseti_union(A, B) -> bitord_union(A, B). + +-spec rdefseti_intersect(rdefseti(), rdefseti()) -> rdefseti(). +rdefseti_intersect(top, D) -> D; +rdefseti_intersect(D, top) -> D; +rdefseti_intersect(A, B) -> bitord_intersect(A, B). + +-type rdefsetf() :: {arr, bitarr()} | top. +-spec rdefset_finalise(rdefseti()) -> rdefsetf(). +rdefset_finalise(top) -> top; +rdefset_finalise(Ord) -> {arr, bitarr_from_bitord(Ord)}. + +%% rdefsetf_top() -> top. +rdefsetf_empty() -> {arr, bitarr_new()}. + +-spec rdefsetf_add_ordset(ordset:ordset(temp()), rdefsetf()) -> rdefsetf(). +rdefsetf_add_ordset(_, top) -> top; +rdefsetf_add_ordset(OS, {arr, Arr}) -> + {arr, lists:foldl(fun bitarr_set/2, Arr, OS)}. + +-spec rdef_step(instr(), rdefsetf()) -> rdefsetf(). +rdef_step(#instr{def=Def}, Defset) -> + %% ?ASSERT(not defines_all_alloc(I, Target)), + rdefsetf_add_ordset(Def, Defset). + +-spec ordset_subtract_rdefsetf(ordsets:ordset(temp()), rdefsetf()) + -> ordsets:ordset(temp()). +ordset_subtract_rdefsetf(_, top) -> []; +ordset_subtract_rdefsetf(OS, {arr, Arr}) -> + %% Lazy implementation; could do better if OS can grow + lists:filter(fun(E) -> not bitarr_get(E, Arr) end, OS). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Integer sets represented as bit sets +%% +%% Two representations; bitord() and bitarr() +-define(LIMB_IX_BITS, 11). +-define(LIMB_BITS, (1 bsl ?LIMB_IX_BITS)). +-define(LIMB_IX(Index), (Index bsr ?LIMB_IX_BITS)). +-define(BIT_IX(Index), (Index band (?LIMB_BITS - 1))). +-define(BIT_MASK(Index), (1 bsl ?BIT_IX(Index))). + +%% bitord(): fast at union/2 and can be compared for equality with '=:=' +-type bitord() :: orddict:orddict(non_neg_integer(), 0..((1 bsl ?LIMB_BITS)-1)). + +-spec bitord_new() -> bitord(). +bitord_new() -> []. + +-spec bitord_union(bitord(), bitord()) -> bitord(). +bitord_union(Lhs, Rhs) -> + orddict:merge(fun(_, L, R) -> L bor R end, Lhs, Rhs). + +-spec bitord_intersect(bitord(), bitord()) -> bitord(). +bitord_intersect([], _) -> []; +bitord_intersect(_, []) -> []; +bitord_intersect([{K, L}|Ls], [{K, R}|Rs]) -> + [{K, L band R} | bitord_intersect(Ls, Rs)]; +bitord_intersect([{LK, _}|Ls], [{RK, _}|_]=Rs) when LK < RK -> + bitord_intersect(Ls, Rs); +bitord_intersect([{LK, _}|_]=Ls, [{RK, _}|Rs]) when LK > RK -> + bitord_intersect(Ls, Rs). + +-spec bitord_from_ordset(ordsets:ordset(non_neg_integer())) -> bitord(). +bitord_from_ordset([]) -> []; +bitord_from_ordset([B|Bs]) -> + bitord_from_ordset_1(Bs, ?LIMB_IX(B), ?BIT_MASK(B)). + +bitord_from_ordset_1([B|Bs], Key, Val) when Key =:= ?LIMB_IX(B) -> + bitord_from_ordset_1(Bs, Key, Val bor ?BIT_MASK(B)); +bitord_from_ordset_1([B|Bs], Key, Val) -> + [{Key,Val} | bitord_from_ordset_1(Bs, ?LIMB_IX(B), ?BIT_MASK(B))]; +bitord_from_ordset_1([], Key, Val) -> [{Key, Val}]. + +%% bitarr(): fast (enough) at get/2 +-type bitarr() :: array:array(0..((1 bsl ?LIMB_BITS)-1)). + +-spec bitarr_new() -> bitarr(). +bitarr_new() -> array:new({default, 0}). + +-spec bitarr_get(non_neg_integer(), bitarr()) -> boolean(). +bitarr_get(Index, Array) -> + Limb = array:get(?LIMB_IX(Index), Array), + 0 =/= (Limb band ?BIT_MASK(Index)). + +-spec bitarr_set(non_neg_integer(), bitarr()) -> bitarr(). +bitarr_set(Index, Array) -> + Limb0 = array:get(?LIMB_IX(Index), Array), + Limb = Limb0 bor ?BIT_MASK(Index), + array:set(?LIMB_IX(Index), Limb, Array). + +-spec bitarr_from_bitord(bitord()) -> bitarr(). +bitarr_from_bitord(Ord) -> + array:from_orddict(Ord, 0). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Sixth pass: Partition-local liveness analysis +%% +%% As temps are not spilled when exiting a partition in mode2, only +%% partition-local uses need to be considered when deciding which temps need +%% restoring at partition entry. + +-type plive() :: #{label() => + {call, liveset(), [label()]} + | {nocall, {liveset(), liveset()}, liveset(), [label()]}}. + +-spec plive_analyse(cfg()) -> plive(). +plive_analyse(CFG) -> + Defs0 = plive_init(CFG), + PO = cfg_postorder(CFG), + plive_dataf(PO, Defs0). + +-spec plive_init(cfg()) -> plive(). +plive_init(#cfg{bbs = BBs}) -> + maps:from_list( + [begin + {L, case HasCall of + true -> + {Gen, _} = plive_init_scan(bb_code(BB)), + {call, Gen, Succs}; + false -> + GenKill = plive_init_scan(bb_code(BB)), + {nocall, GenKill, liveset_empty(), Succs} + end} + end || {L, BB = #bb{has_call=HasCall, succ=Succs}} <- maps:to_list(BBs)]). + +-spec plive_init_scan([instr()]) -> {liveset(), liveset()}. +plive_init_scan([]) -> {liveset_empty(), liveset_empty()}; +plive_init_scan([#instr{def=InstrKill, use=InstrGen}|Is]) -> + {Gen0, Kill0} = plive_init_scan(Is), + Gen1 = liveset_subtract(Gen0, InstrKill), + Gen = liveset_union(Gen1, InstrGen), + Kill1 = liveset_union(Kill0, InstrKill), + Kill = liveset_subtract(Kill1, InstrGen), + {Gen, Kill}. + +-spec plive_dataf([label()], plive()) -> plive(). +plive_dataf(Labels, PLive0) -> + case plive_dataf_once(Labels, PLive0, 0) of + {PLive, 0} -> PLive; + {PLive, _Changed} -> + plive_dataf(Labels, PLive) + end. + +-spec plive_dataf_once([label()], plive(), non_neg_integer()) -> + {plive(), non_neg_integer()}. +plive_dataf_once([], PLive, Changed) -> {PLive, Changed}; +plive_dataf_once([L|Ls], PLive0, Changed0) -> + Liveset = + case Liveset0 = maps:get(L, PLive0) of + {call, Livein, Succs} -> + {call, Livein, Succs}; + {nocall, {Gen, Kill} = GenKill, _OldLivein, Succs} -> + Liveout = pliveout(L, PLive0), + Livein = liveset_union(Gen, liveset_subtract(Liveout, Kill)), + {nocall, GenKill, Livein, Succs} + end, + Changed = case Liveset =:= Liveset0 of + true -> Changed0; + false -> Changed0+1 + end, + plive_dataf_once(Ls, PLive0#{L := Liveset}, Changed). + +-spec pliveout(label(), plive()) -> liveset(). +pliveout(L, PLive) -> + liveset_union([plivein(S, PLive) || S <- psuccs(L, PLive)]). + +-spec psuccs(label(), plive()) -> [label()]. +psuccs(L, PLive) -> psuccs_val(maps:get(L, PLive)). +psuccs_val({call, _Livein, Succs}) -> Succs; +psuccs_val({nocall, _GenKill, _Livein, Succs}) -> Succs. + +-spec plivein(label(), plive()) -> liveset(). +plivein(L, PLive) -> plivein_val(maps:get(L, PLive)). +plivein_val({call, Livein, _Succs}) -> Livein; +plivein_val({nocall, _GenKill, Livein, _Succs}) -> Livein. + +liveset_empty() -> ordsets:new(). +liveset_subtract(A, B) -> ordsets:subtract(A, B). +liveset_union(A, B) -> ordsets:union(A, B). +liveset_union(LivesetList) -> ordsets:union(LivesetList). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Third pass: Compute dataflow analyses required for placing mode3 +%% spills/restores. +%% Reuse analysis implementation in hipe_restore_reuse. +%% XXX: hipe_restore_reuse has it's own "rdef"; we would like to reuse that one +%% too. +-type avail() :: hipe_restore_reuse:avail(). + +-spec avail_analyse(target_cfg(), liveness(), target()) -> avail(). +avail_analyse(CFG, Liveness, Target) -> + hipe_restore_reuse:analyse(CFG, Liveness, Target). + +-spec mode3_split_in_block(label(), avail()) -> ordsets:ordset(temp()). +mode3_split_in_block(L, Avail) -> + hipe_restore_reuse:split_in_block(L, Avail). + +-spec mode3_block_renameset(label(), avail()) -> ordsets:ordset(temp()). +mode3_block_renameset(L, Avail) -> + hipe_restore_reuse:renamed_in_block(L, Avail). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Seventh pass +%% +%% Compute program space partitioning, collect information required by the +%% heuristic. +-type part_key() :: label(). +-type part_dsets() :: hipe_dsets:dsets(part_key()). +-type part_dsets_map() :: #{part_key() => part_key()}. +-type ducounts() :: #{part_key() => ducount()}. + +-spec scan(cfg(), liveness(), plive(), weights(), defs(), rdefs(), avail(), + target()) -> {cfg(), ducounts(), costs(), part_dsets()}. +scan(CFG0, Liveness, PLive, Weights, Defs, RDefs, Avail, Target) -> + #cfg{rpo_labels = Labels, bbs = BBs0} = CFG0, + CFG = CFG0#cfg{bbs=#{}}, % kill reference + DSets0 = hipe_dsets:new(Labels), + Costs0 = costs_new(), + {BBs, DUCounts0, Costs1, DSets1} = + scan_bbs(maps:to_list(BBs0), Liveness, PLive, Weights, Defs, RDefs, Avail, + Target, #{}, Costs0, DSets0, []), + {RLList, DSets2} = hipe_dsets:to_rllist(DSets1), + {Costs, DSets} = costs_map_roots(DSets2, Costs1), + DUCounts = collect_ducounts(RLList, DUCounts0, #{}), + {CFG#cfg{bbs=maps:from_list(BBs)}, DUCounts, Costs, DSets}. + +-spec collect_ducounts([{label(), [label()]}], ducounts(), ducounts()) + -> ducounts(). +collect_ducounts([], _, Acc) -> Acc; +collect_ducounts([{R,Ls}|RLs], DUCounts, Acc) -> + DUCount = lists:foldl( + fun(Key, FAcc) -> + ducount_merge(maps:get(Key, DUCounts, ducount_new()), FAcc) + end, ducount_new(), Ls), + collect_ducounts(RLs, DUCounts, Acc#{R => DUCount}). + +-spec scan_bbs([{label(), bb()}], liveness(), plive(), weights(), defs(), + rdefs(), avail(), target(), ducounts(), costs(), part_dsets(), + [{label(), bb()}]) + -> {[{label(), bb()}], ducounts(), costs(), part_dsets()}. +scan_bbs([], _Liveness, _PLive, _Weights, _Defs, _RDefs, _Avail, _Target, + DUCounts, Costs, DSets, Acc) -> + {Acc, DUCounts, Costs, DSets}; +scan_bbs([{L,BB}|BBs], Liveness, PLive, Weights, Defs, RDefs, Avail, Target, + DUCounts0, Costs0, DSets0, Acc) -> + Wt = weight(L, Weights), + {DSets, Costs5, EntryCode, ExitCode, RDefout, Liveout} = + case bb_has_call(BB) of + false -> + DSets1 = lists:foldl(fun(S, DS) -> hipe_dsets:union(L, S, DS) end, + DSets0, bb_succ(BB)), + {DSets1, Costs0, bb_code(BB), [], rdefout(L, RDefs), + liveout(Liveness, L, Target)}; + true -> + LastI = #instr{def=LastDef} = bb_last(BB), + LiveBefore = ordsets:subtract(liveout(Liveness, L, Target), LastDef), + %% We can omit the spill of a temp that has not been defined since the + %% last time it was spilled + SpillSet = defsetf_intersect_ordset(LiveBefore, defbutlast(L, Defs)), + Costs1 = costs_insert(exit, L, Wt, SpillSet, Costs0), + Costs4 = lists:foldl(fun({S, BranchWt}, Costs2) -> + SLivein = livein(Liveness, S, Target), + SPLivein = plivein(S, PLive), + SWt = weight_scaled(L, BranchWt, Weights), + Costs3 = costs_insert(entry1, S, SWt, SLivein, Costs2), + costs_insert(entry2, S, SWt, SPLivein, Costs3) + end, Costs1, branch_preds(LastI#instr.i, Target)), + {DSets0, Costs4, bb_butlast(BB), [LastI], rdefsetf_empty(), LiveBefore} + end, + Mode3Splits = mode3_split_in_block(L, Avail), + {RevEntryCode, Restored} = scan_bb_fwd(EntryCode, Mode3Splits, [], []), + {Code, DUCount, Mode2Spills} = + scan_bb(RevEntryCode, Wt, RDefout, Liveout, ducount_new(), [], ExitCode), + DUCounts = DUCounts0#{L => DUCount}, + M2SpillSet = ordsets:from_list(Mode2Spills), + Costs6 = costs_insert(spill, L, Wt, M2SpillSet, Costs5), + Mode3Renames = mode3_block_renameset(L, Avail), + Costs7 = costs_insert(restore, L, Wt, ordsets:intersection(M2SpillSet, Mode3Renames), Costs6), + Costs8 = costs_insert(restore, L, Wt, ordsets:from_list(Restored), Costs7), + Costs = add_unsplit_mode3_costs(DUCount, Mode3Renames, L, Costs8), + scan_bbs(BBs, Liveness, PLive, Weights, Defs, RDefs, Avail, Target, DUCounts, + Costs, DSets, [{L,BB#bb{code=Code}}|Acc]). + +-spec add_unsplit_mode3_costs(ducount(), ordsets:ordset(temp()), label(), costs()) + -> costs(). +add_unsplit_mode3_costs(DUCount, Mode3Renames, L, Costs) -> + Unsplit = orddict_without_ordset(Mode3Renames, + orddict:from_list(ducount_to_list(DUCount))), + add_unsplit_mode3_costs_1(Unsplit, L, Costs). + +-spec add_unsplit_mode3_costs_1([{temp(),float()}], label(), costs()) + -> costs(). +add_unsplit_mode3_costs_1([], _L, Costs) -> Costs; +add_unsplit_mode3_costs_1([{T,C}|Cs], L, Costs) -> + add_unsplit_mode3_costs_1(Cs, L, costs_insert(restore, L, C, [T], Costs)). + +%% @doc Returns a new orddict without keys in Set and their associated values. +-spec orddict_without_ordset(ordsets:ordset(K), orddict:orddict(K, V)) + -> orddict:orddict(K, V). +orddict_without_ordset([S|Ss], [{K,_}|_]=Dict) when S < K -> + orddict_without_ordset(Ss, Dict); +orddict_without_ordset([S|_]=Set, [D={K,_}|Ds]) when S > K -> + [D|orddict_without_ordset(Set, Ds)]; +orddict_without_ordset([_S|Ss], [{_K,_}|Ds]) -> % _S == _K + orddict_without_ordset(Ss, Ds); +orddict_without_ordset(_, []) -> []; +orddict_without_ordset([], Dict) -> Dict. + +%% Scans the code forward, collecting and inserting mode3 restores +-spec scan_bb_fwd([instr()], ordsets:ordset(temp()), ordsets:ordset(temp()), + [code_elem()]) + -> {[code_elem()], ordsets:ordset(temp())}. +scan_bb_fwd([], [], Restored, Acc) -> {Acc, Restored}; +scan_bb_fwd([I|Is], SplitHere0, Restored0, Acc0) -> + #instr{def=Def, use=Use} = I, + {ToRestore, SplitHere1} = + lists:partition(fun(R) -> lists:member(R, Use) end, SplitHere0), + SplitHere = lists:filter(fun(R) -> not lists:member(R, Def) end, SplitHere1), + Acc = + case ToRestore of + [] -> [I | Acc0]; + _ -> [I, #mode3_restores{temps=ToRestore} | Acc0] + end, + scan_bb_fwd(Is, SplitHere, ToRestore ++ Restored0, Acc). + +%% Scans the code backwards, collecting def/use counts and mode2 spills +-spec scan_bb([code_elem()], float(), rdefsetf(), liveset(), ducount(), + [temp()], [code_elem()]) + -> {[code_elem()], ducount(), [temp()]}. +scan_bb([], _Wt, _RDefout, _Liveout, DUCount, Spills, Acc) -> + {Acc, DUCount, Spills}; +scan_bb([I=#mode3_restores{}|Is], Wt, RDefout, Liveout, DUCount, Spills, Acc) -> + scan_bb(Is, Wt, RDefout, Liveout, DUCount, Spills, [I|Acc]); +scan_bb([I|Is], Wt, RDefout, Liveout, DUCount0, Spills0, Acc0) -> + #instr{def=Def,use=Use} = I, + DUCount = ducount_add(Use, Wt, ducount_add(Def, Wt, DUCount0)), + Livein = liveness_step(I, Liveout), + RDefin = rdef_step(I, RDefout), + %% The temps that would be spilled after I in mode 2 + NewSpills = ordset_subtract_rdefsetf( + ordsets:intersection(Def, Liveout), + RDefout), + ?ASSERT(NewSpills =:= (NewSpills -- Spills0)), + Spills = NewSpills ++ Spills0, + Acc1 = case NewSpills of + [] -> Acc0; + _ -> [#mode2_spills{temps=NewSpills}|Acc0] + end, + scan_bb(Is, Wt, RDefin, Livein, DUCount, Spills, [I|Acc1]). + +-spec liveness_step(instr(), liveset()) -> liveset(). +liveness_step(#instr{def=Def, use=Use}, Liveout) -> + ordsets:union(Use, ordsets:subtract(Liveout, Def)). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% First pass: compute basic-block weighting + +-type weights() :: no_bb_weights + | {hipe_bb_weights:bb_weights(), float()}. + +-spec weight(label(), weights()) -> float(). +weight(L, Weights) -> weight_scaled(L, 1.0, Weights). + +-spec compute_weights(target_cfg(), target_module(), target_context(), + comp_options()) -> weights(). +compute_weights(CFG, TargetMod, TargetContext, Options) -> + case proplists:get_bool(range_split_weights, Options) of + false -> no_bb_weights; + true -> + {hipe_bb_weights:compute(CFG, TargetMod, TargetContext), + ?WEIGHT_CONST_FUN(proplists:get_value(range_split_weight_power, + Options, ?DEFAULT_WEIGHT_POWER))} + end. + +-spec weight_scaled(label(), float(), weights()) -> float(). +weight_scaled(_L, _Scale, no_bb_weights) -> 1.0; +weight_scaled(L, Scale, {Weights, Const}) -> + Wt0 = hipe_bb_weights:weight(L, Weights) * Scale, + Wt = erlang:min(erlang:max(Wt0, 0.0000000000000000001), 10000.0), + ?WEIGHT_FUN(Wt, Const). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Heuristic splitting decision. +%% +%% Decide which temps to split, in which parts, and pick new names for them. +-type spill_mode() :: mode1 % Spill temps at partition exits + | mode2 % Spill temps at definitions + | mode3.% Spill temps at definitions, restore temps at uses +-type ren() :: #{temp() => {spill_mode(), temp()}}. +-type renames() :: #{label() => ren()}. + +-record(heur_par, { + mode1_fudge :: float(), + min_gain :: float() + }). +-type heur_par() :: #heur_par{}. + +-spec decide(ducounts(), costs(), target(), comp_options()) -> renames(). +decide(DUCounts, Costs, Target, Options) -> + Par = #heur_par{ + mode1_fudge = proplists:get_value(range_split_mode1_fudge, Options, + ?DEFAULT_MODE1_FUDGE), + min_gain = proplists:get_value(range_split_min_gain, Options, + ?DEFAULT_MIN_GAIN)}, + decide_parts(maps:to_list(DUCounts), Costs, Target, Par, #{}). + +-spec decide_parts([{part_key(), ducount()}], costs(), target(), + heur_par(), renames()) + -> renames(). +decide_parts([], _Costs, _Target, _Par, Acc) -> Acc; +decide_parts([{Part,DUCount}|Ps], Costs, Target, Par, Acc) -> + Spills = decide_temps(ducount_to_list(DUCount), Part, Costs, Target, Par, + #{}), + decide_parts(Ps, Costs, Target, Par, Acc#{Part => Spills}). + +-spec decide_temps([{temp(), float()}], part_key(), costs(), target(), + heur_par(), ren()) + -> ren(). +decide_temps([], _Part, _Costs, _Target, _Par, Acc) -> Acc; +decide_temps([{Temp, SpillGain}|Ts], Part, Costs, Target, Par, Acc0) -> + SpillCost1 = costs_query(Temp, entry1, Part, Costs) + + costs_query(Temp, exit, Part, Costs), + SpillCost2 = costs_query(Temp, entry2, Part, Costs) + + costs_query(Temp, spill, Part, Costs), + SpillCost3 = costs_query(Temp, restore, Part, Costs), + Acc = + %% SpillCost1 =:= 0.0 usually means the temp is local to the partition; + %% hence no need to split it + case (SpillCost1 =/= 0.0) %% maps:is_key(Temp, S) + andalso (not is_precoloured(Temp, Target)) + andalso ((Par#heur_par.min_gain*SpillCost1 < SpillGain) + orelse (Par#heur_par.min_gain*SpillCost2 < SpillGain) + orelse (Par#heur_par.min_gain*SpillCost3 < SpillGain)) + of + false -> Acc0; + true -> + Mode = + if Par#heur_par.mode1_fudge*SpillCost1 < SpillCost2, + Par#heur_par.mode1_fudge*SpillCost1 < SpillCost3 -> + mode1; + SpillCost2 < SpillCost3 -> + mode2; + true -> + mode3 + end, + Acc0#{Temp => {Mode, new_reg_nr(Target)}} + end, + decide_temps(Ts, Part, Costs, Target, Par, Acc). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Eighth pass: Rewrite program performing range splitting. + +-spec rewrite(cfg(), target_cfg(), target(), liveness(), plive(), defs(), + avail(), part_dsets_map(), renames(), temps()) + -> target_cfg(). +rewrite(#cfg{bbs=BBs}, TCFG, Target, Liveness, PLive, Defs, Avail, DSets, + Renames, Temps) -> + rewrite_bbs(maps:to_list(BBs), Target, Liveness, PLive, Defs, Avail, DSets, + Renames, Temps, TCFG). + +-spec rewrite_bbs([{label(), bb()}], target(), liveness(), plive(), defs(), + avail(), part_dsets_map(), renames(), temps(), target_cfg()) + -> target_cfg(). +rewrite_bbs([], _Target, _Liveness, _PLive, _Defs, _Avail, _DSets, _Renames, + _Temps, TCFG) -> + TCFG; +rewrite_bbs([{L,BB}|BBs], Target, Liveness, PLive, Defs, Avail, DSets, Renames, + Temps, TCFG0) -> + Code0Rev = lists:reverse(bb_code(BB)), + EntryRen = maps:get(maps:get(L,DSets), Renames), + M3Ren = mode3_block_renameset(L, Avail), + SubstFun = rewrite_subst_fun(Target, EntryRen, M3Ren), + Fun = fun(I) -> subst_temps(SubstFun, I, Target) end, + {Code, TCFG} = + case bb_has_call(BB) of + false -> + Code1 = rewrite_instrs(Code0Rev, Fun, EntryRen, M3Ren, Temps, Target, + []), + {Code1, TCFG0}; + true -> + CallI0 = hd(Code0Rev), + Succ = bb_succ(BB), + {CallTI, TCFG1} = inject_restores(Succ, Target, Liveness, PLive, DSets, + Renames, Temps, CallI0#instr.i, TCFG0), + Liveout1 = liveness_step(CallI0, liveout(Liveness, L, Target)), + Defout = defbutlast(L, Defs), + SpillMap = mk_spillmap(EntryRen, Liveout1, Defout, Temps, Target), + Code1 = rewrite_instrs(tl(Code0Rev), Fun, EntryRen, M3Ren, Temps, + Target, []), + Code2 = lift_spills(lists:reverse(Code1), Target, SpillMap, [CallTI]), + {Code2, TCFG1} + end, + TBB = hipe_bb:code_update(bb(TCFG, L, Target), Code), + rewrite_bbs(BBs, Target, Liveness, PLive, Defs, Avail, DSets, Renames, Temps, + update_bb(TCFG, L, TBB, Target)). + +-spec rewrite_instrs([code_elem()], rewrite_fun(), ren(), + ordsets:ordset(temp()), temps(), target(), + [target_instr()]) + -> [target_instr()]. +rewrite_instrs([], _Fun, _Ren, _M3Ren, _Temps, _Target, Acc) -> Acc; +rewrite_instrs([I|Is], Fun, Ren, M3Ren, Temps, Target, Acc0) -> + Acc = + case I of + #instr{i=TI} -> [Fun(TI)|Acc0]; + #mode2_spills{temps=Mode2Spills} -> + add_mode2_spills(Mode2Spills, Target, Ren, M3Ren, Temps, Acc0); + #mode3_restores{temps=Mode3Restores} -> + add_mode3_restores(Mode3Restores, Target, Ren, Temps, Acc0) + end, + rewrite_instrs(Is, Fun, Ren, M3Ren, Temps, Target, Acc). + +-spec add_mode2_spills(ordsets:ordset(temp()), target(), ren(), + ordsets:ordset(temp()), temps(), [target_instr()]) + -> [target_instr()]. +add_mode2_spills([], _Target, _Ren, _M3Ren, _Temps, Acc) -> Acc; +add_mode2_spills([R|Rs], Target, Ren, M3Ren, Temps, Acc0) -> + Acc = + case Ren of + #{R := {Mode, NewName}} when Mode =:= mode2; Mode =:= mode3 -> + case Mode =/= mode3 orelse lists:member(R, M3Ren) of + false -> Acc0; + true -> + #{R := T} = Temps, + SpillInstr = mk_move(update_reg_nr(NewName, T, Target), T, Target), + [SpillInstr|Acc0] + end; + #{} -> + Acc0 + end, + add_mode2_spills(Rs, Target, Ren, M3Ren, Temps, Acc). + +-spec add_mode3_restores(ordsets:ordset(temp()), target(), ren(), temps(), + [target_instr()]) + -> [target_instr()]. +add_mode3_restores([], _Target, _Ren, _Temps, Acc) -> Acc; +add_mode3_restores([R|Rs], Target, Ren, Temps, Acc) -> + case Ren of + #{R := {mode3, NewName}} -> + #{R := T} = Temps, + RestoreInstr = mk_move(T, update_reg_nr(NewName, T, Target), Target), + add_mode3_restores(Rs, Target, Ren, Temps, [RestoreInstr|Acc]); + #{} -> + add_mode3_restores(Rs, Target, Ren, Temps, Acc) + end. + +-type rewrite_fun() :: fun((target_instr()) -> target_instr()). +-type subst_fun() :: fun((target_temp()) -> target_temp()). +-spec rewrite_subst_fun(target(), ren(), ordsets:ordset(temp())) -> subst_fun(). +rewrite_subst_fun(Target, Ren, M3Ren) -> + fun(Temp) -> + Reg = reg_nr(Temp, Target), + case Ren of + #{Reg := {Mode, NewName}} -> + case Mode =/= mode3 orelse lists:member(Reg, M3Ren) of + false -> Temp; + true -> update_reg_nr(NewName, Temp, Target) + end; + #{} -> Temp + end + end. + +-type spillmap() :: [{temp(), target_instr()}]. +-spec mk_spillmap(ren(), liveset(), defsetf(), temps(), target()) + -> spillmap(). +mk_spillmap(Ren, Livein, Defout, Temps, Target) -> + [begin + Temp = maps:get(Reg, Temps), + {NewName, mk_move(update_reg_nr(NewName, Temp, Target), Temp, Target)} + end || {Reg, {mode1, NewName}} <- maps:to_list(Ren), + lists:member(Reg, Livein), defsetf_member(Reg, Defout)]. + +-spec mk_restores(ren(), liveset(), liveset(), temps(), target()) + -> [target_instr()]. +mk_restores(Ren, Livein, PLivein, Temps, Target) -> + [begin + Temp = maps:get(Reg, Temps), + mk_move(Temp, update_reg_nr(NewName, Temp, Target), Target) + end || {Reg, {Mode, NewName}} <- maps:to_list(Ren), + ( (Mode =:= mode1 andalso lists:member(Reg, Livein )) + orelse (Mode =:= mode2 andalso lists:member(Reg, PLivein)))]. + +-spec inject_restores([label()], target(), liveness(), plive(), + part_dsets_map(), renames(), temps(), target_instr(), + target_cfg()) + -> {target_instr(), target_cfg()}. +inject_restores([], _Target, _Liveness, _PLive, _DSets, _Renames, _Temps, CFTI, + TCFG) -> + {CFTI, TCFG}; +inject_restores([L|Ls], Target, Liveness, PLive, DSets, Renames, Temps, CFTI0, + TCFG0) -> + Ren = maps:get(maps:get(L,DSets), Renames), + Livein = livein(Liveness, L, Target), + PLivein = plivein(L, PLive), + {CFTI, TCFG} = + case mk_restores(Ren, Livein, PLivein, Temps, Target) of + [] -> {CFTI0, TCFG0}; % optimisation + Restores -> + RestBBLbl = new_label(Target), + Code = Restores ++ [mk_goto(L, Target)], + CFTI1 = redirect_jmp(CFTI0, L, RestBBLbl, Target), + TCFG1 = update_bb(TCFG0, RestBBLbl, hipe_bb:mk_bb(Code), Target), + {CFTI1, TCFG1} + end, + inject_restores(Ls, Target, Liveness, PLive, DSets, Renames, Temps, CFTI, + TCFG). + +%% Heuristic. Move spills up until we meet the edge of the BB or a definition of +%% that temp. +-spec lift_spills([target_instr()], target(), spillmap(), [target_instr()]) + -> [target_instr()]. +lift_spills([], _Target, SpillMap, Acc) -> + [SpillI || {_, SpillI} <- SpillMap] ++ Acc; +lift_spills([I|Is], Target, SpillMap0, Acc) -> + Def = reg_defines(I, Target), + {Spills0, SpillMap} = + lists:partition(fun({Reg,_}) -> lists:member(Reg, Def) end, SpillMap0), + Spills = [SpillI || {_, SpillI} <- Spills0], + lift_spills(Is, Target, SpillMap, [I|Spills ++ Acc]). + +reg_defines(I, Target) -> + reg_names(defines(I,Target), Target). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Costs ADT +%% +%% Keeps track of cumulative cost of spilling temps in particular partitions +%% using particular spill modes. +-type cost_map() :: #{[part_key()|temp()] => float()}. +-type cost_key() :: entry1 | entry2 | exit | spill | restore. +-record(costs, {entry1 = #{} :: cost_map() + ,entry2 = #{} :: cost_map() + ,exit = #{} :: cost_map() + ,spill = #{} :: cost_map() + ,restore = #{} :: cost_map() + }). +-type costs() :: #costs{}. + +-spec costs_new() -> costs(). +costs_new() -> #costs{}. + +-spec costs_insert(cost_key(), part_key(), float(), liveset(), costs()) + -> costs(). +costs_insert(entry1, A, Weight, Liveset, Costs=#costs{entry1=Entry1}) -> + Costs#costs{entry1=costs_insert_1(A, Weight, Liveset, Entry1)}; +costs_insert(entry2, A, Weight, Liveset, Costs=#costs{entry2=Entry2}) -> + Costs#costs{entry2=costs_insert_1(A, Weight, Liveset, Entry2)}; +costs_insert(exit, A, Weight, Liveset, Costs=#costs{exit=Exit}) -> + Costs#costs{exit=costs_insert_1(A, Weight, Liveset, Exit)}; +costs_insert(spill, A, Weight, Liveset, Costs=#costs{spill=Spill}) -> + Costs#costs{spill=costs_insert_1(A, Weight, Liveset, Spill)}; +costs_insert(restore, A, Weight, Liveset, Costs=#costs{restore=Restore}) -> + Costs#costs{restore=costs_insert_1(A, Weight, Liveset, Restore)}. + +costs_insert_1(A, Weight, Liveset, CostMap0) when is_float(Weight) -> + lists:foldl(fun(Live, CostMap1) -> + map_update_counter([A|Live], Weight, CostMap1) + end, CostMap0, Liveset). + +-spec costs_map_roots(part_dsets(), costs()) -> {costs(), part_dsets()}. +costs_map_roots(DSets0, Costs) -> + {Entry1, DSets1} = costs_map_roots_1(DSets0, Costs#costs.entry1), + {Entry2, DSets2} = costs_map_roots_1(DSets1, Costs#costs.entry2), + {Exit, DSets3} = costs_map_roots_1(DSets2, Costs#costs.exit), + {Spill, DSets4} = costs_map_roots_1(DSets3, Costs#costs.spill), + {Restore, DSets} = costs_map_roots_1(DSets4, Costs#costs.restore), + {#costs{entry1=Entry1,entry2=Entry2,exit=Exit,spill=Spill,restore=Restore}, + DSets}. + +costs_map_roots_1(DSets0, CostMap) -> + {NewEs, DSets} = lists:mapfoldl(fun({[A|T], Wt}, DSets1) -> + {AR, DSets2} = hipe_dsets:find(A, DSets1), + {{[AR|T], Wt}, DSets2} + end, DSets0, maps:to_list(CostMap)), + {maps_from_list_merge(NewEs, fun erlang:'+'/2, #{}), DSets}. + +maps_from_list_merge([], _MF, Acc) -> Acc; +maps_from_list_merge([{K,V}|Ps], MF, Acc) -> + maps_from_list_merge(Ps, MF, case Acc of + #{K := OV} -> Acc#{K := MF(V, OV)}; + #{} -> Acc#{K => V} + end). + +-spec costs_query(temp(), cost_key(), part_key(), costs()) -> float(). +costs_query(Temp, entry1, Part, #costs{entry1=Entry1}) -> + costs_query_1(Temp, Part, Entry1); +costs_query(Temp, entry2, Part, #costs{entry2=Entry2}) -> + costs_query_1(Temp, Part, Entry2); +costs_query(Temp, exit, Part, #costs{exit=Exit}) -> + costs_query_1(Temp, Part, Exit); +costs_query(Temp, spill, Part, #costs{spill=Spill}) -> + costs_query_1(Temp, Part, Spill); +costs_query(Temp, restore, Part, #costs{restore=Restore}) -> + costs_query_1(Temp, Part, Restore). + +costs_query_1(Temp, Part, CostMap) -> + Key = [Part|Temp], + case CostMap of + #{Key := Wt} -> Wt; + #{} -> 0.0 + end. + +-spec map_update_counter(Key, number(), #{Key => number(), OK => OV}) + -> #{Key := number(), OK => OV}. +map_update_counter(Key, Incr, Map) -> + case Map of + #{Key := Orig} -> Map#{Key := Orig + Incr}; + #{} -> Map#{Key => Incr} + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Def and use counting ADT +-type ducount() :: #{temp() => float()}. + +-spec ducount_new() -> ducount(). +ducount_new() -> #{}. + +-spec ducount_add([temp()], float(), ducount()) -> ducount(). +ducount_add([], _Weight, DUCount) -> DUCount; +ducount_add([T|Ts], Weight, DUCount0) -> + DUCount = + case DUCount0 of + #{T := Count} -> DUCount0#{T := Count + Weight}; + #{} -> DUCount0#{T => Weight} + end, + ducount_add(Ts, Weight, DUCount). + +ducount_to_list(DUCount) -> maps:to_list(DUCount). + +-spec ducount_merge(ducount(), ducount()) -> ducount(). +ducount_merge(DCA, DCB) when map_size(DCA) < map_size(DCB) -> + ducount_merge_1(ducount_to_list(DCA), DCB); +ducount_merge(DCA, DCB) when map_size(DCA) >= map_size(DCB) -> + ducount_merge_1(ducount_to_list(DCB), DCA). + +ducount_merge_1([], DUCount) -> DUCount; +ducount_merge_1([{T,AC}|Ts], DUCount0) -> + DUCount = + case DUCount0 of + #{T := BC} -> DUCount0#{T := AC + BC}; + #{} -> DUCount0#{T => AC} + end, + ducount_merge_1(Ts, DUCount). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Target module interface functions +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +-define(TGT_IFACE_0(N), N( {M,C}) -> M:N( C)). +-define(TGT_IFACE_1(N), N(A1, {M,C}) -> M:N(A1, C)). +-define(TGT_IFACE_2(N), N(A1,A2, {M,C}) -> M:N(A1,A2, C)). +-define(TGT_IFACE_3(N), N(A1,A2,A3,{M,C}) -> M:N(A1,A2,A3,C)). + +?TGT_IFACE_2(bb). +?TGT_IFACE_1(def_use). +?TGT_IFACE_1(defines). +?TGT_IFACE_1(defines_all_alloc). +?TGT_IFACE_1(is_precoloured). +?TGT_IFACE_1(mk_goto). +?TGT_IFACE_2(mk_move). +?TGT_IFACE_0(new_label). +?TGT_IFACE_0(new_reg_nr). +?TGT_IFACE_1(number_of_temporaries). +?TGT_IFACE_3(redirect_jmp). +?TGT_IFACE_1(reg_nr). +?TGT_IFACE_1(reverse_postorder). +?TGT_IFACE_2(subst_temps). +?TGT_IFACE_3(update_bb). +?TGT_IFACE_2(update_reg_nr). + +branch_preds(Instr, {TgtMod,TgtCtx}) -> + merge_sorted_preds(lists:keysort(1, TgtMod:branch_preds(Instr, TgtCtx))). + +livein(Liveness, L, Target={TgtMod,TgtCtx}) -> + ordsets:from_list(reg_names(TgtMod:livein(Liveness, L, TgtCtx), Target)). + +liveout(Liveness, L, Target={TgtMod,TgtCtx}) -> + ordsets:from_list(reg_names(TgtMod:liveout(Liveness, L, TgtCtx), Target)). + +merge_sorted_preds([]) -> []; +merge_sorted_preds([{L, P1}, {L, P2}|LPs]) -> + merge_sorted_preds([{L, P1+P2}|LPs]); +merge_sorted_preds([LP|LPs]) -> [LP|merge_sorted_preds(LPs)]. + +reg_names(Regs, {TgtMod,TgtCtx}) -> + [TgtMod:reg_nr(X,TgtCtx) || X <- Regs]. diff --git a/lib/hipe/regalloc/hipe_regalloc_loop.erl b/lib/hipe/regalloc/hipe_regalloc_loop.erl index 5bbb0ba7c1..29ef3adcc2 100644 --- a/lib/hipe/regalloc/hipe_regalloc_loop.erl +++ b/lib/hipe/regalloc/hipe_regalloc_loop.erl @@ -32,9 +32,11 @@ ra_fp(CFG, Liveness, Options, RegAllocMod, TargetMod, TargetCtx) -> ra_common(CFG0, Liveness0, SpillIndex, Options, RegAllocMod, TargetMod, TargetCtx) -> ?inc_counter(ra_calls_counter, 1), - SpillLimit0 = TargetMod:number_of_temporaries(CFG0, TargetCtx), + {CFG1, Liveness1} = + do_range_split(CFG0, Liveness0, TargetMod, TargetCtx, Options), + SpillLimit0 = TargetMod:number_of_temporaries(CFG1, TargetCtx), {Coloring, _, CFG, Liveness} = - call_allocator_initial(CFG0, Liveness0, SpillLimit0, SpillIndex, Options, + call_allocator_initial(CFG1, Liveness1, SpillLimit0, SpillIndex, Options, RegAllocMod, TargetMod, TargetCtx), %% The first iteration, the hipe_regalloc_prepass may create new temps, these %% should not end up above SpillLimit. @@ -96,3 +98,20 @@ call_allocator(CFG, Liveness, SpillLimit, SpillIndex, Options, RegAllocMod, RegAllocMod:regalloc(CFG, Liveness, SpillIndex, SpillLimit, TargetMod, TargetCtx, Options) end. + +do_range_split(CFG0, Liveness0, TgtMod, TgtCtx, Options) -> + {CFG2, Liveness1} = + case proplists:get_bool(ra_restore_reuse, Options) of + true -> + CFG1 = hipe_restore_reuse:split(CFG0, Liveness0, TgtMod, TgtCtx), + {CFG1, TgtMod:analyze(CFG1, TgtCtx)}; + false -> + {CFG0, Liveness0} + end, + case proplists:get_bool(ra_range_split, Options) of + true -> + CFG3 = hipe_range_split:split(CFG2, Liveness1, TgtMod, TgtCtx, Options), + {CFG3, TgtMod:analyze(CFG3, TgtCtx)}; + false -> + {CFG2, Liveness1} + end. diff --git a/lib/hipe/regalloc/hipe_regalloc_prepass.erl b/lib/hipe/regalloc/hipe_regalloc_prepass.erl index e212420ad2..5024840237 100644 --- a/lib/hipe/regalloc/hipe_regalloc_prepass.erl +++ b/lib/hipe/regalloc/hipe_regalloc_prepass.erl @@ -483,8 +483,8 @@ merge_pointless_splits_1([], _ScanBBs, DSets, Acc) -> {Acc, DSets}; merge_pointless_splits_1([P={_,{single,_}}|Ps], ScanBBs, DSets, Acc) -> merge_pointless_splits_1(Ps, ScanBBs, DSets, [P|Acc]); merge_pointless_splits_1([P0={L,{split,_,_}}|Ps], ScanBBs, DSets0, Acc) -> - {EntryRoot, DSets1} = dsets_find({entry,L}, DSets0), - {ExitRoot, DSets} = dsets_find({exit,L}, DSets1), + {EntryRoot, DSets1} = hipe_dsets:find({entry,L}, DSets0), + {ExitRoot, DSets} = hipe_dsets:find({exit,L}, DSets1), case EntryRoot =:= ExitRoot of false -> merge_pointless_splits_1(Ps, ScanBBs, DSets, [P0|Acc]); true -> @@ -501,7 +501,7 @@ merge_pointless_splits_1([P0={L,{split,_,_}}|Ps], ScanBBs, DSets0, Acc) -> -spec merge_small_parts(bb_dsets()) -> {bb_dsets_rllist(), bb_dsets()}. merge_small_parts(DSets0) -> - {RLList, DSets1} = dsets_to_rllist(DSets0), + {RLList, DSets1} = hipe_dsets:to_rllist(DSets0), RLLList = [{R, length(Elems), Elems} || {R, Elems} <- RLList], merge_small_parts_1(RLLList, DSets1, []). @@ -518,8 +518,8 @@ merge_small_parts_1([Fst,{R, L, Es}|Ps], DSets, Acc) merge_small_parts_1([Fst|Ps], DSets, [{R,Es}|Acc]); merge_small_parts_1([{R1,L1,Es1},{R2,L2,Es2}|Ps], DSets0, Acc) -> ?ASSERT(L1 < ?TUNE_TOO_FEW_BBS andalso L2 < ?TUNE_TOO_FEW_BBS), - DSets1 = dsets_union(R1, R2, DSets0), - {R, DSets} = dsets_find(R1, DSets1), + DSets1 = hipe_dsets:union(R1, R2, DSets0), + {R, DSets} = hipe_dsets:find(R1, DSets1), merge_small_parts_1([{R,L2+L1,Es2++Es1}|Ps], DSets, Acc). %% @doc Partition an ordering over BBs into subsequences for the dsets that @@ -531,8 +531,8 @@ part_order(Lbs, DSets) -> part_order(Lbs, DSets, #{}). part_order([], DSets, Acc) -> {Acc, DSets}; part_order([L|Ls], DSets0, Acc0) -> - {EntryRoot, DSets1} = dsets_find({entry,L}, DSets0), - {ExitRoot, DSets2} = dsets_find({exit,L}, DSets1), + {EntryRoot, DSets1} = hipe_dsets:find({entry,L}, DSets0), + {ExitRoot, DSets2} = hipe_dsets:find({exit,L}, DSets1), Acc1 = map_append(EntryRoot, L, Acc0), %% Only include the label once if both entry and exit is in same partition Acc2 = case EntryRoot =:= ExitRoot of @@ -558,73 +558,26 @@ map_append(Key, Elem, Map) -> %% split point, and one from the end to the last split point. -type bb_dset_key() :: {entry | exit, label()}. --type bb_dsets() :: dsets(bb_dset_key()). +-type bb_dsets() :: hipe_dsets:dsets(bb_dset_key()). -type bb_dsets_rllist() :: [{bb_dset_key(), [bb_dset_key()]}]. -spec initial_dsets(target_cfg(), module(), target_context()) -> bb_dsets(). initial_dsets(CFG, TgtMod, TgtCtx) -> Labels = TgtMod:labels(CFG, TgtCtx), - DSets0 = dsets_new(lists:append([[{entry,L},{exit,L}] || L <- Labels])), + DSets0 = hipe_dsets:new(lists:append([[{entry,L},{exit,L}] || L <- Labels])), Edges = lists:append([[{L, S} || S <- hipe_gen_cfg:succ(CFG, L)] || L <- Labels]), - lists:foldl(fun({X, Y}, DS) -> dsets_union({exit,X}, {entry,Y}, DS) end, + lists:foldl(fun({X, Y}, DS) -> hipe_dsets:union({exit,X}, {entry,Y}, DS) end, DSets0, Edges). -spec join_whole_blocks(part_bb_list(), bb_dsets()) -> bb_dsets(). join_whole_blocks(PartBBList, DSets0) -> - lists:foldl(fun({L, {single, _}}, DS) -> dsets_union({entry,L}, {exit,L}, DS); + lists:foldl(fun({L, {single, _}}, DS) -> + hipe_dsets:union({entry,L}, {exit,L}, DS); ({_, {split, _, _}}, DS) -> DS end, DSets0, PartBBList). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% The disjoint set forests data structure, for elements of arbitrary types. -%% Note that the find operation mutates the set. -%% -%% We could do this more efficiently if we restricted the elements to integers, -%% and used the (mutable) hipe arrays. For arbitrary terms ETS could be used, -%% for a persistent interface (which isn't that nice when even accessors return -%% modified copies), the array module could be used. --type dsets(X) :: #{X => {node, X} | {root, non_neg_integer()}}. - --spec dsets_new([E]) -> dsets(E). -dsets_new(Elems) -> maps:from_list([{E,{root,0}} || E <- Elems]). - --spec dsets_find(E, dsets(E)) -> {E, dsets(E)}. -dsets_find(E, DS0) -> - case DS0 of - #{E := {root,_}} -> {E, DS0}; - #{E := {node,N}} -> - case dsets_find(N, DS0) of - {N, _}=T -> T; - {R, DS1} -> {R, DS1#{E := {node,R}}} - end - ;_ -> error(badarg, [E, DS0]) - end. - --spec dsets_union(E, E, dsets(E)) -> dsets(E). -dsets_union(X, Y, DS0) -> - {XRoot, DS1} = dsets_find(X, DS0), - case dsets_find(Y, DS1) of - {XRoot, DS2} -> DS2; - {YRoot, DS2} -> - #{XRoot := {root,XRR}, YRoot := {root,YRR}} = DS2, - if XRR < YRR -> DS2#{XRoot := {node,YRoot}}; - XRR > YRR -> DS2#{YRoot := {node,XRoot}}; - true -> DS2#{YRoot := {node,XRoot}, XRoot := {root,XRR+1}} - end - end. - --spec dsets_to_rllist(dsets(E)) -> {[{Root::E, Elems::[E]}], dsets(E)}. -dsets_to_rllist(DS0) -> - {Lists, DS} = dsets_to_rllist(maps:keys(DS0), #{}, DS0), - {maps:to_list(Lists), DS}. - -dsets_to_rllist([], Acc, DS) -> {Acc, DS}; -dsets_to_rllist([E|Es], Acc, DS0) -> - {ERoot, DS} = dsets_find(E, DS0), - dsets_to_rllist(Es, map_append(ERoot, E, Acc), DS). - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Third pass %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Collect all referenced temps in each partition. diff --git a/lib/hipe/regalloc/hipe_restore_reuse.erl b/lib/hipe/regalloc/hipe_restore_reuse.erl new file mode 100644 index 0000000000..2158bd185e --- /dev/null +++ b/lib/hipe/regalloc/hipe_restore_reuse.erl @@ -0,0 +1,516 @@ +%% -*- erlang-indent-level: 2 -*- +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%@doc +%% RESTORE REUSE LIVE RANGE SPLITTING PASS +%% +%% This is a simple live range splitter that tries to avoid sequences where a +%% temporary is accessed on stack multiple times by keeping a copy of that temp +%% around in a register. +%% +%% At any point where a temporary that is expected to be spilled (see uses of +%% spills_add_list/2) is defined or used, this pass considers that temporary +%% "available". +%% +%% Limitations: +%% * If a live range part starts with several different restores, this module +%% will introduce a new temp number for each of them, and later be forced to +%% generate phi blocks. It would be more efficient to introduce just a +%% single temp number. That would also remove the need for the phi blocks. +%% * If a live range part ends in a definition, that definition should just +%% define the base temp rather than the substitution, since some CISC +%% targets might be able to inline the memory access in the instruction. +-module(hipe_restore_reuse). + +-export([split/4]). + +%% Exports for hipe_range_split, which uses restore_reuse as one possible spill +%% "mode" +-export([analyse/3 + ,renamed_in_block/2 + ,split_in_block/2 + ]). +-export_type([avail/0]). + +-compile(inline). + +%% -define(DO_ASSERT, 1). +-include("../main/hipe.hrl"). + +-type target_cfg() :: any(). +-type liveness() :: any(). +-type target_module() :: module(). +-type target_context() :: any(). +-type target() :: {target_module(), target_context()}. +-type label() :: non_neg_integer(). +-type reg() :: non_neg_integer(). +-type instr() :: any(). +-type temp() :: any(). + +-spec split(target_cfg(), liveness(), target_module(), target_context()) + -> target_cfg(). +split(CFG, Liveness, TargetMod, TargetContext) -> + Target = {TargetMod, TargetContext}, + Avail = analyse(CFG, Liveness, Target), + rewrite(CFG, Target, Avail). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +-opaque avail() :: #{label() => avail_bb()}. + +-record(avail_bb, { + %% Blocks where HasCall is true are considered to have too high + %% register pressure to support a register copy of a temp + has_call :: boolean(), + %% AvailOut: Temps that can be split (are available) + out :: availset(), + %% Gen: AvailOut generated locally + gen :: availset(), + %% WantIn: Temps that are split + want :: regset(), + %% Self: Temps with avail-want pairs locally + self :: regset(), + %% DefIn: Temps shadowed by later def in same live range part + defin :: regset(), + pred :: [label()], + succ :: [label()] + }). +-type avail_bb() :: #avail_bb{}. + +avail_get(L, Avail) -> maps:get(L, Avail). +avail_set(L, Val, Avail) -> maps:put(L, Val, Avail). +avail_has_call(L, Avail) -> (avail_get(L, Avail))#avail_bb.has_call. +avail_out(L, Avail) -> (avail_get(L, Avail))#avail_bb.out. +avail_self(L, Avail) -> (avail_get(L, Avail))#avail_bb.self. +avail_pred(L, Avail) -> (avail_get(L, Avail))#avail_bb.pred. +avail_succ(L, Avail) -> (avail_get(L, Avail))#avail_bb.succ. + +avail_in(L, Avail) -> + case avail_pred(L, Avail) of + [] -> availset_empty(); % entry + Pred -> + lists:foldl(fun(P, ASet) -> + availset_intersect(avail_out(P, Avail), ASet) + end, availset_top(), Pred) + end. + +want_in(L, Avail) -> (avail_get(L, Avail))#avail_bb.want. +want_out(L, Avail) -> + lists:foldl(fun(S, Set) -> + ordsets:union(want_in(S, Avail), Set) + end, ordsets:new(), avail_succ(L, Avail)). + +def_in(L, Avail) -> (avail_get(L, Avail))#avail_bb.defin. +def_out(L, Avail) -> + case avail_succ(L, Avail) of + [] -> ordsets:new(); % entry + Succ -> + ordsets:intersection([def_in(S, Avail) || S <- Succ]) + end. + +-type regset() :: ordsets:ordset(reg()). +-type availset() :: top | regset(). +availset_empty() -> []. +availset_top() -> top. +availset_intersect(top, B) -> B; +availset_intersect(A, top) -> A; +availset_intersect(A, B) -> ordsets:intersection(A, B). +availset_union(top, _) -> top; +availset_union(_, top) -> top; +availset_union(A, B) -> ordsets:union(A, B). +ordset_intersect_availset(OS, top) -> OS; +ordset_intersect_availset(OS, AS) -> ordsets:intersection(OS, AS). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Analysis pass +%% +%% The analysis pass collects the set of temps we're interested in splitting +%% (Spills), and computes three dataflow analyses for this subset of temps. +%% +%% Avail, which is the set of temps which are available in register from a +%% previous (potential) spill or restore without going through a HasCall +%% block. +%% Want, which is a liveness analysis for the subset of temps used by an +%% instruction that are also in Avail at that point. In other words, Want is +%% the set of temps that are split (has a register copy) at a particular +%% point. +%% Def, which are the temps that are already going to be spilled later, and so +%% need not be spilled when they're defined. +%% +%% Lastly, it computes the set Self for each block, which is the temps that have +%% avail-want pairs in the same block, and so should be split in that block even +%% if they're not in WantIn for the block. + +-spec analyse(target_cfg(), liveness(), target()) -> avail(). +analyse(CFG, Liveness, Target) -> + Avail0 = analyse_init(CFG, Liveness, Target), + RPO = reverse_postorder(CFG, Target), + AvailLs = [L || L <- RPO, not avail_has_call(L, Avail0)], + Avail1 = avail_dataf(AvailLs, Avail0), + Avail2 = analyse_filter_want(maps:keys(Avail1), Avail1), + PO = lists:reverse(RPO), + want_dataf(PO, Avail2). + +-spec analyse_init(target_cfg(), liveness(), target()) -> avail(). +analyse_init(CFG, Liveness, Target) -> + analyse_init(labels(CFG, Target), CFG, Liveness, Target, #{}, []). + +-spec analyse_init([label()], target_cfg(), liveness(), target(), spillset(), + [{label(), avail_bb()}]) + -> avail(). +analyse_init([], _CFG, _Liveness, Target, Spills0, Acc) -> + %% Precoloured temps can't be spilled + Spills = spills_filter(fun(R) -> not is_precoloured(R, Target) end, Spills0), + analyse_init_1(Acc, Spills, []); +analyse_init([L|Ls], CFG, Liveness, Target, Spills0, Acc) -> + {DefIn, Gen, Self, Want, HasCall0} = + analyse_scan(hipe_bb:code(bb(CFG, L, Target)), Target, + ordsets:new(), ordsets:new(), ordsets:new(), + ordsets:new()), + {Spills, Out, HasCall} = + case HasCall0 of + false -> {Spills0, availset_top(), false}; + {true, CallDefs} -> + Spill = ordsets:subtract(liveout(Liveness, L, Target), CallDefs), + {spills_add_list(Spill, Spills0), Gen, true} + end, + Pred = hipe_gen_cfg:pred(CFG, L), + Succ = hipe_gen_cfg:succ(CFG, L), + Val = #avail_bb{gen=Gen, want=Want, self=Self, out=Out, has_call=HasCall, + pred=Pred, succ=Succ, defin=DefIn}, + analyse_init(Ls, CFG, Liveness, Target, Spills, [{L, Val} | Acc]). + +-spec analyse_init_1([{label(), avail_bb()}], spillset(), + [{label(), avail_bb()}]) + -> avail(). +analyse_init_1([], _Spills, Acc) -> maps:from_list(Acc); +analyse_init_1([{L, Val0}|Vs], Spills, Acc) -> + #avail_bb{out=Out,gen=Gen,want=Want,self=Self} = Val0, + Val = Val0#avail_bb{ + out = spills_filter_availset(Out, Spills), + gen = spills_filter_availset(Gen, Spills), + want = spills_filter_availset(Want, Spills), + self = spills_filter_availset(Self, Spills)}, + analyse_init_1(Vs, Spills, [{L, Val} | Acc]). + +-type spillset() :: #{reg() => []}. +-spec spills_add_list([reg()], spillset()) -> spillset(). +spills_add_list([], Spills) -> Spills; +spills_add_list([R|Rs], Spills) -> spills_add_list(Rs, Spills#{R => []}). + +-spec spills_filter_availset(availset(), spillset()) -> availset(). +spills_filter_availset([E|Es], Spills) -> + case Spills of + #{E := _} -> [E|spills_filter_availset(Es, Spills)]; + #{} -> spills_filter_availset(Es, Spills) + end; +spills_filter_availset([], _) -> []; +spills_filter_availset(top, _) -> top. + +spills_filter(Fun, Spills) -> maps:filter(fun(K, _) -> Fun(K) end, Spills). + +-spec analyse_scan([instr()], target(), Defset, Gen, Self, Want) + -> {Defset, Gen, Self, Want, HasCall} when + HasCall :: false | {true, regset()}, + Defset :: regset(), + Gen :: availset(), + Self :: regset(), + Want :: regset(). +analyse_scan([], _Target, Defs, Gen, Self, Want) -> + {Defs, Gen, Self, Want, false}; +analyse_scan([I|Is], Target, Defs0, Gen0, Self0, Want0) -> + {DefL, UseL} = reg_def_use(I, Target), + Use = ordsets:from_list(UseL), + Def = ordsets:from_list(DefL), + Self = ordsets:union(ordsets:intersection(Use, Gen0), Self0), + Want = ordsets:union(ordsets:subtract(Use, Defs0), Want0), + Defs = ordsets:union(Def, Defs0), + case defines_all_alloc(I, Target) of + true -> + [] = Is, %assertion + {Defs, ordsets:new(), Self, Want, {true, Def}}; + false -> + Gen = ordsets:union(ordsets:union(Def, Use), Gen0), + analyse_scan(Is, Target, Defs, Gen, Self, Want) + end. + +-spec avail_dataf([label()], avail()) -> avail(). +avail_dataf(RPO, Avail0) -> + case avail_dataf_once(RPO, Avail0, 0) of + {Avail, 0} -> Avail; + {Avail, _Changed} -> + avail_dataf(RPO, Avail) + end. + +-spec avail_dataf_once([label()], avail(), non_neg_integer()) + -> {avail(), non_neg_integer()}. +avail_dataf_once([], Avail, Changed) -> {Avail, Changed}; +avail_dataf_once([L|Ls], Avail0, Changed0) -> + ABB = #avail_bb{out=OldOut, gen=Gen} = avail_get(L, Avail0), + In = avail_in(L, Avail0), + {Changed, Avail} = + case availset_union(In, Gen) of + OldOut -> {Changed0, Avail0}; + Out -> {Changed0+1, avail_set(L, ABB#avail_bb{out=Out}, Avail0)} + end, + avail_dataf_once(Ls, Avail, Changed). + +-spec analyse_filter_want([label()], avail()) -> avail(). +analyse_filter_want([], Avail) -> Avail; +analyse_filter_want([L|Ls], Avail0) -> + ABB = #avail_bb{want=Want0, defin=DefIn0} = avail_get(L, Avail0), + In = avail_in(L, Avail0), + Want = ordset_intersect_availset(Want0, In), + DefIn = ordset_intersect_availset(DefIn0, In), + Avail = avail_set(L, ABB#avail_bb{want=Want, defin=DefIn}, Avail0), + analyse_filter_want(Ls, Avail). + +-spec want_dataf([label()], avail()) -> avail(). +want_dataf(PO, Avail0) -> + case want_dataf_once(PO, Avail0, 0) of + {Avail, 0} -> Avail; + {Avail, _Changed} -> + want_dataf(PO, Avail) + end. + +-spec want_dataf_once([label()], avail(), non_neg_integer()) + -> {avail(), non_neg_integer()}. +want_dataf_once([], Avail, Changed) -> {Avail, Changed}; +want_dataf_once([L|Ls], Avail0, Changed0) -> + ABB0 = #avail_bb{want=OldIn,defin=OldDef} = avail_get(L, Avail0), + AvailIn = avail_in(L, Avail0), + Out = want_out(L, Avail0), + DefOut = def_out(L, Avail0), + {Changed, Avail} = + case {ordsets:union(ordset_intersect_availset(Out, AvailIn), OldIn), + ordsets:union(ordset_intersect_availset(DefOut, AvailIn), OldDef)} + of + {OldIn, OldDef} -> {Changed0, Avail0}; + {In, DefIn} -> + ABB = ABB0#avail_bb{want=In,defin=DefIn}, + {Changed0+1, avail_set(L, ABB, Avail0)} + end, + want_dataf_once(Ls, Avail, Changed). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Rewrite pass +-type subst_dict() :: orddict:orddict(reg(), reg()). +-type input() :: #{label() => subst_dict()}. + +-spec rewrite(target_cfg(), target(), avail()) -> target_cfg(). +rewrite(CFG, Target, Avail) -> + RPO = reverse_postorder(CFG, Target), + rewrite(RPO, Target, Avail, #{}, CFG). + +-spec rewrite([label()], target(), avail(), input(), target_cfg()) + -> target_cfg(). +rewrite([], _Target, _Avail, _Input, CFG) -> CFG; +rewrite([L|Ls], Target, Avail, Input0, CFG0) -> + SplitHere = split_in_block(L, Avail), + {Input1, LInput} = + case Input0 of + #{L := LInput0} -> {Input0, LInput0}; + #{} -> {Input0#{L => []}, []} % entry block + end, + ?ASSERT([] =:= [X || X <- SplitHere, orddict:is_key(X, LInput)]), + ?ASSERT(want_in(L, Avail) =:= orddict:fetch_keys(LInput)), + {CFG1, LOutput} = + case {SplitHere, LInput} of + {[], []} -> % optimisation (rewrite will do nothing, so skip it) + {CFG0, LInput}; + _ -> + Code0 = hipe_bb:code(BB=bb(CFG0, L, Target)), + DefOut = def_out(L, Avail), + {Code, LOutput0, _DefIn} = + rewrite_instrs(Code0, Target, LInput, DefOut, SplitHere), + {update_bb(CFG0, L, hipe_bb:code_update(BB, Code), Target), LOutput0} + end, + {Input, CFG} = rewrite_succs(avail_succ(L, Avail), Target, L, LOutput, Avail, + Input1, CFG1), + rewrite(Ls, Target, Avail, Input, CFG). + +-spec renamed_in_block(label(), avail()) -> ordsets:ordset(reg()). +renamed_in_block(L, Avail) -> + ordsets:union([avail_self(L, Avail), want_in(L, Avail), + want_out(L, Avail)]). + +-spec split_in_block(label(), avail()) -> ordsets:ordset(reg()). +split_in_block(L, Avail) -> + ordsets:subtract(ordsets:union(avail_self(L, Avail), want_out(L, Avail)), + want_in(L, Avail)). + +-spec rewrite_instrs([instr()], target(), subst_dict(), regset(), [reg()]) + -> {[instr()], subst_dict(), regset()}. +rewrite_instrs([], _Target, Output, DefOut, []) -> + {[], Output, DefOut}; +rewrite_instrs([I|Is], Target, Input0, BBDefOut, SplitHere0) -> + {TDef, TUse} = def_use(I, Target), + {Def, Use} = {reg_names(TDef, Target), reg_names(TUse, Target)}, + %% Restores are generated in forward order by picking temps from SplitHere as + %% they're used or defined. After the last instruction, all temps have been + %% picked. + {ISplits, SplitHere} = + lists:partition(fun(R) -> + lists:member(R, Def) orelse lists:member(R, Use) + end, SplitHere0), + {Input, Restores} = + case ISplits of + [] -> {Input0, []}; + _ -> + make_splits(ISplits, Target, TDef, TUse, Input0, []) + end, + %% Here's the recursive call + {Acc0, Output, DefOut} = + rewrite_instrs(Is, Target, Input, BBDefOut, SplitHere), + %% From here we're processing instructions in reverse order, because to avoid + %% redundant spills we need to walk the 'def' dataflow, which is in reverse. + SubstFun = fun(Temp) -> + case orddict:find(reg_nr(Temp, Target), Input) of + {ok, NewTemp} -> NewTemp; + error -> Temp + end + end, + Acc1 = insert_spills(TDef, Target, Input, DefOut, Acc0), + Acc = Restores ++ [subst_temps(SubstFun, I, Target) | Acc1], + DefIn = ordsets:union(DefOut, ordsets:from_list(Def)), + {Acc, Output, DefIn}. + +-spec make_splits([reg()], target(), [temp()], [temp()], subst_dict(), + [instr()]) + -> {subst_dict(), [instr()]}. +make_splits([], _Target, _TDef, _TUse, Input, Acc) -> + {Input, Acc}; +make_splits([S|Ss], Target, TDef, TUse, Input0, Acc0) -> + SubstReg = new_reg_nr(Target), + {Acc, Subst} = + case find_reg_temp(S, TUse, Target) of + error -> + {ok, Temp} = find_reg_temp(S, TDef, Target), + {Acc0, update_reg_nr(SubstReg, Temp, Target)}; + {ok, Temp} -> + Subst0 = update_reg_nr(SubstReg, Temp, Target), + Acc1 = [mk_move(Temp, Subst0, Target) | Acc0], + {Acc1, Subst0} + end, + Input = orddict:store(S, Subst, Input0), + make_splits(Ss, Target, TDef, TUse, Input, Acc). + +-spec find_reg_temp(reg(), [temp()], target()) -> error | {ok, temp()}. +find_reg_temp(_Reg, [], _Target) -> error; +find_reg_temp(Reg, [T|Ts], Target) -> + case reg_nr(T, Target) of + Reg -> {ok, T}; + _ -> find_reg_temp(Reg, Ts, Target) + end. + +-spec insert_spills([temp()], target(), subst_dict(), regset(), [instr()]) + -> [instr()]. +insert_spills([], _Target, _Input, _DefOut, Acc) -> Acc; +insert_spills([T|Ts], Target, Input, DefOut, Acc0) -> + R = reg_nr(T, Target), + Acc = + case orddict:find(R, Input) of + error -> Acc0; + {ok, Subst} -> + case lists:member(R, DefOut) of + true -> Acc0; + false -> [mk_move(Subst, T, Target) | Acc0] + end + end, + insert_spills(Ts, Target, Input, DefOut, Acc). + +-spec rewrite_succs([label()], target(), label(), subst_dict(), avail(), + input(), target_cfg()) -> {input(), target_cfg()}. +rewrite_succs([], _Target, _P, _POutput, _Avail, Input, CFG) -> {Input, CFG}; +rewrite_succs([L|Ls], Target, P, POutput, Avail, Input0, CFG0) -> + NewLInput = orddict_with_ordset(want_in(L, Avail), POutput), + {Input, CFG} = + case Input0 of + #{L := LInput} -> + CFG2 = + case required_phi_moves(LInput, NewLInput) of + [] -> CFG0; + ReqMovs -> + PhiLb = new_label(Target), + Code = [mk_move(S,D,Target) || {S,D} <- ReqMovs] + ++ [mk_goto(L, Target)], + PhiBB = hipe_bb:mk_bb(Code), + CFG1 = update_bb(CFG0, PhiLb, PhiBB, Target), + bb_redirect_jmp(L, PhiLb, P, CFG1, Target) + end, + {Input0, CFG2}; + #{} -> + {Input0#{L => NewLInput}, CFG0} + end, + rewrite_succs(Ls, Target, P, POutput, Avail, Input, CFG). + +-spec bb_redirect_jmp(label(), label(), label(), target_cfg(), target()) + -> target_cfg(). +bb_redirect_jmp(From, To, Lb, CFG, Target) -> + BB0 = bb(CFG, Lb, Target), + Last = redirect_jmp(hipe_bb:last(BB0), From, To, Target), + BB = hipe_bb:code_update(BB0, hipe_bb:butlast(BB0) ++ [Last]), + update_bb(CFG, Lb, BB, Target). + +-spec required_phi_moves(subst_dict(), subst_dict()) -> [{reg(), reg()}]. +required_phi_moves([], []) -> []; +required_phi_moves([P|Is], [P|Os]) -> required_phi_moves(Is, Os); +required_phi_moves([{K, In}|Is], [{K, Out}|Os]) -> + [{Out, In}|required_phi_moves(Is, Os)]. + +%% @doc Returns a new orddict with the keys in Set and their associated values. +-spec orddict_with_ordset(ordsets:ordset(K), orddict:orddict(K, V)) + -> orddict:orddict(K, V). +orddict_with_ordset([S|Ss], [{K, _}|_]=Dict) when S < K -> + orddict_with_ordset(Ss, Dict); +orddict_with_ordset([S|_]=Set, [{K, _}|Ds]) when S > K -> + orddict_with_ordset(Set, Ds); +orddict_with_ordset([_S|Ss], [{_K, _}=P|Ds]) -> % _S == _K + [P|orddict_with_ordset(Ss, Ds)]; +orddict_with_ordset([], _) -> []; +orddict_with_ordset(_, []) -> []. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Target module interface functions +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +-define(TGT_IFACE_0(N), N( {M,C}) -> M:N( C)). +-define(TGT_IFACE_1(N), N(A1, {M,C}) -> M:N(A1, C)). +-define(TGT_IFACE_2(N), N(A1,A2, {M,C}) -> M:N(A1,A2, C)). +-define(TGT_IFACE_3(N), N(A1,A2,A3,{M,C}) -> M:N(A1,A2,A3,C)). + +?TGT_IFACE_2(bb). +?TGT_IFACE_1(def_use). +?TGT_IFACE_1(defines_all_alloc). +?TGT_IFACE_1(is_precoloured). +?TGT_IFACE_1(labels). +?TGT_IFACE_1(mk_goto). +?TGT_IFACE_2(mk_move). +?TGT_IFACE_0(new_label). +?TGT_IFACE_0(new_reg_nr). +?TGT_IFACE_3(redirect_jmp). +?TGT_IFACE_1(reg_nr). +?TGT_IFACE_1(reverse_postorder). +?TGT_IFACE_2(subst_temps). +?TGT_IFACE_3(update_bb). +?TGT_IFACE_2(update_reg_nr). + +liveout(Liveness, L, Target={TgtMod,TgtCtx}) -> + ordsets:from_list(reg_names(TgtMod:liveout(Liveness, L, TgtCtx), Target)). + +reg_names(Regs, {TgtMod,TgtCtx}) -> + [TgtMod:reg_nr(X,TgtCtx) || X <- Regs]. + +reg_def_use(I, Target) -> + {TDef, TUse} = def_use(I, Target), + {reg_names(TDef, Target), reg_names(TUse, Target)}. diff --git a/lib/hipe/regalloc/hipe_sparc_specific.erl b/lib/hipe/regalloc/hipe_sparc_specific.erl index 31fca81316..78b6379eba 100644 --- a/lib/hipe/regalloc/hipe_sparc_specific.erl +++ b/lib/hipe/regalloc/hipe_sparc_specific.erl @@ -24,6 +24,7 @@ ,reg_nr/2 ,def_use/2 ,is_move/2 + ,is_spill_move/2 ,is_precoloured/2 ,var_range/2 ,allocatable/1 @@ -46,12 +47,19 @@ %% callbacks for hipe_regalloc_loop -export([check_and_rewrite/3]). -%% callbacks for hipe_regalloc_prepass --export([new_reg_nr/1, +%% callbacks for hipe_regalloc_prepass, hipe_range_split +-export([mk_move/3, + mk_goto/2, + redirect_jmp/4, + new_label/1, + new_reg_nr/1, update_reg_nr/3, update_bb/4, subst_temps/3]). +%% callbacks for hipe_bb_weights, hipe_range_split +-export([branch_preds/2]). + check_and_rewrite(CFG, Coloring, no_context) -> hipe_sparc_ra_postconditions:check_and_rewrite(CFG, Coloring, 'normal'). @@ -115,6 +123,9 @@ bb(CFG,L,_) -> update_bb(CFG,L,BB,_) -> hipe_sparc_cfg:bb_add(CFG,L,BB). +branch_preds(Branch,_) -> + hipe_sparc_cfg:branch_preds(Branch). + %% SPARC stuff def_use(Instruction, Ctx) -> @@ -144,9 +155,24 @@ is_move(Instruction, _) -> false -> false end. +is_spill_move(Instruction, _) -> + hipe_sparc:is_pseudo_spill_move(Instruction). + reg_nr(Reg, _) -> hipe_sparc:temp_reg(Reg). +mk_move(Src, Dst, _) -> + hipe_sparc:mk_pseudo_move(Src, Dst). + +mk_goto(Label, _) -> + hipe_sparc:mk_b_label(Label). + +redirect_jmp(Jmp, ToOld, ToNew, _) when is_integer(ToOld), is_integer(ToNew) -> + hipe_sparc_cfg:redirect_jmp(Jmp, ToOld, ToNew). + +new_label(_) -> + hipe_gensym:get_next_label(sparc). + new_reg_nr(_) -> hipe_gensym:get_next_var(sparc). diff --git a/lib/hipe/regalloc/hipe_sparc_specific_fp.erl b/lib/hipe/regalloc/hipe_sparc_specific_fp.erl index 050d65e1a9..485fdc212a 100644 --- a/lib/hipe/regalloc/hipe_sparc_specific_fp.erl +++ b/lib/hipe/regalloc/hipe_sparc_specific_fp.erl @@ -24,6 +24,7 @@ ,reg_nr/2 ,def_use/2 ,is_move/2 + ,is_spill_move/2 ,is_precoloured/2 ,var_range/2 ,allocatable/1 @@ -46,12 +47,19 @@ %% callbacks for hipe_regalloc_loop -export([check_and_rewrite/3]). -%% callbacks for hipe_regalloc_prepass --export([new_reg_nr/1, +%% callbacks for hipe_regalloc_prepass, hipe_range_split +-export([mk_move/3, + mk_goto/2, + redirect_jmp/4, + new_label/1, + new_reg_nr/1, update_reg_nr/3, update_bb/4, subst_temps/3]). +%% callbacks for hipe_bb_weights, hipe_range_split +-export([branch_preds/2]). + check_and_rewrite(CFG, Coloring, no_context) -> hipe_sparc_ra_postconditions_fp:check_and_rewrite(CFG, Coloring). @@ -108,6 +116,9 @@ bb(CFG, L, _) -> update_bb(CFG,L,BB,_) -> hipe_sparc_cfg:bb_add(CFG,L,BB). +branch_preds(Branch,_) -> + hipe_sparc_cfg:branch_preds(Branch). + %% SPARC stuff def_use(I, Ctx) -> @@ -125,9 +136,24 @@ defines_all_alloc(I, _) -> is_move(I, _) -> hipe_sparc:is_pseudo_fmove(I). +is_spill_move(I, _) -> + hipe_sparc:is_pseudo_spill_fmove(I). + reg_nr(Reg, _) -> hipe_sparc:temp_reg(Reg). +mk_move(Src, Dst, _) -> + hipe_sparc:mk_pseudo_fmove(Src, Dst). + +mk_goto(Label, _) -> + hipe_sparc:mk_b_label(Label). + +redirect_jmp(Jmp, ToOld, ToNew, _) when is_integer(ToOld), is_integer(ToNew) -> + hipe_sparc_cfg:redirect_jmp(Jmp, ToOld, ToNew). + +new_label(_) -> + hipe_gensym:get_next_label(sparc). + new_reg_nr(_) -> hipe_gensym:get_next_var(sparc). diff --git a/lib/hipe/regalloc/hipe_x86_specific.erl b/lib/hipe/regalloc/hipe_x86_specific.erl index c1c8dbbcd6..dacfb71b00 100644 --- a/lib/hipe/regalloc/hipe_x86_specific.erl +++ b/lib/hipe/regalloc/hipe_x86_specific.erl @@ -46,6 +46,7 @@ def_use/2, is_arg/2, % used by hipe_ls_regalloc is_move/2, + is_spill_move/2, is_fixed/2, % used by hipe_graph_coloring_regalloc is_global/2, is_precoloured/2, @@ -63,12 +64,19 @@ %% callbacks for hipe_regalloc_loop -export([check_and_rewrite/3]). -%% callbacks for hipe_regalloc_prepass --export([new_reg_nr/1, +%% callbacks for hipe_regalloc_prepass, hipe_range_split +-export([mk_move/3, + mk_goto/2, + redirect_jmp/4, + new_label/1, + new_reg_nr/1, update_reg_nr/3, update_bb/4, subst_temps/3]). +%% callbacks for hipe_bb_weights +-export([branch_preds/2]). + check_and_rewrite(CFG, Coloring, _) -> ?HIPE_X86_RA_POSTCONDITIONS:check_and_rewrite(CFG, Coloring, 'normal'). @@ -156,6 +164,9 @@ bb(CFG,L,_) -> update_bb(CFG,L,BB,_) -> hipe_x86_cfg:bb_add(CFG,L,BB). +branch_preds(Instr,_) -> + hipe_x86_cfg:branch_preds(Instr). + %% X86 stuff def_use(Instruction,_) -> @@ -200,9 +211,33 @@ is_move(Instruction,_) -> false -> false end. +is_spill_move(Instruction,_) -> + hipe_x86:is_pseudo_spill_move(Instruction). + reg_nr(Reg,_) -> hipe_x86:temp_reg(Reg). +mk_move(Src, Dst, _) -> + hipe_x86:mk_move(Src, Dst). + +mk_goto(Label, _) -> + hipe_x86:mk_jmp_label(Label). + +redirect_jmp(Jmp, ToOld, ToNew, _) when is_integer(ToOld), is_integer(ToNew) -> + Ref = make_ref(), + put(Ref, false), + I = hipe_x86_subst:insn_lbls( + fun(Tgt) -> + if Tgt =:= ToOld -> put(Ref, true), ToNew; + is_integer(Tgt) -> Tgt + end + end, Jmp), + true = erase(Ref), % Assert that something was rewritten + I. + +new_label(_) -> + hipe_gensym:get_next_label(x86). + new_reg_nr(_) -> hipe_gensym:get_next_var(x86). diff --git a/lib/hipe/regalloc/hipe_x86_specific_x87.erl b/lib/hipe/regalloc/hipe_x86_specific_x87.erl index 4b4c83f76d..3fe49e1f00 100644 --- a/lib/hipe/regalloc/hipe_x86_specific_x87.erl +++ b/lib/hipe/regalloc/hipe_x86_specific_x87.erl @@ -47,6 +47,7 @@ uses/2, defines/2, defines_all_alloc/2, + is_spill_move/2, is_global/2, reg_nr/2, physical_name/2, @@ -158,6 +159,9 @@ defines(I, _) -> defines_all_alloc(I, _) -> hipe_amd64_defuse:insn_defs_all(I). +is_spill_move(I, _) -> + hipe_x86:is_pseudo_spill_fmove(I). + temp_is_double(Temp) -> hipe_x86:temp_type(Temp) =:= 'double'. diff --git a/lib/hipe/rtl/hipe_icode2rtl.erl b/lib/hipe/rtl/hipe_icode2rtl.erl index 82970f04ab..6da8a76d34 100644 --- a/lib/hipe/rtl/hipe_icode2rtl.erl +++ b/lib/hipe/rtl/hipe_icode2rtl.erl @@ -532,8 +532,12 @@ gen_cond(CondOp, Args, TrueLbl, FalseLbl, Pred) -> FalseLbl, Pred)]; '=:=' -> [Arg1, Arg2] = Args, + TypeTestLbl = hipe_rtl:mk_new_label(), [hipe_rtl:mk_branch(Arg1, eq, Arg2, TrueLbl, - hipe_rtl:label_name(GenLbl), Pred), + hipe_rtl:label_name(TypeTestLbl), Pred), + TypeTestLbl, + hipe_tagscheme:test_either_immed(Arg1, Arg2, FalseLbl, + hipe_rtl:label_name(GenLbl)), GenLbl, hipe_rtl:mk_call([Tmp], op_exact_eqeq_2, Args, TestRetName, [], not_remote), @@ -546,8 +550,12 @@ gen_cond(CondOp, Args, TrueLbl, FalseLbl, Pred) -> TrueLbl, 1-Pred)]; '=/=' -> [Arg1, Arg2] = Args, + TypeTestLbl = hipe_rtl:mk_new_label(), [hipe_rtl:mk_branch(Arg1, eq, Arg2, FalseLbl, - hipe_rtl:label_name(GenLbl), 1-Pred), + hipe_rtl:label_name(TypeTestLbl), 1-Pred), + TypeTestLbl, + hipe_tagscheme:test_either_immed(Arg1, Arg2, TrueLbl, + hipe_rtl:label_name(GenLbl)), GenLbl, hipe_rtl:mk_call([Tmp], op_exact_eqeq_2, Args, TestRetName, [], not_remote), diff --git a/lib/hipe/rtl/hipe_rtl_binary_construct.erl b/lib/hipe/rtl/hipe_rtl_binary_construct.erl index fd0d1f1223..52ea5db382 100644 --- a/lib/hipe/rtl/hipe_rtl_binary_construct.erl +++ b/lib/hipe/rtl/hipe_rtl_binary_construct.erl @@ -137,43 +137,6 @@ gen_rtl(BsOP, Dst, Args, TrueLblName, FalseLblName, SystemLimitLblName, ConstTab end end; - {bs_put_integer, Size, Flags, ConstInfo} -> - Aligned = aligned(Flags), - LittleEndian = littleendian(Flags), - [NewOffset] = get_real(Dst), - case is_illegal_const(Size) of - true -> - [hipe_rtl:mk_goto(FalseLblName)]; - false -> - case ConstInfo of - fail -> - [hipe_rtl:mk_goto(FalseLblName)]; - _ -> - case Args of - [Src, Base, Offset] -> - CCode = static_int_c_code(NewOffset, Src, - Base, Offset, Size, - Flags, TrueLblName, - FalseLblName), - put_static_int(NewOffset, Src, Base, Offset, Size, - CCode, Aligned, LittleEndian, TrueLblName); - [Src, Bits, Base, Offset] -> - {SizeCode, SizeReg} = - hipe_rtl_binary:make_size(Size, Bits, - SystemLimitLblName, - FalseLblName), - CCode = int_c_code(NewOffset, Src, Base, - Offset, SizeReg, Flags, - TrueLblName, FalseLblName), - InCode = - put_dynamic_int(NewOffset, Src, Base, Offset, - SizeReg, CCode, Aligned, - LittleEndian, TrueLblName), - SizeCode ++ InCode - end - end - end; - {unsafe_bs_put_integer, 0, _Flags, _ConstInfo} -> [NewOffset] = get_real(Dst), case Args of @@ -186,44 +149,12 @@ gen_rtl(BsOP, Dst, Args, TrueLblName, FalseLblName, SystemLimitLblName, ConstTab end; {unsafe_bs_put_integer, Size, Flags, ConstInfo} -> - case is_illegal_const(Size) of - true -> - [hipe_rtl:mk_goto(FalseLblName)]; - false -> - Aligned = aligned(Flags), - LittleEndian = littleendian(Flags), - [NewOffset] = get_real(Dst), - case ConstInfo of - fail -> - [hipe_rtl:mk_goto(FalseLblName)]; - _ -> - case Args of - [Src, Base, Offset] -> - CCode = static_int_c_code(NewOffset, Src, - Base, Offset, Size, - Flags, TrueLblName, - FalseLblName), - put_unsafe_static_int(NewOffset, Src, Base, - Offset, Size, - CCode, Aligned, LittleEndian, - TrueLblName); - [Src, Bits, Base, Offset] -> - {SizeCode, SizeReg} = - hipe_rtl_binary:make_size(Size, Bits, - SystemLimitLblName, - FalseLblName), - CCode = int_c_code(NewOffset, Src, Base, - Offset, SizeReg, Flags, - TrueLblName, FalseLblName), - InCode = - put_unsafe_dynamic_int(NewOffset, Src, Base, - Offset, SizeReg, CCode, - Aligned, LittleEndian, - TrueLblName), - SizeCode ++ InCode - end - end - end; + do_bs_put_integer(Dst, Args, Size, Flags, ConstInfo, true, + TrueLblName, FalseLblName, SystemLimitLblName); + + {bs_put_integer, Size, Flags, ConstInfo} -> + do_bs_put_integer(Dst, Args, Size, Flags, ConstInfo, false, + TrueLblName, FalseLblName, SystemLimitLblName); bs_utf8_size -> case Dst of @@ -360,6 +291,40 @@ gen_rtl(BsOP, Dst, Args, TrueLblName, FalseLblName, SystemLimitLblName, ConstTab {Code, ConstTab} end. +%% Common implementation of bs_put_integer and unsafe_bs_put_integer +do_bs_put_integer(Dst, Args, Size, Flags, ConstInfo, SrcUnsafe, + TrueLblName, FalseLblName, SystemLimitLblName) -> + case is_illegal_const(Size) of + true -> + [hipe_rtl:mk_goto(FalseLblName)]; + false -> + Aligned = aligned(Flags), + LittleEndian = littleendian(Flags), + [NewOffset] = get_real(Dst), + case ConstInfo of + fail -> + [hipe_rtl:mk_goto(FalseLblName)]; + _ -> + case Args of + [Src, Base, Offset] -> + CCode = static_int_c_code(NewOffset, Src, Base, Offset, Size, + Flags, TrueLblName, FalseLblName), + put_static_int(NewOffset, Src, Base, Offset, Size, CCode, Aligned, + LittleEndian, SrcUnsafe, TrueLblName); + [Src, Bits, Base, Offset] -> + {SizeCode, SizeReg} = + hipe_rtl_binary:make_size(Size, Bits, SystemLimitLblName, + FalseLblName), + CCode = int_c_code(NewOffset, Src, Base, Offset, SizeReg, Flags, + TrueLblName, FalseLblName), + InCode = put_dynamic_int(NewOffset, Src, Base, Offset, SizeReg, + CCode, Aligned, LittleEndian, SrcUnsafe, + TrueLblName), + SizeCode ++ InCode + end + end + end. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %% Code that is used in the append and init writeable functions @@ -807,28 +772,8 @@ put_float(_NewOffset, _Src, _Base, _Offset, _Size, CCode, _Aligned, CCode. put_static_int(NewOffset, Src, Base, Offset, Size, CCode, Aligned, - LittleEndian, TrueLblName) -> - {Init, End, UntaggedSrc} = make_init_end(Src, CCode, TrueLblName), - case {Aligned, LittleEndian} of - {true, true} -> - Init ++ - copy_int_little(Base, Offset, NewOffset, Size, UntaggedSrc) ++ - End; - {true, false} -> - Init ++ - copy_int_big(Base, Offset, NewOffset, Size, UntaggedSrc) ++ - End; - {false, true} -> - CCode; - {false, false} -> - Init ++ - copy_offset_int_big(Base, Offset, NewOffset, Size, UntaggedSrc) ++ - End - end. - -put_unsafe_static_int(NewOffset, Src, Base, Offset, Size, CCode, Aligned, - LittleEndian, TrueLblName) -> - {Init, End, UntaggedSrc} = make_init_end(Src, TrueLblName), + LittleEndian, SrcUnsafe, TrueLblName) -> + {Init, End, UntaggedSrc} = make_init_end(Src, CCode, SrcUnsafe, TrueLblName), case {Aligned, LittleEndian} of {true, true} -> Init ++ @@ -847,27 +792,8 @@ put_unsafe_static_int(NewOffset, Src, Base, Offset, Size, CCode, Aligned, end. put_dynamic_int(NewOffset, Src, Base, Offset, SizeReg, CCode, Aligned, - LittleEndian, TrueLblName) -> - {Init, End, UntaggedSrc} = make_init_end(Src, CCode, TrueLblName), - case Aligned of - true -> - case LittleEndian of - true -> - Init ++ - copy_int_little(Base, Offset, NewOffset, SizeReg, UntaggedSrc) ++ - End; - false -> - Init ++ - copy_int_big(Base, Offset, NewOffset, SizeReg, UntaggedSrc) ++ - End - end; - false -> - CCode - end. - -put_unsafe_dynamic_int(NewOffset, Src, Base, Offset, SizeReg, CCode, Aligned, - LittleEndian, TrueLblName) -> - {Init, End, UntaggedSrc} = make_init_end(Src, TrueLblName), + LittleEndian, SrcUnsafe, TrueLblName) -> + {Init, End, UntaggedSrc} = make_init_end(Src, CCode, SrcUnsafe, TrueLblName), case Aligned of true -> case LittleEndian of @@ -884,14 +810,13 @@ put_unsafe_dynamic_int(NewOffset, Src, Base, Offset, SizeReg, CCode, Aligned, CCode end. - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %% Help functions used by the above %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -make_init_end(Src, CCode, TrueLblName) -> +make_init_end(Src, CCode, false, TrueLblName) -> [CLbl, SuccessLbl] = create_lbls(2), [UntaggedSrc] = create_regs(1), Init = [hipe_tagscheme:test_fixnum(Src, hipe_rtl:label_name(SuccessLbl), @@ -899,9 +824,8 @@ make_init_end(Src, CCode, TrueLblName) -> SuccessLbl, hipe_tagscheme:untag_fixnum(UntaggedSrc,Src)], End = [hipe_rtl:mk_goto(TrueLblName), CLbl| CCode], - {Init, End, UntaggedSrc}. - -make_init_end(Src, TrueLblName) -> + {Init, End, UntaggedSrc}; +make_init_end(Src, _CCode, true, TrueLblName) -> [UntaggedSrc] = create_regs(1), Init = [hipe_tagscheme:untag_fixnum(UntaggedSrc,Src)], End = [hipe_rtl:mk_goto(TrueLblName)], diff --git a/lib/hipe/rtl/hipe_tagscheme.erl b/lib/hipe/rtl/hipe_tagscheme.erl index 35d1e7c8a4..68cbe75e85 100644 --- a/lib/hipe/rtl/hipe_tagscheme.erl +++ b/lib/hipe/rtl/hipe_tagscheme.erl @@ -40,6 +40,7 @@ fixnum_gt/5, fixnum_lt/5, fixnum_ge/5, fixnum_le/5, fixnum_val/1, fixnum_mul/4, fixnum_addsub/5, fixnum_andorxor/4, fixnum_not/2, fixnum_bsr/3, fixnum_bsl/3]). +-export([test_either_immed/4]). -export([unsafe_car/2, unsafe_cdr/2, unsafe_constant_element/3, unsafe_update_element/3, element/6]). -export([unsafe_closure_element/3]). @@ -363,14 +364,17 @@ test_matchstate(X, TrueLab, FalseLab, Pred) -> mask_and_compare(Tmp, ?TAG_HEADER_MASK, ?TAG_HEADER_BIN_MATCHSTATE, TrueLab, FalseLab, Pred)]. +test_bitstr_header(HdrTmp, TrueLab, FalseLab, Pred) -> + Mask = ?TAG_HEADER_MASK - ?BINARY_XXX_MASK, + mask_and_compare(HdrTmp, Mask, ?TAG_HEADER_REFC_BIN, TrueLab, FalseLab, Pred). + test_bitstr(X, TrueLab, FalseLab, Pred) -> Tmp = hipe_rtl:mk_new_reg_gcsafe(), HalfTrueLab = hipe_rtl:mk_new_label(), - Mask = ?TAG_HEADER_MASK - ?BINARY_XXX_MASK, [test_is_boxed(X, hipe_rtl:label_name(HalfTrueLab), FalseLab, Pred), HalfTrueLab, get_header(Tmp, X), - mask_and_compare(Tmp, Mask, ?TAG_HEADER_REFC_BIN, TrueLab, FalseLab, Pred)]. + test_bitstr_header(Tmp, TrueLab, FalseLab, Pred)]. test_binary(X, TrueLab, FalseLab, Pred) -> Tmp1 = hipe_rtl:mk_new_reg_gcsafe(), @@ -378,12 +382,10 @@ test_binary(X, TrueLab, FalseLab, Pred) -> IsBoxedLab = hipe_rtl:mk_new_label(), IsBitStrLab = hipe_rtl:mk_new_label(), IsSubBinLab = hipe_rtl:mk_new_label(), - Mask = ?TAG_HEADER_MASK - ?BINARY_XXX_MASK, [test_is_boxed(X, hipe_rtl:label_name(IsBoxedLab), FalseLab, Pred), IsBoxedLab, get_header(Tmp1, X), - mask_and_compare(Tmp1, Mask, ?TAG_HEADER_REFC_BIN, - hipe_rtl:label_name(IsBitStrLab), FalseLab, Pred), + test_bitstr_header(Tmp1, hipe_rtl:label_name(IsBitStrLab), FalseLab, Pred), IsBitStrLab, mask_and_compare(Tmp1, ?TAG_HEADER_MASK, ?TAG_HEADER_SUB_BIN, hipe_rtl:label_name(IsSubBinLab), TrueLab, 0.5), @@ -453,6 +455,10 @@ test_fixnums_1([Arg1, Arg2|Args], Acc) -> Tmp = hipe_rtl:mk_new_reg_gcsafe(), test_fixnums_1([Tmp|Args], [hipe_rtl:mk_alu(Tmp, Arg1, 'and', Arg2)|Acc]). +test_two_fixnums(Arg, Arg, FalseLab) -> + TrueLab = hipe_rtl:mk_new_label(), + [test_fixnum(Arg, hipe_rtl:label_name(TrueLab), FalseLab, 0.99), + TrueLab]; test_two_fixnums(Arg1, Arg2, FalseLab) -> TrueLab = hipe_rtl:mk_new_label(), case hipe_rtl:is_imm(Arg1) orelse hipe_rtl:is_imm(Arg2) of @@ -567,8 +573,8 @@ fixnum_andorxor(AluOp, Arg1, Arg2, Res) -> case AluOp of 'xor' -> Tmp = hipe_rtl:mk_new_reg_gcsafe(), - [hipe_rtl:mk_alu(Tmp, Arg1, 'xor', Arg2), % clears tag :-( - hipe_rtl:mk_alu(Res, Tmp, 'or', hipe_rtl:mk_imm(?TAG_IMMED1_SMALL))]; + [hipe_rtl:mk_alu(Tmp, Arg1, 'sub', hipe_rtl:mk_imm(?TAG_IMMED1_SMALL)), + hipe_rtl:mk_alu(Res, Tmp, 'xor', Arg2)]; _ -> hipe_rtl:mk_alu(Res, Arg1, AluOp, Arg2) end. @@ -595,6 +601,21 @@ fixnum_bsl(Arg1, Arg2, Res) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Test if either of two values are immediate (primary tag IMMED1, 0x3) +test_either_immed(Arg1, Arg2, TrueLab, FalseLab) -> + %% This test assumes primary tag 0x0 is reserved and immed has tag 0x3 + 16#0 = ?TAG_PRIMARY_HEADER, + 16#3 = ?TAG_PRIMARY_IMMED1, + Tmp1 = hipe_rtl:mk_new_reg_gcsafe(), + Tmp2 = hipe_rtl:mk_new_reg_gcsafe(), + [hipe_rtl:mk_alu(Tmp1, Arg1, 'sub', hipe_rtl:mk_imm(1)), + hipe_rtl:mk_alu(Tmp2, Arg2, 'sub', hipe_rtl:mk_imm(1)), + hipe_rtl:mk_alu(Tmp2, Tmp2, 'or', Tmp1), + hipe_rtl:mk_branch(Tmp2, 'and', hipe_rtl:mk_imm(2), eq, + FalseLab, TrueLab, 0.01)]. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + unsafe_car(Dst, Arg) -> hipe_rtl:mk_load(Dst, Arg, hipe_rtl:mk_imm(-(?TAG_PRIMARY_LIST))). @@ -631,14 +652,13 @@ unsafe_update_element(Tuple, Index, Value) -> % Index is an immediate element(Dst, Index, Tuple, FailLabName, {tuple, A}, IndexInfo) -> FixnumOkLab = hipe_rtl:mk_new_label(), IndexOkLab = hipe_rtl:mk_new_label(), - Ptr = hipe_rtl:mk_new_reg(), % offset from Tuple UIndex = hipe_rtl:mk_new_reg_gcsafe(), Arity = hipe_rtl:mk_imm(A), - InvIndex = hipe_rtl:mk_new_reg_gcsafe(), - Offset = hipe_rtl:mk_new_reg_gcsafe(), case IndexInfo of valid -> %% This is no branch, 1 load and 3 alus = 4 instr + Offset = hipe_rtl:mk_new_reg_gcsafe(), + Ptr = hipe_rtl:mk_new_reg(), % offset from Tuple [untag_fixnum(UIndex, Index), hipe_rtl:mk_alu(Ptr, Tuple, 'sub', hipe_rtl:mk_imm(?TAG_PRIMARY_BOXED)), hipe_rtl:mk_alu(Offset, UIndex, 'sll', @@ -647,72 +667,56 @@ element(Dst, Index, Tuple, FailLabName, {tuple, A}, IndexInfo) -> fixnums -> %% This is 1 branch, 1 load and 4 alus = 6 instr [untag_fixnum(UIndex, Index), - hipe_rtl:mk_alu(Ptr, Tuple, 'sub',hipe_rtl:mk_imm(?TAG_PRIMARY_BOXED))| - gen_element_tail(Dst, Ptr, InvIndex, Arity, Offset, UIndex, - FailLabName, IndexOkLab)]; + gen_element_tail(Dst, Tuple, Arity, UIndex, FailLabName, IndexOkLab)]; _ -> %% This is 3 branches, 1 load and 5 alus = 9 instr [test_fixnum(Index, hipe_rtl:label_name(FixnumOkLab), FailLabName, 0.99), FixnumOkLab, untag_fixnum(UIndex, Index), - hipe_rtl:mk_alu(Ptr, Tuple, 'sub',hipe_rtl:mk_imm(?TAG_PRIMARY_BOXED))| - gen_element_tail(Dst, Ptr, InvIndex, Arity, Offset, UIndex, - FailLabName, IndexOkLab)] + gen_element_tail(Dst, Tuple, Arity, UIndex, FailLabName, IndexOkLab)] end; element(Dst, Index, Tuple, FailLabName, tuple, IndexInfo) -> FixnumOkLab = hipe_rtl:mk_new_label(), IndexOkLab = hipe_rtl:mk_new_label(), - Ptr = hipe_rtl:mk_new_reg(), % offset from Tuple Header = hipe_rtl:mk_new_reg_gcsafe(), UIndex = hipe_rtl:mk_new_reg_gcsafe(), Arity = hipe_rtl:mk_new_reg_gcsafe(), - InvIndex = hipe_rtl:mk_new_reg_gcsafe(), - Offset = hipe_rtl:mk_new_reg_gcsafe(), case IndexInfo of fixnums -> %% This is 1 branch, 2 loads and 5 alus = 8 instr - [hipe_rtl:mk_alu(Ptr, Tuple, 'sub', hipe_rtl:mk_imm(?TAG_PRIMARY_BOXED)), - hipe_rtl:mk_load(Header, Ptr, hipe_rtl:mk_imm(0)), + [get_header(Header, Tuple), untag_fixnum(UIndex, Index), hipe_rtl:mk_alu(Arity,Header,'srl',hipe_rtl:mk_imm(?HEADER_ARITY_OFFS))| - gen_element_tail(Dst, Ptr, InvIndex, Arity, Offset, UIndex, - FailLabName, IndexOkLab)]; + gen_element_tail(Dst, Tuple, Arity, UIndex, FailLabName, IndexOkLab)]; Num when is_integer(Num) -> %% This is 1 branch, 1 load and 3 alus = 5 instr - [hipe_rtl:mk_alu(Ptr, Tuple, 'sub', hipe_rtl:mk_imm(?TAG_PRIMARY_BOXED))| - gen_element_tail(Dst, Ptr, InvIndex, hipe_rtl:mk_imm(Num), - Offset, UIndex, FailLabName, IndexOkLab)]; + gen_element_tail(Dst, Tuple, hipe_rtl:mk_imm(Num), UIndex, FailLabName, + IndexOkLab); _ -> %% This is 2 branches, 2 loads and 6 alus = 10 instr [test_fixnum(Index, hipe_rtl:label_name(FixnumOkLab), FailLabName, 0.99), FixnumOkLab, - hipe_rtl:mk_alu(Ptr, Tuple, 'sub', hipe_rtl:mk_imm(?TAG_PRIMARY_BOXED)), - hipe_rtl:mk_load(Header, Ptr, hipe_rtl:mk_imm(0)), + get_header(Header, Tuple), untag_fixnum(UIndex, Index), hipe_rtl:mk_alu(Arity,Header,'srl',hipe_rtl:mk_imm(?HEADER_ARITY_OFFS))| - gen_element_tail(Dst, Ptr, InvIndex, Arity, Offset, UIndex, - FailLabName, IndexOkLab)] + gen_element_tail(Dst, Tuple, Arity, UIndex, FailLabName, IndexOkLab)] end; element(Dst, Index, Tuple, FailLabName, unknown, IndexInfo) -> FixnumOkLab = hipe_rtl:mk_new_label(), BoxedOkLab = hipe_rtl:mk_new_label(), TupleOkLab = hipe_rtl:mk_new_label(), IndexOkLab = hipe_rtl:mk_new_label(), - Ptr = hipe_rtl:mk_new_reg(), % offset from Tuple Header = hipe_rtl:mk_new_reg_gcsafe(), UIndex = hipe_rtl:mk_new_reg_gcsafe(), Arity = hipe_rtl:mk_new_reg_gcsafe(), - InvIndex = hipe_rtl:mk_new_reg_gcsafe(), - Offset = hipe_rtl:mk_new_reg_gcsafe(), case IndexInfo of fixnums -> %% This is 3 branches, 2 loads and 5 alus = 10 instr [test_is_boxed(Tuple, hipe_rtl:label_name(BoxedOkLab), FailLabName, 0.99), BoxedOkLab, - hipe_rtl:mk_alu(Ptr, Tuple, 'sub', hipe_rtl:mk_imm(?TAG_PRIMARY_BOXED)), - hipe_rtl:mk_load(Header, Ptr, hipe_rtl:mk_imm(0)), + get_header(Header, Tuple), hipe_rtl:mk_branch(Header, 'and', hipe_rtl:mk_imm(?TAG_HEADER_MASK), 'eq', hipe_rtl:label_name(TupleOkLab), FailLabName, 0.99), @@ -720,23 +724,21 @@ element(Dst, Index, Tuple, FailLabName, unknown, IndexInfo) -> untag_fixnum(UIndex, Index), hipe_rtl:mk_alu(Arity, Header, 'srl', hipe_rtl:mk_imm(?HEADER_ARITY_OFFS))| - gen_element_tail(Dst, Ptr, InvIndex, Arity, Offset, - UIndex, FailLabName, IndexOkLab)]; + gen_element_tail(Dst, Tuple, Arity, UIndex, FailLabName, IndexOkLab)]; Num when is_integer(Num) -> %% This is 3 branches, 2 loads and 4 alus = 9 instr [test_is_boxed(Tuple, hipe_rtl:label_name(BoxedOkLab), FailLabName, 0.99), BoxedOkLab, - hipe_rtl:mk_alu(Ptr, Tuple, 'sub', hipe_rtl:mk_imm(?TAG_PRIMARY_BOXED)), - hipe_rtl:mk_load(Header, Ptr, hipe_rtl:mk_imm(0)), + get_header(Header, Tuple), hipe_rtl:mk_branch(Header, 'and', hipe_rtl:mk_imm(?TAG_HEADER_MASK), 'eq', hipe_rtl:label_name(TupleOkLab), FailLabName, 0.99), TupleOkLab, hipe_rtl:mk_alu(Arity, Header, 'srl', hipe_rtl:mk_imm(?HEADER_ARITY_OFFS))| - gen_element_tail(Dst, Ptr, InvIndex, Arity, Offset, - hipe_rtl:mk_imm(Num), FailLabName, IndexOkLab)]; + gen_element_tail(Dst, Tuple, Arity, hipe_rtl:mk_imm(Num), FailLabName, + IndexOkLab)]; _ -> %% This is 4 branches, 2 loads, and 6 alus = 12 instr :( [test_fixnum(Index, hipe_rtl:label_name(FixnumOkLab), @@ -745,8 +747,7 @@ element(Dst, Index, Tuple, FailLabName, unknown, IndexInfo) -> test_is_boxed(Tuple, hipe_rtl:label_name(BoxedOkLab), FailLabName, 0.99), BoxedOkLab, - hipe_rtl:mk_alu(Ptr, Tuple, 'sub', hipe_rtl:mk_imm(?TAG_PRIMARY_BOXED)), - hipe_rtl:mk_load(Header, Ptr, hipe_rtl:mk_imm(0)), + get_header(Header, Tuple), hipe_rtl:mk_branch(Header, 'and', hipe_rtl:mk_imm(?TAG_HEADER_MASK), 'eq', hipe_rtl:label_name(TupleOkLab), FailLabName, 0.99), @@ -754,20 +755,21 @@ element(Dst, Index, Tuple, FailLabName, unknown, IndexInfo) -> untag_fixnum(UIndex, Index), hipe_rtl:mk_alu(Arity, Header, 'srl', hipe_rtl:mk_imm(?HEADER_ARITY_OFFS))| - gen_element_tail(Dst, Ptr, InvIndex, Arity, Offset, - UIndex, FailLabName, IndexOkLab)] + gen_element_tail(Dst, Tuple, Arity, UIndex, FailLabName, IndexOkLab)] end. -gen_element_tail(Dst, Ptr, InvIndex, Arity, Offset, - UIndex, FailLabName, IndexOkLab) -> +gen_element_tail(Dst, Tuple, Arity, UIndex, FailLabName, IndexOkLab) -> + ZeroIndex = hipe_rtl:mk_new_reg_gcsafe(), + Offset = hipe_rtl:mk_new_reg_gcsafe(), + Ptr = hipe_rtl:mk_new_reg(), % offset from Tuple %% now check that 1 <= UIndex <= Arity - %% if UIndex < 1, then (Arity - UIndex) >= Arity - %% if UIndex > Arity, then (Arity - UIndex) < 0, which is >=u Arity - %% otherwise, 0 <= (Arity - UIndex) < Arity - [hipe_rtl:mk_alu(InvIndex, Arity, 'sub', UIndex), - hipe_rtl:mk_branch(InvIndex, 'geu', Arity, FailLabName, + %% by checking the equivalent (except for when Arity>=2^(WordSize-1)) + %% (UIndex - 1) <u Arity + [hipe_rtl:mk_alu(ZeroIndex, UIndex, 'sub', hipe_rtl:mk_imm(1)), + hipe_rtl:mk_branch(ZeroIndex, 'geu', Arity, FailLabName, hipe_rtl:label_name(IndexOkLab), 0.01), IndexOkLab, + hipe_rtl:mk_alu(Ptr, Tuple, 'sub', hipe_rtl:mk_imm(?TAG_PRIMARY_BOXED)), hipe_rtl:mk_alu(Offset, UIndex, 'sll', hipe_rtl:mk_imm(hipe_rtl_arch:log2_word_size())), hipe_rtl:mk_load(Dst, Ptr, Offset)]. diff --git a/lib/hipe/sparc/hipe_sparc.erl b/lib/hipe/sparc/hipe_sparc.erl index 916857b224..22e0761b69 100644 --- a/lib/hipe/sparc/hipe_sparc.erl +++ b/lib/hipe/sparc/hipe_sparc.erl @@ -87,6 +87,9 @@ mk_pseudo_set/2, + mk_pseudo_spill_move/3, + is_pseudo_spill_move/1, + mk_pseudo_tailcall/4, pseudo_tailcall_funv/1, pseudo_tailcall_linkage/1, @@ -117,6 +120,9 @@ pseudo_fmove_src/1, pseudo_fmove_dst/1, + mk_pseudo_spill_fmove/3, + is_pseudo_spill_fmove/1, + mk_pseudo_fstore/3, mk_fstore/4, @@ -269,6 +275,10 @@ mk_pseudo_ret() -> #pseudo_ret{}. mk_pseudo_set(Imm, Dst) -> #pseudo_set{imm=Imm, dst=Dst}. +mk_pseudo_spill_move(Src, Temp, Dst) -> + #pseudo_spill_move{src=Src, temp=Temp, dst=Dst}. +is_pseudo_spill_move(I) -> is_record(I, pseudo_spill_move). + mk_pseudo_tailcall(FunV, Arity, StkArgs, Linkage) -> #pseudo_tailcall{funv=FunV, arity=Arity, stkargs=StkArgs, linkage=Linkage}. pseudo_tailcall_funv(#pseudo_tailcall{funv=FunV}) -> FunV. @@ -375,6 +385,10 @@ is_pseudo_fmove(I) -> case I of #pseudo_fmove{} -> true; _ -> false end. pseudo_fmove_src(#pseudo_fmove{src=Src}) -> Src. pseudo_fmove_dst(#pseudo_fmove{dst=Dst}) -> Dst. +mk_pseudo_spill_fmove(Src, Temp, Dst) -> + #pseudo_spill_fmove{src=Src, temp=Temp, dst=Dst}. +is_pseudo_spill_fmove(I) -> is_record(I, pseudo_spill_fmove). + mk_pseudo_fstore(Src, Base, Disp) -> #pseudo_fstore{src=Src, base=Base, disp=Disp}. diff --git a/lib/hipe/sparc/hipe_sparc.hrl b/lib/hipe/sparc/hipe_sparc.hrl index 4eae6777a9..f60e516e59 100644 --- a/lib/hipe/sparc/hipe_sparc.hrl +++ b/lib/hipe/sparc/hipe_sparc.hrl @@ -88,6 +88,8 @@ -record(pseudo_move, {src, dst}). -record(pseudo_ret, {}). -record(pseudo_set, {imm, dst}). +-record(pseudo_spill_fmove, {src, temp, dst}). +-record(pseudo_spill_move, {src, temp, dst}). -record(pseudo_tailcall, {funv, arity, stkargs, linkage}). -record(pseudo_tailcall_prepare, {}). -record(rdy, {dst}). diff --git a/lib/hipe/sparc/hipe_sparc_assemble.erl b/lib/hipe/sparc/hipe_sparc_assemble.erl index 08bd47c4d2..2b82f41d23 100644 --- a/lib/hipe/sparc/hipe_sparc_assemble.erl +++ b/lib/hipe/sparc/hipe_sparc_assemble.erl @@ -32,7 +32,7 @@ assemble(CompiledCode, Closures, Exports, Options) -> || {MFA, Defun} <- CompiledCode], %% {ConstAlign,ConstSize,ConstMap,RefsFromConsts} = - hipe_pack_constants:pack_constants(Code, 4), + hipe_pack_constants:pack_constants(Code), %% {CodeSize,CodeBinary,AccRefs,LabelMap,ExportMap} = encode(translate(Code, ConstMap), Options), diff --git a/lib/hipe/sparc/hipe_sparc_cfg.erl b/lib/hipe/sparc/hipe_sparc_cfg.erl index 27374d187b..45c8e887b5 100644 --- a/lib/hipe/sparc/hipe_sparc_cfg.erl +++ b/lib/hipe/sparc/hipe_sparc_cfg.erl @@ -23,6 +23,7 @@ -export([linearise/1]). -export([params/1]). -export([arity/1]). % for linear scan +-export([redirect_jmp/3, branch_preds/1]). -define(SPARC_CFG, true). % needed for cfg.inc @@ -77,28 +78,53 @@ branch_successors(Branch) -> #pseudo_tailcall{} -> [] end. +branch_preds(Branch) -> + case Branch of + #jmp{labels=Labels} -> + Prob = 1.0/length(Labels), + [{L, Prob} || L <- Labels]; + #pseudo_bp{true_label=TrueLab,false_label=FalseLab,pred=Pred} -> + [{FalseLab, 1.0-Pred}, {TrueLab, Pred}]; + #pseudo_call{contlab=ContLab, sdesc=#sparc_sdesc{exnlab=[]}} -> + %% A function can still cause an exception, even if we won't catch it + [{ContLab, 1.0-hipe_bb_weights:call_exn_pred()}]; + #pseudo_call{contlab=ContLab, sdesc=#sparc_sdesc{exnlab=ExnLab}} -> + CallExnPred = hipe_bb_weights:call_exn_pred(), + [{ContLab, 1.0-CallExnPred}, {ExnLab, CallExnPred}]; + _ -> + case branch_successors(Branch) of + [] -> []; + [Single] -> [{Single, 1.0}] + end + end. + -ifdef(REMOVE_TRIVIAL_BBS_NEEDED). fails_to(_Instr) -> []. -endif. --ifdef(notdef). redirect_jmp(I, Old, New) -> case I of - #b_label{label=Label} -> - if Old =:= Label -> I#b_label{label=New}; + #bp{'cond'='a',label=Label} -> + if Old =:= Label -> I#bp{label=New}; true -> I end; - #pseudo_bc{true_label=TrueLab, false_label=FalseLab} -> - I1 = if Old =:= TrueLab -> I#pseudo_bc{true_label=New}; + #pseudo_bp{true_label=TrueLab, false_label=FalseLab} -> + I1 = if Old =:= TrueLab -> I#pseudo_bp{true_label=New}; true -> I end, - if Old =:= FalseLab -> I1#pseudo_bc{false_label=New}; + if Old =:= FalseLab -> I1#pseudo_bp{false_label=New}; true -> I1 end; - %% handle pseudo_call too? - _ -> I + #pseudo_call{contlab=ContLab0, sdesc=SDesc0} -> + SDesc = case SDesc0 of + #sparc_sdesc{exnlab=Old} -> SDesc0#sparc_sdesc{exnlab=New}; + #sparc_sdesc{exnlab=_} -> SDesc0 + end, + ContLab = if Old =:= ContLab0 -> New; + true -> ContLab0 + end, + I#pseudo_call{sdesc=SDesc, contlab=ContLab} end. --endif. mk_goto(Label) -> hipe_sparc:mk_b_label(Label). diff --git a/lib/hipe/sparc/hipe_sparc_defuse.erl b/lib/hipe/sparc/hipe_sparc_defuse.erl index cb75f82e2b..4d4b11e301 100644 --- a/lib/hipe/sparc/hipe_sparc_defuse.erl +++ b/lib/hipe/sparc/hipe_sparc_defuse.erl @@ -39,6 +39,7 @@ insn_def_gpr(I) -> #pseudo_call{} -> call_clobbered_gpr(); #pseudo_move{dst=Dst} -> [Dst]; #pseudo_set{dst=Dst} -> [Dst]; + #pseudo_spill_move{temp=Temp, dst=Dst} -> [Temp, Dst]; #pseudo_tailcall_prepare{} -> tailcall_clobbered_gpr(); #rdy{dst=Dst} -> [Dst]; #sethi{dst=Dst} -> [Dst]; @@ -72,6 +73,7 @@ insn_use_gpr(I) -> funv_use(FunV, arity_use_gpr(Arity)); #pseudo_move{src=Src} -> [Src]; #pseudo_ret{} -> [hipe_sparc:mk_rv()]; + #pseudo_spill_move{src=Src} -> [Src]; #pseudo_tailcall{funv=FunV,arity=Arity,stkargs=StkArgs} -> addsrcs(StkArgs, addtemps(tailcall_clobbered_gpr(), funv_use(FunV, arity_use_gpr(Arity)))); #store{src=Src,base=Base,disp=Disp} -> @@ -112,6 +114,7 @@ insn_def_fpr(I) -> #fp_unary{dst=Dst} -> [Dst]; #pseudo_fload{dst=Dst} -> [Dst]; #pseudo_fmove{dst=Dst} -> [Dst]; + #pseudo_spill_fmove{temp=Temp, dst=Dst} -> [Temp, Dst]; _ -> [] end. @@ -130,6 +133,7 @@ insn_use_fpr(I) -> #fp_unary{src=Src} -> [Src]; #pseudo_fmove{src=Src} -> [Src]; #pseudo_fstore{src=Src} -> [Src]; + #pseudo_spill_fmove{src=Src} -> [Src]; _ -> [] end. diff --git a/lib/hipe/sparc/hipe_sparc_frame.erl b/lib/hipe/sparc/hipe_sparc_frame.erl index 6f29c3c905..1f2a259ca1 100644 --- a/lib/hipe/sparc/hipe_sparc_frame.erl +++ b/lib/hipe/sparc/hipe_sparc_frame.erl @@ -82,6 +82,10 @@ do_insn(I, LiveOut, Context, FPoff) -> {do_pseudo_tailcall(I, Context), context_framesize(Context)}; #pseudo_fmove{} -> {do_pseudo_fmove(I, Context, FPoff), FPoff}; + #pseudo_spill_move{} -> + {do_pseudo_spill_move(I, Context, FPoff), FPoff}; + #pseudo_spill_fmove{} -> + {do_pseudo_spill_fmove(I, Context, FPoff), FPoff}; _ -> {[I], FPoff} end. @@ -110,6 +114,22 @@ do_pseudo_move(I, Context, FPoff) -> end end. +do_pseudo_spill_move(I, Context, FPoff) -> + #pseudo_spill_move{src=Src,temp=Temp,dst=Dst} = I, + case temp_is_pseudo(Src) andalso temp_is_pseudo(Dst) of + false -> % Register allocator changed its mind, turn back to move + do_pseudo_move(hipe_sparc:mk_pseudo_move(Src, Dst), Context, FPoff); + true -> + SrcOffset = pseudo_offset(Src, FPoff, Context), + DstOffset = pseudo_offset(Dst, FPoff, Context), + case SrcOffset =:= DstOffset of + true -> []; % omit move-to-self + false -> + mk_load(hipe_sparc:mk_sp(), SrcOffset, Temp, + mk_store(Temp, hipe_sparc:mk_sp(), DstOffset, [])) + end + end. + do_pseudo_fmove(I, Context, FPoff) -> Dst = hipe_sparc:pseudo_fmove_dst(I), Src = hipe_sparc:pseudo_fmove_src(I), @@ -127,6 +147,22 @@ do_pseudo_fmove(I, Context, FPoff) -> end end. +do_pseudo_spill_fmove(I, Context, FPoff) -> + #pseudo_spill_fmove{src=Src,temp=Temp,dst=Dst} = I, + case temp_is_pseudo(Src) andalso temp_is_pseudo(Dst) of + false -> % Register allocator changed its mind, turn back to fmove + do_pseudo_fmove(hipe_sparc:mk_pseudo_fmove(Src, Dst), Context, FPoff); + true -> + SrcOffset = pseudo_offset(Src, FPoff, Context), + DstOffset = pseudo_offset(Dst, FPoff, Context), + case SrcOffset =:= DstOffset of + true -> []; % omit move-to-self + false -> + mk_fload(hipe_sparc:mk_sp(), SrcOffset, Temp) + ++ mk_fstore(Temp, hipe_sparc:mk_sp(), DstOffset) + end + end. + pseudo_offset(Temp, FPoff, Context) -> FPoff + context_offset(Context, Temp). diff --git a/lib/hipe/sparc/hipe_sparc_ra_finalise.erl b/lib/hipe/sparc/hipe_sparc_ra_finalise.erl index 5fdb73e197..a724821992 100644 --- a/lib/hipe/sparc/hipe_sparc_ra_finalise.erl +++ b/lib/hipe/sparc/hipe_sparc_ra_finalise.erl @@ -38,6 +38,7 @@ ra_insn(I, Map, FPMap) -> #pseudo_call{} -> ra_pseudo_call(I, Map); #pseudo_move{} -> ra_pseudo_move(I, Map); #pseudo_set{} -> ra_pseudo_set(I, Map); + #pseudo_spill_move{} -> ra_pseudo_spill_move(I, Map); #pseudo_tailcall{} -> ra_pseudo_tailcall(I, Map); #rdy{} -> ra_rdy(I, Map); #sethi{} -> ra_sethi(I, Map); @@ -47,6 +48,7 @@ ra_insn(I, Map, FPMap) -> #pseudo_fload{} -> ra_pseudo_fload(I, Map, FPMap); #pseudo_fmove{} -> ra_pseudo_fmove(I, FPMap); #pseudo_fstore{} -> ra_pseudo_fstore(I, Map, FPMap); + #pseudo_spill_fmove{} -> ra_pseudo_spill_fmove(I, FPMap); _ -> I end. @@ -80,6 +82,12 @@ ra_pseudo_set(I=#pseudo_set{dst=Dst}, Map) -> NewDst = ra_temp(Dst, Map), I#pseudo_set{dst=NewDst}. +ra_pseudo_spill_move(I=#pseudo_spill_move{src=Src,temp=Temp,dst=Dst}, Map) -> + NewSrc = ra_temp(Src, Map), + NewTemp = ra_temp(Temp, Map), + NewDst = ra_temp(Dst, Map), + I#pseudo_spill_move{src=NewSrc,temp=NewTemp,dst=NewDst}. + ra_pseudo_tailcall(I=#pseudo_tailcall{funv=FunV,stkargs=StkArgs}, Map) -> NewFunV = ra_funv(FunV, Map), NewStkArgs = ra_args(StkArgs, Map), @@ -120,6 +128,13 @@ ra_pseudo_fmove(I=#pseudo_fmove{src=Src,dst=Dst}, FPMap) -> NewDst = ra_temp_fp(Dst, FPMap), I#pseudo_fmove{src=NewSrc,dst=NewDst}. +ra_pseudo_spill_fmove(I=#pseudo_spill_fmove{src=Src,temp=Temp,dst=Dst}, + FPMap) -> + NewSrc = ra_temp_fp(Src, FPMap), + NewTemp = ra_temp_fp(Temp, FPMap), + NewDst = ra_temp_fp(Dst, FPMap), + I#pseudo_spill_fmove{src=NewSrc,temp=NewTemp,dst=NewDst}. + ra_pseudo_fstore(I=#pseudo_fstore{src=Src,base=Base}, Map, FPMap) -> NewSrc = ra_temp_fp(Src, FPMap), NewBase = ra_temp(Base, Map), diff --git a/lib/hipe/sparc/hipe_sparc_ra_postconditions.erl b/lib/hipe/sparc/hipe_sparc_ra_postconditions.erl index 984c97fbd4..d3ecb43ec6 100644 --- a/lib/hipe/sparc/hipe_sparc_ra_postconditions.erl +++ b/lib/hipe/sparc/hipe_sparc_ra_postconditions.erl @@ -54,6 +54,7 @@ do_insn(I, TempMap, Strategy) -> #pseudo_call{} -> do_pseudo_call(I, TempMap, Strategy); #pseudo_move{} -> do_pseudo_move(I, TempMap, Strategy); #pseudo_set{} -> do_pseudo_set(I, TempMap, Strategy); + #pseudo_spill_move{} -> do_pseudo_spill_move(I, TempMap, Strategy); #pseudo_tailcall{} -> do_pseudo_tailcall(I, TempMap, Strategy); #rdy{} -> do_rdy(I, TempMap, Strategy); #sethi{} -> do_sethi(I, TempMap, Strategy); @@ -92,14 +93,16 @@ do_pseudo_call(I=#pseudo_call{funv=FunV}, TempMap, Strategy) -> do_pseudo_move(I=#pseudo_move{src=Src,dst=Dst}, TempMap, Strategy) -> %% Either Dst or Src (but not both) may be a pseudo temp. - %% pseudo_move is a special case: in [XXX: not pseudo_tailcall] - %% all other instructions, all temps must be non-pseudos - %% after register allocation. - case temp_is_spilled(Dst, TempMap) of - true -> % Src must not be a pseudo - {FixSrc,NewSrc,DidSpill} = fix_src1(Src, TempMap, Strategy), - NewI = I#pseudo_move{src=NewSrc}, - {FixSrc ++ [NewI], DidSpill}; + %% pseudo_move and pseudo_spill_move [XXX: not pseudo_tailcall] + %% are special cases: in all other instructions, all temps must + %% be non-pseudos after register allocation. + case temp_is_spilled(Src, TempMap) + andalso temp_is_spilled(Dst, TempMap) + of + true -> % Turn into pseudo_spill_move + Temp = clone(Src, temp1(Strategy)), + NewI = #pseudo_spill_move{src=Src,temp=Temp,dst=Dst}, + {[NewI], true}; _ -> {[I], false} end. @@ -109,6 +112,11 @@ do_pseudo_set(I=#pseudo_set{dst=Dst}, TempMap, Strategy) -> NewI = I#pseudo_set{dst=NewDst}, {[NewI | FixDst], DidSpill}. +do_pseudo_spill_move(I=#pseudo_spill_move{temp=Temp}, TempMap, _Strategy) -> + %% Temp is above the low water mark and must not have been spilled + false = temp_is_spilled(Temp, TempMap), + {[I], false}. + do_pseudo_tailcall(I=#pseudo_tailcall{funv=FunV}, TempMap, Strategy) -> {FixFunV,NewFunV,DidSpill} = fix_funv(FunV, TempMap, Strategy), NewI = I#pseudo_tailcall{funv=NewFunV}, diff --git a/lib/hipe/sparc/hipe_sparc_ra_postconditions_fp.erl b/lib/hipe/sparc/hipe_sparc_ra_postconditions_fp.erl index 751e91425c..5fa3a5fc59 100644 --- a/lib/hipe/sparc/hipe_sparc_ra_postconditions_fp.erl +++ b/lib/hipe/sparc/hipe_sparc_ra_postconditions_fp.erl @@ -43,6 +43,7 @@ do_insn(I, TempMap) -> #pseudo_fload{} -> do_pseudo_fload(I, TempMap); #pseudo_fmove{} -> do_pseudo_fmove(I, TempMap); #pseudo_fstore{} -> do_pseudo_fstore(I, TempMap); + #pseudo_spill_fmove{} -> do_pseudo_spill_fmove(I, TempMap); _ -> {[I], false} end. @@ -67,11 +68,13 @@ do_pseudo_fload(I=#pseudo_fload{dst=Dst}, TempMap) -> {[NewI | FixDst], DidSpill}. do_pseudo_fmove(I=#pseudo_fmove{src=Src,dst=Dst}, TempMap) -> - case temp_is_spilled(Dst, TempMap) of - true -> - {FixSrc,NewSrc,DidSpill} = fix_src(Src, TempMap), - NewI = I#pseudo_fmove{src=NewSrc}, - {FixSrc ++ [NewI], DidSpill}; + case temp_is_spilled(Src, TempMap) + andalso temp_is_spilled(Dst, TempMap) + of + true -> % Turn into pseudo_spill_fmove + Temp = clone(Src), + NewI = #pseudo_spill_fmove{src=Src,temp=Temp,dst=Dst}, + {[NewI], true}; _ -> {[I], false} end. @@ -81,6 +84,11 @@ do_pseudo_fstore(I=#pseudo_fstore{src=Src}, TempMap) -> NewI = I#pseudo_fstore{src=NewSrc}, {FixSrc ++ [NewI], DidSpill}. +do_pseudo_spill_fmove(I=#pseudo_spill_fmove{temp=Temp}, TempMap) -> + %% Temp is above the low water mark and must not have been spilled + false = temp_is_spilled(Temp, TempMap), + {[I], false}. + %%% Fix Dst and Src operands. fix_src(Src, TempMap) -> diff --git a/lib/hipe/sparc/hipe_sparc_subst.erl b/lib/hipe/sparc/hipe_sparc_subst.erl index 1d0671464e..ce3bbb813a 100644 --- a/lib/hipe/sparc/hipe_sparc_subst.erl +++ b/lib/hipe/sparc/hipe_sparc_subst.erl @@ -44,6 +44,8 @@ insn_temps(T, I) -> #pseudo_move{src=S,dst=D} -> I#pseudo_move{src=T(S),dst=T(D)}; #pseudo_ret{} -> I; #pseudo_set{dst=D}-> I#pseudo_set{dst=T(D)}; + #pseudo_spill_move{src=S,temp=U,dst=D} -> + I#pseudo_spill_move{src=T(S),temp=T(U),dst=T(D)}; #pseudo_tailcall{funv=F,stkargs=Stk} -> I#pseudo_tailcall{funv=funv_temps(T,F),stkargs=lists:map(Arg,Stk)}; #pseudo_tailcall_prepare{} -> I; @@ -57,7 +59,9 @@ insn_temps(T, I) -> I#pseudo_fload{base=T(B),disp=S2(Di),dst=T(Ds)}; #pseudo_fmove{src=S,dst=D} -> I#pseudo_fmove{src=T(S),dst=T(D)}; #pseudo_fstore{src=S,base=B,disp=D} -> - I#pseudo_fstore{src=T(S),base=T(B),disp=S2(D)} + I#pseudo_fstore{src=T(S),base=T(B),disp=S2(D)}; + #pseudo_spill_fmove{src=S,temp=U,dst=D} -> + I#pseudo_spill_fmove{src=T(S),temp=T(U),dst=T(D)} end. -spec src2_temps(subst_fun(), src2()) -> src2(). diff --git a/lib/hipe/test/basic_SUITE_data/basic_bugs_hipe.erl b/lib/hipe/test/basic_SUITE_data/basic_bugs_hipe.erl index caa0e71d0b..430e097b91 100644 --- a/lib/hipe/test/basic_SUITE_data/basic_bugs_hipe.erl +++ b/lib/hipe/test/basic_SUITE_data/basic_bugs_hipe.erl @@ -18,6 +18,7 @@ test() -> ok = test_R12B5_seg_fault(), ok = test_switch_neg_int(), ok = test_icode_range_anal(), + ok = test_icode_range_call(), ok. %%----------------------------------------------------------------------- @@ -461,3 +462,44 @@ g(X, Z) -> test -> non_zero_test; other -> other end. + +%%----------------------------------------------------------------------- +%% From: Rich Neswold +%% Date: Oct 5, 2016 +%% +%% The following was a bug in the HiPE compiler's range analysis. The +%% function range_client/2 below would would not stop when N reached 0, +%% but keep recursing into the second clause forever. +%% +%% The problem turned out to be in hipe_icode_range:analyse_call/2, +%% which would note update the argument ranges of the callee if the +%% result of the call was ignored. +%% ----------------------------------------------------------------------- +-define(TIMEOUT, 42). + +test_icode_range_call() -> + Self = self(), + Client = spawn_link(fun() -> range_client(Self, 4) end), + range_server(4, Client). + +range_server(0, _Client) -> + receive + stopping -> ok; + {called_with, 0} -> error(failure) + after ?TIMEOUT -> error(timeout) + end; +range_server(N, Client) -> + receive + {called_with, N} -> + Client ! proceed + after ?TIMEOUT -> error(timeout) + end, + range_server(N-1, Client). % tailcall (so the bug does not affect it) + +range_client(Server, 0) -> + Server ! stopping; +range_client(Server, N) -> + Server ! {called_with, N}, + receive proceed -> ok end, + range_client(Server, N - 1), % non-tailrecursive call with ignored result + ok. diff --git a/lib/hipe/test/basic_SUITE_data/basic_edge_cases.erl b/lib/hipe/test/basic_SUITE_data/basic_edge_cases.erl new file mode 100644 index 0000000000..9bf5cf52cd --- /dev/null +++ b/lib/hipe/test/basic_SUITE_data/basic_edge_cases.erl @@ -0,0 +1,142 @@ +%%% -*- erlang-indent-level: 2 -*- +%%%---------------------------------------------------------------------- +%%% Contains +%%%---------------------------------------------------------------------- +-module(basic_edge_cases). + +-export([test/0]). + +test() -> + ok = test_float_spills(), + ok = test_infinite_loops(), + ok. + +%% Contains more float temps live at a single point than there are float +%% registers in any backend + +test_float_spills() -> + {{{2942.0,4670.0,3198.0,4926.0,2206.0,4734.0}, + {3118.0,2062.0,5174.0,3038.0,3618.0,3014.0}, + {2542.0,2062.0,4934.0,2590.0,3098.0,3062.0}, + {2950.0,3666.0,2574.0,5038.0,1866.0,2946.0}, + {3126.0,3050.0,3054.0,5070.0,2258.0,2714.0}, + {4734.0,2206.0,4926.0,3198.0,4670.0,2942.0}}, + 58937.0} = + mat66_flip_sum(35.0,86.0,32.0,88.0,33.0,57.0, + 22.0,77.0,91.0,80.0,14.0,33.0, + 51.0,28.0,87.0,20.0,91.0,11.0, + 68.0,83.0,64.0,82.0,10.0,86.0, + 74.0,18.0,08.0,52.0,10.0,14.0, + 89.0,34.0,64.0,66.0,58.0,55.0, + 0.0, 5), + ok. + +mat66_flip_sum(M11, M12, M13, M14, M15, M16, + M21, M22, M23, M24, M25, M26, + M31, M32, M33, M34, M35, M36, + M41, M42, M43, M44, M45, M46, + M51, M52, M53, M54, M55, M56, + M61, M62, M63, M64, M65, M66, + Acc, Ctr) + when is_float(M11), is_float(M12), is_float(M13), + is_float(M14), is_float(M15), is_float(M16), + is_float(M21), is_float(M22), is_float(M23), + is_float(M24), is_float(M25), is_float(M26), + is_float(M31), is_float(M32), is_float(M33), + is_float(M34), is_float(M35), is_float(M36), + is_float(M41), is_float(M42), is_float(M43), + is_float(M44), is_float(M45), is_float(M46), + is_float(M51), is_float(M52), is_float(M53), + is_float(M54), is_float(M55), is_float(M56), + is_float(M61), is_float(M62), is_float(M63), + is_float(M64), is_float(M65), is_float(M66), + is_float(Acc) -> + R11 = M66+M11, R12 = M65+M12, R13 = M64+M13, + R14 = M63+M14, R15 = M62+M15, R16 = M61+M16, + R21 = M56+M21, R22 = M55+M22, R23 = M54+M23, + R24 = M53+M24, R25 = M52+M25, R26 = M51+M26, + R31 = M46+M31, R32 = M45+M32, R33 = M44+M33, + R34 = M43+M34, R35 = M42+M35, R36 = M41+M36, + R41 = M26+M41, R42 = M25+M42, R43 = M24+M43, + R44 = M23+M44, R45 = M22+M45, R46 = M21+M46, + R51 = M36+M51, R52 = M35+M52, R53 = M34+M53, + R54 = M33+M54, R55 = M32+M55, R56 = M31+M56, + R61 = M16+M61, R62 = M15+M62, R63 = M14+M63, + R64 = M13+M64, R65 = M12+M65, R66 = M11+M66, + case Ctr of + 0 -> + {{{R11, R12, R13, R14, R15, R16}, + {R21, R22, R23, R24, R25, R26}, + {R31, R32, R33, R34, R35, R36}, + {R41, R42, R43, R44, R45, R46}, + {R51, R52, R53, R54, R55, R56}, + {R61, R62, R63, R64, R65, R66}}, + Acc}; + _ -> + NewAcc = 0.0 + M11 + M12 + M13 + M14 + M15 + M16 + + + M21 + M22 + M23 + M24 + M25 + M26 + + M31 + M32 + M33 + M34 + M35 + M36 + + M41 + M42 + M43 + M44 + M45 + M46 + + M51 + M52 + M53 + M54 + M55 + M56 + + M61 + M62 + M63 + M64 + M65 + M66 + + Acc, + mat66_flip_sum(R11+1.0, R12+1.0, R13+1.0, R14+1.0, R15+1.0, R16+1.0, + R21+1.0, R22+1.0, R23+1.0, R24+1.0, R25+1.0, R26+1.0, + R31+1.0, R32+1.0, R33+1.0, R34+1.0, R35+1.0, R36+1.0, + R41+1.0, R42+1.0, R43+1.0, R44+1.0, R45+1.0, R46+1.0, + R51+1.0, R52+1.0, R53+1.0, R54+1.0, R55+1.0, R56+1.0, + R61+1.0, R62+1.0, R63+1.0, R64+1.0, R65+1.0, R66+1.0, + NewAcc, Ctr-1) + end. + +%% Infinite loops must receive reduction tests, and might trip up basic block +%% weighting, leading to infinite weights and/or divisions by zero. + +test_infinite_loops() -> + OldTrapExit = process_flag(trap_exit, true), + ok = test_infinite_loop(fun infinite_recursion/0), + ok = test_infinite_loop(fun infinite_corecursion/0), + RecursiveFun = fun RecursiveFun() -> RecursiveFun() end, + ok = test_infinite_loop(RecursiveFun), + CorecursiveFunA = fun CorecursiveFunA() -> + CorecursiveFunA1 = fun () -> CorecursiveFunA() end, + CorecursiveFunA1() + end, + ok = test_infinite_loop(CorecursiveFunA), + CorecursiveFunB1 = fun(CorecursiveFunB) -> CorecursiveFunB() end, + CorecursiveFunB = fun CorecursiveFunB() -> + CorecursiveFunB1(CorecursiveFunB) + end, + ok = test_infinite_loop(CorecursiveFunB), + CorecursiveFunC1 = fun CorecursiveFunC1(Other) -> + Other(CorecursiveFunC1) + end, + CorecursiveFunC = fun CorecursiveFunC(Other) -> + Other(CorecursiveFunC) + end, + ok = test_infinite_loop(fun() -> CorecursiveFunC(CorecursiveFunC1) end), + ok = test_infinite_loop(fun() -> CorecursiveFunC(CorecursiveFunC) end), + true = process_flag(trap_exit, OldTrapExit), + ok. + +-define(INFINITE_LOOP_TIMEOUT, 100). +test_infinite_loop(Fun) -> + Tester = spawn_link(Fun), + kill_soon(Tester), + receive {'EXIT', Tester, awake} -> + undefined = process_info(Tester), + ok + after ?INFINITE_LOOP_TIMEOUT -> error(timeout) + end. + +infinite_recursion() -> infinite_recursion(). + +infinite_corecursion() -> infinite_corecursion_1(). +infinite_corecursion_1() -> infinite_corecursion(). + +kill_soon(Pid) -> + _ = spawn_link(fun() -> + timer:sleep(1), + erlang:exit(Pid, awake) + end), + ok. diff --git a/lib/hipe/test/basic_SUITE_data/basic_tuples.erl b/lib/hipe/test/basic_SUITE_data/basic_tuples.erl index 94c187e364..96e39d565a 100644 --- a/lib/hipe/test/basic_SUITE_data/basic_tuples.erl +++ b/lib/hipe/test/basic_SUITE_data/basic_tuples.erl @@ -55,6 +55,8 @@ test_element(T0, T1, T2, N) -> List = lists:seq(1, N), Tuple = list_to_tuple(List), ok = get_elements(List, Tuple, 1), + %% element/2 of larger tuple with omitted bounds test + true = lists:all(fun(I) -> I * I =:= square(I) end, lists:seq(1, 20)), %% some cases that throw exceptions {'EXIT', _} = (catch my_element(0, T2)), {'EXIT', _} = (catch my_element(3, T2)), @@ -73,6 +75,18 @@ get_elements([Element|Rest], Tuple, Pos) -> get_elements([], _Tuple, _Pos) -> ok. +squares() -> + {1*1, 2*2, 3*3, 4*4, 5*5, 6*6, 7*7, 8*8, 9*9, 10*10, + 11*11, 12*12, 13*13, 14*14, 15*15, 16*16, 17*17, 18*18, 19*19, 20*20}. + +square(N) when is_integer(N), N >= 1, N =< 20 -> + %% The guard tests lets the range analysis conclude N to be an integer in the + %% 1..20 range. 20-1=19 is bigger than ?SET_LIMIT in erl_types.erl, and will + %% thus be represented by an ?int_range() rather than an ?int_set(). + %% Because of the range analysis, the bounds test of this element/2 call + %% should be omitted. + element(N, squares()). + %%-------------------------------------------------------------------- %% Tests set_element/3. diff --git a/lib/hipe/util/Makefile b/lib/hipe/util/Makefile index 04de7f7823..eeb81ac482 100644 --- a/lib/hipe/util/Makefile +++ b/lib/hipe/util/Makefile @@ -48,7 +48,7 @@ HIPE_MODULES = hipe_vectors else HIPE_MODULES = endif -MODULES = hipe_timing hipe_dot hipe_digraph $(HIPE_MODULES) +MODULES = hipe_timing hipe_dot hipe_digraph hipe_dsets $(HIPE_MODULES) HRL_FILES= ERL_FILES= $(MODULES:%=%.erl) diff --git a/lib/hipe/util/hipe_dsets.erl b/lib/hipe/util/hipe_dsets.erl new file mode 100644 index 0000000000..9492cab0ff --- /dev/null +++ b/lib/hipe/util/hipe_dsets.erl @@ -0,0 +1,84 @@ +%% -*- erlang-indent-level: 2 -*- +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%@doc +%% IMMUTABLE DISJOINT SETS OF ARBITRARY TERMS +%% +%% The disjoint set forests data structure, for elements of arbitrary types. +%% Note that the find operation mutates the set. +%% +%% We could do this more efficiently if we restricted the elements to integers, +%% and used the (mutable) hipe arrays. For arbitrary terms ETS could be used, +%% for a persistent interface (which isn't that nice when even accessors return +%% modified copies), the array module could be used. +-module(hipe_dsets). + +-export([new/1, find/2, union/3, to_map/1, to_rllist/1]). +-export_type([dsets/1]). + +-opaque dsets(X) :: #{X => {node, X} | {root, non_neg_integer()}}. + +-spec new([E]) -> dsets(E). +new(Elems) -> maps:from_list([{E,{root,0}} || E <- Elems]). + +-spec find(E, dsets(E)) -> {E, dsets(E)}. +find(E, DS0) -> + case DS0 of + #{E := {root,_}} -> {E, DS0}; + #{E := {node,N}} -> + case find(N, DS0) of + {N, _}=T -> T; + {R, DS1} -> {R, DS1#{E := {node,R}}} + end; + _ -> error(badarg, [E, DS0]) + end. + +-spec union(E, E, dsets(E)) -> dsets(E). +union(X, Y, DS0) -> + {XRoot, DS1} = find(X, DS0), + case find(Y, DS1) of + {XRoot, DS2} -> DS2; + {YRoot, DS2} -> + #{XRoot := {root,XRR}, YRoot := {root,YRR}} = DS2, + if XRR < YRR -> DS2#{XRoot := {node,YRoot}}; + XRR > YRR -> DS2#{YRoot := {node,XRoot}}; + true -> DS2#{YRoot := {node,XRoot}, XRoot := {root,XRR+1}} + end + end. + +-spec to_map(dsets(E)) -> {#{Elem::E => Root::E}, dsets(E)}. +to_map(DS) -> + to_map(maps:keys(DS), DS, #{}). + +to_map([], DS, Acc) -> {Acc, DS}; +to_map([K|Ks], DS0, Acc) -> + {KR, DS} = find(K, DS0), + to_map(Ks, DS, Acc#{K => KR}). + +-spec to_rllist(dsets(E)) -> {[{Root::E, Elems::[E]}], dsets(E)}. +to_rllist(DS0) -> + {Lists, DS} = to_rllist(maps:keys(DS0), #{}, DS0), + {maps:to_list(Lists), DS}. + +to_rllist([], Acc, DS) -> {Acc, DS}; +to_rllist([E|Es], Acc, DS0) -> + {ERoot, DS} = find(E, DS0), + to_rllist(Es, map_append(ERoot, E, Acc), DS). + +map_append(Key, Elem, Map) -> + case Map of + #{Key := List} -> Map#{Key := [Elem|List]}; + #{} -> Map#{Key => [Elem]} + end. diff --git a/lib/hipe/util/hipe_vectors.erl b/lib/hipe/util/hipe_vectors.erl index fc4e4edb24..788dacd11b 100644 --- a/lib/hipe/util/hipe_vectors.erl +++ b/lib/hipe/util/hipe_vectors.erl @@ -116,8 +116,7 @@ get(Vec, Ix) -> %% --------------------------------------------------------------------- -ifdef(USE_ARRAYS). -%%-opaque vector(E) :: array:array(E). --type vector(E) :: array:array(E). % Work around dialyzer bug +-opaque vector(E) :: array:array(E). new(N, V) -> array:new(N, {default, V}). size(V) -> array:size(V). diff --git a/lib/hipe/vsn.mk b/lib/hipe/vsn.mk index cb4174381a..172d976931 100644 --- a/lib/hipe/vsn.mk +++ b/lib/hipe/vsn.mk @@ -1 +1 @@ -HIPE_VSN = 3.15.3 +HIPE_VSN = 3.15.4 diff --git a/lib/hipe/x86/hipe_rtl_to_x86.erl b/lib/hipe/x86/hipe_rtl_to_x86.erl index 29cad6ca51..31e4f6e4ac 100644 --- a/lib/hipe/x86/hipe_rtl_to_x86.erl +++ b/lib/hipe/x86/hipe_rtl_to_x86.erl @@ -124,7 +124,6 @@ conv_insn(I, Map, Data) -> hipe_rtl:call_continuation(I), hipe_rtl:call_fail(I), hipe_rtl:call_type(I)), - %% XXX Fixme: this ++ is probably inefficient. {FixArgs++I2, Map2, Data}; #comment{} -> I2 = [hipe_x86:mk_comment(hipe_rtl:comment_text(I))], diff --git a/lib/hipe/x86/hipe_x86.erl b/lib/hipe/x86/hipe_x86.erl index cc1c75b04d..f514dd1ded 100644 --- a/lib/hipe/x86/hipe_x86.erl +++ b/lib/hipe/x86/hipe_x86.erl @@ -167,6 +167,12 @@ mk_pseudo_spill/1, + mk_pseudo_spill_fmove/3, + is_pseudo_spill_fmove/1, + + mk_pseudo_spill_move/3, + is_pseudo_spill_move/1, + mk_pseudo_tailcall/4, %% is_pseudo_tailcall/1, pseudo_tailcall_fun/1, @@ -425,6 +431,14 @@ mk_pseudo_jcc_simple(Cc, TrueLabel, FalseLabel, Pred) -> mk_pseudo_spill(List) -> #pseudo_spill{args=List}. +mk_pseudo_spill_fmove(Src, Temp, Dst) -> + #pseudo_spill_fmove{src=Src, temp=Temp, dst=Dst}. +is_pseudo_spill_fmove(I) -> is_record(I, pseudo_spill_fmove). + +mk_pseudo_spill_move(Src, Temp, Dst) -> + #pseudo_spill_move{src=Src, temp=Temp, dst=Dst}. +is_pseudo_spill_move(I) -> is_record(I, pseudo_spill_move). + mk_pseudo_tailcall(Fun, Arity, StkArgs, Linkage) -> check_linkage(Linkage), #pseudo_tailcall{'fun'=Fun, arity=Arity, stkargs=StkArgs, linkage=Linkage}. diff --git a/lib/hipe/x86/hipe_x86.hrl b/lib/hipe/x86/hipe_x86.hrl index 567848bae5..6cd69905b2 100644 --- a/lib/hipe/x86/hipe_x86.hrl +++ b/lib/hipe/x86/hipe_x86.hrl @@ -91,6 +91,8 @@ -record(pseudo_call, {'fun', sdesc, contlab, linkage}). -record(pseudo_jcc, {cc, true_label, false_label, pred}). -record(pseudo_spill, {args=[]}). +-record(pseudo_spill_move, {src, temp, dst}). +-record(pseudo_spill_fmove, {src, temp, dst}). -record(pseudo_tailcall, {'fun', arity, stkargs, linkage}). -record(pseudo_tailcall_prepare, {}). -record(push, {src}). diff --git a/lib/hipe/x86/hipe_x86_assemble.erl b/lib/hipe/x86/hipe_x86_assemble.erl index ef9c32ef41..50919bdf4e 100644 --- a/lib/hipe/x86/hipe_x86_assemble.erl +++ b/lib/hipe/x86/hipe_x86_assemble.erl @@ -63,7 +63,7 @@ assemble(CompiledCode, Closures, Exports, Options) -> || {MFA, Defun} <- CompiledCode], %% {ConstAlign,ConstSize,ConstMap,RefsFromConsts} = - hipe_pack_constants:pack_constants(Code, ?HIPE_X86_REGISTERS:alignment()), + hipe_pack_constants:pack_constants(Code), %% {CodeSize,CodeBinary,AccRefs,LabelMap,ExportMap} = encode(translate(Code, ConstMap, Options), Options), @@ -148,6 +148,8 @@ insn_size(I) -> translate_insn(I, Context, Options) -> case I of + #alu{aluop='xor', src=#x86_temp{reg=Reg}=Src, dst=#x86_temp{reg=Reg}=Dst} -> + [{'xor', {temp_to_reg32(Dst), temp_to_rm32(Src)}, I}]; #alu{} -> Arg = resolve_alu_args(hipe_x86:alu_src(I), hipe_x86:alu_dst(I), Context), [{hipe_x86:alu_op(I), Arg, I}]; @@ -228,11 +230,11 @@ translate_insn(I, Context, Options) -> #move64{} -> translate_move64(I, Context); #movsx{} -> - Arg = resolve_movx_args(hipe_x86:movsx_src(I), hipe_x86:movsx_dst(I)), - [{movsx, Arg, I}]; + Src = resolve_movx_src(hipe_x86:movsx_src(I)), + [{movsx, {temp_to_regArch(hipe_x86:movsx_dst(I)), Src}, I}]; #movzx{} -> - Arg = resolve_movx_args(hipe_x86:movzx_src(I), hipe_x86:movzx_dst(I)), - [{movzx, Arg, I}]; + Src = resolve_movx_src(hipe_x86:movzx_src(I)), + [{movzx, {temp_to_reg32(hipe_x86:movzx_dst(I)), Src}, I}]; %% pseudo_call: eliminated before assembly %% pseudo_jcc: eliminated before assembly %% pseudo_tailcall: eliminated before assembly @@ -845,16 +847,15 @@ translate_move64(I, _Context) -> exit({?MODULE, I}). -endif. %%% mov{s,z}x -resolve_movx_args(Src=#x86_mem{type=Type}, Dst=#x86_temp{}) -> - {temp_to_regArch(Dst), - case Type of - byte -> - mem_to_rm8(Src); - int16 -> - mem_to_rm16(Src); - int32 -> - mem_to_rm32(Src) - end}. +resolve_movx_src(Src=#x86_mem{type=Type}) -> + case Type of + byte -> + mem_to_rm8(Src); + int16 -> + mem_to_rm16(Src); + int32 -> + mem_to_rm32(Src) + end. %%% alu/cmp (_not_ test) resolve_alu_args(Src, Dst, Context) -> diff --git a/lib/hipe/x86/hipe_x86_cfg.erl b/lib/hipe/x86/hipe_x86_cfg.erl index a4544e1086..0a3c0fc9d6 100644 --- a/lib/hipe/x86/hipe_x86_cfg.erl +++ b/lib/hipe/x86/hipe_x86_cfg.erl @@ -19,7 +19,7 @@ succ/2, pred/2, bb/2, bb_add/3, map_bbs/2, fold_bbs/3]). -export([postorder/1, reverse_postorder/1]). --export([linearise/1, params/1, arity/1, redirect_jmp/3]). +-export([linearise/1, params/1, arity/1, redirect_jmp/3, branch_preds/1]). %%% these tell cfg.inc what to define (ugly as hell) -define(PRED_NEEDED,true). @@ -72,6 +72,26 @@ branch_successors(Branch) -> #ret{} -> [] end. +branch_preds(Branch) -> + case Branch of + #jmp_switch{labels=Labels} -> + Prob = 1.0/length(Labels), + [{L, Prob} || L <- Labels]; + #pseudo_call{contlab=ContLab, sdesc=#x86_sdesc{exnlab=[]}} -> + %% A function can still cause an exception, even if we won't catch it + [{ContLab, 1.0-hipe_bb_weights:call_exn_pred()}]; + #pseudo_call{contlab=ContLab, sdesc=#x86_sdesc{exnlab=ExnLab}} -> + CallExnPred = hipe_bb_weights:call_exn_pred(), + [{ContLab, 1.0-CallExnPred}, {ExnLab, CallExnPred}]; + #pseudo_jcc{true_label=TrueLab,false_label=FalseLab,pred=Pred} -> + [{FalseLab, 1.0-Pred}, {TrueLab, Pred}]; + _ -> + case branch_successors(Branch) of + [] -> []; + [Single] -> [{Single, 1.0}] + end + end. + -ifdef(REMOVE_TRIVIAL_BBS_NEEDED). fails_to(_Instr) -> []. -endif. diff --git a/lib/hipe/x86/hipe_x86_defuse.erl b/lib/hipe/x86/hipe_x86_defuse.erl index 5d7fadf8e5..2731836dc1 100644 --- a/lib/hipe/x86/hipe_x86_defuse.erl +++ b/lib/hipe/x86/hipe_x86_defuse.erl @@ -51,6 +51,8 @@ insn_def(I) -> #movzx{dst=Dst} -> dst_def(Dst); #pseudo_call{} -> call_clobbered(); #pseudo_spill{} -> []; + #pseudo_spill_fmove{temp=Temp, dst=Dst} -> [Temp, Dst]; + #pseudo_spill_move{temp=Temp, dst=Dst} -> [Temp, Dst]; #pseudo_tailcall_prepare{} -> tailcall_clobbered(); #shift{dst=Dst} -> dst_def(Dst); %% call, cmp, comment, jcc, jmp_fun, jmp_label, jmp_switch, label @@ -108,6 +110,8 @@ insn_use(I) -> #pseudo_call{'fun'=Fun,sdesc=#x86_sdesc{arity=Arity}} -> addtemp(Fun, arity_use(Arity)); #pseudo_spill{args=Args} -> Args; + #pseudo_spill_fmove{src=Src} -> [Src]; + #pseudo_spill_move{src=Src} -> [Src]; #pseudo_tailcall{'fun'=Fun,arity=Arity,stkargs=StkArgs} -> addtemp(Fun, addtemps(StkArgs, addtemps(tailcall_clobbered(), arity_use(Arity)))); diff --git a/lib/hipe/x86/hipe_x86_frame.erl b/lib/hipe/x86/hipe_x86_frame.erl index 3c2b67967a..558321d0c3 100644 --- a/lib/hipe/x86/hipe_x86_frame.erl +++ b/lib/hipe/x86/hipe_x86_frame.erl @@ -95,13 +95,17 @@ do_insn(I, LiveOut, Context, FPoff) -> #imul{} -> {[do_imul(I, Context, FPoff)], FPoff}; #move{} -> - {[do_move(I, Context, FPoff)], FPoff}; + {do_move(I, Context, FPoff), FPoff}; #movsx{} -> {[do_movsx(I, Context, FPoff)], FPoff}; #movzx{} -> {[do_movzx(I, Context, FPoff)], FPoff}; #pseudo_call{} -> do_pseudo_call(I, LiveOut, Context, FPoff); + #pseudo_spill_fmove{} -> + {do_pseudo_spill_fmove(I, Context, FPoff), FPoff}; + #pseudo_spill_move{} -> + {do_pseudo_spill_move(I, Context, FPoff), FPoff}; #pseudo_tailcall{} -> {do_pseudo_tailcall(I, Context), context_framesize(Context)}; #push{} -> @@ -144,22 +148,50 @@ do_fp_binop(I, Context, FPoff) -> Dst = conv_opnd(Dst0, FPoff, Context), [I#fp_binop{src=Src,dst=Dst}]. -do_fmove(I, Context, FPoff) -> - #fmove{src=Src0,dst=Dst0} = I, +do_fmove(I0, Context, FPoff) -> + #fmove{src=Src0,dst=Dst0} = I0, Src = conv_opnd(Src0, FPoff, Context), Dst = conv_opnd(Dst0, FPoff, Context), - I#fmove{src=Src,dst=Dst}. + I = I0#fmove{src=Src,dst=Dst}, + case Src =:= Dst of + true -> []; % omit move-to-self + false -> [I] + end. + +do_pseudo_spill_fmove(I0, Context, FPoff) -> + #pseudo_spill_fmove{src=Src0,temp=Temp0,dst=Dst0} = I0, + Src = conv_opnd(Src0, FPoff, Context), + Temp = conv_opnd(Temp0, FPoff, Context), + Dst = conv_opnd(Dst0, FPoff, Context), + case Src =:= Dst of + true -> []; % omit move-to-self + false -> [#fmove{src=Src, dst=Temp}, #fmove{src=Temp, dst=Dst}] + end. do_imul(I, Context, FPoff) -> #imul{src=Src0} = I, Src = conv_opnd(Src0, FPoff, Context), I#imul{src=Src}. -do_move(I, Context, FPoff) -> - #move{src=Src0,dst=Dst0} = I, +do_move(I0, Context, FPoff) -> + #move{src=Src0,dst=Dst0} = I0, Src = conv_opnd(Src0, FPoff, Context), Dst = conv_opnd(Dst0, FPoff, Context), - I#move{src=Src,dst=Dst}. + I = I0#move{src=Src,dst=Dst}, + case Src =:= Dst of + true -> []; % omit move-to-self + false -> [I] + end. + +do_pseudo_spill_move(I0, Context, FPoff) -> + #pseudo_spill_move{src=Src0,temp=Temp0,dst=Dst0} = I0, + Src = conv_opnd(Src0, FPoff, Context), + Temp = conv_opnd(Temp0, FPoff, Context), + Dst = conv_opnd(Dst0, FPoff, Context), + case Src =:= Dst of + true -> []; % omit move-to-self + false -> [#move{src=Src, dst=Temp}, #move{src=Temp, dst=Dst}] + end. do_movsx(I, Context, FPoff) -> #movsx{src=Src0,dst=Dst0} = I, diff --git a/lib/hipe/x86/hipe_x86_postpass.erl b/lib/hipe/x86/hipe_x86_postpass.erl index b84e9bed91..925054dd68 100644 --- a/lib/hipe/x86/hipe_x86_postpass.erl +++ b/lib/hipe/x86/hipe_x86_postpass.erl @@ -57,9 +57,10 @@ postpass(#defun{code=Code0}=Defun, Options) -> peephole_optimization(Insns) -> peep(Insns, [], []). -%% MoveSelf related peep-opts + +%% MoveSelf related peep-opts %% ------------------------------ -peep([#fmove{src=Src, dst=Src} | Insns], Res,Lst) -> +peep([#fmove{src=Src, dst=Src} | Insns], Res,Lst) -> peep(Insns, Res, [moveSelf1|Lst]); peep([I=#fmove{src=Src, dst=Dst}, #fmove{src=Dst, dst=Src} | Insns], Res,Lst) -> @@ -159,8 +160,7 @@ peep([#jcc{label=Lab}, I=#label{label=Lab}|Insns], Res, Lst) -> %% ElimSet0 %% -------- -peep([#move{src=#x86_imm{value=0},dst=Dst}|Insns],Res,Lst) -when (Dst==#x86_temp{}) -> +peep([#move{src=#x86_imm{value=0},dst=Dst=#x86_temp{}}|Insns],Res,Lst) -> peep(Insns, [#alu{aluop='xor', src=Dst, dst=Dst}|Res], [elimSet0|Lst]); %% ElimMDPow2 diff --git a/lib/hipe/x86/hipe_x86_ra_finalise.erl b/lib/hipe/x86/hipe_x86_ra_finalise.erl index 4273e3cee8..e8abe78e00 100644 --- a/lib/hipe/x86/hipe_x86_ra_finalise.erl +++ b/lib/hipe/x86/hipe_x86_ra_finalise.erl @@ -140,6 +140,16 @@ ra_insn(I, Map, FpMap) -> I#pseudo_call{'fun'=Fun}; #pseudo_jcc{} -> I; + #pseudo_spill_fmove{src=Src0, temp=Temp0, dst=Dst0} -> + Src = ra_opnd(Src0, Map, FpMap), + Temp = ra_opnd(Temp0, Map, FpMap), + Dst = ra_opnd(Dst0, Map, FpMap), + I#pseudo_spill_fmove{src=Src, temp=Temp, dst=Dst}; + #pseudo_spill_move{src=Src0, temp=Temp0, dst=Dst0} -> + Src = ra_opnd(Src0, Map), + Temp = ra_opnd(Temp0, Map), + Dst = ra_opnd(Dst0, Map), + I#pseudo_spill_move{src=Src, temp=Temp, dst=Dst}; #pseudo_tailcall{'fun'=Fun0,stkargs=StkArgs0} -> Fun = ra_opnd(Fun0, Map), StkArgs = ra_args(StkArgs0, Map), diff --git a/lib/hipe/x86/hipe_x86_ra_postconditions.erl b/lib/hipe/x86/hipe_x86_ra_postconditions.erl index 28ec9c4277..db6391d5c1 100644 --- a/lib/hipe/x86/hipe_x86_ra_postconditions.erl +++ b/lib/hipe/x86/hipe_x86_ra_postconditions.erl @@ -74,6 +74,8 @@ do_insn(I, TempMap, Strategy) -> % Insn -> {Insn list, DidSpill} do_movx(I, TempMap, Strategy); #fmove{} -> do_fmove(I, TempMap, Strategy); + #pseudo_spill_move{} -> + do_pseudo_spill_move(I, TempMap, Strategy); #shift{} -> do_shift(I, TempMap, Strategy); #test{} -> @@ -190,10 +192,19 @@ do_lea(I, TempMap, Strategy) -> do_move(I, TempMap, Strategy) -> #move{src=Src0,dst=Dst0} = I, - {FixSrc, Src, FixDst, Dst, DidSpill} = - do_check_byte_move(Src0, Dst0, TempMap, Strategy), - {FixSrc ++ FixDst ++ [I#move{src=Src,dst=Dst}], - DidSpill}. + case + is_record(Src0, x86_temp) andalso is_record(Dst0, x86_temp) + andalso is_spilled(Src0, TempMap) andalso is_spilled(Dst0, TempMap) + of + true -> + Tmp = clone(Src0, Strategy), + {[hipe_x86:mk_pseudo_spill_move(Src0, Tmp, Dst0)], true}; + false -> + {FixSrc, Src, FixDst, Dst, DidSpill} = + do_check_byte_move(Src0, Dst0, TempMap, Strategy), + {FixSrc ++ FixDst ++ [I#move{src=Src,dst=Dst}], + DidSpill} + end. -ifdef(HIPE_AMD64). @@ -287,6 +298,13 @@ do_fmove(I, TempMap, Strategy) -> {FixSrc ++ FixDst ++ [I#fmove{src=Src,dst=Dst}], DidSpill1 or DidSpill2}. +%%% Fix an pseudo_spill_move op. + +do_pseudo_spill_move(I = #pseudo_spill_move{temp=Temp}, TempMap, _Strategy) -> + %% Temp is above the low water mark and must not have been spilled + false = is_spilled(Temp, TempMap), + {[I], false}. % nothing to do + %%% Fix a shift operation. %%% 1. remove pseudos from any explicit memory operands %%% 2. if the source is a register or memory position diff --git a/lib/hipe/x86/hipe_x86_subst.erl b/lib/hipe/x86/hipe_x86_subst.erl index 7b5fb1352b..7db3b23d92 100644 --- a/lib/hipe/x86/hipe_x86_subst.erl +++ b/lib/hipe/x86/hipe_x86_subst.erl @@ -19,7 +19,7 @@ -endif. -module(?HIPE_X86_SUBST). --export([insn_temps/2]). +-export([insn_temps/2, insn_lbls/2]). -include("../x86/hipe_x86.hrl"). %% These should be moved to hipe_x86 and exported @@ -28,6 +28,7 @@ -type mfarec() :: #x86_mfa{}. -type prim() :: #x86_prim{}. -type funv() :: mfarec() | prim() | temp(). +-type label() :: non_neg_integer(). -type insn() :: tuple(). % for now -type subst_fun() :: fun((temp()) -> temp()). @@ -49,14 +50,19 @@ insn_temps(SubstTemp, I) -> #movzx {src=S, dst=D} -> I#movzx {src=O(S), dst=O(D)}; #shift {src=S, dst=D} -> I#shift {src=O(S), dst=O(D)}; #test {src=S, dst=D} -> I#test {src=O(S), dst=O(D)}; - #fp_unop{arg=A} -> I#fp_unop{arg=O(A)}; - #move64 {dst=D} -> I#move64 {dst=O(D)}; - #push {src=S} -> I#push {src=O(S)}; - #pop {dst=D} -> I#pop {dst=O(D)}; + #fp_unop{arg=[]} -> I; + #fp_unop{arg=A} -> I#fp_unop{arg=O(A)}; + #move64 {dst=D} -> I#move64 {dst=O(D)}; + #push {src=S} -> I#push {src=O(S)}; + #pop {dst=D} -> I#pop {dst=O(D)}; #jmp_switch{temp=T, jtab=J} -> I#jmp_switch{temp=O(T), jtab=jtab_temps(SubstTemp, J)}; #pseudo_call{'fun'=F} -> I#pseudo_call{'fun'=funv_temps(SubstTemp, F)}; + #pseudo_spill_fmove{src=S, temp=T, dst=D} -> + I#pseudo_spill_fmove{src=O(S), temp=O(T), dst=O(D)}; + #pseudo_spill_move{src=S, temp=T, dst=D} -> + I#pseudo_spill_move{src=O(S), temp=O(T), dst=O(D)}; #pseudo_tailcall{'fun'=F, stkargs=Stk} -> I#pseudo_tailcall{'fun'=funv_temps(SubstTemp, F), stkargs=lists:map(O, Stk)}; @@ -85,3 +91,22 @@ jtab_temps(SubstTemp, T=#x86_temp{}) -> SubstTemp(T). -else. jtab_temps(_SubstTemp, DataLbl) when is_integer(DataLbl) -> DataLbl. -endif. + +-type lbl_subst_fun() :: fun((label()) -> label()). + +%% @doc Maps over the branch targets in an instruction +-spec insn_lbls(lbl_subst_fun(), insn()) -> insn(). +insn_lbls(SubstLbl, I) -> + case I of + #jmp_label{label=Label} -> + I#jmp_label{label=SubstLbl(Label)}; + #pseudo_call{sdesc=Sdesc, contlab=Contlab} -> + I#pseudo_call{sdesc=sdesc_lbls(SubstLbl, Sdesc), + contlab=SubstLbl(Contlab)}; + #pseudo_jcc{true_label=T, false_label=F} -> + I#pseudo_jcc{true_label=SubstLbl(T), false_label=SubstLbl(F)} + end. + +sdesc_lbls(_SubstLbl, Sdesc=#x86_sdesc{exnlab=[]}) -> Sdesc; +sdesc_lbls(SubstLbl, Sdesc=#x86_sdesc{exnlab=Exnlab}) -> + Sdesc#x86_sdesc{exnlab=SubstLbl(Exnlab)}. diff --git a/lib/ic/test/c_client_erl_server_SUITE_data/c_client.c b/lib/ic/test/c_client_erl_server_SUITE_data/c_client.c index af189a74f7..b3a18e03d4 100644 --- a/lib/ic/test/c_client_erl_server_SUITE_data/c_client.c +++ b/lib/ic/test/c_client_erl_server_SUITE_data/c_client.c @@ -58,7 +58,7 @@ #include "erl_interface.h" #include "m_i.h" -#define HOSTNAMESZ 256 +#define HOSTNAMESZ 255 #define NODENAMESZ 512 #define INBUFSZ 10 @@ -295,7 +295,7 @@ int main(int argc, char **argv) progname = argv[0]; host[HOSTNAMESZ] = '\0'; - if (gethostname(host, HOSTNAMESZ) < 0) { + if (gethostname(host, HOSTNAMESZ + 1) < 0) { fprintf(stderr, "Can't find own hostname\n"); done(1); } diff --git a/lib/ic/test/c_client_erl_server_proto_SUITE_data/c_client.c b/lib/ic/test/c_client_erl_server_proto_SUITE_data/c_client.c index b7609d63e5..40c7328f03 100644 --- a/lib/ic/test/c_client_erl_server_proto_SUITE_data/c_client.c +++ b/lib/ic/test/c_client_erl_server_proto_SUITE_data/c_client.c @@ -61,7 +61,7 @@ #include "erl_interface.h" #include "m_i.h" -#define HOSTNAMESZ 256 +#define HOSTNAMESZ 255 #define NODENAMESZ 512 #define INBUFSZ 10 @@ -298,7 +298,7 @@ int main(int argc, char **argv) progname = argv[0]; host[HOSTNAMESZ] = '\0'; - if (gethostname(host, HOSTNAMESZ) < 0) { + if (gethostname(host, HOSTNAMESZ + 1) < 0) { fprintf(stderr, "Can't find own hostname\n"); done(1); } diff --git a/lib/ic/test/c_client_erl_server_proto_tmo_SUITE_data/c_client.c b/lib/ic/test/c_client_erl_server_proto_tmo_SUITE_data/c_client.c index 23dc089555..33cfe71322 100644 --- a/lib/ic/test/c_client_erl_server_proto_tmo_SUITE_data/c_client.c +++ b/lib/ic/test/c_client_erl_server_proto_tmo_SUITE_data/c_client.c @@ -61,7 +61,7 @@ #include "erl_interface.h" #include "m_i.h" -#define HOSTNAMESZ 256 +#define HOSTNAMESZ 255 #define NODENAMESZ 512 #define INBUFSZ 10 @@ -298,7 +298,7 @@ int main(int argc, char **argv) progname = argv[0]; host[HOSTNAMESZ] = '\0'; - if (gethostname(host, HOSTNAMESZ) < 0) { + if (gethostname(host, HOSTNAMESZ + 1) < 0) { fprintf(stderr, "Can't find own hostname\n"); done(1); } diff --git a/lib/ic/test/erl_client_c_server_SUITE_data/c_server.c b/lib/ic/test/erl_client_c_server_SUITE_data/c_server.c index 53345d561b..f48480e8dc 100644 --- a/lib/ic/test/erl_client_c_server_SUITE_data/c_server.c +++ b/lib/ic/test/erl_client_c_server_SUITE_data/c_server.c @@ -81,7 +81,7 @@ static void showtime(MyTimeval *start, MyTimeval *stop); static void usage(void); static void done(int r); -#define HOSTNAMESZ 256 +#define HOSTNAMESZ 255 #define NODENAMESZ 512 #define INBUFSZ 10 #define OUTBUFSZ 0 @@ -122,7 +122,7 @@ int main(int argc, char **argv) progname = argv[0]; host[HOSTNAMESZ] = '\0'; - if (gethostname(host, HOSTNAMESZ) < 0) { + if (gethostname(host, HOSTNAMESZ + 1) < 0) { fprintf(stderr, "Can't find own hostname\n"); done(1); } diff --git a/lib/ic/test/erl_client_c_server_proto_SUITE_data/c_server.c b/lib/ic/test/erl_client_c_server_proto_SUITE_data/c_server.c index a18f0e7ba9..e2ba5bd5b6 100644 --- a/lib/ic/test/erl_client_c_server_proto_SUITE_data/c_server.c +++ b/lib/ic/test/erl_client_c_server_proto_SUITE_data/c_server.c @@ -81,7 +81,7 @@ static void showtime(MyTimeval *start, MyTimeval *stop); static void usage(void); static void done(int r); -#define HOSTNAMESZ 256 +#define HOSTNAMESZ 255 #define NODENAMESZ 512 #define INBUFSZ 10 #define OUTBUFSZ 0 @@ -122,7 +122,7 @@ int main(int argc, char **argv) progname = argv[0]; host[HOSTNAMESZ] = '\0'; - if (gethostname(host, HOSTNAMESZ) < 0) { + if (gethostname(host, HOSTNAMESZ + 1) < 0) { fprintf(stderr, "Can't find own hostname\n"); done(1); } diff --git a/lib/inets/doc/src/notes.xml b/lib/inets/doc/src/notes.xml index 5c3b5a2d3c..2aa48cd50a 100644 --- a/lib/inets/doc/src/notes.xml +++ b/lib/inets/doc/src/notes.xml @@ -33,7 +33,61 @@ <file>notes.xml</file> </header> - <section><title>Inets 6.3.5</title> + <section><title>Inets 6.3.7</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed a bug in ftp that made further operations after a + recv_chunk operation impossible.</p> + <p> + Own Id: OTP-14242</p> + </item> + </list> + </section> + +</section> + +<section><title>Inets 6.3.6</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Chunk size decoding could fail. The symptom was that + chunk decoding sometimes failed depending on timing of + the received stream. If chunk size was split into two + different packets decoding would fail.</p> + <p> + Own Id: OTP-13571 Aux Id: ERL-116 </p> + </item> + <item> + <p> + Prevent httpc user process to hang if httpc_handler + process terminates unexpectedly</p> + <p> + Own Id: OTP-14091</p> + </item> + <item> + <p> + Correct Host header, to include port number, when + redirecting requests.</p> + <p> + Own Id: OTP-14097</p> + </item> + <item> + <p> + Shutdown gracefully on connection or TLS handshake errors</p> + <p> + Own Id: OTP-14173 Aux Id: seq13262 </p> + </item> + </list> + </section> + +</section> + +<section><title>Inets 6.3.5</title> <section><title>Fixed Bugs and Malfunctions</title> <list> @@ -713,7 +767,7 @@ <list> <item> <p> - Gracefully handle invalid content-lenght headers instead + Gracefully handle invalid content-length headers instead of crashing in list_to_integer.</p> <p> Own Id: OTP-12429</p> diff --git a/lib/inets/src/ftp/ftp.erl b/lib/inets/src/ftp/ftp.erl index 23d6483291..de869e3204 100644 --- a/lib/inets/src/ftp/ftp.erl +++ b/lib/inets/src/ftp/ftp.erl @@ -108,7 +108,7 @@ -define(DBG(F,A), 'n/a'). %%-define(DBG(F,A), io:format(F,A)). -%%-define(DBG(F,A), if is_list(F) -> ct:pal(F,A); is_atom(F)->ct:pal(atom_to_list(F),A) end). +%%-define(DBG(F,A), ct:pal("~p:~p " ++ if is_list(F) -> F; is_atom(F) -> atom_to_list(F) end, [?MODULE,?LINE|A])). %%%========================================================================= %%% API - CLIENT FUNCTIONS @@ -1482,13 +1482,13 @@ handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, activate_ctrl_connection(State), {noreply, State#state{dsock = undefined, data = <<>>}}; -handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, client = From, +handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, caller = recv_chunk} = State) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> - gen_server:reply(From, ok), - {noreply, State#state{dsock = undefined, client = undefined, - data = <<>>, caller = undefined, - chunk = false}}; + activate_ctrl_connection(State), + {noreply, State#state{dsock = undefined, data = <<>>, + caller = recv_chunk_closed + }}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, caller = recv_bin, data = Data} = State) @@ -1601,13 +1601,13 @@ terminate(normal, State) -> %% If terminate reason =/= normal the progress reporting process will %% be killed by the exit signal. progress_report(stop, State), - do_termiante({error, econn}, State); + do_terminate({error, econn}, State); terminate(Reason, State) -> Report = io_lib:format("Ftp connection closed due to: ~p~n", [Reason]), error_logger:error_report(Report), - do_termiante({error, eclosed}, State). + do_terminate({error, eclosed}, State). -do_termiante(ErrorMsg, State) -> +do_terminate(ErrorMsg, State) -> close_data_connection(State), close_ctrl_connection(State), case State#state.client of @@ -2046,6 +2046,16 @@ handle_ctrl_result({pos_prel, _}, #state{client = From, end; %%-------------------------------------------------------------------------- +%% File handling - chunk_transfer complete +handle_ctrl_result({pos_compl, _}, #state{client = From, + caller = recv_chunk_closed} + = State0) -> + gen_server:reply(From, ok), + {noreply, State0#state{caller = undefined, + chunk = false, + client = undefined}}; + +%%-------------------------------------------------------------------------- %% File handling - recv_file handle_ctrl_result({pos_prel, _}, #state{caller = {recv_file, _}} = State0) -> case accept_data_connection(State0) of diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl index bd5f6df39e..418b6247b0 100644 --- a/lib/inets/src/http_client/httpc.erl +++ b/lib/inets/src/http_client/httpc.erl @@ -524,7 +524,7 @@ handle_request(Method, Url, Options = request_options(Options0), Sync = proplists:get_value(sync, Options), Stream = proplists:get_value(stream, Options), - Host2 = header_host(Scheme, Host, Port), + Host2 = http_request:normalize_host(Scheme, Host, Port), HeadersRecord = header_record(NewHeaders, Host2, HTTPOptions), Receiver = proplists:get_value(receiver, Options), SocketOpts = proplists:get_value(socket_opts, Options), @@ -1035,14 +1035,6 @@ bad_option(Option, BadValue) -> throw({error, {bad_option, Option, BadValue}}). -header_host(https, Host, 443 = _Port) -> - Host; -header_host(http, Host, 80 = _Port) -> - Host; -header_host(_Scheme, Host, Port) -> - Host ++ ":" ++ integer_to_list(Port). - - header_record(NewHeaders, Host, #http_options{version = Version}) -> header_record(NewHeaders, #http_request_h{}, Host, Version). diff --git a/lib/inets/src/http_client/httpc_response.erl b/lib/inets/src/http_client/httpc_response.erl index 0fd5faa466..d81afde5fe 100644 --- a/lib/inets/src/http_client/httpc_response.erl +++ b/lib/inets/src/http_client/httpc_response.erl @@ -362,8 +362,9 @@ redirect(Response = {StatusLine, Headers, Body}, Request) -> {ok, error(Request, Reason), Data}; %% Automatic redirection {ok, {Scheme, _, Host, Port, Path, Query}} -> + HostPort = http_request:normalize_host(Scheme, Host, Port), NewHeaders = - (Request#request.headers)#http_request_h{host = Host++":"++integer_to_list(Port)}, + (Request#request.headers)#http_request_h{host = HostPort}, NewRequest = Request#request{redircount = Request#request.redircount+1, @@ -434,7 +435,7 @@ format_response({StatusLine, Headers, Body}) -> Length = list_to_integer(Headers#http_response_h.'content-length'), {NewBody, Data} = case Length of - -1 -> % When no lenght indicator is provided + -1 -> % When no length indicator is provided {Body, <<>>}; Length when (Length =< size(Body)) -> <<BodyThisReq:Length/binary, Next/binary>> = Body, diff --git a/lib/inets/src/http_lib/http_request.erl b/lib/inets/src/http_lib/http_request.erl index c77b616f0d..4c50edb5ef 100644 --- a/lib/inets/src/http_lib/http_request.erl +++ b/lib/inets/src/http_lib/http_request.erl @@ -22,7 +22,7 @@ -include("http_internal.hrl"). --export([headers/2, http_headers/1, is_absolut_uri/1, key_value/1]). +-export([headers/2, http_headers/1, is_absolut_uri/1, key_value/1, normalize_host/3]). key_value(KeyValueStr) -> @@ -85,6 +85,22 @@ is_absolut_uri("https://" ++ _) -> is_absolut_uri(_) -> false. +%%------------------------------------------------------------------------- +%% normalize_host(Scheme, Host, Port) -> string() +%% Scheme - http | https +%% Host - string() +%% Port - integer() +%% +%% Description: returns a normalized Host header value, with the port +%% number omitted for well-known ports +%%------------------------------------------------------------------------- +normalize_host(https, Host, 443 = _Port) -> + Host; +normalize_host(http, Host, 80 = _Port) -> + Host; +normalize_host(_Scheme, Host, Port) -> + Host ++ ":" ++ integer_to_list(Port). + %%%======================================================================== %%% Internal functions %%%======================================================================== diff --git a/lib/inets/test/ftp_SUITE.erl b/lib/inets/test/ftp_SUITE.erl index e2dec0c42a..0053ba6edd 100644 --- a/lib/inets/test/ftp_SUITE.erl +++ b/lib/inets/test/ftp_SUITE.erl @@ -93,8 +93,10 @@ ftp_tests()-> append_chunk, recv, recv_3, - recv_bin, + recv_bin, + recv_bin_twice, recv_chunk, + recv_chunk_twice, type, quote, error_elogin, @@ -191,9 +193,22 @@ end_per_suite(Config) -> ok. %%-------------------------------------------------------------------- -init_per_group(_Group, Config) -> Config. - -end_per_group(_Group, Config) -> Config. +init_per_group(Group, Config) when Group == ftps_active, + Group == ftps_passive -> + catch crypto:stop(), + try crypto:start() of + ok -> + Config + catch + _:_ -> + {skip, "Crypto did not start"} + end; + +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, Config) -> + Config. %%-------------------------------------------------------------------- init_per_testcase(Case, Config0) -> @@ -533,15 +548,19 @@ append_chunk(Config0) -> recv() -> [{doc, "Receive a file using recv/2"}]. recv(Config0) -> - File = "f_dst.txt", + File1 = "f_dst1.txt", + File2 = "f_dst2.txt", SrcDir = "a_dir", - Contents = <<"ftp_SUITE test ...">>, - Config = set_state([reset, {mkfile,[SrcDir,File],Contents}], Config0), + Contents1 = <<"1 ftp_SUITE test ...">>, + Contents2 = <<"2 ftp_SUITE test ...">>, + Config = set_state([reset, {mkfile,[SrcDir,File1],Contents1}, {mkfile,[SrcDir,File2],Contents2}], Config0), Pid = proplists:get_value(ftp, Config), ok = ftp:cd(Pid, id2ftp(SrcDir,Config)), ok = ftp:lcd(Pid, id2ftp("",Config)), - ok = ftp:recv(Pid, File), - chk_file(File, Contents, Config), + ok = ftp:recv(Pid, File1), + chk_file(File1, Contents1, Config), + ok = ftp:recv(Pid, File2), + chk_file(File2, Contents2, Config), {error,epath} = ftp:recv(Pid, "non_existing_file"), ok. @@ -572,6 +591,25 @@ recv_bin(Config0) -> ok. %%------------------------------------------------------------------------- +recv_bin_twice() -> + [{doc, "Receive two files as a binaries."}]. +recv_bin_twice(Config0) -> + File1 = "f_dst1.txt", + File2 = "f_dst2.txt", + Contents1 = <<"1 ftp_SUITE test ...">>, + Contents2 = <<"2 ftp_SUITE test ...">>, + Config = set_state([reset, {mkfile,File1,Contents1}, {mkfile,File2,Contents2}], Config0), + ct:log("First transfer",[]), + Pid = proplists:get_value(ftp, Config), + {ok,Received1} = ftp:recv_bin(Pid, id2ftp(File1,Config)), + find_diff(Received1, Contents1), + ct:log("Second transfer",[]), + {ok,Received2} = ftp:recv_bin(Pid, id2ftp(File2,Config)), + find_diff(Received2, Contents2), + ct:log("Transfers ready!",[]), + {error,epath} = ftp:recv_bin(Pid, id2ftp("non_existing_file",Config)), + ok. +%%------------------------------------------------------------------------- recv_chunk() -> [{doc, "Receive a file using chunk-wise."}]. recv_chunk(Config0) -> @@ -584,6 +622,23 @@ recv_chunk(Config0) -> {ok, ReceivedContents, _Ncunks} = recv_chunk(Pid, <<>>), find_diff(ReceivedContents, Contents). +recv_chunk_twice() -> + [{doc, "Receive two files using chunk-wise."}]. +recv_chunk_twice(Config0) -> + File1 = "big_file1.txt", + File2 = "big_file2.txt", + Contents1 = list_to_binary( lists:duplicate(1000, lists:seq(0,255)) ), + Contents2 = crypto:strong_rand_bytes(1200), + Config = set_state([reset, {mkfile,File1,Contents1}, {mkfile,File2,Contents2}], Config0), + Pid = proplists:get_value(ftp, Config), + {{error, "ftp:recv_chunk_start/2 not called"},_} = recv_chunk(Pid, <<>>), + ok = ftp:recv_chunk_start(Pid, id2ftp(File1,Config)), + {ok, ReceivedContents1, _Ncunks1} = recv_chunk(Pid, <<>>), + ok = ftp:recv_chunk_start(Pid, id2ftp(File2,Config)), + {ok, ReceivedContents2, _Ncunks2} = recv_chunk(Pid, <<>>), + find_diff(ReceivedContents1, Contents1), + find_diff(ReceivedContents2, Contents2). + recv_chunk(Pid, Acc) -> recv_chunk(Pid, Acc, 0). diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 8aea38037d..67aa78aa06 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -163,21 +163,17 @@ init_per_group(misc = Group, Config) -> ok = httpc:set_options([{ipfamily, Inet}]), Config; + init_per_group(Group, Config0) when Group =:= sim_https; Group =:= https-> - ct:timetrap({seconds, 30}), - start_apps(Group), - StartSsl = try ssl:start() + catch crypto:stop(), + try crypto:start() of + ok -> + ct:timetrap({seconds, 30}), + start_apps(Group), + do_init_per_group(Group, Config0) catch - Error:Reason -> - {skip, lists:flatten(io_lib:format("Failed to start apps for https Error=~p Reason=~p", [Error, Reason]))} - end, - case StartSsl of - {error, {already_started, _}} -> - do_init_per_group(Group, Config0); - ok -> - do_init_per_group(Group, Config0); - _ -> - StartSsl + _:_ -> + {skip, "Crypto did not start"} end; init_per_group(Group, Config0) -> diff --git a/lib/inets/test/httpd_1_1.erl b/lib/inets/test/httpd_1_1.erl index 3755ed117b..2b5968ca12 100644 --- a/lib/inets/test/httpd_1_1.erl +++ b/lib/inets/test/httpd_1_1.erl @@ -405,11 +405,11 @@ getRangeSize(Head)-> {multiPart, BoundaryString}; _X1 -> case re:run(Head, ?CONTENT_RANGE "bytes=.*\r\n", [{capture, first}]) of - {match, [{Start, Lenght}]} -> + {match, [{Start, Length}]} -> %% Get the range data remove the fieldname and the %% end of line. RangeInfo = string:substr(Head, Start + 1 + 20, - Lenght - (20 +2)), + Length - (20 +2)), rangeSize(string:strip(RangeInfo)); _X2 -> error diff --git a/lib/inets/test/httpd_SUITE.erl b/lib/inets/test/httpd_SUITE.erl index aae4ce5256..44b1e09cbc 100644 --- a/lib/inets/test/httpd_SUITE.erl +++ b/lib/inets/test/httpd_SUITE.erl @@ -197,7 +197,14 @@ init_per_group(Group, Config0) when Group == https_basic; Group == https_security; Group == https_reload -> - init_ssl(Group, Config0); + catch crypto:stop(), + try crypto:start() of + ok -> + init_ssl(Group, Config0) + catch + _:_ -> + {skip, "Crypto did not start"} + end; init_per_group(Group, Config0) when Group == http_basic; Group == http_limit; Group == http_custom; @@ -232,7 +239,14 @@ init_per_group(https_htaccess = Group, Config) -> Path = proplists:get_value(doc_root, Config), catch remove_htaccess(Path), create_htaccess_data(Path, proplists:get_value(address, Config)), - init_ssl(Group, Config); + catch crypto:stop(), + try crypto:start() of + ok -> + init_ssl(Group, Config) + catch + _:_ -> + {skip, "Crypto did not start"} + end; init_per_group(auth_api, Config) -> [{auth_prefix, ""} | Config]; init_per_group(auth_api_dets, Config) -> diff --git a/lib/inets/vsn.mk b/lib/inets/vsn.mk index 9591ab22ed..411bbfc043 100644 --- a/lib/inets/vsn.mk +++ b/lib/inets/vsn.mk @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = inets -INETS_VSN = 6.3.5 +INETS_VSN = 6.3.7 PRE_VSN = APP_VSN = "$(APPLICATION)-$(INETS_VSN)$(PRE_VSN)" diff --git a/lib/kernel/doc/src/gen_tcp.xml b/lib/kernel/doc/src/gen_tcp.xml index e97db20062..bef8096aed 100644 --- a/lib/kernel/doc/src/gen_tcp.xml +++ b/lib/kernel/doc/src/gen_tcp.xml @@ -140,6 +140,23 @@ do_recv(Sock, Bs) -> <fsummary>Close a TCP socket.</fsummary> <desc> <p>Closes a TCP socket.</p> + <p>Note that in most implementations of TCP, doing a <c>close</c> does + not guarantee that any data sent is delivered to the recipient before + the close is detected at the remote side. If you want to guarantee + delivery of the data to the recipient there are two common ways to + achieve this.</p> + <list type="ordered"> + <item><p>Use <seealso marker="#shutdown/2"> + <c>gen_tcp:shutdown(Sock, write)</c></seealso> to signal that + no more data is to be sent and wait for the read side of the + socket to be closed.</p> + </item> + <item><p>Use the socket option <seealso marker="inet#packet"> + <c>{packet, N}</c></seealso> (or something similar) to make + it possible for the receiver to close the connection when it + knowns it has received all the data.</p> + </item> + </list> </desc> </func> diff --git a/lib/kernel/doc/src/inet.xml b/lib/kernel/doc/src/inet.xml index 4c4a5c39cb..076e50cd10 100644 --- a/lib/kernel/doc/src/inet.xml +++ b/lib/kernel/doc/src/inet.xml @@ -659,7 +659,8 @@ get_tcpi_sacked(Sock) -> <tag><c>{buffer, Size}</c></tag> <item> <p>The size of the user-level software buffer used by - the driver. Not to be confused with options <c>sndbuf</c> + the driver. + Not to be confused with options <c>sndbuf</c> and <c>recbuf</c>, which correspond to the Kernel socket buffers. It is recommended to have <c>val(buffer) >= max(val(sndbuf),val(recbuf))</c> to @@ -670,6 +671,9 @@ get_tcpi_sacked(Sock) -> usually become larger, you are encouraged to use <seealso marker="#getopts/2"><c>getopts/2</c></seealso> to analyze the behavior of your operating system.</p> + <p>Note that this is also the maximum amount of data that can be + received from a single recv call. If you are using higher than + normal MTU consider setting buffer higher.</p> </item> <tag><c>{delay_send, Boolean}</c></tag> <item> @@ -909,7 +913,7 @@ setcap cap_sys_admin,cap_sys_ptrace,cap_dac_read_search+epi beam.smp</code> </item> <tag><c>{packet, PacketType}</c>(TCP/IP sockets)</tag> <item> - <p>Defines the type of packets to use for a socket. + <p><marker id="packet"/>Defines the type of packets to use for a socket. Possible values:</p> <taglist> <tag><c>raw | 0</c></tag> diff --git a/lib/kernel/doc/src/notes.xml b/lib/kernel/doc/src/notes.xml index d80c4f077c..ad349c5aaf 100644 --- a/lib/kernel/doc/src/notes.xml +++ b/lib/kernel/doc/src/notes.xml @@ -31,6 +31,45 @@ </header> <p>This document describes the changes made to the Kernel application.</p> +<section><title>Kernel 5.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix a race during cleanup of os:cmd that would cause + os:cmd to hang indefinitely.</p> + <p> + Own Id: OTP-14232 Aux Id: seq13275 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p>The functions in the '<c>file</c>' module that take a + list of paths (e.g. <c>file:path_consult/2</c>) will now + continue to search in the path if the path contains + something that is not a directory.</p> + <p> + Own Id: OTP-14191</p> + </item> + <item> + <p>Two OTP processes that are known to receive many + messages are 'rex' (used by 'rpc') and 'error_logger'. + Those processes will now store unprocessed messages + outside the process heap, which will potentially decrease + the cost of garbage collections.</p> + <p> + Own Id: OTP-14192</p> + </item> + </list> + </section> + +</section> + <section><title>Kernel 5.1.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/kernel/src/application_controller.erl b/lib/kernel/src/application_controller.erl index 0e61153613..3b642f5873 100644 --- a/lib/kernel/src/application_controller.erl +++ b/lib/kernel/src/application_controller.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1620,7 +1620,7 @@ conv(_) -> []. make_term(Str) -> case erl_scan:string(Str) of {ok, Tokens, _} -> - case erl_parse:parse_term(Tokens ++ [{dot, 1}]) of + case erl_parse:parse_term(Tokens ++ [{dot, erl_anno:new(1)}]) of {ok, Term} -> Term; {error, {_,M,Reason}} -> diff --git a/lib/kernel/src/kernel.appup.src b/lib/kernel/src/kernel.appup.src index b505524471..2dc90e2b3e 100644 --- a/lib/kernel/src/kernel.appup.src +++ b/lib/kernel/src/kernel.appup.src @@ -18,7 +18,7 @@ %% %CopyrightEnd% {"%VSN%", %% Up from - max one major revision back - [{<<"5\\.[0-1](\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-19.* + [{<<"5\\.[0-2](\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-19.* %% Down to - max one major revision back - [{<<"5\\.[0-1](\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-19.* + [{<<"5\\.[0-2](\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-19.* }. diff --git a/lib/kernel/src/kernel.erl b/lib/kernel/src/kernel.erl index 59eca242b1..b901da95b8 100644 --- a/lib/kernel/src/kernel.erl +++ b/lib/kernel/src/kernel.erl @@ -100,63 +100,112 @@ get_error_logger_type() -> %%%----------------------------------------------------------------- init([]) -> - SupFlags = {one_for_all, 0, 1}, - - Config = {kernel_config, - {kernel_config, start_link, []}, - permanent, 2000, worker, [kernel_config]}, - Code = {code_server, - {code, start_link, []}, - permanent, 2000, worker, [code]}, - File = {file_server_2, - {file_server, start_link, []}, - permanent, 2000, worker, - [file, file_server, file_io_server, prim_file]}, - StdError = {standard_error, - {standard_error, start_link, []}, - temporary, 2000, supervisor, [user_sup]}, - User = {user, - {user_sup, start, []}, - temporary, 2000, supervisor, [user_sup]}, - + SupFlags = #{strategy => one_for_all, + intensity => 0, + period => 1}, + + Config = #{id => kernel_config, + start => {kernel_config, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [kernel_config]}, + + Code = #{id => code_server, + start => {code, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [code]}, + + File = #{id => file_server_2, + start => {file_server, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modeules => [file, file_server, file_io_server, prim_file]}, + + StdError = #{id => standard_error, + start => {standard_error, start_link, []}, + restart => temporary, + shutdown => 2000, + type => supervisor, + modules => [user_sup]}, + + User = #{id => user, + start => {user_sup, start, []}, + restart => temporary, + shutdown => 2000, + type => supervisor, + modules => [user_sup]}, + + SafeSup = #{id => kernel_safe_sup, + start =>{supervisor, start_link, [{local, kernel_safe_sup}, ?MODULE, safe]}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [?MODULE]}, + case init:get_argument(mode) of - {ok, [["minimal"]]} -> - SafeSupervisor = {kernel_safe_sup, - {supervisor, start_link, - [{local, kernel_safe_sup}, ?MODULE, safe]}, - permanent, infinity, supervisor, [?MODULE]}, - {ok, {SupFlags, - [Code, File, StdError, User, - Config, SafeSupervisor]}}; - _ -> - Rpc = {rex, {rpc, start_link, []}, - permanent, 2000, worker, [rpc]}, - Global = {global_name_server, {global, start_link, []}, - permanent, 2000, worker, [global]}, - Glo_grp = {global_group, {global_group,start_link,[]}, - permanent, 2000, worker, [global_group]}, - InetDb = {inet_db, {inet_db, start_link, []}, - permanent, 2000, worker, [inet_db]}, - NetSup = {net_sup, {erl_distribution, start_link, []}, - permanent, infinity, supervisor,[erl_distribution]}, + {ok, [["minimal"]]} -> + {ok, {SupFlags, [Code, File, StdError, User, Config, SafeSup]}}; + _ -> + Rpc = #{id => rex, + start => {rpc, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [rpc]}, + + Global = #{id => global_name_server, + start => {global, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [global]}, + + GlGroup = #{id => global_group, + start => {global_group,start_link,[]}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [global_group]}, + + InetDb = #{id => inet_db, + start => {inet_db, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [inet_db]}, + + NetSup = #{id => net_sup, + start => {erl_distribution, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [erl_distribution]}, + SigSrv = #{id => erl_signal_server, start => {gen_event, start_link, [{local, erl_signal_server}]}, - type => worker, restart => permanent, shutdown => 2000, modules => dynamic}, - DistAC = start_dist_ac(), - - Timer = start_timer(), - - SafeSupervisor = {kernel_safe_sup, - {supervisor, start_link, - [{local, kernel_safe_sup}, ?MODULE, safe]}, - permanent, infinity, supervisor, [?MODULE]}, - {ok, {SupFlags, - [Code, Rpc, Global, InetDb | DistAC] ++ - [NetSup, Glo_grp, File, SigSrv, - StdError, User, Config, SafeSupervisor] ++ Timer}} + restart => permanent, + shutdown => 2000, + type => worker, + modules => dynamic}, + + DistAC = start_dist_ac(), + + Timer = start_timer(), + + {ok, {SupFlags, + [Code, Rpc, Global, InetDb | DistAC] ++ + [NetSup, GlGroup, File, SigSrv, + StdError, User, Config, SafeSup] ++ Timer}} end; init(safe) -> - SupFlags = {one_for_one, 4, 3600}, + SupFlags = #{strategy => one_for_one, + intensity => 4, + period => 3600}, + Boot = start_boot_server(), DiskLog = start_disk_log(), Pg2 = start_pg2(), @@ -170,60 +219,85 @@ init(safe) -> {ok, {SupFlags, Boot ++ DiskLog ++ Pg2}}. start_dist_ac() -> - Spec = [{dist_ac,{dist_ac,start_link,[]},permanent,2000,worker,[dist_ac]}], + Spec = [#{id => dist_ac, + start => {dist_ac,start_link,[]}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [dist_ac]}], case application:get_env(kernel, start_dist_ac) of - {ok, true} -> Spec; - {ok, false} -> []; - undefined -> - case application:get_env(kernel, distributed) of - {ok, _} -> Spec; - _ -> [] - end + {ok, true} -> Spec; + {ok, false} -> []; + undefined -> + case application:get_env(kernel, distributed) of + {ok, _} -> Spec; + _ -> [] + end end. start_boot_server() -> case application:get_env(kernel, start_boot_server) of - {ok, true} -> - Args = get_boot_args(), - [{boot_server, {erl_boot_server, start_link, [Args]}, permanent, - 1000, worker, [erl_boot_server]}]; - _ -> - [] + {ok, true} -> + Args = get_boot_args(), + [#{id => boot_server, + start => {erl_boot_server, start_link, [Args]}, + restart => permanent, + shutdown => 1000, + type => worker, + modules => [erl_boot_server]}]; + _ -> + [] end. get_boot_args() -> case application:get_env(kernel, boot_server_slaves) of - {ok, Slaves} -> Slaves; - _ -> [] + {ok, Slaves} -> Slaves; + _ -> [] end. start_disk_log() -> case application:get_env(kernel, start_disk_log) of - {ok, true} -> - [{disk_log_server, - {disk_log_server, start_link, []}, - permanent, 2000, worker, [disk_log_server]}, - {disk_log_sup, {disk_log_sup, start_link, []}, permanent, - 1000, supervisor, [disk_log_sup]}]; - _ -> - [] + {ok, true} -> + [#{id => disk_log_server, + start => {disk_log_server, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [disk_log_server]}, + #{id => disk_log_sup, + start => {disk_log_sup, start_link, []}, + restart => permanent, + shutdown => 1000, + type => supervisor, + modules => [disk_log_sup]}]; + _ -> + [] end. start_pg2() -> case application:get_env(kernel, start_pg2) of - {ok, true} -> - [{pg2, {pg2, start_link, []}, permanent, 1000, worker, [pg2]}]; - _ -> - [] + {ok, true} -> + [#{id => pg2, + start => {pg2, start_link, []}, + restart => permanent, + shutdown => 1000, + type => worker, + modules => [pg2]}]; + _ -> + [] end. start_timer() -> case application:get_env(kernel, start_timer) of - {ok, true} -> - [{timer_server, {timer, start_link, []}, permanent, 1000, worker, - [timer]}]; - _ -> - [] + {ok, true} -> + [#{id => timer_server, + start => {timer, start_link, []}, + restart => permanent, + shutdown => 1000, + type => worker, + modules => [timer]}]; + _ -> + [] end. %%----------------------------------------------------------------- diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl index b665d7e592..088d851b09 100644 --- a/lib/kernel/test/code_SUITE.erl +++ b/lib/kernel/test/code_SUITE.erl @@ -1366,9 +1366,8 @@ create_big_boot(Config) -> %% corresponding beam file (if hipe is not enabled). filter_app("hipe",_) -> false; -%% Dialyzer and typer depends on hipe +%% Dialyzer depends on hipe filter_app("dialyzer",_) -> false; -filter_app("typer",_) -> false; %% Orber requires explicit configuration filter_app("orber",_) -> false; diff --git a/lib/kernel/vsn.mk b/lib/kernel/vsn.mk index 8d2517e680..76b020e8ed 100644 --- a/lib/kernel/vsn.mk +++ b/lib/kernel/vsn.mk @@ -1 +1 @@ -KERNEL_VSN = 5.1.1 +KERNEL_VSN = 5.2 diff --git a/lib/observer/doc/src/etop.xml b/lib/observer/doc/src/etop.xml index d70d9d1d23..059f9f05d8 100644 --- a/lib/observer/doc/src/etop.xml +++ b/lib/observer/doc/src/etop.xml @@ -35,7 +35,7 @@ <file></file> </header> <module>etop</module> - <modulesummary>Erlang Top is a tool for presenting information about Erlang + <modulesummary>Erlang Top is a tool for presenting information about Erlang processes similar to the information presented by "top" in UNIX.</modulesummary> <description> @@ -60,11 +60,11 @@ <p>Value: <c>atom()</c></p> <p>Mandatory</p></item> <tag><c>setcookie</c></tag> - <item><p>Cookie to use for the <c>etop</c> node. Must be same as the + <item><p>Cookie to use for the <c>etop</c> node. Must be same as the cookie on the measured node.</p> <p>Value: <c>atom()</c></p></item> <tag><c>lines</c></tag> - <item><p>Number of lines (processes) to display.</p> + <item><p>Number of lines (processes) to display.</p> <p>Value: <c>integer()</c></p> <p>Default: <c>10</c></p></item> <tag><c>interval</c></tag> @@ -92,7 +92,7 @@ <p>Default: <c>on</c></p></item> </taglist> - <p>For detalis about Erlang Top, see the + <p>For details about Erlang Top, see the <seealso marker="etop_ug">User's Guide</seealso>.</p> </description> diff --git a/lib/observer/doc/src/notes.xml b/lib/observer/doc/src/notes.xml index 8f3ebcb4de..79e2b2b9db 100644 --- a/lib/observer/doc/src/notes.xml +++ b/lib/observer/doc/src/notes.xml @@ -32,6 +32,47 @@ <p>This document describes the changes made to the Observer application.</p> +<section><title>Observer 2.3.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + etop erroneously reported the average scheduler + utilization since the tool was first started instead of + the scheduler utilization since last update. This is now + corrected.</p> + <p> + Own Id: OTP-14090 Aux Id: seq13232 </p> + </item> + <item> + <p> + crashdump_viewer crashed when the 'Slogan' had more than + one line. This is now corrected.</p> + <p> + Own Id: OTP-14093 Aux Id: ERL-318 </p> + </item> + <item> + <p> + When clicking an HTML-link to a port before the port tab + has been opened for the first time, observer would crash + since port info is not initiated. This is now corrected.</p> + <p> + Own Id: OTP-14151 Aux Id: PR-1296 </p> + </item> + <item> + <p>The dialyzer and observer applications will now use a + portable way to find the home directory. That means that + there is no longer any need to manually set the HOME + environment variable on Windows.</p> + <p> + Own Id: OTP-14249 Aux Id: ERL-161 </p> + </item> + </list> + </section> + +</section> + <section><title>Observer 2.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/observer/doc/src/observer.xml b/lib/observer/doc/src/observer.xml index 4d43ffe39f..fc6395d2c0 100644 --- a/lib/observer/doc/src/observer.xml +++ b/lib/observer/doc/src/observer.xml @@ -43,7 +43,7 @@ <seealso marker="ttb"><c>ttb</c></seealso>. </p> - <p>For detalis about how to get started, see the + <p>For details about how to get started, see the <seealso marker="observer_ug"><c>User's Guide</c></seealso>.</p> </description> <funcs> diff --git a/lib/observer/doc/src/observer_ug.xml b/lib/observer/doc/src/observer_ug.xml index 6eb72f3e58..ae85ab7a29 100644 --- a/lib/observer/doc/src/observer_ug.xml +++ b/lib/observer/doc/src/observer_ug.xml @@ -107,6 +107,11 @@ see module <seealso marker="erts:erts_alloc"><c>erts_alloc</c></seealso> in application ERTS.</p> + <p>The <c>Max Carrier size</c> column shows the maximum value seen by observer + since the last node change or since the start of the application, i.e. switching + nodes will reset the max column. Values are sampled so higher values may have + existed than what is shown. + </p> </section> <section> diff --git a/lib/observer/src/cdv_ets_cb.erl b/lib/observer/src/cdv_ets_cb.erl index ddd2d42df6..18f0c86fd3 100644 --- a/lib/observer/src/cdv_ets_cb.erl +++ b/lib/observer/src/cdv_ets_cb.erl @@ -30,26 +30,23 @@ -include("crashdump_viewer.hrl"). %% Defines --define(COL_ID, 0). --define(COL_NAME, ?COL_ID+1). --define(COL_SLOT, ?COL_NAME+1). --define(COL_OWNER, ?COL_SLOT+1). +-define(COL_NAME, 0). +-define(COL_IS_NAMED, ?COL_NAME+1). +-define(COL_OWNER, ?COL_IS_NAMED+1). -define(COL_OBJ, ?COL_OWNER+1). -define(COL_MEM, ?COL_OBJ+1). %% Callbacks for cdv_virtual_list_wx -col_to_elem(id) -> col_to_elem(?COL_ID); -col_to_elem(?COL_ID) -> #ets_table.id; +col_to_elem(id) -> col_to_elem(?COL_NAME); +col_to_elem(?COL_IS_NAMED) -> #ets_table.is_named; col_to_elem(?COL_NAME) -> #ets_table.name; -col_to_elem(?COL_SLOT) -> #ets_table.slot; col_to_elem(?COL_OWNER) -> #ets_table.pid; col_to_elem(?COL_OBJ) -> #ets_table.size; col_to_elem(?COL_MEM) -> #ets_table.memory. col_spec() -> - [{"Id", ?wxLIST_FORMAT_LEFT, 200}, - {"Name", ?wxLIST_FORMAT_LEFT, 200}, - {"Slot", ?wxLIST_FORMAT_RIGHT, 50}, + [{"Name", ?wxLIST_FORMAT_LEFT, 200}, + {"Is Named", ?wxLIST_FORMAT_CENTRE, 70}, {"Owner", ?wxLIST_FORMAT_CENTRE, 120}, {"Objects", ?wxLIST_FORMAT_RIGHT, 80}, {"Memory", ?wxLIST_FORMAT_RIGHT, 80} @@ -68,7 +65,7 @@ get_details(Id, Data) -> {ok,{"Table:" ++ Id,Proplist,""}}. get_detail_cols(all) -> - {[{ets, ?COL_ID}, {process, ?COL_OWNER}],true}; + {[{ets, ?COL_NAME}, {process, ?COL_OWNER}],true}; get_detail_cols(_W) -> {[],true}. diff --git a/lib/observer/src/crashdump_viewer.erl b/lib/observer/src/crashdump_viewer.erl index 13e73f027d..e21f1c501b 100644 --- a/lib/observer/src/crashdump_viewer.erl +++ b/lib/observer/src/crashdump_viewer.erl @@ -1555,10 +1555,14 @@ split_pid_list_no_space([],[],Pids) -> %% Page with external ets tables get_ets_tables(File,Pid,WS) -> ParseFun = fun(Fd,Id) -> - get_etsinfo(Fd,#ets_table{pid=list_to_pid(Id)},WS) + ET = get_etsinfo(Fd,#ets_table{pid=list_to_pid(Id)},WS), + ET#ets_table{is_named=tab_is_named(ET)} end, lookup_and_parse_index(File,{?ets,Pid},ParseFun,"ets"). +tab_is_named(#ets_table{id=Name,name=Name}) -> "yes"; +tab_is_named(#ets_table{}) -> "no". + get_etsinfo(Fd,EtsTable = #ets_table{details=Ds},WS) -> case line_head(Fd) of "Slot" -> diff --git a/lib/observer/src/crashdump_viewer.hrl b/lib/observer/src/crashdump_viewer.hrl index a08659efd6..742e145641 100644 --- a/lib/observer/src/crashdump_viewer.hrl +++ b/lib/observer/src/crashdump_viewer.hrl @@ -118,6 +118,7 @@ slot, id, name, + is_named, data_type="hash", buckets="-", size, diff --git a/lib/observer/src/observer_alloc_wx.erl b/lib/observer/src/observer_alloc_wx.erl index ca54080e15..9e1442a5ca 100644 --- a/lib/observer/src/observer_alloc_wx.erl +++ b/lib/observer/src/observer_alloc_wx.erl @@ -18,7 +18,7 @@ %% %CopyrightEnd% -module(observer_alloc_wx). --export([start_link/2]). +-export([start_link/3]). %% wx_object callbacks -export([init/1, handle_info/2, terminate/2, code_change/3, handle_call/3, @@ -36,6 +36,7 @@ wins, mem, samples, + max, panel, paint, appmon, @@ -48,10 +49,10 @@ [make_win/4, setup_graph_drawing/1, refresh_panel/4, interval_dialog/2, add_data/5, precalc/4]). -start_link(Notebook, Parent) -> - wx_object:start_link(?MODULE, [Notebook, Parent], []). +start_link(Notebook, Parent, Config) -> + wx_object:start_link(?MODULE, [Notebook, Parent, Config], []). -init([Notebook, Parent]) -> +init([Notebook, Parent, Config]) -> try TopP = wxPanel:new(Notebook), Main = wxBoxSizer:new(?wxVERTICAL), @@ -74,7 +75,8 @@ init([Notebook, Parent]) -> wins = Windows, mem = MemWin, paint = PaintInfo, - time = setup_time() + time = setup_time(Config), + max = #{} } } catch _:Err -> @@ -82,9 +84,11 @@ init([Notebook, Parent]) -> {stop, Err} end. -setup_time() -> - Freq = 1, - #ti{fetch=Freq, disp=?DISP_FREQ/Freq}. +setup_time(Config) -> + Freq = maps:get(fetch, Config, 1), + #ti{disp=?DISP_FREQ/Freq, + fetch=Freq, + secs=maps:get(secs, Config, ?DISP_SECONDS)}. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% handle_event(#wx{id=?ID_REFRESH_INTERVAL, event=#wxCommand{type=command_menu_selected}}, @@ -115,6 +119,10 @@ handle_sync_event(#wx{obj=Panel, event = #wxPaint{}},_, refresh_panel(Active, Win, Ti, Paint), ok. %%%%%%%%%% +handle_call(get_config, _, #state{time=Ti}=State) -> + #ti{fetch=Fetch, secs=Range} = Ti, + {reply, #{fetch=>Fetch, secs=>Range}, State}; + handle_call(Event, From, _State) -> error({unhandled_call, Event, From}). @@ -126,16 +134,17 @@ handle_info({Key, {promise_reply, {badrpc, _}}}, #state{async=Key} = State) -> {noreply, State#state{active=false, appmon=undefined}}; handle_info({Key, {promise_reply, SysInfo}}, - #state{async=Key, panel=_Panel, samples=Data, active=Active, wins=Wins0, - time=#ti{tick=Tick, disp=Disp0}=Ti} = S0) -> + #state{async=Key, samples=Data, max=Max0, + active=Active, wins=Wins0, time=#ti{tick=Tick, disp=Disp0}=Ti} = S0) -> Disp = trunc(Disp0), Next = max(Tick - Disp, 0), erlang:send_after(1000 div ?DISP_FREQ, self(), {refresh, Next}), Info = alloc_info(SysInfo), + Max = lists:foldl(fun calc_max/2, Max0, Info), {Wins, Samples} = add_data(Info, Data, Wins0, Ti, Active), - S1 = S0#state{time=Ti#ti{tick=Next}, wins=Wins, samples=Samples, async=undefined}, + S1 = S0#state{time=Ti#ti{tick=Next}, wins=Wins, samples=Samples, max=Max, async=undefined}, if Active -> - update_alloc(S0, Info), + update_alloc(S0, Info, Max), State = precalc(S1), {noreply, State}; true -> @@ -187,25 +196,35 @@ code_change(_, _, State) -> restart_fetcher(Node, #state{panel=Panel, wins=Wins0, time=Ti} = State) -> SysInfo = observer_wx:try_rpc(Node, observer_backend, sys_info, []), Info = alloc_info(SysInfo), + Max = lists:foldl(fun calc_max/2, #{}, Info), {Wins, Samples} = add_data(Info, {0, queue:new()}, Wins0, Ti, true), erlang:send_after(1000 div ?DISP_FREQ, self(), {refresh, 0}), wxWindow:refresh(Panel), precalc(State#state{active=true, appmon=Node, time=Ti#ti{tick=0}, - wins=Wins, samples=Samples}). + wins=Wins, samples=Samples, max=Max}). precalc(#state{samples=Data0, paint=Paint, time=Ti, wins=Wins0}=State) -> Wins = [precalc(Ti, Data0, Paint, Win) || Win <- Wins0], State#state{wins=Wins}. +calc_max({Name, _, Cs}, Max0) -> + case maps:get(Name, Max0, 0) of + Value when Value < Cs -> + Max0#{Name=>Cs}; + _V -> + Max0 + end. -update_alloc(#state{mem=Grid}, Fields) -> +update_alloc(#state{mem=Grid}, Fields, Max) -> wxWindow:freeze(Grid), - Max = wxListCtrl:getItemCount(Grid), + Last = wxListCtrl:getItemCount(Grid), Update = fun({Name, BS, CS}, Row) -> - (Row >= Max) andalso wxListCtrl:insertItem(Grid, Row, ""), + (Row >= Last) andalso wxListCtrl:insertItem(Grid, Row, ""), + MaxV = maps:get(Name, Max, CS), wxListCtrl:setItem(Grid, Row, 0, observer_lib:to_str(Name)), wxListCtrl:setItem(Grid, Row, 1, observer_lib:to_str(BS div 1024)), wxListCtrl:setItem(Grid, Row, 2, observer_lib:to_str(CS div 1024)), + wxListCtrl:setItem(Grid, Row, 3, observer_lib:to_str(MaxV div 1024)), Row + 1 end, wx:foldl(Update, 0, Fields), @@ -269,7 +288,9 @@ create_mem_info(Parent) -> end, ListItems = [{"Allocator Type", ?wxLIST_FORMAT_LEFT, 200}, {"Block size (kB)", ?wxLIST_FORMAT_RIGHT, 150}, - {"Carrier size (kB)",?wxLIST_FORMAT_RIGHT, 150}], + {"Carrier size (kB)",?wxLIST_FORMAT_RIGHT, 150}, + {"Max Carrier size (kB)",?wxLIST_FORMAT_RIGHT, 150} + ], lists:foldl(AddListEntry, 0, ListItems), wxListItem:destroy(Li), diff --git a/lib/observer/src/observer_app_wx.erl b/lib/observer/src/observer_app_wx.erl index 80a41fdde9..63ca3aeba7 100644 --- a/lib/observer/src/observer_app_wx.erl +++ b/lib/observer/src/observer_app_wx.erl @@ -18,7 +18,7 @@ %% %CopyrightEnd% -module(observer_app_wx). --export([start_link/2]). +-export([start_link/3]). %% wx_object callbacks -export([init/1, handle_info/2, terminate/2, code_change/3, handle_call/3, @@ -73,10 +73,10 @@ -define(wxGC, wxGraphicsContext). -start_link(Notebook, Parent) -> - wx_object:start_link(?MODULE, [Notebook, Parent], []). +start_link(Notebook, Parent, Config) -> + wx_object:start_link(?MODULE, [Notebook, Parent, Config], []). -init([Notebook, Parent]) -> +init([Notebook, Parent, _Config]) -> Panel = wxPanel:new(Notebook, [{size, wxWindow:getClientSize(Notebook)}, {winid, 1} ]), @@ -258,6 +258,8 @@ handle_sync_event(#wx{event = #wxPaint{}},_, destroy_gc(GC), ok. %%%%%%%%%% +handle_call(get_config, _, State) -> + {reply, #{}, State}; handle_call(Event, From, _State) -> error({unhandled_call, Event, From}). diff --git a/lib/observer/src/observer_lib.erl b/lib/observer/src/observer_lib.erl index 47844c1307..68095d7f58 100644 --- a/lib/observer/src/observer_lib.erl +++ b/lib/observer/src/observer_lib.erl @@ -24,7 +24,7 @@ display_progress_dialog/2, destroy_progress_dialog/0, wait_for_progress/0, report_progress/1, user_term/3, user_term_multiline/3, - interval_dialog/4, start_timer/1, stop_timer/1, + interval_dialog/4, start_timer/1, start_timer/2, stop_timer/1, timer_config/1, display_info/2, display_info/3, fill_info/2, update_info/2, to_str/1, create_menus/3, create_menu_item/3, create_attrs/0, @@ -90,6 +90,12 @@ stop_timer(Timer = {true, _}) -> Timer; stop_timer(Timer = {_, Intv}) -> setup_timer(false, Timer), {true, Intv}. + +start_timer(#{interval:=Intv}, _Def) -> + setup_timer(true, {false, Intv}); +start_timer(_, Def) -> + setup_timer(true, {false, Def}). + start_timer(Intv) when is_integer(Intv) -> setup_timer(true, {true, Intv}); start_timer(Timer) -> @@ -105,6 +111,11 @@ setup_timer(Bool, {Timer, Old}) -> timer:cancel(Timer), setup_timer(Bool, {false, Old}). +timer_config({_, Interval}) -> + #{interval=>Interval}; +timer_config(#{}=Config) -> + Config. + display_info_dialog(Parent,Str) -> display_info_dialog(Parent,"",Str). display_info_dialog(Parent,Title,Str) -> diff --git a/lib/observer/src/observer_perf_wx.erl b/lib/observer/src/observer_perf_wx.erl index b0ead42e3f..fc5fb226db 100644 --- a/lib/observer/src/observer_perf_wx.erl +++ b/lib/observer/src/observer_perf_wx.erl @@ -18,7 +18,7 @@ %% %CopyrightEnd% -module(observer_perf_wx). --export([start_link/2]). +-export([start_link/3]). %% wx_object callbacks -export([init/1, handle_info/2, terminate/2, code_change/3, handle_call/3, @@ -55,12 +55,12 @@ -define(wxGC, wxGraphicsContext). --record(paint, {font, small, pen, pen2, pens, usegc = false}). +-record(paint, {font, small, pen, pen2, pens, dot_pens, usegc = false}). -start_link(Notebook, Parent) -> - wx_object:start_link(?MODULE, [Notebook, Parent], []). +start_link(Notebook, Parent, Config) -> + wx_object:start_link(?MODULE, [Notebook, Parent, Config], []). -init([Notebook, Parent]) -> +init([Notebook, Parent, Config]) -> try Panel = wxPanel:new(Notebook), Main = wxBoxSizer:new(?wxVERTICAL), @@ -81,7 +81,9 @@ init([Notebook, Parent]) -> panel =Panel, wins = Windows, paint=PaintInfo, - samples=reset_data() + samples=reset_data(), + time=#ti{fetch=maps:get(fetch, Config, ?FETCH_DATA), + secs=maps:get(secs, Config, ?DISP_SECONDS)} }, {Panel, State0} catch _:Err -> @@ -124,13 +126,17 @@ setup_graph_drawing(Panels) -> {F, SF} end, BlackPen = wxPen:new({0,0,0}, [{width, 1}]), - Pens = [wxPen:new(Col, [{width, 1}]) || Col <- tuple_to_list(colors())], + Pens = [wxPen:new(Col, [{width, 1}, {style, ?wxSOLID}]) + || Col <- tuple_to_list(colors())], + DotPens = [wxPen:new(Col, [{width, 1}, {style, ?wxDOT}]) + || Col <- tuple_to_list(colors())], #paint{usegc = UseGC, font = Font, small = SmallFont, pen = ?wxGREY_PEN, pen2 = BlackPen, - pens = list_to_tuple(Pens) + pens = list_to_tuple(Pens), + dot_pens = list_to_tuple(DotPens) }. @@ -173,6 +179,10 @@ refresh_panel(Active, #win{name=_Id, panel=Panel}=Win, Ti, #paint{usegc=UseGC}=P destroy_gc(GC). %%%%%%%%%% +handle_call(get_config, _, #state{time=Ti}=State) -> + #ti{fetch=Fetch, secs=Range} = Ti, + {reply, #{fetch=>Fetch, secs=>Range}, State}; + handle_call(Event, From, _State) -> error({unhandled_call, Event, From}). @@ -181,17 +191,17 @@ handle_cast(Event, _State) -> %%%%%%%%%% handle_info({stats, 1, _, _, _} = Stats, #state{panel=Panel, samples=Data, active=Active, wins=Wins0, - time=#ti{tick=Tick, disp=Disp0}=Ti} = State0) -> + appmon=Node, time=#ti{tick=Tick, disp=Disp0}=Ti} = State0) -> if Active -> Disp = trunc(Disp0), Next = max(Tick - Disp, 0), erlang:send_after(1000 div ?DISP_FREQ, self(), {refresh, Next}), - {Wins, Samples} = add_data(Stats, Data, Wins0, Ti, Active), + {Wins, Samples} = add_data(Stats, Data, Wins0, Ti, Active, Node), State = precalc(State0#state{time=Ti#ti{tick=Next}, wins=Wins, samples=Samples}), wxWindow:refresh(Panel), {noreply, State}; true -> - {Wins1, Samples} = add_data(Stats, Data, Wins0, Ti, Active), + {Wins1, Samples} = add_data(Stats, Data, Wins0, Ti, Active, Node), Wins = [W#win{max=undefined} || W <- Wins1], {noreply, State0#state{samples=Samples, wins=Wins, time=Ti#ti{tick=0}}} end; @@ -206,7 +216,7 @@ handle_info({refresh, Seq}, #state{panel=Panel, time=#ti{tick=Seq, disp=DispF}=T handle_info({refresh, _}, State) -> {noreply, State}; -handle_info({active, Node}, #state{parent=Parent, panel=Panel, appmon=Old, time=_Ti} = State) -> +handle_info({active, Node}, #state{parent=Parent, panel=Panel, appmon=Old} = State) -> create_menus(Parent, []), try Node = node(Old), @@ -247,13 +257,17 @@ restart_fetcher(Node, #state{appmon=Old, panel=Panel, time=#ti{fetch=Freq}=Ti, w reset_data() -> {0, queue:new()}. -add_data(Stats, {N, Q0}, Wins, #ti{fetch=Fetch, secs=Secs}, Active) when N > (Secs*Fetch+1) -> +add_data(Stats, Q, Wins, Ti, Active) -> + add_data(Stats, Q, Wins, Ti, Active, ignore). + +add_data(Stats, {N, Q0}, Wins, #ti{fetch=Fetch, secs=Secs}, Active, Node) + when N > (Secs*Fetch+1) -> {{value, Drop}, Q} = queue:out(Q0), - add_data_1(Wins, Stats, N, {Drop,Q}, Active); -add_data(Stats, {N, Q}, Wins, _, Active) -> - add_data_1(Wins, Stats, N+1, {empty, Q}, Active). + add_data_1(Wins, Stats, N, {Drop,Q}, Active, Node); +add_data(Stats, {N, Q}, Wins, _, Active, Node) -> + add_data_1(Wins, Stats, N+1, {empty, Q}, Active, Node). -add_data_1([#win{state={_,St}}|_]=Wins0, Last, N, {Drop, Q}, Active) +add_data_1([#win{state={_,St}}|_]=Wins0, Last, N, {Drop, Q}, Active, Node) when St /= undefined -> try {Wins, Stat} = @@ -269,14 +283,12 @@ add_data_1([#win{state={_,St}}|_]=Wins0, Last, N, {Drop, Q}, Active) end, #{}, Wins0), {Wins, {N,queue:in(Stat#{}, Q)}} catch no_scheduler_change -> - {[Win#win{state=init_data(Id, Last), - info = info(Id, Last)} + {[Win#win{state=init_data(Id, Last), info=info(Id, Last, Node)} || #win{name=Id}=Win <- Wins0], {0,queue:new()}} end; -add_data_1(Wins, Stats, 1, {_, Q}, _) -> - {[Win#win{state=init_data(Id, Stats), - info = info(Id, Stats)} +add_data_1(Wins, Stats, 1, {_, Q}, _, Node) -> + {[Win#win{state=init_data(Id, Stats), info=info(Id, Stats, Node)} || #win{name=Id}=Win <- Wins], {0,Q}}. add_data_2(#win{name=Id, state=S0}=Win, Stats, Map) -> @@ -382,16 +394,24 @@ lmax(MState, Values, State) -> init_data(runq, {stats, _, T0, _, _}) -> {mk_max(),lists:sort(T0)}; init_data(io, {stats, _, _, {{_,In0}, {_,Out0}}, _}) -> {mk_max(), {In0,Out0}}; -init_data(memory, _) -> {mk_max(), info(memory, undefined)}; +init_data(memory, _) -> {mk_max(), info(memory, undefined, undefined)}; init_data(alloc, _) -> {mk_max(), unused}; init_data(utilz, _) -> {mk_max(), unused}. -info(runq, {stats, _, T0, _, _}) -> lists:seq(1, length(T0)); -info(memory, _) -> [total, processes, atom, binary, code, ets]; -info(io, _) -> [input, output]; -info(alloc, First) -> [Type || {Type, _, _} <- First]; -info(utilz, First) -> [Type || {Type, _, _} <- First]; -info(_, []) -> []. +info(runq, {stats, _, T0, _, _}, Node) -> + Dirty = get_dirty_cpu(Node), + {lists:seq(1, length(T0)-Dirty), Dirty}; +info(memory, _, _) -> [total, processes, atom, binary, code, ets]; +info(io, _, _) -> [input, output]; +info(alloc, First, _) -> [Type || {Type, _, _} <- First]; +info(utilz, First, _) -> [Type || {Type, _, _} <- First]; +info(_, [], _) -> []. + +get_dirty_cpu(Node) -> + case rpc:call(node(Node), erlang, system_info, [dirty_cpu_schedulers]) of + {badrpc,_R} -> 0; + N -> N + end. collect_data(runq, {stats, _, T0, _, _}, {Max,S0}) -> S1 = lists:sort(T0), @@ -471,9 +491,10 @@ window_geom({W,H}, {_, Max, _Unit, MaxUnit}, %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -draw_win(DC, #win{no_samples=Samples, geom=#{scale:={WS,HS}}, graphs=Graphs, max={_,Max,_,_}}=Win, +draw_win(DC, #win{name=Name, no_samples=Samples, geom=#{scale:={WS,HS}}, + graphs=Graphs, max={_,Max,_,_}, info=Info}=Win, #ti{tick=Tick, fetch=FetchFreq, secs=Secs, disp=DispFreq}=Ti, - Paint=#paint{pens=Pens}) when Samples >= 2, Graphs =/= [] -> + Paint=#paint{pens=Pens, dot_pens=Dots}) when Samples >= 2, Graphs =/= [] -> %% Draw graphs {X0,Y0,DrawBs} = draw_borders(DC, Ti, Win, Paint), Offset = Tick / DispFreq, @@ -483,14 +504,23 @@ draw_win(DC, #win{no_samples=Samples, geom=#{scale:={WS,HS}}, graphs=Graphs, max end, Start = X0 + (max(Secs*FetchFreq+Full-Samples, 0) - Offset)*WS, Last = Secs*FetchFreq*WS+X0, + Dirty = case {Name, Info} of + {runq, {_, DCpu}} -> DCpu; + _ -> 0 + end, + NoGraphs = length(Graphs), + NoCpu = NoGraphs - Dirty, Draw = fun(Lines0, N) -> - setPen(DC, element(1+ ((N-1) rem tuple_size(Pens)), Pens)), + case Dirty > 0 andalso N > NoCpu of + true -> setPen(DC, element(1+ ((N-NoCpu-1) rem tuple_size(Dots)), Dots)); + false -> setPen(DC, element(1+ ((N-1) rem tuple_size(Pens)), Pens)) + end, Order = lists:reverse(Lines0), [{_,Y}|Lines] = translate(Order, {Start, Y0}, 0, WS, {X0,Max*HS,Last}, []), strokeLines(DC, [{Last,Y}|Lines]), N-1 end, - lists:foldl(Draw, length(Graphs), Graphs), + lists:foldl(Draw, NoGraphs, Graphs), DrawBs(), ok; @@ -655,11 +685,17 @@ draw_borders(DC, #ti{secs=Secs, fetch=FetchFreq}, case Type of runq -> + {TextInfo, DirtyCpus} = Info, drawText(DC, "Scheduler Utilization (%) ", TopTextX, ?BH), TN0 = Text(TopTextX, BottomTextY, "Scheduler: ", 0), - lists:foldl(fun(Id, Pos0) -> - Text(Pos0, BottomTextY, integer_to_list(Id), Id) - end, TN0, Info); + Id = fun(Id, Pos0) -> + Text(Pos0, BottomTextY, integer_to_list(Id), Id) + end, + TN1 = lists:foldl(Id, TN0, TextInfo), + TN2 = Text(TN1, BottomTextY, "Dirty cpu: ", 0), + TN3 = lists:foldl(Id, TN2, lists:seq(1, DirtyCpus)), + _ = Text(TN3, BottomTextY, "(dotted)", 0), + ok; memory -> drawText(DC, "Memory Usage " ++ Unit, TopTextX,?BH), lists:foldl(fun(MType, {PenId, Pos0}) -> @@ -748,10 +784,10 @@ calc_max1(Max) -> end. colors() -> - {{240, 100, 100}, {100, 240, 100}, {100, 100, 240}, - {220, 220, 80}, {100, 240, 240}, {240, 100, 240}, - {100, 25, 25}, {25, 100, 25}, {25, 25, 100}, - {120, 120, 0}, {25, 100, 100}, {100, 50, 100} + {{240, 100, 100}, {0, 128, 0}, {25, 45, 170}, {255, 165, 0}, + {220, 220, 40}, {100, 240, 240},{240, 100, 240}, {160, 40, 40}, + {100, 100, 240}, {140, 140, 0}, {25, 200, 100}, {120, 25, 240}, + {255, 140, 163}, {25, 120, 120}, {120, 25, 120}, {110, 90, 60} }. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/observer/src/observer_port_wx.erl b/lib/observer/src/observer_port_wx.erl index c21d2705c0..db5e6ceb38 100644 --- a/lib/observer/src/observer_port_wx.erl +++ b/lib/observer/src/observer_port_wx.erl @@ -18,7 +18,7 @@ %% %CopyrightEnd% -module(observer_port_wx). --export([start_link/2]). +-export([start_link/3]). %% wx_object callbacks -export([init/1, handle_info/2, terminate/2, code_change/3, handle_call/3, @@ -77,10 +77,10 @@ open_wins=[] }). -start_link(Notebook, Parent) -> - wx_object:start_link(?MODULE, [Notebook, Parent], []). +start_link(Notebook, Parent, Config) -> + wx_object:start_link(?MODULE, [Notebook, Parent, Config], []). -init([Notebook, Parent]) -> +init([Notebook, Parent, Config]) -> Panel = wxPanel:new(Notebook), Sizer = wxBoxSizer:new(?wxVERTICAL), Style = ?wxLC_REPORT bor ?wxLC_HRULES, @@ -110,12 +110,12 @@ init([Notebook, Parent]) -> wxListCtrl:connect(Grid, size, [{skip, true}]), wxWindow:setFocus(Grid), - {Panel, #state{grid=Grid, parent=Parent, panel=Panel, timer={false, 10}}}. + {Panel, #state{grid=Grid, parent=Parent, panel=Panel, timer=Config}}. handle_event(#wx{id=?ID_REFRESH}, State = #state{node=Node, grid=Grid, opt=Opt}) -> Ports0 = get_ports(Node), - Ports = update_grid(Grid, Opt, Ports0), + Ports = update_grid(Grid, sel(State), Opt, Ports0), {noreply, State#state{ports=Ports}}; handle_event(#wx{obj=Obj, event=#wxClose{}}, #state{open_wins=Opened} = State) -> @@ -134,7 +134,7 @@ handle_event(#wx{event=#wxList{type=command_list_col_click, col=Col}}, NewKey -> Opt0#opt{sort_key=NewKey} end, Ports0 = get_ports(Node), - Ports = update_grid(Grid, Opt, Ports0), + Ports = update_grid(Grid, sel(State), Opt, Ports0), wxWindow:setFocus(Grid), {noreply, State#state{opt=Opt, ports=Ports}}; @@ -260,6 +260,9 @@ handle_event(Event, _State) -> handle_sync_event(_Event, _Obj, _State) -> ok. +handle_call(get_config, _, #state{timer=Timer}=State) -> + {reply, observer_lib:timer_config(Timer), State}; + handle_call(Event, From, _State) -> error({unhandled_call, Event, From}). @@ -269,7 +272,7 @@ handle_cast(Event, _State) -> handle_info({portinfo_open, PortIdStr}, State = #state{node=Node, grid=Grid, opt=Opt, open_wins=Opened}) -> Ports0 = get_ports(Node), - Ports = update_grid(Grid, Opt, Ports0), + Ports = update_grid(Grid, sel(State), Opt, Ports0), Port = lists:keyfind(PortIdStr, #port.id_str, Ports), NewOpened = case Port of @@ -288,17 +291,17 @@ handle_info(refresh_interval, State = #state{node=Node, grid=Grid, opt=Opt, %% no change {noreply, State}; Ports0 -> - Ports = update_grid(Grid, Opt, Ports0), + Ports = update_grid(Grid, sel(State), Opt, Ports0), {noreply, State#state{ports=Ports}} end; handle_info({active, Node}, State = #state{parent=Parent, grid=Grid, opt=Opt, timer=Timer0}) -> Ports0 = get_ports(Node), - Ports = update_grid(Grid, Opt, Ports0), + Ports = update_grid(Grid, sel(State), Opt, Ports0), wxWindow:setFocus(Grid), create_menus(Parent), - Timer = observer_lib:start_timer(Timer0), + Timer = observer_lib:start_timer(Timer0, 10), {noreply, State#state{node=Node, ports=Ports, timer=Timer}}; handle_info(not_active, State = #state{timer = Timer0}) -> @@ -511,9 +514,9 @@ filter_monitor_info() -> [Pid || {process, Pid} <- Ms] end. -update_grid(Grid, Opt, Ports) -> - wx:batch(fun() -> update_grid2(Grid, Opt, Ports) end). -update_grid2(Grid, #opt{sort_key=Sort,sort_incr=Dir}, Ports) -> +update_grid(Grid, Sel, Opt, Ports) -> + wx:batch(fun() -> update_grid2(Grid, Sel, Opt, Ports) end). +update_grid2(Grid, Sel, #opt{sort_key=Sort,sort_incr=Dir}, Ports) -> wxListCtrl:deleteAllItems(Grid), Update = fun(#port{id = Id, @@ -533,6 +536,12 @@ update_grid2(Grid, #opt{sort_key=Sort,sort_incr=Dir}, Ports) -> observer_lib:to_str(Val)) end, [{0,Id},{1,Connected},{2,Name},{3,Ctrl},{4,Slot}]), + case lists:member(Id, Sel) of + true -> + wxListCtrl:setItemState(Grid, Row, 16#FFFF, ?wxLIST_STATE_SELECTED); + false -> + wxListCtrl:setItemState(Grid, Row, 0, ?wxLIST_STATE_SELECTED) + end, Row + 1 end, PortInfo = case Dir of @@ -542,6 +551,8 @@ update_grid2(Grid, #opt{sort_key=Sort,sort_incr=Dir}, Ports) -> lists:foldl(Update, 0, PortInfo), PortInfo. +sel(#state{grid=Grid, ports=Ports}) -> + [Id || #port{id=Id} <- get_selected_items(Grid, Ports)]. get_selected_items(Grid, Data) -> get_indecies(get_selected_items(Grid, -1, []), Data). diff --git a/lib/observer/src/observer_pro_wx.erl b/lib/observer/src/observer_pro_wx.erl index f07b9e295a..3ecf8bdd92 100644 --- a/lib/observer/src/observer_pro_wx.erl +++ b/lib/observer/src/observer_pro_wx.erl @@ -20,7 +20,7 @@ -behaviour(wx_object). --export([start_link/2]). +-export([start_link/3]). %% wx_object callbacks -export([init/1, handle_info/2, terminate/2, code_change/3, handle_call/3, @@ -86,18 +86,19 @@ right_clicked_pid, holder}). -start_link(Notebook, Parent) -> - wx_object:start_link(?MODULE, [Notebook, Parent], []). +start_link(Notebook, Parent, Config) -> + wx_object:start_link(?MODULE, [Notebook, Parent, Config], []). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -init([Notebook, Parent]) -> +init([Notebook, Parent, Config]) -> Attrs = observer_lib:create_attrs(), Self = self(), - Holder = spawn_link(fun() -> init_table_holder(Self, Attrs) end), - {ProPanel, State} = setup(Notebook, Parent, Holder), + Acc = maps:get(acc, Config, false), + Holder = spawn_link(fun() -> init_table_holder(Self, Acc, Attrs) end), + {ProPanel, State} = setup(Notebook, Parent, Holder, Config), {ProPanel, State#state{holder=Holder}}. -setup(Notebook, Parent, Holder) -> +setup(Notebook, Parent, Holder, Config) -> ProPanel = wxPanel:new(Notebook, []), Grid = create_list_box(ProPanel, Holder), @@ -113,7 +114,7 @@ setup(Notebook, Parent, Holder) -> panel=ProPanel, parent_notebook=Notebook, holder=Holder, - timer={false, 10} + timer=Config }, {ProPanel, State}. @@ -246,7 +247,7 @@ handle_info({active, Node}, #state{holder=Holder, timer=Timer, parent=Parent}=State) -> create_pro_menu(Parent, Holder), Holder ! {change_node, Node}, - {noreply, State#state{timer=observer_lib:start_timer(Timer)}}; + {noreply, State#state{timer=observer_lib:start_timer(Timer, 10)}}; handle_info(not_active, #state{timer=Timer0}=State) -> Timer = observer_lib:stop_timer(Timer0), @@ -264,11 +265,15 @@ terminate(_Reason, #state{holder=Holder}) -> code_change(_, _, State) -> {ok, State}. +handle_call(get_config, _, #state{holder=Holder, timer=Timer}=State) -> + Conf = observer_lib:timer_config(Timer), + Accum = call(Holder, {get_accum, self()}), + {reply, Conf#{acc=>Accum}, State}; + handle_call(Msg, _From, State) -> io:format("~p:~p: Unhandled call ~p~n",[?MODULE, ?LINE, Msg]), {reply, ok, State}. - handle_cast(Msg, State) -> io:format("~p:~p: Unhandled cast ~p~n", [?MODULE, ?LINE, Msg]), {noreply, State}. @@ -453,14 +458,19 @@ rm_selected(_, [], [], AccIds, AccPids) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%TABLE HOLDER%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -init_table_holder(Parent, Attrs) -> +init_table_holder(Parent, Accum0, Attrs) -> Backend = spawn_link(node(), observer_backend,etop_collect,[self()]), + Accum = case Accum0 of + true -> true; + false -> [] + end, table_holder(#holder{parent=Parent, etop=#etop_info{}, info=array:new(), node=node(), backend_pid=Backend, - attrs=Attrs + attrs=Attrs, + accum=Accum }). table_holder(#holder{info=Info, attrs=Attrs, diff --git a/lib/observer/src/observer_sys_wx.erl b/lib/observer/src/observer_sys_wx.erl index fa824995f7..2529e79e20 100644 --- a/lib/observer/src/observer_sys_wx.erl +++ b/lib/observer/src/observer_sys_wx.erl @@ -20,7 +20,7 @@ -behaviour(wx_object). --export([start_link/2]). +-export([start_link/3]). %% wx_object callbacks -export([init/1, handle_info/2, terminate/2, code_change/3, handle_call/3, handle_event/2, handle_cast/2]). @@ -41,12 +41,12 @@ fields, timer}). -start_link(Notebook, Parent) -> - wx_object:start_link(?MODULE, [Notebook, Parent], []). +start_link(Notebook, Parent, Config) -> + wx_object:start_link(?MODULE, [Notebook, Parent, Config], []). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -init([Notebook, Parent]) -> +init([Notebook, Parent, Config]) -> SysInfo = observer_backend:sys_info(), {Sys, Mem, Cpu, Stats} = info_fields(), Panel = wxPanel:new(Notebook), @@ -69,7 +69,7 @@ init([Notebook, Parent]) -> wxSizer:add(Sizer, HSizer1, [{flag, ?wxEXPAND bor BorderFlags bor ?wxBOTTOM}, {proportion, 0}, {border, 5}]), wxPanel:setSizer(Panel, Sizer), - Timer = observer_lib:start_timer(10), + Timer = observer_lib:start_timer(Config, 10), {Panel, #sys_wx_state{parent=Parent, parent_notebook=Notebook, panel=Panel, sizer=Sizer, @@ -167,6 +167,9 @@ terminate(_Reason, _State) -> code_change(_, _, State) -> {ok, State}. +handle_call(get_config, _, #sys_wx_state{timer=Timer}=State) -> + {reply, observer_lib:timer_config(Timer), State}; + handle_call(Msg, _From, State) -> io:format("~p~p: Unhandled Call ~p~n",[?MODULE, ?LINE, Msg]), {reply, ok, State}. diff --git a/lib/observer/src/observer_trace_wx.erl b/lib/observer/src/observer_trace_wx.erl index af90e2100c..247a4608d5 100644 --- a/lib/observer/src/observer_trace_wx.erl +++ b/lib/observer/src/observer_trace_wx.erl @@ -19,7 +19,7 @@ -module(observer_trace_wx). --export([start_link/2, add_processes/1, add_ports/1]). +-export([start_link/3, add_processes/1, add_ports/1]). -export([init/1, handle_info/2, terminate/2, code_change/3, handle_call/3, handle_event/2, handle_cast/2]). @@ -88,8 +88,8 @@ -record(titem, {id, opts}). -start_link(Notebook, ParentPid) -> - wx_object:start_link(?MODULE, [Notebook, ParentPid], []). +start_link(Notebook, ParentPid, Config) -> + wx_object:start_link(?MODULE, [Notebook, ParentPid, Config], []). add_processes(Pids) when is_list(Pids) -> wx_object:cast(observer_wx:get_tracer(), {add_processes, Pids}). @@ -99,10 +99,10 @@ add_ports(Ports) when is_list(Ports) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -init([Notebook, ParentPid]) -> - wx:batch(fun() -> create_window(Notebook, ParentPid) end). +init([Notebook, ParentPid, Config]) -> + wx:batch(fun() -> create_window(Notebook, ParentPid, Config) end). -create_window(Notebook, ParentPid) -> +create_window(Notebook, ParentPid, Config) -> %% Create the window Panel = wxPanel:new(Notebook, [{size, wxWindow:getClientSize(Notebook)}]), Sizer = wxBoxSizer:new(?wxVERTICAL), @@ -130,11 +130,16 @@ create_window(Notebook, ParentPid) -> wxSizer:add(Sizer, Buttons, [{flag, ?wxLEFT bor ?wxRIGHT bor ?wxDOWN}, {border, 5}, {proportion,0}]), wxWindow:setSizer(Panel, Sizer), + MS = parse_ms(maps:get(match_specs, Config, []), default_matchspecs()), {Panel, #state{parent=ParentPid, panel=Panel, n_view=NodeView, proc_view=ProcessView, port_view=PortView, m_view=ModView, f_view=FuncView, toggle_button = ToggleButton, - match_specs=default_matchspecs()}}. + output=maps:get(output, Config, []), + def_proc_flags=maps:get(procflags, Config, []), + def_port_flags=maps:get(portflags, Config, []), + match_specs=MS + }}. default_matchspecs() -> [{Key,default_matchspecs(Key)} || Key <- [funcs,send,'receive']]. @@ -397,27 +402,19 @@ handle_event(#wx{id=?LOG_SAVE, userData=TCtrl}, #state{panel=Panel} = State) -> {noreply, State}; handle_event(#wx{id = ?SAVE_TRACEOPTS}, - #state{panel = Panel, - def_proc_flags = ProcFlags, - def_port_flags = PortFlags, - match_specs = MatchSpecs, - tpatterns = TracePatterns, - output = Output - } = State) -> + #state{panel = Panel} = State) -> Dialog = wxFileDialog:new(Panel, [{style, ?wxFD_SAVE bor ?wxFD_OVERWRITE_PROMPT}]), case wxFileDialog:showModal(Dialog) of ?wxID_OK -> Path = wxFileDialog:getPath(Dialog), - write_file(Panel, Path, - ProcFlags, PortFlags, MatchSpecs, Output, - dict:to_list(TracePatterns) - ); + write_file(Panel, Path, get_config(State)); _ -> ok end, wxDialog:destroy(Dialog), {noreply, State}; + handle_event(#wx{id = ?LOAD_TRACEOPTS}, #state{panel = Panel} = State) -> Dialog = wxFileDialog:new(Panel, [{style, ?wxFD_FILE_MUST_EXIST}]), State2 = case wxFileDialog:showModal(Dialog) of @@ -690,6 +687,10 @@ handle_event(#wx{id=ID, event = What}, State) -> {noreply, State}. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +handle_call(get_config, _, State) -> + Config0 = get_config(State), + Config = lists:keydelete(trace_p, 1, Config0), + {reply, maps:from_list(Config), State}; handle_call(Msg, From, _State) -> error({unhandled_call, Msg, From}). @@ -1101,26 +1102,38 @@ ftup(Trace, Index, Size) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -write_file(Frame, Filename, ProcFlags, PortFlags, MatchSpecs, Output, TPs) -> +get_config(#state{def_proc_flags = ProcFlags, + def_port_flags = PortFlags, + match_specs = MatchSpecs0, + tpatterns = TracePatterns, + output = Output}) -> MSToList = fun(#match_spec{name=Id, term=T, func=F}) -> [{name,Id},{term,T},{func,F}] end, - MSTermList = [{ms,Key,[MSToList(MS) || MS <- MSs]} || - {Key,MSs} <- MatchSpecs], + MatchSpecs = [{ms,Key,[MSToList(MS) || MS <- MSs]} || + {Key,MSs} <- MatchSpecs0], TPToTuple = fun(#tpattern{fa={F,A}, ms=Ms}) -> - {F,A,MSToList(Ms)} + {F,A,MSToList(Ms)} end, ModuleTermList = [{tp, Module, [TPToTuple(FTP) || FTP <- FTPs]} || - {Module,FTPs} <- TPs], - + {Module,FTPs} <- dict:to_list(TracePatterns)], + [{procflags,ProcFlags}, + {portflags,PortFlags}, + {match_specs,MatchSpecs}, + {output,Output}, + {trace_p,ModuleTermList}]. + +write_file(Frame, Filename, Config) -> Str = ["%%%\n%%% This file is generated by Observer\n", "%%%\n%%% DO NOT EDIT!\n%%%\n", - [io_lib:format("~p.~n",[MSTerm]) || MSTerm <- MSTermList], - io_lib:format("~p.~n",[{procflags,ProcFlags}]), - io_lib:format("~p.~n",[{portflags,PortFlags}]), - io_lib:format("~p.~n",[{output,Output}]), - [io_lib:format("~p.~n",[ModuleTerm]) || ModuleTerm <- ModuleTermList] + [io_lib:format("~p.~n",[MSTerm]) || + MSTerm <- proplists:get_value(match_specs, Config)], + io_lib:format("~p.~n",[lists:keyfind(procflags, 1, Config)]), + io_lib:format("~p.~n",[lists:keyfind(portflags, 1, Config)]), + io_lib:format("~p.~n",[lists:keyfind(output, 1, Config)]), + [io_lib:format("~p.~n",[ModuleTerm]) || + ModuleTerm <- proplists:get_value(trace_p, Config)] ], case file:write_file(Filename, list_to_binary(Str)) of diff --git a/lib/observer/src/observer_tv_table.erl b/lib/observer/src/observer_tv_table.erl index 75e6919642..46da65e005 100644 --- a/lib/observer/src/observer_tv_table.erl +++ b/lib/observer/src/observer_tv_table.erl @@ -233,9 +233,22 @@ handle_event(#wx{id=?ID_REFRESH},State = #state{pid=Pid}) -> {noreply, State}; handle_event(#wx{event=#wxList{type=command_list_col_click, col=Col}}, - State = #state{pid=Pid}) -> + State = #state{pid=Pid, grid=Grid, selected=OldSel}) -> + SelObj = case OldSel of + undefined -> undefined; + _ -> get_row(Pid, OldSel, term) + end, Pid ! {sort, Col+1}, - {noreply, State}; + case SelObj =/= undefined andalso search(Pid, SelObj, -1, true, term) of + false when is_integer(OldSel) -> + wxListCtrl:setItemState(Grid, OldSel, 0, ?wxLIST_STATE_SELECTED), + {noreply, State#state{selected=undefined}}; + false -> + {noreply, State#state{selected=undefined}}; + Row -> + wxListCtrl:setItemState(Grid, Row, 16#FFFF, ?wxLIST_STATE_SELECTED), + {noreply, State#state{selected=Row}} + end; handle_event(#wx{event=#wxSize{size={W,_}}}, State=#state{grid=Grid}) -> observer_lib:set_listctrl_col_size(Grid, W), @@ -607,6 +620,17 @@ keysort(Col, Table) -> end, lists:sort(Sort, Table). +search([Term, -1, true, term], S=#holder{parent=Parent, table=Table}) -> + Search = fun(Idx, [Tuple|_]) -> + Tuple =:= Term andalso throw(Idx), + Tuple + end, + try array:map(Search, Table) of + _ -> Parent ! {self(), false} + catch Index -> + Parent ! {self(), Index} + end, + S; search([Str, Row, Dir0, CaseSens], S=#holder{parent=Parent, n=N, table=Table}) -> Opt = case CaseSens of @@ -642,6 +666,8 @@ get_row(From, Row, Col, Table) -> From ! {self(), format(Object)}; [Object|_] when Col =:= all_multiline -> From ! {self(), io_lib:format("~p", [Object])}; + [Object|_] when Col =:= term -> + From ! {self(), Object}; [Object|_] when tuple_size(Object) >= Col -> From ! {self(), format(element(Col, Object))}; _ -> diff --git a/lib/observer/src/observer_tv_wx.erl b/lib/observer/src/observer_tv_wx.erl index 4356cb890c..e112c54534 100644 --- a/lib/observer/src/observer_tv_wx.erl +++ b/lib/observer/src/observer_tv_wx.erl @@ -18,7 +18,7 @@ %% %CopyrightEnd% -module(observer_tv_wx). --export([start_link/2, display_table_info/4]). +-export([start_link/3, display_table_info/4]). %% wx_object callbacks -export([init/1, handle_info/2, terminate/2, code_change/3, handle_call/3, @@ -58,10 +58,10 @@ timer }). -start_link(Notebook, Parent) -> - wx_object:start_link(?MODULE, [Notebook, Parent], []). +start_link(Notebook, Parent, Config) -> + wx_object:start_link(?MODULE, [Notebook, Parent, Config], []). -init([Notebook, Parent]) -> +init([Notebook, Parent, Config]) -> Panel = wxPanel:new(Notebook), Sizer = wxBoxSizer:new(?wxVERTICAL), Style = ?wxLC_REPORT bor ?wxLC_SINGLE_SEL bor ?wxLC_HRULES, @@ -78,11 +78,11 @@ init([Notebook, Parent]) -> Col + 1 end, ListItems = [{"Table Name", ?wxLIST_FORMAT_LEFT, 200}, - {"Table Id", ?wxLIST_FORMAT_RIGHT, 100}, {"Objects", ?wxLIST_FORMAT_RIGHT, 100}, {"Size (kB)", ?wxLIST_FORMAT_RIGHT, 100}, {"Owner Pid", ?wxLIST_FORMAT_CENTER, 150}, - {"Owner Name", ?wxLIST_FORMAT_LEFT, 200} + {"Owner Name", ?wxLIST_FORMAT_LEFT, 200}, + {"Table Id", ?wxLIST_FORMAT_LEFT, 250} ], lists:foldl(AddListEntry, 0, ListItems), wxListItem:destroy(Li), @@ -94,25 +94,31 @@ init([Notebook, Parent]) -> wxListCtrl:connect(Grid, size, [{skip, true}]), wxWindow:setFocus(Grid), - {Panel, #state{grid=Grid, parent=Parent, panel=Panel, timer={false, 10}}}. + {Panel, #state{grid=Grid, parent=Parent, panel=Panel, + timer=Config, + opt=#opt{type=maps:get(type, Config, ets), + sys_hidden=maps:get(sys_hidden, Config, true), + unread_hidden=maps:get(unread_hidden, Config, true)} + }}. handle_event(#wx{id=?ID_REFRESH}, State = #state{node=Node, grid=Grid, opt=Opt}) -> Tables = get_tables(Node, Opt), - Tabs = update_grid(Grid, Opt, Tables), - {noreply, State#state{tabs=Tabs}}; + {Tabs,Sel} = update_grid(Grid, sel(State), Opt, Tables), + Sel =/= undefined andalso wxListCtrl:ensureVisible(Grid, Sel), + {noreply, State#state{tabs=Tabs, selected=Sel}}; handle_event(#wx{event=#wxList{type=command_list_col_click, col=Col}}, State = #state{node=Node, grid=Grid, opt=Opt0=#opt{sort_key=Key, sort_incr=Bool}}) -> - Opt = case Col+2 of + Opt = case col2key(Col) of Key -> Opt0#opt{sort_incr=not Bool}; NewKey -> Opt0#opt{sort_key=NewKey} end, Tables = get_tables(Node, Opt), - Tabs = update_grid(Grid, Opt, Tables), + {Tabs,Sel} = update_grid(Grid, sel(State), Opt, Tables), wxWindow:setFocus(Grid), - {noreply, State#state{opt=Opt, tabs=Tabs}}; + {noreply, State#state{opt=Opt, tabs=Tabs, selected=Sel}}; handle_event(#wx{id=Id}, State = #state{node=Node, grid=Grid, opt=Opt0}) when Id >= ?ID_ETS, Id =< ?ID_SYSTEM_TABLES -> @@ -129,9 +135,9 @@ handle_event(#wx{id=Id}, State = #state{node=Node, grid=Grid, opt=Opt0}) self() ! Error, {noreply, State}; Tables -> - Tabs = update_grid(Grid, Opt, Tables), + {Tabs, Sel} = update_grid(Grid, sel(State), Opt, Tables), wxWindow:setFocus(Grid), - {noreply, State#state{opt=Opt, tabs=Tabs}} + {noreply, State#state{opt=Opt, tabs=Tabs, selected=Sel}} end; handle_event(#wx{event=#wxSize{size={W,_}}}, State=#state{grid=Grid}) -> @@ -202,6 +208,12 @@ handle_event(Event, _State) -> handle_sync_event(_Event, _Obj, _State) -> ok. +handle_call(get_config, _, #state{timer=Timer, opt=Opt}=State) -> + #opt{type=Type, sys_hidden=Sys, unread_hidden=Unread} = Opt, + Conf0 = observer_lib:timer_config(Timer), + Conf = Conf0#{type=>Type, sys_hidden=>Sys, unread_hidden=>Unread}, + {reply, Conf, State}; + handle_call(Event, From, _State) -> error({unhandled_call, Event, From}). @@ -215,8 +227,9 @@ handle_info(refresh_interval, State = #state{node=Node, grid=Grid, opt=Opt, %% no change {noreply, State}; Tables -> - Tabs = update_grid(Grid, Opt, Tables), - {noreply, State#state{tabs=Tabs}} + {Tabs, Sel} = update_grid(Grid, sel(State), Opt, Tables), + Sel =/= undefined andalso wxListCtrl:ensureVisible(Grid, Sel), + {noreply, State#state{tabs=Tabs, selected=Sel}} end; handle_info({active, Node}, State = #state{parent=Parent, grid=Grid, opt=Opt0, @@ -228,11 +241,11 @@ handle_info({active, Node}, State = #state{parent=Parent, grid=Grid, opt=Opt0, Opt1 = Opt0#opt{type=ets}, {get_tables(Node, Opt1), Opt1} end, - Tabs = update_grid(Grid, Opt, Tables), + {Tabs,Sel} = update_grid(Grid, sel(State), Opt, Tables), wxWindow:setFocus(Grid), create_menus(Parent, Opt), - Timer = observer_lib:start_timer(Timer0), - {noreply, State#state{node=Node, tabs=Tabs, timer=Timer, opt=Opt}}; + Timer = observer_lib:start_timer(Timer0, 10), + {noreply, State#state{node=Node, tabs=Tabs, timer=Timer, opt=Opt, selected=Sel}}; handle_info(not_active, State = #state{timer = Timer0}) -> Timer = observer_lib:stop_timer(Timer0), @@ -296,6 +309,13 @@ get_tables2(Node, #opt{type=Type, sys_hidden=Sys, unread_hidden=Unread}) -> [list_to_tabrec(Tab) || Tab <- Result] end. +col2key(0) -> #tab.name; +col2key(1) -> #tab.size; +col2key(2) -> #tab.memory; +col2key(3) -> #tab.owner; +col2key(4) -> #tab.reg_name; +col2key(5) -> #tab.id. + list_to_tabrec(PL) -> #tab{name = proplists:get_value(name, PL), id = proplists:get_value(id, PL, ignore), @@ -366,13 +386,15 @@ list_to_strings([A]) -> integer_to_list(A); list_to_strings([A|B]) -> integer_to_list(A) ++ " ," ++ list_to_strings(B). -update_grid(Grid, Opt, Tables) -> - wx:batch(fun() -> update_grid2(Grid, Opt, Tables) end). -update_grid2(Grid, #opt{sort_key=Sort,sort_incr=Dir}, Tables) -> +update_grid(Grid, Selected, Opt, Tables) -> + wx:batch(fun() -> update_grid2(Grid, Selected, Opt, Tables) end). + +update_grid2(Grid, {SelName,SelId}, #opt{sort_key=Sort,sort_incr=Dir}, Tables) -> wxListCtrl:deleteAllItems(Grid), Update = fun(#tab{name = Name, id = Id, owner = Owner, size = Size, memory = Memory, - protection = Protection, reg_name = RegName}, Row) -> + protection = Protection, reg_name = RegName}, + {Row, Sel}) -> _Item = wxListCtrl:insertItem(Grid, Row, ""), if (Row rem 2) =:= 0 -> wxListCtrl:setItemBackgroundColour(Grid, Row, ?BG_EVEN); @@ -387,13 +409,26 @@ update_grid2(Grid, #opt{sort_key=Sort,sort_incr=Dir}, Tables) -> ({Col, Val}) -> wxListCtrl:setItem(Grid, Row, Col, observer_lib:to_str(Val)) end, - [{0,Name}, {1,Id}, {2,Size}, {3, Memory div 1024}, - {4,Owner}, {5,RegName}]), - Row + 1 + [{0,Name}, {1,Size}, {2, Memory div 1024}, + {3,Owner}, {4,RegName}, {5,Id}]), + if SelName =:= Name, SelId =:= Id -> + wxListCtrl:setItemState(Grid, Row, 16#FFFF, ?wxLIST_STATE_SELECTED), + {Row+1, Row}; + true -> + wxListCtrl:setItemState(Grid, Row, 0, ?wxLIST_STATE_SELECTED), + {Row+1, Sel} + end end, ProcInfo = case Dir of false -> lists:reverse(lists:keysort(Sort, Tables)); true -> lists:keysort(Sort, Tables) end, - lists:foldl(Update, 0, ProcInfo), - ProcInfo. + {_, Sel} = lists:foldl(Update, {0, undefined}, ProcInfo), + {ProcInfo, Sel}. + +sel(#state{selected=Sel, tabs=Tabs}) -> + try lists:nth(Sel+1, Tabs) of + #tab{name=Name, id=Id} -> {Name, Id} + catch _:_ -> + {undefined, undefined} + end. diff --git a/lib/observer/src/observer_wx.erl b/lib/observer/src/observer_wx.erl index 3031a1f90d..0a591babdd 100644 --- a/lib/observer/src/observer_wx.erl +++ b/lib/observer/src/observer_wx.erl @@ -54,20 +54,14 @@ status_bar, notebook, main_panel, - pro_panel, - port_panel, - tv_panel, - sys_panel, - trace_panel, - app_panel, - perf_panel, - allc_panel, + panels, active_tab, node, nodes, prev_node="", log = false, - reply_to=false + reply_to=false, + config }). start() -> @@ -118,6 +112,10 @@ init(_Args) -> setup(#state{frame = Frame} = State) -> %% Setup Menubar & Menus + Config = load_config(), + Cnf = fun(Who) -> + proplists:get_value(Who, Config, #{}) + end, MenuBar = wxMenuBar:new(), {Nodes, NodeMenus} = get_nodes(), @@ -131,7 +129,7 @@ setup(#state{frame = Frame} = State) -> Notebook = wxNotebook:new(Panel, ?ID_NOTEBOOK, [{style, ?wxBK_DEFAULT}]), %% System Panel - SysPanel = observer_sys_wx:start_link(Notebook, self()), + SysPanel = observer_sys_wx:start_link(Notebook, self(), Cnf(sys_panel)), wxNotebook:addPage(Notebook, SysPanel, "System", []), %% Setup sizer create early to get it when window shows @@ -145,43 +143,44 @@ setup(#state{frame = Frame} = State) -> wxFrame:setTitle(Frame, atom_to_list(node())), wxStatusBar:setStatusText(StatusBar, atom_to_list(node())), - wxNotebook:connect(Notebook, command_notebook_page_changing), - wxFrame:connect(Frame, close_window, [{skip, true}]), + wxNotebook:connect(Notebook, command_notebook_page_changed, [{skip, true}]), + wxFrame:connect(Frame, close_window, []), wxMenu:connect(Frame, command_menu_selected), wxFrame:show(Frame), %% Freeze and thaw is buggy currently - DoFreeze = [?wxMAJOR_VERSION,?wxMINOR_VERSION] < [2,9], + DoFreeze = [?wxMAJOR_VERSION,?wxMINOR_VERSION] < [2,9] + orelse element(1, os:type()) =:= win32, DoFreeze andalso wxWindow:freeze(Panel), %% I postpone the creation of the other tabs so they can query/use %% the window size %% Perf Viewer Panel - PerfPanel = observer_perf_wx:start_link(Notebook, self()), + PerfPanel = observer_perf_wx:start_link(Notebook, self(), Cnf(perf_panel)), wxNotebook:addPage(Notebook, PerfPanel, "Load Charts", []), %% Memory Allocator Viewer Panel - AllcPanel = observer_alloc_wx:start_link(Notebook, self()), + AllcPanel = observer_alloc_wx:start_link(Notebook, self(), Cnf(allc_panel)), wxNotebook:addPage(Notebook, AllcPanel, ?ALLOC_STR, []), %% App Viewer Panel - AppPanel = observer_app_wx:start_link(Notebook, self()), + AppPanel = observer_app_wx:start_link(Notebook, self(), Cnf(app_panel)), wxNotebook:addPage(Notebook, AppPanel, "Applications", []), %% Process Panel - ProPanel = observer_pro_wx:start_link(Notebook, self()), + ProPanel = observer_pro_wx:start_link(Notebook, self(), Cnf(pro_panel)), wxNotebook:addPage(Notebook, ProPanel, "Processes", []), %% Port Panel - PortPanel = observer_port_wx:start_link(Notebook, self()), + PortPanel = observer_port_wx:start_link(Notebook, self(), Cnf(port_panel)), wxNotebook:addPage(Notebook, PortPanel, "Ports", []), %% Table Viewer Panel - TVPanel = observer_tv_wx:start_link(Notebook, self()), + TVPanel = observer_tv_wx:start_link(Notebook, self(), Cnf(tv_panel)), wxNotebook:addPage(Notebook, TVPanel, "Table Viewer", []), %% Trace Viewer Panel - TracePanel = observer_trace_wx:start_link(Notebook, self()), + TracePanel = observer_trace_wx:start_link(Notebook, self(), Cnf(trace_panel)), wxNotebook:addPage(Notebook, TracePanel, ?TRACE_STR, []), %% Force redraw (windows needs it) @@ -193,19 +192,21 @@ setup(#state{frame = Frame} = State) -> SysPid = wx_object:get_pid(SysPanel), SysPid ! {active, node()}, + Panels = [{sys_panel, SysPanel, "System"}, %% In order + {perf_panel, PerfPanel, "Load Charts"}, + {allc_panel, AllcPanel, ?ALLOC_STR}, + {app_panel, AppPanel, "Applications"}, + {pro_panel, ProPanel, "Processes"}, + {port_panel, PortPanel, "Ports"}, + {tv_panel, TVPanel, "Table Viewer"}, + {trace_panel, TracePanel, ?TRACE_STR}], + UpdState = State#state{main_panel = Panel, notebook = Notebook, menubar = MenuBar, status_bar = StatusBar, - sys_panel = SysPanel, - pro_panel = ProPanel, - port_panel = PortPanel, - tv_panel = TVPanel, - trace_panel = TracePanel, - app_panel = AppPanel, - perf_panel = PerfPanel, - allc_panel = AllcPanel, active_tab = SysPid, + panels = Panels, node = node(), nodes = Nodes }, @@ -228,10 +229,12 @@ setup(#state{frame = Frame} = State) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%Callbacks -handle_event(#wx{event=#wxNotebook{type=command_notebook_page_changing}}, - #state{active_tab=Previous, node=Node} = State) -> - case get_active_pid(State) of - Previous -> {noreply, State}; +handle_event(#wx{event=#wxNotebook{type=command_notebook_page_changed, nSel=Next}}, + #state{active_tab=Previous, node=Node, panels=Panels} = State) -> + {_, Obj, _} = lists:nth(Next+1, Panels), + case wx_object:get_pid(Obj) of + Previous -> + {noreply, State}; Pid -> Previous ! not_active, Pid ! {active, Node}, @@ -362,8 +365,7 @@ handle_event(#wx{id = Id, event = #wxCommand{type = command_menu_selected}}, end, {noreply, change_node_view(Node, LState)}; -handle_event(Event, State) -> - Pid = get_active_pid(State), +handle_event(Event, #state{active_tab=Pid} = State) -> Pid ! Event, {noreply, State}. @@ -388,7 +390,8 @@ handle_call({create_menus, TabMenus}, _From, handle_call({get_attrib, Attrib}, _From, State) -> {reply, get(Attrib), State}; -handle_call(get_tracer, _From, State=#state{trace_panel=TraceP}) -> +handle_call(get_tracer, _From, State=#state{panels=Panels}) -> + {_, TraceP, _} = lists:keyfind(trace_panel, 1, Panels), {reply, TraceP, State}; handle_call(get_active_node, _From, State=#state{node=Node}) -> @@ -424,9 +427,7 @@ handle_info({nodedown, Node}, create_txt_dialog(Frame, Msg, "Node down", ?wxICON_EXCLAMATION), {noreply, State3}; -handle_info({open_link, Id0}, State = #state{pro_panel=ProcViewer, - port_panel=PortViewer, - frame=Frame}) -> +handle_info({open_link, Id0}, State = #state{panels=Panels,frame=Frame}) -> Id = case Id0 of [_|_] -> try list_to_pid(Id0) catch _:_ -> Id0 end; _ -> Id0 @@ -434,8 +435,10 @@ handle_info({open_link, Id0}, State = #state{pro_panel=ProcViewer, %% Forward to process tab case Id of Pid when is_pid(Pid) -> + {pro_panel, ProcViewer, _} = lists:keyfind(pro_panel, 1, Panels), wx_object:get_pid(ProcViewer) ! {procinfo_open, Pid}; "#Port" ++ _ = Port -> + {port_panel, PortViewer, _} = lists:keyfind(port_panel, 1, Panels), wx_object:get_pid(PortViewer) ! {portinfo_open, Port}; _ -> Msg = io_lib:format("Information about ~p is not available or implemented",[Id]), @@ -465,15 +468,13 @@ handle_info({stop, Me}, State) when Me =:= self() -> handle_info(_Info, State) -> {noreply, State}. -stop_servers(#state{node=Node, log=LogOn, sys_panel=Sys, pro_panel=Procs, tv_panel=TVs, - trace_panel=Trace, app_panel=Apps, perf_panel=Perfs, - allc_panel=Alloc, port_panel=Ports} = _State) -> +stop_servers(#state{node=Node, log=LogOn, panels=Panels} = _State) -> LogOn andalso rpc:block_call(Node, rb, stop, []), Me = self(), - Tabs = [Sys, Procs, Ports, TVs, Trace, Apps, Perfs, Alloc], + save_config(Panels), Stop = fun() -> try - _ = [wx_object:stop(Panel) || Panel <- Tabs], + _ = [wx_object:stop(Panel) || {_, Panel, _} <- Panels], ok catch _:_ -> ok end, @@ -490,6 +491,27 @@ terminate(_Reason, #state{frame = Frame, reply_to=From}) -> end, ok. +load_config() -> + case file:consult(config_file()) of + {ok, Config} -> Config; + _ -> [] + end. + +save_config(Panels) -> + Configs = [{Name, wx_object:call(Panel, get_config)} || {Name, Panel, _} <- Panels], + File = config_file(), + case filelib:ensure_dir(File) of + ok -> + Format = [io_lib:format("~p.~n",[Conf]) || Conf <- Configs], + _ = file:write_file(File, Format); + _ -> + ignore + end. + +config_file() -> + Dir = filename:basedir(user_config, "erl_observer"), + filename:join(Dir, "config.txt"). + code_change(_, _, State) -> {ok, State}. @@ -549,8 +571,7 @@ connect2(NodeName, Opts, Cookie) -> {error, net_kernel, Reason} end. -change_node_view(Node, State) -> - Tab = get_active_pid(State), +change_node_view(Node, #state{active_tab=Tab} = State) -> Tab ! not_active, Tab ! {active, Node}, StatusText = ["Observer - " | atom_to_list(Node)], @@ -562,38 +583,13 @@ check_page_title(Notebook) -> Selection = wxNotebook:getSelection(Notebook), wxNotebook:getPageText(Notebook, Selection). -get_active_pid(#state{notebook=Notebook, pro_panel=Pro, sys_panel=Sys, - tv_panel=Tv, trace_panel=Trace, app_panel=App, - perf_panel=Perf, allc_panel=Alloc, port_panel=Port - }) -> - Panel = case check_page_title(Notebook) of - "Processes" -> Pro; - "Ports" -> Port; - "System" -> Sys; - "Table Viewer" -> Tv; - ?TRACE_STR -> Trace; - "Load Charts" -> Perf; - "Applications" -> App; - ?ALLOC_STR -> Alloc - end, - wx_object:get_pid(Panel). - -pid2panel(Pid, #state{pro_panel=Pro, sys_panel=Sys, - tv_panel=Tv, trace_panel=Trace, app_panel=App, - perf_panel=Perf, allc_panel=Alloc, port_panel=Port}) -> - case Pid of - Pro -> "Processes"; - Port -> "Ports"; - Sys -> "System"; - Tv -> "Table Viewer" ; - Trace -> ?TRACE_STR; - Perf -> "Load Charts"; - App -> "Applications"; - Alloc -> ?ALLOC_STR; - _ -> "unknown" +pid2panel(Pid, #state{panels=Panels}) -> + PanelPids = [{Name, wx_object:get_pid(Obj)} || {Name, Obj, _} <- Panels], + case lists:keyfind(Pid, 2, PanelPids) of + false -> "unknown"; + {Name,_} -> Name end. - create_connect_dialog(ping, #state{frame = Frame, prev_node=Prev}) -> Dialog = wxTextEntryDialog:new(Frame, "Connect to node", [{value, Prev}]), case wxDialog:showModal(Dialog) of @@ -636,7 +632,8 @@ create_connect_dialog(connect, #state{frame = Frame}) -> wxWindow:setSizerAndFit(Dialog, VSizer), wxSizer:setSizeHints(VSizer, Dialog), - CookiePath = filename:join(os:getenv("HOME"), ".erlang.cookie"), + {ok,[[HomeDir]]} = init:get_argument(home), + CookiePath = filename:join(HomeDir, ".erlang.cookie"), DefaultCookie = case filelib:is_file(CookiePath) of true -> {ok, Bin} = file:read_file(CookiePath), diff --git a/lib/observer/test/crashdump_helper.erl b/lib/observer/test/crashdump_helper.erl index 4239a3d0d1..e57c8162e4 100644 --- a/lib/observer/test/crashdump_helper.erl +++ b/lib/observer/test/crashdump_helper.erl @@ -44,7 +44,7 @@ n1_proc(Creator,_N2,Pid2,Port2,_L) -> Ref = make_ref(), Pid = self(), Bin = list_to_binary(lists:seq(1, 255)), - SubBin = element(1, split_binary(element(2, split_binary(Bin, 8)), 17)), + <<_:2,SubBin:17/binary,_/bits>> = Bin, register(named_port,Port), diff --git a/lib/observer/vsn.mk b/lib/observer/vsn.mk index dd23b08484..ca9ad72473 100644 --- a/lib/observer/vsn.mk +++ b/lib/observer/vsn.mk @@ -1 +1 @@ -OBSERVER_VSN = 2.3 +OBSERVER_VSN = 2.3.1 diff --git a/lib/orber/src/cdr_encode.erl b/lib/orber/src/cdr_encode.erl index f922b330a0..d8d1809f9d 100644 --- a/lib/orber/src/cdr_encode.erl +++ b/lib/orber/src/cdr_encode.erl @@ -683,7 +683,7 @@ enc_fixed(_Env, Digits, Scale, Fixed, _Bytes, _Len) -> orber:dbg("[~p] cdr_encode:enc_fixed(~p, ~p, ~p)~n" "The supplied fixed type incorrect. Check that the 'digits' and 'scale' field~n" "match the definition in the IDL-specification. The value field must be~n" - "a list of Digits lenght.", + "a list of Digits length.", [?LINE, Digits, Scale, Fixed], ?DEBUG_LEVEL), corba:raise(#'MARSHAL'{completion_status=?COMPLETED_MAYBE}). diff --git a/lib/os_mon/doc/src/notes.xml b/lib/os_mon/doc/src/notes.xml index e6e80b046d..df4151147c 100644 --- a/lib/os_mon/doc/src/notes.xml +++ b/lib/os_mon/doc/src/notes.xml @@ -31,6 +31,21 @@ </header> <p>This document describes the changes made to the OS_Mon application.</p> +<section><title>Os_Mon 2.4.2</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Support s390x in os_mon.</p> + <p> + Own Id: OTP-14161 Aux Id: PR-1309 </p> + </item> + </list> + </section> + +</section> + <section><title>Os_Mon 2.4.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/os_mon/vsn.mk b/lib/os_mon/vsn.mk index 1ac0fb1d27..59a3d9dee4 100644 --- a/lib/os_mon/vsn.mk +++ b/lib/os_mon/vsn.mk @@ -1 +1 @@ -OS_MON_VSN = 2.4.1 +OS_MON_VSN = 2.4.2 diff --git a/lib/parsetools/doc/src/yecc.xml b/lib/parsetools/doc/src/yecc.xml index 9188bd2a22..004fc1668d 100644 --- a/lib/parsetools/doc/src/yecc.xml +++ b/lib/parsetools/doc/src/yecc.xml @@ -207,7 +207,7 @@ <code> Header "%% Copyright (C)" "%% @private" -"%% @Author John"</code> +"%% @Author John".</code> <p>Next comes a declaration of the <c>nonterminal categories</c> to be used in the rules. For example:</p> <code type="none"> diff --git a/lib/parsetools/src/yecc.erl b/lib/parsetools/src/yecc.erl index f6b80eb1b4..05446c1a85 100644 --- a/lib/parsetools/src/yecc.erl +++ b/lib/parsetools/src/yecc.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -81,7 +81,7 @@ -record(rule, { n, % rule n in the grammar file - line, + anno, symbols, % the names of symbols tokens, is_guard, % the action is a guard (not used) @@ -105,7 +105,7 @@ -record(user_code, {state, terminal, funname, action}). --record(symbol, {line = none, name}). +-record(symbol, {anno = none, name}). %% ACCEPT is neither an atom nor a non-terminal. -define(ACCEPT, {}). @@ -517,7 +517,7 @@ parse_grammar(Grammar, Inport, NextLine, St0) -> parse_grammar(Inport, NextLine, St). parse_grammar({error,ErrorLine,Error}, St) -> - add_error(ErrorLine, Error, St); + add_error(erl_anno:new(ErrorLine), Error, St); parse_grammar({rule, Rule, Tokens}, St0) -> NmbrOfDaughters = case Rule of [_, #symbol{name = '$empty'}] -> 0; @@ -534,15 +534,15 @@ parse_grammar({rule, Rule, Tokens}, St0) -> St#yecc{rules_list = [RuleDef | St#yecc.rules_list]}; parse_grammar({prec, Prec}, St) -> St#yecc{prec = Prec ++ St#yecc.prec}; -parse_grammar({#symbol{}, [{string,Line,String}]}, St) -> - add_error(Line, {bad_symbol, String}, St); -parse_grammar({#symbol{line = Line, name = Name}, Symbols}, St) -> +parse_grammar({#symbol{}, [{string,Anno,String}]}, St) -> + add_error(Anno, {bad_symbol, String}, St); +parse_grammar({#symbol{anno = Anno, name = Name}, Symbols}, St) -> CF = fun(I) -> case element(I, St) of [] -> setelement(I, St, Symbols); _ -> - add_error(Line, {duplicate_declaration, Name}, St) + add_error(Anno, {duplicate_declaration, Name}, St) end end, OneSymbol = length(Symbols) =:= 1, @@ -553,7 +553,7 @@ parse_grammar({#symbol{line = Line, name = Name}, Symbols}, St) -> 'Endsymbol' when OneSymbol -> CF(#yecc.endsymbol); 'Expect' when OneSymbol -> CF(#yecc.expect_shift_reduce); 'States' when OneSymbol -> CF(#yecc.expect_n_states); % undocumented - _ -> add_warning(Line, bad_declaration, St) + _ -> add_warning(Anno, bad_declaration, St) end. read_grammar(Inport, St, Line) -> @@ -599,7 +599,7 @@ precedence(_) -> unknown. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% check_grammar(St0) -> - Empty = #symbol{line = none, name = '$empty'}, + Empty = #symbol{anno = none, name = '$empty'}, AllSymbols = St0#yecc.nonterminals ++ St0#yecc.terminals ++ [Empty], St1 = St0#yecc{all_symbols = AllSymbols}, Cs = [fun check_nonterminals/1, fun check_terminals/1, @@ -640,12 +640,12 @@ check_rootsymbol(St) -> case St#yecc.rootsymbol of [] -> add_error(rootsymbol_missing, St); - [#symbol{line = Line, name = SymName}] -> + [#symbol{anno = Anno, name = SymName}] -> case kind_of_symbol(St, SymName) of nonterminal -> St#yecc{rootsymbol = SymName}; _ -> - add_error(Line, {bad_rootsymbol, SymName}, St) + add_error(Anno, {bad_rootsymbol, SymName}, St) end end. @@ -653,12 +653,12 @@ check_endsymbol(St) -> case St#yecc.endsymbol of [] -> St#yecc{endsymbol = '$end'}; - [#symbol{line = Line, name = SymName}] -> + [#symbol{anno = Anno, name = SymName}] -> case kind_of_symbol(St, SymName) of nonterminal -> - add_error(Line, {endsymbol_is_nonterminal, SymName}, St); + add_error(Anno, {endsymbol_is_nonterminal, SymName}, St); terminal -> - add_error(Line, {endsymbol_is_terminal, SymName}, St); + add_error(Anno, {endsymbol_is_terminal, SymName}, St); _ -> St#yecc{endsymbol = SymName} end @@ -670,8 +670,8 @@ check_expect(St0) -> St0#yecc{expect_shift_reduce = 0}; [#symbol{name = Expect}] when is_integer(Expect) -> St0#yecc{expect_shift_reduce = Expect}; - [#symbol{line = Line, name = Name}] -> - St1 = add_error(Line, {bad_expect, Name}, St0), + [#symbol{anno = Anno, name = Name}] -> + St1 = add_error(Anno, {bad_expect, Name}, St0), St1#yecc{expect_shift_reduce = 0} end. @@ -681,27 +681,27 @@ check_states(St) -> St; [#symbol{name = NStates}] when is_integer(NStates) -> St#yecc{expect_n_states = NStates}; - [#symbol{line = Line, name = Name}] -> - add_error(Line, {bad_states, Name}, St) + [#symbol{anno = Anno, name = Name}] -> + add_error(Anno, {bad_states, Name}, St) end. check_precedences(St0) -> {St1, _} = - foldr(fun({#symbol{line = Line, name = Op},_I,_A}, {St,Ps}) -> + foldr(fun({#symbol{anno = Anno, name = Op},_I,_A}, {St,Ps}) -> case member(Op, Ps) of true -> - {add_error(Line, {duplicate_precedence,Op}, St), + {add_error(Anno, {duplicate_precedence,Op}, St), Ps}; false -> {St, [Op | Ps]} end end, {St0,[]}, St0#yecc.prec), - foldl(fun({#symbol{line = Line, name = Op},I,A}, St) -> + foldl(fun({#symbol{anno = Anno, name = Op},I,A}, St) -> case kind_of_symbol(St, Op) of endsymbol -> - add_error(Line,{precedence_op_is_endsymbol,Op}, St); + add_error(Anno,{precedence_op_is_endsymbol,Op}, St); unknown -> - add_error(Line, {precedence_op_is_unknown, Op}, St); + add_error(Anno, {precedence_op_is_unknown, Op}, St); _ -> St#yecc{prec = [{Op,I,A} | St#yecc.prec]} end @@ -709,13 +709,13 @@ check_precedences(St0) -> check_rule(Rule0, {St0,Rules}) -> Symbols = Rule0#rule.symbols, - #symbol{line = HeadLine, name = Head} = hd(Symbols), + #symbol{anno = HeadAnno, name = Head} = hd(Symbols), case member(Head, St0#yecc.nonterminals) of false -> - {add_error(HeadLine, {undefined_nonterminal, Head}, St0), Rules}; + {add_error(HeadAnno, {undefined_nonterminal, Head}, St0), Rules}; true -> St = check_rhs(tl(Symbols), St0), - Rule = Rule0#rule{line = HeadLine, symbols = names(Symbols)}, + Rule = Rule0#rule{anno = HeadAnno, symbols = names(Symbols)}, {St, [Rule | Rules]} end. @@ -725,7 +725,7 @@ check_rules(St0) -> [] -> add_error(no_grammar_rules, St); _ -> - Rule = #rule{line = none, + Rule = #rule{anno = none, symbols = [?ACCEPT, St#yecc.rootsymbol], tokens = []}, Rules1 = [Rule | Rules0], @@ -740,9 +740,9 @@ duplicates(List) -> names(Symbols) -> map(fun(Symbol) -> Symbol#symbol.name end, Symbols). -symbol_line(Name, St) -> - #symbol{line = Line} = symbol_find(Name, St#yecc.all_symbols), - Line. +symbol_anno(Name, St) -> + #symbol{anno = Anno} = symbol_find(Name, St#yecc.all_symbols), + Anno. symbol_member(Symbol, Symbols) -> symbol_find(Symbol#symbol.name, Symbols) =/= false. @@ -894,31 +894,33 @@ report_warnings(St) -> add_error(E, St) -> add_error(none, E, St). -add_error(Line, E, St) -> - add_error(St#yecc.infile, Line, E, St). +add_error(Anno, E, St) -> + add_error(St#yecc.infile, Anno, E, St). -add_error(File, Line, E, St) -> - St#yecc{errors = [{File,{Line,?MODULE,E}}|St#yecc.errors]}. +add_error(File, Anno, E, St) -> + Loc = location(Anno), + St#yecc{errors = [{File,{Loc,?MODULE,E}}|St#yecc.errors]}. add_errors(SymNames, E0, St0) -> foldl(fun(SymName, St) -> - add_error(symbol_line(SymName, St), {E0, SymName}, St) + add_error(symbol_anno(SymName, St), {E0, SymName}, St) end, St0, SymNames). -add_warning(Line, W, St) -> - St#yecc{warnings = [{St#yecc.infile,{Line,?MODULE,W}}|St#yecc.warnings]}. +add_warning(Anno, W, St) -> + Loc = location(Anno), + St#yecc{warnings = [{St#yecc.infile,{Loc,?MODULE,W}}|St#yecc.warnings]}. add_warnings(SymNames, W0, St0) -> foldl(fun(SymName, St) -> - add_warning(symbol_line(SymName, St), {W0, SymName}, St) + add_warning(symbol_anno(SymName, St), {W0, SymName}, St) end, St0, SymNames). check_rhs([#symbol{name = '$empty'}], St) -> St; check_rhs(Rhs, St0) -> case symbol_find('$empty', Rhs) of - #symbol{line = Line} -> - add_error(Line, illegal_empty, St0); + #symbol{anno = Anno} -> + add_error(Anno, illegal_empty, St0); false -> foldl(fun(Sym, St) -> case symbol_member(Sym, St#yecc.all_symbols) of @@ -926,13 +928,13 @@ check_rhs(Rhs, St0) -> St; false -> E = {undefined_symbol,Sym#symbol.name}, - add_error(Sym#symbol.line, E, St) + add_error(Sym#symbol.anno, E, St) end end, St0, Rhs) end. check_action(Tokens) -> - case erl_parse:parse_exprs(add_roberts_dot(Tokens, 0)) of + case erl_parse:parse_exprs(add_roberts_dot(Tokens, erl_anno:new(0))) of {error, _Error} -> {false, false}; {ok, [Expr | Exprs]} -> @@ -940,10 +942,10 @@ check_action(Tokens) -> {IsGuard, true} end. -add_roberts_dot([], Line) -> - [{'dot', Line}]; -add_roberts_dot([{'dot', Line} | _], _) -> - [{'dot', Line}]; +add_roberts_dot([], Anno) -> + [{'dot', Anno}]; +add_roberts_dot([{'dot', Anno} | _], _) -> + [{'dot', Anno}]; add_roberts_dot([Token | Tokens], _) -> [Token | add_roberts_dot(Tokens, element(2, Token))]. @@ -953,21 +955,22 @@ subst_pseudo_vars([H0 | T0], NmbrOfDaughters, St0) -> {H, St1} = subst_pseudo_vars(H0, NmbrOfDaughters, St0), {T, St} = subst_pseudo_vars(T0, NmbrOfDaughters, St1), {[H | T], St}; -subst_pseudo_vars({atom, Line, Atom}, NmbrOfDaughters, St0) -> +subst_pseudo_vars({atom, Anno, Atom}, NmbrOfDaughters, St0) -> case atom_to_list(Atom) of [$$ | Rest] -> try list_to_integer(Rest) of N when N > 0, N =< NmbrOfDaughters -> - {{var, Line, list_to_atom(append("__", Rest))}, St0}; + {{var, Anno, list_to_atom(append("__", Rest))}, St0}; _ -> - St = add_error(Line, {undefined_pseudo_variable, Atom}, + St = add_error(Anno, + {undefined_pseudo_variable, Atom}, St0), - {{atom, Line, '$undefined'}, St} + {{atom, Anno, '$undefined'}, St} catch - error: _ -> {{atom, Line, Atom}, St0} + error: _ -> {{atom, Anno, Atom}, St0} end; _ -> - {{atom, Line, Atom}, St0} + {{atom, Anno, Atom}, St0} end; subst_pseudo_vars(Tuple, NmbrOfDaughters, St0) when is_tuple(Tuple) -> {L, St} = subst_pseudo_vars(tuple_to_list(Tuple), NmbrOfDaughters, St0), @@ -2295,9 +2298,9 @@ function_name(Name, Suf) -> list_to_atom(concat([Name, '_' | quoted_atom(Suf)])). rule(RulePointer, St) -> - #rule{n = N, line = Line, symbols = Symbols} = + #rule{n = N, anno = Anno, symbols = Symbols} = dict:fetch(RulePointer, St#yecc.rule_pointer2rule), - {Symbols, Line, N}. + {Symbols, Anno, N}. get_rule(RuleNmbr, St) -> dict:fetch(RuleNmbr, St#yecc.rule_pointer2rule). @@ -2463,7 +2466,7 @@ include(St, File, Outport) -> include1(eof, _, _, _File, L, _St) -> L; include1({error, _}=_Error, _Inport, _Outport, File, L, St) -> - throw(add_error(File, L, cannot_parse, St)); + throw(add_error(File, erl_anno:new(L), cannot_parse, St)); include1(Line, Inport, Outport, File, L, St) -> Incr = case member($\n, Line) of true -> 1; @@ -2488,7 +2491,7 @@ includefile_version(Includefile) -> parse_file(Epp) -> case epp:parse_erl_form(Epp) of - {ok, {function,_Line,yeccpars1,7,_Clauses}} -> + {ok, {function,_Anno,yeccpars1,7,_Clauses}} -> {1,4}; {eof,_Line} -> {1,1}; @@ -2503,7 +2506,7 @@ pp_tokens(Tokens, Line0, Enc) -> pp_tokens1([], _Line0, _Enc, _T0) -> []; pp_tokens1([T | Ts], Line0, Enc, T0) -> - Line = element(2, T), + Line = location(anno(T)), [pp_sep(Line, Line0, T0), pp_symbol(T, Enc)|pp_tokens1(Ts, Line, Enc, T)]. pp_symbol({var,_,Var}, _Enc) -> Var; @@ -2538,10 +2541,17 @@ output_file_directive(St, _Filename, _Line) -> St. first_line(Tokens) -> - element(2, hd(Tokens)). + location(anno(hd(Tokens))). last_line(Tokens) -> - element(2, lists:last(Tokens)). + location(anno(lists:last(Tokens))). + +location(none) -> none; +location(Anno) -> + erl_anno:line(Anno). + +anno(Token) -> + element(2, Token). %% Keep track of the current line in the generated file. fwrite(#yecc{outport = Outport, line = Line}=St, Format, Args) -> diff --git a/lib/parsetools/src/yeccgramm.yrl b/lib/parsetools/src/yeccgramm.yrl index c7b2ef6a86..40aa85a43e 100644 --- a/lib/parsetools/src/yeccgramm.yrl +++ b/lib/parsetools/src/yeccgramm.yrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2015. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -39,43 +39,38 @@ rule -> head '->' symbols attached_code dot: {rule, ['$1' | '$3'], '$4'}. head -> symbol : '$1'. symbols -> symbol : ['$1']. symbols -> symbol symbols : ['$1' | '$2']. -strings -> string : [string('$1')]. -strings -> string strings : [string('$1') | '$2']. +strings -> string : ['$1']. +strings -> string strings : ['$1' | '$2']. attached_code -> ':' tokens : {erlang_code, '$2'}. -attached_code -> '$empty' : {erlang_code, [{atom, 0, '$undefined'}]}. +attached_code -> '$empty' : {erlang_code, + [{atom, erl_anno:new(0), '$undefined'}]}. tokens -> token : ['$1']. tokens -> token tokens : ['$1' | '$2']. symbol -> var : symbol('$1'). symbol -> atom : symbol('$1'). symbol -> integer : symbol('$1'). symbol -> reserved_word : symbol('$1'). -token -> var : token('$1'). -token -> atom : token('$1'). -token -> float : token('$1'). -token -> integer : token('$1'). -token -> string : token('$1'). -token -> char : token('$1'). -token -> reserved_symbol : {value_of('$1'), line_of('$1')}. -token -> reserved_word : {value_of('$1'), line_of('$1')}. -token -> '->' : {'->', line_of('$1')}. % Have to be treated in this -token -> ':' : {':', line_of('$1')}. % manner, because they are also - % special symbols of the metagrammar +token -> var : '$1'. +token -> atom : '$1'. +token -> float : '$1'. +token -> integer : '$1'. +token -> string : '$1'. +token -> char : '$1'. +token -> reserved_symbol : {value_of('$1'), anno_of('$1')}. +token -> reserved_word : {value_of('$1'), anno_of('$1')}. +token -> '->' : {'->', anno_of('$1')}. % Have to be treated in this +token -> ':' : {':', anno_of('$1')}. % manner, because they are also + % special symbols of the metagrammar Erlang code. --record(symbol, {line, name}). +-record(symbol, {anno, name}). symbol(Symbol) -> - #symbol{line = line_of(Symbol), name = value_of(Symbol)}. - -token(Token) -> - setelement(2, Token, line_of(Token)). - -string(Token) -> - setelement(2, Token, line_of(Token)). + #symbol{anno = anno_of(Symbol), name = value_of(Symbol)}. value_of(Token) -> element(3, Token). -line_of(Token) -> - erl_anno:line(element(2, Token)). +anno_of(Token) -> + element(2, Token). diff --git a/lib/parsetools/src/yeccparser.erl b/lib/parsetools/src/yeccparser.erl index 0025284ccf..6f6f66d56c 100644 --- a/lib/parsetools/src/yeccparser.erl +++ b/lib/parsetools/src/yeccparser.erl @@ -1,29 +1,23 @@ -module(yeccparser). -export([parse/1, parse_and_scan/1, format_error/1]). --file("yeccgramm.yrl", 63). +-file("yeccgramm.yrl", 65). --record(symbol, {line, name}). +-record(symbol, {anno, name}). symbol(Symbol) -> - #symbol{line = line_of(Symbol), name = value_of(Symbol)}. - -token(Token) -> - setelement(2, Token, line_of(Token)). - -string(Token) -> - setelement(2, Token, line_of(Token)). + #symbol{anno = anno_of(Symbol), name = value_of(Symbol)}. value_of(Token) -> element(3, Token). -line_of(Token) -> - erl_anno:line(element(2, Token)). +anno_of(Token) -> + element(2, Token). --file("lib/parsetools/include/yeccpre.hrl", 0). +-file("/ldisk/hasse/otp/lib/parsetools/include/yeccpre.hrl", 0). %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2015. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -147,21 +141,10 @@ yecc_end(Line) -> {'$end', Line}. yecctoken_end_location(Token) -> - try - Str = erl_scan:text(Token), - Line = erl_scan:line(Token), - Parts = re:split(Str, "\n"), - Dline = length(Parts) - 1, - Yline = Line + Dline, - case erl_scan:column(Token) of - Column when is_integer(Column) -> - Col = byte_size(lists:last(Parts)), - {Yline, Col + if Dline =:= 0 -> Column; true -> 1 end}; - undefined -> - Yline - end - catch _:_ -> - yecctoken_location(Token) + try erl_anno:end_location(element(2, Token)) of + undefined -> yecctoken_location(Token); + Loc -> Loc + catch _:_ -> yecctoken_location(Token) end. -compile({nowarn_unused_function, yeccerror/1}). @@ -172,15 +155,15 @@ yeccerror(Token) -> -compile({nowarn_unused_function, yecctoken_to_string/1}). yecctoken_to_string(Token) -> - case catch erl_scan:text(Token) of - Txt when is_list(Txt) -> Txt; - _ -> yecctoken2string(Token) + try erl_scan:text(Token) of + undefined -> yecctoken2string(Token); + Txt -> Txt + catch _:_ -> yecctoken2string(Token) end. yecctoken_location(Token) -> - case catch erl_scan:location(Token) of - Loc when Loc =/= undefined -> Loc; - _ -> element(2, Token) + try erl_scan:location(Token) + catch _:_ -> element(2, Token) end. -compile({nowarn_unused_function, yecctoken2string/1}). @@ -204,8 +187,9 @@ yecctoken2string(Other) -> --file("yeccgramm.erl", 207). +-file("yeccgramm.erl", 190). +-dialyzer({nowarn_function, yeccpars2/7}). yeccpars2(0=S, Cat, Ss, Stack, T, Ts, Tzr) -> yeccpars2_0(S, Cat, Ss, Stack, T, Ts, Tzr); %% yeccpars2(1=S, Cat, Ss, Stack, T, Ts, Tzr) -> @@ -281,6 +265,7 @@ yeccpars2(35=S, Cat, Ss, Stack, T, Ts, Tzr) -> yeccpars2(Other, _, _, _, _, _, _) -> erlang:error({yecc_bug,"1.4",{missing_state_in_action_table, Other}}). +-dialyzer({nowarn_function, yeccpars2_0/7}). yeccpars2_0(S, atom, Ss, Stack, T, Ts, Tzr) -> yeccpars1(S, 6, Ss, Stack, T, Ts, Tzr); yeccpars2_0(S, integer, Ss, Stack, T, Ts, Tzr) -> @@ -308,11 +293,13 @@ yeccpars2_1(_S, Cat, Ss, Stack, T, Ts, Tzr) -> yeccpars2_2(_S, Cat, Ss, Stack, T, Ts, Tzr) -> yeccgoto_grammar(hd(Ss), Cat, Ss, Stack, T, Ts, Tzr). +-dialyzer({nowarn_function, yeccpars2_3/7}). yeccpars2_3(S, '->', Ss, Stack, T, Ts, Tzr) -> yeccpars1(S, 10, Ss, Stack, T, Ts, Tzr); yeccpars2_3(_, _, _, _, T, _, _) -> yeccerror(T). +-dialyzer({nowarn_function, yeccpars2_4/7}). yeccpars2_4(_S, '$end', _Ss, Stack, _T, _Ts, _Tzr) -> {ok, hd(Stack)}; yeccpars2_4(_, _, _, _, T, _, _) -> @@ -362,11 +349,13 @@ yeccpars2_13(_S, Cat, Ss, Stack, T, Ts, Tzr) -> NewStack = yeccpars2_13_(Stack), yeccgoto_symbols(hd(Nss), Cat, Nss, NewStack, T, Ts, Tzr). +-dialyzer({nowarn_function, yeccpars2_14/7}). yeccpars2_14(S, dot, Ss, Stack, T, Ts, Tzr) -> yeccpars1(S, 29, Ss, Stack, T, Ts, Tzr); yeccpars2_14(_, _, _, _, T, _, _) -> yeccerror(T). +-dialyzer({nowarn_function, yeccpars2_15/7}). yeccpars2_15(S, '->', Ss, Stack, T, Ts, Tzr) -> yeccpars1(S, 18, Ss, Stack, T, Ts, Tzr); yeccpars2_15(S, ':', Ss, Stack, T, Ts, Tzr) -> @@ -428,20 +417,16 @@ yeccpars2_19(_S, Cat, Ss, Stack, T, Ts, Tzr) -> yeccgoto_token(hd(Ss), Cat, Ss, NewStack, T, Ts, Tzr). yeccpars2_20(_S, Cat, Ss, Stack, T, Ts, Tzr) -> - NewStack = yeccpars2_20_(Stack), - yeccgoto_token(hd(Ss), Cat, Ss, NewStack, T, Ts, Tzr). + yeccgoto_token(hd(Ss), Cat, Ss, Stack, T, Ts, Tzr). yeccpars2_21(_S, Cat, Ss, Stack, T, Ts, Tzr) -> - NewStack = yeccpars2_21_(Stack), - yeccgoto_token(hd(Ss), Cat, Ss, NewStack, T, Ts, Tzr). + yeccgoto_token(hd(Ss), Cat, Ss, Stack, T, Ts, Tzr). yeccpars2_22(_S, Cat, Ss, Stack, T, Ts, Tzr) -> - NewStack = yeccpars2_22_(Stack), - yeccgoto_token(hd(Ss), Cat, Ss, NewStack, T, Ts, Tzr). + yeccgoto_token(hd(Ss), Cat, Ss, Stack, T, Ts, Tzr). yeccpars2_23(_S, Cat, Ss, Stack, T, Ts, Tzr) -> - NewStack = yeccpars2_23_(Stack), - yeccgoto_token(hd(Ss), Cat, Ss, NewStack, T, Ts, Tzr). + yeccgoto_token(hd(Ss), Cat, Ss, Stack, T, Ts, Tzr). yeccpars2_24(_S, Cat, Ss, Stack, T, Ts, Tzr) -> NewStack = yeccpars2_24_(Stack), @@ -452,12 +437,10 @@ yeccpars2_25(_S, Cat, Ss, Stack, T, Ts, Tzr) -> yeccgoto_token(hd(Ss), Cat, Ss, NewStack, T, Ts, Tzr). yeccpars2_26(_S, Cat, Ss, Stack, T, Ts, Tzr) -> - NewStack = yeccpars2_26_(Stack), - yeccgoto_token(hd(Ss), Cat, Ss, NewStack, T, Ts, Tzr). + yeccgoto_token(hd(Ss), Cat, Ss, Stack, T, Ts, Tzr). yeccpars2_27(_S, Cat, Ss, Stack, T, Ts, Tzr) -> - NewStack = yeccpars2_27_(Stack), - yeccgoto_token(hd(Ss), Cat, Ss, NewStack, T, Ts, Tzr). + yeccgoto_token(hd(Ss), Cat, Ss, Stack, T, Ts, Tzr). yeccpars2_28(_S, Cat, Ss, Stack, T, Ts, Tzr) -> [_|Nss] = Ss, @@ -469,11 +452,13 @@ yeccpars2_29(_S, Cat, Ss, Stack, T, Ts, Tzr) -> NewStack = yeccpars2_29_(Stack), yeccgoto_rule(hd(Nss), Cat, Nss, NewStack, T, Ts, Tzr). +-dialyzer({nowarn_function, yeccpars2_30/7}). yeccpars2_30(S, dot, Ss, Stack, T, Ts, Tzr) -> yeccpars1(S, 35, Ss, Stack, T, Ts, Tzr); yeccpars2_30(_, _, _, _, T, _, _) -> yeccerror(T). +-dialyzer({nowarn_function, yeccpars2_31/7}). yeccpars2_31(S, dot, Ss, Stack, T, Ts, Tzr) -> yeccpars1(S, 34, Ss, Stack, T, Ts, Tzr); yeccpars2_31(_, _, _, _, T, _, _) -> @@ -500,26 +485,33 @@ yeccpars2_35(_S, Cat, Ss, Stack, T, Ts, Tzr) -> NewStack = yeccpars2_35_(Stack), yeccgoto_declaration(hd(Nss), Cat, Nss, NewStack, T, Ts, Tzr). +-dialyzer({nowarn_function, yeccgoto_attached_code/7}). yeccgoto_attached_code(11, Cat, Ss, Stack, T, Ts, Tzr) -> yeccpars2_14(14, Cat, Ss, Stack, T, Ts, Tzr). +-dialyzer({nowarn_function, yeccgoto_declaration/7}). yeccgoto_declaration(0=_S, Cat, Ss, Stack, T, Ts, Tzr) -> yeccpars2_5(_S, Cat, Ss, Stack, T, Ts, Tzr). +-dialyzer({nowarn_function, yeccgoto_grammar/7}). yeccgoto_grammar(0, Cat, Ss, Stack, T, Ts, Tzr) -> yeccpars2_4(4, Cat, Ss, Stack, T, Ts, Tzr). +-dialyzer({nowarn_function, yeccgoto_head/7}). yeccgoto_head(0, Cat, Ss, Stack, T, Ts, Tzr) -> yeccpars2_3(3, Cat, Ss, Stack, T, Ts, Tzr). +-dialyzer({nowarn_function, yeccgoto_rule/7}). yeccgoto_rule(0=_S, Cat, Ss, Stack, T, Ts, Tzr) -> yeccpars2_2(_S, Cat, Ss, Stack, T, Ts, Tzr). +-dialyzer({nowarn_function, yeccgoto_strings/7}). yeccgoto_strings(1, Cat, Ss, Stack, T, Ts, Tzr) -> yeccpars2_31(31, Cat, Ss, Stack, T, Ts, Tzr); yeccgoto_strings(32=_S, Cat, Ss, Stack, T, Ts, Tzr) -> yeccpars2_33(_S, Cat, Ss, Stack, T, Ts, Tzr). +-dialyzer({nowarn_function, yeccgoto_symbol/7}). yeccgoto_symbol(0, Cat, Ss, Stack, T, Ts, Tzr) -> yeccpars2_1(1, Cat, Ss, Stack, T, Ts, Tzr); yeccgoto_symbol(1, Cat, Ss, Stack, T, Ts, Tzr) -> @@ -529,6 +521,7 @@ yeccgoto_symbol(10, Cat, Ss, Stack, T, Ts, Tzr) -> yeccgoto_symbol(12, Cat, Ss, Stack, T, Ts, Tzr) -> yeccpars2_12(12, Cat, Ss, Stack, T, Ts, Tzr). +-dialyzer({nowarn_function, yeccgoto_symbols/7}). yeccgoto_symbols(1, Cat, Ss, Stack, T, Ts, Tzr) -> yeccpars2_30(30, Cat, Ss, Stack, T, Ts, Tzr); yeccgoto_symbols(10, Cat, Ss, Stack, T, Ts, Tzr) -> @@ -536,18 +529,20 @@ yeccgoto_symbols(10, Cat, Ss, Stack, T, Ts, Tzr) -> yeccgoto_symbols(12=_S, Cat, Ss, Stack, T, Ts, Tzr) -> yeccpars2_13(_S, Cat, Ss, Stack, T, Ts, Tzr). +-dialyzer({nowarn_function, yeccgoto_token/7}). yeccgoto_token(15, Cat, Ss, Stack, T, Ts, Tzr) -> yeccpars2_17(17, Cat, Ss, Stack, T, Ts, Tzr); yeccgoto_token(17, Cat, Ss, Stack, T, Ts, Tzr) -> yeccpars2_17(17, Cat, Ss, Stack, T, Ts, Tzr). +-dialyzer({nowarn_function, yeccgoto_tokens/7}). yeccgoto_tokens(15=_S, Cat, Ss, Stack, T, Ts, Tzr) -> yeccpars2_16(_S, Cat, Ss, Stack, T, Ts, Tzr); yeccgoto_tokens(17=_S, Cat, Ss, Stack, T, Ts, Tzr) -> yeccpars2_28(_S, Cat, Ss, Stack, T, Ts, Tzr). -compile({inline,yeccpars2_6_/1}). --file("yeccgramm.yrl", 44). +-file("yeccgramm.yrl", 46). yeccpars2_6_(__Stack0) -> [__1 | __Stack] = __Stack0, [begin @@ -555,7 +550,7 @@ yeccpars2_6_(__Stack0) -> end | __Stack]. -compile({inline,yeccpars2_7_/1}). --file("yeccgramm.yrl", 45). +-file("yeccgramm.yrl", 47). yeccpars2_7_(__Stack0) -> [__1 | __Stack] = __Stack0, [begin @@ -563,7 +558,7 @@ yeccpars2_7_(__Stack0) -> end | __Stack]. -compile({inline,yeccpars2_8_/1}). --file("yeccgramm.yrl", 46). +-file("yeccgramm.yrl", 48). yeccpars2_8_(__Stack0) -> [__1 | __Stack] = __Stack0, [begin @@ -571,7 +566,7 @@ yeccpars2_8_(__Stack0) -> end | __Stack]. -compile({inline,yeccpars2_9_/1}). --file("yeccgramm.yrl", 43). +-file("yeccgramm.yrl", 45). yeccpars2_9_(__Stack0) -> [__1 | __Stack] = __Stack0, [begin @@ -579,14 +574,15 @@ yeccpars2_9_(__Stack0) -> end | __Stack]. -compile({inline,yeccpars2_11_/1}). --file("yeccgramm.yrl", 40). +-file("yeccgramm.yrl", 41). yeccpars2_11_(__Stack0) -> [begin - { erlang_code , [ { atom , 0 , '$undefined' } ] } + { erlang_code , + [ { atom , erl_anno : new ( 0 ) , '$undefined' } ] } end | __Stack0]. -compile({inline,yeccpars2_12_/1}). --file("yeccgramm.yrl", 35). +-file("yeccgramm.yrl", 36). yeccpars2_12_(__Stack0) -> [__1 | __Stack] = __Stack0, [begin @@ -594,7 +590,7 @@ yeccpars2_12_(__Stack0) -> end | __Stack]. -compile({inline,yeccpars2_13_/1}). --file("yeccgramm.yrl", 36). +-file("yeccgramm.yrl", 37). yeccpars2_13_(__Stack0) -> [__2,__1 | __Stack] = __Stack0, [begin @@ -602,7 +598,7 @@ yeccpars2_13_(__Stack0) -> end | __Stack]. -compile({inline,yeccpars2_16_/1}). --file("yeccgramm.yrl", 39). +-file("yeccgramm.yrl", 40). yeccpars2_16_(__Stack0) -> [__2,__1 | __Stack] = __Stack0, [begin @@ -610,7 +606,7 @@ yeccpars2_16_(__Stack0) -> end | __Stack]. -compile({inline,yeccpars2_17_/1}). --file("yeccgramm.yrl", 41). +-file("yeccgramm.yrl", 43). yeccpars2_17_(__Stack0) -> [__1 | __Stack] = __Stack0, [begin @@ -618,87 +614,39 @@ yeccpars2_17_(__Stack0) -> end | __Stack]. -compile({inline,yeccpars2_18_/1}). --file("yeccgramm.yrl", 55). +-file("yeccgramm.yrl", 57). yeccpars2_18_(__Stack0) -> [__1 | __Stack] = __Stack0, [begin - { '->' , line_of ( __1 ) } + { '->' , anno_of ( __1 ) } end | __Stack]. -compile({inline,yeccpars2_19_/1}). --file("yeccgramm.yrl", 56). +-file("yeccgramm.yrl", 58). yeccpars2_19_(__Stack0) -> [__1 | __Stack] = __Stack0, [begin - { ':' , line_of ( __1 ) } - end | __Stack]. - --compile({inline,yeccpars2_20_/1}). --file("yeccgramm.yrl", 48). -yeccpars2_20_(__Stack0) -> - [__1 | __Stack] = __Stack0, - [begin - token ( __1 ) - end | __Stack]. - --compile({inline,yeccpars2_21_/1}). --file("yeccgramm.yrl", 52). -yeccpars2_21_(__Stack0) -> - [__1 | __Stack] = __Stack0, - [begin - token ( __1 ) - end | __Stack]. - --compile({inline,yeccpars2_22_/1}). --file("yeccgramm.yrl", 49). -yeccpars2_22_(__Stack0) -> - [__1 | __Stack] = __Stack0, - [begin - token ( __1 ) - end | __Stack]. - --compile({inline,yeccpars2_23_/1}). --file("yeccgramm.yrl", 50). -yeccpars2_23_(__Stack0) -> - [__1 | __Stack] = __Stack0, - [begin - token ( __1 ) + { ':' , anno_of ( __1 ) } end | __Stack]. -compile({inline,yeccpars2_24_/1}). --file("yeccgramm.yrl", 53). +-file("yeccgramm.yrl", 55). yeccpars2_24_(__Stack0) -> [__1 | __Stack] = __Stack0, [begin - { value_of ( __1 ) , line_of ( __1 ) } + { value_of ( __1 ) , anno_of ( __1 ) } end | __Stack]. -compile({inline,yeccpars2_25_/1}). --file("yeccgramm.yrl", 54). +-file("yeccgramm.yrl", 56). yeccpars2_25_(__Stack0) -> [__1 | __Stack] = __Stack0, [begin - { value_of ( __1 ) , line_of ( __1 ) } - end | __Stack]. - --compile({inline,yeccpars2_26_/1}). --file("yeccgramm.yrl", 51). -yeccpars2_26_(__Stack0) -> - [__1 | __Stack] = __Stack0, - [begin - token ( __1 ) - end | __Stack]. - --compile({inline,yeccpars2_27_/1}). --file("yeccgramm.yrl", 47). -yeccpars2_27_(__Stack0) -> - [__1 | __Stack] = __Stack0, - [begin - token ( __1 ) + { value_of ( __1 ) , anno_of ( __1 ) } end | __Stack]. -compile({inline,yeccpars2_28_/1}). --file("yeccgramm.yrl", 42). +-file("yeccgramm.yrl", 44). yeccpars2_28_(__Stack0) -> [__2,__1 | __Stack] = __Stack0, [begin @@ -706,7 +654,7 @@ yeccpars2_28_(__Stack0) -> end | __Stack]. -compile({inline,yeccpars2_29_/1}). --file("yeccgramm.yrl", 33). +-file("yeccgramm.yrl", 34). yeccpars2_29_(__Stack0) -> [__5,__4,__3,__2,__1 | __Stack] = __Stack0, [begin @@ -714,23 +662,23 @@ yeccpars2_29_(__Stack0) -> end | __Stack]. -compile({inline,yeccpars2_32_/1}). --file("yeccgramm.yrl", 37). +-file("yeccgramm.yrl", 38). yeccpars2_32_(__Stack0) -> [__1 | __Stack] = __Stack0, [begin - [ string ( __1 ) ] + [ __1 ] end | __Stack]. -compile({inline,yeccpars2_33_/1}). --file("yeccgramm.yrl", 38). +-file("yeccgramm.yrl", 39). yeccpars2_33_(__Stack0) -> [__2,__1 | __Stack] = __Stack0, [begin - [ string ( __1 ) | __2 ] + [ __1 | __2 ] end | __Stack]. -compile({inline,yeccpars2_34_/1}). --file("yeccgramm.yrl", 32). +-file("yeccgramm.yrl", 33). yeccpars2_34_(__Stack0) -> [__3,__2,__1 | __Stack] = __Stack0, [begin @@ -738,7 +686,7 @@ yeccpars2_34_(__Stack0) -> end | __Stack]. -compile({inline,yeccpars2_35_/1}). --file("yeccgramm.yrl", 31). +-file("yeccgramm.yrl", 32). yeccpars2_35_(__Stack0) -> [__3,__2,__1 | __Stack] = __Stack0, [begin @@ -746,4 +694,4 @@ yeccpars2_35_(__Stack0) -> end | __Stack]. --file("yeccgramm.yrl", 82). +-file("yeccgramm.yrl", 77). diff --git a/lib/parsetools/test/yecc_SUITE.erl b/lib/parsetools/test/yecc_SUITE.erl index 5bd71d5d19..2c37278d4b 100644 --- a/lib/parsetools/test/yecc_SUITE.erl +++ b/lib/parsetools/test/yecc_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2016. All Rights Reserved. +%% Copyright Ericsson AB 2005-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1524,7 +1524,7 @@ otp_7945(suite) -> []; otp_7945(Config) when is_list(Config) -> A2 = erl_anno:new(2), A3 = erl_anno:new(3), - {error,_} = erl_parse:parse([{atom,3,foo},{'.',A2,9,9}]), + {error,_} = erl_parse:parse([{atom,A3,foo},{'.',A2,9,9}]), ok. otp_8483(doc) -> diff --git a/lib/public_key/doc/src/notes.xml b/lib/public_key/doc/src/notes.xml index 74d1a57936..92e314186e 100644 --- a/lib/public_key/doc/src/notes.xml +++ b/lib/public_key/doc/src/notes.xml @@ -35,6 +35,34 @@ <file>notes.xml</file> </header> +<section><title>Public_Key 1.4</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + New function <c>pkix_verify_hostname/2,3</c> Implements + certificate hostname checking. See the manual and RFC + 6125.</p> + <p> + Own Id: OTP-13009</p> + </item> + <item> + <p> + The ssh host key fingerprint generation now also takes a + list of algorithms and returns a list of corresponding + fingerprints. See + <c>public_key:ssh_hostkey_fingerprint/2</c> and the + option <c>silently_accept_hosts</c> in + <c>ssh:connect</c>.</p> + <p> + Own Id: OTP-14223</p> + </item> + </list> + </section> + +</section> + <section><title>Public_Key 1.3</title> <section><title>Improvements and New Features</title> diff --git a/lib/public_key/doc/src/public_key.xml b/lib/public_key/doc/src/public_key.xml index c97ec361d1..2300ce3937 100644 --- a/lib/public_key/doc/src/public_key.xml +++ b/lib/public_key/doc/src/public_key.xml @@ -331,13 +331,16 @@ </func> <func> - <name>generate_key(Params) -> {Public::binary(), Private::binary()} | #'ECPrivateKey'{} </name> + <name>generate_key(Params) -> {Public::binary(), Private::binary()} | #'ECPrivateKey'{} | {#'RSAPublicKey'{}, #'RSAPrivateKey'{}}</name> <fsummary>Generates a new keypair.</fsummary> <type> - <v>Params = #'DHParameter'{} | {namedCurve, oid()} | #'ECParameters'{}</v> + <v>Params = #'DHParameter'{} | {namedCurve, oid()} | #'ECParameters'{} + | {rsa, Size::integer(), PubExp::integer} </v> </type> <desc> - <p>Generates a new keypair.</p> + <p>Generates a new keypair. See also + <seealso marker="crypto:crypto#generate_key/2">crypto:generate_key/2</seealso> + </p> </desc> </func> diff --git a/lib/public_key/src/public_key.app.src b/lib/public_key/src/public_key.app.src index 88ef07c5a6..dbd732c384 100644 --- a/lib/public_key/src/public_key.app.src +++ b/lib/public_key/src/public_key.app.src @@ -14,7 +14,7 @@ {applications, [asn1, crypto, kernel, stdlib]}, {registered, []}, {env, []}, - {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0","crypto-3.3", + {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0","crypto-3.8", "asn1-3.0"]} ] }. diff --git a/lib/public_key/src/public_key.erl b/lib/public_key/src/public_key.erl index 50d4d82d15..965606045d 100644 --- a/lib/public_key/src/public_key.erl +++ b/lib/public_key/src/public_key.erl @@ -395,9 +395,15 @@ dh_gex_group(Min, N, Max, Groups) -> pubkey_ssh:dh_gex_group(Min, N, Max, Groups). %%-------------------------------------------------------------------- --spec generate_key(#'DHParameter'{} | {namedCurve, Name ::oid()} | - #'ECParameters'{}) -> {Public::binary(), Private::binary()} | - #'ECPrivateKey'{}. +-spec generate_key(#'DHParameter'{}) -> + {Public::binary(), Private::binary()}; + ({namedCurve, Name ::oid()}) -> + #'ECPrivateKey'{}; + (#'ECParameters'{}) -> + #'ECPrivateKey'{}; + ({rsa, Size::pos_integer(), PubExp::pos_integer()}) -> + {#'RSAPublicKey'{}, #'RSAPrivateKey'{}}. + %% Description: Generates a new keypair %%-------------------------------------------------------------------- generate_key(#'DHParameter'{prime = P, base = G}) -> @@ -405,7 +411,49 @@ generate_key(#'DHParameter'{prime = P, base = G}) -> generate_key({namedCurve, _} = Params) -> ec_generate_key(Params); generate_key(#'ECParameters'{} = Params) -> - ec_generate_key(Params). + ec_generate_key(Params); +generate_key({rsa, ModulusSize, PublicExponent}) -> + case crypto:generate_key(rsa, {ModulusSize,PublicExponent}) of + {[E, N], [E, N, D, P, Q, D_mod_P_1, D_mod_Q_1, InvQ_mod_P]} -> + Nint = crypto:bytes_to_integer(N), + Eint = crypto:bytes_to_integer(E), + {#'RSAPublicKey'{modulus = Nint, + publicExponent = Eint}, + #'RSAPrivateKey'{version = 0, % Two-factor (I guess since otherPrimeInfos is not given) + modulus = Nint, + publicExponent = Eint, + privateExponent = crypto:bytes_to_integer(D), + prime1 = crypto:bytes_to_integer(P), + prime2 = crypto:bytes_to_integer(Q), + exponent1 = crypto:bytes_to_integer(D_mod_P_1), + exponent2 = crypto:bytes_to_integer(D_mod_Q_1), + coefficient = crypto:bytes_to_integer(InvQ_mod_P)} + }; + + {[E, N], [E, N, D]} -> % FIXME: what to set the other fields in #'RSAPrivateKey'? + % Answer: Miller [Mil76] + % G.L. Miller. Riemann's hypothesis and tests for primality. + % Journal of Computer and Systems Sciences, + % 13(3):300-307, + % 1976. + Nint = crypto:bytes_to_integer(N), + Eint = crypto:bytes_to_integer(E), + {#'RSAPublicKey'{modulus = Nint, + publicExponent = Eint}, + #'RSAPrivateKey'{version = 0, % Two-factor (I guess since otherPrimeInfos is not given) + modulus = Nint, + publicExponent = Eint, + privateExponent = crypto:bytes_to_integer(D), + prime1 = '?', + prime2 = '?', + exponent1 = '?', + exponent2 = '?', + coefficient = '?'} + }; + + Other -> + Other + end. %%-------------------------------------------------------------------- -spec compute_key(#'ECPoint'{} , #'ECPrivateKey'{}) -> binary(). @@ -562,7 +610,7 @@ pkix_match_dist_point(#'CertificateList'{ %%-------------------------------------------------------------------- -spec pkix_sign(#'OTPTBSCertificate'{}, - rsa_private_key() | dsa_private_key()) -> Der::binary(). + rsa_private_key() | dsa_private_key() | ec_private_key()) -> Der::binary(). %% %% Description: Sign a pkix x.509 certificate. Returns the corresponding %% der encoded 'Certificate'{} @@ -1198,8 +1246,11 @@ ec_curve_spec( #'ECParameters'{fieldID = FieldId, curve = PCurve, base = Base, o FieldId#'FieldID'.parameters}, Curve = {PCurve#'Curve'.a, PCurve#'Curve'.b, none}, {Field, Curve, Base, Order, CoFactor}; -ec_curve_spec({namedCurve, OID}) -> - pubkey_cert_records:namedCurves(OID). +ec_curve_spec({namedCurve, OID}) when is_tuple(OID), is_integer(element(1,OID)) -> + ec_curve_spec({namedCurve, pubkey_cert_records:namedCurves(OID)}); +ec_curve_spec({namedCurve, Name}) when is_atom(Name) -> + crypto:ec_curve(Name). + ec_key({PubKey, PrivateKey}, Params) -> #'ECPrivateKey'{version = 1, diff --git a/lib/public_key/test/erl_make_certs.erl b/lib/public_key/test/erl_make_certs.erl index 3dab70784c..00be7dd5b3 100644 --- a/lib/public_key/test/erl_make_certs.erl +++ b/lib/public_key/test/erl_make_certs.erl @@ -346,10 +346,24 @@ make_key(ec, _Opts) -> %% RSA key generation (OBS: for testing only) %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +gen_rsa2(Size) -> + try + %% The numbers 2048,17 is choosen to not cause the cryptolib on + %% FIPS-enabled test machines be mad at us. + public_key:generate_key({rsa, 2048, 17}) + of + {_Public, Private} -> Private + catch + error:notsup -> + %% Disabled dirty_schedulers => crypto:generate_key not working + weak_gen_rsa2(Size) + end. + + -define(SMALL_PRIMES, [65537,97,89,83,79,73,71,67,61,59,53, 47,43,41,37,31,29,23,19,17,13,11,7,5,3]). -gen_rsa2(Size) -> +weak_gen_rsa2(Size) -> P = prime(Size), Q = prime(Size), N = P*Q, diff --git a/lib/public_key/vsn.mk b/lib/public_key/vsn.mk index 2f541d8d84..b94768ae77 100644 --- a/lib/public_key/vsn.mk +++ b/lib/public_key/vsn.mk @@ -1 +1 @@ -PUBLIC_KEY_VSN = 1.3 +PUBLIC_KEY_VSN = 1.4 diff --git a/lib/reltool/doc/src/notes.xml b/lib/reltool/doc/src/notes.xml index 2365a68feb..b47d451055 100644 --- a/lib/reltool/doc/src/notes.xml +++ b/lib/reltool/doc/src/notes.xml @@ -38,7 +38,22 @@ thus constitutes one section in this document. The title of each section is the version number of Reltool.</p> - <section><title>Reltool 0.7.2</title> + <section><title>Reltool 0.7.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed xml issues in old release notes</p> + <p> + Own Id: OTP-14269</p> + </item> + </list> + </section> + +</section> + +<section><title>Reltool 0.7.2</title> <section><title>Fixed Bugs and Malfunctions</title> <list> @@ -52,13 +67,13 @@ Some dependency chains would even be missed for applications that are included in a 'rel' spec in the reltool config. E.g.</p> - <p> + <list> <item>Application x has y as included application, and y in turn has z as included application. Then z is not included. </item> <item>Application x has y in its 'applications' tag in the .app file, and y in turn has z as included application. Then z is not included.</item> - </list></p> + </list> <p> These bugs are now corrected.</p> <p> diff --git a/lib/reltool/src/reltool.hrl b/lib/reltool/src/reltool.hrl index 3b1e868757..c61c3a0c71 100644 --- a/lib/reltool/src/reltool.hrl +++ b/lib/reltool/src/reltool.hrl @@ -289,8 +289,8 @@ "^lib", "^releases"]). -define(EMBEDDED_EXCL_SYS_FILTERS, - ["^bin/(erlc|dialyzer|typer)(|\\.exe)\$", - "^erts.*/bin/(erlc|dialyzer|typer)(|\\.exe)\$", + ["^bin/(erlc|dialyzer)(|\\.exe)\$", + "^erts.*/bin/(erlc|dialyzer)(|\\.exe)\$", "^erts.*/bin/.*(debug|pdb)"]). -define(EMBEDDED_INCL_APP_FILTERS, ["^ebin", "^include", @@ -303,7 +303,7 @@ "^erts.*/bin", "^lib\$"]). -define(STANDALONE_EXCL_SYS_FILTERS, - ["^erts.*/bin/(erlc|dialyzer|typer)(|\\.exe)\$", + ["^erts.*/bin/(erlc|dialyzer)(|\\.exe)\$", "^erts.*/bin/(start|escript|to_erl|run_erl)(|\\.exe)\$", "^erts.*/bin/.*(debug|pdb)"]). -define(STANDALONE_INCL_APP_FILTERS, ["^ebin", diff --git a/lib/reltool/vsn.mk b/lib/reltool/vsn.mk index 2b23ff6f20..2d07eeb8f0 100644 --- a/lib/reltool/vsn.mk +++ b/lib/reltool/vsn.mk @@ -1 +1 @@ -RELTOOL_VSN = 0.7.2 +RELTOOL_VSN = 0.7.3 diff --git a/lib/runtime_tools/doc/src/notes.xml b/lib/runtime_tools/doc/src/notes.xml index 4c79a560ec..0eafc437cc 100644 --- a/lib/runtime_tools/doc/src/notes.xml +++ b/lib/runtime_tools/doc/src/notes.xml @@ -32,6 +32,24 @@ <p>This document describes the changes made to the Runtime_Tools application.</p> +<section><title>Runtime_Tools 1.11.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + etop erroneously reported the average scheduler + utilization since the tool was first started instead of + the scheduler utilization since last update. This is now + corrected.</p> + <p> + Own Id: OTP-14090 Aux Id: seq13232 </p> + </item> + </list> + </section> + +</section> + <section><title>Runtime_Tools 1.11</title> <section><title>Improvements and New Features</title> diff --git a/lib/runtime_tools/src/dyntrace.erl b/lib/runtime_tools/src/dyntrace.erl index 58c5a773c3..5fe62a46f6 100644 --- a/lib/runtime_tools/src/dyntrace.erl +++ b/lib/runtime_tools/src/dyntrace.erl @@ -61,8 +61,8 @@ enabled_garbage_collection/3, enabled/3]). - -export([user_trace_i4s4/9]). % Know what you're doing! +-compile(no_native). -on_load(on_load/0). -type probe_arg() :: integer() | iolist(). diff --git a/lib/runtime_tools/vsn.mk b/lib/runtime_tools/vsn.mk index 53fc51c198..8ec532de76 100644 --- a/lib/runtime_tools/vsn.mk +++ b/lib/runtime_tools/vsn.mk @@ -1 +1 @@ -RUNTIME_TOOLS_VSN = 1.11 +RUNTIME_TOOLS_VSN = 1.11.1 diff --git a/lib/sasl/doc/src/notes.xml b/lib/sasl/doc/src/notes.xml index 190b937f03..cd3f0e1864 100644 --- a/lib/sasl/doc/src/notes.xml +++ b/lib/sasl/doc/src/notes.xml @@ -31,6 +31,28 @@ </header> <p>This document describes the changes made to the SASL application.</p> +<section><title>SASL 3.0.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + When both options 'warnings_as_errors' and 'silent' were + given to systools:make_script or systools:make_relup, no + error reason would be returned if warnings occurred. + Instead only the atom 'error' was returned. This is now + corrected.</p> + <p> + Options 'warnings_as_errors' and 'no_warn_sasl' are now + also allowed for systools:make_tar.</p> + <p> + Own Id: OTP-14170</p> + </item> + </list> + </section> + +</section> + <section><title>SASL 3.0.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/sasl/vsn.mk b/lib/sasl/vsn.mk index e35a0c2977..6aa662a743 100644 --- a/lib/sasl/vsn.mk +++ b/lib/sasl/vsn.mk @@ -1 +1 @@ -SASL_VSN = 3.0.2 +SASL_VSN = 3.0.3 diff --git a/lib/snmp/doc/src/notes.xml b/lib/snmp/doc/src/notes.xml index 3323d32878..f1919a6bb1 100644 --- a/lib/snmp/doc/src/notes.xml +++ b/lib/snmp/doc/src/notes.xml @@ -34,7 +34,27 @@ </header> - <section><title>SNMP 5.2.4</title> + <section><title>SNMP 5.2.5</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The SNMP MIB compiler has been fixed to compile MIBS with + refinements on user types such as in RFC 4669 + RADIUS-AUTH-SERVER-MIB.mib. Problem reported and + researched by Kenneth Lakin and Daniel Goertzen.</p> + <p> + See also: https://bugs.erlang.org/browse/ERL-325</p> + <p> + Own Id: OTP-14145 Aux Id: ERL-325 </p> + </item> + </list> + </section> + +</section> + +<section><title>SNMP 5.2.4</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index 1837350284..c8c6e61cc8 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -30,6 +30,102 @@ <file>notes.xml</file> </header> +<section><title>Ssh 4.4.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + ssh:daemon_info/1 crashed if the listening IP was not + 'any'</p> + <p> + Own Id: OTP-14298 Aux Id: seq13294 </p> + </item> + </list> + </section> + +</section> + +<section><title>Ssh 4.4.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix bug when opening connections. If the tcp setup + failed, that would in some cases not result in an error + return value.</p> + <p> + Own Id: OTP-14108</p> + </item> + <item> + <p> + Reduce information leakage in case of decryption errors.</p> + <p> + Own Id: OTP-14109</p> + </item> + <item> + <p> + The key exchange algorithm + diffie-hellman-group-exchange-sha* has a server-option + <c>{dh_gex_limits,{Min,Max}}</c>. There was a hostkey + signature validation error on the client side if the + option was used and the <c>Min</c> or the <c>Max</c> + differed from the corresponding values obtained from the + client.</p> + <p> + This bug is now corrected.</p> + <p> + Own Id: OTP-14166</p> + </item> + <item> + <p> + The sftpd server now correctly uses <c>root_dir</c> and + <c>cwd</c> when resolving file paths if both are + provided. The <c>cwd</c> handling is also corrected.</p> + <p> + Thanks to kape1395!</p> + <p> + Own Id: OTP-14225 Aux Id: PR-1331, PR-1335 </p> + </item> + <item> + <p> + Ssh_cli used a function that does not handle non-utf8 + unicode correctly.</p> + <p> + Own Id: OTP-14230 Aux Id: ERL-364 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + The implementation of the key exchange algorithms + diffie-hellman-group-exchange-sha* are optimized, up to a + factor of 11 for the slowest ( = biggest and safest) + group size.</p> + <p> + Own Id: OTP-14169 Aux Id: seq-13261 </p> + </item> + <item> + <p> + The ssh host key fingerprint generation now also takes a + list of algorithms and returns a list of corresponding + fingerprints. See + <c>public_key:ssh_hostkey_fingerprint/2</c> and the + option <c>silently_accept_hosts</c> in + <c>ssh:connect</c>.</p> + <p> + Own Id: OTP-14223</p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssh/doc/src/ssh.xml b/lib/ssh/doc/src/ssh.xml index f6e26f5ee8..88d402cf38 100644 --- a/lib/ssh/doc/src/ssh.xml +++ b/lib/ssh/doc/src/ssh.xml @@ -243,21 +243,6 @@ <p><c>Peer</c> is in the format of <c>{Host,Port}</c>.</p> </item> - <tag><c><![CDATA[{public_key_alg, 'ssh-rsa' | 'ssh-dss'}]]></c></tag> - <item> - <note> - <p>This option will be removed in OTP 20, but is kept for compatibility. It is ignored if - the preferred <c>pref_public_key_algs</c> option is used.</p> - </note> - <p>Sets the preferred public key algorithm to use for user - authentication. If the preferred algorithm fails, - the other algorithm is tried. If <c>{public_key_alg, 'ssh-rsa'}</c> is set, it is translated - to <c>{pref_public_key_algs, ['ssh-rsa','ssh-dss']}</c>. If it is - <c>{public_key_alg, 'ssh-dss'}</c>, it is translated - to <c>{pref_public_key_algs, ['ssh-dss','ssh-rsa']}</c>. - </p> - </item> - <tag><c><![CDATA[{pref_public_key_algs, list()}]]></c></tag> <item> <p>List of user (client) public key algorithms to try to use.</p> @@ -714,6 +699,12 @@ <p><c>Peer</c> is in the format of <c>{Host,Port}</c>.</p> </item> + <tag><c><![CDATA[{idle_time, integer()}]]></c></tag> + <item> + <p>Sets a time-out on a connection when no channels are active. + Defaults to <c>infinity</c>.</p> + </item> + <tag><c><![CDATA[{ssh_msg_debug_fun, fun(ConnectionRef::ssh_connection_ref(), AlwaysDisplay::boolean(), Msg::binary(), LanguageTag::binary()) -> _}]]></c></tag> <item> <p>Provide a fun to implement your own logging of the SSH message SSH_MSG_DEBUG. The last three parameters are from the message, see RFC4253, section 11.3. The <c>ConnectionRef</c> is the reference to the connection on which the message arrived. The return value from the fun is not checked.</p> @@ -726,9 +717,10 @@ </func> <func> - <name>daemon_info(Daemon) -> {ok, [{port,Port}]} | {error,Error}</name> + <name>daemon_info(Daemon) -> {ok, [DaemonInfo]} | {error,Error}</name> <fsummary>Get info about a daemon</fsummary> <type> + <v>DaemonInfo = {port,Port::pos_integer()} | {listen_address, any|ip_address()} | {profile,atom()}</v> <v>Port = integer()</v> <v>Error = bad_daemon_ref</v> </type> diff --git a/lib/ssh/doc/src/ssh_app.xml b/lib/ssh/doc/src/ssh_app.xml index 5f710decc1..515b0639d5 100644 --- a/lib/ssh/doc/src/ssh_app.xml +++ b/lib/ssh/doc/src/ssh_app.xml @@ -109,7 +109,7 @@ </section> <section> <title>Host Keys</title> - <p>RSA and DSA host keys are supported and are + <p>RSA, DSA and ECDSA host keys are supported and are expected to be found in files named <c>ssh_host_rsa_key</c>, <c>ssh_host_dsa_key</c> and <c>ssh_host_ecdsa_key</c>. </p> @@ -160,7 +160,7 @@ <item>ecdsa-sha2-nistp384</item> <item>ecdsa-sha2-nistp521</item> <item>ssh-rsa</item> - <item>(ssh-dss, retired: can be enabled with the <c>preferred_algorithms</c> option)</item> + <item>ssh-dss</item> </list> </item> diff --git a/lib/ssh/src/Makefile b/lib/ssh/src/Makefile index 7ab6f22424..f826fdfd9b 100644 --- a/lib/ssh/src/Makefile +++ b/lib/ssh/src/Makefile @@ -51,6 +51,7 @@ MODULES= \ ssh_sup \ sshc_sup \ sshd_sup \ + ssh_options \ ssh_connection_sup \ ssh_connection \ ssh_connection_handler \ diff --git a/lib/ssh/src/ssh.app.src b/lib/ssh/src/ssh.app.src index 2bb7491b0c..974292fde1 100644 --- a/lib/ssh/src/ssh.app.src +++ b/lib/ssh/src/ssh.app.src @@ -7,6 +7,7 @@ ssh_app, ssh_acceptor, ssh_acceptor_sup, + ssh_options, ssh_auth, ssh_message, ssh_bits, @@ -41,10 +42,10 @@ {env, []}, {mod, {ssh_app, []}}, {runtime_dependencies, [ - "crypto-3.3", + "crypto-3.7.3", "erts-6.0", "kernel-3.0", - "public_key-1.1", - "stdlib-3.1" + "public_key-1.4", + "stdlib-3.3" ]}]}. diff --git a/lib/ssh/src/ssh.erl b/lib/ssh/src/ssh.erl index 68d98d3875..369a00ac40 100644 --- a/lib/ssh/src/ssh.erl +++ b/lib/ssh/src/ssh.erl @@ -40,10 +40,24 @@ ]). %%% Type exports --export_type([connection_ref/0, - channel_id/0 +-export_type([ssh_daemon_ref/0, + ssh_connection_ref/0, + ssh_channel_id/0, + role/0, + subsystem_spec/0, + subsystem_name/0, + channel_callback/0, + channel_init_args/0, + algs_list/0, + alg_entry/0, + simple_algs/0, + double_algs/0 ]). +-opaque ssh_daemon_ref() :: daemon_ref() . +-opaque ssh_connection_ref() :: connection_ref() . +-opaque ssh_channel_id() :: channel_id(). + %%-------------------------------------------------------------------- -spec start() -> ok | {error, term()}. -spec start(permanent | transient | temporary) -> ok | {error, term()}. @@ -71,55 +85,63 @@ stop() -> application:stop(ssh). %%-------------------------------------------------------------------- --spec connect(port(), proplists:proplist()) -> {ok, pid()} | {error, term()}. +-spec connect(inet:socket(), proplists:proplist()) -> ok_error(connection_ref()). + +-spec connect(inet:socket(), proplists:proplist(), timeout()) -> ok_error(connection_ref()) + ; (string(), inet:port_number(), proplists:proplist()) -> ok_error(connection_ref()). --spec connect(port(), proplists:proplist(), timeout()) -> {ok, pid()} | {error, term()} - ; (string(), integer(), proplists:proplist()) -> {ok, pid()} | {error, term()}. +-spec connect(string(), inet:port_number(), proplists:proplist(), timeout()) -> ok_error(connection_ref()). --spec connect(string(), integer(), proplists:proplist(), timeout()) -> {ok, pid()} | {error, term()}. %% %% Description: Starts an ssh connection. %%-------------------------------------------------------------------- -connect(Socket, Options) -> - connect(Socket, Options, infinity). +connect(Socket, UserOptions) when is_port(Socket), + is_list(UserOptions) -> + connect(Socket, UserOptions, infinity). -connect(Socket, Options, Timeout) when is_port(Socket) -> - case handle_options(Options) of +connect(Socket, UserOptions, Timeout) when is_port(Socket), + is_list(UserOptions) -> + case ssh_options:handle_options(client, UserOptions) of {error, Error} -> {error, Error}; - {_SocketOptions, SshOptions} -> - case valid_socket_to_use(Socket, Options) of + Options -> + case valid_socket_to_use(Socket, ?GET_OPT(transport,Options)) of ok -> {ok, {Host,_Port}} = inet:sockname(Socket), - Opts = [{user_pid,self()}, {host,fmt_host(Host)} | SshOptions], + Opts = ?PUT_INTERNAL_OPT([{user_pid,self()}, {host,fmt_host(Host)}], Options), ssh_connection_handler:start_connection(client, Socket, Opts, Timeout); {error,SockError} -> {error,SockError} end end; -connect(Host, Port, Options) when is_integer(Port), Port>0 -> - connect(Host, Port, Options, infinity). +connect(Host, Port, UserOptions) when is_integer(Port), + Port>0, + is_list(UserOptions) -> + connect(Host, Port, UserOptions, infinity). -connect(Host, Port, Options, Timeout) -> - case handle_options(Options) of +connect(Host, Port, UserOptions, Timeout) when is_integer(Port), + Port>0, + is_list(UserOptions) -> + case ssh_options:handle_options(client, UserOptions) of {error, _Reason} = Error -> Error; - {SocketOptions, SshOptions} -> - {_, Transport, _} = TransportOpts = - proplists:get_value(transport, Options, {tcp, gen_tcp, tcp_closed}), - ConnectionTimeout = proplists:get_value(connect_timeout, Options, infinity), - try Transport:connect(Host, Port, [ {active, false} | SocketOptions], ConnectionTimeout) of + Options -> + {_, Transport, _} = TransportOpts = ?GET_OPT(transport, Options), + ConnectionTimeout = ?GET_OPT(connect_timeout, Options), + SocketOpts = [{active,false} | ?GET_OPT(socket_options,Options)], + try Transport:connect(Host, Port, SocketOpts, ConnectionTimeout) of {ok, Socket} -> - Opts = [{user_pid,self()}, {host,Host} | SshOptions], + Opts = ?PUT_INTERNAL_OPT([{user_pid,self()}, {host,Host}], Options), ssh_connection_handler:start_connection(client, Socket, Opts, Timeout); {error, Reason} -> {error, Reason} catch - exit:{function_clause, _} -> + exit:{function_clause, _F} -> + io:format('function_clause ~p~n',[_F]), {error, {options, {transport, TransportOpts}}}; exit:badarg -> - {error, {options, {socket_options, SocketOptions}}} + {error, {options, {socket_options, SocketOpts}}} end end. @@ -148,9 +170,11 @@ channel_info(ConnectionRef, ChannelId, Options) -> ssh_connection_handler:channel_info(ConnectionRef, ChannelId, Options). %%-------------------------------------------------------------------- --spec daemon(integer()) -> {ok, pid()} | {error, term()}. --spec daemon(integer()|port(), proplists:proplist()) -> {ok, pid()} | {error, term()}. --spec daemon(any | inet:ip_address(), integer(), proplists:proplist()) -> {ok, pid()} | {error, term()}. +-spec daemon(inet:port_number()) -> ok_error(daemon_ref()). +-spec daemon(inet:port_number()|inet:socket(), proplists:proplist()) -> ok_error(daemon_ref()). +-spec daemon(any | inet:ip_address(), inet:port_number(), proplists:proplist()) -> ok_error(daemon_ref()) + ;(socket, inet:socket(), proplists:proplist()) -> ok_error(daemon_ref()) + . %% Description: Starts a server listening for SSH connections %% on the given port. @@ -158,34 +182,38 @@ channel_info(ConnectionRef, ChannelId, Options) -> daemon(Port) -> daemon(Port, []). -daemon(Port, Options) when is_integer(Port) -> - daemon(any, Port, Options); -daemon(Socket, Options0) when is_port(Socket) -> - Options = daemon_shell_opt(Options0), - start_daemon(Socket, Options). +daemon(Port, UserOptions) when is_integer(Port), Port >= 0 -> + daemon(any, Port, UserOptions); + +daemon(Socket, UserOptions) when is_port(Socket) -> + daemon(socket, Socket, UserOptions). -daemon(HostAddr, Port, Options0) -> - Options1 = daemon_shell_opt(Options0), - {Host, Inet, Options} = daemon_host_inet_opt(HostAddr, Options1), - start_daemon(Host, Port, Options, Inet). + +daemon(Host0, Port, UserOptions0) -> + {Host, UserOptions} = handle_daemon_args(Host0, UserOptions0), + start_daemon(Host, Port, ssh_options:handle_options(server, UserOptions)). %%-------------------------------------------------------------------- +-spec daemon_info(daemon_ref()) -> ok_error( [{atom(), term()}] ). + daemon_info(Pid) -> case catch ssh_system_sup:acceptor_supervisor(Pid) of AsupPid when is_pid(AsupPid) -> - [Port] = - [Prt || {{ssh_acceptor_sup,any,Prt,default}, - _WorkerPid,worker,[ssh_acceptor]} <- supervisor:which_children(AsupPid)], - {ok, [{port,Port}]}; - + [{ListenAddr,Port,Profile}] = + [{LA,Prt,Prf} || {{ssh_acceptor_sup,LA,Prt,Prf}, + _WorkerPid,worker,[ssh_acceptor]} <- supervisor:which_children(AsupPid)], + {ok, [{port,Port}, + {listen_address,ListenAddr}, + {profile,Profile} + ]}; _ -> {error,bad_daemon_ref} end. %%-------------------------------------------------------------------- --spec stop_listener(pid()) -> ok. --spec stop_listener(inet:ip_address(), integer()) -> ok. +-spec stop_listener(daemon_ref()) -> ok. +-spec stop_listener(inet:ip_address(), inet:port_number()) -> ok. %% %% Description: Stops the listener, but leaves %% existing connections started by the listener up and running. @@ -198,8 +226,9 @@ stop_listener(Address, Port, Profile) -> ssh_system_sup:stop_listener(Address, Port, Profile). %%-------------------------------------------------------------------- --spec stop_daemon(pid()) -> ok. --spec stop_daemon(inet:ip_address(), integer()) -> ok. +-spec stop_daemon(daemon_ref()) -> ok. +-spec stop_daemon(inet:ip_address(), inet:port_number()) -> ok. +-spec stop_daemon(inet:ip_address(), inet:port_number(), atom()) -> ok. %% %% Description: Stops the listener and all connections started by %% the listener. @@ -210,10 +239,11 @@ stop_daemon(Address, Port) -> ssh_system_sup:stop_system(Address, Port, ?DEFAULT_PROFILE). stop_daemon(Address, Port, Profile) -> ssh_system_sup:stop_system(Address, Port, Profile). + %%-------------------------------------------------------------------- --spec shell(port() | string()) -> _. --spec shell(port() | string(), proplists:proplist()) -> _. --spec shell(string(), integer(), proplists:proplist()) -> _. +-spec shell(inet:socket() | string()) -> _. +-spec shell(inet:socket() | string(), proplists:proplist()) -> _. +-spec shell(string(), inet:port_number(), proplists:proplist()) -> _. %% Host = string() %% Port = integer() @@ -254,6 +284,7 @@ start_shell(Error) -> Error. %%-------------------------------------------------------------------- +-spec default_algorithms() -> algs_list() . %%-------------------------------------------------------------------- default_algorithms() -> ssh_transport:default_algorithms(). @@ -261,112 +292,96 @@ default_algorithms() -> %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- -valid_socket_to_use(Socket, Options) -> - case proplists:get_value(transport, Options, {tcp, gen_tcp, tcp_closed}) of - {tcp,_,_} -> - %% Is this tcp-socket a valid socket? - case {is_tcp_socket(Socket), - {ok,[{active,false}]} == inet:getopts(Socket, [active]) - } - of - {true, true} -> - ok; - {true, false} -> - {error, not_passive_mode}; - _ -> - {error, not_tcp_socket} - end; - {L4,_,_} -> - {error, {unsupported,L4}} +handle_daemon_args(Host, UserOptions0) -> + case Host of + socket -> + {Host, UserOptions0}; + any -> + {ok, Host0} = inet:gethostname(), + Inet = proplists:get_value(inet, UserOptions0, inet), + {Host0, [Inet | UserOptions0]}; + {_,_,_,_} -> + {Host, [inet, {ip,Host} | UserOptions0]}; + {_,_,_,_,_,_,_,_} -> + {Host, [inet6, {ip,Host} | UserOptions0]}; + _ -> + error(badarg) end. +%%%---------------------------------------------------------------- +valid_socket_to_use(Socket, {tcp,_,_}) -> + %% Is this tcp-socket a valid socket? + case {is_tcp_socket(Socket), + {ok,[{active,false}]} == inet:getopts(Socket, [active]) + } + of + {true, true} -> + ok; + {true, false} -> + {error, not_passive_mode}; + _ -> + {error, not_tcp_socket} + end; + +valid_socket_to_use(_, {L4,_,_}) -> + {error, {unsupported,L4}}. + + is_tcp_socket(Socket) -> case inet:getopts(Socket, [delay_send]) of {ok,[_]} -> true; _ -> false end. -daemon_shell_opt(Options) -> - case proplists:get_value(shell, Options) of - undefined -> - [{shell, {shell, start, []}} | Options]; - _ -> - Options - end. - -daemon_host_inet_opt(HostAddr, Options1) -> - case HostAddr of - any -> - {ok, Host0} = inet:gethostname(), - {Host0, proplists:get_value(inet, Options1, inet), Options1}; - {_,_,_,_} -> - {HostAddr, inet, - [{ip, HostAddr} | Options1]}; - {_,_,_,_,_,_,_,_} -> - {HostAddr, inet6, - [{ip, HostAddr} | Options1]} - end. - +%%%---------------------------------------------------------------- +start_daemon(_, _, {error,Error}) -> + {error,Error}; + +start_daemon(socket, Socket, Options) -> + case valid_socket_to_use(Socket, ?GET_OPT(transport,Options)) of + ok -> + try + do_start_daemon(Socket, Options) + catch + throw:bad_fd -> {error,bad_fd}; + throw:bad_socket -> {error,bad_socket}; + _C:_E -> {error,{cannot_start_daemon,_C,_E}} + end; + {error,SockError} -> + {error,SockError} + end; -start_daemon(Socket, Options) -> - case handle_options(Options) of - {error, Error} -> - {error, Error}; - {SocketOptions, SshOptions} -> - case valid_socket_to_use(Socket, Options) of - ok -> - try - do_start_daemon(Socket, [{role,server}|SshOptions], SocketOptions) - catch - throw:bad_fd -> {error,bad_fd}; - throw:bad_socket -> {error,bad_socket}; - _C:_E -> {error,{cannot_start_daemon,_C,_E}} - end; - {error,SockError} -> - {error,SockError} - end +start_daemon(Host, Port, Options) -> + try + do_start_daemon(Host, Port, Options) + catch + throw:bad_fd -> {error,bad_fd}; + throw:bad_socket -> {error,bad_socket}; + _C:_E -> {error,{cannot_start_daemon,_C,_E}} end. -start_daemon(Host, Port, Options, Inet) -> - case handle_options(Options) of - {error, _Reason} = Error -> - Error; - {SocketOptions, SshOptions}-> - try - do_start_daemon(Host, Port, [{role,server}|SshOptions] , [Inet|SocketOptions]) - catch - throw:bad_fd -> {error,bad_fd}; - throw:bad_socket -> {error,bad_socket}; - _C:_E -> {error,{cannot_start_daemon,_C,_E}} - end - end. -do_start_daemon(Socket, SshOptions, SocketOptions) -> +do_start_daemon(Socket, Options) -> {ok, {IP,Port}} = try {ok,_} = inet:sockname(Socket) catch _:_ -> throw(bad_socket) end, Host = fmt_host(IP), - Profile = proplists:get_value(profile, SshOptions, ?DEFAULT_PROFILE), - Opts = [{asocket, Socket}, - {asock_owner,self()}, - {address, Host}, - {port, Port}, - {role, server}, - {socket_opts, SocketOptions}, - {ssh_opts, SshOptions}], - {_, Callback, _} = proplists:get_value(transport, SshOptions, {tcp, gen_tcp, tcp_closed}), + Opts = ?PUT_INTERNAL_OPT([{asocket, Socket}, + {asock_owner,self()}, + {address, Host}, + {port, Port}, + {role, server}], Options), + + Profile = ?GET_OPT(profile, Options), case ssh_system_sup:system_supervisor(Host, Port, Profile) of undefined -> - %% It would proably make more sense to call the - %% address option host but that is a too big change at the - %% monent. The name is a legacy name! try sshd_sup:start_child(Opts) of {error, {already_started, _}} -> {error, eaddrinuse}; Result = {ok,_} -> - call_ssh_acceptor_handle_connection(Callback, Host, Port, Opts, Socket, Result); + call_ssh_acceptor_handle_connection(Host, Port, Opts, Socket, Result); Result = {error, _} -> Result catch @@ -379,56 +394,47 @@ do_start_daemon(Socket, SshOptions, SocketOptions) -> {error, {already_started, _}} -> {error, eaddrinuse}; {ok, _} -> - call_ssh_acceptor_handle_connection(Callback, Host, Port, Opts, Socket, {ok, Sup}); + call_ssh_acceptor_handle_connection(Host, Port, Opts, Socket, {ok,Sup}); Other -> Other end end. -do_start_daemon(Host0, Port0, SshOptions, SocketOptions) -> +do_start_daemon(Host0, Port0, Options0) -> {Host,Port1} = try - case proplists:get_value(fd, SocketOptions) of + case ?GET_SOCKET_OPT(fd, Options0) of undefined -> {Host0,Port0}; Fd when Port0==0 -> - find_hostport(Fd); - _ -> - {Host0,Port0} + find_hostport(Fd) end catch _:_ -> throw(bad_fd) end, - Profile = proplists:get_value(profile, SshOptions, ?DEFAULT_PROFILE), - {Port, WaitRequestControl, Opts0} = + {Port, WaitRequestControl, Options1} = case Port1 of 0 -> %% Allocate the socket here to get the port number... - {_, Callback, _} = - proplists:get_value(transport, SshOptions, {tcp, gen_tcp, tcp_closed}), - {ok,LSock} = ssh_acceptor:callback_listen(Callback, 0, SocketOptions), + {ok,LSock} = ssh_acceptor:callback_listen(0, Options0), {ok,{_,LPort}} = inet:sockname(LSock), {LPort, - {LSock,Callback}, - [{lsocket,LSock},{lsock_owner,self()}] + LSock, + ?PUT_INTERNAL_OPT({lsocket,{LSock,self()}}, Options0) }; _ -> - {Port1, false, []} + {Port1, false, Options0} end, - Opts = [{address, Host}, - {port, Port}, - {role, server}, - {socket_opts, SocketOptions}, - {ssh_opts, SshOptions} | Opts0], + Options = ?PUT_INTERNAL_OPT([{address, Host}, + {port, Port}, + {role, server}], Options1), + Profile = ?GET_OPT(profile, Options0), case ssh_system_sup:system_supervisor(Host, Port, Profile) of undefined -> - %% It would proably make more sense to call the - %% address option host but that is a too big change at the - %% monent. The name is a legacy name! - try sshd_sup:start_child(Opts) of + try sshd_sup:start_child(Options) of {error, {already_started, _}} -> {error, eaddrinuse}; Result = {ok,_} -> - sync_request_control(WaitRequestControl), + sync_request_control(WaitRequestControl, Options), Result; Result = {error, _} -> Result @@ -436,21 +442,22 @@ do_start_daemon(Host0, Port0, SshOptions, SocketOptions) -> exit:{noproc, _} -> {error, ssh_not_started} end; - Sup -> + Sup -> AccPid = ssh_system_sup:acceptor_supervisor(Sup), - case ssh_acceptor_sup:start_child(AccPid, Opts) of + case ssh_acceptor_sup:start_child(AccPid, Options) of {error, {already_started, _}} -> {error, eaddrinuse}; {ok, _} -> - sync_request_control(WaitRequestControl), + sync_request_control(WaitRequestControl, Options), {ok, Sup}; Other -> Other end end. -call_ssh_acceptor_handle_connection(Callback, Host, Port, Opts, Socket, DefaultResult) -> - try ssh_acceptor:handle_connection(Callback, Host, Port, Opts, Socket) +call_ssh_acceptor_handle_connection(Host, Port, Options, Socket, DefaultResult) -> + {_, Callback, _} = ?GET_OPT(transport, Options), + try ssh_acceptor:handle_connection(Callback, Host, Port, Options, Socket) of {error,Error} -> {error,Error}; _ -> DefaultResult @@ -459,9 +466,10 @@ call_ssh_acceptor_handle_connection(Callback, Host, Port, Opts, Socket, DefaultR end. -sync_request_control(false) -> +sync_request_control(false, _Options) -> ok; -sync_request_control({LSock,Callback}) -> +sync_request_control(LSock, Options) -> + {_, Callback, _} = ?GET_OPT(transport, Options), receive {request_control,LSock,ReqPid} -> ok = Callback:controlling_process(LSock, ReqPid), @@ -477,523 +485,6 @@ find_hostport(Fd) -> ok = inet:close(S), HostPort. - -handle_options(Opts) -> - try handle_option(algs_compatibility(proplists:unfold(Opts)), [], []) of - {Inet, Ssh} -> - {handle_ip(Inet), Ssh} - catch - throw:Error -> - Error - end. - - -algs_compatibility(Os0) -> - %% Take care of old options 'public_key_alg' and 'pref_public_key_algs' - case proplists:get_value(public_key_alg, Os0) of - undefined -> - Os0; - A when is_atom(A) -> - %% Skip public_key_alg if pref_public_key_algs is defined: - Os = lists:keydelete(public_key_alg, 1, Os0), - case proplists:get_value(pref_public_key_algs,Os) of - undefined when A == 'ssh-rsa' ; A==ssh_rsa -> - [{pref_public_key_algs,['ssh-rsa','ssh-dss']} | Os]; - undefined when A == 'ssh-dss' ; A==ssh_dsa -> - [{pref_public_key_algs,['ssh-dss','ssh-rsa']} | Os]; - undefined -> - throw({error, {eoptions, {public_key_alg,A} }}); - _ -> - Os - end; - V -> - throw({error, {eoptions, {public_key_alg,V} }}) - end. - - -handle_option([], SocketOptions, SshOptions) -> - {SocketOptions, SshOptions}; -handle_option([{system_dir, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{user_dir, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{user_dir_fun, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{silently_accept_hosts, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{user_interaction, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{connect_timeout, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{user, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{dsa_pass_phrase, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{rsa_pass_phrase, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{password, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{user_passwords, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{pwdfun, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{key_cb, {Module, Options}} | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option({key_cb, Module}), - handle_ssh_priv_option({key_cb_private, Options}) | - SshOptions]); -handle_option([{key_cb, Module} | Rest], SocketOptions, SshOptions) -> - handle_option([{key_cb, {Module, []}} | Rest], SocketOptions, SshOptions); -handle_option([{keyboard_interact_fun, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -%%Backwards compatibility -handle_option([{allow_user_interaction, Value} | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option({user_interaction, Value}) | SshOptions]); -handle_option([{infofun, _} = Opt | Rest],SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{connectfun, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{disconnectfun, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{unexpectedfun, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{failfun, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{ssh_msg_debug_fun, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -%%Backwards compatibility should not be underscore between ip and v6 in API -handle_option([{ip_v6_disabled, Value} | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option({ipv6_disabled, Value}) | SshOptions]); -handle_option([{ipv6_disabled, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{transport, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{subsystems, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{ssh_cli, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{shell, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{exec, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{auth_methods, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{auth_method_kb_interactive_data, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{pref_public_key_algs, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{preferred_algorithms,_} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{dh_gex_groups,_} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{dh_gex_limits,_} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{quiet_mode, _} = Opt|Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{idle_time, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{rekey_limit, _} = Opt|Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{max_sessions, _} = Opt|Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{max_channels, _} = Opt|Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{negotiation_timeout, _} = Opt|Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{parallel_login, _} = Opt|Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -%% (Is handled by proplists:unfold above:) -%% handle_option([parallel_login|Rest], SocketOptions, SshOptions) -> -%% handle_option(Rest, SocketOptions, [handle_ssh_option({parallel_login,true}) | SshOptions]); -handle_option([{minimal_remote_max_packet_size, _} = Opt|Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{id_string, _ID} = Opt|Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{profile, _ID} = Opt|Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{max_random_length_padding, _Bool} = Opt|Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{tstflg, _} = Opt|Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, [handle_inet_option(Opt) | SocketOptions], SshOptions). - - -handle_ssh_option({tstflg,_F} = Opt) -> Opt; -handle_ssh_option({minimal_remote_max_packet_size, Value} = Opt) when is_integer(Value), Value >=0 -> - Opt; -handle_ssh_option({system_dir, Value} = Opt) when is_list(Value) -> - check_dir(Opt); -handle_ssh_option({user_dir, Value} = Opt) when is_list(Value) -> - check_dir(Opt); -handle_ssh_option({user_dir_fun, Value} = Opt) when is_function(Value) -> - Opt; -handle_ssh_option({silently_accept_hosts, Value} = Opt) when is_boolean(Value) -> - Opt; -handle_ssh_option({silently_accept_hosts, Value} = Opt) when is_function(Value,2) -> - Opt; -handle_ssh_option({silently_accept_hosts, {DigestAlg,Value}} = Opt) when is_function(Value,2) -> - Algs = if is_atom(DigestAlg) -> [DigestAlg]; - is_list(DigestAlg) -> DigestAlg; - true -> throw({error, {eoptions, Opt}}) - end, - case [A || A <- Algs, - not lists:member(A, [md5, sha, sha224, sha256, sha384, sha512])] of - [_|_] = UnSup1 -> - throw({error, {{eoptions, Opt}, {not_fingerprint_algos,UnSup1}}}); - [] -> - CryptoHashAlgs = proplists:get_value(hashs, crypto:supports(), []), - case [A || A <- Algs, - not lists:member(A, CryptoHashAlgs)] of - [_|_] = UnSup2 -> - throw({error, {{eoptions, Opt}, {unsupported_algo,UnSup2}}}); - [] -> Opt - end - end; -handle_ssh_option({user_interaction, Value} = Opt) when is_boolean(Value) -> - Opt; -handle_ssh_option({preferred_algorithms,[_|_]} = Opt) -> - handle_pref_algs(Opt); - -handle_ssh_option({dh_gex_groups,L0}) when is_list(L0) -> - {dh_gex_groups, - collect_per_size( - lists:foldl( - fun({N,G,P}, Acc) when is_integer(N),N>0, - is_integer(G),G>0, - is_integer(P),P>0 -> - [{N,{G,P}} | Acc]; - ({N,{G,P}}, Acc) when is_integer(N),N>0, - is_integer(G),G>0, - is_integer(P),P>0 -> - [{N,{G,P}} | Acc]; - ({N,GPs}, Acc) when is_list(GPs) -> - lists:foldr(fun({Gi,Pi}, Acci) when is_integer(Gi),Gi>0, - is_integer(Pi),Pi>0 -> - [{N,{Gi,Pi}} | Acci] - end, Acc, GPs) - end, [], L0))}; - -handle_ssh_option({dh_gex_groups,{Tag,File=[C|_]}}=Opt) when is_integer(C), C>0, - Tag == file ; - Tag == ssh_moduli_file -> - {ok,GroupDefs} = - case Tag of - file -> - file:consult(File); - ssh_moduli_file -> - case file:open(File,[read]) of - {ok,D} -> - try - {ok,Moduli} = read_moduli_file(D, 1, []), - file:close(D), - {ok, Moduli} - catch - _:_ -> - throw({error, {{eoptions, Opt}, "Bad format in file "++File}}) - end; - {error,enoent} -> - throw({error, {{eoptions, Opt}, "File not found:"++File}}); - {error,Error} -> - throw({error, {{eoptions, Opt}, io_lib:format("Error reading file ~s: ~p",[File,Error])}}) - end - end, - - try - handle_ssh_option({dh_gex_groups,GroupDefs}) - catch - _:_ -> - throw({error, {{eoptions, Opt}, "Bad format in file: "++File}}) - end; - - -handle_ssh_option({dh_gex_limits,{Min,Max}} = Opt) when is_integer(Min), Min>0, - is_integer(Max), Max>=Min -> - %% Server - Opt; -handle_ssh_option({dh_gex_limits,{Min,I,Max}} = Opt) when is_integer(Min), Min>0, - is_integer(I), I>=Min, - is_integer(Max), Max>=I -> - %% Client - Opt; -handle_ssh_option({pref_public_key_algs, Value} = Opt) when is_list(Value), length(Value) >= 1 -> - case handle_user_pref_pubkey_algs(Value, []) of - {true, NewOpts} -> - {pref_public_key_algs, NewOpts}; - _ -> - throw({error, {eoptions, Opt}}) - end; -handle_ssh_option({connect_timeout, Value} = Opt) when is_integer(Value); Value == infinity -> - Opt; -handle_ssh_option({max_sessions, Value} = Opt) when is_integer(Value), Value>0 -> - Opt; -handle_ssh_option({max_channels, Value} = Opt) when is_integer(Value), Value>0 -> - Opt; -handle_ssh_option({negotiation_timeout, Value} = Opt) when is_integer(Value); Value == infinity -> - Opt; -handle_ssh_option({parallel_login, Value} = Opt) when Value==true ; Value==false -> - Opt; -handle_ssh_option({user, Value} = Opt) when is_list(Value) -> - Opt; -handle_ssh_option({dsa_pass_phrase, Value} = Opt) when is_list(Value) -> - Opt; -handle_ssh_option({rsa_pass_phrase, Value} = Opt) when is_list(Value) -> - Opt; -handle_ssh_option({password, Value} = Opt) when is_list(Value) -> - Opt; -handle_ssh_option({user_passwords, Value} = Opt) when is_list(Value)-> - Opt; -handle_ssh_option({pwdfun, Value} = Opt) when is_function(Value,2) -> - Opt; -handle_ssh_option({pwdfun, Value} = Opt) when is_function(Value,4) -> - Opt; -handle_ssh_option({key_cb, Value} = Opt) when is_atom(Value) -> - Opt; -handle_ssh_option({key_cb, {CallbackMod, CallbackOptions}} = Opt) when is_atom(CallbackMod), - is_list(CallbackOptions) -> - Opt; -handle_ssh_option({keyboard_interact_fun, Value} = Opt) when is_function(Value,3) -> - Opt; -handle_ssh_option({compression, Value} = Opt) when is_atom(Value) -> - Opt; -handle_ssh_option({exec, {Module, Function, _}} = Opt) when is_atom(Module), - is_atom(Function) -> - Opt; -handle_ssh_option({exec, Function} = Opt) when is_function(Function) -> - Opt; -handle_ssh_option({auth_methods, Value} = Opt) when is_list(Value) -> - Opt; -handle_ssh_option({auth_method_kb_interactive_data, {Name,Instruction,Prompt,Echo}} = Opt) when is_list(Name), - is_list(Instruction), - is_list(Prompt), - is_boolean(Echo) -> - Opt; -handle_ssh_option({auth_method_kb_interactive_data, F} = Opt) when is_function(F,3) -> - Opt; -handle_ssh_option({infofun, Value} = Opt) when is_function(Value) -> - Opt; -handle_ssh_option({connectfun, Value} = Opt) when is_function(Value) -> - Opt; -handle_ssh_option({disconnectfun, Value} = Opt) when is_function(Value) -> - Opt; -handle_ssh_option({unexpectedfun, Value} = Opt) when is_function(Value,2) -> - Opt; -handle_ssh_option({failfun, Value} = Opt) when is_function(Value) -> - Opt; -handle_ssh_option({ssh_msg_debug_fun, Value} = Opt) when is_function(Value,4) -> - Opt; - -handle_ssh_option({ipv6_disabled, Value} = Opt) when is_boolean(Value) -> - throw({error, {{ipv6_disabled, Opt}, option_no_longer_valid_use_inet_option_instead}}); -handle_ssh_option({transport, {Protocol, Cb, ClosTag}} = Opt) when is_atom(Protocol), - is_atom(Cb), - is_atom(ClosTag) -> - Opt; -handle_ssh_option({subsystems, Value} = Opt) when is_list(Value) -> - Opt; -handle_ssh_option({ssh_cli, {Cb, _}}= Opt) when is_atom(Cb) -> - Opt; -handle_ssh_option({ssh_cli, no_cli} = Opt) -> - Opt; -handle_ssh_option({shell, {Module, Function, _}} = Opt) when is_atom(Module), - is_atom(Function) -> - Opt; -handle_ssh_option({shell, Value} = Opt) when is_function(Value) -> - Opt; -handle_ssh_option({quiet_mode, Value} = Opt) when is_boolean(Value) -> - Opt; -handle_ssh_option({idle_time, Value} = Opt) when is_integer(Value), Value > 0 -> - Opt; -handle_ssh_option({rekey_limit, Value} = Opt) when is_integer(Value) -> - Opt; -handle_ssh_option({id_string, random}) -> - {id_string, {random,2,5}}; %% 2 - 5 random characters -handle_ssh_option({id_string, ID} = Opt) when is_list(ID) -> - Opt; -handle_ssh_option({max_random_length_padding, Value} = Opt) when is_integer(Value), - Value =< 255 -> - Opt; -handle_ssh_option({profile, Value} = Opt) when is_atom(Value) -> - Opt; -handle_ssh_option(Opt) -> - throw({error, {eoptions, Opt}}). - -handle_ssh_priv_option({key_cb_private, Value} = Opt) when is_list(Value) -> - Opt. - -handle_inet_option({active, _} = Opt) -> - throw({error, {{eoptions, Opt}, "SSH has built in flow control, " - "and active is handled internally, user is not allowed" - "to specify this option"}}); - -handle_inet_option({inet, Value}) when (Value == inet) or (Value == inet6) -> - Value; -handle_inet_option({reuseaddr, _} = Opt) -> - throw({error, {{eoptions, Opt},"Is set internally, user is not allowed" - "to specify this option"}}); -%% Option verified by inet -handle_inet_option(Opt) -> - Opt. - - -%% Check preferred algs - -handle_pref_algs({preferred_algorithms,Algs}) -> - try alg_duplicates(Algs, [], []) of - [] -> - {preferred_algorithms, - [try ssh_transport:supported_algorithms(Key) - of - DefAlgs -> handle_pref_alg(Key,Vals,DefAlgs) - catch - _:_ -> throw({error, {{eoptions, {preferred_algorithms,Key}}, - "Bad preferred_algorithms key"}}) - end || {Key,Vals} <- Algs] - }; - - Dups -> - throw({error, {{eoptions, {preferred_algorithms,Dups}}, "Duplicates found"}}) - catch - _:_ -> - throw({error, {{eoptions, preferred_algorithms}, "Malformed"}}) - end. - -alg_duplicates([{K,V}|KVs], Ks, Dups0) -> - Dups = - case lists:member(K,Ks) of - true -> - [K|Dups0]; - false -> - Dups0 - end, - case V--lists:usort(V) of - [] -> - alg_duplicates(KVs, [K|Ks], Dups); - Ds -> - alg_duplicates(KVs, [K|Ks], Dups++Ds) - end; -alg_duplicates([], _Ks, Dups) -> - Dups. - -handle_pref_alg(Key, - Vs=[{client2server,C2Ss=[_|_]},{server2client,S2Cs=[_|_]}], - [{client2server,Sup_C2Ss},{server2client,Sup_S2Cs}] - ) -> - chk_alg_vs(Key, C2Ss, Sup_C2Ss), - chk_alg_vs(Key, S2Cs, Sup_S2Cs), - {Key, Vs}; - -handle_pref_alg(Key, - Vs=[{server2client,[_|_]},{client2server,[_|_]}], - Sup=[{client2server,_},{server2client,_}] - ) -> - handle_pref_alg(Key, lists:reverse(Vs), Sup); - -handle_pref_alg(Key, - Vs=[V|_], - Sup=[{client2server,_},{server2client,_}] - ) when is_atom(V) -> - handle_pref_alg(Key, [{client2server,Vs},{server2client,Vs}], Sup); - -handle_pref_alg(Key, - Vs=[V|_], - Sup=[S|_] - ) when is_atom(V), is_atom(S) -> - chk_alg_vs(Key, Vs, Sup), - {Key, Vs}; - -handle_pref_alg(Key, Vs, _) -> - throw({error, {{eoptions, {preferred_algorithms,[{Key,Vs}]}}, "Badly formed list"}}). - -chk_alg_vs(OptKey, Values, SupportedValues) -> - case (Values -- SupportedValues) of - [] -> Values; - Bad -> throw({error, {{eoptions, {OptKey,Bad}}, "Unsupported value(s) found"}}) - end. - -handle_ip(Inet) -> %% Default to ipv4 - case lists:member(inet, Inet) of - true -> - Inet; - false -> - case lists:member(inet6, Inet) of - true -> - Inet; - false -> - [inet | Inet] - end - end. - -check_dir({_,Dir} = Opt) -> - case directory_exist_readable(Dir) of - ok -> - Opt; - {error,Error} -> - throw({error, {eoptions,{Opt,Error}}}) - end. - -directory_exist_readable(Dir) -> - case file:read_file_info(Dir) of - {ok, #file_info{type = directory, - access = Access}} -> - case Access of - read -> ok; - read_write -> ok; - _ -> {error, eacces} - end; - - {ok, #file_info{}}-> - {error, enotdir}; - - {error, Error} -> - {error, Error} - end. - - - -collect_per_size(L) -> - lists:foldr( - fun({Sz,GP}, [{Sz,GPs}|Acc]) -> [{Sz,[GP|GPs]}|Acc]; - ({Sz,GP}, Acc) -> [{Sz,[GP]}|Acc] - end, [], lists:sort(L)). - -read_moduli_file(D, I, Acc) -> - case io:get_line(D,"") of - {error,Error} -> - {error,Error}; - eof -> - {ok, Acc}; - "#" ++ _ -> read_moduli_file(D, I+1, Acc); - <<"#",_/binary>> -> read_moduli_file(D, I+1, Acc); - Data -> - Line = if is_binary(Data) -> binary_to_list(Data); - is_list(Data) -> Data - end, - try - [_Time,_Type,_Tests,_Tries,Size,G,P] = string:tokens(Line," \r\n"), - M = {list_to_integer(Size), - {list_to_integer(G), list_to_integer(P,16)} - }, - read_moduli_file(D, I+1, [M|Acc]) - catch - _:_ -> - read_moduli_file(D, I+1, Acc) - end - end. - -handle_user_pref_pubkey_algs([], Acc) -> - {true, lists:reverse(Acc)}; -handle_user_pref_pubkey_algs([H|T], Acc) -> - case lists:member(H, ?SUPPORTED_USER_KEYS) of - true -> - handle_user_pref_pubkey_algs(T, [H| Acc]); - - false when H==ssh_dsa -> handle_user_pref_pubkey_algs(T, ['ssh-dss'| Acc]); - false when H==ssh_rsa -> handle_user_pref_pubkey_algs(T, ['ssh-rsa'| Acc]); - - false -> - false - end. - fmt_host({A,B,C,D}) -> lists:concat([A,".",B,".",C,".",D]); fmt_host(T={_,_,_,_,_,_,_,_}) -> diff --git a/lib/ssh/src/ssh.hrl b/lib/ssh/src/ssh.hrl index 4cd91177f6..c1ba58ed40 100644 --- a/lib/ssh/src/ssh.hrl +++ b/lib/ssh/src/ssh.hrl @@ -33,6 +33,10 @@ -define(REKEY_DATA_TIMOUT, 60000). -define(DEFAULT_PROFILE, default). +-define(DEFAULT_TRANSPORT, {tcp, gen_tcp, tcp_closed} ). + +-define(MAX_RND_PADDING_LEN, 15). + -define(SUPPORTED_AUTH_METHODS, "publickey,keyboard-interactive,password"). -define(SUPPORTED_USER_KEYS, ['ssh-rsa','ssh-dss','ecdsa-sha2-nistp256','ecdsa-sha2-nistp384','ecdsa-sha2-nistp521']). @@ -64,10 +68,49 @@ -define(string_utf8(X), << ?STRING(unicode:characters_to_binary(X)) >> ). -define(binary(X), << ?STRING(X) >>). +%% Cipher details -define(SSH_CIPHER_NONE, 0). -define(SSH_CIPHER_3DES, 3). -define(SSH_CIPHER_AUTHFILE, ?SSH_CIPHER_3DES). +%% Option access macros +-define(do_get_opt(C,K,O), ssh_options:get_value(C,K,O, ?MODULE,?LINE)). +-define(do_get_opt(C,K,O,D), ssh_options:get_value(C,K,O,D,?MODULE,?LINE)). + +-define(GET_OPT(Key,Opts), ?do_get_opt(user_options, Key,Opts ) ). +-define(GET_INTERNAL_OPT(Key,Opts), ?do_get_opt(internal_options,Key,Opts ) ). +-define(GET_INTERNAL_OPT(Key,Opts,Def), ?do_get_opt(internal_options,Key,Opts,Def) ). +-define(GET_SOCKET_OPT(Key,Opts), ?do_get_opt(socket_options, Key,Opts ) ). +-define(GET_SOCKET_OPT(Key,Opts,Def), ?do_get_opt(socket_options, Key,Opts,Def) ). + +-define(do_put_opt(C,KV,O), ssh_options:put_value(C,KV,O, ?MODULE,?LINE)). + +-define(PUT_OPT(KeyVal,Opts), ?do_put_opt(user_options, KeyVal,Opts) ). +-define(PUT_INTERNAL_OPT(KeyVal,Opts), ?do_put_opt(internal_options,KeyVal,Opts) ). +-define(PUT_SOCKET_OPT(KeyVal,Opts), ?do_put_opt(socket_options, KeyVal,Opts) ). + +%% Types +-type role() :: client | server . +-type ok_error(SuccessType) :: {ok, SuccessType} | {error, any()} . +-type daemon_ref() :: pid() . + +-type subsystem_spec() :: {subsystem_name(), {channel_callback(), channel_init_args()}} . +-type subsystem_name() :: string() . +-type channel_callback() :: atom() . +-type channel_init_args() :: list() . + +-type algs_list() :: list( alg_entry() ). +-type alg_entry() :: {kex, simple_algs()} + | {public_key, simple_algs()} + | {cipher, double_algs()} + | {mac, double_algs()} + | {compression, double_algs()} . +-type simple_algs() :: list( atom() ) . +-type double_algs() :: list( {client2serverlist,simple_algs()} | {server2client,simple_algs()} ) + | simple_algs() . + + +%% Records -record(ssh, { role, %% client | server @@ -127,7 +170,7 @@ recv_sequence = 0, keyex_key, keyex_info, - random_length_padding = 15, % From RFC 4253 section 6. + random_length_padding = ?MAX_RND_PADDING_LEN, % From RFC 4253 section 6. %% User auth user, diff --git a/lib/ssh/src/ssh_acceptor.erl b/lib/ssh/src/ssh_acceptor.erl index 13c9d9af4a..42be18f2ad 100644 --- a/lib/ssh/src/ssh_acceptor.erl +++ b/lib/ssh/src/ssh_acceptor.erl @@ -25,56 +25,63 @@ -include("ssh.hrl"). %% Internal application API --export([start_link/5, +-export([start_link/4, number_of_connections/1, - callback_listen/3, + callback_listen/2, handle_connection/5]). %% spawn export --export([acceptor_init/6, acceptor_loop/6]). +-export([acceptor_init/5, acceptor_loop/6]). -define(SLEEP_TIME, 200). %%==================================================================== %% Internal application API %%==================================================================== -start_link(Port, Address, SockOpts, Opts, AcceptTimeout) -> - Args = [self(), Port, Address, SockOpts, Opts, AcceptTimeout], +start_link(Port, Address, Options, AcceptTimeout) -> + Args = [self(), Port, Address, Options, AcceptTimeout], proc_lib:start_link(?MODULE, acceptor_init, Args). %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- -acceptor_init(Parent, Port, Address, SockOpts, Opts, AcceptTimeout) -> - {_, Callback, _} = - proplists:get_value(transport, Opts, {tcp, gen_tcp, tcp_closed}), - - SockOwner = proplists:get_value(lsock_owner, Opts), - LSock = proplists:get_value(lsocket, Opts), - UseExistingSocket = - case catch inet:sockname(LSock) of - {ok,{_,Port}} -> is_pid(SockOwner); - _ -> false - end, - - case UseExistingSocket of - true -> - proc_lib:init_ack(Parent, {ok, self()}), +acceptor_init(Parent, Port, Address, Opts, AcceptTimeout) -> + {_, Callback, _} = ?GET_OPT(transport, Opts), + try + {LSock0,SockOwner0} = ?GET_INTERNAL_OPT(lsocket, Opts), + true = is_pid(SockOwner0), + {ok,{_,Port}} = inet:sockname(LSock0), + {LSock0, SockOwner0} + of + {LSock, SockOwner} -> + %% Use existing socket + proc_lib:init_ack(Parent, {ok, self()}), request_ownership(LSock, SockOwner), - acceptor_loop(Callback, Port, Address, Opts, LSock, AcceptTimeout); - - false -> - case (catch do_socket_listen(Callback, Port, SockOpts)) of - {ok, ListenSocket} -> - proc_lib:init_ack(Parent, {ok, self()}), - acceptor_loop(Callback, - Port, Address, Opts, ListenSocket, AcceptTimeout); - Error -> - proc_lib:init_ack(Parent, Error), - error - end + acceptor_loop(Callback, Port, Address, Opts, LSock, AcceptTimeout) + catch + error:{badkey,lsocket} -> + %% Open new socket + try + socket_listen(Port, Opts) + of + {ok, ListenSocket} -> + proc_lib:init_ack(Parent, {ok, self()}), + {_, Callback, _} = ?GET_OPT(transport, Opts), + acceptor_loop(Callback, + Port, Address, Opts, ListenSocket, AcceptTimeout); + {error,Error} -> + proc_lib:init_ack(Parent, Error), + {error,Error} + catch + _:_ -> + {error,listen_socket_failed} + end; + + _:_ -> + {error,use_existing_socket_failed} end. + request_ownership(LSock, SockOwner) -> SockOwner ! {request_control,LSock,self()}, receive @@ -82,23 +89,25 @@ request_ownership(LSock, SockOwner) -> end. -do_socket_listen(Callback, Port0, Opts) -> - Port = - case proplists:get_value(fd, Opts) of - undefined -> Port0; - _ -> 0 - end, - callback_listen(Callback, Port, Opts). - -callback_listen(Callback, Port, Opts0) -> - Opts = [{active, false}, {reuseaddr,true} | Opts0], - case Callback:listen(Port, Opts) of +socket_listen(Port0, Opts) -> + Port = case ?GET_SOCKET_OPT(fd, Opts) of + undefined -> Port0; + _ -> 0 + end, + callback_listen(Port, Opts). + + +callback_listen(Port, Opts0) -> + {_, Callback, _} = ?GET_OPT(transport, Opts0), + Opts = ?PUT_SOCKET_OPT([{active, false}, {reuseaddr,true}], Opts0), + SockOpts = ?GET_OPT(socket_options, Opts), + case Callback:listen(Port, SockOpts) of {error, nxdomain} -> - Callback:listen(Port, lists:delete(inet6, Opts)); + Callback:listen(Port, lists:delete(inet6, SockOpts)); {error, enetunreach} -> - Callback:listen(Port, lists:delete(inet6, Opts)); + Callback:listen(Port, lists:delete(inet6, SockOpts)); {error, eafnosupport} -> - Callback:listen(Port, lists:delete(inet6, Opts)); + Callback:listen(Port, lists:delete(inet6, SockOpts)); Other -> Other end. @@ -120,21 +129,21 @@ acceptor_loop(Callback, Port, Address, Opts, ListenSocket, AcceptTimeout) -> end. handle_connection(Callback, Address, Port, Options, Socket) -> - SSHopts = proplists:get_value(ssh_opts, Options, []), - Profile = proplists:get_value(profile, SSHopts, ?DEFAULT_PROFILE), + Profile = ?GET_OPT(profile, Options), SystemSup = ssh_system_sup:system_supervisor(Address, Port, Profile), - MaxSessions = proplists:get_value(max_sessions,SSHopts,infinity), + MaxSessions = ?GET_OPT(max_sessions, Options), case number_of_connections(SystemSup) < MaxSessions of true -> {ok, SubSysSup} = ssh_system_sup:start_subsystem(SystemSup, Options), ConnectionSup = ssh_subsystem_sup:connection_supervisor(SubSysSup), - Timeout = proplists:get_value(negotiation_timeout, SSHopts, 2*60*1000), + NegTimeout = ?GET_OPT(negotiation_timeout, Options), ssh_connection_handler:start_connection(server, Socket, - [{supervisors, [{system_sup, SystemSup}, - {subsystem_sup, SubSysSup}, - {connection_sup, ConnectionSup}]} - | Options], Timeout); + ?PUT_INTERNAL_OPT( + {supervisors, [{system_sup, SystemSup}, + {subsystem_sup, SubSysSup}, + {connection_sup, ConnectionSup}]}, + Options), NegTimeout); false -> Callback:close(Socket), IPstr = if is_tuple(Address) -> inet:ntoa(Address); diff --git a/lib/ssh/src/ssh_acceptor_sup.erl b/lib/ssh/src/ssh_acceptor_sup.erl index 129f85a3e0..77f7826918 100644 --- a/lib/ssh/src/ssh_acceptor_sup.erl +++ b/lib/ssh/src/ssh_acceptor_sup.erl @@ -44,14 +44,13 @@ start_link(Servers) -> supervisor:start_link(?MODULE, [Servers]). -start_child(AccSup, ServerOpts) -> - Spec = child_spec(ServerOpts), +start_child(AccSup, Options) -> + Spec = child_spec(Options), case supervisor:start_child(AccSup, Spec) of {error, already_present} -> - Address = proplists:get_value(address, ServerOpts), - Port = proplists:get_value(port, ServerOpts), - Profile = proplists:get_value(profile, - proplists:get_value(ssh_opts, ServerOpts), ?DEFAULT_PROFILE), + Address = ?GET_INTERNAL_OPT(address, Options), + Port = ?GET_INTERNAL_OPT(port, Options), + Profile = ?GET_OPT(profile, Options), stop_child(AccSup, Address, Port, Profile), supervisor:start_child(AccSup, Spec); Reply -> @@ -70,24 +69,23 @@ stop_child(AccSup, Address, Port, Profile) -> %%%========================================================================= %%% Supervisor callback %%%========================================================================= -init([ServerOpts]) -> +init([Options]) -> RestartStrategy = one_for_one, MaxR = 10, MaxT = 3600, - Children = [child_spec(ServerOpts)], + Children = [child_spec(Options)], {ok, {{RestartStrategy, MaxR, MaxT}, Children}}. %%%========================================================================= %%% Internal functions %%%========================================================================= -child_spec(ServerOpts) -> - Address = proplists:get_value(address, ServerOpts), - Port = proplists:get_value(port, ServerOpts), - Timeout = proplists:get_value(timeout, ServerOpts, ?DEFAULT_TIMEOUT), - Profile = proplists:get_value(profile, proplists:get_value(ssh_opts, ServerOpts), ?DEFAULT_PROFILE), +child_spec(Options) -> + Address = ?GET_INTERNAL_OPT(address, Options), + Port = ?GET_INTERNAL_OPT(port, Options), + Timeout = ?GET_INTERNAL_OPT(timeout, Options, ?DEFAULT_TIMEOUT), + Profile = ?GET_OPT(profile, Options), Name = id(Address, Port, Profile), - SocketOpts = proplists:get_value(socket_opts, ServerOpts), - StartFunc = {ssh_acceptor, start_link, [Port, Address, SocketOpts, ServerOpts, Timeout]}, + StartFunc = {ssh_acceptor, start_link, [Port, Address, Options, Timeout]}, Restart = transient, Shutdown = brutal_kill, Modules = [ssh_acceptor], diff --git a/lib/ssh/src/ssh_auth.erl b/lib/ssh/src/ssh_auth.erl index 9b54ecb2dd..88c8144063 100644 --- a/lib/ssh/src/ssh_auth.erl +++ b/lib/ssh/src/ssh_auth.erl @@ -96,14 +96,14 @@ unique(L) -> password_msg([#ssh{opts = Opts, io_cb = IoCb, user = User, service = Service} = Ssh0]) -> {Password,Ssh} = - case proplists:get_value(password, Opts) of + case ?GET_OPT(password, Opts) of undefined when IoCb == ssh_no_io -> {not_ok, Ssh0}; undefined -> - {IoCb:read_password("ssh password: ",Ssh0), Ssh0}; + {IoCb:read_password("ssh password: ",Opts), Ssh0}; PW -> %% If "password" option is given it should not be tried again - {PW, Ssh0#ssh{opts = lists:keyreplace(password,1,Opts,{password,not_ok})}} + {PW, Ssh0#ssh{opts = ?PUT_OPT({password,not_ok}, Opts)}} end, case Password of not_ok -> @@ -123,7 +123,7 @@ password_msg([#ssh{opts = Opts, io_cb = IoCb, keyboard_interactive_msg([#ssh{user = User, opts = Opts, service = Service} = Ssh]) -> - case proplists:get_value(password, Opts) of + case ?GET_OPT(password, Opts) of not_ok -> {not_ok,Ssh}; % No need to use a failed pwd once more _ -> @@ -141,8 +141,9 @@ publickey_msg([Alg, #ssh{user = User, service = Service, opts = Opts} = Ssh]) -> Hash = ssh_transport:sha(Alg), - KeyCb = proplists:get_value(key_cb, Opts, ssh_file), - case KeyCb:user_key(Alg, Opts) of + {KeyCb,KeyCbOpts} = ?GET_OPT(key_cb, Opts), + UserOpts = ?GET_OPT(user_options, Opts), + case KeyCb:user_key(Alg, [{key_cb_private,KeyCbOpts}|UserOpts]) of {ok, PrivKey} -> StrAlgo = atom_to_list(Alg), case encode_public_key(StrAlgo, ssh_transport:extract_public_key(PrivKey)) of @@ -174,13 +175,19 @@ service_request_msg(Ssh) -> %%%---------------------------------------------------------------- init_userauth_request_msg(#ssh{opts = Opts} = Ssh) -> - case user_name(Opts) of - {ok, User} -> + case ?GET_OPT(user, Opts) of + undefined -> + ErrStr = "Could not determine the users name", + ssh_connection_handler:disconnect( + #ssh_msg_disconnect{code = ?SSH_DISCONNECT_ILLEGAL_USER_NAME, + description = ErrStr}); + + User -> Msg = #ssh_msg_userauth_request{user = User, service = "ssh-connection", method = "none", data = <<>>}, - Algs0 = proplists:get_value(pref_public_key_algs, Opts, ?SUPPORTED_USER_KEYS), + Algs0 = ?GET_OPT(pref_public_key_algs, Opts), %% The following line is not strictly correct. The call returns the %% supported HOST key types while we are interested in USER keys. However, %% they "happens" to be the same (for now). This could change.... @@ -194,12 +201,7 @@ init_userauth_request_msg(#ssh{opts = Opts} = Ssh) -> ssh_transport:ssh_packet(Msg, Ssh#ssh{user = User, userauth_preference = Prefs, userauth_methods = none, - service = "ssh-connection"}); - {error, no_user} -> - ErrStr = "Could not determine the users name", - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{code = ?SSH_DISCONNECT_ILLEGAL_USER_NAME, - description = ErrStr}) + service = "ssh-connection"}) end. %%%---------------------------------------------------------------- @@ -342,7 +344,7 @@ handle_userauth_request(#ssh_msg_userauth_request{user = User, false}, {Name, Instruction, Prompt, Echo} = - case proplists:get_value(auth_method_kb_interactive_data, Opts) of + case ?GET_OPT(auth_method_kb_interactive_data, Opts) of undefined -> Default; {_,_,_,_}=V -> @@ -407,9 +409,9 @@ handle_userauth_info_response(#ssh_msg_userauth_info_response{num_responses = 1, user = User, userauth_supported_methods = Methods} = Ssh) -> SendOneEmpty = - (proplists:get_value(tstflg,Opts) == one_empty) + (?GET_OPT(tstflg,Opts) == one_empty) orelse - proplists:get_value(one_empty, proplists:get_value(tstflg,Opts,[]), false), + proplists:get_value(one_empty, ?GET_OPT(tstflg,Opts), false), case check_password(User, unicode:characters_to_list(Password), Opts, Ssh) of {true,Ssh1} when SendOneEmpty==true -> @@ -460,27 +462,8 @@ method_preference(Algs) -> ], Algs). -user_name(Opts) -> - Env = case os:type() of - {win32, _} -> - "USERNAME"; - {unix, _} -> - "LOGNAME" - end, - case proplists:get_value(user, Opts, os:getenv(Env)) of - false -> - case os:getenv("USER") of - false -> - {error, no_user}; - User -> - {ok, User} - end; - User -> - {ok, User} - end. - check_password(User, Password, Opts, Ssh) -> - case proplists:get_value(pwdfun, Opts) of + case ?GET_OPT(pwdfun, Opts) of undefined -> Static = get_password_option(Opts, User), {Password == Static, Ssh}; @@ -510,17 +493,18 @@ check_password(User, Password, Opts, Ssh) -> end. get_password_option(Opts, User) -> - Passwords = proplists:get_value(user_passwords, Opts, []), + Passwords = ?GET_OPT(user_passwords, Opts), case lists:keysearch(User, 1, Passwords) of {value, {User, Pw}} -> Pw; - false -> proplists:get_value(password, Opts, false) + false -> ?GET_OPT(password, Opts) end. pre_verify_sig(User, Alg, KeyBlob, Opts) -> try {ok, Key} = decode_public_key_v2(KeyBlob, Alg), - KeyCb = proplists:get_value(key_cb, Opts, ssh_file), - KeyCb:is_auth_key(Key, User, Opts) + {KeyCb,KeyCbOpts} = ?GET_OPT(key_cb, Opts), + UserOpts = ?GET_OPT(user_options, Opts), + KeyCb:is_auth_key(Key, User, [{key_cb_private,KeyCbOpts}|UserOpts]) catch _:_ -> false @@ -529,9 +513,10 @@ pre_verify_sig(User, Alg, KeyBlob, Opts) -> verify_sig(SessionId, User, Service, Alg, KeyBlob, SigWLen, Opts) -> try {ok, Key} = decode_public_key_v2(KeyBlob, Alg), - KeyCb = proplists:get_value(key_cb, Opts, ssh_file), - case KeyCb:is_auth_key(Key, User, Opts) of + {KeyCb,KeyCbOpts} = ?GET_OPT(key_cb, Opts), + UserOpts = ?GET_OPT(user_options, Opts), + case KeyCb:is_auth_key(Key, User, [{key_cb_private,KeyCbOpts}|UserOpts]) of true -> PlainText = build_sig_data(SessionId, User, Service, KeyBlob, Alg), @@ -565,9 +550,9 @@ decode_keyboard_interactive_prompts(_NumPrompts, Data) -> keyboard_interact_get_responses(IoCb, Opts, Name, Instr, PromptInfos) -> NumPrompts = length(PromptInfos), - keyboard_interact_get_responses(proplists:get_value(user_interaction, Opts, true), - proplists:get_value(keyboard_interact_fun, Opts), - proplists:get_value(password, Opts, undefined), IoCb, Name, + keyboard_interact_get_responses(?GET_OPT(user_interaction, Opts), + ?GET_OPT(keyboard_interact_fun, Opts), + ?GET_OPT(password, Opts), IoCb, Name, Instr, PromptInfos, Opts, NumPrompts). diff --git a/lib/ssh/src/ssh_cli.erl b/lib/ssh/src/ssh_cli.erl index 6f8c050486..4c4f61e036 100644 --- a/lib/ssh/src/ssh_cli.erl +++ b/lib/ssh/src/ssh_cli.erl @@ -499,14 +499,12 @@ start_shell(ConnectionHandler, State) -> [peer, user]), ShellFun = case is_function(Shell) of true -> - User = - proplists:get_value(user, ConnectionInfo), + User = proplists:get_value(user, ConnectionInfo), case erlang:fun_info(Shell, arity) of {arity, 1} -> fun() -> Shell(User) end; {arity, 2} -> - {_, PeerAddr} = - proplists:get_value(peer, ConnectionInfo), + {_, PeerAddr} = proplists:get_value(peer, ConnectionInfo), fun() -> Shell(User, PeerAddr) end; _ -> Shell @@ -525,8 +523,7 @@ start_shell(ConnectionHandler, Cmd, #state{exec=Shell} = State) when is_function ConnectionInfo = ssh_connection_handler:connection_info(ConnectionHandler, [peer, user]), - User = - proplists:get_value(user, ConnectionInfo), + User = proplists:get_value(user, ConnectionInfo), ShellFun = case erlang:fun_info(Shell, arity) of {arity, 1} -> @@ -534,8 +531,7 @@ start_shell(ConnectionHandler, Cmd, #state{exec=Shell} = State) when is_function {arity, 2} -> fun() -> Shell(Cmd, User) end; {arity, 3} -> - {_, PeerAddr} = - proplists:get_value(peer, ConnectionInfo), + {_, PeerAddr} = proplists:get_value(peer, ConnectionInfo), fun() -> Shell(Cmd, User, PeerAddr) end; _ -> Shell diff --git a/lib/ssh/src/ssh_connect.hrl b/lib/ssh/src/ssh_connect.hrl index 4fb6bc39f3..c91c56435e 100644 --- a/lib/ssh/src/ssh_connect.hrl +++ b/lib/ssh/src/ssh_connect.hrl @@ -22,9 +22,9 @@ %%% Description : SSH connection protocol --type role() :: client | server . --type connection_ref() :: pid(). -type channel_id() :: pos_integer(). +-type connection_ref() :: pid(). + -define(DEFAULT_PACKET_SIZE, 65536). -define(DEFAULT_WINDOW_SIZE, 10*?DEFAULT_PACKET_SIZE). diff --git a/lib/ssh/src/ssh_connection.erl b/lib/ssh/src/ssh_connection.erl index c7a2c92670..930ccecb4c 100644 --- a/lib/ssh/src/ssh_connection.erl +++ b/lib/ssh/src/ssh_connection.erl @@ -56,8 +56,8 @@ %%-------------------------------------------------------------------- %%-------------------------------------------------------------------- --spec session_channel(pid(), timeout()) -> {ok, channel_id()} | {error, timeout | closed}. --spec session_channel(pid(), integer(), integer(), timeout()) -> {ok, channel_id()} | {error, timeout | closed}. +-spec session_channel(connection_ref(), timeout()) -> {ok, channel_id()} | {error, timeout | closed}. +-spec session_channel(connection_ref(), integer(), integer(), timeout()) -> {ok, channel_id()} | {error, timeout | closed}. %% Description: Opens a channel for a ssh session. A session is a %% remote execution of a program. The program may be a shell, an @@ -81,7 +81,7 @@ session_channel(ConnectionHandler, InitialWindowSize, end. %%-------------------------------------------------------------------- --spec exec(pid(), channel_id(), string(), timeout()) -> +-spec exec(connection_ref(), channel_id(), string(), timeout()) -> success | failure | {error, timeout | closed}. %% Description: Will request that the server start the @@ -92,7 +92,7 @@ exec(ConnectionHandler, ChannelId, Command, TimeOut) -> true, [?string(Command)], TimeOut). %%-------------------------------------------------------------------- --spec shell(pid(), channel_id()) -> _. +-spec shell(connection_ref(), channel_id()) -> _. %% Description: Will request that the user's default shell (typically %% defined in /etc/passwd in UNIX systems) be started at the other @@ -102,7 +102,7 @@ shell(ConnectionHandler, ChannelId) -> ssh_connection_handler:request(ConnectionHandler, self(), ChannelId, "shell", false, <<>>, 0). %%-------------------------------------------------------------------- --spec subsystem(pid(), channel_id(), string(), timeout()) -> +-spec subsystem(connection_ref(), channel_id(), string(), timeout()) -> success | failure | {error, timeout | closed}. %% %% Description: Executes a predefined subsystem. @@ -112,11 +112,11 @@ subsystem(ConnectionHandler, ChannelId, SubSystem, TimeOut) -> ChannelId, "subsystem", true, [?string(SubSystem)], TimeOut). %%-------------------------------------------------------------------- --spec send(pid(), channel_id(), iodata()) -> +-spec send(connection_ref(), channel_id(), iodata()) -> ok | {error, closed}. --spec send(pid(), channel_id(), integer()| iodata(), timeout() | iodata()) -> +-spec send(connection_ref(), channel_id(), integer()| iodata(), timeout() | iodata()) -> ok | {error, timeout} | {error, closed}. --spec send(pid(), channel_id(), integer(), iodata(), timeout()) -> +-spec send(connection_ref(), channel_id(), integer(), iodata(), timeout()) -> ok | {error, timeout} | {error, closed}. %% %% @@ -134,7 +134,7 @@ send(ConnectionHandler, ChannelId, Type, Data, TimeOut) -> ssh_connection_handler:send(ConnectionHandler, ChannelId, Type, Data, TimeOut). %%-------------------------------------------------------------------- --spec send_eof(pid(), channel_id()) -> ok | {error, closed}. +-spec send_eof(connection_ref(), channel_id()) -> ok | {error, closed}. %% %% %% Description: Sends eof on the channel <ChannelId>. @@ -143,7 +143,7 @@ send_eof(ConnectionHandler, Channel) -> ssh_connection_handler:send_eof(ConnectionHandler, Channel). %%-------------------------------------------------------------------- --spec adjust_window(pid(), channel_id(), integer()) -> ok | {error, closed}. +-spec adjust_window(connection_ref(), channel_id(), integer()) -> ok | {error, closed}. %% %% %% Description: Adjusts the ssh flowcontrol window. @@ -152,7 +152,7 @@ adjust_window(ConnectionHandler, Channel, Bytes) -> ssh_connection_handler:adjust_window(ConnectionHandler, Channel, Bytes). %%-------------------------------------------------------------------- --spec setenv(pid(), channel_id(), string(), string(), timeout()) -> +-spec setenv(connection_ref(), channel_id(), string(), string(), timeout()) -> success | failure | {error, timeout | closed}. %% %% @@ -165,7 +165,7 @@ setenv(ConnectionHandler, ChannelId, Var, Value, TimeOut) -> %%-------------------------------------------------------------------- --spec close(pid(), channel_id()) -> ok. +-spec close(connection_ref(), channel_id()) -> ok. %% %% %% Description: Sends a close message on the channel <ChannelId>. @@ -174,7 +174,7 @@ close(ConnectionHandler, ChannelId) -> ssh_connection_handler:close(ConnectionHandler, ChannelId). %%-------------------------------------------------------------------- --spec reply_request(pid(), boolean(), success | failure, channel_id()) -> ok. +-spec reply_request(connection_ref(), boolean(), success | failure, channel_id()) -> ok. %% %% %% Description: Send status replies to requests that want such replies. @@ -185,9 +185,9 @@ reply_request(_,false, _, _) -> ok. %%-------------------------------------------------------------------- --spec ptty_alloc(pid(), channel_id(), proplists:proplist()) -> +-spec ptty_alloc(connection_ref(), channel_id(), proplists:proplist()) -> success | failiure | {error, closed}. --spec ptty_alloc(pid(), channel_id(), proplists:proplist(), timeout()) -> +-spec ptty_alloc(connection_ref(), channel_id(), proplists:proplist(), timeout()) -> success | failiure | {error, timeout} | {error, closed}. %% @@ -197,16 +197,16 @@ reply_request(_,false, _, _) -> ptty_alloc(ConnectionHandler, Channel, Options) -> ptty_alloc(ConnectionHandler, Channel, Options, infinity). ptty_alloc(ConnectionHandler, Channel, Options0, TimeOut) -> - Options = backwards_compatible(Options0, []), - {Width, PixWidth} = pty_default_dimensions(width, Options), - {Height, PixHeight} = pty_default_dimensions(height, Options), + TermData = backwards_compatible(Options0, []), % FIXME + {Width, PixWidth} = pty_default_dimensions(width, TermData), + {Height, PixHeight} = pty_default_dimensions(height, TermData), pty_req(ConnectionHandler, Channel, - proplists:get_value(term, Options, os:getenv("TERM", ?DEFAULT_TERMINAL)), - proplists:get_value(width, Options, Width), - proplists:get_value(height, Options, Height), - proplists:get_value(pixel_widh, Options, PixWidth), - proplists:get_value(pixel_height, Options, PixHeight), - proplists:get_value(pty_opts, Options, []), TimeOut + proplists:get_value(term, TermData, os:getenv("TERM", ?DEFAULT_TERMINAL)), + proplists:get_value(width, TermData, Width), + proplists:get_value(height, TermData, Height), + proplists:get_value(pixel_widh, TermData, PixWidth), + proplists:get_value(pixel_height, TermData, PixHeight), + proplists:get_value(pty_opts, TermData, []), TimeOut ). %%-------------------------------------------------------------------- %% Not yet officialy supported! The following functions are part of the @@ -417,7 +417,8 @@ handle_msg(#ssh_msg_channel_open{channel_type = "session" = Type, maximum_packet_size = PacketSz}, #connection{options = SSHopts} = Connection0, server) -> - MinAcceptedPackSz = proplists:get_value(minimal_remote_max_packet_size, SSHopts, 0), + MinAcceptedPackSz = + ?GET_OPT(minimal_remote_max_packet_size, SSHopts), if MinAcceptedPackSz =< PacketSz -> @@ -574,7 +575,6 @@ handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, PixWidth, PixHeight, decode_pty_opts(Modes)}, Channel = ssh_channel:cache_lookup(Cache, ChannelId), - handle_cli_msg(Connection, Channel, {pty, ChannelId, WantReply, PtyRequest}); @@ -691,7 +691,6 @@ handle_cli_msg(#connection{channel_cache = Cache} = Connection, #channel{user = undefined, remote_id = RemoteId, local_id = ChannelId} = Channel0, Reply0) -> - case (catch start_cli(Connection, ChannelId)) of {ok, Pid} -> erlang:monitor(process, Pid), @@ -819,7 +818,7 @@ start_channel(Cb, Id, Args, SubSysSup, Exec, Opts) -> ssh_channel_sup:start_child(ChannelSup, ChildSpec). assert_limit_num_channels_not_exceeded(ChannelSup, Opts) -> - MaxNumChannels = proplists:get_value(max_channels, Opts, infinity), + MaxNumChannels = ?GET_OPT(max_channels, Opts), NumChannels = length([x || {_,_,worker,[ssh_channel]} <- supervisor:which_children(ChannelSup)]), if @@ -858,8 +857,8 @@ setup_session(#connection{channel_cache = Cache check_subsystem("sftp"= SsName, Options) -> - case proplists:get_value(subsystems, Options, no_subsys) of - no_subsys -> + case ?GET_OPT(subsystems, Options) of + no_subsys -> % FIXME: Can 'no_subsys' ever be matched? {SsName, {Cb, Opts}} = ssh_sftpd:subsystem_spec([]), {Cb, Opts}; SubSystems -> @@ -867,7 +866,7 @@ check_subsystem("sftp"= SsName, Options) -> end; check_subsystem(SsName, Options) -> - Subsystems = proplists:get_value(subsystems, Options, []), + Subsystems = ?GET_OPT(subsystems, Options), case proplists:get_value(SsName, Subsystems, {none, []}) of Fun when is_function(Fun) -> {Fun, []}; @@ -1022,12 +1021,13 @@ pty_req(ConnectionHandler, Channel, Term, Width, Height, ?uint32(PixWidth),?uint32(PixHeight), encode_pty_opts(PtyOpts)], TimeOut). -pty_default_dimensions(Dimension, Options) -> - case proplists:get_value(Dimension, Options, 0) of +pty_default_dimensions(Dimension, TermData) -> + case proplists:get_value(Dimension, TermData, 0) of N when is_integer(N), N > 0 -> {N, 0}; _ -> - case proplists:get_value(list_to_atom("pixel_" ++ atom_to_list(Dimension)), Options, 0) of + PixelDim = list_to_atom("pixel_" ++ atom_to_list(Dimension)), + case proplists:get_value(PixelDim, TermData, 0) of N when is_integer(N), N > 0 -> {0, N}; _ -> diff --git a/lib/ssh/src/ssh_connection_handler.erl b/lib/ssh/src/ssh_connection_handler.erl index dcf509ca09..5a13209ae3 100644 --- a/lib/ssh/src/ssh_connection_handler.erl +++ b/lib/ssh/src/ssh_connection_handler.erl @@ -76,7 +76,7 @@ %%-------------------------------------------------------------------- -spec start_link(role(), inet:socket(), - proplists:proplist() + ssh_options:options() ) -> {ok, pid()}. %% . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . start_link(Role, Socket, Options) -> @@ -99,12 +99,10 @@ stop(ConnectionHandler)-> %% Internal application API %%==================================================================== --define(DefaultTransport, {tcp, gen_tcp, tcp_closed} ). - %%-------------------------------------------------------------------- -spec start_connection(role(), inet:socket(), - proplists:proplist(), + ssh_options:options(), timeout() ) -> {ok, connection_ref()} | {error, term()}. %% . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . @@ -121,9 +119,8 @@ start_connection(client = Role, Socket, Options, Timeout) -> end; start_connection(server = Role, Socket, Options, Timeout) -> - SSH_Opts = proplists:get_value(ssh_opts, Options, []), try - case proplists:get_value(parallel_login, SSH_Opts, false) of + case ?GET_OPT(parallel_login, Options) of true -> HandshakerPid = spawn_link(fun() -> @@ -346,7 +343,7 @@ renegotiate_data(ConnectionHandler) -> | undefined, last_size_rekey = 0 :: non_neg_integer(), event_queue = [] :: list(), - opts :: proplists:proplist(), + opts :: ssh_options:options(), inet_initial_recbuf_size :: pos_integer() | undefined }). @@ -357,15 +354,14 @@ renegotiate_data(ConnectionHandler) -> %%-------------------------------------------------------------------- -spec init_connection_handler(role(), inet:socket(), - proplists:proplist() + ssh_options:options() ) -> no_return(). %% . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . init_connection_handler(Role, Socket, Opts) -> process_flag(trap_exit, true), S0 = init_process_state(Role, Socket, Opts), try - {Protocol, Callback, CloseTag} = - proplists:get_value(transport, Opts, ?DefaultTransport), + {Protocol, Callback, CloseTag} = ?GET_OPT(transport, Opts), S0#data{ssh_params = init_ssh_record(Role, Socket, Opts), transport_protocol = Protocol, transport_cb = Callback, @@ -393,7 +389,7 @@ init_process_state(Role, Socket, Opts) -> port_bindings = [], requests = [], options = Opts}, - starter = proplists:get_value(user_pid, Opts), + starter = ?GET_INTERNAL_OPT(user_pid, Opts), socket = Socket, opts = Opts }, @@ -404,18 +400,25 @@ init_process_state(Role, Socket, Opts) -> timer:apply_after(?REKEY_DATA_TIMOUT, gen_statem, cast, [self(), data_size]), cache_init_idle_timer(D); server -> - D#data{connection_state = init_connection(Role, C, Opts)} + cache_init_idle_timer( + D#data{connection_state = init_connection(Role, C, Opts)} + ) end. init_connection(server, C = #connection{}, Opts) -> - Sups = proplists:get_value(supervisors, Opts), - SystemSup = proplists:get_value(system_sup, Sups), - SubSystemSup = proplists:get_value(subsystem_sup, Sups), + Sups = ?GET_INTERNAL_OPT(supervisors, Opts), + + SystemSup = proplists:get_value(system_sup, Sups), + SubSystemSup = proplists:get_value(subsystem_sup, Sups), ConnectionSup = proplists:get_value(connection_sup, Sups), - Shell = proplists:get_value(shell, Opts), - Exec = proplists:get_value(exec, Opts), - CliSpec = proplists:get_value(ssh_cli, Opts, {ssh_cli, [Shell]}), + + Shell = ?GET_OPT(shell, Opts), + Exec = ?GET_OPT(exec, Opts), + CliSpec = case ?GET_OPT(ssh_cli, Opts) of + undefined -> {ssh_cli, [Shell]}; + Spec -> Spec + end, C#connection{cli_spec = CliSpec, exec = Exec, system_supervisor = SystemSup, @@ -426,41 +429,38 @@ init_connection(server, C = #connection{}, Opts) -> init_ssh_record(Role, Socket, Opts) -> {ok, PeerAddr} = inet:peername(Socket), - KeyCb = proplists:get_value(key_cb, Opts, ssh_file), - AuthMethods = proplists:get_value(auth_methods, - Opts, - case Role of - server -> ?SUPPORTED_AUTH_METHODS; - client -> undefined - end), + KeyCb = ?GET_OPT(key_cb, Opts), + AuthMethods = + case Role of + server -> ?GET_OPT(auth_methods, Opts); + client -> undefined + end, S0 = #ssh{role = Role, key_cb = KeyCb, opts = Opts, userauth_supported_methods = AuthMethods, available_host_keys = supported_host_keys(Role, KeyCb, Opts), - random_length_padding = proplists:get_value(max_random_length_padding, - Opts, - (#ssh{})#ssh.random_length_padding) + random_length_padding = ?GET_OPT(max_random_length_padding, Opts) }, {Vsn, Version} = ssh_transport:versions(Role, Opts), case Role of client -> - PeerName = proplists:get_value(host, Opts), + PeerName = ?GET_INTERNAL_OPT(host, Opts), S0#ssh{c_vsn = Vsn, c_version = Version, - io_cb = case proplists:get_value(user_interaction, Opts, true) of + io_cb = case ?GET_OPT(user_interaction, Opts) of true -> ssh_io; false -> ssh_no_io end, - userauth_quiet_mode = proplists:get_value(quiet_mode, Opts, false), + userauth_quiet_mode = ?GET_OPT(quiet_mode, Opts), peer = {PeerName, PeerAddr} }; server -> S0#ssh{s_vsn = Vsn, s_version = Version, - io_cb = proplists:get_value(io_cb, Opts, ssh_io), + io_cb = ?GET_INTERNAL_OPT(io_cb, Opts, ssh_io), userauth_methods = string:tokens(AuthMethods, ","), kb_tries_left = 3, peer = {undefined, PeerAddr} @@ -849,14 +849,12 @@ handle_event(_, Msg = #ssh_msg_userauth_failure{}, {userauth_keyboard_interactiv handle_event(_, Msg=#ssh_msg_userauth_failure{}, {userauth_keyboard_interactive_info_response, client}, #data{ssh_params = Ssh0} = D0) -> Opts = Ssh0#ssh.opts, - D = case proplists:get_value(password, Opts) of + D = case ?GET_OPT(password, Opts) of undefined -> D0; _ -> D0#data{ssh_params = - Ssh0#ssh{opts = - lists:keyreplace(password,1,Opts, - {password,not_ok})}} % FIXME:intermodule dependency + Ssh0#ssh{opts = ?PUT_OPT({password,not_ok}, Opts)}} % FIXME:intermodule dependency end, {next_state, {userauth,client}, D, [{next_event, internal, Msg}]}; @@ -923,6 +921,9 @@ handle_event(internal, Msg=#ssh_msg_channel_extended_data{}, StateName, D) - handle_event(internal, Msg=#ssh_msg_channel_eof{}, StateName, D) -> handle_connection_msg(Msg, StateName, D); +handle_event(internal, Msg=#ssh_msg_channel_close{}, {connected,server} = StateName, D) -> + handle_connection_msg(Msg, StateName, cache_request_idle_timer_check(D)); + handle_event(internal, Msg=#ssh_msg_channel_close{}, StateName, D) -> handle_connection_msg(Msg, StateName, D); @@ -954,7 +955,7 @@ handle_event(cast, renegotiate, _, _) -> handle_event(cast, data_size, {connected,Role}, D) -> {ok, [{send_oct,Sent0}]} = inet:getstat(D#data.socket, [send_oct]), Sent = Sent0 - D#data.last_size_rekey, - MaxSent = proplists:get_value(rekey_limit, D#data.opts, 1024000000), + MaxSent = ?GET_OPT(rekey_limit, D#data.opts), timer:apply_after(?REKEY_DATA_TIMOUT, gen_statem, cast, [self(), data_size]), case Sent >= MaxSent of true -> @@ -1294,11 +1295,12 @@ handle_event(info, UnexpectedMessage, StateName, D = #data{ssh_params = Ssh}) -> "Unexpected message '~p' received in state '~p'\n" "Role: ~p\n" "Peer: ~p\n" - "Local Address: ~p\n", [UnexpectedMessage, - StateName, - Ssh#ssh.role, - Ssh#ssh.peer, - proplists:get_value(address, Ssh#ssh.opts)])), + "Local Address: ~p\n", + [UnexpectedMessage, + StateName, + Ssh#ssh.role, + Ssh#ssh.peer, + ?GET_INTERNAL_OPT(address, Ssh#ssh.opts)])), error_logger:info_report(Msg), keep_state_and_data; @@ -1312,11 +1314,12 @@ handle_event(info, UnexpectedMessage, StateName, D = #data{ssh_params = Ssh}) -> "Message: ~p\n" "Role: ~p\n" "Peer: ~p\n" - "Local Address: ~p\n", [Other, - UnexpectedMessage, - Ssh#ssh.role, - element(2,Ssh#ssh.peer), - proplists:get_value(address, Ssh#ssh.opts)] + "Local Address: ~p\n", + [Other, + UnexpectedMessage, + Ssh#ssh.role, + element(2,Ssh#ssh.peer), + ?GET_INTERNAL_OPT(address, Ssh#ssh.opts)] )), error_logger:error_report(Msg), keep_state_and_data @@ -1438,11 +1441,11 @@ code_change(_OldVsn, StateName, State, _Extra) -> %%-------------------------------------------------------------------- %% Starting -start_the_connection_child(UserPid, Role, Socket, Options) -> - Sups = proplists:get_value(supervisors, Options), +start_the_connection_child(UserPid, Role, Socket, Options0) -> + Sups = ?GET_INTERNAL_OPT(supervisors, Options0), ConnectionSup = proplists:get_value(connection_sup, Sups), - Opts = [{supervisors, Sups}, {user_pid, UserPid} | proplists:get_value(ssh_opts, Options, [])], - {ok, Pid} = ssh_connection_sup:start_child(ConnectionSup, [Role, Socket, Opts]), + Options = ?PUT_INTERNAL_OPT({user_pid,UserPid}, Options0), + {ok, Pid} = ssh_connection_sup:start_child(ConnectionSup, [Role, Socket, Options]), ok = socket_control(Socket, Pid, Options), Pid. @@ -1499,7 +1502,7 @@ supported_host_keys(server, KeyCb, Options) -> find_sup_hkeys(Options) -> case proplists:get_value(public_key, - proplists:get_value(preferred_algorithms,Options,[]) + ?GET_OPT(preferred_algorithms,Options) ) of undefined -> @@ -1512,9 +1515,10 @@ find_sup_hkeys(Options) -> %% Alg :: atom() -available_host_key(KeyCb, Alg, Opts) -> - element(1, catch KeyCb:host_key(Alg, Opts)) == ok. - +available_host_key({KeyCb,KeyCbOpts}, Alg, Opts) -> + UserOpts = ?GET_OPT(user_options, Opts), + element(1, + catch KeyCb:host_key(Alg, [{key_cb_private,KeyCbOpts}|UserOpts])) == ok. send_msg(Msg, State=#data{ssh_params=Ssh0}) when is_tuple(Msg) -> {Bytes, Ssh} = ssh_transport:ssh_packet(Msg, Ssh0), @@ -1770,47 +1774,24 @@ get_repl(X, Acc) -> exit({get_repl,X,Acc}). %%%---------------------------------------------------------------- -disconnect_fun({disconnect,Msg}, D) -> - disconnect_fun(Msg, D); -disconnect_fun(Reason, #data{opts=Opts}) -> - case proplists:get_value(disconnectfun, Opts) of - undefined -> - ok; - Fun -> - catch Fun(Reason) - end. - -unexpected_fun(UnexpectedMessage, #data{opts = Opts, - ssh_params = #ssh{peer = {_,Peer} } - } ) -> - case proplists:get_value(unexpectedfun, Opts) of - undefined -> - report; - Fun -> - catch Fun(UnexpectedMessage, Peer) - end. +-define(CALL_FUN(Key,D), catch (?GET_OPT(Key, D#data.opts)) ). +disconnect_fun({disconnect,Msg}, D) -> ?CALL_FUN(disconnectfun,D)(Msg); +disconnect_fun(Reason, D) -> ?CALL_FUN(disconnectfun,D)(Reason). + +unexpected_fun(UnexpectedMessage, #data{ssh_params = #ssh{peer = {_,Peer} }} = D) -> + ?CALL_FUN(unexpectedfun,D)(UnexpectedMessage, Peer). debug_fun(#ssh_msg_debug{always_display = Display, message = DbgMsg, language = Lang}, - #data{opts = Opts}) -> - case proplists:get_value(ssh_msg_debug_fun, Opts) of - undefined -> - ok; - Fun -> - catch Fun(self(), Display, DbgMsg, Lang) - end. + D) -> + ?CALL_FUN(ssh_msg_debug_fun,D)(self(), Display, DbgMsg, Lang). -connected_fun(User, Method, #data{ssh_params = #ssh{peer = {_,Peer}}, - opts = Opts}) -> - case proplists:get_value(connectfun, Opts) of - undefined -> - ok; - Fun -> - catch Fun(User, Peer, Method) - end. +connected_fun(User, Method, #data{ssh_params = #ssh{peer = {_,Peer}}} = D) -> + ?CALL_FUN(connectfun,D)(User, Peer, Method). + retry_fun(_, undefined, _) -> ok; @@ -1824,7 +1805,7 @@ retry_fun(User, Reason, #data{ssh_params = #ssh{opts = Opts, _ -> {infofun, Reason} end, - Fun = proplists:get_value(Tag, Opts, fun(_,_)-> ok end), + Fun = ?GET_OPT(Tag, Opts), try erlang:fun_info(Fun, arity) of {arity, 2} -> %% Backwards compatible @@ -1843,7 +1824,7 @@ retry_fun(User, Reason, #data{ssh_params = #ssh{opts = Opts, %%% channels open for a while. cache_init_idle_timer(D) -> - case proplists:get_value(idle_time, D#data.opts, infinity) of + case ?GET_OPT(idle_time, D#data.opts) of infinity -> D#data{idle_timer_value = infinity, idle_timer_ref = infinity % A flag used later... @@ -1906,9 +1887,8 @@ start_channel_request_timer(Channel, From, Time) -> %%% Connection start and initalization helpers socket_control(Socket, Pid, Options) -> - {_, TransportCallback, _} = % For example {_,gen_tcp,_} - proplists:get_value(transport, Options, ?DefaultTransport), - case TransportCallback:controlling_process(Socket, Pid) of + {_, Callback, _} = ?GET_OPT(transport, Options), + case Callback:controlling_process(Socket, Pid) of ok -> gen_statem:cast(Pid, socket_control); {error, Reason} -> diff --git a/lib/ssh/src/ssh_file.erl b/lib/ssh/src/ssh_file.erl index 216f65f33a..898b4cc5c4 100644 --- a/lib/ssh/src/ssh_file.erl +++ b/lib/ssh/src/ssh_file.erl @@ -192,8 +192,8 @@ lookup_user_key(Key, User, Opts) -> ssh_dir({remoteuser, User}, Opts) -> case proplists:get_value(user_dir_fun, Opts) of undefined -> - case proplists:get_value(user_dir, Opts) of - undefined -> + case proplists:get_value(user_dir, Opts, false) of + false -> default_user_dir(); Dir -> Dir diff --git a/lib/ssh/src/ssh_io.erl b/lib/ssh/src/ssh_io.erl index 1d8f370884..6828fd4760 100644 --- a/lib/ssh/src/ssh_io.erl +++ b/lib/ssh/src/ssh_io.erl @@ -27,17 +27,17 @@ -export([yes_no/2, read_password/2, read_line/2, format/2]). -include("ssh.hrl"). -read_line(Prompt, Ssh) -> +read_line(Prompt, Opts) -> format("~s", [listify(Prompt)]), - proplists:get_value(user_pid, Ssh) ! {self(), question}, + ?GET_INTERNAL_OPT(user_pid, Opts) ! {self(), question}, receive Answer when is_list(Answer) -> Answer end. -yes_no(Prompt, Ssh) -> +yes_no(Prompt, Opts) -> format("~s [y/n]?", [Prompt]), - proplists:get_value(user_pid, Ssh#ssh.opts) ! {self(), question}, + ?GET_INTERNAL_OPT(user_pid, Opts) ! {self(), question}, receive %% I can't see that the atoms y and n are ever received, but it must %% be investigated before removing @@ -52,15 +52,13 @@ yes_no(Prompt, Ssh) -> "N" -> no; _ -> format("please answer y or n\n",[]), - yes_no(Prompt, Ssh) + yes_no(Prompt, Opts) end end. - -read_password(Prompt, #ssh{opts=Opts}) -> read_password(Prompt, Opts); -read_password(Prompt, Opts) when is_list(Opts) -> +read_password(Prompt, Opts) -> format("~s", [listify(Prompt)]), - proplists:get_value(user_pid, Opts) ! {self(), user_password}, + ?GET_INTERNAL_OPT(user_pid, Opts) ! {self(), user_password}, receive Answer when is_list(Answer) -> case trim(Answer) of diff --git a/lib/ssh/src/ssh_options.erl b/lib/ssh/src/ssh_options.erl new file mode 100644 index 0000000000..55f9c6bdc8 --- /dev/null +++ b/lib/ssh/src/ssh_options.erl @@ -0,0 +1,884 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2004-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% + +-module(ssh_options). + +-include("ssh.hrl"). +-include_lib("kernel/include/file.hrl"). + +-export([default/1, + get_value/5, get_value/6, + put_value/5, + handle_options/2 + ]). + +-export_type([options/0 + ]). + +%%%================================================================ +%%% Types + +-type options() :: #{socket_options := socket_options(), + internal_options := internal_options(), + option_key() => any() + }. + +-type socket_options() :: proplists:proplist(). +-type internal_options() :: #{option_key() => any()}. + +-type option_key() :: atom(). + +-type option_in() :: proplists:property() | proplists:proplist() . + +-type option_class() :: internal_options | socket_options | user_options . + +-type option_declaration() :: #{class := user_options, + chk := fun((any) -> boolean() | {true,any()}), + default => any() + }. + +-type option_declarations() :: #{ {option_key(),def} := option_declaration() }. + +-type error() :: {error,{eoptions,any()}} . + +%%%================================================================ +%%% +%%% Get an option +%%% + +-spec get_value(option_class(), option_key(), options(), + atom(), non_neg_integer()) -> any() | no_return(). + +get_value(Class, Key, Opts, _CallerMod, _CallerLine) when is_map(Opts) -> + case Class of + internal_options -> maps:get(Key, maps:get(internal_options,Opts)); + socket_options -> proplists:get_value(Key, maps:get(socket_options,Opts)); + user_options -> maps:get(Key, Opts) + end; +get_value(Class, Key, Opts, _CallerMod, _CallerLine) -> + io:format("*** Bad Opts GET OPT ~p ~p:~p Key=~p,~n Opts=~p~n",[Class,_CallerMod,_CallerLine,Key,Opts]), + error({bad_options,Class, Key, Opts, _CallerMod, _CallerLine}). + + +-spec get_value(option_class(), option_key(), options(), any(), + atom(), non_neg_integer()) -> any() | no_return(). + +get_value(socket_options, Key, Opts, Def, _CallerMod, _CallerLine) when is_map(Opts) -> + proplists:get_value(Key, maps:get(socket_options,Opts), Def); +get_value(Class, Key, Opts, Def, CallerMod, CallerLine) when is_map(Opts) -> + try get_value(Class, Key, Opts, CallerMod, CallerLine) + catch + error:{badkey,Key} -> Def + end; +get_value(Class, Key, Opts, _Def, _CallerMod, _CallerLine) -> + io:format("*** Bad Opts GET OPT ~p ~p:~p Key=~p,~n Opts=~p~n",[Class,_CallerMod,_CallerLine,Key,Opts]), + error({bad_options,Class, Key, Opts, _CallerMod, _CallerLine}). + + +%%%================================================================ +%%% +%%% Put an option +%%% + +-spec put_value(option_class(), option_in(), options(), + atom(), non_neg_integer()) -> options(). + +put_value(user_options, KeyVal, Opts, _CallerMod, _CallerLine) when is_map(Opts) -> + put_user_value(KeyVal, Opts); + +put_value(internal_options, KeyVal, Opts, _CallerMod, _CallerLine) when is_map(Opts) -> + InternalOpts = maps:get(internal_options,Opts), + Opts#{internal_options := put_internal_value(KeyVal, InternalOpts)}; + +put_value(socket_options, KeyVal, Opts, _CallerMod, _CallerLine) when is_map(Opts) -> + SocketOpts = maps:get(socket_options,Opts), + Opts#{socket_options := put_socket_value(KeyVal, SocketOpts)}. + + +%%%---------------- +put_user_value(L, Opts) when is_list(L) -> + lists:foldl(fun put_user_value/2, Opts, L); +put_user_value({Key,Value}, Opts) -> + Opts#{Key := Value}. + +%%%---------------- +put_internal_value(L, IntOpts) when is_list(L) -> + lists:foldl(fun put_internal_value/2, IntOpts, L); +put_internal_value({Key,Value}, IntOpts) -> + IntOpts#{Key => Value}. + +%%%---------------- +put_socket_value(L, SockOpts) when is_list(L) -> + L ++ SockOpts; +put_socket_value({Key,Value}, SockOpts) -> + [{Key,Value} | SockOpts]; +put_socket_value(A, SockOpts) when is_atom(A) -> + [A | SockOpts]. + +%%%================================================================ +%%% +%%% Initialize the options +%%% + +-spec handle_options(role(), proplists:proplist()) -> options() | error() . + +-spec handle_options(role(), proplists:proplist(), options()) -> options() | error() . + +handle_options(Role, PropList0) -> + handle_options(Role, PropList0, #{socket_options => [], + internal_options => #{}, + user_options => [] + }). + +handle_options(Role, PropList0, Opts0) when is_map(Opts0), + is_list(PropList0) -> + PropList1 = proplists:unfold(PropList0), + try + OptionDefinitions = default(Role), + InitialMap = + maps:fold( + fun({K,def}, #{default:=V}, M) -> M#{K=>V}; + (_,_,M) -> M + end, + Opts0#{user_options => + maps:get(user_options,Opts0) ++ PropList1 + }, + OptionDefinitions), + %% Enter the user's values into the map; unknown keys are + %% treated as socket options + lists:foldl(fun(KV, Vals) -> + save(KV, OptionDefinitions, Vals) + end, InitialMap, PropList1) + catch + error:{eoptions, KV, undefined} -> + {error, {eoptions,KV}}; + + error:{eoptions, KV, Txt} when is_list(Txt) -> + {error, {eoptions,{KV,lists:flatten(Txt)}}}; + + error:{eoptions, KV, Extra} -> + {error, {eoptions,{KV,Extra}}} + end. + + +check_fun(Key, Defs) -> + #{chk := Fun} = maps:get({Key,def}, Defs), + Fun. + +%%%================================================================ +%%% +%%% Check and save one option +%%% + + +%%% First some prohibited inet options: +save({K,V}, _, _) when K == reuseaddr ; + K == active + -> + forbidden_option(K, V); + +%%% then compatibility conversions: +save({allow_user_interaction,V}, Opts, Vals) -> + save({user_interaction,V}, Opts, Vals); + +%% Special case for socket options 'inet' and 'inet6' +save(Inet, Defs, OptMap) when Inet==inet ; Inet==inet6 -> + save({inet,Inet}, Defs, OptMap); + +%% Two clauses to prepare for a proplists:unfold +save({Inet,true}, Defs, OptMap) when Inet==inet ; Inet==inet6 -> save({inet,Inet}, Defs, OptMap); +save({Inet,false}, _Defs, OptMap) when Inet==inet ; Inet==inet6 -> OptMap; + +%% and finaly the 'real stuff': +save({Key,Value}, Defs, OptMap) when is_map(OptMap) -> + try (check_fun(Key,Defs))(Value) + of + true -> + OptMap#{Key := Value}; + {true, ModifiedValue} -> + OptMap#{Key := ModifiedValue}; + false -> + error({eoptions, {Key,Value}, "Bad value"}) + catch + %% An unknown Key (= not in the definition map) is + %% regarded as an inet option: + error:{badkey,{inet,def}} -> + %% atomic (= non-tuple) options 'inet' and 'inet6': + OptMap#{socket_options := [Value | maps:get(socket_options,OptMap)]}; + error:{badkey,{Key,def}} -> + OptMap#{socket_options := [{Key,Value} | maps:get(socket_options,OptMap)]}; + + %% But a Key that is known but the value does not validate + %% by the check fun will give an error exception: + error:{check,{BadValue,Extra}} -> + error({eoptions, {Key,BadValue}, Extra}) + end. + +%%%================================================================ +%%% +%%% Default options +%%% + +-spec default(role() | common) -> option_declarations() . + +default(server) -> + (default(common)) + #{ + {subsystems, def} => + #{default => [ssh_sftpd:subsystem_spec([])], + chk => fun(L) -> + is_list(L) andalso + lists:all(fun({Name,{CB,Args}}) -> + check_string(Name) andalso + is_atom(CB) andalso + is_list(Args); + (_) -> + false + end, L) + end, + class => user_options + }, + + {shell, def} => + #{default => {shell, start, []}, + chk => fun({M,F,A}) -> is_atom(M) andalso is_atom(F) andalso is_list(A); + (V) -> check_function1(V) orelse check_function2(V) + end, + class => user_options + }, + + {exec, def} => % FIXME: need some archeology.... + #{default => undefined, + chk => fun({M,F,_}) -> is_atom(M) andalso is_atom(F); + (V) -> is_function(V) + end, + class => user_options + }, + + {ssh_cli, def} => + #{default => undefined, + chk => fun({Cb, As}) -> is_atom(Cb) andalso is_list(As); + (V) -> V == no_cli + end, + class => user_options + }, + + {system_dir, def} => + #{default => "/etc/ssh", + chk => fun(V) -> check_string(V) andalso check_dir(V) end, + class => user_options + }, + + {auth_methods, def} => + #{default => ?SUPPORTED_AUTH_METHODS, + chk => fun check_string/1, + class => user_options + }, + + {auth_method_kb_interactive_data, def} => + #{default => undefined, % Default value can be constructed when User is known + chk => fun({S1,S2,S3,B}) -> + check_string(S1) andalso + check_string(S2) andalso + check_string(S3) andalso + is_boolean(B); + (F) -> + check_function3(F) + end, + class => user_options + }, + + {user_passwords, def} => + #{default => [], + chk => fun(V) -> + is_list(V) andalso + lists:all(fun({S1,S2}) -> + check_string(S1) andalso + check_string(S2) + end, V) + end, + class => user_options + }, + + {password, def} => + #{default => undefined, + chk => fun check_string/1, + class => user_options + }, + + {dh_gex_groups, def} => + #{default => undefined, + chk => fun check_dh_gex_groups/1, + class => user_options + }, + + {dh_gex_limits, def} => + #{default => {0, infinity}, + chk => fun({I1,I2}) -> + check_pos_integer(I1) andalso + check_pos_integer(I2) andalso + I1 < I2; + (_) -> + false + end, + class => user_options + }, + + {pwdfun, def} => + #{default => undefined, + chk => fun(V) -> check_function4(V) orelse check_function2(V) end, + class => user_options + }, + + {negotiation_timeout, def} => + #{default => 2*60*1000, + chk => fun check_timeout/1, + class => user_options + }, + + {max_sessions, def} => + #{default => infinity, + chk => fun check_pos_integer/1, + class => user_options + }, + + {max_channels, def} => + #{default => infinity, + chk => fun check_pos_integer/1, + class => user_options + }, + + {parallel_login, def} => + #{default => false, + chk => fun erlang:is_boolean/1, + class => user_options + }, + + {minimal_remote_max_packet_size, def} => + #{default => 0, + chk => fun check_pos_integer/1, + class => user_options + }, + + {failfun, def} => + #{default => fun(_,_,_) -> void end, + chk => fun(V) -> check_function3(V) orelse + check_function2(V) % Backwards compatibility + end, + class => user_options + }, + + {connectfun, def} => + #{default => fun(_,_,_) -> void end, + chk => fun check_function3/1, + class => user_options + }, + +%%%%% Undocumented + {infofun, def} => + #{default => fun(_,_,_) -> void end, + chk => fun(V) -> check_function3(V) orelse + check_function2(V) % Backwards compatibility + end, + class => user_options + } + }; + +default(client) -> + (default(common)) + #{ + {dsa_pass_phrase, def} => + #{default => undefined, + chk => fun check_string/1, + class => user_options + }, + + {rsa_pass_phrase, def} => + #{default => undefined, + chk => fun check_string/1, + class => user_options + }, + + {silently_accept_hosts, def} => + #{default => false, + chk => fun check_silently_accept_hosts/1, + class => user_options + }, + + {user_interaction, def} => + #{default => true, + chk => fun erlang:is_boolean/1, + class => user_options + }, + + {pref_public_key_algs, def} => + #{default => + %% Get dynamically supported keys in the order of the ?SUPPORTED_USER_KEYS + [A || A <- ?SUPPORTED_USER_KEYS, + lists:member(A, ssh_transport:supported_algorithms(public_key))], + chk => + fun check_pref_public_key_algs/1, + class => + ssh + }, + + {dh_gex_limits, def} => + #{default => {1024, 6144, 8192}, % FIXME: Is this true nowadays? + chk => fun({Min,I,Max}) -> + lists:all(fun check_pos_integer/1, + [Min,I,Max]); + (_) -> false + end, + class => user_options + }, + + {connect_timeout, def} => + #{default => infinity, + chk => fun check_timeout/1, + class => user_options + }, + + {user, def} => + #{default => + begin + Env = case os:type() of + {win32, _} -> "USERNAME"; + {unix, _} -> "LOGNAME" + end, + case os:getenv(Env) of + false -> + case os:getenv("USER") of + false -> undefined; + User -> User + end; + User -> + User + end + end, + chk => fun check_string/1, + class => user_options + }, + + {password, def} => + #{default => undefined, + chk => fun check_string/1, + class => user_options + }, + + {quiet_mode, def} => + #{default => false, + chk => fun erlang:is_boolean/1, + class => user_options + }, + +%%%%% Undocumented + {keyboard_interact_fun, def} => + #{default => undefined, + chk => fun check_function3/1, + class => user_options + } + }; + +default(common) -> + #{ + {user_dir, def} => + #{default => false, % FIXME: TBD ~/.ssh at time of call when user is known + chk => fun(V) -> check_string(V) andalso check_dir(V) end, + class => user_options + }, + + {preferred_algorithms, def} => + #{default => ssh:default_algorithms(), + chk => fun check_preferred_algorithms/1, + class => user_options + }, + + {id_string, def} => + #{default => undefined, % FIXME: see ssh_transport:ssh_vsn/0 + chk => fun(random) -> + {true, {random,2,5}}; % 2 - 5 random characters + ({random,I1,I2}) -> + %% Undocumented + check_pos_integer(I1) andalso + check_pos_integer(I2) andalso + I1=<I2; + (V) -> + check_string(V) + end, + class => user_options + }, + + {key_cb, def} => + #{default => {ssh_file, []}, + chk => fun({Mod,Opts}) -> is_atom(Mod) andalso is_list(Opts); + (Mod) when is_atom(Mod) -> {true, {Mod,[]}}; + (_) -> false + end, + class => user_options + }, + + {profile, def} => + #{default => ?DEFAULT_PROFILE, + chk => fun erlang:is_atom/1, + class => user_options + }, + + {idle_time, def} => + #{default => infinity, + chk => fun check_timeout/1, + class => user_options + }, + + %% This is a "SocketOption"... + %% {fd, def} => + %% #{default => undefined, + %% chk => fun erlang:is_integer/1, + %% class => user_options + %% }, + + {disconnectfun, def} => + #{default => fun(_) -> void end, + chk => fun check_function1/1, + class => user_options + }, + + {unexpectedfun, def} => + #{default => fun(_,_) -> report end, + chk => fun check_function2/1, + class => user_options + }, + + {ssh_msg_debug_fun, def} => + #{default => fun(_,_,_,_) -> void end, + chk => fun check_function4/1, + class => user_options + }, + + {rekey_limit, def} => % FIXME: Why not common? + #{default => 1024000000, + chk => fun check_non_neg_integer/1, + class => user_options + }, + +%%%%% Undocumented + {transport, def} => + #{default => ?DEFAULT_TRANSPORT, + chk => fun({A,B,C}) -> + is_atom(A) andalso is_atom(B) andalso is_atom(C) + end, + class => user_options + }, + + {vsn, def} => + #{default => {2,0}, + chk => fun({Maj,Min}) -> check_non_neg_integer(Maj) andalso check_non_neg_integer(Min); + (_) -> false + end, + class => user_options + }, + + {tstflg, def} => + #{default => [], + chk => fun erlang:is_list/1, + class => user_options + }, + + {user_dir_fun, def} => + #{default => undefined, + chk => fun check_function1/1, + class => user_options + }, + + {max_random_length_padding, def} => + #{default => ?MAX_RND_PADDING_LEN, + chk => fun check_non_neg_integer/1, + class => user_options + } + }. + + +%%%================================================================ +%%%================================================================ +%%%================================================================ + +%%% +%%% check_*/1 -> true | false | error({check,Spec}) +%%% See error_in_check/2,3 +%%% + +%%% error_in_check(BadValue) -> error_in_check(BadValue, undefined). + +error_in_check(BadValue, Extra) -> error({check,{BadValue,Extra}}). + + +%%%---------------------------------------------------------------- +check_timeout(infinity) -> true; +check_timeout(I) -> check_pos_integer(I). + +%%%---------------------------------------------------------------- +check_pos_integer(I) -> is_integer(I) andalso I>0. + +%%%---------------------------------------------------------------- +check_non_neg_integer(I) -> is_integer(I) andalso I>=0. + +%%%---------------------------------------------------------------- +check_function1(F) -> is_function(F,1). +check_function2(F) -> is_function(F,2). +check_function3(F) -> is_function(F,3). +check_function4(F) -> is_function(F,4). + +%%%---------------------------------------------------------------- +check_pref_public_key_algs(V) -> + %% Get the dynamically supported keys, that is, thoose + %% that are stored + PKs = ssh_transport:supported_algorithms(public_key), + CHK = fun(A, Ack) -> + case lists:member(A, PKs) of + true -> + [A|Ack]; + false -> + %% Check with the documented options, that is, + %% the one we can handle + case lists:member(A,?SUPPORTED_USER_KEYS) of + false -> + %% An algorithm ssh never can handle + error_in_check(A, "Not supported public key"); + true -> + %% An algorithm ssh can handle, but not in + %% this very call + Ack + end + end + end, + case lists:foldr( + fun(ssh_dsa, Ack) -> CHK('ssh-dss', Ack); % compatibility + (ssh_rsa, Ack) -> CHK('ssh-rsa', Ack); % compatibility + (X, Ack) -> CHK(X, Ack) + end, [], V) + of + V -> true; + [] -> false; + V1 -> {true,V1} + end. + + +%%%---------------------------------------------------------------- +%% Check that it is a directory and is readable +check_dir(Dir) -> + case file:read_file_info(Dir) of + {ok, #file_info{type = directory, + access = Access}} -> + case Access of + read -> true; + read_write -> true; + _ -> error_in_check(Dir, eacces) + end; + + {ok, #file_info{}}-> + error_in_check(Dir, enotdir); + + {error, Error} -> + error_in_check(Dir, Error) + end. + +%%%---------------------------------------------------------------- +check_string(S) -> is_list(S). % FIXME: stub + +%%%---------------------------------------------------------------- +check_dh_gex_groups({file,File}) when is_list(File) -> + case file:consult(File) of + {ok, GroupDefs} -> + check_dh_gex_groups(GroupDefs); + {error, Error} -> + error_in_check({file,File},Error) + end; + +check_dh_gex_groups({ssh_moduli_file,File}) when is_list(File) -> + case file:open(File,[read]) of + {ok,D} -> + try + read_moduli_file(D, 1, []) + of + {ok,Moduli} -> + check_dh_gex_groups(Moduli); + {error,Error} -> + error_in_check({ssh_moduli_file,File}, Error) + catch + _:_ -> + error_in_check({ssh_moduli_file,File}, "Bad format in file "++File) + after + file:close(D) + end; + + {error, Error} -> + error_in_check({ssh_moduli_file,File}, Error) + end; + +check_dh_gex_groups(L0) when is_list(L0), is_tuple(hd(L0)) -> + {true, + collect_per_size( + lists:foldl( + fun({N,G,P}, Acc) when is_integer(N),N>0, + is_integer(G),G>0, + is_integer(P),P>0 -> + [{N,{G,P}} | Acc]; + ({N,{G,P}}, Acc) when is_integer(N),N>0, + is_integer(G),G>0, + is_integer(P),P>0 -> + [{N,{G,P}} | Acc]; + ({N,GPs}, Acc) when is_list(GPs) -> + lists:foldr(fun({Gi,Pi}, Acci) when is_integer(Gi),Gi>0, + is_integer(Pi),Pi>0 -> + [{N,{Gi,Pi}} | Acci] + end, Acc, GPs) + end, [], L0))}; + +check_dh_gex_groups(_) -> + false. + + + +collect_per_size(L) -> + lists:foldr( + fun({Sz,GP}, [{Sz,GPs}|Acc]) -> [{Sz,[GP|GPs]}|Acc]; + ({Sz,GP}, Acc) -> [{Sz,[GP]}|Acc] + end, [], lists:sort(L)). + +read_moduli_file(D, I, Acc) -> + case io:get_line(D,"") of + {error,Error} -> + {error,Error}; + eof -> + {ok, Acc}; + "#" ++ _ -> read_moduli_file(D, I+1, Acc); + <<"#",_/binary>> -> read_moduli_file(D, I+1, Acc); + Data -> + Line = if is_binary(Data) -> binary_to_list(Data); + is_list(Data) -> Data + end, + try + [_Time,_Class,_Tests,_Tries,Size,G,P] = string:tokens(Line," \r\n"), + M = {list_to_integer(Size), + {list_to_integer(G), list_to_integer(P,16)} + }, + read_moduli_file(D, I+1, [M|Acc]) + catch + _:_ -> + read_moduli_file(D, I+1, Acc) + end + end. + +%%%---------------------------------------------------------------- +-define(SHAs, [md5, sha, sha224, sha256, sha384, sha512]). + +check_silently_accept_hosts(B) when is_boolean(B) -> true; +check_silently_accept_hosts(F) when is_function(F,2) -> true; +check_silently_accept_hosts({S,F}) when is_atom(S), + is_function(F,2) -> + lists:member(S, ?SHAs) andalso + lists:member(S, proplists:get_value(hashs,crypto:supports())); +check_silently_accept_hosts({L,F}) when is_list(L), + is_function(F,2) -> + lists:all(fun(S) -> + lists:member(S, ?SHAs) andalso + lists:member(S, proplists:get_value(hashs,crypto:supports())) + end, L); +check_silently_accept_hosts(_) -> false. + +%%%---------------------------------------------------------------- +check_preferred_algorithms(Algs) -> + try alg_duplicates(Algs, [], []) + of + [] -> + {true, + [try ssh_transport:supported_algorithms(Key) + of + DefAlgs -> handle_pref_alg(Key,Vals,DefAlgs) + catch + _:_ -> error_in_check(Key,"Bad preferred_algorithms key") + end || {Key,Vals} <- Algs] + }; + + Dups -> + error_in_check(Dups, "Duplicates") + catch + _:_ -> + false + end. + +alg_duplicates([{K,V}|KVs], Ks, Dups0) -> + Dups = + case lists:member(K,Ks) of + true -> [K|Dups0]; + false -> Dups0 + end, + case V--lists:usort(V) of + [] -> alg_duplicates(KVs, [K|Ks], Dups); + Ds -> alg_duplicates(KVs, [K|Ks], Dups++Ds) + end; +alg_duplicates([], _Ks, Dups) -> + Dups. + +handle_pref_alg(Key, + Vs=[{client2server,C2Ss=[_|_]},{server2client,S2Cs=[_|_]}], + [{client2server,Sup_C2Ss},{server2client,Sup_S2Cs}] + ) -> + chk_alg_vs(Key, C2Ss, Sup_C2Ss), + chk_alg_vs(Key, S2Cs, Sup_S2Cs), + {Key, Vs}; + +handle_pref_alg(Key, + Vs=[{server2client,[_|_]},{client2server,[_|_]}], + Sup=[{client2server,_},{server2client,_}] + ) -> + handle_pref_alg(Key, lists:reverse(Vs), Sup); + +handle_pref_alg(Key, + Vs=[V|_], + Sup=[{client2server,_},{server2client,_}] + ) when is_atom(V) -> + handle_pref_alg(Key, [{client2server,Vs},{server2client,Vs}], Sup); + +handle_pref_alg(Key, + Vs=[V|_], + Sup=[S|_] + ) when is_atom(V), is_atom(S) -> + chk_alg_vs(Key, Vs, Sup), + {Key, Vs}; + +handle_pref_alg(Key, Vs, _) -> + error_in_check({Key,Vs}, "Badly formed list"). + +chk_alg_vs(OptKey, Values, SupportedValues) -> + case (Values -- SupportedValues) of + [] -> Values; + Bad -> error_in_check({OptKey,Bad}, "Unsupported value(s) found") + end. + +%%%---------------------------------------------------------------- +forbidden_option(K,V) -> + Txt = io_lib:format("The option '~s' is used internally. The " + "user is not allowed to specify this option.", + [K]), + error({eoptions, {K,V}, Txt}). + +%%%---------------------------------------------------------------- diff --git a/lib/ssh/src/ssh_sftp.erl b/lib/ssh/src/ssh_sftp.erl index 8d994cdb43..140856c8e3 100644 --- a/lib/ssh/src/ssh_sftp.erl +++ b/lib/ssh/src/ssh_sftp.erl @@ -100,18 +100,14 @@ start_channel(Socket) when is_port(Socket) -> start_channel(Host) when is_list(Host) -> start_channel(Host, []). -start_channel(Socket, Options) when is_port(Socket) -> - Timeout = - %% A mixture of ssh:connect and ssh_sftp:start_channel: - case proplists:get_value(connect_timeout, Options, undefined) of - undefined -> - proplists:get_value(timeout, Options, infinity); - TO -> - TO - end, - case ssh:connect(Socket, Options, Timeout) of +start_channel(Socket, UserOptions) when is_port(Socket) -> + {SshOpts, _ChanOpts, SftpOpts} = handle_options(UserOptions), + Timeout = % A mixture of ssh:connect and ssh_sftp:start_channel: + proplists:get_value(connect_timeout, SshOpts, + proplists:get_value(timeout, SftpOpts, infinity)), + case ssh:connect(Socket, SshOpts, Timeout) of {ok,Cm} -> - case start_channel(Cm, Options) of + case start_channel(Cm, UserOptions) of {ok, Pid} -> {ok, Pid, Cm}; Error -> @@ -120,9 +116,9 @@ start_channel(Socket, Options) when is_port(Socket) -> Error -> Error end; -start_channel(Cm, Opts) when is_pid(Cm) -> - Timeout = proplists:get_value(timeout, Opts, infinity), - {_, ChanOpts, SftpOpts} = handle_options(Opts, [], [], []), +start_channel(Cm, UserOptions) when is_pid(Cm) -> + Timeout = proplists:get_value(timeout, UserOptions, infinity), + {_SshOpts, ChanOpts, SftpOpts} = handle_options(UserOptions), case ssh_xfer:attach(Cm, [], ChanOpts) of {ok, ChannelId, Cm} -> case ssh_channel:start(Cm, ChannelId, @@ -143,15 +139,17 @@ start_channel(Cm, Opts) when is_pid(Cm) -> Error end; -start_channel(Host, Opts) -> - start_channel(Host, 22, Opts). -start_channel(Host, Port, Opts) -> - {SshOpts, ChanOpts, SftpOpts} = handle_options(Opts, [], [], []), - Timeout = proplists:get_value(timeout, SftpOpts, infinity), +start_channel(Host, UserOptions) -> + start_channel(Host, 22, UserOptions). + +start_channel(Host, Port, UserOptions) -> + {SshOpts, ChanOpts, SftpOpts} = handle_options(UserOptions), + Timeout = % A mixture of ssh:connect and ssh_sftp:start_channel: + proplists:get_value(connect_timeout, SshOpts, + proplists:get_value(timeout, SftpOpts, infinity)), case ssh_xfer:connect(Host, Port, SshOpts, ChanOpts, Timeout) of {ok, ChannelId, Cm} -> - case ssh_channel:start(Cm, ChannelId, ?MODULE, [Cm, - ChannelId, SftpOpts]) of + case ssh_channel:start(Cm, ChannelId, ?MODULE, [Cm,ChannelId,SftpOpts]) of {ok, Pid} -> case wait_for_version_negotiation(Pid, Timeout) of ok -> @@ -865,6 +863,9 @@ terminate(_Reason, State) -> %%==================================================================== %% Internal functions %%==================================================================== +handle_options(UserOptions) -> + handle_options(UserOptions, [], [], []). + handle_options([], Sftp, Chan, Ssh) -> {Ssh, Chan, Sftp}; handle_options([{timeout, _} = Opt | Rest], Sftp, Chan, Ssh) -> diff --git a/lib/ssh/src/ssh_subsystem_sup.erl b/lib/ssh/src/ssh_subsystem_sup.erl index 637f5f398f..cf82db458f 100644 --- a/lib/ssh/src/ssh_subsystem_sup.erl +++ b/lib/ssh/src/ssh_subsystem_sup.erl @@ -26,6 +26,8 @@ -behaviour(supervisor). +-include("ssh.hrl"). + -export([start_link/1, connection_supervisor/1, channel_supervisor/1 @@ -37,8 +39,8 @@ %%%========================================================================= %%% API %%%========================================================================= -start_link(Opts) -> - supervisor:start_link(?MODULE, [Opts]). +start_link(Options) -> + supervisor:start_link(?MODULE, [Options]). connection_supervisor(SupPid) -> Children = supervisor:which_children(SupPid), @@ -53,42 +55,42 @@ channel_supervisor(SupPid) -> %%%========================================================================= -spec init( [term()] ) -> {ok,{supervisor:sup_flags(),[supervisor:child_spec()]}} | ignore . -init([Opts]) -> +init([Options]) -> RestartStrategy = one_for_all, MaxR = 0, MaxT = 3600, - Children = child_specs(Opts), + Children = child_specs(Options), {ok, {{RestartStrategy, MaxR, MaxT}, Children}}. %%%========================================================================= %%% Internal functions %%%========================================================================= -child_specs(Opts) -> - case proplists:get_value(role, Opts) of +child_specs(Options) -> + case ?GET_INTERNAL_OPT(role, Options) of client -> []; server -> - [ssh_channel_child_spec(Opts), ssh_connectinon_child_spec(Opts)] + [ssh_channel_child_spec(Options), ssh_connectinon_child_spec(Options)] end. -ssh_connectinon_child_spec(Opts) -> - Address = proplists:get_value(address, Opts), - Port = proplists:get_value(port, Opts), - Role = proplists:get_value(role, Opts), +ssh_connectinon_child_spec(Options) -> + Address = ?GET_INTERNAL_OPT(address, Options), + Port = ?GET_INTERNAL_OPT(port, Options), + Role = ?GET_INTERNAL_OPT(role, Options), Name = id(Role, ssh_connection_sup, Address, Port), - StartFunc = {ssh_connection_sup, start_link, [Opts]}, + StartFunc = {ssh_connection_sup, start_link, [Options]}, Restart = temporary, Shutdown = 5000, Modules = [ssh_connection_sup], Type = supervisor, {Name, StartFunc, Restart, Shutdown, Type, Modules}. -ssh_channel_child_spec(Opts) -> - Address = proplists:get_value(address, Opts), - Port = proplists:get_value(port, Opts), - Role = proplists:get_value(role, Opts), +ssh_channel_child_spec(Options) -> + Address = ?GET_INTERNAL_OPT(address, Options), + Port = ?GET_INTERNAL_OPT(port, Options), + Role = ?GET_INTERNAL_OPT(role, Options), Name = id(Role, ssh_channel_sup, Address, Port), - StartFunc = {ssh_channel_sup, start_link, [Opts]}, + StartFunc = {ssh_channel_sup, start_link, [Options]}, Restart = temporary, Shutdown = infinity, Modules = [ssh_channel_sup], diff --git a/lib/ssh/src/ssh_system_sup.erl b/lib/ssh/src/ssh_system_sup.erl index e97ac7b01a..b0bbd3aae5 100644 --- a/lib/ssh/src/ssh_system_sup.erl +++ b/lib/ssh/src/ssh_system_sup.erl @@ -45,12 +45,12 @@ %%%========================================================================= %%% Internal API %%%========================================================================= -start_link(ServerOpts) -> - Address = proplists:get_value(address, ServerOpts), - Port = proplists:get_value(port, ServerOpts), - Profile = proplists:get_value(profile, proplists:get_value(ssh_opts, ServerOpts), ?DEFAULT_PROFILE), +start_link(Options) -> + Address = ?GET_INTERNAL_OPT(address, Options), + Port = ?GET_INTERNAL_OPT(port, Options), + Profile = ?GET_OPT(profile, Options), Name = make_name(Address, Port, Profile), - supervisor:start_link({local, Name}, ?MODULE, [ServerOpts]). + supervisor:start_link({local, Name}, ?MODULE, [Options]). stop_listener(SysSup) -> stop_acceptor(SysSup). @@ -127,12 +127,12 @@ restart_acceptor(Address, Port, Profile) -> %%%========================================================================= -spec init( [term()] ) -> {ok,{supervisor:sup_flags(),[supervisor:child_spec()]}} | ignore . -init([ServerOpts]) -> +init([Options]) -> RestartStrategy = one_for_one, MaxR = 0, MaxT = 3600, - Children = case proplists:get_value(asocket,ServerOpts) of - undefined -> child_specs(ServerOpts); + Children = case ?GET_INTERNAL_OPT(asocket,Options,undefined) of + undefined -> child_specs(Options); _ -> [] end, {ok, {{RestartStrategy, MaxR, MaxT}, Children}}. @@ -140,24 +140,24 @@ init([ServerOpts]) -> %%%========================================================================= %%% Internal functions %%%========================================================================= -child_specs(ServerOpts) -> - [ssh_acceptor_child_spec(ServerOpts)]. +child_specs(Options) -> + [ssh_acceptor_child_spec(Options)]. -ssh_acceptor_child_spec(ServerOpts) -> - Address = proplists:get_value(address, ServerOpts), - Port = proplists:get_value(port, ServerOpts), - Profile = proplists:get_value(profile, proplists:get_value(ssh_opts, ServerOpts), ?DEFAULT_PROFILE), +ssh_acceptor_child_spec(Options) -> + Address = ?GET_INTERNAL_OPT(address, Options), + Port = ?GET_INTERNAL_OPT(port, Options), + Profile = ?GET_OPT(profile, Options), Name = id(ssh_acceptor_sup, Address, Port, Profile), - StartFunc = {ssh_acceptor_sup, start_link, [ServerOpts]}, + StartFunc = {ssh_acceptor_sup, start_link, [Options]}, Restart = transient, Shutdown = infinity, Modules = [ssh_acceptor_sup], Type = supervisor, {Name, StartFunc, Restart, Shutdown, Type, Modules}. -ssh_subsystem_child_spec(ServerOpts) -> +ssh_subsystem_child_spec(Options) -> Name = make_ref(), - StartFunc = {ssh_subsystem_sup, start_link, [ServerOpts]}, + StartFunc = {ssh_subsystem_sup, start_link, [Options]}, Restart = temporary, Shutdown = infinity, Modules = [ssh_subsystem_sup], diff --git a/lib/ssh/src/ssh_transport.erl b/lib/ssh/src/ssh_transport.erl index a17ad560d1..02c995399a 100644 --- a/lib/ssh/src/ssh_transport.erl +++ b/lib/ssh/src/ssh_transport.erl @@ -153,14 +153,14 @@ supported_algorithms(compression) -> %%%---------------------------------------------------------------------------- versions(client, Options)-> - Vsn = proplists:get_value(vsn, Options, ?DEFAULT_CLIENT_VERSION), + Vsn = ?GET_INTERNAL_OPT(vsn, Options, ?DEFAULT_CLIENT_VERSION), {Vsn, format_version(Vsn, software_version(Options))}; versions(server, Options) -> - Vsn = proplists:get_value(vsn, Options, ?DEFAULT_SERVER_VERSION), + Vsn = ?GET_INTERNAL_OPT(vsn, Options, ?DEFAULT_SERVER_VERSION), {Vsn, format_version(Vsn, software_version(Options))}. software_version(Options) -> - case proplists:get_value(id_string, Options) of + case ?GET_OPT(id_string, Options) of undefined -> "Erlang"++ssh_vsn(); {random,Nlo,Nup} -> @@ -171,7 +171,7 @@ software_version(Options) -> ssh_vsn() -> try {ok,L} = application:get_all_key(ssh), - proplists:get_value(vsn,L,"") + proplists:get_value(vsn, L, "") of "" -> ""; VSN when is_list(VSN) -> "/" ++ VSN; @@ -232,13 +232,7 @@ key_exchange_init_msg(Ssh0) -> kex_init(#ssh{role = Role, opts = Opts, available_host_keys = HostKeyAlgs}) -> Random = ssh_bits:random(16), - PrefAlgs = - case proplists:get_value(preferred_algorithms,Opts) of - undefined -> - default_algorithms(); - Algs0 -> - Algs0 - end, + PrefAlgs = ?GET_OPT(preferred_algorithms, Opts), kexinit_message(Role, Random, PrefAlgs, HostKeyAlgs). key_init(client, Ssh, Value) -> @@ -341,10 +335,7 @@ key_exchange_first_msg(Kex, Ssh0) when Kex == 'diffie-hellman-group1-sha1' ; key_exchange_first_msg(Kex, Ssh0=#ssh{opts=Opts}) when Kex == 'diffie-hellman-group-exchange-sha1' ; Kex == 'diffie-hellman-group-exchange-sha256' -> - {Min,NBits0,Max} = - proplists:get_value(dh_gex_limits, Opts, {?DEFAULT_DH_GROUP_MIN, - ?DEFAULT_DH_GROUP_NBITS, - ?DEFAULT_DH_GROUP_MAX}), + {Min,NBits0,Max} = ?GET_OPT(dh_gex_limits, Opts), DhBits = dh_bits(Ssh0#ssh.algorithms), NBits1 = %% NIST Special Publication 800-57 Part 1 Revision 4: Recommendation for Key Management @@ -458,7 +449,7 @@ handle_kex_dh_gex_request(#ssh_msg_kex_dh_gex_request{min = Min0, %% server {Min, Max} = adjust_gex_min_max(Min0, Max0, Opts), case public_key:dh_gex_group(Min, NBits, Max, - proplists:get_value(dh_gex_groups,Opts)) of + ?GET_OPT(dh_gex_groups,Opts)) of {ok, {_, {G,P}}} -> {SshPacket, Ssh} = ssh_packet(#ssh_msg_kex_dh_gex_group{p = P, g = G}, Ssh0), @@ -492,7 +483,7 @@ handle_kex_dh_gex_request(#ssh_msg_kex_dh_gex_request_old{n = NBits}, Max0 = 8192, {Min, Max} = adjust_gex_min_max(Min0, Max0, Opts), case public_key:dh_gex_group(Min, NBits, Max, - proplists:get_value(dh_gex_groups,Opts)) of + ?GET_OPT(dh_gex_groups,Opts)) of {ok, {_, {G,P}}} -> {SshPacket, Ssh} = ssh_packet(#ssh_msg_kex_dh_gex_group{p = P, g = G}, Ssh0), @@ -517,22 +508,18 @@ handle_kex_dh_gex_request(_, _) -> adjust_gex_min_max(Min0, Max0, Opts) -> - case proplists:get_value(dh_gex_limits, Opts) of - undefined -> - {Min0, Max0}; - {Min1, Max1} -> - Min2 = max(Min0, Min1), - Max2 = min(Max0, Max1), - if - Min2 =< Max2 -> - {Min2, Max2}; - Max2 < Min2 -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{ - code = ?SSH_DISCONNECT_PROTOCOL_ERROR, - description = "No possible diffie-hellman-group-exchange group possible" - }) - end + {Min1, Max1} = ?GET_OPT(dh_gex_limits, Opts), + Min2 = max(Min0, Min1), + Max2 = min(Max0, Max1), + if + Min2 =< Max2 -> + {Min2, Max2}; + Max2 < Min2 -> + ssh_connection_handler:disconnect( + #ssh_msg_disconnect{ + code = ?SSH_DISCONNECT_PROTOCOL_ERROR, + description = "No possible diffie-hellman-group-exchange group possible" + }) end. @@ -719,9 +706,9 @@ sid(#ssh{session_id = Id}, _) -> %% The host key should be read from storage %% get_host_key(SSH) -> - #ssh{key_cb = Mod, opts = Opts, algorithms = ALG} = SSH, - - case Mod:host_key(ALG#alg.hkey, Opts) of + #ssh{key_cb = {KeyCb,KeyCbOpts}, opts = Opts, algorithms = ALG} = SSH, + UserOpts = ?GET_OPT(user_options, Opts), + case KeyCb:host_key(ALG#alg.hkey, [{key_cb_private,KeyCbOpts}|UserOpts]) of {ok, #'RSAPrivateKey'{} = Key} -> Key; {ok, #'DSAPrivateKey'{} = Key} -> Key; {ok, #'ECPrivateKey'{} = Key} -> Key; @@ -767,7 +754,7 @@ public_algo({#'ECPoint'{},{namedCurve,OID}}) -> accepted_host(Ssh, PeerName, Public, Opts) -> - case proplists:get_value(silently_accept_hosts, Opts, false) of + case ?GET_OPT(silently_accept_hosts, Opts) of F when is_function(F,2) -> true == (catch F(PeerName, public_key:ssh_hostkey_fingerprint(Public))); {DigestAlg,F} when is_function(F,2) -> @@ -778,15 +765,16 @@ accepted_host(Ssh, PeerName, Public, Opts) -> yes == yes_no(Ssh, "New host " ++ PeerName ++ " accept") end. -known_host_key(#ssh{opts = Opts, key_cb = Mod, peer = {PeerName,_}} = Ssh, +known_host_key(#ssh{opts = Opts, key_cb = {KeyCb,KeyCbOpts}, peer = {PeerName,_}} = Ssh, Public, Alg) -> - case Mod:is_host_key(Public, PeerName, Alg, Opts) of + UserOpts = ?GET_OPT(user_options, Opts), + case KeyCb:is_host_key(Public, PeerName, Alg, [{key_cb_private,KeyCbOpts}|UserOpts]) of true -> ok; false -> case accepted_host(Ssh, PeerName, Public, Opts) of true -> - Mod:add_host_key(PeerName, Public, Opts); + KeyCb:add_host_key(PeerName, Public, [{key_cb_private,KeyCbOpts}|UserOpts]); false -> {error, rejected} end @@ -1822,10 +1810,6 @@ len_supported(Name, Len) -> same(Algs) -> [{client2server,Algs}, {server2client,Algs}]. - -%% default_algorithms(kex) -> % Example of how to disable an algorithm -%% supported_algorithms(kex, ['ecdh-sha2-nistp521']); - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %% Other utils diff --git a/lib/ssh/src/sshd_sup.erl b/lib/ssh/src/sshd_sup.erl index 04d2df30f7..14f1937abd 100644 --- a/lib/ssh/src/sshd_sup.erl +++ b/lib/ssh/src/sshd_sup.erl @@ -41,13 +41,13 @@ start_link(Servers) -> supervisor:start_link({local, ?MODULE}, ?MODULE, [Servers]). -start_child(ServerOpts) -> - Address = proplists:get_value(address, ServerOpts), - Port = proplists:get_value(port, ServerOpts), - Profile = proplists:get_value(profile, proplists:get_value(ssh_opts, ServerOpts), ?DEFAULT_PROFILE), +start_child(Options) -> + Address = ?GET_INTERNAL_OPT(address, Options), + Port = ?GET_INTERNAL_OPT(port, Options), + Profile = ?GET_OPT(profile, Options), case ssh_system_sup:system_supervisor(Address, Port, Profile) of undefined -> - Spec = child_spec(Address, Port, ServerOpts), + Spec = child_spec(Address, Port, Options), case supervisor:start_child(?MODULE, Spec) of {error, already_present} -> Name = id(Address, Port, Profile), @@ -58,7 +58,7 @@ start_child(ServerOpts) -> end; Pid -> AccPid = ssh_system_sup:acceptor_supervisor(Pid), - ssh_acceptor_sup:start_child(AccPid, ServerOpts) + ssh_acceptor_sup:start_child(AccPid, Options) end. stop_child(Name) -> @@ -82,8 +82,8 @@ init([Servers]) -> MaxR = 10, MaxT = 3600, Fun = fun(ServerOpts) -> - Address = proplists:get_value(address, ServerOpts), - Port = proplists:get_value(port, ServerOpts), + Address = ?GET_INTERNAL_OPT(address, ServerOpts), + Port = ?GET_INTERNAL_OPT(port, ServerOpts), child_spec(Address, Port, ServerOpts) end, Children = lists:map(Fun, Servers), @@ -92,10 +92,10 @@ init([Servers]) -> %%%========================================================================= %%% Internal functions %%%========================================================================= -child_spec(Address, Port, ServerOpts) -> - Profile = proplists:get_value(profile, proplists:get_value(ssh_opts, ServerOpts), ?DEFAULT_PROFILE), +child_spec(Address, Port, Options) -> + Profile = ?GET_OPT(profile, Options), Name = id(Address, Port,Profile), - StartFunc = {ssh_system_sup, start_link, [ServerOpts]}, + StartFunc = {ssh_system_sup, start_link, [Options]}, Restart = temporary, Shutdown = infinity, Modules = [ssh_system_sup], diff --git a/lib/ssh/test/Makefile b/lib/ssh/test/Makefile index 3fca78237c..fab79a7a43 100644 --- a/lib/ssh/test/Makefile +++ b/lib/ssh/test/Makefile @@ -36,7 +36,7 @@ MODULES= \ ssh_options_SUITE \ ssh_renegotiate_SUITE \ ssh_basic_SUITE \ - ssh_benchmark_SUITE \ + ssh_bench_SUITE \ ssh_connection_SUITE \ ssh_protocol_SUITE \ ssh_sftp_SUITE \ @@ -50,6 +50,7 @@ MODULES= \ ssh_key_cb_options \ ssh_trpt_test_lib \ ssh_echo_server \ + ssh_bench_dev_null \ ssh_peername_sockname_server \ ssh_test_cli \ ssh_relay \ diff --git a/lib/ssh/test/ssh.spec b/lib/ssh/test/ssh.spec index 0076fc275e..68268cb20d 100644 --- a/lib/ssh/test/ssh.spec +++ b/lib/ssh/test/ssh.spec @@ -1,6 +1,7 @@ {suites,"../ssh_test",all}. -{skip_suites, "../ssh_test", [ssh_benchmark_SUITE], +{skip_suites, "../ssh_test", [ssh_bench_SUITE + ], "Benchmarks run separately"}. diff --git a/lib/ssh/test/ssh_algorithms_SUITE.erl b/lib/ssh/test/ssh_algorithms_SUITE.erl index 313b7fc559..6f75d83c4a 100644 --- a/lib/ssh/test/ssh_algorithms_SUITE.erl +++ b/lib/ssh/test/ssh_algorithms_SUITE.erl @@ -200,6 +200,9 @@ try_exec_simple_group(Group, Config) -> %%-------------------------------------------------------------------- %% Testing all default groups +simple_exec_groups() -> + [{timetrap,{seconds,120}}]. + simple_exec_groups(Config) -> Sizes = interpolate( public_key:dh_gex_group_sizes() ), lists:foreach( diff --git a/lib/ssh/test/ssh_basic_SUITE.erl b/lib/ssh/test/ssh_basic_SUITE.erl index cdf6cf9ae1..a9b6be222e 100644 --- a/lib/ssh/test/ssh_basic_SUITE.erl +++ b/lib/ssh/test/ssh_basic_SUITE.erl @@ -46,7 +46,8 @@ exec_key_differs2/1, exec_key_differs3/1, exec_key_differs_fail/1, - idle_time/1, + idle_time_client/1, + idle_time_server/1, inet6_option/1, inet_option/1, internal_error/1, @@ -139,7 +140,7 @@ basic_tests() -> exec, exec_compressed, shell, shell_no_unicode, shell_unicode_string, cli, known_hosts, - idle_time, openssh_zlib_basic_test, + idle_time_client, idle_time_server, openssh_zlib_basic_test, misc_ssh_options, inet_option, inet6_option]. @@ -522,8 +523,8 @@ exec_compressed(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- -%%% Idle timeout test -idle_time(Config) -> +%%% Idle timeout test, client +idle_time_client(Config) -> SystemDir = filename:join(proplists:get_value(priv_dir, Config), system), UserDir = proplists:get_value(priv_dir, Config), @@ -544,6 +545,28 @@ idle_time(Config) -> ssh:stop_daemon(Pid). %%-------------------------------------------------------------------- +%%% Idle timeout test, server +idle_time_server(Config) -> + SystemDir = filename:join(proplists:get_value(priv_dir, Config), system), + UserDir = proplists:get_value(priv_dir, Config), + + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, + {user_dir, UserDir}, + {idle_time, 2000}, + {failfun, fun ssh_test_lib:failfun/2}]), + ConnectionRef = + ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user_dir, UserDir}, + {user_interaction, false}]), + {ok, Id} = ssh_connection:session_channel(ConnectionRef, 1000), + ssh_connection:close(ConnectionRef, Id), + receive + after 10000 -> + {error, closed} = ssh_connection:session_channel(ConnectionRef, 1000) + end, + ssh:stop_daemon(Pid). + +%%-------------------------------------------------------------------- %%% Test that ssh:shell/2 works shell(Config) when is_list(Config) -> process_flag(trap_exit, true), diff --git a/lib/ssh/test/ssh_bench.spec b/lib/ssh/test/ssh_bench.spec index 029f0bd074..b0b64713cf 100644 --- a/lib/ssh/test/ssh_bench.spec +++ b/lib/ssh/test/ssh_bench.spec @@ -1 +1,2 @@ -{suites,"../ssh_test",[ssh_benchmark_SUITE]}. +{suites,"../ssh_test",[ssh_bench_SUITE + ]}. diff --git a/lib/ssh/test/ssh_bench_SUITE.erl b/lib/ssh/test/ssh_bench_SUITE.erl new file mode 100644 index 0000000000..ac52bb7e28 --- /dev/null +++ b/lib/ssh/test/ssh_bench_SUITE.erl @@ -0,0 +1,252 @@ +%%%------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2015-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(ssh_bench_SUITE). +-compile(export_all). + +-include_lib("common_test/include/ct_event.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-include_lib("ssh/src/ssh.hrl"). +-include_lib("ssh/src/ssh_transport.hrl"). +-include_lib("ssh/src/ssh_connect.hrl"). +-include_lib("ssh/src/ssh_userauth.hrl"). + +%%%================================================================ +%%% +%%% Suite declarations +%%% + +suite() -> [{ct_hooks,[{ts_install_cth,[{nodenames,2}]}]}, + {timetrap,{minutes,1}} + ]. +all() -> [connect, + transfer_text + ]. + +-define(UID, "foo"). +-define(PWD, "bar"). +-define(Nruns, 8). + +%%%================================================================ +%%% +%%% Init per suite +%%% + +init_per_suite(Config) -> + catch ssh:stop(), + try + ok = ssh:start() + of + ok -> + DataSize = 1000000, + SystemDir = proplists:get_value(data_dir, Config), + Algs = insert_none(ssh:default_algorithms()), + {_ServerPid, _Host, Port} = + ssh_test_lib:daemon([{system_dir, SystemDir}, + {user_passwords, [{?UID,?PWD}]}, + {failfun, fun ssh_test_lib:failfun/2}, + {preferred_algorithms, Algs}, + {max_random_length_padding, 0}, + {subsystems, [{"/dev/null", {ssh_bench_dev_null,[DataSize]}}]} + ]), + [{host,"localhost"}, {port,Port}, {uid,?UID}, {pwd,?PWD}, {data_size,DataSize} | Config] + catch + C:E -> + {skip, io_lib:format("Couldn't start ~p:~p",[C,E])} + end. + +end_per_suite(_Config) -> + catch ssh:stop(), + ok. + +%%%================================================================ +%%% +%%% Init per testcase +%%% + +init_per_testcase(_Func, Conf) -> + Conf. + +end_per_testcase(_Func, _Conf) -> + ok. + +%%%================================================================ +%%% +%%% Testcases +%%% + +%%%---------------------------------------------------------------- +%%% Measure the time for an Erlang client to connect to an Erlang +%%% server on the localhost + +connect(Config) -> + KexAlgs = proplists:get_value(kex, ssh:default_algorithms()), + ct:pal("KexAlgs = ~p",[KexAlgs]), + lists:foreach( + fun(KexAlg) -> + PrefAlgs = preferred_algorithms(KexAlg), + report([{value, measure_connect(Config, + [{preferred_algorithms,PrefAlgs}])}, + {suite, ?MODULE}, + {name, mk_name(["Connect erlc erld ",KexAlg," [µs]"])} + ]) + end, KexAlgs). + + +measure_connect(Config, Opts) -> + Port = proplists:get_value(port, Config), + ConnectOptions = [{user, proplists:get_value(uid, Config)}, + {password, proplists:get_value(pwd, Config)}, + {user_dir, proplists:get_value(priv_dir, Config)}, + {silently_accept_hosts, true}, + {user_interaction, false}, + {max_random_length_padding, 0} + ] ++ Opts, + median( + [begin + {Time, {ok,Pid}} = timer:tc(ssh,connect,["localhost", Port, ConnectOptions]), + ssh:close(Pid), + Time + end || _ <- lists:seq(1,?Nruns)]). + +%%%---------------------------------------------------------------- +%%% Measure the time to transfer a set of data with +%%% and without crypto + +transfer_text(Config) -> + Port = proplists:get_value(port, Config), + Options = [{user, proplists:get_value(uid, Config)}, + {password, proplists:get_value(pwd, Config)}, + {user_dir, proplists:get_value(priv_dir, Config)}, + {silently_accept_hosts, true}, + {user_interaction, false}, + {max_random_length_padding, 0} + ], + Data = gen_data(proplists:get_value(data_size,Config)), + + [connect_measure(Port, Crypto, Mac, Data, Options) + || {Crypto,Mac} <- [{ none, none}, + {'aes128-ctr', 'hmac-sha1'}, + {'aes256-ctr', 'hmac-sha1'}, +%% {'[email protected]', 'hmac-sha1'}, + {'aes128-cbc', 'hmac-sha1'}, + {'3des-cbc', 'hmac-sha1'}, + {'aes128-ctr', 'hmac-sha2-256'}, + {'aes128-ctr', 'hmac-sha2-512'} + ], + crypto_mac_supported(Crypto,Mac)]. + + +crypto_mac_supported(none, none) -> + true; +crypto_mac_supported(C, M) -> + Algs = ssh:default_algorithms(), + [{_,Cs},_] = proplists:get_value(cipher, Algs), + [{_,Ms},_] = proplists:get_value(mac, Algs), + lists:member(C,Cs) andalso lists:member(M,Ms). + + +gen_data(DataSz) -> + Data0 = << <<C>> || _ <- lists:seq(1,DataSz div 256), + C <- lists:seq(0,255) >>, + Data1 = << <<C>> || C <- lists:seq(0,(DataSz rem 256) - 1) >>, + <<Data0/binary, Data1/binary>>. + + +%% connect_measure(Port, Cipher, Mac, Data, Options) -> +%% report([{value, 1}, +%% {suite, ?MODULE}, +%% {name, mk_name(["Transfer 1M bytes ",Cipher,"/",Mac," [µs]"])}]); +connect_measure(Port, Cipher, Mac, Data, Options) -> + Times = + [begin + {ok,C} = ssh:connect("localhost", Port, [{preferred_algorithms, [{cipher,[Cipher]}, + {mac,[Mac]}]} + |Options]), + {ok,Ch} = ssh_connection:session_channel(C, 10000), + success = ssh_connection:subsystem(C, Ch, "/dev/null", 10000), + {Time,ok} = timer:tc(?MODULE, send_wait_acc, [C, Ch, Data]), + ok = ssh_connection:send_eof(C, Ch), + ssh:close(C), + Time + end || _ <- lists:seq(1,?Nruns)], + + report([{value, median(Times)}, + {suite, ?MODULE}, + {name, mk_name(["Transfer 1M bytes ",Cipher,"/",Mac," [µs]"])}]). + +send_wait_acc(C, Ch, Data) -> + ssh_connection:send(C, Ch, Data), + receive + {ssh_cm, C, {data, Ch, 0, <<"READY">>}} -> ok + end. + + +%%%================================================================ +%%% +%%% Private +%%% + +%%%---------------------------------------------------------------- +insert_none(L) -> + lists:foldl(fun insert_none/2, [], L). + +insert_none({T,L}, Acc) when T==cipher ; + T==mac -> + [{T, [{T1,L1++[none]} || {T1,L1} <- L]} | Acc]; +insert_none(_, Acc) -> + Acc. + +%%%---------------------------------------------------------------- +mk_name(Name) -> [char(C) || C <- lists:concat(Name)]. + +char($-) -> $_; +char(C) -> C. + +%%%---------------------------------------------------------------- +preferred_algorithms(KexAlg) -> + [{kex, [KexAlg]}, + {public_key, ['ssh-rsa']}, + {cipher, ['aes128-ctr']}, + {mac, ['hmac-sha1']}, + {compression, [none]} + ]. + +%%%---------------------------------------------------------------- +median(Data) when is_list(Data) -> + SortedData = lists:sort(Data), + N = length(Data), + Median = + case N rem 2 of + 0 -> + MeanOfMiddle = (lists:nth(N div 2, SortedData) + + lists:nth(N div 2 + 1, SortedData)) / 2, + round(MeanOfMiddle); + 1 -> + lists:nth(N div 2 + 1, SortedData) + end, + ct:pal("median(~p) = ~p",[SortedData,Median]), + Median. + + +report(Data) -> + ct:pal("EventData = ~p",[Data]), + ct_event:notify(#event{name = benchmark_data, + data = Data}). diff --git a/lib/ssh/test/ssh_benchmark_SUITE_data/id_dsa b/lib/ssh/test/ssh_bench_SUITE_data/id_dsa index d306f8b26e..d306f8b26e 100644 --- a/lib/ssh/test/ssh_benchmark_SUITE_data/id_dsa +++ b/lib/ssh/test/ssh_bench_SUITE_data/id_dsa diff --git a/lib/ssh/test/ssh_benchmark_SUITE_data/id_ecdsa256 b/lib/ssh/test/ssh_bench_SUITE_data/id_ecdsa256 index 4b1eb12eaa..4b1eb12eaa 100644 --- a/lib/ssh/test/ssh_benchmark_SUITE_data/id_ecdsa256 +++ b/lib/ssh/test/ssh_bench_SUITE_data/id_ecdsa256 diff --git a/lib/ssh/test/ssh_benchmark_SUITE_data/id_ecdsa256.pub b/lib/ssh/test/ssh_bench_SUITE_data/id_ecdsa256.pub index a0147e60fa..a0147e60fa 100644 --- a/lib/ssh/test/ssh_benchmark_SUITE_data/id_ecdsa256.pub +++ b/lib/ssh/test/ssh_bench_SUITE_data/id_ecdsa256.pub diff --git a/lib/ssh/test/ssh_benchmark_SUITE_data/id_ecdsa384 b/lib/ssh/test/ssh_bench_SUITE_data/id_ecdsa384 index 4e8aa40959..4e8aa40959 100644 --- a/lib/ssh/test/ssh_benchmark_SUITE_data/id_ecdsa384 +++ b/lib/ssh/test/ssh_bench_SUITE_data/id_ecdsa384 diff --git a/lib/ssh/test/ssh_benchmark_SUITE_data/id_ecdsa384.pub b/lib/ssh/test/ssh_bench_SUITE_data/id_ecdsa384.pub index 41e722e545..41e722e545 100644 --- a/lib/ssh/test/ssh_benchmark_SUITE_data/id_ecdsa384.pub +++ b/lib/ssh/test/ssh_bench_SUITE_data/id_ecdsa384.pub diff --git a/lib/ssh/test/ssh_benchmark_SUITE_data/id_ecdsa521 b/lib/ssh/test/ssh_bench_SUITE_data/id_ecdsa521 index 7196f46e97..7196f46e97 100644 --- a/lib/ssh/test/ssh_benchmark_SUITE_data/id_ecdsa521 +++ b/lib/ssh/test/ssh_bench_SUITE_data/id_ecdsa521 diff --git a/lib/ssh/test/ssh_benchmark_SUITE_data/id_ecdsa521.pub b/lib/ssh/test/ssh_bench_SUITE_data/id_ecdsa521.pub index 8f059120bc..8f059120bc 100644 --- a/lib/ssh/test/ssh_benchmark_SUITE_data/id_ecdsa521.pub +++ b/lib/ssh/test/ssh_bench_SUITE_data/id_ecdsa521.pub diff --git a/lib/ssh/test/ssh_benchmark_SUITE_data/id_rsa b/lib/ssh/test/ssh_bench_SUITE_data/id_rsa index 9d7e0dd5fb..9d7e0dd5fb 100644 --- a/lib/ssh/test/ssh_benchmark_SUITE_data/id_rsa +++ b/lib/ssh/test/ssh_bench_SUITE_data/id_rsa diff --git a/lib/ssh/test/ssh_benchmark_SUITE_data/ssh_host_dsa_key b/lib/ssh/test/ssh_bench_SUITE_data/ssh_host_dsa_key index 51ab6fbd88..51ab6fbd88 100644 --- a/lib/ssh/test/ssh_benchmark_SUITE_data/ssh_host_dsa_key +++ b/lib/ssh/test/ssh_bench_SUITE_data/ssh_host_dsa_key diff --git a/lib/ssh/test/ssh_benchmark_SUITE_data/ssh_host_dsa_key.pub b/lib/ssh/test/ssh_bench_SUITE_data/ssh_host_dsa_key.pub index 4dbb1305b0..4dbb1305b0 100644 --- a/lib/ssh/test/ssh_benchmark_SUITE_data/ssh_host_dsa_key.pub +++ b/lib/ssh/test/ssh_bench_SUITE_data/ssh_host_dsa_key.pub diff --git a/lib/ssh/test/ssh_benchmark_SUITE_data/ssh_host_ecdsa_key256 b/lib/ssh/test/ssh_bench_SUITE_data/ssh_host_ecdsa_key256 index 2979ea88ed..2979ea88ed 100644 --- a/lib/ssh/test/ssh_benchmark_SUITE_data/ssh_host_ecdsa_key256 +++ b/lib/ssh/test/ssh_bench_SUITE_data/ssh_host_ecdsa_key256 diff --git a/lib/ssh/test/ssh_benchmark_SUITE_data/ssh_host_ecdsa_key256.pub b/lib/ssh/test/ssh_bench_SUITE_data/ssh_host_ecdsa_key256.pub index 85dc419345..85dc419345 100644 --- a/lib/ssh/test/ssh_benchmark_SUITE_data/ssh_host_ecdsa_key256.pub +++ b/lib/ssh/test/ssh_bench_SUITE_data/ssh_host_ecdsa_key256.pub diff --git a/lib/ssh/test/ssh_benchmark_SUITE_data/ssh_host_ecdsa_key384 b/lib/ssh/test/ssh_bench_SUITE_data/ssh_host_ecdsa_key384 index fb1a862ded..fb1a862ded 100644 --- a/lib/ssh/test/ssh_benchmark_SUITE_data/ssh_host_ecdsa_key384 +++ b/lib/ssh/test/ssh_bench_SUITE_data/ssh_host_ecdsa_key384 diff --git a/lib/ssh/test/ssh_benchmark_SUITE_data/ssh_host_ecdsa_key384.pub b/lib/ssh/test/ssh_bench_SUITE_data/ssh_host_ecdsa_key384.pub index 428d5fb7d7..428d5fb7d7 100644 --- a/lib/ssh/test/ssh_benchmark_SUITE_data/ssh_host_ecdsa_key384.pub +++ b/lib/ssh/test/ssh_bench_SUITE_data/ssh_host_ecdsa_key384.pub diff --git a/lib/ssh/test/ssh_benchmark_SUITE_data/ssh_host_ecdsa_key521 b/lib/ssh/test/ssh_bench_SUITE_data/ssh_host_ecdsa_key521 index 3e51ec2ecd..3e51ec2ecd 100644 --- a/lib/ssh/test/ssh_benchmark_SUITE_data/ssh_host_ecdsa_key521 +++ b/lib/ssh/test/ssh_bench_SUITE_data/ssh_host_ecdsa_key521 diff --git a/lib/ssh/test/ssh_benchmark_SUITE_data/ssh_host_ecdsa_key521.pub b/lib/ssh/test/ssh_bench_SUITE_data/ssh_host_ecdsa_key521.pub index 017a29f4da..017a29f4da 100644 --- a/lib/ssh/test/ssh_benchmark_SUITE_data/ssh_host_ecdsa_key521.pub +++ b/lib/ssh/test/ssh_bench_SUITE_data/ssh_host_ecdsa_key521.pub diff --git a/lib/ssh/test/ssh_benchmark_SUITE_data/ssh_host_rsa_key b/lib/ssh/test/ssh_bench_SUITE_data/ssh_host_rsa_key index 79968bdd7d..79968bdd7d 100644 --- a/lib/ssh/test/ssh_benchmark_SUITE_data/ssh_host_rsa_key +++ b/lib/ssh/test/ssh_bench_SUITE_data/ssh_host_rsa_key diff --git a/lib/ssh/test/ssh_benchmark_SUITE_data/ssh_host_rsa_key.pub b/lib/ssh/test/ssh_bench_SUITE_data/ssh_host_rsa_key.pub index 75d2025c71..75d2025c71 100644 --- a/lib/ssh/test/ssh_benchmark_SUITE_data/ssh_host_rsa_key.pub +++ b/lib/ssh/test/ssh_bench_SUITE_data/ssh_host_rsa_key.pub diff --git a/lib/ssh/test/ssh_bench_dev_null.erl b/lib/ssh/test/ssh_bench_dev_null.erl new file mode 100644 index 0000000000..0e390b7712 --- /dev/null +++ b/lib/ssh/test/ssh_bench_dev_null.erl @@ -0,0 +1,58 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2005-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% + +%%% Description: Example ssh server +-module(ssh_bench_dev_null). +-behaviour(ssh_daemon_channel). + +-record(state, { + cm, + chid, + n, + sum = 0 + }). + +-export([init/1, handle_msg/2, handle_ssh_msg/2, terminate/2]). + +init([N]) -> {ok, #state{n=N}}. + +handle_msg({ssh_channel_up, ChId, CM}, S) -> + {ok, S#state{cm = CM, + chid = ChId}}. + + + +handle_ssh_msg({ssh_cm, CM, {data,ChId,0,Data}}, #state{n=N, sum=Sum0, cm=CM, chid=ChId} = S) -> + Sum = Sum0 + size(Data), + if Sum == N -> + %% Got all + ssh_connection:send(CM, ChId, <<"READY">>), + {ok, S#state{sum=Sum}}; + Sum < N -> + %% Expects more + {ok, S#state{sum=Sum}} + end; +handle_ssh_msg({ssh_cm, _, {exit_signal,ChId,_,_,_}}, S) -> {stop, ChId, S}; +handle_ssh_msg({ssh_cm, _, {exit_status,ChId,_} }, S) -> {stop, ChId, S}; +handle_ssh_msg({ssh_cm, _, _ }, S) -> {ok, S}. + +terminate(_, _) -> ok. diff --git a/lib/ssh/test/ssh_benchmark_SUITE.erl b/lib/ssh/test/ssh_benchmark_SUITE.erl deleted file mode 100644 index 85750f8fbd..0000000000 --- a/lib/ssh/test/ssh_benchmark_SUITE.erl +++ /dev/null @@ -1,573 +0,0 @@ -%%%------------------------------------------------------------------- -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2015-2016. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% --module(ssh_benchmark_SUITE). --compile(export_all). - --include_lib("common_test/include/ct_event.hrl"). --include_lib("common_test/include/ct.hrl"). - --include_lib("ssh/src/ssh.hrl"). --include_lib("ssh/src/ssh_transport.hrl"). --include_lib("ssh/src/ssh_connect.hrl"). --include_lib("ssh/src/ssh_userauth.hrl"). - - -suite() -> [{ct_hooks,[{ts_install_cth,[{nodenames,2}]}]}, - {timetrap,{minutes,6}} - ]. -%%suite() -> [{ct_hooks,[ts_install_cth]}]. - -all() -> [{group, opensshc_erld} -%% {group, erlc_opensshd} - ]. - -groups() -> - [{opensshc_erld, [{repeat, 3}], [openssh_client_shell, - openssh_client_sftp]} - ]. - - -init_per_suite(Config) -> - catch ssh:stop(), - try - report_client_algorithms(), - ok = ssh:start(), - {ok,TracerPid} = erlang_trace(), - [{tracer_pid,TracerPid} | init_sftp_dirs(Config)] - catch - C:E -> - {skip, io_lib:format("Couldn't start ~p:~p",[C,E])} - end. - -end_per_suite(_Config) -> - catch ssh:stop(), - ok. - - - -init_per_group(opensshc_erld, Config) -> - case ssh_test_lib:ssh_type() of - openSSH -> - DataDir = proplists:get_value(data_dir, Config), - UserDir = proplists:get_value(priv_dir, Config), - ssh_test_lib:setup_dsa(DataDir, UserDir), - ssh_test_lib:setup_rsa(DataDir, UserDir), - ssh_test_lib:setup_ecdsa("256", DataDir, UserDir), - AlgsD = ssh:default_algorithms(), - AlgsC = ssh_test_lib:default_algorithms(sshc), - Common = ssh_test_lib:intersect_bi_dir( - ssh_test_lib:intersection(AlgsD, AlgsC)), - ct:pal("~p~n~nErld:~n~p~n~nOpenSSHc:~n~p~n~nCommon:~n~p", - [inet:gethostname(), AlgsD, AlgsC, Common]), - [{c_kexs, ssh_test_lib:sshc(kex)}, - {c_ciphers, ssh_test_lib:sshc(cipher)}, - {common_algs, Common} - | Config]; - _ -> - {skip, "No OpenSsh client found"} - end; - -init_per_group(erlc_opensshd, _) -> - {skip, "Group erlc_opensshd not implemented"}; - -init_per_group(_GroupName, Config) -> - Config. - -end_per_group(_GroupName, _Config) -> - ok. - - -init_per_testcase(_Func, Conf) -> - Conf. - -end_per_testcase(_Func, _Conf) -> - ok. - - -init_sftp_dirs(Config) -> - UserDir = proplists:get_value(priv_dir, Config), - SrcDir = filename:join(UserDir, "sftp_src"), - ok = file:make_dir(SrcDir), - SrcFile = "big_data", - DstDir = filename:join(UserDir, "sftp_dst"), - ok = file:make_dir(DstDir), - N = 100 * 1024*1024, - ok = file:write_file(filename:join(SrcDir,SrcFile), crypto:strong_rand_bytes(N)), - [{sftp_src_dir,SrcDir}, {sftp_dst_dir,DstDir}, {src_file,SrcFile}, {sftp_size,N} - | Config]. - -%%%================================================================ -openssh_client_shell(Config) -> - lists:foreach( - fun(PrefAlgs=[{kex,[Kex]}]) when Kex == 'diffie-hellman-group-exchange-sha256' -> - lists:foreach( - fun(Grp) -> - openssh_client_shell(Config, - [{preferred_algorithms, PrefAlgs}, - {dh_gex_groups, [Grp]} - ]) - end, moduli()); - (PrefAlgs) -> - openssh_client_shell(Config, - [{preferred_algorithms, PrefAlgs}]) - end, variants(kex,Config) ++ variants(cipher,Config) - ). - - -openssh_client_shell(Config, Options) -> - SystemDir = proplists:get_value(data_dir, Config), - UserDir = proplists:get_value(priv_dir, Config), - KnownHosts = filename:join(UserDir, "known_hosts"), - - {ok, TracerPid} = erlang_trace(), - {ServerPid, _Host, Port} = - ssh_test_lib:daemon([{system_dir, SystemDir}, - {public_key_alg, ssh_dsa}, - {failfun, fun ssh_test_lib:failfun/2} | - Options]), - ct:sleep(500), - - Data = lists:duplicate(100000, $a), - Cmd = lists:concat(["ssh -p ",Port, - " -o UserKnownHostsFile=", KnownHosts, - " -o \"StrictHostKeyChecking no\"", - " localhost '\"",Data,"\"'."]), -%% ct:pal("Cmd ="++Cmd), - - Parent = self(), - SlavePid = spawn(fun() -> - Parent ! {self(),os:cmd(Cmd)} - end), - receive - {SlavePid, _ClientResponse} -> -%% ct:pal("ClientResponse = ~p",[_ClientResponse]), - {ok, List} = get_trace_list(TracerPid), - Times = find_times(List, [accept_to_hello, kex, kex_to_auth, auth, to_prompt]), - Algs = find_algs(List), - ct:pal("Algorithms = ~p~n~nTimes = ~p",[Algs,Times]), - lists:foreach( - fun({Tag,Value,Unit}) -> - EventData = - case Tag of - {A,B} when A==encrypt ; A==decrypt -> - [{value, Value}, - {suite, ?MODULE}, - {name, mk_name(["Cipher ",A," ",B," [",Unit,"]"])} - ]; - kex -> - KexAlgStr = fmt_alg(Algs#alg.kex, List), - [{value, Value}, - {suite, ?MODULE}, - {name, mk_name(["Erl server kex ",KexAlgStr," [",Unit,"]"])} - ]; - _ when is_atom(Tag) -> - [{value, Value}, - {suite, ?MODULE}, - {name, mk_name(["Erl server ",Tag," [",Unit,"]"])} - ] - end, - ct:pal("ct_event:notify ~p",[EventData]), - ct_event:notify(#event{name = benchmark_data, - data = EventData}) - end, Times), - ssh:stop_daemon(ServerPid), - ok - after 60*1000 -> - ssh:stop_daemon(ServerPid), - exit(SlavePid, kill), - {fail, timeout} - end. - - -%%%================================================================ -openssh_client_sftp(Config) -> - lists:foreach( - fun(PrefAlgs) -> - openssh_client_sftp(Config, [{preferred_algorithms,PrefAlgs}]) - end, variants(cipher,Config)). - - -openssh_client_sftp(Config, Options) -> - SystemDir = proplists:get_value(data_dir, Config), - UserDir = proplists:get_value(priv_dir, Config), - SftpSrcDir = proplists:get_value(sftp_src_dir, Config), - SrcFile = proplists:get_value(src_file, Config), - SrcSize = proplists:get_value(sftp_size, Config), - KnownHosts = filename:join(UserDir, "known_hosts"), - - {ok, TracerPid} = erlang_trace(), - {ServerPid, _Host, Port} = - ssh_test_lib:daemon([{system_dir, SystemDir}, - {public_key_alg, ssh_dsa}, - {subsystems,[ssh_sftpd:subsystem_spec([%{cwd, SftpSrcDir}, - {root, SftpSrcDir}])]}, - {failfun, fun ssh_test_lib:failfun/2} - | Options]), - ct:pal("ServerPid = ~p",[ServerPid]), - ct:sleep(500), - Cmd = lists:concat(["sftp", - " -b -", - " -P ",Port, - " -o UserKnownHostsFile=", KnownHosts, - " -o \"StrictHostKeyChecking no\"", - " localhost:",SrcFile - ]), -%% ct:pal("Cmd = ~p",[Cmd]), - - Parent = self(), - SlavePid = spawn(fun() -> - Parent ! {self(),os:cmd(Cmd)} - end), - receive - {SlavePid, _ClientResponse} -> - ct:pal("ClientResponse = ~p~nServerPid = ~p",[_ClientResponse,ServerPid]), - {ok, List} = get_trace_list(TracerPid), -%%ct:pal("List=~p",[List]), - Times = find_times(List, [channel_open_close]), - Algs = find_algs(List), - ct:pal("Algorithms = ~p~n~nTimes = ~p",[Algs,Times]), - lists:foreach( - fun({{A,B},Value,Unit}) when A==encrypt ; A==decrypt -> - Data = [{value, Value}, - {suite, ?MODULE}, - {name, mk_name(["Sftp Cipher ",A," ",B," [",Unit,"]"])} - ], - ct:pal("sftp ct_event:notify ~p",[Data]), - ct_event:notify(#event{name = benchmark_data, - data = Data}); - ({channel_open_close,Value,Unit}) -> - Cipher = fmt_alg(Algs#alg.encrypt, List), - Data = [{value, round( (1024*Value) / SrcSize )}, - {suite, ?MODULE}, - {name, mk_name(["Sftp transfer ",Cipher," [",Unit," per kbyte]"])} - ], - ct:pal("sftp ct_event:notify ~p",[Data]), - ct_event:notify(#event{name = benchmark_data, - data = Data}); - (_) -> - skip - end, Times), - ssh:stop_daemon(ServerPid), - ok - after 2*60*1000 -> - ssh:stop_daemon(ServerPid), - exit(SlavePid, kill), - {fail, timeout} - end. - -%%%================================================================ -variants(Tag, Config) -> - TagType = - case proplists:get_value(Tag, ssh:default_algorithms()) of - [{_,_}|_] -> one_way; - [A|_] when is_atom(A) -> two_way - end, - [ [{Tag,tag_value(TagType,Alg)}] - || Alg <- proplists:get_value(Tag, proplists:get_value(common_algs,Config)) - ]. - -tag_value(two_way, Alg) -> [Alg]; -tag_value(one_way, Alg) -> [{client2server,[Alg]}, - {server2client,[Alg]}]. - -%%%---------------------------------------------------------------- -fmt_alg(Alg, List) when is_atom(Alg) -> - fmt_alg(atom_to_list(Alg), List); -fmt_alg(Alg = "diffie-hellman-group-exchange-sha" ++ _, List) -> - try - integer_to_list(find_gex_size_string(List)) - of - GexSize -> lists:concat([Alg," ",GexSize]) - catch - _:_ -> Alg - end; -fmt_alg(Alg, _List) -> - Alg. - -%%%---------------------------------------------------------------- -mk_name(Name) -> [char(C) || C <- lists:concat(Name)]. - -char($-) -> $_; -char(C) -> C. - -%%%---------------------------------------------------------------- -find_times(L, Xs) -> - [find_time(X,L) || X <- Xs] ++ - function_algs_times_sizes([{ssh_transport,encrypt,2}, - {ssh_transport,decrypt,2}, - {ssh_message,decode,1}, - {ssh_message,encode,1}], L). - --record(call, { - mfa, - pid, - t_call, - t_return, - args, - result - }). - -%%%---------------- --define(send(M), fun(C=#call{mfa = {ssh_message,encode,1}, - args = [M]}) -> - C#call.t_return - end). - --define(recv(M), fun(C=#call{mfa = {ssh_message,decode,1}, - result = M}) -> - C#call.t_call - end). - -find_time(accept_to_hello, L) -> - [T0,T1] = find([fun(C=#call{mfa = {ssh_acceptor,handle_connection,5}}) -> - C#call.t_call - end, - ?LINE, - fun(C=#call{mfa = {ssh_connection_handler,handle_event,4}, - args = [_, {version_exchange,_}, {hello,_}, _]}) -> - C#call.t_call - end, - ?LINE - ], L, []), - {accept_to_hello, now2micro_sec(now_diff(T1,T0)), microsec}; -find_time(kex, L) -> - [T0,T1] = find([fun(C=#call{mfa = {ssh_connection_handler,handle_event,4}, - args = [_, {version_exchange,_}, {hello,_}, _]}) -> - C#call.t_call - end, - ?LINE, - ?send(#ssh_msg_newkeys{}), - ?LINE - ], L, []), - {kex, now2micro_sec(now_diff(T1,T0)), microsec}; -find_time(kex_to_auth, L) -> - [T0,T1] = find([?send(#ssh_msg_newkeys{}), - ?LINE, - ?recv(#ssh_msg_userauth_request{}), - ?LINE - ], L, []), - {kex_to_auth, now2micro_sec(now_diff(T1,T0)), microsec}; -find_time(auth, L) -> - [T0,T1] = find([?recv(#ssh_msg_userauth_request{}), - ?LINE, - ?send(#ssh_msg_userauth_success{}), - ?LINE - ], L, []), - {auth, now2micro_sec(now_diff(T1,T0)), microsec}; -find_time(to_prompt, L) -> - [T0,T1] = find([fun(C=#call{mfa = {ssh_acceptor,handle_connection,5}}) -> - C#call.t_call - end, - ?LINE, - ?recv(#ssh_msg_channel_request{request_type="env"}), - ?LINE - ], L, []), - {to_prompt, now2micro_sec(now_diff(T1,T0)), microsec}; -find_time(channel_open_close, L) -> - [T0,T1] = find([?recv(#ssh_msg_channel_request{request_type="subsystem"}), - ?LINE, - ?send(#ssh_msg_channel_close{}), - ?LINE - ], L, []), - {channel_open_close, now2micro_sec(now_diff(T1,T0)), microsec}. - - - -find([F,Id|Fs], [C|Cs], Acc) when is_function(F,1) -> - try - F(C) - of - T -> find(Fs, Cs, [T|Acc]) - catch - _:_ -> find([F,Id|Fs], Cs, Acc) - end; -find([], _, Acc) -> - lists:reverse(Acc). - - -find_algs(L) -> - {value, #call{result={ok,Algs}}} = - lists:keysearch({ssh_transport,select_algorithm,3}, #call.mfa, L), - Algs. - -find_gex_size_string(L) -> - %% server - {value, #call{result={ok,{Size, _}}}} = - lists:keysearch({public_key,dh_gex_group,4}, #call.mfa, L), - Size. - -%%%---------------- -function_algs_times_sizes(EncDecs, L) -> - Raw = [begin - {Tag,Size} = function_ats_result(EncDec, C), - {Tag, Size, now2micro_sec(now_diff(T1,T0))} - end - || EncDec <- EncDecs, - C = #call{mfa = ED, - % args = Args, %%[S,Data], - t_call = T0, - t_return = T1} <- L, - ED == EncDec - ], - [{Alg, round(1024*Time/Size), "microsec per kbyte"} % Microseconds per 1k bytes. - || {Alg,Size,Time} <- lists:foldl(fun increment/2, [], Raw)]. - -function_ats_result({ssh_transport,encrypt,2}, #call{args=[S,Data]}) -> - {{encrypt,S#ssh.encrypt}, binsize(Data)}; -function_ats_result({ssh_transport,decrypt,2}, #call{args=[S,Data]}) -> - {{decrypt,S#ssh.decrypt}, binsize(Data)}; -function_ats_result({ssh_message,encode,1}, #call{result=Data}) -> - {encode, size(Data)}; -function_ats_result({ssh_message,decode,1}, #call{args=[Data]}) -> - {decode, size(Data)}. - -binsize(B) when is_binary(B) -> size(B); -binsize({B1,B2}) when is_binary(B1), is_binary(B2) -> size(B1) + size(B2); -binsize({B1,B2,_}) when is_binary(B1), is_binary(B2) -> size(B1) + size(B2). - - - - - -increment({Alg,Sz,T}, [{Alg,SumSz,SumT}|Acc]) -> - [{Alg,SumSz+Sz,SumT+T} | Acc]; -increment(Spec, [X|Acc]) -> - [X | increment(Spec,Acc)]; % Not so many Alg, 2 or 3 -increment({Alg,Sz,T},[]) -> - [{Alg,Sz,T}]. - -%%%---------------------------------------------------------------- -%%% -%%% API for the traceing -%%% -get_trace_list(TracerPid) -> - MonRef = monitor(process, TracerPid), - TracerPid ! {get_trace_list,self()}, - receive - {trace_list,L} -> - demonitor(MonRef), - {ok, pair_events(lists:reverse(L))}; - {'DOWN', MonRef, process, TracerPid, Info} -> - {error, {tracer_down,Info}} - - after 3*60*1000 -> - demonitor(MonRef), - {error,no_reply} - end. - -erlang_trace() -> - TracerPid = spawn(fun trace_loop/0), - 0 = erlang:trace(new, true, [call,timestamp,{tracer,TracerPid}]), - [init_trace(MFA, tp(MFA)) - || MFA <- [{ssh_acceptor,handle_connection,5}, -%% {ssh_connection_handler,hello,2}, - {ssh_message,encode,1}, - {ssh_message,decode,1}, - {ssh_transport,select_algorithm,3}, - {ssh_transport,encrypt,2}, - {ssh_transport,decrypt,2}, - {ssh_message,encode,1}, - {ssh_message,decode,1}, - {public_key,dh_gex_group,4} % To find dh_gex group size - ]], - init_trace({ssh_connection_handler,handle_event,4}, - [{['_', {version_exchange,'_'}, {hello,'_'}, '_'], - [], - [return_trace]}]), - {ok, TracerPid}. - -tp({_M,_F,Arity}) -> - [{lists:duplicate(Arity,'_'), [], [{return_trace}]}]. - -%%%---------------------------------------------------------------- -init_trace(MFA = {Module,_,_}, TP) -> - case code:is_loaded(Module) of - false -> code:load_file(Module); - _ -> ok - end, - erlang:trace_pattern(MFA, TP, [local]). - - -trace_loop() -> - trace_loop([]). - -trace_loop(L) -> - receive - {get_trace_list, From} -> - From ! {trace_list, L}, - trace_loop(L); - Ev -> - trace_loop([Ev|L]) - end. - -pair_events(L) -> - pair_events(L, []). - -pair_events([{trace_ts,Pid,call,{M,F,Args},TS0} | L], Acc) -> - Arity = length(Args), - {ReturnValue,TS1} = find_return(Pid, {M,F,Arity}, L), - pair_events(L, [#call{mfa = {M,F,Arity}, - pid = Pid, - t_call = TS0, - t_return = TS1, - args = Args, - result = ReturnValue} | Acc]); -pair_events([_|L], Acc) -> - pair_events(L, Acc); -pair_events([], Acc) -> - lists:reverse(Acc). - - -find_return(Pid, MFA, - [{trace_ts, Pid, return_from, MFA, ReturnValue, TS}|_]) -> - {ReturnValue, TS}; -find_return(Pid, MFA, [_|L]) -> - find_return(Pid, MFA, L); -find_return(_, _, []) -> - {undefined, undefined}. - -%%%---------------------------------------------------------------- -report_client_algorithms() -> - try - ssh_test_lib:extract_algos( ssh_test_lib:default_algorithms(sshc) ) - of - ClientAlgs -> - ct:pal("The client supports:~n~p",[ClientAlgs]) - catch - Cls:Err -> - ct:pal("Testing client about algorithms failed:~n~p ~p",[Cls,Err]) - end. - -%%%---------------------------------------------------------------- - - -now2sec({A,B,C}) -> A*1000000 + B + C/1000000. - -now2micro_sec({A,B,C}) -> (A*1000000 + B)*1000000 + C. - -now_diff({A1,B1,C1}, {A0,B0,C0}) -> {A1-A0, B1-B0, C1-C0}. - -%%%================================================================ -moduli() -> - [{1023, 5, 16#CF973CD39DC7D62F2C45AAC5180491104C76E0FE5D80A10E6C06AE442F1F373167B0FCBC931F3C157B10A5557008FDE20D68051E6A4DB11CEE0B0749F76D7134B937A59DA998C42BC234A5C1A3CFCD70E624D253D7694076F7B1FD7B8D3427849C9377B3555796ACA58C69DFF542EEEC9859D3ADCE5CC88DF6F7817C9D182EB7}, - {2047, 5, 16#F7693FC11FDDEAA493D3BA36F1FFF9264AA9952209203192A88A697BE9D0E306E306A27430BD87AB9EE9DB4BC78C41950C2EB0E5E4C686E8B1BA6D6A2B1FE91EF40C5EA32C51018323E1D305FE637F35ACABDBFC40AD683F779570A76869EB90015A342B2D1F7C81602688081FCAAA8D623090258D9C5C729C8CDDC0C12CA2D561DD987DB79B6AD7A2A509EBC383BF223FD95BC5A2FCC26FB3F3A0DD3FDC1228E338D3290235A596F9465F7BF490974847E616229A9E60B8F4AA161C52F655843CCCAE8821B40C426B535DE087964778652BBD4EC601C0456AE7128B593FCC64402C891227AE6EE88CC839416FBF462B4852999C646BE0BED7D8CF2BE5E381EF}, - {4095, 2, 16#C8842271626E53546E0C712FA265713F2EE073C20A0723C96B6B182B1EAACC96233D4A199BD0E85F264078A513AD2454F284B8DF543D85019D1E70F2FF54BA43EFBC64AF465C170C3E376F5EC328F98E33E1ED8BED84FA097ABE584152B0E9827ED5CC2B1D4F5ECF2DC46F45C59816D02698EA26F319311E2B6973E83C37021CC8B416AEF653896A1764EE0CEE718A45E8B47CB960BD5907D0E843E8A8E7D4698363C3C3FB3ADC512368B72CAF16510C69052EA2AF51BE00BC8CA04DF1F00A00CC2CA4D74254A1E8738460FD244DDB446CB36554B0A24EEF3710E44DBCF39881E7D3F9AE223388084E7A49A3CB12612AE36416C0EB5628DF1477FEE4A5CF77CDC09AA0E2C989C0B7D1310AFA44B81DA79A65226C7EA510057991EABF9388DC5EA9F52FEA5D3B0872843F50878740794E523E9DC60E0EA1FC8746A7B2AA31FCA89AAA2FA907BED116C69D98F912DD5089BECF28577064225DE96FC214ED1794E7CCE8024F94036D915A123A464C951DA96A5ED7F286F205BEE71BDE2D133FD1891B31178FF25D31611A5B7839F0E68EAF0F8901A571E6917C580F31842A9F19C47E0638483B7947DDCD7864660AC2F8B2C430F1E7FC0F22FA51F96F0499332C5AD3FF9DC7F4332DD5BCCA820CC779B90C0F4C5F0CA52E96FAA187361753FBADC5C80D0492CD80A3EEA5D578772DA9FC1C0E10A0203098AF36D0ED2156BA7321EB}, - {6143, 5, 16#FD9E6B52785CD7BE64D396A599DA4B97CD0BB49183F932A97694D80CA553354DBC26E77B8A0EC002257AADDF6AD27819CE64A06416E4A80B6EA92F28EA8D5B96C774109EEE5816B4B18F84368D1B41864C11AA73D6881675D779B174F6B4E344303F3EFD11BD7DE468467242372FD00908F296F5A2B20E2684F9122D08A46D647B05E298F0BCDAB60468349CCA6DA1B9FEBBC69D256FB9A3F1980F68466364FCEF1C98C1405191A6737A3627BA7F7313A8A18FC0B8521BF3430B1C6805CB44BCEB39904DD30130D24B225B598ED83C5FD757B80189FD9D5C2F9596687C40BAB1C6ED6244944629849D074A4C33FB15DDB3F9760FC59C44BEBB0EC032177147F61789769DAAAE2123CE488F7ECF19BDA051925BA9ED11EAA72DF70C9ECC8F714B4C35728E6679E66A1B56CCAE0FBBD3F9EBF950D4D623ED78E77CC3AD604E91F304EA78CE876F036214BD6F1977BD04C9ADD707D7A3BCCE87AD5D5A11C95E7025B0EA9C649DCB37942A3970A4FB04C284E4DDB4DC90163353B98B1C254FFD28443353F17A87C02E0BDB9F05424CC44C86309F1D73706F039CDAAC3EDC1A64F38FB42707D351DB5360C2680ADC1CC8D1C4AD312ACC904382C26BE33DA0E61429A5940820356ED28586BEB629ED1521D12D25B4DA01926295F3DA504DC9F431B719AC63277BE675E6F6DD4F7499CA11A23744577D653941963E8DAB610F7F226DB52CE5C683F72AEED2B6CE35ED07C29410397A6F7F606477CCC0EDE18CD0D96A7863BC4606193A8799B5AC1EEE6AC5EE36AC3077EC8DAB30EE94434B45B78BC13D96F74D6C4056EAA528CD3C68D308344808819B12F2BFB95A5C1A7DEEE188BF139216DDB7D757D7A50D3C46CE18881D776D617DCFFAA62276045373AA4D9446D7570338F99C0CA8A08851B4F9D388B4C275D3F9B7BA25F235D4329F63F7457C2EB5C68CE2A96D19766F0ED8E19F66DF3C5E29A38795B2F92291BB6EAB6F70A7E89DC9691F28486E9CF87FF11D5DF2E6B030A30B5D476AD59A34EE7262712ED96CEF4A5CAC3F08B3563D44683F746DA094C9CDB34427AF8D8CC2AE1B23C3BEB637}, - {8191, 2, 16#DC61EF13E4F3FC10CC946EEABC33F83EFCB35E0F47E4EC25C1CCBB2C7B502B2EFB0691AA231C8476DD51BA73204E6EA10B1A970FE2CF14AF01E72E1AEA87519A91D00D1499189F94A6CDA9E29C05F11F17FE74A4919A710A2787E180744465DF81C62AA65662FDA46FA6175E8A31E5B29E66DED6701C8FC4217E91D733FE94380F046680967D4CEA7BAC8F3916CDF96AA2C474FAD9650F48403FD0B5B756D34667D36A07767FA33027AE55484D0F701C3CA16632F413A14E4B8645AFAF15B78978C19A7661EDC569BEC72394B1204B166A48FCD5F56BE29840C7794CA6D3440356F15858CDCA9B429C7EA92E17242893FDC8C9C63841A382C32F20CFAB121B4BCAFD7BF9EF07FBF7CDFFECA0CEF3A49C3E2B24FA836F3318435255655E1B281071F62D5E4CD63361299B7828F72936E3FEA9E8044562A6F6ADD5321187C3101E4669C6271598FE1A866C93FE2870A4CEB9254BA32A4719E439317EA42200A335B5CFFA7946A7D0F1BD1A69AA11288B73C71C80B77FE3707CB077DDDEA5CA36A449FAB230C9625A0B12F8275D3FF82F5DA380E7A3F11B6F155FE7E91AC960BD95D9B13F7423AB9B15CC3C4DC34EF296033F009468EA16A721AD659F56C18516025050749ABF05E6D3EBD9778142A530979291F46DAA399A86B7BCDF09CC3E6EEF101419762A306DB45AEFC96C64E83F28338D55905F6A387E0F515E580C3A9B35330E21C32198CDEE3AFB355967A098F635FCA7C49CB4E1E82464B2B390EF1F259E40B9A06235C0273F76284FE6BD534EF3AF7CB01A4A5252B8B94CADC2850B2E56D53F9A31D7C029DF967D0A30C05BC64E119BED6076818FABC8CDD93F3255693E14EFC1A740A5D63A5E847FFE87BAB1DDE0506E1762EA61EFA9F9756151ECCCADD91B98A961A901A2D8B01ABDDD29EC804E8C8D28214BBA26048F924CA66316696E51A49D02FF034D20E44914B1115339CAD3819E0CB1640F0084886FEDDE5E28C29DC48ED30A8C3D789734338F5A9DF42584326E536FD1CF30BC85B8DCBD6120D127C98FE4B3614074F13C2CA4854E6D794156C185C40EB3DA7619CE96ADAF0941BD5499848B034C2B11DFECC0BDFA81C594241F759EF53FC7CDE7F2DE4F23CF81A5A0B7D62E31DABB9198D40307F7824DD130B7D1B80E9B6D322FEEDB5ACE34944F0BFB7D016762A9B2E173BFDD69303766AFBAB45FAB75D05430B4A3515858C4B7F04E23414E4AD03842CB0A20D8FF4B59B7C852BA9A5BE982A8ADA5CB70C36CE2A4D2C31A7015C9F3275E43D192C1B2924424088907A057DA7F2D32A2149922AB2E33F2147D637A3508911CB3FEA5E1AAB4525BACF27B6DD7A3E0AFA978FC3A39DE8882FB22688C3CCC92B6E69ACB0BBF575AB3368E51A2F6A20C414C6F146727CC0045F29061E695D29F7C030CE6929EB3AD11A5CBD0CDEE37347869A3}]. diff --git a/lib/ssh/test/ssh_sftpd_SUITE.erl b/lib/ssh/test/ssh_sftpd_SUITE.erl index 6d18a980ee..b167f98ac8 100644 --- a/lib/ssh/test/ssh_sftpd_SUITE.erl +++ b/lib/ssh/test/ssh_sftpd_SUITE.erl @@ -158,8 +158,7 @@ init_per_testcase(TestCase, Config) -> [{user_dir, ClientUserDir}, {user, ?USER}, {password, ?PASSWD}, {user_interaction, false}, - {silently_accept_hosts, true}, - {pwdfun, fun(_,_) -> true end}]), + {silently_accept_hosts, true}]), {ok, Channel} = ssh_connection:session_channel(Cm, ?XFER_WINDOW_SIZE, ?XFER_PACKET_SIZE, ?TIMEOUT), diff --git a/lib/ssh/test/ssh_sftpd_erlclient_SUITE.erl b/lib/ssh/test/ssh_sftpd_erlclient_SUITE.erl index fd5157d603..b4d7eadfa4 100644 --- a/lib/ssh/test/ssh_sftpd_erlclient_SUITE.erl +++ b/lib/ssh/test/ssh_sftpd_erlclient_SUITE.erl @@ -189,7 +189,6 @@ quit(Config) when is_list(Config) -> timer:sleep(5000), {ok, NewSftp, _Conn} = ssh_sftp:start_channel(Host, Port, [{silently_accept_hosts, true}, - {pwdfun, fun(_,_) -> true end}, {user_dir, UserDir}, {user, ?USER}, {password, ?PASSWD}]), diff --git a/lib/ssh/test/ssh_to_openssh_SUITE.erl b/lib/ssh/test/ssh_to_openssh_SUITE.erl index 4d5e5be2a1..7eda009552 100644 --- a/lib/ssh/test/ssh_to_openssh_SUITE.erl +++ b/lib/ssh/test/ssh_to_openssh_SUITE.erl @@ -333,7 +333,7 @@ erlang_client_openssh_server_publickey_rsa(Config) when is_list(Config) -> [{_,_, not_encrypted}] -> ConnectionRef = ssh_test_lib:connect(?SSH_DEFAULT_PORT, - [{public_key_alg, ssh_rsa}, + [{pref_public_key_algs, ['ssh-rsa','ssh-dss']}, {user_interaction, false}, silently_accept_hosts]), {ok, Channel} = @@ -354,7 +354,7 @@ erlang_client_openssh_server_publickey_dsa() -> erlang_client_openssh_server_publickey_dsa(Config) when is_list(Config) -> ConnectionRef = ssh_test_lib:connect(?SSH_DEFAULT_PORT, - [{public_key_alg, ssh_dsa}, + [{pref_public_key_algs, ['ssh-dss','ssh-rsa']}, {user_interaction, false}, silently_accept_hosts]), {ok, Channel} = @@ -381,7 +381,6 @@ erlang_server_openssh_client_public_key_X(Config, PubKeyAlg) -> PrivDir = proplists:get_value(priv_dir, Config), KnownHosts = filename:join(PrivDir, "known_hosts"), {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, - {public_key_alg, PubKeyAlg}, {failfun, fun ssh_test_lib:failfun/2}]), ct:sleep(500), @@ -402,7 +401,6 @@ erlang_server_openssh_client_renegotiate(Config) -> KnownHosts = filename:join(PrivDir, "known_hosts"), {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, - {public_key_alg, PubKeyAlg}, {failfun, fun ssh_test_lib:failfun/2}]), ct:sleep(500), @@ -464,6 +462,7 @@ erlang_client_openssh_server_renegotiate(_Config) -> {silently_accept_hosts,true}], group_leader(IO, self()), {ok, ConnRef} = ssh:connect(Host, ?SSH_DEFAULT_PORT, Options), + ct:pal("Parent = ~p, IO = ~p, Shell = ~p, ConnRef = ~p~n",[Parent, IO, self(), ConnRef]), case ssh_connection:session_channel(ConnRef, infinity) of {ok,ChannelId} -> success = ssh_connection:ptty_alloc(ConnRef, ChannelId, []), diff --git a/lib/ssh/test/ssh_trpt_test_lib.erl b/lib/ssh/test/ssh_trpt_test_lib.erl index 0fa0f0c0e4..261239c152 100644 --- a/lib/ssh/test/ssh_trpt_test_lib.erl +++ b/lib/ssh/test/ssh_trpt_test_lib.erl @@ -85,18 +85,18 @@ exec(Op, S0=#s{}) -> throw:Term -> report_trace(throw, Term, S1), - throw(Term); + throw({Term,Op}); error:Error -> report_trace(error, Error, S1), - error(Error); + error({Error,Op}); exit:Exit -> report_trace(exit, Exit, S1), - exit(Exit); + exit({Exit,Op}); Cls:Err -> ct:pal("Class=~p, Error=~p", [Cls,Err]), - error("fooooooO") + error({"fooooooO",Op}) end; exec(Op, {ok,S=#s{}}) -> exec(Op, S); exec(_, Error) -> Error. @@ -114,20 +114,20 @@ op({accept,Opts}, S) when ?role(S) == server -> {ok,Socket} = gen_tcp:accept(S#s.listen_socket, S#s.timeout), {Host,_Port} = ok(inet:sockname(Socket)), S#s{socket = Socket, - ssh = init_ssh(server,Socket,[{host,host(Host)}|Opts]), + ssh = init_ssh(server, Socket, host(Host), Opts), return_value = ok}; %%%---- Client ops op({connect,Host,Port,Opts}, S) when ?role(S) == undefined -> Socket = ok(gen_tcp:connect(host(Host), Port, mangle_opts([]))), S#s{socket = Socket, - ssh = init_ssh(client, Socket, [{host,host(Host)}|Opts]), + ssh = init_ssh(client, Socket, host(Host), Opts), return_value = ok}; %%%---- ops for both client and server op(close_socket, S) -> - catch tcp_gen:close(S#s.socket), - catch tcp_gen:close(S#s.listen_socket), + catch gen_tcp:close(S#s.socket), + catch gen_tcp:close(S#s.listen_socket), S#s{socket = undefined, listen_socket = undefined, return_value = ok}; @@ -296,12 +296,14 @@ instantiate(X, _S) -> %%%================================================================ %%% -init_ssh(Role, Socket, Options0) -> - Options = [{user_interaction, false}, - {vsn, {2,0}}, - {id_string, "ErlangTestLib"} - | Options0], - ssh_connection_handler:init_ssh_record(Role, Socket, Options). +init_ssh(Role, Socket, Host, UserOptions0) -> + UserOptions = [{user_interaction, false}, + {vsn, {2,0}}, + {id_string, "ErlangTestLib"} + | UserOptions0], + Opts = ?PUT_INTERNAL_OPT({host,Host}, + ssh_options:handle_options(Role, UserOptions)), + ssh_connection_handler:init_ssh_record(Role, Socket, Opts). mangle_opts(Options) -> SysOpts = [{reuseaddr, true}, diff --git a/lib/ssh/vsn.mk b/lib/ssh/vsn.mk index c6a5990f41..48332d2e5a 100644 --- a/lib/ssh/vsn.mk +++ b/lib/ssh/vsn.mk @@ -1,5 +1,5 @@ #-*-makefile-*- ; force emacs to enter makefile-mode -SSH_VSN = 4.4 +SSH_VSN = 4.4.2 APP_VSN = "ssh-$(SSH_VSN)" diff --git a/lib/ssl/doc/src/notes.xml b/lib/ssl/doc/src/notes.xml index 29b8e8ff67..7ffb9c0e88 100644 --- a/lib/ssl/doc/src/notes.xml +++ b/lib/ssl/doc/src/notes.xml @@ -28,6 +28,88 @@ <p>This document describes the changes made to the SSL application.</p> +<section><title>SSL 8.1.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Correct active once emulation, for TLS. Now all data + received by the connection process will be delivered + through active once, even when the active once arrives + after that the gen_tcp socket is closed by the peer.</p> + <p> + Own Id: OTP-14300</p> + </item> + </list> + </section> + +</section> + +<section><title>SSL 8.1.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Corrected termination behavior, that caused a PEM cache + bug and sometimes resulted in connection failures.</p> + <p> + Own Id: OTP-14100</p> + </item> + <item> + <p> + Fix bug that could hang ssl connection processes when + failing to require more data for very large handshake + packages. Add option max_handshake_size to mitigate DoS + attacks.</p> + <p> + Own Id: OTP-14138</p> + </item> + <item> + <p> + Improved support for CRL handling that could fail to work + as intended when an id-ce-extKeyUsage was present in the + certificate. Also improvements where needed to + distributionpoint handling so that all revocations + actually are found and not deemed to be not determinable.</p> + <p> + Own Id: OTP-14141</p> + </item> + <item> + <p> + A TLS handshake might accidentally match old sslv2 format + and ssl application would incorrectly aborted TLS + handshake with ssl_v2_client_hello_no_supported. Parsing + was altered to avoid this problem.</p> + <p> + Own Id: OTP-14222</p> + </item> + <item> + <p> + Correct default cipher list to prefer AES 128 before 3DES</p> + <p> + Own Id: OTP-14235</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Move PEM cache to a dedicated process, to avoid making + the SSL manager process a bottleneck. This improves + scalability of TLS connections.</p> + <p> + Own Id: OTP-13874</p> + </item> + </list> + </section> + +</section> + <section><title>SSL 8.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index 916b41742e..91c590c247 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -935,13 +935,14 @@ fun(srp, Username :: string(), UserState :: term()) -> <fsummary>Returns all the connection information. </fsummary> <type> - <v>Item = protocol | cipher_suite | sni_hostname | ecc | atom()</v> + <v>Item = protocol | cipher_suite | sni_hostname | ecc | session_id | atom()</v> <d>Meaningful atoms, not specified above, are the ssl option names.</d> <v>Result = [{Item::atom(), Value::term()}]</v> <v>Reason = term()</v> </type> - <desc><p>Returns all relevant information about the connection, ssl options that - are undefined will be filtered out.</p> + <desc><p>Returns the most relevant information about the connection, ssl options that + are undefined will be filtered out. Note that values that affect the security of the + connection will only be returned if explicitly requested by connection_information/2.</p> </desc> </func> @@ -952,8 +953,10 @@ fun(srp, Username :: string(), UserState :: term()) -> </fsummary> <type> <v>Items = [Item]</v> - <v>Item = protocol | cipher_suite | sni_hostname | atom()</v> - <d>Meaningful atoms, not specified above, are the ssl option names.</d> + <v>Item = protocol | cipher_suite | sni_hostname | ecc | session_id | client_random + | server_random | master_secret | atom()</v> + <d>Note that client_random, server_random and master_secret are values + that affect the security of connection. Meaningful atoms, not specified above, are the ssl option names.</d> <v>Result = [{Item::atom(), Value::term()}]</v> <v>Reason = term()</v> </type> diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl index 070a90d481..16e0df2101 100644 --- a/lib/ssl/src/dtls_connection.erl +++ b/lib/ssl/src/dtls_connection.erl @@ -39,7 +39,7 @@ -export([start_fsm/8, start_link/7, init/1]). %% State transition handling --export([next_record/1, next_event/3]). +-export([next_record/1, next_event/3, next_event/4]). %% Handshake handling -export([renegotiate/2, @@ -53,7 +53,7 @@ %% Data handling -export([encode_data/3, passive_receive/2, next_record_if_active/1, handle_common_event/4, - send/3]). + send/3, socket/5]). %% gen_statem state functions -export([init/3, error/3, downgrade/3, %% Initiation and take down states @@ -77,20 +77,6 @@ start_fsm(Role, Host, Port, Socket, {#ssl_options{erl_dist = false},_, Tracker} catch error:{badmatch, {error, _} = Error} -> Error - end; - -start_fsm(Role, Host, Port, Socket, {#ssl_options{erl_dist = true},_, Tracker} = Opts, - User, {CbModule, _,_, _} = CbInfo, - Timeout) -> - try - {ok, Pid} = dtls_connection_sup:start_child_dist([Role, Host, Port, Socket, - Opts, User, CbInfo]), - {ok, SslSocket} = ssl_connection:socket_control(?MODULE, Socket, Pid, CbModule, Tracker), - ok = ssl_connection:handshake(SslSocket, Timeout), - {ok, SslSocket} - catch - error:{badmatch, {error, _} = Error} -> - Error end. send_handshake(Handshake, #state{connection_states = ConnectionStates} = States) -> @@ -201,6 +187,7 @@ reinit_handshake_data(#state{protocol_buffers = Buffers} = State) -> State#state{premaster_secret = undefined, public_key_info = undefined, tls_handshake_history = ssl_handshake:init_handshake_history(), + flight_state = {retransmit, ?INITIAL_RETRANSMIT_TIMEOUT}, protocol_buffers = Buffers#protocol_buffers{ dtls_handshake_next_seq = 0, @@ -213,6 +200,9 @@ select_sni_extension(#client_hello{extensions = HelloExtensions}) -> select_sni_extension(_) -> undefined. +socket(Pid, Transport, Socket, Connection, _) -> + dtls_socket:socket(Pid, Transport, Socket, Connection). + %%==================================================================== %% tls_connection_sup API %%==================================================================== @@ -243,7 +233,7 @@ callback_mode() -> state_functions. %%-------------------------------------------------------------------- -%% State functionsconnection/2 +%% State functions %%-------------------------------------------------------------------- init({call, From}, {start, Timeout}, @@ -262,18 +252,25 @@ init({call, From}, {start, Timeout}, Version = Hello#client_hello.client_version, HelloVersion = dtls_record:lowest_protocol_version(SslOpts#ssl_options.versions), State1 = prepare_flight(State0#state{negotiated_version = Version}), - State2 = send_handshake(Hello, State1#state{negotiated_version = HelloVersion}), + {State2, Actions} = send_handshake(Hello, State1#state{negotiated_version = HelloVersion}), State3 = State2#state{negotiated_version = Version, %% Requested version session = Session0#session{session_id = Hello#client_hello.session_id}, start_or_recv_from = From, - timer = Timer}, + timer = Timer, + flight_state = {retransmit, ?INITIAL_RETRANSMIT_TIMEOUT} + }, {Record, State} = next_record(State3), - next_event(hello, Record, State); + next_event(hello, Record, State, Actions); init({call, _} = Type, Event, #state{role = server, transport_cb = gen_udp} = State) -> - ssl_connection:init(Type, Event, - State#state{flight_state = {waiting, undefined, ?INITIAL_RETRANSMIT_TIMEOUT}}, - ?MODULE); + Result = ssl_connection:init(Type, Event, + State#state{flight_state = {retransmit, ?INITIAL_RETRANSMIT_TIMEOUT}, + protocol_specific = #{current_cookie_secret => dtls_v1:cookie_secret(), + previous_cookie_secret => <<>>}}, + ?MODULE), + erlang:send_after(dtls_v1:cookie_timeout(), self(), new_cookie_secret), + Result; + init({call, _} = Type, Event, #state{role = server} = State) -> %% I.E. DTLS over sctp ssl_connection:init(Type, Event, State#state{flight_state = reliable}, ?MODULE); @@ -296,26 +293,32 @@ error(_, _, _) -> hello(internal, #client_hello{cookie = <<>>, client_version = Version} = Hello, #state{role = server, transport_cb = Transport, - socket = Socket} = State0) -> - %% TODO: not hard code key + socket = Socket, + protocol_specific = #{current_cookie_secret := Secret}} = State0) -> {ok, {IP, Port}} = dtls_socket:peername(Transport, Socket), - Cookie = dtls_handshake:cookie(<<"secret">>, IP, Port, Hello), + Cookie = dtls_handshake:cookie(Secret, IP, Port, Hello), VerifyRequest = dtls_handshake:hello_verify_request(Cookie, Version), State1 = prepare_flight(State0#state{negotiated_version = Version}), - State2 = send_handshake(VerifyRequest, State1), + {State2, Actions} = send_handshake(VerifyRequest, State1), {Record, State} = next_record(State2), - next_event(hello, Record, State#state{tls_handshake_history = ssl_handshake:init_handshake_history()}); + next_event(hello, Record, State#state{tls_handshake_history = ssl_handshake:init_handshake_history()}, Actions); hello(internal, #client_hello{cookie = Cookie} = Hello, #state{role = server, transport_cb = Transport, - socket = Socket} = State0) -> + socket = Socket, + protocol_specific = #{current_cookie_secret := Secret, + previous_cookie_secret := PSecret}} = State0) -> {ok, {IP, Port}} = dtls_socket:peername(Transport, Socket), - %% TODO: not hard code key - case dtls_handshake:cookie(<<"secret">>, IP, Port, Hello) of + case dtls_handshake:cookie(Secret, IP, Port, Hello) of Cookie -> handle_client_hello(Hello, State0); _ -> - %% Handle bad cookie as new cookie request RFC 6347 4.1.2 - hello(internal, Hello#client_hello{cookie = <<>>}, State0) + case dtls_handshake:cookie(PSecret, IP, Port, Hello) of + Cookie -> + handle_client_hello(Hello, State0); + _ -> + %% Handle bad cookie as new cookie request RFC 6347 4.1.2 + hello(internal, Hello#client_hello{cookie = <<>>}, State0) + end end; hello(internal, #hello_verify_request{cookie = Cookie}, #state{role = client, host = Host, port = Port, @@ -333,13 +336,13 @@ hello(internal, #hello_verify_request{cookie = Cookie}, #state{role = client, Cache, CacheCb, Renegotiation, OwnCert), Version = Hello#client_hello.client_version, HelloVersion = dtls_record:lowest_protocol_version(SslOpts#ssl_options.versions), - State2 = send_handshake(Hello, State1#state{negotiated_version = HelloVersion}), + {State2, Actions} = send_handshake(Hello, State1#state{negotiated_version = HelloVersion}), State3 = State2#state{negotiated_version = Version, %% Requested version session = Session0#session{session_id = Hello#client_hello.session_id}}, {Record, State} = next_record(State3), - next_event(hello, Record, State); + next_event(hello, Record, State, Actions); hello(internal, #server_hello{} = Hello, #state{connection_states = ConnectionStates0, negotiated_version = ReqVersion, @@ -356,13 +359,13 @@ hello(internal, #server_hello{} = Hello, hello(internal, {handshake, {#client_hello{cookie = <<>>} = Handshake, _}}, State) -> %% Initial hello should not be in handshake history {next_state, hello, State, [{next_event, internal, Handshake}]}; - hello(internal, {handshake, {#hello_verify_request{} = Handshake, _}}, State) -> %% hello_verify should not be in handshake history {next_state, hello, State, [{next_event, internal, Handshake}]}; - hello(info, Event, State) -> handle_info(Event, hello, State); +hello(state_timeout, Event, State) -> + handle_state_timeout(Event, hello, State); hello(Type, Event, State) -> ssl_connection:hello(Type, Event, State, ?MODULE). @@ -375,7 +378,11 @@ abbreviated(internal = Type, ConnectionStates = dtls_record:next_epoch(ConnectionStates1, read), ssl_connection:abbreviated(Type, Event, State#state{connection_states = ConnectionStates}, ?MODULE); abbreviated(internal = Type, #finished{} = Event, #state{connection_states = ConnectionStates} = State) -> - ssl_connection:cipher(Type, Event, prepare_flight(State#state{connection_states = ConnectionStates}), ?MODULE); + ssl_connection:abbreviated(Type, Event, + prepare_flight(State#state{connection_states = ConnectionStates, + flight_state = connection}), ?MODULE); +abbreviated(state_timeout, Event, State) -> + handle_state_timeout(Event, abbreviated, State); abbreviated(Type, Event, State) -> ssl_connection:abbreviated(Type, Event, State, ?MODULE). @@ -383,6 +390,8 @@ certify(info, Event, State) -> handle_info(Event, certify, State); certify(internal = Type, #server_hello_done{} = Event, State) -> ssl_connection:certify(Type, Event, prepare_flight(State), ?MODULE); +certify(state_timeout, Event, State) -> + handle_state_timeout(Event, certify, State); certify(Type, Event, State) -> ssl_connection:certify(Type, Event, State, ?MODULE). @@ -395,7 +404,11 @@ cipher(internal = Type, #change_cipher_spec{type = <<1>>} = Event, ssl_connection:cipher(Type, Event, State#state{connection_states = ConnectionStates}, ?MODULE); cipher(internal = Type, #finished{} = Event, #state{connection_states = ConnectionStates} = State) -> ssl_connection:cipher(Type, Event, - prepare_flight(State#state{connection_states = ConnectionStates}), ?MODULE); + prepare_flight(State#state{connection_states = ConnectionStates, + flight_state = connection}), + ?MODULE); +cipher(state_timeout, Event, State) -> + handle_state_timeout(Event, cipher, State); cipher(Type, Event, State) -> ssl_connection:cipher(Type, Event, State, ?MODULE). @@ -409,12 +422,12 @@ connection(internal, #hello_request{}, #state{host = Host, port = Port, renegotiation = {Renegotiation, _}} = State0) -> Hello = dtls_handshake:client_hello(Host, Port, ConnectionStates0, SslOpts, Cache, CacheCb, Renegotiation, Cert), - State1 = send_handshake(Hello, State0), + {State1, Actions} = send_handshake(Hello, State0), {Record, State} = next_record( State1#state{session = Session0#session{session_id = Hello#client_hello.session_id}}), - next_event(hello, Record, State); + next_event(hello, Record, State, Actions); connection(internal, #client_hello{} = Hello, #state{role = server, allow_renegotiate = true} = State) -> %% Mitigate Computational DoS attack %% http://www.educatedguesswork.org/2011/10/ssltls_and_computational_dos.html @@ -434,7 +447,6 @@ connection(Type, Event, State) -> downgrade(Type, Event, State) -> ssl_connection:downgrade(Type, Event, State, ?MODULE). - %%-------------------------------------------------------------------- %% Description: This function is called by a gen_fsm when it receives any %% other message than a synchronous or asynchronous event @@ -442,16 +454,6 @@ downgrade(Type, Event, State) -> %%-------------------------------------------------------------------- %% raw data from socket, unpack records -handle_info({_,flight_retransmission_timeout}, connection, _) -> - {next_state, keep_state_and_data}; -handle_info({Ref, flight_retransmission_timeout}, StateName, - #state{flight_state = {waiting, Ref, NextTimeout}} = State0) -> - State1 = send_handshake_flight(State0#state{flight_state = {retransmit_timer, NextTimeout}}, - retransmit_epoch(StateName, State0)), - {Record, State} = next_record(State1), - next_event(StateName, Record, State); -handle_info({_, flight_retransmission_timeout}, _, _) -> - {next_state, keep_state_and_data}; handle_info({Protocol, _, _, _, Data}, StateName, #state{data_tag = Protocol} = State0) -> case next_dtls_record(Data, State0) of @@ -480,16 +482,20 @@ handle_info({CloseTag, Socket}, StateName, end, ssl_connection:handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), StateName, State), {stop, {shutdown, transport_closed}}; +handle_info(new_cookie_secret, StateName, #state{protocol_specific = #{cookie_secret := Secret} = CookieInfo} = State) -> + erlang:send_after(dtls_v1:cookie_timeout(), self(), new_cookie_secret), + {next_state, StateName, State#state{protocol_specific = CookieInfo#{cookie_secret => dtls_v1:cookie_secret(), + previous_cookie_secret => Secret}}}; handle_info(Msg, StateName, State) -> ssl_connection:handle_info(Msg, StateName, State). + handle_call(Event, From, StateName, State) -> ssl_connection:handle_call(Event, From, StateName, State, ?MODULE). handle_common_event(internal, #alert{} = Alert, StateName, #state{negotiated_version = Version} = State) -> ssl_connection:handle_own_alert(Alert, Version, StateName, State); - %%% DTLS record protocol level handshake messages handle_common_event(internal, #ssl_tls{type = ?HANDSHAKE, fragment = Data}, @@ -498,19 +504,14 @@ handle_common_event(internal, #ssl_tls{type = ?HANDSHAKE, negotiated_version = Version} = State0) -> try case dtls_handshake:get_dtls_handshake(Version, Data, Buffers0) of - {more_data, Buffers} -> + {[], Buffers} -> {Record, State} = next_record(State0#state{protocol_buffers = Buffers}), next_event(StateName, Record, State); {Packets, Buffers} -> State = State0#state{protocol_buffers = Buffers}, Events = dtls_handshake_events(Packets), - case StateName of - connection -> - ssl_connection:hibernate_after(StateName, State, Events); - _ -> - {next_state, StateName, - State#state{unprocessed_handshake_events = unprocessed_events(Events)}, Events} - end + {next_state, StateName, + State#state{unprocessed_handshake_events = unprocessed_events(Events)}, Events} end catch throw:#alert{} = Alert -> ssl_connection:handle_own_alert(Alert, Version, StateName, State0) @@ -534,6 +535,13 @@ handle_common_event(internal, #ssl_tls{type = ?ALERT, fragment = EncAlerts}, Sta handle_common_event(internal, #ssl_tls{type = _Unknown}, StateName, State) -> {next_state, StateName, State}. +handle_state_timeout(flight_retransmission_timeout, StateName, + #state{flight_state = {retransmit, NextTimeout}} = State0) -> + {State1, Actions} = send_handshake_flight(State0#state{flight_state = {retransmit, NextTimeout}}, + retransmit_epoch(StateName, State0)), + {Record, State} = next_record(State1), + next_event(StateName, Record, State, Actions). + send(Transport, {_, {{_,_}, _} = Socket}, Data) -> send(Transport, Socket, Data); send(Transport, Socket, Data) -> @@ -645,7 +653,8 @@ initial_state(Role, Host, Port, Socket, {SSLOptions, SocketOptions, _}, User, allow_renegotiate = SSLOptions#ssl_options.client_renegotiation, start_or_recv_from = undefined, protocol_cb = ?MODULE, - flight_buffer = new_flight() + flight_buffer = new_flight(), + flight_state = {retransmit, ?INITIAL_RETRANSMIT_TIMEOUT} }. next_dtls_record(Data, #state{protocol_buffers = #protocol_buffers{ @@ -714,14 +723,14 @@ next_event(connection = StateName, no_record, #state{connection_states = #{current_read := #{epoch := CurrentEpoch}}} = State0, Actions) -> case next_record_if_active(State0) of {no_record, State} -> - ssl_connection:hibernate_after(StateName, State, Actions); + ssl_connection:hibernate_after(StateName, State, Actions); {#ssl_tls{epoch = CurrentEpoch} = Record, State} -> {next_state, StateName, State, [{next_event, internal, {protocol_record, Record}} | Actions]}; {#ssl_tls{epoch = Epoch, type = ?HANDSHAKE, version = _Version}, State1} = _Record when Epoch == CurrentEpoch-1 -> - State = send_handshake_flight(State1, Epoch), - {next_state, StateName, State, Actions}; + {State, MoreActions} = send_handshake_flight(State1, Epoch), + {next_state, StateName, State, Actions ++ MoreActions}; {#ssl_tls{epoch = _Epoch, version = _Version}, State} -> %% TODO maybe buffer later epoch @@ -772,17 +781,20 @@ next_flight(Flight) -> Flight#{handshakes => [], change_cipher_spec => undefined, handshakes_after_change_cipher_spec => []}. - start_flight(#state{transport_cb = gen_udp, - flight_state = {retransmit_timer, Timeout}} = State) -> - Ref = erlang:make_ref(), - _ = erlang:send_after(Timeout, self(), {Ref, flight_retransmission_timeout}), - State#state{flight_state = {waiting, Ref, new_timeout(Timeout)}}; - + flight_state = {retransmit, Timeout}} = State) -> + start_retransmision_timer(Timeout, State); +start_flight(#state{transport_cb = gen_udp, + flight_state = connection} = State) -> + {State, []}; start_flight(State) -> %% No retransmision needed i.e DTLS over SCTP - State#state{flight_state = reliable}. + {State#state{flight_state = reliable}, []}. + +start_retransmision_timer(Timeout, State) -> + {State#state{flight_state = {retransmit, new_timeout(Timeout)}}, + [{state_timeout, Timeout, flight_retransmission_timeout}]}. new_timeout(N) when N =< 30 -> N * 2; @@ -806,13 +818,13 @@ renegotiate(#state{role = server, connection_states = CS0} = State0, Actions) -> HelloRequest = ssl_handshake:hello_request(), CS = CS0#{write_msg_seq => 0}, - State1 = send_handshake(HelloRequest, - State0#state{connection_states = - CS}), + {State1, MoreActions} = send_handshake(HelloRequest, + State0#state{connection_states = + CS}), Hs0 = ssl_handshake:init_handshake_history(), {Record, State} = next_record(State1#state{tls_handshake_history = Hs0, protocol_buffers = #protocol_buffers{}}), - next_event(hello, Record, State, Actions). + next_event(hello, Record, State, Actions ++ MoreActions). handle_alerts([], Result) -> Result; @@ -823,15 +835,11 @@ handle_alerts([Alert | Alerts], {next_state, StateName, State}) -> handle_alerts([Alert | Alerts], {next_state, StateName, State, _Actions}) -> handle_alerts(Alerts, ssl_connection:handle_alert(Alert, StateName, State)). -retransmit_epoch(StateName, #state{connection_states = ConnectionStates}) -> +retransmit_epoch(_StateName, #state{connection_states = ConnectionStates}) -> #{epoch := Epoch} = ssl_record:current_connection_state(ConnectionStates, write), - case StateName of - connection -> - Epoch-1; - _ -> - Epoch - end. + Epoch. + update_handshake_history(#hello_verify_request{}, _, Hist) -> Hist; @@ -846,3 +854,4 @@ unprocessed_events(Events) -> %% handshake events left to process before we should %% process more TLS-records received on the socket. erlang:length(Events)-1. + diff --git a/lib/ssl/src/dtls_handshake.erl b/lib/ssl/src/dtls_handshake.erl index af3708ddb7..d3ba90a226 100644 --- a/lib/ssl/src/dtls_handshake.erl +++ b/lib/ssl/src/dtls_handshake.erl @@ -136,9 +136,11 @@ handshake_bin([Type, Length, Data], Seq) -> %%-------------------------------------------------------------------- -spec get_dtls_handshake(dtls_record:dtls_version(), binary(), #protocol_buffers{}) -> - {[{dtls_handshake(), binary()}], #protocol_buffers{}} | {more_data, #protocol_buffers{}}. + {[dtls_handshake()], #protocol_buffers{}}. %% -%% Description: ... +%% Description: Given buffered and new data from dtls_record, collects +%% and returns it as a list of handshake messages, also returns +%% possible leftover data in the new "protocol_buffers". %%-------------------------------------------------------------------- get_dtls_handshake(Version, Fragment, ProtocolBuffers) -> handle_fragments(Version, Fragment, ProtocolBuffers, []). @@ -288,8 +290,6 @@ do_handle_fragments(_, [], Buffers, Acc) -> {lists:reverse(Acc), Buffers}; do_handle_fragments(Version, [Fragment | Fragments], Buffers0, Acc) -> case reassemble(Version, Fragment, Buffers0) of - {more_data, _} = More when Acc == []-> - More; {more_data, Buffers} when Fragments == [] -> {lists:reverse(Acc), Buffers}; {more_data, Buffers} -> @@ -455,7 +455,7 @@ merge_fragments(#handshake_fragment{ fragment_offset = PreviousOffSet, fragment_length = CurrentLen}) when CurrentLen < PreviousLen -> Previous; -%% Next fragment +%% Next fragment, might be overlapping merge_fragments(#handshake_fragment{ fragment_offset = PreviousOffSet, fragment_length = PreviousLen, @@ -464,10 +464,26 @@ merge_fragments(#handshake_fragment{ #handshake_fragment{ fragment_offset = CurrentOffSet, fragment_length = CurrentLen, - fragment = CurrentData}) when PreviousOffSet + PreviousLen == CurrentOffSet-> - Previous#handshake_fragment{ - fragment_length = PreviousLen + CurrentLen, - fragment = <<PreviousData/binary, CurrentData/binary>>}; + fragment = CurrentData}) + when PreviousOffSet + PreviousLen >= CurrentOffSet andalso + PreviousOffSet + PreviousLen < CurrentOffSet + CurrentLen -> + CurrentStart = PreviousOffSet + PreviousLen - CurrentOffSet, + <<_:CurrentStart/bytes, Data/binary>> = CurrentData, + Previous#handshake_fragment{ + fragment_length = PreviousLen + CurrentLen - CurrentStart, + fragment = <<PreviousData/binary, Data/binary>>}; +%% already fully contained fragment +merge_fragments(#handshake_fragment{ + fragment_offset = PreviousOffSet, + fragment_length = PreviousLen + } = Previous, + #handshake_fragment{ + fragment_offset = CurrentOffSet, + fragment_length = CurrentLen}) + when PreviousOffSet + PreviousLen >= CurrentOffSet andalso + PreviousOffSet + PreviousLen >= CurrentOffSet + CurrentLen -> + Previous; + %% No merge there is a gap merge_fragments(Previous, Current) -> [Previous, Current]. diff --git a/lib/ssl/src/dtls_socket.erl b/lib/ssl/src/dtls_socket.erl index 570b3ae83a..ac1a7b37c6 100644 --- a/lib/ssl/src/dtls_socket.erl +++ b/lib/ssl/src/dtls_socket.erl @@ -71,11 +71,14 @@ connect(Address, Port, #config{transport_info = {Transport, _, _, _} = CbInfo, close(gen_udp, {_Client, _Socket}) -> ok. +socket(Pid, gen_udp = Transport, {{_, _}, Socket}, ConnectionCb) -> + #sslsocket{pid = Pid, + %% "The name "fd" is keept for backwards compatibility + fd = {Transport, Socket, ConnectionCb}}; socket(Pid, Transport, Socket, ConnectionCb) -> #sslsocket{pid = Pid, %% "The name "fd" is keept for backwards compatibility - fd = {Transport, Socket, ConnectionCb}}. - + fd = {Transport, Socket, ConnectionCb}}. %% Vad göra med emulerade setopts(gen_udp, #sslsocket{pid = {Socket, _}}, Options) -> {SockOpts, _} = tls_socket:split_options(Options), @@ -108,11 +111,15 @@ getstat(gen_udp, {_,Socket}, Options) -> inet:getstat(Socket, Options); getstat(Transport, Socket, Options) -> Transport:getstat(Socket, Options). +peername(udp, _) -> + {error, enotconn}; peername(gen_udp, {_, {Client, _Socket}}) -> {ok, Client}; peername(Transport, Socket) -> Transport:peername(Socket). -sockname(gen_udp, {_,Socket}) -> +sockname(gen_udp, {_, {_,Socket}}) -> + inet:sockname(Socket); +sockname(gen_udp, Socket) -> inet:sockname(Socket); sockname(Transport, Socket) -> Transport:sockname(Socket). diff --git a/lib/ssl/src/dtls_udp_listener.erl b/lib/ssl/src/dtls_udp_listener.erl index b7f115582e..ab3d0783bd 100644 --- a/lib/ssl/src/dtls_udp_listener.erl +++ b/lib/ssl/src/dtls_udp_listener.erl @@ -24,7 +24,8 @@ -behaviour(gen_server). %% API --export([start_link/4, active_once/3, accept/2, sockname/1]). +-export([start_link/4, active_once/3, accept/2, sockname/1, close/1, + get_all_opts/1]). %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, @@ -39,7 +40,8 @@ clients = set_new(), dtls_processes = kv_new(), accepters = queue:new(), - first + first, + close }). %%%=================================================================== @@ -53,10 +55,14 @@ active_once(UDPConnection, Client, Pid) -> gen_server:cast(UDPConnection, {active_once, Client, Pid}). accept(UDPConnection, Accepter) -> - gen_server:call(UDPConnection, {accept, Accepter}, infinity). + call(UDPConnection, {accept, Accepter}). sockname(UDPConnection) -> - gen_server:call(UDPConnection, sockname, infinity). + call(UDPConnection, sockname). +close(UDPConnection) -> + call(UDPConnection, close). +get_all_opts(UDPConnection) -> + call(UDPConnection, get_all_opts). %%%=================================================================== %%% gen_server callbacks @@ -69,10 +75,13 @@ init([Port, EmOpts, InetOptions, DTLSOptions]) -> first = true, dtls_options = DTLSOptions, emulated_options = EmOpts, - listner = Socket}} + listner = Socket, + close = false}} catch _:_ -> {error, closed} end. +handle_call({accept, _}, _, #state{close = true} = State) -> + {reply, {error, closed}, State}; handle_call({accept, Accepter}, From, #state{first = true, accepters = Accepters, @@ -87,7 +96,21 @@ handle_call({accept, Accepter}, From, #state{accepters = Accepters} = State0) -> {noreply, State}; handle_call(sockname, _, #state{listner = Socket} = State) -> Reply = inet:sockname(Socket), - {reply, Reply, State}. + {reply, Reply, State}; +handle_call(close, _, #state{dtls_processes = Processes, + accepters = Accepters} = State) -> + case kv_empty(Processes) of + true -> + {stop, normal, ok, State#state{close=true}}; + false -> + lists:foreach(fun({_, From}) -> + gen_server:reply(From, {error, closed}) + end, queue:to_list(Accepters)), + {reply, ok, State#state{close = true, accepters = queue:new()}} + end; +handle_call(get_all_opts, _, #state{dtls_options = DTLSOptions, + emulated_options = EmOpts} = State) -> + {reply, {ok, EmOpts, DTLSOptions}, State}. handle_cast({active_once, Client, Pid}, State0) -> State = handle_active_once(Client, Pid, State0), @@ -99,11 +122,17 @@ handle_info({udp, Socket, IP, InPortNo, _} = Msg, #state{listner = Socket} = Sta {noreply, State}; handle_info({'DOWN', _, process, Pid, _}, #state{clients = Clients, - dtls_processes = Processes0} = State) -> + dtls_processes = Processes0, + close = ListenClosed} = State) -> Client = kv_get(Pid, Processes0), Processes = kv_delete(Pid, Processes0), - {noreply, State#state{clients = set_delete(Client, Clients), - dtls_processes = Processes}}. + case ListenClosed andalso kv_empty(Processes) of + true -> + {stop, normal, State}; + false -> + {noreply, State#state{clients = set_delete(Client, Clients), + dtls_processes = Processes}} + end. terminate(_Reason, _State) -> ok. @@ -182,6 +211,7 @@ setup_new_connection(User, From, Client, Msg, #state{dtls_processes = Processes, gen_server:reply(From, {error, Reason}), State end. + kv_update(Key, Value, Store) -> gb_trees:update(Key, Value, Store). kv_lookup(Key, Store) -> @@ -194,6 +224,8 @@ kv_delete(Key, Store) -> gb_trees:delete(Key, Store). kv_new() -> gb_trees:empty(). +kv_empty(Store) -> + gb_trees:is_empty(Store). set_new() -> gb_sets:empty(). @@ -203,3 +235,15 @@ set_delete(Item, Set) -> gb_sets:delete(Item, Set). set_is_member(Item, Set) -> gb_sets:is_member(Item, Set). + +call(Server, Msg) -> + try + gen_server:call(Server, Msg, infinity) + catch + exit:{noproc, _} -> + {error, closed}; + exit:{normal, _} -> + {error, closed}; + exit:{{shutdown, _},_} -> + {error, closed} + end. diff --git a/lib/ssl/src/dtls_v1.erl b/lib/ssl/src/dtls_v1.erl index ffd3e4b833..4aaf8baa6c 100644 --- a/lib/ssl/src/dtls_v1.erl +++ b/lib/ssl/src/dtls_v1.erl @@ -21,12 +21,24 @@ -include("ssl_cipher.hrl"). --export([suites/1, mac_hash/7, ecc_curves/1, corresponding_tls_version/1, corresponding_dtls_version/1]). +-export([suites/1, all_suites/1, mac_hash/7, ecc_curves/1, + corresponding_tls_version/1, corresponding_dtls_version/1, + cookie_secret/0, cookie_timeout/0]). + +-define(COOKIE_BASE_TIMEOUT, 30000). -spec suites(Minor:: 253|255) -> [ssl_cipher:cipher_suite()]. suites(Minor) -> - tls_v1:suites(corresponding_minor_tls_version(Minor)). + lists:filter(fun(Cipher) -> + is_acceptable_cipher(ssl_cipher:suite_definition(Cipher)) + end, + tls_v1:suites(corresponding_minor_tls_version(Minor))). +all_suites(Version) -> + lists:filter(fun(Cipher) -> + is_acceptable_cipher(ssl_cipher:suite_definition(Cipher)) + end, + ssl_cipher:all_suites(corresponding_tls_version(Version))). mac_hash(Version, MacAlg, MacSecret, SeqNo, Type, Length, Fragment) -> tls_v1:mac_hash(MacAlg, MacSecret, SeqNo, Type, Version, @@ -38,6 +50,13 @@ ecc_curves({_Major, Minor}) -> corresponding_tls_version({254, Minor}) -> {3, corresponding_minor_tls_version(Minor)}. +cookie_secret() -> + crypto:strong_rand_bytes(32). + +cookie_timeout() -> + %% Cookie will live for two timeouts periods + round(rand:uniform() * ?COOKIE_BASE_TIMEOUT/2). + corresponding_minor_tls_version(255) -> 2; corresponding_minor_tls_version(253) -> @@ -50,3 +69,5 @@ corresponding_minor_dtls_version(2) -> 255; corresponding_minor_dtls_version(3) -> 253. +is_acceptable_cipher(Suite) -> + not ssl_cipher:is_stream_ciphersuite(Suite). diff --git a/lib/ssl/src/ssl.app.src b/lib/ssl/src/ssl.app.src index 148989174d..064dcd6892 100644 --- a/lib/ssl/src/ssl.app.src +++ b/lib/ssl/src/ssl.app.src @@ -63,7 +63,7 @@ {applications, [crypto, public_key, kernel, stdlib]}, {env, []}, {mod, {ssl_app, []}}, - {runtime_dependencies, ["stdlib-3.1","public_key-1.2","kernel-3.0", + {runtime_dependencies, ["stdlib-3.2","public_key-1.2","kernel-3.0", "erts-7.0","crypto-3.3", "inets-5.10.7"]}]}. diff --git a/lib/ssl/src/ssl.appup.src b/lib/ssl/src/ssl.appup.src index 32252386b4..2eda9d9491 100644 --- a/lib/ssl/src/ssl.appup.src +++ b/lib/ssl/src/ssl.appup.src @@ -1,11 +1,21 @@ %% -*- erlang -*- {"%VSN%", [ - {<<"^8[.]0([.][0-9]+)?$">>, [{restart_application, ssl}]}, - {<<"^[3-7][.][^.].*">>, [{restart_application, ssl}]} + {<<"8.1.1">>, [{load_module, tls_connection, soft_purge, soft_purge, []}]}, + {<<"8\\..*">>, [{restart_application, ssl}]}, + {<<"7\\..*">>, [{restart_application, ssl}]}, + {<<"6\\..*">>, [{restart_application, ssl}]}, + {<<"5\\..*">>, [{restart_application, ssl}]}, + {<<"4\\..*">>, [{restart_application, ssl}]}, + {<<"3\\..*">>, [{restart_application, ssl}]} ], [ - {<<"^8[.]0([.][0-9]+)?$">>, [{restart_application, ssl}]}, - {<<"^[3-7][.][^.].*">>, [{restart_application, ssl}]} - ] + {<<"8.1.1">>, [{load_module, tls_connection, soft_purge, soft_purge, []}]}, + {<<"8\\..*">>, [{restart_application, ssl}]}, + {<<"7\\..*">>, [{restart_application, ssl}]}, + {<<"6\\..*">>, [{restart_application, ssl}]}, + {<<"5\\..*">>, [{restart_application, ssl}]}, + {<<"4\\..*">>, [{restart_application, ssl}]}, + {<<"3\\..*">>, [{restart_application, ssl}]} + ] }. diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index 4a5a7e25ea..b3d08bdfbe 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -38,16 +38,13 @@ getopts/2, setopts/2, getstat/1, getstat/2 ]). %% SSL/TLS protocol handling --export([cipher_suites/0, cipher_suites/1, eccs/0, eccs/1, - connection_info/1, versions/0, session_info/1, format_error/1, - renegotiate/1, prf/5, negotiated_protocol/1, negotiated_next_protocol/1, + +-export([cipher_suites/0, cipher_suites/1, eccs/0, eccs/1, versions/0, + format_error/1, renegotiate/1, prf/5, negotiated_protocol/1, connection_information/1, connection_information/2]). %% Misc -export([handle_options/2, tls_version/1]). --deprecated({negotiated_next_protocol, 1, next_major_release}). --deprecated({connection_info, 1, next_major_release}). - -include("ssl_api.hrl"). -include("ssl_internal.hrl"). -include("ssl_record.hrl"). @@ -187,16 +184,24 @@ ssl_accept(ListenSocket, SslOptions) when is_port(ListenSocket) -> ssl_accept(#sslsocket{} = Socket, [], Timeout) when (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity)-> ssl_accept(Socket, Timeout); -ssl_accept(#sslsocket{fd = {_, _, _, Tracker}} = Socket, SslOpts0, Timeout) when +ssl_accept(#sslsocket{fd = {_, _, _, Tracker}} = Socket, SslOpts, Timeout) when (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity)-> try - {ok, EmOpts, InheritedSslOpts} = tls_socket:get_all_opts(Tracker), - SslOpts = handle_options(SslOpts0, InheritedSslOpts), + {ok, EmOpts, _} = tls_socket:get_all_opts(Tracker), ssl_connection:handshake(Socket, {SslOpts, tls_socket:emulated_socket_options(EmOpts, #socket_options{})}, Timeout) catch Error = {error, _Reason} -> Error end; +ssl_accept(#sslsocket{pid = Pid, fd = {_, _, _}} = Socket, SslOpts, Timeout) when + (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity)-> + try + {ok, EmOpts, _} = dtls_udp_listener:get_all_opts(Pid), + ssl_connection:handshake(Socket, {SslOpts, + tls_socket:emulated_socket_options(EmOpts, #socket_options{})}, Timeout) + catch + Error = {error, _Reason} -> Error + end; ssl_accept(Socket, SslOptions, Timeout) when is_port(Socket), (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> {Transport,_,_,_} = @@ -215,7 +220,6 @@ ssl_accept(Socket, SslOptions, Timeout) when is_port(Socket), catch Error = {error, _Reason} -> Error end. - %%-------------------------------------------------------------------- -spec close(#sslsocket{}) -> term(). %% @@ -223,6 +227,8 @@ ssl_accept(Socket, SslOptions, Timeout) when is_port(Socket), %%-------------------------------------------------------------------- close(#sslsocket{pid = Pid}) when is_pid(Pid) -> ssl_connection:close(Pid, {close, ?DEFAULT_TIMEOUT}); +close(#sslsocket{pid = {udp, #config{udp_handler = {Pid, _}}}}) -> + dtls_udp_listener:close(Pid); close(#sslsocket{pid = {ListenSocket, #config{transport_info={Transport,_, _, _}}}}) -> Transport:close(ListenSocket). @@ -251,6 +257,8 @@ send(#sslsocket{pid = Pid}, Data) when is_pid(Pid) -> ssl_connection:send(Pid, Data); send(#sslsocket{pid = {_, #config{transport_info={gen_udp, _, _, _}}}}, _) -> {error,enotconn}; %% Emulate connection behaviour +send(#sslsocket{pid = {udp,_}}, _) -> + {error,enotconn}; send(#sslsocket{pid = {ListenSocket, #config{transport_info={Transport, _, _, _}}}}, Data) -> Transport:send(ListenSocket, Data). %% {error,enotconn} @@ -265,6 +273,8 @@ recv(Socket, Length) -> recv(#sslsocket{pid = Pid}, Length, Timeout) when is_pid(Pid), (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity)-> ssl_connection:recv(Pid, Length, Timeout); +recv(#sslsocket{pid = {udp,_}}, _, _) -> + {error,enotconn}; recv(#sslsocket{pid = {Listen, #config{transport_info = {Transport, _, _, _}}}}, _,_) when is_port(Listen)-> Transport:recv(Listen, 0). %% {error,enotconn} @@ -277,10 +287,14 @@ recv(#sslsocket{pid = {Listen, %%-------------------------------------------------------------------- controlling_process(#sslsocket{pid = Pid}, NewOwner) when is_pid(Pid), is_pid(NewOwner) -> ssl_connection:new_user(Pid, NewOwner); +controlling_process(#sslsocket{pid = {udp, _}}, + NewOwner) when is_pid(NewOwner) -> + ok; %% Meaningless but let it be allowed to conform with TLS controlling_process(#sslsocket{pid = {Listen, #config{transport_info = {Transport, _, _, _}}}}, NewOwner) when is_port(Listen), is_pid(NewOwner) -> + %% Meaningless but let it be allowed to conform with normal sockets Transport:controlling_process(Listen, NewOwner). @@ -290,22 +304,24 @@ controlling_process(#sslsocket{pid = {Listen, %% Description: Return SSL information for the connection %%-------------------------------------------------------------------- connection_information(#sslsocket{pid = Pid}) when is_pid(Pid) -> - case ssl_connection:connection_information(Pid) of + case ssl_connection:connection_information(Pid, false) of {ok, Info} -> {ok, [Item || Item = {_Key, Value} <- Info, Value =/= undefined]}; Error -> Error end; connection_information(#sslsocket{pid = {Listen, _}}) when is_port(Listen) -> - {error, enotconn}. + {error, enotconn}; +connection_information(#sslsocket{pid = {udp,_}}) -> + {error,enotconn}. %%-------------------------------------------------------------------- -spec connection_information(#sslsocket{}, [atom()]) -> {ok, list()} | {error, reason()}. %% %% Description: Return SSL information for the connection %%-------------------------------------------------------------------- -connection_information(#sslsocket{} = SSLSocket, Items) -> - case connection_information(SSLSocket) of +connection_information(#sslsocket{pid = Pid}, Items) when is_pid(Pid) -> + case ssl_connection:connection_information(Pid, include_security_info(Items)) of {ok, Info} -> {ok, [Item || Item = {Key, Value} <- Info, lists:member(Key, Items), Value =/= undefined]}; @@ -314,29 +330,22 @@ connection_information(#sslsocket{} = SSLSocket, Items) -> end. %%-------------------------------------------------------------------- -%% Deprecated --spec connection_info(#sslsocket{}) -> {ok, {tls_record:tls_atom_version(), ssl_cipher:erl_cipher_suite()}} | - {error, reason()}. -%% -%% Description: Returns ssl protocol and cipher used for the connection -%%-------------------------------------------------------------------- -connection_info(#sslsocket{} = SSLSocket) -> - case connection_information(SSLSocket) of - {ok, Result} -> - {ok, {proplists:get_value(protocol, Result), proplists:get_value(cipher_suite, Result)}}; - Error -> - Error - end. - -%%-------------------------------------------------------------------- -spec peername(#sslsocket{}) -> {ok, {inet:ip_address(), inet:port_number()}} | {error, reason()}. %% %% Description: same as inet:peername/1. %%-------------------------------------------------------------------- +peername(#sslsocket{pid = Pid, fd = {Transport, Socket, _}}) when is_pid(Pid)-> + dtls_socket:peername(Transport, Socket); peername(#sslsocket{pid = Pid, fd = {Transport, Socket, _, _}}) when is_pid(Pid)-> tls_socket:peername(Transport, Socket); +peername(#sslsocket{pid = {udp = Transport, #config{udp_handler = {_Pid, _}}}}) -> + dtls_socket:peername(Transport, undefined); +peername(#sslsocket{pid = Pid, fd = {gen_udp= Transport, Socket, _, _}}) when is_pid(Pid) -> + dtls_socket:peername(Transport, Socket); peername(#sslsocket{pid = {ListenSocket, #config{transport_info = {Transport,_,_,_}}}}) -> - tls_socket:peername(Transport, ListenSocket). %% Will return {error, enotconn} + tls_socket:peername(Transport, ListenSocket); %% Will return {error, enotconn} +peername(#sslsocket{pid = {udp,_}}) -> + {error,enotconn}. %%-------------------------------------------------------------------- -spec peercert(#sslsocket{}) ->{ok, DerCert::binary()} | {error, reason()}. @@ -350,6 +359,8 @@ peercert(#sslsocket{pid = Pid}) when is_pid(Pid) -> Result -> Result end; +peercert(#sslsocket{pid = {udp, _}}) -> + {error, enotconn}; peercert(#sslsocket{pid = {Listen, _}}) when is_port(Listen) -> {error, enotconn}. @@ -363,20 +374,6 @@ negotiated_protocol(#sslsocket{pid = Pid}) -> ssl_connection:negotiated_protocol(Pid). %%-------------------------------------------------------------------- --spec negotiated_next_protocol(#sslsocket{}) -> {ok, binary()} | {error, reason()}. -%% -%% Description: Returns the next protocol that has been negotiated. If no -%% protocol has been negotiated will return {error, next_protocol_not_negotiated} -%%-------------------------------------------------------------------- -negotiated_next_protocol(Socket) -> - case negotiated_protocol(Socket) of - {error, protocol_not_negotiated} -> - {error, next_protocol_not_negotiated}; - Res -> - Res - end. - -%%-------------------------------------------------------------------- -spec cipher_suites() -> [ssl_cipher:erl_cipher_suite()] | [string()]. %%-------------------------------------------------------------------- cipher_suites() -> @@ -506,6 +503,8 @@ getstat(#sslsocket{pid = Pid, fd = {Transport, Socket, _, _}}, Options) when is_ shutdown(#sslsocket{pid = {Listen, #config{transport_info = {Transport,_, _, _}}}}, How) when is_port(Listen) -> Transport:shutdown(Listen, How); +shutdown(#sslsocket{pid = {udp,_}},_) -> + {error, enotconn}; shutdown(#sslsocket{pid = Pid}, How) -> ssl_connection:shutdown(Pid, How). @@ -518,23 +517,12 @@ sockname(#sslsocket{pid = {Listen, #config{transport_info = {Transport, _, _, _ tls_socket:sockname(Transport, Listen); sockname(#sslsocket{pid = {udp, #config{udp_handler = {Pid, _}}}}) -> dtls_udp_listener:sockname(Pid); -sockname(#sslsocket{pid = Pid, fd = {gen_udp= Transport, Socket, _, _}}) when is_pid(Pid) -> +sockname(#sslsocket{pid = Pid, fd = {Transport, Socket, _}}) when is_pid(Pid) -> dtls_socket:sockname(Transport, Socket); sockname(#sslsocket{pid = Pid, fd = {Transport, Socket, _, _}}) when is_pid(Pid) -> tls_socket:sockname(Transport, Socket). %%--------------------------------------------------------------- --spec session_info(#sslsocket{}) -> {ok, list()} | {error, reason()}. -%% -%% Description: Returns list of session info currently [{session_id, session_id(), -%% {cipher_suite, cipher_suite()}] -%%-------------------------------------------------------------------- -session_info(#sslsocket{pid = Pid}) when is_pid(Pid) -> - ssl_connection:session_info(Pid); -session_info(#sslsocket{pid = {Listen,_}}) when is_port(Listen) -> - {error, enotconn}. - -%%--------------------------------------------------------------- -spec versions() -> [{ssl_app, string()} | {supported, [tls_record:tls_atom_version()]} | {available, [tls_record:tls_atom_version()]}]. %% @@ -555,6 +543,8 @@ versions() -> %%-------------------------------------------------------------------- renegotiate(#sslsocket{pid = Pid}) when is_pid(Pid) -> ssl_connection:renegotiation(Pid); +renegotiate(#sslsocket{pid = {udp,_}}) -> + {error, enotconn}; renegotiate(#sslsocket{pid = {Listen,_}}) when is_port(Listen) -> {error, enotconn}. @@ -568,6 +558,8 @@ renegotiate(#sslsocket{pid = {Listen,_}}) when is_port(Listen) -> prf(#sslsocket{pid = Pid}, Secret, Label, Seed, WantedLength) when is_pid(Pid) -> ssl_connection:prf(Pid, Secret, Label, Seed, WantedLength); +prf(#sslsocket{pid = {udp,_}}, _,_,_,_) -> + {error, enotconn}; prf(#sslsocket{pid = {Listen,_}}, _,_,_,_) when is_port(Listen) -> {error, enotconn}. @@ -696,7 +688,7 @@ handle_options(Opts0, Role) -> [RecordCb:protocol_version(Vsn) || Vsn <- Vsns] end, - Protocol = proplists:get_value(protocol, Opts, tls), + Protocol = handle_option(protocol, Opts, tls), SSLOptions = #ssl_options{ versions = Versions, @@ -755,7 +747,7 @@ handle_options(Opts0, Role) -> honor_ecc_order = handle_option(honor_ecc_order, Opts, default_option_role(server, false, Role), server, Role), - protocol = Protocol, + protocol = Protocol, padding_check = proplists:get_value(padding_check, Opts, true), beast_mitigation = handle_option(beast_mitigation, Opts, one_n_minus_one), fallback = handle_option(fallback, Opts, @@ -1032,6 +1024,10 @@ validate_option(v2_hello_compatible, Value) when is_boolean(Value) -> Value; validate_option(max_handshake_size, Value) when is_integer(Value) andalso Value =< ?MAX_UNIT24 -> Value; +validate_option(protocol, Value = tls) -> + Value; +validate_option(protocol, Value = dtls) -> + Value; validate_option(Opt, Value) -> throw({error, {options, {Opt, Value}}}). @@ -1069,17 +1065,37 @@ validate_binary_list(Opt, List) -> (Bin) -> throw({error, {options, {Opt, {invalid_protocol, Bin}}}}) end, List). - validate_versions([], Versions) -> Versions; validate_versions([Version | Rest], Versions) when Version == 'tlsv1.2'; Version == 'tlsv1.1'; Version == tlsv1; Version == sslv3 -> - validate_versions(Rest, Versions); + tls_validate_versions(Rest, Versions); +validate_versions([Version | Rest], Versions) when Version == 'dtlsv1'; + Version == 'dtlsv1.2'-> + dtls_validate_versions(Rest, Versions); validate_versions([Ver| _], Versions) -> throw({error, {options, {Ver, {versions, Versions}}}}). +tls_validate_versions([], Versions) -> + Versions; +tls_validate_versions([Version | Rest], Versions) when Version == 'tlsv1.2'; + Version == 'tlsv1.1'; + Version == tlsv1; + Version == sslv3 -> + tls_validate_versions(Rest, Versions); +tls_validate_versions([Ver| _], Versions) -> + throw({error, {options, {Ver, {versions, Versions}}}}). + +dtls_validate_versions([], Versions) -> + Versions; +dtls_validate_versions([Version | Rest], Versions) when Version == 'dtlsv1'; + Version == 'dtlsv1.2'-> + dtls_validate_versions(Rest, Versions); +dtls_validate_versions([Ver| _], Versions) -> + throw({error, {options, {Ver, {versions, Versions}}}}). + validate_inet_option(mode, Value) when Value =/= list, Value =/= binary -> throw({error, {options, {mode,Value}}}); @@ -1151,18 +1167,18 @@ handle_cipher_option(Value, Version) when is_list(Value) -> binary_cipher_suites(Version, []) -> %% Defaults to all supported suites that does %% not require explicit configuration - ssl_cipher:filter_suites(ssl_cipher:suites(Version)); + ssl_cipher:filter_suites(ssl_cipher:suites(tls_version(Version))); binary_cipher_suites(Version, [Tuple|_] = Ciphers0) when is_tuple(Tuple) -> Ciphers = [ssl_cipher:suite(C) || C <- Ciphers0], binary_cipher_suites(Version, Ciphers); binary_cipher_suites(Version, [Cipher0 | _] = Ciphers0) when is_binary(Cipher0) -> - All = ssl_cipher:all_suites(Version), + All = ssl_cipher:all_suites(tls_version(Version)), case [Cipher || Cipher <- Ciphers0, lists:member(Cipher, All)] of [] -> %% Defaults to all supported suites that does %% not require explicit configuration - ssl_cipher:filter_suites(ssl_cipher:suites(Version)); + ssl_cipher:filter_suites(ssl_cipher:suites(tls_version(Version))); Ciphers -> Ciphers end; @@ -1175,7 +1191,8 @@ binary_cipher_suites(Version, Ciphers0) -> Ciphers = [ssl_cipher:openssl_suite(C) || C <- string:tokens(Ciphers0, ":")], binary_cipher_suites(Version, Ciphers). -handle_eccs_option(Value, {_Major, Minor}) when is_list(Value) -> +handle_eccs_option(Value, Version) when is_list(Value) -> + {_Major, Minor} = tls_version(Version), try tls_v1:ecc_curves(Minor, Value) of Curves -> #elliptic_curves{elliptic_curve_list = Curves} catch @@ -1348,7 +1365,10 @@ new_ssl_options([{signature_algs, Value} | Rest], #ssl_options{} = Opts, RecordC handle_hashsigns_option(Value, tls_version(RecordCB:highest_protocol_version()))}, RecordCB); - +new_ssl_options([{protocol, dtls = Value} | Rest], #ssl_options{} = Opts, dtls_record = RecordCB) -> + new_ssl_options(Rest, Opts#ssl_options{protocol = Value}, RecordCB); +new_ssl_options([{protocol, tls = Value} | Rest], #ssl_options{} = Opts, tls_record = RecordCB) -> + new_ssl_options(Rest, Opts#ssl_options{protocol = Value}, RecordCB); new_ssl_options([{Key, Value} | _Rest], #ssl_options{}, _) -> throw({error, {options, {Key, Value}}}). @@ -1415,3 +1435,13 @@ default_cb_info(tls) -> {gen_tcp, tcp, tcp_closed, tcp_error}; default_cb_info(dtls) -> {gen_udp, udp, udp_closed, udp_error}. + +include_security_info([]) -> + false; +include_security_info([Item | Items]) -> + case lists:member(Item, [client_random, server_random, master_secret]) of + true -> + true; + false -> + include_security_info(Items) + end. diff --git a/lib/ssl/src/ssl_cipher.erl b/lib/ssl/src/ssl_cipher.erl index 32fec03b8e..8e6860e9dc 100644 --- a/lib/ssl/src/ssl_cipher.erl +++ b/lib/ssl/src/ssl_cipher.erl @@ -40,7 +40,8 @@ ec_keyed_suites/0, anonymous_suites/1, psk_suites/1, srp_suites/0, rc4_suites/1, des_suites/1, openssl_suite/1, openssl_suite_name/1, filter/2, filter_suites/1, hash_algorithm/1, sign_algorithm/1, is_acceptable_hash/2, is_fallback/1, - random_bytes/1, calc_aad/3, calc_mac_hash/4]). + random_bytes/1, calc_aad/3, calc_mac_hash/4, + is_stream_ciphersuite/1]). -export_type([cipher_suite/0, erl_cipher_suite/0, openssl_cipher_suite/0, @@ -310,18 +311,21 @@ aead_decipher(Type, #cipher_state{key = Key, iv = IV} = CipherState, %%-------------------------------------------------------------------- suites({3, 0}) -> ssl_v3:suites(); -suites({3, N}) -> - tls_v1:suites(N); -suites(Version) -> - suites(dtls_v1:corresponding_tls_version(Version)). +suites({3, Minor}) -> + tls_v1:suites(Minor); +suites({_, Minor}) -> + dtls_v1:suites(Minor). -all_suites(Version) -> +all_suites({3, _} = Version) -> suites(Version) ++ anonymous_suites(Version) ++ psk_suites(Version) ++ srp_suites() ++ rc4_suites(Version) - ++ des_suites(Version). + ++ des_suites(Version); +all_suites(Version) -> + dtls_v1:all_suites(Version). + %%-------------------------------------------------------------------- -spec anonymous_suites(ssl_record:ssl_version() | integer()) -> [cipher_suite()]. %% @@ -1541,6 +1545,10 @@ calc_mac_hash(Type, Version, MacSecret, SeqNo, Type, Length, PlainFragment). +is_stream_ciphersuite({_, rc4_128, _, _}) -> + true; +is_stream_ciphersuite(_) -> + false. %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index 0c17891fbc..df9b9e8a63 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -42,9 +42,9 @@ %% User Events -export([send/2, recv/3, close/2, shutdown/2, - new_user/2, get_opts/2, set_opts/2, session_info/1, + new_user/2, get_opts/2, set_opts/2, peer_certificate/1, renegotiation/1, negotiated_protocol/1, prf/5, - connection_information/1, handle_common_event/5 + connection_information/2, handle_common_event/5 ]). %% General gen_statem state functions with extra callback argument @@ -148,19 +148,19 @@ socket_control(Connection, Socket, Pid, Transport) -> %%-------------------------------------------------------------------- socket_control(Connection, Socket, Pid, Transport, udp_listner) -> %% dtls listner process must have the socket control - {ok, dtls_socket:socket(Pid, Transport, Socket, Connection)}; + {ok, Connection:socket(Pid, Transport, Socket, Connection, undefined)}; socket_control(tls_connection = Connection, Socket, Pid, Transport, ListenTracker) -> case Transport:controlling_process(Socket, Pid) of ok -> - {ok, tls_socket:socket(Pid, Transport, Socket, Connection, ListenTracker)}; + {ok, Connection:socket(Pid, Transport, Socket, Connection, ListenTracker)}; {error, Reason} -> {error, Reason} end; socket_control(dtls_connection = Connection, {_, Socket}, Pid, Transport, ListenTracker) -> case Transport:controlling_process(Socket, Pid) of ok -> - {ok, tls_socket:socket(Pid, Transport, Socket, Connection, ListenTracker)}; + {ok, Connection:socket(Pid, Transport, Socket, Connection, ListenTracker)}; {error, Reason} -> {error, Reason} end. @@ -185,12 +185,12 @@ recv(Pid, Length, Timeout) -> call(Pid, {recv, Length, Timeout}). %%-------------------------------------------------------------------- --spec connection_information(pid()) -> {ok, list()} | {error, reason()}. +-spec connection_information(pid(), boolean()) -> {ok, list()} | {error, reason()}. %% %% Description: Get the SNI hostname %%-------------------------------------------------------------------- -connection_information(Pid) when is_pid(Pid) -> - call(Pid, connection_information). +connection_information(Pid, IncludeSecrityInfo) when is_pid(Pid) -> + call(Pid, {connection_information, IncludeSecrityInfo}). %%-------------------------------------------------------------------- -spec close(pid(), {close, Timeout::integer() | @@ -247,14 +247,6 @@ set_opts(ConnectionPid, Options) -> call(ConnectionPid, {set_opts, Options}). %%-------------------------------------------------------------------- --spec session_info(pid()) -> {ok, list()} | {error, reason()}. -%% -%% Description: Returns info about the ssl session -%%-------------------------------------------------------------------- -session_info(ConnectionPid) -> - call(ConnectionPid, session_info). - -%%-------------------------------------------------------------------- -spec peer_certificate(pid()) -> {ok, binary()| undefined} | {error, reason()}. %% %% Description: Returns the peer cert @@ -363,11 +355,13 @@ init({call, From}, {start, Timeout}, State0, Connection) -> timer = Timer}), Connection:next_event(hello, Record, State); init({call, From}, {start, {Opts, EmOpts}, Timeout}, - #state{role = Role} = State0, Connection) -> + #state{role = Role, ssl_options = OrigSSLOptions, + socket_options = SockOpts} = State0, Connection) -> try - State = ssl_config(Opts, Role, State0), + SslOpts = ssl:handle_options(Opts, OrigSSLOptions), + State = ssl_config(SslOpts, Role, State0), init({call, From}, {start, Timeout}, - State#state{ssl_options = Opts, socket_options = EmOpts}, Connection) + State#state{ssl_options = SslOpts, socket_options = new_emulated(EmOpts, SockOpts)}, Connection) catch throw:Error -> {stop_and_reply, normal, {reply, From, {error, Error}}} end; @@ -432,11 +426,11 @@ abbreviated(internal, #finished{verify_data = Data} = Finished, verified -> ConnectionStates1 = ssl_record:set_server_verify_data(current_read, Data, ConnectionStates0), - State1 = + {State1, Actions} = finalize_handshake(State0#state{connection_states = ConnectionStates1}, abbreviated, Connection), {Record, State} = prepare_connection(State1#state{expecting_finished = false}, Connection), - Connection:next_event(connection, Record, State); + Connection:next_event(connection, Record, State, Actions); #alert{} = Alert -> handle_own_alert(Alert, Version, abbreviated, State0) end; @@ -773,14 +767,12 @@ connection({call, From}, renegotiate, #state{protocol_cb = Connection} = State, connection({call, From}, peer_certificate, #state{session = #session{peer_certificate = Cert}} = State, _) -> hibernate_after(connection, State, [{reply, From, {ok, Cert}}]); -connection({call, From}, connection_information, State, _) -> +connection({call, From}, {connection_information, true}, State, _) -> + Info = connection_info(State) ++ security_info(State), + hibernate_after(connection, State, [{reply, From, {ok, Info}}]); +connection({call, From}, {connection_information, false}, State, _) -> Info = connection_info(State), hibernate_after(connection, State, [{reply, From, {ok, Info}}]); -connection({call, From}, session_info, #state{session = #session{session_id = Id, - cipher_suite = Suite}} = State, _) -> - SessionInfo = [{session_id, Id}, - {cipher_suite, ssl_cipher:erl_suite_definition(Suite)}], - hibernate_after(connection, State, [{reply, From, SessionInfo}]); connection({call, From}, negotiated_protocol, #state{negotiated_protocol = undefined} = State, _) -> hibernate_after(connection, State, [{reply, From, {error, protocol_not_negotiated}}]); @@ -856,6 +848,7 @@ handle_common_event(internal, #change_cipher_spec{type = <<1>>}, StateName, StateName, State); handle_common_event(_Type, Msg, StateName, #state{negotiated_version = Version} = State, _) -> + ct:pal("Unexpected msg ~p", [Msg]), Alert = ?ALERT_REC(?FATAL,?UNEXPECTED_MESSAGE), handle_own_alert(Alert, Version, {StateName, Msg}, State). @@ -1192,7 +1185,8 @@ handle_alert(#alert{level = ?WARNING} = Alert, StateName, %%% Internal functions %%-------------------------------------------------------------------- connection_info(#state{sni_hostname = SNIHostname, - session = #session{cipher_suite = CipherSuite, ecc = ECCCurve}, + session = #session{session_id = SessionId, + cipher_suite = CipherSuite, ecc = ECCCurve}, protocol_cb = Connection, negotiated_version = {_,_} = Version, ssl_options = Opts}) -> @@ -1207,9 +1201,18 @@ connection_info(#state{sni_hostname = SNIHostname, [] end, [{protocol, RecordCB:protocol_version(Version)}, + {session_id, SessionId}, {cipher_suite, CipherSuiteDef}, {sni_hostname, SNIHostname} | CurveInfo] ++ ssl_options_list(Opts). +security_info(#state{connection_states = ConnectionStates}) -> + #{security_parameters := + #security_parameters{client_random = ClientRand, + server_random = ServerRand, + master_secret = MasterSecret}} = + ssl_record:current_connection_state(ConnectionStates, read), + [{client_random, ClientRand}, {server_random, ServerRand}, {master_secret, MasterSecret}]. + do_server_hello(Type, #hello_extensions{next_protocol_negotiation = NextProtocols} = ServerHelloExt, #state{negotiated_version = Version, @@ -1236,13 +1239,13 @@ new_server_hello(#server_hello{cipher_suite = CipherSuite, negotiated_version = Version} = State0, Connection) -> try server_certify_and_key_exchange(State0, Connection) of #state{} = State1 -> - State2 = server_hello_done(State1, Connection), + {State2, Actions} = server_hello_done(State1, Connection), Session = Session0#session{session_id = SessionId, cipher_suite = CipherSuite, compression_method = Compression}, {Record, State} = Connection:next_record(State2#state{session = Session}), - Connection:next_event(certify, Record, State) + Connection:next_event(certify, Record, State, Actions) catch #alert{} = Alert -> handle_own_alert(Alert, Version, hello, State0) @@ -1257,10 +1260,10 @@ resumed_server_hello(#state{session = Session, {_, ConnectionStates1} -> State1 = State0#state{connection_states = ConnectionStates1, session = Session}, - State2 = + {State2, Actions} = finalize_handshake(State1, abbreviated, Connection), {Record, State} = Connection:next_record(State2), - Connection:next_event(abbreviated, Record, State); + Connection:next_event(abbreviated, Record, State, Actions); #alert{} = Alert -> handle_own_alert(Alert, Version, hello, State0) end. @@ -1343,12 +1346,12 @@ client_certify_and_key_exchange(#state{negotiated_version = Version} = State0, Connection) -> try do_client_certify_and_key_exchange(State0, Connection) of State1 = #state{} -> - State2 = finalize_handshake(State1, certify, Connection), + {State2, Actions} = finalize_handshake(State1, certify, Connection), State3 = State2#state{ %% Reinitialize client_certificate_requested = false}, {Record, State} = Connection:next_record(State3), - Connection:next_event(cipher, Record, State) + Connection:next_event(cipher, Record, State, Actions) catch throw:#alert{} = Alert -> handle_own_alert(Alert, Version, certify, State0) @@ -1870,11 +1873,11 @@ cipher_role(server, Data, Session, #state{connection_states = ConnectionStates0 Connection) -> ConnectionStates1 = ssl_record:set_client_verify_data(current_read, Data, ConnectionStates0), - State1 = + {State1, Actions} = finalize_handshake(State0#state{connection_states = ConnectionStates1, session = Session}, cipher, Connection), {Record, State} = prepare_connection(State1, Connection), - Connection:next_event(connection, Record, State). + Connection:next_event(connection, Record, State, Actions). is_anonymous(Algo) when Algo == dh_anon; Algo == ecdh_anon; @@ -2305,7 +2308,7 @@ format_reply(_, _,#socket_options{active = false, mode = Mode, packet = Packet, {ok, do_format_reply(Mode, Packet, Header, Data)}; format_reply(Transport, Socket, #socket_options{active = _, mode = Mode, packet = Packet, header = Header}, Data, Tracker, Connection) -> - {ssl, tls_socket:socket(self(), Transport, Socket, Connection, Tracker), + {ssl, Connection:socket(self(), Transport, Socket, Connection, Tracker), do_format_reply(Mode, Packet, Header, Data)}. deliver_packet_error(Transport, Socket, SO= #socket_options{active = Active}, Data, Pid, From, Tracker, Connection) -> @@ -2314,7 +2317,7 @@ deliver_packet_error(Transport, Socket, SO= #socket_options{active = Active}, Da format_packet_error(_, _,#socket_options{active = false, mode = Mode}, Data, _, _) -> {error, {invalid_packet, do_format_reply(Mode, raw, 0, Data)}}; format_packet_error(Transport, Socket, #socket_options{active = _, mode = Mode}, Data, Tracker, Connection) -> - {ssl_error, tls_socket:socket(self(), Transport, Socket, Connection, Tracker), + {ssl_error, Connection:socket(self(), Transport, Socket, Connection, Tracker), {invalid_packet, do_format_reply(Mode, raw, 0, Data)}}. do_format_reply(binary, _, N, Data) when N > 0 -> % Header mode @@ -2369,11 +2372,11 @@ alert_user(Transport, Tracker, Socket, Active, Pid, From, Alert, Role, Connectio case ssl_alert:reason_code(Alert, Role) of closed -> send_or_reply(Active, Pid, From, - {ssl_closed, tls_socket:socket(self(), + {ssl_closed, Connection:socket(self(), Transport, Socket, Connection, Tracker)}); ReasonCode -> send_or_reply(Active, Pid, From, - {ssl_error, tls_socket:socket(self(), + {ssl_error, Connection:socket(self(), Transport, Socket, Connection, Tracker), ReasonCode}) end. @@ -2472,3 +2475,8 @@ update_ssl_options_from_sni(OrigSSLOptions, SNIHostname) -> _ -> ssl:handle_options(SSLOption, OrigSSLOptions) end. + +new_emulated([], EmOpts) -> + EmOpts; +new_emulated(NewEmOpts, _) -> + NewEmOpts. diff --git a/lib/ssl/src/ssl_connection.hrl b/lib/ssl/src/ssl_connection.hrl index b597c059af..368eaf6090 100644 --- a/lib/ssl/src/ssl_connection.hrl +++ b/lib/ssl/src/ssl_connection.hrl @@ -91,7 +91,8 @@ %% underlaying packet format. Introduced by DTLS - RFC 4347. %% The mecahnism is also usefull in TLS although we do not %% need to worry about packet loss in TLS. In DTLS we need to track DTLS handshake seqnr - flight_state = reliable %% reliable | {retransmit, integer()}| {waiting, ref(), integer()} - last two is used in DTLS over udp. + flight_state = reliable, %% reliable | {retransmit, integer()}| {waiting, ref(), integer()} - last two is used in DTLS over udp. + protocol_specific = #{} :: map() }). -define(DEFAULT_DIFFIE_HELLMAN_PARAMS, #'DHParameter'{prime = ?DEFAULT_DIFFIE_HELLMAN_PRIME, diff --git a/lib/ssl/src/ssl_internal.hrl b/lib/ssl/src/ssl_internal.hrl index c34af9f82c..0fbaa82b6a 100644 --- a/lib/ssl/src/ssl_internal.hrl +++ b/lib/ssl/src/ssl_internal.hrl @@ -76,7 +76,7 @@ -define(ALL_SUPPORTED_VERSIONS, ['tlsv1.2', 'tlsv1.1', tlsv1]). -define(MIN_SUPPORTED_VERSIONS, ['tlsv1.1', tlsv1]). -define(ALL_DATAGRAM_SUPPORTED_VERSIONS, ['dtlsv1.2', dtlsv1]). --define(MIN_DATAGRAM_SUPPORTED_VERSIONS, ['dtlsv1.2', dtlsv1]). +-define(MIN_DATAGRAM_SUPPORTED_VERSIONS, [dtlsv1]). -define('24H_in_msec', 86400000). -define('24H_in_sec', 86400). @@ -144,7 +144,7 @@ honor_ecc_order :: boolean(), v2_hello_compatible :: boolean(), max_handshake_size :: integer() - }). + }). -record(socket_options, { diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index 77606911be..bda6bf0349 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -48,7 +48,7 @@ -export([encode_data/3, encode_alert/3]). %% State transition handling --export([next_record/1, next_event/3]). +-export([next_record/1, next_event/3, next_event/4]). %% Handshake handling -export([renegotiate/2, send_handshake/2, @@ -59,7 +59,8 @@ -export([send_alert/2, close/5]). %% Data handling --export([passive_receive/2, next_record_if_active/1, handle_common_event/4, send/3]). +-export([passive_receive/2, next_record_if_active/1, handle_common_event/4, send/3, + socket/5]). %% gen_statem state functions -export([init/3, error/3, downgrade/3, %% Initiation and take down states @@ -117,7 +118,7 @@ send_handshake_flight(#state{socket = Socket, transport_cb = Transport, flight_buffer = Flight} = State0) -> send(Transport, Socket, Flight), - State0#state{flight_buffer = []}. + {State0#state{flight_buffer = []}, []}. queue_change_cipher(Msg, #state{negotiated_version = Version, flight_buffer = Flight0, @@ -191,6 +192,10 @@ init([Role, Host, Port, Socket, Options, User, CbInfo]) -> callback_mode() -> state_functions. +socket(Pid, Transport, Socket, Connection, Tracker) -> + tls_socket:socket(Pid, Transport, Socket, Connection, Tracker). + + %%-------------------------------------------------------------------- %% State functions %%-------------------------------------------------------------------- @@ -340,12 +345,12 @@ connection(internal, #hello_request{}, renegotiation = {Renegotiation, _}} = State0) -> Hello = tls_handshake:client_hello(Host, Port, ConnectionStates0, SslOpts, Cache, CacheCb, Renegotiation, Cert), - State1 = send_handshake(Hello, State0), + {State1, Actions} = send_handshake(Hello, State0), {Record, State} = next_record( State1#state{session = Session0#session{session_id = Hello#client_hello.session_id}}), - next_event(hello, Record, State); + next_event(hello, Record, State, Actions); connection(internal, #client_hello{} = Hello, #state{role = server, allow_renegotiate = true} = State0) -> %% Mitigate Computational DoS attack @@ -392,23 +397,36 @@ handle_info({Protocol, _, Data}, StateName, end; handle_info({CloseTag, Socket}, StateName, #state{socket = Socket, close_tag = CloseTag, + socket_options = #socket_options{active = Active}, + protocol_buffers = #protocol_buffers{tls_cipher_texts = CTs}, negotiated_version = Version} = State) -> + %% Note that as of TLS 1.1, %% failure to properly close a connection no longer requires that a %% session not be resumed. This is a change from TLS 1.0 to conform %% with widespread implementation practice. - case Version of - {1, N} when N >= 1 -> - ok; - _ -> - %% As invalidate_sessions here causes performance issues, - %% we will conform to the widespread implementation - %% practice and go aginst the spec - %%invalidate_session(Role, Host, Port, Session) - ok - end, - ssl_connection:handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), StateName, State), - {stop, {shutdown, transport_closed}}; + + case (Active == false) andalso (CTs =/= []) of + false -> + case Version of + {1, N} when N >= 1 -> + ok; + _ -> + %% As invalidate_sessions here causes performance issues, + %% we will conform to the widespread implementation + %% practice and go aginst the spec + %%invalidate_session(Role, Host, Port, Session) + ok + end, + + ssl_connection:handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), StateName, State), + {stop, {shutdown, transport_closed}}; + true -> + %% Fixes non-delivery of final TLS record in {active, once}. + %% Basically allows the application the opportunity to set {active, once} again + %% and then receive the final message. + next_event(StateName, no_record, State) + end; handle_info(Msg, StateName, State) -> ssl_connection:handle_info(Msg, StateName, State). diff --git a/lib/ssl/test/Makefile b/lib/ssl/test/Makefile index a2eb4ce449..55d45c98f6 100644 --- a/lib/ssl/test/Makefile +++ b/lib/ssl/test/Makefile @@ -56,7 +56,8 @@ MODULES = \ ssl_upgrade_SUITE\ ssl_sni_SUITE \ make_certs\ - erl_make_certs + erl_make_certs\ + x509_test ERL_FILES = $(MODULES:%=%.erl) diff --git a/lib/ssl/test/erl_make_certs.erl b/lib/ssl/test/erl_make_certs.erl index a6657be995..af217efc11 100644 --- a/lib/ssl/test/erl_make_certs.erl +++ b/lib/ssl/test/erl_make_certs.erl @@ -179,7 +179,7 @@ make_tbs(SubjectKey, Opts) -> subject(proplists:get_value(subject, Opts),false) end, - {#'OTPTBSCertificate'{serialNumber = trunc(random:uniform()*100000000)*10000 + 1, + {#'OTPTBSCertificate'{serialNumber = trunc(rand:uniform()*100000000)*10000 + 1, signature = SignAlgo, issuer = Issuer, validity = validity(Opts), diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index f0a3c42e8d..4eabe544d7 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -53,7 +53,8 @@ all() -> {group, options_tls}, {group, session}, {group, 'dtlsv1.2'}, - %%{group, 'dtlsv1'}, + %% {group, 'dtlsv1'}, Breaks dtls in cert_verify_SUITE enable later when + %% problem is identified and fixed {group, 'tlsv1.2'}, {group, 'tlsv1.1'}, {group, 'tlsv1'}, @@ -65,15 +66,15 @@ groups() -> {basic_tls, [], basic_tests_tls()}, {options, [], options_tests()}, {options_tls, [], options_tests_tls()}, - %%{'dtlsv1.2', [], all_versions_groups()}, - {'dtlsv1.2', [], [connection_information]}, - %%{'dtlsv1', [], all_versions_groups()}, + {'dtlsv1.2', [], all_versions_groups()}, + {'dtlsv1', [], all_versions_groups()}, {'tlsv1.2', [], all_versions_groups() ++ tls_versions_groups() ++ [conf_signature_algs, no_common_signature_algs]}, {'tlsv1.1', [], all_versions_groups() ++ tls_versions_groups()}, {'tlsv1', [], all_versions_groups() ++ tls_versions_groups() ++ rizzo_tests()}, {'sslv3', [], all_versions_groups() ++ tls_versions_groups() ++ rizzo_tests() ++ [tls_ciphersuite_vs_version]}, {api,[], api_tests()}, {api_tls,[], api_tests_tls()}, + {tls_ciphers,[], tls_cipher_tests()}, {session, [], session_tests()}, {renegotiate, [], renegotiate_tests()}, {ciphers, [], cipher_tests()}, @@ -83,12 +84,13 @@ groups() -> ]. tls_versions_groups ()-> - [{group, api_tls}, + [{group, renegotiate}, %% Should be in all_versions_groups not fixed for DTLS yet + {group, api_tls}, + {group, tls_ciphers}, {group, error_handling_tests_tls}]. all_versions_groups ()-> [{group, api}, - {group, renegotiate}, {group, ciphers}, {group, ciphers_ec}, {group, error_handling_tests}]. @@ -146,11 +148,10 @@ options_tests_tls() -> api_tests() -> [connection_info, + secret_connection_info, connection_information, - peername, peercert, peercert_with_client_cert, - sockname, versions, eccs, controlling_process, @@ -162,7 +163,6 @@ api_tests() -> ssl_recv_timeout, server_name_indication_option, accept_pool, - new_options_in_accept, prf ]. @@ -175,7 +175,10 @@ api_tests_tls() -> tls_shutdown, tls_shutdown_write, tls_shutdown_both, - tls_shutdown_error + tls_shutdown_error, + peername, + sockname, + new_options_in_accept ]. session_tests() -> @@ -197,6 +200,11 @@ renegotiate_tests() -> renegotiate_dos_mitigate_passive, renegotiate_dos_mitigate_absolute]. +tls_cipher_tests() -> + [rc4_rsa_cipher_suites, + rc4_ecdh_rsa_cipher_suites, + rc4_ecdsa_cipher_suites]. + cipher_tests() -> [cipher_suites, cipher_suites_mix, @@ -212,9 +220,6 @@ cipher_tests() -> srp_cipher_suites, srp_anon_cipher_suites, srp_dsa_cipher_suites, - rc4_rsa_cipher_suites, - rc4_ecdh_rsa_cipher_suites, - rc4_ecdsa_cipher_suites, des_rsa_cipher_suites, des_ecdh_rsa_cipher_suites, default_reject_anonymous]. @@ -226,15 +231,15 @@ cipher_tests_ec() -> ciphers_ecdh_rsa_signed_certs_openssl_names]. error_handling_tests()-> - [controller_dies, - close_transport_accept, + [close_transport_accept, recv_active, recv_active_once, recv_error_handling ]. error_handling_tests_tls()-> - [tls_client_closes_socket, + [controller_dies, + tls_client_closes_socket, tls_tcp_error_propagation_in_active_mode, tls_tcp_connect, tls_tcp_connect_big, @@ -607,7 +612,7 @@ prf(Config) when is_list(Config) -> %%-------------------------------------------------------------------- connection_info() -> - [{doc,"Test the API function ssl:connection_information/1"}]. + [{doc,"Test the API function ssl:connection_information/2"}]. connection_info(Config) when is_list(Config) -> ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), @@ -641,6 +646,38 @@ connection_info(Config) when is_list(Config) -> %%-------------------------------------------------------------------- +secret_connection_info() -> + [{doc,"Test the API function ssl:connection_information/2"}]. +secret_connection_info(Config) when is_list(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, secret_connection_info_result, []}}, + {options, ServerOpts}]), + + Port = ssl_test_lib:inet_port(Server), + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, secret_connection_info_result, []}}, + {options, ClientOpts}]), + + ct:log("Testcase ~p, Client ~p Server ~p ~n", + [self(), Client, Server]), + + Version = ssl_test_lib:protocol_version(Config), + + ssl_test_lib:check_result(Server, true, Client, true), + + ssl_test_lib:close(Server), + ssl_test_lib:close(Client). + + +%%-------------------------------------------------------------------- + connection_information() -> [{doc,"Test the API function ssl:connection_information/1"}]. connection_information(Config) when is_list(Config) -> @@ -843,8 +880,7 @@ controller_dies(Config) when is_list(Config) -> Server ! listen, Tester = self(), Connect = fun(Pid) -> - {ok, Socket} = ssl:connect(Hostname, Port, - [{reuseaddr,true},{ssl_imp,new}]), + {ok, Socket} = ssl:connect(Hostname, Port, ClientOpts), %% Make sure server finishes and verification %% and is in coonection state before %% killing client @@ -2194,8 +2230,9 @@ ciphers_dsa_signed_certs() -> [{doc,"Test all dsa ssl cipher suites in highest support ssl/tls version"}]. ciphers_dsa_signed_certs(Config) when is_list(Config) -> + NVersion = ssl_test_lib:protocol_version(Config, tuple), Version = ssl_test_lib:protocol_version(Config), - Ciphers = ssl_test_lib:dsa_suites(tls_record:protocol_version(Version)), + Ciphers = ssl_test_lib:dsa_suites(NVersion), ct:log("~p erlang cipher suites ~p~n", [Version, Ciphers]), run_suites(Ciphers, Version, Config, dsa). %%------------------------------------------------------------------- @@ -2218,29 +2255,33 @@ anonymous_cipher_suites(Config) when is_list(Config) -> psk_cipher_suites() -> [{doc, "Test the PSK ciphersuites WITHOUT server supplied identity hint"}]. psk_cipher_suites(Config) when is_list(Config) -> + NVersion = tls_record:highest_protocol_version([]), Version = ssl_test_lib:protocol_version(Config), - Ciphers = ssl_test_lib:psk_suites(), + Ciphers = ssl_test_lib:psk_suites(NVersion), run_suites(Ciphers, Version, Config, psk). %%------------------------------------------------------------------- psk_with_hint_cipher_suites()-> [{doc, "Test the PSK ciphersuites WITH server supplied identity hint"}]. psk_with_hint_cipher_suites(Config) when is_list(Config) -> + NVersion = tls_record:highest_protocol_version([]), Version = ssl_test_lib:protocol_version(Config), - Ciphers = ssl_test_lib:psk_suites(), + Ciphers = ssl_test_lib:psk_suites(NVersion), run_suites(Ciphers, Version, Config, psk_with_hint). %%------------------------------------------------------------------- psk_anon_cipher_suites() -> [{doc, "Test the anonymous PSK ciphersuites WITHOUT server supplied identity hint"}]. psk_anon_cipher_suites(Config) when is_list(Config) -> + NVersion = tls_record:highest_protocol_version([]), Version = ssl_test_lib:protocol_version(Config), - Ciphers = ssl_test_lib:psk_anon_suites(), + Ciphers = ssl_test_lib:psk_anon_suites(NVersion), run_suites(Ciphers, Version, Config, psk_anon). %%------------------------------------------------------------------- psk_anon_with_hint_cipher_suites()-> [{doc, "Test the anonymous PSK ciphersuites WITH server supplied identity hint"}]. psk_anon_with_hint_cipher_suites(Config) when is_list(Config) -> + NVersion = tls_record:highest_protocol_version([]), Version = ssl_test_lib:protocol_version(Config), - Ciphers = ssl_test_lib:psk_anon_suites(), + Ciphers = ssl_test_lib:psk_anon_suites(NVersion), run_suites(Ciphers, Version, Config, psk_anon_with_hint). %%------------------------------------------------------------------- srp_cipher_suites()-> @@ -2291,18 +2332,17 @@ rc4_ecdsa_cipher_suites(Config) when is_list(Config) -> %%------------------------------------------------------------------- des_rsa_cipher_suites()-> - [{doc, "Test the RC4 ciphersuites"}]. + [{doc, "Test the des_rsa ciphersuites"}]. des_rsa_cipher_suites(Config) when is_list(Config) -> - NVersion = tls_record:highest_protocol_version([]), - Version = tls_record:protocol_version(NVersion), - Ciphers = ssl_test_lib:des_suites(NVersion), + Version = ssl_test_lib:protocol_version(Config), + Ciphers = ssl_test_lib:des_suites(Config), run_suites(Ciphers, Version, Config, des_rsa). %------------------------------------------------------------------- des_ecdh_rsa_cipher_suites()-> - [{doc, "Test the RC4 ciphersuites"}]. + [{doc, "Test ECDH rsa signed ciphersuites"}]. des_ecdh_rsa_cipher_suites(Config) when is_list(Config) -> - NVersion = tls_record:highest_protocol_version([]), - Version = tls_record:protocol_version(NVersion), + NVersion = ssl_test_lib:protocol_version(Config, tuple), + Version = ssl_test_lib:protocol_version(Config), Ciphers = ssl_test_lib:des_suites(NVersion), run_suites(Ciphers, Version, Config, des_dhe_rsa). @@ -2313,9 +2353,11 @@ default_reject_anonymous(Config) when is_list(Config) -> {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), - Version = tls_record:highest_protocol_version(tls_record:supported_protocol_versions()), - [CipherSuite | _] = ssl_test_lib:anonymous_suites(Version), - + Version = ssl_test_lib:protocol_version(Config), + TLSVersion = ssl_test_lib:tls_version(Version), + + [CipherSuite | _] = ssl_test_lib:anonymous_suites(TLSVersion), + Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, {from, self()}, {options, ServerOpts}]), @@ -2335,8 +2377,9 @@ ciphers_ecdsa_signed_certs() -> [{doc, "Test all ecdsa ssl cipher suites in highest support ssl/tls version"}]. ciphers_ecdsa_signed_certs(Config) when is_list(Config) -> + NVersion = ssl_test_lib:protocol_version(Config, tuple), Version = ssl_test_lib:protocol_version(Config), - Ciphers = ssl_test_lib:ecdsa_suites(tls_record:protocol_version(Version)), + Ciphers = ssl_test_lib:ecdsa_suites(NVersion), ct:log("~p erlang cipher suites ~p~n", [Version, Ciphers]), run_suites(Ciphers, Version, Config, ecdsa). %%-------------------------------------------------------------------- @@ -2353,8 +2396,9 @@ ciphers_ecdh_rsa_signed_certs() -> [{doc, "Test all ecdh_rsa ssl cipher suites in highest support ssl/tls version"}]. ciphers_ecdh_rsa_signed_certs(Config) when is_list(Config) -> + NVersion = ssl_test_lib:protocol_version(Config, tuple), Version = ssl_test_lib:protocol_version(Config), - Ciphers = ssl_test_lib:ecdh_rsa_suites(tls_record:protocol_version(Version)), + Ciphers = ssl_test_lib:ecdh_rsa_suites(NVersion), ct:log("~p erlang cipher suites ~p~n", [Version, Ciphers]), run_suites(Ciphers, Version, Config, ecdh_rsa). %%-------------------------------------------------------------------- @@ -3326,11 +3370,11 @@ hibernate(Config) -> process_info(Pid, current_function), ssl_test_lib:check_result(Server, ok, Client, ok), - timer:sleep(1100), - + + timer:sleep(1500), {current_function, {erlang, hibernate, 3}} = process_info(Pid, current_function), - + ssl_test_lib:close(Server), ssl_test_lib:close(Client). @@ -3363,13 +3407,12 @@ hibernate_right_away(Config) -> [{port, Port1}, {options, [{hibernate_after, 0}|ClientOpts]}]), ssl_test_lib:check_result(Server1, ok, Client1, ok), - - {current_function, {erlang, hibernate, 3}} = + + {current_function, {erlang, hibernate, 3}} = process_info(Pid1, current_function), - ssl_test_lib:close(Server1), ssl_test_lib:close(Client1), - + Server2 = ssl_test_lib:start_server(StartServerOpts), Port2 = ssl_test_lib:inet_port(Server2), {Client2, #sslsocket{pid = Pid2}} = ssl_test_lib:start_client(StartClientOpts ++ @@ -3377,8 +3420,8 @@ hibernate_right_away(Config) -> ssl_test_lib:check_result(Server2, ok, Client2, ok), - ct:sleep(100), %% Schedule out - + ct:sleep(1000), %% Schedule out + {current_function, {erlang, hibernate, 3}} = process_info(Pid2, current_function), @@ -3404,7 +3447,6 @@ listen_socket(Config) -> {error, enotconn} = ssl:connection_information(ListenSocket), {error, enotconn} = ssl:peername(ListenSocket), {error, enotconn} = ssl:peercert(ListenSocket), - {error, enotconn} = ssl:session_info(ListenSocket), {error, enotconn} = ssl:renegotiate(ListenSocket), {error, enotconn} = ssl:prf(ListenSocket, 'master_secret', <<"Label">>, client_random, 256), {error, enotconn} = ssl:shutdown(ListenSocket, read_write), @@ -4030,11 +4072,11 @@ prf_create_plan(TlsVersions, PRFs, Results) -> prf_ciphers_and_expected(TlsVer, PRFs, Results) -> case TlsVer of TlsVer when TlsVer == sslv3 orelse TlsVer == tlsv1 - orelse TlsVer == 'tlsv1.1' -> + orelse TlsVer == 'tlsv1.1' orelse TlsVer == 'dtlsv1' -> Ciphers = ssl:cipher_suites(), {_, Expected} = lists:keyfind(md5sha, 1, Results), [[{tls_ver, TlsVer}, {ciphers, Ciphers}, {expected, Expected}, {prf, md5sha}]]; - 'tlsv1.2' -> + TlsVer when TlsVer == 'tlsv1.2' orelse TlsVer == 'dtlsv1.2'-> lists:foldl( fun(PRF, Acc) -> Ciphers = prf_get_ciphers(TlsVer, PRF), @@ -4049,21 +4091,20 @@ prf_ciphers_and_expected(TlsVer, PRFs, Results) -> end end, [], PRFs) end. -prf_get_ciphers(TlsVer, PRF) -> - case TlsVer of - 'tlsv1.2' -> - lists:filter( - fun(C) when tuple_size(C) == 4 andalso - element(4, C) == PRF -> - true; - (_) -> false - end, ssl:cipher_suites()) - end. +prf_get_ciphers(_, PRF) -> + lists:filter( + fun(C) when tuple_size(C) == 4 andalso + element(4, C) == PRF -> + true; + (_) -> + false + end, + ssl:cipher_suites()). prf_run_test(_, TlsVer, [], _, Prf) -> ct:fail({error, cipher_list_empty, TlsVer, Prf}); prf_run_test(Config, TlsVer, Ciphers, Expected, Prf) -> {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), - BaseOpts = [{active, true}, {versions, [TlsVer]}, {ciphers, Ciphers}], + BaseOpts = [{active, true}, {versions, [TlsVer]}, {ciphers, Ciphers}, {protocol, tls_or_dtls(TlsVer)}], ServerOpts = BaseOpts ++ proplists:get_value(server_opts, Config), ClientOpts = BaseOpts ++ proplists:get_value(client_opts, Config), Server = ssl_test_lib:start_server( @@ -4507,16 +4548,21 @@ run_suites(Ciphers, Version, Config, Type) -> [{reuseaddr, true}, {ciphers, ssl_test_lib:anonymous_suites(Version)}]}; psk -> {ssl_test_lib:ssl_options(client_psk, Config), - ssl_test_lib:ssl_options(server_psk, Config)}; + [{ciphers, ssl_test_lib:psk_suites(Version)} | + ssl_test_lib:ssl_options(server_psk, Config)]}; psk_with_hint -> {ssl_test_lib:ssl_options(client_psk, Config), - ssl_test_lib:ssl_options(server_psk_hint, Config)}; + [{ciphers, ssl_test_lib:psk_suites(Version)} | + ssl_test_lib:ssl_options(server_psk_hint, Config) + ]}; psk_anon -> {ssl_test_lib:ssl_options(client_psk, Config), - ssl_test_lib:ssl_options(server_psk_anon, Config)}; + [{ciphers, ssl_test_lib:psk_anon_suites(Version)} | + ssl_test_lib:ssl_options(server_psk_anon, Config)]}; psk_anon_with_hint -> {ssl_test_lib:ssl_options(client_psk, Config), - ssl_test_lib:ssl_options(server_psk_anon_hint, Config)}; + [{ciphers, ssl_test_lib:psk_anon_suites(Version)} | + ssl_test_lib:ssl_options(server_psk_anon_hint, Config)]}; srp -> {ssl_test_lib:ssl_options(client_srp, Config), ssl_test_lib:ssl_options(server_srp, Config)}; @@ -4556,7 +4602,7 @@ run_suites(Ciphers, Version, Config, Type) -> Result = lists:map(fun(Cipher) -> cipher(Cipher, Version, Config, ClientOpts, ServerOpts) end, - ssl_test_lib:filter_suites(Ciphers)), + ssl_test_lib:filter_suites(Ciphers, Version)), case lists:flatten(Result) of [] -> ok; @@ -4624,6 +4670,11 @@ version_info_result(Socket) -> {ok, [{version, Version}]} = ssl:connection_information(Socket, [version]), {ok, Version}. +secret_connection_info_result(Socket) -> + {ok, [{client_random, ClientRand}, {server_random, ServerRand}, {master_secret, MasterSecret}]} + = ssl:connection_information(Socket, [client_random, server_random, master_secret]), + is_binary(ClientRand) andalso is_binary(ServerRand) andalso is_binary(MasterSecret). + connect_dist_s(S) -> Msg = term_to_binary({erlang,term}), ok = ssl:send(S, Msg). @@ -4756,3 +4807,9 @@ wait_for_send(Socket) -> %% Make sure TLS process processed send message event _ = ssl:connection_information(Socket). +tls_or_dtls('dtlsv1') -> + dtls; +tls_or_dtls('dtlsv1.2') -> + dtls; +tls_or_dtls(_) -> + tls. diff --git a/lib/ssl/test/ssl_certificate_verify_SUITE.erl b/lib/ssl/test/ssl_certificate_verify_SUITE.erl index 5265c87e29..66b0c09b73 100644 --- a/lib/ssl/test/ssl_certificate_verify_SUITE.erl +++ b/lib/ssl/test/ssl_certificate_verify_SUITE.erl @@ -39,17 +39,26 @@ %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- all() -> - [{group, active}, - {group, passive}, - {group, active_once}, - {group, error_handling}]. - + [ + {group, tls}, + {group, dtls} + ]. groups() -> - [{active, [], tests()}, + [ + {tls, [], all_protocol_groups()}, + {dtls, [], all_protocol_groups()}, + {active, [], tests()}, {active_once, [], tests()}, {passive, [], tests()}, - {error_handling, [],error_handling_tests()}]. + {error_handling, [],error_handling_tests()} + ]. + +all_protocol_groups() -> + [{group, active}, + {group, passive}, + {group, active_once}, + {group, error_handling}]. tests() -> [verify_peer, @@ -85,7 +94,7 @@ init_per_suite(Config0) -> catch crypto:stop(), try crypto:start() of ok -> - ssl_test_lib:clean_start(), + ssl_test_lib:clean_start(), %% make rsa certs using oppenssl {ok, _} = make_certs:all(proplists:get_value(data_dir, Config0), proplists:get_value(priv_dir, Config0)), @@ -99,6 +108,26 @@ end_per_suite(_Config) -> ssl:stop(), application:stop(crypto). +init_per_group(tls, Config) -> + Version = tls_record:protocol_version(tls_record:highest_protocol_version([])), + ssl:stop(), + application:load(ssl), + application:set_env(ssl, protocol_version, Version), + application:set_env(ssl, bypass_pem_cache, Version), + ssl:start(), + NewConfig = proplists:delete(protocol, Config), + [{protocol, tls}, {version, tls_record:protocol_version(Version)} | NewConfig]; + +init_per_group(dtls, Config) -> + Version = dtls_record:protocol_version(dtls_record:highest_protocol_version([])), + ssl:stop(), + application:load(ssl), + application:set_env(ssl, protocol_version, Version), + application:set_env(ssl, bypass_pem_cache, Version), + ssl:start(), + NewConfig = proplists:delete(protocol_opts, proplists:delete(protocol, Config)), + [{protocol, dtls}, {protocol_opts, [{protocol, dtls}]}, {version, dtls_record:protocol_version(Version)} | NewConfig]; + init_per_group(active, Config) -> [{active, true}, {receive_function, send_recv_result_active} | Config]; init_per_group(active_once, Config) -> @@ -126,7 +155,7 @@ init_per_testcase(_TestCase, Config) -> ssl:stop(), ssl:start(), ssl_test_lib:ct_log_supported_protocol_versions(Config), - ct:timetrap({seconds, 5}), + ct:timetrap({seconds, 10}), Config. end_per_testcase(_TestCase, Config) -> @@ -262,7 +291,7 @@ server_require_peer_cert_fail() -> server_require_peer_cert_fail(Config) when is_list(Config) -> ServerOpts = [{verify, verify_peer}, {fail_if_no_peer_cert, true} | ssl_test_lib:ssl_options(server_verification_opts, Config)], - BadClientOpts = ssl_test_lib:ssl_options(client_opts, []), + BadClientOpts = ssl_test_lib:ssl_options(empty_client_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, @@ -411,7 +440,7 @@ server_require_peer_cert_partial_chain_fun_fail() -> server_require_peer_cert_partial_chain_fun_fail(Config) when is_list(Config) -> ServerOpts = [{verify, verify_peer}, {fail_if_no_peer_cert, true} | ssl_test_lib:ssl_options(server_verification_opts, Config)], - ClientOpts = proplists:get_value(client_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), {ok, ServerCAs} = file:read_file(proplists:get_value(cacertfile, ServerOpts)), @@ -1091,6 +1120,7 @@ client_with_cert_cipher_suites_handshake() -> client_with_cert_cipher_suites_handshake(Config) when is_list(Config) -> ClientOpts = ssl_test_lib:ssl_options(client_verification_opts_digital_signature_only, Config), ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, @@ -1098,7 +1128,7 @@ client_with_cert_cipher_suites_handshake(Config) when is_list(Config) -> send_recv_result_active, []}}, {options, [{active, true}, {ciphers, - ssl_test_lib:rsa_non_signed_suites(tls_record:highest_protocol_version([]))} + ssl_test_lib:rsa_non_signed_suites(proplists:get_value(version, Config))} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, @@ -1132,7 +1162,7 @@ server_verify_no_cacerts(Config) when is_list(Config) -> unknown_server_ca_fail() -> [{doc,"Test that the client fails if the ca is unknown in verify_peer mode"}]. unknown_server_ca_fail(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_opts, []), + ClientOpts = ssl_test_lib:ssl_options(empty_client_opts, Config), ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, @@ -1176,7 +1206,7 @@ unknown_server_ca_fail(Config) when is_list(Config) -> unknown_server_ca_accept_verify_none() -> [{doc,"Test that the client succeds if the ca is unknown in verify_none mode"}]. unknown_server_ca_accept_verify_none(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_opts, []), + ClientOpts = ssl_test_lib:ssl_options(empty_client_opts, Config), ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, @@ -1201,8 +1231,8 @@ unknown_server_ca_accept_verify_peer() -> [{doc, "Test that the client succeds if the ca is unknown in verify_peer mode" " with a verify_fun that accepts the unknown ca error"}]. unknown_server_ca_accept_verify_peer(Config) when is_list(Config) -> - ClientOpts =ssl_test_lib:ssl_options(client_opts, []), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(empty_client_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, @@ -1240,7 +1270,7 @@ unknown_server_ca_accept_verify_peer(Config) when is_list(Config) -> unknown_server_ca_accept_backwardscompatibility() -> [{doc,"Test that old style verify_funs will work"}]. unknown_server_ca_accept_backwardscompatibility(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_opts, []), + ClientOpts = ssl_test_lib:ssl_options(empty_client_opts, Config), ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, diff --git a/lib/ssl/test/ssl_npn_hello_SUITE.erl b/lib/ssl/test/ssl_npn_hello_SUITE.erl index 69aeea10c5..0b1de1dc1c 100644 --- a/lib/ssl/test/ssl_npn_hello_SUITE.erl +++ b/lib/ssl/test/ssl_npn_hello_SUITE.erl @@ -50,6 +50,10 @@ init_per_suite(Config) -> {skip, "Crypto did not start"} end. +end_per_suite(_Config) -> + %% This function is required since init_per_suite/1 exists. + ok. + init_per_testcase(_TestCase, Config) -> ssl_test_lib:ct_log_supported_protocol_versions(Config), ct:timetrap({seconds, 5}), diff --git a/lib/ssl/test/ssl_packet_SUITE.erl b/lib/ssl/test/ssl_packet_SUITE.erl index 3446a566c4..c8caa9c11a 100644 --- a/lib/ssl/test/ssl_packet_SUITE.erl +++ b/lib/ssl/test/ssl_packet_SUITE.erl @@ -1973,14 +1973,14 @@ passive_recv_packet(Socket, _, 0) -> {error, timeout} = ssl:recv(Socket, 0, 500), ok; Other -> - {other, Other, ssl:session_info(Socket), 0} + {other, Other, ssl:connection_information(Socket, [session_id, cipher_suite]), 0} end; passive_recv_packet(Socket, Data, N) -> case ssl:recv(Socket, 0) of {ok, Data} -> passive_recv_packet(Socket, Data, N-1); Other -> - {other, Other, ssl:session_info(Socket), N} + {other, Other, ssl:connection_information(Socket, [session_id, cipher_suite]), N} end. send(Socket,_, 0) -> @@ -2032,7 +2032,7 @@ active_once_packet(Socket,_, 0) -> {ssl, Socket, []} -> ok; {ssl, Socket, Other} -> - {other, Other, ssl:session_info(Socket), 0} + {other, Other, ssl:connection_information(Socket, [session_id, cipher_suite]), 0} end; active_once_packet(Socket, Data, N) -> receive @@ -2077,7 +2077,7 @@ active_packet(Socket, _, 0) -> {ssl, Socket, []} -> ok; Other -> - {other, Other, ssl:session_info(Socket), 0} + {other, Other, ssl:connection_information(Socket, [session_id, cipher_suite]), 0} end; active_packet(Socket, Data, N) -> receive @@ -2089,7 +2089,7 @@ active_packet(Socket, Data, N) -> {ssl, Socket, Data} -> active_packet(Socket, Data, N -1); Other -> - {other, Other, ssl:session_info(Socket),N} + {other, Other, ssl:connection_information(Socket, [session_id, cipher_suite]),N} end. assert_packet_opt(Socket, Type) -> diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index 49d2b5c1b8..ae378037dd 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -401,27 +401,22 @@ cert_options(Config) -> {ssl_imp, new}]}, {server_opts, [{ssl_imp, new},{reuseaddr, true}, {cacertfile, ServerCaCertFile}, {certfile, ServerCertFile}, {keyfile, ServerKeyFile}]}, - %%{server_anon, [{ssl_imp, new},{reuseaddr, true}, {ciphers, anonymous_suites()}]}, - {client_psk, [{ssl_imp, new},{reuseaddr, true}, + {client_psk, [{ssl_imp, new}, {psk_identity, "Test-User"}, {user_lookup_fun, {fun user_lookup/3, PskSharedSecret}}]}, {server_psk, [{ssl_imp, new},{reuseaddr, true}, {certfile, ServerCertFile}, {keyfile, ServerKeyFile}, - {user_lookup_fun, {fun user_lookup/3, PskSharedSecret}}, - {ciphers, psk_suites()}]}, + {user_lookup_fun, {fun user_lookup/3, PskSharedSecret}}]}, {server_psk_hint, [{ssl_imp, new},{reuseaddr, true}, {certfile, ServerCertFile}, {keyfile, ServerKeyFile}, {psk_identity, "HINT"}, - {user_lookup_fun, {fun user_lookup/3, PskSharedSecret}}, - {ciphers, psk_suites()}]}, + {user_lookup_fun, {fun user_lookup/3, PskSharedSecret}}]}, {server_psk_anon, [{ssl_imp, new},{reuseaddr, true}, - {user_lookup_fun, {fun user_lookup/3, PskSharedSecret}}, - {ciphers, psk_anon_suites()}]}, + {user_lookup_fun, {fun user_lookup/3, PskSharedSecret}}]}, {server_psk_anon_hint, [{ssl_imp, new},{reuseaddr, true}, {psk_identity, "HINT"}, - {user_lookup_fun, {fun user_lookup/3, PskSharedSecret}}, - {ciphers, psk_anon_suites()}]}, - {client_srp, [{ssl_imp, new},{reuseaddr, true}, + {user_lookup_fun, {fun user_lookup/3, PskSharedSecret}}]}, + {client_srp, [{ssl_imp, new}, {srp_identity, {"Test-User", "secret"}}]}, {server_srp, [{ssl_imp, new},{reuseaddr, true}, {certfile, ServerCertFile}, {keyfile, ServerKeyFile}, @@ -476,7 +471,7 @@ make_dsa_cert(Config) -> {cacertfile, ClientCaCertFile}, {certfile, ServerCertFile}, {keyfile, ServerKeyFile}, {verify, verify_peer}]}, - {client_dsa_opts, [{ssl_imp, new},{reuseaddr, true}, + {client_dsa_opts, [{ssl_imp, new}, {cacertfile, ClientCaCertFile}, {certfile, ClientCertFile}, {keyfile, ClientKeyFile}]}, {server_srp_dsa, [{ssl_imp, new},{reuseaddr, true}, @@ -484,7 +479,7 @@ make_dsa_cert(Config) -> {certfile, ServerCertFile}, {keyfile, ServerKeyFile}, {user_lookup_fun, {fun user_lookup/3, undefined}}, {ciphers, srp_dss_suites()}]}, - {client_srp_dsa, [{ssl_imp, new},{reuseaddr, true}, + {client_srp_dsa, [{ssl_imp, new}, {srp_identity, {"Test-User", "secret"}}, {cacertfile, ClientCaCertFile}, {certfile, ClientCertFile}, {keyfile, ClientKeyFile}]} @@ -493,23 +488,32 @@ make_dsa_cert(Config) -> make_ecdsa_cert(Config) -> CryptoSupport = crypto:supports(), case proplists:get_bool(ecdsa, proplists:get_value(public_keys, CryptoSupport)) of - true -> - {ServerCaCertFile, ServerCertFile, ServerKeyFile} = - make_cert_files("server", Config, ec, ec, "", [{digest, appropriate_sha(CryptoSupport)}]), - {ClientCaCertFile, ClientCertFile, ClientKeyFile} = - make_cert_files("client", Config, ec, ec, "", [{digest, appropriate_sha(CryptoSupport)}]), - [{server_ecdsa_opts, [{ssl_imp, new},{reuseaddr, true}, - {cacertfile, ServerCaCertFile}, - {certfile, ServerCertFile}, {keyfile, ServerKeyFile}]}, - {server_ecdsa_verify_opts, [{ssl_imp, new},{reuseaddr, true}, - {cacertfile, ClientCaCertFile}, - {certfile, ServerCertFile}, {keyfile, ServerKeyFile}, - {verify, verify_peer}]}, - {client_ecdsa_opts, [{ssl_imp, new},{reuseaddr, true}, - {cacertfile, ClientCaCertFile}, - {certfile, ClientCertFile}, {keyfile, ClientKeyFile}]} + true -> + %% {ServerCaCertFile, ServerCertFile, ServerKeyFile} = + %% make_cert_files("server", Config, ec, ec, "", [{digest, appropriate_sha(CryptoSupport)}]), + %% {ClientCaCertFile, ClientCertFile, ClientKeyFile} = + %% make_cert_files("client", Config, ec, ec, "", [{digest, appropriate_sha(CryptoSupport)}]), + CertFileBase = filename:join([proplists:get_value(priv_dir, Config), "ecdsa_cert.pem"]), + KeyFileBase = filename:join([proplists:get_value(priv_dir, Config), "ecdsa_key.pem"]), + CaCertFileBase = filename:join([proplists:get_value(priv_dir, Config), "ecdsa_cacerts.pem"]), + CurveOid = hd(tls_v1:ecc_curves(0)), + GenCertData = x509_test:gen_test_certs([{server_key_gen, {namedCurve, CurveOid}}, + {client_key_gen, {namedCurve, CurveOid}}, + {server_key_gen_chain, [{namedCurve, CurveOid}, + {namedCurve, CurveOid}]}, + {client_key_gen_chain, [{namedCurve, CurveOid}, + {namedCurve, CurveOid}]}, + {digest, appropriate_sha(CryptoSupport)}]), + [{server_config, ServerConf}, + {client_config, ClientConf}] = + x509_test:gen_pem_config_files(GenCertData, CertFileBase, KeyFileBase, CaCertFileBase), + [{server_ecdsa_opts, [{ssl_imp, new},{reuseaddr, true} | ServerConf]}, + + {server_ecdsa_verify_opts, [{ssl_imp, new}, {reuseaddr, true}, + {verify, verify_peer} | ServerConf]}, + {client_ecdsa_opts, [{ssl_imp, new}, {reuseaddr, true} | ClientConf]} | Config]; - _ -> + false -> Config end. @@ -540,7 +544,7 @@ make_ecdh_rsa_cert(Config) -> {cacertfile, ClientCaCertFile}, {certfile, ServerCertFile}, {keyfile, ServerKeyFile}, {verify, verify_peer}]}, - {client_ecdh_rsa_opts, [{ssl_imp, new},{reuseaddr, true}, + {client_ecdh_rsa_opts, [{ssl_imp, new}, {cacertfile, ClientCaCertFile}, {certfile, ClientCertFile}, {keyfile, ClientKeyFile}]} | Config]; @@ -560,7 +564,7 @@ make_mix_cert(Config) -> {cacertfile, ClientCaCertFile}, {certfile, ServerCertFile}, {keyfile, ServerKeyFile}, {verify, verify_peer}]}, - {client_mix_opts, [{ssl_imp, new},{reuseaddr, true}, + {client_mix_opts, [{ssl_imp, new}, {cacertfile, ClientCaCertFile}, {certfile, ClientCertFile}, {keyfile, ClientKeyFile}]} | Config]. @@ -787,18 +791,18 @@ no_result(_) -> no_result_msg. trigger_renegotiate(Socket, [ErlData, N]) -> - [{session_id, Id} | _ ] = ssl:session_info(Socket), + {ok, [{session_id, Id}]} = ssl:connection_information(Socket, [session_id]), trigger_renegotiate(Socket, ErlData, N, Id). trigger_renegotiate(Socket, _, 0, Id) -> ct:sleep(1000), - case ssl:session_info(Socket) of - [{session_id, Id} | _ ] -> + case ssl:connection_information(Socket, [session_id]) of + {ok, [{session_id, Id}]} -> fail_session_not_renegotiated; %% Tests that uses this function will not reuse %% sessions so if we get a new session id the %% renegotiation has succeeded. - [{session_id, _} | _ ] -> + {ok, [{session_id, _}]} -> ok; {error, closed} -> fail_session_fatal_alert_during_renegotiation; @@ -830,17 +834,17 @@ rsa_suites(CounterPart) -> ({dhe_rsa, des_cbc, sha}) when FIPS == true -> false; ({rsa, Cipher, _}) -> - lists:member(Cipher, Ciphers); + lists:member(cipher_atom(Cipher), Ciphers); ({dhe_rsa, Cipher, _}) -> - lists:member(Cipher, Ciphers); + lists:member(cipher_atom(Cipher), Ciphers); ({ecdhe_rsa, Cipher, _}) when ECC == true -> - lists:member(Cipher, Ciphers); + lists:member(cipher_atom(Cipher), Ciphers); ({rsa, Cipher, _, _}) -> - lists:member(Cipher, Ciphers); + lists:member(cipher_atom(Cipher), Ciphers); ({dhe_rsa, Cipher, _,_}) -> - lists:member(Cipher, Ciphers); + lists:member(cipher_atom(Cipher), Ciphers); ({ecdhe_rsa, Cipher, _,_}) when ECC == true -> - lists:member(Cipher, Ciphers); + lists:member(cipher_atom(Cipher), Ciphers); (_) -> false end, @@ -933,44 +937,12 @@ anonymous_suites(Version) -> Suites = ssl_cipher:anonymous_suites(Version), ssl_cipher:filter_suites(Suites). -psk_suites() -> - Suites = - [{psk, rc4_128, sha}, - {psk, '3des_ede_cbc', sha}, - {psk, aes_128_cbc, sha}, - {psk, aes_256_cbc, sha}, - {psk, aes_128_cbc, sha256}, - {psk, aes_256_cbc, sha384}, - {dhe_psk, rc4_128, sha}, - {dhe_psk, '3des_ede_cbc', sha}, - {dhe_psk, aes_128_cbc, sha}, - {dhe_psk, aes_256_cbc, sha}, - {dhe_psk, aes_128_cbc, sha256}, - {dhe_psk, aes_256_cbc, sha384}, - {rsa_psk, rc4_128, sha}, - {rsa_psk, '3des_ede_cbc', sha}, - {rsa_psk, aes_128_cbc, sha}, - {rsa_psk, aes_256_cbc, sha}, - {rsa_psk, aes_128_cbc, sha256}, - {rsa_psk, aes_256_cbc, sha384}, - {psk, aes_128_gcm, null, sha256}, - {psk, aes_256_gcm, null, sha384}, - {dhe_psk, aes_128_gcm, null, sha256}, - {dhe_psk, aes_256_gcm, null, sha384}, - {rsa_psk, aes_128_gcm, null, sha256}, - {rsa_psk, aes_256_gcm, null, sha384}], +psk_suites(Version) -> + Suites = ssl_cipher:psk_suites(Version), ssl_cipher:filter_suites(Suites). -psk_anon_suites() -> - Suites = - [{psk, rc4_128, sha}, - {psk, '3des_ede_cbc', sha}, - {psk, aes_128_cbc, sha}, - {psk, aes_256_cbc, sha}, - {dhe_psk, rc4_128, sha}, - {dhe_psk, '3des_ede_cbc', sha}, - {dhe_psk, aes_128_cbc, sha}, - {dhe_psk, aes_256_cbc, sha}], +psk_anon_suites(Version) -> + Suites = [Suite || Suite <- psk_suites(Version), is_psk_anon_suite(Suite)], ssl_cipher:filter_suites(Suites). srp_suites() -> @@ -1035,8 +1007,8 @@ cipher_result(Socket, Result) -> end. session_info_result(Socket) -> - ssl:session_info(Socket). - + {ok, Info} = ssl:connection_information(Socket, [session_id, cipher_suite]), + Info. public_key(#'PrivateKeyInfo'{privateKeyAlgorithm = #'PrivateKeyInfo_privateKeyAlgorithm'{algorithm = ?rsaEncryption}, @@ -1092,14 +1064,16 @@ init_tls_version(Version, Config) application:load(ssl), application:set_env(ssl, dtls_protocol_version, Version), ssl:start(), - [{protocol, dtls}, {protocol_opts, [{protocol, dtls}]}|Config]; + NewConfig = proplists:delete(protocol_opts, proplists:delete(protocol, Config)), + [{protocol, dtls}, {protocol_opts, [{protocol, dtls}]} | NewConfig]; init_tls_version(Version, Config) -> ssl:stop(), application:load(ssl), application:set_env(ssl, protocol_version, Version), ssl:start(), - [{protocol, tls}|Config]. + NewConfig = proplists:delete(protocol_opts, proplists:delete(protocol, Config)), + [{protocol, tls} | NewConfig]. sufficient_crypto_support(Version) when Version == 'tlsv1.2'; Version == 'dtlsv1.2' -> @@ -1225,6 +1199,10 @@ check_sane_openssl_version(Version) -> false; {'tlsv1.1', "OpenSSL 0" ++ _} -> false; + {'dtlsv1', "OpenSSL 0" ++ _} -> + false; + {'dtlsv1.2', "OpenSSL 0" ++ _} -> + false; {_, _} -> true end; @@ -1234,19 +1212,37 @@ check_sane_openssl_version(Version) -> enough_openssl_crl_support("OpenSSL 0." ++ _) -> false; enough_openssl_crl_support(_) -> true. -wait_for_openssl_server(Port) -> - wait_for_openssl_server(Port, 10). -wait_for_openssl_server(_, 0) -> +wait_for_openssl_server(Port, tls) -> + do_wait_for_openssl_tls_server(Port, 10); +wait_for_openssl_server(Port, dtls) -> + do_wait_for_openssl_dtls_server(Port, 10). + +do_wait_for_openssl_tls_server(_, 0) -> exit(failed_to_connect_to_openssl); -wait_for_openssl_server(Port, N) -> +do_wait_for_openssl_tls_server(Port, N) -> case gen_tcp:connect("localhost", Port, []) of {ok, S} -> gen_tcp:close(S); _ -> ct:sleep(?SLEEP), - wait_for_openssl_server(Port, N-1) + do_wait_for_openssl_tls_server(Port, N-1) end. +do_wait_for_openssl_dtls_server(_, 0) -> + %%exit(failed_to_connect_to_openssl); + ok; +do_wait_for_openssl_dtls_server(Port, N) -> + %% case gen_udp:open(0) of + %% {ok, S} -> + %% gen_udp:connect(S, "localhost", Port), + %% gen_udp:close(S); + %% _ -> + %% ct:sleep(?SLEEP), + %% do_wait_for_openssl_dtls_server(Port, N-1) + %% end. + ct:sleep(500), + do_wait_for_openssl_dtls_server(Port, N-1). + version_flag(tlsv1) -> "-tls1"; version_flag('tlsv1.1') -> @@ -1256,10 +1252,14 @@ version_flag('tlsv1.2') -> version_flag(sslv3) -> "-ssl3"; version_flag(sslv2) -> - "-ssl2". - -filter_suites(Ciphers0) -> - Version = tls_record:highest_protocol_version([]), + "-ssl2"; +version_flag('dtlsv1.2') -> + "-dtls1_2"; +version_flag('dtlsv1') -> + "-dtls1". + +filter_suites(Ciphers0, AtomVersion) -> + Version = tls_version(AtomVersion), Supported0 = ssl_cipher:suites(Version) ++ ssl_cipher:anonymous_suites(Version) ++ ssl_cipher:psk_suites(Version) @@ -1341,7 +1341,7 @@ protocol_version(Config) -> protocol_version(Config, tuple) -> case proplists:get_value(protocol, Config) of dtls -> - dtls_record:protocol_version(dtls_record:highest_protocol_version([])); + dtls_record:highest_protocol_version(dtls_record:supported_protocol_versions()); _ -> tls_record:highest_protocol_version(tls_record:supported_protocol_versions()) end; @@ -1375,6 +1375,7 @@ clean_env() -> application:unset_env(ssl, session_cache_client_max), application:unset_env(ssl, session_cache_server_max), application:unset_env(ssl, ssl_pem_cache_clean), + application:unset_env(ssl, bypass_pem_cache), application:unset_env(ssl, alert_timeout). clean_start() -> @@ -1382,3 +1383,105 @@ clean_start() -> application:load(ssl), clean_env(), ssl:start(). + +is_psk_anon_suite({psk, _,_}) -> + true; +is_psk_anon_suite({dhe_psk,_,_}) -> + true; +is_psk_anon_suite({psk, _,_,_}) -> + true; +is_psk_anon_suite({dhe_psk, _,_,_}) -> + true; +is_psk_anon_suite(_) -> + false. + +cipher_atom(aes_256_cbc) -> + aes_cbc256; +cipher_atom(aes_128_cbc) -> + aes_cbc128; +cipher_atom('3des_ede_cbc') -> + des_ede3; +cipher_atom(Atom) -> + Atom. +tls_version('dtlsv1' = Atom) -> + dtls_v1:corresponding_tls_version(dtls_record:protocol_version(Atom)); +tls_version('dtlsv1.2' = Atom) -> + dtls_v1:corresponding_tls_version(dtls_record:protocol_version(Atom)); +tls_version(Atom) -> + tls_record:protocol_version(Atom). + +dtls_hello() -> + [1, + <<0,1,4>>, + <<0,0>>, + <<0,0,0>>, + <<0,1,4>>, + <<254,253,88, + 156,129,61, + 131,216,15, + 131,194,242, + 46,154,190, + 20,228,234, + 234,150,44, + 62,96,96,103, + 127,95,103, + 23,24,42,138, + 13,142,32,57, + 230,177,32, + 210,154,152, + 188,121,134, + 136,53,105, + 118,96,106, + 103,231,223, + 133,10,165, + 50,32,211, + 227,193,14, + 181,143,48, + 66,0,0,100,0, + 255,192,44, + 192,48,192, + 36,192,40, + 192,46,192, + 50,192,38, + 192,42,0,159, + 0,163,0,107, + 0,106,0,157, + 0,61,192,43, + 192,47,192, + 35,192,39, + 192,45,192, + 49,192,37, + 192,41,0,158, + 0,162,0,103, + 0,64,0,156,0, + 60,192,10, + 192,20,0,57, + 0,56,192,5, + 192,15,0,53, + 192,8,192,18, + 0,22,0,19, + 192,3,192,13, + 0,10,192,9, + 192,19,0,51, + 0,50,192,4, + 192,14,0,47, + 1,0,0,86,0,0, + 0,14,0,12,0, + 0,9,108,111, + 99,97,108, + 104,111,115, + 116,0,10,0, + 58,0,56,0,14, + 0,13,0,25,0, + 28,0,11,0,12, + 0,27,0,24,0, + 9,0,10,0,26, + 0,22,0,23,0, + 8,0,6,0,7,0, + 20,0,21,0,4, + 0,5,0,18,0, + 19,0,1,0,2,0, + 3,0,15,0,16, + 0,17,0,11,0, + 2,1,0>>]. + diff --git a/lib/ssl/test/ssl_to_openssl_SUITE.erl b/lib/ssl/test/ssl_to_openssl_SUITE.erl index e99340822d..48fd2b7eab 100644 --- a/lib/ssl/test/ssl_to_openssl_SUITE.erl +++ b/lib/ssl/test/ssl_to_openssl_SUITE.erl @@ -42,7 +42,9 @@ all() -> {group, 'tlsv1.2'}, {group, 'tlsv1.1'}, {group, 'tlsv1'}, - {group, 'sslv3'} + {group, 'sslv3'}, + {group, 'dtlsv1.2'}, + {group, 'dtlsv1'} ]. groups() -> @@ -50,7 +52,10 @@ groups() -> {'tlsv1.2', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'tlsv1.1', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'tlsv1', [], all_versions_tests()++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, - {'sslv3', [], all_versions_tests()}]. + {'sslv3', [], all_versions_tests()}, + {'dtlsv1.2', [], dtls_all_versions_tests()}, + {'dtlsv1', [], dtls_all_versions_tests()} + ]. basic_tests() -> [basic_erlang_client_openssl_server, @@ -78,6 +83,24 @@ all_versions_tests() -> expired_session, ssl2_erlang_server_openssl_client ]. +dtls_all_versions_tests() -> + [ + %%erlang_client_openssl_server, + erlang_server_openssl_client, + %%erlang_client_openssl_server_dsa_cert, + erlang_server_openssl_client_dsa_cert, + erlang_server_openssl_client_reuse_session + %%erlang_client_openssl_server_renegotiate, + %%erlang_client_openssl_server_nowrap_seqnum, + %%erlang_server_openssl_client_nowrap_seqnum, + %%erlang_client_openssl_server_no_server_ca_cert, + %%erlang_client_openssl_server_client_cert, + %%erlang_server_openssl_client_client_cert + %%ciphers_rsa_signed_certs, + %%ciphers_dsa_signed_certs, + %%erlang_client_bad_openssl_server, + %%expired_session + ]. alpn_tests() -> [erlang_client_alpn_openssl_server_alpn, @@ -144,13 +167,18 @@ init_per_group(basic, Config) -> init_per_group(GroupName, Config) -> case ssl_test_lib:is_tls_version(GroupName) of true -> - case ssl_test_lib:check_sane_openssl_version(GroupName) of - true -> - ssl_test_lib:init_tls_version(GroupName, Config); - false -> - {skip, openssl_does_not_support_version} - end; - _ -> + case ssl_test_lib:supports_ssl_tls_version(GroupName) of + true -> + case ssl_test_lib:check_sane_openssl_version(GroupName) of + true -> + ssl_test_lib:init_tls_version(GroupName, Config); + false -> + {skip, openssl_does_not_support_version} + end; + false -> + {skip, openssl_does_not_support_version} + end; + _ -> ssl:start(), Config end. @@ -284,7 +312,8 @@ basic_erlang_client_openssl_server(Config) when is_list(Config) -> OpensslPort = ssl_test_lib:portable_open_port(Exe, Args), - ssl_test_lib:wait_for_openssl_server(Port), + + ssl_test_lib:wait_for_openssl_server(Port, tls), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, @@ -357,7 +386,7 @@ erlang_client_openssl_server(Config) when is_list(Config) -> OpensslPort = ssl_test_lib:portable_open_port(Exe, Args), - ssl_test_lib:wait_for_openssl_server(Port), + ssl_test_lib:wait_for_openssl_server(Port, proplists:get_value(protocol, Config)), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, @@ -431,7 +460,7 @@ erlang_client_openssl_server_dsa_cert(Config) when is_list(Config) -> OpensslPort = ssl_test_lib:portable_open_port(Exe, Args), - ssl_test_lib:wait_for_openssl_server(Port), + ssl_test_lib:wait_for_openssl_server(Port, proplists:get_value(protocol, Config)), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, @@ -551,7 +580,7 @@ erlang_client_openssl_server_renegotiate(Config) when is_list(Config) -> OpensslPort = ssl_test_lib:portable_open_port(Exe, Args), - ssl_test_lib:wait_for_openssl_server(Port), + ssl_test_lib:wait_for_openssl_server(Port, proplists:get_value(protocol, Config)), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, @@ -600,7 +629,7 @@ erlang_client_openssl_server_nowrap_seqnum(Config) when is_list(Config) -> OpensslPort = ssl_test_lib:portable_open_port(Exe, Args), - ssl_test_lib:wait_for_openssl_server(Port), + ssl_test_lib:wait_for_openssl_server(Port, proplists:get_value(protocol, Config)), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, @@ -681,7 +710,7 @@ erlang_client_openssl_server_no_server_ca_cert(Config) when is_list(Config) -> OpensslPort = ssl_test_lib:portable_open_port(Exe, Args), - ssl_test_lib:wait_for_openssl_server(Port), + ssl_test_lib:wait_for_openssl_server(Port, proplists:get_value(protocol, Config)), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, @@ -724,7 +753,7 @@ erlang_client_openssl_server_client_cert(Config) when is_list(Config) -> OpensslPort = ssl_test_lib:portable_open_port(Exe, Args), - ssl_test_lib:wait_for_openssl_server(Port), + ssl_test_lib:wait_for_openssl_server(Port, proplists:get_value(protocol, Config)), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, @@ -856,7 +885,7 @@ erlang_client_bad_openssl_server(Config) when is_list(Config) -> "-cert", CertFile, "-key", KeyFile], OpensslPort = ssl_test_lib:portable_open_port(Exe, Args), - ssl_test_lib:wait_for_openssl_server(Port), + ssl_test_lib:wait_for_openssl_server(Port, proplists:get_value(protocol, Config)), Client0 = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, @@ -911,7 +940,7 @@ expired_session(Config) when is_list(Config) -> OpensslPort = ssl_test_lib:portable_open_port(Exe, Args), - ssl_test_lib:wait_for_openssl_server(Port), + ssl_test_lib:wait_for_openssl_server(Port, tls), Client0 = ssl_test_lib:start_client([{node, ClientNode}, @@ -970,20 +999,7 @@ ssl2_erlang_server_openssl_client(Config) when is_list(Config) -> true = port_command(OpenSslPort, Data), ct:log("Ports ~p~n", [[erlang:port_info(P) || P <- erlang:ports()]]), - receive - {'EXIT', OpenSslPort, _} = Exit -> - ct:log("Received: ~p ~n", [Exit]), - ok - end, - receive - {'EXIT', _, _} = UnkownExit -> - Msg = lists:flatten(io_lib:format("Received: ~p ~n", [UnkownExit])), - ct:log(Msg), - ct:comment(Msg), - ok - after 0 -> - ok - end, + consume_port_exit(OpenSslPort), ssl_test_lib:check_result(Server, {error, {tls_alert, "handshake failure"}}), process_flag(trap_exit, false). %%-------------------------------------------------------------------- @@ -1014,20 +1030,7 @@ ssl2_erlang_server_openssl_client_comp(Config) when is_list(Config) -> true = port_command(OpenSslPort, Data), ct:log("Ports ~p~n", [[erlang:port_info(P) || P <- erlang:ports()]]), - receive - {'EXIT', OpenSslPort, _} = Exit -> - ct:log("Received: ~p ~n", [Exit]), - ok - end, - receive - {'EXIT', _, _} = UnkownExit -> - Msg = lists:flatten(io_lib:format("Received: ~p ~n", [UnkownExit])), - ct:log(Msg), - ct:comment(Msg), - ok - after 0 -> - ok - end, + consume_port_exit(OpenSslPort), ssl_test_lib:check_result(Server, {error, {tls_alert, "protocol version"}}), process_flag(trap_exit, false). @@ -1399,7 +1402,7 @@ cipher(CipherSuite, Version, Config, ClientOpts, ServerOpts) -> OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), - ssl_test_lib:wait_for_openssl_server(Port), + ssl_test_lib:wait_for_openssl_server(Port, proplists:get_value(protocol, Config)), ConnectionInfo = {ok, {Version, CipherSuite}}, @@ -1469,7 +1472,7 @@ start_erlang_client_and_openssl_server_with_opts(Config, ErlangClientOpts, Opens OpensslPort = ssl_test_lib:portable_open_port(Exe, Args), - ssl_test_lib:wait_for_openssl_server(Port), + ssl_test_lib:wait_for_openssl_server(Port, proplists:get_value(protocol, Config)), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, @@ -1505,7 +1508,7 @@ start_erlang_client_and_openssl_server_for_alpn_negotiation(Config, Data, Callba Args = ["s_server", "-msg", "-alpn", "http/1.1,spdy/2", "-accept", integer_to_list(Port), ssl_test_lib:version_flag(Version), "-cert", CertFile, "-key", KeyFile], OpensslPort = ssl_test_lib:portable_open_port(Exe, Args), - ssl_test_lib:wait_for_openssl_server(Port), + ssl_test_lib:wait_for_openssl_server(Port, proplists:get_value(protocol, Config)), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, @@ -1574,7 +1577,7 @@ start_erlang_client_and_openssl_server_for_alpn_npn_negotiation(Config, Data, Ca OpensslPort = ssl_test_lib:portable_open_port(Exe, Args), - ssl_test_lib:wait_for_openssl_server(Port), + ssl_test_lib:wait_for_openssl_server(Port, proplists:get_value(protocol, Config)), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, @@ -1639,7 +1642,7 @@ start_erlang_client_and_openssl_server_for_npn_negotiation(Config, Data, Callbac "-cert", CertFile, "-key", KeyFile], OpensslPort = ssl_test_lib:portable_open_port(Exe, Args), - ssl_test_lib:wait_for_openssl_server(Port), + ssl_test_lib:wait_for_openssl_server(Port, proplists:get_value(protocol, Config)), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, @@ -1848,3 +1851,9 @@ openssl_client_args(false, Hostname, Port, ServerName) -> openssl_client_args(true, Hostname, Port, ServerName) -> ["s_client", "-no_ssl2", "-connect", Hostname ++ ":" ++ integer_to_list(Port), "-servername", ServerName]. + +consume_port_exit(OpenSSLPort) -> + receive + {'EXIT', OpenSSLPort, _} -> + ok + end. diff --git a/lib/ssl/test/x509_test.erl b/lib/ssl/test/x509_test.erl new file mode 100644 index 0000000000..5cd5c8eca7 --- /dev/null +++ b/lib/ssl/test/x509_test.erl @@ -0,0 +1,310 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% + +-module(x509_test). + +-include_lib("public_key/include/public_key.hrl"). + +-export([gen_test_certs/1, gen_pem_config_files/4]). + +gen_test_certs(Opts) -> + SRootKey = gen_key(proplists:get_value(server_key_gen, Opts)), + CRootKey = gen_key(proplists:get_value(client_key_gen, Opts)), + ServerRoot = root_cert("server", SRootKey, Opts), + ClientRoot = root_cert("client", CRootKey, Opts), + [{ServerCert, ServerKey} | ServerCAsKeys] = config(server, ServerRoot, SRootKey, Opts), + [{ClientCert, ClientKey} | ClientCAsKeys] = config(client, ClientRoot, CRootKey, Opts), + ServerCAs = ca_config(ClientRoot, ServerCAsKeys), + ClientCAs = ca_config(ServerRoot, ClientCAsKeys), + [{server_config, [{cert, ServerCert}, {key, ServerKey}, {cacerts, ServerCAs}]}, + {client_config, [{cert, ClientCert}, {key, ClientKey}, {cacerts, ClientCAs}]}]. + +gen_pem_config_files(GenCertData, CertFileBase, KeyFileBase, CAFileBase) -> + ServerConf = proplists:get_value(server_config, GenCertData), + ClientConf = proplists:get_value(client_config, GenCertData), + + ServerCaCertFile = filename:join("server_", CAFileBase), + ServerCertFile = filename:join("server_", CertFileBase), + ServerKeyFile = filename:join("server_", KeyFileBase), + + ClientCaCertFile = filename:join("client_", CAFileBase), + ClientCertFile = filename:join("client_", CertFileBase), + ClientKeyFile = filename:join("client_", KeyFileBase), + + do_gen_pem_config_files(ServerConf, + ServerCertFile, + ServerKeyFile, + ServerCaCertFile), + do_gen_pem_config_files(ClientConf, + ClientCertFile, + ClientKeyFile, + ClientCaCertFile), + [{server_config, [{certfile, ServerCertFile}, {keyfile, ServerKeyFile}, {cacertfile, ServerCaCertFile}]}, + {client_config, [{certfile, ClientCertFile}, {keyfile, ClientKeyFile}, {cacertfile, ClientCaCertFile}]}]. + + +do_gen_pem_config_files(Config, CertFile, KeyFile, CAFile) -> + CAs = proplists:get_value(cacerts, Config), + Cert = proplists:get_value(cert, Config), + Key = proplists:get_value(key, Config), + der_to_pem(CertFile, [cert_entry(Cert)]), + der_to_pem(KeyFile, [key_entry(Key)]), + der_to_pem(CAFile, ca_entries(CAs)). + +cert_entry(Cert) -> + {'Certificate', Cert, not_encrypted}. + +key_entry(Key = #'RSAPrivateKey'{}) -> + Der = public_key:der_encode('RSAPrivateKey', Key), + {'RSAPrivateKey', Der, not_encrypted}; +key_entry(Key = #'DSAPrivateKey'{}) -> + Der = public_key:der_encode('DSAPrivateKey', Key), + {'DSAPrivateKey', Der, not_encrypted}; +key_entry(Key = #'ECPrivateKey'{}) -> + Der = public_key:der_encode('ECPrivateKey', Key), + {'ECPrivateKey', Der, not_encrypted}. + +ca_entries(CAs) -> + [{'Certificate', CACert, not_encrypted} || CACert <- CAs]. + +gen_key(KeyGen) -> + case is_key(KeyGen) of + true -> + KeyGen; + false -> + public_key:generate_key(KeyGen) + end. + +root_cert(Role, PrivKey, Opts) -> + TBS = cert_template(), + Issuer = issuer("root", Role, " ROOT CA"), + OTPTBS = TBS#'OTPTBSCertificate'{ + signature = sign_algorithm(PrivKey, Opts), + issuer = Issuer, + validity = validity(Opts), + subject = Issuer, + subjectPublicKeyInfo = public_key(PrivKey), + extensions = extensions(Opts) + }, + public_key:pkix_sign(OTPTBS, PrivKey). + +config(Role, Root, Key, Opts) -> + KeyGenOpt = list_to_atom(atom_to_list(Role) ++ "key_gen_chain"), + KeyGens = proplists:get_value(KeyGenOpt, Opts, [{namedCurve, hd(tls_v1:ecc_curves(0))}, + {namedCurve, hd(tls_v1:ecc_curves(0))}]), + Keys = lists:map(fun gen_key/1, KeyGens), + cert_chain(Role, Root, Key, Opts, Keys). + +cert_template() -> + #'OTPTBSCertificate'{ + version = v3, + serialNumber = trunc(rand:uniform()*100000000)*10000 + 1, + issuerUniqueID = asn1_NOVALUE, + subjectUniqueID = asn1_NOVALUE + }. + +issuer(Contact, Role, Name) -> + subject(Contact, Role ++ Name). + +subject(Contact, Name) -> + Opts = [{email, Contact ++ "@erlang.org"}, + {name, Name}, + {city, "Stockholm"}, + {country, "SE"}, + {org, "erlang"}, + {org_unit, "automated testing"}], + subject(Opts). + +subject(SubjectOpts) when is_list(SubjectOpts) -> + Encode = fun(Opt) -> + {Type,Value} = subject_enc(Opt), + [#'AttributeTypeAndValue'{type=Type, value=Value}] + end, + {rdnSequence, [Encode(Opt) || Opt <- SubjectOpts]}. + +subject_enc({name, Name}) -> + {?'id-at-commonName', {printableString, Name}}; +subject_enc({email, Email}) -> + {?'id-emailAddress', Email}; +subject_enc({city, City}) -> + {?'id-at-localityName', {printableString, City}}; +subject_enc({state, State}) -> + {?'id-at-stateOrProvinceName', {printableString, State}}; +subject_enc({org, Org}) -> + {?'id-at-organizationName', {printableString, Org}}; +subject_enc({org_unit, OrgUnit}) -> + {?'id-at-organizationalUnitName', {printableString, OrgUnit}}; +subject_enc({country, Country}) -> + {?'id-at-countryName', Country}; +subject_enc({serial, Serial}) -> + {?'id-at-serialNumber', Serial}; +subject_enc({title, Title}) -> + {?'id-at-title', {printableString, Title}}; +subject_enc({dnQualifer, DnQ}) -> + {?'id-at-dnQualifier', DnQ}; +subject_enc(Other) -> + Other. + +validity(Opts) -> + DefFrom0 = calendar:gregorian_days_to_date(calendar:date_to_gregorian_days(date())-1), + DefTo0 = calendar:gregorian_days_to_date(calendar:date_to_gregorian_days(date())+7), + {DefFrom, DefTo} = proplists:get_value(validity, Opts, {DefFrom0, DefTo0}), + Format = fun({Y,M,D}) -> + lists:flatten(io_lib:format("~w~2..0w~2..0w000000Z",[Y,M,D])) + end, + #'Validity'{notBefore={generalTime, Format(DefFrom)}, + notAfter ={generalTime, Format(DefTo)}}. + +extensions(Opts) -> + case proplists:get_value(extensions, Opts, []) of + false -> + asn1_NOVALUE; + Exts -> + lists:flatten([extension(Ext) || Ext <- default_extensions(Exts)]) + end. + +default_extensions(Exts) -> + Def = [{key_usage,undefined}, + {subject_altname, undefined}, + {issuer_altname, undefined}, + {basic_constraints, default}, + {name_constraints, undefined}, + {policy_constraints, undefined}, + {ext_key_usage, undefined}, + {inhibit_any, undefined}, + {auth_key_id, undefined}, + {subject_key_id, undefined}, + {policy_mapping, undefined}], + Filter = fun({Key, _}, D) -> + lists:keydelete(Key, 1, D) + end, + Exts ++ lists:foldl(Filter, Def, Exts). + +extension({_, undefined}) -> + []; +extension({basic_constraints, Data}) -> + case Data of + default -> + #'Extension'{extnID = ?'id-ce-basicConstraints', + extnValue = #'BasicConstraints'{cA=true}, + critical=true}; + false -> + []; + Len when is_integer(Len) -> + #'Extension'{extnID = ?'id-ce-basicConstraints', + extnValue = #'BasicConstraints'{cA=true, pathLenConstraint = Len}, + critical = true}; + _ -> + #'Extension'{extnID = ?'id-ce-basicConstraints', + extnValue = Data} + end; +extension({Id, Data, Critical}) -> + #'Extension'{extnID = Id, extnValue = Data, critical = Critical}. + +public_key(#'RSAPrivateKey'{modulus=N, publicExponent=E}) -> + Public = #'RSAPublicKey'{modulus=N, publicExponent=E}, + Algo = #'PublicKeyAlgorithm'{algorithm= ?rsaEncryption, parameters='NULL'}, + #'OTPSubjectPublicKeyInfo'{algorithm = Algo, + subjectPublicKey = Public}; +public_key(#'DSAPrivateKey'{p=P, q=Q, g=G, y=Y}) -> + Algo = #'PublicKeyAlgorithm'{algorithm= ?'id-dsa', + parameters={params, #'Dss-Parms'{p=P, q=Q, g=G}}}, + #'OTPSubjectPublicKeyInfo'{algorithm = Algo, subjectPublicKey = Y}; +public_key(#'ECPrivateKey'{version = _Version, + privateKey = _PrivKey, + parameters = Params, + publicKey = PubKey}) -> + Algo = #'PublicKeyAlgorithm'{algorithm= ?'id-ecPublicKey', parameters=Params}, + #'OTPSubjectPublicKeyInfo'{algorithm = Algo, + subjectPublicKey = #'ECPoint'{point = PubKey}}. + +sign_algorithm(#'RSAPrivateKey'{}, Opts) -> + Type = rsa_digest_oid(proplists:get_value(digest, Opts, sha1)), + #'SignatureAlgorithm'{algorithm = Type, + parameters = 'NULL'}; +sign_algorithm(#'DSAPrivateKey'{p=P, q=Q, g=G}, _Opts) -> + #'SignatureAlgorithm'{algorithm = ?'id-dsa-with-sha1', + parameters = {params,#'Dss-Parms'{p=P, q=Q, g=G}}}; +sign_algorithm(#'ECPrivateKey'{parameters = Parms}, Opts) -> + Type = ecdsa_digest_oid(proplists:get_value(digest, Opts, sha1)), + #'SignatureAlgorithm'{algorithm = Type, + parameters = Parms}. + +rsa_digest_oid(sha1) -> + ?'sha1WithRSAEncryption'; +rsa_digest_oid(sha512) -> + ?'sha512WithRSAEncryption'; +rsa_digest_oid(sha384) -> + ?'sha384WithRSAEncryption'; +rsa_digest_oid(sha256) -> + ?'sha256WithRSAEncryption'; +rsa_digest_oid(md5) -> + ?'md5WithRSAEncryption'. + +ecdsa_digest_oid(sha1) -> + ?'ecdsa-with-SHA1'; +ecdsa_digest_oid(sha512) -> + ?'ecdsa-with-SHA512'; +ecdsa_digest_oid(sha384) -> + ?'ecdsa-with-SHA384'; +ecdsa_digest_oid(sha256) -> + ?'ecdsa-with-SHA256'. + +ca_config(Root, CAsKeys) -> + [Root | [CA || {CA, _} <- CAsKeys]]. + +cert_chain(Role, Root, RootKey, Opts, Keys) -> + cert_chain(Role, Root, RootKey, Opts, Keys, 0, []). + +cert_chain(Role, IssuerCert, IssuerKey, Opts, [Key], _, Acc) -> + Cert = cert(Role, public_key:pkix_decode_cert(IssuerCert, otp), IssuerKey, Key, "admin", " Peer cert", Opts), + [{Cert, Key}, {IssuerCert, IssuerKey} | Acc]; +cert_chain(Role, IssuerCert, IssuerKey, Opts, [Key | Keys], N, Acc) -> + Cert = cert(Role, public_key:pkix_decode_cert(IssuerCert, otp), IssuerKey, Key, "webadmin", + " Intermidiate CA " ++ integer_to_list(N), Opts), + cert_chain(Role, Cert, Key, Opts, Keys, N+1, [{IssuerCert, IssuerKey} | Acc]). + +cert(Role, #'OTPCertificate'{tbsCertificate = #'OTPTBSCertificate'{subject = Issuer}}, + PrivKey, Key, Contact, Name, Opts) -> + TBS = cert_template(), + OTPTBS = TBS#'OTPTBSCertificate'{ + signature = sign_algorithm(PrivKey, Opts), + issuer = Issuer, + validity = validity(Opts), + subject = subject(Contact, atom_to_list(Role) ++ Name), + subjectPublicKeyInfo = public_key(Key), + extensions = extensions(Opts) + }, + public_key:pkix_sign(OTPTBS, PrivKey). + +is_key(#'DSAPrivateKey'{}) -> + true; +is_key(#'RSAPrivateKey'{}) -> + true; +is_key(#'ECPrivateKey'{}) -> + true; +is_key(_) -> + false. + +der_to_pem(File, Entries) -> + PemBin = public_key:pem_encode(Entries), + file:write_file(File, PemBin). diff --git a/lib/ssl/vsn.mk b/lib/ssl/vsn.mk index 2cdb825d75..82184f5c74 100644 --- a/lib/ssl/vsn.mk +++ b/lib/ssl/vsn.mk @@ -1 +1 @@ -SSL_VSN = 8.1 +SSL_VSN = 8.1.2 diff --git a/lib/stdlib/doc/src/assert_hrl.xml b/lib/stdlib/doc/src/assert_hrl.xml index 57bb5207df..ea23cca2ee 100644 --- a/lib/stdlib/doc/src/assert_hrl.xml +++ b/lib/stdlib/doc/src/assert_hrl.xml @@ -4,7 +4,7 @@ <fileref> <header> <copyright> - <year>2012</year><year>2016</year> + <year>2012</year><year>2017</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -92,18 +92,21 @@ erlc -DNOASSERT=true *.erl</code> <title>Macros</title> <taglist> <tag><c>assert(BoolExpr)</c></tag> - <tag><c>assert(BoolExpr, Comment)</c></tag> + <item></item> + <tag><c>URKAassert(BoolExpr, Comment)</c></tag> <item> <p>Tests that <c>BoolExpr</c> completes normally returning <c>true</c>.</p> </item> <tag><c>assertNot(BoolExpr)</c></tag> + <item></item> <tag><c>assertNot(BoolExpr, Comment)</c></tag> <item> <p>Tests that <c>BoolExpr</c> completes normally returning <c>false</c>.</p> </item> <tag><c>assertMatch(GuardedPattern, Expr)</c></tag> + <item></item> <tag><c>assertMatch(GuardedPattern, Expr, Comment)</c></tag> <item> <p>Tests that <c>Expr</c> completes normally yielding a value that @@ -115,6 +118,7 @@ erlc -DNOASSERT=true *.erl</code> ?assertMatch({bork, X} when X > 0, f())</code> </item> <tag><c>assertNotMatch(GuardedPattern, Expr)</c></tag> + <item></item> <tag><c>assertNotMatch(GuardedPattern, Expr, Comment)</c></tag> <item> <p>Tests that <c>Expr</c> completes normally yielding a value that does @@ -123,18 +127,21 @@ erlc -DNOASSERT=true *.erl</code> <c>when</c> part.</p> </item> <tag><c>assertEqual(ExpectedValue, Expr)</c></tag> + <item></item> <tag><c>assertEqual(ExpectedValue, Expr, Comment)</c></tag> <item> <p>Tests that <c>Expr</c> completes normally yielding a value that is exactly equal to <c>ExpectedValue</c>.</p> </item> <tag><c>assertNotEqual(ExpectedValue, Expr)</c></tag> + <item></item> <tag><c>assertNotEqual(ExpectedValue, Expr, Comment)</c></tag> <item> <p>Tests that <c>Expr</c> completes normally yielding a value that is not exactly equal to <c>ExpectedValue</c>.</p> </item> <tag><c>assertException(Class, Term, Expr)</c></tag> + <item></item> <tag><c>assertException(Class, Term, Expr, Comment)</c></tag> <item> <p>Tests that <c>Expr</c> completes abnormally with an exception of type @@ -145,6 +152,7 @@ erlc -DNOASSERT=true *.erl</code> patterns, as in <c>assertMatch</c>.</p> </item> <tag><c>assertNotException(Class, Term, Expr)</c></tag> + <item></item> <tag><c>assertNotException(Class, Term, Expr, Comment)</c></tag> <item> <p>Tests that <c>Expr</c> does not evaluate abnormally with an @@ -155,16 +163,19 @@ erlc -DNOASSERT=true *.erl</code> be guarded patterns.</p> </item> <tag><c>assertError(Term, Expr)</c></tag> + <item></item> <tag><c>assertError(Term, Expr, Comment)</c></tag> <item> <p>Equivalent to <c>assertException(error, Term, Expr)</c></p> </item> <tag><c>assertExit(Term, Expr)</c></tag> + <item></item> <tag><c>assertExit(Term, Expr, Comment)</c></tag> <item> <p>Equivalent to <c>assertException(exit, Term, Expr)</c></p> </item> <tag><c>assertThrow(Term, Expr)</c></tag> + <item></item> <tag><c>assertThrow(Term, Expr, Comment)</c></tag> <item> <p>Equivalent to <c>assertException(throw, Term, Expr)</c></p> diff --git a/lib/stdlib/doc/src/ets.xml b/lib/stdlib/doc/src/ets.xml index 05401a2d40..d1ec176f81 100644 --- a/lib/stdlib/doc/src/ets.xml +++ b/lib/stdlib/doc/src/ets.xml @@ -1491,6 +1491,25 @@ is_integer(X), is_integer(Y), X + Y < 4711]]></code> </func> <func> + <name name="select_replace" arity="2"/> + <fsummary>Match and replace objects atomically in an ETS table</fsummary> + <desc> + <p>Matches the objects in the table <c><anno>Tab</anno></c> using a + <seealso marker="#match_spec">match specification</seealso>. If + an object is matched, the existing object is replaced with + the match specification result, which <em>must</em> retain + the original key or the operation will fail with <c>badarg</c>.</p> + <p>For the moment, due to performance and semantic constraints, + tables of type <c>bag</c> are not yet supported.</p> + <p>The function returns the total number of replaced objects.</p> + <note> + <p>The match/replacement operation atomicity scope is limited + to each individual object.</p> + </note> + </desc> + </func> + + <func> <name name="select_reverse" arity="1"/> <fsummary>Continue matching objects in an ETS table.</fsummary> <desc> diff --git a/lib/stdlib/doc/src/gen_statem.xml b/lib/stdlib/doc/src/gen_statem.xml index fd498ee82e..5eb13db1aa 100644 --- a/lib/stdlib/doc/src/gen_statem.xml +++ b/lib/stdlib/doc/src/gen_statem.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2016</year> + <year>2016-2017</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -587,8 +587,8 @@ handle_event(_, _, State, Data) -> <name name="state_enter"/> <desc> <p> - If the state machine should use <em>state enter calls</em> - is selected when starting the <c>gen_statem</c> + Whether the state machine should use <em>state enter calls</em> + or not is selected when starting the <c>gen_statem</c> and after code change using the return value from <seealso marker="#Module:callback_mode/0"><c>Module:callback_mode/0</c></seealso>. </p> @@ -606,7 +606,16 @@ handle_event(_, _, State, Data) -> See <seealso marker="#Module:StateName/3"><c>Module:StateName/3</c></seealso> and - <seealso marker="#Module:handle_event/4"><c>Module:handle_event/4</c></seealso>. + <seealso marker="#Module:handle_event/4"><c>Module:handle_event/4</c></seealso>. + Such a call can be repeated by returning a + <seealso marker="#type-state_callback_result"> + <c>repeat_state</c> + </seealso> + or + <seealso marker="#type-state_callback_result"> + <c>repeat_state_and_data</c> + </seealso> + tuple from the state callback. </p> <p> If @@ -625,7 +634,8 @@ handle_event(_, _, State, Data) -> right before entering the initial state even though this formally is not a state change. In this case <c>OldState</c> will be the same as <c>State</c>, - which can not happen for a subsequent state change. + which can not happen for a subsequent state change, + but will happen when repeating the state enter call. </p> </desc> </datatype> @@ -640,7 +650,15 @@ handle_event(_, _, State, Data) -> <list type="ordered"> <item> <p> - If the state changes or is the initial state, and + If the state changes, is the initial state, + <seealso marker="#type-state_callback_result"> + <c>repeat_state</c> + </seealso> + or + <seealso marker="#type-state_callback_result"> + <c>repeat_state_and_data</c> + </seealso> + is used, and also <seealso marker="#type-state_enter"><em>state enter calls</em></seealso> are used, the <c>gen_statem</c> calls the new state callback with arguments @@ -983,6 +1001,33 @@ handle_event(_, _, State, Data) -> </desc> </datatype> <datatype> + <name name="init_result"/> + <desc> + <p> + For a succesful initialization, + <c><anno>State</anno></c> is the initial + <seealso marker="#type-state"><c>state()</c></seealso> + and <c><anno>Data</anno></c> the initial server + <seealso marker="#type-data"><c>data()</c></seealso> + of the <c>gen_statem</c>. + </p> + <p> + The <seealso marker="#type-action"><c>Actions</c></seealso> + are executed when entering the first + <seealso marker="#type-state">state</seealso> just as for a + <seealso marker="#state callback">state callback</seealso>, + except that the action <c>postpone</c> is forced to + <c>false</c> since there is no event to postpone. + </p> + <p> + For an unsuccesful initialization, + <c>{stop,<anno>Reason</anno>}</c> + or <c>ignore</c> should be used; see + <seealso marker="#start_link/3"><c>start_link/3,4</c></seealso>. + </p> + </desc> + </datatype> + <datatype> <name name="state_enter_result"/> <desc> <p> @@ -1068,6 +1113,37 @@ handle_event(_, _, State, Data) -> <c>{next_state,CurrentState,CurrentData,<anno>Actions</anno>}</c>. </p> </item> + <tag><c>repeat_state</c></tag> + <item> + <p> + The <c>gen_statem</c> keeps the current state, or + does a state transition to the current state if you like, + sets <c><anno>NewData</anno></c>, + and executes all <c><anno>Actions</anno></c>. + If the <c>gen_statem</c> runs with + <seealso marker="#type-state_enter"><em>state enter calls</em></seealso>, + the state enter call is repeated, see type + <seealso marker="#type-transition_option"><c>transition_option()</c></seealso>, + otherwise <c>repeat_state</c> is the same as + <c>keep_state</c>. + </p> + </item> + <tag><c>repeat_state_and_data</c></tag> + <item> + <p> + The <c>gen_statem</c> keeps the current state and data, or + does a state transition to the current state if you like, + and executes all <c><anno>Actions</anno></c>. + This is the same as + <c>{repeat_state,CurrentData,<anno>Actions</anno>}</c>. + If the <c>gen_statem</c> runs with + <seealso marker="#type-state_enter"><em>state enter calls</em></seealso>, + the state enter call is repeated, see type + <seealso marker="#type-transition_option"><c>transition_option()</c></seealso>, + otherwise <c>repeat_state_and_data</c> is the same as + <c>keep_state_and_data</c>. + </p> + </item> <tag><c>stop</c></tag> <item> <p> @@ -1609,29 +1685,33 @@ handle_event(_, _, State, Data) -> It is recommended to use an atom as <c>Reason</c> since it will be wrapped in an <c>{error,Reason}</c> tuple. </p> + <p> + Also note when upgrading a <c>gen_statem</c>, + this function and hence + the <c>Change={advanced,Extra}</c> parameter in the + <seealso marker="sasl:appup"><c>appup</c></seealso> file + is not only needed to update the internal state + or to act on the <c>Extra</c> argument. + It is also needed if an upgrade or downgrade should change + <seealso marker="#type-callback_mode"><em>callback mode</em></seealso>, + or else the callback mode after the code change + will not be honoured, + most probably causing a server crash. + </p> </desc> </func> <func> - <name>Module:init(Args) -> Result</name> + <name>Module:init(Args) -> Result(StateType)</name> <fsummary> Optional function for initializing process and internal state. </fsummary> <type> <v>Args = term()</v> - <v>Result = {ok,State,Data}</v> - <v> | {ok,State,Data,Actions}</v> - <v> | {stop,Reason} | ignore</v> - <v>State = <seealso marker="#type-state">state()</seealso></v> - <v> - Data = <seealso marker="#type-data">data()</seealso> - </v> <v> - Actions = - [<seealso marker="#type-action">action()</seealso>] | - <seealso marker="#type-action">action()</seealso> + Result(StateType) = + <seealso marker="#type-init_result">init_result(StateType)</seealso> </v> - <v>Reason = term()</v> </type> <desc> <marker id="Module:init-1"/> @@ -1644,30 +1724,9 @@ handle_event(_, _, State, Data) -> the implementation state and server data. </p> <p> - <c>Args</c> is the <c>Args</c> argument provided to the start + <c>Args</c> is the <c>Args</c> argument provided to that start function. </p> - <p> - If the initialization is successful, the function is to - return <c>{ok,State,Data}</c> or - <c>{ok,State,Data,Actions}</c>. - <c>State</c> is the initial - <seealso marker="#type-state"><c>state()</c></seealso> - and <c>Data</c> the initial server - <seealso marker="#type-data"><c>data()</c></seealso>. - </p> - <p> - The <seealso marker="#type-action"><c>Actions</c></seealso> - are executed when entering the first - <seealso marker="#type-state">state</seealso> just as for a - <seealso marker="#state callback">state callback</seealso>. - </p> - <p> - If the initialization fails, - the function is to return <c>{stop,Reason}</c> - or <c>ignore</c>; see - <seealso marker="#start_link/3"><c>start_link/3,4</c></seealso>. - </p> <note> <p> This callback is optional, so a callback module does not need @@ -1873,22 +1932,33 @@ handle_event(_, _, State, Data) -> <seealso marker="#type-enter_action">actions</seealso> that may be returned: <seealso marker="#type-postpone"><c>postpone()</c></seealso> - and + is not allowed since a <em>state enter call</em> is not + an event so there is no event to postpone, and <seealso marker="#type-action"><c>{next_event,_,_}</c></seealso> - are not allowed. + is not allowed since using <em>state enter calls</em> + should not affect how events are consumed and produced. You may also not change states from this call. Should you return <c>{next_state,NextState, ...}</c> with <c>NextState =/= State</c> the <c>gen_statem</c> crashes. - You are advised to use <c>{keep_state,...}</c> or - <c>keep_state_and_data</c>. + It is possible to use <c>{repeat_state, ...}</c>, + <c>{repeat_state_and_data,_}</c> or + <c>repeat_state_and_data</c> but all of them makes little + sense since you immediately will be called again with a new + <em>state enter call</em> making this just a weird way + of looping, and there are better ways to loop in Erlang. + You are advised to use <c>{keep_state,...}</c>, + <c>{keep_state_and_data,_}</c> or + <c>keep_state_and_data</c> since you can not change states + from a <em>state enter call</em> anyway. </p> <p> Note the fact that you can use <seealso marker="erts:erlang#throw/1"><c>throw</c></seealso> to return the result, which can be useful. For example to bail out with <c>throw(keep_state_and_data)</c> - from deep within complex code that is in no position to - return <c>{next_state,State,Data}</c>. + from deep within complex code that can not + return <c>{next_state,State,Data}</c> because + <c>State</c> or <c>Data</c> is no longer in scope. </p> </desc> </func> @@ -1903,6 +1973,11 @@ handle_event(_, _, State, Data) -> <v>Ignored = term()</v> </type> <desc> + <note> + <p>This callback is optional, so callback modules need not + export it. The <c>gen_statem</c> module provides a default + implementation without cleanup.</p> + </note> <p> This function is called by a <c>gen_statem</c> when it is about to terminate. It is to be the opposite of diff --git a/lib/stdlib/doc/src/notes.xml b/lib/stdlib/doc/src/notes.xml index 0143686bb2..428d8a6e70 100644 --- a/lib/stdlib/doc/src/notes.xml +++ b/lib/stdlib/doc/src/notes.xml @@ -31,6 +31,110 @@ </header> <p>This document describes the changes made to the STDLIB application.</p> +<section><title>STDLIB 3.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>An escript with only two lines would not work.</p> + <p> + Own Id: OTP-14098</p> + </item> + <item> + <p> Characters (<c>$char</c>) can be used in constant + pattern expressions. They can also be used in types and + contracts. </p> + <p> + Own Id: OTP-14103 Aux Id: ERL-313 </p> + </item> + <item> + <p> The signatures of <c>erl_parse:anno_to_term/1</c> and + <c>erl_parse:anno_from_term/1</c> are corrected. Using + these functions no longer results in false Dialyzer + warnings. </p> + <p> + Own Id: OTP-14131</p> + </item> + <item> + <p>Pretty-printing of maps is improved. </p> + <p> + Own Id: OTP-14175 Aux Id: seq13277 </p> + </item> + <item> + <p>If any of the following functions in the <c>zip</c> + module crashed, a file would be left open: + <c>extract()</c>, <c>unzip()</c>, <c>create()</c>, or + <c>zip()</c>. This has been corrected.</p> + <p>A <c>zip</c> file having a "Unix header" could not be + unpacked.</p> + <p> + Own Id: OTP-14189 Aux Id: ERL-348, ERL-349 </p> + </item> + <item> + <p> Improve the Erlang shell's tab-completion of long + names. </p> + <p> + Own Id: OTP-14200 Aux Id: ERL-352 </p> + </item> + <item> + <p> + The reference manual for <c>sys</c> had some faulty + information about the 'get_modules' message used by + processes where modules change dynamically during + runtime. The documentation is now corrected.</p> + <p> + Own Id: OTP-14248 Aux Id: ERL-367 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Bug fixes, new features and improvements to gen_statem:</p> + <p> + A new type init_result/1 has replaced the old + init_result/0, so if you used that old type (that was + never documented) you have to change your code, which may + be regarded as a potential incompatibility.</p> + <p> + Changing callback modes after code change did not work + since the new callback mode was not recorded. This bug + has been fixed.</p> + <p> + The event types state_timeout and {call,From} could not + be generated with a {next_event,EventType,EventContent} + action since they did not pass the runtime type check. + This bug has now been corrected.</p> + <p> + State entry calls can now be repeated using (new) state + callback returns {repeat_state,...}, + {repeat_state_and_data,_} and repeat_state_and_data.</p> + <p> + There have been lots of code cleanup in particular + regarding timer handling. For example is async + cancel_timer now used. Error handling has also been + cleaned up.</p> + <p> + To align with probable future changes to the rest of + gen_*, terminate/3 has now got a fallback and + code_change/4 is not mandatory.</p> + <p> + Own Id: OTP-14114</p> + </item> + <item> + <p><c>filename:safe_relative_path/1</c> to sanitize a + relative path has been added.</p> + <p> + Own Id: OTP-14215</p> + </item> + </list> + </section> + +</section> + <section><title>STDLIB 3.2</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -3163,7 +3267,7 @@ <p> Two bugs in io:format for ~F.~Ps has been corrected. When length(S) >= abs(F) > P, the precision P was incorrectly - ignored. When F == P > lenght(S) the result was + ignored. When F == P > length(S) the result was incorrectly left adjusted. Bug found by Ali Yakout who also provided a fix.</p> <p> diff --git a/lib/stdlib/doc/src/proplists.xml b/lib/stdlib/doc/src/proplists.xml index fe6b8cc3bf..990d47b313 100644 --- a/lib/stdlib/doc/src/proplists.xml +++ b/lib/stdlib/doc/src/proplists.xml @@ -344,7 +344,7 @@ split([{c, 2}, {e, 1}, a, {c, 3, 4}, d, {b, 5}, b], [a, b, c])</code> with <c>{K2, true}</c>, thus changing the name of the option and simultaneously negating the value specified by <seealso marker="#get_bool/2"> - <c>get_bool(Key, <anno>ListIn</anno></c></seealso>. + <c>get_bool(Key, <anno>ListIn</anno>)</c></seealso>. If the same <c>K1</c> occurs more than once in <c><anno>Negations</anno></c>, only the first occurrence is used.</p> <p>For example, <c>substitute_negations([{no_foo, foo}], L)</c> diff --git a/lib/stdlib/doc/src/rand.xml b/lib/stdlib/doc/src/rand.xml index 8745e16908..2ddf3021ac 100644 --- a/lib/stdlib/doc/src/rand.xml +++ b/lib/stdlib/doc/src/rand.xml @@ -120,27 +120,50 @@ S0 = rand:seed_s(exsplus), {SND0, S2} = rand:normal_s(S1),</pre> <note> - <p>This random number generator is not cryptographically - strong. If a strong cryptographic random number generator is - needed, use one of functions in the - <seealso marker="crypto:crypto"><c>crypto</c></seealso> - module, for example, <seealso marker="crypto:crypto"> - <c>crypto:strong_rand_bytes/1</c></seealso>.</p> + <p>The builtin random number generator algorithms are not + cryptographically strong. If a cryptographically strong + random number generator is needed, use something like + <seealso marker="crypto:crypto#rand_seed-0"><c>crypto:rand_seed/0</c></seealso>. + </p> </note> </description> <datatypes> <datatype> + <name name="builtin_alg"/> + </datatype> + <datatype> <name name="alg"/> </datatype> <datatype> + <name name="alg_handler"/> + </datatype> + <datatype> + <name name="alg_state"/> + </datatype> + <datatype> + <name name="exs64_state"/> + <desc><p>Algorithm specific internal state</p></desc> + </datatype> + <datatype> + <name name="exsplus_state"/> + <desc><p>Algorithm specific internal state</p></desc> + </datatype> + <datatype> + <name name="exs1024_state"/> + <desc><p>Algorithm specific internal state</p></desc> + </datatype> + <datatype> <name name="state"/> <desc><p>Algorithm-dependent state.</p></desc> </datatype> <datatype> <name name="export_state"/> - <desc><p>Algorithm-dependent state that can be printed or saved to - file.</p></desc> + <desc> + <p> + Algorithm-dependent state that can be printed or saved to file. + </p> + </desc> </datatype> </datatypes> @@ -215,8 +238,11 @@ S0 = rand:seed_s(exsplus), <fsummary>Seed random number generator.</fsummary> <desc> <marker id="seed-1"/> - <p>Seeds random number generation with the specifed algorithm and - time-dependent data if <anno>AlgOrExpState</anno> is an algorithm.</p> + <p> + Seeds random number generation with the specifed algorithm and + time-dependent data if <c><anno>AlgOrStateOrExpState</anno></c> + is an algorithm. + </p> <p>Otherwise recreates the exported seed in the process dictionary, and returns the state. See also <seealso marker="#export_seed-0"><c>export_seed/0</c></seealso>.</p> @@ -236,8 +262,11 @@ S0 = rand:seed_s(exsplus), <name name="seed_s" arity="1"/> <fsummary>Seed random number generator.</fsummary> <desc> - <p>Seeds random number generation with the specifed algorithm and - time-dependent data if <anno>AlgOrExpState</anno> is an algorithm.</p> + <p> + Seeds random number generation with the specifed algorithm and + time-dependent data if <c><anno>AlgOrStateOrExpState</anno></c> + is an algorithm. + </p> <p>Otherwise recreates the exported seed and returns the state. See also <seealso marker="#export_seed-0"> <c>export_seed/0</c></seealso>.</p> @@ -258,7 +287,7 @@ S0 = rand:seed_s(exsplus), <fsummary>Return a random float.</fsummary> <desc><marker id="uniform-0"/> <p>Returns a random float uniformly distributed in the value - range <c>0.0 < <anno>X</anno> < 1.0</c> and + range <c>0.0 =< <anno>X</anno> < 1.0</c> and updates the state in the process dictionary.</p> </desc> </func> @@ -269,7 +298,7 @@ S0 = rand:seed_s(exsplus), <desc><marker id="uniform-1"/> <p>Returns, for a specified integer <c><anno>N</anno> >= 1</c>, a random integer uniformly distributed in the value range - <c>1 <= <anno>X</anno> <= <anno>N</anno></c> and + <c>1 =< <anno>X</anno> =< <anno>N</anno></c> and updates the state in the process dictionary.</p> </desc> </func> @@ -279,7 +308,7 @@ S0 = rand:seed_s(exsplus), <fsummary>Return a random float.</fsummary> <desc> <p>Returns, for a specified state, random float - uniformly distributed in the value range <c>0.0 < + uniformly distributed in the value range <c>0.0 =< <anno>X</anno> < 1.0</c> and a new state.</p> </desc> </func> @@ -290,7 +319,7 @@ S0 = rand:seed_s(exsplus), <desc> <p>Returns, for a specified integer <c><anno>N</anno> >= 1</c> and a state, a random integer uniformly distributed in the value - range <c>1 <= <anno>X</anno> <= <anno>N</anno></c> and a + range <c>1 =< <anno>X</anno> =< <anno>N</anno></c> and a new state.</p> </desc> </func> diff --git a/lib/stdlib/doc/src/sys.xml b/lib/stdlib/doc/src/sys.xml index 9091a46df9..45171f814d 100644 --- a/lib/stdlib/doc/src/sys.xml +++ b/lib/stdlib/doc/src/sys.xml @@ -83,8 +83,8 @@ <p>If the modules used to implement the process change dynamically during runtime, the process must understand one more message. An example is the <seealso marker="gen_event"><c>gen_event</c></seealso> - processes. The message is <c>{get_modules, From}</c>. - The reply to this message is <c>From ! {modules, Modules}</c>, where + processes. The message is <c>{_Label, {From, Ref}, get_modules}</c>. + The reply to this message is <c>From ! {Ref, Modules}</c>, where <c>Modules</c> is a list of the currently active modules in the process.</p> <p>This message is used by the release handler to find which diff --git a/lib/stdlib/doc/src/unicode_usage.xml b/lib/stdlib/doc/src/unicode_usage.xml index efc8b75075..a8ef8ff5c5 100644 --- a/lib/stdlib/doc/src/unicode_usage.xml +++ b/lib/stdlib/doc/src/unicode_usage.xml @@ -62,6 +62,10 @@ <item><p>In Erlang/OTP 17.0, the encoding default for Erlang source files was switched to UTF-8.</p></item> + + <item><p>In Erlang/OTP 20.0, atoms and function can contain + Unicode characters. Module names are still restricted to + the ISO-Latin-1 range.</p></item> </list> <p>This section outlines the current Unicode support and gives some @@ -339,9 +343,10 @@ <tag>The language</tag> <item> <p>Having the source code in UTF-8 also allows you to write string - literals containing Unicode characters with code points > 255, - although atoms, module names, and function names are restricted to - the ISO Latin-1 range. Binary literals, where you use type + literals, function names, and atoms containing Unicode + characters with code points > 255. + Module names are still restricted to the ISO Latin-1 range. + Binary literals, where you use type <c>/utf8</c>, can also be expressed using Unicode characters > 255. Having module names using characters other than 7-bit ASCII can cause trouble on operating systems with inconsistent file naming schemes, @@ -432,15 +437,17 @@ external_charlist() = maybe_improper_list(char() | external_unicode_binary() | <section> <title>Basic Language Support</title> - <p><marker id="unicode_in_erlang"/>As from Erlang/OTP R16, Erlang source - files can be written in UTF-8 or bytewise (<c>latin1</c>) encoding. For - information about how to state the encoding of an Erlang source file, see - the <seealso marker="stdlib:epp#encoding"><c>epp(3)</c></seealso> module. - Strings and comments can be written using Unicode, but functions must - still be named using characters from the ISO Latin-1 character set, and - atoms are restricted to the same ISO Latin-1 range. These restrictions in - the language are of course independent of the encoding of the source - file.</p> + <p><marker id="unicode_in_erlang"/>As from Erlang/OTP R16, Erlang + source files can be written in UTF-8 or bytewise (<c>latin1</c>) + encoding. For information about how to state the encoding of an + Erlang source file, see the <seealso + marker="stdlib:epp#encoding"><c>epp(3)</c></seealso> module. As + from Erlang/OTP R16, strings and comments can be written using + Unicode. As from Erlang/OTP 20, also atoms and functions can be + written using Unicode. Modules names must still be named using + characters from the ISO Latin-1 character set. (These + restrictions in the language are independent of the encoding of + the source file.)</p> <section> <title>Bit Syntax</title> diff --git a/lib/stdlib/src/base64.erl b/lib/stdlib/src/base64.erl index bf259e6691..0c8d817910 100644 --- a/lib/stdlib/src/base64.erl +++ b/lib/stdlib/src/base64.erl @@ -219,38 +219,49 @@ mime_decode_binary(Result, <<0:8,T/bits>>) -> mime_decode_binary(Result, T); mime_decode_binary(Result0, <<C:8,T/bits>>) -> case element(C, ?DECODE_MAP) of - Bits when is_integer(Bits) -> - mime_decode_binary(<<Result0/bits,Bits:6>>, T); - eq -> - case tail_contains_more(T, false) of - {<<>>, Eq} -> - %% No more valid data. - case bit_size(Result0) rem 8 of - 0 -> - %% '====' is not uncommon. - Result0; - 4 when Eq -> - %% enforce at least one more '=' only ignoring illegals and spacing - Split = byte_size(Result0) - 1, - <<Result:Split/bytes,_:4>> = Result0, - Result; - 2 -> - %% remove 2 bits - Split = byte_size(Result0) - 1, - <<Result:Split/bytes,_:2>> = Result0, - Result - end; - {More, _} -> - %% More valid data, skip the eq as invalid - mime_decode_binary(Result0, More) - end; - _ -> - mime_decode_binary(Result0, T) + Bits when is_integer(Bits) -> + mime_decode_binary(<<Result0/bits,Bits:6>>, T); + eq -> + mime_decode_binary_after_eq(Result0, T, false); + _ -> + mime_decode_binary(Result0, T) end; -mime_decode_binary(Result, <<>>) -> +mime_decode_binary(Result, _) -> true = is_binary(Result), Result. +mime_decode_binary_after_eq(Result, <<0:8,T/bits>>, Eq) -> + mime_decode_binary_after_eq(Result, T, Eq); +mime_decode_binary_after_eq(Result0, <<C:8,T/bits>>, Eq) -> + case element(C, ?DECODE_MAP) of + bad -> + mime_decode_binary_after_eq(Result0, T, Eq); + ws -> + mime_decode_binary_after_eq(Result0, T, Eq); + eq -> + mime_decode_binary_after_eq(Result0, T, true); + Bits when is_integer(Bits) -> + %% More valid data, skip the eq as invalid + mime_decode_binary(<<Result0/bits,Bits:6>>, T) + end; +mime_decode_binary_after_eq(Result0, <<>>, Eq) -> + %% No more valid data. + case bit_size(Result0) rem 8 of + 0 -> + %% '====' is not uncommon. + Result0; + 4 when Eq -> + %% enforce at least one more '=' only ignoring illegals and spacing + Split = byte_size(Result0) - 1, + <<Result:Split/bytes,_:4>> = Result0, + Result; + 2 -> + %% remove 2 bits + Split = byte_size(Result0) - 1, + <<Result:Split/bytes,_:2>> = Result0, + Result + end. + decode([], A) -> A; decode([$=,$=,C2,C1|Cs], A) -> Bits2x6 = (b64d(C1) bsl 18) bor (b64d(C2) bsl 12), diff --git a/lib/stdlib/src/c.erl b/lib/stdlib/src/c.erl index 52df2319dd..bb7b485490 100644 --- a/lib/stdlib/src/c.erl +++ b/lib/stdlib/src/c.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -400,7 +400,7 @@ split_def([], Res) -> {d, list_to_atom(reverse(Res))}. make_term(Str) -> case erl_scan:string(Str) of {ok, Tokens, _} -> - case erl_parse:parse_term(Tokens ++ [{dot, 1}]) of + case erl_parse:parse_term(Tokens ++ [{dot, erl_anno:new(1)}]) of {ok, Term} -> Term; {error, {_,_,Reason}} -> io:format("~ts: ~ts~n", [Reason, Str]), diff --git a/lib/stdlib/src/epp.erl b/lib/stdlib/src/epp.erl index 40eba4ad67..61d755ba55 100644 --- a/lib/stdlib/src/epp.erl +++ b/lib/stdlib/src/epp.erl @@ -286,7 +286,7 @@ parse_file(Epp) -> {warning,W} -> [{warning,W}|parse_file(Epp)]; {eof,Location} -> - [{eof,erl_anno:new(Location)}] + [{eof,Location}] end. -spec default_encoding() -> source_encoding(). diff --git a/lib/stdlib/src/erl_anno.erl b/lib/stdlib/src/erl_anno.erl index d32c34dabd..d0310f52e2 100644 --- a/lib/stdlib/src/erl_anno.erl +++ b/lib/stdlib/src/erl_anno.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2015. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -42,7 +42,7 @@ %% Debug: define DEBUG to make sure that annotations are handled as an %% opaque type. Note that all abstract code need to be compiled with -%% DEBUG=true. See also ./erl_pp.erl. +%% DEBUG=true. See also ./erl_pp.erl and ./erl_parse.yrl. %-define(DEBUG, true). @@ -52,7 +52,11 @@ | {'record', record()} | {'text', string()}. +-ifdef(DEBUG). +-opaque anno() :: [annotation(), ...]. +-else. -opaque anno() :: location() | [annotation(), ...]. +-endif. -type anno_term() :: term(). -type column() :: pos_integer(). diff --git a/lib/stdlib/src/erl_compile.erl b/lib/stdlib/src/erl_compile.erl index a6ae398d03..76db2eeacd 100644 --- a/lib/stdlib/src/erl_compile.erl +++ b/lib/stdlib/src/erl_compile.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2016. All Rights Reserved. +%% Copyright Ericsson AB 1997-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -337,7 +337,7 @@ file_or_directory(Name) -> make_term(Str) -> case erl_scan:string(Str) of {ok, Tokens, _} -> - case erl_parse:parse_term(Tokens ++ [{dot, 1}]) of + case erl_parse:parse_term(Tokens ++ [{dot, erl_anno:new(1)}]) of {ok, Term} -> Term; {error, {_,_,Reason}} -> io:format(?STDERR, "~ts: ~ts~n", [Reason, Str]), diff --git a/lib/stdlib/src/erl_lint.erl b/lib/stdlib/src/erl_lint.erl index 1b84234fac..0789f5dfb7 100644 --- a/lib/stdlib/src/erl_lint.erl +++ b/lib/stdlib/src/erl_lint.erl @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -156,6 +156,8 @@ format_error(pmod_unsupported) -> "parameterized modules are no longer supported"; %% format_error({redefine_mod_import, M, P}) -> %% io_lib:format("module '~s' already imported from package '~s'", [M, P]); +format_error(non_latin1_module_unsupported) -> + "module names with non-latin1 characters are not supported"; format_error(invalid_call) -> "invalid function call"; @@ -733,13 +735,27 @@ form(Form, #lint{state=State}=St) -> start_state({attribute,Line,module,{_,_}}=Form, St0) -> St1 = add_error(Line, pmod_unsupported, St0), attribute_state(Form, St1#lint{state=attribute}); -start_state({attribute,_,module,M}, St0) -> +start_state({attribute,Line,module,M}, St0) -> St1 = St0#lint{module=M}, - St1#lint{state=attribute}; + St2 = St1#lint{state=attribute}, + case is_non_latin1_name(M) of + true -> + add_error(Line, non_latin1_module_unsupported, St2); + false -> + St2 + end; start_state(Form, St) -> - St1 = add_error(element(2, Form), undefined_module, St), + Anno = case Form of + {eof, L} -> erl_anno:new(L); + %% {warning, Warning} and {error, Error} not possible here. + _ -> element(2, Form) + end, + St1 = add_error(Anno, undefined_module, St), attribute_state(Form, St1#lint{state=attribute}). +is_non_latin1_name(Name) -> + lists:any(fun(C) -> C > 255 end, atom_to_list(Name)). + %% attribute_state(Form, State) -> %% State' diff --git a/lib/stdlib/src/erl_parse.yrl b/lib/stdlib/src/erl_parse.yrl index 922455a6f2..2dcddeb8c2 100644 --- a/lib/stdlib/src/erl_parse.yrl +++ b/lib/stdlib/src/erl_parse.yrl @@ -981,6 +981,16 @@ Erlang code. %% keep track of annotation info in tokens -define(anno(Tup), element(2, Tup)). +%-define(DEBUG, true). + +-ifdef(DEBUG). +%% Assumes that erl_anno has been compiled with DEBUG=true. +-define(ANNO_CHECK(Tokens), + [] = [T || T <- Tokens, not is_list(element(2, T))]). +-else. +-define(ANNO_CHECK(Tokens), ok). +-endif. + %% Entry points compatible to old erl_parse. %% These really suck and are only here until Calle gets multiple %% entry points working. @@ -990,10 +1000,15 @@ Erlang code. AbsForm :: abstract_form(), ErrorInfo :: error_info(). parse_form([{'-',A1},{atom,A2,spec}|Tokens]) -> - parse([{'-',A1},{'spec',A2}|Tokens]); + NewTokens = [{'-',A1},{'spec',A2}|Tokens], + ?ANNO_CHECK(NewTokens), + parse(NewTokens); parse_form([{'-',A1},{atom,A2,callback}|Tokens]) -> - parse([{'-',A1},{'callback',A2}|Tokens]); + NewTokens = [{'-',A1},{'callback',A2}|Tokens], + ?ANNO_CHECK(NewTokens), + parse(NewTokens); parse_form(Tokens) -> + ?ANNO_CHECK(Tokens), parse(Tokens). -spec parse_exprs(Tokens) -> {ok, ExprList} | {error, ErrorInfo} when @@ -1001,6 +1016,7 @@ parse_form(Tokens) -> ExprList :: [abstract_expr()], ErrorInfo :: error_info(). parse_exprs(Tokens) -> + ?ANNO_CHECK(Tokens), A = erl_anno:new(0), case parse([{atom,A,f},{'(',A},{')',A},{'->',A}|Tokens]) of {ok,{function,_Lf,f,0,[{clause,_Lc,[],[],Exprs}]}} -> @@ -1013,6 +1029,7 @@ parse_exprs(Tokens) -> Term :: term(), ErrorInfo :: error_info(). parse_term(Tokens) -> + ?ANNO_CHECK(Tokens), A = erl_anno:new(0), case parse([{atom,A,f},{'(',A},{')',A},{'->',A}|Tokens]) of {ok,{function,_Af,f,0,[{clause,_Ac,[],[],[Expr]}]}} -> @@ -1531,8 +1548,8 @@ type_preop_prec('#') -> {700,800}. Fun :: fun((Anno) -> NewAnno), Anno :: erl_anno:anno(), NewAnno :: erl_anno:anno(), - Abstr :: erl_parse_tree(), - NewAbstr :: erl_parse_tree(). + Abstr :: erl_parse_tree() | form_info(), + NewAbstr :: erl_parse_tree() | form_info(). map_anno(F0, Abstr) -> F = fun(A, Acc) -> {F0(A), Acc} end, @@ -1546,7 +1563,7 @@ map_anno(F0, Abstr) -> Acc1 :: term(), AccIn :: term(), AccOut :: term(), - Abstr :: erl_parse_tree(). + Abstr :: erl_parse_tree() | form_info(). fold_anno(F0, Acc0, Abstr) -> F = fun(A, Acc) -> {A, F0(A, Acc)} end, @@ -1561,15 +1578,15 @@ fold_anno(F0, Acc0, Abstr) -> Acc1 :: term(), AccIn :: term(), AccOut :: term(), - Abstr :: erl_parse_tree(), - NewAbstr :: erl_parse_tree(). + Abstr :: erl_parse_tree() | form_info(), + NewAbstr :: erl_parse_tree() | form_info(). mapfold_anno(F, Acc0, Abstr) -> modify_anno1(Abstr, Acc0, F). -spec new_anno(Term) -> Abstr when Term :: term(), - Abstr :: erl_parse_tree(). + Abstr :: erl_parse_tree() | form_info(). new_anno(Term) -> F = fun(L, Acc) -> {erl_anno:new(L), Acc} end, @@ -1577,14 +1594,14 @@ new_anno(Term) -> NewAbstr. -spec anno_to_term(Abstr) -> term() when - Abstr :: erl_parse_tree(). + Abstr :: erl_parse_tree() | form_info(). anno_to_term(Abstract) -> F = fun(Anno, Acc) -> {erl_anno:to_term(Anno), Acc} end, {NewAbstract, []} = modify_anno1(Abstract, [], F), NewAbstract. --spec anno_from_term(Term) -> erl_parse_tree() when +-spec anno_from_term(Term) -> erl_parse_tree() | form_info() when Term :: term(). anno_from_term(Term) -> @@ -1629,6 +1646,8 @@ modify_anno1({warning,W}, Ac, _Mf) -> {{warning,W},Ac}; modify_anno1({error,W}, Ac, _Mf) -> {{error,W},Ac}; +modify_anno1({eof,L}, Ac, _Mf) -> + {{eof,L},Ac}; %% Expressions. modify_anno1({clauses,Cs}, Ac, Mf) -> {Cs1,Ac1} = modify_anno1(Cs, Ac, Mf), diff --git a/lib/stdlib/src/erl_pp.erl b/lib/stdlib/src/erl_pp.erl index d30cd508c1..6068afb293 100644 --- a/lib/stdlib/src/erl_pp.erl +++ b/lib/stdlib/src/erl_pp.erl @@ -51,6 +51,15 @@ %-define(DEBUG, true). -ifdef(DEBUG). +-define(FORM_TEST(T), + _ = case T of + {eof, _Line} -> ok; + {warning, _W} -> ok; + {error, _E} -> ok; + _ -> ?TEST(T) + end). +-define(EXPRS_TEST(L), + [?TEST(E) || E <- L]). -define(TEST(T), %% Assumes that erl_anno has been compiled with DEBUG=true. %% erl_pp does not use the annoations, but test it anyway. @@ -62,6 +71,8 @@ erlang:error(badarg, [T]) end). -else. +-define(FORM_TEST(T), ok). +-define(EXPRS_TEST(T), ok). -define(TEST(T), ok). -endif. @@ -80,7 +91,7 @@ form(Thing) -> Options :: options()). form(Thing, Options) -> - ?TEST(Thing), + ?FORM_TEST(Thing), State = state(Options), frmt(lform(Thing, options(Options)), State). @@ -124,7 +135,7 @@ guard(Gs) -> Options :: options()). guard(Gs, Options) -> - ?TEST(Gs), + ?EXPRS_TEST(Gs), frmt(lguard(Gs, options(Options)), state(Options)). -spec(exprs(Expressions) -> io_lib:chars() when @@ -146,7 +157,7 @@ exprs(Es, Options) -> Options :: options()). exprs(Es, I, Options) -> - ?TEST(Es), + ?EXPRS_TEST(Es), frmt({seq,[],[],[$,],lexprs(Es, options(Options))}, I, state(Options)). -spec(expr(Expression) -> io_lib:chars() when diff --git a/lib/stdlib/src/erl_tar.erl b/lib/stdlib/src/erl_tar.erl index 086e77cd28..a54df939bf 100644 --- a/lib/stdlib/src/erl_tar.erl +++ b/lib/stdlib/src/erl_tar.erl @@ -1321,7 +1321,11 @@ foldl_read(TarName, Fun, Accu, #read_opts{}=Opts) when is_function(Fun,4) -> try open(TarName, [read|Opts#read_opts.open_mode]) of {ok, #reader{access=read}=Reader} -> - foldl_read(Reader, Fun, Accu, Opts); + try + foldl_read(Reader, Fun, Accu, Opts) + after + _ = close(Reader) + end; {error, _} = Err -> Err catch diff --git a/lib/stdlib/src/escript.erl b/lib/stdlib/src/escript.erl index c42ae981e7..6e8f780f7c 100644 --- a/lib/stdlib/src/escript.erl +++ b/lib/stdlib/src/escript.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2016. All Rights Reserved. +%% Copyright Ericsson AB 2007-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -629,8 +629,7 @@ parse_source(S, File, Fd, StartLine, HeaderSz, CheckOnly) -> {error, _} -> epp_parse_file2(Epp, S2, [FileForm], OptModRes); {eof, LastLine} -> - Anno = anno(LastLine), - S#state{forms_or_bin = [FileForm, {eof, Anno}]} + S#state{forms_or_bin = [FileForm, {eof, LastLine}]} end, ok = epp:close(Epp), ok = file:close(Fd), @@ -728,8 +727,7 @@ epp_parse_file2(Epp, S, Forms, Parsed) -> [S#state.file,Ln,Mod:format_error(Args)]), epp_parse_file(Epp, S#state{n_errors = S#state.n_errors + 1}, [Form | Forms]); {eof, LastLine} -> - Anno = anno(LastLine), - S#state{forms_or_bin = lists:reverse([{eof, Anno} | Forms])} + S#state{forms_or_bin = lists:reverse([{eof, LastLine} | Forms])} end. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/stdlib/src/ets.erl b/lib/stdlib/src/ets.erl index d6fd1e3ea1..195a407570 100644 --- a/lib/stdlib/src/ets.erl +++ b/lib/stdlib/src/ets.erl @@ -70,15 +70,33 @@ match_object/2, match_object/3, match_spec_compile/1, match_spec_run_r/3, member/2, new/2, next/2, prev/2, rename/2, safe_fixtable/2, select/1, select/2, select/3, - select_count/2, select_delete/2, select_reverse/1, + select_count/2, select_delete/2, select_replace/2, select_reverse/1, select_reverse/2, select_reverse/3, setopts/2, slot/2, take/2, update_counter/3, update_counter/4, update_element/3]). +%% internal exports +-export([internal_request_all/0]). + -spec all() -> [Tab] when Tab :: tab(). all() -> + receive_all(ets:internal_request_all(), + erlang:system_info(schedulers), + []). + +receive_all(_Ref, 0, All) -> + All; +receive_all(Ref, N, All) -> + receive + {Ref, SchedAll} -> + receive_all(Ref, N-1, SchedAll ++ All) + end. + +-spec internal_request_all() -> reference(). + +internal_request_all() -> erlang:nif_error(undef). -spec delete(Tab) -> true when @@ -361,6 +379,14 @@ select_count(_, _) -> select_delete(_, _) -> erlang:nif_error(undef). +-spec select_replace(Tab, MatchSpec) -> NumReplaced when + Tab :: tab(), + MatchSpec :: match_spec(), + NumReplaced :: non_neg_integer(). + +select_replace(_, _) -> + erlang:nif_error(undef). + -spec select_reverse(Tab, MatchSpec) -> [Match] when Tab :: tab(), MatchSpec :: match_spec(), diff --git a/lib/stdlib/src/gen_event.erl b/lib/stdlib/src/gen_event.erl index 4839fe4f2c..0aebf1bdc5 100644 --- a/lib/stdlib/src/gen_event.erl +++ b/lib/stdlib/src/gen_event.erl @@ -778,7 +778,7 @@ stop_handlers([], _) -> []. %% Message from the release_handler. -%% The list of modules got to be a set ! +%% The list of modules got to be a set, i.e. no duplicate elements! get_modules(MSL) -> Mods = [Handler#handler.module || Handler <- MSL], ordsets:to_list(ordsets:from_list(Mods)). diff --git a/lib/stdlib/src/gen_statem.erl b/lib/stdlib/src/gen_statem.erl index 018aca90e6..cacc932ec4 100644 --- a/lib/stdlib/src/gen_statem.erl +++ b/lib/stdlib/src/gen_statem.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2016. All Rights Reserved. +%% Copyright Ericsson AB 2016-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -47,15 +47,17 @@ %% Type exports for templates and callback modules -export_type( [event_type/0, - init_result/0, callback_mode_result/0, - state_function_result/0, - handle_event_result/0, + init_result/1, state_enter_result/1, event_handler_result/1, reply_action/0, enter_action/0, action/0]). +%% Old types, not advertised +-export_type( + [state_function_result/0, + handle_event_result/0]). %% Type that is exported just to be documented -export_type([transition_option/0]). @@ -143,9 +145,10 @@ {'reply', % Reply to a caller From :: from(), Reply :: term()}. --type init_result() :: - {ok, state(), data()} | - {ok, state(), data(), [action()] | action()} | +-type init_result(StateType) :: + {ok, State :: StateType, Data :: data()} | + {ok, State :: StateType, Data :: data(), + Actions :: [action()] | action()} | 'ignore' | {'stop', Reason :: term()}. @@ -182,12 +185,23 @@ 'keep_state_and_data' | % {keep_state_and_data,[]} {'keep_state_and_data', % Keep state and data -> only actions Actions :: [ActionType] | ActionType} | + %% + {'repeat_state', % {repeat_state,NewData,[]} + NewData :: data()} | + {'repeat_state', % Repeat state, change data + NewData :: data(), + Actions :: [ActionType] | ActionType} | + 'repeat_state_and_data' | % {repeat_state_and_data,[]} + {'repeat_state_and_data', % Repeat state and data -> only actions + Actions :: [ActionType] | ActionType} | + %% 'stop' | % {stop,normal} {'stop', % Stop the server Reason :: term()} | {'stop', % Stop the server Reason :: term(), NewData :: data()} | + %% {'stop_and_reply', % Reply then stop the server Reason :: term(), Replies :: [reply_action()] | reply_action()} | @@ -201,7 +215,7 @@ %% the server is not running until this function has returned %% an {ok, ...} tuple. Thereafter the state callbacks are called %% for all events to this server. --callback init(Args :: term()) -> init_result(). +-callback init(Args :: term()) -> init_result(state()). %% This callback shall return the callback mode of the callback module. %% @@ -275,6 +289,8 @@ -optional_callbacks( [init/1, % One may use enter_loop/5,6,7 instead format_status/2, % Has got a default implementation + terminate/3, % Has got a default implementation + code_change/4, % Only needed by advanced soft upgrade %% state_name/3, % Example for callback_mode() =:= state_functions: %% there has to be a StateName/3 callback function @@ -304,12 +320,16 @@ event_type({call,From}) -> from(From); event_type(Type) -> case Type of + {call,From} -> + from(From); cast -> true; info -> true; timeout -> true; + state_timeout -> + true; internal -> true; _ -> @@ -588,6 +608,22 @@ enter(Module, Opts, State, Data, Server, Actions, Parent) -> true -> [Actions,{postpone,false}] end, + TimerRefs = #{}, + %% Key: timer ref + %% Value: the timer type i.e the timer's event type + %% + TimerTypes = #{}, + %% Key: timer type i.e the timer's event type + %% Value: timer ref + %% + %% We add a timer to both timer_refs and timer_types + %% when we start it. When we request an asynchronous + %% timer cancel we remove it from timer_types. When + %% the timer cancel message arrives we remove it from + %% timer_refs. + %% + Hibernate = false, + CancelTimers = 0, S = #{ callback_mode => undefined, state_enter => false, @@ -596,25 +632,25 @@ enter(Module, Opts, State, Data, Server, Actions, Parent) -> state => State, data => Data, postponed => P, - %% The rest of the fields are set from to the arguments to - %% loop_event_actions/10 when it finally loops back to loop/3 - %% in loop_events/10 %% - %% Marker for initial state, cleared immediately when used - init_state => true + %% The following fields are finally set from to the arguments to + %% loop_event_actions/9 when it finally loops back to loop/3 + %% in loop_event_result/11 + timer_refs => TimerRefs, + timer_types => TimerTypes, + hibernate => Hibernate, + cancel_timers => CancelTimers }, NewDebug = sys_debug(Debug, S, State, {enter,Event,State}), case call_callback_mode(S) of {ok,NewS} -> - TimerRefs = #{}, - TimerTypes = #{}, loop_event_actions( - Parent, NewDebug, NewS, TimerRefs, TimerTypes, - Events, Event, State, Data, NewActions); + Parent, NewDebug, NewS, + Events, Event, State, Data, NewActions, true); {Class,Reason,Stacktrace} -> terminate( - Class, Reason, Stacktrace, - NewDebug, S, [Event|Events]) + Class, Reason, Stacktrace, NewDebug, + S, [Event|Events]) end. %%%========================================================================== @@ -683,9 +719,7 @@ system_continue(Parent, Debug, S) -> loop(Parent, Debug, S). system_terminate(Reason, _Parent, Debug, S) -> - terminate( - exit, Reason, ?STACKTRACE(), - Debug, S, []). + terminate(exit, Reason, ?STACKTRACE(), Debug, S, []). system_code_change( #{module := Module, @@ -796,23 +830,22 @@ wakeup_from_hibernate(Parent, Debug, S) -> %% and detours through sys:handle_system_message/7 and proc_lib:hibernate/3 %% Entry point for system_continue/3 -loop(Parent, Debug, #{hibernate := Hibernate} = S) -> - case Hibernate of - true -> - %% Does not return but restarts process at - %% wakeup_from_hibernate/3 that jumps to loop_receive/3 - proc_lib:hibernate( - ?MODULE, wakeup_from_hibernate, [Parent,Debug,S]), - error( - {should_not_have_arrived_here_but_instead_in, - {wakeup_from_hibernate,3}}); - false -> - loop_receive(Parent, Debug, S) - end. +loop(Parent, Debug, #{hibernate := true, cancel_timers := 0} = S) -> + loop_hibernate(Parent, Debug, S); +loop(Parent, Debug, S) -> + loop_receive(Parent, Debug, S). + +loop_hibernate(Parent, Debug, S) -> + %% Does not return but restarts process at + %% wakeup_from_hibernate/3 that jumps to loop_receive/3 + proc_lib:hibernate( + ?MODULE, wakeup_from_hibernate, [Parent,Debug,S]), + error( + {should_not_have_arrived_here_but_instead_in, + {wakeup_from_hibernate,3}}). %% Entry point for wakeup_from_hibernate/3 -loop_receive( - Parent, Debug, #{timer_refs := TimerRefs, timer_types := TimerTypes} = S) -> +loop_receive(Parent, Debug, S) -> receive Msg -> case Msg of @@ -821,30 +854,87 @@ loop_receive( %% Does not return but tail recursively calls %% system_continue/3 that jumps to loop/3 sys:handle_system_msg( - Req, Pid, Parent, ?MODULE, Debug, S, Hibernate); + Req, Pid, Parent, ?MODULE, Debug, S, + Hibernate); {'EXIT',Parent,Reason} = EXIT -> - %% EXIT is not a 2-tuple and therefore - %% not an event and has no event_type(), - %% but this will stand out in the crash report... - terminate( - exit, Reason, ?STACKTRACE(), Debug, S, [EXIT]); + %% EXIT is not a 2-tuple therefore + %% not an event but this will stand out + %% in the crash report... + Q = [EXIT], + terminate(exit, Reason, ?STACKTRACE(), Debug, S, Q); {timeout,TimerRef,TimerMsg} -> + #{timer_refs := TimerRefs, + timer_types := TimerTypes, + hibernate := Hibernate} = S, case TimerRefs of #{TimerRef := TimerType} -> - Event = {TimerType,TimerMsg}, - %% Unregister the triggered timeout + %% We know of this timer; is it a running + %% timer or a timer being cancelled that + %% managed to send a late timeout message? + case TimerTypes of + #{TimerType := TimerRef} -> + %% The timer type maps back to this + %% timer ref, so it was a running timer + Event = {TimerType,TimerMsg}, + %% Unregister the triggered timeout + NewTimerRefs = + maps:remove(TimerRef, TimerRefs), + NewTimerTypes = + maps:remove(TimerType, TimerTypes), + loop_receive_result( + Parent, Debug, + S#{ + timer_refs := NewTimerRefs, + timer_types := NewTimerTypes}, + Hibernate, + Event); + _ -> + %% This was a late timeout message + %% from timer being cancelled, so + %% ignore it and expect a cancel_timer + %% msg shortly + loop_receive(Parent, Debug, S) + end; + _ -> + %% Not our timer; present it as an event + Event = {info,Msg}, loop_receive_result( - Parent, Debug, S, - maps:remove(TimerRef, TimerRefs), - maps:remove(TimerType, TimerTypes), - Event); + Parent, Debug, S, Hibernate, Event) + end; + {cancel_timer,TimerRef,_} -> + #{timer_refs := TimerRefs, + cancel_timers := CancelTimers, + hibernate := Hibernate} = S, + case TimerRefs of + #{TimerRef := _} -> + %% We must have requested a cancel + %% of this timer so it is already + %% removed from TimerTypes + NewTimerRefs = + maps:remove(TimerRef, TimerRefs), + NewCancelTimers = CancelTimers - 1, + NewS = + S#{ + timer_refs := NewTimerRefs, + cancel_timers := NewCancelTimers}, + if + Hibernate =:= true, NewCancelTimers =:= 0 -> + %% No more cancel_timer msgs to expect; + %% we can hibernate + loop_hibernate(Parent, Debug, NewS); + NewCancelTimers >= 0 -> % Assert + loop_receive(Parent, Debug, NewS) + end; _ -> + %% Not our cancel_timer msg; + %% present it as an event Event = {info,Msg}, loop_receive_result( - Parent, Debug, S, - TimerRefs, TimerTypes, Event) + Parent, Debug, S, Hibernate, Event) end; _ -> + %% External msg + #{hibernate := Hibernate} = S, Event = case Msg of {'$gen_call',From,Request} -> @@ -855,208 +945,212 @@ loop_receive( {info,Msg} end, loop_receive_result( - Parent, Debug, S, - TimerRefs, TimerTypes, Event) + Parent, Debug, S, Hibernate, Event) end end. loop_receive_result( - Parent, Debug, #{state := State} = S, - TimerRefs, TimerTypes, Event) -> - %% The fields 'timer_refs', 'timer_types' and 'hibernate' - %% are now invalid in state map S - they will be recalculated - %% and restored when we return to loop/3 - %% + Parent, Debug, + #{state := State, + timer_types := TimerTypes, cancel_timers := CancelTimers} = S, + Hibernate, Event) -> + %% From now the 'hibernate' field in S is invalid + %% and will be restored when looping back + %% in loop_event_result/11 NewDebug = sys_debug(Debug, S, State, {in,Event}), - %% Here the queue of not yet handled events is created + %% Here is the queue of not yet handled events created Events = [], - Hibernate = false, - loop_event( - Parent, NewDebug, S, TimerRefs, TimerTypes, Events, Event, Hibernate). + %% Cancel any running event timer + case + cancel_timer_by_type(timeout, TimerTypes, CancelTimers) + of + {_,CancelTimers} -> + %% No timer cancelled + loop_event(Parent, NewDebug, S, Events, Event, Hibernate); + {NewTimerTypes,NewCancelTimers} -> + %% The timer is removed from NewTimerTypes but + %% remains in TimerRefs until we get + %% the cancel_timer msg + NewS = + S#{ + timer_types := NewTimerTypes, + cancel_timers := NewCancelTimers}, + loop_event(Parent, NewDebug, NewS, Events, Event, Hibernate) + end. %% Entry point for handling an event, received or enqueued loop_event( - Parent, Debug, #{state := State, data := Data} = S, TimerRefs, TimerTypes, + Parent, Debug, + #{state := State, data := Data} = S, Events, {Type,Content} = Event, Hibernate) -> %% - %% If Hibernate is true here it can only be + %% If (this old) Hibernate is true here it can only be %% because it was set from an event action - %% and we did not go into hibernation since there - %% were events in queue, so we do what the user + %% and we did not go into hibernation since there were + %% events in queue, so we do what the user %% might rely on i.e collect garbage which %% would have happened if we actually hibernated %% and immediately was awakened Hibernate andalso garbage_collect(), case call_state_function(S, Type, Content, State, Data) of {ok,Result,NewS} -> - %% Cancel event timeout - {NewTimerRefs,NewTimerTypes} = - cancel_timer_by_type( - timeout, TimerRefs, TimerTypes), - {NewData,NextState,Actions} = + {NextState,NewData,Actions,EnterCall} = parse_event_result( - true, Debug, NewS, Result, - Events, Event, State, Data), + true, Debug, NewS, + Events, Event, State, Data, Result), loop_event_actions( - Parent, Debug, S, NewTimerRefs, NewTimerTypes, - Events, Event, NextState, NewData, Actions); + Parent, Debug, NewS, + Events, Event, NextState, NewData, Actions, EnterCall); {Class,Reason,Stacktrace} -> terminate( - Class, Reason, Stacktrace, Debug, S, [Event|Events]) + Class, Reason, Stacktrace, Debug, S, + [Event|Events]) end. loop_event_actions( Parent, Debug, - #{state := State, state_enter := StateEnter} = S, TimerRefs, TimerTypes, - Events, Event, NextState, NewData, Actions) -> + #{state := State, state_enter := StateEnter} = S, + Events, Event, NextState, NewData, + Actions, EnterCall) -> + %% Hibernate is reborn here as false being + %% the default value from parse_actions/4 case parse_actions(Debug, S, State, Actions) of {ok,NewDebug,Hibernate,TimeoutsR,Postpone,NextEventsR} -> if - StateEnter, NextState =/= State -> + StateEnter, EnterCall -> loop_event_enter( - Parent, NewDebug, S, TimerRefs, TimerTypes, + Parent, NewDebug, S, Events, Event, NextState, NewData, Hibernate, TimeoutsR, Postpone, NextEventsR); - StateEnter -> - case maps:is_key(init_state, S) of - true -> - %% Avoid infinite loop in initial state - %% with state entry events - NewS = maps:remove(init_state, S), - loop_event_enter( - Parent, NewDebug, NewS, TimerRefs, TimerTypes, - Events, Event, NextState, NewData, - Hibernate, TimeoutsR, Postpone, NextEventsR); - false -> - loop_event_result( - Parent, NewDebug, S, TimerRefs, TimerTypes, - Events, Event, NextState, NewData, - Hibernate, TimeoutsR, Postpone, NextEventsR) - end; true -> loop_event_result( - Parent, NewDebug, S, TimerRefs, TimerTypes, + Parent, NewDebug, S, Events, Event, NextState, NewData, Hibernate, TimeoutsR, Postpone, NextEventsR) end; {Class,Reason,Stacktrace} -> terminate( - Class, Reason, Stacktrace, - Debug, S#{data := NewData}, [Event|Events]) + Class, Reason, Stacktrace, Debug, S, + [Event|Events]) end. loop_event_enter( - Parent, Debug, #{state := State} = S, TimerRefs, TimerTypes, + Parent, Debug, #{state := State} = S, Events, Event, NextState, NewData, Hibernate, TimeoutsR, Postpone, NextEventsR) -> case call_state_function(S, enter, State, NextState, NewData) of {ok,Result,NewS} -> - {NewerData,_,Actions} = - parse_event_result( - false, Debug, NewS, Result, - Events, Event, NextState, NewData), - loop_event_enter_actions( - Parent, Debug, NewS, TimerRefs, TimerTypes, - Events, Event, NextState, NewerData, - Hibernate, TimeoutsR, Postpone, NextEventsR, Actions); + case parse_event_result( + false, Debug, NewS, + Events, Event, NextState, NewData, Result) of + {_,NewerData,Actions,EnterCall} -> + loop_event_enter_actions( + Parent, Debug, NewS, + Events, Event, NextState, NewerData, + Hibernate, TimeoutsR, Postpone, NextEventsR, + Actions, EnterCall) + end; {Class,Reason,Stacktrace} -> terminate( - Class, Reason, Stacktrace, - Debug, S#{state := NextState, data := NewData}, + Class, Reason, Stacktrace, Debug, + S#{ + state := NextState, + data := NewData, + hibernate := Hibernate}, [Event|Events]) end. loop_event_enter_actions( - Parent, Debug, S, TimerRefs, TimerTypes, + Parent, Debug, #{state_enter := StateEnter} = S, Events, Event, NextState, NewData, - Hibernate, TimeoutsR, Postpone, NextEventsR, Actions) -> + Hibernate, TimeoutsR, Postpone, NextEventsR, + Actions, EnterCall) -> case parse_enter_actions( - Debug, S, NextState, Actions, - Hibernate, TimeoutsR) + Debug, S, NextState, Actions, Hibernate, TimeoutsR) of {ok,NewDebug,NewHibernate,NewTimeoutsR,_,_} -> - loop_event_result( - Parent, NewDebug, S, TimerRefs, TimerTypes, - Events, Event, NextState, NewData, - NewHibernate, NewTimeoutsR, Postpone, NextEventsR); + if + StateEnter, EnterCall -> + loop_event_enter( + Parent, NewDebug, S, + Events, Event, NextState, NewData, + NewHibernate, NewTimeoutsR, Postpone, NextEventsR); + true -> + loop_event_result( + Parent, NewDebug, S, + Events, Event, NextState, NewData, + NewHibernate, NewTimeoutsR, Postpone, NextEventsR) + end; {Class,Reason,Stacktrace} -> terminate( - Class, Reason, Stacktrace, - Debug, S#{state := NextState, data := NewData}, + Class, Reason, Stacktrace, Debug, + S#{ + state := NextState, + data := NewData, + hibernate := Hibernate}, [Event|Events]) end. loop_event_result( - Parent, Debug, - #{state := State, postponed := P_0} = S, TimerRefs_0, TimerTypes_0, - Events, Event, NextState, NewData, + Parent, Debug_0, + #{state := State, postponed := P_0, + timer_refs := TimerRefs_0, timer_types := TimerTypes_0, + cancel_timers := CancelTimers_0} = S_0, + Events_0, Event_0, NextState, NewData, Hibernate, TimeoutsR, Postpone, NextEventsR) -> %% %% All options have been collected and next_events are buffered. %% Do the actual state transition. %% - {NewDebug,P_1} = % Move current event to postponed if Postpone + {Debug_1,P_1} = % Move current event to postponed if Postpone case Postpone of true -> - {sys_debug(Debug, S, State, {postpone,Event,State}), - [Event|P_0]}; + {sys_debug(Debug_0, S_0, State, {postpone,Event_0,State}), + [Event_0|P_0]}; false -> - {sys_debug(Debug, S, State, {consume,Event,State}), + {sys_debug(Debug_0, S_0, State, {consume,Event_0,State}), P_0} end, - {Events_1,NewP,{TimerRefs_1,TimerTypes_1}} = + {Events_1,P_2,{TimerTypes_1,CancelTimers_1}} = %% Move all postponed events to queue and cancel the %% state timeout if the state changes if NextState =:= State -> - {Events,P_1,{TimerRefs_0,TimerTypes_0}}; + {Events_0,P_1,{TimerTypes_0,CancelTimers_0}}; true -> - {lists:reverse(P_1, Events),[], + {lists:reverse(P_1, Events_0), + [], cancel_timer_by_type( - state_timeout, TimerRefs_0, TimerTypes_0)} + state_timeout, TimerTypes_0, CancelTimers_0)} + %% The state timer is removed from TimerTypes_1 + %% but remains in TimerRefs_0 until we get + %% the cancel_timer msg end, - {TimerRefs_2,TimerTypes_2,TimeoutEvents} = - %% Stop and start timers non-event timers - parse_timers(TimerRefs_1, TimerTypes_1, TimeoutsR), + {TimerRefs_2,TimerTypes_2,CancelTimers_2,TimeoutEvents} = + %% Stop and start non-event timers + parse_timers(TimerRefs_0, TimerTypes_1, CancelTimers_1, TimeoutsR), %% Place next events last in reversed queue Events_2R = lists:reverse(Events_1, NextEventsR), %% Enqueue immediate timeout events and start event timer - {NewTimerRefs,NewTimerTypes,Events_3R} = - process_timeout_events( - TimerRefs_2, TimerTypes_2, TimeoutEvents, Events_2R), - NewEvents = lists:reverse(Events_3R), - loop_events( - Parent, NewDebug, S, NewTimerRefs, NewTimerTypes, - NewEvents, Hibernate, NextState, NewData, NewP). - -%% Loop until out of enqueued events -%% -loop_events( - Parent, Debug, S, TimerRefs, TimerTypes, - [] = _Events, Hibernate, State, Data, P) -> - %% Update S and loop back to loop/3 to receive a new event - NewS = - S#{ - state := State, - data := Data, - postponed := P, - hibernate => Hibernate, - timer_refs => TimerRefs, - timer_types => TimerTypes}, - loop(Parent, Debug, NewS); -loop_events( - Parent, Debug, S, TimerRefs, TimerTypes, - [Event|Events], Hibernate, State, Data, P) -> - %% Update S and continue with enqueued events - NewS = - S#{ - state := State, - data := Data, - postponed := P}, - loop_event( - Parent, Debug, NewS, TimerRefs, TimerTypes, Events, Event, Hibernate). - + Events_3R = prepend_timeout_events(TimeoutEvents, Events_2R), + S_1 = + S_0#{ + state := NextState, + data := NewData, + postponed := P_2, + timer_refs := TimerRefs_2, + timer_types := TimerTypes_2, + cancel_timers := CancelTimers_2, + hibernate := Hibernate}, + case lists:reverse(Events_3R) of + [] -> + %% Get a new event + loop(Parent, Debug_1, S_1); + [Event|Events] -> + %% Loop until out of enqueued events + loop_event(Parent, Debug_1, S_1, Events, Event, Hibernate) + end. %%--------------------------------------------------------------------------- @@ -1069,19 +1163,6 @@ call_callback_mode(#{module := Module} = S) -> catch CallbackMode -> callback_mode_result(S, CallbackMode); - error:undef -> - %% Process undef to check for the simple mistake - %% of calling a nonexistent state function - %% to make the undef more precise - case erlang:get_stacktrace() of - [{Module,callback_mode,[]=Args,_} - |Stacktrace] -> - {error, - {undef_callback,{Module,callback_mode,Args}}, - Stacktrace}; - Stacktrace -> - {error,undef,Stacktrace} - end; Class:Reason -> {Class,Reason,erlang:get_stacktrace()} end. @@ -1126,8 +1207,7 @@ parse_callback_mode(_, _CBMode, StateEnter) -> call_state_function( - #{callback_mode := undefined} = S, - Type, Content, State, Data) -> + #{callback_mode := undefined} = S, Type, Content, State, Data) -> case call_callback_mode(S) of {ok,NewS} -> call_state_function(NewS, Type, Content, State, Data); @@ -1135,13 +1215,12 @@ call_state_function( Error end; call_state_function( - #{callback_mode := CallbackMode, - module := Module} = S, + #{callback_mode := CallbackMode, module := Module} = S, Type, Content, State, Data) -> try case CallbackMode of state_functions -> - erlang:apply(Module, State, [Type,Content,Data]); + Module:State(Type, Content, Data); handle_event_function -> Module:handle_event(Type, Content, State, Data) end @@ -1151,41 +1230,6 @@ call_state_function( catch Result -> {ok,Result,S}; - error:badarg -> - case erlang:get_stacktrace() of - [{erlang,apply, - [Module,State,[Type,Content,Data]=Args], - _} - |Stacktrace] - when CallbackMode =:= state_functions -> - %% We get here e.g if apply fails - %% due to State not being an atom - {error, - {undef_state_function,{Module,State,Args}}, - Stacktrace}; - Stacktrace -> - {error,badarg,Stacktrace} - end; - error:undef -> - %% Process undef to check for the simple mistake - %% of calling a nonexistent state function - %% to make the undef more precise - case erlang:get_stacktrace() of - [{Module,State,[Type,Content,Data]=Args,_} - |Stacktrace] - when CallbackMode =:= state_functions -> - {error, - {undef_state_function,{Module,State,Args}}, - Stacktrace}; - [{Module,handle_event,[Type,Content,State,Data]=Args,_} - |Stacktrace] - when CallbackMode =:= handle_event_function -> - {error, - {undef_state_function,{Module,handle_event,Args}}, - Stacktrace}; - Stacktrace -> - {error,undef,Stacktrace} - end; Class:Reason -> {Class,Reason,erlang:get_stacktrace()} end. @@ -1193,65 +1237,83 @@ call_state_function( %% Interpret all callback return variants parse_event_result( - AllowStateChange, Debug, S, Result, Events, Event, State, Data) -> + AllowStateChange, Debug, S, + Events, Event, State, Data, Result) -> case Result of stop -> terminate( - exit, normal, ?STACKTRACE(), Debug, S, [Event|Events]); + exit, normal, ?STACKTRACE(), Debug, + S#{state := State, data := Data}, + [Event|Events]); {stop,Reason} -> terminate( - exit, Reason, ?STACKTRACE(), Debug, S, [Event|Events]); + exit, Reason, ?STACKTRACE(), Debug, + S#{state := State, data := Data}, + [Event|Events]); {stop,Reason,NewData} -> terminate( - exit, Reason, ?STACKTRACE(), - Debug, S#{data := NewData}, [Event|Events]); + exit, Reason, ?STACKTRACE(), Debug, + S#{state := State, data := NewData}, + [Event|Events]); + %% {stop_and_reply,Reason,Replies} -> - Q = [Event|Events], reply_then_terminate( - exit, Reason, ?STACKTRACE(), - Debug, S, Q, Replies); + exit, Reason, ?STACKTRACE(), Debug, + S#{state := State, data := Data}, + [Event|Events], Replies); {stop_and_reply,Reason,Replies,NewData} -> - Q = [Event|Events], reply_then_terminate( - exit, Reason, ?STACKTRACE(), - Debug, S#{data := NewData}, Q, Replies); + exit, Reason, ?STACKTRACE(), Debug, + S#{state := State, data := NewData}, + [Event|Events], Replies); + %% {next_state,State,NewData} -> - {NewData,State,[]}; + {State,NewData,[],false}; {next_state,NextState,NewData} when AllowStateChange -> - {NewData,NextState,[]}; + {NextState,NewData,[],true}; {next_state,State,NewData,Actions} -> - {NewData,State,Actions}; + {State,NewData,Actions,false}; {next_state,NextState,NewData,Actions} when AllowStateChange -> - {NewData,NextState,Actions}; + {NextState,NewData,Actions,true}; + %% {keep_state,NewData} -> - {NewData,State,[]}; + {State,NewData,[],false}; {keep_state,NewData,Actions} -> - {NewData,State,Actions}; + {State,NewData,Actions,false}; keep_state_and_data -> - {Data,State,[]}; + {State,Data,[],false}; {keep_state_and_data,Actions} -> - {Data,State,Actions}; + {State,Data,Actions,false}; + %% + {repeat_state,NewData} -> + {State,NewData,[],true}; + {repeat_state,NewData,Actions} -> + {State,NewData,Actions,true}; + repeat_state_and_data -> + {State,Data,[],true}; + {repeat_state_and_data,Actions} -> + {State,Data,Actions,true}; + %% _ -> terminate( error, {bad_return_from_state_function,Result}, - ?STACKTRACE(), - Debug, S, [Event|Events]) + ?STACKTRACE(), Debug, + S#{state := State, data := Data}, + [Event|Events]) end. -parse_enter_actions( - Debug, S, State, Actions, - Hibernate, TimeoutsR) -> +parse_enter_actions(Debug, S, State, Actions, Hibernate, TimeoutsR) -> Postpone = forbidden, NextEventsR = forbidden, parse_actions( Debug, S, State, listify(Actions), Hibernate, TimeoutsR, Postpone, NextEventsR). - + parse_actions(Debug, S, State, Actions) -> Hibernate = false, - TimeoutsR = [], + TimeoutsR = [{timeout,infinity,infinity}], %% Will cancel event timer Postpone = false, NextEventsR = [], parse_actions( @@ -1279,64 +1341,29 @@ parse_actions( {bad_action_from_state_function,Action}, ?STACKTRACE()} end; + %% %% Actions that set options {hibernate,NewHibernate} when is_boolean(NewHibernate) -> parse_actions( Debug, S, State, Actions, NewHibernate, TimeoutsR, Postpone, NextEventsR); - {hibernate,_} -> - {error, - {bad_action_from_state_function,Action}, - ?STACKTRACE()}; hibernate -> + NewHibernate = true, parse_actions( Debug, S, State, Actions, - true, TimeoutsR, Postpone, NextEventsR); - {state_timeout,Time,_} = StateTimeout - when is_integer(Time), Time >= 0; - Time =:= infinity -> - parse_actions( - Debug, S, State, Actions, - Hibernate, [StateTimeout|TimeoutsR], Postpone, NextEventsR); - {state_timeout,_,_} -> - {error, - {bad_action_from_state_function,Action}, - ?STACKTRACE()}; - {timeout,infinity,_} -> - %% Ignore - timeout will never happen and already cancelled - parse_actions( - Debug, S, State, Actions, - Hibernate, TimeoutsR, Postpone, NextEventsR); - {timeout,Time,_} = Timeout when is_integer(Time), Time >= 0 -> - parse_actions( - Debug, S, State, Actions, - Hibernate, [Timeout|TimeoutsR], Postpone, NextEventsR); - {timeout,_,_} -> - {error, - {bad_action_from_state_function,Action}, - ?STACKTRACE()}; - infinity -> % Ignore - timeout will never happen - parse_actions( - Debug, S, State, Actions, - Hibernate, TimeoutsR, Postpone, NextEventsR); - Time when is_integer(Time), Time >= 0 -> - Timeout = {timeout,Time,Time}, - parse_actions( - Debug, S, State, Actions, - Hibernate, [Timeout|TimeoutsR], Postpone, NextEventsR); + NewHibernate, TimeoutsR, Postpone, NextEventsR); + %% {postpone,NewPostpone} when is_boolean(NewPostpone), Postpone =/= forbidden -> parse_actions( Debug, S, State, Actions, Hibernate, TimeoutsR, NewPostpone, NextEventsR); - {postpone,_} -> - {error, - {bad_action_from_state_function,Action}, - ?STACKTRACE()}; postpone when Postpone =/= forbidden -> + NewPostpone = true, parse_actions( Debug, S, State, Actions, - Hibernate, TimeoutsR, true, NextEventsR); + Hibernate, TimeoutsR, NewPostpone, NextEventsR); + %% {next_event,Type,Content} -> case event_type(Type) of true when NextEventsR =/= forbidden -> @@ -1351,96 +1378,150 @@ parse_actions( {bad_action_from_state_function,Action}, ?STACKTRACE()} end; - _ -> + %% + {state_timeout,_,_} = Timeout -> + parse_actions_timeout( + Debug, S, State, Actions, + Hibernate, TimeoutsR, Postpone, NextEventsR, Timeout); + {timeout,_,_} = Timeout -> + parse_actions_timeout( + Debug, S, State, Actions, + Hibernate, TimeoutsR, Postpone, NextEventsR, Timeout); + Time -> + parse_actions_timeout( + Debug, S, State, Actions, + Hibernate, TimeoutsR, Postpone, NextEventsR, Time) + end. + +parse_actions_timeout( + Debug, S, State, Actions, + Hibernate, TimeoutsR, Postpone, NextEventsR, Timeout) -> + Time = + case Timeout of + {_,T,_} -> T; + T -> T + end, + case validate_time(Time) of + true -> + parse_actions( + Debug, S, State, Actions, + Hibernate, [Timeout|TimeoutsR], + Postpone, NextEventsR); + false -> {error, - {bad_action_from_state_function,Action}, + {bad_action_from_state_function,Timeout}, ?STACKTRACE()} end. +validate_time(Time) when is_integer(Time), Time >= 0 -> true; +validate_time(infinity) -> true; +validate_time(_) -> false. %% Stop and start timers as well as create timeout zero events %% and pending event timer %% %% Stop and start timers non-event timers -parse_timers(TimerRefs, TimerTypes, TimeoutsR) -> - parse_timers(TimerRefs, TimerTypes, TimeoutsR, #{}, []). +parse_timers(TimerRefs, TimerTypes, CancelTimers, TimeoutsR) -> + parse_timers(TimerRefs, TimerTypes, CancelTimers, TimeoutsR, #{}, []). %% -parse_timers(TimerRefs, TimerTypes, [], _Seen, TimeoutEvents) -> - {TimerRefs,TimerTypes,TimeoutEvents}; parse_timers( - TimerRefs, TimerTypes, [Timeout|TimeoutsR], Seen, TimeoutEvents) -> - {TimerType,Time,TimerMsg} = Timeout, + TimerRefs, TimerTypes, CancelTimers, [], _Seen, TimeoutEvents) -> + {TimerRefs,TimerTypes,CancelTimers,TimeoutEvents}; +parse_timers( + TimerRefs, TimerTypes, CancelTimers, [Timeout|TimeoutsR], + Seen, TimeoutEvents) -> + case Timeout of + {TimerType,Time,TimerMsg} -> + parse_timers( + TimerRefs, TimerTypes, CancelTimers, TimeoutsR, + Seen, TimeoutEvents, + TimerType, Time, TimerMsg); + Time -> + parse_timers( + TimerRefs, TimerTypes, CancelTimers, TimeoutsR, + Seen, TimeoutEvents, + timeout, Time, Time) + end. + +parse_timers( + TimerRefs, TimerTypes, CancelTimers, TimeoutsR, + Seen, TimeoutEvents, + TimerType, Time, TimerMsg) -> case Seen of #{TimerType := _} -> %% Type seen before - ignore parse_timers( - TimerRefs, TimerTypes, TimeoutsR, Seen, TimeoutEvents); + TimerRefs, TimerTypes, CancelTimers, TimeoutsR, + Seen, TimeoutEvents); #{} -> %% Unseen type - handle NewSeen = Seen#{TimerType => true}, - %% Cancel any running timer - {NewTimerRefs,NewTimerTypes} = - cancel_timer_by_type(TimerType, TimerRefs, TimerTypes), - if - Time =:= infinity -> - %% Ignore - timer will never fire + case Time of + infinity -> + %% Cancel any running timer + {NewTimerTypes,NewCancelTimers} = + cancel_timer_by_type( + TimerType, TimerTypes, CancelTimers), parse_timers( - NewTimerRefs, NewTimerTypes, TimeoutsR, + TimerRefs, NewTimerTypes, NewCancelTimers, TimeoutsR, NewSeen, TimeoutEvents); - TimerType =:= timeout -> - %% Handle event timer later - parse_timers( - NewTimerRefs, NewTimerTypes, TimeoutsR, - NewSeen, [Timeout|TimeoutEvents]); - Time =:= 0 -> + 0 -> + %% Cancel any running timer + {NewTimerTypes,NewCancelTimers} = + cancel_timer_by_type( + TimerType, TimerTypes, CancelTimers), %% Handle zero time timeouts later TimeoutEvent = {TimerType,TimerMsg}, parse_timers( - NewTimerRefs, NewTimerTypes, TimeoutsR, + TimerRefs, NewTimerTypes, NewCancelTimers, TimeoutsR, NewSeen, [TimeoutEvent|TimeoutEvents]); - true -> - %% Start a new timer - TimerRef = erlang:start_timer(Time, self(), TimerMsg), - parse_timers( - NewTimerRefs#{TimerRef => TimerType}, - NewTimerTypes#{TimerType => TimerRef}, - TimeoutsR, NewSeen, TimeoutEvents) + _ -> + %% (Re)start the timer + TimerRef = + erlang:start_timer(Time, self(), TimerMsg), + case TimerTypes of + #{TimerType := OldTimerRef} -> + %% Cancel the running timer + cancel_timer(OldTimerRef), + NewCancelTimers = CancelTimers + 1, + %% Insert the new timer into + %% both TimerRefs and TimerTypes + parse_timers( + TimerRefs#{TimerRef => TimerType}, + TimerTypes#{TimerType => TimerRef}, + NewCancelTimers, TimeoutsR, + NewSeen, TimeoutEvents); + #{} -> + parse_timers( + TimerRefs#{TimerRef => TimerType}, + TimerTypes#{TimerType => TimerRef}, + CancelTimers, TimeoutsR, + NewSeen, TimeoutEvents) + end end end. -%% Enqueue immediate timeout events and start event timer -process_timeout_events(TimerRefs, TimerTypes, [], EventsR) -> - {TimerRefs, TimerTypes, EventsR}; -process_timeout_events( - TimerRefs, TimerTypes, - [{timeout,0,TimerMsg}|TimeoutEvents], []) -> - %% No enqueued events - insert a timeout zero event - TimeoutEvent = {timeout,TimerMsg}, - process_timeout_events( - TimerRefs, TimerTypes, - TimeoutEvents, [TimeoutEvent]); -process_timeout_events( - TimerRefs, TimerTypes, - [{timeout,Time,TimerMsg}], []) -> - %% No enqueued events - start event timer - TimerRef = erlang:start_timer(Time, self(), TimerMsg), - process_timeout_events( - TimerRefs#{TimerRef => timeout}, TimerTypes#{timeout => TimerRef}, - [], []); -process_timeout_events( - TimerRefs, TimerTypes, - [{timeout,_Time,_TimerMsg}|TimeoutEvents], EventsR) -> - %% There will be some other event so optimize by not starting - %% an event timer to just have to cancel it again - process_timeout_events( - TimerRefs, TimerTypes, - TimeoutEvents, EventsR); -process_timeout_events( - TimerRefs, TimerTypes, - [{_TimeoutType,_TimeoutMsg} = TimeoutEvent|TimeoutEvents], EventsR) -> - process_timeout_events( - TimerRefs, TimerTypes, - TimeoutEvents, [TimeoutEvent|EventsR]). +%% Enqueue immediate timeout events (timeout 0 events) +%% +%% Event timer timeout 0 events gets special treatment since +%% an event timer is cancelled by any received event, +%% so if there are enqueued events before the event timer +%% timeout 0 event - the event timer is cancelled hence no event. +%% +%% Other (state_timeout) timeout 0 events that are after +%% the event timer timeout 0 events are considered to +%% belong to timers that were started after the event timer +%% timeout 0 event fired, so they do not cancel the event timer. +%% +prepend_timeout_events([], EventsR) -> + EventsR; +prepend_timeout_events([{timeout,_} = TimeoutEvent|TimeoutEvents], []) -> + prepend_timeout_events(TimeoutEvents, [TimeoutEvent]); +prepend_timeout_events([{timeout,_}|TimeoutEvents], EventsR) -> + prepend_timeout_events(TimeoutEvents, EventsR); +prepend_timeout_events([TimeoutEvent|TimeoutEvents], EventsR) -> + %% Just prepend all others + prepend_timeout_events(TimeoutEvents, [TimeoutEvent|EventsR]). @@ -1448,18 +1529,11 @@ process_timeout_events( %% Server helpers reply_then_terminate( - Class, Reason, Stacktrace, - Debug, #{state := State} = S, Q, Replies) -> - if - is_list(Replies) -> - do_reply_then_terminate( - Class, Reason, Stacktrace, - Debug, S, Q, Replies, State); - true -> - do_reply_then_terminate( - Class, Reason, Stacktrace, - Debug, S, Q, [Replies], State) - end. + Class, Reason, Stacktrace, Debug, + #{state := State} = S, Q, Replies) -> + do_reply_then_terminate( + Class, Reason, Stacktrace, Debug, + S, Q, listify(Replies), State). %% do_reply_then_terminate( Class, Reason, Stacktrace, Debug, S, Q, [], _State) -> @@ -1485,21 +1559,25 @@ do_reply(Debug, S, State, From, Reply) -> terminate( - Class, Reason, Stacktrace, - Debug, + Class, Reason, Stacktrace, Debug, #{module := Module, state := State, data := Data, postponed := P} = S, Q) -> - try Module:terminate(Reason, State, Data) of - _ -> ok - catch - _ -> ok; - C:R -> - ST = erlang:get_stacktrace(), - error_info( - C, R, ST, S, Q, P, - format_status(terminate, get(), S)), - sys:print_log(Debug), - erlang:raise(C, R, ST) + case erlang:function_exported(Module, terminate, 3) of + true -> + try Module:terminate(Reason, State, Data) of + _ -> ok + catch + _ -> ok; + C:R -> + ST = erlang:get_stacktrace(), + error_info( + C, R, ST, S, Q, P, + format_status(terminate, get(), S)), + sys:print_log(Debug), + erlang:raise(C, R, ST) + end; + false -> + ok end, _ = case Reason of @@ -1637,28 +1715,21 @@ listify(Item) -> [Item]. %% Cancel timer if running, otherwise no op -cancel_timer_by_type(TimerType, TimerRefs, TimerTypes) -> +%% +%% This is an asynchronous cancel so the timer is not really cancelled +%% until we get a cancel_timer msg i.e {cancel_timer,TimerRef,_}. +%% In the mean time we might get a timeout message. +%% +%% Remove the timer from TimerTypes. +%% When we get the cancel_timer msg we remove it from TimerRefs. +cancel_timer_by_type(TimerType, TimerTypes, CancelTimers) -> case TimerTypes of #{TimerType := TimerRef} -> cancel_timer(TimerRef), - {maps:remove(TimerRef, TimerRefs), - maps:remove(TimerType, TimerTypes)}; + {maps:remove(TimerType, TimerTypes),CancelTimers + 1}; #{} -> - {TimerRefs,TimerTypes} + {TimerTypes,CancelTimers} end. -%%cancel_timer(undefined) -> -%% ok; -cancel_timer(TRef) -> - case erlang:cancel_timer(TRef) of - false -> - %% We have to assume that TRef is the ref of a running timer - %% and if so the timer has expired - %% hence we must wait for the timeout message - receive - {timeout,TRef,_} -> - ok - end; - _TimeLeft -> - ok - end. +cancel_timer(TimerRef) -> + ok = erlang:cancel_timer(TimerRef, [{async,true}]). diff --git a/lib/stdlib/src/otp_internal.erl b/lib/stdlib/src/otp_internal.erl index 2a0e3118d0..d89ff4a624 100644 --- a/lib/stdlib/src/otp_internal.erl +++ b/lib/stdlib/src/otp_internal.erl @@ -55,6 +55,11 @@ obsolete_1(erlang, now, 0) -> obsolete_1(calendar, local_time_to_universal_time, 1) -> {deprecated, {calendar, local_time_to_universal_time_dst, 1}}; +%% *** CRYPTO added in OTP 20 *** + +obsolete_1(crypto, rand_uniform, 2) -> + {deprecated, {rand, uniform, 1}}; + %% *** CRYPTO added in OTP 19 *** obsolete_1(crypto, rand_bytes, 1) -> @@ -63,178 +68,178 @@ obsolete_1(crypto, rand_bytes, 1) -> %% *** CRYPTO added in R16B01 *** obsolete_1(crypto, md4, 1) -> - {deprecated, {crypto, hash, 2}}; + {removed, {crypto, hash, 2}, "20.0"}; obsolete_1(crypto, md5, 1) -> - {deprecated, {crypto, hash, 2}}; + {removed, {crypto, hash, 2}, "20.0"}; obsolete_1(crypto, sha, 1) -> - {deprecated, {crypto, hash, 2}}; + {removed, {crypto, hash, 2}, "20.0"}; obsolete_1(crypto, md4_init, 0) -> - {deprecated, {crypto, hash_init, 1}}; + {removed, {crypto, hash_init, 1}, "20.0"}; obsolete_1(crypto, md5_init, 0) -> - {deprecated, {crypto, hash_init, 1}}; + {removed, {crypto, hash_init, 1}, "20.0"}; obsolete_1(crypto, sha_init, 0) -> - {deprecated, {crypto, hash_init, 1}}; + {removed, {crypto, hash_init, 1}, "20.0"}; obsolete_1(crypto, md4_update, 2) -> - {deprecated, {crypto, hash_update, 2}}; + {removed, {crypto, hash_update, 2}, "20.0"}; obsolete_1(crypto, md5_update, 2) -> - {deprecated, {crypto, hash_update, 2}}; + {removed, {crypto, hash_update, 2}, "20.0"}; obsolete_1(crypto, sha_update, 2) -> - {deprecated, {crypto, hash_update, 2}}; + {removed, {crypto, hash_update, 2}, "20.0"}; obsolete_1(crypto, md4_final, 1) -> - {deprecated, {crypto, hash_final, 1}}; + {removed, {crypto, hash_final, 1}, "20.0"}; obsolete_1(crypto, md5_final, 1) -> - {deprecated, {crypto, hash_final, 1}}; + {removed, {crypto, hash_final, 1}, "20.0"}; obsolete_1(crypto, sha_final, 1) -> - {deprecated, {crypto, hash_final, 1}}; + {removed, {crypto, hash_final, 1}, "20.0"}; obsolete_1(crypto, md5_mac, 2) -> - {deprecated, {crypto, hmac, 3}}; + {removed, {crypto, hmac, 3}, "20.0"}; obsolete_1(crypto, sha_mac, 2) -> - {deprecated, {crypto, hmac, 3}}; + {removed, {crypto, hmac, 3}, "20.0"}; obsolete_1(crypto, sha_mac, 3) -> - {deprecated, {crypto, hmac, 4}}; + {removed, {crypto, hmac, 4}, "20.0"}; obsolete_1(crypto, sha_mac_96, 2) -> - {deprecated, {crypto, hmac, 4}}; + {removed, {crypto, hmac, 4}, "20.0"}; obsolete_1(crypto, md5_mac_96, 2) -> - {deprecated, {crypto, hmac, 4}}; + {removed, {crypto, hmac, 4}, "20.0"}; obsolete_1(crypto, rsa_sign, 2) -> - {deprecated, {crypto, sign, 4}}; + {removed, {crypto, sign, 4}, "20.0"}; obsolete_1(crypto, rsa_sign, 3) -> - {deprecated, {crypto, sign, 4}}; + {removed, {crypto, sign, 4}, "20.0"}; obsolete_1(crypto, rsa_verify, 3) -> - {deprecated, {crypto, verify, 5}}; + {removed, {crypto, verify, 5}, "20.0"}; obsolete_1(crypto, rsa_verify, 4) -> - {deprecated, {crypto, verify, 5}}; + {removed, {crypto, verify, 5}, "20.0"}; obsolete_1(crypto, dss_sign, 2) -> - {deprecated, {crypto, sign, 4}}; + {removed, {crypto, sign, 4}, "20.0"}; obsolete_1(crypto, dss_sign, 3) -> - {deprecated, {crypto, sign, 4}}; + {removed, {crypto, sign, 4}, "20.0"}; obsolete_1(crypto, dss_verify, 3) -> - {deprecated, {crypto, verify, 5}}; + {removed, {crypto, verify, 5}, "20.0"}; obsolete_1(crypto, dss_verify, 4) -> - {deprecated, {crypto, verify, 5}}; + {removed, {crypto, verify, 5}, "20.0"}; obsolete_1(crypto, mod_exp, 3) -> - {deprecated, {crypto, mod_pow, 3}}; + {removed, {crypto, mod_pow, 3}, "20.0"}; obsolete_1(crypto, dh_compute_key, 3) -> - {deprecated, {crypto, compute_key, 4}}; + {removed, {crypto, compute_key, 4}, "20.0"}; obsolete_1(crypto, dh_generate_key, 1) -> - {deprecated, {crypto, generate_key, 2}}; + {removed, {crypto, generate_key, 2}, "20.0"}; obsolete_1(crypto, dh_generate_key, 2) -> - {deprecated, {crypto, generate_key, 3}}; + {removed, {crypto, generate_key, 3}, "20.0"}; obsolete_1(crypto, des_cbc_encrypt, 3) -> - {deprecated, {crypto, block_encrypt, 4}}; + {removed, {crypto, block_encrypt, 4}, "20.0"}; obsolete_1(crypto, des3_cbc_encrypt, 5) -> - {deprecated, {crypto, block_encrypt, 4}}; + {removed, {crypto, block_encrypt, 4}, "20.0"}; obsolete_1(crypto, des_ecb_encrypt, 2) -> - {deprecated, {crypto, block_encrypt, 3}}; + {removed, {crypto, block_encrypt, 3}, "20.0"}; obsolete_1(crypto, des_ede3_cbc_encrypt, 5) -> - {deprecated, {crypto, block_encrypt, 4}}; + {removed, {crypto, block_encrypt, 4}, "20.0"}; obsolete_1(crypto, des_cfb_encrypt, 3) -> - {deprecated, {crypto, block_encrypt, 4}}; + {removed, {crypto, block_encrypt, 4}, "20.0"}; obsolete_1(crypto, des3_cfb_encrypt, 5) -> - {deprecated, {crypto, block_encrypt, 4}}; + {removed, {crypto, block_encrypt, 4}, "20.0"}; obsolete_1(crypto, blowfish_ecb_encrypt, 2) -> - {deprecated, {crypto, block_encrypt, 3}}; + {removed, {crypto, block_encrypt, 3}, "20.0"}; obsolete_1(crypto, blowfish_cbc_encrypt, 3) -> - {deprecated, {crypto, block_encrypt, 4}}; + {removed, {crypto, block_encrypt, 4}, "20.0"}; obsolete_1(crypto, blowfish_cfb64_encrypt, 3) -> - {deprecated, {crypto, block_encrypt, 4}}; + {removed, {crypto, block_encrypt, 4}, "20.0"}; obsolete_1(crypto, blowfish_ofb64_encrypt, 3) -> - {deprecated, {crypto, block_encrypt, 4}}; + {removed, {crypto, block_encrypt, 4}, "20.0"}; obsolete_1(crypto, aes_cfb_128_encrypt, 3) -> - {deprecated, {crypto, block_encrypt, 4}}; + {removed, {crypto, block_encrypt, 4}, "20.0"}; obsolete_1(crypto, aes_cbc_128_encrypt, 3) -> - {deprecated, {crypto, block_encrypt, 4}}; + {removed, {crypto, block_encrypt, 4}, "20.0"}; obsolete_1(crypto, aes_cbc_256_encrypt, 3) -> - {deprecated, {crypto, block_encrypt, 4}}; + {removed, {crypto, block_encrypt, 4}, "20.0"}; obsolete_1(crypto,rc2_cbc_encrypt, 3) -> - {deprecated, {crypto, block_encrypt, 4}}; + {removed, {crypto, block_encrypt, 4}, "20.0"}; obsolete_1(crypto,rc2_40_cbc_encrypt, 3) -> - {deprecated, {crypto, block_encrypt, 4}}; + {removed, {crypto, block_encrypt, 4}, "20.0"}; obsolete_1(crypto, des_cbc_decrypt, 3) -> - {deprecated, {crypto, block_decrypt, 4}}; + {removed, {crypto, block_decrypt, 4}, "20.0"}; obsolete_1(crypto, des3_cbc_decrypt, 5) -> - {deprecated, {crypto, block_decrypt, 4}}; + {removed, {crypto, block_decrypt, 4}, "20.0"}; obsolete_1(crypto, des_ecb_decrypt, 2) -> - {deprecated, {crypto, block_decrypt, 3}}; + {removed, {crypto, block_decrypt, 3}, "20.0"}; obsolete_1(crypto, des_ede3_cbc_decrypt, 5) -> - {deprecated, {crypto, block_decrypt, 4}}; + {removed, {crypto, block_decrypt, 4}, "20.0"}; obsolete_1(crypto, des_cfb_decrypt, 3) -> - {deprecated, {crypto, block_decrypt, 4}}; + {removed, {crypto, block_decrypt, 4}, "20.0"}; obsolete_1(crypto, des3_cfb_decrypt, 5) -> - {deprecated, {crypto, block_decrypt, 4}}; + {removed, {crypto, block_decrypt, 4}, "20.0"}; obsolete_1(crypto, blowfish_ecb_decrypt, 2) -> - {deprecated, {crypto, block_decrypt, 3}}; + {removed, {crypto, block_decrypt, 3}, "20.0"}; obsolete_1(crypto, blowfish_cbc_decrypt, 3) -> - {deprecated, {crypto, block_decrypt, 4}}; + {removed, {crypto, block_decrypt, 4}, "20.0"}; obsolete_1(crypto, blowfish_cfb64_decrypt, 3) -> - {deprecated, {crypto, block_decrypt, 4}}; + {removed, {crypto, block_decrypt, 4}, "20.0"}; obsolete_1(crypto, blowfish_ofb64_decrypt, 3) -> - {deprecated, {crypto, block_decrypt, 4}}; + {removed, {crypto, block_decrypt, 4}, "20.0"}; obsolete_1(crypto, aes_cfb_128_decrypt, 3) -> - {deprecated, {crypto, block_decrypt, 4}}; + {removed, {crypto, block_decrypt, 4}, "20.0"}; obsolete_1(crypto, aes_cbc_128_decrypt, 3) -> - {deprecated, {crypto, block_decrypt, 4}}; + {removed, {crypto, block_decrypt, 4}, "20.0"}; obsolete_1(crypto, aes_cbc_256_decrypt, 3) -> - {deprecated, {crypto, block_decrypt, 4}}; + {removed, {crypto, block_decrypt, 4}, "20.0"}; obsolete_1(crypto,rc2_cbc_decrypt, 3) -> - {deprecated, {crypto, block_decrypt, 4}}; + {removed, {crypto, block_decrypt, 4}, "20.0"}; obsolete_1(crypto,rc2_40_cbc_decrypt, 3) -> - {deprecated, {crypto, block_decrypt, 4}}; + {removed, {crypto, block_decrypt, 4}, "20.0"}; obsolete_1(crypto, aes_ctr_stream_decrypt, 2) -> - {deprecated, {crypto, stream_decrypt, 2}}; + {removed, {crypto, stream_decrypt, 2}, "20.0"}; obsolete_1(crypto, aes_ctr_stream_encrypt, 2) -> - {deprecated, {crypto, stream_encrypt, 2}}; + {removed, {crypto, stream_encrypt, 2}, "20.0"}; obsolete_1(crypto, aes_ctr_decrypt, 3) -> - {deprecated, {crypto, stream_decrypt, 2}}; + {removed, {crypto, stream_decrypt, 2}, "20.0"}; obsolete_1(crypto, aes_ctr_encrypt, 3) -> - {deprecated, {crypto, stream_encrypt, 2}}; + {removed, {crypto, stream_encrypt, 2}, "20.0"}; obsolete_1(crypto, rc4_encrypt, 2) -> - {deprecated, {crypto, stream_encrypt, 2}}; + {removed, {crypto, stream_encrypt, 2}, "20.0"}; obsolete_1(crypto, rc4_encrypt_with_state, 2) -> - {deprecated, {crypto, stream_encrypt, 2}}; + {removed, {crypto, stream_encrypt, 2}, "20.0"}; obsolete_1(crypto, aes_ctr_stream_init, 2) -> - {deprecated, {crypto, stream_init, 3}}; + {removed, {crypto, stream_init, 3}, "20.0"}; obsolete_1(crypto, rc4_set_key, 1) -> - {deprecated, {crypto, stream_init, 2}}; + {removed, {crypto, stream_init, 2}, "20.0"}; obsolete_1(crypto, rsa_private_decrypt, 3) -> - {deprecated, {crypto, private_decrypt, 4}}; + {removed, {crypto, private_decrypt, 4}, "20.0"}; obsolete_1(crypto, rsa_public_decrypt, 3) -> - {deprecated, {crypto, public_decrypt, 4}}; + {removed, {crypto, public_decrypt, 4}, "20.0"}; obsolete_1(crypto, rsa_private_encrypt, 3) -> - {deprecated, {crypto, private_encrypt, 4}}; + {removed, {crypto, private_encrypt, 4}, "20.0"}; obsolete_1(crypto, rsa_public_encrypt, 3) -> - {deprecated, {crypto, public_encrypt, 4}}; + {removed, {crypto, public_encrypt, 4}, "20.0"}; obsolete_1(crypto, des_cfb_ivec, 2) -> - {deprecated, {crypto, next_iv, 3}}; + {removed, {crypto, next_iv, 3}, "20.0"}; obsolete_1(crypto,des_cbc_ivec, 1) -> - {deprecated, {crypto, next_iv, 2}}; + {removed, {crypto, next_iv, 2}, "20.0"}; obsolete_1(crypto, aes_cbc_ivec, 1) -> - {deprecated, {crypto, next_iv, 2}}; + {removed, {crypto, next_iv, 2}, "20.0"}; obsolete_1(crypto,info, 0) -> - {deprecated, {crypto, module_info, 0}}; + {removed, {crypto, module_info, 0}, "20.0"}; obsolete_1(crypto, strong_rand_mpint, 3) -> - {deprecated, "needed only by deprecated functions"}; + {removed, "removed in 20.0; only needed by removed functions"}; obsolete_1(crypto, erlint, 1) -> - {deprecated, "needed only by deprecated functions"}; + {removed, "removed in 20.0; only needed by removed functions"}; obsolete_1(crypto, mpint, 1) -> - {deprecated, "needed only by deprecated functions"}; + {removed, "removed in 20.0; only needed by removed functions"}; %% *** SNMP *** @@ -387,13 +392,13 @@ obsolete_1(erlang, concat_binary, 1) -> %% Added in R14A. obsolete_1(ssl, peercert, 2) -> - {deprecated,"deprecated (will be removed in R15A); use ssl:peercert/1 and public_key:pkix_decode_cert/2 instead"}; + {removed ,"removed in R15A; use ssl:peercert/1 and public_key:pkix_decode_cert/2 instead"}; %% Added in R14B. obsolete_1(public_key, pem_to_der, 1) -> - {deprecated,"deprecated (will be removed in R15A); use file:read_file/1 and public_key:pem_decode/1"}; + {removed,"removed in R15A; use file:read_file/1 and public_key:pem_decode/1"}; obsolete_1(public_key, decode_private_key, A) when A =:= 1; A =:= 2 -> - {deprecated,{public_key,pem_entry_decode,1},"R15A"}; + {removed, "removed in R15A; use public_key:pem_entry_decode/1"}; %% Added in R14B03. obsolete_1(docb_gen, _, _) -> @@ -415,10 +420,10 @@ obsolete_1(inviso, _, _) -> obsolete_1(gs, _, _) -> {removed,"the gs application has been removed; use the wx application instead"}; obsolete_1(ssh, sign_data, 2) -> - {deprecated,"deprecated (will be removed in R16A); use public_key:pem_decode/1, public_key:pem_entry_decode/1 " + {removed,"removed in R16A; use public_key:pem_decode/1, public_key:pem_entry_decode/1 " "and public_key:sign/3 instead"}; obsolete_1(ssh, verify_data, 3) -> - {deprecated,"deprecated (will be removed in R16A); use public_key:ssh_decode/1, and public_key:verify/4 instead"}; + {removed,"removed in R16A; use public_key:ssh_decode/1, and public_key:verify/4 instead"}; %% Added in R16 obsolete_1(wxCalendarCtrl, enableYearChange, _) -> %% wx bug documented? @@ -515,10 +520,9 @@ obsolete_1(erl_parse, get_attribute, 2) -> obsolete_1(erl_lint, modify_line, 2) -> {removed,{erl_parse,map_anno,2},"19.0"}; obsolete_1(ssl, negotiated_next_protocol, 1) -> - {deprecated,{ssl,negotiated_protocol,1}}; - + {removed,"removed in 20.0; use ssl:negotiated_protocol/1 instead"}; obsolete_1(ssl, connection_info, 1) -> - {deprecated, "deprecated; use connection_information/[1,2] instead"}; + {removed, "removed in 20.0; use ssl:connection_information/[1,2] instead"}; obsolete_1(httpd_conf, check_enum, 2) -> {deprecated, "deprecated; use lists:member/2 instead"}; @@ -548,7 +552,7 @@ obsolete_1(queue, lait, 1) -> obsolete_1(overload, _, _) -> {removed, "removed in OTP 19"}; obsolete_1(rpc, safe_multi_server_call, A) when A =:= 2; A =:= 3 -> - {removed, {rpc, multi_server_call, A}}; + {removed, {rpc, multi_server_call, A}, "removed in OTP 19"}; %% Added in OTP 20. diff --git a/lib/stdlib/src/qlc.erl b/lib/stdlib/src/qlc.erl index f3665824f2..8c4d835432 100644 --- a/lib/stdlib/src/qlc.erl +++ b/lib/stdlib/src/qlc.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2016. All Rights Reserved. +%% Copyright Ericsson AB 2004-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1292,6 +1292,10 @@ abstr_term(Fun, Line) when is_function(Fun) -> end; abstr_term(PPR, Line) when is_pid(PPR); is_port(PPR); is_reference(PPR) -> {special, Line, lists:flatten(io_lib:write(PPR))}; +abstr_term(Map, Line) when is_map(Map) -> + {map,Line, + [{map_field_assoc,Line,abstr_term(K, Line),abstr_term(V, Line)} || + {K,V} <- maps:to_list(Map)]}; abstr_term(Simple, Line) -> erl_parse:abstract(Simple, erl_anno:line(Line)). diff --git a/lib/stdlib/src/qlc_pt.erl b/lib/stdlib/src/qlc_pt.erl index 28221ea75f..4a39f8ae9d 100644 --- a/lib/stdlib/src/qlc_pt.erl +++ b/lib/stdlib/src/qlc_pt.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2016. All Rights Reserved. +%% Copyright Ericsson AB 2004-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -439,7 +439,7 @@ compile_forms(Forms0, Options) -> (_) -> false end, Forms = ([F || F <- Forms0, not Exclude(element(1, F))] - ++ [{eof,anno0()}]), + ++ [{eof,0}]), try case compile:noenv_forms(Forms, compile_options(Options)) of {ok, _ModName, Ws0} -> diff --git a/lib/stdlib/src/rand.erl b/lib/stdlib/src/rand.erl index 1f457b9e0e..dfd102f9ef 100644 --- a/lib/stdlib/src/rand.erl +++ b/lib/stdlib/src/rand.erl @@ -45,20 +45,31 @@ %% ===================================================================== %% This depends on the algorithm handler function --type alg_seed() :: exs64_state() | exsplus_state() | exs1024_state(). +-type alg_state() :: + exs64_state() | exsplus_state() | exs1024_state() | term(). + %% This is the algorithm handler function within this module --type alg_handler() :: #{type := alg(), - max := integer(), - next := fun(), - uniform := fun(), - uniform_n := fun(), - jump := fun()}. - -%% Internal state --opaque state() :: {alg_handler(), alg_seed()}. --type alg() :: exs64 | exsplus | exs1024. --opaque export_state() :: {alg(), alg_seed()}. --export_type([alg/0, state/0, export_state/0]). +-type alg_handler() :: + #{type := alg(), + max := integer() | infinity, + next := + fun((alg_state()) -> {non_neg_integer(), alg_state()}), + uniform := + fun((state()) -> {float(), state()}), + uniform_n := + fun((pos_integer(), state()) -> {pos_integer(), state()}), + jump := + fun((state()) -> state())}. + +%% Algorithm state +-type state() :: {alg_handler(), alg_state()}. +-type builtin_alg() :: exs64 | exsplus | exs1024. +-type alg() :: builtin_alg() | atom(). +-type export_state() :: {alg(), alg_state()}. +-export_type( + [builtin_alg/0, alg/0, alg_handler/0, alg_state/0, + state/0, export_state/0]). +-export_type([exs64_state/0, exsplus_state/0, exs1024_state/0]). %% ===================================================================== %% API @@ -72,7 +83,7 @@ export_seed() -> _ -> undefined end. --spec export_seed_s(state()) -> export_state(). +-spec export_seed_s(State :: state()) -> export_state(). export_seed_s({#{type:=Alg}, Seed}) -> {Alg, Seed}. %% seed(Alg) seeds RNG with runtime dependent values @@ -81,27 +92,37 @@ export_seed_s({#{type:=Alg}, Seed}) -> {Alg, Seed}. %% seed({Alg,Seed}) setup RNG with a previously exported seed %% and return the NEW state --spec seed(AlgOrExpState::alg() | export_state()) -> state(). +-spec seed( + AlgOrStateOrExpState :: builtin_alg() | state() | export_state()) -> + state(). seed(Alg) -> seed_put(seed_s(Alg)). --spec seed_s(AlgOrExpState::alg() | export_state()) -> state(). -seed_s(Alg) when is_atom(Alg) -> - seed_s(Alg, {erlang:phash2([{node(),self()}]), - erlang:system_time(), - erlang:unique_integer()}); +-spec seed_s( + AlgOrStateOrExpState :: builtin_alg() | state() | export_state()) -> + state(). +seed_s({AlgHandler, _Seed} = State) when is_map(AlgHandler) -> + State; seed_s({Alg0, Seed}) -> {Alg,_SeedFun} = mk_alg(Alg0), - {Alg, Seed}. + {Alg, Seed}; +seed_s(Alg) -> + seed_s(Alg, {erlang:phash2([{node(),self()}]), + erlang:system_time(), + erlang:unique_integer()}). %% seed/2: seeds RNG with the algorithm and given values %% and returns the NEW state. --spec seed(Alg :: alg(), {integer(), integer(), integer()}) -> state(). +-spec seed( + Alg :: builtin_alg(), Seed :: {integer(), integer(), integer()}) -> + state(). seed(Alg0, S0) -> seed_put(seed_s(Alg0, S0)). --spec seed_s(Alg :: alg(), {integer(), integer(), integer()}) -> state(). +-spec seed_s( + Alg :: builtin_alg(), Seed :: {integer(), integer(), integer()}) -> + state(). seed_s(Alg0, S0 = {_, _, _}) -> {Alg, Seed} = mk_alg(Alg0), AS = Seed(S0), @@ -113,7 +134,7 @@ seed_s(Alg0, S0 = {_, _, _}) -> %% uniform/0: returns a random float X where 0.0 < X < 1.0, %% updating the state in the process dictionary. --spec uniform() -> X::float(). +-spec uniform() -> X :: float(). uniform() -> {X, Seed} = uniform_s(seed_get()), _ = seed_put(Seed), @@ -123,7 +144,7 @@ uniform() -> %% uniform/1 returns a random integer X where 1 =< X =< N, %% updating the state in the process dictionary. --spec uniform(N :: pos_integer()) -> X::pos_integer(). +-spec uniform(N :: pos_integer()) -> X :: pos_integer(). uniform(N) -> {X, Seed} = uniform_s(N, seed_get()), _ = seed_put(Seed), @@ -133,7 +154,7 @@ uniform(N) -> %% returns a random float X where 0.0 < X < 1.0, %% and a new state. --spec uniform_s(state()) -> {X::float(), NewS :: state()}. +-spec uniform_s(State :: state()) -> {X :: float(), NewState :: state()}. uniform_s(State = {#{uniform:=Uniform}, _}) -> Uniform(State). @@ -141,7 +162,8 @@ uniform_s(State = {#{uniform:=Uniform}, _}) -> %% uniform_s/2 returns a random integer X where 1 =< X =< N, %% and a new state. --spec uniform_s(N::pos_integer(), state()) -> {X::pos_integer(), NewS::state()}. +-spec uniform_s(N :: pos_integer(), State :: state()) -> + {X :: pos_integer(), NewState :: state()}. uniform_s(N, State = {#{uniform_n:=Uniform, max:=Max}, _}) when 0 < N, N =< Max -> Uniform(N, State); @@ -155,7 +177,7 @@ uniform_s(N, State0 = {#{uniform:=Uniform}, _}) %% after a large number of call defined for each algorithm. %% The large number is algorithm dependent. --spec jump(state()) -> NewS :: state(). +-spec jump(state()) -> NewState :: state(). jump(State = {#{jump:=Jump}, _}) -> Jump(State). @@ -164,7 +186,7 @@ jump(State = {#{jump:=Jump}, _}) -> %% and write back the new value to the internal state, %% then returns the new value. --spec jump() -> NewS :: state(). +-spec jump() -> NewState :: state(). jump() -> seed_put(jump(seed_get())). @@ -182,7 +204,7 @@ normal() -> %% The Ziggurat Method for generating random variables - Marsaglia and Tsang %% Paper and reference code: http://www.jstatsoft.org/v05/i08/ --spec normal_s(state()) -> {float(), NewS :: state()}. +-spec normal_s(State :: state()) -> {float(), NewState :: state()}. normal_s(State0) -> {Sign, R, State} = get_52(State0), Idx = R band 16#FF, @@ -245,7 +267,7 @@ mk_alg(exs1024) -> %% Reference URL: http://xorshift.di.unimi.it/ %% ===================================================================== --type exs64_state() :: uint64(). +-opaque exs64_state() :: uint64(). exs64_seed({A1, A2, A3}) -> {V1, _} = exs64_next(((A1 band ?UINT32MASK) * 4294967197 + 1)), @@ -280,7 +302,7 @@ exs64_jump(_) -> %% Modification of the original Xorshift128+ algorithm to 116 %% by Sebastiano Vigna, a lot of thanks for his help and work. %% ===================================================================== --type exsplus_state() :: nonempty_improper_list(uint58(), uint58()). +-opaque exsplus_state() :: nonempty_improper_list(uint58(), uint58()). -dialyzer({no_improper_lists, exsplus_seed/1}). @@ -349,7 +371,7 @@ exsplus_jump(S, [AS0|AS1], J, N) -> %% Reference URL: http://xorshift.di.unimi.it/ %% ===================================================================== --type exs1024_state() :: {list(uint64()), list(uint64())}. +-opaque exs1024_state() :: {list(uint64()), list(uint64())}. exs1024_seed({A1, A2, A3}) -> B1 = (((A1 band ?UINT21MASK) + 1) * 2097131) band ?UINT21MASK, diff --git a/lib/stdlib/src/stdlib.appup.src b/lib/stdlib/src/stdlib.appup.src index 979161fef7..3c9e95e3a9 100644 --- a/lib/stdlib/src/stdlib.appup.src +++ b/lib/stdlib/src/stdlib.appup.src @@ -18,7 +18,7 @@ %% %CopyrightEnd% {"%VSN%", %% Up from - max one major revision back - [{<<"3\\.[0-1](\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-19.* + [{<<"3\\.[0-3](\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-19.* %% Down to - max one major revision back - [{<<"3\\.[0-1](\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-19.* + [{<<"3\\.[0-3](\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-19.* }. diff --git a/lib/stdlib/src/zip.erl b/lib/stdlib/src/zip.erl index 340cc21390..fadf96146e 100644 --- a/lib/stdlib/src/zip.erl +++ b/lib/stdlib/src/zip.erl @@ -179,19 +179,6 @@ external_attr, local_header_offset}). -%% Unix extra fields (not yet supported) --define(UNIX_EXTRA_FIELD_TAG, 16#000d). --record(unix_extra_field, {atime, - mtime, - uid, - gid}). - -%% extended timestamps (not yet supported) --define(EXTENDED_TIMESTAMP_TAG, 16#5455). -%% -record(extended_timestamp, {mtime, -%% atime, -%% ctime}). - -define(END_OF_CENTRAL_DIR_MAGIC, 16#06054b50). -define(END_OF_CENTRAL_DIR_SZ, (4+2+2+2+2+4+4+2)). @@ -381,9 +368,12 @@ do_unzip(F, Options) -> {Info, In1} = get_central_dir(In0, RawIterator, Input), %% get rid of zip-comment Z = zlib:open(), - Files = get_z_files(Info, Z, In1, Opts, []), - zlib:close(Z), - Input(close, In1), + Files = try + get_z_files(Info, Z, In1, Opts, []) + after + zlib:close(Z), + Input(close, In1) + end, {ok, Files}. %% Iterate over all files in a zip archive @@ -460,11 +450,20 @@ do_zip(F, Files, Options) -> #zip_opts{output = Output, open_opts = OpO} = Opts, Out0 = Output({open, F, OpO}, []), Z = zlib:open(), - {Out1, LHS, Pos} = put_z_files(Files, Z, Out0, 0, Opts, []), - zlib:close(Z), - Out2 = put_central_dir(LHS, Pos, Out1, Opts), - Out3 = Output({close, F}, Out2), - {ok, Out3}. + try + {Out1, LHS, Pos} = put_z_files(Files, Z, Out0, 0, Opts, []), + zlib:close(Z), + Out2 = put_central_dir(LHS, Pos, Out1, Opts), + Out3 = Output({close, F}, Out2), + {ok, Out3} + catch + C:R -> + Stk = erlang:get_stacktrace(), + zlib:close(Z), + Output({close, F}, Out0), + erlang:raise(C, R, Stk) + end. + %% List zip directory contents %% @@ -1379,12 +1378,7 @@ cd_file_header_to_file_info(FileName, gid = 0}, add_extra_info(FI, ExtraField). -%% add extra info to file (some day when we implement it) -add_extra_info(FI, <<?EXTENDED_TIMESTAMP_TAG:16/little, _Rest/binary>>) -> - FI; % not yet supported, some other day... -add_extra_info(FI, <<?UNIX_EXTRA_FIELD_TAG:16/little, Rest/binary>>) -> - _UnixExtra = unix_extra_field_and_var_from_bin(Rest), - FI; % not yet supported, and not widely used +%% Currently, we ignore all the extra fields. add_extra_info(FI, _) -> FI. @@ -1572,20 +1566,6 @@ dos_date_time_from_datetime({{Year, Month, Day}, {Hour, Min, Sec}}) -> <<DosDate:16>> = <<YearFrom1980:7, Month:4, Day:5>>, {DosDate, DosTime}. -unix_extra_field_and_var_from_bin(<<TSize:16/little, - ATime:32/little, - MTime:32/little, - UID:16/little, - GID:16/little, - Var:TSize/binary>>) -> - {#unix_extra_field{atime = ATime, - mtime = MTime, - uid = UID, - gid = GID}, - Var}; -unix_extra_field_and_var_from_bin(_) -> - throw(bad_unix_extra_field). - %% A pwrite-like function for iolists (used by memory-option) pwrite_binary(B, Pos, Bin) when byte_size(B) =:= Pos -> diff --git a/lib/stdlib/test/erl_lint_SUITE.erl b/lib/stdlib/test/erl_lint_SUITE.erl index c7dcd9ae16..fd7de65302 100644 --- a/lib/stdlib/test/erl_lint_SUITE.erl +++ b/lib/stdlib/test/erl_lint_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2016. All Rights Reserved. +%% Copyright Ericsson AB 1999-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -64,7 +64,8 @@ predef/1, maps/1,maps_type/1,maps_parallel_match/1, otp_11851/1,otp_11879/1,otp_13230/1, - record_errors/1, otp_xxxxx/1]). + record_errors/1, otp_xxxxx/1, + non_latin1_module/1]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -84,7 +85,7 @@ all() -> too_many_arguments, basic_errors, bin_syntax_errors, predef, maps, maps_type, maps_parallel_match, otp_11851, otp_11879, otp_13230, - record_errors, otp_xxxxx]. + record_errors, otp_xxxxx, non_latin1_module]. groups() -> [{unused_vars_warn, [], @@ -2098,11 +2099,11 @@ otp_5362(Config) when is_list(Config) -> [{2,erl_lint,disallowed_nowarn_bif_clash}],[]}}, {call_deprecated_function, - <<"t(X) -> crypto:md5(X).">>, + <<"t(X) -> calendar:local_time_to_universal_time(X).">>, [], {warnings, - [{1,erl_lint,{deprecated,{crypto,md5,1}, - {crypto,hash,2}, "a future release"}}]}}, + [{1,erl_lint,{deprecated,{calendar,local_time_to_universal_time,1}, + {calendar,local_time_to_universal_time_dst,1}, "a future release"}}]}}, {call_removed_function, <<"t(X) -> regexp:match(X).">>, @@ -2549,7 +2550,7 @@ otp_5878(Config) when is_list(Config) -> {function,9,t,0,[{clause,9,[],[],[{record,10,r,[]}]}]}, {eof,11}], {error,[{"rec.erl",[{7,erl_lint,old_abstract_code}]}],[]} = - compile:forms(OldAbstract, [return, report]), + compile_forms(OldAbstract, [return, report]), ok. @@ -3848,9 +3849,13 @@ otp_11879(_Config) -> [{1,erl_lint,{spec_fun_undefined,{f,1}}}, {2,erl_lint,spec_wrong_arity}, {22,erl_lint,callback_wrong_arity}]}], - []} = compile:forms(Fs, [return,report]), + []} = compile_forms(Fs, [return,report]), ok. +compile_forms(Terms, Opts) -> + Forms = [erl_parse:anno_from_term(Term) || Term <- Terms], + compile:forms(Forms, Opts). + %% OTP-13230: -deprecated without -module. otp_13230(Config) when is_list(Config) -> Abstr = <<"-deprecated([{frutt,0,next_version}]).">>, @@ -3919,6 +3924,24 @@ otp_xxxxx(Config) -> []}], run(Config, Ts). +%% OTP-14285: We currently don't support non-latin1 module names. + +non_latin1_module(_Config) -> + do_non_latin1_module('юникод'), + do_non_latin1_module(list_to_atom([256,$a,$b,$c])), + do_non_latin1_module(list_to_atom([$a,$b,256,$c])), + ok. + +do_non_latin1_module(Mod) -> + File = atom_to_list(Mod) ++ ".erl", + Forms = [{attribute,1,file,{File,1}}, + {attribute,1,module,Mod}, + {eof,2}], + error = compile:forms(Forms), + {error,_,[]} = compile:forms(Forms, [return]), + ok. + + run(Config, Tests) -> F = fun({N,P,Ws,E}, BadL) -> case catch run_test(Config, P, Ws) of diff --git a/lib/stdlib/test/erl_pp_SUITE.erl b/lib/stdlib/test/erl_pp_SUITE.erl index 31ea3210a8..1a028204b4 100644 --- a/lib/stdlib/test/erl_pp_SUITE.erl +++ b/lib/stdlib/test/erl_pp_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2016. All Rights Reserved. +%% Copyright Ericsson AB 2006-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1068,10 +1068,10 @@ otp_11100(Config) when is_list(Config) -> %% There are a few places where the added code ("options(none)") %% doesn't make a difference (pp:bit_elem_type/1 is an example). + A1 = erl_anno:new(1), %% Cannot trigger the use of the hook function with export/import. "-export([{fy,a}/b]).\n" = - pf({attribute,1,export,[{{fy,a},b}]}), - A1 = erl_anno:new(1), + pf({attribute,A1,export,[{{fy,a},b}]}), "-type foo() :: integer(INVALID-FORM:{foo,bar}:).\n" = pf({attribute,A1,type,{foo,{type,A1,integer,[{foo,bar}]},[]}}), pf({attribute,A1,type, @@ -1100,10 +1100,11 @@ otp_11100(Config) when is_list(Config) -> %% OTP-11861. behaviour_info() and -callback. otp_11861(Config) when is_list(Config) -> + A3 = erl_anno:new(3), "-optional_callbacks([bar/0]).\n" = - pf({attribute,3,optional_callbacks,[{bar,0}]}), + pf({attribute,A3,optional_callbacks,[{bar,0}]}), "-optional_callbacks([{bar,1,bad}]).\n" = - pf({attribute,4,optional_callbacks,[{bar,1,bad}]}), + pf({attribute,A3,optional_callbacks,[{bar,1,bad}]}), ok. pf(Form) -> diff --git a/lib/stdlib/test/erl_scan_SUITE.erl b/lib/stdlib/test/erl_scan_SUITE.erl index 7d0ba967f9..aca5b1e54f 100644 --- a/lib/stdlib/test/erl_scan_SUITE.erl +++ b/lib/stdlib/test/erl_scan_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2016. All Rights Reserved. +%% Copyright Ericsson AB 1998-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -904,9 +904,9 @@ more_chars() -> otp_10302(Config) when is_list(Config) -> %% From unicode(): {ok,[{atom,1,'aсb'}],1} = - erl_scan:string("'a"++[1089]++"b'", 1), + erl_scan_string("'a"++[1089]++"b'", 1), {ok,[{atom,{1,1},'qaપ'}],{1,12}} = - erl_scan:string("'qa\\x{aaa}'",{1,1}), + erl_scan_string("'qa\\x{aaa}'",{1,1}), {ok,[{char,1,1089}],1} = erl_scan_string([$$,1089], 1), {ok,[{char,1,1089}],1} = erl_scan_string([$$,$\\,1089],1), diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 8581440d58..ac68fdcc34 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -39,8 +39,9 @@ -export([lookup_element_mult/1]). -export([foldl_ordered/1, foldr_ordered/1, foldl/1, foldr/1, fold_empty/1]). -export([t_delete_object/1, t_init_table/1, t_whitebox/1, + select_bound_chunk/1, t_delete_all_objects/1, t_insert_list/1, t_test_ms/1, - t_select_delete/1,t_ets_dets/1]). + t_select_delete/1,t_select_replace/1,t_ets_dets/1]). -export([ordered/1, ordered_match/1, interface_equality/1, fixtable_next/1, fixtable_insert/1, rename/1, rename_unnamed/1, evil_rename/1, @@ -64,7 +65,7 @@ meta_lookup_named_read/1, meta_lookup_named_write/1, meta_newdel_unnamed/1, meta_newdel_named/1]). -export([smp_insert/1, smp_fixed_delete/1, smp_unfix_fix/1, smp_select_delete/1, - otp_8166/1, otp_8732/1]). + smp_select_replace/1, otp_8166/1, otp_8732/1]). -export([exit_large_table_owner/1, exit_many_large_table_owner/1, exit_many_tables_owner/1, @@ -75,7 +76,7 @@ -export([otp_9423/1]). -export([otp_10182/1]). -export([ets_all/1]). --export([memory_check_summary/1]). +-export([massive_ets_all/1]). -export([take/1]). -export([init_per_testcase/2, end_per_testcase/2]). @@ -87,13 +88,13 @@ -include_lib("common_test/include/ct.hrl"). -define(m(A,B), assert_eq(A,B)). +-define(heap_binary_size, 64). init_per_testcase(Case, Config) -> rand:seed(exsplus), io:format("*** SEED: ~p ***\n", [rand:export_seed()]), start_spawn_logger(), wait_for_test_procs(), %% Ensure previous case cleaned up - put('__ETS_TEST_CASE__', Case), [{test_case, Case} | Config]. end_per_testcase(_Func, _Config) -> @@ -118,15 +119,16 @@ all() -> update_counter_with_default, partly_bound, update_counter_table_growth, match_heavy, {group, fold}, member, t_delete_object, + select_bound_chunk, t_init_table, t_whitebox, t_delete_all_objects, - t_insert_list, t_test_ms, t_select_delete, t_ets_dets, - memory, t_select_reverse, t_bucket_disappears, + t_insert_list, t_test_ms, t_select_delete, t_select_replace, + t_ets_dets, memory, t_select_reverse, t_bucket_disappears, select_fail, t_insert_new, t_repair_continuation, otp_5340, otp_6338, otp_6842_select_1000, otp_7665, otp_8732, meta_wb, grow_shrink, grow_pseudo_deleted, shrink_pseudo_deleted, {group, meta_smp}, smp_insert, - smp_fixed_delete, smp_unfix_fix, smp_select_delete, - otp_8166, exit_large_table_owner, + smp_fixed_delete, smp_unfix_fix, smp_select_replace, + smp_select_delete, otp_8166, exit_large_table_owner, exit_many_large_table_owner, exit_many_tables_owner, exit_many_many_tables_owner, write_concurrency, heir, give_away, setopts, bad_table, types, @@ -134,9 +136,8 @@ all() -> otp_9932, otp_9423, ets_all, - take, - - memory_check_summary]. % MUST BE LAST + massive_ets_all, + take]. groups() -> [{new, [], @@ -181,27 +182,6 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. -%% Test that we did not have "too many" failed verify_etsmem()'s -%% in the test suite. -%% verify_etsmem() may give a low number of false positives -%% as concurrent activities, such as lingering processes -%% from earlier test suites, may do unrelated ets (de)allocations. -memory_check_summary(_Config) -> - case whereis(ets_test_spawn_logger) of - undefined -> - ct:fail("No spawn logger exist"); - _ -> - ets_test_spawn_logger ! {self(), get_failed_memchecks}, - receive {get_failed_memchecks, FailedMemchecks} -> ok end, - io:format("Failed memchecks: ~p\n",[FailedMemchecks]), - NoFailedMemchecks = length(FailedMemchecks), - if NoFailedMemchecks > 1 -> - ct:fail("Too many failed (~p) memchecks", [NoFailedMemchecks]); - true -> - ok - end - end. - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -718,6 +698,15 @@ whitebox_2(Opts) -> ets:delete(T2), ok. +select_bound_chunk(Config) -> + repeat_for_opts(fun select_bound_chunk_do/1, [all_types]). + +select_bound_chunk_do(Opts) -> + T = ets:new(x, Opts), + ets:insert(T, [{key, 1}]), + {[{key, 1}], '$end_of_table'} = ets:select(T, [{{key,1},[],['$_']}], 100000), + ok. + %% Test ets:to/from_dets. t_ets_dets(Config) when is_list(Config) -> @@ -1159,6 +1148,211 @@ t_select_delete(Config) when is_list(Config) -> lists:foreach(fun(Tab) -> ets:delete(Tab) end,Tables), verify_etsmem(EtsMem). +%% Tests the ets:select_replace/2 BIF +t_select_replace(Config) when is_list(Config) -> + EtsMem = etsmem(), + Tables = fill_sets_int(10000) ++ fill_sets_int(10000, [{write_concurrency,true}]), + + TestFun = fun (Table, TableType) when TableType =:= bag -> + % Operation not supported; bag implementation + % presented both semantic consistency and performance issues. + 10000 = ets:select_delete(Table, [{'_',[],[true]}]); + + (Table, TableType) -> + % Invalid replacement doesn't keep the key + MatchSpec1 = [{{'$1', '$2'}, + [{'=:=', {'band', '$1', 2#11}, 2#11}, + {'=/=', {'hd', '$2'}, $x}], + [{{'$2', '$1'}}]}], + {'EXIT',{badarg,_}} = (catch ets:select_replace(Table, MatchSpec1)), + + % Invalid replacement doesn't keep the key (even though it would be the same value) + MatchSpec2 = [{{'$1', '$2'}, + [{'=:=', {'band', '$1', 2#11}, 2#11}], + [{{{'+', '$1', 0}, '$2'}}]}, + {{'$1', '$2'}, + [{'=/=', {'band', '$1', 2#11}, 2#11}], + [{{{'-', '$1', 0}, '$2'}}]}], + {'EXIT',{badarg,_}} = (catch ets:select_replace(Table, MatchSpec2)), + + % Invalid replacement changes key to float equivalent + MatchSpec3 = [{{'$1', '$2'}, + [{'=:=', {'band', '$1', 2#11}, 2#11}, + {'=/=', {'hd', '$2'}, $x}], + [{{{'*', '$1', 1.0}, '$2'}}]}], + {'EXIT',{badarg,_}} = (catch ets:select_replace(Table, MatchSpec3)), + + % Replacements are differently-sized tuples + MatchSpec4_A = [{{'$1','$2'}, + [{'<', {'rem', '$1', 5}, 2}], + [{{'$1', [$x | '$2'], stuff}}]}], + MatchSpec4_B = [{{'$1','$2','_'}, + [], + [{{'$1','$2'}}]}], + 4000 = ets:select_replace(Table, MatchSpec4_A), + 4000 = ets:select_replace(Table, MatchSpec4_B), + + % Replacement is the same tuple + MatchSpec5 = [{{'$1', '$2'}, + [{'>', {'rem', '$1', 5}, 3}], + ['$_']}], + 2000 = ets:select_replace(Table, MatchSpec5), + + % Replacement reconstructs an equal tuple + MatchSpec6 = [{{'$1', '$2'}, + [{'>', {'rem', '$1', 5}, 3}], + [{{'$1', '$2'}}]}], + 2000 = ets:select_replace(Table, MatchSpec6), + + % Replacement uses {element,KeyPos,T} for key + 2000 = ets:select_replace(Table, + [{{'$1', '$2'}, + [{'>', {'rem', '$1', 5}, 3}], + [{{{element, 1, '$_'}, '$2'}}]}]), + + % Replacement uses wrong {element,KeyPos,T} for key + {'EXIT',{badarg,_}} = (catch ets:select_replace(Table, + [{{'$1', '$2'}, + [], + [{{{element, 2, '$_'}, '$2'}}]}])), + + check(Table, + fun ({N, [$x, C | _]}) when ((N rem 5) < 2) -> (C >= $0) andalso (C =< $9); + ({N, [C | _]}) when is_float(N) -> (C >= $0) andalso (C =< $9); + ({N, [C | _]}) when ((N rem 5) > 3) -> (C >= $0) andalso (C =< $9); + ({_, [C | _]}) -> (C >= $0) andalso (C =< $9) + end, + 10000), + + % Replace unbound range (>) + MatchSpec7 = [{{'$1', '$2'}, + [{'>', '$1', 7000}], + [{{'$1', {{gt_range, '$2'}}}}]}], + 3000 = ets:select_replace(Table, MatchSpec7), + + % Replace unbound range (<) + MatchSpec8 = [{{'$1', '$2'}, + [{'<', '$1', 3000}], + [{{'$1', {{le_range, '$2'}}}}]}], + case TableType of + ordered_set -> 2999 = ets:select_replace(Table, MatchSpec8); + set -> 2999 = ets:select_replace(Table, MatchSpec8); + duplicate_bag -> 2998 = ets:select_replace(Table, MatchSpec8) + end, + + % Replace bound range + MatchSpec9 = [{{'$1', '$2'}, + [{'>=', '$1', 3001}, + {'<', '$1', 7000}], + [{{'$1', {{range, '$2'}}}}]}], + case TableType of + ordered_set -> 3999 = ets:select_replace(Table, MatchSpec9); + set -> 3999 = ets:select_replace(Table, MatchSpec9); + duplicate_bag -> 3998 = ets:select_replace(Table, MatchSpec9) + end, + + % Replace particular keys + MatchSpec10 = [{{'$1', '$2'}, + [{'==', '$1', 3000}], + [{{'$1', {{specific1, '$2'}}}}]}, + {{'$1', '$2'}, + [{'==', '$1', 7000}], + [{{'$1', {{specific2, '$2'}}}}]}], + case TableType of + ordered_set -> 2 = ets:select_replace(Table, MatchSpec10); + set -> 2 = ets:select_replace(Table, MatchSpec10); + duplicate_bag -> 4 = ets:select_replace(Table, MatchSpec10) + end, + + check(Table, + fun ({N, {gt_range, _}}) -> N > 7000; + ({N, {le_range, _}}) -> N < 3000; + ({N, {range, _}}) -> (N >= 3001) andalso (N < 7000); + ({N, {specific1, _}}) -> N == 3000; + ({N, {specific2, _}}) -> N == 7000 + end, + 10000), + + 10000 = ets:select_delete(Table, [{'_',[],[true]}]), + check(Table, fun (_) -> false end, 0) + end, + + lists:foreach( + fun(Table) -> + TestFun(Table, ets:info(Table, type)), + ets:delete(Table) + end, + Tables), + + %% Test key-safe match-specs are accepted + BigNum = (123 bsl 123), + RefcBin = list_to_binary(lists:seq(1,?heap_binary_size+1)), + Terms = [a, "hej", 123, 1.23, BigNum , <<"123">>, RefcBin, TestFun, self()], + EqPairs = fun(X,Y) -> + [{ '$1', '$1'}, + { {X, Y}, {{X, Y}}}, + { {'$1', Y}, {{'$1', Y}}}, + { {{X, Y}}, {{{{X, Y}}}}}, + { {X}, {{X}}}, + { X, {const, X}}, + { {X,Y}, {const, {X,Y}}}, + { {X}, {const, {X}}}, + { {X, Y}, {{X, {const, Y}}}}, + { {X, {Y,'$1'}}, {{{const, X}, {{Y,'$1'}}}}}, + { [X, Y | '$1'], [X, Y | '$1']}, + { [{X, '$1'}, Y], [{{X, '$1'}}, Y]}, + { [{X, Y} | '$1'], [{const, {X, Y}} | '$1']}, + { [$p,$r,$e,$f,$i,$x | '$1'], [$p,$r,$e,$f,$i,$x | '$1']}, + { {[{X,Y}]}, {{[{{X,Y}}]}}}, + { {[{X,Y}]}, {{{const, [{X,Y}]}}}}, + { {[{X,Y}]}, {{[{const,{X,Y}}]}}} + ] + end, + + T2 = ets:new(x, []), + [lists:foreach(fun({A, B}) -> + %% just check that matchspec is accepted + 0 = ets:select_replace(T2, [{{A, '$2', '$3'}, [], [{{B, '$3', '$2'}}]}]) + end, + EqPairs(X,Y)) || X <- Terms, Y <- Terms], + + %% Test key-unsafe matchspecs are rejected + NeqPairs = fun(X, Y) -> + [{'$1', '$2'}, + {{X, Y}, {X, Y}}, + {{{X, Y}}, {{{X, Y}}}}, + {{X}, {{{X}}}}, + {{const, X}, {const, X}}, + {{const, {X,Y}}, {const, {X,Y}}}, + {'$1', {const, '$1'}}, + {{X}, {const, {{X}}}}, + {{X, {Y,'$1'}}, {{{const, X}, {Y,'$1'}}}}, + {[X, Y | '$1'], [X, Y]}, + {[X, Y], [X, Y | '$1']}, + {[{X, '$1'}, Y], [{X, '$1'}, Y]}, + {[$p,$r,$e,$f,$i,$x | '$1'], [$p,$r,$e,$f,$I,$x | '$1']}, + { {[{X,Y}]}, {{[{X,Y}]}}}, + { {[{X,Y}]}, {{{const, [{{X,Y}}]}}}}, + { {[{X,Y}]}, {{[{const,{{X,Y}}}]}}}, + {'_', '_'}, + {'$_', '$_'}, + {'$$', '$$'}, + {#{}, #{}}, + {#{X => '$1'}, #{X => '$1'}} + ] + end, + + [lists:foreach(fun({A, B}) -> + %% just check that matchspec is rejected + {'EXIT',{badarg,_}} = (catch ets:select_replace(T2, [{{A, '$2', '$3'}, [], [{{B, '$3', '$2'}}]}])) + end, + NeqPairs(X,Y)) || X <- Terms, Y <- Terms], + + + ets:delete(T2), + + verify_etsmem(EtsMem). + %% Test that partly bound keys gives faster matches. partly_bound(Config) when is_list(Config) -> case os:type() of @@ -5442,6 +5636,42 @@ smp_select_delete(Config) when is_list(Config) -> false = ets:info(T,fixed), ets:delete(T). +smp_select_replace(Config) when is_list(Config) -> + lists:foreach( + fun (TableType) -> + T = ets_new(smp_select_replace, [TableType, named_table, public, + {write_concurrency, true}]), + WorkerCount = 20, + CounterIterations = 10000, + InitF = fun (_) -> no_state end, + ExecF = fun (State) -> + lists:foreach( + fun F(IterId) -> + CounterId = rand:uniform(WorkerCount), + Match = [{{'$1', '$2'}, + [{'=:=', '$1', CounterId}], + [{{'$1', {'+', '$2', 1}}}]}], + case ets:select_replace(T, Match) of + 1 -> ok; + 0 -> + ets:insert_new(T, {CounterId, 1}) orelse + F(IterId) + end + end, + lists:seq(1, CounterIterations)), + State + end, + FiniF = fun (State) -> State end, + run_workers_do(InitF, ExecF, FiniF, WorkerCount), + FinalCounts = ets:select(T, [{{'_', '$1'}, [], ['$1']}]), + TotalIterations = WorkerCount * CounterIterations * erlang:system_info(schedulers), + TotalIterations = lists:sum(FinalCounts), + WorkerCount = ets:select_delete(T, [{{'_', '_'}, [], [true]}]), + 0 = ets:info(T, size), + ets:delete(T) + end, + [ordered_set, set, duplicate_bag]). + %% Test different types. types(Config) when is_list(Config) -> init_externals(), @@ -5545,6 +5775,68 @@ ets_all_run() -> false = lists:member(Table, ets:all()), ets_all_run(). +create_tables(N) -> + create_tables(N, []). + +create_tables(0, Ts) -> + Ts; +create_tables(N, Ts) -> + create_tables(N-1, [ets:new(tjo, [])|Ts]). + +massive_ets_all(Config) when is_list(Config) -> + Me = self(), + InitTables = lists:sort(ets:all()), + io:format("InitTables=~p~n", [InitTables]), + PMs0 = lists:map(fun (Sid) -> + my_spawn_opt(fun () -> + Ts = create_tables(250), + Me ! {self(), up, Ts}, + receive {Me, die} -> ok end + end, + [link, monitor, {scheduler, Sid}]) + end, + lists:seq(1, erlang:system_info(schedulers_online))), + AllRes = lists:sort(lists:foldl(fun ({P, _M}, Ts) -> + receive + {P, up, PTs} -> + PTs ++ Ts + end + end, + InitTables, + PMs0)), + AllRes = lists:sort(ets:all()), + PMs1 = lists:map(fun (_) -> + my_spawn_opt(fun () -> + AllRes = lists:sort(ets:all()) + end, + [link, monitor]) + end, lists:seq(1, 50)), + lists:foreach(fun ({P, M}) -> + receive + {'DOWN', M, process, P, _} -> + ok + end + end, PMs1), + PMs2 = lists:map(fun (_) -> + my_spawn_opt(fun () -> + _ = ets:all() + end, + [link, monitor]) + end, lists:seq(1, 50)), + lists:foreach(fun ({P, _M}) -> + P ! {Me, die} + end, PMs0), + lists:foreach(fun ({P, M}) -> + receive + {'DOWN', M, process, P, _} -> + ok + end + end, PMs0 ++ PMs2), + EndTables = lists:sort(ets:all()), + io:format("EndTables=~p~n", [EndTables]), + InitTables = EndTables, + ok. + take(Config) when is_list(Config) -> %% Simple test for set tables. @@ -5712,45 +6004,27 @@ etsmem() -> {Bl0+Bl,BlSz0+BlSz} end, {0,0}, CS) end}, - {Mem,AllTabs, erts_debug:get_internal_state('DbTable_meta')}. + {Mem,AllTabs}. -verify_etsmem(EtsMem) -> +verify_etsmem({MemInfo,AllTabs}) -> wait_for_test_procs(), - verify_etsmem(EtsMem, false). - -verify_etsmem({MemInfo,AllTabs,MetaState}=EtsMem, Adjusted) -> case etsmem() of - {MemInfo,_,_} -> + {MemInfo,_} -> io:format("Ets mem info: ~p", [MemInfo]), case MemInfo of {ErlMem,EtsAlloc} when ErlMem == notsup; EtsAlloc == undefined -> %% Use 'erl +Mea max' to do more complete memory leak testing. {comment,"Incomplete or no mem leak testing"}; _ -> - case Adjusted of - true -> - {comment, "Meta state adjusted"}; - false -> - ok - end + ok end; - {MemInfo2, AllTabs2, MetaState2} -> + {MemInfo2, AllTabs2} -> io:format("Expected: ~p", [MemInfo]), io:format("Actual: ~p", [MemInfo2]), io:format("Changed tables before: ~p\n",[AllTabs -- AllTabs2]), io:format("Changed tables after: ~p\n", [AllTabs2 -- AllTabs]), - io:format("Meta state before: ~p\n", [MetaState]), - io:format("Meta state after: ~p\n", [MetaState2]), - case {MetaState =:= MetaState2, Adjusted} of - {false, false} -> - io:format("Adjust meta state and retry...\n\n",[]), - {ok,ok} = erts_debug:set_internal_state('DbTable_meta', MetaState), - verify_etsmem(EtsMem, true); - _ -> - ets_test_spawn_logger ! {failed_memcheck, get('__ETS_TEST_CASE__')}, - {comment, "Failed memory check"} - end + ct:fail("Failed memory check") end. @@ -5772,10 +6046,10 @@ stop_loopers(Loopers) -> looper(Fun, State) -> looper(Fun, Fun(State)). -spawn_logger(Procs, FailedMemchecks) -> +spawn_logger(Procs) -> receive {new_test_proc, Proc} -> - spawn_logger([Proc|Procs], FailedMemchecks); + spawn_logger([Proc|Procs]); {sync_test_procs, Kill, From} -> lists:foreach(fun (Proc) when From == Proc -> ok; @@ -5799,14 +6073,7 @@ spawn_logger(Procs, FailedMemchecks) -> end end, Procs), From ! test_procs_synced, - spawn_logger([From], FailedMemchecks); - - {failed_memcheck, TestCase} -> - spawn_logger(Procs, [TestCase|FailedMemchecks]); - - {Pid, get_failed_memchecks} -> - Pid ! {get_failed_memchecks, FailedMemchecks}, - spawn_logger(Procs, FailedMemchecks) + spawn_logger([From]) end. pid_status(Pid) -> @@ -5822,7 +6089,7 @@ start_spawn_logger() -> case whereis(ets_test_spawn_logger) of Pid when is_pid(Pid) -> true; _ -> register(ets_test_spawn_logger, - spawn_opt(fun () -> spawn_logger([], []) end, + spawn_opt(fun () -> spawn_logger([]) end, [{priority, max}])) end. @@ -5945,7 +6212,6 @@ only_if_smp(Schedulers, Func) -> end. %% Copy-paste from emulator/test/binary_SUITE.erl --define(heap_binary_size, 64). test_terms(Test_Func, Mode) -> garbage_collect(), Pib0 = process_info(self(),binary), diff --git a/lib/stdlib/test/gen_statem_SUITE.erl b/lib/stdlib/test/gen_statem_SUITE.erl index 8f2ba0cab2..ac27c9fc79 100644 --- a/lib/stdlib/test/gen_statem_SUITE.erl +++ b/lib/stdlib/test/gen_statem_SUITE.erl @@ -38,7 +38,7 @@ all() -> {group, abnormal}, {group, abnormal_handle_event}, shutdown, stop_and_reply, state_enter, event_order, - state_timeout, code_change, + state_timeout, event_types, code_change, {group, sys}, hibernate, enter_loop]. @@ -600,15 +600,26 @@ state_enter(_Config) -> (internal, Prev, N) -> Self ! {internal,start,Prev,N}, {keep_state,N + 1}; + ({call,From}, repeat, N) -> + {repeat_state,N + 1, + [{reply,From,{repeat,start,N}}]}; ({call,From}, echo, N) -> - {next_state,wait,N + 1,{reply,From,{echo,start,N}}}; + {next_state,wait,N + 1, + {reply,From,{echo,start,N}}}; ({call,From}, {stop,Reason}, N) -> - {stop_and_reply,Reason,[{reply,From,{stop,N}}],N + 1} + {stop_and_reply,Reason, + [{reply,From,{stop,N}}],N + 1} end, wait => - fun (enter, Prev, N) -> + fun (enter, Prev, N) when N < 5 -> + {repeat_state,N + 1, + {reply,{Self,N},{enter,Prev}}}; + (enter, Prev, N) -> Self ! {enter,wait,Prev,N}, {keep_state,N + 1}; + ({call,From}, repeat, N) -> + {repeat_state_and_data, + [{reply,From,{repeat,wait,N}}]}; ({call,From}, echo, N) -> {next_state,start,N + 1, [{next_event,internal,wait}, @@ -620,11 +631,15 @@ state_enter(_Config) -> [{enter,start,start,1}] = flush(), {echo,start,2} = gen_statem:call(STM, echo), - [{enter,wait,start,3}] = flush(), - {wait,[4|_]} = sys:get_state(STM), - {echo,wait,4} = gen_statem:call(STM, echo), - [{enter,start,wait,5},{internal,start,wait,6}] = flush(), - {stop,7} = gen_statem:call(STM, {stop,bye}), + [{3,{enter,start}},{4,{enter,start}},{enter,wait,start,5}] = flush(), + {wait,[6|_]} = sys:get_state(STM), + {repeat,wait,6} = gen_statem:call(STM, repeat), + [{enter,wait,wait,6}] = flush(), + {echo,wait,7} = gen_statem:call(STM, echo), + [{enter,start,wait,8},{internal,start,wait,9}] = flush(), + {repeat,start,10} = gen_statem:call(STM, repeat), + [{enter,start,start,11}] = flush(), + {stop,12} = gen_statem:call(STM, {stop,bye}), [{'EXIT',STM,bye}] = flush(), {noproc,_} = @@ -801,6 +816,74 @@ state_timeout(_Config) -> +%% Test that all event types can be sent with {next_event,EventType,_} +event_types(_Config) -> + process_flag(trap_exit, true), + + Machine = + %% Abusing the internal format of From... + #{init => + fun () -> + {ok, start, undefined} + end, + start => + fun ({call,_} = Call, Req, undefined) -> + {next_state, state1, undefined, + [{next_event,internal,1}, + {next_event,state_timeout,2}, + {next_event,timeout,3}, + {next_event,info,4}, + {next_event,cast,5}, + {next_event,Call,Req}]} + end, + state1 => + fun (internal, 1, undefined) -> + {next_state, state2, undefined} + end, + state2 => + fun (state_timeout, 2, undefined) -> + {next_state, state3, undefined} + end, + state3 => + fun (timeout, 3, undefined) -> + {next_state, state4, undefined} + end, + state4 => + fun (info, 4, undefined) -> + {next_state, state5, undefined} + end, + state5 => + fun (cast, 5, undefined) -> + {next_state, state6, undefined} + end, + state6 => + fun ({call,From}, stop, undefined) -> + {stop_and_reply, shutdown, + [{reply,From,stopped}]} + end}, + {ok,STM} = + gen_statem:start_link( + ?MODULE, {map_statem,Machine,[]}, [{debug,[trace]}]), + + stopped = gen_statem:call(STM, stop), + receive + {'EXIT',STM,shutdown} -> + ok + after 500 -> + ct:fail(did_not_stop) + end, + + {noproc,_} = + ?EXPECT_FAILURE(gen_statem:call(STM, hej), Reason), + case flush() of + [] -> + ok; + Other2 -> + ct:fail({unexpected,Other2}) + end. + + + sys1(Config) -> {ok,Pid} = gen_statem:start(?MODULE, start_arg(Config, []), []), {status, Pid, {module,gen_statem}, _} = sys:get_status(Pid), @@ -1722,6 +1805,10 @@ handle_event( {keep_state,[NewData|Machine]}; {keep_state,NewData,Ops} -> {keep_state,[NewData|Machine],Ops}; + {repeat_state,NewData} -> + {repeat_state,[NewData|Machine]}; + {repeat_state,NewData,Ops} -> + {repeat_state,[NewData|Machine],Ops}; Other -> Other end; diff --git a/lib/stdlib/test/qlc_SUITE.erl b/lib/stdlib/test/qlc_SUITE.erl index c08e138ad3..2b5d52287e 100644 --- a/lib/stdlib/test/qlc_SUITE.erl +++ b/lib/stdlib/test/qlc_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2016. All Rights Reserved. +%% Copyright Ericsson AB 2004-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -886,11 +886,12 @@ eval_unique(Config) when is_list(Config) -> [a] = qlc:e(Q2, {unique_all, true}) ">>, - <<"Q = qlc:q([SQV || SQV <- qlc:q([X || X <- [1,2,1]],unique)], + <<"Q = qlc:q([SQV || SQV <- qlc:q([X || X <- [1,2,1,#{a => 1}]], + unique)], unique), {call,_,_,[{lc,_,{var,_,'X'},[{generate,_,{var,_,'X'},_}]},_]} = qlc:info(Q, [{format,abstract_code},unique_all]), - [1,2] = qlc:e(Q)">>, + [1,2,#{a := 1}] = qlc:e(Q)">>, <<"Q = qlc:q([X || X <- [1,2,1]]), {call,_,_,[{lc,_,{var,_,'X'},[{generate,_,{var,_,'X'},_}]},_]} = @@ -2637,7 +2638,16 @@ info(Config) when is_list(Config) -> {cons, _, _, _}]}, {nil,_}}]}]} = i(QH, {format, abstract_code}), [{5},{6}] = qlc:e(QH), - [{4},{5},{6}] = qlc:e(F(3))">> + [{4},{5},{6}] = qlc:e(F(3))">>, + + <<"Fun = fun ?MODULE:i/2, + L = [{#{k => #{v => Fun}}, Fun}], + H = qlc:q([Q || Q <- L, Q =:= {#{k => #{v => Fun}}, Fun}]), + L = qlc:e(H), + {call,_,_,[{lc,_,{var,_,'Q'}, + [{generate,_,_,_}, + {op,_,_,_,_}]}]} = + qlc:info(H, [{format,abstract_code}])">> ], run(Config, Ts), diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl index fe5eaccda5..098eefeb61 100644 --- a/lib/stdlib/test/rand_SUITE.erl +++ b/lib/stdlib/test/rand_SUITE.erl @@ -356,14 +356,23 @@ basic_normal_1(0, {#{type:=Alg}, _}, Sum, SumSq) -> %% Test that the user can write algorithms. plugin(Config) when is_list(Config) -> - _ = lists:foldl(fun(_, S0) -> - {V1, S1} = rand:uniform_s(10000, S0), - true = is_integer(V1), - {V2, S2} = rand:uniform_s(S1), - true = is_float(V2), - S2 - end, crypto_seed(), lists:seq(1, 200)), - ok. + try crypto:strong_rand_bytes(1) of + <<_>> -> + _ = lists:foldl( + fun(_, S0) -> + {V1, S1} = rand:uniform_s(10000, S0), + true = is_integer(V1), + {V2, S2} = rand:uniform_s(S1), + true = is_float(V2), + S2 + end, crypto_seed(), lists:seq(1, 200)), + ok + catch + error:low_entropy -> + {skip,low_entropy}; + error:undef -> + {skip,no_crypto} + end. %% Test implementation crypto_seed() -> @@ -397,7 +406,13 @@ crypto_uniform_n(N, State0) -> measure(Suite) when is_atom(Suite) -> []; measure(_Config) -> ct:timetrap({minutes,15}), %% valgrind needs a lot of time - Algos = [crypto64|algs()], + Algos = + try crypto:strong_rand_bytes(1) of + <<_>> -> [crypto64] + catch + error:low_entropy -> []; + error:undef -> [] + end ++ algs(), io:format("RNG uniform integer performance~n",[]), _ = measure_1(random, fun(State) -> {int, random:uniform_s(10000, State)} end), _ = [measure_1(Algo, fun(State) -> {int, rand:uniform_s(10000, State)} end) || Algo <- Algos], diff --git a/lib/stdlib/test/tar_SUITE.erl b/lib/stdlib/test/tar_SUITE.erl index d6b6d3f80c..2e1ae7bcff 100644 --- a/lib/stdlib/test/tar_SUITE.erl +++ b/lib/stdlib/test/tar_SUITE.erl @@ -20,7 +20,9 @@ -module(tar_SUITE). -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, - init_per_group/2,end_per_group/2, borderline/1, atomic/1, long_names/1, + init_per_group/2, end_per_group/2, + init_per_testcase/2, + borderline/1, atomic/1, long_names/1, create_long_names/1, bad_tar/1, errors/1, extract_from_binary/1, extract_from_binary_compressed/1, extract_filtered/1, extract_from_open_file/1, symlinks/1, open_add_close/1, cooked_compressed/1, @@ -56,6 +58,9 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. +init_per_testcase(_Case, Config) -> + Ports = ordsets:from_list(erlang:ports()), + [{ports,Ports}|Config]. %% Test creating, listing and extracting one file from an archive, %% multiple times with different file sizes. Also check that the file @@ -85,7 +90,7 @@ borderline(Config) when is_list(Config) -> %% Clean up. delete_files([TempDir]), - ok. + verify_ports(Config). borderline_test(Size, TempDir) -> io:format("Testing size ~p", [Size]), @@ -270,7 +275,7 @@ atomic(Config) when is_list(Config) -> %% Clean up. delete_files([Tar1,Tar2,Tar3,Tar4|Names]), - ok. + verify_ports(Config). %% Returns a sequence of characters. @@ -304,7 +309,9 @@ long_names(Config) when is_list(Config) -> DataDir = proplists:get_value(data_dir, Config), Long = filename:join(DataDir, "long_names.tar"), run_in_short_tempdir(Config, - fun() -> do_long_names(Long) end). + fun() -> do_long_names(Long) end), + verify_ports(Config). + do_long_names(Long) -> %% Try table/2 and extract/2. @@ -336,7 +343,8 @@ do_long_names(Long) -> %% Creates a tar file from a deep directory structure (filenames are %% longer than 100 characters). create_long_names(Config) when is_list(Config) -> - run_in_short_tempdir(Config, fun create_long_names/0). + run_in_short_tempdir(Config, fun create_long_names/0), + verify_ports(Config). create_long_names() -> {ok,Dir} = file:get_cwd(), @@ -383,7 +391,7 @@ bad_tar(Config) when is_list(Config) -> try_bad("bad_octal", invalid_tar_checksum, Config), try_bad("bad_too_short", eof, Config), try_bad("bad_even_shorter", eof, Config), - ok. + verify_ports(Config). try_bad(Name0, Reason, Config) -> %% Intentionally no macros here. @@ -433,7 +441,7 @@ errors(Config) when is_list(Config) -> %% Clean up. delete_files([GoodTar,BadTar]), - ok. + verify_ports(Config). try_error(M, F, A, Error) -> io:format("Trying ~p:~p(~p)", [M, F, A]), @@ -483,7 +491,7 @@ extract_from_binary(Config) when is_list(Config) -> %% Clean up. delete_files([ExtractDir]), - ok. + verify_ports(Config). extract_from_binary_compressed(Config) when is_list(Config) -> %% Test extracting a compressed tar archive from a binary. @@ -516,7 +524,7 @@ extract_from_binary_compressed(Config) when is_list(Config) -> %% Clean up the rest. delete_files([ExtractDir]), - ok. + verify_ports(Config). %% Test extracting a tar archive from a binary. extract_filtered(Config) when is_list(Config) -> @@ -537,7 +545,7 @@ extract_filtered(Config) when is_list(Config) -> %% Clean up. delete_files([ExtractDir]), - ok. + verify_ports(Config). %% Test extracting a tar archive from an open file. extract_from_open_file(Config) when is_list(Config) -> @@ -562,7 +570,7 @@ extract_from_open_file(Config) when is_list(Config) -> %% Clean up. delete_files([ExtractDir]), - ok. + verify_ports(Config). %% Test that archives containing symlinks can be created and extracted. symlinks(Config) when is_list(Config) -> @@ -581,6 +589,7 @@ symlinks(Config) when is_list(Config) -> %% Clean up. delete_files([Dir]), + verify_ports(Config), Res. make_symlink(Path, Link) -> @@ -697,7 +706,8 @@ init(Config) when is_list(Config) -> ok = erl_tar:add(Tar, FileOne, []), ok = erl_tar:close(Tar), {ok, [FileOne]} = erl_tar:table(TarOne), - ok. + + verify_ports(Config). file_op_bad(_) -> throw({error, should_never_be_called}). @@ -751,7 +761,7 @@ open_add_close(Config) when is_list(Config) -> delete_files(["oac_file","oac_small","oac_big",Dir,AnotherDir,ADir]), - ok. + verify_ports(Config). oac_files() -> Files = [{"oac_file", 1459, $x}, @@ -782,7 +792,8 @@ cooked_compressed(Config) when is_list(Config) -> %% Clean up. delete_files([filename:join(PrivDir, "ddll_SUITE_data")]), - ok. + + verify_ports(Config). %% Test that an archive can be created directly from binaries and %% that an archive can be extracted into binaries. @@ -810,13 +821,15 @@ memory(Config) when is_list(Config) -> %% Clean up. ok = delete_files([Name1,Name2]), - ok. + + verify_ports(Config). read_other_implementations(Config) when is_list(Config) -> DataDir = proplists:get_value(data_dir, Config), Files = ["v7.tar", "gnu.tar", "bsd.tar", "star.tar", "pax_mtime.tar"], - do_read_other_implementations(Files, DataDir). + do_read_other_implementations(Files, DataDir), + verify_ports(Config). do_read_other_implementations([], _DataDir) -> ok; @@ -836,7 +849,8 @@ sparse(Config) when is_list(Config) -> Sparse01 = "sparse01.tar", Sparse10Empty = "sparse10_empty.tar", Sparse10 = "sparse10.tar", - do_sparse([Sparse01Empty, Sparse01, Sparse10Empty, Sparse10], DataDir, PrivDir). + do_sparse([Sparse01Empty, Sparse01, Sparse10Empty, Sparse10], DataDir, PrivDir), + verify_ports(Config). do_sparse([], _DataDir, _PrivDir) -> ok; @@ -994,3 +1008,14 @@ is_ustar(File) -> $g -> false; _ -> true end. + + +verify_ports(Config) -> + PortsBefore = proplists:get_value(ports, Config), + PortsAfter = ordsets:from_list(erlang:ports()), + case ordsets:subtract(PortsAfter, PortsBefore) of + [] -> + ok; + [_|_]=Rem -> + error({leaked_ports,Rem}) + end. diff --git a/lib/stdlib/test/zip_SUITE.erl b/lib/stdlib/test/zip_SUITE.erl index 7d90795c9e..f0feda217a 100644 --- a/lib/stdlib/test/zip_SUITE.erl +++ b/lib/stdlib/test/zip_SUITE.erl @@ -27,7 +27,7 @@ openzip_api/1, zip_api/1, open_leak/1, unzip_jar/1, unzip_traversal_exploit/1, compress_control/1, - foldl/1]). + foldl/1,fd_leak/1]). -include_lib("common_test/include/ct.hrl"). -include_lib("kernel/include/file.hrl"). @@ -40,7 +40,7 @@ all() -> unzip_to_binary, zip_to_binary, unzip_options, zip_options, list_dir_options, aliases, openzip_api, zip_api, open_leak, unzip_jar, compress_control, foldl, - unzip_traversal_exploit]. + unzip_traversal_exploit,fd_leak]. groups() -> []. @@ -882,3 +882,35 @@ foldl(Config) -> {error, enoent} = zip:foldl(ZipFun, [], File), ok. + +fd_leak(Config) -> + ok = file:set_cwd(proplists:get_value(priv_dir, Config)), + DataDir = proplists:get_value(data_dir, Config), + Name = filename:join(DataDir, "bad_file_header.zip"), + BadExtract = fun() -> + {error,bad_file_header} = zip:extract(Name), + ok + end, + do_fd_leak(BadExtract, 1), + + BadCreate = fun() -> + {error,enoent} = zip:zip("failed.zip", + ["none"]), + ok + end, + do_fd_leak(BadCreate, 1), + + ok. + +do_fd_leak(_Bad, 10000) -> + ok; +do_fd_leak(Bad, N) -> + try Bad() of + ok -> + do_fd_leak(Bad, N + 1) + catch + C:R -> + Stk = erlang:get_stacktrace(), + io:format("Bad error after ~p attempts\n", [N]), + erlang:raise(C, R, Stk) + end. diff --git a/lib/stdlib/vsn.mk b/lib/stdlib/vsn.mk index e67cb9b08d..f7bd21472c 100644 --- a/lib/stdlib/vsn.mk +++ b/lib/stdlib/vsn.mk @@ -1 +1 @@ -STDLIB_VSN = 3.2 +STDLIB_VSN = 3.3 diff --git a/lib/tools/doc/src/make.xml b/lib/tools/doc/src/make.xml index fddf5ebd7b..6b878f72fb 100644 --- a/lib/tools/doc/src/make.xml +++ b/lib/tools/doc/src/make.xml @@ -43,15 +43,15 @@ <fsummary>Compile a set of modules.</fsummary> <type> <v>Options = [Option]</v> - <v> Option = noexec | load | netload | <compiler option></v> + <v> Option = noexec | load | netload | {emake, Emake} | <compiler option></v> </type> <desc> - <p>This function first looks in the current working directory - for a file named <c>Emakefile</c> (see below) specifying the - set of modules to compile and the compile options to use. If - no such file is found, the set of modules to compile - defaults to all modules in the current working - directory.</p> + <p>This function determines the set of modules to compile and the + compile options to use, by first looking for the <c>emake</c> make + option, if not present reads the configuration from a file named + <c>Emakefile</c> (see below). If no such file is found, the + set of modules to compile defaults to all modules in the + current working directory.</p> <p>Traversing the set of modules, it then recompiles every module for which at least one of the following conditions apply:</p> <list type="bulleted"> @@ -77,6 +77,9 @@ <item><c>netload</c> <br></br> Net load mode. Loads all recompiled modules on all known nodes.</item> + <item><c>{emake, Emake}</c> <br></br> + + Rather than reading the <c>Emakefile</c> specify configuration explicitly.</item> </list> <p>All items in <c>Options</c> that are not make options are assumed to be compiler options and are passed as-is to @@ -108,9 +111,10 @@ <section> <title>Emakefile</title> - <p><c>make:all/0,1</c> and <c>make:files/1,2</c> looks in the - current working directory for a file named <c>Emakefile</c>. If - it exists, <c>Emakefile</c> should contain elements like this:</p> + <p><c>make:all/0,1</c> and <c>make:files/1,2</c> first looks for + <c>{emake, Emake}</c> in options, then in the current working directory + for a file named <c>Emakefile</c>. If present <c>Emake</c> should + contain elements like this:</p> <code type="none"> Modules. {Modules,Options}. </code> diff --git a/lib/tools/doc/src/notes.xml b/lib/tools/doc/src/notes.xml index 415f1b8516..af20200d49 100644 --- a/lib/tools/doc/src/notes.xml +++ b/lib/tools/doc/src/notes.xml @@ -31,6 +31,21 @@ </header> <p>This document describes the changes made to the Tools application.</p> +<section><title>Tools 2.9.1</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Improved edoc support in emacs mode.</p> + <p> + Own Id: OTP-14217 Aux Id: PR-1282 </p> + </item> + </list> + </section> + +</section> + <section><title>Tools 2.9</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/tools/emacs/erlang-skels.el b/lib/tools/emacs/erlang-skels.el index eeba7f34e9..bdb3d9ad4a 100644 --- a/lib/tools/emacs/erlang-skels.el +++ b/lib/tools/emacs/erlang-skels.el @@ -1,7 +1,7 @@ ;; ;; %CopyrightBegin% ;; -;; Copyright Ericsson AB 2010-2016. All Rights Reserved. +;; Copyright Ericsson AB 2010-2017. All Rights Reserved. ;; ;; Licensed under the Apache License, Version 2.0 (the "License"); ;; you may not use this file except in compliance with the License. @@ -915,11 +915,7 @@ Please see the function `tempo-define-template'.") "%% process to initialize." n (erlang-skel-separator-end 2) "-spec init(Args :: term()) ->" n> - "{ok, State :: term(), Data :: term()} |" n> - "{ok, State :: term(), Data :: term()," n> - "[gen_statem:action()] | gen_statem:action()} |" n> - "ignore |" n> - "{stop, Reason :: term()}." n + "gen_statem:init_result(atom())." n "init([]) ->" n> "process_flag(trap_exit, true)," n> "{ok, state_name, #data{}}." n @@ -1028,11 +1024,7 @@ Please see the function `tempo-define-template'.") "%% process to initialize." n (erlang-skel-separator-end 2) "-spec init(Args :: term()) ->" n> - "{ok, State :: term(), Data :: term()} |" n> - "{ok, State :: term(), Data :: term()," n> - "[gen_statem:action()] | gen_statem:action()} |" n> - "ignore |" n> - "{stop, Reason :: term()}." n + "gen_statem:init_result(term())." n "init([]) ->" n> "process_flag(trap_exit, true)," n> "{ok, state_name, #data{}}." n diff --git a/lib/tools/emacs/erldoc.el b/lib/tools/emacs/erldoc.el index e1fd661348..348800f880 100644 --- a/lib/tools/emacs/erldoc.el +++ b/lib/tools/emacs/erldoc.el @@ -407,7 +407,7 @@ up the indexing." (defvar erldoc-user-guides nil) (defvar erldoc-missing-user-guides - '("compiler" "hipe" "kernel" "os_mon" "parsetools" "typer") + '("compiler" "hipe" "kernel" "os_mon" "parsetools") "List of standard Erlang applications with no user guides.") ;; Search in `code:lib_dir/0' using find LIB_DIR -type f -name @@ -417,7 +417,7 @@ up the indexing." "runtime_tools" "sasl" "snmp" "ssl" "test_server" ("ssh" . "SSH") ("stdlib" . "STDLIB") - ("hipe" . "HiPE") ("typer" . "TypEr")) + ("hipe" . "HiPE")) "List of applications that come with a manual.") (defun erldoc-user-guide-chapters (user-guide) diff --git a/lib/tools/examples/xref_examples.erl b/lib/tools/examples/xref_examples.erl index 4c082195a2..f7e71c9708 100644 --- a/lib/tools/examples/xref_examples.erl +++ b/lib/tools/examples/xref_examples.erl @@ -7,7 +7,7 @@ %% ${HOME}/unused_locals.txt. script() -> Root = code:root_dir(), - Dir = os:getenv("HOME"), + {ok,[[Dir]]} = init:get_argument(home), Server = s, xref:start(Server), {ok, _Relname} = xref:add_release(Server, code:lib_dir(), {name,otp}), diff --git a/lib/tools/src/make.erl b/lib/tools/src/make.erl index 37e67cbe34..60695febb4 100644 --- a/lib/tools/src/make.erl +++ b/lib/tools/src/make.erl @@ -29,7 +29,7 @@ -include_lib("kernel/include/file.hrl"). --define(MakeOpts,[noexec,load,netload,noload]). +-define(MakeOpts,[noexec,load,netload,noload,emake]). all_or_nothing() -> case all() of @@ -43,29 +43,30 @@ all() -> all([]). all(Options) -> - {MakeOpts,CompileOpts} = sort_options(Options,[],[]), - case read_emakefile('Emakefile',CompileOpts) of - Files when is_list(Files) -> - do_make_files(Files,MakeOpts); - error -> - error - end. + run_emake(undefined, Options). files(Fs) -> files(Fs, []). files(Fs0, Options) -> Fs = [filename:rootname(F,".erl") || F <- Fs0], + run_emake(Fs, Options). + +run_emake(Mods, Options) -> {MakeOpts,CompileOpts} = sort_options(Options,[],[]), - case get_opts_from_emakefile(Fs,'Emakefile',CompileOpts) of + Emake = get_emake(Options), + case normalize_emake(Emake, Mods, CompileOpts) of Files when is_list(Files) -> - do_make_files(Files,MakeOpts); - error -> error + do_make_files(Files,MakeOpts); + error -> + error end. do_make_files(Fs, Opts) -> process(Fs, lists:member(noexec, Opts), load_opt(Opts)). +sort_options([{emake, _}=H|T],Make,Comp) -> + sort_options(T,[H|Make],Comp); sort_options([H|T],Make,Comp) -> case lists:member(H,?MakeOpts) of @@ -89,20 +90,35 @@ sort_options([],Make,Comp) -> %%% %%% These elements are converted to [{ModList,OptList},...] %%% ModList is a list of modulenames (strings) -read_emakefile(Emakefile,Opts) -> - case file:consult(Emakefile) of - {ok,Emake} -> + +normalize_emake(EmakeRaw, Mods, Opts) -> + case EmakeRaw of + {ok, Emake} when Mods =:= undefined -> transform(Emake,Opts,[],[]); - {error,enoent} -> + {ok, Emake} when is_list(Mods) -> + ModsOpts = transform(Emake,Opts,[],[]), + ModStrings = [coerce_2_list(M) || M <- Mods], + get_opts_from_emakefile(ModsOpts,ModStrings,Opts,[]); + {error,enoent} when Mods =:= undefined -> %% No Emakefile found - return all modules in current %% directory and the options given at command line - Mods = [filename:rootname(F) || F <- filelib:wildcard("*.erl")], + CwdMods = [filename:rootname(F) || F <- filelib:wildcard("*.erl")], + [{CwdMods, Opts}]; + {error,enoent} when is_list(Mods) -> [{Mods, Opts}]; - {error,Other} -> - io:format("make: Trouble reading 'Emakefile':~n~tp~n",[Other]), + {error, Error} -> + io:format("make: Trouble reading 'Emakefile':~n~tp~n",[Error]), error end. +get_emake(Opts) -> + case proplists:get_value(emake, Opts, false) of + false -> + file:consult('Emakefile'); + OptsEmake -> + {ok, OptsEmake} + end. + transform([{Mod,ModOpts}|Emake],Opts,Files,Already) -> case expand(Mod,Already) of [] -> @@ -143,31 +159,19 @@ expand(Mod,Already) -> end end. -%%% Reads the given Emakefile to see if there are any specific compile +%%% Reads the given Emake to see if there are any specific compile %%% options given for the modules. -get_opts_from_emakefile(Mods,Emakefile,Opts) -> - case file:consult(Emakefile) of - {ok,Emake} -> - Modsandopts = transform(Emake,Opts,[],[]), - ModStrings = [coerce_2_list(M) || M <- Mods], - get_opts_from_emakefile2(Modsandopts,ModStrings,Opts,[]); - {error,enoent} -> - [{Mods, Opts}]; - {error,Other} -> - io:format("make: Trouble reading 'Emakefile':~n~tp~n",[Other]), - error - end. -get_opts_from_emakefile2([{MakefileMods,O}|Rest],Mods,Opts,Result) -> +get_opts_from_emakefile([{MakefileMods,O}|Rest],Mods,Opts,Result) -> case members(Mods,MakefileMods,[],Mods) of {[],_} -> - get_opts_from_emakefile2(Rest,Mods,Opts,Result); + get_opts_from_emakefile(Rest,Mods,Opts,Result); {I,RestOfMods} -> - get_opts_from_emakefile2(Rest,RestOfMods,Opts,[{I,O}|Result]) + get_opts_from_emakefile(Rest,RestOfMods,Opts,[{I,O}|Result]) end; -get_opts_from_emakefile2([],[],_Opts,Result) -> +get_opts_from_emakefile([],[],_Opts,Result) -> Result; -get_opts_from_emakefile2([],RestOfMods,Opts,Result) -> +get_opts_from_emakefile([],RestOfMods,Opts,Result) -> [{RestOfMods,Opts}|Result]. members([H|T],MakefileMods,I,Rest) -> diff --git a/lib/tools/src/tools.app.src b/lib/tools/src/tools.app.src index 4c7dd24006..17b1d06686 100644 --- a/lib/tools/src/tools.app.src +++ b/lib/tools/src/tools.app.src @@ -41,7 +41,6 @@ ] }, {runtime_dependencies, ["stdlib-3.1","runtime_tools-1.8.14", - "kernel-3.0","inets-5.10","erts-7.0", - "compiler-5.0"]} + "kernel-3.0","erts-7.0","compiler-5.0"]} ] }. diff --git a/lib/tools/test/Makefile b/lib/tools/test/Makefile index 84c4e56aff..fe65d1484d 100644 --- a/lib/tools/test/Makefile +++ b/lib/tools/test/Makefile @@ -52,8 +52,8 @@ RELSYSDIR = $(RELEASE_PATH)/tools_test # ---------------------------------------------------- # FLAGS # ---------------------------------------------------- -ERL_MAKE_FLAGS += -ERL_COMPILE_FLAGS += -I$(ERL_TOP)/lib/percept/include +ERL_MAKE_FLAGS += +ERL_COMPILE_FLAGS += EBIN = . diff --git a/lib/tools/test/make_SUITE.erl b/lib/tools/test/make_SUITE.erl index e6284db8b8..2a94ead329 100644 --- a/lib/tools/test/make_SUITE.erl +++ b/lib/tools/test/make_SUITE.erl @@ -20,7 +20,7 @@ -module(make_SUITE). -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, - init_per_group/2,end_per_group/2, make_all/1, make_files/1]). + init_per_group/2,end_per_group/2, make_all/1, make_files/1, emake_opts/1]). -export([otp_6057_init/1, otp_6057_a/1, otp_6057_b/1, otp_6057_c/1, otp_6057_end/1]). @@ -40,7 +40,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [make_all, make_files, {group, otp_6057}]. + [make_all, make_files, emake_opts, {group, otp_6057}]. groups() -> [{otp_6057,[],[otp_6057_a, otp_6057_b, @@ -86,6 +86,20 @@ make_files(Config) when is_list(Config) -> ensure_no_messages(), ok. +emake_opts(Config) when is_list(Config) -> + Current = prepare_data_dir(Config), + + %% prove that emake is used in opts instead of local Emakefile + Opts = [{emake, [test8, test9]}], + error = make:all(Opts), + error = make:files([test9], Opts), + "test8.beam" = ensure_exists([test8]), + "test9.beam" = ensure_exists([test9]), + "test5.S" = ensure_exists(["test5"],".S"), + + file:set_cwd(Current), + ensure_no_messages(), + ok. %% Moves to the data directory of this suite, clean it from any object %% files (*.jam for a JAM emulator). Returns the previous directory. diff --git a/lib/tools/vsn.mk b/lib/tools/vsn.mk index 07bc39f76e..f60da27c44 100644 --- a/lib/tools/vsn.mk +++ b/lib/tools/vsn.mk @@ -1 +1 @@ -TOOLS_VSN = 2.9 +TOOLS_VSN = 2.9.1 diff --git a/lib/typer/Makefile b/lib/typer/Makefile deleted file mode 100644 index bd1b6458a8..0000000000 --- a/lib/typer/Makefile +++ /dev/null @@ -1,44 +0,0 @@ -# -# %CopyrightBegin% -# -# Copyright Ericsson AB 2006-2016. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# %CopyrightEnd% -# -#============================================================================= -# -# File: lib/typer/Makefile -# Authors: Bingwen He, Tobias Lindahl, and Kostis Sagonas -# -#============================================================================= -include $(ERL_TOP)/make/target.mk -include $(ERL_TOP)/make/$(TARGET)/otp.mk - -# -# Macros -# - -SUB_DIRECTORIES = src doc/src - -include vsn.mk -VSN = $(TYPER_VSN) - -SPECIAL_TARGETS = - -# -# Default Subdir Targets -# -include $(ERL_TOP)/make/otp_subdir.mk - diff --git a/lib/typer/RELEASE_NOTES b/lib/typer/RELEASE_NOTES deleted file mode 100644 index d91a815ee9..0000000000 --- a/lib/typer/RELEASE_NOTES +++ /dev/null @@ -1,22 +0,0 @@ -============================================================================== - Major features, additions and changes between Typer versions - (in reversed chronological order) -============================================================================== - -Version 0.9 (in Erlang/OTP R14B02) ----------------------------------- - - Major rewrite; all code has been cleaned up and placed in one file. - The only reason why this is not version 1.0 yet is that there is no proper - documentation for typer which can be displayed in the www.erlang.org site. - - Added ability to receive the set of exported types and report unknown ones. - - Better handling of overloaded contracts; especially erroneous ones on which - typer does not crash anymore. - - Fixed problem that caused typer to hang when given a file whose module name - did not correspond to the file name. - - Added two undocumented options that may come very handy when trying to - understand why typer reports some particular set of types for the functions - in a module. These options are mainly for typer developers at this point, - but may become documented in some future version. - -Older versions --------------- diff --git a/lib/typer/doc/Makefile b/lib/typer/doc/Makefile deleted file mode 100644 index 1015ca78eb..0000000000 --- a/lib/typer/doc/Makefile +++ /dev/null @@ -1,40 +0,0 @@ -# -# %CopyrightBegin% -# -# Copyright Ericsson AB 2006-2016. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# %CopyrightEnd% -# -SHELL=/bin/sh - -include $(ERL_TOP)/make/target.mk -include $(ERL_TOP)/make/$(TARGET)/otp.mk - -clean: - -rm -f *.html edoc-info stylesheet.css erlang.png - -distclean: clean -realclean: clean - -# ---------------------------------------------------- -# Special Build Targets -# ---------------------------------------------------- - - - -# ---------------------------------------------------- -# Release Target -# ---------------------------------------------------- -include $(ERL_TOP)/make/otp_release_targets.mk diff --git a/lib/typer/doc/html/.gitignore b/lib/typer/doc/html/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 --- a/lib/typer/doc/html/.gitignore +++ /dev/null diff --git a/lib/typer/doc/pdf/.gitignore b/lib/typer/doc/pdf/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 --- a/lib/typer/doc/pdf/.gitignore +++ /dev/null diff --git a/lib/typer/doc/src/Makefile b/lib/typer/doc/src/Makefile deleted file mode 100644 index 3724a2e4d1..0000000000 --- a/lib/typer/doc/src/Makefile +++ /dev/null @@ -1,118 +0,0 @@ -# -# %CopyrightBegin% -# -# Copyright Ericsson AB 2006-2016. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# %CopyrightEnd% -# -include $(ERL_TOP)/make/target.mk -include $(ERL_TOP)/make/$(TARGET)/otp.mk - -# ---------------------------------------------------- -# Application version -# ---------------------------------------------------- -include ../../vsn.mk -VSN=$(TYPER_VSN) -APPLICATION=typer - -# ---------------------------------------------------- -# Release directory specification -# ---------------------------------------------------- -RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) - -# ---------------------------------------------------- -# Target Specs -# ---------------------------------------------------- -XML_APPLICATION_FILES = ref_man.xml -XML_REF3_FILES = - -XML_PART_FILES = part_notes.xml -XML_CHAPTER_FILES = notes.xml - -BOOK_FILES = book.xml - -XML_FILES = \ - $(BOOK_FILES) $(XML_CHAPTER_FILES) \ - $(XML_PART_FILES) $(XML_REF3_FILES) $(XML_APPLICATION_FILES) - -GIF_FILES = - -# ---------------------------------------------------- - -HTML_FILES = $(XML_APPLICATION_FILES:%.xml=$(HTMLDIR)/%.html) \ - $(XML_PART_FILES:%.xml=$(HTMLDIR)/%.html) - -INFO_FILE = ../../info -EXTRA_FILES = \ - $(DEFAULT_GIF_FILES) \ - $(DEFAULT_HTML_FILES) \ - $(XML_REF3_FILES:%.xml=$(HTMLDIR)/%.html) \ - $(XML_CHAPTER_FILES:%.xml=$(HTMLDIR)/%.html) - -MAN3_FILES = $(XML_REF3_FILES:%.xml=$(MAN3DIR)/%.3) - -HTML_REF_MAN_FILE = $(HTMLDIR)/index.html - -TOP_PDF_FILE = $(PDFDIR)/$(APPLICATION)-$(VSN).pdf - -# ---------------------------------------------------- -# FLAGS -# ---------------------------------------------------- -XML_FLAGS += - -# ---------------------------------------------------- -# Targets -# ---------------------------------------------------- -$(HTMLDIR)/%.gif: %.gif - $(INSTALL_DATA) $< $@ - -docs: pdf html man - -$(TOP_PDF_FILE): $(XML_FILES) - -pdf: $(TOP_PDF_FILE) - -html: gifs $(HTML_REF_MAN_FILE) - -man: $(MAN3_FILES) - -gifs: $(GIF_FILES:%=$(HTMLDIR)/%) - -debug opt: - -clean clean_docs: - rm -rf $(HTMLDIR)/* - rm -f $(MAN3DIR)/* - rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) - rm -f errs core *~ - -distclean: clean -realclean: clean - -# ---------------------------------------------------- -# Release Target -# ---------------------------------------------------- -include $(ERL_TOP)/make/otp_release_targets.mk - -release_docs_spec: docs - $(INSTALL_DIR) "$(RELSYSDIR)/doc/pdf" - $(INSTALL_DATA) $(TOP_PDF_FILE) "$(RELSYSDIR)/doc/pdf" - $(INSTALL_DIR) "$(RELSYSDIR)/doc/html" - $(INSTALL_DATA) $(HTMLDIR)/* \ - "$(RELSYSDIR)/doc/html" - $(INSTALL_DATA) $(INFO_FILE) "$(RELSYSDIR)" - - -release_spec: diff --git a/lib/typer/doc/src/book.xml b/lib/typer/doc/src/book.xml deleted file mode 100644 index 20da44ae04..0000000000 --- a/lib/typer/doc/src/book.xml +++ /dev/null @@ -1,42 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE book SYSTEM "book.dtd"> - -<book xmlns:xi="http://www.w3.org/2001/XInclude"> - <header titlestyle="normal"> - <copyright> - <year>2006</year><year>2016</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>TypEr</title> - <prepared></prepared> - <docno></docno> - <date></date> - <rev></rev> - </header> - <pagetext></pagetext> - <preamble> - </preamble> - <pagetext>TypEr</pagetext> - <applications> - <xi:include href="ref_man.xml"/> - </applications> - <releasenotes> - <xi:include href="notes.xml"/> - </releasenotes> -</book> - diff --git a/lib/typer/doc/src/fascicules.xml b/lib/typer/doc/src/fascicules.xml deleted file mode 100644 index b15610fa8b..0000000000 --- a/lib/typer/doc/src/fascicules.xml +++ /dev/null @@ -1,12 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE fascicules SYSTEM "fascicules.dtd"> - -<fascicules> - <fascicule file="part_notes" href="part_notes_frame.html" entry="yes"> - Release Notes - </fascicule> - <fascicule file="" href="../../../../doc/print.html" entry="no"> - Off-Print - </fascicule> -</fascicules> - diff --git a/lib/typer/doc/src/notes.xml b/lib/typer/doc/src/notes.xml deleted file mode 100644 index 9ef5ca1c70..0000000000 --- a/lib/typer/doc/src/notes.xml +++ /dev/null @@ -1,111 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE chapter SYSTEM "chapter.dtd"> - -<chapter> - <header> - <copyright> - <year>2014</year><year>2016</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>TypEr Release Notes</title> - <prepared>otp_appnotes</prepared> - <docno>nil</docno> - <date>nil</date> - <rev>nil</rev> - <file>notes.xml</file> - </header> - <p>This document describes the changes made to TypEr.</p> - -<section><title>TypEr 0.9.11</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Internal changes</p> - <p> - Own Id: OTP-13551</p> - </item> - </list> - </section> - -</section> - -<section><title>TypEr 0.9.10</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p>Fix a bug that could result in a crash when printing - warnings onto standard error. </p> - <p> - Own Id: OTP-13010</p> - </item> - </list> - </section> - -</section> - -<section><title>TypEr 0.9.9</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> Properly extract annotations from core code. </p> - <p> - Own Id: OTP-12727</p> - </item> - </list> - </section> - -</section> - -<section><title>TypEr 0.9.8</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> The name of a compiler option has been fixed in the - Makefile. </p> - <p> - Own Id: OTP-11996</p> - </item> - </list> - </section> - -</section> - -<section><title>TypEr 0.9.7</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Added initial documentation framework for TypEr.</p> - <p> - Own Id: OTP-11860</p> - </item> - </list> - </section> - -</section> - - - -</chapter> - diff --git a/lib/typer/doc/src/part_notes.xml b/lib/typer/doc/src/part_notes.xml deleted file mode 100644 index 3234f0903e..0000000000 --- a/lib/typer/doc/src/part_notes.xml +++ /dev/null @@ -1,36 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE part SYSTEM "part.dtd"> - -<part xmlns:xi="http://www.w3.org/2001/XInclude"> - <header> - <copyright> - <year>2006</year><year>2016</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>TypEr Release Notes</title> - <prepared></prepared> - <docno></docno> - <date></date> - <rev></rev> - </header> - <description> - <p><em>TypEr</em></p> - </description> - <xi:include href="notes.xml"/> -</part> - diff --git a/lib/typer/doc/src/ref_man.xml b/lib/typer/doc/src/ref_man.xml deleted file mode 100644 index c793207443..0000000000 --- a/lib/typer/doc/src/ref_man.xml +++ /dev/null @@ -1,36 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE application SYSTEM "application.dtd"> - -<application xmlns:xi="http://www.w3.org/2001/XInclude"> - <header> - <copyright> - <year>2014</year><year>2016</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>TypEr</title> - <prepared></prepared> - <docno></docno> - <date></date> - <rev></rev> - <file>ref_man.xml</file> - </header> - <description> - </description> - <xi:include href="typer_app.xml"/> -</application> - diff --git a/lib/typer/doc/src/typer_app.xml b/lib/typer/doc/src/typer_app.xml deleted file mode 100644 index d52df5d0da..0000000000 --- a/lib/typer/doc/src/typer_app.xml +++ /dev/null @@ -1,44 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE appref SYSTEM "appref.dtd"> - -<appref> - <header> - <copyright> - <year>2014</year><year>2016</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>TypEr</title> - <prepared></prepared> - <responsible></responsible> - <docno></docno> - <approved></approved> - <checked></checked> - <date></date> - <rev></rev> - <file>typer.xml</file> - </header> - <app>TypEr</app> - <appsummary>The TypEr Application</appsummary> - <description> - <p>An Erlang/OTP application that shows type information - for Erlang modules to the user. Additionally, it can - annotate the code of files with such type information.</p> - </description> - -</appref> - diff --git a/lib/typer/ebin/.gitignore b/lib/typer/ebin/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 --- a/lib/typer/ebin/.gitignore +++ /dev/null diff --git a/lib/typer/info b/lib/typer/info deleted file mode 100644 index 5145fbcfff..0000000000 --- a/lib/typer/info +++ /dev/null @@ -1,2 +0,0 @@ -group: tools -short: TypEr diff --git a/lib/typer/src/Makefile b/lib/typer/src/Makefile deleted file mode 100644 index 6c5d8b0726..0000000000 --- a/lib/typer/src/Makefile +++ /dev/null @@ -1,111 +0,0 @@ -# -# %CopyrightBegin% -# -# Copyright Ericsson AB 2006-2016. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# %CopyrightEnd% -# -#============================================================================= -# -# File: lib/typer/src/Makefile -# Authors: Kostis Sagonas -# -#============================================================================= - -include $(ERL_TOP)/make/target.mk -include $(ERL_TOP)/make/$(TARGET)/otp.mk - -# ---------------------------------------------------- -# Application version -# ---------------------------------------------------- -include ../vsn.mk -VSN=$(TYPER_VSN) - -# ---------------------------------------------------- -# Release directory specification -# ---------------------------------------------------- -RELSYSDIR = $(RELEASE_PATH)/lib/typer-$(VSN) - -# ---------------------------------------------------- -# Orientation information -- find dialyzer's dir -# ---------------------------------------------------- -DIALYZER_DIR = $(ERL_TOP)/lib/dialyzer - -# ---------------------------------------------------- -# Target Specs -# ---------------------------------------------------- -MODULES = typer - -HRL_FILES= -ERL_FILES= $(MODULES:%=%.erl) -INSTALL_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) $(APP_TARGET) $(APPUP_TARGET) -TARGET_FILES= $(INSTALL_FILES) - -APP_FILE= typer.app -APP_SRC= $(APP_FILE).src -APP_TARGET= $(EBIN)/$(APP_FILE) - -APPUP_FILE= typer.appup -APPUP_SRC= $(APPUP_FILE).src -APPUP_TARGET= $(EBIN)/$(APPUP_FILE) - -# ---------------------------------------------------- -# FLAGS -# ---------------------------------------------------- -ERL_COMPILE_FLAGS += +warn_export_vars +warn_untyped_record +warn_missing_spec - -# ---------------------------------------------------- -# Targets -# ---------------------------------------------------- - -debug opt: $(TARGET_FILES) - -docs: - -clean: - rm -f $(TARGET_FILES) - rm -f core - -# ---------------------------------------------------- -# Special Build Targets -# ---------------------------------------------------- - -$(EBIN)/typer.$(EMULATOR): typer.erl ../vsn.mk Makefile - $(erlc_verbose)erlc -W $(ERL_COMPILE_FLAGS) -DVSN="\"v$(VSN)\"" -o$(EBIN) typer.erl - -$(APP_TARGET): $(APP_SRC) ../vsn.mk - $(vsn_verbose)sed -e 's;%VSN%;$(VSN);' $< > $@ - -$(APPUP_TARGET): $(APPUP_SRC) ../vsn.mk - $(vsn_verbose)sed -e 's;%VSN%;$(VSN);' $< > $@ - -# --------------------------------------------------------------------- -# dependencies -# --------------------------------------------------------------------- - - -# ---------------------------------------------------- -# Release Target -# ---------------------------------------------------- -include $(ERL_TOP)/make/otp_release_targets.mk - -release_spec: opt - $(INSTALL_DIR) "$(RELSYSDIR)/src" - $(INSTALL_DATA) $(ERL_FILES) $(HRL_FILES) $(YRL_FILES) \ - "$(RELSYSDIR)/src" - $(INSTALL_DIR) "$(RELSYSDIR)/ebin" - $(INSTALL_DATA) $(INSTALL_FILES) "$(RELSYSDIR)/ebin" - -release_docs_spec: diff --git a/lib/typer/src/typer.app.src b/lib/typer/src/typer.app.src deleted file mode 100644 index 974091b44c..0000000000 --- a/lib/typer/src/typer.app.src +++ /dev/null @@ -1,11 +0,0 @@ -% This is an -*- erlang -*- file. - -{application, typer, - [{description, "TYPe annotator for ERlang programs, version %VSN%"}, - {vsn, "%VSN%"}, - {modules, [typer]}, - {registered, []}, - {applications, [compiler, dialyzer, hipe, kernel, stdlib]}, - {env, []}, - {runtime_dependencies, ["stdlib-2.0","kernel-3.0","hipe-3.10.3","erts-6.0", - "dialyzer-2.7","compiler-5.0"]}]}. diff --git a/lib/typer/src/typer.erl b/lib/typer/src/typer.erl deleted file mode 100644 index 18c4fe902d..0000000000 --- a/lib/typer/src/typer.erl +++ /dev/null @@ -1,1110 +0,0 @@ -%% -*- erlang-indent-level: 2 -*- -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. - -%%----------------------------------------------------------------------- -%% File : typer.erl -%% Author(s) : The first version of typer was written by Bingwen He -%% with guidance from Kostis Sagonas and Tobias Lindahl. -%% Since June 2008 typer is maintained by Kostis Sagonas. -%% Description : An Erlang/OTP application that shows type information -%% for Erlang modules to the user. Additionally, it can -%% annotate the code of files with such type information. -%%----------------------------------------------------------------------- - --module(typer). - --export([start/0]). - -%%----------------------------------------------------------------------- - --define(SHOW, show). --define(SHOW_EXPORTED, show_exported). --define(ANNOTATE, annotate). --define(ANNOTATE_INC_FILES, annotate_inc_files). - --type mode() :: ?SHOW | ?SHOW_EXPORTED | ?ANNOTATE | ?ANNOTATE_INC_FILES. - -%%----------------------------------------------------------------------- - --type files() :: [file:filename()]. --type callgraph() :: dialyzer_callgraph:callgraph(). --type codeserver() :: dialyzer_codeserver:codeserver(). --type plt() :: dialyzer_plt:plt(). - --record(analysis, - {mode :: mode() | 'undefined', - macros = [] :: [{atom(), term()}], - includes = [] :: files(), - codeserver = dialyzer_codeserver:new():: codeserver(), - callgraph = dialyzer_callgraph:new() :: callgraph(), - files = [] :: files(), % absolute names - plt = none :: 'none' | file:filename(), - no_spec = false :: boolean(), - show_succ = false :: boolean(), - %% For choosing between specs or edoc @spec comments - edoc = false :: boolean(), - %% Files in 'fms' are compilable with option 'to_pp'; we keep them - %% as {FileName, ModuleName} in case the ModuleName is different - fms = [] :: [{file:filename(), module()}], - ex_func = map__new() :: map_dict(), - record = map__new() :: map_dict(), - func = map__new() :: map_dict(), - inc_func = map__new() :: map_dict(), - trust_plt = dialyzer_plt:new() :: plt()}). --type analysis() :: #analysis{}. - --record(args, {files = [] :: files(), - files_r = [] :: files(), - trusted = [] :: files()}). --type args() :: #args{}. - -%%-------------------------------------------------------------------- - --spec start() -> no_return(). - -start() -> - {Args, Analysis} = process_cl_args(), - %% io:format("Args: ~p\n", [Args]), - %% io:format("Analysis: ~p\n", [Analysis]), - Timer = dialyzer_timing:init(false), - TrustedFiles = filter_fd(Args#args.trusted, [], fun is_erl_file/1), - Analysis2 = extract(Analysis, TrustedFiles), - All_Files = get_all_files(Args), - %% io:format("All_Files: ~p\n", [All_Files]), - Analysis3 = Analysis2#analysis{files = All_Files}, - Analysis4 = collect_info(Analysis3), - %% io:format("Final: ~p\n", [Analysis4#analysis.fms]), - TypeInfo = get_type_info(Analysis4), - dialyzer_timing:stop(Timer), - show_or_annotate(TypeInfo), - %% io:format("\nTyper analysis finished\n"), - erlang:halt(0). - -%%-------------------------------------------------------------------- - --spec extract(analysis(), files()) -> analysis(). - -extract(#analysis{macros = Macros, - includes = Includes, - trust_plt = TrustPLT} = Analysis, TrustedFiles) -> - %% io:format("--- Extracting trusted typer_info... "), - Ds = [{d, Name, Value} || {Name, Value} <- Macros], - CodeServer = dialyzer_codeserver:new(), - Fun = - fun(File, CS) -> - %% We include one more dir; the one above the one we are trusting - %% E.g, for /home/tests/typer_ann/test.ann.erl, we should include - %% /home/tests/ rather than /home/tests/typer_ann/ - AllIncludes = [filename:dirname(filename:dirname(File)) | Includes], - Is = [{i, Dir} || Dir <- AllIncludes], - CompOpts = dialyzer_utils:src_compiler_opts() ++ Is ++ Ds, - case dialyzer_utils:get_abstract_code_from_src(File, CompOpts) of - {ok, AbstractCode} -> - case dialyzer_utils:get_record_and_type_info(AbstractCode) of - {ok, RecDict} -> - Mod = list_to_atom(filename:basename(File, ".erl")), - case dialyzer_utils:get_spec_info(Mod, AbstractCode, RecDict) of - {ok, SpecDict, CbDict} -> - CS1 = dialyzer_codeserver:store_temp_records(Mod, RecDict, CS), - dialyzer_codeserver:store_temp_contracts(Mod, SpecDict, CbDict, CS1); - {error, Reason} -> compile_error([Reason]) - end; - {error, Reason} -> compile_error([Reason]) - end; - {error, Reason} -> compile_error(Reason) - end - end, - CodeServer1 = lists:foldl(Fun, CodeServer, TrustedFiles), - %% Process remote types - NewCodeServer = - try - CodeServer2 = - dialyzer_utils:merge_types(CodeServer1, - TrustPLT), % XXX change to the PLT? - NewExpTypes = dialyzer_codeserver:get_temp_exported_types(CodeServer1), - case sets:size(NewExpTypes) of 0 -> ok end, - CodeServer3 = dialyzer_codeserver:finalize_exported_types(NewExpTypes, CodeServer2), - CodeServer4 = dialyzer_utils:process_record_remote_types(CodeServer3), - dialyzer_contracts:process_contract_remote_types(CodeServer4) - catch - throw:{error, ErrorMsg} -> - compile_error(ErrorMsg) - end, - %% Create TrustPLT - ContractsDict = dialyzer_codeserver:get_contracts(NewCodeServer), - Contracts = orddict:from_list(dict:to_list(ContractsDict)), - NewTrustPLT = dialyzer_plt:insert_contract_list(TrustPLT, Contracts), - Analysis#analysis{trust_plt = NewTrustPLT}. - -%%-------------------------------------------------------------------- - --spec get_type_info(analysis()) -> analysis(). - -get_type_info(#analysis{callgraph = CallGraph, - trust_plt = TrustPLT, - codeserver = CodeServer} = Analysis) -> - StrippedCallGraph = remove_external(CallGraph, TrustPLT), - %% io:format("--- Analyzing callgraph... "), - try - NewMiniPlt = dialyzer_succ_typings:analyze_callgraph(StrippedCallGraph, - TrustPLT, - CodeServer), - NewPlt = dialyzer_plt:restore_full_plt(NewMiniPlt), - Analysis#analysis{callgraph = StrippedCallGraph, trust_plt = NewPlt} - catch - error:What -> - fatal_error(io_lib:format("Analysis failed with message: ~p", - [{What, erlang:get_stacktrace()}])); - throw:{dialyzer_succ_typing_error, Msg} -> - fatal_error(io_lib:format("Analysis failed with message: ~s", [Msg])) - end. - --spec remove_external(callgraph(), plt()) -> callgraph(). - -remove_external(CallGraph, PLT) -> - {StrippedCG0, Ext} = dialyzer_callgraph:remove_external(CallGraph), - case get_external(Ext, PLT) of - [] -> ok; - Externals -> - msg(io_lib:format(" Unknown functions: ~p\n", [lists:usort(Externals)])), - ExtTypes = rcv_ext_types(), - case ExtTypes of - [] -> ok; - _ -> msg(io_lib:format(" Unknown types: ~p\n", [ExtTypes])) - end - end, - StrippedCG0. - --spec get_external([{mfa(), mfa()}], plt()) -> [mfa()]. - -get_external(Exts, Plt) -> - Fun = fun ({_From, To = {M, F, A}}, Acc) -> - case dialyzer_plt:contains_mfa(Plt, To) of - false -> - case erl_bif_types:is_known(M, F, A) of - true -> Acc; - false -> [To|Acc] - end; - true -> Acc - end - end, - lists:foldl(Fun, [], Exts). - -%%-------------------------------------------------------------------- -%% Showing type information or annotating files with such information. -%%-------------------------------------------------------------------- - --define(TYPER_ANN_DIR, "typer_ann"). - --type line() :: non_neg_integer(). --type fa() :: {atom(), arity()}. --type func_info() :: {line(), atom(), arity()}. - --record(info, {records = maps:new() :: erl_types:type_table(), - functions = [] :: [func_info()], - types = map__new() :: map_dict(), - edoc = false :: boolean()}). --record(inc, {map = map__new() :: map_dict(), filter = [] :: files()}). --type inc() :: #inc{}. - --spec show_or_annotate(analysis()) -> 'ok'. - -show_or_annotate(#analysis{mode = Mode, fms = Files} = Analysis) -> - case Mode of - ?SHOW -> show(Analysis); - ?SHOW_EXPORTED -> show(Analysis); - ?ANNOTATE -> - Fun = fun ({File, Module}) -> - Info = get_final_info(File, Module, Analysis), - write_typed_file(File, Info) - end, - lists:foreach(Fun, Files); - ?ANNOTATE_INC_FILES -> - IncInfo = write_and_collect_inc_info(Analysis), - write_inc_files(IncInfo) - end. - -write_and_collect_inc_info(Analysis) -> - Fun = fun ({File, Module}, Inc) -> - Info = get_final_info(File, Module, Analysis), - write_typed_file(File, Info), - IncFuns = get_functions(File, Analysis), - collect_imported_functions(IncFuns, Info#info.types, Inc) - end, - NewInc = lists:foldl(Fun, #inc{}, Analysis#analysis.fms), - clean_inc(NewInc). - -write_inc_files(Inc) -> - Fun = - fun (File) -> - Val = map__lookup(File, Inc#inc.map), - %% Val is function with its type info - %% in form [{{Line,F,A},Type}] - Functions = [Key || {Key, _} <- Val], - Val1 = [{{F,A},Type} || {{_Line,F,A},Type} <- Val], - Info = #info{types = map__from_list(Val1), - records = maps:new(), - %% Note we need to sort functions here! - functions = lists:keysort(1, Functions)}, - %% io:format("Types ~p\n", [Info#info.types]), - %% io:format("Functions ~p\n", [Info#info.functions]), - %% io:format("Records ~p\n", [Info#info.records]), - write_typed_file(File, Info) - end, - lists:foreach(Fun, dict:fetch_keys(Inc#inc.map)). - -show(Analysis) -> - Fun = fun ({File, Module}) -> - Info = get_final_info(File, Module, Analysis), - show_type_info(File, Info) - end, - lists:foreach(Fun, Analysis#analysis.fms). - -get_final_info(File, Module, Analysis) -> - Records = get_records(File, Analysis), - Types = get_types(Module, Analysis, Records), - Functions = get_functions(File, Analysis), - Edoc = Analysis#analysis.edoc, - #info{records = Records, functions = Functions, types = Types, edoc = Edoc}. - -collect_imported_functions(Functions, Types, Inc) -> - %% Coming from other sourses, including: - %% FIXME: How to deal with yecc-generated file???? - %% --.yrl (yecc-generated file)??? - %% -- yeccpre.hrl (yecc-generated file)??? - %% -- other cases - Fun = fun ({File, _} = Obj, I) -> - case is_yecc_gen(File, I) of - {true, NewI} -> NewI; - {false, NewI} -> - check_imported_functions(Obj, NewI, Types) - end - end, - lists:foldl(Fun, Inc, Functions). - --spec is_yecc_gen(file:filename(), inc()) -> {boolean(), inc()}. - -is_yecc_gen(File, #inc{filter = Fs} = Inc) -> - case lists:member(File, Fs) of - true -> {true, Inc}; - false -> - case filename:extension(File) of - ".yrl" -> - Rootname = filename:rootname(File, ".yrl"), - Obj = Rootname ++ ".erl", - case lists:member(Obj, Fs) of - true -> {true, Inc}; - false -> - NewInc = Inc#inc{filter = [Obj|Fs]}, - {true, NewInc} - end; - _ -> - case filename:basename(File) of - "yeccpre.hrl" -> {true, Inc}; - _ -> {false, Inc} - end - end - end. - -check_imported_functions({File, {Line, F, A}}, Inc, Types) -> - IncMap = Inc#inc.map, - FA = {F, A}, - Type = get_type_info(FA, Types), - case map__lookup(File, IncMap) of - none -> %% File is not added. Add it - Obj = {File,[{FA, {Line, Type}}]}, - NewMap = map__insert(Obj, IncMap), - Inc#inc{map = NewMap}; - Val -> %% File is already in. Check. - case lists:keyfind(FA, 1, Val) of - false -> - %% Function is not in; add it - Obj = {File, Val ++ [{FA, {Line, Type}}]}, - NewMap = map__insert(Obj, IncMap), - Inc#inc{map = NewMap}; - Type -> - %% Function is in and with same type - Inc; - _ -> - %% Function is in but with diff type - inc_warning(FA, File), - Elem = lists:keydelete(FA, 1, Val), - NewMap = case Elem of - [] -> map__remove(File, IncMap); - _ -> map__insert({File, Elem}, IncMap) - end, - Inc#inc{map = NewMap} - end - end. - -inc_warning({F, A}, File) -> - io:format(" ***Warning: Skip function ~p/~p ", [F, A]), - io:format("in file ~p because of inconsistent type\n", [File]). - -clean_inc(Inc) -> - Inc1 = remove_yecc_generated_file(Inc), - normalize_obj(Inc1). - -remove_yecc_generated_file(#inc{filter = Filter} = Inc) -> - Fun = fun (Key, #inc{map = Map} = I) -> - I#inc{map = map__remove(Key, Map)} - end, - lists:foldl(Fun, Inc, Filter). - -normalize_obj(TmpInc) -> - Fun = fun (Key, Val, Inc) -> - NewVal = [{{Line,F,A},Type} || {{F,A},{Line,Type}} <- Val], - map__insert({Key, NewVal}, Inc) - end, - TmpInc#inc{map = map__fold(Fun, map__new(), TmpInc#inc.map)}. - -get_records(File, Analysis) -> - map__lookup(File, Analysis#analysis.record). - -get_types(Module, Analysis, Records) -> - TypeInfoPlt = Analysis#analysis.trust_plt, - TypeInfo = - case dialyzer_plt:lookup_module(TypeInfoPlt, Module) of - none -> []; - {value, List} -> List - end, - CodeServer = Analysis#analysis.codeserver, - TypeInfoList = - case Analysis#analysis.show_succ of - true -> - [convert_type_info(I) || I <- TypeInfo]; - false -> - [get_type(I, CodeServer, Records) || I <- TypeInfo] - end, - map__from_list(TypeInfoList). - -convert_type_info({{_M, F, A}, Range, Arg}) -> - {{F, A}, {Range, Arg}}. - -get_type({{M, F, A} = MFA, Range, Arg}, CodeServer, Records) -> - case dialyzer_codeserver:lookup_mfa_contract(MFA, CodeServer) of - error -> - {{F, A}, {Range, Arg}}; - {ok, {_FileLine, Contract, _Xtra}} -> - Sig = erl_types:t_fun(Arg, Range), - case dialyzer_contracts:check_contract(Contract, Sig) of - ok -> {{F, A}, {contract, Contract}}; - {error, {extra_range, _, _}} -> - {{F, A}, {contract, Contract}}; - {error, {overlapping_contract, []}} -> - {{F, A}, {contract, Contract}}; - {error, invalid_contract} -> - CString = dialyzer_contracts:contract_to_string(Contract), - SigString = dialyzer_utils:format_sig(Sig, Records), - Msg = io_lib:format("Error in contract of function ~w:~w/~w\n" - "\t The contract is: " ++ CString ++ "\n" ++ - "\t but the inferred signature is: ~s", - [M, F, A, SigString]), - fatal_error(Msg); - {error, ErrorStr} when is_list(ErrorStr) -> % ErrorStr is a string() - Msg = io_lib:format("Error in contract of function ~w:~w/~w: ~s", - [M, F, A, ErrorStr]), - fatal_error(Msg) - end - end. - -get_functions(File, Analysis) -> - case Analysis#analysis.mode of - ?SHOW -> - Funcs = map__lookup(File, Analysis#analysis.func), - Inc_Funcs = map__lookup(File, Analysis#analysis.inc_func), - remove_module_info(Funcs) ++ normalize_incFuncs(Inc_Funcs); - ?SHOW_EXPORTED -> - Ex_Funcs = map__lookup(File, Analysis#analysis.ex_func), - remove_module_info(Ex_Funcs); - ?ANNOTATE -> - Funcs = map__lookup(File, Analysis#analysis.func), - remove_module_info(Funcs); - ?ANNOTATE_INC_FILES -> - map__lookup(File, Analysis#analysis.inc_func) - end. - -normalize_incFuncs(Functions) -> - [FunInfo || {_FileName, FunInfo} <- Functions]. - --spec remove_module_info([func_info()]) -> [func_info()]. - -remove_module_info(FunInfoList) -> - F = fun ({_,module_info,0}) -> false; - ({_,module_info,1}) -> false; - ({Line,F,A}) when is_integer(Line), is_atom(F), is_integer(A) -> true - end, - lists:filter(F, FunInfoList). - -write_typed_file(File, Info) -> - io:format(" Processing file: ~p\n", [File]), - Dir = filename:dirname(File), - RootName = filename:basename(filename:rootname(File)), - Ext = filename:extension(File), - TyperAnnDir = filename:join(Dir, ?TYPER_ANN_DIR), - TmpNewFilename = lists:concat([RootName, ".ann", Ext]), - NewFileName = filename:join(TyperAnnDir, TmpNewFilename), - case file:make_dir(TyperAnnDir) of - {error, Reason} -> - case Reason of - eexist -> %% TypEr dir exists; remove old typer files if they exist - case file:delete(NewFileName) of - ok -> ok; - {error, enoent} -> ok; - {error, _} -> - Msg = io_lib:format("Error in deleting file ~s\n", [NewFileName]), - fatal_error(Msg) - end, - write_typed_file(File, Info, NewFileName); - enospc -> - Msg = io_lib:format("Not enough space in ~p\n", [Dir]), - fatal_error(Msg); - eacces -> - Msg = io_lib:format("No write permission in ~p\n", [Dir]), - fatal_error(Msg); - _ -> - Msg = io_lib:format("Unhandled error ~s when writing ~p\n", - [Reason, Dir]), - fatal_error(Msg) - end; - ok -> %% Typer dir does NOT exist - write_typed_file(File, Info, NewFileName) - end. - -write_typed_file(File, Info, NewFileName) -> - {ok, Binary} = file:read_file(File), - Chars = binary_to_list(Binary), - write_typed_file(Chars, NewFileName, Info, 1, []), - io:format(" Saved as: ~p\n", [NewFileName]). - -write_typed_file(Chars, File, #info{functions = []}, _LNo, _Acc) -> - ok = file:write_file(File, list_to_binary(Chars), [append]); -write_typed_file([Ch|Chs] = Chars, File, Info, LineNo, Acc) -> - [{Line,F,A}|RestFuncs] = Info#info.functions, - case Line of - 1 -> %% This will happen only for inc files - ok = raw_write(F, A, Info, File, []), - NewInfo = Info#info{functions = RestFuncs}, - NewAcc = [], - write_typed_file(Chars, File, NewInfo, Line, NewAcc); - _ -> - case Ch of - 10 -> - NewLineNo = LineNo + 1, - {NewInfo, NewAcc} = - case NewLineNo of - Line -> - ok = raw_write(F, A, Info, File, [Ch|Acc]), - {Info#info{functions = RestFuncs}, []}; - _ -> - {Info, [Ch|Acc]} - end, - write_typed_file(Chs, File, NewInfo, NewLineNo, NewAcc); - _ -> - write_typed_file(Chs, File, Info, LineNo, [Ch|Acc]) - end - end. - -raw_write(F, A, Info, File, Content) -> - TypeInfo = get_type_string(F, A, Info, file), - ContentList = lists:reverse(Content) ++ TypeInfo ++ "\n", - ContentBin = list_to_binary(ContentList), - file:write_file(File, ContentBin, [append]). - -get_type_string(F, A, Info, Mode) -> - Type = get_type_info({F,A}, Info#info.types), - TypeStr = - case Type of - {contract, C} -> - dialyzer_contracts:contract_to_string(C); - {RetType, ArgType} -> - Sig = erl_types:t_fun(ArgType, RetType), - dialyzer_utils:format_sig(Sig, Info#info.records) - end, - case Info#info.edoc of - false -> - case {Mode, Type} of - {file, {contract, _}} -> ""; - _ -> - Prefix = lists:concat(["-spec ", erl_types:atom_to_string(F)]), - lists:concat([Prefix, TypeStr, "."]) - end; - true -> - Prefix = lists:concat(["%% @spec ", F]), - lists:concat([Prefix, TypeStr, "."]) - end. - -show_type_info(File, Info) -> - io:format("\n%% File: ~p\n%% ", [File]), - OutputString = lists:concat(["~.", length(File)+8, "c~n"]), - io:fwrite(OutputString, [$-]), - Fun = fun ({_LineNo, F, A}) -> - TypeInfo = get_type_string(F, A, Info, show), - io:format("~s\n", [TypeInfo]) - end, - lists:foreach(Fun, Info#info.functions). - -get_type_info(Func, Types) -> - case map__lookup(Func, Types) of - none -> - %% Note: Typeinfo of any function should exist in - %% the result offered by dialyzer, otherwise there - %% *must* be something wrong with the analysis - Msg = io_lib:format("No type info for function: ~p\n", [Func]), - fatal_error(Msg); - {contract, _Fun} = C -> C; - {_RetType, _ArgType} = RA -> RA - end. - -%%-------------------------------------------------------------------- -%% Processing of command-line options and arguments. -%%-------------------------------------------------------------------- - --spec process_cl_args() -> {args(), analysis()}. - -process_cl_args() -> - ArgList = init:get_plain_arguments(), - %% io:format("Args is ~p\n", [ArgList]), - {Args, Analysis} = analyze_args(ArgList, #args{}, #analysis{}), - %% if the mode has not been set, set it to the default mode (show) - {Args, case Analysis#analysis.mode of - undefined -> Analysis#analysis{mode = ?SHOW}; - Mode when is_atom(Mode) -> Analysis - end}. - -analyze_args([], Args, Analysis) -> - {Args, Analysis}; -analyze_args(ArgList, Args, Analysis) -> - {Result, Rest} = cl(ArgList), - {NewArgs, NewAnalysis} = analyze_result(Result, Args, Analysis), - analyze_args(Rest, NewArgs, NewAnalysis). - -cl(["-h"|_]) -> help_message(); -cl(["--help"|_]) -> help_message(); -cl(["-v"|_]) -> version_message(); -cl(["--version"|_]) -> version_message(); -cl(["--edoc"|Opts]) -> {edoc, Opts}; -cl(["--show"|Opts]) -> {{mode, ?SHOW}, Opts}; -cl(["--show_exported"|Opts]) -> {{mode, ?SHOW_EXPORTED}, Opts}; -cl(["--show-exported"|Opts]) -> {{mode, ?SHOW_EXPORTED}, Opts}; -cl(["--show_success_typings"|Opts]) -> {show_succ, Opts}; -cl(["--show-success-typings"|Opts]) -> {show_succ, Opts}; -cl(["--annotate"|Opts]) -> {{mode, ?ANNOTATE}, Opts}; -cl(["--annotate-inc-files"|Opts]) -> {{mode, ?ANNOTATE_INC_FILES}, Opts}; -cl(["--no_spec"|Opts]) -> {no_spec, Opts}; -cl(["--plt",Plt|Opts]) -> {{plt, Plt}, Opts}; -cl(["-D"++Def|Opts]) -> - case Def of - "" -> fatal_error("no variable name specified after -D"); - _ -> - DefPair = process_def_list(re:split(Def, "=", [{return, list}])), - {{def, DefPair}, Opts} - end; -cl(["-I",Dir|Opts]) -> {{inc, Dir}, Opts}; -cl(["-I"++Dir|Opts]) -> - case Dir of - "" -> fatal_error("no include directory specified after -I"); - _ -> {{inc, Dir}, Opts} - end; -cl(["-T"|Opts]) -> - {Files, RestOpts} = dialyzer_cl_parse:collect_args(Opts), - case Files of - [] -> fatal_error("no file or directory specified after -T"); - [_|_] -> {{trusted, Files}, RestOpts} - end; -cl(["-r"|Opts]) -> - {Files, RestOpts} = dialyzer_cl_parse:collect_args(Opts), - {{files_r, Files}, RestOpts}; -cl(["-pa",Dir|Opts]) -> {{pa,Dir}, Opts}; -cl(["-pz",Dir|Opts]) -> {{pz,Dir}, Opts}; -cl(["-"++H|_]) -> fatal_error("unknown option -"++H); -cl(Opts) -> - {Files, RestOpts} = dialyzer_cl_parse:collect_args(Opts), - {{files, Files}, RestOpts}. - -process_def_list(L) -> - case L of - [Name, Value] -> - {ok, Tokens, _} = erl_scan:string(Value ++ "."), - {ok, ErlValue} = erl_parse:parse_term(Tokens), - {list_to_atom(Name), ErlValue}; - [Name] -> - {list_to_atom(Name), true} - end. - -%% Get information about files that the user trusts and wants to analyze -analyze_result({files, Val}, Args, Analysis) -> - NewVal = Args#args.files ++ Val, - {Args#args{files = NewVal}, Analysis}; -analyze_result({files_r, Val}, Args, Analysis) -> - NewVal = Args#args.files_r ++ Val, - {Args#args{files_r = NewVal}, Analysis}; -analyze_result({trusted, Val}, Args, Analysis) -> - NewVal = Args#args.trusted ++ Val, - {Args#args{trusted = NewVal}, Analysis}; -analyze_result(edoc, Args, Analysis) -> - {Args, Analysis#analysis{edoc = true}}; -%% Get useful information for actual analysis -analyze_result({mode, Mode}, Args, Analysis) -> - case Analysis#analysis.mode of - undefined -> {Args, Analysis#analysis{mode = Mode}}; - OldMode -> mode_error(OldMode, Mode) - end; -analyze_result({def, Val}, Args, Analysis) -> - NewVal = Analysis#analysis.macros ++ [Val], - {Args, Analysis#analysis{macros = NewVal}}; -analyze_result({inc, Val}, Args, Analysis) -> - NewVal = Analysis#analysis.includes ++ [Val], - {Args, Analysis#analysis{includes = NewVal}}; -analyze_result({plt, Plt}, Args, Analysis) -> - {Args, Analysis#analysis{plt = Plt}}; -analyze_result(show_succ, Args, Analysis) -> - {Args, Analysis#analysis{show_succ = true}}; -analyze_result(no_spec, Args, Analysis) -> - {Args, Analysis#analysis{no_spec = true}}; -analyze_result({pa, Dir}, Args, Analysis) -> - true = code:add_patha(Dir), - {Args, Analysis}; -analyze_result({pz, Dir}, Args, Analysis) -> - true = code:add_pathz(Dir), - {Args, Analysis}. - -%%-------------------------------------------------------------------- -%% File processing. -%%-------------------------------------------------------------------- - --spec get_all_files(args()) -> [file:filename(),...]. - -get_all_files(#args{files = Fs, files_r = Ds}) -> - case filter_fd(Fs, Ds, fun test_erl_file_exclude_ann/1) of - [] -> fatal_error("no file(s) to analyze"); - AllFiles -> AllFiles - end. - --spec test_erl_file_exclude_ann(file:filename()) -> boolean(). - -test_erl_file_exclude_ann(File) -> - case is_erl_file(File) of - true -> %% Exclude files ending with ".ann.erl" - case re:run(File, "[\.]ann[\.]erl$") of - {match, _} -> false; - nomatch -> true - end; - false -> false - end. - --spec is_erl_file(file:filename()) -> boolean(). - -is_erl_file(File) -> - filename:extension(File) =:= ".erl". - --type test_file_fun() :: fun((file:filename()) -> boolean()). - --spec filter_fd(files(), files(), test_file_fun()) -> files(). - -filter_fd(File_Dir, Dir_R, Fun) -> - All_File_1 = process_file_and_dir(File_Dir, Fun), - All_File_2 = process_dir_rec(Dir_R, Fun), - remove_dup(All_File_1 ++ All_File_2). - --spec process_file_and_dir(files(), test_file_fun()) -> files(). - -process_file_and_dir(File_Dir, TestFun) -> - Fun = - fun (Elem, Acc) -> - case filelib:is_regular(Elem) of - true -> process_file(Elem, TestFun, Acc); - false -> check_dir(Elem, false, Acc, TestFun) - end - end, - lists:foldl(Fun, [], File_Dir). - --spec process_dir_rec(files(), test_file_fun()) -> files(). - -process_dir_rec(Dirs, TestFun) -> - Fun = fun (Dir, Acc) -> check_dir(Dir, true, Acc, TestFun) end, - lists:foldl(Fun, [], Dirs). - --spec check_dir(file:filename(), boolean(), files(), test_file_fun()) -> files(). - -check_dir(Dir, Recursive, Acc, Fun) -> - case file:list_dir(Dir) of - {ok, Files} -> - {TmpDirs, TmpFiles} = split_dirs_and_files(Files, Dir), - case Recursive of - false -> - FinalFiles = process_file_and_dir(TmpFiles, Fun), - Acc ++ FinalFiles; - true -> - TmpAcc1 = process_file_and_dir(TmpFiles, Fun), - TmpAcc2 = process_dir_rec(TmpDirs, Fun), - Acc ++ TmpAcc1 ++ TmpAcc2 - end; - {error, eacces} -> - fatal_error("no access permission to dir \""++Dir++"\""); - {error, enoent} -> - fatal_error("cannot access "++Dir++": No such file or directory"); - {error, _Reason} -> - fatal_error("error involving a use of file:list_dir/1") - end. - -%% Same order as the input list --spec process_file(file:filename(), test_file_fun(), files()) -> files(). - -process_file(File, TestFun, Acc) -> - case TestFun(File) of - true -> Acc ++ [File]; - false -> Acc - end. - -%% Same order as the input list --spec split_dirs_and_files(files(), file:filename()) -> {files(), files()}. - -split_dirs_and_files(Elems, Dir) -> - Test_Fun = - fun (Elem, {DirAcc, FileAcc}) -> - File = filename:join(Dir, Elem), - case filelib:is_regular(File) of - false -> {[File|DirAcc], FileAcc}; - true -> {DirAcc, [File|FileAcc]} - end - end, - {Dirs, Files} = lists:foldl(Test_Fun, {[], []}, Elems), - {lists:reverse(Dirs), lists:reverse(Files)}. - -%% Removes duplicate filenames but keeps the order of the input list --spec remove_dup(files()) -> files(). - -remove_dup(Files) -> - Test_Dup = fun (File, Acc) -> - case lists:member(File, Acc) of - true -> Acc; - false -> [File|Acc] - end - end, - Reversed_Elems = lists:foldl(Test_Dup, [], Files), - lists:reverse(Reversed_Elems). - -%%-------------------------------------------------------------------- -%% Collect information. -%%-------------------------------------------------------------------- - --type inc_file_info() :: {file:filename(), func_info()}. - --record(tmpAcc, {file :: file:filename(), - module :: atom(), - funcAcc = [] :: [func_info()], - incFuncAcc = [] :: [inc_file_info()], - dialyzerObj = [] :: [{mfa(), {_, _}}]}). - --spec collect_info(analysis()) -> analysis(). - -collect_info(Analysis) -> - NewPlt = - try get_dialyzer_plt(Analysis) of - DialyzerPlt -> - dialyzer_plt:merge_plts([Analysis#analysis.trust_plt, DialyzerPlt]) - catch - throw:{dialyzer_error,_Reason} -> - fatal_error("Dialyzer's PLT is missing or is not up-to-date; please (re)create it") - end, - NewAnalysis = lists:foldl(fun collect_one_file_info/2, - Analysis#analysis{trust_plt = NewPlt}, - Analysis#analysis.files), - %% Process Remote Types - TmpCServer = NewAnalysis#analysis.codeserver, - NewCServer = - try - TmpCServer1 = dialyzer_utils:merge_types(TmpCServer, NewPlt), - NewExpTypes = dialyzer_codeserver:get_temp_exported_types(TmpCServer), - OldExpTypes = dialyzer_plt:get_exported_types(NewPlt), - MergedExpTypes = sets:union(NewExpTypes, OldExpTypes), - TmpCServer2 = - dialyzer_codeserver:finalize_exported_types(MergedExpTypes, TmpCServer1), - TmpCServer3 = dialyzer_utils:process_record_remote_types(TmpCServer2), - dialyzer_contracts:process_contract_remote_types(TmpCServer3) - catch - throw:{error, ErrorMsg} -> - fatal_error(ErrorMsg) - end, - NewAnalysis#analysis{codeserver = NewCServer}. - -collect_one_file_info(File, Analysis) -> - Ds = [{d,Name,Val} || {Name,Val} <- Analysis#analysis.macros], - %% Current directory should also be included in "Includes". - Includes = [filename:dirname(File)|Analysis#analysis.includes], - Is = [{i,Dir} || Dir <- Includes], - Options = dialyzer_utils:src_compiler_opts() ++ Is ++ Ds, - case dialyzer_utils:get_abstract_code_from_src(File, Options) of - {error, Reason} -> - %% io:format("File=~p\n,Options=~p\n,Error=~p\n", [File,Options,Reason]), - compile_error(Reason); - {ok, AbstractCode} -> - case dialyzer_utils:get_core_from_abstract_code(AbstractCode, Options) of - error -> compile_error(["Could not get core erlang for "++File]); - {ok, Core} -> - case dialyzer_utils:get_record_and_type_info(AbstractCode) of - {error, Reason} -> compile_error([Reason]); - {ok, Records} -> - Mod = cerl:concrete(cerl:module_name(Core)), - case dialyzer_utils:get_spec_info(Mod, AbstractCode, Records) of - {error, Reason} -> compile_error([Reason]); - {ok, SpecInfo, CbInfo} -> - ExpTypes = get_exported_types_from_core(Core), - analyze_core_tree(Core, Records, SpecInfo, CbInfo, - ExpTypes, Analysis, File) - end - end - end - end. - -analyze_core_tree(Core, Records, SpecInfo, CbInfo, ExpTypes, Analysis, File) -> - Module = cerl:concrete(cerl:module_name(Core)), - TmpTree = cerl:from_records(Core), - CS1 = Analysis#analysis.codeserver, - NextLabel = dialyzer_codeserver:get_next_core_label(CS1), - {Tree, NewLabel} = cerl_trees:label(TmpTree, NextLabel), - CS2 = dialyzer_codeserver:insert(Module, Tree, CS1), - CS3 = dialyzer_codeserver:set_next_core_label(NewLabel, CS2), - CS4 = dialyzer_codeserver:store_temp_records(Module, Records, CS3), - CS5 = - case Analysis#analysis.no_spec of - true -> CS4; - false -> - dialyzer_codeserver:store_temp_contracts(Module, SpecInfo, CbInfo, CS4) - end, - OldExpTypes = dialyzer_codeserver:get_temp_exported_types(CS5), - MergedExpTypes = sets:union(ExpTypes, OldExpTypes), - CS6 = dialyzer_codeserver:insert_temp_exported_types(MergedExpTypes, CS5), - Ex_Funcs = [{0,F,A} || {_,_,{F,A}} <- cerl:module_exports(Tree)], - CG = Analysis#analysis.callgraph, - {V, E} = dialyzer_callgraph:scan_core_tree(Tree, CG), - dialyzer_callgraph:add_edges(E, V, CG), - Fun = fun analyze_one_function/2, - All_Defs = cerl:module_defs(Tree), - Acc = lists:foldl(Fun, #tmpAcc{file = File, module = Module}, All_Defs), - Exported_FuncMap = map__insert({File, Ex_Funcs}, Analysis#analysis.ex_func), - %% we must sort all functions in the file which - %% originate from this file by *numerical order* of lineNo - Sorted_Functions = lists:keysort(1, Acc#tmpAcc.funcAcc), - FuncMap = map__insert({File, Sorted_Functions}, Analysis#analysis.func), - %% we do not need to sort functions which are imported from included files - IncFuncMap = map__insert({File, Acc#tmpAcc.incFuncAcc}, - Analysis#analysis.inc_func), - FMs = Analysis#analysis.fms ++ [{File, Module}], - RecordMap = map__insert({File, Records}, Analysis#analysis.record), - Analysis#analysis{fms = FMs, - callgraph = CG, - codeserver = CS6, - ex_func = Exported_FuncMap, - inc_func = IncFuncMap, - record = RecordMap, - func = FuncMap}. - -analyze_one_function({Var, FunBody} = Function, Acc) -> - F = cerl:fname_id(Var), - A = cerl:fname_arity(Var), - TmpDialyzerObj = {{Acc#tmpAcc.module, F, A}, Function}, - NewDialyzerObj = Acc#tmpAcc.dialyzerObj ++ [TmpDialyzerObj], - Anno = cerl:get_ann(FunBody), - LineNo = get_line(Anno), - FileName = get_file(Anno), - BaseName = filename:basename(FileName), - FuncInfo = {LineNo, F, A}, - OriginalName = Acc#tmpAcc.file, - {FuncAcc, IncFuncAcc} = - case (FileName =:= OriginalName) orelse (BaseName =:= OriginalName) of - true -> %% Coming from original file - %% io:format("Added function ~p\n", [{LineNo, F, A}]), - {Acc#tmpAcc.funcAcc ++ [FuncInfo], Acc#tmpAcc.incFuncAcc}; - false -> - %% Coming from other sourses, including: - %% -- .yrl (yecc-generated file) - %% -- yeccpre.hrl (yecc-generated file) - %% -- other cases - {Acc#tmpAcc.funcAcc, Acc#tmpAcc.incFuncAcc ++ [{FileName, FuncInfo}]} - end, - Acc#tmpAcc{funcAcc = FuncAcc, - incFuncAcc = IncFuncAcc, - dialyzerObj = NewDialyzerObj}. - -get_line([Line|_]) when is_integer(Line) -> Line; -get_line([_|T]) -> get_line(T); -get_line([]) -> none. - -get_file([{file,File}|_]) -> File; -get_file([_|T]) -> get_file(T); -get_file([]) -> "no_file". % should not happen - --spec get_dialyzer_plt(analysis()) -> plt(). - -get_dialyzer_plt(#analysis{plt = PltFile0}) -> - PltFile = - case PltFile0 =:= none of - true -> dialyzer_plt:get_default_plt(); - false -> PltFile0 - end, - dialyzer_plt:from_file(PltFile). - -%% Exported Types - -get_exported_types_from_core(Core) -> - Attrs = cerl:module_attrs(Core), - ExpTypes1 = [cerl:concrete(L2) || {L1, L2} <- Attrs, - cerl:is_literal(L1), - cerl:is_literal(L2), - cerl:concrete(L1) =:= 'export_type'], - ExpTypes2 = lists:flatten(ExpTypes1), - M = cerl:atom_val(cerl:module_name(Core)), - sets:from_list([{M, F, A} || {F, A} <- ExpTypes2]). - -%%-------------------------------------------------------------------- -%% Utilities for error reporting. -%%-------------------------------------------------------------------- - --spec fatal_error(string()) -> no_return(). - -fatal_error(Slogan) -> - msg(io_lib:format("typer: ~s\n", [Slogan])), - erlang:halt(1). - --spec mode_error(mode(), mode()) -> no_return(). - -mode_error(OldMode, NewMode) -> - Msg = io_lib:format("Mode was previously set to '~s'; " - "can not set it to '~s' now", - [OldMode, NewMode]), - fatal_error(Msg). - --spec compile_error([string()]) -> no_return(). - -compile_error(Reason) -> - JoinedString = lists:flatten([X ++ "\n" || X <- Reason]), - Msg = "Analysis failed with error report:\n" ++ JoinedString, - fatal_error(Msg). - --spec msg(string()) -> 'ok'. - -msg(Msg) -> - io:format(standard_error, "~s", [Msg]). - -%%-------------------------------------------------------------------- -%% Version and help messages. -%%-------------------------------------------------------------------- - --spec version_message() -> no_return(). - -version_message() -> - io:format("TypEr version "++?VSN++"\n"), - erlang:halt(0). - --spec help_message() -> no_return(). - -help_message() -> - S = <<" Usage: typer [--help] [--version] [--plt PLT] [--edoc] - [--show | --show-exported | --annotate | --annotate-inc-files] - [-Ddefine]* [-I include_dir]* [-pa dir]* [-pz dir]* - [-T application]* [-r] file* - - Options: - -r dir* - search directories recursively for .erl files below them - --show - Prints type specifications for all functions on stdout. - (this is the default behaviour; this option is not really needed) - --show-exported (or --show_exported) - Same as --show, but prints specifications for exported functions only - Specs are displayed sorted alphabetically on the function's name - --annotate - Annotates the specified files with type specifications - --annotate-inc-files - Same as --annotate but annotates all -include() files as well as - all .erl files (use this option with caution - has not been tested much) - --edoc - Prints type information as Edoc @spec comments, not as type specs - --plt PLT - Use the specified dialyzer PLT file rather than the default one - -T file* - The specified file(s) already contain type specifications and these - are to be trusted in order to print specs for the rest of the files - (Multiple files or dirs, separated by spaces, can be specified.) - -Dname (or -Dname=value) - pass the defined name(s) to TypEr - (The syntax of defines is the same as that used by \"erlc\".) - -I include_dir - pass the include_dir to TypEr - (The syntax of includes is the same as that used by \"erlc\".) - -pa dir - -pz dir - Set code path options to TypEr - (This is useful for files that use parse tranforms.) - --version (or -v) - prints the Typer version and exits - --help (or -h) - prints this message and exits - - Note: - * denotes that multiple occurrences of these options are possible. -">>, - io:put_chars(S), - erlang:halt(0). - -%%-------------------------------------------------------------------- -%% Handle messages. -%%-------------------------------------------------------------------- - -rcv_ext_types() -> - Self = self(), - Self ! {Self, done}, - rcv_ext_types(Self, []). - -rcv_ext_types(Self, ExtTypes) -> - receive - {Self, ext_types, ExtType} -> - rcv_ext_types(Self, [ExtType|ExtTypes]); - {Self, done} -> - lists:usort(ExtTypes) - end. - -%%-------------------------------------------------------------------- -%% A convenient abstraction of a Key-Value mapping data structure -%% specialized for the uses in this module -%%-------------------------------------------------------------------- - --type map_dict() :: dict:dict(). - --spec map__new() -> map_dict(). -map__new() -> - dict:new(). - --spec map__insert({term(), term()}, map_dict()) -> map_dict(). -map__insert(Object, Map) -> - {Key, Value} = Object, - dict:store(Key, Value, Map). - --spec map__lookup(term(), map_dict()) -> term(). -map__lookup(Key, Map) -> - try dict:fetch(Key, Map) catch error:_ -> none end. - --spec map__from_list([{fa(), term()}]) -> map_dict(). -map__from_list(List) -> - dict:from_list(List). - --spec map__remove(term(), map_dict()) -> map_dict(). -map__remove(Key, Dict) -> - dict:erase(Key, Dict). - --spec map__fold(fun((term(), term(), term()) -> map_dict()), map_dict(), map_dict()) -> map_dict(). -map__fold(Fun, Acc0, Dict) -> - dict:fold(Fun, Acc0, Dict). diff --git a/lib/typer/test/Makefile b/lib/typer/test/Makefile deleted file mode 100644 index fb5570d9f0..0000000000 --- a/lib/typer/test/Makefile +++ /dev/null @@ -1,65 +0,0 @@ -include $(ERL_TOP)/make/target.mk -include $(ERL_TOP)/make/$(TARGET)/otp.mk - -# ---------------------------------------------------- -# Target Specs -# ---------------------------------------------------- - -MODULES= \ - typer_SUITE - -ERL_FILES= $(MODULES:%=%.erl) - -TARGET_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) -INSTALL_PROGS= $(TARGET_FILES) - -EMAKEFILE=Emakefile - -# ---------------------------------------------------- -# Release directory specification -# ---------------------------------------------------- -RELSYSDIR = $(RELEASE_PATH)/typer_test - -# ---------------------------------------------------- -# FLAGS -# ---------------------------------------------------- - -ERL_MAKE_FLAGS += -ERL_COMPILE_FLAGS += - -EBIN = . - -# ---------------------------------------------------- -# Targets -# ---------------------------------------------------- - -make_emakefile: - $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) -o$(EBIN) $(MODULES) \ - > $(EMAKEFILE) - $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) -o$(EBIN) '*_SUITE_make' \ - >> $(EMAKEFILE) - -tests debug opt: make_emakefile - erl $(ERL_MAKE_FLAGS) -make - -clean: - rm -f $(EMAKEFILE) - rm -f $(TARGET_FILES) $(GEN_FILES) - rm -f core - -docs: - -# ---------------------------------------------------- -# Release Target -# ---------------------------------------------------- -include $(ERL_TOP)/make/otp_release_targets.mk - -release_spec: opt - -release_tests_spec: make_emakefile - $(INSTALL_DIR) "$(RELSYSDIR)" - $(INSTALL_DATA) $(EMAKEFILE) $(ERL_FILES) "$(RELSYSDIR)" - $(INSTALL_DATA) typer.spec "$(RELSYSDIR)" - chmod -R u+w "$(RELSYSDIR)" - -release_docs_spec: diff --git a/lib/typer/test/typer.spec b/lib/typer/test/typer.spec deleted file mode 100644 index 79f51b6781..0000000000 --- a/lib/typer/test/typer.spec +++ /dev/null @@ -1 +0,0 @@ -{suites,"../typer_test",all}. diff --git a/lib/typer/test/typer_SUITE.erl b/lib/typer/test/typer_SUITE.erl deleted file mode 100644 index 25f0229640..0000000000 --- a/lib/typer/test/typer_SUITE.erl +++ /dev/null @@ -1,57 +0,0 @@ -%% ``Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% The Initial Developer of the Original Code is Ericsson Utvecklings AB. -%% Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings -%% AB. All Rights Reserved.'' -%% --module(typer_SUITE). - --compile([export_all]). --include_lib("common_test/include/ct.hrl"). - -suite() -> - [{ct_hooks, [ts_install_cth]}]. - -all() -> - case application:ensure_all_started(typer) of - {ok, Apps} -> - [application:stop(App) || App <- lists:reverse(Apps)], - [app, appup]; - _ -> - [appup] - end. - -groups() -> - []. - -init_per_suite(Config) -> - Config. - -end_per_suite(_Config) -> - ok. - -init_per_group(_GroupName, Config) -> - Config. - -end_per_group(_GroupName, Config) -> - Config. - -app() -> - [{doc, "Test that the typer app file is ok"}]. -app(Config) when is_list(Config) -> - ok = ?t:app_test(typer). - -appup() -> - [{doc, "Test that the typer appup file is ok"}]. -appup(Config) when is_list(Config) -> - ok = ?t:appup_test(typer). diff --git a/lib/typer/vsn.mk b/lib/typer/vsn.mk deleted file mode 100644 index ed12e067c1..0000000000 --- a/lib/typer/vsn.mk +++ /dev/null @@ -1 +0,0 @@ -TYPER_VSN = 0.9.11 diff --git a/lib/wx/c_src/wxe_impl.cpp b/lib/wx/c_src/wxe_impl.cpp index 0d2da5d4a7..05d56667ab 100644 --- a/lib/wx/c_src/wxe_impl.cpp +++ b/lib/wx/c_src/wxe_impl.cpp @@ -128,7 +128,7 @@ bool WxeApp::OnInit() delayed_cleanup = new wxList; wxe_ps_init2(); - // wxIdleEvent::SetMode(wxIDLE_PROCESS_SPECIFIED); // Hmm printpreview doesn't work in 2.9 with this + wxIdleEvent::SetMode(wxIDLE_PROCESS_SPECIFIED); Connect(wxID_ANY, wxEVT_IDLE, (wxObjectEventFunction) (wxEventFunction) &WxeApp::idle); Connect(CREATE_PORT, wxeEVT_META_COMMAND,(wxObjectEventFunction) (wxEventFunction) &WxeApp::newMemEnv); @@ -200,7 +200,8 @@ void WxeApp::OnAssertFailure(const wxChar *file, int line, const wxChar *cfunc, // Called by wx thread void WxeApp::idle(wxIdleEvent& event) { event.Skip(true); - dispatch_cmds(); + if(dispatch_cmds()) + event.RequestMore(); } /* ************************************************************ @@ -233,14 +234,15 @@ void handle_event_callback(ErlDrvPort port, ErlDrvTermData process) } } -void WxeApp::dispatch_cmds() +int WxeApp::dispatch_cmds() { + int more = 0; if(wxe_status != WXE_INITIATED) - return; + return more; recurse_level++; // fprintf(stderr, "\r\ndispatch_normal %d\r\n", recurse_level);fflush(stderr); wxe_queue->cb_start = 0; - dispatch(wxe_queue); + more = dispatch(wxe_queue); // fprintf(stderr, "\r\ndispatch_done %d\r\n", recurse_level);fflush(stderr); recurse_level--; @@ -262,12 +264,14 @@ void WxeApp::dispatch_cmds() delete event; } } + return more; } int WxeApp::dispatch(wxeFifo * batch) { int ping = 0; int blevel = 0; + int wait = 0; // Let event handling generate events sometime wxeCommand *event; erl_drv_mutex_lock(wxe_batch_locker_m); while(true) { @@ -275,10 +279,10 @@ int WxeApp::dispatch(wxeFifo * batch) erl_drv_mutex_unlock(wxe_batch_locker_m); switch(event->op) { case WXE_BATCH_END: - {--blevel; } + if(blevel>0) blevel--; break; case WXE_BATCH_BEGIN: - {blevel++; } + blevel++; break; case WXE_DEBUG_PING: // When in debugger we don't want to hang waiting for a BATCH_END @@ -293,7 +297,7 @@ int WxeApp::dispatch(wxeFifo * batch) memcpy(cb_buff, event->buffer, event->len); } event->Delete(); - return blevel; + return 1; default: if(event->op < OPENGL_START) { // fprintf(stderr, " c %d (%d) \r\n", event->op, blevel); @@ -307,13 +311,15 @@ int WxeApp::dispatch(wxeFifo * batch) erl_drv_mutex_lock(wxe_batch_locker_m); batch->Cleanup(); } - if(blevel <= 0) { + if(blevel <= 0 || wait > 3) { erl_drv_mutex_unlock(wxe_batch_locker_m); - return blevel; + if(blevel > 0) return 1; // We are still in a batch but we can let wx check for events + else return 0; } // sleep until something happens - //fprintf(stderr, "%s:%d sleep %d %d\r\n", __FILE__, __LINE__, batch->m_n, blevel);fflush(stderr); + // fprintf(stderr, "%s:%d sleep %d %d %d\r\n", __FILE__, __LINE__, batch->m_n, blevel, wait);fflush(stderr); wxe_needs_signal = 1; + wait += 1; while(batch->m_n == 0) { erl_drv_cond_wait(wxe_batch_locker_c, wxe_batch_locker_m); } diff --git a/lib/wx/c_src/wxe_impl.h b/lib/wx/c_src/wxe_impl.h index 57dac997ab..68f5deb336 100644 --- a/lib/wx/c_src/wxe_impl.h +++ b/lib/wx/c_src/wxe_impl.h @@ -73,7 +73,7 @@ public: void wxe_dispatch(wxeCommand& event); void idle(wxIdleEvent& event); - void dispatch_cmds(); + int dispatch_cmds(); void dummy_close(wxEvent& Ev); bool sendevent(wxEvent *event); diff --git a/lib/wx/examples/demo/demo.erl b/lib/wx/examples/demo/demo.erl index 8b7412017a..0258202a67 100644 --- a/lib/wx/examples/demo/demo.erl +++ b/lib/wx/examples/demo/demo.erl @@ -243,6 +243,9 @@ handle_event(#wx{id = Id, %% If you are going to printout mainly text it is easier if %% you generate HTML code and use a wxHtmlEasyPrint %% instead of using DCs + + %% Printpreview doesn't work in >2.9 without this + wxIdleEvent:setMode(?wxIDLE_PROCESS_ALL), Module = "ex_" ++ wxListBox:getStringSelection(State#state.selector) ++ ".erl", HEP = wxHtmlEasyPrinting:new([{name, "Print"}, {parentWindow, State#state.win}]), diff --git a/lib/wx/src/wxe_master.erl b/lib/wx/src/wxe_master.erl index e17a3327ac..913bf4d41b 100644 --- a/lib/wx/src/wxe_master.erl +++ b/lib/wx/src/wxe_master.erl @@ -82,8 +82,14 @@ init_port(SilentStart) -> %% Initalizes the opengl library %%-------------------------------------------------------------------- init_opengl() -> - GLLib = wxe_util:wxgl_dl(), - wxe_util:call(?WXE_INIT_OPENGL, <<(list_to_binary(GLLib))/binary, 0:8>>). + case get(wx_init_opengl) of + true -> {ok, "already initialized"}; + _ -> + GLLib = wxe_util:wxgl_dl(), + Res = wxe_util:call(?WXE_INIT_OPENGL, <<(list_to_binary(GLLib))/binary, 0:8>>), + element(1, Res) =:= ok andalso put(wx_init_opengl, true), + Res + end. %%-------------------------------------------------------------------- %% Fetch early messages, hack to get start up args on mac diff --git a/lib/xmerl/doc/src/notes.xml b/lib/xmerl/doc/src/notes.xml index 12e64537ed..652560f60c 100644 --- a/lib/xmerl/doc/src/notes.xml +++ b/lib/xmerl/doc/src/notes.xml @@ -32,6 +32,61 @@ <p>This document describes the changes made to the Xmerl application.</p> +<section><title>Xmerl 1.3.13</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The namespace_conformant option in xmerl_scan did not + work when parsing documents without explicit XML + namespace declaration.</p> + <p> + Own Id: OTP-14139</p> + </item> + <item> + <p> Fix a "well-formedness" bug in the XML Sax parser so + it returns an error if there are something more in the + file after the matching document. If one using the + xmerl_sax_parser:stream() a rest is allowed which then + can be sent to a new call of xmerl_sax_parser:stream() to + parse next document. </p> <p> This is done to be + compliant with XML conformance tests. </p> + <p> + Own Id: OTP-14211</p> + </item> + <item> + <p> Fixed compiler and dialyzer warnings in the XML SAX + parser. </p> + <p> + Own Id: OTP-14212</p> + </item> + <item> + <p> Change how to interpret end of document in the XML + SAX parser to comply with Tim Brays comment on the + standard. This makes it possible to handle more than one + doc on a stream, the standard makes it impossible to know + when the document is ended without waiting for the next + document (and not always even that). </p> <p> Tim Brays + comment: </p> <p> Trailing "Misc"<br/> The fact that + you're allowed some trailing junk after the root element, + I decided (but unfortunately too late) is a real design + error in XML. If I'm writing a network client, I'm + probably going to close the link as soon as a I see the + root element end-tag, and not depend on the other end + closing it down properly.<br/> Furthermore, if I want to + send a succession of XML documents over a network link, + if I find a processing instruction after a root element, + is it a trailer on the previous document, or part of the + prolog of the next? </p> + <p> + Own Id: OTP-14213</p> + </item> + </list> + </section> + +</section> + <section><title>Xmerl 1.3.12</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/xmerl/src/xmerl_sax_parser_base.erlsrc b/lib/xmerl/src/xmerl_sax_parser_base.erlsrc index 2b9b37b5f3..f3470b2809 100644 --- a/lib/xmerl/src/xmerl_sax_parser_base.erlsrc +++ b/lib/xmerl/src/xmerl_sax_parser_base.erlsrc @@ -145,10 +145,11 @@ parse_dtd(Xml, State) -> %% [1] document ::= prolog element Misc* %%---------------------------------------------------------------------- parse_document(Rest, State) when is_record(State, xmerl_sax_parser_state) -> - {Rest1, State1} = parse_xml_decl(Rest, State), + {Rest1, State1} = parse_byte_order_mark(Rest, State), {Rest2, State2} = parse_misc(Rest1, State1, true), {ok, Rest2, State2}. +?PARSE_BYTE_ORDER_MARK(Bytes, State). %%---------------------------------------------------------------------- %% Function: parse_xml_decl(Rest, State) -> Result @@ -159,15 +160,8 @@ parse_document(Rest, State) when is_record(State, xmerl_sax_parser_state) -> %% [22] prolog ::= XMLDecl? Misc* (doctypedecl Misc*)? %% [23] XMLDecl ::= '<?xml' VersionInfo EncodingDecl? SDDecl? S? '?>' %%---------------------------------------------------------------------- --dialyzer({[no_fail_call, no_match], parse_xml_decl/2}). parse_xml_decl(?STRING_EMPTY, State) -> cf(?STRING_EMPTY, State, fun parse_xml_decl/2); -parse_xml_decl(?BYTE_ORDER_MARK_1, State) -> - cf(?BYTE_ORDER_MARK_1, State, fun parse_xml_decl/2); -parse_xml_decl(?BYTE_ORDER_MARK_2, State) -> - cf(?BYTE_ORDER_MARK_2, State, fun parse_xml_decl/2); -parse_xml_decl(?BYTE_ORDER_MARK_REST(Rest), State) -> - cf(Rest, State, fun parse_xml_decl/2); parse_xml_decl(?STRING("<") = Bytes, State) -> cf(Bytes, State, fun parse_xml_decl/2); parse_xml_decl(?STRING("<?") = Bytes, State) -> @@ -179,31 +173,19 @@ parse_xml_decl(?STRING("<?xm") = Bytes, State) -> parse_xml_decl(?STRING("<?xml") = Bytes, State) -> cf(Bytes, State, fun parse_xml_decl/2); parse_xml_decl(?STRING_REST("<?xml", Rest1), State) -> - parse_xml_decl_1(Rest1, State); -parse_xml_decl(Bytes, #xmerl_sax_parser_state{encoding=Enc} = State) when is_binary(Bytes) -> - case unicode:characters_to_list(Bytes, Enc) of - {incomplete, _, _} -> - cf(Bytes, State, fun parse_xml_decl/2); - {error, _Encoded, _Rest} -> - ?fatal_error(State, lists:flatten(io_lib:format("Bad character, not in ~p\n", [Enc]))); - _ -> - parse_prolog(Bytes, State) - end; -parse_xml_decl(Bytes, State) -> - parse_prolog(Bytes, State). - + parse_xml_decl_rest(Rest1, State); +?PARSE_XML_DECL(Bytes, State). -parse_xml_decl_1(?STRING_UNBOUND_REST(C, Rest) = Bytes, State) -> +parse_xml_decl_rest(?STRING_UNBOUND_REST(C, Rest) = Bytes, State) -> if ?is_whitespace(C) -> {_XmlAttributes, Rest1, State1} = parse_version_info(Rest, State, []), - %State2 = event_callback({processingInstruction, "xml", XmlAttributes}, State1),% The XML decl. should not be reported as a PI parse_prolog(Rest1, State1); true -> parse_prolog(?STRING_REST("<?xml", Bytes), State) end; -parse_xml_decl_1(Bytes, State) -> - unicode_incomplete_check([Bytes, State, fun parse_xml_decl_1/2], undefined). +parse_xml_decl_rest(Bytes, State) -> + unicode_incomplete_check([Bytes, State, fun parse_xml_decl_rest/2], undefined). @@ -225,8 +207,6 @@ parse_prolog(?STRING_REST("<?", Rest), State) -> parse_prolog(Rest1, State1); {endDocument, Rest1, State1} -> parse_prolog(Rest1, State1) - % IValue = ?TO_INPUT_FORMAT("<?"), - % {?APPEND_STRING(IValue, Rest1), State1} end; parse_prolog(?STRING_REST("<!", Rest), State) -> parse_prolog_1(Rest, State); @@ -239,7 +219,6 @@ parse_prolog(Bytes, State) -> unicode_incomplete_check([Bytes, State, fun parse_prolog/2], "expecting < or whitespace"). - parse_prolog_1(?STRING_EMPTY, State) -> cf(?STRING_EMPTY, State, fun parse_prolog_1/2); parse_prolog_1(?STRING("D") = Bytes, State) -> @@ -1232,7 +1211,6 @@ send_character_event(_, true, String, State) -> %% Description: Parse whitespaces. %% [3] S ::= (#x20 | #x9 | #xD | #xA)+ %%---------------------------------------------------------------------- --dialyzer({no_fail_call, whitespace/3}). whitespace(?STRING_EMPTY, State, Acc) -> case cf(?STRING_EMPTY, State, Acc, fun whitespace/3) of {?STRING_EMPTY, State} -> @@ -1258,16 +1236,7 @@ whitespace(?STRING_REST("\r", Rest), State, Acc) -> whitespace(Rest, State#xmerl_sax_parser_state{line_no=N+1}, [?lf |Acc]); whitespace(?STRING_UNBOUND_REST(C, Rest), State, Acc) when ?is_whitespace(C) -> whitespace(Rest, State, [C|Acc]); -whitespace(?STRING_UNBOUND_REST(_C, _) = Bytes, State, Acc) -> - {lists:reverse(Acc), Bytes, State}; -whitespace(Bytes, #xmerl_sax_parser_state{encoding=Enc} = State, Acc) when is_binary(Bytes) -> - case unicode:characters_to_list(Bytes, Enc) of - {incomplete, _, _} -> - cf(Bytes, State, Acc, fun whitespace/3); - {error, _Encoded, _Rest} -> - ?fatal_error(State, lists:flatten(io_lib:format("Bad character, not in ~p\n", [Enc]))) - end. - +?WHITESPACE(Bytes, State, Acc). %%---------------------------------------------------------------------- %% Function: parse_reference(Rest, State, HaveToExist) -> Result @@ -1390,7 +1359,6 @@ parse_pe_reference_1(Bytes, State, Name) -> "missing ; after reference " ++ Name). - %%---------------------------------------------------------------------- %% Function: insert_reference(Reference, State) -> Result %% Parameters: Reference = string() @@ -1406,7 +1374,6 @@ insert_reference({Name, Type, Value}, Table) -> end. - %%---------------------------------------------------------------------- %% Function: look_up_reference(Reference, State) -> Result %% Parameters: Reference = string() @@ -1721,7 +1688,7 @@ handle_external_entity({http, Url}, State) -> ++ file:format_error(Reason)); {ok, FD} -> {?STRING_EMPTY, EntityState} = - parse_external_entity_1(<<>>, + parse_external_entity_byte_order_mark(<<>>, State#xmerl_sax_parser_state{continuation_state=FD, current_location=filename:dirname(Url), entity=filename:basename(Url), @@ -1737,6 +1704,8 @@ handle_external_entity({http, Url}, State) -> handle_external_entity({Tag, _Url}, State) -> ?fatal_error(State, "Unsupported URI type: " ++ atom_to_list(Tag)). +?PARSE_EXTERNAL_ENTITY_BYTE_ORDER_MARK(Bytes, State). + %%---------------------------------------------------------------------- %% Function : parse_external_entity_1(Rest, State) -> Result %% Parameters: Rest = string() | binary() @@ -1744,7 +1713,6 @@ handle_external_entity({Tag, _Url}, State) -> %% Result : {Rest, State} %% Description: Parse the external entity. %%---------------------------------------------------------------------- --dialyzer({[no_fail_call, no_match], parse_external_entity_1/2}). parse_external_entity_1(?STRING_EMPTY, #xmerl_sax_parser_state{file_type=Type} = State) -> case catch cf(?STRING_EMPTY, State, fun parse_external_entity_1/2) of {Rest, State1} when is_record(State1, xmerl_sax_parser_state) -> @@ -1754,12 +1722,6 @@ parse_external_entity_1(?STRING_EMPTY, #xmerl_sax_parser_state{file_type=Type} = Other -> throw(Other) end; -parse_external_entity_1(?BYTE_ORDER_MARK_1, State) -> - cf(?BYTE_ORDER_MARK_1, State, fun parse_external_entity_1/2); -parse_external_entity_1(?BYTE_ORDER_MARK_2, State) -> - cf(?BYTE_ORDER_MARK_2, State, fun parse_external_entity_1/2); -parse_external_entity_1(?BYTE_ORDER_MARK_REST(Rest), State) -> - parse_external_entity_1(Rest, State); parse_external_entity_1(?STRING("<") = Bytes, State) -> cf(Bytes, State, fun parse_external_entity_1/2); parse_external_entity_1(?STRING("<?") = Bytes, State) -> diff --git a/lib/xmerl/src/xmerl_sax_parser_latin1.erlsrc b/lib/xmerl/src/xmerl_sax_parser_latin1.erlsrc index 961806bf4c..6e59347fb8 100644 --- a/lib/xmerl/src/xmerl_sax_parser_latin1.erlsrc +++ b/lib/xmerl/src/xmerl_sax_parser_latin1.erlsrc @@ -2,7 +2,7 @@ %%-------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2016. All Rights Reserved. +%% Copyright Ericsson AB 2008-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -34,8 +34,36 @@ -define(APPEND_STRING(Rest, New), <<Rest/binary, New/binary>>). -define(TO_INPUT_FORMAT(Val), unicode:characters_to_binary(Val, unicode, latin1)). -%% STRING_REST and STRING_UNBOUND_REST is only different in the list case -define(STRING_UNBOUND_REST(MatchChar, Rest), <<MatchChar, Rest/binary>>). --define(BYTE_ORDER_MARK_1, undefined_bom1). --define(BYTE_ORDER_MARK_2, undefined_bom2). --define(BYTE_ORDER_MARK_REST(Rest), <<undefined, Rest/binary>>). + +-define(PARSE_BYTE_ORDER_MARK(Bytes, State), + parse_byte_order_mark(Bytes, State) -> + parse_xml_decl(Bytes, State)). + +-define(PARSE_XML_DECL(Bytes, State), + parse_xml_decl(Bytes, #xmerl_sax_parser_state{encoding=Enc} = State) when is_binary(Bytes) -> + case unicode:characters_to_list(Bytes, Enc) of + {incomplete, _, _} -> + cf(Bytes, State, fun parse_xml_decl/2); + {error, _Encoded, _Rest} -> + ?fatal_error(State, lists:flatten(io_lib:format("Bad character, not in ~p\n", [Enc]))); + _ -> + parse_prolog(Bytes, State) + end; + parse_xml_decl(Bytes, State) -> + parse_prolog(Bytes, State)). + +-define(WHITESPACE(Bytes, State, Acc), + whitespace(?STRING_UNBOUND_REST(_C, _) = Bytes, State, Acc) -> + {lists:reverse(Acc), Bytes, State}; + whitespace(Bytes, #xmerl_sax_parser_state{encoding=Enc} = State, Acc) when is_binary(Bytes) -> + case unicode:characters_to_list(Bytes, Enc) of + {incomplete, _, _} -> + cf(Bytes, State, Acc, fun whitespace/3); + {error, _Encoded, _Rest} -> + ?fatal_error(State, lists:flatten(io_lib:format("Bad character, not in ~p\n", [Enc]))) + end). + +-define(PARSE_EXTERNAL_ENTITY_BYTE_ORDER_MARK(Bytes, State), + parse_external_entity_byte_order_mark(Bytes, State) -> + parse_external_entity_1(Bytes, State)). diff --git a/lib/xmerl/src/xmerl_sax_parser_list.erlsrc b/lib/xmerl/src/xmerl_sax_parser_list.erlsrc index 624a621d92..6a4435b1d9 100644 --- a/lib/xmerl/src/xmerl_sax_parser_list.erlsrc +++ b/lib/xmerl/src/xmerl_sax_parser_list.erlsrc @@ -36,6 +36,19 @@ %% In the list case we can't use a '++' when matchin against an unbound variable -define(STRING_UNBOUND_REST(MatchChar, Rest), [MatchChar | Rest]). --define(BYTE_ORDER_MARK_1, undefined_bom1). --define(BYTE_ORDER_MARK_2, undefined_bom2). --define(BYTE_ORDER_MARK_REST(Rest), [undefined|Rest]). + +-define(PARSE_BYTE_ORDER_MARK(Bytes, State), + parse_byte_order_mark(Bytes, State) -> + parse_xml_decl(Bytes, State)). + +-define(PARSE_XML_DECL(Bytes, State), + parse_xml_decl(Bytes, State) -> + parse_prolog(Bytes, State)). + +-define(WHITESPACE(Bytes, State, Acc), + whitespace(?STRING_UNBOUND_REST(_C, _) = Bytes, State, Acc) -> + {lists:reverse(Acc), Bytes, State}). + +-define(PARSE_EXTERNAL_ENTITY_BYTE_ORDER_MARK(Bytes, State), + parse_external_entity_byte_order_mark(Bytes, State) -> + parse_external_entity_1(Bytes, State)). diff --git a/lib/xmerl/src/xmerl_sax_parser_utf16be.erlsrc b/lib/xmerl/src/xmerl_sax_parser_utf16be.erlsrc index ff84ece97a..ec89024729 100644 --- a/lib/xmerl/src/xmerl_sax_parser_utf16be.erlsrc +++ b/lib/xmerl/src/xmerl_sax_parser_utf16be.erlsrc @@ -2,7 +2,7 @@ %%-------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2016. All Rights Reserved. +%% Copyright Ericsson AB 2008-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -34,8 +34,50 @@ -define(APPEND_STRING(Rest, New), <<Rest/binary, New/binary>>). -define(TO_INPUT_FORMAT(Val), unicode:characters_to_binary(Val, unicode, {utf16, big})). -%% STRING_REST and STRING_UNBOUND_REST is only different in the list case -define(STRING_UNBOUND_REST(MatchChar, Rest), <<MatchChar/big-utf16, Rest/binary>>). --define(BYTE_ORDER_MARK_1, undefined_bom1). --define(BYTE_ORDER_MARK_2, <<16#FE>>). +-define(BYTE_ORDER_MARK_1, <<16#FE>>). -define(BYTE_ORDER_MARK_REST(Rest), <<16#FE, 16#FF, Rest/binary>>). + +-define(PARSE_BYTE_ORDER_MARK(Bytes, State), + parse_byte_order_mark(?STRING_EMPTY, State) -> + cf(?STRING_EMPTY, State, fun parse_byte_order_mark/2); + parse_byte_order_mark(?BYTE_ORDER_MARK_1, State) -> + cf(?BYTE_ORDER_MARK_1, State, fun parse_byte_order_mark/2); + parse_byte_order_mark(?BYTE_ORDER_MARK_REST(Rest), State) -> + parse_xml_decl(Rest, State); + parse_byte_order_mark(Bytes, State) -> + parse_xml_decl(Bytes, State)). + +-define(PARSE_XML_DECL(Bytes, State), + parse_xml_decl(Bytes, #xmerl_sax_parser_state{encoding=Enc} = State) when is_binary(Bytes) -> + case unicode:characters_to_list(Bytes, Enc) of + {incomplete, _, _} -> + cf(Bytes, State, fun parse_xml_decl/2); + {error, _Encoded, _Rest} -> + ?fatal_error(State, lists:flatten(io_lib:format("Bad character, not in ~p\n", [Enc]))); + _ -> + parse_prolog(Bytes, State) + end; + parse_xml_decl(Bytes, State) -> + parse_prolog(Bytes, State)). + +-define(WHITESPACE(Bytes, State, Acc), + whitespace(?STRING_UNBOUND_REST(_C, _) = Bytes, State, Acc) -> + {lists:reverse(Acc), Bytes, State}; + whitespace(Bytes, #xmerl_sax_parser_state{encoding=Enc} = State, Acc) when is_binary(Bytes) -> + case unicode:characters_to_list(Bytes, Enc) of + {incomplete, _, _} -> + cf(Bytes, State, Acc, fun whitespace/3); + {error, _Encoded, _Rest} -> + ?fatal_error(State, lists:flatten(io_lib:format("Bad character, not in ~p\n", [Enc]))) + end). + +-define(PARSE_EXTERNAL_ENTITY_BYTE_ORDER_MARK(Bytes, State), + parse_external_entity_byte_order_mark(?STRING_EMPTY, State) -> + cf(?STRING_EMPTY, State, fun parse_external_entity_byte_order_mark/2); + parse_external_entity_byte_order_mark(?BYTE_ORDER_MARK_1, State) -> + cf(?BYTE_ORDER_MARK_1, State, fun parse_external_entity_byte_order_mark/2); + parse_external_entity_byte_order_mark(?BYTE_ORDER_MARK_REST(Rest), State) -> + parse_external_entity_1(Rest, State); + parse_external_entity_byte_order_mark(Bytes, State) -> + parse_external_entity_1(Bytes, State)). diff --git a/lib/xmerl/src/xmerl_sax_parser_utf16le.erlsrc b/lib/xmerl/src/xmerl_sax_parser_utf16le.erlsrc index a330fce8d0..566333a045 100644 --- a/lib/xmerl/src/xmerl_sax_parser_utf16le.erlsrc +++ b/lib/xmerl/src/xmerl_sax_parser_utf16le.erlsrc @@ -2,7 +2,7 @@ %%-------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2016. All Rights Reserved. +%% Copyright Ericsson AB 2008-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -34,8 +34,50 @@ -define(APPEND_STRING(Rest, New), <<Rest/binary, New/binary>>). -define(TO_INPUT_FORMAT(Val), unicode:characters_to_binary(Val, unicode, {utf16, little})). -%% STRING_REST and STRING_UNBOUND_REST is only different in the list case -define(STRING_UNBOUND_REST(MatchChar, Rest), <<MatchChar/little-utf16, Rest/binary>>). --define(BYTE_ORDER_MARK_1, undefined_bom1). --define(BYTE_ORDER_MARK_2, <<16#FF>>). +-define(BYTE_ORDER_MARK_1, <<16#FF>>). -define(BYTE_ORDER_MARK_REST(Rest), <<16#FF, 16#FE, Rest/binary>>). + +-define(PARSE_BYTE_ORDER_MARK(Bytes, State), + parse_byte_order_mark(?STRING_EMPTY, State) -> + cf(?STRING_EMPTY, State, fun parse_byte_order_mark/2); + parse_byte_order_mark(?BYTE_ORDER_MARK_1, State) -> + cf(?BYTE_ORDER_MARK_1, State, fun parse_byte_order_mark/2); + parse_byte_order_mark(?BYTE_ORDER_MARK_REST(Rest), State) -> + parse_xml_decl(Rest, State); + parse_byte_order_mark(Bytes, State) -> + parse_xml_decl(Bytes, State)). + +-define(PARSE_XML_DECL(Bytes, State), + parse_xml_decl(Bytes, #xmerl_sax_parser_state{encoding=Enc} = State) when is_binary(Bytes) -> + case unicode:characters_to_list(Bytes, Enc) of + {incomplete, _, _} -> + cf(Bytes, State, fun parse_xml_decl/2); + {error, _Encoded, _Rest} -> + ?fatal_error(State, lists:flatten(io_lib:format("Bad character, not in ~p\n", [Enc]))); + _ -> + parse_prolog(Bytes, State) + end; + parse_xml_decl(Bytes, State) -> + parse_prolog(Bytes, State)). + +-define(WHITESPACE(Bytes, State, Acc), + whitespace(?STRING_UNBOUND_REST(_C, _) = Bytes, State, Acc) -> + {lists:reverse(Acc), Bytes, State}; + whitespace(Bytes, #xmerl_sax_parser_state{encoding=Enc} = State, Acc) when is_binary(Bytes) -> + case unicode:characters_to_list(Bytes, Enc) of + {incomplete, _, _} -> + cf(Bytes, State, Acc, fun whitespace/3); + {error, _Encoded, _Rest} -> + ?fatal_error(State, lists:flatten(io_lib:format("Bad character, not in ~p\n", [Enc]))) + end). + +-define(PARSE_EXTERNAL_ENTITY_BYTE_ORDER_MARK(Bytes, State), + parse_external_entity_byte_order_mark(?STRING_EMPTY, State) -> + cf(?STRING_EMPTY, State, fun parse_external_entity_byte_order_mark/2); + parse_external_entity_byte_order_mark(?BYTE_ORDER_MARK_1, State) -> + cf(?BYTE_ORDER_MARK_1, State, fun parse_external_entity_byte_order_mark/2); + parse_external_entity_byte_order_mark(?BYTE_ORDER_MARK_REST(Rest), State) -> + parse_external_entity_1(Rest, State); + parse_external_entity_byte_order_mark(Bytes, State) -> + parse_external_entity_1(Bytes, State)). diff --git a/lib/xmerl/src/xmerl_sax_parser_utf8.erlsrc b/lib/xmerl/src/xmerl_sax_parser_utf8.erlsrc index d46d60d237..f41d06d013 100644 --- a/lib/xmerl/src/xmerl_sax_parser_utf8.erlsrc +++ b/lib/xmerl/src/xmerl_sax_parser_utf8.erlsrc @@ -2,7 +2,7 @@ %%-------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2016. All Rights Reserved. +%% Copyright Ericsson AB 2008-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -34,11 +34,55 @@ -define(APPEND_STRING(Rest, New), <<Rest/binary, New/binary>>). -define(TO_INPUT_FORMAT(Val), unicode:characters_to_binary(Val, unicode, utf8)). - -%% STRING_REST and STRING_UNBOUND_REST is only different in the list case -define(STRING_UNBOUND_REST(MatchChar, Rest), <<MatchChar/utf8, Rest/binary>>). -define(BYTE_ORDER_MARK_1, <<16#EF>>). -define(BYTE_ORDER_MARK_2, <<16#EF, 16#BB>>). -define(BYTE_ORDER_MARK_REST(Rest), <<16#EF, 16#BB, 16#BF, Rest/binary>>). +-define(PARSE_BYTE_ORDER_MARK(Bytes, State), + parse_byte_order_mark(?STRING_EMPTY, State) -> + cf(?STRING_EMPTY, State, fun parse_byte_order_mark/2); + parse_byte_order_mark(?BYTE_ORDER_MARK_1, State) -> + cf(?BYTE_ORDER_MARK_1, State, fun parse_byte_order_mark/2); + parse_byte_order_mark(?BYTE_ORDER_MARK_2, State) -> + cf(?BYTE_ORDER_MARK_2, State, fun parse_byte_order_mark/2); + parse_byte_order_mark(?BYTE_ORDER_MARK_REST(Rest), State) -> + parse_xml_decl(Rest, State); + parse_byte_order_mark(Bytes, State) -> + parse_xml_decl(Bytes, State)). + +-define(PARSE_XML_DECL(Bytes, State), + parse_xml_decl(Bytes, #xmerl_sax_parser_state{encoding=Enc} = State) when is_binary(Bytes) -> + case unicode:characters_to_list(Bytes, Enc) of + {incomplete, _, _} -> + cf(Bytes, State, fun parse_xml_decl/2); + {error, _Encoded, _Rest} -> + ?fatal_error(State, lists:flatten(io_lib:format("Bad character, not in ~p\n", [Enc]))); + _ -> + parse_prolog(Bytes, State) + end; + parse_xml_decl(Bytes, State) -> + parse_prolog(Bytes, State)). + +-define(WHITESPACE(Bytes, State, Acc), + whitespace(?STRING_UNBOUND_REST(_C, _) = Bytes, State, Acc) -> + {lists:reverse(Acc), Bytes, State}; + whitespace(Bytes, #xmerl_sax_parser_state{encoding=Enc} = State, Acc) when is_binary(Bytes) -> + case unicode:characters_to_list(Bytes, Enc) of + {incomplete, _, _} -> + cf(Bytes, State, Acc, fun whitespace/3); + {error, _Encoded, _Rest} -> + ?fatal_error(State, lists:flatten(io_lib:format("Bad character, not in ~p\n", [Enc]))) + end). +-define(PARSE_EXTERNAL_ENTITY_BYTE_ORDER_MARK(Bytes, State), + parse_external_entity_byte_order_mark(?STRING_EMPTY, State) -> + cf(?STRING_EMPTY, State, fun parse_external_entity_byte_order_mark/2); + parse_external_entity_byte_order_mark(?BYTE_ORDER_MARK_1, State) -> + cf(?BYTE_ORDER_MARK_1, State, fun parse_external_entity_byte_order_mark/2); + parse_external_entity_byte_order_mark(?BYTE_ORDER_MARK_2, State) -> + cf(?BYTE_ORDER_MARK_2, State, fun parse_external_entity_byte_order_mark/2); + parse_external_entity_byte_order_mark(?BYTE_ORDER_MARK_REST(Rest), State) -> + parse_external_entity_1(Rest, State); + parse_external_entity_byte_order_mark(Bytes, State) -> + parse_external_entity_1(Bytes, State)). diff --git a/lib/xmerl/vsn.mk b/lib/xmerl/vsn.mk index 95adaa5bb0..1515a4e37d 100644 --- a/lib/xmerl/vsn.mk +++ b/lib/xmerl/vsn.mk @@ -1 +1 @@ -XMERL_VSN = 1.3.12 +XMERL_VSN = 1.3.13 |