diff options
Diffstat (limited to 'lib')
46 files changed, 2223 insertions, 909 deletions
diff --git a/lib/compiler/src/sys_core_fold.erl b/lib/compiler/src/sys_core_fold.erl index 43ce9a7172..ab67c8164b 100644 --- a/lib/compiler/src/sys_core_fold.erl +++ b/lib/compiler/src/sys_core_fold.erl @@ -277,7 +277,7 @@ expr(#c_fun{}=Fun, effect, _) -> add_warning(Fun, useless_building), void(); expr(#c_fun{vars=Vs0,body=B0}=Fun, Ctxt0, Sub0) -> - {Vs1,Sub1} = pattern_list(Vs0, Sub0), + {Vs1,Sub1} = var_list(Vs0, Sub0), Ctxt = case Ctxt0 of {letrec,Ctxt1} -> Ctxt1; value -> value @@ -420,13 +420,13 @@ expr(#c_try{anno=A,arg=E0,vars=Vs0,body=B0,evars=Evs0,handler=H0}=Try, _, Sub0) %% Here is the general try/catch construct outside of guards. %% We can remove try if the value is simple and replace it with a let. E1 = body(E0, value, Sub0), - {Vs1,Sub1} = pattern_list(Vs0, Sub0), + {Vs1,Sub1} = var_list(Vs0, Sub0), B1 = body(B0, value, Sub1), case is_safe_simple(E1, Sub0) of true -> expr(#c_let{anno=A,vars=Vs1,arg=E1,body=B1}, value, Sub0); false -> - {Evs1,Sub2} = pattern_list(Evs0, Sub0), + {Evs1,Sub2} = var_list(Evs0, Sub0), H1 = body(H0, value, Sub2), Try#c_try{arg=E1,vars=Vs1,body=B1,evars=Evs1,handler=H1} end. @@ -1078,15 +1078,28 @@ is_atom_or_var(_) -> false. %% clause(Clause, Cepxr, Context, Sub) -> Clause. -clause(#c_clause{pats=Ps0,guard=G0,body=B0}=Cl, Cexpr, Ctxt, Sub0) -> - {Ps1,Sub1} = pattern_list(Ps0, Sub0), +clause(#c_clause{pats=Ps0}=Cl, Cexpr, Ctxt, Sub0) -> + try pattern_list(Ps0, Sub0) of + {Ps1,Sub1} -> + clause_1(Cl, Ps1, Cexpr, Ctxt, Sub1) + catch + nomatch -> + Cl#c_clause{anno=[compiler_generated], + guard=#c_literal{val=false}} + end. + +clause_1(#c_clause{guard=G0,body=B0}=Cl, Ps1, Cexpr, Ctxt, Sub1) -> Sub2 = update_types(Cexpr, Ps1, Sub1), - GSub = case {Cexpr,Ps1} of - {#c_var{name='_'},_} -> + GSub = case {Cexpr,Ps1,G0} of + {_,_,#c_literal{}} -> + %% No need for substitution tricks when the guard + %% does not contain any variables. + Sub2; + {#c_var{name='_'},_,_} -> %% In a 'receive', Cexpr is the variable '_', which represents the %% message being matched. We must NOT do any extra substiutions. Sub2; - {#c_var{},[#c_var{}=Var]} -> + {#c_var{},[#c_var{}=Var],_} -> %% The idea here is to optimize expressions such as %% %% case A of A -> ... @@ -1120,7 +1133,7 @@ clause(#c_clause{pats=Ps0,guard=G0,body=B0}=Cl, Cexpr, Ctxt, Sub0) -> %% the unsubstituted variables and values. let_substs(Vs0, As0, Sub0) -> - {Vs1,Sub1} = pattern_list(Vs0, Sub0), + {Vs1,Sub1} = var_list(Vs0, Sub0), {Vs2,As1,Ss} = let_substs_1(Vs1, As0, Sub1), Sub2 = sub_add_scope([V || #c_var{name=V} <- Vs2], Sub1), {Vs2,As1, @@ -1206,20 +1219,132 @@ bin_pattern_list(Ps0, Isub, Osub0) -> {Ps,{_,Osub}} = mapfoldl(fun bin_pattern/2, {Isub,Osub0}, Ps0), {Ps,Osub}. -bin_pattern(#c_bitstr{val=E0,size=Size0}=Pat, {Isub0,Osub0}) -> +bin_pattern(#c_bitstr{val=E0,size=Size0}=Pat0, {Isub0,Osub0}) -> Size1 = expr(Size0, Isub0), {E1,Osub} = pattern(E0, Isub0, Osub0), Isub = case E0 of #c_var{} -> sub_set_var(E0, E1, Isub0); _ -> Isub0 end, - {Pat#c_bitstr{val=E1,size=Size1},{Isub,Osub}}. + Pat = Pat0#c_bitstr{val=E1,size=Size1}, + bin_pat_warn(Pat), + {Pat,{Isub,Osub}}. pattern_list(Ps, Sub) -> pattern_list(Ps, Sub, Sub). pattern_list(Ps0, Isub, Osub0) -> mapfoldl(fun (P, Osub) -> pattern(P, Isub, Osub) end, Osub0, Ps0). +%% var_list([Var], InSub) -> {Pattern,OutSub}. +%% Works like pattern_list/2 but only accept variables and is +%% guaranteed not to throw an exception. + +var_list(Vs, Sub0) -> + mapfoldl(fun (#c_var{}=V, Sub) -> + pattern(V, Sub, Sub) + end, Sub0, Vs). + + +%%% +%%% Generate warnings for binary patterns that will not match. +%%% + +bin_pat_warn(#c_bitstr{type=#c_literal{val=Type}, + val=Val0, + size=#c_literal{val=Sz}, + unit=#c_literal{val=Unit}, + flags=Fl}=Pat) -> + case {Type,Sz} of + {_,_} when is_integer(Sz), Sz >= 0 -> ok; + {binary,all} -> ok; + {utf8,undefined} -> ok; + {utf16,undefined} -> ok; + {utf32,undefined} -> ok; + {_,_} -> + add_warning(Pat, {nomatch_bit_syntax_size,Sz}), + throw(nomatch) + end, + case {Type,Val0} of + {integer,#c_literal{val=Val}} when is_integer(Val) -> + Signedness = signedness(Fl), + TotalSz = Sz * Unit, + bit_pat_warn_int(Val, TotalSz, Signedness, Pat); + {float,#c_literal{val=Val}} when is_float(Val) -> + ok; + {utf8,#c_literal{val=Val}} when is_integer(Val) -> + bit_pat_warn_unicode(Val, Pat); + {utf16,#c_literal{val=Val}} when is_integer(Val) -> + bit_pat_warn_unicode(Val, Pat); + {utf32,#c_literal{val=Val}} when is_integer(Val) -> + bit_pat_warn_unicode(Val, Pat); + {_,#c_literal{val=Val}} -> + add_warning(Pat, {nomatch_bit_syntax_type,Val,Type}), + throw(nomatch); + {_,_} -> + ok + end; +bin_pat_warn(#c_bitstr{type=#c_literal{val=Type},val=Val0,flags=Fl}=Pat) -> + %% Size is variable. Not much that we can check. + case {Type,Val0} of + {integer,#c_literal{val=Val}} when is_integer(Val) -> + case signedness(Fl) of + unsigned when Val < 0 -> + add_warning(Pat, {nomatch_bit_syntax_unsigned,Val}), + throw(nomatch); + _ -> + ok + end; + {float,#c_literal{val=Val}} when is_float(Val) -> + ok; + {_,#c_literal{val=Val}} -> + add_warning(Pat, {nomatch_bit_syntax_type,Val,Type}), + throw(nomatch); + {_,_} -> + ok + end. + +bit_pat_warn_int(Val, 0, signed, Pat) -> + if + Val =:= 0 -> + ok; + true -> + add_warning(Pat, {nomatch_bit_syntax_truncated,signed,Val,0}), + throw(nomatch) + end; +bit_pat_warn_int(Val, Sz, signed, Pat) -> + if + Val < 0, Val bsr (Sz - 1) =/= -1 -> + add_warning(Pat, {nomatch_bit_syntax_truncated,signed,Val,Sz}), + throw(nomatch); + Val > 0, Val bsr (Sz - 1) =/= 0 -> + add_warning(Pat, {nomatch_bit_syntax_truncated,signed,Val,Sz}), + throw(nomatch); + true -> + ok + end; +bit_pat_warn_int(Val, _Sz, unsigned, Pat) when Val < 0 -> + add_warning(Pat, {nomatch_bit_syntax_unsigned,Val}), + throw(nomatch); +bit_pat_warn_int(Val, Sz, unsigned, Pat) -> + if + Val bsr Sz =:= 0 -> + ok; + true -> + add_warning(Pat, {nomatch_bit_syntax_truncated,unsigned,Val,Sz}), + throw(nomatch) + end. + +bit_pat_warn_unicode(U, _Pat) when 0 =< U, U =< 16#10FFFF -> + ok; +bit_pat_warn_unicode(U, Pat) -> + add_warning(Pat, {nomatch_bit_syntax_unicode,U}), + throw(nomatch). + +signedness(#c_literal{val=Flags}) -> + [S] = [F || F <- Flags, F =:= signed orelse F =:= unsigned], + S. + + %% is_subst(Expr) -> true | false. %% Test whether an expression is a suitable substitution. @@ -2251,11 +2376,11 @@ move_let_into_expr(#c_let{vars=InnerVs0,body=InnerBody0}=Inner, %% Arg = body(Arg0, Sub0), ScopeSub0 = sub_subst_scope(Sub0#sub{t=#{}}), - {OuterVs,ScopeSub} = pattern_list(OuterVs0, ScopeSub0), + {OuterVs,ScopeSub} = var_list(OuterVs0, ScopeSub0), OuterBody = body(OuterBody0, ScopeSub), - {InnerVs,Sub} = pattern_list(InnerVs0, Sub0), + {InnerVs,Sub} = var_list(InnerVs0, Sub0), InnerBody = body(InnerBody0, Sub), Outer#c_let{vars=OuterVs,arg=Arg, body=Inner#c_let{vars=InnerVs,arg=OuterBody,body=InnerBody}}; @@ -2271,39 +2396,49 @@ move_let_into_expr(#c_let{vars=Lvs0,body=Lbody0}=Let, case {TwoClauses,is_failing_clause(Ca0),is_failing_clause(Cb0)} of {true,false,true} -> %% let <Lvars> = case <Case-expr> of - %% <Cvars> -> <Clause-body>; - %% <OtherCvars> -> erlang:error(...) + %% <Cpats> -> <Clause-body>; + %% <OtherCpats> -> erlang:error(...) %% end %% in <Let-body> %% %% ==> %% %% case <Case-expr> of - %% <Cvars> -> + %% <Cpats> -> %% let <Lvars> = <Clause-body> %% in <Let-body>; - %% <OtherCvars> -> erlang:error(...) + %% <OtherCpats> -> erlang:error(...) %% end Cexpr = body(Cexpr0, Sub0), - CaVars0 = Ca0#c_clause.pats, + CaPats0 = Ca0#c_clause.pats, G0 = Ca0#c_clause.guard, B0 = Ca0#c_clause.body, ScopeSub0 = sub_subst_scope(Sub0#sub{t=#{}}), - {CaVars,ScopeSub} = pattern_list(CaVars0, ScopeSub0), - G = guard(G0, ScopeSub), - - B1 = body(B0, ScopeSub), - - {Lvs,B2,Sub1} = let_substs(Lvs0, B1, Sub0), - Sub2 = Sub1#sub{s=cerl_sets:union(ScopeSub#sub.s, - Sub1#sub.s)}, - Lbody = body(Lbody0, Sub2), - B = Let#c_let{vars=Lvs,arg=core_lib:make_values(B2),body=Lbody}, - - Ca = Ca0#c_clause{pats=CaVars,guard=G,body=B}, - Cb = clause(Cb0, Cexpr, value, Sub0), - Case#c_case{arg=Cexpr,clauses=[Ca,Cb]}; + try pattern_list(CaPats0, ScopeSub0) of + {CaPats,ScopeSub} -> + G = guard(G0, ScopeSub), + + B1 = body(B0, ScopeSub), + + {Lvs,B2,Sub1} = let_substs(Lvs0, B1, Sub0), + Sub2 = Sub1#sub{s=cerl_sets:union(ScopeSub#sub.s, + Sub1#sub.s)}, + Lbody = body(Lbody0, Sub2), + B = Let#c_let{vars=Lvs, + arg=core_lib:make_values(B2), + body=Lbody}, + + Ca = Ca0#c_clause{pats=CaPats,guard=G,body=B}, + Cb = clause(Cb0, Cexpr, value, Sub0), + Case#c_case{arg=Cexpr,clauses=[Ca,Cb]} + catch + nomatch -> + %% This is not a defeat. The code will eventually + %% be optimized to erlang:error(...) by the other + %% optimizations done in this module. + impossible + end; {_,_,_} -> impossible end; move_let_into_expr(#c_let{vars=Lvs0,body=Lbody0}=Let, @@ -2595,7 +2730,7 @@ move_case_into_arg(#c_case{arg=#c_let{vars=OuterVars0,arg=OuterArg, %% in case <InnerArg> of <InnerClauses> end %% ScopeSub0 = sub_subst_scope(Sub#sub{t=#{}}), - {OuterVars,ScopeSub} = pattern_list(OuterVars0, ScopeSub0), + {OuterVars,ScopeSub} = var_list(OuterVars0, ScopeSub0), InnerArg = body(InnerArg0, ScopeSub), Outer#c_let{vars=OuterVars,arg=OuterArg, body=Inner#c_case{arg=InnerArg,clauses=InnerClauses}}; @@ -2624,14 +2759,18 @@ move_case_into_arg(#c_case{arg=#c_case{arg=OuterArg, %% end %% ScopeSub0 = sub_subst_scope(Sub#sub{t=#{}}), - {OuterPats,ScopeSub} = pattern_list(OuterPats0, ScopeSub0), - OuterGuard = guard(OuterGuard0, ScopeSub), - InnerArg = body(InnerArg0, ScopeSub), - Inner = Inner0#c_case{arg=InnerArg,clauses=InnerClauses}, - OuterCa = OuterCa0#c_clause{pats=OuterPats,guard=OuterGuard, - body=Inner}, - Outer#c_case{arg=OuterArg, - clauses=[OuterCa,OuterCb]}; + + %% We KNOW that pattern_list/2 has already been called for OuterPats0; + %% therefore, it cannot throw an exception. + {OuterPats,ScopeSub} = pattern_list(OuterPats0, ScopeSub0), + OuterGuard = guard(OuterGuard0, ScopeSub), + InnerArg = body(InnerArg0, ScopeSub), + Inner = Inner0#c_case{arg=InnerArg,clauses=InnerClauses}, + OuterCa = OuterCa0#c_clause{pats=OuterPats, + guard=OuterGuard, + body=Inner}, + Outer#c_case{arg=OuterArg, + clauses=[OuterCa,OuterCb]}; false -> impossible end; @@ -3213,6 +3352,29 @@ format_error(nomatch_shadow) -> "this clause cannot match because a previous clause always matches"; format_error(nomatch_guard) -> "the guard for this clause evaluates to 'false'"; +format_error({nomatch_bit_syntax_truncated,Signess,Val,Sz}) -> + S = case Signess of + signed -> "a 'signed'"; + unsigned -> "an 'unsigned'" + end, + F = "this clause cannot match because the value ~P" + " will not fit in ~s binary segment of size ~p", + flatten(io_lib:format(F, [Val,10,S,Sz])); +format_error({nomatch_bit_syntax_unsigned,Val}) -> + F = "this clause cannot match because the negative value ~P" + " will never match the value of an 'unsigned' binary segment", + flatten(io_lib:format(F, [Val,10])); +format_error({nomatch_bit_syntax_size,Sz}) -> + F = "this clause cannot match because '~P' is not a valid size for a binary segment", + flatten(io_lib:format(F, [Sz,10])); +format_error({nomatch_bit_syntax_type,Val,Type}) -> + F = "this clause cannot match because '~P' is not of the" + " expected type '~p'", + flatten(io_lib:format(F, [Val,10,Type])); +format_error({nomatch_bit_syntax_unicode,Val}) -> + F = "this clause cannot match because the value ~p" + " is not a valid Unicode code point", + flatten(io_lib:format(F, [Val])); format_error(no_clause_match) -> "no clause will ever match"; format_error(nomatch_clause_type) -> diff --git a/lib/compiler/src/v3_core.erl b/lib/compiler/src/v3_core.erl index 72649e5c9f..68c9f964d8 100644 --- a/lib/compiler/src/v3_core.erl +++ b/lib/compiler/src/v3_core.erl @@ -1653,10 +1653,12 @@ pat_alias_map_pairs_1([]) -> []. pat_bin(Ps, St) -> [pat_segment(P, St) || P <- Ps]. -pat_segment({bin_element,_,Val,Size,[Type,{unit,Unit}|Flags]}, St) -> +pat_segment({bin_element,L,Val,Size,[Type,{unit,Unit}|Flags]}, St) -> + Anno = lineno_anno(L, St), {Pval,[],St1} = pattern(Val,St), {Psize,[],_St2} = pattern(Size,St1), - #c_bitstr{val=Pval,size=Psize, + #c_bitstr{anno=Anno, + val=Pval,size=Psize, unit=#c_literal{val=Unit}, type=#c_literal{val=Type}, flags=#c_literal{val=Flags}}. diff --git a/lib/compiler/test/bs_match_SUITE.erl b/lib/compiler/test/bs_match_SUITE.erl index 1f34e6947c..7fa26b6c26 100644 --- a/lib/compiler/test/bs_match_SUITE.erl +++ b/lib/compiler/test/bs_match_SUITE.erl @@ -37,11 +37,13 @@ cover_beam_bool/1,matched_out_size/1,follow_fail_branch/1, no_partition/1,calling_a_binary/1,binary_in_map/1, match_string_opt/1,select_on_integer/1, - map_and_binary/1,unsafe_branch_caching/1]). + map_and_binary/1,unsafe_branch_caching/1, + bad_literals/1,good_literals/1]). -export([coverage_id/1,coverage_external_ignore/2]). -include_lib("common_test/include/ct.hrl"). +-include_lib("syntax_tools/include/merl.hrl"). suite() -> @@ -66,7 +68,8 @@ groups() -> cover_beam_bool,matched_out_size,follow_fail_branch, no_partition,calling_a_binary,binary_in_map, match_string_opt,select_on_integer, - map_and_binary,unsafe_branch_caching]}]. + map_and_binary,unsafe_branch_caching, + bad_literals,good_literals]}]. init_per_suite(Config) -> @@ -1294,6 +1297,96 @@ do_unsafe_branch_caching(<<Code/integer, Bin/binary>>) -> _ -> Bin2 end. +bad_literals(_Config) -> + Mod = list_to_atom(?MODULE_STRING ++ "_" ++ + atom_to_list(?FUNCTION_NAME)), + S = [signed_lit_match(V, Sz) || V <- lists:seq(-8, 8), + Sz <- [0,1,2,3]] ++ + [unsigned_lit_match(V, Sz) || V <- lists:seq(-2, 8), + Sz <- [0,1,2]] ++ + [unicode_match(V) || + V <- [-100,-1,0,1,2|lists:seq(16#10FFFC, 16#110004)]], + Code = ?Q(["-module('@Mod@').\n" + "-export([f/0]).\n" + "f() ->\n" + "_@S,\n" + "ok.\n"]), + merl:print(Code), + Opts = test_lib:opt_opts(?MODULE), + {ok,_} = merl:compile_and_load(Code, Opts), + Mod:f(), + + {'EXIT',<<42>>} = (catch bad_literals_1()), + + Sz = id(8), + {'EXIT',{{badmatch,_},_}} = (catch <<-1:Sz>> = <<-1>>), + ok. + +bad_literals_1() -> + BadSz = bad, + case case <<42>> of + <<42:BadSz>> -> ok; + Val -> exit(Val) + end of + ok -> ok; + error -> error + end. + +signed_lit_match(V, Sz) -> + case <<V:Sz>> of + <<V:Sz/signed>> -> + ?Q("<<_@V@:_@Sz@/signed>> = <<_@V@:_@Sz@>>"); + _ -> + ?Q(["case <<_@V@:_@Sz@>> of\n", + " <<_@V@:_@Sz@/signed>> ->\n", + " ct:fail(should_not_match);\n", + " _ ->\n", + " ok\n", + "end\n"]) + end. + +unsigned_lit_match(V, Sz) -> + case <<V:Sz>> of + <<V:Sz/unsigned>> -> + ?Q("<<_@V@:_@Sz@>> = <<_@V@:_@Sz@>>"); + _ -> + ?Q(["case <<_@V@:_@Sz@>> of\n", + " <<_@V@:_@Sz@/unsigned>> ->\n", + " ct:fail(should_not_match);\n", + " _ ->\n", + " ok\n", + "end\n"]) + end. + +unicode_match(V) -> + try <<V/utf8>> of + <<V/utf8>> -> + ?Q(["<<_@V@/utf8>> = <<_@V@/utf8>>,\n", + "<<_@V@/utf16>> = <<_@V@/utf16>>,\n", + "<<_@V@/utf32>> = <<_@V@/utf32>>\n"]) + catch + error:badarg -> + ?Q(["case <<_@V@:32>> of\n", + " <<_@V@/utf32>> ->\n", + " ct:fail(should_not_match);\n", + " _ ->\n", + " ok\n", + "end\n"]) + end. + +%% Test a few legal but rare cases. + +good_literals(_Config) -> + Sz = id(64), + + %% Variable size. + <<42:Sz>> = id(<<42:Sz>>), + <<42.0:Sz/float>> = id(<<42:Sz/float>>), + + %% unit > 1 + <<16#cafebeef:4/unit:8>> = id(<<16#cafebeef:32>>), + ok. + check(F, R) -> R = F(). diff --git a/lib/compiler/test/warnings_SUITE.erl b/lib/compiler/test/warnings_SUITE.erl index ba4628cad3..d66f2d5053 100644 --- a/lib/compiler/test/warnings_SUITE.erl +++ b/lib/compiler/test/warnings_SUITE.erl @@ -41,7 +41,8 @@ files/1,effect/1,bin_opt_info/1,bin_construction/1, comprehensions/1,maps/1,maps_bin_opt_info/1, redundant_boolean_clauses/1, - latin1_fallback/1,underscore/1,no_warnings/1]). + latin1_fallback/1,underscore/1,no_warnings/1, + bit_syntax/1]). init_per_testcase(_Case, Config) -> Config. @@ -64,7 +65,7 @@ groups() -> bin_opt_info,bin_construction,comprehensions,maps, maps_bin_opt_info, redundant_boolean_clauses,latin1_fallback, - underscore,no_warnings]}]. + underscore,no_warnings,bit_syntax]}]. init_per_suite(Config) -> Config. @@ -779,6 +780,50 @@ no_warnings(Config) when is_list(Config) -> run(Config, Ts), ok. +bit_syntax(Config) -> + Ts = [{?FUNCTION_NAME, + <<"a(<<-1>>) -> ok; + a(<<1023>>) -> ok; + a(<<777/signed>>) -> ok; + a(<<a/binary>>) -> ok; + a(<<a/integer>>) -> ok; + a(<<a/float>>) -> ok; + a(<<a/utf8>>) -> ok; + a(<<a/utf16>>) -> ok; + a(<<a/utf32>>) -> ok; + a(<<a/utf32>>) -> ok. + b(Bin) -> Sz = bad, <<42:Sz>> = Bin. + c(Sz, Bin) -> + case Bin of + <<-42:Sz/unsigned>> -> ok; + <<42:Sz/float>> -> ok; + <<42:Sz/binary>> -> ok + end. + ">>, + [], + {warnings,[{1,sys_core_fold,no_clause_match}, + {1,sys_core_fold,{nomatch_bit_syntax_unsigned,-1}}, + {2,sys_core_fold,{nomatch_bit_syntax_truncated, + unsigned,1023,8}}, + {3,sys_core_fold,{nomatch_bit_syntax_truncated, + signed,777,8}}, + {4,sys_core_fold,{nomatch_bit_syntax_type,a,binary}}, + {5,sys_core_fold,{nomatch_bit_syntax_type,a,integer}}, + {6,sys_core_fold,{nomatch_bit_syntax_type,a,float}}, + {7,sys_core_fold,{nomatch_bit_syntax_type,a,utf8}}, + {8,sys_core_fold,{nomatch_bit_syntax_type,a,utf16}}, + {9,sys_core_fold,{nomatch_bit_syntax_type,a,utf32}}, + {10,sys_core_fold,{nomatch_bit_syntax_type,a,utf32}}, + {11,sys_core_fold,no_clause_match}, + {11,sys_core_fold,{nomatch_bit_syntax_size,bad}}, + {14,sys_core_fold,{nomatch_bit_syntax_unsigned,-42}}, + {16,sys_core_fold,{nomatch_bit_syntax_type,42,binary}} + ]} + }], + run(Config, Ts), + ok. + + %%% %%% End of test cases. %%% diff --git a/lib/debugger/src/int.erl b/lib/debugger/src/int.erl index 4f54beb45b..1e43d1858a 100644 --- a/lib/debugger/src/int.erl +++ b/lib/debugger/src/int.erl @@ -547,7 +547,7 @@ load({Mod, Src, Beam, BeamBin, Exp, Abst}, Dist) -> check_module(Mod) -> case code:which(Mod) of Beam when is_list(Beam) -> - case find_src(Beam) of + case find_src(Mod, Beam) of Src when is_list(Src) -> check_application(Src), case check_beam(Beam) of @@ -608,7 +608,7 @@ check_application2("gs-"++_) -> throw({error,{app,gs}}); check_application2("debugger-"++_) -> throw({error,{app,debugger}}); check_application2(_) -> ok. -find_src(Beam) -> +find_src(Mod, Beam) -> Src0 = filename:rootname(Beam) ++ ".erl", case is_file(Src0) of true -> Src0; @@ -618,10 +618,22 @@ find_src(Beam) -> filename:basename(Src0)]), case is_file(Src) of true -> Src; - false -> error + false -> find_src_from_module(Mod) end end. +find_src_from_module(Mod) -> + Compile = Mod:module_info(compile), + case lists:keyfind(source, 1, Compile) of + {source, Src} -> + case is_file(Src) of + true -> Src; + false -> error + end; + false -> + error + end. + find_beam(Mod, Src) -> SrcDir = filename:dirname(Src), BeamFile = atom_to_list(Mod) ++ code:objfile_extension(), diff --git a/lib/debugger/test/int_SUITE.erl b/lib/debugger/test/int_SUITE.erl index 8b16b7631b..ec4de88331 100644 --- a/lib/debugger/test/int_SUITE.erl +++ b/lib/debugger/test/int_SUITE.erl @@ -245,14 +245,13 @@ interpretable(Config) when is_list(Config) -> ?line true = int:interpretable(filename:join([DataDir,lists1])), ?line true = code:del_path(DataDir), - %% {error, no_src} - ?line PrivDir = filename:join(?config(priv_dir, Config), ""), - ?line {ok, _} = file:copy(filename:join([DataDir,"lists1.beam"]), + %% true (from source) + PrivDir = filename:join(?config(priv_dir, Config), ""), + {ok, _} = file:copy(filename:join([DataDir,"lists1.beam"]), filename:join([PrivDir,"lists1.beam"])), - ?line true = code:add_patha(PrivDir), - - ?line {error, no_src} = int:interpretable(lists1), - ?line ok = file:delete(filename:join([PrivDir,"lists1.beam"])), + true = code:add_patha(PrivDir), + true = int:interpretable(lists1), + ok = file:delete(filename:join([PrivDir,"lists1.beam"])), %% {error, no_beam} Src = filename:join([PrivDir,"lists1.erl"]), @@ -267,6 +266,11 @@ interpretable(Config) when is_list(Config) -> ?line ok = file:delete(Src), ?line true = code:del_path(PrivDir), + %% {error, no_src} + {ok, lists2, Binary} = compile:forms([{attribute,1,module,lists2}], []), + code:load_binary(lists2, "unknown", Binary), + {error, no_src} = int:interpretable(lists2), + %% {error, badarg} ?line {error, badarg} = int:interpretable(pride), ?line {error, badarg} = int:interpretable("prejudice.erl"), diff --git a/lib/eldap/doc/src/eldap.xml b/lib/eldap/doc/src/eldap.xml index 8f4479a730..43873e44e2 100644 --- a/lib/eldap/doc/src/eldap.xml +++ b/lib/eldap/doc/src/eldap.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2012</year><year>2013</year> + <year>2012</year><year>2016</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -29,7 +29,7 @@ <rev>B</rev> </header> <module>eldap</module> - <modulesummary>Eldap Functions</modulesummary> + <modulesummary>LDAP Client</modulesummary> <description> <p>This module provides a client api to the Lightweight Directory Access Protocol (LDAP). </p> @@ -40,20 +40,67 @@ </list> <p>The above publications can be found at <url href="http://www.ietf.org">IETF</url>. </p> - <p><em>Types</em></p> - <pre> -handle() Connection handle -attribute() {Type = string(), Values=[string()]} -modify_op() See mod_add/2, mod_delete/2, mod_replace/2 -scope() See baseObject/0, singleLevel/0, wholeSubtree/0 -dereference() See neverDerefAliases/0, derefInSearching/0, derefFindingBaseObj/0, derefAlways/0 -filter() See present/1, substrings/2, - equalityMatch/2, greaterOrEqual/2, lessOrEqual/2, - approxMatch/2, extensibleMatch/2, - 'and'/1, 'or'/1, 'not'/1. - </pre> - <p></p> </description> + + <section> + <title>DATA TYPES</title> + <p>Type definitions that are used more than once in this module: + </p> + <taglist> + <tag><c>handle()</c></tag> + <item><p>Connection handle</p></item> + + <tag><c>attribute() =</c></tag> + <item><p><c>{Type = string(), Values=[string()]}</c></p></item> + + <tag><c>modify_op()</c></tag> + <item><p>See + <seealso marker="#mod_add/2">mod_add/2</seealso>, + <seealso marker="#mod_delete/2">mod_delete/2</seealso>, + <seealso marker="#mod_replace/2">mod_replace/2</seealso> + </p></item> + + <tag><c>scope()</c></tag> + <item><p>See + <seealso marker="#baseObject/0">baseObject/0</seealso>, + <seealso marker="#singleLevel/0">singleLevel/0</seealso>, + <seealso marker="#wholeSubtree/0">wholeSubtree/0</seealso> + </p></item> + + <tag><c>dereference()</c></tag> + <item><p>See + <seealso marker="#neverDerefAliases/0">neverDerefAliases/0</seealso>, + <seealso marker="#derefInSearching/0">derefInSearching/0</seealso>, + <seealso marker="#derefFindingBaseObj/0">derefFindingBaseObj/0</seealso>, + <seealso marker="#derefAlways/0">derefAlways/0</seealso> + </p></item> + + <tag><c>filter()</c></tag> + <item><p>See + <seealso marker="#present/1">present/1</seealso>, + <seealso marker="#substrings/2">substrings/2</seealso>, + <seealso marker="#equalityMatch/2">equalityMatch/2</seealso>, + <seealso marker="#greaterOrEqual/2">greaterOrEqual/2</seealso>, + <seealso marker="#lessOrEqual/2">lessOrEqual/2</seealso>, + <seealso marker="#approxMatch/2">approxMatch/2</seealso>, + <seealso marker="#extensibleMatch/2">extensibleMatch/2</seealso>, + <seealso marker="#'and'/1">'and'/1</seealso>, + <seealso marker="#'or'/1">'or'/1</seealso>, + <seealso marker="#'not'/1">'not'/1</seealso> + </p></item> + + <tag><c>return_value() = </c></tag> + <item><p><c>ok | {ok, {referral,referrals()}} | {error,Error}</c> + </p></item> + + <tag><c>referrals() =</c></tag> + <item><p><c>[Address = string()]</c> The contents of <c>Address</c> is server dependent. + </p></item> + + </taglist> + </section> + + <funcs> <func> <name>open([Host]) -> {ok, Handle} | {error, Reason}</name> @@ -88,18 +135,19 @@ filter() See present/1, substrings/2, <v>Handle = handle()</v> </type> <desc> - <p>Shutdown the connection.</p> + <p>Shutdown the connection after sending an unbindRequest to the server. If the connection is tls the connection + will be closed with <c>ssl:close/1</c>, otherwise with <c>gen_tcp:close/1</c>.</p> </desc> </func> <func> - <name>start_tls(Handle, Options) -> ok | {error,Error}</name> + <name>start_tls(Handle, Options) -> return_value()</name> <fsummary>Upgrade a connection to TLS.</fsummary> <desc> <p>Same as start_tls(Handle, Options, infinity)</p> </desc> </func> <func> - <name>start_tls(Handle, Options, Timeout) -> ok | {error,Error}</name> + <name>start_tls(Handle, Options, Timeout) -> return_value()</name> <fsummary>Upgrade a connection to TLS.</fsummary> <type> <v>Handle = handle()</v> @@ -128,7 +176,7 @@ filter() See present/1, substrings/2, </desc> </func> <func> - <name>simple_bind(Handle, Dn, Password) -> ok | {error, Reason}</name> + <name>simple_bind(Handle, Dn, Password) -> return_value()</name> <fsummary>Authenticate the connection.</fsummary> <type> <v>Handle = handle()</v> @@ -140,7 +188,7 @@ filter() See present/1, substrings/2, </desc> </func> <func> - <name>add(Handle, Dn, [Attribute]) -> ok | {error, Reason}</name> + <name>add(Handle, Dn, [Attribute]) -> return_value()</name> <fsummary>Add an entry.</fsummary> <type> <v>Handle = handle()</v> @@ -161,7 +209,7 @@ filter() See present/1, substrings/2, </desc> </func> <func> - <name>delete(Handle, Dn) -> ok | {error, Reason}</name> + <name>delete(Handle, Dn) -> return_value()</name> <fsummary>Delete an entry.</fsummary> <type> <v>Dn = string()</v> @@ -203,7 +251,7 @@ filter() See present/1, substrings/2, </func> <func> - <name>modify(Handle, Dn, [ModifyOp]) -> ok | {error, Reason}</name> + <name>modify(Handle, Dn, [ModifyOp]) -> return_value()</name> <fsummary>Modify an entry.</fsummary> <type> <v>Dn = string()</v> @@ -219,7 +267,7 @@ filter() See present/1, substrings/2, </desc> </func> <func> - <name>modify_password(Handle, Dn, NewPasswd) -> ok | {ok, GenPasswd} | {error, Reason}</name> + <name>modify_password(Handle, Dn, NewPasswd) -> return_value() | {ok, GenPasswd}</name> <fsummary>Modify the password of a user.</fsummary> <type> <v>Dn = string()</v> @@ -230,7 +278,7 @@ filter() See present/1, substrings/2, </desc> </func> <func> - <name>modify_password(Handle, Dn, NewPasswd, OldPasswd) -> ok | {ok, GenPasswd} | {error, Reason}</name> + <name>modify_password(Handle, Dn, NewPasswd, OldPasswd) -> return_value() | {ok, GenPasswd}</name> <fsummary>Modify the password of a user.</fsummary> <type> <v>Dn = string()</v> @@ -259,7 +307,7 @@ filter() See present/1, substrings/2, </desc> </func> <func> - <name>modify_dn(Handle, Dn, NewRDN, DeleteOldRDN, NewSupDN) -> ok | {error, Reason}</name> + <name>modify_dn(Handle, Dn, NewRDN, DeleteOldRDN, NewSupDN) -> return_value()</name> <fsummary>Modify the DN of an entry.</fsummary> <type> <v>Dn = string()</v> @@ -279,7 +327,7 @@ filter() See present/1, substrings/2, </desc> </func> <func> - <name>search(Handle, SearchOptions) -> {ok, #eldap_search_result{}} | {error, Reason}</name> + <name>search(Handle, SearchOptions) -> {ok, #eldap_search_result{}} | {ok, {referral,referrals()}} | {error, Reason}</name> <fsummary>Search the Directory</fsummary> <type> <v>SearchOptions = #eldap_search{} | [SearchOption]</v> diff --git a/lib/eldap/src/eldap.erl b/lib/eldap/src/eldap.erl index df87ddde08..dc236f8a44 100644 --- a/lib/eldap/src/eldap.erl +++ b/lib/eldap/src/eldap.erl @@ -10,16 +10,23 @@ %%% See MIT-LICENSE at the top dir for licensing information. %%% -------------------------------------------------------------------- -vc('$Id$ '). --export([open/1,open/2,simple_bind/3,controlling_process/2, - start_tls/2, start_tls/3, - modify_password/3, modify_password/4, +-export([open/1, open/2, + simple_bind/3, simple_bind/4, + controlling_process/2, + start_tls/2, start_tls/3, start_tls/4, + modify_password/3, modify_password/4, modify_password/5, getopts/2, baseObject/0,singleLevel/0,wholeSubtree/0,close/1, equalityMatch/2,greaterOrEqual/2,lessOrEqual/2, extensibleMatch/2, - approxMatch/2,search/2,substrings/2,present/1, - 'and'/1,'or'/1,'not'/1,modify/3, mod_add/2, mod_delete/2, - mod_replace/2, add/3, delete/2, modify_dn/5,parse_dn/1, + search/2, search/3, + approxMatch/2,substrings/2,present/1, + 'and'/1,'or'/1,'not'/1,mod_add/2, mod_delete/2, + mod_replace/2, + modify/3, modify/4, + add/3, add/4, + delete/2, delete/3, + modify_dn/5,parse_dn/1, parse_ldap_url/1]). -export([neverDerefAliases/0, derefInSearching/0, @@ -91,7 +98,10 @@ start_tls(Handle, TlsOptions) -> start_tls(Handle, TlsOptions, infinity). start_tls(Handle, TlsOptions, Timeout) -> - send(Handle, {start_tls,TlsOptions,Timeout}), + start_tls(Handle, TlsOptions, Timeout, asn1_NOVALUE). + +start_tls(Handle, TlsOptions, Timeout, Controls) -> + send(Handle, {start_tls,TlsOptions,Timeout,Controls}), recv(Handle). %%% -------------------------------------------------------------------- @@ -108,7 +118,11 @@ modify_password(Handle, Dn, NewPasswd) -> modify_password(Handle, Dn, NewPasswd, OldPasswd) when is_pid(Handle), is_list(Dn), is_list(NewPasswd), is_list(OldPasswd) -> - send(Handle, {passwd_modify,optional(Dn),optional(NewPasswd),optional(OldPasswd)}), + modify_password(Handle, Dn, NewPasswd, OldPasswd, asn1_NOVALUE). + +modify_password(Handle, Dn, NewPasswd, OldPasswd, Controls) + when is_pid(Handle), is_list(Dn), is_list(NewPasswd), is_list(OldPasswd) -> + send(Handle, {passwd_modify,optional(Dn),optional(NewPasswd),optional(OldPasswd),Controls}), recv(Handle). %%% -------------------------------------------------------------------- @@ -147,7 +161,10 @@ controlling_process(Handle, Pid) when is_pid(Handle), is_pid(Pid) -> %%% Returns: ok | {error, Error} %%% -------------------------------------------------------------------- simple_bind(Handle, Dn, Passwd) when is_pid(Handle) -> - send(Handle, {simple_bind, Dn, Passwd}), + simple_bind(Handle, Dn, Passwd, asn1_NOVALUE). + +simple_bind(Handle, Dn, Passwd, Controls) when is_pid(Handle) -> + send(Handle, {simple_bind, Dn, Passwd, Controls}), recv(Handle). %%% -------------------------------------------------------------------- @@ -164,7 +181,10 @@ simple_bind(Handle, Dn, Passwd) when is_pid(Handle) -> %%% ) %%% -------------------------------------------------------------------- add(Handle, Entry, Attributes) when is_pid(Handle),is_list(Entry),is_list(Attributes) -> - send(Handle, {add, Entry, add_attrs(Attributes)}), + add(Handle, Entry, Attributes, asn1_NOVALUE). + +add(Handle, Entry, Attributes, Controls) when is_pid(Handle),is_list(Entry),is_list(Attributes) -> + send(Handle, {add, Entry, add_attrs(Attributes), Controls}), recv(Handle). %%% Do sanity check ! @@ -188,7 +208,10 @@ add_attrs(Attrs) -> %%% ) %%% -------------------------------------------------------------------- delete(Handle, Entry) when is_pid(Handle), is_list(Entry) -> - send(Handle, {delete, Entry}), + delete(Handle, Entry, asn1_NOVALUE). + +delete(Handle, Entry, Controls) when is_pid(Handle), is_list(Entry) -> + send(Handle, {delete, Entry, Controls}), recv(Handle). %%% -------------------------------------------------------------------- @@ -203,7 +226,10 @@ delete(Handle, Entry) when is_pid(Handle), is_list(Entry) -> %%% ) %%% -------------------------------------------------------------------- modify(Handle, Object, Mods) when is_pid(Handle), is_list(Object), is_list(Mods) -> - send(Handle, {modify, Object, Mods}), + modify(Handle, Object, Mods, asn1_NOVALUE). + +modify(Handle, Object, Mods, Controls) when is_pid(Handle), is_list(Object), is_list(Mods) -> + send(Handle, {modify, Object, Mods, Controls}), recv(Handle). %%% @@ -236,8 +262,12 @@ m(Operation, Type, Values) -> %%% -------------------------------------------------------------------- modify_dn(Handle, Entry, NewRDN, DelOldRDN, NewSup) when is_pid(Handle),is_list(Entry),is_list(NewRDN),is_atom(DelOldRDN),is_list(NewSup) -> + modify_dn(Handle, Entry, NewRDN, DelOldRDN, NewSup, asn1_NOVALUE). + +modify_dn(Handle, Entry, NewRDN, DelOldRDN, NewSup, Controls) + when is_pid(Handle),is_list(Entry),is_list(NewRDN),is_atom(DelOldRDN),is_list(NewSup) -> send(Handle, {modify_dn, Entry, NewRDN, - bool_p(DelOldRDN), optional(NewSup)}), + bool_p(DelOldRDN), optional(NewSup), Controls}), recv(Handle). %%% Sanity checks ! @@ -272,16 +302,19 @@ optional(Value) -> Value. %%% []}} %%% %%% -------------------------------------------------------------------- -search(Handle, A) when is_pid(Handle), is_record(A, eldap_search) -> - call_search(Handle, A); -search(Handle, L) when is_pid(Handle), is_list(L) -> +search(Handle, X) when is_pid(Handle), is_record(X,eldap_search) ; is_list(X) -> + search(Handle, X, asn1_NOVALUE). + +search(Handle, A, Controls) when is_pid(Handle), is_record(A, eldap_search) -> + call_search(Handle, A, Controls); +search(Handle, L, Controls) when is_pid(Handle), is_list(L) -> case catch parse_search_args(L) of {error, Emsg} -> {error, Emsg}; - A when is_record(A, eldap_search) -> call_search(Handle, A) + A when is_record(A, eldap_search) -> call_search(Handle, A, Controls) end. -call_search(Handle, A) -> - send(Handle, {search, A}), +call_search(Handle, A, Controls) -> + send(Handle, {search, A, Controls}), recv(Handle). parse_search_args(Args) -> @@ -484,33 +517,33 @@ do_connect(Host, Data, Opts) when Data#eldap.ldaps == true -> loop(Cpid, Data) -> receive - {From, {search, A}} -> - {Res,NewData} = do_search(Data, A), + {From, {search, A, Controls}} -> + {Res,NewData} = do_search(Data, A, Controls), send(From,Res), ?MODULE:loop(Cpid, NewData); - {From, {modify, Obj, Mod}} -> - {Res,NewData} = do_modify(Data, Obj, Mod), + {From, {modify, Obj, Mod, Controls}} -> + {Res,NewData} = do_modify(Data, Obj, Mod, Controls), send(From,Res), ?MODULE:loop(Cpid, NewData); - {From, {modify_dn, Obj, NewRDN, DelOldRDN, NewSup}} -> - {Res,NewData} = do_modify_dn(Data, Obj, NewRDN, DelOldRDN, NewSup), + {From, {modify_dn, Obj, NewRDN, DelOldRDN, NewSup, Controls}} -> + {Res,NewData} = do_modify_dn(Data, Obj, NewRDN, DelOldRDN, NewSup, Controls), send(From,Res), ?MODULE:loop(Cpid, NewData); - {From, {add, Entry, Attrs}} -> - {Res,NewData} = do_add(Data, Entry, Attrs), + {From, {add, Entry, Attrs, Controls}} -> + {Res,NewData} = do_add(Data, Entry, Attrs, Controls), send(From,Res), ?MODULE:loop(Cpid, NewData); - {From, {delete, Entry}} -> - {Res,NewData} = do_delete(Data, Entry), + {From, {delete, Entry, Controls}} -> + {Res,NewData} = do_delete(Data, Entry, Controls), send(From,Res), ?MODULE:loop(Cpid, NewData); - {From, {simple_bind, Dn, Passwd}} -> - {Res,NewData} = do_simple_bind(Data, Dn, Passwd), + {From, {simple_bind, Dn, Passwd, Controls}} -> + {Res,NewData} = do_simple_bind(Data, Dn, Passwd, Controls), send(From,Res), ?MODULE:loop(Cpid, NewData); @@ -520,17 +553,18 @@ loop(Cpid, Data) -> ?PRINT("New Cpid is: ~p~n",[NewCpid]), ?MODULE:loop(NewCpid, Data); - {From, {start_tls,TlsOptions,Timeout}} -> - {Res,NewData} = do_start_tls(Data, TlsOptions, Timeout), + {From, {start_tls,TlsOptions,Timeout,Controls}} -> + {Res,NewData} = do_start_tls(Data, TlsOptions, Timeout, Controls), send(From,Res), ?MODULE:loop(Cpid, NewData); - {From, {passwd_modify,Dn,NewPasswd,OldPasswd}} -> - {Res,NewData} = do_passwd_modify(Data, Dn, NewPasswd, OldPasswd), + {From, {passwd_modify,Dn,NewPasswd,OldPasswd,Controls}} -> + {Res,NewData} = do_passwd_modify(Data, Dn, NewPasswd, OldPasswd, Controls), send(From, Res), ?MODULE:loop(Cpid, NewData); {_From, close} -> + {no_reply,_NewData} = do_unbind(Data), unlink(Cpid), exit(closed); @@ -578,11 +612,10 @@ loop(Cpid, Data) -> %%% -------------------------------------------------------------------- %%% startTLS Request %%% -------------------------------------------------------------------- - -do_start_tls(Data=#eldap{using_tls=true}, _, _) -> +do_start_tls(Data=#eldap{using_tls=true}, _, _, _) -> {{error,tls_already_started}, Data}; -do_start_tls(Data=#eldap{fd=FD} , TlsOptions, Timeout) -> - case catch exec_start_tls(Data) of +do_start_tls(Data=#eldap{fd=FD} , TlsOptions, Timeout, Controls) -> + case catch exec_start_tls(Data, Controls) of {ok,NewData} -> case ssl:connect(FD,TlsOptions,Timeout) of {ok, SslSocket} -> @@ -593,15 +626,16 @@ do_start_tls(Data=#eldap{fd=FD} , TlsOptions, Timeout) -> {error,Error} -> {{error,Error}, Data} end; - {error,Error} -> {{error,Error},Data}; - Else -> {{error,Else},Data} + {{ok,Val},NewData} -> {{ok,Val},NewData}; + {error,Error} -> {{error,Error},Data}; + Else -> {{error,Else},Data} end. -define(START_TLS_OID, "1.3.6.1.4.1.1466.20037"). -exec_start_tls(Data) -> +exec_start_tls(Data, Controls) -> Req = #'ExtendedRequest'{requestName = ?START_TLS_OID}, - Reply = request(Data#eldap.fd, Data, Data#eldap.id, {extendedReq, Req}), + Reply = request(Data#eldap.fd, Data, Data#eldap.id, {extendedReq, Req, Controls}), exec_extended_req_reply(Data, Reply). exec_extended_req_reply(Data, {ok,Msg}) when @@ -611,6 +645,8 @@ exec_extended_req_reply(Data, {ok,Msg}) when case Result#'ExtendedResponse'.resultCode of success -> {ok,Data}; + referral -> + {{ok, {referral,Result#'ExtendedResponse'.referral}}, Data}; Error -> {error, {response,Error}} end; @@ -626,30 +662,32 @@ exec_extended_req_reply(_, Error) -> %%% Authenticate ourselves to the directory using %%% simple authentication. -do_simple_bind(Data, anon, anon) -> %% For testing - do_the_simple_bind(Data, "", ""); -do_simple_bind(Data, Dn, _Passwd) when Dn=="",Data#eldap.anon_auth==false -> +do_simple_bind(Data, anon, anon, Controls) -> %% For testing + do_the_simple_bind(Data, "", "", Controls); +do_simple_bind(Data, Dn, _Passwd,_) when Dn=="",Data#eldap.anon_auth==false -> {{error,anonymous_auth},Data}; -do_simple_bind(Data, _Dn, Passwd) when Passwd=="",Data#eldap.anon_auth==false -> +do_simple_bind(Data, _Dn, Passwd,_) when Passwd=="",Data#eldap.anon_auth==false -> {{error,anonymous_auth},Data}; -do_simple_bind(Data, Dn, Passwd) -> - do_the_simple_bind(Data, Dn, Passwd). +do_simple_bind(Data, Dn, Passwd, Controls) -> + do_the_simple_bind(Data, Dn, Passwd, Controls). -do_the_simple_bind(Data, Dn, Passwd) -> +do_the_simple_bind(Data, Dn, Passwd, Controls) -> case catch exec_simple_bind(Data#eldap{binddn = Dn, passwd = Passwd, - id = bump_id(Data)}) of - {ok,NewData} -> {ok,NewData}; - {error,Emsg} -> {{error,Emsg},Data}; - Else -> {{error,Else},Data} + id = bump_id(Data)}, + Controls) of + {ok,NewData} -> {ok,NewData}; + {{ok,Val},NewData} -> {{ok,Val},NewData}; + {error,Emsg} -> {{error,Emsg},Data}; + Else -> {{error,Else},Data} end. -exec_simple_bind(Data) -> +exec_simple_bind(Data, Controls) -> Req = #'BindRequest'{version = Data#eldap.version, name = Data#eldap.binddn, authentication = {simple, Data#eldap.passwd}}, log2(Data, "bind request = ~p~n", [Req]), - Reply = request(Data#eldap.fd, Data, Data#eldap.id, {bindRequest, Req}), + Reply = request(Data#eldap.fd, Data, Data#eldap.id, {bindRequest, Req, Controls}), log2(Data, "bind reply = ~p~n", [Reply]), exec_simple_bind_reply(Data, Reply). @@ -659,6 +697,7 @@ exec_simple_bind_reply(Data, {ok,Msg}) when {bindResponse, Result} -> case Result#'BindResponse'.resultCode of success -> {ok,Data}; + referral -> {{ok, {referral,Result#'BindResponse'.referral}}, Data}; Error -> {error, Error} end; Other -> {error, Other} @@ -671,10 +710,11 @@ exec_simple_bind_reply(_, Error) -> %%% searchRequest %%% -------------------------------------------------------------------- -do_search(Data, A) -> - case catch do_search_0(Data, A) of +do_search(Data, A, Controls) -> + case catch do_search_0(Data, A, Controls) of {error,Emsg} -> {ldap_closed_p(Data, Emsg),Data}; {'EXIT',Error} -> {ldap_closed_p(Data, Error),Data}; + {{ok,Val},NewData} -> {{ok,Val},NewData}; {ok,Res,Ref,NewData} -> {{ok,polish(Res, Ref)},NewData}; {{error,Reason},NewData} -> {{error,Reason},NewData}; Else -> {ldap_closed_p(Data, Else),Data} @@ -700,7 +740,7 @@ polish_result([H|T]) when is_record(H, 'SearchResultEntry') -> polish_result([]) -> []. -do_search_0(Data, A) -> +do_search_0(Data, A, Controls) -> Req = #'SearchRequest'{baseObject = A#eldap_search.base, scope = v_scope(A#eldap_search.scope), derefAliases = v_deref(A#eldap_search.deref), @@ -711,15 +751,15 @@ do_search_0(Data, A) -> attributes = v_attributes(A#eldap_search.attributes) }, Id = bump_id(Data), - collect_search_responses(Data#eldap{id=Id}, Req, Id). + collect_search_responses(Data#eldap{id=Id}, Req, Id, Controls). %%% The returned answers cames in one packet per entry %%% mixed with possible referals -collect_search_responses(Data, Req, ID) -> +collect_search_responses(Data, Req, ID, Controls) -> S = Data#eldap.fd, log2(Data, "search request = ~p~n", [Req]), - send_request(S, Data, ID, {searchRequest, Req}), + send_request(S, Data, ID, {searchRequest, Req, Controls}), Resp = recv_response(S, Data), log2(Data, "search reply = ~p~n", [Resp]), collect_search_responses(Data, S, ID, Resp, [], []). @@ -732,6 +772,8 @@ collect_search_responses(Data, S, ID, {ok,Msg}, Acc, Ref) success -> log2(Data, "search reply = searchResDone ~n", []), {ok,Acc,Ref,Data}; + referral -> + {{ok, {referral,R#'LDAPResult'.referral}}, Data}; Reason -> {{error,Reason},Data} end; @@ -756,21 +798,22 @@ collect_search_responses(_, _, _, Else, _, _) -> %%% addRequest %%% -------------------------------------------------------------------- -do_add(Data, Entry, Attrs) -> - case catch do_add_0(Data, Entry, Attrs) of +do_add(Data, Entry, Attrs, Controls) -> + case catch do_add_0(Data, Entry, Attrs, Controls) of {error,Emsg} -> {ldap_closed_p(Data, Emsg),Data}; {'EXIT',Error} -> {ldap_closed_p(Data, Error),Data}; {ok,NewData} -> {ok,NewData}; + {{ok,Val},NewData} -> {{ok,Val},NewData}; Else -> {ldap_closed_p(Data, Else),Data} end. -do_add_0(Data, Entry, Attrs) -> +do_add_0(Data, Entry, Attrs, Controls) -> Req = #'AddRequest'{entry = Entry, attributes = Attrs}, S = Data#eldap.fd, Id = bump_id(Data), log2(Data, "add request = ~p~n", [Req]), - Resp = request(S, Data, Id, {addRequest, Req}), + Resp = request(S, Data, Id, {addRequest, Req, Controls}), log2(Data, "add reply = ~p~n", [Resp]), check_reply(Data#eldap{id = Id}, Resp, addResponse). @@ -779,19 +822,20 @@ do_add_0(Data, Entry, Attrs) -> %%% deleteRequest %%% -------------------------------------------------------------------- -do_delete(Data, Entry) -> - case catch do_delete_0(Data, Entry) of +do_delete(Data, Entry, Controls) -> + case catch do_delete_0(Data, Entry, Controls) of {error,Emsg} -> {ldap_closed_p(Data, Emsg),Data}; {'EXIT',Error} -> {ldap_closed_p(Data, Error),Data}; {ok,NewData} -> {ok,NewData}; + {{ok,Val},NewData} -> {{ok,Val},NewData}; Else -> {ldap_closed_p(Data, Else),Data} end. -do_delete_0(Data, Entry) -> +do_delete_0(Data, Entry, Controls) -> S = Data#eldap.fd, Id = bump_id(Data), log2(Data, "del request = ~p~n", [Entry]), - Resp = request(S, Data, Id, {delRequest, Entry}), + Resp = request(S, Data, Id, {delRequest, Entry, Controls}), log2(Data, "del reply = ~p~n", [Resp]), check_reply(Data#eldap{id = Id}, Resp, delResponse). @@ -800,22 +844,23 @@ do_delete_0(Data, Entry) -> %%% modifyRequest %%% -------------------------------------------------------------------- -do_modify(Data, Obj, Mod) -> - case catch do_modify_0(Data, Obj, Mod) of +do_modify(Data, Obj, Mod, Controls) -> + case catch do_modify_0(Data, Obj, Mod, Controls) of {error,Emsg} -> {ldap_closed_p(Data, Emsg),Data}; {'EXIT',Error} -> {ldap_closed_p(Data, Error),Data}; {ok,NewData} -> {ok,NewData}; + {{ok,Val},NewData} -> {{ok,Val},NewData}; Else -> {ldap_closed_p(Data, Else),Data} end. -do_modify_0(Data, Obj, Mod) -> +do_modify_0(Data, Obj, Mod, Controls) -> v_modifications(Mod), Req = #'ModifyRequest'{object = Obj, changes = Mod}, S = Data#eldap.fd, Id = bump_id(Data), log2(Data, "modify request = ~p~n", [Req]), - Resp = request(S, Data, Id, {modifyRequest, Req}), + Resp = request(S, Data, Id, {modifyRequest, Req, Controls}), log2(Data, "modify reply = ~p~n", [Resp]), check_reply(Data#eldap{id = Id}, Resp, modifyResponse). @@ -825,16 +870,17 @@ do_modify_0(Data, Obj, Mod) -> -define(PASSWD_MODIFY_OID, "1.3.6.1.4.1.4203.1.11.1"). -do_passwd_modify(Data, Dn, NewPasswd, OldPasswd) -> - case catch do_passwd_modify_0(Data, Dn, NewPasswd, OldPasswd) of +do_passwd_modify(Data, Dn, NewPasswd, OldPasswd, Controls) -> + case catch do_passwd_modify_0(Data, Dn, NewPasswd, OldPasswd, Controls) of {error,Emsg} -> {ldap_closed_p(Data, Emsg),Data}; {'EXIT',Error} -> {ldap_closed_p(Data, Error),Data}; {ok,NewData} -> {ok,NewData}; + {{ok,Val},NewData} -> {{ok,Val},NewData}; {ok,Passwd,NewData} -> {{ok, Passwd},NewData}; Else -> {ldap_closed_p(Data, Else),Data} end. -do_passwd_modify_0(Data, Dn, NewPasswd, OldPasswd) -> +do_passwd_modify_0(Data, Dn, NewPasswd, OldPasswd, Controls) -> Req = #'PasswdModifyRequestValue'{userIdentity = Dn, oldPasswd = OldPasswd, newPasswd = NewPasswd}, @@ -844,7 +890,7 @@ do_passwd_modify_0(Data, Dn, NewPasswd, OldPasswd) -> requestValue = Bytes}, Id = bump_id(Data), log2(Data, "extended request = ~p~n", [ExtReq]), - Reply = request(Data#eldap.fd, Data, Id, {extendedReq, ExtReq}), + Reply = request(Data#eldap.fd, Data, Id, {extendedReq, ExtReq, Controls}), log2(Data, "modify password reply = ~p~n", [Reply]), exec_passwd_modify_reply(Data#eldap{id = Id}, Reply). @@ -865,6 +911,8 @@ exec_passwd_modify_reply(Data, {ok,Msg}) when throw(Error) end end; + referral -> + {{ok, {referral,Result#'ExtendedResponse'.referral}}, Data}; Error -> {error, {response,Error}} end; @@ -877,15 +925,16 @@ exec_passwd_modify_reply(_, Error) -> %%% modifyDNRequest %%% -------------------------------------------------------------------- -do_modify_dn(Data, Entry, NewRDN, DelOldRDN, NewSup) -> - case catch do_modify_dn_0(Data, Entry, NewRDN, DelOldRDN, NewSup) of +do_modify_dn(Data, Entry, NewRDN, DelOldRDN, NewSup, Controls) -> + case catch do_modify_dn_0(Data, Entry, NewRDN, DelOldRDN, NewSup, Controls) of {error,Emsg} -> {ldap_closed_p(Data, Emsg),Data}; {'EXIT',Error} -> {ldap_closed_p(Data, Error),Data}; {ok,NewData} -> {ok,NewData}; + {{ok,Val},NewData} -> {{ok,Val},NewData}; Else -> {ldap_closed_p(Data, Else),Data} end. -do_modify_dn_0(Data, Entry, NewRDN, DelOldRDN, NewSup) -> +do_modify_dn_0(Data, Entry, NewRDN, DelOldRDN, NewSup, Controls) -> Req = #'ModifyDNRequest'{entry = Entry, newrdn = NewRDN, deleteoldrdn = DelOldRDN, @@ -893,22 +942,51 @@ do_modify_dn_0(Data, Entry, NewRDN, DelOldRDN, NewSup) -> S = Data#eldap.fd, Id = bump_id(Data), log2(Data, "modify DN request = ~p~n", [Req]), - Resp = request(S, Data, Id, {modDNRequest, Req}), + Resp = request(S, Data, Id, {modDNRequest, Req, Controls}), log2(Data, "modify DN reply = ~p~n", [Resp]), check_reply(Data#eldap{id = Id}, Resp, modDNResponse). +%%%-------------------------------------------------------------------- +%%% unbindRequest +%%%-------------------------------------------------------------------- +do_unbind(Data) -> + Req = "", + log2(Data, "unbind request = ~p (has no reply)~n", [Req]), + send_request(Data#eldap.fd, Data, Data#eldap.id, {unbindRequest, Req}), + case Data#eldap.using_tls of + true -> ssl:close(Data#eldap.fd); + false -> gen_tcp:close(Data#eldap.fd) + end, + {no_reply, Data#eldap{binddn = (#eldap{})#eldap.binddn, + passwd = (#eldap{})#eldap.passwd, + fd = (#eldap{})#eldap.fd, + using_tls = false + }}. + + %%% -------------------------------------------------------------------- %%% Send an LDAP request and receive the answer %%% -------------------------------------------------------------------- - request(S, Data, ID, Request) -> send_request(S, Data, ID, Request), recv_response(S, Data). -send_request(S, Data, ID, Request) -> - Message = #'LDAPMessage'{messageID = ID, - protocolOp = Request}, - {ok,Bytes} = 'ELDAPv3':encode('LDAPMessage', Message), +send_request(S, Data, Id, {T,P}) -> + send_the_LDAPMessage(S, Data, #'LDAPMessage'{messageID = Id, + protocolOp = {T,P}}); +send_request(S, Data, Id, {T,P,asn1_NOVALUE}) -> + send_the_LDAPMessage(S, Data, #'LDAPMessage'{messageID = Id, + protocolOp = {T,P}}); +send_request(S, Data, Id, {T,P,Controls0}) -> + Controls = [#'Control'{controlType=F1, + criticality=F2, + controlValue=F3} || {control,F1,F2,F3} <- Controls0], + send_the_LDAPMessage(S, Data, #'LDAPMessage'{messageID = Id, + protocolOp = {T,P}, + controls = Controls}). + +send_the_LDAPMessage(S, Data, LDAPMessage) -> + {ok,Bytes} = 'ELDAPv3':encode('LDAPMessage', LDAPMessage), case do_send(S, Data, Bytes) of {error,Reason} -> throw({gen_tcp_error,Reason}); Else -> Else @@ -942,6 +1020,7 @@ check_reply(Data, {ok,Msg}, Op) when {Op, Result} -> case Result#'LDAPResult'.resultCode of success -> {ok,Data}; + referral -> {{ok, {referral,Result#'LDAPResult'.referral}}, Data}; Error -> {error, Error} end; Other -> {error, Other} diff --git a/lib/eldap/test/eldap_basic_SUITE.erl b/lib/eldap/test/eldap_basic_SUITE.erl index 8efed3cc18..4c1c2f8144 100644 --- a/lib/eldap/test/eldap_basic_SUITE.erl +++ b/lib/eldap/test/eldap_basic_SUITE.erl @@ -30,6 +30,11 @@ -define(TIMEOUT, 120000). % 2 min + +%% Control to delete a referral object: +-define(manageDsaIT, {control,"2.16.840.1.113730.3.4.2",false,asn1_NOVALUE}). + + all() -> [app, appup, @@ -59,6 +64,7 @@ groups() -> {api_bound, [], [add_when_bound, add_already_exists, more_add, + add_referral, search_filter_equalityMatch, search_filter_substring_any, search_filter_initial, @@ -67,8 +73,11 @@ groups() -> search_filter_or, search_filter_and_not, search_two_hits, + search_referral, modify, + modify_referral, delete, + delete_referral, modify_dn_delete_old, modify_dn_keep_old]}, {v4_connections, [], connection_tests()}, @@ -92,11 +101,16 @@ connection_tests() -> init_per_suite(Config) -> SSL_available = init_ssl_certs_et_al(Config), - LDAP_server = find_first_server(false, [{config,eldap_server}, {config,ldap_server}, {"localhost",9876}]), + LDAP_server = find_first_server(false, [{config,eldap_server}, + {config,ldap_server}, + {"localhost",9876}, + {"aramis.otp.ericsson.se",9876}]), LDAPS_server = case SSL_available of true -> - find_first_server(true, [{config,ldaps_server}, {"localhost",9877}]); + find_first_server(true, [{config,ldaps_server}, + {"localhost",9877}, + {"aramis.otp.ericsson.se",9877}]); false -> undefined end, @@ -454,6 +468,16 @@ more_add(Config) -> [{"objectclass", ["organizationalUnit"]}, {"ou", ["Team"]}]). +%%%---------------------------------------------------------------- +add_referral(Config) -> + H = ?config(handle, Config), + BasePath = ?config(eldap_path, Config), + {ok,{referral,["ldap://nowhere.example.com"++_]}} = + eldap:add(H, "cn=Foo Bar,dc=notHere," ++ BasePath, + [{"objectclass", ["person"]}, + {"cn", ["Foo Bar"]}, + {"sn", ["Bar"]}, + {"telephoneNumber", ["555-1232", "555-5432"]}]). %%%---------------------------------------------------------------- search_filter_equalityMatch(Config) -> @@ -569,6 +593,16 @@ search_two_hits(Config) -> [ok=eldap:delete(H,DN) || DN <- ExpectedDNs]. %%%---------------------------------------------------------------- +search_referral(Config) -> + H = ?config(handle, Config), + BasePath = ?config(eldap_path, Config), + DN = "cn=Santa Claus,dc=notHere," ++ BasePath, + {ok,{referral,["ldap://nowhere.example.com"++_]}} = + eldap:search(H, #eldap_search{base = DN, + filter = eldap:present("description"), + scope=eldap:singleLevel()}). + +%%%---------------------------------------------------------------- modify(Config) -> H = ?config(handle, Config), BasePath = ?config(eldap_path, Config), @@ -602,6 +636,19 @@ modify(Config) -> restore_original_object(H, DN, OriginalAttrs). %%%---------------------------------------------------------------- +modify_referral(Config) -> + H = ?config(handle, Config), + BasePath = ?config(eldap_path, Config), + %% The object to modify + DN = "cn=Foo Bar,dc=notHere," ++ BasePath, + + %% Do a change + Mod = [eldap:mod_replace("telephoneNumber", ["555-12345"]), + eldap:mod_add("description", ["Nice guy"])], + {ok,{referral,["ldap://nowhere.example.com"++_]}} = + eldap:modify(H, DN, Mod). + +%%%---------------------------------------------------------------- delete(Config) -> H = ?config(handle, Config), BasePath = ?config(eldap_path, Config), @@ -620,6 +667,14 @@ delete(Config) -> restore_original_object(H, DN, OriginalAttrs). %%%---------------------------------------------------------------- +delete_referral(Config) -> + H = ?config(handle, Config), + BasePath = ?config(eldap_path, Config), + %% The element to play with: + DN = "cn=Jonas Jonsson,dc=notHere," ++ BasePath, + {ok,{referral,["ldap://nowhere.example.com"++_]}} = eldap:delete(H, DN). + +%%%---------------------------------------------------------------- modify_dn_delete_old(Config) -> H = ?config(handle, Config), BasePath = ?config(eldap_path, Config), @@ -817,25 +872,44 @@ delete_old_contents(H, Path) -> {filter, eldap:present("objectclass")}, {scope, eldap:wholeSubtree()}]) of - {ok, #eldap_search_result{entries=Entries}} -> + {ok, _R=#eldap_search_result{entries=Entries}} -> + case eldap:delete(H, "dc=notHere,"++Path, [?manageDsaIT]) of + ok -> ok; + {error,noSuchObject} -> ok; + Other -> ct:fail("eldap:delete notHere ret ~p",[Other]) + end, [ok = eldap:delete(H,DN) || #eldap_entry{object_name=DN} <- Entries]; _Res -> ignore end. + +-define(ok(X), ok(?MODULE,?LINE,X)). + add_new_contents(H, Path, MyHost) -> - ok(eldap:add(H,"dc=ericsson,dc=se", + ?ok(eldap:add(H,"dc=ericsson,dc=se", [{"objectclass", ["dcObject", "organization"]}, {"dc", ["ericsson"]}, {"o", ["Testing"]}])), - ok(eldap:add(H,Path, + ?ok(eldap:add(H,Path, [{"objectclass", ["dcObject", "organization"]}, {"dc", [MyHost]}, - {"o", ["Test machine"]}])). - - -ok({error,entryAlreadyExists}) -> ok; -ok(X) -> ok=X. + {"o", ["Test machine"]}])), + ?ok(eldap:add(H, "dc=notHere,"++Path, + [{"objectclass", ["referral", + "dcObject" + ]}, + {"ref", ["ldap://nowhere.example.com/notHere,"++Path]}, + {"dc", ["notHere"]} + ])). + + + +ok(_, _, {error,entryAlreadyExists}) -> ok; +ok(_, _, ok) -> ok; +ok(MODULE, LINE, X) -> + ct:pal("~p:~p add_new_contents: ret from eldap:add = ~p",[MODULE,LINE,X]), + X. diff --git a/lib/eldap/vsn.mk b/lib/eldap/vsn.mk index 105a2bcdbb..99c474d588 100644 --- a/lib/eldap/vsn.mk +++ b/lib/eldap/vsn.mk @@ -1 +1 @@ -ELDAP_VSN = 1.2 +ELDAP_VSN = 1.2.1 diff --git a/lib/hipe/icode/hipe_icode_primops.erl b/lib/hipe/icode/hipe_icode_primops.erl index 84aae30291..a0deb31c42 100644 --- a/lib/hipe/icode/hipe_icode_primops.erl +++ b/lib/hipe/icode/hipe_icode_primops.erl @@ -504,16 +504,16 @@ type(Primop, Args) -> NewBinType = match_bin(erl_types:t_bitstr(0, Size), BinType), NewMatchState = erl_types:t_matchstate_update_present(NewBinType, MatchState), - if Signed =:= 0 -> - UpperBound = inf_add(safe_bsl(1, Size), -1), - erl_types:t_product([erl_types:t_from_range(0, UpperBound), - NewMatchState]); - Signed =:= 4 -> - erl_types:t_product([erl_types:t_from_range( - inf_inv(safe_bsl(1, Size-1)), - inf_add(safe_bsl(1, Size-1), -1)), - NewMatchState]) - end; + Range = + case Signed of + 0 -> + UpperBound = inf_add(safe_bsl_1(Size), -1), + erl_types:t_from_range(0, UpperBound); + 4 -> + Bound = safe_bsl_1(Size - 1), + erl_types:t_from_range(inf_inv(Bound), inf_add(Bound, -1)) + end, + erl_types:t_product([Range, NewMatchState]); [_Arg] -> NewBinType = match_bin(erl_types:t_bitstr(Size, 0), BinType), NewMatchState = @@ -969,18 +969,19 @@ check_fun_args(_, _) -> match_bin(Pattern, Match) -> erl_types:t_bitstr_match(Pattern, Match). -safe_bsl(0, _) -> 0; -safe_bsl(Base, Shift) when Shift =< 128 -> Base bsl Shift; -safe_bsl(Base, _Shift) when Base > 0 -> pos_inf; -safe_bsl(Base, _Shift) when Base < 0 -> neg_inf. +-spec safe_bsl_1(non_neg_integer()) -> non_neg_integer() | 'pos_inf'. + +safe_bsl_1(Shift) when Shift =< 128 -> 1 bsl Shift; +safe_bsl_1(_Shift) -> pos_inf. + +%% +%% The following two functions are stripped-down versions of more +%% general functions that exist in hipe_icode_range.erl +%% inf_inv(pos_inf) -> neg_inf; -inf_inv(neg_inf) -> pos_inf; -inf_inv(Number) -> -Number. +inf_inv(Number) when is_integer(Number) -> -Number. inf_add(pos_inf, _Number) -> pos_inf; -inf_add(neg_inf, _Number) -> neg_inf; -inf_add(_Number, pos_inf) -> pos_inf; -inf_add(_Number, neg_inf) -> neg_inf; inf_add(Number1, Number2) when is_integer(Number1), is_integer(Number2) -> Number1 + Number2. diff --git a/lib/kernel/doc/src/code.xml b/lib/kernel/doc/src/code.xml index d4c9a48901..819da544c3 100644 --- a/lib/kernel/doc/src/code.xml +++ b/lib/kernel/doc/src/code.xml @@ -310,6 +310,10 @@ <datatype> <name name="load_error_rsn"/> </datatype> + <datatype> + <name name="prepared_code"/> + <desc>An opaque term holding prepared code.</desc> + </datatype> </datatypes> <funcs> @@ -479,6 +483,138 @@ </desc> </func> <func> + <name name="atomic_load" arity="1"/> + <fsummary>Load a list of modules atomically</fsummary> + <desc> + <p>Tries to load all of the modules in the list + <c><anno>Modules</anno></c> atomically. That means that + either all modules are loaded at the same time, or + none of the modules are loaded if there is a problem with any + of the modules.</p> + <p>Loading can fail for one the following reasons:</p> + <taglist> + <tag><c>badfile</c></tag> + <item> + <p>The object code has an incorrect format or the module + name in the object code is not the expected module name.</p> + </item> + <tag><c>nofile</c></tag> + <item> + <p>No file with object code exists.</p> + </item> + <tag><c>on_load_not_allowed</c></tag> + <item> + <p>A module contains an + <seealso marker="doc/reference_manual:code_loading#on_load">-on_load function</seealso>.</p> + </item> + <tag><c>duplicated</c></tag> + <item> + <p>A module is included more than once in + <c><anno>Modules</anno></c>.</p> + </item> + <tag><c>not_purged</c></tag> + <item> + <p>The object code can not be loaded because an old version + of the code already exists.</p> + </item> + <tag><c>sticky_directory</c></tag> + <item> + <p>The object code resides in a sticky directory.</p> + </item> + <tag><c>pending_on_load</c></tag> + <item> + <p>A previously loaded module contains an + <c>-on_load</c> function that never finished.</p> + </item> + </taglist> + <p>If it is important to minimize the time that an application + is inactive while changing code, use + <seealso marker="#prepare_loading/1">prepare_loading/1</seealso> + and + <seealso marker="#finish_loading/1">finish_loading/1</seealso> + instead of <c>atomic_load/1</c>. Here is an example:</p> +<pre> +{ok,Prepared} = code:prepare_loading(Modules), +%% Put the application into an inactive state or do any +%% other preparation needed before changing the code. +ok = code:finish_loading(Prepared), +%% Resume the application.</pre> + </desc> + </func> + <func> + <name name="prepare_loading" arity="1"/> + <fsummary>Prepare a list of modules atomically</fsummary> + <desc> + <p>Prepares to load the modules in the list + <c><anno>Modules</anno></c>. + Finish the loading by calling + <seealso marker="#finish_loading/1">finish_loading(Prepared)</seealso>.</p> + <p>This function can fail with one of the following error reasons:</p> + <taglist> + <tag><c>badfile</c></tag> + <item> + <p>The object code has an incorrect format or the module + name in the object code is not the expected module name.</p> + </item> + <tag><c>nofile</c></tag> + <item> + <p>No file with object code exists.</p> + </item> + <tag><c>on_load_not_allowed</c></tag> + <item> + <p>A module contains an + <seealso marker="doc/reference_manual:code_loading#on_load">-on_load function</seealso>.</p> + </item> + <tag><c>duplicated</c></tag> + <item> + <p>A module is included more than once in + <c><anno>Modules</anno></c>.</p> + </item> + </taglist> + </desc> + </func> + <func> + <name name="finish_loading" arity="1"/> + <fsummary>Finish loading a list of prepared modules atomically</fsummary> + <desc> + <p>Tries to load code for all modules that have been previously + prepared by + <seealso marker="#prepare_loading/1">prepare_loading/1</seealso>. + The loading occurs atomically, meaning that + either all modules are loaded at the same time, or + none of the modules are loaded.</p> + <p>This function can fail with one of the following error reasons:</p> + <taglist> + <tag><c>not_purged</c></tag> + <item> + <p>The object code can not be loaded because an old version + of the code already exists.</p> + </item> + <tag><c>sticky_directory</c></tag> + <item> + <p>The object code resides in a sticky directory.</p> + </item> + <tag><c>pending_on_load</c></tag> + <item> + <p>A previously loaded module contains an + <c>-on_load</c> function that never finished.</p> + </item> + </taglist> + </desc> + </func> + <func> + <name name="ensure_modules_loaded" arity="1"/> + <fsummary>Ensure that a list of modules is loaded</fsummary> + <desc> + <p>Tries to load any modules not already loaded in the list + <c><anno>Modules</anno></c> in the same way as + <seealso marker="#load_file/1">load_file/1</seealso>.</p> + <p>Returns <c>ok</c> if successful, or + <c>{error,[{Module,Reason}]}</c> if loading of some modules fails. + See <seealso marker="#error_reasons">Error Reasons for Code-Loading Functions</seealso> for a description of other possible error reasons.</p> + </desc> + </func> + <func> <name name="delete" arity="1"/> <fsummary>Removes current code for a module</fsummary> <desc> diff --git a/lib/kernel/doc/src/heart.xml b/lib/kernel/doc/src/heart.xml index b9fad17ce1..9da4773f2d 100644 --- a/lib/kernel/doc/src/heart.xml +++ b/lib/kernel/doc/src/heart.xml @@ -118,6 +118,13 @@ <p>In the following descriptions, all function fails with reason <c>badarg</c> if <c>heart</c> is not started.</p> </description> + + <datatypes> + <datatype> + <name name="heart_option"/> + </datatype> + </datatypes> + <funcs> <func> <name name="set_cmd" arity="1"/> @@ -154,6 +161,62 @@ the empty string will be returned.</p> </desc> </func> + + <func> + <name name="set_callback" arity="2"/> + <fsummary>Set a validation callback</fsummary> + <desc> + <p> This validation callback will be executed before any heartbeat sent + to the port program. For the validation to succeed it needs to return + with the value <c>ok</c>. + </p> + <p> An exception within the callback will be treated as a validation failure. </p> + <p> The callback will be removed if the system reboots. </p> + </desc> + </func> + <func> + <name name="clear_callback" arity="0"/> + <fsummary>Clear the validation callback</fsummary> + <desc> + <p>Removes the validation callback call before heartbeats.</p> + </desc> + </func> + <func> + <name name="get_callback" arity="0"/> + <fsummary>Get the validation callback</fsummary> + <desc> + <p>Get the validation callback. If the callback is cleared, <c>none</c> will be returned.</p> + </desc> + </func> + + <func> + <name name="set_options" arity="1"/> + <fsummary>Set a list of options</fsummary> + <desc> + <p> Valid options <c>set_options</c> are: </p> + <taglist> + <tag><c>check_schedulers</c></tag> + <item> + <p>If enabled, a signal will be sent to each scheduler to check its + responsiveness. The system check occurs before any heartbeat sent + to the port program. If any scheduler is not responsive enough the + heart program will not receive its heartbeat and thus eventually terminate the node. + </p> + </item> + </taglist> + <p> Returns with the value <c>ok</c> if the options are valid.</p> + </desc> + </func> + <func> + <name name="get_options" arity="0"/> + <fsummary>Get the temporary reboot command</fsummary> + <desc> + <p>Returns <c>{ok, Options}</c> where <c>Options</c> is a list of current options enabled for heart. + If the callback is cleared, <c>none</c> will be returned.</p> + </desc> + </func> + + </funcs> </erlref> diff --git a/lib/kernel/src/code.erl b/lib/kernel/src/code.erl index 59e226df43..0882cb170c 100644 --- a/lib/kernel/src/code.erl +++ b/lib/kernel/src/code.erl @@ -28,11 +28,15 @@ get_path/0, load_file/1, ensure_loaded/1, + ensure_modules_loaded/1, load_abs/1, load_abs/2, load_binary/3, load_native_partial/2, load_native_sticky/3, + atomic_load/1, + prepare_loading/1, + finish_loading/1, delete/1, purge/1, soft_purge/1, @@ -71,6 +75,7 @@ -deprecated({rehash,0,next_major_release}). -export_type([load_error_rsn/0, load_ret/0]). +-export_type([prepared_code/0]). -include_lib("kernel/include/file.hrl"). @@ -88,6 +93,11 @@ -type loaded_ret_atoms() :: 'cover_compiled' | 'preloaded'. -type loaded_filename() :: (Filename :: file:filename()) | loaded_ret_atoms(). +-define(PREPARED, '$prepared$'). +-opaque prepared_code() :: + {?PREPARED,[{module(),{binary(),string(),_}}]}. + + %%% BIFs -export([get_chunk/2, is_module_native/1, make_stub_module/3, module_md5/1]). @@ -303,6 +313,313 @@ rehash() -> -spec get_mode() -> 'embedded' | 'interactive'. get_mode() -> call(get_mode). +%%% +%%% Loading of several modules in parallel. +%%% + +-spec ensure_modules_loaded([Module]) -> + 'ok' | {'error',[{Module,What}]} when + Module :: module(), + What :: badfile | nofile | on_load_failure. + +ensure_modules_loaded(Modules) when is_list(Modules) -> + case prepare_ensure(Modules, []) of + Ms when is_list(Ms) -> + ensure_modules_loaded_1(Ms); + error -> + error(function_clause, [Modules]) + end. + +ensure_modules_loaded_1(Ms0) -> + Ms = lists:usort(Ms0), + {Prep,Error0} = load_mods(Ms), + {OnLoad,Normal} = partition_on_load(Prep), + Error1 = case finish_loading(Normal, true) of + ok -> Error0; + {error,Err} -> Err ++ Error0 + end, + ensure_modules_loaded_2(OnLoad, Error1). + +ensure_modules_loaded_2([{M,_}|Ms], Errors) -> + case ensure_loaded(M) of + {module,M} -> + ensure_modules_loaded_2(Ms, Errors); + {error,Err} -> + ensure_modules_loaded_2(Ms, [{M,Err}|Errors]) + end; +ensure_modules_loaded_2([], []) -> + ok; +ensure_modules_loaded_2([], [_|_]=Errors) -> + {error,Errors}. + +prepare_ensure([M|Ms], Acc) when is_atom(M) -> + case erlang:module_loaded(M) of + true -> + prepare_ensure(Ms, Acc); + false -> + prepare_ensure(Ms, [M|Acc]) + end; +prepare_ensure([], Acc) -> + Acc; +prepare_ensure(_, _) -> + error. + +-spec atomic_load(Modules) -> 'ok' | {'error',[{Module,What}]} when + Modules :: [Module | {Module, Filename, Binary}], + Module :: module(), + Filename :: file:filename(), + Binary :: binary(), + What :: 'badfile' | 'nofile' | 'on_load_not_allowed' | 'duplicated' | + 'not_purged' | 'sticky_directory' | 'pending_on_load'. + +atomic_load(Modules) -> + case do_prepare_loading(Modules) of + {ok,Prep} -> + finish_loading(Prep, false); + {error,_}=Error -> + Error; + badarg -> + error(function_clause, [Modules]) + end. + +-spec prepare_loading(Modules) -> + {'ok',Prepared} | {'error',[{Module,What}]} when + Modules :: [Module | {Module, Filename, Binary}], + Module :: module(), + Filename :: file:filename(), + Binary :: binary(), + Prepared :: prepared_code(), + What :: 'badfile' | 'nofile' | 'on_load_not_allowed' | 'duplicated'. + +prepare_loading(Modules) -> + case do_prepare_loading(Modules) of + {ok,Prep} -> + {ok,{?PREPARED,Prep}}; + {error,_}=Error -> + Error; + badarg -> + error(function_clause, [Modules]) + end. + +-spec finish_loading(Prepared) -> 'ok' | {'error',[{Module,What}]} when + Prepared :: prepared_code(), + Module :: module(), + What :: 'not_purged' | 'sticky_directory' | 'pending_on_load'. + +finish_loading({?PREPARED,Prepared}=Arg) when is_list(Prepared) -> + case verify_prepared(Prepared) of + ok -> + finish_loading(Prepared, false); + error -> + error(function_clause, [Arg]) + end. + +partition_load([Item|T], Bs, Ms) -> + case Item of + {M,File,Bin} when is_atom(M) andalso + is_list(File) andalso + is_binary(Bin) -> + partition_load(T, [Item|Bs], Ms); + M when is_atom(M) -> + partition_load(T, Bs, [Item|Ms]); + _ -> + error + end; +partition_load([], Bs, Ms) -> + {Bs,Ms}. + +do_prepare_loading(Modules) -> + case partition_load(Modules, [], []) of + {ModBins,Ms} -> + case prepare_loading_1(ModBins, Ms) of + {error,_}=Error -> + Error; + Prep when is_list(Prep) -> + {ok,Prep} + end; + error -> + badarg + end. + +prepare_loading_1(ModBins, Ms) -> + %% erlang:finish_loading/1 *will* detect duplicates. + %% However, we want to detect all errors that can be detected + %% by only examining the input data before call the LastAction + %% fun. + case prepare_check_uniq(ModBins, Ms) of + ok -> + prepare_loading_2(ModBins, Ms); + Error -> + Error + end. + +prepare_loading_2(ModBins, Ms) -> + {Prep0,Error0} = load_bins(ModBins), + {Prep1,Error1} = load_mods(Ms), + case Error0 ++ Error1 of + [] -> + prepare_loading_3(Prep0 ++ Prep1); + [_|_]=Error -> + {error,Error} + end. + +prepare_loading_3(Prep) -> + case partition_on_load(Prep) of + {[_|_]=OnLoad,_} -> + Error = [{M,on_load_not_allowed} || {M,_} <- OnLoad], + {error,Error}; + {[],_} -> + Prep + end. + +prepare_check_uniq([{M,_,_}|T], Ms) -> + prepare_check_uniq(T, [M|Ms]); +prepare_check_uniq([], Ms) -> + prepare_check_uniq_1(lists:sort(Ms), []). + +prepare_check_uniq_1([M|[M|_]=Ms], Acc) -> + prepare_check_uniq_1(Ms, [{M,duplicated}|Acc]); +prepare_check_uniq_1([_|Ms], Acc) -> + prepare_check_uniq_1(Ms, Acc); +prepare_check_uniq_1([], []) -> + ok; +prepare_check_uniq_1([], [_|_]=Errors) -> + {error,Errors}. + +partition_on_load(Prep) -> + P = fun({_,{Bin,_,_}}) -> + erlang:has_prepared_code_on_load(Bin) + end, + lists:partition(P, Prep). + +verify_prepared([{M,{Prep,Name,_Native}}|T]) + when is_atom(M), is_binary(Prep), is_list(Name) -> + try erlang:has_prepared_code_on_load(Prep) of + false -> + verify_prepared(T); + _ -> + error + catch + error:_ -> + error + end; +verify_prepared([]) -> + ok; +verify_prepared(_) -> + error. + +finish_loading(Prepared0, EnsureLoaded) -> + Prepared = [{M,{Bin,File}} || {M,{Bin,File,_}} <- Prepared0], + Native0 = [{M,Code} || {M,{_,_,Code}} <- Prepared0, + Code =/= undefined], + case call({finish_loading,Prepared,EnsureLoaded}) of + ok -> + finish_loading_native(Native0); + {error,Errors}=E when EnsureLoaded -> + S0 = sofs:relation(Errors), + S1 = sofs:domain(S0), + R0 = sofs:relation(Native0), + R1 = sofs:drestriction(R0, S1), + Native = sofs:to_external(R1), + finish_loading_native(Native), + E; + {error,_}=E -> + E + end. + +finish_loading_native([{Mod,Code}|Ms]) -> + _ = load_native_partial(Mod, Code), + finish_loading_native(Ms); +finish_loading_native([]) -> + ok. + +load_mods([]) -> + {[],[]}; +load_mods(Mods) -> + Path = get_path(), + F = prepare_loading_fun(), + {ok,{Succ,Error0}} = erl_prim_loader:get_modules(Mods, F, Path), + Error = [case E of + badfile -> {M,E}; + _ -> {M,nofile} + end || {M,E} <- Error0], + {Succ,Error}. + +load_bins([]) -> + {[],[]}; +load_bins(BinItems) -> + F = prepare_loading_fun(), + do_par(F, BinItems). + +-type prep_fun_type() :: fun((module(), file:filename(), binary()) -> + {ok,_} | {error,_}). + +-spec prepare_loading_fun() -> prep_fun_type(). + +prepare_loading_fun() -> + GetNative = get_native_fun(), + fun(Mod, FullName, Beam) -> + case erlang:prepare_loading(Mod, Beam) of + Prepared when is_binary(Prepared) -> + {ok,{Prepared,FullName,GetNative(Beam)}}; + {error,_}=Error -> + Error + end + end. + +get_native_fun() -> + Architecture = erlang:system_info(hipe_architecture), + try hipe_unified_loader:chunk_name(Architecture) of + ChunkTag -> + fun(Beam) -> code:get_chunk(Beam, ChunkTag) end + catch _:_ -> + fun(_) -> undefined end + end. + +do_par(Fun, L) -> + {_,Ref} = spawn_monitor(do_par_fun(Fun, L)), + receive + {'DOWN',Ref,process,_,Res} -> + Res + end. + +-spec do_par_fun(prep_fun_type(), list()) -> fun(() -> no_return()). + +do_par_fun(Fun, L) -> + fun() -> + _ = [spawn_monitor(do_par_fun_2(Fun, Item)) || + Item <- L], + exit(do_par_recv(length(L), [], [])) + end. + +-spec do_par_fun_2(prep_fun_type(), + {module(),file:filename(),binary()}) -> + fun(() -> no_return()). + +do_par_fun_2(Fun, Item) -> + fun() -> + {Mod,Filename,Bin} = Item, + try Fun(Mod, Filename, Bin) of + {ok,Res} -> + exit({good,{Mod,Res}}); + {error,Error} -> + exit({bad,{Mod,Error}}) + catch + _:Error -> + exit({bad,{Mod,Error}}) + end + end. + +do_par_recv(0, Good, Bad) -> + {Good,Bad}; +do_par_recv(N, Good, Bad) -> + receive + {'DOWN',_,process,_,{good,Res}} -> + do_par_recv(N-1, [Res|Good], Bad); + {'DOWN',_,process,_,{bad,Res}} -> + do_par_recv(N-1, Good, [Res|Bad]) + end. + %%----------------------------------------------------------------- call(Req) -> diff --git a/lib/kernel/src/code_server.erl b/lib/kernel/src/code_server.erl index b52def8777..6262407354 100644 --- a/lib/kernel/src/code_server.erl +++ b/lib/kernel/src/code_server.erl @@ -28,11 +28,10 @@ ]). -include_lib("kernel/include/file.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). -import(lists, [foreach/2]). --define(ANY_NATIVE_CODE_LOADED, any_native_code_loaded). - -type on_load_item() :: {reference(),module(),file:name_all(),[pid()]}. -record(state, {supervisor :: pid(), @@ -90,8 +89,6 @@ init(Ref, Parent, [Root,Mode]) -> namedb = init_namedb(Path), mode = Mode}, - put(?ANY_NATIVE_CODE_LOADED, false), - Parent ! {Ref,{ok,self()}}, loop(State). @@ -289,14 +286,14 @@ handle_call({load_binary,Mod,File,Bin}, Caller, S) when is_atom(Mod) -> handle_call({load_native_partial,Mod,Bin}, {_From,_Tag}, S) -> Architecture = erlang:system_info(hipe_architecture), Result = (catch hipe_unified_loader:load(Mod, Bin, Architecture)), - Status = hipe_result_to_status(Result), + Status = hipe_result_to_status(Result, S), {reply,Status,S}; handle_call({load_native_sticky,Mod,Bin,WholeModule}, {_From,_Tag}, S) -> Architecture = erlang:system_info(hipe_architecture), Result = (catch hipe_unified_loader:load_module(Mod, Bin, WholeModule, Architecture)), - Status = hipe_result_to_status(Result), + Status = hipe_result_to_status(Result, S), {reply,Status,S}; handle_call({ensure_loaded,Mod}, Caller, St) when is_atom(Mod) -> @@ -356,6 +353,9 @@ handle_call({set_primary_archive, File, ArchiveBin, FileInfo, ParserFun}, {_From handle_call(get_mode, {_From,_Tag}, S=#state{mode=Mode}) -> {reply, Mode, S}; +handle_call({finish_loading,Prepared,EnsureLoaded}, {_,_}, S) -> + {reply,finish_loading(Prepared, EnsureLoaded, S),S}; + handle_call(Other,{_From,_Tag}, S) -> error_msg(" ** Codeserver*** ignoring ~w~n ",[Other]), {noreply,S}. @@ -1107,8 +1107,7 @@ try_load_module_2(File, Mod, Bin, Caller, Architecture, #state{moddb=Db}=St) -> case catch hipe_unified_loader:load_native_code(Mod, Bin, Architecture) of {module,Mod} = Module -> - put(?ANY_NATIVE_CODE_LOADED, true), - ets:insert(Db, {Mod,File}), + ets:insert(Db, [{{native,Mod},true},{Mod,File}]), {reply,Module,St}; no_native -> try_load_module_3(File, Mod, Bin, Caller, Architecture, St); @@ -1122,7 +1121,7 @@ try_load_module_3(File, Mod, Bin, Caller, Architecture, case erlang:load_module(Mod, Bin) of {module,Mod} = Module -> ets:insert(Db, {Mod,File}), - post_beam_load(Mod, Architecture), + post_beam_load([Mod], Architecture, St), {reply,Module,St}; {error,on_load} -> handle_on_load(Mod, File, Caller, St); @@ -1131,23 +1130,24 @@ try_load_module_3(File, Mod, Bin, Caller, Architecture, {reply,Error,St} end. -hipe_result_to_status(Result) -> +hipe_result_to_status(Result, #state{moddb=Db}) -> case Result of - {module,_} -> - put(?ANY_NATIVE_CODE_LOADED, true), + {module,Mod} -> + ets:insert(Db, [{{native,Mod},true}]), Result; _ -> {error,Result} end. -post_beam_load(Mod, Architecture) -> +post_beam_load(_, undefined, _) -> + %% HiPE is disabled. + ok; +post_beam_load(Mods0, _Architecture, #state{moddb=Db}) -> %% post_beam_load/2 can potentially be very expensive because it - %% blocks multi-scheduling; thus we want to avoid the call if we - %% know that it is not needed. - case get(?ANY_NATIVE_CODE_LOADED) of - true -> hipe_unified_loader:post_beam_load(Mod, Architecture); - false -> ok - end. + %% blocks multi-scheduling. Therefore, we only want to call + %% it with modules that are known to have native code loaded. + Mods = [M || M <- Mods0, ets:member(Db, {native,M})], + hipe_unified_loader:post_beam_load(Mods). int_list([H|T]) when is_integer(H) -> int_list(T); int_list([_|_]) -> false; @@ -1221,7 +1221,6 @@ absname_vr([[X, $:]|Name], _, _AbsBase) -> absname(filename:join(Name), Dcwd). - is_loaded(M, Db) -> case ets:lookup(Db, M) of [{M,File}] -> {file,File}; @@ -1236,6 +1235,64 @@ do_soft_purge(Mod) -> erts_code_purger:soft_purge(Mod). +%%% +%%% Loading of multiple modules in parallel. +%%% + +finish_loading(Prepared, EnsureLoaded, #state{moddb=Db}=St) -> + Ps = [fun(L) -> finish_loading_ensure(L, EnsureLoaded) end, + fun(L) -> abort_if_pending_on_load(L, St) end, + fun(L) -> abort_if_sticky(L, Db) end, + fun(L) -> do_finish_loading(L, St) end], + run(Ps, Prepared). + +finish_loading_ensure(Prepared, true) -> + {ok,[P || {M,_}=P <- Prepared, not erlang:module_loaded(M)]}; +finish_loading_ensure(Prepared, false) -> + {ok,Prepared}. + +abort_if_pending_on_load(L, #state{on_load=[]}) -> + {ok,L}; +abort_if_pending_on_load(L, #state{on_load=OnLoad}) -> + Pending = [{M,pending_on_load} || + {M,_} <- L, + lists:keymember(M, 2, OnLoad)], + case Pending of + [] -> {ok,L}; + [_|_] -> {error,Pending} + end. + +abort_if_sticky(L, Db) -> + Sticky = [{M,sticky_directory} || {M,_} <- L, is_sticky(M, Db)], + case Sticky of + [] -> {ok,L}; + [_|_] -> {error,Sticky} + end. + +do_finish_loading(Prepared, #state{moddb=Db}=St) -> + MagicBins = [B || {_,{B,_}} <- Prepared], + case erlang:finish_loading(MagicBins) of + ok -> + MFs = [{M,F} || {M,{_,F}} <- Prepared], + true = ets:insert(Db, MFs), + Ms = [M || {M,_} <- MFs], + Architecture = erlang:system_info(hipe_architecture), + post_beam_load(Ms, Architecture, St), + ok; + {Reason,Ms} -> + {error,[{M,Reason} || M <- Ms]} + end. + +run([F], Data) -> + F(Data); +run([F|Fs], Data0) -> + case F(Data0) of + {ok,Data} -> + run(Fs, Data); + {error,_}=Error -> + Error + end. + %% ------------------------------------------------------- %% The on_load functionality. %% ------------------------------------------------------- @@ -1317,18 +1374,8 @@ finish_on_load_report(Mod, Term) -> %% ------------------------------------------------------- all_loaded(Db) -> - all_l(Db, ets:slot(Db, 0), 1, []). - -all_l(_Db, '$end_of_table', _, Acc) -> - Acc; -all_l(Db, ModInfo, N, Acc) -> - NewAcc = strip_mod_info(ModInfo,Acc), - all_l(Db, ets:slot(Db, N), N + 1, NewAcc). - - -strip_mod_info([{{sticky,_},_}|T], Acc) -> strip_mod_info(T, Acc); -strip_mod_info([H|T], Acc) -> strip_mod_info(T, [H|Acc]); -strip_mod_info([], Acc) -> Acc. + Ms = ets:fun2ms(fun({M,_}=T) when is_atom(M) -> T end), + ets:select(Db, Ms). -spec error_msg(io:format(), [term()]) -> 'ok'. error_msg(Format, Args) -> diff --git a/lib/kernel/src/gen_tcp.erl b/lib/kernel/src/gen_tcp.erl index d7dba4ac80..8cb2a725e8 100644 --- a/lib/kernel/src/gen_tcp.erl +++ b/lib/kernel/src/gen_tcp.erl @@ -114,7 +114,8 @@ option(). -type socket() :: port(). --export_type([option/0, option_name/0, connect_option/0, listen_option/0]). +-export_type([option/0, option_name/0, connect_option/0, listen_option/0, + socket/0]). %% %% Connect a socket diff --git a/lib/kernel/src/heart.erl b/lib/kernel/src/heart.erl index 137fad706f..eea78aabdf 100644 --- a/lib/kernel/src/heart.erl +++ b/lib/kernel/src/heart.erl @@ -34,7 +34,11 @@ %%% %%% It recognizes the flag '-heart' %%%-------------------------------------------------------------------- --export([start/0, init/2, set_cmd/1, clear_cmd/0, get_cmd/0, cycle/0]). +-export([start/0, init/2, + set_cmd/1, clear_cmd/0, get_cmd/0, + set_callback/2, clear_callback/0, get_callback/0, + set_options/1, get_options/0, + cycle/0]). -define(START_ACK, 1). -define(HEART_BEAT, 2). @@ -49,6 +53,16 @@ -define(CYCLE_TIMEOUT, 10000). -define(HEART_PORT_NAME, heart_port). +%% valid heart options +-define(SCHEDULER_CHECK_OPT, check_schedulers). + +-type heart_option() :: ?SCHEDULER_CHECK_OPT. + +-record(state,{port :: port(), + cmd :: [] | binary(), + options :: [heart_option()], + callback :: 'undefined' | {atom(), atom()}}). + %%--------------------------------------------------------------------- -spec start() -> 'ignore' | {'error', term()} | {'ok', pid()}. @@ -81,11 +95,11 @@ wait_for_init_ack(From) -> init(Starter, Parent) -> process_flag(trap_exit, true), process_flag(priority, max), - register(heart, self()), + register(?MODULE, self()), case catch start_portprogram() of {ok, Port} -> Starter ! {ok, self()}, - loop(Parent, Port, ""); + loop(Parent, #state{port=Port, cmd=[], options=[]}); no_heart -> Starter ! {no_heart, self()}; error -> @@ -96,33 +110,68 @@ init(Starter, Parent) -> Cmd :: string(). set_cmd(Cmd) -> - heart ! {self(), set_cmd, Cmd}, + ?MODULE ! {self(), set_cmd, Cmd}, wait(). -spec get_cmd() -> {ok, Cmd} when Cmd :: string(). get_cmd() -> - heart ! {self(), get_cmd}, + ?MODULE ! {self(), get_cmd}, wait(). -spec clear_cmd() -> ok. clear_cmd() -> - heart ! {self(), clear_cmd}, + ?MODULE ! {self(), clear_cmd}, + wait(). + +-spec set_callback(Module,Function) -> 'ok' | {'error', {'bad_callback', {Module, Function}}} when + Module :: atom(), + Function :: atom(). + +set_callback(Module, Function) -> + ?MODULE ! {self(), set_callback, {Module,Function}}, + wait(). + +-spec get_callback() -> {'ok', {Module, Function}} | 'none' when + Module :: atom(), + Function :: atom(). + +get_callback() -> + ?MODULE ! {self(), get_callback}, + wait(). + +-spec clear_callback() -> ok. + +clear_callback() -> + ?MODULE ! {self(), clear_callback}, + wait(). + +-spec set_options(Options) -> 'ok' | {'error', {'bad_options', Options}} when + Options :: [heart_option()]. + +set_options(Options) -> + ?MODULE ! {self(), set_options, Options}, wait(). +-spec get_options() -> {'ok', Options} | 'none' when + Options :: [atom()]. + +get_options() -> + ?MODULE ! {self(), get_options}, + wait(). %%% Should be used solely by the release handler!!!!!!! -spec cycle() -> 'ok' | {'error', term()}. cycle() -> - heart ! {self(), cycle}, + ?MODULE ! {self(), cycle}, wait(). wait() -> receive - {heart, Res} -> + {?MODULE, Res} -> Res end. @@ -182,8 +231,8 @@ wait_ack(Port) -> {error, Reason} end. -loop(Parent, Port, Cmd) -> - _ = send_heart_beat(Port), +loop(Parent, #state{port=Port}=S) -> + _ = send_heart_beat(S), receive {From, set_cmd, NewCmd0} -> Enc = file:native_name_encoding(), @@ -191,37 +240,72 @@ loop(Parent, Port, Cmd) -> NewCmd when is_binary(NewCmd), byte_size(NewCmd) < 2047 -> _ = send_heart_cmd(Port, NewCmd), _ = wait_ack(Port), - From ! {heart, ok}, - loop(Parent, Port, NewCmd); + From ! {?MODULE, ok}, + loop(Parent, S#state{cmd=NewCmd}); _ -> - From ! {heart, {error, {bad_cmd, NewCmd0}}}, - loop(Parent, Port, Cmd) + From ! {?MODULE, {error, {bad_cmd, NewCmd0}}}, + loop(Parent, S) end; {From, clear_cmd} -> - From ! {heart, ok}, - _ = send_heart_cmd(Port, ""), + From ! {?MODULE, ok}, + _ = send_heart_cmd(Port, []), _ = wait_ack(Port), - loop(Parent, Port, ""); + loop(Parent, S#state{cmd = []}); {From, get_cmd} -> - From ! {heart, get_heart_cmd(Port)}, - loop(Parent, Port, Cmd); + From ! {?MODULE, get_heart_cmd(Port)}, + loop(Parent, S); + {From, set_callback, Callback} -> + case Callback of + {M,F} when is_atom(M), is_atom(F) -> + From ! {?MODULE, ok}, + loop(Parent, S#state{callback=Callback}); + _ -> + From ! {?MODULE, {error, {bad_callback, Callback}}}, + loop(Parent, S) + end; + {From, get_callback} -> + Res = case S#state.callback of + undefined -> none; + Cb -> {ok, Cb} + end, + From ! {?MODULE, Res}, + loop(Parent, S); + {From, clear_callback} -> + From ! {?MODULE, ok}, + loop(Parent, S#state{callback=undefined}); + {From, set_options, Options} -> + case validate_options(Options) of + Validated when is_list(Validated) -> + From ! {?MODULE, ok}, + loop(Parent, S#state{options=Validated}); + _ -> + From ! {?MODULE, {error, {bad_options, Options}}}, + loop(Parent, S) + end; + {From, get_options} -> + Res = case S#state.options of + [] -> none; + Cb -> {ok, Cb} + end, + From ! {?MODULE, Res}, + loop(Parent, S); {From, cycle} -> %% Calls back to loop - do_cycle_port_program(From, Parent, Port, Cmd); + do_cycle_port_program(From, Parent, S); {'EXIT', Parent, shutdown} -> no_reboot_shutdown(Port); {'EXIT', Parent, Reason} -> exit(Port, Reason), exit(Reason); {'EXIT', Port, badsig} -> % we can ignore badsig-messages! - loop(Parent, Port, Cmd); + loop(Parent, S); {'EXIT', Port, _Reason} -> - exit({port_terminated, {heart, loop, [Parent, Port, Cmd]}}); + exit({port_terminated, {?MODULE, loop, [Parent, S]}}); _ -> - loop(Parent, Port, Cmd) + loop(Parent, S) after ?TIMEOUT -> - loop(Parent, Port, Cmd) + loop(Parent, S) end. -spec no_reboot_shutdown(port()) -> no_return(). @@ -233,36 +317,44 @@ no_reboot_shutdown(Port) -> exit(normal) end. -do_cycle_port_program(Caller, Parent, Port, Cmd) -> +validate_options(Opts) -> validate_options(Opts,[]). +validate_options([],Res) -> Res; +validate_options([?SCHEDULER_CHECK_OPT=Opt|Opts],Res) -> validate_options(Opts,[Opt|Res]); +validate_options(_,_) -> error. + +do_cycle_port_program(Caller, Parent, #state{port=Port} = S) -> unregister(?HEART_PORT_NAME), case catch start_portprogram() of {ok, NewPort} -> _ = send_shutdown(Port), receive {'EXIT', Port, _Reason} -> - _ = send_heart_cmd(NewPort, Cmd), - Caller ! {heart, ok}, - loop(Parent, NewPort, Cmd) + _ = send_heart_cmd(NewPort, S#state.cmd), + Caller ! {?MODULE, ok}, + loop(Parent, S#state{port=NewPort}) after ?CYCLE_TIMEOUT -> %% Huh! Two heart port programs running... %% well, the old one has to be sick not to respond %% so we'll settle for the new one... - _ = send_heart_cmd(NewPort, Cmd), - Caller ! {heart, {error, stop_error}}, - loop(Parent, NewPort, Cmd) + _ = send_heart_cmd(NewPort, S#state.cmd), + Caller ! {?MODULE, {error, stop_error}}, + loop(Parent, S#state{port=NewPort}) end; no_heart -> - Caller ! {heart, {error, no_heart}}, - loop(Parent, Port, Cmd); + Caller ! {?MODULE, {error, no_heart}}, + loop(Parent, S); error -> - Caller ! {heart, {error, start_error}}, - loop(Parent, Port, Cmd) + Caller ! {?MODULE, {error, start_error}}, + loop(Parent, S) end. %% "Beates" the heart once. -send_heart_beat(Port) -> Port ! {self(), {command, [?HEART_BEAT]}}. +send_heart_beat(#state{port=Port, callback=Cb, options=Opts}) -> + ok = check_system(Opts), + ok = check_callback(Cb), + Port ! {self(), {command, [?HEART_BEAT]}}. %% Set a new HEART_COMMAND. -dialyzer({no_improper_lists, send_heart_cmd/2}). @@ -278,6 +370,24 @@ get_heart_cmd(Port) -> {ok, Cmd} end. +check_system([]) -> ok; +check_system([?SCHEDULER_CHECK_OPT|Opts]) -> + ok = erts_internal:system_check(schedulers), + check_system(Opts). + +%% validate system by performing a check before the heartbeat +%% return 'ok' if everything is alright. +%% Terminate if with reason if something is a miss. +%% It is fine to timeout in the callback, in fact that is the intention +%% if something goes wront -> no heartbeat. + +check_callback(Callback) -> + case Callback of + undefined -> ok; + {M,F} -> + erlang:apply(M,F,[]) + end. + %% Sends shutdown command to the port. send_shutdown(Port) -> Port ! {self(), {command, [?SHUT_DOWN]}}. diff --git a/lib/kernel/src/hipe_unified_loader.erl b/lib/kernel/src/hipe_unified_loader.erl index ddbbc548dd..73fcb2469c 100644 --- a/lib/kernel/src/hipe_unified_loader.erl +++ b/lib/kernel/src/hipe_unified_loader.erl @@ -44,7 +44,7 @@ -export([chunk_name/1, %% Only the code and code_server modules may call the entries below! load_native_code/3, - post_beam_load/2, + post_beam_load/1, load_module/4, load/3]). @@ -120,15 +120,15 @@ load_native_code(Mod, Bin, Architecture) when is_atom(Mod), is_binary(Bin) -> %%======================================================================== --spec post_beam_load(atom(), hipe_architecture()) -> 'ok'. +-spec post_beam_load([module()]) -> 'ok'. -%% does nothing on a hipe-disabled system -post_beam_load(_Mod, undefined) -> +post_beam_load([])-> ok; -post_beam_load(Mod, _) when is_atom(Mod) -> +post_beam_load([_|_]=Mods) -> erlang:system_flag(multi_scheduling, block), try - patch_to_emu(Mod) + _ = [patch_to_emu(Mod) || Mod <- Mods], + ok after erlang:system_flag(multi_scheduling, unblock) end, diff --git a/lib/kernel/test/Makefile b/lib/kernel/test/Makefile index ba40dd3168..7b233741e0 100644 --- a/lib/kernel/test/Makefile +++ b/lib/kernel/test/Makefile @@ -79,7 +79,8 @@ MODULES= \ zlib_SUITE \ loose_node \ sendfile_SUITE \ - standard_error_SUITE + standard_error_SUITE \ + multi_load_SUITE APP_FILES = \ appinc.app \ diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl index cf8863fcd0..00f29aa8ed 100644 --- a/lib/kernel/test/code_SUITE.erl +++ b/lib/kernel/test/code_SUITE.erl @@ -798,7 +798,7 @@ analyse2(MFA={_,_,_}, Path, Visited0) -> analyse(FL, [MFA|Path], my_usort([MFA|Visited0]), 0). %%%% We need to check these manually... -% fun's are ok as long as they are defined locally. +%% fun's are ok as long as they are defined locally. check_funs({'$M_EXPR','$F_EXPR',_}, [{unicode,characters_to_binary_int,3}, {unicode,characters_to_binary,3}, @@ -870,6 +870,8 @@ check_funs({'$M_EXPR','$F_EXPR',1}, {hipe_unified_loader,get_refs_from,2}| _]) -> 0; check_funs({'$M_EXPR',warning_msg,2}, [{code_server,finish_on_load_report,2} | _]) -> 0; +check_funs({'$M_EXPR','$F_EXPR',1}, + [{code_server,run,2}|_]) -> 0; %% This is cheating! /raimo %% %% check_funs(This = {M,_,_}, Path) -> diff --git a/lib/kernel/test/erl_prim_loader_SUITE.erl b/lib/kernel/test/erl_prim_loader_SUITE.erl index bc14a2764d..bccca59b93 100644 --- a/lib/kernel/test/erl_prim_loader_SUITE.erl +++ b/lib/kernel/test/erl_prim_loader_SUITE.erl @@ -29,7 +29,8 @@ inet_existing/1, inet_coming_up/1, inet_disconnects/1, multiple_slaves/1, file_requests/1, local_archive/1, remote_archive/1, - primary_archive/1, virtual_dir_in_archive/1]). + primary_archive/1, virtual_dir_in_archive/1, + get_modules/1]). -export([init_per_testcase/2, end_per_testcase/2]). @@ -44,7 +45,8 @@ all() -> normalize_and_backslash, inet_existing, inet_coming_up, inet_disconnects, multiple_slaves, file_requests, local_archive, remote_archive, - primary_archive, virtual_dir_in_archive]. + primary_archive, virtual_dir_in_archive, + get_modules]. groups() -> []. @@ -109,6 +111,37 @@ get_file(Config) when is_list(Config) -> ?line error = erl_prim_loader:get_file({dummy}), ok. +get_modules(_Config) -> + MsGood = lists:sort([lists,gen_server,gb_trees,code_server]), + Ms = [certainly_not_existing|MsGood], + SuccExp = [begin + F = code:which(M), + {ok,Code} = file:read_file(F), + {M,{F,erlang:md5(Code)}} + end || M <- MsGood], + FailExp = [{certainly_not_existing,enoent}], + + Path = code:get_path(), + Process = fun(_, F, Code) -> {ok,{F,erlang:md5(Code)}} end, + {ok,{Succ,FailExp}} = erl_prim_loader:get_modules(Ms, Process, Path), + SuccExp = lists:sort(Succ), + + Name = inet_get_modules, + {ok, Node, BootPid} = complete_start_node(Name), + ThisDir = filename:dirname(code:which(?MODULE)), + true = rpc:call(Node, code, add_patha, [ThisDir]), + _ = rpc:call(Node, code, ensure_loaded, [?MODULE]), + {ok,{InetSucc,FailExp}} = rpc:call(Node, erl_prim_loader, + get_modules, [Ms,Process,Path]), + SuccExp = lists:sort(InetSucc), + + stop_node(Node), + unlink(BootPid), + exit(BootPid, kill), + + ok. + + normalize_and_backslash(Config) -> %% Test OTP-11170 case os:type() of diff --git a/lib/kernel/test/heart_SUITE.erl b/lib/kernel/test/heart_SUITE.erl index 4447475833..eb6cb06622 100644 --- a/lib/kernel/test/heart_SUITE.erl +++ b/lib/kernel/test/heart_SUITE.erl @@ -27,6 +27,8 @@ node_start_immediately_after_crash/1, node_start_soon_after_crash/1, set_cmd/1, clear_cmd/1, get_cmd/1, + callback_api/1, + options_api/1, dont_drop/1, kill_pid/1]). -export([init_per_testcase/2, end_per_testcase/2]). @@ -66,6 +68,8 @@ all() -> [ node_start_immediately_after_crash, node_start_soon_after_crash, set_cmd, clear_cmd, get_cmd, + callback_api, + options_api, kill_pid ]. @@ -358,6 +362,69 @@ get_cmd(Config) when is_list(Config) -> stop_node(Node), ok. +callback_api(Config) when is_list(Config) -> + {ok, Node} = start_check(slave, heart_test), + none = rpc:call(Node, heart, get_callback, []), + M0 = self(), + F0 = ok, + {error, {bad_callback, {M0,F0}}} = rpc:call(Node, heart, set_callback, [M0,F0]), + none = rpc:call(Node, heart, get_callback, []), + M1 = lists:duplicate(28, $a), + F1 = lists:duplicate(28, $b), + {error, {bad_callback, {M1,F1}}} = rpc:call(Node, heart, set_callback, [M1,F1]), + none = rpc:call(Node, heart, get_callback, []), + + M2 = heart_check_module, + F2 = cb_ok, + F3 = cb_error, + Code0 = generate(M2, [], [ + atom_to_list(F2) ++ "() -> ok.", + atom_to_list(F3) ++ "() -> exit(\"callback_error (as intended)\")." + ]), + {module, M2} = rpc:call(Node, erlang, load_module, [M2, Code0]), + ok = rpc:call(Node, M2, F2, []), + ok = rpc:call(Node, heart, set_callback, [M2,F2]), + {ok, {M2,F2}} = rpc:call(Node, heart, get_callback, []), + ok = rpc:call(Node, heart, clear_callback, []), + none = rpc:call(Node, heart, get_callback, []), + ok = rpc:call(Node, heart, set_callback, [M2,F2]), + {ok, {M2,F2}} = rpc:call(Node, heart, get_callback, []), + ok = rpc:call(Node, heart, set_callback, [M2,F3]), + receive {nodedown, Node} -> ok + after 5000 -> test_server:fail(node_not_killed) + end, + stop_node(Node), + ok. + +options_api(Config) when is_list(Config) -> + {ok, Node} = start_check(slave, heart_test), + none = rpc:call(Node, heart, get_options, []), + M0 = self(), + F0 = ok, + {error, {bad_options, {M0,F0}}} = rpc:call(Node, heart, set_options, [{M0,F0}]), + none = rpc:call(Node, heart, get_options, []), + Ls = lists:duplicate(28, $b), + {error, {bad_options, Ls}} = rpc:call(Node, heart, set_options, [Ls]), + none = rpc:call(Node, heart, get_options, []), + + ok = rpc:call(Node, heart, set_options, [[check_schedulers]]), + {ok, [check_schedulers]} = rpc:call(Node, heart, get_options, []), + ok = rpc:call(Node, heart, set_options, [[]]), + none = rpc:call(Node, heart, get_options, []), + + ok = rpc:call(Node, heart, set_options, [[check_schedulers]]), + {ok, [check_schedulers]} = rpc:call(Node, heart, get_options, []), + {error, {bad_options, Ls}} = rpc:call(Node, heart, set_options, [Ls]), + {ok, [check_schedulers]} = rpc:call(Node, heart, get_options, []), + + receive after 3000 -> ok end, %% wait 3 secs + + ok = rpc:call(Node, heart, set_options, [[]]), + none = rpc:call(Node, heart, get_options, []), + stop_node(Node), + ok. + + dont_drop(suite) -> %%% Removed as it may crash epmd/distribution in colourful %%% ways. While we ARE finding out WHY, it would diff --git a/lib/kernel/test/multi_load_SUITE.erl b/lib/kernel/test/multi_load_SUITE.erl new file mode 100644 index 0000000000..bb87443e36 --- /dev/null +++ b/lib/kernel/test/multi_load_SUITE.erl @@ -0,0 +1,412 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1999-2012. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(multi_load_SUITE). +-export([all/0,suite/0,groups/0,init_per_suite/1,end_per_suite/1, + init_per_group/2,end_per_group/2, + basic_atomic_load/1,basic_errors/1,sticky_dir/1, + on_load_failing/1,ensure_modules_loaded/1, + native_code/1]). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("syntax_tools/include/merl.hrl"). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [basic_atomic_load,basic_errors,sticky_dir,on_load_failing, + ensure_modules_loaded,native_code]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + +basic_atomic_load(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + Dir = filename:join(PrivDir, multi_load_sticky_dir), + _ = file:make_dir(Dir), + + OldPath = code:get_path(), + try + code:add_patha(Dir), + do_basic(Dir) + after + code:set_path(OldPath) + end, + + ok. + +do_basic(Dir) -> + MsVer1_0 = make_modules(5, versioned_module(1)), + MsVer1 = [{M,filename:absname(F, Dir),Bin} || {M,F,Bin} <- MsVer1_0], + _ = [ok = file:write_file(F, Bin) || {_,F,Bin} <- MsVer1], + + Ms = [M || {M,_,_} <- MsVer1], + [] = [loaded || M <- Ms, is_loaded(M)], + + ok = code:atomic_load(Ms), + _ = [1 = M:M() || M <- Ms], + _ = [F = code:which(M) || {M,F,_} <- MsVer1], + [] = [not_loaded || M <- Ms, not is_loaded(M)], + + MsVer2 = update_modules(Ms, versioned_module(2)), + {ok,Prepared} = code:prepare_loading(MsVer2), + ok = code:finish_loading(Prepared), + _ = [2 = M:M() || M <- Ms], + _ = [F = code:which(M) || {M,F,_} <- MsVer2], + [] = [not_loaded || M <- Ms, not is_loaded(M)], + + MsVer3 = update_modules(Ms, versioned_module(2)), + NotPurged = lists:sort([{M,not_purged} || M <- Ms]), + NotPurged = atomic_load_error(MsVer3, true), + + ok. + +versioned_module(Ver) -> + fun(Mod) -> + ?Q(["-module('@Mod@').\n", + "-export(['@Mod@'/0]).\n", + "'@Mod@'() -> _@Ver@.\n"]) + end. + +basic_errors(_Config) -> + atomic_load_fc([42]), + atomic_load_fc([{"mod","file","bin"}]), + + finish_loading_fc(atom), + {ok,{PrepTag,_}} = code:prepare_loading([code]), + finish_loading_fc({PrepTag,[x]}), + finish_loading_fc({PrepTag,[{m,{<<>>,"",<<>>}}]}), + Prep = prepared_with_wrong_magic_bin(), + finish_loading_fc(Prep), + + [{x,badfile}] = atomic_load_error([{x,"x",<<"bad">>}], false), + [{a,badfile},{some_nonexistent_file,nofile}] = + atomic_load_error([some_nonexistent_file,{a,"a",<<>>}], + false), + + %% Modules mentioned more than once. + Mods = make_modules(2, fun basic_module/1), + Ms = [M || {M,_,_} <- Mods], + DupMods = Mods ++ [mnesia] ++ Mods ++ [mnesia], + DupErrors0 = lists:sort([mnesia|Ms]), + DupErrors = [{M,duplicated} || M <- DupErrors0], + DupErrors = atomic_load_error(DupMods, false), + + ok. + +atomic_load_fc(L) -> + {'EXIT',{function_clause,[{code,atomic_load,[L],_}|_]}} = + (catch code:atomic_load(L)), + {'EXIT',{function_clause,[{code,prepare_loading,[L],_}|_]}} = + (catch code:prepare_loading(L)). + +finish_loading_fc(Term) -> + {'EXIT',{function_clause,[{code,finish_loading,[Term],_}|_]}} = + (catch code:finish_loading(Term)). + +prepared_with_wrong_magic_bin() -> + {ok,Prep} = code:prepare_loading([?MODULE]), + prep_magic(Prep). + +prep_magic([H|T]) -> + [prep_magic(H)|prep_magic(T)]; +prep_magic(Tuple) when is_tuple(Tuple) -> + L = prep_magic(tuple_to_list(Tuple)), + list_to_tuple(L); +prep_magic(Bin) when is_binary(Bin) -> + try erlang:has_prepared_code_on_load(Bin) of + false -> + %% Create a different kind of magic binary. + ets:match_spec_compile([{'_',[true],['$_']}]) + catch + _:_ -> + Bin + end; +prep_magic(Other) -> + Other. + +sticky_dir(_Config) -> + Mod0 = make_module(lists, fun basic_module/1), + Mod1 = make_module(gen_server, fun basic_module/1), + Ms = [Mod0,Mod1], + SD = sticky_directory, + StickyErrors = [{gen_server,SD},{lists,SD}], + StickyErrors = atomic_load_error(Ms, true), + + ok. + +on_load_failing(_Config) -> + OnLoad = make_modules(1, fun on_load_module/1), + [{OnLoadMod,_,_}] = OnLoad, + Ms = make_modules(10, fun basic_module/1) ++ OnLoad, + + %% Fail because there is a module with on_load in the list. + on_load_failure(OnLoadMod, Ms), + on_load_failure(OnLoadMod, [lists:last(Ms)]), + + %% Fail because there already is a pending on_load. + [{HangingOnLoad,_,_}|_] = Ms, + spawn_hanging_on_load(HangingOnLoad), + NoOnLoadMs = lists:droplast(Ms), + {error,[{HangingOnLoad,pending_on_load}]} = + code:atomic_load(NoOnLoadMs), + hanging_on_load ! stop_hanging_and_unload, + + ok. + +on_load_failure(OnLoadMod, Ms) -> + [{OnLoadMod,on_load_not_allowed}] = atomic_load_error(Ms, false). + +spawn_hanging_on_load(Mod) -> + {Mod,Name,Bin} = make_module(Mod, "unknown", + fun(_) -> + hanging_on_load_module(Mod) + end), + spawn_link(fun() -> + {error,on_load_failure} = + code:load_binary(Mod, Name, Bin) + end). + +hanging_on_load_module(Mod) -> + ?Q(["-module('@Mod@').\n", + "-on_load(hang/0).\n", + "hang() ->\n" + " register(hanging_on_load, self()),\n" + " receive _ -> unload end.\n"]). + +ensure_modules_loaded(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + Dir = filename:join(PrivDir, multi_load_ensure_modules_loaded), + _ = file:make_dir(Dir), + + OldPath = code:get_path(), + try + code:add_patha(Dir), + do_ensure_modules_loaded(Dir) + after + code:set_path(OldPath) + end, + + ok. + +do_ensure_modules_loaded(Dir) -> + %% Create a dummy "lists" module and place it in our code path. + {lists,ListsFile,ListsCode} = make_module(lists, fun basic_module/1), + ok = file:write_file(filename:absname(ListsFile, Dir), ListsCode), + {error,sticky_directory} = code:load_file(lists), + + %% Make a new module that we can load. + Mod = make_module_file(Dir, fun basic_module/1), + false = is_loaded(Mod), + + %% Make a new module with an on_load function. + OLMod = make_module_file(Dir, fun on_load_module/1), + false = is_loaded(OLMod), + + %% lists should not be loaded again; Mod and OLMod should be + %% loaded. ?MODULE should not be reloaded, but there is no easy + %% way to test that. Repeating modules is OK. + ok = code:ensure_modules_loaded([?MODULE,lists,Mod,OLMod, + Mod,OLMod,Mod,lists]), + last = lists:last([last]), + true = is_loaded(Mod), + ok = Mod:Mod(), + true = is_loaded(OLMod), + _ = OLMod:module_info(), + + %% Unload the modules that were loaded. + [begin + code:purge(M), + code:delete(M), + code:purge(M), + false = is_loaded(M) + end || M <- [Mod,OLMod]], + + %% If there are some errors, all other modules should be loaded + %% anyway. + [{BadMod,BadFile,_}] = make_modules(1, fun basic_module/1), + ok = file:write_file(filename:absname(BadFile, Dir), <<"bad_code">>), + BadOLMod = make_module_file(Dir, fun failing_on_load_module/1), + BadEgg = bad__egg, + NativeMod = a_native_module, + NativeModFile = atom_to_list(NativeMod) ++ ".beam", + {NativeMod,_,NativeCode} = make_module(NativeMod, NativeModFile, + fun basic_module/1, [native]), + ok = file:write_file(filename:absname(NativeModFile, Dir), NativeCode), + ModulesToLoad = [OLMod,?MODULE,Mod,BadOLMod,NativeMod, + BadEgg,BadMod,lists], + {error,Error0} = code:ensure_modules_loaded(ModulesToLoad), + Error = lists:sort([{BadEgg,nofile}, + {BadMod,badfile}, + {BadOLMod,on_load_failure}]), + Error = lists:sort(Error0), + true = is_loaded(Mod), + true = is_loaded(OLMod), + true = is_loaded(NativeMod), + true = NativeMod:module_info(native), + + ok. + +failing_on_load_module(Mod) -> + ?Q(["-module('@Mod@').\n", + "-on_load(f/0).\n", + "f() -> fail.\n"]). + +native_code(_Config) -> + case erlang:system_info(hipe_architecture) of + undefined -> + {skip,"No native support"}; + _ -> + do_native_code() + end. + +do_native_code() -> + CalledMod = native_called_module, + CallingMod = native_calling_module, + + %% Create a module in native code that calls another module. + CallingMod = make_and_load(CallingMod, + calling_module_fun(CalledMod), + [native]), + + %% Create a threaded-code module. + _ = make_and_load(CalledMod, called_module_fun(42), []), + 42 = CallingMod:call(), + + %% Now replace it with a changed module in native code. + code:purge(CalledMod), + make_and_load(CalledMod, called_module_fun(43), [native]), + true = test_server:is_native(CalledMod), + 43 = CallingMod:call(), + + %% Reload the called module and call it. + code:purge(CalledMod), + ModVer3 = make_module(CalledMod, "", called_module_fun(changed)), + ok = code:atomic_load([ModVer3]), + false = test_server:is_native(CalledMod), + changed = CallingMod:call(), + code:purge(CalledMod), + + ok. + +make_and_load(Mod, Fun, Opts) -> + {Mod,_,Code} = make_module(Mod, "", Fun, Opts), + {module,Mod} = code:load_binary(Mod, "", Code), + Mod. + +calling_module_fun(Called) -> + fun(Mod) -> + ?Q(["-module('@Mod@').\n", + "-export([call/0]).\n", + "call() -> _@Called@:f().\n"]) + end. + +called_module_fun(Ret) -> + fun(Mod) -> + ?Q(["-module('@Mod@').\n", + "-export([f/0]).\n", + "f() -> _@Ret@.\n"]) + end. + +%%% +%%% Common utilities +%%% + +atomic_load_error(Modules, ErrorInFinishLoading) -> + {error,Errors0} = code:atomic_load(Modules), + {Errors1,Bool} = + case code:prepare_loading(Modules) of + {ok,Prepared} -> + {error,Es0} = code:finish_loading(Prepared), + {Es0,true}; + {error,Es0} -> + {Es0,false} + end, + Errors = lists:sort(Errors0), + Errors = lists:sort(Errors1), + case {ErrorInFinishLoading,Bool} of + {B,B} -> + Errors; + {false,true} -> + ct:fail("LastAction fun must not be called"); + {true,false} -> + ct:fail("LastAction fun was not called") + end. + +is_loaded(Mod) -> + case erlang:module_loaded(Mod) of + false -> + false = code:is_loaded(Mod); + true -> + {file,_} = code:is_loaded(Mod), + true + end. + +basic_module(Mod) -> + ?Q(["-module('@Mod@').\n" + "-export(['@Mod@'/0]).\n", + "'@Mod@'() -> ok."]). + +on_load_module(Mod) -> + ?Q(["-module('@Mod@').\n", + "-on_load(f/0).\n", + "f() -> ok.\n"]). + +make_module_file(Dir, Fun) -> + [{Mod,File,Code}] = make_modules(1, Fun), + ok = file:write_file(filename:absname(File, Dir), Code), + Mod. + +make_modules(0, _) -> + []; +make_modules(N, Fun) -> + U = erlang:unique_integer([positive]), + ModName = "m__" ++ integer_to_list(N) ++ "_" ++ integer_to_list(U), + Mod = list_to_atom(ModName), + ModItem = make_module(Mod, Fun), + [ModItem|make_modules(N-1, Fun)]. + +update_modules(Ms, Fun) -> + [make_module(M, Fun) || M <- Ms]. + +make_module(Mod, Fun) -> + Filename = atom_to_list(Mod) ++ ".beam", + make_module(Mod, Filename, Fun). + +make_module(Mod, Filename, Fun) -> + make_module(Mod, Filename, Fun, []). + +make_module(Mod, Filename, Fun, Opts) -> + Tree = Fun(Mod), + merl:print(Tree), + {ok,Mod,Code} = merl:compile(Tree, Opts), + {Mod,Filename,Code}. diff --git a/lib/sasl/doc/src/Makefile b/lib/sasl/doc/src/Makefile index 1ee48af338..a66b1f8bcb 100644 --- a/lib/sasl/doc/src/Makefile +++ b/lib/sasl/doc/src/Makefile @@ -36,7 +36,6 @@ RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) # ---------------------------------------------------- XML_APPLICATION_FILES = ref_man.xml XML_REF3_FILES = alarm_handler.xml \ - overload.xml \ rb.xml \ release_handler.xml \ systools.xml diff --git a/lib/sasl/doc/src/overload.xml b/lib/sasl/doc/src/overload.xml deleted file mode 100644 index 2f19cd9088..0000000000 --- a/lib/sasl/doc/src/overload.xml +++ /dev/null @@ -1,152 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE erlref SYSTEM "erlref.dtd"> - -<erlref> - <header> - <copyright> - <year>1996</year><year>2013</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>overload</title> - <prepared>Peter Högfeldt</prepared> - <responsible>Peter Högfeldt</responsible> - <docno></docno> - <approved>(Joe Armstrong)</approved> - <checked></checked> - <date>1996-10-29</date> - <rev>A</rev> - <file>overload.sgml</file> - </header> - <module>overload</module> - <modulesummary>An Overload Regulation Process</modulesummary> - <description> - <warning> - <p> - All functions in this module are deprecated and will be - removed in a future release. - </p> - </warning> - <p><c>overload</c> is a process that indirectly regulates the CPU - usage in the system. The idea is that a main application calls - function - <seealso marker="#request/0"><c>request/0</c></seealso> - before starting a major job and - proceeds with the job if the return value is positive; otherwise - the job must not be started.</p> - <p><c>overload</c> is part of the <c>SASL</c> application and all - configuration parameters are defined there.</p> - <p>A set of two intensities are maintained, the <c>total intensity</c> - and the <c>accept intensity</c>. For that purpose, - there are two configuration parameters, <c>MaxIntensity</c> - and <c>Weight</c>; both are measured in 1/second.</p> - <p>Then total and accept intensities are calculated as - follows. Assume that the time of the current call to - <c>request/0</c> is <c>T(n)</c> and that the time of the - previous call was <c>T(n-1)</c>.</p> - <list type="bulleted"> - <item> - <p>The current <c>total intensity</c>, denoted - <c>TI(n)</c>, is calculated according to the formula</p> - <p><c>TI(n) = exp(-Weight*(T(n) - T(n-1)) * TI(n-1) + Weight</c>,</p> - <p>where <c>TI(n-1)</c> is the previous <c>total intensity</c>.</p> - </item> - <item> - <p>The current <c>accept intensity</c>, denoted - <c>AI(n)</c>, is determined by the formula</p> - <p><c>AI(n) = exp(-Weight*(T(n) - T(n-1)) * AI(n-1) + Weight</c>,</p> - <p>where <c>AI(n-1)</c> is the previous <c>accept intensity</c>, - if the value of <c>exp(-Weight*(T(n) - T(n-1)) * AI(n-1)</c> - is less than <c>MaxIntensity</c>. Otherwise the value is</p> - <p><c>AI(n) = exp(-Weight*(T(n) - T(n-1)) * AI(n-1)</c></p> - </item> - </list> - <p>The value of configuration parameter <c>Weight</c> controls the - speed with which the calculations of intensities react to - changes in the underlying input intensity. The inverted value of - <c>Weight</c>, <c>T = 1/Weight</c>, can be thought of as the - "time constant" of the intensity calculation formulas. For example, - if <c>Weight = 0.1</c>, a change in the underlying input intensity is - reflected in <c>total intensity</c> and <c>accept intensity</c> within - about 10 seconds.</p> - <p>The overload process defines one alarm, which it sets using - <c>alarm_handler:set_alarm(Alarm)</c>. <c>Alarm</c> is defined - as follows:</p> - <taglist> - <tag><c>{overload, []}</c></tag> - <item> - <p>This alarm is set when the current <c>accept intensity</c> exceeds - <c>MaxIntensity</c>.</p> - </item> - </taglist> - <p>A new request is not accepted until the current <c>accept - intensity</c> has fallen below <c>MaxIntensity</c>. To prevent the - overload process from generating many set/reset alarms, the - alarm is not reset until the current <c>accept intensity</c> has fallen - below 75% of <c>MaxIntensity</c>; it is not until then that - the alarm can be set again.</p> - </description> - - <funcs> - <func> - <name>request() -> accept | reject</name> - <fsummary>Requests to proceed with current job.</fsummary> - <desc> - <p>Returns <c>accept</c> or <c>reject</c> depending on the - current value of the <c>accept intensity</c>.</p> - <p>The application - calling this function is to proceed with the job in - question if the return value is <c>accept</c>; otherwise it - is not to continue with that job.</p> - </desc> - </func> - - <func> - <name>get_overload_info() -> OverloadInfo</name> - <fsummary>Returns current overload information data.</fsummary> - <type> - <v>OverloadInfo = [{total_intensity, TotalIntensity}, - {accept_intensity, AcceptIntensity}, {max_intensity, - MaxIntensity}, {weight, Weight}, {total_requests, - TotalRequests}, {accepted_requests, AcceptedRequests}].</v> - <v>TotalIntensity = float() > 0</v> - <v>AcceptIntensity = float() > 0</v> - <v>MaxIntensity = float() > 0</v> - <v>Weight = float() > 0</v> - <v>TotalRequests = integer()</v> - <v>AcceptedRequests = integer()</v> - </type> - <desc> - <p>Returns:</p> - <list type="bulleted"> - <item>Current total and accept intensities</item> - <item>Configuration parameters</item> - <item>Absolute counts of the total number of requests</item> - <item>Accepted number of requests (since the overload - process was started)</item> - </list> - </desc> - </func> - </funcs> - - <section> - <title>See Also</title> - <p><seealso marker="alarm_handler"><c>alarm_handler(3)</c></seealso>, - <seealso marker="sasl_app"><c>sasl(6)</c></seealso></p> - </section> -</erlref> - diff --git a/lib/sasl/doc/src/ref_man.xml b/lib/sasl/doc/src/ref_man.xml index a80e5a2a00..74cd92a9b0 100644 --- a/lib/sasl/doc/src/ref_man.xml +++ b/lib/sasl/doc/src/ref_man.xml @@ -35,7 +35,6 @@ </description> <xi:include href="sasl_app.xml"/> <xi:include href="alarm_handler.xml"/> - <xi:include href="overload.xml"/> <xi:include href="rb.xml"/> <xi:include href="release_handler.xml"/> <xi:include href="systools.xml"/> diff --git a/lib/sasl/doc/src/sasl_app.xml b/lib/sasl/doc/src/sasl_app.xml index bcd446a868..044c016302 100644 --- a/lib/sasl/doc/src/sasl_app.xml +++ b/lib/sasl/doc/src/sasl_app.xml @@ -34,7 +34,6 @@ <p>The <c>SASL</c> application provides the following services:</p> <list type="bulleted"> <item><c>alarm_handler</c></item> - <item><c>overload</c> (deprecated)</item> <item><c>rb</c></item> <item><c>release_handler</c></item> <item><c>systools</c></item> @@ -140,21 +139,6 @@ this parameter is undefined, the <c>log_mf_h</c> handler is not installed.</p> </item> - <tag><c><![CDATA[overload_max_intensity = float() > 0 ]]></c></tag> - <item> - <p>Specifies the maximum intensity - for <seealso marker="overload"><c>overload</c></seealso>. Default - is <c>0.8</c>.</p> - <p>Note that the <c>overload</c> module is deprected and - will be removed in a future release.</p> - </item> - <tag><c><![CDATA[overload_weight = float() > 0 ]]></c></tag> - <item> - <p>Specifies the <seealso marker="overload"><c>overload</c></seealso> - weight. Default is <c>0.1</c>.</p> - <p>Note that the <c>overload</c> module is deprected and - will be removed in a future release.</p> - </item> <tag><c><![CDATA[start_prg = string() ]]></c></tag> <item> <p>Specifies the program to be used when restarting the system @@ -205,7 +189,6 @@ <p><seealso marker="alarm_handler"><c>alarm_handler(3)</c></seealso>, <seealso marker="kernel:error_logger"><c>error_logger(3)</c></seealso>, <seealso marker="stdlib:log_mf_h"><c>log_mf_h(3)</c></seealso>, - <seealso marker="overload"><c>overload(3)</c></seealso>, <seealso marker="rb"><c>rb(3)</c></seealso>, <seealso marker="release_handler"><c>release_handler(3)</c></seealso>, <seealso marker="systools"><c>systools(3)</c></seealso></p> diff --git a/lib/sasl/doc/src/sasl_intro.xml b/lib/sasl/doc/src/sasl_intro.xml index bbc9457103..237580977c 100644 --- a/lib/sasl/doc/src/sasl_intro.xml +++ b/lib/sasl/doc/src/sasl_intro.xml @@ -36,7 +36,6 @@ <list type="bulleted"> <item>Error logging</item> <item>Alarm handling</item> - <item>Overload regulation</item> <item>Release handling</item> <item>Report browsing</item> </list> diff --git a/lib/sasl/src/Makefile b/lib/sasl/src/Makefile index 7ff6a03a50..92d4818f2e 100644 --- a/lib/sasl/src/Makefile +++ b/lib/sasl/src/Makefile @@ -36,7 +36,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/sasl-$(VSN) # ---------------------------------------------------- MODULES= alarm_handler sasl sasl_report \ sasl_report_file_h sasl_report_tty_h format_lib_supp \ - misc_supp overload rb rb_format_supp release_handler \ + misc_supp rb rb_format_supp release_handler \ release_handler_1 si si_sasl_supp systools \ systools_make systools_rc systools_relup systools_lib \ erlsrv diff --git a/lib/sasl/src/misc_supp.erl b/lib/sasl/src/misc_supp.erl index 42de7eedec..39656f6e65 100644 --- a/lib/sasl/src/misc_supp.erl +++ b/lib/sasl/src/misc_supp.erl @@ -27,7 +27,7 @@ %%% 2) Very generic functions such as, multi_map, is_string... %%% %%% This module is a part of the BOS. (format_pdict is called from -%%% init, memsup, disksup, overload (but not the fileserver since it +%%% init, memsup, disksup (but not the fileserver since it %%% formats its pdict its own way).) %%%--------------------------------------------------------------------- diff --git a/lib/sasl/src/overload.erl b/lib/sasl/src/overload.erl deleted file mode 100644 index bc8ab7d5e4..0000000000 --- a/lib/sasl/src/overload.erl +++ /dev/null @@ -1,233 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% --module(overload). - --deprecated(module). - --export([start_link/0, request/0, set_config_data/2, - get_overload_info/0]). - --export([init/1, handle_call/3, handle_info/2, terminate/2, - format_status/2]). - -%%%----------------------------------------------------------------- -%%% This is a rewrite of overload from BS.3, by Peter Högfeldt. -%%% -%%% DESCRIPTION -%%% -%%% This module implements a server process that keeps record of the -%%% intensity of calls of the request/0 function, and answers accept or -%%% reject depending on if the current average intensity is not greater -%%% than a specified maximum intensity. -%%% -%%% The intensity i is calculated according to the formula: -%%% i(n) = exp(-K*(T(n) - T(n-1)))*i(n-1) + Kappa -%%% where i(n) is the intensity at event n, Kappa is a constant, and -%%% T(n) is the time at event n. -%%% -%%% The constant Kappa can be thought of as 1 / T, where T is the time -%%% constant. Kappa is externally referred to as Weight. -%%% -%%% We keep track of two intensities: the total call intensity and the -%%% intensity of accepted calls. -%%%----------------------------------------------------------------- -%%% TODO -%%% -%%% 3. Hysteresis. -%%% -%%%----------------------------------------------------------------- - --record(state, {total = 0, accept = 0, max, prev_t = get_now(), - kappa, call_counts = {0, 0}, alarm = clear}). - --define(clear_timeout, 30000). - -start_link() -> - gen_server:start_link({local, overload}, overload, [], []). - -init([]) -> - process_flag(priority, high), - MaxIntensity = fetch_config_data(overload_max_intensity), - Kappa = fetch_config_data(overload_weight), - {ok, #state{max = MaxIntensity, kappa = Kappa}}. - -%%----------------------------------------------------------------- -%% Func: request/0 -%% Purpose: This is a request to proceed, e.g. a request to -%% establish a call. -%% Returns: accept | reject -%%----------------------------------------------------------------- -request() -> call(request). - -%%----------------------------------------------------------------- -%% Func: set_config_data/2 -%% Purpose: Set configuration data, and reset intensities. -%% Arguments: MaxIntensity (real > 0), Weight (real > 0). -%% Returns: ok | {error, What} -%% This function is for debugging purposes and is therefore not -%% documented at all. -%%----------------------------------------------------------------- -set_config_data(MaxIntensity, Weight) -> - call({set_config_data, MaxIntensity, Weight}). -%%----------------------------------------------------------------- -%% Func: get_overload_info/0 -%% Returns: A list of tagged items: TotalIntensity, AcceptIntensity, -%% MaxIntensity, Weight, TotalRequests, AcceptedRequests. -%%----------------------------------------------------------------- -get_overload_info() -> call(get_overload_info). - -%%----------------------------------------------------------------- -%% call(Request) -> Term -%%----------------------------------------------------------------- -call(Req) -> - gen_server:call(overload, Req, infinity). - -%%%----------------------------------------------------------------- -%%% Callback functions from gen_server -%%%----------------------------------------------------------------- -handle_call(request, _From, State) -> - #state{total = TI, accept = AI, kappa = Kappa, prev_t = PrevT, - alarm = Alarm} = State, - {TR, AR} = State#state.call_counts, - T = get_now(), - CurI = new_intensity(AI, T, PrevT, Kappa), - NewTI = new_intensity(TI, T, PrevT, Kappa) + Kappa, - if - CurI =< State#state.max -> - %% Hysteresis: If alarm is set, and current intensity has - %% fallen below 75% of max intensity, clear alarm. - NewAlarm = if - CurI =< 0.75*State#state.max -> - clear_alarm(Alarm); - true -> - Alarm - end, - {reply, accept, State#state{call_counts = {TR+1, AR+1}, - prev_t = T, total = NewTI, - accept = CurI + Kappa, - alarm = NewAlarm}, - ?clear_timeout}; - true -> - %% Set alarm if not already set. - NewAlarm = set_alarm(Alarm), - {reply, reject, - State#state{call_counts = {TR+1, AR}, prev_t = T, - total = NewTI, accept = CurI, - alarm = NewAlarm}, - ?clear_timeout} - end; -handle_call({set_config_data, MaxIntensity, Weight}, _From, _State) -> - {reply, ok, #state{max = MaxIntensity, kappa = Weight}, - ?clear_timeout}; -handle_call(get_overload_info, _From, State) -> - #state{max = MI, total = TI, accept = AI, kappa = Kappa, - prev_t = PrevT, call_counts = {TR, AR}} = State, - T = get_now(), - CurI = new_intensity(AI, T, PrevT, Kappa), - NewTI = new_intensity(TI, T, PrevT, Kappa), - Reply = [{total_intensity, NewTI}, {accept_intensity, CurI}, - {max_intensity, MI}, {weight, Kappa}, - {total_requests, TR}, {accepted_requests, AR}], - {reply, Reply, State#state{total = NewTI, accept = CurI}, - ?clear_timeout}. - -handle_info(timeout, State) -> - #state{total = TI, accept = AI, kappa = Kappa, prev_t = PrevT, - alarm = Alarm} = State, - T = get_now(), - CurI = new_intensity(AI, T, PrevT, Kappa), - NewTI = new_intensity(TI, T, PrevT, Kappa), - if - CurI < 0.75* State#state.max -> - NewAlarm = clear_alarm(Alarm), - {noreply, State#state{total = NewTI, accept = CurI, - alarm = NewAlarm}}; - - true -> - {noreply, State#state{total = NewTI, accept = CurI}, - ?clear_timeout} - end; - -handle_info(_, State) -> - {noreply, State, ?clear_timeout}. - -terminate(_Reason, _State) -> - ok. - -%%----------------------------------------------------------------- -%% Internal functions -%%----------------------------------------------------------------- -fetch_config_data(Tag) -> - case application:get_env(sasl, Tag) of - {ok, Value} -> Value; - _ -> fetch_default_data(Tag) - end. - -fetch_default_data(overload_max_intensity) -> 0.8; -fetch_default_data(overload_weight) -> 0.1. - -set_alarm(clear) -> - alarm_handler:set_alarm({overload, []}), - set; -set_alarm(Alarm) -> - Alarm. - -clear_alarm(set) -> - alarm_handler:clear_alarm(overload), - clear; -clear_alarm(Alarm) -> - Alarm. - -%%----------------------------------------------------------------- -%% The catch protects against floating-point exception. -%% -new_intensity(I, T, PrevT, K) -> - Diff = sub(T, PrevT)/1000, - case catch (I*math:exp(-K*Diff)) of - {'EXIT', _} -> % Assume zero. - 0.0; - Res -> - Res - end. - -%% Mask equal to 2^27 - 1, used below. --define(mask27, 16#7ffffff). - -%% Returns number of milliseconds in the range [0, 2^27 - 1]. Must have -%% this since statistics(wall_clock) wraps. Having 2^27 -1 as the max -%% assures that we always get non-negative integers. 2^27 milliseconds -%% are approx. 37.28 hours. -get_now() -> - element(1, statistics(wall_clock)) band ?mask27. - -%% Returns (X - Y) mod 2^27 (which is in the range [0, 2^27 - 1]). -sub(X, Y) -> - (X + (bnot Y) + 1) band ?mask27. - -format_status(Opt, [PDict, #state{max = MI, total = TI, accept = AI, - kappa = K, - call_counts = {TR, AR}}]) -> - [{data, [{"Total Intensity", TI}, - {"Accept Intensity", AI}, - {"Max Intensity", MI}, - {"Weight", K}, - {"Total requests", TR}, - {"Accepted requests", AR}]} | - misc_supp:format_pdict(Opt, PDict, [])]. diff --git a/lib/sasl/src/sasl.app.src b/lib/sasl/src/sasl.app.src index 507e2dc229..16e8e44ba2 100644 --- a/lib/sasl/src/sasl.app.src +++ b/lib/sasl/src/sasl.app.src @@ -24,7 +24,6 @@ alarm_handler, format_lib_supp, misc_supp, - overload, rb, rb_format_supp, release_handler, @@ -41,7 +40,7 @@ systools_relup, systools_lib ]}, - {registered, [sasl_sup, alarm_handler, overload, release_handler]}, + {registered, [sasl_sup, alarm_handler, release_handler]}, {applications, [kernel, stdlib]}, {env, [{sasl_error_logger, tty}, {errlog_type, all}]}, diff --git a/lib/sasl/src/sasl.erl b/lib/sasl/src/sasl.erl index fc49fc2465..0fb85682c6 100644 --- a/lib/sasl/src/sasl.erl +++ b/lib/sasl/src/sasl.erl @@ -170,7 +170,4 @@ init(safe) -> AlarmH = {alarm_handler, {alarm_handler, start_link, []}, permanent, 2000, worker, dynamic}, - Overload = {overload, - {overload, start_link, []}, - permanent, 2000, worker, [overload]}, - {ok, {SupFlags, [AlarmH, Overload]}}. + {ok, {SupFlags, [AlarmH]}}. diff --git a/lib/sasl/test/Makefile b/lib/sasl/test/Makefile index 6f67498714..2e9e107efd 100644 --- a/lib/sasl/test/Makefile +++ b/lib/sasl/test/Makefile @@ -33,7 +33,6 @@ MODULES= \ sasl_report_suite_supervisor \ systools_SUITE \ systools_rc_SUITE \ - overload_SUITE \ rb_SUITE \ rh_test_lib \ diff --git a/lib/sasl/test/overload_SUITE.erl b/lib/sasl/test/overload_SUITE.erl deleted file mode 100644 index 02d1056698..0000000000 --- a/lib/sasl/test/overload_SUITE.erl +++ /dev/null @@ -1,168 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2011. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% - --module(overload_SUITE). --include_lib("common_test/include/ct.hrl"). - --compile(export_all). - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -all() -> [info, set_config_data, set_env_vars, request, timeout]. - -init_per_testcase(_Case,Config) -> - restart_sasl(), - Config. - -end_per_testcase(Case,Config) -> - try apply(?MODULE,Case,[cleanup,Config]) - catch error:undef -> ok - end, - ok. - -%%%----------------------------------------------------------------- -info(_Config) -> - Info = overload:get_overload_info(), - [{total_intensity,0.0}, - {accept_intensity,0.0}, - {max_intensity,0.8}, - {weight,0.1}, - {total_requests,0}, - {accepted_requests,0}] = Info. - -%%%----------------------------------------------------------------- -set_config_data(_Config) -> - InfoDefault = overload:get_overload_info(), - ok = check_info(0.8,0.1,InfoDefault), - ok = overload:set_config_data(0.5,0.4), - Info1 = overload:get_overload_info(), - ok = check_info(0.5,0.4,Info1), - ok. - -%%%----------------------------------------------------------------- -set_env_vars(_Config) -> - InfoDefault = overload:get_overload_info(), - ok = check_info(0.8,0.1,InfoDefault), - ok = application:set_env(sasl,overload_max_intensity,0.5), - ok = application:set_env(sasl,overload_weight,0.4), - ok = application:stop(sasl), - ok = application:start(sasl), - Info1 = overload:get_overload_info(), - ok = check_info(0.5,0.4,Info1), - ok. -set_env_vars(cleanup,_Config) -> - application:unset_env(sasl,overload_max_intensity), - application:unset_env(sasl,overload_weight), - ok. - -%%%----------------------------------------------------------------- -request(_Config) -> - %% Find number of request that can be done with default settings - %% and no delay - overload:set_config_data(0.8, 0.1), - NDefault = do_many_requests(0), - restart_sasl(), - ?t:format("NDefault: ~p",[NDefault]), - - %% Check that the number of requests increases when max_intensity - %% increases - overload:set_config_data(2, 0.1), - NLargeMI = do_many_requests(0), - restart_sasl(), - ?t:format("NLargeMI: ~p",[NLargeMI]), - true = NLargeMI > NDefault, - - %% Check that the number of requests decreases when weight - %% increases - overload:set_config_data(0.8, 1), - NLargeWeight = do_many_requests(0), - restart_sasl(), - ?t:format("NLargeWeight: ~p",[NLargeWeight]), - true = NLargeWeight < NDefault, - - %% Check that number of requests increases when delay between - %% requests increases. - %% (Keeping same config and comparing to large weight in order to - %% minimize the time needed for this case.) - overload:set_config_data(0.8, 1), - NLargeTime = do_many_requests(500), - restart_sasl(), - ?t:format("NLargeTime: ~p",[NLargeTime]), - true = NLargeTime > NLargeWeight, - ok. - -%%%----------------------------------------------------------------- -timeout(_Config) -> - overload:set_config_data(0.8, 1), - _N = do_many_requests(0), - - %% Check that the overload alarm is raised - [{overload,_}] = alarm_handler:get_alarms(), - - %% Fake a clear timeout in overload.erl and check that, since it - %% came very soon after the overload situation, the alarm is not - %% cleared - overload ! timeout, - timer:sleep(1000), - [{overload,_}] = alarm_handler:get_alarms(), - - %% A bit later, try again and check that this time the alarm is - %% cleared - overload ! timeout, - timer:sleep(1000), - [] = alarm_handler:get_alarms(), - - ok. - - -%%%----------------------------------------------------------------- -%%% INTERNAL FUNCTIONS - -%%%----------------------------------------------------------------- -%%% Call overload:request/0 up to 30 times with the given time delay -%%% between. Stop when 'reject' is returned. -do_many_requests(T) -> - 30 - do_requests(30,T). - -do_requests(0,_) -> - ?t:fail(never_rejected); -do_requests(N,T) -> - case overload:request() of - accept -> - timer:sleep(T), - do_requests(N-1,T); - reject -> - N - end. - -%%%----------------------------------------------------------------- -%%% Restart the sasl application -restart_sasl() -> - application:stop(sasl), - application:start(sasl), - ok. - -%%%----------------------------------------------------------------- -%%% Check that max_intensity and weight is set as expected -check_info(MI,W,Info) -> - case {lists:keyfind(max_intensity,1,Info), lists:keyfind(weight,1,Info)} of - {{_,MI},{_,W}} -> ok; - _ -> ?t:fail({unexpected_info,MI,W,Info}) - end. diff --git a/lib/ssh/doc/src/ssh_sftp.xml b/lib/ssh/doc/src/ssh_sftp.xml index c6ca0f161a..f4b41b74f3 100644 --- a/lib/ssh/doc/src/ssh_sftp.xml +++ b/lib/ssh/doc/src/ssh_sftp.xml @@ -333,7 +333,7 @@ <func> <name>position(ChannelPid, Handle, Location) -></name> - <name>position(ChannelPid, Handle, Location, Timeout) -> {ok, NewPosition | {error, Error}</name> + <name>position(ChannelPid, Handle, Location, Timeout) -> {ok, NewPosition} | {error, Reason}</name> <fsummary>Sets the file position of a file.</fsummary> <type> <v>ChannelPid = pid()</v> @@ -399,7 +399,7 @@ <func> <name>pwrite(ChannelPid, Handle, Position, Data) -> ok</name> - <name>pwrite(ChannelPid, Handle, Position, Data, Timeout) -> ok | {error, Error}</name> + <name>pwrite(ChannelPid, Handle, Position, Data, Timeout) -> ok | {error, Reason}</name> <fsummary>Writes to an open file.</fsummary> <type> <v>ChannelPid = pid()</v> @@ -592,7 +592,7 @@ <func> <name>write(ChannelPid, Handle, Data) -></name> - <name>write(ChannelPid, Handle, Data, Timeout) -> ok | {error, Error}</name> + <name>write(ChannelPid, Handle, Data, Timeout) -> ok | {error, Reason}</name> <fsummary>Writes to an open file.</fsummary> <type> <v>ChannelPid = pid()</v> diff --git a/lib/ssh/vsn.mk b/lib/ssh/vsn.mk index 55d12abffe..41b42d454b 100644 --- a/lib/ssh/vsn.mk +++ b/lib/ssh/vsn.mk @@ -1,5 +1,5 @@ #-*-makefile-*- ; force emacs to enter makefile-mode -SSH_VSN = 4.2.1 +SSH_VSN = 4.2.2 APP_VSN = "ssh-$(SSH_VSN)" diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index d3881ad117..a76d46ee9b 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -271,7 +271,11 @@ atom()}} | terminate regarding verification failures and the connection is established.</p></item> <item><p>If called with an extension unknown to the user application, - return value <c>{unknown, UserState}</c> is to be used.</p></item> + return value <c>{unknown, UserState}</c> is to be used.</p> + + <p>Note that if the fun returns <c>unknown</c> for an extension marked + as critical, validation will fail.</p> + </item> </list> <p>Default option <c>verify_fun</c> in <c>verify_peer mode</c>:</p> @@ -293,6 +297,8 @@ atom()}} | <code> {fun(_,{bad_cert, _}, UserState) -> {valid, UserState}; + (_,{extension, #'Extension'{critical = true}}, UserState) -> + {valid, UserState}; (_,{extension, _}, UserState) -> {unknown, UserState}; (_, valid, UserState) -> diff --git a/lib/ssl/examples/src/client_server.erl b/lib/ssl/examples/src/client_server.erl index 799027123f..019b5130d2 100644 --- a/lib/ssl/examples/src/client_server.erl +++ b/lib/ssl/examples/src/client_server.erl @@ -26,9 +26,7 @@ start() -> %% Start ssl application - application:start(crypto), - application:start(public_key), - application:start(ssl), + {ok, StartedApps} = application:ensure_all_started(ssl), %% Let the current process be the server that listens and accepts %% Listen @@ -52,7 +50,8 @@ start() -> ssl:close(ASock), io:fwrite("Listen: closing and terminating.~n"), ssl:close(LSock), - application:stop(ssl). + + lists:foreach(fun application:stop/1, lists:reverse(StartedApps)). %% Client connect diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index c1bc90559e..780bef5877 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -60,22 +60,19 @@ -spec start() -> ok | {error, reason()}. -spec start(permanent | transient | temporary) -> ok | {error, reason()}. %% -%% Description: Utility function that starts the ssl, -%% crypto and public_key applications. Default type -%% is temporary. see application(3) +%% Description: Utility function that starts the ssl and applications +%% that it depends on. +%% see application(3) %%-------------------------------------------------------------------- start() -> - application:start(crypto), - application:start(asn1), - application:start(public_key), - application:start(ssl). - + start(temporary). start(Type) -> - application:start(crypto, Type), - application:start(asn1), - application:start(public_key, Type), - application:start(ssl, Type). - + case application:ensure_all_started(ssl, Type) of + {ok, _} -> + ok; + Other -> + Other + end. %%-------------------------------------------------------------------- -spec stop() -> ok. %% @@ -1296,6 +1293,12 @@ handle_verify_options(Opts, CaCerts) -> DefaultVerifyNoneFun = {fun(_,{bad_cert, _}, UserState) -> {valid, UserState}; + (_,{extension, #'Extension'{critical = true}}, UserState) -> + %% This extension is marked as critical, so + %% certificate verification should fail if we don't + %% understand the extension. However, this is + %% `verify_none', so let's accept it anyway. + {valid, UserState}; (_,{extension, _}, UserState) -> {unknown, UserState}; (_, valid, UserState) -> diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index e9e140836b..e98073080a 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -2072,12 +2072,9 @@ crl_check(OtpCert, Check, CertDbHandle, CertDbRef, {Callback, CRLDbHandle}, _) - ], case dps_and_crls(OtpCert, Callback, CRLDbHandle, ext) of no_dps -> - case dps_and_crls(OtpCert, Callback, CRLDbHandle, same_issuer) of - [] -> - valid; %% No relevant CRL existed - DpsAndCRls -> - crl_check_same_issuer(OtpCert, Check, DpsAndCRls, Options) - end; + crl_check_same_issuer(OtpCert, Check, + dps_and_crls(OtpCert, Callback, CRLDbHandle, same_issuer), + Options); DpsAndCRLs -> %% This DP list may be empty if relevant CRLs existed %% but could not be retrived, will result in {bad_cert, revocation_status_undetermined} case public_key:pkix_crls_validate(OtpCert, DpsAndCRLs, Options) of diff --git a/lib/ssl/test/ssl_certificate_verify_SUITE.erl b/lib/ssl/test/ssl_certificate_verify_SUITE.erl index 968ef30791..d10506cb69 100644 --- a/lib/ssl/test/ssl_certificate_verify_SUITE.erl +++ b/lib/ssl/test/ssl_certificate_verify_SUITE.erl @@ -66,7 +66,9 @@ tests() -> invalid_signature_client, invalid_signature_server, extended_key_usage_verify_peer, - extended_key_usage_verify_none]. + extended_key_usage_verify_none, + critical_extension_verify_peer, + critical_extension_verify_none]. error_handling_tests()-> [client_with_cert_cipher_suites_handshake, @@ -795,6 +797,121 @@ extended_key_usage_verify_none(Config) when is_list(Config) -> ssl_test_lib:close(Client). %%-------------------------------------------------------------------- +critical_extension_verify_peer() -> + [{doc,"Test cert that has a critical unknown extension in verify_peer mode"}]. + +critical_extension_verify_peer(Config) when is_list(Config) -> + ClientOpts = ?config(client_verification_opts, Config), + ServerOpts = ?config(server_verification_opts, Config), + PrivDir = ?config(priv_dir, Config), + Active = ?config(active, Config), + ReceiveFunction = ?config(receive_function, Config), + + KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), + NewCertName = integer_to_list(erlang:unique_integer()) ++ ".pem", + + ServerCertFile = proplists:get_value(certfile, ServerOpts), + NewServerCertFile = filename:join([PrivDir, "server", NewCertName]), + add_critical_netscape_cert_type(ServerCertFile, NewServerCertFile, KeyFile), + NewServerOpts = [{certfile, NewServerCertFile} | proplists:delete(certfile, ServerOpts)], + + ClientCertFile = proplists:get_value(certfile, ClientOpts), + NewClientCertFile = filename:join([PrivDir, "client", NewCertName]), + add_critical_netscape_cert_type(ClientCertFile, NewClientCertFile, KeyFile), + NewClientOpts = [{certfile, NewClientCertFile} | proplists:delete(certfile, ClientOpts)], + + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server_error( + [{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, ReceiveFunction, []}}, + {options, [{verify, verify_peer}, {active, Active} | NewServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + Client = ssl_test_lib:start_client_error( + [{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {ssl_test_lib, ReceiveFunction, []}}, + {options, [{verify, verify_peer}, {active, Active} | NewClientOpts]}]), + + %% This certificate has a critical extension that we don't + %% understand. Therefore, verification should fail. + tcp_delivery_workaround(Server, {error, {tls_alert, "unsupported certificate"}}, + Client, {error, {tls_alert, "unsupported certificate"}}), + + ssl_test_lib:close(Server), + ok. + +%%-------------------------------------------------------------------- +critical_extension_verify_none() -> + [{doc,"Test cert that has a critical unknown extension in verify_none mode"}]. + +critical_extension_verify_none(Config) when is_list(Config) -> + ClientOpts = ?config(client_verification_opts, Config), + ServerOpts = ?config(server_verification_opts, Config), + PrivDir = ?config(priv_dir, Config), + Active = ?config(active, Config), + ReceiveFunction = ?config(receive_function, Config), + + KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), + NewCertName = integer_to_list(erlang:unique_integer()) ++ ".pem", + + ServerCertFile = proplists:get_value(certfile, ServerOpts), + NewServerCertFile = filename:join([PrivDir, "server", NewCertName]), + add_critical_netscape_cert_type(ServerCertFile, NewServerCertFile, KeyFile), + NewServerOpts = [{certfile, NewServerCertFile} | proplists:delete(certfile, ServerOpts)], + + ClientCertFile = proplists:get_value(certfile, ClientOpts), + NewClientCertFile = filename:join([PrivDir, "client", NewCertName]), + add_critical_netscape_cert_type(ClientCertFile, NewClientCertFile, KeyFile), + NewClientOpts = [{certfile, NewClientCertFile} | proplists:delete(certfile, ClientOpts)], + + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server( + [{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, ReceiveFunction, []}}, + {options, [{verify, verify_none}, {active, Active} | NewServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + Client = ssl_test_lib:start_client( + [{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {ssl_test_lib, ReceiveFunction, []}}, + {options, [{verify, verify_none}, {active, Active} | NewClientOpts]}]), + + %% This certificate has a critical extension that we don't + %% understand. But we're using `verify_none', so verification + %% shouldn't fail. + ssl_test_lib:check_result(Server, ok, Client, ok), + + ssl_test_lib:close(Server), + ssl_test_lib:close(Client), + ok. + +add_critical_netscape_cert_type(CertFile, NewCertFile, KeyFile) -> + [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), + Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), + + [{'Certificate', DerCert, _}] = ssl_test_lib:pem_to_der(CertFile), + OTPCert = public_key:pkix_decode_cert(DerCert, otp), + %% This is the "Netscape Cert Type" extension, telling us that the + %% certificate can be used for SSL clients and SSL servers. + NetscapeCertTypeExt = #'Extension'{ + extnID = {2,16,840,1,113730,1,1}, + critical = true, + extnValue = <<3,2,6,192>>}, + OTPTbsCert = OTPCert#'OTPCertificate'.tbsCertificate, + Extensions = OTPTbsCert#'OTPTBSCertificate'.extensions, + NewOTPTbsCert = OTPTbsCert#'OTPTBSCertificate'{ + extensions = [NetscapeCertTypeExt] ++ Extensions}, + NewDerCert = public_key:pkix_sign(NewOTPTbsCert, Key), + ssl_test_lib:der_to_pem(NewCertFile, [{'Certificate', NewDerCert, not_encrypted}]), + ok. + +%%-------------------------------------------------------------------- no_authority_key_identifier() -> [{doc, "Test cert that does not have authorityKeyIdentifier extension" " but are present in trusted certs db."}]. diff --git a/lib/ssl/test/ssl_crl_SUITE.erl b/lib/ssl/test/ssl_crl_SUITE.erl index 44580be1ff..5b86027210 100644 --- a/lib/ssl/test/ssl_crl_SUITE.erl +++ b/lib/ssl/test/ssl_crl_SUITE.erl @@ -53,7 +53,7 @@ groups() -> {idp_crl, [], basic_tests()}]. basic_tests() -> - [crl_verify_valid, crl_verify_revoked]. + [crl_verify_valid, crl_verify_revoked, crl_verify_no_crl]. init_per_suite(Config) -> @@ -186,11 +186,6 @@ crl_verify_revoked(Config) when is_list(Config) -> {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), - Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, - {from, self()}, - {options, ServerOpts}]), - Port = ssl_test_lib:inet_port(Server), - ssl_crl_cache:insert({file, filename:join([PrivDir, "erlangCA", "crl.pem"])}), ssl_crl_cache:insert({file, filename:join([PrivDir, "otpCA", "crl.pem"])}), @@ -206,16 +201,55 @@ crl_verify_revoked(Config) when is_list(Config) -> {verify, verify_peer}] end, - Client = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port}, - {host, Hostname}, - {from, self()}, - {options, ClientOpts}]), - receive - {Server, AlertOrColse} -> - ct:pal("Server Alert or Close ~p", [AlertOrColse]) - end, - ssl_test_lib:check_result(Client, {error, {tls_alert, "certificate revoked"}}). + crl_verify_error(Hostname, ServerNode, ServerOpts, ClientNode, ClientOpts, + "certificate revoked"). +crl_verify_no_crl() -> + [{doc,"Verify a simple CRL chain when the CRL is missing"}]. +crl_verify_no_crl(Config) when is_list(Config) -> + PrivDir = ?config(cert_dir, Config), + Check = ?config(crl_check, Config), + ServerOpts = [{keyfile, filename:join([PrivDir, "server", "key.pem"])}, + {certfile, filename:join([PrivDir, "server", "cert.pem"])}, + {cacertfile, filename:join([PrivDir, "server", "cacerts.pem"])}], + ClientOpts = case ?config(idp_crl, Config) of + true -> + [{cacertfile, filename:join([PrivDir, "server", "cacerts.pem"])}, + {crl_check, Check}, + {crl_cache, {ssl_crl_cache, {internal, [{http, 5000}]}}}, + {verify, verify_peer}]; + false -> + [{cacertfile, filename:join([PrivDir, "server", "cacerts.pem"])}, + {crl_check, Check}, + {verify, verify_peer}] + end, + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + %% In case we're running an HTTP server that serves CRLs, let's + %% rename those files, so the CRL is absent when we try to verify + %% it. + %% + %% If we're not using an HTTP server, we just need to refrain from + %% adding the CRLs to the cache manually. + rename_crl(filename:join([PrivDir, "erlangCA", "crl.pem"])), + rename_crl(filename:join([PrivDir, "otpCA", "crl.pem"])), + + %% The expected outcome when the CRL is missing depends on the + %% crl_check setting. + case Check of + true -> + %% The error "revocation status undetermined" gets turned + %% into "bad certificate". + crl_verify_error(Hostname, ServerNode, ServerOpts, ClientNode, ClientOpts, + "bad certificate"); + peer -> + crl_verify_error(Hostname, ServerNode, ServerOpts, ClientNode, ClientOpts, + "bad certificate"); + best_effort -> + %% In "best effort" mode, we consider the certificate not + %% to be revoked if we can't find the appropriate CRL. + crl_verify_valid(Hostname, ServerNode, ServerOpts, ClientNode, ClientOpts) + end. crl_verify_valid(Hostname, ServerNode, ServerOpts, ClientNode, ClientOpts) -> Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, @@ -236,6 +270,22 @@ crl_verify_valid(Hostname, ServerNode, ServerOpts, ClientNode, ClientOpts) -> ssl_test_lib:close(Server), ssl_test_lib:close(Client). +crl_verify_error(Hostname, ServerNode, ServerOpts, ClientNode, ClientOpts, ExpectedAlert) -> + Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, + {from, self()}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {options, ClientOpts}]), + receive + {Server, AlertOrClose} -> + ct:pal("Server Alert or Close ~p", [AlertOrClose]) + end, + ssl_test_lib:check_result(Client, {error, {tls_alert, ExpectedAlert}}). + %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- @@ -259,3 +309,5 @@ make_dir_path(PathComponents) -> "", PathComponents). +rename_crl(Filename) -> + file:rename(Filename, Filename ++ ".notfound"). diff --git a/lib/stdlib/src/maps.erl b/lib/stdlib/src/maps.erl index 3c798b7a04..43d10f4800 100644 --- a/lib/stdlib/src/maps.erl +++ b/lib/stdlib/src/maps.erl @@ -205,7 +205,7 @@ size(Val) -> K :: term(). without(Ks,M) when is_list(Ks), is_map(M) -> - maps:from_list([{K,V}||{K,V} <- maps:to_list(M), not lists:member(K, Ks)]); + lists:foldl(fun(K, M1) -> ?MODULE:remove(K, M1) end, M, Ks); without(Ks,M) -> erlang:error(error_type(M),[Ks,M]). @@ -216,8 +216,16 @@ without(Ks,M) -> Map2 :: map(), K :: term(). -with(Ks,M) when is_list(Ks), is_map(M) -> - maps:from_list([{K,V}||{K,V} <- maps:to_list(M), lists:member(K, Ks)]); +with(Ks,Map1) when is_list(Ks), is_map(Map1) -> + Fun = fun(K, List) -> + case ?MODULE:find(K, Map1) of + {ok, V} -> + [{K, V} | List]; + error -> + List + end + end, + ?MODULE:from_list(lists:foldl(Fun, [], Ks)); with(Ks,M) -> erlang:error(error_type(M),[Ks,M]). diff --git a/lib/stdlib/src/otp_internal.erl b/lib/stdlib/src/otp_internal.erl index 0dc2626ae8..6f546da7b8 100644 --- a/lib/stdlib/src/otp_internal.erl +++ b/lib/stdlib/src/otp_internal.erl @@ -650,7 +650,7 @@ obsolete_1(code, rehash, 0) -> {deprecated, "deprecated because the code path cache feature has been removed"}; obsolete_1(overload, _, _) -> - {deprecated, "deprecated; will be removed in OTP 19"}; + {removed, "removed in OTP 19"}; obsolete_1(_, _, _) -> no. diff --git a/lib/tools/emacs/erlang.el b/lib/tools/emacs/erlang.el index 466bf139b9..2ff14dea02 100644 --- a/lib/tools/emacs/erlang.el +++ b/lib/tools/emacs/erlang.el @@ -897,6 +897,7 @@ resulting regexp is surrounded by \\_< and \\_>." "get_module_info" "get_stacktrace" "hash" + "has_prepared_code_on_load" "hibernate" "insert_element" "is_builtin" |