diff options
Diffstat (limited to 'lib')
77 files changed, 2366 insertions, 1387 deletions
diff --git a/lib/common_test/src/ct_cover.erl b/lib/common_test/src/ct_cover.erl index c7f446dee9..b630a51835 100644 --- a/lib/common_test/src/ct_cover.erl +++ b/lib/common_test/src/ct_cover.erl @@ -174,7 +174,7 @@ get_spec_test(File) -> [] -> [#cover{app=none, level=details}]; _ -> Res end, - case get_cover_opts(Apps, Terms, []) of + case get_cover_opts(Apps, Terms, Dir, []) of E = {error,_} -> E; [CoverSpec] -> @@ -205,124 +205,125 @@ collect_apps([], Apps) -> %% get_cover_opts(Terms) -> AppCoverInfo %% AppCoverInfo: [#cover{app=App,...}] -get_cover_opts([App | Apps], Terms, CoverInfo) -> - case get_app_info(App, Terms) of +get_cover_opts([App | Apps], Terms, Dir, CoverInfo) -> + case get_app_info(App, Terms, Dir) of E = {error,_} -> E; AppInfo -> AppInfo1 = files2mods(AppInfo), - get_cover_opts(Apps, Terms, [AppInfo1|CoverInfo]) + get_cover_opts(Apps, Terms, Dir, [AppInfo1|CoverInfo]) end; -get_cover_opts([], _, CoverInfo) -> +get_cover_opts([], _, _, CoverInfo) -> lists:reverse(CoverInfo). -%% get_app_info(App, Terms) -> App1 +%% get_app_info(App, Terms, Dir) -> App1 -get_app_info(App=#cover{app=none}, [{incl_dirs,Dirs}|Terms]) -> - get_app_info(App, [{incl_dirs,none,Dirs}|Terms]); -get_app_info(App=#cover{app=Name}, [{incl_dirs,Name,Dirs}|Terms]) -> - case get_files(Dirs, ".beam", false, []) of +get_app_info(App=#cover{app=none}, [{incl_dirs,Dirs}|Terms], Dir) -> + get_app_info(App, [{incl_dirs,none,Dirs}|Terms], Dir); +get_app_info(App=#cover{app=Name}, [{incl_dirs,Name,Dirs}|Terms], Dir) -> + case get_files(Dirs, Dir, ".beam", false, []) of E = {error,_} -> E; Mods1 -> Mods = App#cover.incl_mods, - get_app_info(App#cover{incl_mods=Mods++Mods1},Terms) + get_app_info(App#cover{incl_mods=Mods++Mods1},Terms,Dir) end; -get_app_info(App=#cover{app=none}, [{incl_dirs_r,Dirs}|Terms]) -> - get_app_info(App, [{incl_dirs_r,none,Dirs}|Terms]); -get_app_info(App=#cover{app=Name}, [{incl_dirs_r,Name,Dirs}|Terms]) -> - case get_files(Dirs, ".beam", true, []) of +get_app_info(App=#cover{app=none}, [{incl_dirs_r,Dirs}|Terms], Dir) -> + get_app_info(App, [{incl_dirs_r,none,Dirs}|Terms], Dir); +get_app_info(App=#cover{app=Name}, [{incl_dirs_r,Name,Dirs}|Terms], Dir) -> + case get_files(Dirs, Dir, ".beam", true, []) of E = {error,_} -> E; Mods1 -> Mods = App#cover.incl_mods, - get_app_info(App#cover{incl_mods=Mods++Mods1},Terms) + get_app_info(App#cover{incl_mods=Mods++Mods1},Terms,Dir) end; -get_app_info(App=#cover{app=none}, [{incl_mods,Mods1}|Terms]) -> - get_app_info(App, [{incl_mods,none,Mods1}|Terms]); -get_app_info(App=#cover{app=Name}, [{incl_mods,Name,Mods1}|Terms]) -> +get_app_info(App=#cover{app=none}, [{incl_mods,Mods1}|Terms], Dir) -> + get_app_info(App, [{incl_mods,none,Mods1}|Terms], Dir); +get_app_info(App=#cover{app=Name}, [{incl_mods,Name,Mods1}|Terms], Dir) -> Mods = App#cover.incl_mods, - get_app_info(App#cover{incl_mods=Mods++Mods1},Terms); + get_app_info(App#cover{incl_mods=Mods++Mods1},Terms,Dir); -get_app_info(App=#cover{app=none}, [{excl_dirs,Dirs}|Terms]) -> - get_app_info(App, [{excl_dirs,none,Dirs}|Terms]); -get_app_info(App=#cover{app=Name}, [{excl_dirs,Name,Dirs}|Terms]) -> - case get_files(Dirs, ".beam", false, []) of +get_app_info(App=#cover{app=none}, [{excl_dirs,Dirs}|Terms], Dir) -> + get_app_info(App, [{excl_dirs,none,Dirs}|Terms], Dir); +get_app_info(App=#cover{app=Name}, [{excl_dirs,Name,Dirs}|Terms], Dir) -> + case get_files(Dirs, Dir, ".beam", false, []) of E = {error,_} -> E; Mods1 -> Mods = App#cover.excl_mods, - get_app_info(App#cover{excl_mods=Mods++Mods1},Terms) + get_app_info(App#cover{excl_mods=Mods++Mods1},Terms,Dir) end; -get_app_info(App=#cover{app=none}, [{excl_dirs_r,Dirs}|Terms]) -> - get_app_info(App, [{excl_dirs_r,none,Dirs}|Terms]); -get_app_info(App=#cover{app=Name}, [{excl_dirs_r,Name,Dirs}|Terms]) -> - case get_files(Dirs, ".beam", true, []) of +get_app_info(App=#cover{app=none}, [{excl_dirs_r,Dirs}|Terms],Dir) -> + get_app_info(App, [{excl_dirs_r,none,Dirs}|Terms],Dir); +get_app_info(App=#cover{app=Name}, [{excl_dirs_r,Name,Dirs}|Terms],Dir) -> + case get_files(Dirs, Dir, ".beam", true, []) of E = {error,_} -> E; Mods1 -> Mods = App#cover.excl_mods, - get_app_info(App#cover{excl_mods=Mods++Mods1},Terms) + get_app_info(App#cover{excl_mods=Mods++Mods1},Terms,Dir) end; -get_app_info(App=#cover{app=none}, [{excl_mods,Mods1}|Terms]) -> - get_app_info(App, [{excl_mods,none,Mods1}|Terms]); -get_app_info(App=#cover{app=Name}, [{excl_mods,Name,Mods1}|Terms]) -> +get_app_info(App=#cover{app=none}, [{excl_mods,Mods1}|Terms], Dir) -> + get_app_info(App, [{excl_mods,none,Mods1}|Terms], Dir); +get_app_info(App=#cover{app=Name}, [{excl_mods,Name,Mods1}|Terms], Dir) -> Mods = App#cover.excl_mods, - get_app_info(App#cover{excl_mods=Mods++Mods1},Terms); + get_app_info(App#cover{excl_mods=Mods++Mods1},Terms,Dir); -get_app_info(App=#cover{app=none}, [{cross,Cross}|Terms]) -> - get_app_info(App, [{cross,none,Cross}|Terms]); -get_app_info(App=#cover{app=Name}, [{cross,Name,Cross1}|Terms]) -> +get_app_info(App=#cover{app=none}, [{cross,Cross}|Terms], Dir) -> + get_app_info(App, [{cross,none,Cross}|Terms], Dir); +get_app_info(App=#cover{app=Name}, [{cross,Name,Cross1}|Terms], Dir) -> Cross = App#cover.cross, - get_app_info(App#cover{cross=Cross++Cross1},Terms); + get_app_info(App#cover{cross=Cross++Cross1},Terms,Dir); -get_app_info(App=#cover{app=none}, [{src_dirs,Dirs}|Terms]) -> - get_app_info(App, [{src_dirs,none,Dirs}|Terms]); -get_app_info(App=#cover{app=Name}, [{src_dirs,Name,Dirs}|Terms]) -> - case get_files(Dirs, ".erl", false, []) of +get_app_info(App=#cover{app=none}, [{src_dirs,Dirs}|Terms], Dir) -> + get_app_info(App, [{src_dirs,none,Dirs}|Terms], Dir); +get_app_info(App=#cover{app=Name}, [{src_dirs,Name,Dirs}|Terms], Dir) -> + case get_files(Dirs, Dir, ".erl", false, []) of E = {error,_} -> E; Src1 -> Src = App#cover.src, - get_app_info(App#cover{src=Src++Src1},Terms) + get_app_info(App#cover{src=Src++Src1},Terms,Dir) end; -get_app_info(App=#cover{app=none}, [{src_dirs_r,Dirs}|Terms]) -> - get_app_info(App, [{src_dirs_r,none,Dirs}|Terms]); -get_app_info(App=#cover{app=Name}, [{src_dirs_r,Name,Dirs}|Terms]) -> - case get_files(Dirs, ".erl", true, []) of +get_app_info(App=#cover{app=none}, [{src_dirs_r,Dirs}|Terms], Dir) -> + get_app_info(App, [{src_dirs_r,none,Dirs}|Terms], Dir); +get_app_info(App=#cover{app=Name}, [{src_dirs_r,Name,Dirs}|Terms], Dir) -> + case get_files(Dirs, Dir, ".erl", true, []) of E = {error,_} -> E; Src1 -> Src = App#cover.src, - get_app_info(App#cover{src=Src++Src1},Terms) + get_app_info(App#cover{src=Src++Src1},Terms,Dir) end; -get_app_info(App=#cover{app=none}, [{src_files,Src1}|Terms]) -> - get_app_info(App, [{src_files,none,Src1}|Terms]); -get_app_info(App=#cover{app=Name}, [{src_files,Name,Src1}|Terms]) -> +get_app_info(App=#cover{app=none}, [{src_files,Src1}|Terms], Dir) -> + get_app_info(App, [{src_files,none,Src1}|Terms], Dir); +get_app_info(App=#cover{app=Name}, [{src_files,Name,Src1}|Terms], Dir) -> Src = App#cover.src, - get_app_info(App#cover{src=Src++Src1},Terms); + get_app_info(App#cover{src=Src++Src1},Terms,Dir); -get_app_info(App, [_|Terms]) -> - get_app_info(App, Terms); +get_app_info(App, [_|Terms], Dir) -> + get_app_info(App, Terms, Dir); -get_app_info(App, []) -> +get_app_info(App, [], _) -> App. %% get_files(...) -get_files([Dir|Dirs], Ext, Recurse, Files) -> - case file:list_dir(Dir) of +get_files([Dir|Dirs], RootDir, Ext, Recurse, Files) -> + DirAbs = filename:absname(Dir, RootDir), + case file:list_dir(DirAbs) of {ok,Entries} -> - {SubDirs,Matches} = analyse_files(Entries, Dir, Ext, [], []), + {SubDirs,Matches} = analyse_files(Entries, DirAbs, Ext, [], []), if Recurse == false -> - get_files(Dirs, Ext, Recurse, Files++Matches); + get_files(Dirs, RootDir, Ext, Recurse, Files++Matches); true -> - Files1 = get_files(SubDirs, Ext, Recurse, Files++Matches), - get_files(Dirs, Ext, Recurse, Files1) + Files1 = get_files(SubDirs, RootDir, Ext, Recurse, Files++Matches), + get_files(Dirs, RootDir, Ext, Recurse, Files1) end; {error,Reason} -> - {error,{Reason,Dir}} + {error,{Reason,DirAbs}} end; -get_files([], _Ext, _R, Files) -> +get_files([], _RootDir, _Ext, _R, Files) -> Files. %% analyse_files(...) diff --git a/lib/common_test/test/ct_cover_SUITE.erl b/lib/common_test/test/ct_cover_SUITE.erl index 87ba4ae1b9..1dab425509 100644 --- a/lib/common_test/test/ct_cover_SUITE.erl +++ b/lib/common_test/test/ct_cover_SUITE.erl @@ -77,7 +77,11 @@ all() -> ct_cover_add_remove_nodes, otp_9956, cross, - export_import + export_import, + relative_incl_dirs, + absolute_incl_dirs, + relative_excl_dirs, + absolute_excl_dirs ]. %%-------------------------------------------------------------------- @@ -215,6 +219,45 @@ export_import(Config) -> check_calls(Events2,2), ok. +relative_incl_dirs(Config) -> + false = check_cover(Config), + RelDir = rel_path(?config(priv_dir, Config), ?config(data_dir, Config)), + CoverSpec = [{incl_dirs, [RelDir]}], + CoverFile = create_cover_file(rel_incl_dirs, CoverSpec, Config), + Opts = [{cover, CoverFile}], + {ok, Events} = run_test(rel_incl_dirs, default, Opts, Config), + check_calls(Events, 1), + ok. + +absolute_incl_dirs(Config) -> + false = check_cover(Config), + CoverSpec = [{incl_dirs, [?config(data_dir, Config)]}], + CoverFile = create_cover_file(abs_incl_dirs, CoverSpec, Config), + Opts = [{cover, CoverFile}], + {ok, Events} = run_test(abs_incl_dirs, default, Opts, Config), + check_calls(Events, 1), + ok. + +relative_excl_dirs(Config) -> + false = check_cover(Config), + RelDir = rel_path(?config(priv_dir, Config), ?config(data_dir, Config)), + CoverSpec = default_cover_file_content() ++ [{excl_dirs, [RelDir]}], + CoverFile = create_cover_file(rel_excl_dirs, CoverSpec, Config), + Opts = [{cover, CoverFile}], + {ok, Events} = run_test(rel_excl_dirs, default_no_cover, Opts, Config), + check_no_cover_compiled(Events), + ok. + +absolute_excl_dirs(Config) -> + false = check_cover(Config), + AbsDir = ?config(data_dir, Config), + CoverSpec = default_cover_file_content() ++ [{excl_dirs, [AbsDir]}], + CoverFile = create_cover_file(abs_excl_dirs, CoverSpec, Config), + Opts = [{cover, CoverFile}], + {ok, Events} = run_test(abs_excl_dirs, default_no_cover, Opts, Config), + check_no_cover_compiled(Events), + ok. + %%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- @@ -288,23 +331,36 @@ get_log_dirs(Events) -> {ct_test_support_eh, {event,start_logging,_Node,LogDir}} <- Events]. +%% Check if a module was compiled without cover +check_no_cover_compiled(Events) -> + check_no_cover_compiled(Events, ?mod). +check_no_cover_compiled(Events, Mod) -> + [ {error, {not_cover_compiled, Mod}} = analyse_log(CoverLog, Mod) + || CoverLog <- cover_logs(Events) ]. + %% Check that each coverlog includes N calls to ?mod:foo/0 check_calls(Events,N) -> check_calls(Events,{?mod,foo,0},N). check_calls(Events,MFA,N) -> - CoverLogs = [filename:join(D,"all.coverdata") || D <- get_log_dirs(Events)], - do_check_logs(CoverLogs,MFA,N). + do_check_logs(cover_logs(Events),MFA,N). do_check_logs([CoverLog|CoverLogs],{Mod,_,_} = MFA,N) -> - {ok,_} = cover:start(), - ok = cover:import(CoverLog), - {ok,Calls} = cover:analyse(Mod,calls,function), - ok = cover:stop(), + {ok, Calls} = analyse_log(CoverLog, Mod), {MFA,N} = lists:keyfind(MFA,1,Calls), do_check_logs(CoverLogs,MFA,N); do_check_logs([],_,_) -> ok. +cover_logs(Events) -> + [filename:join(D,"all.coverdata") || D <- get_log_dirs(Events)]. + +analyse_log(CoverLog, Mod) -> + {ok, _} = cover:start(), + ok = cover:import(CoverLog), + Result = cover:analyse(Mod, calls, function), + ok = cover:stop(), + Result. + fullname(Name) -> {ok,Host} = inet:gethostname(), list_to_atom(atom_to_list(Name) ++ "@" ++ Host). @@ -333,3 +389,12 @@ start_slave(Name,Args) -> {boot_timeout,10}, % extending some timers for slow test hosts {init_timeout,10}, {startup_timeout,10}]). + +rel_path(From, To) -> + Segments = do_rel_path(filename:split(From), filename:split(To)), + filename:join(Segments). + +do_rel_path([Seg|RestA], [Seg|RestB]) -> + do_rel_path(RestA, RestB); +do_rel_path(PathA, PathB) -> + lists:duplicate(length(PathA), "..") ++ PathB. diff --git a/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl index 83d368c53d..789e48bd96 100644 --- a/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl +++ b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl @@ -71,6 +71,10 @@ default(_Config) -> cover_test_mod:foo(), ok. +default_no_cover(_Config) -> + cover_test_mod:foo(), + ok. + slave(_Config) -> cover_compiled = code:which(cover_test_mod), cover_test_mod:foo(), diff --git a/lib/compiler/src/Makefile b/lib/compiler/src/Makefile index c6d09d85eb..2032392821 100644 --- a/lib/compiler/src/Makefile +++ b/lib/compiler/src/Makefile @@ -81,6 +81,7 @@ MODULES = \ rec_env \ sys_core_dsetel \ sys_core_fold \ + sys_core_fold_lists \ sys_core_inline \ sys_pre_attributes \ sys_pre_expand \ @@ -187,6 +188,7 @@ $(EBIN)/core_parse.beam: core_parse.hrl $(EGEN)/core_parse.erl $(EBIN)/core_pp.beam: core_parse.hrl $(EBIN)/sys_core_dsetel.beam: core_parse.hrl $(EBIN)/sys_core_fold.beam: core_parse.hrl +$(EBIN)/sys_core_fold_lists.beam: core_parse.hrl $(EBIN)/sys_core_inline.beam: core_parse.hrl $(EBIN)/sys_pre_expand.beam: ../../stdlib/include/erl_bits.hrl $(EBIN)/v3_codegen.beam: v3_life.hrl diff --git a/lib/compiler/src/beam_z.erl b/lib/compiler/src/beam_z.erl index c2a6ef604e..0c7bef9183 100644 --- a/lib/compiler/src/beam_z.erl +++ b/lib/compiler/src/beam_z.erl @@ -79,17 +79,9 @@ undo_rename({put_map,Fail,assoc,S,D,R,L}) -> undo_rename({put_map,Fail,exact,S,D,R,L}) -> {put_map_exact,Fail,S,D,R,L}; undo_rename({test,has_map_fields,Fail,[Src|List]}) -> - {test,has_map_fields,Fail,Src,{list,[to_typed_literal(V)||V<-List]}}; -undo_rename({get_map_elements,Fail,Src,{list, List}}) -> - {get_map_elements,Fail,Src,{list,[to_typed_literal(V)||V<-List]}}; + {test,has_map_fields,Fail,Src,{list,List}}; +undo_rename({get_map_elements,Fail,Src,{list,List}}) -> + {get_map_elements,Fail,Src,{list,List}}; undo_rename({select,I,Reg,Fail,List}) -> {I,Reg,Fail,{list,List}}; undo_rename(I) -> I. - -%% to_typed_literal(Arg) -%% transform Arg to specific literal i.e. float | integer | atom if applicable -to_typed_literal({literal, V}) when is_float(V) -> {float, V}; -to_typed_literal({literal, V}) when is_atom(V) -> {atom, V}; -to_typed_literal({literal, V}) when is_integer(V) -> {integer, V}; -to_typed_literal({literal, []}) -> nil; -to_typed_literal(V) -> V. diff --git a/lib/compiler/src/cerl.erl b/lib/compiler/src/cerl.erl index 7a2c3d70de..3d4b9ee0c6 100644 --- a/lib/compiler/src/cerl.erl +++ b/lib/compiler/src/cerl.erl @@ -123,11 +123,14 @@ bitstr_flags/1, %% keep map exports here for now + c_map_pattern/1, + is_c_map/1, map_es/1, map_arg/1, update_c_map/3, c_map/1, is_c_map_empty/1, ann_c_map/2, ann_c_map/3, + ann_c_map_pattern/2, map_pair_op/1,map_pair_key/1,map_pair_val/1, update_c_map_pair/4, c_map_pair/2, @@ -431,6 +434,8 @@ is_literal_term([H | T]) -> is_literal_term(T) when is_tuple(T) -> is_literal_term_list(tuple_to_list(T)); is_literal_term(B) when is_bitstring(B) -> true; +is_literal_term(M) when is_map(M) -> + is_literal_term_list(maps:to_list(M)); is_literal_term(_) -> false. @@ -1577,6 +1582,20 @@ ann_make_list(_, [], Node) -> %% --------------------------------------------------------------------- %% maps +%% @spec is_c_map(Node::cerl()) -> boolean() +%% +%% @doc Returns <code>true</code> if <code>Node</code> is an abstract +%% map constructor, otherwise <code>false</code>. + +-spec is_c_map(cerl()) -> boolean(). + +is_c_map(#c_map{}) -> + true; +is_c_map(#c_literal{val = V}) when is_map(V) -> + true; +is_c_map(_) -> + false. + -spec map_es(c_map()) -> [c_map_pair()]. map_es(#c_map{es = Es}) -> @@ -1590,7 +1609,17 @@ map_arg(#c_map{arg=M}) -> -spec c_map([c_map_pair()]) -> c_map(). c_map(Pairs) -> - #c_map{es=Pairs}. + ann_c_map([], Pairs). + +-spec c_map_pattern([c_map_pair()]) -> c_map(). + +c_map_pattern(Pairs) -> + #c_map{es=Pairs, is_pat=true}. + +-spec ann_c_map_pattern([term()], [c_map_pair()]) -> c_map(). + +ann_c_map_pattern(As, Pairs) -> + #c_map{anno=As, es=Pairs, is_pat=true}. -spec is_c_map_empty(c_map() | c_literal()) -> boolean(). @@ -1598,25 +1627,13 @@ is_c_map_empty(#c_map{ es=[] }) -> true; is_c_map_empty(#c_literal{val=M}) when is_map(M),map_size(M) =:= 0 -> true; is_c_map_empty(_) -> false. --spec ann_c_map([term()], [cerl()]) -> c_map() | c_literal(). +-spec ann_c_map([term()], [c_map_pair()]) -> c_map() | c_literal(). -ann_c_map(As,Es) -> +ann_c_map(As, Es) -> ann_c_map(As, #c_literal{val=#{}}, Es). -spec ann_c_map([term()], c_map() | c_literal(), [c_map_pair()]) -> c_map() | c_literal(). -ann_c_map(As,#c_literal{val=Mval}=M,Es) when is_map(Mval), map_size(Mval) =:= 0 -> - Pairs = [[Ck,Cv]||#c_map_pair{key=Ck,val=Cv}<-Es], - IsLit = lists:foldl(fun(Pair,Res) -> - Res andalso is_lit_list(Pair) - end, true, Pairs), - Fun = fun(Pair) -> [K,V] = lit_list_vals(Pair), {K,V} end, - case IsLit of - false -> - #c_map{arg=M, es=Es, anno=As }; - true -> - #c_literal{anno=As, val=maps:from_list(lists:map(Fun, Pairs))} - end; ann_c_map(As,#c_literal{val=M},Es) when is_map(M) -> fold_map_pairs(As,Es,M); ann_c_map(As,M,Es) -> @@ -1644,14 +1661,14 @@ fold_map_pairs(As,[#c_map_pair{op=#c_literal{val=exact},key=Ck,val=Cv}=E|Es],M) end; false -> #c_map{arg=#c_literal{val=M,anno=As}, es=[E|Es], anno=As } - end; -fold_map_pairs(As,Es,M) -> - #c_map{arg=#c_literal{val=M,anno=As}, es=Es, anno=As }. + end. -%-spec update_c_map(c_map() | c_literal(), [c_map_pair()]) -> c_map() | c_literal(). +-spec update_c_map(c_map(), cerl(), [cerl()]) -> c_map() | c_literal(). -update_c_map(Old,M,Es) -> - #c_map{arg=M, es = Es, anno = get_ann(Old)}. +update_c_map(#c_map{is_pat=true}=Old, M, Es) -> + Old#c_map{arg=M, es=Es}; +update_c_map(#c_map{is_pat=false}=Old, M, Es) -> + ann_c_map(get_ann(Old), M, Es). map_pair_key(#c_map_pair{key=K}) -> K. map_pair_val(#c_map_pair{val=V}) -> V. diff --git a/lib/compiler/src/cerl_clauses.erl b/lib/compiler/src/cerl_clauses.erl index 87bd47c08b..ef74c5b76f 100644 --- a/lib/compiler/src/cerl_clauses.erl +++ b/lib/compiler/src/cerl_clauses.erl @@ -354,29 +354,29 @@ match(P, E, Bs) -> {false, Bs} end end; - map -> - %% The most we can do is to say "definitely no match" if a - %% map pattern is matched against non-map data. - case E of - any -> - {false, Bs}; - _ -> - case type(E) of - literal -> - case is_map(concrete(E)) of - false -> - none; - true -> - {false, Bs} - end; - cons -> - none; - tuple -> - none; - _ -> - {false, Bs} - end - end; + map -> + %% The most we can do is to say "definitely no match" if a + %% map pattern is matched against non-map data. + case E of + any -> + {false, Bs}; + _ -> + case type(E) of + literal -> + case is_map(concrete(E)) of + false -> + none; + true -> + {false, Bs} + end; + cons -> + none; + tuple -> + none; + _ -> + {false, Bs} + end + end; _ -> match_1(P, E, Bs) end. diff --git a/lib/compiler/src/compiler.app.src b/lib/compiler/src/compiler.app.src index 8f68915f8e..fbaa7a96fe 100644 --- a/lib/compiler/src/compiler.app.src +++ b/lib/compiler/src/compiler.app.src @@ -56,6 +56,7 @@ rec_env, sys_core_dsetel, sys_core_fold, + sys_core_fold_lists, sys_core_inline, sys_pre_attributes, sys_pre_expand, diff --git a/lib/compiler/src/core_lib.erl b/lib/compiler/src/core_lib.erl index 0d95971f91..66319dbd36 100644 --- a/lib/compiler/src/core_lib.erl +++ b/lib/compiler/src/core_lib.erl @@ -20,6 +20,12 @@ -module(core_lib). +-deprecated({get_anno,1,next_major_release}). +-deprecated({set_anno,2,next_major_release}). +-deprecated({is_literal,1,next_major_release}). +-deprecated({is_literal_list,1,next_major_release}). +-deprecated({literal_value,1,next_major_release}). + -export([get_anno/1,set_anno/2]). -export([is_literal/1,is_literal_list/1]). -export([literal_value/1]). @@ -33,59 +39,27 @@ %% -spec get_anno(cerl:cerl()) -> term(). -get_anno(C) -> element(2, C). +get_anno(C) -> cerl:get_ann(C). -spec set_anno(cerl:cerl(), term()) -> cerl:cerl(). -set_anno(C, A) -> setelement(2, C, A). +set_anno(C, A) -> cerl:set_ann(C, A). -spec is_literal(cerl:cerl()) -> boolean(). -is_literal(#c_literal{}) -> true; -is_literal(#c_cons{hd=H,tl=T}) -> - is_literal(H) andalso is_literal(T); -is_literal(#c_tuple{es=Es}) -> is_literal_list(Es); -is_literal(#c_binary{segments=Es}) -> is_lit_bin(Es); -is_literal(_) -> false. +is_literal(Cerl) -> + cerl:is_literal(cerl:fold_literal(Cerl)). -spec is_literal_list([cerl:cerl()]) -> boolean(). is_literal_list(Es) -> lists:all(fun is_literal/1, Es). -is_lit_bin(Es) -> - lists:all(fun (#c_bitstr{val=E,size=S}) -> - is_literal(E) andalso is_literal(S) - end, Es). - %% Return the value of LitExpr. -spec literal_value(cerl:c_literal() | cerl:c_binary() | cerl:c_map() | cerl:c_cons() | cerl:c_tuple()) -> term(). -literal_value(#c_literal{val=V}) -> V; -literal_value(#c_binary{segments=Es}) -> - list_to_binary([literal_value_bin(Bit) || Bit <- Es]); -literal_value(#c_cons{hd=H,tl=T}) -> - [literal_value(H)|literal_value(T)]; -literal_value(#c_tuple{es=Es}) -> - list_to_tuple(literal_value_list(Es)); -literal_value(#c_map{arg=Cm,es=Cmps}) -> - M = literal_value(Cm), - lists:foldl(fun(#c_map_pair{ key=Ck, val=Cv },Mi) -> - K = literal_value(Ck), - V = literal_value(Cv), - maps:put(K,V,Mi) - end, M, Cmps). - -literal_value_list(Vals) -> [literal_value(V) || V <- Vals]. - -literal_value_bin(#c_bitstr{val=Val,size=Sz,unit=U,type=T,flags=Fs}) -> - %% We will only handle literals constructed by make_literal/1. - %% Could be made more general in the future if the need arises. - 8 = literal_value(Sz), - 1 = literal_value(U), - integer = literal_value(T), - [unsigned,big] = literal_value(Fs), - literal_value(Val). +literal_value(Cerl) -> + cerl:concrete(cerl:fold_literal(Cerl)). %% Make a suitable values structure, expr or values, depending on Expr. -spec make_values([cerl:cerl()] | cerl:cerl()) -> cerl:cerl(). @@ -236,10 +210,15 @@ vu_pat_seg_list(V, Ss, St) -> end end, St, Ss). -vu_map_pairs(V, [#c_map_pair{val=Pat}|T], St0) -> - case vu_pattern(V, Pat, St0) of - {true,_}=St -> St; - St -> vu_map_pairs(V, T, St) +vu_map_pairs(V, [#c_map_pair{key=Key,val=Pat}|T], St0) -> + case vu_expr(V, Key) of + true -> + {true,false}; + false -> + case vu_pattern(V, Pat, St0) of + {true,_}=St -> St; + St -> vu_map_pairs(V, T, St) + end end; vu_map_pairs(_, [], St) -> St. diff --git a/lib/compiler/src/core_lint.erl b/lib/compiler/src/core_lint.erl index c0e2bdaba0..f62b2bb0ba 100644 --- a/lib/compiler/src/core_lint.erl +++ b/lib/compiler/src/core_lint.erl @@ -173,7 +173,7 @@ check_exports(Es, St) -> end. check_attrs(As, St) -> - case all(fun ({#c_literal{},V}) -> core_lib:is_literal(V); + case all(fun ({#c_literal{},#c_literal{}}) -> true; (_) -> false end, As) of true -> St; diff --git a/lib/compiler/src/core_parse.hrl b/lib/compiler/src/core_parse.hrl index 4a00535360..7fd4a82e4e 100644 --- a/lib/compiler/src/core_parse.hrl +++ b/lib/compiler/src/core_parse.hrl @@ -72,7 +72,8 @@ -record(c_map, {anno=[], arg=#c_literal{val=#{}} :: cerl:c_var() | cerl:c_literal(), - es :: [cerl:c_map_pair()]}). + es :: [cerl:c_map_pair()], + is_pat=false :: boolean()}). -record(c_map_pair, {anno=[], op :: #c_literal{val::'assoc'} | #c_literal{val::'exact'}, diff --git a/lib/compiler/src/core_parse.yrl b/lib/compiler/src/core_parse.yrl index a66ad4235f..eeb9f5dba7 100644 --- a/lib/compiler/src/core_parse.yrl +++ b/lib/compiler/src/core_parse.yrl @@ -58,7 +58,8 @@ Terminals %% Separators -'(' ')' '{' '}' '[' ']' '|' ',' '->' '=' '/' '<' '>' ':' '-|' '#' '~' '::' +'(' ')' '{' '}' '[' ']' '|' ',' '->' '=' '/' '<' '>' ':' '-|' '#' +'~' '=>' ':=' %% Keywords (atoms are assumed to always be single-quoted). @@ -123,7 +124,7 @@ function_definition -> {'$1','$3'}. anno_fun -> '(' fun_expr '-|' annotation ')' : - core_lib:set_anno('$2', '$4'). + cerl:set_ann('$2', '$4'). anno_fun -> fun_expr : '$1'. %% Constant terms for annotations and attributes. @@ -162,7 +163,7 @@ tail_constant -> ',' constant tail_constant : ['$2'|'$3']. %% ( ( V -| <anno> ) = ( {a} -| <anno> ) -| <anno> ) anno_pattern -> '(' other_pattern '-|' annotation ')' : - core_lib:set_anno('$2', '$4'). + cerl:set_ann('$2', '$4'). anno_pattern -> other_pattern : '$1'. anno_pattern -> anno_variable : '$1'. @@ -182,23 +183,24 @@ atomic_pattern -> atomic_literal : '$1'. tuple_pattern -> '{' '}' : c_tuple([]). tuple_pattern -> '{' anno_patterns '}' : c_tuple('$2'). -map_pattern -> '~' '{' '}' '~' : #c_map{es=[]}. +map_pattern -> '~' '{' '}' '~' : c_map_pattern([]). map_pattern -> '~' '{' map_pair_patterns '}' '~' : - #c_map{es=lists:sort('$3')}. + c_map_pattern(lists:sort('$3')). map_pair_patterns -> map_pair_pattern : ['$1']. map_pair_patterns -> map_pair_pattern ',' map_pair_patterns : ['$1' | '$3']. -map_pair_pattern -> '~' '<' anno_pattern ',' anno_pattern '>' : - #c_map_pair{op=#c_literal{val=exact},key='$3',val='$5'}. +map_pair_pattern -> anno_expression ':=' anno_pattern : + #c_map_pair{op=#c_literal{val=exact}, + key='$1',val='$3'}. cons_pattern -> '[' anno_pattern tail_pattern : - #c_cons{hd='$2',tl='$3'}. + c_cons('$2', '$3'). tail_pattern -> ']' : #c_literal{val=[]}. tail_pattern -> '|' anno_pattern ']' : '$2'. tail_pattern -> ',' anno_pattern tail_pattern : - #c_cons{hd='$2',tl='$3'}. + c_cons('$2', '$3'). binary_pattern -> '#' '{' '}' '#' : #c_binary{segments=[]}. binary_pattern -> '#' '{' segment_patterns '}' '#' : #c_binary{segments='$3'}. @@ -206,7 +208,7 @@ binary_pattern -> '#' '{' segment_patterns '}' '#' : #c_binary{segments='$3'}. segment_patterns -> segment_pattern ',' segment_patterns : ['$1' | '$3']. segment_patterns -> segment_pattern : ['$1']. -segment_pattern -> '#' '<' anno_pattern '>' '(' anno_patterns ')': +segment_pattern -> '#' '<' anno_pattern '>' '(' anno_expressions ')': case '$6' of [S,U,T,Fs] -> #c_bitstr{val='$3',size=S,unit=U,type=T,flags=Fs}; @@ -222,7 +224,7 @@ anno_variables -> anno_variable : ['$1']. anno_variable -> variable : '$1'. anno_variable -> '(' variable '-|' annotation ')' : - core_lib:set_anno('$2', '$4'). + cerl:set_ann('$2', '$4'). %% Expressions %% Must split expressions into two levels as nested value expressions @@ -230,7 +232,7 @@ anno_variable -> '(' variable '-|' annotation ')' : anno_expression -> expression : '$1'. anno_expression -> '(' expression '-|' annotation ')' : - core_lib:set_anno('$2', '$4'). + cerl:set_ann('$2', '$4'). anno_expressions -> anno_expression ',' anno_expressions : ['$1' | '$3']. anno_expressions -> anno_expression : ['$1']. @@ -279,15 +281,15 @@ cons_literal -> '[' literal tail_literal : c_cons('$2', '$3'). tail_literal -> ']' : #c_literal{val=[]}. tail_literal -> '|' literal ']' : '$2'. -tail_literal -> ',' literal tail_literal : #c_cons{hd='$2',tl='$3'}. +tail_literal -> ',' literal tail_literal : c_cons('$2', '$3'). tuple -> '{' '}' : c_tuple([]). tuple -> '{' anno_expressions '}' : c_tuple('$2'). -map_expr -> '~' '{' '}' '~' : #c_map{es=[]}. -map_expr -> '~' '{' map_pairs '}' '~' : #c_map{es='$3'}. -map_expr -> '~' '{' map_pairs '|' variable '}' '~' : #c_map{arg='$5',es='$3'}. -map_expr -> '~' '{' map_pairs '|' map_expr '}' '~' : #c_map{arg='$5',es='$3'}. +map_expr -> '~' '{' '}' '~' : c_map([]). +map_expr -> '~' '{' map_pairs '}' '~' : c_map('$3'). +map_expr -> '~' '{' map_pairs '|' variable '}' '~' : ann_c_map([], '$5', '$3'). +map_expr -> '~' '{' map_pairs '|' map_expr '}' '~' : ann_c_map([], '$5', '$3'). map_pairs -> map_pair : ['$1']. map_pairs -> map_pair ',' map_pairs : ['$1' | '$3']. @@ -295,10 +297,10 @@ map_pairs -> map_pair ',' map_pairs : ['$1' | '$3']. map_pair -> map_pair_assoc : '$1'. map_pair -> map_pair_exact : '$1'. -map_pair_assoc -> '::' '<' anno_expression ',' anno_expression'>' : - #c_map_pair{op=#c_literal{val=assoc},key='$3',val='$5'}. -map_pair_exact -> '~' '<' anno_expression ',' anno_expression'>' : - #c_map_pair{op=#c_literal{val=exact},key='$3',val='$5'}. +map_pair_assoc -> anno_expression '=>' anno_expression : + #c_map_pair{op=#c_literal{val=assoc},key='$1',val='$3'}. +map_pair_exact -> anno_expression ':=' anno_expression : + #c_map_pair{op=#c_literal{val=exact},key='$1',val='$3'}. cons -> '[' anno_expression tail : c_cons('$2', '$3'). @@ -307,7 +309,7 @@ tail -> '|' anno_expression ']' : '$2'. tail -> ',' anno_expression tail : c_cons('$2', '$3'). binary -> '#' '{' '}' '#' : #c_literal{val = <<>>}. -binary -> '#' '{' segments '}' '#' : #c_binary{segments='$3'}. +binary -> '#' '{' segments '}' '#' : make_binary('$3'). segments -> segment ',' segments : ['$1' | '$3']. segments -> segment : ['$1']. @@ -326,7 +328,7 @@ function_name -> atom '/' integer : anno_function_name -> function_name : '$1'. anno_function_name -> '(' function_name '-|' annotation ')' : - core_lib:set_anno('$2', '$4'). + cerl:set_ann('$2', '$4'). let_vars -> anno_variable : ['$1']. let_vars -> '<' '>' : []. @@ -354,7 +356,7 @@ anno_clauses -> anno_clause : ['$1']. anno_clause -> clause : '$1'. anno_clause -> '(' clause '-|' annotation ')' : - core_lib:set_anno('$2', '$4'). + cerl:set_ann('$2', '$4'). clause -> clause_pattern 'when' anno_expression '->' anno_expression : #c_clause{pats='$1',guard='$3',body='$5'}. @@ -410,9 +412,55 @@ Erlang code. -include("core_parse.hrl"). --import(cerl, [c_cons/2,c_tuple/1]). +-import(cerl, [ann_c_map/3,c_cons/2,c_map/1,c_map_pattern/1,c_tuple/1]). tok_val(T) -> element(3, T). tok_line(T) -> element(2, T). +%% make_binary([#c_bitstr{}]) -> #c_binary{} | #c_literal{} +%% Create either #c_binary{} or #c_literal{} from the binary segments. +%% In certain contexts, such as keys for maps, only literals and +%% variables are allowed, so we must not create a #c_binary{} +%% record in those situation. +%% +%% To keep this function simple, we use a crude heuristic. We will +%% assume that Core Erlang has been produced by core_pp. If the +%% segments *could* have been output from a literal binary by +%% core_pp, we will create a #c_literal{}. Otherwise we will create a +%% #c_binary{} record. + +make_binary(Segs) -> + try make_lit_bin(<<>>, Segs) of + Bs when is_bitstring(Bs) -> + #c_literal{val=Bs} + catch + throw:impossible -> + #c_binary{segments=Segs} + end. + +make_lit_bin(Acc, [#c_bitstr{val=I0,size=Sz0,unit=U0,type=Type0,flags=F0}|T]) -> + I = get_lit_val(I0), + Sz = get_lit_val(Sz0), + U = get_lit_val(U0), + Type = get_lit_val(Type0), + F = get_lit_val(F0), + if + is_integer(I), U =:= 1, Type =:= integer, F =:= [unsigned,big] -> + ok; + true -> + throw(impossible) + end, + if + Sz =< 8, T =:= [] -> + <<Acc/binary,I:Sz>>; + Sz =:= 8 -> + make_lit_bin(<<Acc/binary,I:8>>, T); + true -> + throw(impossible) + end; +make_lit_bin(Acc, []) -> Acc. + +get_lit_val(#c_literal{val=Val}) -> Val; +get_lit_val(_) -> throw(impossible). + %% vim: syntax=erlang diff --git a/lib/compiler/src/core_pp.erl b/lib/compiler/src/core_pp.erl index 03801a9b6d..9cfca88e8c 100644 --- a/lib/compiler/src/core_pp.erl +++ b/lib/compiler/src/core_pp.erl @@ -45,7 +45,7 @@ format(Node) -> format(Node, #ctxt{}). maybe_anno(Node, Fun, Ctxt) -> - As = core_lib:get_anno(Node), + As = cerl:get_ann(Node), case get_line(As) of none -> maybe_anno(Node, Fun, Ctxt, As); @@ -183,15 +183,9 @@ format_1(#c_map{arg=Var,es=Es}, Ctxt) -> "}~" ]; format_1(#c_map_pair{op=#c_literal{val=assoc},key=K,val=V}, Ctxt) -> - ["::<", - format_hseq([K,V], ",", add_indent(Ctxt, 1), fun format/2), - ">" - ]; + format_map_pair("=>", K, V, Ctxt); format_1(#c_map_pair{op=#c_literal{val=exact},key=K,val=V}, Ctxt) -> - ["~<", - format_hseq([K,V], ",", add_indent(Ctxt, 1), fun format/2), - ">" - ]; + format_map_pair(":=", K, V, Ctxt); format_1(#c_cons{hd=H,tl=T}, Ctxt) -> Txt = ["["|format(H, add_indent(Ctxt, 1))], [Txt|format_list_tail(T, add_indent(Ctxt, width(Txt, Ctxt)))]; @@ -201,7 +195,7 @@ format_1(#c_alias{var=V,pat=P}, Ctxt) -> Txt = [format(V, Ctxt)|" = "], [Txt|format(P, add_indent(Ctxt, width(Txt, Ctxt)))]; format_1(#c_let{vars=Vs0,arg=A,body=B}, Ctxt) -> - Vs = [core_lib:set_anno(V, []) || V <- Vs0], + Vs = [cerl:set_ann(V, []) || V <- Vs0], case is_simple_term(A) of false -> Ctxt1 = add_indent(Ctxt, Ctxt#ctxt.body_indent), @@ -219,7 +213,7 @@ format_1(#c_let{vars=Vs0,arg=A,body=B}, Ctxt) -> ["let ", format_values(Vs, add_indent(Ctxt, 4)), " = ", - format(core_lib:set_anno(A, []), Ctxt1), + format(cerl:set_ann(A, []), Ctxt1), nl_indent(Ctxt), "in " | format(B, add_indent(Ctxt, 4)) @@ -448,6 +442,12 @@ format_list_tail(#c_cons{anno=[],hd=H,tl=T}, Ctxt) -> format_list_tail(Tail, Ctxt) -> ["|",format(Tail, add_indent(Ctxt, 1)),"]"]. +format_map_pair(Op, K, V, Ctxt0) -> + Ctxt1 = add_indent(Ctxt0, 1), + Txt = format(K, set_class(Ctxt1, expr)), + Ctxt2 = add_indent(Ctxt0, width(Txt, Ctxt1)), + [Txt,Op,format(V, Ctxt2)]. + indent(Ctxt) -> indent(Ctxt#ctxt.indent, Ctxt). indent(N, _) when N =< 0 -> ""; diff --git a/lib/compiler/src/core_scan.erl b/lib/compiler/src/core_scan.erl index b7799b373a..8ab20b1982 100644 --- a/lib/compiler/src/core_scan.erl +++ b/lib/compiler/src/core_scan.erl @@ -271,8 +271,10 @@ scan1("->" ++ Cs, Toks, Pos) -> scan1(Cs, [{'->',Pos}|Toks], Pos); scan1("-|" ++ Cs, Toks, Pos) -> scan1(Cs, [{'-|',Pos}|Toks], Pos); -scan1("::" ++ Cs, Toks, Pos) -> - scan1(Cs, [{'::',Pos}|Toks], Pos); +scan1(":=" ++ Cs, Toks, Pos) -> + scan1(Cs, [{':=',Pos}|Toks], Pos); +scan1("=>" ++ Cs, Toks, Pos) -> + scan1(Cs, [{'=>',Pos}|Toks], Pos); scan1([C|Cs], Toks, Pos) -> %Punctuation character P = list_to_atom([C]), scan1(Cs, [{P,Pos}|Toks], Pos); diff --git a/lib/compiler/src/sys_core_fold.erl b/lib/compiler/src/sys_core_fold.erl index 09716d0866..7d1819ea15 100644 --- a/lib/compiler/src/sys_core_fold.erl +++ b/lib/compiler/src/sys_core_fold.erl @@ -96,6 +96,10 @@ t=[], %Types in_guard=false}). %In guard or not. +-type type_info() :: cerl:cerl() | 'bool'. +-type yes_no_maybe() :: 'yes' | 'no' | 'maybe'. +-type sub() :: #sub{}. + -spec module(cerl:c_module(), [compile:option()]) -> {'ok', cerl:c_module(), [_]}. @@ -313,7 +317,7 @@ expr(#c_letrec{defs=Fs0,body=B0}=Letrec, Ctxt, Sub) -> Fs1 = map(fun ({Name,Fb}) -> {Name,expr(Fb, {letrec,Ctxt}, Sub)} end, Fs0), - B1 = body(B0, value, Sub), + B1 = body(B0, Ctxt, Sub), Letrec#c_letrec{defs=Fs1,body=B1}; expr(#c_case{}=Case0, Ctxt, Sub) -> %% Ideally, the compiler should only emit warnings when there is @@ -462,10 +466,7 @@ is_safe_simple(#c_call{module=#c_literal{val=erlang}, case erl_internal:bool_op(Name, NumArgs) of true -> %% Boolean operators are safe if the arguments are boolean. - all(fun(#c_var{name=V}) -> is_boolean_type(V, Sub); - (#c_literal{val=Lit}) -> is_boolean(Lit); - (_) -> false - end, Args); + all(fun(C) -> is_boolean_type(C, Sub) =:= yes end, Args); false -> %% We need a rather complicated test to ensure that %% we only allow safe calls that are allowed in a guard. @@ -607,14 +608,6 @@ eval_binary_1([#c_bitstr{val=#c_literal{val=Val},size=#c_literal{val=Sz}, error:_ -> throw(impossible) end; -eval_binary_1([#c_bitstr{val=#c_literal{},size=#c_literal{}, - unit=#c_literal{},type=#c_literal{}, - flags=#c_cons{}=Flags}=Bitstr|Ss], Acc0) -> - case cerl:fold_literal(Flags) of - #c_literal{} = Flags1 -> - eval_binary_1([Bitstr#c_bitstr{flags=Flags1}|Ss], Acc0); - _ -> throw(impossible) - end; eval_binary_1([], Acc) -> Acc; eval_binary_1(_, _) -> throw(impossible). @@ -688,23 +681,15 @@ count_bits_1(Int, Bits) -> count_bits_1(Int bsr 64, Bits+64). %% a rewritten expression consisting of a sequence of %% the arguments only is returned. -useless_call(effect, #c_call{anno=Anno, - module=#c_literal{val=Mod}, +useless_call(effect, #c_call{module=#c_literal{val=Mod}, name=#c_literal{val=Name}, args=Args}=Call) -> A = length(Args), case erl_bifs:is_safe(Mod, Name, A) of false -> case erl_bifs:is_pure(Mod, Name, A) of - true -> - case member(result_not_wanted, Anno) of - false -> - add_warning(Call, result_ignored); - true -> - ok - end; - false -> - ok + true -> add_warning(Call, result_ignored); + false -> ok end, no; true -> @@ -730,385 +715,23 @@ make_effect_seq([], _) -> void(). call(#c_call{args=As}=Call, #c_literal{val=M}=M0, #c_literal{val=N}=N0, Sub) -> case get(no_inline_list_funcs) of true -> - call_0(Call, M0, N0, As, Sub); + call_1(Call, M0, N0, As, Sub); false -> - call_1(Call, M, N, As, Sub) + case sys_core_fold_lists:call(Call, M, N, As) of + none -> + call_1(Call, M, N, As, Sub); + Core -> + expr(Core, Sub) + end + end; call(#c_call{args=As}=Call, M, N, Sub) -> - call_0(Call, M, N, As, Sub). + call_1(Call, M, N, As, Sub). -call_0(Call, M, N, As0, Sub) -> +call_1(Call, M, N, As0, Sub) -> As1 = expr_list(As0, value, Sub), fold_call(Call#c_call{args=As1}, M, N, As1, Sub). -%% We inline some very common higher order list operations. -%% We use the same evaluation order as the library function. - -call_1(#c_call{anno=Anno}, lists, all, [Arg1,Arg2], Sub) -> - Loop = #c_var{name={'lists^all',1}}, - F = #c_var{name='F'}, - Xs = #c_var{name='Xs'}, - X = #c_var{name='X'}, - Err1 = #c_tuple{es=[#c_literal{val='case_clause'}, X]}, - CC1 = #c_clause{pats=[#c_literal{val=true}], guard=#c_literal{val=true}, - body=#c_apply{anno=Anno, op=Loop, args=[Xs]}}, - CC2 = #c_clause{pats=[#c_literal{val=false}], guard=#c_literal{val=true}, - body=#c_literal{val=false}}, - CC3 = #c_clause{pats=[X], guard=#c_literal{val=true}, - body=match_fail(Anno, Err1)}, - C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, - body=#c_case{arg=#c_apply{anno=Anno, op=F, args=[X]}, - clauses = [CC1, CC2, CC3]}}, - C2 = #c_clause{pats=[#c_literal{val=[]}], - guard=#c_call{module=#c_literal{val=erlang}, - name=#c_literal{val=is_function}, - args=[F, #c_literal{val=1}]}, - body=#c_literal{val=true}}, - Err2 = #c_tuple{es=[#c_literal{val='function_clause'}, F, Xs]}, - C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, - body=match_fail([{function_name,{'lists^all',1}}|Anno], Err2)}, - Fun = #c_fun{vars=[Xs], - body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, - L = #c_var{name='L'}, - expr(#c_let{vars=[F, L], arg=#c_values{es=[Arg1, Arg2]}, - body=#c_letrec{defs=[{Loop,Fun}], - body=#c_apply{anno=Anno, op=Loop, args=[L]}}}, - Sub); -call_1(#c_call{anno=Anno}, lists, any, [Arg1,Arg2], Sub) -> - Loop = #c_var{name={'lists^any',1}}, - F = #c_var{name='F'}, - Xs = #c_var{name='Xs'}, - X = #c_var{name='X'}, - Err1 = #c_tuple{es=[#c_literal{val='case_clause'}, X]}, - CC1 = #c_clause{pats=[#c_literal{val=true}], guard=#c_literal{val=true}, - body=#c_literal{val=true}}, - CC2 = #c_clause{pats=[#c_literal{val=false}], guard=#c_literal{val=true}, - body=#c_apply{anno=Anno, op=Loop, args=[Xs]}}, - CC3 = #c_clause{pats=[X], guard=#c_literal{val=true}, - body=match_fail(Anno, Err1)}, - C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, - body=#c_case{arg=#c_apply{anno=Anno, op=F, args=[X]}, - clauses = [CC1, CC2, CC3]}}, - C2 = #c_clause{pats=[#c_literal{val=[]}], - guard=#c_call{module=#c_literal{val=erlang}, - name=#c_literal{val=is_function}, - args=[F, #c_literal{val=1}]}, - body=#c_literal{val=false}}, - Err2 = #c_tuple{es=[#c_literal{val='function_clause'}, F, Xs]}, - C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, - body=match_fail([{function_name,{'lists^any',1}}|Anno], Err2)}, - Fun = #c_fun{vars=[Xs], - body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, - L = #c_var{name='L'}, - expr(#c_let{vars=[F, L], arg=#c_values{es=[Arg1, Arg2]}, - body=#c_letrec{defs=[{Loop,Fun}], - body=#c_apply{anno=Anno, op=Loop, args=[L]}}}, - Sub); -call_1(#c_call{anno=Anno}, lists, foreach, [Arg1,Arg2], Sub) -> - Loop = #c_var{name={'lists^foreach',1}}, - F = #c_var{name='F'}, - Xs = #c_var{name='Xs'}, - X = #c_var{name='X'}, - C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, - body=#c_seq{arg=#c_apply{anno=Anno, op=F, args=[X]}, - body=#c_apply{anno=Anno, op=Loop, args=[Xs]}}}, - C2 = #c_clause{pats=[#c_literal{val=[]}], - guard=#c_call{module=#c_literal{val=erlang}, - name=#c_literal{val=is_function}, - args=[F, #c_literal{val=1}]}, - body=#c_literal{val=ok}}, - Err = #c_tuple{es=[#c_literal{val='function_clause'}, F, Xs]}, - C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, - body=match_fail([{function_name,{'lists^foreach',1}}|Anno], Err)}, - Fun = #c_fun{vars=[Xs], - body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, - L = #c_var{name='L'}, - expr(#c_let{vars=[F, L], arg=#c_values{es=[Arg1, Arg2]}, - body=#c_letrec{defs=[{Loop,Fun}], - body=#c_apply{anno=Anno, op=Loop, args=[L]}}}, - Sub); -call_1(#c_call{anno=Anno}, lists, map, [Arg1,Arg2], Sub) -> - Loop = #c_var{name={'lists^map',1}}, - F = #c_var{name='F'}, - Xs = #c_var{name='Xs'}, - X = #c_var{name='X'}, - H = #c_var{name='H'}, - C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, - body=#c_let{vars=[H], arg=#c_apply{anno=Anno, - op=F, - args=[X]}, - body=#c_cons{hd=H, - anno=[compiler_generated], - tl=#c_apply{anno=Anno, - op=Loop, - args=[Xs]}}}}, - C2 = #c_clause{pats=[#c_literal{val=[]}], - guard=#c_call{module=#c_literal{val=erlang}, - name=#c_literal{val=is_function}, - args=[F, #c_literal{val=1}]}, - body=#c_literal{val=[]}}, - Err = #c_tuple{es=[#c_literal{val='function_clause'}, F, Xs]}, - C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, - body=match_fail([{function_name,{'lists^map',1}}|Anno], Err)}, - Fun = #c_fun{vars=[Xs], - body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, - L = #c_var{name='L'}, - expr(#c_let{vars=[F, L], arg=#c_values{es=[Arg1, Arg2]}, - body=#c_letrec{defs=[{Loop,Fun}], - body=#c_apply{anno=Anno, op=Loop, args=[L]}}}, - Sub); -call_1(#c_call{anno=Anno}, lists, flatmap, [Arg1,Arg2], Sub) -> - Loop = #c_var{name={'lists^flatmap',1}}, - F = #c_var{name='F'}, - Xs = #c_var{name='Xs'}, - X = #c_var{name='X'}, - H = #c_var{name='H'}, - C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, - body=#c_let{vars=[H], - arg=#c_apply{anno=Anno, op=F, args=[X]}, - body=#c_call{anno=[compiler_generated|Anno], - module=#c_literal{val=erlang}, - name=#c_literal{val='++'}, - args=[H, - #c_apply{anno=Anno, - op=Loop, - args=[Xs]}]}}}, - C2 = #c_clause{pats=[#c_literal{val=[]}], - guard=#c_call{module=#c_literal{val=erlang}, - name=#c_literal{val=is_function}, - args=[F, #c_literal{val=1}]}, - body=#c_literal{val=[]}}, - Err = #c_tuple{es=[#c_literal{val='function_clause'}, F, Xs]}, - C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, - body=match_fail([{function_name,{'lists^flatmap',1}}|Anno], Err)}, - Fun = #c_fun{vars=[Xs], - body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, - L = #c_var{name='L'}, - expr(#c_let{vars=[F, L], arg=#c_values{es=[Arg1, Arg2]}, - body=#c_letrec{defs=[{Loop,Fun}], - body=#c_apply{anno=Anno, op=Loop, args=[L]}}}, - Sub); -call_1(#c_call{anno=Anno}, lists, filter, [Arg1,Arg2], Sub) -> - Loop = #c_var{name={'lists^filter',1}}, - F = #c_var{name='F'}, - Xs = #c_var{name='Xs'}, - X = #c_var{name='X'}, - B = #c_var{name='B'}, - Err1 = #c_tuple{es=[#c_literal{val='case_clause'}, X]}, - CC1 = #c_clause{pats=[#c_literal{val=true}], guard=#c_literal{val=true}, - body=#c_cons{anno=[compiler_generated], hd=X, tl=Xs}}, - CC2 = #c_clause{pats=[#c_literal{val=false}], guard=#c_literal{val=true}, - body=Xs}, - CC3 = #c_clause{pats=[X], guard=#c_literal{val=true}, - body=match_fail(Anno, Err1)}, - Case = #c_case{arg=B, clauses = [CC1, CC2, CC3]}, - C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, - body=#c_let{vars=[B], - arg=#c_apply{anno=Anno, op=F, args=[X]}, - body=#c_let{vars=[Xs], - arg=#c_apply{anno=Anno, - op=Loop, - args=[Xs]}, - body=Case}}}, - C2 = #c_clause{pats=[#c_literal{val=[]}], - guard=#c_call{module=#c_literal{val=erlang}, - name=#c_literal{val=is_function}, - args=[F, #c_literal{val=1}]}, - body=#c_literal{val=[]}}, - Err2 = #c_tuple{es=[#c_literal{val='function_clause'}, F, Xs]}, - C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, - body=match_fail([{function_name,{'lists^filter',1}}|Anno], Err2)}, - Fun = #c_fun{vars=[Xs], - body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, - L = #c_var{name='L'}, - expr(#c_let{vars=[F, L], arg=#c_values{es=[Arg1, Arg2]}, - body=#c_letrec{defs=[{Loop,Fun}], - body=#c_apply{anno=Anno, op=Loop, args=[L]}}}, - Sub); -call_1(#c_call{anno=Anno}, lists, foldl, [Arg1,Arg2,Arg3], Sub) -> - Loop = #c_var{name={'lists^foldl',2}}, - F = #c_var{name='F'}, - Xs = #c_var{name='Xs'}, - X = #c_var{name='X'}, - A = #c_var{name='A'}, - C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, - body=#c_apply{anno=Anno, - op=Loop, - args=[Xs, #c_apply{anno=Anno, - op=F, - args=[X, A]}]}}, - C2 = #c_clause{pats=[#c_literal{val=[]}], - guard=#c_call{module=#c_literal{val=erlang}, - name=#c_literal{val=is_function}, - args=[F, #c_literal{val=2}]}, - body=A}, - Err = #c_tuple{es=[#c_literal{val='function_clause'}, F, A, Xs]}, - C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, - body=match_fail([{function_name,{'lists^foldl',2}}|Anno], Err)}, - Fun = #c_fun{vars=[Xs, A], - body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, - L = #c_var{name='L'}, - expr(#c_let{vars=[F, A, L], arg=#c_values{es=[Arg1, Arg2, Arg3]}, - body=#c_letrec{defs=[{Loop,Fun}], - body=#c_apply{anno=Anno, op=Loop, args=[L, A]}}}, - Sub); -call_1(#c_call{anno=Anno}, lists, foldr, [Arg1,Arg2,Arg3], Sub) -> - Loop = #c_var{name={'lists^foldr',2}}, - F = #c_var{name='F'}, - Xs = #c_var{name='Xs'}, - X = #c_var{name='X'}, - A = #c_var{name='A'}, - C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, - body=#c_apply{anno=Anno, - op=F, - args=[X, #c_apply{anno=Anno, - op=Loop, - args=[Xs, A]}]}}, - C2 = #c_clause{pats=[#c_literal{val=[]}], - guard=#c_call{module=#c_literal{val=erlang}, - name=#c_literal{val=is_function}, - args=[F, #c_literal{val=2}]}, - body=A}, - Err = #c_tuple{es=[#c_literal{val='function_clause'}, F, A, Xs]}, - C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, - body=match_fail([{function_name,{'lists^foldr',2}}|Anno], Err)}, - Fun = #c_fun{vars=[Xs, A], - body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, - L = #c_var{name='L'}, - expr(#c_let{vars=[F, A, L], arg=#c_values{es=[Arg1, Arg2, Arg3]}, - body=#c_letrec{defs=[{Loop,Fun}], - body=#c_apply{anno=Anno, op=Loop, args=[L, A]}}}, - Sub); -call_1(#c_call{anno=Anno}, lists, mapfoldl, [Arg1,Arg2,Arg3], Sub) -> - Loop = #c_var{name={'lists^mapfoldl',2}}, - F = #c_var{name='F'}, - Xs = #c_var{name='Xs'}, - X = #c_var{name='X'}, - Avar = #c_var{name='A'}, - Match = - fun (A, P, E) -> - C1 = #c_clause{pats=[P], guard=#c_literal{val=true}, body=E}, - Err = #c_tuple{es=[#c_literal{val='badmatch'}, X]}, - C2 = #c_clause{pats=[X], guard=#c_literal{val=true}, - body=match_fail(Anno, Err)}, - #c_case{arg=A, clauses=[C1, C2]} - end, - C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, - body=Match(#c_apply{anno=Anno, op=F, args=[X, Avar]}, - #c_tuple{es=[X, Avar]}, -%%% Tuple passing version - Match(#c_apply{anno=Anno, - op=Loop, - args=[Xs, Avar]}, - #c_tuple{es=[Xs, Avar]}, - #c_tuple{anno=[compiler_generated], - es=[#c_cons{anno=[compiler_generated], - hd=X, tl=Xs}, - Avar]}) -%%% Multiple-value version -%%% #c_let{vars=[Xs,A], -%%% %% The tuple here will be optimised -%%% %% away later; no worries. -%%% arg=#c_apply{op=Loop, args=[Xs, A]}, -%%% body=#c_values{es=[#c_cons{hd=X, tl=Xs}, -%%% A]}} - )}, - C2 = #c_clause{pats=[#c_literal{val=[]}], - guard=#c_call{module=#c_literal{val=erlang}, - name=#c_literal{val=is_function}, - args=[F, #c_literal{val=2}]}, -%%% Tuple passing version - body=#c_tuple{anno=[compiler_generated], - es=[#c_literal{val=[]}, Avar]}}, -%%% Multiple-value version -%%% body=#c_values{es=[#c_literal{val=[]}, A]}}, - Err = #c_tuple{es=[#c_literal{val='function_clause'}, F, Avar, Xs]}, - C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, - body=match_fail([{function_name,{'lists^mapfoldl',2}}|Anno], Err)}, - Fun = #c_fun{vars=[Xs, Avar], - body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, - L = #c_var{name='L'}, - expr(#c_let{vars=[F, Avar, L], arg=#c_values{es=[Arg1, Arg2, Arg3]}, - body=#c_letrec{defs=[{Loop,Fun}], -%%% Tuple passing version - body=#c_apply{anno=Anno, - op=Loop, - args=[L, Avar]}}}, -%%% Multiple-value version -%%% body=#c_let{vars=[Xs, A], -%%% arg=#c_apply{op=Loop, -%%% args=[L, A]}, -%%% body=#c_tuple{es=[Xs, A]}}}}, - Sub); -call_1(#c_call{anno=Anno}, lists, mapfoldr, [Arg1,Arg2,Arg3], Sub) -> - Loop = #c_var{name={'lists^mapfoldr',2}}, - F = #c_var{name='F'}, - Xs = #c_var{name='Xs'}, - X = #c_var{name='X'}, - Avar = #c_var{name='A'}, - Match = - fun (A, P, E) -> - C1 = #c_clause{pats=[P], guard=#c_literal{val=true}, body=E}, - Err = #c_tuple{es=[#c_literal{val='badmatch'}, X]}, - C2 = #c_clause{pats=[X], guard=#c_literal{val=true}, - body=match_fail(Anno, Err)}, - #c_case{arg=A, clauses=[C1, C2]} - end, - C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, -%%% Tuple passing version - body=Match(#c_apply{anno=Anno, - op=Loop, - args=[Xs, Avar]}, - #c_tuple{es=[Xs, Avar]}, - Match(#c_apply{anno=Anno, op=F, args=[X, Avar]}, - #c_tuple{es=[X, Avar]}, - #c_tuple{anno=[compiler_generated], - es=[#c_cons{anno=[compiler_generated], - hd=X, tl=Xs}, Avar]})) -%%% Multiple-value version -%%% body=#c_let{vars=[Xs,A], -%%% %% The tuple will be optimised away -%%% arg=#c_apply{op=Loop, args=[Xs, A]}, -%%% body=Match(#c_apply{op=F, args=[X, A]}, -%%% #c_tuple{es=[X, A]}, -%%% #c_values{es=[#c_cons{hd=X, tl=Xs}, -%%% A]})} - }, - C2 = #c_clause{pats=[#c_literal{val=[]}], - guard=#c_call{module=#c_literal{val=erlang}, - name=#c_literal{val=is_function}, - args=[F, #c_literal{val=2}]}, -%%% Tuple passing version - body=#c_tuple{anno=[compiler_generated], - es=[#c_literal{val=[]}, Avar]}}, -%%% Multiple-value version -%%% body=#c_values{es=[#c_literal{val=[]}, A]}}, - Err = #c_tuple{es=[#c_literal{val='function_clause'}, F, Avar, Xs]}, - C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, - body=match_fail([{function_name,{'lists^mapfoldr',2}}|Anno], Err)}, - Fun = #c_fun{vars=[Xs, Avar], - body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, - L = #c_var{name='L'}, - expr(#c_let{vars=[F, Avar, L], arg=#c_values{es=[Arg1, Arg2, Arg3]}, - body=#c_letrec{defs=[{Loop,Fun}], -%%% Tuple passing version - body=#c_apply{anno=Anno, - op=Loop, - args=[L, Avar]}}}, -%%% Multiple-value version -%%% body=#c_let{vars=[Xs, A], -%%% arg=#c_apply{op=Loop, -%%% args=[L, A]}, -%%% body=#c_tuple{es=[Xs, A]}}}}, - Sub); -call_1(#c_call{module=M, name=N}=Call, _, _, As, Sub) -> - call_0(Call, M, N, As, Sub). - -match_fail(Anno, Arg) -> - #c_primop{anno=Anno, - name=#c_literal{val='match_fail'}, - args=[Arg]}. - %% fold_call(Call, Mod, Name, Args, Sub) -> Expr. %% Try to safely evaluate the call. Just try to evaluate arguments, %% do the call and convert return values to literals. If this @@ -1133,29 +756,33 @@ fold_call_1(Call, Mod, Name, Args, Sub) -> true -> fold_call_2(Call, Mod, Name, Args, Sub) end. -fold_call_2(Call, Module, Name, Args0, Sub) -> - try - Args = [core_lib:literal_value(A) || A <- Args0], - try apply(Module, Name, Args) of - Val -> - case cerl:is_literal_term(Val) of - true -> - #c_literal{val=Val}; - false -> - %% Successful evaluation, but it was not - %% possible to express the computed value as a literal. - Call - end - catch - error:Reason -> - %% Evaluation of the function failed. Warn and replace - %% the call with a call to erlang:error/1. - eval_failure(Call, Reason) - end +fold_call_2(Call, Module, Name, Args, Sub) -> + case all(fun cerl:is_literal/1, Args) of + true -> + %% All arguments are literals. + fold_lit_args(Call, Module, Name, Args); + false -> + %% At least one non-literal argument. + fold_non_lit_args(Call, Module, Name, Args, Sub) + end. + +fold_lit_args(Call, Module, Name, Args0) -> + Args = [cerl:concrete(A) || A <- Args0], + try apply(Module, Name, Args) of + Val -> + case cerl:is_literal_term(Val) of + true -> + cerl:abstract(Val); + false -> + %% Successful evaluation, but it was not possible + %% to express the computed value as a literal. + Call + end catch - error:_ -> - %% There was at least one non-literal argument. - fold_non_lit_args(Call, Module, Name, Args0, Sub) + error:Reason -> + %% Evaluation of the function failed. Warn and replace + %% the call with a call to erlang:error/1. + eval_failure(Call, Reason) end. %% fold_non_lit_args(Call, Module, Name, Args, Sub) -> Expr. @@ -1194,17 +821,18 @@ fold_non_lit_args(Call, _, _, _, _) -> Call. %% Evaluate a relational operation using type information. eval_rel_op(Call, Op, [#c_var{name=V},#c_var{name=V}], _) -> Bool = erlang:Op(same, same), - #c_literal{anno=core_lib:get_anno(Call),val=Bool}; -eval_rel_op(Call, '=:=', [#c_var{name=V}=Var,#c_literal{val=true}], Sub) -> + #c_literal{anno=cerl:get_ann(Call),val=Bool}; +eval_rel_op(Call, '=:=', [Term,#c_literal{val=true}], Sub) -> %% BoolVar =:= true ==> BoolVar - case is_boolean_type(V, Sub) of - true -> Var; - false -> Call + case is_boolean_type(Term, Sub) of + yes -> Term; + maybe -> Call; + no -> #c_literal{val=false} end; eval_rel_op(Call, '==', Ops, _Sub) -> case is_exact_eq_ok(Ops) of true -> - Name = #c_literal{anno=core_lib:get_anno(Call),val='=:='}, + Name = #c_literal{anno=cerl:get_ann(Call),val='=:='}, Call#c_call{name=Name}; false -> Call @@ -1212,7 +840,7 @@ eval_rel_op(Call, '==', Ops, _Sub) -> eval_rel_op(Call, '/=', Ops, _Sub) -> case is_exact_eq_ok(Ops) of true -> - Name = #c_literal{anno=core_lib:get_anno(Call),val='=/='}, + Name = #c_literal{anno=cerl:get_ann(Call),val='=/='}, Call#c_call{name=Name}; false -> Call @@ -1229,6 +857,11 @@ is_non_numeric([H|T]) -> is_non_numeric(H) andalso is_non_numeric(T); is_non_numeric(Tuple) when is_tuple(Tuple) -> is_non_numeric_tuple(Tuple, tuple_size(Tuple)); +is_non_numeric(Map) when is_map(Map) -> + %% Note that 17.x and 18.x compare keys in different ways. + %% Be very conservative -- require that both keys and values + %% are non-numeric. + is_non_numeric(maps:to_list(Map)); is_non_numeric(Num) when is_number(Num) -> false; is_non_numeric(_) -> true. @@ -1242,40 +875,31 @@ is_non_numeric_tuple(_Tuple, 0) -> true. %% there must be at least one non-literal argument (i.e. %% there is no need to handle the case that all argments %% are literal). -eval_bool_op(Call, 'and', [#c_literal{val=true},#c_var{name=V}=Res], Sub) -> - case is_boolean_type(V, Sub) of - true -> Res; - false-> Call - end; -eval_bool_op(Call, 'and', [#c_var{name=V}=Res,#c_literal{val=true}], Sub) -> - case is_boolean_type(V, Sub) of - true -> Res; - false-> Call - end; -eval_bool_op(Call, 'and', [#c_literal{val=false}=Res,#c_var{name=V}], Sub) -> - case is_boolean_type(V, Sub) of - true -> Res; - false-> Call - end; -eval_bool_op(Call, 'and', [#c_var{name=V},#c_literal{val=false}=Res], Sub) -> - case is_boolean_type(V, Sub) of - true -> Res; - false-> Call - end; + +eval_bool_op(Call, 'and', [#c_literal{val=true},Term], Sub) -> + eval_bool_op_1(Call, Term, Term, Sub); +eval_bool_op(Call, 'and', [Term,#c_literal{val=true}], Sub) -> + eval_bool_op_1(Call, Term, Term, Sub); +eval_bool_op(Call, 'and', [#c_literal{val=false}=Res,Term], Sub) -> + eval_bool_op_1(Call, Res, Term, Sub); +eval_bool_op(Call, 'and', [Term,#c_literal{val=false}=Res], Sub) -> + eval_bool_op_1(Call, Res, Term, Sub); eval_bool_op(Call, _, _, _) -> Call. +eval_bool_op_1(Call, Res, Term, Sub) -> + case is_boolean_type(Term, Sub) of + yes -> Res; + no -> eval_failure(Call, badarg); + maybe -> Call + end. + %% Evaluate is_boolean/1 using type information. -eval_is_boolean(Call, #c_var{name=V}, Sub) -> - case is_boolean_type(V, Sub) of - true -> #c_literal{val=true}; - false -> Call - end; -eval_is_boolean(_, #c_cons{}, _) -> - #c_literal{val=false}; -eval_is_boolean(_, #c_tuple{}, _) -> - #c_literal{val=false}; -eval_is_boolean(Call, _, _) -> - Call. +eval_is_boolean(Call, Term, Sub) -> + case is_boolean_type(Term, Sub) of + no -> #c_literal{val=false}; + yes -> #c_literal{val=true}; + maybe -> Call + end. %% eval_length(Call, List) -> Val. %% Evaluates the length for the prefix of List which has a known @@ -1325,33 +949,33 @@ eval_append(Call, X, Y) -> %% Evaluates element/2 if the position Pos is a literal and %% the shape of the tuple Tuple is known. %% -eval_element(Call, #c_literal{val=Pos}, #c_tuple{es=Es}, _Types) when is_integer(Pos) -> - if - 1 =< Pos, Pos =< length(Es) -> - lists:nth(Pos, Es); - true -> - eval_failure(Call, badarg) - end; -eval_element(Call, #c_literal{val=Pos}, #c_var{name=V}, Types) +eval_element(Call, #c_literal{val=Pos}, Tuple, Types) when is_integer(Pos) -> - case orddict:find(V, Types#sub.t) of - {ok,#c_tuple{es=Elements}} -> + case get_type(Tuple, Types) of + none -> + Call; + Type -> + Es = case cerl:is_c_tuple(Type) of + false -> []; + true -> cerl:tuple_es(Type) + end, if - 1 =< Pos, Pos =< length(Elements) -> - case lists:nth(Pos, Elements) of - #c_alias{var=Alias} -> Alias; - Res -> Res + 1 =< Pos, Pos =< length(Es) -> + El = lists:nth(Pos, Es), + try + pat_to_expr(El) + catch + throw:impossible -> + Call end; true -> + %% Index outside tuple or not a tuple. eval_failure(Call, badarg) - end; - {ok,_} -> - eval_failure(Call, badarg); - error -> - Call + end end; -eval_element(Call, Pos, Tuple, _Types) -> - case is_not_integer(Pos) orelse is_not_tuple(Tuple) of +eval_element(Call, Pos, Tuple, Sub) -> + case is_int_type(Pos, Sub) =:= no orelse + is_tuple_type(Tuple, Sub) =:= no of true -> eval_failure(Call, badarg); false -> @@ -1361,34 +985,27 @@ eval_element(Call, Pos, Tuple, _Types) -> %% eval_is_record(Call, Var, Tag, Size, Types) -> Val. %% Evaluates is_record/3 using type information. %% -eval_is_record(Call, #c_var{name=V}, #c_literal{val=NeededTag}=Lit, +eval_is_record(Call, Term, #c_literal{val=NeededTag}, #c_literal{val=Size}, Types) -> - case orddict:find(V, Types#sub.t) of - {ok,#c_tuple{es=[#c_literal{val=Tag}|_]=Es}} -> - Lit#c_literal{val=Tag =:= NeededTag andalso - length(Es) =:= Size}; - _ -> - Call + case get_type(Term, Types) of + none -> + Call; + Type -> + Es = case cerl:is_c_tuple(Type) of + false -> []; + true -> cerl:tuple_es(Type) + end, + case Es of + [#c_literal{val=Tag}|_] -> + Bool = Tag =:= NeededTag andalso + length(Es) =:= Size, + #c_literal{val=Bool}; + _ -> + #c_literal{val=false} + end end; eval_is_record(Call, _, _, _, _) -> Call. -%% is_not_integer(Core) -> true | false. -%% Returns true if Core is definitely not an integer. - -is_not_integer(#c_literal{val=Val}) when not is_integer(Val) -> true; -is_not_integer(#c_tuple{}) -> true; -is_not_integer(#c_cons{}) -> true; -is_not_integer(#c_map{}) -> true; -is_not_integer(_) -> false. - -%% is_not_tuple(Core) -> true | false. -%% Returns true if Core is definitely not a tuple. - -is_not_tuple(#c_literal{val=Val}) when not is_tuple(Val) -> true; -is_not_tuple(#c_cons{}) -> true; -is_not_tuple(#c_map{}) -> true; -is_not_tuple(_) -> false. - %% eval_setelement(Call, Pos, Tuple, NewVal) -> Core. %% Evaluates setelement/3 if position Pos is an integer %% the shape of the tuple Tuple is known. @@ -1492,7 +1109,7 @@ clause(#c_clause{pats=Ps0,guard=G0,body=B0}=Cl, Cexpr, Ctxt, Sub0) -> let_substs(Vs0, As0, Sub0) -> {Vs1,Sub1} = pattern_list(Vs0, Sub0), {Vs2,As1,Ss} = let_substs_1(Vs1, As0, Sub1), - Sub2 = scope_add([V || #c_var{name=V} <- Vs2], Sub1), + Sub2 = sub_add_scope([V || #c_var{name=V} <- Vs2], Sub1), {Vs2,As1, foldl(fun ({V,S}, Sub) -> sub_set_name(V, S, Sub) end, Sub2, Ss)}. @@ -1527,7 +1144,7 @@ pattern(#c_var{}=Pat, Isub, Osub) -> true -> V1 = make_var_name(), Pat1 = #c_var{name=V1}, - {Pat1,sub_set_var(Pat, Pat1, scope_add([V1], Osub))}; + {Pat1,sub_set_var(Pat, Pat1, sub_add_scope([V1], Osub))}; false -> {Pat,sub_del_var(Pat, Osub)} end; @@ -1597,6 +1214,7 @@ is_subst(_) -> false. %% sub_del_var(Var, #sub{}) -> #sub{}. %% sub_subst_var(Var, Value, #sub{}) -> [{Name,Value}]. %% sub_is_val(Var, #sub{}) -> boolean(). +%% sub_add_scope(#sub{}) -> #sub{} %% sub_subst_scope(#sub{}) -> #sub{} %% %% We use the variable name as key so as not have problems with @@ -1607,9 +1225,10 @@ is_subst(_) -> false. %% In addition to the list of substitutions, we also keep track of %% all variable currently live (the scope). %% -%% sub_subst_scope/1 adds dummy substitutions for all variables -%% in the scope in order to force renaming if variables in the -%% scope occurs as pattern variables. +%% sub_add_scope/2 adds variables to the scope. sub_subst_scope/1 +%% adds dummy substitutions for all variables in the scope in order +%% to force renaming if variables in the scope occurs as pattern +%% variables. sub_new() -> #sub{v=orddict:new(),s=gb_trees:empty(),t=[]}. @@ -1649,6 +1268,12 @@ sub_subst_var(#c_var{name=V}, Val, #sub{v=S0}) -> %% Fold chained substitutions. [{V,Val}] ++ [ {K,Val} || {K,#c_var{name=V1}} <- S0, V1 =:= V]. +sub_add_scope(Vs, #sub{s=Scope0}=Sub) -> + Scope = foldl(fun(V, S) when is_integer(V); is_atom(V) -> + gb_sets:add(V, S) + end, Scope0, Vs), + Sub#sub{s=Scope}. + sub_subst_scope(#sub{v=S0,s=Scope}=Sub) -> S = [{-1,#c_var{name=Sv}} || Sv <- gb_sets:to_list(Scope)]++S0, Sub#sub{v=S}. @@ -1696,7 +1321,7 @@ clauses(E, [C0|Cs], Ctxt, Sub, LitExpr) -> {yes,yes} -> case LitExpr of false -> - Line = get_line(core_lib:get_anno(C1)), + Line = get_line(cerl:get_ann(C1)), shadow_warning(Cs, Line); true -> %% If the case expression is a literal, @@ -1930,7 +1555,7 @@ opt_bool_case_guard(#c_case{arg=Arg,clauses=Cs0}=Case) -> Case; true -> Cs = opt_bool_case_guard(Arg, Cs0), - Case#c_case{arg=#c_values{anno=core_lib:get_anno(Arg),es=[]}, + Case#c_case{arg=#c_values{anno=cerl:get_ann(Arg),es=[]}, clauses=Cs} end. @@ -2041,31 +1666,75 @@ case_opt_args([], Cs, _Sub, _LitExpr, Acc) -> %% or to remove a literal argument. %% case_opt_arg(E0, Sub, Cs, LitExpr) -> - E = maybe_replace_var(E0, Sub), - case cerl:is_data(E) of + case cerl:is_c_var(E0) of + false -> + case_opt_arg_1(E0, Cs, LitExpr); + true -> + case case_will_var_match(Cs) of + true -> + %% All clauses will match a variable in the + %% current position. Don't expand this variable + %% (that can only make the code worse). + {error,Cs}; + false -> + %% If possible, expand this variable to a previously + %% matched term. + E = case_expand_var(E0, Sub), + case_opt_arg_1(E, Cs, LitExpr) + end + end. + +case_opt_arg_1(E0, Cs0, LitExpr) -> + case cerl:is_data(E0) of false -> - {error,Cs}; + {error,Cs0}; true -> + E = case_opt_compiler_generated(E0), + Cs = case_opt_nomatch(E, Cs0, LitExpr), case cerl:data_type(E) of {atomic,_} -> - case_opt_lit(E, Cs, LitExpr); + case_opt_lit(E, Cs); _ -> - case_opt_data(E, Cs, LitExpr) + case_opt_data(E, Cs) end end. -%% maybe_replace_var(Expr0, Sub) -> Expr +%% case_will_var_match([Clause]) -> true | false. +%% Return if all clauses will match a variable in the +%% current position. +%% +case_will_var_match(Cs) -> + all(fun({[P|_],_,_,_}) -> + case cerl_clauses:match(P, any) of + {true,_} -> true; + _ -> false + end + end, Cs). + + +%% case_opt_compiler_generated(Core) -> Core' +%% Mark Core expressions as compiler generated to ensure that +%% no warnings are generated if they turn out to be unused. +%% To pretty-printed Core Erlang easier to read, don't mark +%% constructs that can't cause warnings to be emitted. +%% +case_opt_compiler_generated(Core) -> + F = fun(C) -> + case cerl:type(C) of + alias -> C; + var -> C; + _ -> cerl:set_ann(C, [compiler_generated]) + end + end, + cerl_trees:map(F, Core). + + +%% case_expand_var(Expr0, Sub) -> Expr %% If Expr0 is a variable that has been previously matched and %% is known to be a tuple, return the tuple instead. Otherwise %% return Expr0 unchanged. %% -maybe_replace_var(E, Sub) -> - case cerl:is_c_var(E) of - false -> E; - true -> maybe_replace_var_1(E, Sub) - end. - -maybe_replace_var_1(E, #sub{t=Tdb}) -> +case_expand_var(E, #sub{t=Tdb}) -> case orddict:find(cerl:var_name(E), Tdb) of {ok,T0} -> case cerl:is_c_tuple(T0) of @@ -2082,9 +1751,8 @@ maybe_replace_var_1(E, #sub{t=Tdb}) -> %% operator will fail when used in map %% construction (only the '=>' operator is allowed %% when constructing a map from scratch). - ToData = fun coerce_to_data/1, try - cerl_trees:map(ToData, T0) + cerl_trees:map(fun coerce_to_data/1, T0) catch throw:impossible -> %% Something unsuitable was found (map or @@ -2113,140 +1781,144 @@ coerce_to_data(C) -> coerce_to_data(cerl:alias_pat(C)) end. -%% case_opt_lit(Literal, Clauses0, LitExpr) -> -%% {ok,[],Clauses} | error +%% case_opt_nomatch(E, Clauses, LitExpr) -> Clauses' +%% Remove all clauses that cannot possibly match. + +case_opt_nomatch(E, [{[P|_],C,_,_}=Current|Cs], LitExpr) -> + case cerl_clauses:match(P, E) of + none -> + %% The pattern will not match the case expression. Remove + %% the clause. Unless the entire case expression is a + %% literal, also emit a warning. + case LitExpr of + false -> add_warning(C, nomatch_clause_type); + true -> ok + end, + case_opt_nomatch(E, Cs, LitExpr); + _ -> + [Current|case_opt_nomatch(E, Cs, LitExpr)] + end; +case_opt_nomatch(_, [], _) -> []. + +%% case_opt_lit(Literal, Clauses0) -> {ok,[],Clauses} | error %% The current part of the case expression is a literal. That %% means that we will know at compile-time whether a clause %% will match, and we can remove the corresponding pattern from %% each clause. %% -%% The only complication is if the literal is a binary. Binary -%% pattern matching is tricky, so we will give up in that case. +%% The only complication is if the literal is a binary or map. +%% In general, it is difficult to know whether a binary or +%% map pattern will match, so we give up in that case. -case_opt_lit(Lit, Cs0, LitExpr) -> - Cs1 = case_opt_lit_1(Lit, Cs0, LitExpr), - try case_opt_lit_2(Lit, Cs1) of +case_opt_lit(Lit, Cs0) -> + try case_opt_lit_1(Lit, Cs0) of Cs -> {ok,[],Cs} catch throw:impossible -> - {error,Cs1} + {error,Cs0} end. -case_opt_lit_1(E, [{[P|_],C,_,_}=Current|Cs], LitExpr) -> - case cerl_clauses:match(P, E) of - none -> - %% The pattern will not match the literal. Remove the clause. - %% Unless the entire case expression is a literal, also - %% emit a warning. - case LitExpr of - false -> add_warning(C, nomatch_clause_type); - true -> ok - end, - case_opt_lit_1(E, Cs, LitExpr); - _ -> - [Current|case_opt_lit_1(E, Cs, LitExpr)] - end; -case_opt_lit_1(_, [], _) -> []. - -case_opt_lit_2(E, [{[P|Ps],C,PsAcc,Bs0}|Cs]) -> - %% Non-matching clauses have already been removed in case_opt_lit_1/3. +case_opt_lit_1(E, [{[P|Ps],C,PsAcc,Bs0}|Cs]) -> + %% Non-matching clauses have already been removed + %% in case_opt_nomatch/3. case cerl_clauses:match(P, E) of {true,Bs} -> %% The pattern matches the literal. Remove the pattern %% and update the bindings. - [{Ps,C,PsAcc,Bs++Bs0}|case_opt_lit_2(E, Cs)]; + [{Ps,C,PsAcc,Bs++Bs0}|case_opt_lit_1(E, Cs)]; {false,_} -> %% Binary literal and pattern. We are not sure whether %% the pattern will match. throw(impossible) end; -case_opt_lit_2(_, []) -> []. +case_opt_lit_1(_, []) -> []. %% case_opt_data(Expr, Clauses0, LitExpr) -> {ok,Exprs,Clauses} +%% The case expression is a non-atomic data constructor (cons +%% or tuple). We can know at compile time whether each clause +%% will match, and we can delay the building of the data to +%% the clauses where it is actually needed. -case_opt_data(E, Cs0, LitExpr) -> +case_opt_data(E, Cs0) -> Es = cerl:data_es(E), - Cs = case_opt_data_1(Cs0, Es, - {cerl:data_type(E),cerl:data_arity(E)}, - LitExpr), - {ok,Es,Cs}. - -case_opt_data_1([{[P|Ps0],C,PsAcc,Bs0}|Cs], Es, TypeSig, LitExpr) -> - case case_data_pat(P, TypeSig) of - {ok,Ps1,Bs1} -> - [{Ps1++Ps0,C,PsAcc,Bs1++Bs0}| - case_opt_data_1(Cs, Es, TypeSig,LitExpr)]; - error -> - case LitExpr of - false -> add_warning(C, nomatch_clause_type); - true -> ok - end, - case_opt_data_1(Cs, Es, TypeSig, LitExpr) - end; -case_opt_data_1([], _, _, _) -> []. - -%% case_data_pat(Pattern, Type, Arity) -> {ok,[Pattern],[{AliasVar,Pat}]} | error. - -case_data_pat(P, TypeSig) -> - case cerl:is_data(P) of - false -> - case_data_pat_var(P, TypeSig); - true -> - case {cerl:data_type(P),cerl:data_arity(P)} of - TypeSig -> - {ok,cerl:data_es(P),[]}; - {_,_} -> - error - end + TypeSig = {cerl:data_type(E),cerl:data_arity(E)}, + try case_opt_data_1(Cs0, Es, TypeSig) of + Cs -> + {ok,Es,Cs} + catch + throw:impossible -> + %% The pattern contained a binary or map. + {error,Cs0} end. -%% case_data_pat_var(Pattern, {DataType,ArityType}) -> -%% {ok,[Pattern],[{AliasVar,Pat}]} +case_opt_data_1([{[P0|Ps0],C,PsAcc,Bs0}|Cs], Es, TypeSig) -> + P = case_opt_compiler_generated(P0), + BindTo = #c_var{name=dummy}, + {Ps1,[{BindTo,_}|Bs1]} = case_data_pat_alias(P, BindTo, TypeSig, []), + [{Ps1++Ps0,C,PsAcc,Bs1++Bs0}|case_opt_data_1(Cs, Es, TypeSig)]; +case_opt_data_1([], _, _) -> []. -case_data_pat_var(P, {Type,Arity}=TypeSig) -> - %% If the entire case statement is evaluated in an effect - %% context (e.g. "case {A,B} of ... end, ok"), there will - %% be a warning that a term is constructed but never used. - %% To avoid that warning, we must annotate the data - %% constructor as compiler generated. - Ann = [compiler_generated|cerl:get_ann(P)], +case_data_pat_alias(P, BindTo0, TypeSig, Bs0) -> case cerl:type(P) of - var -> - Vars = make_vars(cerl:get_ann(P), Arity), - {ok,Vars,[{P,cerl:ann_make_data(Ann, Type, Vars)}]}; alias -> - V = cerl:alias_var(P), - Apat = cerl:alias_pat(P), - case case_data_pat(Apat, TypeSig) of - {ok,Ps,Bs} -> - {ok,Ps,[{V,cerl:ann_make_data(Ann, Type, unalias_pat_list(Ps))}|Bs]}; - error -> - error - end; + %% Recursively handle the pattern and bind to + %% the alias variable. + BindTo = cerl:alias_var(P), + Apat0 = cerl:alias_pat(P), + Ann = [compiler_generated], + Apat = cerl:set_ann(Apat0, Ann), + {Ps,Bs} = case_data_pat_alias(Apat, BindTo, TypeSig, Bs0), + {Ps,[{BindTo0,BindTo}|Bs]}; + var -> + %% Here we will need to actually build the data and bind + %% it to the variable. + {Type,Arity} = TypeSig, + Vars = make_vars([], Arity), + Ann = [compiler_generated], + Data = cerl:ann_make_data(Ann, Type, Vars), + Bs = [{BindTo0,P},{P,Data}|Bs0], + {Vars,Bs}; _ -> - error + %% Since case_opt_nomatch/3 has removed all clauses that + %% cannot match, we KNOW that this clause must match and + %% that the pattern must be a data constructor. + %% Here we must build the data and bind it to the variable. + {Type,_} = TypeSig, + DataEs = cerl:data_es(P), + Vars = pat_to_expr_list(DataEs), + Ann = [compiler_generated], + Data = cerl:ann_make_data(Ann, Type, Vars), + {DataEs,[{BindTo0,Data}]} end. -%% unalias_pat(Pattern) -> Pattern. -%% Remove all the aliases in a pattern but using the alias variables -%% instead of the values. We KNOW they will be bound. +%% pat_to_expr(Pattern) -> Expression. +%% Convert a pattern to an expression if possible. We KNOW that +%% all variables in the pattern will be bound. +%% +%% Throw an 'impossible' exception if a map or (non-literal) +%% binary is encountered. Trying to use a map pattern as an +%% expression is incorrect, while rebuilding a potentially +%% huge binary in an expression would be wasteful. -unalias_pat(P) -> - case cerl:is_c_alias(P) of - true -> +pat_to_expr(P) -> + case cerl:type(P) of + alias -> cerl:alias_var(P); - false -> + var -> + P; + _ -> case cerl:is_data(P) of false -> - P; + %% Map or binary. + throw(impossible); true -> - Es = unalias_pat_list(cerl:data_es(P)), + Es = pat_to_expr_list(cerl:data_es(P)), cerl:update_data(P, cerl:data_type(P), Es) end end. -unalias_pat_list(Ps) -> [unalias_pat(P) || P <- Ps]. +pat_to_expr_list(Ps) -> [pat_to_expr(P) || P <- Ps]. make_vars(A, Max) -> make_vars(A, 1, Max). @@ -2264,18 +1936,11 @@ make_var_name() -> list_to_atom("fol"++integer_to_list(N)). letify(Bs, Body) -> + Ann = cerl:get_ann(Body), foldr(fun({V,Val}, B) -> - letify(V, Val, B) + cerl:ann_c_let(Ann, [V], Val, B) end, Body, Bs). -letify(#c_var{name=Vname}=Var, Val, Body) -> - case core_lib:is_var_used(Vname, Body) of - true -> - A = element(2, Body), - #c_let{anno=A,vars=[Var],arg=Val,body=Body}; - false -> Body - end. - %% opt_case_in_let(LetExpr) -> LetExpr' opt_case_in_let(#c_let{vars=Vs,arg=Arg,body=B}=Let, Sub) -> @@ -2364,11 +2029,8 @@ is_bool_expr(#c_let{vars=[V],arg=Arg,body=B}, Sub0) -> is_bool_expr(#c_let{body=B}, Sub) -> %% Binding of multiple variables. is_bool_expr(B, Sub); -is_bool_expr(#c_literal{val=Bool}, _) when is_boolean(Bool) -> - true; -is_bool_expr(#c_var{name=V}, Sub) -> - is_boolean_type(V, Sub); -is_bool_expr(_, _) -> false. +is_bool_expr(C, Sub) -> + is_boolean_type(C, Sub) =:= yes. is_bool_expr_list([C|Cs], Sub) -> is_bool_expr(C, Sub) andalso is_bool_expr_list(Cs, Sub); @@ -2582,12 +2244,6 @@ move_let_into_expr(_Let, _Expr, _Sub) -> impossible. is_failing_clause(#c_clause{body=B}) -> will_fail(B). -scope_add(Vs, #sub{s=Scope0}=Sub) -> - Scope = foldl(fun(V, S) when is_integer(V); is_atom(V) -> - gb_sets:add(V, S) - end, Scope0, Vs), - Sub#sub{s=Scope}. - %% opt_simple_let(#c_let{}, Context, Sub) -> CoreTerm %% Optimize a let construct that does not contain any lets in %% in its argument. @@ -2616,31 +2272,7 @@ opt_simple_let_1(#c_let{vars=Vs0,body=B0}=Let, Arg0, Ctxt, Sub0) -> Arg = core_lib:make_values(Args), opt_simple_let_2(Let, Vs, Arg, B, Ctxt, Sub1). -opt_simple_let_2(Let0, Vs0, Arg0, Body0, effect, Sub) -> - case {Vs0,Arg0,Body0} of - {[],#c_values{es=[]},Body} -> - %% No variables left (because of substitutions). - Body; - {[_|_],Arg,#c_literal{}} -> - %% The body is a literal. That means that we can ignore - %% it and that the return value is Arg revisited in - %% effect context. - body(Arg, effect, sub_new_preserve_types(Sub)); - {Vs,Arg,Body} -> - %% Since we are in effect context, there is a chance - %% that the body no longer references the variables. - %% In that case we can construct a sequence and visit - %% that in effect context: - %% let <Var> = Arg in BodyWithoutVar ==> seq Arg BodyWithoutVar - case is_any_var_used(Vs, Body) of - false -> - expr(#c_seq{arg=Arg,body=Body}, effect, sub_new_preserve_types(Sub)); - true -> - Let = Let0#c_let{vars=Vs,arg=Arg,body=Body}, - opt_case_in_let_arg(opt_case_in_let(Let, Sub), effect, Sub) - end - end; -opt_simple_let_2(Let, Vs0, Arg0, Body, value, Sub) -> +opt_simple_let_2(Let0, Vs0, Arg0, Body, Ctxt, Sub) -> case {Vs0,Arg0,Body} of {[#c_var{name=N1}],Arg,#c_var{name=N2}} -> case N1 =:= N2 of @@ -2649,19 +2281,31 @@ opt_simple_let_2(Let, Vs0, Arg0, Body, value, Sub) -> Arg; false -> %% let <Var> = Arg in <OtherVar> ==> seq Arg OtherVar - expr(#c_seq{arg=Arg,body=Body}, value, sub_new_preserve_types(Sub)) + expr(#c_seq{arg=Arg,body=Body}, Ctxt, + sub_new_preserve_types(Sub)) end; {[],#c_values{es=[]},_} -> %% No variables left. Body; {_,Arg,#c_literal{}} -> - %% The variable is not used in the body. The argument - %% can be evaluated in effect context to simplify it. - expr(#c_seq{arg=Arg,body=Body}, value, sub_new_preserve_types(Sub)); + %% Since the variable is not used in the body, we can rewrite the + %% let to a sequence: + %% let <Var> = Arg in Literal ==> seq Arg Literal + expr(#c_seq{arg=Arg,body=Body}, Ctxt, sub_new_preserve_types(Sub)); {Vs,Arg,Body} -> - opt_case_in_let_arg( - opt_case_in_let(Let#c_let{vars=Vs,arg=Arg,body=Body}, Sub), - value, Sub) + %% If none of the variables are used in the body, we can + %% rewrite the let to a sequence: + %% let <Var> = Arg in BodyWithoutVar ==> + %% seq Arg BodyWithoutVar + case is_any_var_used(Vs, Body) of + false -> + expr(#c_seq{arg=Arg,body=Body}, Ctxt, + sub_new_preserve_types(Sub)); + true -> + Let1 = Let0#c_let{vars=Vs,arg=Arg,body=Body}, + Let2 = opt_case_in_let(Let1, Sub), + opt_case_in_let_arg(Let2, Ctxt, Sub) + end end. move_case_into_arg(#c_case{arg=#c_let{vars=OuterVars0,arg=OuterArg, @@ -2784,12 +2428,61 @@ is_any_var_used([#c_var{name=V}|Vs], Expr) -> end; is_any_var_used([], _) -> false. -is_boolean_type(V, #sub{t=Tdb}) -> +%%% +%%% Retrieving information about types. +%%% + +-spec get_type(cerl:cerl(), #sub{}) -> type_info() | 'none'. + +get_type(#c_var{name=V}, #sub{t=Tdb}) -> case orddict:find(V, Tdb) of - {ok,bool} -> true; - _ -> false + {ok,Type} -> Type; + error -> none + end; +get_type(C, _) -> + case cerl:type(C) of + binary -> C; + map -> C; + _ -> + case cerl:is_data(C) of + true -> C; + false -> none + end + end. + +-spec is_boolean_type(cerl:cerl(), sub()) -> yes_no_maybe(). + +is_boolean_type(Var, Sub) -> + case get_type(Var, Sub) of + none -> + maybe; + bool -> + yes; + C -> + B = cerl:is_c_atom(C) andalso + is_boolean(cerl:atom_val(C)), + yes_no(B) + end. + +-spec is_int_type(cerl:cerl(), sub()) -> yes_no_maybe(). + +is_int_type(Var, Sub) -> + case get_type(Var, Sub) of + none -> maybe; + C -> yes_no(cerl:is_c_int(C)) + end. + +-spec is_tuple_type(cerl:cerl(), sub()) -> yes_no_maybe(). + +is_tuple_type(Var, Sub) -> + case get_type(Var, Sub) of + none -> maybe; + C -> yes_no(cerl:is_c_tuple(C)) end. +yes_no(true) -> yes; +yes_no(false) -> no. + %% update_types(Expr, Pattern, Sub) -> Sub' %% Update the type database. update_types(Expr, Pat, #sub{t=Tdb0}=Sub) -> @@ -3111,11 +2804,11 @@ add_bin_opt_info(Core, Term) -> end. add_warning(Core, Term) -> - case is_compiler_generated(Core) of + case suppress_warning(Core) of true -> ok; false -> - Anno = core_lib:get_anno(Core), + Anno = cerl:get_ann(Core), Line = get_line(Anno), File = get_file(Anno), Key = {?MODULE,warnings}, @@ -3136,9 +2829,17 @@ get_file([{file,File}|_]) -> File; get_file([_|T]) -> get_file(T); get_file([]) -> "no_file". % should not happen +suppress_warning(Core) -> + is_compiler_generated(Core) orelse + is_result_unwanted(Core). + is_compiler_generated(Core) -> - Anno = core_lib:get_anno(Core), - member(compiler_generated, Anno). + Ann = cerl:get_ann(Core), + member(compiler_generated, Ann). + +is_result_unwanted(Core) -> + Ann = cerl:get_ann(Core), + member(result_not_wanted, Ann). get_warnings() -> ordsets:from_list((erase({?MODULE,warnings}))). diff --git a/lib/compiler/src/sys_core_fold_lists.erl b/lib/compiler/src/sys_core_fold_lists.erl new file mode 100644 index 0000000000..49dc59052a --- /dev/null +++ b/lib/compiler/src/sys_core_fold_lists.erl @@ -0,0 +1,386 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2015. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% Purpose : Inline high order lists functions from the lists module. + +-module(sys_core_fold_lists). + +-export([call/4]). + +-include("core_parse.hrl"). + +%% We inline some very common higher order list operations. +%% We use the same evaluation order as the library function. + +-spec call(cerl:c_call(), atom(), atom(), [cerl:cerl()]) -> + 'none' | cerl:cerl(). + +call(#c_call{anno=Anno}, lists, all, [Arg1,Arg2]) -> + Loop = #c_var{name={'lists^all',1}}, + F = #c_var{name='F'}, + Xs = #c_var{name='Xs'}, + X = #c_var{name='X'}, + Err1 = #c_tuple{es=[#c_literal{val='case_clause'}, X]}, + CC1 = #c_clause{pats=[#c_literal{val=true}], guard=#c_literal{val=true}, + body=#c_apply{anno=Anno, op=Loop, args=[Xs]}}, + CC2 = #c_clause{pats=[#c_literal{val=false}], guard=#c_literal{val=true}, + body=#c_literal{val=false}}, + CC3 = #c_clause{pats=[X], guard=#c_literal{val=true}, + body=match_fail(Anno, Err1)}, + C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, + body=#c_case{arg=#c_apply{anno=Anno, op=F, args=[X]}, + clauses = [CC1, CC2, CC3]}}, + C2 = #c_clause{pats=[#c_literal{val=[]}], + guard=#c_call{module=#c_literal{val=erlang}, + name=#c_literal{val=is_function}, + args=[F, #c_literal{val=1}]}, + body=#c_literal{val=true}}, + Err2 = #c_tuple{es=[#c_literal{val='function_clause'}, F, Xs]}, + C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, + body=match_fail([{function_name,{'lists^all',1}}|Anno], Err2)}, + Fun = #c_fun{vars=[Xs], + body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, + L = #c_var{name='L'}, + #c_let{vars=[F, L], arg=#c_values{es=[Arg1, Arg2]}, + body=#c_letrec{defs=[{Loop,Fun}], + body=#c_apply{anno=Anno, op=Loop, args=[L]}}}; +call(#c_call{anno=Anno}, lists, any, [Arg1,Arg2]) -> + Loop = #c_var{name={'lists^any',1}}, + F = #c_var{name='F'}, + Xs = #c_var{name='Xs'}, + X = #c_var{name='X'}, + Err1 = #c_tuple{es=[#c_literal{val='case_clause'}, X]}, + CC1 = #c_clause{pats=[#c_literal{val=true}], guard=#c_literal{val=true}, + body=#c_literal{val=true}}, + CC2 = #c_clause{pats=[#c_literal{val=false}], guard=#c_literal{val=true}, + body=#c_apply{anno=Anno, op=Loop, args=[Xs]}}, + CC3 = #c_clause{pats=[X], guard=#c_literal{val=true}, + body=match_fail(Anno, Err1)}, + C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, + body=#c_case{arg=#c_apply{anno=Anno, op=F, args=[X]}, + clauses = [CC1, CC2, CC3]}}, + C2 = #c_clause{pats=[#c_literal{val=[]}], + guard=#c_call{module=#c_literal{val=erlang}, + name=#c_literal{val=is_function}, + args=[F, #c_literal{val=1}]}, + body=#c_literal{val=false}}, + Err2 = #c_tuple{es=[#c_literal{val='function_clause'}, F, Xs]}, + C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, + body=match_fail([{function_name,{'lists^any',1}}|Anno], Err2)}, + Fun = #c_fun{vars=[Xs], + body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, + L = #c_var{name='L'}, + #c_let{vars=[F, L], arg=#c_values{es=[Arg1, Arg2]}, + body=#c_letrec{defs=[{Loop,Fun}], + body=#c_apply{anno=Anno, op=Loop, args=[L]}}}; +call(#c_call{anno=Anno}, lists, foreach, [Arg1,Arg2]) -> + Loop = #c_var{name={'lists^foreach',1}}, + F = #c_var{name='F'}, + Xs = #c_var{name='Xs'}, + X = #c_var{name='X'}, + C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, + body=#c_seq{arg=#c_apply{anno=Anno, op=F, args=[X]}, + body=#c_apply{anno=Anno, op=Loop, args=[Xs]}}}, + C2 = #c_clause{pats=[#c_literal{val=[]}], + guard=#c_call{module=#c_literal{val=erlang}, + name=#c_literal{val=is_function}, + args=[F, #c_literal{val=1}]}, + body=#c_literal{val=ok}}, + Err = #c_tuple{es=[#c_literal{val='function_clause'}, F, Xs]}, + C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, + body=match_fail([{function_name,{'lists^foreach',1}}|Anno], Err)}, + Fun = #c_fun{vars=[Xs], + body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, + L = #c_var{name='L'}, + #c_let{vars=[F, L], arg=#c_values{es=[Arg1, Arg2]}, + body=#c_letrec{defs=[{Loop,Fun}], + body=#c_apply{anno=Anno, op=Loop, args=[L]}}}; +call(#c_call{anno=Anno}, lists, map, [Arg1,Arg2]) -> + Loop = #c_var{name={'lists^map',1}}, + F = #c_var{name='F'}, + Xs = #c_var{name='Xs'}, + X = #c_var{name='X'}, + H = #c_var{name='H'}, + C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, + body=#c_let{vars=[H], arg=#c_apply{anno=Anno, + op=F, + args=[X]}, + body=#c_cons{hd=H, + anno=[compiler_generated], + tl=#c_apply{anno=Anno, + op=Loop, + args=[Xs]}}}}, + C2 = #c_clause{pats=[#c_literal{val=[]}], + guard=#c_call{module=#c_literal{val=erlang}, + name=#c_literal{val=is_function}, + args=[F, #c_literal{val=1}]}, + body=#c_literal{val=[]}}, + Err = #c_tuple{es=[#c_literal{val='function_clause'}, F, Xs]}, + C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, + body=match_fail([{function_name,{'lists^map',1}}|Anno], Err)}, + Fun = #c_fun{vars=[Xs], + body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, + L = #c_var{name='L'}, + #c_let{vars=[F, L], arg=#c_values{es=[Arg1, Arg2]}, + body=#c_letrec{defs=[{Loop,Fun}], + body=#c_apply{anno=Anno, op=Loop, args=[L]}}}; +call(#c_call{anno=Anno}, lists, flatmap, [Arg1,Arg2]) -> + Loop = #c_var{name={'lists^flatmap',1}}, + F = #c_var{name='F'}, + Xs = #c_var{name='Xs'}, + X = #c_var{name='X'}, + H = #c_var{name='H'}, + C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, + body=#c_let{vars=[H], + arg=#c_apply{anno=Anno, op=F, args=[X]}, + body=#c_call{anno=[compiler_generated|Anno], + module=#c_literal{val=erlang}, + name=#c_literal{val='++'}, + args=[H, + #c_apply{anno=Anno, + op=Loop, + args=[Xs]}]}}}, + C2 = #c_clause{pats=[#c_literal{val=[]}], + guard=#c_call{module=#c_literal{val=erlang}, + name=#c_literal{val=is_function}, + args=[F, #c_literal{val=1}]}, + body=#c_literal{val=[]}}, + Err = #c_tuple{es=[#c_literal{val='function_clause'}, F, Xs]}, + C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, + body=match_fail([{function_name,{'lists^flatmap',1}}|Anno], Err)}, + Fun = #c_fun{vars=[Xs], + body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, + L = #c_var{name='L'}, + #c_let{vars=[F, L], arg=#c_values{es=[Arg1, Arg2]}, + body=#c_letrec{defs=[{Loop,Fun}], + body=#c_apply{anno=Anno, op=Loop, args=[L]}}}; +call(#c_call{anno=Anno}, lists, filter, [Arg1,Arg2]) -> + Loop = #c_var{name={'lists^filter',1}}, + F = #c_var{name='F'}, + Xs = #c_var{name='Xs'}, + X = #c_var{name='X'}, + B = #c_var{name='B'}, + Err1 = #c_tuple{es=[#c_literal{val='case_clause'}, X]}, + CC1 = #c_clause{pats=[#c_literal{val=true}], guard=#c_literal{val=true}, + body=#c_cons{anno=[compiler_generated], hd=X, tl=Xs}}, + CC2 = #c_clause{pats=[#c_literal{val=false}], guard=#c_literal{val=true}, + body=Xs}, + CC3 = #c_clause{pats=[X], guard=#c_literal{val=true}, + body=match_fail(Anno, Err1)}, + Case = #c_case{arg=B, clauses = [CC1, CC2, CC3]}, + C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, + body=#c_let{vars=[B], + arg=#c_apply{anno=Anno, op=F, args=[X]}, + body=#c_let{vars=[Xs], + arg=#c_apply{anno=Anno, + op=Loop, + args=[Xs]}, + body=Case}}}, + C2 = #c_clause{pats=[#c_literal{val=[]}], + guard=#c_call{module=#c_literal{val=erlang}, + name=#c_literal{val=is_function}, + args=[F, #c_literal{val=1}]}, + body=#c_literal{val=[]}}, + Err2 = #c_tuple{es=[#c_literal{val='function_clause'}, F, Xs]}, + C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, + body=match_fail([{function_name,{'lists^filter',1}}|Anno], Err2)}, + Fun = #c_fun{vars=[Xs], + body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, + L = #c_var{name='L'}, + #c_let{vars=[F, L], arg=#c_values{es=[Arg1, Arg2]}, + body=#c_letrec{defs=[{Loop,Fun}], + body=#c_apply{anno=Anno, op=Loop, args=[L]}}}; +call(#c_call{anno=Anno}, lists, foldl, [Arg1,Arg2,Arg3]) -> + Loop = #c_var{name={'lists^foldl',2}}, + F = #c_var{name='F'}, + Xs = #c_var{name='Xs'}, + X = #c_var{name='X'}, + A = #c_var{name='A'}, + C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, + body=#c_apply{anno=Anno, + op=Loop, + args=[Xs, #c_apply{anno=Anno, + op=F, + args=[X, A]}]}}, + C2 = #c_clause{pats=[#c_literal{val=[]}], + guard=#c_call{module=#c_literal{val=erlang}, + name=#c_literal{val=is_function}, + args=[F, #c_literal{val=2}]}, + body=A}, + Err = #c_tuple{es=[#c_literal{val='function_clause'}, F, A, Xs]}, + C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, + body=match_fail([{function_name,{'lists^foldl',2}}|Anno], Err)}, + Fun = #c_fun{vars=[Xs, A], + body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, + L = #c_var{name='L'}, + #c_let{vars=[F, A, L], arg=#c_values{es=[Arg1, Arg2, Arg3]}, + body=#c_letrec{defs=[{Loop,Fun}], + body=#c_apply{anno=Anno, op=Loop, args=[L, A]}}}; +call(#c_call{anno=Anno}, lists, foldr, [Arg1,Arg2,Arg3]) -> + Loop = #c_var{name={'lists^foldr',2}}, + F = #c_var{name='F'}, + Xs = #c_var{name='Xs'}, + X = #c_var{name='X'}, + A = #c_var{name='A'}, + C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, + body=#c_apply{anno=Anno, + op=F, + args=[X, #c_apply{anno=Anno, + op=Loop, + args=[Xs, A]}]}}, + C2 = #c_clause{pats=[#c_literal{val=[]}], + guard=#c_call{module=#c_literal{val=erlang}, + name=#c_literal{val=is_function}, + args=[F, #c_literal{val=2}]}, + body=A}, + Err = #c_tuple{es=[#c_literal{val='function_clause'}, F, A, Xs]}, + C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, + body=match_fail([{function_name,{'lists^foldr',2}}|Anno], Err)}, + Fun = #c_fun{vars=[Xs, A], + body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, + L = #c_var{name='L'}, + #c_let{vars=[F, A, L], arg=#c_values{es=[Arg1, Arg2, Arg3]}, + body=#c_letrec{defs=[{Loop,Fun}], + body=#c_apply{anno=Anno, op=Loop, args=[L, A]}}}; +call(#c_call{anno=Anno}, lists, mapfoldl, [Arg1,Arg2,Arg3]) -> + Loop = #c_var{name={'lists^mapfoldl',2}}, + F = #c_var{name='F'}, + Xs = #c_var{name='Xs'}, + X = #c_var{name='X'}, + Avar = #c_var{name='A'}, + Match = + fun (A, P, E) -> + C1 = #c_clause{pats=[P], guard=#c_literal{val=true}, body=E}, + Err = #c_tuple{es=[#c_literal{val='badmatch'}, X]}, + C2 = #c_clause{pats=[X], guard=#c_literal{val=true}, + body=match_fail(Anno, Err)}, + #c_case{arg=A, clauses=[C1, C2]} + end, + C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, + body=Match(#c_apply{anno=Anno, op=F, args=[X, Avar]}, + #c_tuple{es=[X, Avar]}, +%%% Tuple passing version + Match(#c_apply{anno=Anno, + op=Loop, + args=[Xs, Avar]}, + #c_tuple{es=[Xs, Avar]}, + #c_tuple{anno=[compiler_generated], + es=[#c_cons{anno=[compiler_generated], + hd=X, tl=Xs}, + Avar]}) +%%% Multiple-value version +%%% #c_let{vars=[Xs,A], +%%% %% The tuple here will be optimised +%%% %% away later; no worries. +%%% arg=#c_apply{op=Loop, args=[Xs, A]}, +%%% body=#c_values{es=[#c_cons{hd=X, tl=Xs}, +%%% A]}} + )}, + C2 = #c_clause{pats=[#c_literal{val=[]}], + guard=#c_call{module=#c_literal{val=erlang}, + name=#c_literal{val=is_function}, + args=[F, #c_literal{val=2}]}, +%%% Tuple passing version + body=#c_tuple{anno=[compiler_generated], + es=[#c_literal{val=[]}, Avar]}}, +%%% Multiple-value version +%%% body=#c_values{es=[#c_literal{val=[]}, A]}}, + Err = #c_tuple{es=[#c_literal{val='function_clause'}, F, Avar, Xs]}, + C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, + body=match_fail([{function_name,{'lists^mapfoldl',2}}|Anno], Err)}, + Fun = #c_fun{vars=[Xs, Avar], + body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, + L = #c_var{name='L'}, + #c_let{vars=[F, Avar, L], arg=#c_values{es=[Arg1, Arg2, Arg3]}, + body=#c_letrec{defs=[{Loop,Fun}], +%%% Tuple passing version + body=#c_apply{anno=Anno, + op=Loop, + args=[L, Avar]}}}; +%%% Multiple-value version +%%% body=#c_let{vars=[Xs, A], +%%% arg=#c_apply{op=Loop, +%%% args=[L, A]}, +%%% body=#c_tuple{es=[Xs, A]}}}}; +call(#c_call{anno=Anno}, lists, mapfoldr, [Arg1,Arg2,Arg3]) -> + Loop = #c_var{name={'lists^mapfoldr',2}}, + F = #c_var{name='F'}, + Xs = #c_var{name='Xs'}, + X = #c_var{name='X'}, + Avar = #c_var{name='A'}, + Match = + fun (A, P, E) -> + C1 = #c_clause{pats=[P], guard=#c_literal{val=true}, body=E}, + Err = #c_tuple{es=[#c_literal{val='badmatch'}, X]}, + C2 = #c_clause{pats=[X], guard=#c_literal{val=true}, + body=match_fail(Anno, Err)}, + #c_case{arg=A, clauses=[C1, C2]} + end, + C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, +%%% Tuple passing version + body=Match(#c_apply{anno=Anno, + op=Loop, + args=[Xs, Avar]}, + #c_tuple{es=[Xs, Avar]}, + Match(#c_apply{anno=Anno, op=F, args=[X, Avar]}, + #c_tuple{es=[X, Avar]}, + #c_tuple{anno=[compiler_generated], + es=[#c_cons{anno=[compiler_generated], + hd=X, tl=Xs}, Avar]})) +%%% Multiple-value version +%%% body=#c_let{vars=[Xs,A], +%%% %% The tuple will be optimised away +%%% arg=#c_apply{op=Loop, args=[Xs, A]}, +%%% body=Match(#c_apply{op=F, args=[X, A]}, +%%% #c_tuple{es=[X, A]}, +%%% #c_values{es=[#c_cons{hd=X, tl=Xs}, +%%% A]})} + }, + C2 = #c_clause{pats=[#c_literal{val=[]}], + guard=#c_call{module=#c_literal{val=erlang}, + name=#c_literal{val=is_function}, + args=[F, #c_literal{val=2}]}, +%%% Tuple passing version + body=#c_tuple{anno=[compiler_generated], + es=[#c_literal{val=[]}, Avar]}}, +%%% Multiple-value version +%%% body=#c_values{es=[#c_literal{val=[]}, A]}}, + Err = #c_tuple{es=[#c_literal{val='function_clause'}, F, Avar, Xs]}, + C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, + body=match_fail([{function_name,{'lists^mapfoldr',2}}|Anno], Err)}, + Fun = #c_fun{vars=[Xs, Avar], + body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, + L = #c_var{name='L'}, + #c_let{vars=[F, Avar, L], arg=#c_values{es=[Arg1, Arg2, Arg3]}, + body=#c_letrec{defs=[{Loop,Fun}], +%%% Tuple passing version + body=#c_apply{anno=Anno, + op=Loop, + args=[L, Avar]}}}; +%%% Multiple-value version +%%% body=#c_let{vars=[Xs, A], +%%% arg=#c_apply{op=Loop, +%%% args=[L, A]}, +%%% body=#c_tuple{es=[Xs, A]}}}}; +call(_, _, _, _) -> + none. + +match_fail(Ann, Arg) -> + Name = cerl:abstract(match_fail), + Args = [Arg], + cerl:ann_c_primop(Ann, Name, Args). diff --git a/lib/compiler/src/sys_core_inline.erl b/lib/compiler/src/sys_core_inline.erl index 9f93acb666..1e3a735e9b 100644 --- a/lib/compiler/src/sys_core_inline.erl +++ b/lib/compiler/src/sys_core_inline.erl @@ -195,10 +195,10 @@ kill_id_anns(Body) -> A = kill_id_anns_1(A0), CFun#c_fun{anno=A}; (Expr) -> - %% Mark everything as compiler generated to suppress - %% bogus warnings. - A = compiler_generated(core_lib:get_anno(Expr)), - core_lib:set_anno(Expr, A) + %% Mark everything as compiler generated to + %% suppress bogus warnings. + A = compiler_generated(cerl:get_ann(Expr)), + cerl:set_ann(Expr, A) end, Body). kill_id_anns_1([{'id',_}|As]) -> diff --git a/lib/compiler/src/v3_core.erl b/lib/compiler/src/v3_core.erl index 612660c2d6..9dd6b319a3 100644 --- a/lib/compiler/src/v3_core.erl +++ b/lib/compiler/src/v3_core.erl @@ -66,6 +66,7 @@ %% match arguments are novars %% case arguments are novars %% receive timeouts are novars +%% binaries and maps are novars %% let/set arguments are expressions %% fun is not a safe @@ -105,7 +106,9 @@ -record(iset, {anno=#a{},var,arg}). -record(itry, {anno=#a{},args,vars,body,evars,handler}). -record(ifilter, {anno=#a{},arg}). --record(igen, {anno=#a{},acc_pat,acc_guard,skip_pat,tail,tail_pat,arg}). +-record(igen, {anno=#a{},ceps=[],acc_pat,acc_guard, + skip_pat,tail,tail_pat,arg}). +-record(isimple, {anno=#a{},term :: cerl:cerl()}). -type iapply() :: #iapply{}. -type ibinary() :: #ibinary{}. @@ -124,11 +127,12 @@ -type itry() :: #itry{}. -type ifilter() :: #ifilter{}. -type igen() :: #igen{}. +-type isimple() :: #isimple{}. -type i() :: iapply() | ibinary() | icall() | icase() | icatch() | iclause() | ifun() | iletrec() | imatch() | iprimop() | iprotect() | ireceive1() | ireceive2() | iset() | itry() - | ifilter() | igen(). + | ifilter() | igen() | isimple(). -type warning() :: {file:filename(), [{integer(), module(), term()}]}. @@ -287,13 +291,15 @@ gexpr({protect,Line,Arg}, Bools0, St0) -> {#iprotect{anno=#a{anno=Anno},body=Eps++[E]},[],Bools0,St} end; gexpr({op,L,'andalso',E1,E2}, Bools, St0) -> - {#c_var{name=V0},St} = new_var(L, St0), + Anno = lineno_anno(L, St0), + {#c_var{name=V0},St} = new_var(Anno, St0), V = {var,L,V0}, False = {atom,L,false}, E = make_bool_switch_guard(L, E1, V, E2, False), gexpr(E, Bools, St); gexpr({op,L,'orelse',E1,E2}, Bools, St0) -> - {#c_var{name=V0},St} = new_var(L, St0), + Anno = lineno_anno(L, St0), + {#c_var{name=V0},St} = new_var(Anno, St0), V = {var,L,V0}, True = {atom,L,true}, E = make_bool_switch_guard(L, E1, V, True, E2), @@ -382,33 +388,30 @@ gexpr_test(E0, Bools0, St0) -> Lanno = Anno#a.anno, {New,St2} = new_var(Lanno, St1), Bools = [New|Bools0], - {#icall{anno=Anno, %Must have an #a{} - module=#c_literal{anno=Lanno,val=erlang}, - name=#c_literal{anno=Lanno,val='=:='}, - args=[New,#c_literal{anno=Lanno,val=true}]}, + {icall_eq_true(New), Eps0 ++ [#iset{anno=Anno,var=New,arg=E1}],Bools,St2} end; _ -> - Anno = get_ianno(E1), Lanno = get_lineno_anno(E1), + ACompGen = #a{anno=[compiler_generated]}, case is_simple(E1) of true -> Bools = [E1|Bools0], - {#icall{anno=Anno, %Must have an #a{} - module=#c_literal{anno=Lanno,val=erlang}, - name=#c_literal{anno=Lanno,val='=:='}, - args=[E1,#c_literal{anno=Lanno,val=true}]},Eps0,Bools,St1}; + {icall_eq_true(E1),Eps0,Bools,St1}; false -> {New,St2} = new_var(Lanno, St1), Bools = [New|Bools0], - {#icall{anno=Anno, %Must have an #a{} - module=#c_literal{anno=Lanno,val=erlang}, - name=#c_literal{anno=Lanno,val='=:='}, - args=[New,#c_literal{anno=Lanno,val=true}]}, - Eps0 ++ [#iset{anno=Anno,var=New,arg=E1}],Bools,St2} + {icall_eq_true(New), + Eps0 ++ [#iset{anno=ACompGen,var=New,arg=E1}],Bools,St2} end end. +icall_eq_true(Arg) -> + #icall{anno=#a{anno=[compiler_generated]}, + module=#c_literal{val=erlang}, + name=#c_literal{val='=:='}, + args=[Arg,#c_literal{val=true}]}. + force_booleans(Vs0, E, Eps, St) -> Vs1 = [set_anno(V, []) || V <- Vs0], Vs = unforce(E, Eps, Vs1), @@ -418,16 +421,15 @@ force_booleans_1([], E, Eps, St) -> {E,Eps,St}; force_booleans_1([V|Vs], E0, Eps0, St0) -> {E1,Eps1,St1} = force_safe(E0, St0), - Lanno = element(2, V), - Anno = #a{anno=Lanno}, - Call = #icall{anno=Anno,module=#c_literal{anno=Lanno,val=erlang}, - name=#c_literal{anno=Lanno,val=is_boolean}, + ACompGen = #a{anno=[compiler_generated]}, + Call = #icall{anno=ACompGen,module=#c_literal{val=erlang}, + name=#c_literal{val=is_boolean}, args=[V]}, - {New,St} = new_var(Lanno, St1), - Iset = #iset{anno=Anno,var=New,arg=Call}, + {New,St} = new_var([], St1), + Iset = #iset{var=New,arg=Call}, Eps = Eps0 ++ Eps1 ++ [Iset], - E = #icall{anno=Anno, - module=#c_literal{anno=Lanno,val=erlang},name=#c_literal{anno=Lanno,val='and'}, + E = #icall{anno=ACompGen, + module=#c_literal{val=erlang},name=#c_literal{val='and'}, args=[E1,New]}, force_booleans_1(Vs, E, Eps, St). @@ -514,28 +516,28 @@ exprs([], St) -> {[],St}. %% Generate an internal core expression. expr({var,L,V}, St) -> {#c_var{anno=lineno_anno(L, St),name=V},[],St}; -expr({char,L,C}, St) -> {#c_literal{anno=lineno_anno(L, St),val=C},[],St}; -expr({integer,L,I}, St) -> {#c_literal{anno=lineno_anno(L, St),val=I},[],St}; -expr({float,L,F}, St) -> {#c_literal{anno=lineno_anno(L, St),val=F},[],St}; -expr({atom,L,A}, St) -> {#c_literal{anno=lineno_anno(L, St),val=A},[],St}; -expr({nil,L}, St) -> {#c_literal{anno=lineno_anno(L, St),val=[]},[],St}; -expr({string,L,S}, St) -> {#c_literal{anno=lineno_anno(L, St),val=S},[],St}; +expr({char,L,C}, St) -> {#c_literal{anno=full_anno(L, St),val=C},[],St}; +expr({integer,L,I}, St) -> {#c_literal{anno=full_anno(L, St),val=I},[],St}; +expr({float,L,F}, St) -> {#c_literal{anno=full_anno(L, St),val=F},[],St}; +expr({atom,L,A}, St) -> {#c_literal{anno=full_anno(L, St),val=A},[],St}; +expr({nil,L}, St) -> {#c_literal{anno=full_anno(L, St),val=[]},[],St}; +expr({string,L,S}, St) -> {#c_literal{anno=full_anno(L, St),val=S},[],St}; expr({cons,L,H0,T0}, St0) -> {H1,Hps,St1} = safe(H0, St0), {T1,Tps,St2} = safe(T0, St1), - A = lineno_anno(L, St2), + A = full_anno(L, St2), {annotate_cons(A, H1, T1, St2),Hps ++ Tps,St2}; expr({lc,L,E,Qs0}, St0) -> {Qs1,St1} = preprocess_quals(L, Qs0, St0), lc_tq(L, E, Qs1, #c_literal{anno=lineno_anno(L, St1),val=[]}, St1); expr({bc,L,E,Qs}, St) -> - bc_tq(L, E, Qs, {nil,L}, St); + bc_tq(L, E, Qs, St); expr({tuple,L,Es0}, St0) -> {Es1,Eps,St1} = safe_list(Es0, St0), A = record_anno(L, St1), {annotate_tuple(A, Es1, St1),Eps,St1}; expr({map,L,Es0}, St0) -> - map_build_pair_chain(#c_literal{val=#{}},Es0,lineno_anno(L,St0),St0); + map_build_pairs(#c_literal{val=#{}}, Es0, full_anno(L, St0), St0); expr({map,L,M0,Es0}, St0) -> try expr_map(M0,Es0,lineno_anno(L, St0),St0) of {_,_,_}=Res -> Res @@ -550,7 +552,7 @@ expr({map,L,M0,Es0}, St0) -> args=As},[],St} end; expr({bin,L,Es0}, St0) -> - try expr_bin(Es0, lineno_anno(L, St0), St0) of + try expr_bin(Es0, full_anno(L, St0), St0) of {_,_,_}=Res -> Res catch throw:bad_binary -> @@ -640,11 +642,11 @@ expr({'catch',L,E0}, St0) -> Lanno = lineno_anno(L, St1), {#icatch{anno=#a{anno=Lanno},body=Eps ++ [E1]},[],St1}; expr({'fun',L,{function,F,A},{_,_,_}=Id}, St) -> - Lanno = lineno_anno(L, St), + Lanno = full_anno(L, St), {#c_var{anno=Lanno++[{id,Id}],name={F,A}},[],St}; expr({'fun',L,{function,M,F,A}}, St0) -> {As,Aps,St1} = safe_list([M,F,A], St0), - Lanno = lineno_anno(L, St1), + Lanno = full_anno(L, St1), {#icall{anno=#a{anno=Lanno}, module=#c_literal{val=erlang}, name=#c_literal{val=make_fun}, @@ -655,13 +657,9 @@ expr({named_fun,L,'_',Cs,Id}, St) -> fun_tq(Id, Cs, L, St, unnamed); expr({named_fun,L,Name,Cs,Id}, St) -> fun_tq(Id, Cs, L, St, {named,Name}); -expr({call,L,{remote,_,M,F},As0}, #core{wanted=Wanted}=St0) -> +expr({call,L,{remote,_,M,F},As0}, St0) -> {[M1,F1|As1],Aps,St1} = safe_list([M,F|As0], St0), - Lanno = lineno_anno(L, St1), - Anno = case Wanted of - false -> [result_not_wanted|Lanno]; - true -> Lanno - end, + Anno = full_anno(L, St1), {#icall{anno=#a{anno=Anno},module=M1,name=F1,args=As1},Aps,St1}; expr({call,Lc,{atom,Lf,F},As0}, St0) -> {As1,Aps,St1} = safe_list(As0, St0), @@ -710,26 +708,28 @@ expr({op,_,'++',{lc,Llc,E,Qs0},More}, St0) -> {Y,Yps,St} = lc_tq(Llc, E, Qs, Mc, St2), {Y,Mps++Yps,St}; expr({op,L,'andalso',E1,E2}, St0) -> - {#c_var{name=V0},St} = new_var(L, St0), + Anno = lineno_anno(L, St0), + {#c_var{name=V0},St} = new_var(Anno, St0), V = {var,L,V0}, False = {atom,L,false}, E = make_bool_switch(L, E1, V, E2, False, St0), expr(E, St); expr({op,L,'orelse',E1,E2}, St0) -> - {#c_var{name=V0},St} = new_var(L, St0), + Anno = lineno_anno(L, St0), + {#c_var{name=V0},St} = new_var(Anno, St0), V = {var,L,V0}, True = {atom,L,true}, E = make_bool_switch(L, E1, V, True, E2, St0), expr(E, St); expr({op,L,Op,A0}, St0) -> {A1,Aps,St1} = safe(A0, St0), - LineAnno = lineno_anno(L, St1), + LineAnno = full_anno(L, St1), {#icall{anno=#a{anno=LineAnno}, %Must have an #a{} module=#c_literal{anno=LineAnno,val=erlang}, name=#c_literal{anno=LineAnno,val=Op},args=[A1]},Aps,St1}; expr({op,L,Op,L0,R0}, St0) -> {As,Aps,St1} = safe_list([L0,R0], St0), - LineAnno = lineno_anno(L, St1), + LineAnno = full_anno(L, St1), {#icall{anno=#a{anno=LineAnno}, %Must have an #a{} module=#c_literal{anno=LineAnno,val=erlang}, name=#c_literal{anno=LineAnno,val=Op},args=As},Aps,St1}. @@ -778,66 +778,29 @@ expr_map(M0,Es0,A,St0) -> Fc = fail_clause([Fpat], A, #c_literal{val=badarg}), {#icase{anno=#a{anno=A},args=[M1],clauses=Cs,fc=Fc},Mps,St3}; {_,_} -> - {M2,Eps,St2} = map_build_pair_chain(M1,Es0,A,St1), + {M2,Eps,St2} = map_build_pairs(M1, Es0, A, St1), {M2,Mps++Eps,St2} end; false -> throw({bad_map,bad_map}) end. -%% Group continuous literal blocks and single variables, i.e. -%% M0#{ a := 1, b := V1, K1 := V2, K2 := 42} -%% becomes equivalent to -%% M1 = M0#{ a := 1, b := V1 }, -%% M2 = M1#{ K1 := V1 }, -%% M3 = M2#{ K2 := 42 } - -map_build_pair_chain(M,Es,A,St) -> - %% hack, remove iset if only literal - case map_build_pair_chain(M,Es,A,St,[]) of - {_,[#iset{arg=#c_literal{}=Val}],St1} -> {Val,[],St1}; - Normal -> Normal - end. - -map_build_pair_chain(M0,[],_,St,Mps) -> - {M0,Mps,St}; -map_build_pair_chain(M0,Es0,A,St0,Mps) -> - % group continuous literal blocks - % Anno = #a{anno=[compiler_generated]}, - % order is important, we need to reverse the literals - case map_pair_block(Es0,[],[],St0) of - {{CesL,EspL},{[],[]},Es1,St1} -> - {MVar,St2} = new_var(St1), - Pre = [#iset{var=MVar, arg=ann_c_map(A,M0,reverse(CesL))}], - map_build_pair_chain(MVar,Es1,A,St2,Mps++EspL++Pre); - {{[],[]},{CesV,EspV},Es1,St1} -> - {MVar,St2} = new_var(St1), - Pre = [#iset{var=MVar, arg=#c_map{arg=M0,es=CesV, anno=A}}], - map_build_pair_chain(MVar,Es1,A,St2,Mps ++ EspV++Pre); - {{CesL,EspL},{CesV,EspV},Es1,St1} -> - {MVarL,St2} = new_var(St1), - {MVarV,St3} = new_var(St2), - Pre = [#iset{var=MVarL, arg=ann_c_map(A,M0,reverse(CesL))}, - #iset{var=MVarV, arg=#c_map{arg=MVarL,es=CesV,anno=A}}], - map_build_pair_chain(MVarV,Es1,A,St3,Mps++EspL++EspV++Pre) - end. - -map_pair_block([{Op,L,K0,V0}|Es],Ces,Esp,St0) -> - {K,Ep0,St1} = safe(K0, St0), - {V,Ep1,St2} = safe(V0, St1), - A = lineno_anno(L, St2), - Pair0 = map_op_to_c_map_pair(Op), - Pair1 = Pair0#c_map_pair{anno=A,key=K,val=V}, - case cerl:is_literal(K) of - true -> - map_pair_block(Es,[Pair1|Ces],Ep0 ++ Ep1 ++ Esp,St2); - false -> - {{Ces,Esp},{[Pair1],Ep0++Ep1},Es,St2} - end; -map_pair_block([],Ces,Esp,St) -> - {{Ces,Esp},{[],[]},[],St}. +map_build_pairs(Map, Es0, Ann, St0) -> + {Es,Pre,St1} = map_build_pairs_1(Es0, St0), + {ann_c_map(Ann, Map, Es),Pre,St1}. + +map_build_pairs_1([{Op0,L,K0,V0}|Es], St0) -> + {K,Pre0,St1} = safe(K0, St0), + {V,Pre1,St2} = safe(V0, St1), + {Pairs,Pre2,St3} = map_build_pairs_1(Es, St2), + As = lineno_anno(L, St3), + Op = map_op(Op0), + Pair = cerl:ann_c_map_pair(As, Op, K, V), + {[Pair|Pairs],Pre0++Pre1++Pre2,St3}; +map_build_pairs_1([], St) -> + {[],[],St}. -map_op_to_c_map_pair(map_field_assoc) -> #c_map_pair{op=#c_literal{val=assoc}}; -map_op_to_c_map_pair(map_field_exact) -> #c_map_pair{op=#c_literal{val=exact}}. +map_op(map_field_assoc) -> #c_literal{val=assoc}; +map_op(map_field_exact) -> #c_literal{val=exact}. is_valid_map_src(#c_literal{val = M}) when is_map(M) -> true; is_valid_map_src(#c_var{}) -> true; @@ -1001,7 +964,7 @@ fun_tq({_,_,Name}=Id, Cs0, L, St0, NameInfo) -> {Cs1,Ceps,St1} = clauses(Cs0, St0), {Args,St2} = new_vars(Arity, St1), {Ps,St3} = new_vars(Arity, St2), %Need new variables here - Anno = lineno_anno(L, St3), + Anno = full_anno(L, St3), Fc = function_clause(Ps, Anno, {Name,Arity}), Fun = #ifun{anno=#a{anno=Anno}, id=[{id,Id}], %We KNOW! @@ -1011,7 +974,8 @@ fun_tq({_,_,Name}=Id, Cs0, L, St0, NameInfo) -> %% lc_tq(Line, Exp, [Qualifier], Mc, State) -> {LetRec,[PreExp],State}. %% This TQ from Simon PJ pp 127-138. -lc_tq(Line, E, [#igen{anno=GAnno,acc_pat=AccPat,acc_guard=AccGuard, +lc_tq(Line, E, [#igen{anno=GAnno,ceps=Ceps, + acc_pat=AccPat,acc_guard=AccGuard, skip_pat=SkipPat,tail=Tail,tail_pat=TailPat, arg={Pre,Arg}}|Qs], Mc, St0) -> {Name,St1} = new_fun_name("lc", St0), @@ -1046,7 +1010,7 @@ lc_tq(Line, E, [#igen{anno=GAnno,acc_pat=AccPat,acc_guard=AccGuard, Fun = #ifun{anno=LAnno,id=[],vars=[Var],clauses=Cs,fc=Fc}, {#iletrec{anno=LAnno#a{anno=[list_comprehension|LA]},defs=[{{Name,1},Fun}], body=Pre ++ [#iapply{anno=LAnno,op=F,args=[Arg]}]}, - [],St4}; + Ceps,St4}; lc_tq(Line, E, [#ifilter{}=Filter|Qs], Mc, St) -> filter_tq(Line, E, Filter, Mc, St, Qs, fun lc_tq/5); lc_tq(Line, E0, [], Mc0, St0) -> @@ -1060,7 +1024,7 @@ lc_tq(Line, E0, [], Mc0, St0) -> %% This TQ from Gustafsson ERLANG'05. %% More could be transformed before calling bc_tq. -bc_tq(Line, Exp, Qs0, _, St0) -> +bc_tq(Line, Exp, Qs0, St0) -> {BinVar,St1} = new_var(St0), {Sz,SzPre,St2} = bc_initial_size(Exp, Qs0, St1), {Qs,St3} = preprocess_quals(Line, Qs0, St2), @@ -1071,7 +1035,8 @@ bc_tq(Line, Exp, Qs0, _, St0) -> args=[Sz]}}] ++ BcPre, {E,Pre,St}. -bc_tq1(Line, E, [#igen{anno=GAnno,acc_pat=AccPat,acc_guard=AccGuard, +bc_tq1(Line, E, [#igen{anno=GAnno,ceps=Ceps, + acc_pat=AccPat,acc_guard=AccGuard, skip_pat=SkipPat,tail=Tail,tail_pat=TailPat, arg={Pre,Arg}}|Qs], Mc, St0) -> {Name,St1} = new_fun_name("lbc", St0), @@ -1109,7 +1074,7 @@ bc_tq1(Line, E, [#igen{anno=GAnno,acc_pat=AccPat,acc_guard=AccGuard, Fun = #ifun{anno=LAnno,id=[],vars=Vars,clauses=Cs,fc=Fc}, {#iletrec{anno=LAnno#a{anno=[list_comprehension|LA]},defs=[{{Name,2},Fun}], body=Pre ++ [#iapply{anno=LAnno,op=F,args=[Arg,Mc]}]}, - [],St4}; + Ceps,St4}; bc_tq1(Line, E, [#ifilter{}=Filter|Qs], Mc, St) -> filter_tq(Line, E, Filter, Mc, St, Qs, fun bc_tq1/5); bc_tq1(_, {bin,Bl,Elements}, [], AccVar, St0) -> @@ -1245,8 +1210,9 @@ generator(Line, {generate,Lg,P0,E}, Gs, St0) -> ann_c_cons(LA, Skip, Tail)} end, {Ce,Pre,St4} = safe(E, St3), - Gen = #igen{anno=#a{anno=GA},acc_pat=AccPat,acc_guard=Cg,skip_pat=SkipPat, - tail=Tail,tail_pat=#c_literal{anno=LA,val=[]},arg={Ceps++Pre,Ce}}, + Gen = #igen{anno=#a{anno=GA},ceps=Ceps, + acc_pat=AccPat,acc_guard=Cg,skip_pat=SkipPat, + tail=Tail,tail_pat=#c_literal{anno=LA,val=[]},arg={Pre,Ce}}, {Gen,St4}; generator(Line, {b_generate,Lg,P,E}, Gs, St0) -> LA = lineno_anno(Line, St0), @@ -1515,6 +1481,7 @@ force_novars(#iapply{}=App, St) -> {App,[],St}; force_novars(#icall{}=Call, St) -> {Call,[],St}; force_novars(#ifun{}=Fun, St) -> {Fun,[],St}; %These are novars too force_novars(#ibinary{}=Bin, St) -> {Bin,[],St}; +force_novars(#c_map{}=Bin, St) -> {Bin,[],St}; force_novars(Ce, St) -> force_safe(Ce, St). @@ -1634,7 +1601,7 @@ pattern({tuple,L,Ps}, St) -> {annotate_tuple(record_anno(L, St), Ps1, St),Eps,St1}; pattern({map,L,Pairs}, St0) -> {Ps,Eps,St1} = pattern_map_pairs(Pairs, St0), - {#c_map{anno=lineno_anno(L, St1), es=Ps},Eps,St1}; + {#c_map{anno=lineno_anno(L, St1),es=Ps,is_pat=true},Eps,St1}; pattern({bin,L,Ps}, St) -> %% We don't create a #ibinary record here, since there is %% no need to hold any used/new annotations in a pattern. @@ -1794,7 +1761,7 @@ new_var_name(#core{vcount=C}=St) -> new_var(St) -> new_var([], St). -new_var(Anno, St0) -> +new_var(Anno, St0) when is_list(Anno) -> {New,St} = new_var_name(St0), {#c_var{anno=Anno,name=New},St}. @@ -2021,11 +1988,11 @@ uexpr(#ibinary{anno=A,segments=Ss}, _, St) -> uexpr(#c_literal{}=Lit, _, St) -> Anno = get_anno(Lit), {set_anno(Lit, #a{us=[],anno=Anno}),St}; -uexpr(Lit, _, St) -> - true = is_simple(Lit), %Sanity check! - Vs = lit_vars(Lit), - Anno = get_anno(Lit), - {set_anno(Lit, #a{us=Vs,anno=Anno}),St}. +uexpr(Simple, _, St) -> + true = is_simple(Simple), %Sanity check! + Vs = lit_vars(Simple), + Anno = get_anno(Simple), + {#isimple{anno=#a{us=Vs,anno=Anno},term=Simple},St}. uexpr_list(Les0, Ks, St0) -> mapfoldl(fun (Le, St) -> uexpr(Le, Ks, St) end, St0, Les0). @@ -2202,7 +2169,8 @@ cguard(Gs, St0) -> cexprs([#iset{var=#c_var{name=Name}=Var}=Iset], As, St) -> %% Make return value explicit, and make Var true top level. - cexprs([Iset,Var#c_var{anno=#a{us=[Name]}}], As, St); + Isimple = #isimple{anno=#a{us=[Name]},term=Var}, + cexprs([Iset,Isimple], As, St); cexprs([Le], As, St0) -> {Ce,Es,Us,St1} = cexpr(Le, As, St0), Exp = make_vars(As), %The export variables @@ -2317,12 +2285,9 @@ cexpr(#c_literal{}=Lit, _As, St) -> Anno = get_anno(Lit), Vs = Anno#a.us, {set_anno(Lit, Anno#a.anno),[],Vs,St}; -cexpr(Lit, _As, St) -> - true = is_simple(Lit), %Sanity check! - Anno = get_anno(Lit), - Vs = Anno#a.us, - %%Vs = lit_vars(Lit), - {set_anno(Lit, Anno#a.anno),[],Vs,St}. +cexpr(#isimple{anno=#a{us=Vs},term=Simple}, _As, St) -> + true = is_simple(Simple), %Sanity check! + {Simple,[],Vs,St}. cfun(#ifun{anno=A,id=Id,vars=Args,clauses=Lcs,fc=Lfc}, _As, St0) -> {Ccs,St1} = cclauses(Lcs, [], St0), %NEVER export! @@ -2345,11 +2310,6 @@ lit_vars(#c_map_pair{key=K,val=V}, Vs) -> lit_vars(K, lit_vars(V, Vs)); lit_vars(#c_var{name=V}, Vs) -> add_element(V, Vs); lit_vars(_, Vs) -> Vs. %These are atomic -% lit_bin_vars(Segs, Vs) -> -% foldl(fun (#c_bitstr{val=V,size=S}, Vs0) -> -% lit_vars(V, lit_vars(S, Vs0)) -% end, Vs, Segs). - lit_list_vars(Ls) -> lit_list_vars(Ls, []). lit_list_vars(Ls, Vs) -> @@ -2368,16 +2328,21 @@ record_anno(L, St) when L >= ?REC_OFFSET -> true -> [record | lineno_anno(L - ?REC_OFFSET, St)]; false -> - lineno_anno(L, St) + full_anno(L, St) end; record_anno(L, St) when L < -?REC_OFFSET -> case member(dialyzer, St#core.opts) of true -> [record | lineno_anno(L + ?REC_OFFSET, St)]; false -> - lineno_anno(L, St) + full_anno(L, St) end; record_anno(L, St) -> + full_anno(L, St). + +full_anno(L, #core{wanted=false}=St) -> + [result_not_wanted|lineno_anno(L, St)]; +full_anno(L, #core{wanted=true}=St) -> lineno_anno(L, St). lineno_anno(L, St) -> diff --git a/lib/compiler/src/v3_kernel.erl b/lib/compiler/src/v3_kernel.erl index 6504351c02..08e84efc1b 100644 --- a/lib/compiler/src/v3_kernel.erl +++ b/lib/compiler/src/v3_kernel.erl @@ -131,12 +131,12 @@ module(#c_module{anno=A,name=M,exports=Es,attrs=As,defs=Fs}, _Options) -> {ok,#k_mdef{anno=A,name=M#c_literal.val,exports=Kes,attributes=Kas, body=Kfs ++ St#kern.funs},lists:sort(St#kern.ws)}. -attributes([{#c_literal{val=Name},Val}|As]) -> +attributes([{#c_literal{val=Name},#c_literal{val=Val}}|As]) -> case include_attribute(Name) of false -> attributes(As); true -> - [{Name,core_lib:literal_value(Val)}|attributes(As)] + [{Name,Val}|attributes(As)] end; attributes([]) -> []. @@ -521,62 +521,76 @@ is_valid_map_src(#k_var{}) -> true; is_valid_map_src(_) -> false. map_split_pairs(A, Var, Ces, Sub, St0) -> - %% two steps - %% 1. force variables - %% 2. remove multiples - Pairs0 = [{Op,K,V} || #c_map_pair{op=#c_literal{val=Op},key=K,val=V} <- Ces], + %% 1. Force variables. + %% 2. Group adjacent pairs with literal keys. + %% 3. Within each such group, remove multiple assignments to the same key. + %% 4. Partition each group according to operator ('=>' and ':='). + Pairs0 = [{Op,K,V} || + #c_map_pair{op=#c_literal{val=Op},key=K,val=V} <- Ces], {Pairs,Esp,St1} = foldr(fun ({Op,K0,V0}, {Ops,Espi,Sti0}) when Op =:= assoc; Op =:= exact -> {K,Eps1,Sti1} = atomic(K0, Sub, Sti0), {V,Eps2,Sti2} = atomic(V0, Sub, Sti1), {[{Op,K,V}|Ops],Eps1 ++ Eps2 ++ Espi,Sti2} end, {[],[],St0}, Pairs0), - - case map_group_pairs(Pairs) of - {Assoc,[]} -> - Kes = [#k_map_pair{key=K,val=V}||{_,{assoc,K,V}} <- Assoc], - {#k_map{anno=A,op=assoc,var=Var,es=Kes},Esp,St1}; - {[],Exact} -> - Kes = [#k_map_pair{key=K,val=V}||{_,{exact,K,V}} <- Exact], - {#k_map{anno=A,op=exact,var=Var,es=Kes},Esp,St1}; - {Assoc,Exact} -> - Kes1 = [#k_map_pair{key=K,val=V}||{_,{assoc,K,V}} <- Assoc], - {Mvar,Em,St2} = force_atomic(#k_map{anno=A,op=assoc,var=Var,es=Kes1},St1), - Kes2 = [#k_map_pair{key=K,val=V}||{_,{exact,K,V}} <- Exact], - {#k_map{anno=A,op=exact,var=Mvar,es=Kes2},Esp ++ Em,St2} - + map_split_pairs_1(A, Var, Pairs, Esp, St1). + +map_split_pairs_1(A, Map0, [{Op,Key,Val}|Pairs1]=Pairs0, Esp0, St0) -> + {Map1,Em,St1} = force_atomic(Map0, St0), + case Key of + #k_var{} -> + %% Don't combine variable keys with other keys. + Kes = [#k_map_pair{key=Key,val=Val}], + Map = #k_map{anno=A,op=Op,var=Map1,es=Kes}, + map_split_pairs_1(A, Map, Pairs1, Esp0 ++ Em, St1); + _ -> + %% Literal key. Split off all literal keys. + {L,Pairs} = splitwith(fun({_,#k_var{},_}) -> false; + ({_,_,_}) -> true + end, Pairs0), + {Map,Esp,St2} = map_group_pairs(A, Map1, L, Esp0 ++ Em, St1), + map_split_pairs_1(A, Map, Pairs, Esp, St2) + end; +map_split_pairs_1(_, Map, [], Esp, St0) -> + {Map,Esp,St0}. + +map_group_pairs(A, Var, Pairs0, Esp, St0) -> + Pairs = map_remove_dup_keys(Pairs0), + Assoc = [#k_map_pair{key=K,val=V} || {_,{assoc,K,V}} <- Pairs], + Exact = [#k_map_pair{key=K,val=V} || {_,{exact,K,V}} <- Pairs], + case {Assoc,Exact} of + {[_|_],[]} -> + {#k_map{anno=A,op=assoc,var=Var,es=Assoc},Esp,St0}; + {[],[_|_]} -> + {#k_map{anno=A,op=exact,var=Var,es=Exact},Esp,St0}; + {[_|_],[_|_]} -> + Map = #k_map{anno=A,op=assoc,var=Var,es=Assoc}, + {Mvar,Em,St1} = force_atomic(Map, St0), + {#k_map{anno=A,op=exact,var=Mvar,es=Exact},Esp ++ Em,St1} end. -%% Group map by Assoc operations and Exact operations - -map_group_pairs(Es) -> - Groups = dict:to_list(map_group_pairs(Es,dict:new())), - partition(fun({_,{Op,_,_}}) -> Op =:= assoc end, Groups). - -map_group_pairs([{assoc,K,V}|Es0],Used0) -> - Used1 = case map_key_is_used(K,Used0) of - {ok, {assoc,_,_}} -> map_key_set_used(K,{assoc,K,V},Used0); - {ok, {exact,_,_}} -> map_key_set_used(K,{exact,K,V},Used0); - _ -> map_key_set_used(K,{assoc,K,V},Used0) - end, - map_group_pairs(Es0,Used1); -map_group_pairs([{exact,K,V}|Es0],Used0) -> - Used1 = case map_key_is_used(K,Used0) of - {ok, {assoc,_,_}} -> map_key_set_used(K,{assoc,K,V},Used0); - {ok, {exact,_,_}} -> map_key_set_used(K,{exact,K,V},Used0); - _ -> map_key_set_used(K,{exact,K,V},Used0) - end, - map_group_pairs(Es0,Used1); -map_group_pairs([],Used) -> - Used. +map_remove_dup_keys(Es) -> + dict:to_list(map_remove_dup_keys(Es, dict:new())). -map_key_set_used(K,How,Used) -> - dict:store(map_key_clean(K),How,Used). - -map_key_is_used(K,Used) -> - dict:find(map_key_clean(K),Used). +map_remove_dup_keys([{assoc,K0,V}|Es0],Used0) -> + K = map_key_clean(K0), + Op = case dict:find(K, Used0) of + {ok,{exact,_,_}} -> exact; + _ -> assoc + end, + Used1 = dict:store(K, {Op,K0,V}, Used0), + map_remove_dup_keys(Es0, Used1); +map_remove_dup_keys([{exact,K0,V}|Es0],Used0) -> + K = map_key_clean(K0), + Op = case dict:find(K, Used0) of + {ok,{assoc,_,_}} -> assoc; + _ -> exact + end, + Used1 = dict:store(K, {Op,K0,V}, Used0), + map_remove_dup_keys(Es0, Used1); +map_remove_dup_keys([], Used) -> Used. -%% Be explicit instead of using set_kanno(K,[]) +%% Be explicit instead of using set_kanno(K, []). map_key_clean(#k_var{name=V}) -> {var,V}; map_key_clean(#k_literal{val=V}) -> {lit,V}; map_key_clean(#k_int{val=V}) -> {lit,V}; @@ -661,12 +675,12 @@ atomic_bin([#c_bitstr{anno=A,val=E0,size=S0,unit=U0,type=T,flags=Fs0}|Es0], {E,Ap1,St1} = atomic(E0, Sub, St0), {S1,Ap2,St2} = atomic(S0, Sub, St1), validate_bin_element_size(S1), - U1 = core_lib:literal_value(U0), - Fs1 = core_lib:literal_value(Fs0), + U1 = cerl:concrete(U0), + Fs1 = cerl:concrete(Fs0), {Es,Ap3,St3} = atomic_bin(Es0, Sub, St2), {#k_bin_seg{anno=A,size=S1, unit=U1, - type=core_lib:literal_value(T), + type=cerl:concrete(T), flags=Fs1, seg=E,next=Es}, Ap1++Ap2++Ap3,St3}; @@ -793,8 +807,8 @@ pattern_bin_1([#c_bitstr{anno=A,val=E0,size=S0,unit=U,type=T,flags=Fs}|Es0], %% problems. #k_atom{val=bad_size} end, - U0 = core_lib:literal_value(U), - Fs0 = core_lib:literal_value(Fs), + U0 = cerl:concrete(U), + Fs0 = cerl:concrete(Fs), %%ok= io:fwrite("~w: ~p~n", [?LINE,{B0,S,U0,Fs0}]), {E,Osub1,St2} = pattern(E0, Isub0, Osub0, St1), Isub1 = case E0 of @@ -805,7 +819,7 @@ pattern_bin_1([#c_bitstr{anno=A,val=E0,size=S0,unit=U,type=T,flags=Fs}|Es0], {Es,{Isub,Osub},St3} = pattern_bin_1(Es0, Isub1, Osub1, St2), {#k_bin_seg{anno=A,size=S, unit=U0, - type=core_lib:literal_value(T), + type=cerl:concrete(T), flags=Fs0, seg=E,next=Es}, {Isub,Osub},St3}; diff --git a/lib/compiler/test/andor_SUITE.erl b/lib/compiler/test/andor_SUITE.erl index b5408ecd8f..3199440d84 100644 --- a/lib/compiler/test/andor_SUITE.erl +++ b/lib/compiler/test/andor_SUITE.erl @@ -33,7 +33,7 @@ all() -> [{group,p}]. groups() -> - [{p,test_lib:parallel(), + [{p,[parallel], [t_case,t_and_or,t_andalso,t_orelse,inside,overlap, combined,in_case,before_and_inside_if]}]. @@ -173,7 +173,13 @@ t_and_or(Config) when is_list(Config) -> true = (fun (X = true) when X or true or X -> true end)(True), - ok. + Tuple = id({a,b}), + case Tuple of + {_,_} -> + {'EXIT',{badarg,_}} = (catch true and Tuple) + end, + + ok. t_andalso(Config) when is_list(Config) -> Bs = [true,false], diff --git a/lib/compiler/test/beam_utils_SUITE.erl b/lib/compiler/test/beam_utils_SUITE.erl index 9142ca6485..d2e24cb5ae 100644 --- a/lib/compiler/test/beam_utils_SUITE.erl +++ b/lib/compiler/test/beam_utils_SUITE.erl @@ -32,7 +32,7 @@ all() -> [{group,p}]. groups() -> - [{p,test_lib:parallel(), + [{p,[parallel], [apply_fun, apply_mf, bs_init, diff --git a/lib/compiler/test/bs_bit_binaries_SUITE.erl b/lib/compiler/test/bs_bit_binaries_SUITE.erl index 8609a490f5..2433e7621e 100644 --- a/lib/compiler/test/bs_bit_binaries_SUITE.erl +++ b/lib/compiler/test/bs_bit_binaries_SUITE.erl @@ -37,7 +37,7 @@ all() -> [{group,p}]. groups() -> - [{p,test_lib:parallel(), + [{p,[parallel], [misc,horrid_match,test_bitstr,test_bit_size, asymmetric_tests,big_asymmetric_tests, binary_to_and_from_list,big_binary_to_and_from_list, diff --git a/lib/compiler/test/bs_construct_SUITE.erl b/lib/compiler/test/bs_construct_SUITE.erl index ce39de2a82..9df874c387 100644 --- a/lib/compiler/test/bs_construct_SUITE.erl +++ b/lib/compiler/test/bs_construct_SUITE.erl @@ -39,7 +39,7 @@ all() -> [{group,p}]. groups() -> - [{p,test_lib:parallel(), + [{p,[parallel], [two,test1,fail,float_bin,in_guard,in_catch, nasty_literals,side_effect,opt,otp_7556,float_arith, otp_8054]}]. diff --git a/lib/compiler/test/bs_match_SUITE.erl b/lib/compiler/test/bs_match_SUITE.erl index e1710988d8..f7af56afcc 100644 --- a/lib/compiler/test/bs_match_SUITE.erl +++ b/lib/compiler/test/bs_match_SUITE.erl @@ -48,7 +48,7 @@ all() -> [{group,p}]. groups() -> - [{p,test_lib:parallel(), + [{p,[parallel], [fun_shadow,int_float,otp_5269,null_fields,wiger, bin_tail,save_restore,shadowed_size_var, partitioned_bs_match,function_clause,unit, diff --git a/lib/compiler/test/compilation_SUITE.erl b/lib/compiler/test/compilation_SUITE.erl index 8711f35e8e..296774e083 100644 --- a/lib/compiler/test/compilation_SUITE.erl +++ b/lib/compiler/test/compilation_SUITE.erl @@ -611,12 +611,10 @@ otp_7345(Config) when is_list(Config) -> otp_7345(ObjRef, _RdEnv, Args) -> Cid = ObjRef#contextId.cid, - _DpRef = - #dpRef{cid = Cid, + _ = #dpRef{cid = Cid, ms_device_context_id = cid_id, tlli = #ptmsi{value = 0}}, - _QosProfile = - #qosProfileBssgp{peak_bit_rate_msb = 0, + _ = #qosProfileBssgp{peak_bit_rate_msb = 0, peak_bit_rate_lsb = 80, t_a_precedence = 49}, [Cpdu|_] = Args, diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl index 0ff03c4295..1c96abe017 100644 --- a/lib/compiler/test/compile_SUITE.erl +++ b/lib/compiler/test/compile_SUITE.erl @@ -748,42 +748,65 @@ env_1(Simple, Target) -> %% compile the generated Core Erlang files. core(Config) when is_list(Config) -> - ?line Dog = test_server:timetrap(test_server:minutes(5)), - ?line PrivDir = ?config(priv_dir, Config), - ?line Outdir = filename:join(PrivDir, "core"), - ?line ok = file:make_dir(Outdir), + PrivDir = ?config(priv_dir, Config), + Outdir = filename:join(PrivDir, "core"), + ok = file:make_dir(Outdir), - ?line Wc = filename:join(filename:dirname(code:which(?MODULE)), "*.beam"), - ?line TestBeams = filelib:wildcard(Wc), - ?line Abstr = [begin {ok,{Mod,[{abstract_code, + Wc = filename:join(filename:dirname(code:which(?MODULE)), "*.beam"), + TestBeams = filelib:wildcard(Wc), + Abstr = [begin {ok,{Mod,[{abstract_code, {raw_abstract_v1,Abstr}}]}} = beam_lib:chunks(Beam, [abstract_code]), {Mod,Abstr} end || Beam <- TestBeams], - ?line Res = test_lib:p_run(fun(F) -> do_core(F, Outdir) end, Abstr), - ?line test_server:timetrap_cancel(Dog), - Res. - + test_lib:p_run(fun(F) -> do_core(F, Outdir) end, Abstr). do_core({M,A}, Outdir) -> try - {ok,M,Core} = compile:forms(A, [to_core,report]), - CoreFile = filename:join(Outdir, atom_to_list(M)++".core"), - CorePP = core_pp:format(Core), - ok = file:write_file(CoreFile, CorePP), - case compile:file(CoreFile, [clint,from_core,binary]) of - {ok,M,_} -> - ok = file:delete(CoreFile); - Other -> - io:format("*** core_lint failure '~p' for ~s\n", - [Other,CoreFile]), - error - end - catch Class:Error -> + do_core_1(M, A, Outdir) + catch + throw:{error,Error} -> + io:format("*** compilation failure '~p' for module ~s\n", + [Error,M]), + error; + Class:Error -> io:format("~p: ~p ~p\n~p\n", [M,Class,Error,erlang:get_stacktrace()]), error end. +do_core_1(M, A, Outdir) -> + {ok,M,Core0} = compile:forms(A, [to_core]), + CoreFile = filename:join(Outdir, atom_to_list(M)++".core"), + CorePP = core_pp:format(Core0), + ok = file:write_file(CoreFile, CorePP), + + %% Parse the .core file and return the result as Core Erlang Terms. + Core = case compile:file(CoreFile, [report_errors,from_core,no_copt,to_core,binary]) of + {ok,M,Core1} -> Core1; + Other -> throw({error,Other}) + end, + ok = file:delete(CoreFile), + + %% Compile as usual (including optimizations). + compile_forms(Core, [clint,from_core,binary]), + + %% Don't optimize to test that we are not dependent + %% on the Core Erlang optmimization passes. + %% (Example of a previous bug: The core_parse pass + %% would not turn map literals into #c_literal{} + %% records; if sys_core_fold was run it would fix + %% that; if sys_core_fold was not run v3_kernel would + %% crash.) + compile_forms(Core, [clint,from_core,no_copt,binary]), + + ok. + +compile_forms(Forms, Opts) -> + case compile:forms(Forms, [report_errors|Opts]) of + {ok,[],_} -> ok; + Other -> throw({error,Other}) + end. + %% Compile to Beam assembly language (.S) and then try to %% run .S through the compiler again. diff --git a/lib/compiler/test/core_SUITE_data/map_core_test.core b/lib/compiler/test/core_SUITE_data/map_core_test.core index 2aa853d450..a75f6cf24f 100644 --- a/lib/compiler/test/core_SUITE_data/map_core_test.core +++ b/lib/compiler/test/core_SUITE_data/map_core_test.core @@ -7,11 +7,11 @@ module 'map_core_test' ['map_core_test'/0, fun () -> let <_cor0> = %% Line 15 - ~{::<'check','ok'>,::<1337,#{#<104>(8,1,'integer',['unsigned'|['big']]), + ~{'check'=>'ok',1337=>#{#<104>(8,1,'integer',['unsigned'|['big']]), #<101>(8,1,'integer',['unsigned'|['big']]), #<108>(8,1,'integer',['unsigned'|['big']]), #<108>(8,1,'integer',['unsigned'|['big']]), - #<111>(8,1,'integer',['unsigned'|['big']])}#>,::<'val',0>}~ + #<111>(8,1,'integer',['unsigned'|['big']])}#,'val'=>0}~ in let <M> = %% Line 15 apply 'id'/1 @@ -23,7 +23,7 @@ module 'map_core_test' ['map_core_test'/0, in %% Line 16 case apply 'call'/2 (M, _cor2) of - <~{~<1337,#{#<104>(8,1,'integer',['unsigned'|['big']]), + <~{1337:=#{#<104>(8,1,'integer',['unsigned'|['big']]), #<101>(8,1,'integer',['unsigned'|['big']]), #<108>(8,1,'integer',['unsigned'|['big']]), #<108>(8,1,'integer',['unsigned'|['big']]), @@ -39,7 +39,7 @@ module 'map_core_test' ['map_core_test'/0, #<32>(8,1,'integer',['unsigned'|['big']]), #<53>(8,1,'integer',['unsigned'|['big']]), #<32>(8,1,'integer',['unsigned'|['big']]), - #<54>(8,1,'integer',['unsigned'|['big']])}#>,~<'check','ok'>,~<'val',21>}~> when 'true' -> + #<54>(8,1,'integer',['unsigned'|['big']])}#,'check':='ok','val':=21}~> when 'true' -> %% Line 17 'ok' ( <_cor3> when 'true' -> @@ -51,7 +51,7 @@ module 'map_core_test' ['map_core_test'/0, %% Line 20 fun (_cor1,_cor0) -> case <_cor1,_cor0> of - <M = ~{~<1337,Bin>,~<'check',_cor8>,~<'val',Val>}~,[V|Vs]> when 'true' -> + <M = ~{1337:=Bin,'check':=_cor8,'val':=Val}~,[V|Vs]> when 'true' -> let <_cor3> = %% Line 21 call 'erlang':'+' @@ -67,7 +67,7 @@ module 'map_core_test' ['map_core_test'/0, (Val, V) in let <_cor5> = %% Line 21 - ~{~<1337,_cor4>,~<'val',_cor2>|M}~ + ~{1337:=_cor4,'val':=_cor2|M}~ in %% Line 21 apply 'call'/2 (_cor5, Vs) diff --git a/lib/compiler/test/core_fold_SUITE.erl b/lib/compiler/test/core_fold_SUITE.erl index 6a7036d728..512aada203 100644 --- a/lib/compiler/test/core_fold_SUITE.erl +++ b/lib/compiler/test/core_fold_SUITE.erl @@ -23,7 +23,8 @@ t_element/1,setelement/1,t_length/1,append/1,t_apply/1,bifs/1, eq/1,nested_call_in_case/1,guard_try_catch/1,coverage/1, unused_multiple_values_error/1,unused_multiple_values/1, - multiple_aliases/1,redundant_boolean_clauses/1,mixed_matching_clauses/1]). + multiple_aliases/1,redundant_boolean_clauses/1, + mixed_matching_clauses/1,unnecessary_building/1]). -export([foo/0,foo/1,foo/2,foo/3]). @@ -36,11 +37,12 @@ all() -> [{group,p}]. groups() -> - [{p,test_lib:parallel(), + [{p,[parallel], [t_element,setelement,t_length,append,t_apply,bifs, eq,nested_call_in_case,guard_try_catch,coverage, unused_multiple_values_error,unused_multiple_values, - multiple_aliases,redundant_boolean_clauses,mixed_matching_clauses]}]. + multiple_aliases,redundant_boolean_clauses, + mixed_matching_clauses,unnecessary_building]}]. init_per_suite(Config) -> @@ -60,6 +62,12 @@ t_element(Config) when is_list(Config) -> X = make_ref(), ?line X = id(element(1, {X,y,z})), ?line b = id(element(2, {a,b,c,d})), + (fun() -> + case {a,#{k=>X}} of + {a,#{k:=X}}=Tuple -> + #{k:=X} = id(element(2, Tuple)) + end + end)(), %% No optimization, but should work. Tuple = id({x,y,z}), @@ -189,7 +197,10 @@ foo(A, B, C) -> A + B + C. bifs(Config) when is_list(Config) -> - ?line <<1,2,3,4>> = id(list_to_binary([1,2,3,4])), + <<1,2,3,4>> = id(list_to_binary([1,2,3,4])), + K = {a,key}, + V = {a,value}, + {ok,#{K:=V}} = id(list_to_tuple([ok,#{K=>V}])), ok. -define(CMP_SAME(A0, B), (fun(A) -> true = A == B, false = A /= B end)(id(A0))). @@ -204,6 +215,16 @@ eq(Config) when is_list(Config) -> ?line ?CMP_DIFF(a, [a]), ?line ?CMP_DIFF(a, {1,2,3}), + ?CMP_SAME(#{a=>1.0,b=>2}, #{b=>2.0,a=>1}), + ?CMP_SAME(#{a=>[1.0],b=>[2]}, #{b=>[2.0],a=>[1]}), + + %% The rule for comparing keys are different in 17.x and 18.x. + %% Just test that the results are consistent. + Bool = id(#{1=>a}) == id(#{1.0=>a}), %Unoptimizable. + Bool = id(#{1=>a}) == #{1.0=>a}, %Optimizable. + Bool = #{1=>a} == #{1.0=>a}, %Optimizable. + io:format("Bool = ~p\n", [Bool]), + ok. %% OTP-7117. @@ -236,6 +257,8 @@ do_guard_try_catch(K, V) -> false end. +-record(cover_opt_guard_try, {list=[]}). + coverage(Config) when is_list(Config) -> ?line {'EXIT',{{case_clause,{a,b,c}},_}} = (catch cover_will_match_list_type({a,b,c})), @@ -245,6 +268,9 @@ coverage(Config) when is_list(Config) -> ?line error = cover_will_match_lit_list(), {ok,[a]} = cover_is_safe_bool_expr(a), + ok = cover_opt_guard_try(#cover_opt_guard_try{list=[a]}), + error = cover_opt_guard_try(#cover_opt_guard_try{list=[]}), + %% Make sure that we don't attempt to make literals %% out of pids. (Putting a pid into a #c_literal{} %% would crash later compiler passes.) @@ -257,6 +283,12 @@ coverage(Config) when is_list(Config) -> error = bsm_an_inlined(<<1,2,3>>, Config), error = bsm_an_inlined([], Config), + %% Cover eval_rel_op/4. + Tuple = id({a,b}), + false = case Tuple of + {_,_} -> + Tuple =:= true + end, ok. cover_will_match_list_type(A) -> @@ -298,6 +330,14 @@ cover_is_safe_bool_expr(X) -> false end. +cover_opt_guard_try(Msg) -> + if + length(Msg#cover_opt_guard_try.list) =/= 1 -> + error; + true -> + ok + end. + bsm_an_inlined(<<_:8>>, _) -> ok; bsm_an_inlined(_, _) -> error. @@ -384,4 +424,29 @@ mixed_matching_clauses(Config) when is_list(Config) -> end, ok. +unnecessary_building(Config) when is_list(Config) -> + Term1 = do_unnecessary_building_1(test_lib:id(a)), + [{a,a},{a,a}] = Term1, + 7 = erts_debug:size(Term1), + + %% The Input term should not be rebuilt (thus, it should + %% only be counted once in the size of the combined term). + Input = test_lib:id({a,b,c}), + Term2 = test_lib:id(do_unnecessary_building_2(Input)), + {b,[{a,b,c},none],x} = Term2, + 4+4+4+2 = erts_debug:size([Term2|Input]), + + ok. + +do_unnecessary_building_1(S) -> + %% The tuple must only be built once. + F0 = F1 = {S,S}, + [F0,F1]. + +do_unnecessary_building_2({a,_,_}=T) -> + %% The T term should not be rebuilt. + {b, + [_,_] = [T,none], + x}. + id(I) -> I. diff --git a/lib/compiler/test/guard_SUITE.erl b/lib/compiler/test/guard_SUITE.erl index 882e2c3f8b..8db47ffa40 100644 --- a/lib/compiler/test/guard_SUITE.erl +++ b/lib/compiler/test/guard_SUITE.erl @@ -42,7 +42,7 @@ all() -> [{group,p}]. groups() -> - [{p,test_lib:parallel(), + [{p,[parallel], [misc,const_cond,basic_not,complex_not,nested_nots, semicolon,complex_semicolon,comma,or_guard, more_or_guards,complex_or_guards,and_guard,xor_guard, diff --git a/lib/compiler/test/lc_SUITE.erl b/lib/compiler/test/lc_SUITE.erl index 398398a397..6c5b34498b 100644 --- a/lib/compiler/test/lc_SUITE.erl +++ b/lib/compiler/test/lc_SUITE.erl @@ -18,12 +18,12 @@ %% -module(lc_SUITE). --author('[email protected]'). -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2, basic/1,deeply_nested/1,no_generator/1, - empty_generator/1,no_export/1]). + empty_generator/1,no_export/1,shadow/1, + effect/1]). -include_lib("test_server/include/test_server.hrl"). @@ -31,10 +31,18 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> test_lib:recompile(?MODULE), - [basic, deeply_nested, no_generator, empty_generator, no_export]. + [{group,p}]. groups() -> - []. + [{p,test_lib:parallel(), + [basic, + deeply_nested, + no_generator, + empty_generator, + no_export, + shadow, + effect + ]}]. init_per_suite(Config) -> Config. @@ -59,34 +67,34 @@ end_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> ok. basic(Config) when is_list(Config) -> - ?line L0 = lists:seq(1, 10), - ?line L1 = my_map(fun(X) -> {x,X} end, L0), - ?line L1 = [{x,X} || X <- L0], - ?line L0 = my_map(fun({x,X}) -> X end, L1), - ?line [1,2,3,4,5] = [X || X <- L0, X < 6], - ?line [4,5,6] = [X || X <- L0, X > 3, X < 7], - ?line [] = [X || X <- L0, X > 32, X < 7], - ?line [1,3,5,7,9] = [X || X <- L0, odd(X)], - ?line [2,4,6,8,10] = [X || X <- L0, not odd(X)], - ?line [1,3,5,9] = [X || X <- L0, odd(X), X =/= 7], - ?line [2,4,8,10] = [X || X <- L0, not odd(X), X =/= 6], + L0 = lists:seq(1, 10), + L1 = my_map(fun(X) -> {x,X} end, L0), + L1 = [{x,X} || X <- L0], + L0 = my_map(fun({x,X}) -> X end, L1), + [1,2,3,4,5] = [X || X <- L0, X < 6], + [4,5,6] = [X || X <- L0, X > 3, X < 7], + [] = [X || X <- L0, X > 32, X < 7], + [1,3,5,7,9] = [X || X <- L0, odd(X)], + [2,4,6,8,10] = [X || X <- L0, not odd(X)], + [1,3,5,9] = [X || X <- L0, odd(X), X =/= 7], + [2,4,8,10] = [X || X <- L0, not odd(X), X =/= 6], %% Append is specially handled. - ?line [1,3,5,9,2,4,8,10] = [X || X <- L0, odd(X), X =/= 7] ++ + [1,3,5,9,2,4,8,10] = [X || X <- L0, odd(X), X =/= 7] ++ [X || X <- L0, not odd(X), X =/= 6], %% Guards BIFs are evaluated in guard context. Weird, but true. - ?line [{a,b,true},{x,y,true,true}] = [X || X <- tuple_list(), element(3, X)], + [{a,b,true},{x,y,true,true}] = [X || X <- tuple_list(), element(3, X)], %% Filter expressions with andalso/orelse. - ?line "abc123" = alphanum("?abc123.;"), + "abc123" = alphanum("?abc123.;"), %% Error cases. - ?line [] = [{xx,X} || X <- L0, element(2, X) == no_no_no], - ?line {'EXIT',_} = (catch [X || X <- L1, list_to_atom(X) == dum]), - ?line [] = [X || X <- L1, X+1 < 2], - ?line {'EXIT',_} = (catch [X || X <- L1, odd(X)]), - ?line fc([x], catch [E || E <- id(x)]), + [] = [{xx,X} || X <- L0, element(2, X) == no_no_no], + {'EXIT',_} = (catch [X || X <- L1, list_to_atom(X) == dum]), + [] = [X || X <- L1, X+1 < 2], + {'EXIT',_} = (catch [X || X <- L1, odd(X)]), + fc([x], catch [E || E <- id(x)]), ok. tuple_list() -> @@ -116,12 +124,12 @@ deeply_nested_1() -> X16 <- [4],X17 <- [3],X18 <- [fun() -> X16+X17 end],X19 <- [2],X20 <- [1]]. no_generator(Config) when is_list(Config) -> - ?line Seq = lists:seq(-10, 17), - ?line [no_gen_verify(no_gen(A, B), A, B) || A <- Seq, B <- Seq], + Seq = lists:seq(-10, 17), + [no_gen_verify(no_gen(A, B), A, B) || A <- Seq, B <- Seq], %% Literal expression, for coverage. - ?line [a] = [a || true], - ?line [a,b,c] = [a || true] ++ [b,c], + [a] = [a || true], + [a,b,c] = [a || true] ++ [b,c], ok. no_gen(A, B) -> @@ -174,13 +182,40 @@ no_gen_eval(Fun, Res) -> no_gen_one_more(A, B) -> A + 1 =:= B. empty_generator(Config) when is_list(Config) -> - ?line [] = [X || {X} <- [], (false or (X/0 > 3))], + [] = [X || {X} <- [], (false or (X/0 > 3))], ok. no_export(Config) when is_list(Config) -> [] = [ _X = a || false ] ++ [ _X = a || false ], ok. +%% Test that variables in list comprehensions are +%% correctly shadowed. + +shadow(Config) when is_list(Config) -> + Shadowed = nomatch, + _ = id(Shadowed), %Eliminate warning. + L = [{Shadowed,Shadowed+1} || Shadowed <- lists:seq(7, 9)], + [{7,8},{8,9},{9,10}] = id(L), + [8,9] = id([Shadowed || {_,Shadowed} <- id(L), + Shadowed < 10]), + ok. + +effect(Config) when is_list(Config) -> + [{42,{a,b,c}}] = + do_effect(fun(F, L) -> + [F({V1,V2}) || + #{<<1:500>>:=V1,<<2:301>>:=V2} <- L], + ok + end, id([#{},x,#{<<1:500>>=>42,<<2:301>>=>{a,b,c}}])), + ok. + +do_effect(Lc, L) -> + put(?MODULE, []), + F = fun(V) -> put(?MODULE, [V|get(?MODULE)]) end, + ok = Lc(F, L), + lists:reverse(erase(?MODULE)). + id(I) -> I. fc(Args, {'EXIT',{function_clause,[{?MODULE,_,Args,_}|_]}}) -> ok; diff --git a/lib/compiler/test/map_SUITE.erl b/lib/compiler/test/map_SUITE.erl index dbac61765b..bc5ae803c6 100644 --- a/lib/compiler/test/map_SUITE.erl +++ b/lib/compiler/test/map_SUITE.erl @@ -319,6 +319,12 @@ t_update_exact(Config) when is_list(Config) -> {'EXIT',{badarg,_}} = (catch M0#{42=>v1,42.0:=v2,42:=v3}), {'EXIT',{badarg,_}} = (catch <<>>#{nonexisting:=val}), {'EXIT',{badarg,_}} = (catch M0#{<<0:257>> := val}), %% limitation + + %% A workaround for a bug allowed an empty map to be updated. + {'EXIT',{badarg,_}} = (catch (id(#{}))#{a:=1}), + {'EXIT',{badarg,_}} = (catch #{}#{a:=1}), + Empty = #{}, + {'EXIT',{badarg,_}} = (catch Empty#{a:=1}), ok. t_update_values(Config) when is_list(Config) -> diff --git a/lib/compiler/test/match_SUITE.erl b/lib/compiler/test/match_SUITE.erl index e5aaf49d6f..7522ee985f 100644 --- a/lib/compiler/test/match_SUITE.erl +++ b/lib/compiler/test/match_SUITE.erl @@ -22,7 +22,8 @@ init_per_group/2,end_per_group/2, pmatch/1,mixed/1,aliases/1,match_in_call/1, untuplify/1,shortcut_boolean/1,letify_guard/1, - selectify/1,underscore/1,match_map/1,coverage/1]). + selectify/1,underscore/1,match_map/1,map_vars_used/1, + coverage/1]). -include_lib("test_server/include/test_server.hrl"). @@ -33,10 +34,10 @@ all() -> [{group,p}]. groups() -> - [{p,test_lib:parallel(), + [{p,[parallel], [pmatch,mixed,aliases,match_in_call,untuplify, shortcut_boolean,letify_guard,selectify, - underscore,match_map,coverage]}]. + underscore,match_map,map_vars_used,coverage]}]. init_per_suite(Config) -> @@ -406,12 +407,31 @@ underscore(Config) when is_list(Config) -> match_map(Config) when is_list(Config) -> Map = #{key=>{x,y},ignore=>anything}, #s{map=Map,t={x,y}} = do_match_map(#s{map=Map}), + {a,#{k:={a,b,c}}} = do_match_map_2(#{k=>{a,b,c}}), ok. do_match_map(#s{map=#{key:=Val}}=S) -> %% Would crash with a 'badarg' exception. S#s{t=Val}. +do_match_map_2(Map) -> + case {a,Map} of + {a,#{k:=_}}=Tuple -> + Tuple + end. + +map_vars_used(Config) when is_list(Config) -> + {some,value} = do_map_vars_used(a, b, #{{a,b}=>42,v=>{some,value}}), + ok. + +do_map_vars_used(X, Y, Map) -> + case {X,Y} of + T -> + %% core_lib:is_var_used/2 would not consider T used. + #{T:=42,v:=Val} = Map, + Val + end. + coverage(Config) when is_list(Config) -> %% Cover beam_dead. ok = coverage_1(x, a), diff --git a/lib/compiler/test/misc_SUITE.erl b/lib/compiler/test/misc_SUITE.erl index 5416e8b6c7..d721a86f5a 100644 --- a/lib/compiler/test/misc_SUITE.erl +++ b/lib/compiler/test/misc_SUITE.erl @@ -60,7 +60,7 @@ all() -> [{group,p}]. groups() -> - [{p,[],%%test_lib:parallel(), + [{p,[], [tobias,empty_string,md5,silly_coverage, confused_literals,integer_encoding,override_bif]}]. diff --git a/lib/compiler/test/record_SUITE.erl b/lib/compiler/test/record_SUITE.erl index f736e14bf6..8cc90026ec 100644 --- a/lib/compiler/test/record_SUITE.erl +++ b/lib/compiler/test/record_SUITE.erl @@ -246,6 +246,14 @@ record_test_2(Config) when is_list(Config) -> ?line Barf = update_barf(Barf0), ?line #barf{a="abc",b=1} = id(Barf), + %% Test optimization of is_record/3. + false = case id({a,b}) of + {_,_}=Tuple -> is_record(Tuple, foo) + end, + false = case id(true) of + true=Bool -> is_record(Bool, foo) + end, + ok. record_test_3(Config) when is_list(Config) -> diff --git a/lib/compiler/test/test_lib.erl b/lib/compiler/test/test_lib.erl index a8befbecd9..a5e2855f8c 100644 --- a/lib/compiler/test/test_lib.erl +++ b/lib/compiler/test/test_lib.erl @@ -18,11 +18,13 @@ %% -module(test_lib). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -compile({no_auto_import,[binary_part/2]}). --export([recompile/1,parallel/0,uniq/0,opt_opts/1,get_data_dir/1, +-export([id/1,recompile/1,parallel/0,uniq/0,opt_opts/1,get_data_dir/1, smoke_disasm/1,p_run/2,binary_part/2]). +id(I) -> I. + recompile(Mod) when is_atom(Mod) -> case whereis(cover_server) of undefined -> ok; @@ -44,6 +46,10 @@ smoke_disasm(File) when is_list(File) -> Res = beam_disasm:file(File), {beam_file,_Mod} = {element(1, Res),element(2, Res)}. +%% If we are running cover, we don't want to run test cases that +%% invokes the compiler in parallel, as doing so would probably +%% be slower than running them sequentially. + parallel() -> case ?t:is_cover() orelse erlang:system_info(schedulers) =:= 1 of true -> []; @@ -90,13 +96,18 @@ get_data_dir(Config) -> %% Will fail the test case if there were any errors. p_run(Test, List) -> + S = erlang:system_info(schedulers), N = case ?t:is_cover() of false -> - erlang:system_info(schedulers); + S + 1; true -> - %% Cover is running. Using more than one process - %% will probably only slow down compilation. - 1 + %% Cover is running. Using too many processes + %% could slow us down. Measurements on my computer + %% showed that using 4 parallel processes was + %% slightly faster than using 3. Using more than + %% 4 would not buy us much and could actually be + %% slower. + max(S, 4) end, p_run_loop(Test, List, N, [], 0, 0). diff --git a/lib/compiler/test/trycatch_SUITE.erl b/lib/compiler/test/trycatch_SUITE.erl index 4530d08c77..8ab618bb01 100644 --- a/lib/compiler/test/trycatch_SUITE.erl +++ b/lib/compiler/test/trycatch_SUITE.erl @@ -35,7 +35,7 @@ all() -> [{group,p}]. groups() -> - [{p,test_lib:parallel(), + [{p,[parallel], [basic,lean_throw,try_of,try_after,catch_oops, after_oops,eclectic,rethrow,nested_of,nested_catch, nested_after,nested_horrid,last_call_optimization, @@ -790,7 +790,6 @@ nested_after_1({X1,C1,V1}, nested_horrid(Config) when is_list(Config) -> - _V = {make_ref(),nested_horrid,4.711}, {[true,true],{[true,1.0],1.0}} = nested_horrid_1({true,void,void}, 1.0), ok. diff --git a/lib/compiler/test/warnings_SUITE.erl b/lib/compiler/test/warnings_SUITE.erl index be0348a92d..6663985ad7 100644 --- a/lib/compiler/test/warnings_SUITE.erl +++ b/lib/compiler/test/warnings_SUITE.erl @@ -39,7 +39,7 @@ guard/1,bad_arith/1,bool_cases/1,bad_apply/1, files/1,effect/1,bin_opt_info/1,bin_construction/1, comprehensions/1,maps/1,redundant_boolean_clauses/1, - latin1_fallback/1]). + latin1_fallback/1,underscore/1]). % Default timetrap timeout (set in init_per_testcase). -define(default_timeout, ?t:minutes(2)). @@ -64,7 +64,8 @@ groups() -> [pattern,pattern2,pattern3,pattern4,guard, bad_arith,bool_cases,bad_apply,files,effect, bin_opt_info,bin_construction,comprehensions,maps, - redundant_boolean_clauses,latin1_fallback]}]. + redundant_boolean_clauses,latin1_fallback, + underscore]}]. init_per_suite(Config) -> Config. @@ -280,11 +281,12 @@ bad_arith(Config) when is_list(Config) -> {3,sys_core_fold,{eval_failure,badarith}}, {9,sys_core_fold,nomatch_guard}, {9,sys_core_fold,{eval_failure,badarith}}, + {9,sys_core_fold,{no_effect,{erlang,is_integer,1}}}, {10,sys_core_fold,nomatch_guard}, {10,sys_core_fold,{eval_failure,badarith}}, {15,sys_core_fold,{eval_failure,badarith}} ] }}], - ?line [] = run(Config, Ts), + [] = run(Config, Ts), ok. bool_cases(Config) when is_list(Config) -> @@ -678,6 +680,45 @@ latin1_fallback(Conf) when is_list(Conf) -> ok. +underscore(Config) when is_list(Config) -> + S0 = <<"f(A) -> + _VAR1 = <<A>>, + _VAR2 = {ok,A}, + _VAR3 = [A], + ok. + g(A) -> + _VAR1 = A/0, + _VAR2 = date(), + ok. + h() -> + _VAR1 = fun() -> ok end, + ok. + i(A) -> + _VAR1 = #{A=>42}, + ok. + ">>, + Ts0 = [{underscore0, + S0, + [], + {warnings,[{2,sys_core_fold,useless_building}, + {3,sys_core_fold,useless_building}, + {4,sys_core_fold,useless_building}, + {7,sys_core_fold,result_ignored}, + {8,sys_core_fold,{no_effect,{erlang,date,0}}}, + {11,sys_core_fold,useless_building}, + {14,sys_core_fold,useless_building} + ]}}], + [] = run(Config, Ts0), + + %% Replace all "_VAR<digit>" variables with a plain underscore. + %% Now there should be no warnings. + S1 = re:replace(S0, "_VAR\\d+", "_", [global]), + io:format("~s\n", [S1]), + Ts1 = [{underscore1,S1,[],[]}], + [] = run(Config, Ts1), + + ok. + %%% %%% End of test cases. %%% @@ -699,10 +740,10 @@ run(Config, Tests) -> %% Compiles a test module and returns the list of errors and warnings. run_test(Conf, Test0, Warnings) -> - Mod = "warnings_"++test_lib:uniq(), - Filename = Mod ++ ".erl", + Module = "warnings_"++test_lib:uniq(), + Filename = Module ++ ".erl", ?line DataDir = ?privdir, - Test = ["-module(", Mod, "). ", Test0], + Test = ["-module(", Module, "). ", Test0], ?line File = filename:join(DataDir, Filename), ?line Opts = [binary,export_all,return|Warnings], ?line ok = file:write_file(File, Test), diff --git a/lib/dialyzer/test/small_SUITE_data/src/maps_redef2.erl b/lib/dialyzer/test/small_SUITE_data/src/maps_redef2.erl new file mode 100644 index 0000000000..945b2a9144 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/maps_redef2.erl @@ -0,0 +1,23 @@ +%% In 17, the linter says that map(A) redefines 'type map', which is +%% allowed until next release. However, Dialyzer used to replace +%% map(A) with #{}, which resulted in warnings. + +-module(maps_redef2). + +-export([t/0]). + +-type map(_A) :: integer(). + +t() -> + M = new(), + t1(M). + +-spec t1(map(_)) -> map(_). + +t1(A) -> + A + A. + +-spec new() -> map(_). + +new() -> + 3. diff --git a/lib/eldap/src/eldap.erl b/lib/eldap/src/eldap.erl index 977f7d2809..ae47c815c9 100644 --- a/lib/eldap/src/eldap.erl +++ b/lib/eldap/src/eldap.erl @@ -413,7 +413,7 @@ parse_args([{port, Port}|T], Cpid, Data) when is_integer(Port) -> parse_args([{timeout, Timeout}|T], Cpid, Data) when is_integer(Timeout),Timeout>0 -> parse_args(T, Cpid, Data#eldap{timeout = Timeout}); parse_args([{anon_auth, true}|T], Cpid, Data) -> - parse_args(T, Cpid, Data#eldap{anon_auth = false}); + parse_args(T, Cpid, Data#eldap{anon_auth = true}); parse_args([{anon_auth, _}|T], Cpid, Data) -> parse_args(T, Cpid, Data); parse_args([{ssl, true}|T], Cpid, Data) -> diff --git a/lib/hipe/cerl/erl_bif_types.erl b/lib/hipe/cerl/erl_bif_types.erl index 74e93bf098..5b1401b34a 100644 --- a/lib/hipe/cerl/erl_bif_types.erl +++ b/lib/hipe/cerl/erl_bif_types.erl @@ -1070,9 +1070,6 @@ type(hipe_bifs, find_na_or_make_stub, 2, Xs, Opaques) -> type(hipe_bifs, fun_to_address, 1, Xs, Opaques) -> strict(hipe_bifs, fun_to_address, 1, Xs, fun (_) -> t_integer() end, Opaques); -%% type(hipe_bifs, get_emu_address, 1, Xs, Opaques) -> -%% strict(hipe_bifs, get_emu_address, 1, Xs, -%% fun (_) -> t_integer() end, Opaques); % address type(hipe_bifs, get_fe, 2, Xs, Opaques) -> strict(hipe_bifs, get_fe, 2, Xs, fun (_) -> t_integer() end, Opaques); type(hipe_bifs, get_rts_param, 1, Xs, Opaques) -> @@ -1081,9 +1078,6 @@ type(hipe_bifs, get_rts_param, 1, Xs, Opaques) -> type(hipe_bifs, invalidate_funinfo_native_addresses, 1, Xs, Opaques) -> strict(hipe_bifs, invalidate_funinfo_native_addresses, 1, Xs, fun (_) -> t_nil() end, Opaques); -%% type(hipe_bifs, make_native_stub, 2, Xs, Opaques) -> -%% strict(hipe_bifs, make_native_stub, 2, Xs, -%% fun (_) -> t_integer() end, Opaques); % address type(hipe_bifs, mark_referred_from, 1, Xs, Opaques) -> strict(hipe_bifs, mark_referred_from, 1, Xs, fun (_) -> t_nil() end, Opaques); @@ -2462,16 +2456,12 @@ arg_types(hipe_bifs, find_na_or_make_stub, 2) -> [t_mfa(), t_boolean()]; arg_types(hipe_bifs, fun_to_address, 1) -> [t_mfa()]; -%% arg_types(hipe_bifs, get_emu_address, 1) -> -%% [t_mfa()]; arg_types(hipe_bifs, get_fe, 2) -> [t_atom(), t_tuple([t_integer(), t_integer(), t_integer()])]; arg_types(hipe_bifs, get_rts_param, 1) -> [t_fixnum()]; arg_types(hipe_bifs, invalidate_funinfo_native_addresses, 1) -> [t_list(t_mfa())]; -%% arg_types(hipe_bifs, make_native_stub, 2) -> -%% [t_integer(), t_arity()]; arg_types(hipe_bifs, mark_referred_from, 1) -> [t_mfa()]; arg_types(hipe_bifs, merge_term, 1) -> diff --git a/lib/inets/doc/src/httpd.xml b/lib/inets/doc/src/httpd.xml index 4ca038cc99..20c8a6b1b1 100644 --- a/lib/inets/doc/src/httpd.xml +++ b/lib/inets/doc/src/httpd.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1997</year><year>2013</year> + <year>1997</year><year>2015</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -249,7 +249,16 @@ <p>Limits the size of the message header of HTTP request. Defaults to 10240. </p> </item> - + + <marker id="prop_max_content_length"></marker> + <tag>{max_content_length, integer()}</tag> + <item> + <p>Maximum Content-Length in an incoming request, in bytes. Requests + with content larger than this are answered with Status 413. + Defaults to 100000000 (100 MB). + </p> + </item> + <marker id="prop_max_uri"></marker> <tag>{max_uri_size, integer()}</tag> <item> diff --git a/lib/inets/doc/src/notes.xml b/lib/inets/doc/src/notes.xml index fb7034498c..7f73aa5e7b 100644 --- a/lib/inets/doc/src/notes.xml +++ b/lib/inets/doc/src/notes.xml @@ -32,7 +32,40 @@ <file>notes.xml</file> </header> - <section><title>Inets 5.10.4</title> + <section><title>Inets 5.10.5</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + mod_alias now handles https-URIs properly</p> + <p> + Consistent view of configuration parameter + keep_alive_timeout, should be presented in the + httpd:info/[1,2] function in the same unit as it is + inputted.</p> + <p> + Own Id: OTP-12436 Aux Id: seq12786 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Gracefully handle invalid content-lenght headers instead + of crashing in list_to_integer.</p> + <p> + Own Id: OTP-12429</p> + </item> + </list> + </section> + +</section> + +<section><title>Inets 5.10.4</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/inets/examples/httpd_load_test/hdlt_slave.erl b/lib/inets/examples/httpd_load_test/hdlt_slave.erl index 52af9b5b90..41361418bc 100644 --- a/lib/inets/examples/httpd_load_test/hdlt_slave.erl +++ b/lib/inets/examples/httpd_load_test/hdlt_slave.erl @@ -180,7 +180,7 @@ ssh_slave_start(Host, ErlCmd) -> ?DEBUG("ssh_exec_erl -> done", []), {ok, Connection, Channel}; Error3 -> - ?LOG("failed exec comand: ~p", [Error3]), + ?LOG("failed exec command: ~p", [Error3]), throw({error, {ssh_exec_failed, Error3}}) end. diff --git a/lib/inets/src/http_client/httpc_cookie.erl b/lib/inets/src/http_client/httpc_cookie.erl index 5d71a0bb8f..ed306a84f5 100644 --- a/lib/inets/src/http_client/httpc_cookie.erl +++ b/lib/inets/src/http_client/httpc_cookie.erl @@ -362,6 +362,8 @@ parse_set_cookie(CookieHeader, {DefaultPath, DefaultDomain}) -> Name = string:substr(CookieHeader, 1, Pos - 1), {Value, Attrs} = case string:substr(CookieHeader, Pos + 1) of + [] -> + {"", ""}; [$;|ValueAndAttrs] -> {"", string:tokens(ValueAndAttrs, ";")}; ValueAndAttrs -> diff --git a/lib/inets/src/http_lib/http_internal.hrl b/lib/inets/src/http_lib/http_internal.hrl index 53b776c4e7..54425740b5 100644 --- a/lib/inets/src/http_lib/http_internal.hrl +++ b/lib/inets/src/http_lib/http_internal.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2014. All Rights Reserved. +%% Copyright Ericsson AB 2002-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -28,6 +28,7 @@ -define(HTTP_MAX_URI_SIZE, nolimit). -define(HTTP_MAX_VERSION_STRING, 8). -define(HTTP_MAX_METHOD_STRING, 20). +-define(HTTP_MAX_CONTENT_LENGTH, 100000000). -ifndef(HTTP_DEFAULT_SSL_KIND). -define(HTTP_DEFAULT_SSL_KIND, essl). diff --git a/lib/inets/src/http_lib/http_request.erl b/lib/inets/src/http_lib/http_request.erl index f295453bdd..a0833ddf01 100644 --- a/lib/inets/src/http_lib/http_request.erl +++ b/lib/inets/src/http_lib/http_request.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2014. All Rights Reserved. +%% Copyright Ericsson AB 2005-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -21,8 +21,16 @@ -include("http_internal.hrl"). --export([headers/2, http_headers/1, is_absolut_uri/1]). +-export([headers/2, http_headers/1, is_absolut_uri/1, key_value/1]). + +key_value(KeyValueStr) -> + case lists:splitwith(fun($:) -> false; (_) -> true end, KeyValueStr) of + {Key, [$: | Value]} -> + {http_util:to_lower(string:strip(Key)), string:strip(Value)}; + {_, []} -> + undefined + end. %%------------------------------------------------------------------------- %% headers(HeaderList, #http_request_h{}) -> #http_request_h{} %% HeaderList - ["HeaderField:Value"] @@ -34,14 +42,12 @@ %%------------------------------------------------------------------------- headers([], Headers) -> Headers; -headers([Header | Tail], Headers) -> - case lists:splitwith(fun($:) -> false; (_) -> true end, Header) of - {Key, [$: | Value]} -> - headers(Tail, headers(http_util:to_lower(string:strip(Key)), - string:strip(Value), Headers)); - {_, []} -> - headers(Tail, Headers) - end. +headers([{Key, Value} | Tail], Headers) -> + headers(Tail, headers(Key, Value, Headers)); +headers([undefined], Headers) -> + Headers; +headers(KeyValues, Headers) -> + headers([key_value(KeyValue) || KeyValue <- KeyValues], Headers). %%------------------------------------------------------------------------- %% headers(#http_request_h{}) -> HeaderList diff --git a/lib/inets/src/http_server/httpd_conf.erl b/lib/inets/src/http_server/httpd_conf.erl index 8f68d9fcd5..78dda794db 100644 --- a/lib/inets/src/http_server/httpd_conf.erl +++ b/lib/inets/src/http_server/httpd_conf.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2013. All Rights Reserved. +%% Copyright Ericsson AB 1997-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -205,13 +205,13 @@ load("MaxURISize " ++ MaxHeaderSize, []) -> " is an invalid number of MaxHeaderSize")} end; -load("MaxBodySize " ++ MaxBodySize, []) -> - case make_integer(MaxBodySize) of +load("MaxContentLength " ++ Max, []) -> + case make_integer(Max) of {ok, Integer} -> - {ok, [], {max_body_size,Integer}}; + {ok, [], {max_content_length, Integer}}; {error, _} -> - {error, ?NICE(clean(MaxBodySize) ++ - " is an invalid number of MaxBodySize")} + {error, ?NICE(clean(Max) ++ + " is an invalid number of MaxContentLength")} end; load("ServerName " ++ ServerName, []) -> @@ -337,7 +337,7 @@ load("MaxKeepAliveRequest " ++ MaxRequests, []) -> load("KeepAliveTimeout " ++ Timeout, []) -> case make_integer(Timeout) of {ok, Integer} -> - {ok, [], {keep_alive_timeout, Integer*1000}}; + {ok, [], {keep_alive_timeout, Integer}}; {error, _} -> {error, ?NICE(clean(Timeout)++" is an invalid KeepAliveTimeout")} end; @@ -569,6 +569,12 @@ validate_config_params([{max_body_size, Value} | Rest]) validate_config_params([{max_body_size, Value} | _]) -> throw({max_body_size, Value}); +validate_config_params([{max_content_length, Value} | Rest]) + when is_integer(Value) andalso (Value > 0) -> + validate_config_params(Rest); +validate_config_params([{max_content_length, Value} | _]) -> + throw({max_content_length, Value}); + validate_config_params([{server_name, Value} | Rest]) when is_list(Value) -> validate_config_params(Rest); @@ -635,7 +641,7 @@ validate_config_params([{max_keep_alive_request, Value} | Rest]) when is_integer(Value) andalso (Value > 0) -> validate_config_params(Rest); validate_config_params([{max_keep_alive_request, Value} | _]) -> - throw({max_header_size, Value}); + throw({max_keep_alive_request, Value}); validate_config_params([{keep_alive_timeout, Value} | Rest]) when is_integer(Value) andalso (Value >= 0) -> @@ -799,7 +805,7 @@ store({server_tokens, ServerTokens} = Entry, _ConfigList) -> Server = server(ServerTokens), {ok, [Entry, {server, Server}]}; store({keep_alive_timeout, KeepAliveTimeout}, _ConfigList) -> - {ok, {keep_alive_timeout, KeepAliveTimeout * 1000}}; + {ok, {keep_alive_timeout, KeepAliveTimeout}}; store(ConfigListEntry, _ConfigList) -> {ok, ConfigListEntry}. diff --git a/lib/inets/src/http_server/httpd_request.erl b/lib/inets/src/http_server/httpd_request.erl index 712c73599f..6985065c3e 100644 --- a/lib/inets/src/http_server/httpd_request.erl +++ b/lib/inets/src/http_server/httpd_request.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2014. All Rights Reserved. +%% Copyright Ericsson AB 2005-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -118,18 +118,17 @@ validate(Method, Uri, Version) -> %% create it. %% ---------------------------------------------------------------------- update_mod_data(ModData, Method, RequestURI, HTTPVersion, Headers)-> - ParsedHeaders = tagup_header(Headers), - PersistentConn = get_persistens(HTTPVersion, ParsedHeaders, + PersistentConn = get_persistens(HTTPVersion, Headers, ModData#mod.config_db), {ok, ModData#mod{data = [], method = Method, absolute_uri = format_absolute_uri(RequestURI, - ParsedHeaders), + Headers), request_uri = format_request_uri(RequestURI), http_version = HTTPVersion, request_line = Method ++ " " ++ RequestURI ++ " " ++ HTTPVersion, - parsed_header = ParsedHeaders, + parsed_header = Headers, connection = PersistentConn}}. %%%======================================================================== @@ -146,14 +145,14 @@ parse_method(_, _, _, Max, _, _) -> %% We do not know the version of the client as it comes after the %% method send the lowest version in the response so that the client %% will be able to handle it. - {error, {too_long, Max, 413, "Method unreasonably long"}, lowest_version()}. + {error, {size_error, Max, 413, "Method unreasonably long"}, lowest_version()}. parse_uri(_, _, Current, MaxURI, _, _) when (Current > MaxURI) andalso (MaxURI =/= nolimit) -> %% We do not know the version of the client as it comes after the %% uri send the lowest version in the response so that the client %% will be able to handle it. - {error, {too_long, MaxURI, 414, "URI unreasonably long"},lowest_version()}; + {error, {size_error, MaxURI, 414, "URI unreasonably long"},lowest_version()}; parse_uri(<<>>, URI, Current, Max, MaxSizes, Result) -> {?MODULE, parse_uri, [URI, Current, Max, MaxSizes, Result]}; parse_uri(<<?SP, Rest/binary>>, URI, _, _, MaxSizes, Result) -> @@ -179,12 +178,12 @@ parse_version(<<?CR>> = Data, Version, Current, Max, MaxSizes, Result) -> parse_version(<<Octet, Rest/binary>>, Version, Current, Max, MaxSizes, Result) when Current =< Max -> parse_version(Rest, [Octet | Version], Current + 1, Max, MaxSizes, Result); parse_version(_, _, _, Max,_,_) -> - {error, {too_long, Max, 413, "Version string unreasonably long"}, lowest_version()}. + {error, {size_error, Max, 413, "Version string unreasonably long"}, lowest_version()}. parse_headers(_, _, _, Current, Max, _, Result) when Max =/= nolimit andalso Current > Max -> HttpVersion = lists:nth(3, lists:reverse(Result)), - {error, {too_long, Max, 413, "Headers unreasonably long"}, HttpVersion}; + {error, {size_error, Max, 413, "Headers unreasonably long"}, HttpVersion}; parse_headers(<<>>, Header, Headers, Current, Max, MaxSizes, Result) -> {?MODULE, parse_headers, [<<>>, Header, Headers, Current, Max, @@ -204,14 +203,22 @@ parse_headers(<<?CR,?LF,?CR,?LF,Body/binary>>, [], [], _, _, _, Result) -> Result])), {ok, NewResult}; parse_headers(<<?CR,?LF,?CR,?LF,Body/binary>>, Header, Headers, _, _, - _, Result) -> - HTTPHeaders = [lists:reverse(Header) | Headers], - RequestHeaderRcord = - http_request:headers(HTTPHeaders, #http_request_h{}), - NewResult = - list_to_tuple(lists:reverse([Body, {RequestHeaderRcord, - HTTPHeaders} | Result])), - {ok, NewResult}; + MaxSizes, Result) -> + case http_request:key_value(lists:reverse(Header)) of + undefined -> %% Skip headers with missing : + {ok, list_to_tuple(lists:reverse([Body, {http_request:headers(Headers, #http_request_h{}), Headers} | Result]))}; + NewHeader -> + case check_header(NewHeader, MaxSizes) of + ok -> + {ok, list_to_tuple(lists:reverse([Body, {http_request:headers([NewHeader | Headers], + #http_request_h{}), + [NewHeader | Headers]} | Result]))}; + + {error, Reason} -> + HttpVersion = lists:nth(3, lists:reverse(Result)), + {error, Reason, HttpVersion} + end + end; parse_headers(<<?CR,?LF,?CR>> = Data, Header, Headers, Current, Max, MaxSizes, Result) -> @@ -243,8 +250,21 @@ parse_headers(<<?LF, Octet, Rest/binary>>, Header, Headers, Current, Max, MaxSizes, Result); parse_headers(<<?CR,?LF, Octet, Rest/binary>>, Header, Headers, _, Max, MaxSizes, Result) -> - parse_headers(Rest, [Octet], [lists:reverse(Header) | Headers], - 0, Max, MaxSizes, Result); + case http_request:key_value(lists:reverse(Header)) of + undefined -> %% Skip headers with missing : + parse_headers(Rest, [Octet], Headers, + 0, Max, MaxSizes, Result); + NewHeader -> + case check_header(NewHeader, MaxSizes) of + ok -> + parse_headers(Rest, [Octet], [NewHeader | Headers], + 0, Max, MaxSizes, Result); + {error, Reason} -> + HttpVersion = lists:nth(3, lists:reverse(Result)), + {error, Reason, HttpVersion} + end + end; + parse_headers(<<?CR>> = Data, Header, Headers, Current, Max, MaxSizes, Result) -> {?MODULE, parse_headers, [Data, Header, Headers, Current, Max, @@ -388,29 +408,25 @@ get_persistens(HTTPVersion,ParsedHeader,ConfigDB)-> false end. - -%%---------------------------------------------------------------------- -%% tagup_header -%% -%% Parses the header of a HTTP request and returns a key,value tuple -%% list containing Name and Value of each header directive as of: -%% -%% Content-Type: multipart/mixed -> {"Content-Type", "multipart/mixed"} -%% -%% But in http/1.1 the field-names are case insencitive so now it must be -%% Content-Type: multipart/mixed -> {"content-type", "multipart/mixed"} -%% The standard furthermore says that leading and traling white space -%% is not a part of the fieldvalue and shall therefore be removed. -%%---------------------------------------------------------------------- -tagup_header([]) -> []; -tagup_header([Line|Rest]) -> [tag(Line, [])|tagup_header(Rest)]. - -tag([], Tag) -> - {http_util:to_lower(lists:reverse(Tag)), ""}; -tag([$:|Rest], Tag) -> - {http_util:to_lower(lists:reverse(Tag)), string:strip(Rest)}; -tag([Chr|Rest], Tag) -> - tag(Rest, [Chr|Tag]). - lowest_version()-> "HTTP/0.9". + +check_header({"content-length", Value}, Maxsizes) -> + Max = proplists:get_value(max_content_length, Maxsizes), + MaxLen = length(integer_to_list(Max)), + case length(Value) =< MaxLen of + true -> + try + _ = list_to_integer(Value), + ok + catch _:_ -> + {error, {size_error, Max, 411, "content-length not an integer"}} + end; + false -> + {error, {size_error, Max, 413, "content-length unreasonably long"}} + end; +check_header(_, _) -> + ok. + + + diff --git a/lib/inets/src/http_server/httpd_request_handler.erl b/lib/inets/src/http_server/httpd_request_handler.erl index 9bea58cc9e..f7a9fe5d49 100644 --- a/lib/inets/src/http_server/httpd_request_handler.erl +++ b/lib/inets/src/http_server/httpd_request_handler.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2014. All Rights Reserved. +%% Copyright Ericsson AB 1997-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -96,8 +96,9 @@ init([Manager, ConfigDB, AcceptTimeout]) -> proc_lib:init_ack({ok, self()}), {SocketType, Socket} = await_socket_ownership_transfer(AcceptTimeout), - - KeepAliveTimeOut = httpd_util:lookup(ConfigDB, keep_alive_timeout, 150000), + + %%Timeout value is in seconds we want it in milliseconds + KeepAliveTimeOut = 1000 * httpd_util:lookup(ConfigDB, keep_alive_timeout, 150), case http_transport:negotiate(SocketType, Socket, ?HANDSHAKE_TIMEOUT) of {error, _Error} -> @@ -119,11 +120,15 @@ continue_init(Manager, ConfigDB, SocketType, Socket, TimeOut) -> MaxHeaderSize = max_header_size(ConfigDB), MaxURISize = max_uri_size(ConfigDB), NrOfRequest = max_keep_alive_request(ConfigDB), - + MaxContentLen = max_content_length(ConfigDB), + {_, Status} = httpd_manager:new_connection(Manager), MFA = {httpd_request, parse, [[{max_uri, MaxURISize}, {max_header, MaxHeaderSize}, - {max_version, ?HTTP_MAX_VERSION_STRING}, {max_method, ?HTTP_MAX_METHOD_STRING}]]}, + {max_version, ?HTTP_MAX_VERSION_STRING}, + {max_method, ?HTTP_MAX_METHOD_STRING}, + {max_content_length, MaxContentLen} + ]]}, State = #state{mod = Mod, manager = Manager, @@ -207,7 +212,7 @@ handle_info({Proto, Socket, Data}, set_new_data_size(cancel_request_timeout(State), NewDataSize) end, handle_http_msg(Result, NewState); - {error, {too_long, MaxSize, ErrCode, ErrStr}, Version} -> + {error, {size_error, MaxSize, ErrCode, ErrStr}, Version} -> NewModData = ModData#mod{http_version = Version}, httpd_response:send_status(NewModData, ErrCode, ErrStr), Reason = io_lib:format("~p: ~p max size is ~p~n", @@ -444,8 +449,7 @@ handle_body(#state{headers = Headers, body = Body, mod = ModData} = State, error_log(Reason, ModData), {stop, normal, State#state{response_sent = true}}; _ -> - Length = - list_to_integer(Headers#http_request_h.'content-length'), + Length = list_to_integer(Headers#http_request_h.'content-length'), case ((Length =< MaxBodySize) or (MaxBodySize == nolimit)) of true -> case httpd_request:whole_body(Body, Length) of @@ -454,7 +458,7 @@ handle_body(#state{headers = Headers, body = Body, mod = ModData} = State, ModData#mod.socket, [{active, once}]), {noreply, State#state{mfa = - {Module, Function, Args}}}; + {Module, Function, Args}}}; {ok, NewBody} -> handle_response( @@ -471,7 +475,7 @@ handle_body(#state{headers = Headers, body = Body, mod = ModData} = State, handle_expect(#state{headers = Headers, mod = #mod{config_db = ConfigDB} = ModData} = State, MaxBodySize) -> - Length = Headers#http_request_h.'content-length', + Length = list_to_integer(Headers#http_request_h.'content-length'), case expect(Headers, ModData#mod.http_version, ConfigDB) of continue when (MaxBodySize > Length) orelse (MaxBodySize =:= nolimit) -> httpd_response:send_status(ModData, 100, ""), @@ -545,9 +549,13 @@ handle_next_request(#state{mod = #mod{connection = true} = ModData, init_data = ModData#mod.init_data}, MaxHeaderSize = max_header_size(ModData#mod.config_db), MaxURISize = max_uri_size(ModData#mod.config_db), + MaxContentLen = max_content_length(ModData#mod.config_db), MFA = {httpd_request, parse, [[{max_uri, MaxURISize}, {max_header, MaxHeaderSize}, - {max_version, ?HTTP_MAX_VERSION_STRING}, {max_method, ?HTTP_MAX_METHOD_STRING}]]}, + {max_version, ?HTTP_MAX_VERSION_STRING}, + {max_method, ?HTTP_MAX_METHOD_STRING}, + {max_content_length, MaxContentLen} + ]]}, TmpState = State#state{mod = NewModData, mfa = MFA, max_keep_alive_request = decrease(Max), @@ -630,3 +638,5 @@ max_body_size(ConfigDB) -> max_keep_alive_request(ConfigDB) -> httpd_util:lookup(ConfigDB, max_keep_alive_request, infinity). +max_content_length(ConfigDB) -> + httpd_util:lookup(ConfigDB, max_content_length, ?HTTP_MAX_CONTENT_LENGTH). diff --git a/lib/inets/src/http_server/mod_alias.erl b/lib/inets/src/http_server/mod_alias.erl index 0b9fe4cfe0..5039cd56b5 100644 --- a/lib/inets/src/http_server/mod_alias.erl +++ b/lib/inets/src/http_server/mod_alias.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% Copyright Ericsson AB 1997-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -55,6 +55,7 @@ do(#mod{data = Data} = Info) -> do_alias(#mod{config_db = ConfigDB, request_uri = ReqURI, + socket_type = SocketType, data = Data}) -> {ShortPath, Path, AfterPath} = real_name(ConfigDB, ReqURI, which_alias(ConfigDB)), @@ -70,8 +71,9 @@ do_alias(#mod{config_db = ConfigDB, (LastChar =/= $/)) -> ?hdrt("directory and last-char is a /", []), ServerName = which_server_name(ConfigDB), - Port = port_string( which_port(ConfigDB) ), - URL = "http://" ++ ServerName ++ Port ++ ReqURI ++ "/", + Port = port_string(which_port(ConfigDB)), + Protocol = get_protocol(SocketType), + URL = Protocol ++ ServerName ++ Port ++ ReqURI ++ "/", ReasonPhrase = httpd_util:reason_phrase(301), Message = httpd_util:message(301, URL, ConfigDB), {proceed, @@ -94,6 +96,12 @@ port_string(80) -> port_string(Port) -> ":" ++ integer_to_list(Port). +get_protocol(ip_comm) -> + "http://"; +get_protocol(_) -> + %% Should clean up to have only one ssl type essl vs ssl is not relevant any more + "https://". + %% real_name real_name(ConfigDB, RequestURI, []) -> diff --git a/lib/inets/test/http_format_SUITE.erl b/lib/inets/test/http_format_SUITE.erl index d4a3f28f38..5952e9fd6e 100644 --- a/lib/inets/test/http_format_SUITE.erl +++ b/lib/inets/test/http_format_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2014. All Rights Reserved. +%% Copyright Ericsson AB 2004-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -355,10 +355,12 @@ http_request(Config) when is_list(Config) -> "http://www.erlang.org", "HTTP/1.1", {#http_request_h{host = "www.erlang.org", te = []}, - ["te: ","host:www.erlang.org"]}, <<>>} = + [{"te", []}, {"host", "www.erlang.org"}]}, <<>>} = parse(httpd_request, parse, [[{max_header, ?HTTP_MAX_HEADER_SIZE}, {max_version, ?HTTP_MAX_VERSION_STRING}, - {max_method, ?HTTP_MAX_METHOD_STRING}]], + {max_method, ?HTTP_MAX_METHOD_STRING}, + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH} + ]], HttpHead), HttpHead1 = ["GET http://www.erlang.org HTTP/1.1" ++ @@ -369,7 +371,9 @@ http_request(Config) when is_list(Config) -> {#http_request_h{}, []}, <<>>} = parse(httpd_request, parse, [[{max_header, ?HTTP_MAX_HEADER_SIZE}, {max_version, ?HTTP_MAX_VERSION_STRING}, - {max_method, ?HTTP_MAX_METHOD_STRING}]], HttpHead1), + {max_method, ?HTTP_MAX_METHOD_STRING}, + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH} + ]], HttpHead1), HttpHead2 = ["GET http://www.erlang.org HTTP/1.1" ++ @@ -380,7 +384,9 @@ http_request(Config) when is_list(Config) -> {#http_request_h{}, []}, <<>>} = parse(httpd_request, parse, [[{max_header, ?HTTP_MAX_HEADER_SIZE}, {max_version, ?HTTP_MAX_VERSION_STRING}, - {max_method, ?HTTP_MAX_METHOD_STRING}]], HttpHead2), + {max_method, ?HTTP_MAX_METHOD_STRING}, + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH} + ]], HttpHead2), %% Note the following body is not related to the headers above HttpBody = ["<HTML>\n<HEAD>\n<TITLE> dummy </TITLE>\n</HEAD>\n<BODY>\n", diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 82e08a52cd..0e89e831fb 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2014. All Rights Reserved. +%% Copyright Ericsson AB 2004-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -1288,8 +1288,9 @@ dummy_server_init(Caller, ip_comm, Inet, _) -> dummy_ipcomm_server_loop({httpd_request, parse, [[{max_uri, ?HTTP_MAX_URI_SIZE}, {max_header, ?HTTP_MAX_HEADER_SIZE}, {max_version,?HTTP_MAX_VERSION_STRING}, - {max_method, ?HTTP_MAX_METHOD_STRING}]]}, - [], ListenSocket); + {max_method, ?HTTP_MAX_METHOD_STRING}, + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH}]]}, + [], ListenSocket); dummy_server_init(Caller, ssl, Inet, SSLOptions) -> BaseOpts = [binary, {reuseaddr,true}, {active, false} | @@ -1303,7 +1304,9 @@ dummy_ssl_server_init(Caller, BaseOpts, Inet) -> dummy_ssl_server_loop({httpd_request, parse, [[{max_uri, ?HTTP_MAX_URI_SIZE}, {max_method, ?HTTP_MAX_METHOD_STRING}, {max_version,?HTTP_MAX_VERSION_STRING}, - {max_method, ?HTTP_MAX_METHOD_STRING}]]}, + {max_method, ?HTTP_MAX_METHOD_STRING}, + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH} + ]]}, [], ListenSocket). dummy_ipcomm_server_loop(MFA, Handlers, ListenSocket) -> @@ -1380,16 +1383,20 @@ handle_request(Module, Function, Args, Socket) -> stop -> stop; <<>> -> - {httpd_request, parse, [[<<>>, [{max_uri, ?HTTP_MAX_URI_SIZE}, + {httpd_request, parse, [[{max_uri,?HTTP_MAX_URI_SIZE}, {max_header, ?HTTP_MAX_HEADER_SIZE}, {max_version,?HTTP_MAX_VERSION_STRING}, - {max_method, ?HTTP_MAX_METHOD_STRING}]]]}; + {max_method, ?HTTP_MAX_METHOD_STRING}, + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH} + ]]}; Data -> handle_request(httpd_request, parse, [Data, [{max_uri, ?HTTP_MAX_URI_SIZE}, - {max_header, ?HTTP_MAX_HEADER_SIZE}, - {max_version,?HTTP_MAX_VERSION_STRING}, - {max_method, ?HTTP_MAX_METHOD_STRING}]], Socket) + {max_header, ?HTTP_MAX_HEADER_SIZE}, + {max_version,?HTTP_MAX_VERSION_STRING}, + {max_method, ?HTTP_MAX_METHOD_STRING}, + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH} + ]], Socket) end; NewMFA -> NewMFA @@ -1479,7 +1486,7 @@ dummy_ssl_server_hang_loop(_) -> ensure_host_header_with_port([]) -> false; -ensure_host_header_with_port(["host: " ++ Host| _]) -> +ensure_host_header_with_port([{"host", Host}| _]) -> case string:tokens(Host, [$:]) of [_ActualHost, _Port] -> true; @@ -1491,7 +1498,7 @@ ensure_host_header_with_port([_|T]) -> auth_header([]) -> auth_header_not_found; -auth_header(["authorization:" ++ Value | _]) -> +auth_header([{"authorization", Value} | _]) -> {ok, string:strip(Value)}; auth_header([_ | Tail]) -> auth_header(Tail). @@ -1508,7 +1515,7 @@ handle_auth("Basic " ++ UserInfo, Challange, DefaultResponse) -> check_cookie([]) -> ct:fail(no_cookie_header); -check_cookie(["cookie:" ++ _Value | _]) -> +check_cookie([{"cookie", _} | _]) -> ok; check_cookie([_Head | Tail]) -> check_cookie(Tail). @@ -1731,6 +1738,7 @@ handle_uri(_,"/empty_set_cookie.html",_,_,_,_) -> handle_uri(_,"/invalid_set_cookie.html",_,_,_,_) -> "HTTP/1.1 200 ok\r\n" ++ "set-cookie: =\r\n" ++ + "set-cookie: name=\r\n" ++ "set-cookie: name-or-value\r\n" ++ "Content-Length:32\r\n\r\n"++ "<HTML><BODY>foobar</BODY></HTML>"; diff --git a/lib/inets/test/httpd_SUITE.erl b/lib/inets/test/httpd_SUITE.erl index 4010597657..342004f19b 100644 --- a/lib/inets/test/httpd_SUITE.erl +++ b/lib/inets/test/httpd_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2014. All Rights Reserved. +%% Copyright Ericsson AB 2013-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -132,6 +132,7 @@ http_get() -> bad_hex, missing_CR, max_header, + max_content_length, ipv6 ]. @@ -979,13 +980,22 @@ max_header(Config) when is_list(Config) -> Host = ?config(host, Config), case Version of "HTTP/0.9" -> - {skip, no_implemented}; + {skip, not_implemented}; _ -> dos_hostname(?config(type, Config), ?config(port, Config), Host, ?config(node, Config), Version, ?MAX_HEADER_SIZE) end. %%------------------------------------------------------------------------- +max_content_length() -> + ["Denial Of Service (DOS) attack, prevented by max_content_length"]. +max_content_length(Config) when is_list(Config) -> + Version = ?config(http_version, Config), + Host = ?config(host, Config), + garbage_content_length(?config(type, Config), ?config(port, Config), Host, + ?config(node, Config), Version). + +%%------------------------------------------------------------------------- security_1_1(Config) when is_list(Config) -> security([{http_version, "HTTP/1.1"} | Config]). @@ -1368,7 +1378,9 @@ server_config(http_reload, Config) -> server_config(https_reload, Config) -> [{keep_alive_timeout, 2}] ++ server_config(https, Config); server_config(http_limit, Config) -> - [{max_clients, 1}] ++ server_config(http, Config); + [{max_clients, 1}, + %% Make sure option checking code is run + {max_content_length, 100000002}] ++ server_config(http, Config); server_config(https_limit, Config) -> [{max_clients, 1}] ++ server_config(https, Config); server_config(http_basic_auth, Config) -> @@ -1814,7 +1826,7 @@ dos_hostname(Type, Port, Host, Node, Version, Max) -> ok = httpd_test_lib:verify_request(Type, Host, Port, Node, dos_hostname_request(TooLongHeader, Version), - [{statuscode, dos_code(Version)}, + [{statuscode, request_entity_too_large_code(Version)}, {version, Version}]). dos_hostname_request(Host, Version) -> dos_http_request("GET / ", Version, Host). @@ -1824,11 +1836,32 @@ dos_http_request(Request, "HTTP/1.1" = Version, Host) -> dos_http_request(Request, Version, Host) -> Request ++ Version ++ "\r\nhost:" ++ Host ++ "\r\n\r\n". -dos_code("HTTP/1.0") -> +request_entity_too_large_code("HTTP/1.0") -> 403; %% 413 not defined in HTTP/1.0 -dos_code(_) -> +request_entity_too_large_code(_) -> 413. +length_required_code("HTTP/1.0") -> + 403; %% 411 not defined in HTTP/1.0 +length_required_code(_) -> + 411. + +garbage_content_length(Type, Port, Host, Node, Version) -> + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + garbage_content_length_request("GET / ", Version, Host, "aaaa"), + [{statuscode, length_required_code(Version)}, + {version, Version}]), + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + garbage_content_length_request("GET / ", Version, Host, + lists:duplicate($a, 100)), + [{statuscode, request_entity_too_large_code(Version)}, + {version, Version}]). + +garbage_content_length_request(Request, Version, Host, Garbage) -> + http_request(Request, Version, Host, + {"content-length:" ++ Garbage, "Body with garbage content length indicator"}). + + update_password(Node, ServerRoot, _Address, Port, AuthPrefix, Dir, Old, New)-> Directory = filename:join([ServerRoot, "htdocs", AuthPrefix ++ Dir]), rpc:call(Node, mod_auth, update_password, diff --git a/lib/inets/vsn.mk b/lib/inets/vsn.mk index dbae5e4b3c..7d11916454 100644 --- a/lib/inets/vsn.mk +++ b/lib/inets/vsn.mk @@ -18,6 +18,6 @@ # %CopyrightEnd% APPLICATION = inets -INETS_VSN = 5.10.4 +INETS_VSN = 5.10.5 PRE_VSN = APP_VSN = "$(APPLICATION)-$(INETS_VSN)$(PRE_VSN)" diff --git a/lib/kernel/src/hipe_unified_loader.erl b/lib/kernel/src/hipe_unified_loader.erl index e5928c7b63..2d124d95b7 100644 --- a/lib/kernel/src/hipe_unified_loader.erl +++ b/lib/kernel/src/hipe_unified_loader.erl @@ -827,7 +827,6 @@ patch_to_emu_step1(Mod) -> %% were added as the result of dynamic apply calls. We must %% purge them too, but we have no explicit record of them. %% Therefore invalidate all native addresses for the module. - %% emu_make_stubs/1 will repair the ones for compiled static calls. hipe_bifs:invalidate_funinfo_native_addresses(MFAs), %% Find all call sites that call these MFAs. As a side-effect, %% create native stubs for any MFAs that are referred. @@ -841,7 +840,6 @@ patch_to_emu_step1(Mod) -> %% Step 2 must occur after the new BEAM stub module is created. patch_to_emu_step2(ReferencesToPatch) -> - emu_make_stubs(ReferencesToPatch), redirect(ReferencesToPatch). -spec is_loaded(Module::atom()) -> boolean(). @@ -852,21 +850,6 @@ is_loaded(M) when is_atom(M) -> catch _:_ -> false end. --ifdef(notdef). -emu_make_stubs([{MFA,_Refs}|Rest]) -> - make_stub(MFA), - emu_make_stubs(Rest); -emu_make_stubs([]) -> - []. - -make_stub({_,_,A} = MFA) -> - EmuAddress = hipe_bifs:get_emu_address(MFA), - StubAddress = hipe_bifs:make_native_stub(EmuAddress, A), - hipe_bifs:set_funinfo_native_address(MFA, StubAddress). --else. -emu_make_stubs(_) -> []. --endif. - %%-------------------------------------------------------------------- %% Given a list of MFAs, tag them with their referred_from references. %% The resulting {MFA,Refs} list is later passed to redirect/1, once diff --git a/lib/mnesia/doc/src/Mnesia_chap5.xmlsrc b/lib/mnesia/doc/src/Mnesia_chap5.xmlsrc index 65b950bd46..127c23e0f7 100644 --- a/lib/mnesia/doc/src/Mnesia_chap5.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_chap5.xmlsrc @@ -867,6 +867,7 @@ ok </section> <section> + <marker id="event_handling"></marker> <title>Mnesia Event Handling</title> <p>System events and table events are the two categories of events that Mnesia will generate in various situations. diff --git a/lib/mnesia/doc/src/mnesia.xml b/lib/mnesia/doc/src/mnesia.xml index c23c2cb226..856a7594a7 100644 --- a/lib/mnesia/doc/src/mnesia.xml +++ b/lib/mnesia/doc/src/mnesia.xml @@ -863,7 +863,7 @@ mnesia:create_table(person, {attributes, record_info(fields,person)}]). </code> <p>The specification of <c>index</c> and <c>attributes</c> may be - hard coded as <c>{index, [2]}</c> and + hard coded as <c>{index, [4]}</c> and <c>{attributes, [name, age, address, salary, children]}</c> respectively. </p> @@ -2188,12 +2188,13 @@ mnesia:create_table(employee, </desc> </func> <func> - <name>subscribe(EventCategory)</name> + <name>subscribe(EventCategory) -> {ok, Node} | {error, Reason} </name> <fsummary>Subscribe to events of type <c>EventCategory</c>.</fsummary> <desc> <p>Ensures that a copy of all events of type <c>EventCategory</c> are sent to the caller. The event - types available are described in the Mnesia User's Guide.</p> + types available are described in the Mnesia User's Guide at <seealso marker="Mnesia_chap5#event_handling">Mnesia Event Handling</seealso>.</p> + <p><c>Node</c> is the local node. For table events to be subscribed, mnesia must have a readable local copy of the table on the node.</p> </desc> </func> <func> @@ -2861,11 +2862,12 @@ raise(Name, Amount) -> </desc> </func> <func> - <name>unsubscribe(EventCategory)</name> + <name>unsubscribe(EventCategory) -> {ok, Node} | {error, Reason} </name> <fsummary>Subscribe to events of type <c>EventCategory</c>.</fsummary> <desc> <p>Stops sending events of type <c>EventCategory</c> to the caller.</p> + <p><c>Node</c> is the local node.</p> </desc> </func> <func> @@ -3017,6 +3019,12 @@ raise(Name, Amount) -> totally unpredictable.</p> </item> <item> + <p><c>-mnesia dump_disc_copies_at_startup true | false</c>. + If set to false, this disables the dumping of <c>disc_copies</c> + tables during startup while tables are being loaded. The default + is true.</p> + </item> + <item> <p><c>-mnesia dump_log_load_regulation true | false</c>. Controls if the log dumps should be performed as fast as possible or if the dumper should do its own load diff --git a/lib/mnesia/src/mnesia_controller.erl b/lib/mnesia/src/mnesia_controller.erl index 5a9bae54da..aa72de7594 100644 --- a/lib/mnesia/src/mnesia_controller.erl +++ b/lib/mnesia/src/mnesia_controller.erl @@ -51,6 +51,7 @@ force_load_table/1, async_dump_log/1, sync_dump_log/1, + snapshot_dcd/1, connect_nodes/1, connect_nodes/2, wait_for_schema_commit_lock/0, @@ -139,7 +140,8 @@ max_loaders() -> -record(block_controller, {owner}). -record(dump_log, {initiated_by, - opt_reply_to + opt_reply_to, + operation = dump_log }). -record(net_load, {table, @@ -201,6 +203,15 @@ async_dump_log(InitBy) -> ?SERVER_NAME ! {async_dump_log, InitBy}, ok. +snapshot_dcd(Tables) when is_list(Tables) -> + case [T || T <- Tables, + mnesia_lib:storage_type_at_node(node(), T) =/= disc_copies] of + [] -> + call({snapshot_dcd, Tables}); + BadTabs -> + {error, {not_disc_copies, BadTabs}} + end. + %% Wait for tables to be active %% If needed, we will wait for Mnesia to start %% If Mnesia stops, we will wait for Mnesia to restart @@ -646,6 +657,15 @@ handle_call({sync_dump_log, InitBy}, From, State) -> State2 = add_worker(Worker, State), noreply(State2); +handle_call({snapshot_dcd, Tables}, From, State) -> + Worker = #dump_log{initiated_by = user, + opt_reply_to = From, + operation = fun() -> + mnesia_dumper:snapshot_dcd(Tables) + end}, + State2 = add_worker(Worker, State), + noreply(State2); + handle_call(wait_for_schema_commit_lock, From, State) -> Worker = #schema_commit_lock{owner = From}, State2 = add_worker(Worker, State), @@ -2089,7 +2109,12 @@ start_remote_sender(Node, Tab, Receiver, Storage) -> dump_and_reply(ReplyTo, Worker) -> %% No trap_exit, die intentionally instead - Res = mnesia_dumper:opt_dump_log(Worker#dump_log.initiated_by), + Res = case Worker#dump_log.operation of + dump_log -> + mnesia_dumper:opt_dump_log(Worker#dump_log.initiated_by); + F when is_function(F, 0) -> + F() + end, ReplyTo ! #dumper_done{worker_pid = self(), worker_res = Res}, unlink(ReplyTo), diff --git a/lib/mnesia/src/mnesia_dumper.erl b/lib/mnesia/src/mnesia_dumper.erl index 14665797a0..509b765dee 100644 --- a/lib/mnesia/src/mnesia_dumper.erl +++ b/lib/mnesia/src/mnesia_dumper.erl @@ -34,11 +34,13 @@ -export([ get_log_writes/0, incr_log_writes/0, + needs_dump_ets/1, raw_dump_table/2, raw_named_dump_table/2, start_regulator/0, opt_dump_log/1, - update/3 + update/3, + snapshot_dcd/1 ]). %% Internal stuff @@ -99,6 +101,19 @@ opt_dump_log(InitBy) -> end, perform_dump(InitBy, Reg). +snapshot_dcd(Tables) -> + lists:foreach( + fun(Tab) -> + case mnesia_lib:storage_type_at_node(node(), Tab) of + disc_copies -> + mnesia_log:ets2dcd(Tab); + _ -> + %% Storage type was checked before queueing the op, though + skip + end + end, Tables), + dumped. + %% Scan for decisions perform_dump(InitBy, Regulator) when InitBy == scan_decisions -> ?eval_debug_fun({?MODULE, perform_dump}, [InitBy]), @@ -981,28 +996,10 @@ open_files(_Tab, _Storage, _UpdateInPlace, _InitBy) -> false. open_disc_copies(Tab, InitBy) -> - DclF = mnesia_lib:tab2dcl(Tab), - DumpEts = - case file:read_file_info(DclF) of - {error, enoent} -> - false; - {ok, DclInfo} -> - DcdF = mnesia_lib:tab2dcd(Tab), - case file:read_file_info(DcdF) of - {error, Reason} -> - mnesia_lib:dbg_out("File ~p info_error ~p ~n", - [DcdF, Reason]), - true; - {ok, DcdInfo} -> - Mul = case ?catch_val(dc_dump_limit) of - {'EXIT', _} -> ?DumpToEtsMultiplier; - Val -> Val - end, - DcdInfo#file_info.size =< (DclInfo#file_info.size * Mul) - end - end, + DumpEts = needs_dump_ets(Tab), if DumpEts == false; InitBy == startup -> + DclF = mnesia_lib:tab2dcl(Tab), mnesia_log:open_log({?MODULE,Tab}, mnesia_log:dcl_log_header(), DclF, @@ -1017,6 +1014,27 @@ open_disc_copies(Tab, InitBy) -> false end. +needs_dump_ets(Tab) -> + DclF = mnesia_lib:tab2dcl(Tab), + case file:read_file_info(DclF) of + {error, enoent} -> + false; + {ok, DclInfo} -> + DcdF = mnesia_lib:tab2dcd(Tab), + case file:read_file_info(DcdF) of + {error, Reason} -> + mnesia_lib:dbg_out("File ~p info_error ~p ~n", + [DcdF, Reason]), + true; + {ok, DcdInfo} -> + Mul = case ?catch_val(dc_dump_limit) of + {'EXIT', _} -> ?DumpToEtsMultiplier; + Val -> Val + end, + DcdInfo#file_info.size =< (DclInfo#file_info.size * Mul) + end + end. + %% Always opens the dcl file for writing overriding already_dumped %% mechanismen, used for schema transactions. open_dcl(Tab) -> diff --git a/lib/mnesia/src/mnesia_loader.erl b/lib/mnesia/src/mnesia_loader.erl index 530317bcdd..cbb3d7e430 100644 --- a/lib/mnesia/src/mnesia_loader.erl +++ b/lib/mnesia/src/mnesia_loader.erl @@ -69,9 +69,10 @@ do_get_disc_copy2(Tab, Reason, Storage, Type) when Storage == disc_copies -> ignore; _ -> mnesia_monitor:mktab(Tab, Args), - Count = mnesia_log:dcd2ets(Tab, Repair), - case ets:info(Tab, size) of - X when X < Count * 4 -> + _Count = mnesia_log:dcd2ets(Tab, Repair), + case mnesia_monitor:get_env(dump_disc_copies_at_startup) + andalso mnesia_dumper:needs_dump_ets(Tab) of + true -> ok = mnesia_log:ets2dcd(Tab); _ -> ignore diff --git a/lib/mnesia/src/mnesia_monitor.erl b/lib/mnesia/src/mnesia_monitor.erl index 6fc1a394a6..a0e0e630ec 100644 --- a/lib/mnesia/src/mnesia_monitor.erl +++ b/lib/mnesia/src/mnesia_monitor.erl @@ -664,6 +664,7 @@ env() -> backup_module, debug, dir, + dump_disc_copies_at_startup, dump_log_load_regulation, dump_log_time_threshold, dump_log_update_in_place, @@ -692,6 +693,8 @@ default_env(debug) -> default_env(dir) -> Name = lists:concat(["Mnesia.", node()]), filename:absname(Name); +default_env(dump_disc_copies_at_startup) -> + true; default_env(dump_log_load_regulation) -> false; default_env(dump_log_time_threshold) -> @@ -741,6 +744,7 @@ do_check_type(debug, trace) -> trace; do_check_type(debug, true) -> debug; do_check_type(debug, verbose) -> verbose; do_check_type(dir, V) -> filename:absname(V); +do_check_type(dump_disc_copies_at_startup, B) -> bool(B); do_check_type(dump_log_load_regulation, B) -> bool(B); do_check_type(dump_log_time_threshold, I) when is_integer(I), I > 0 -> I; do_check_type(dump_log_update_in_place, B) -> bool(B); diff --git a/lib/mnesia/src/mnesia_recover.erl b/lib/mnesia/src/mnesia_recover.erl index b6492707e2..eeb4fa0ced 100644 --- a/lib/mnesia/src/mnesia_recover.erl +++ b/lib/mnesia/src/mnesia_recover.erl @@ -689,12 +689,29 @@ handle_call({connect_nodes, Ns}, From, State) -> %% called from handle_info gen_server:reply(From, {[], AlreadyConnected}), {noreply, State}; - GoodNodes -> + ProbablyGoodNodes -> %% Now we have agreed upon a protocol with some new nodes - %% and we may use them when we recover transactions + %% and we may use them when we recover transactions. + %% + %% Just in case Mnesia was stopped on some of those nodes + %% between the protocol negotiation and now, we check one + %% more time the state of Mnesia. + %% + %% Of course, there is still a chance that mnesia_down + %% events occur during this check and we miss them. To + %% prevent it, handle_cast({mnesia_down, ...}, ...) removes + %% the down node again, in addition to mnesia_down/1. + %% + %% See a comment in handle_cast({mnesia_down, ...}, ...). + Verify = fun(N) -> + Run = mnesia_lib:is_running(N), + Run =:= yes orelse Run =:= starting + end, + GoodNodes = [N || N <- ProbablyGoodNodes, Verify(N)], + mnesia_lib:add_list(recover_nodes, GoodNodes), cast({announce_all, GoodNodes}), - case get_master_nodes(schema) of + case get_master_nodes(schema) of [] -> Context = starting_partitioned_network, mnesia_monitor:detect_inconcistency(GoodNodes, Context); @@ -842,6 +859,14 @@ handle_cast({what_decision, Node, OtherD}, State) -> {noreply, State}; handle_cast({mnesia_down, Node}, State) -> + %% The node was already removed from recover_nodes in mnesia_down/1, + %% but we do it again here in the mnesia_recover process, in case + %% another event incorrectly added it back. This can happen during + %% Mnesia startup which takes time betweenthe connection, the + %% protocol negotiation and the merge of the schema. + %% + %% See a comment in handle_call({connect_nodes, ...), ...). + mnesia_lib:del(recover_nodes, Node), case State#state.unclear_decision of undefined -> {noreply, State}; diff --git a/lib/ssh/src/ssh_connection_handler.erl b/lib/ssh/src/ssh_connection_handler.erl index 915060c426..68523aa72b 100644 --- a/lib/ssh/src/ssh_connection_handler.erl +++ b/lib/ssh/src/ssh_connection_handler.erl @@ -1711,7 +1711,7 @@ handshake(Pid, Ref, Timeout) -> {error, Reason} after Timeout -> stop(Pid), - {error, Timeout} + {error, timeout} end. start_timeout(_,_, infinity) -> diff --git a/lib/ssh/test/ssh_basic_SUITE.erl b/lib/ssh/test/ssh_basic_SUITE.erl index 415cb9fc9c..cb1b4ae945 100644 --- a/lib/ssh/test/ssh_basic_SUITE.erl +++ b/lib/ssh/test/ssh_basic_SUITE.erl @@ -723,7 +723,7 @@ ssh_connect_arg4_timeout(_Config) -> %% Wait for client reaction on the connection try: receive - {done, Client, {error,_E}, T0} -> + {done, Client, {error,timeout}, T0} -> Msp = ms_passed(T0, now()), exit(Server,hasta_la_vista___baby), Low = 0.9*Timeout, @@ -733,6 +733,11 @@ ssh_connect_arg4_timeout(_Config) -> Low<Msp, Msp<High -> ok; true -> {fail, "timeout not within limits"} end; + + {done, Client, {error,Other}, _T0} -> + ct:log("Error message \"~p\" from the client is unexpected.",[{error,Other}]), + {fail, "Unexpected error message"}; + {done, Client, {ok,_Ref}, _T0} -> {fail,"ssh-connected ???"} after diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index d77c6440b0..9e6d294f09 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -165,7 +165,7 @@ is supplied it will override the certfile option.</item> <tag>{certfile, path()}</tag> - <item>Path to a file containing the user's certificate.</item> + <item>Path to a file containing the user's PEM encoded certificate.</item> <tag>{key, {'RSAPrivateKey'| 'DSAPrivateKey' | 'ECPrivateKey' |'PrivateKeyInfo', der_encoded()}}</tag> <item> The DER encoded users private key. If this option @@ -552,7 +552,19 @@ fun(srp, Username :: string(), UserState :: term()) -> </p> </desc> </func> - + + <func> + <name>clear_pem_cache() -> ok </name> + <fsummary> Clears the pem cache</fsummary> + + <desc><p>PEM files, used by ssl API-functions, are cached. The + cache is regularly checked to see if any cache entries should be + invalidated, however this function provides a way to + unconditionally clear the whole cache. + </p> + </desc> + </func> + <func> <name>connect(Socket, SslOptions) -> </name> <name>connect(Socket, SslOptions, Timeout) -> {ok, SslSocket} diff --git a/lib/ssl/doc/src/ssl_app.xml b/lib/ssl/doc/src/ssl_app.xml index c8024548b5..e3a3fc27f2 100644 --- a/lib/ssl/doc/src/ssl_app.xml +++ b/lib/ssl/doc/src/ssl_app.xml @@ -4,7 +4,7 @@ <appref> <header> <copyright> - <year>1999</year><year>2014</year> + <year>1999</year><year>2015</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -82,7 +82,16 @@ callback module, defaults to []. </p> </item> - + + <tag><c><![CDATA[ssl_pem_cache_clean = integer() <optional>]]></c></tag> + <item> + <p> + Number of milliseconds between PEM cache validations. + </p> + <seealso + marker="ssl#clear_pem_cache-0">ssl:clear_pem_cache/0</seealso> + + </item> </taglist> </section> diff --git a/lib/ssl/src/ssl.appup.src b/lib/ssl/src/ssl.appup.src index 7986722094..1476336039 100644 --- a/lib/ssl/src/ssl.appup.src +++ b/lib/ssl/src/ssl.appup.src @@ -1,14 +1,14 @@ %% -*- erlang -*- {"%VSN%", [ - {<<"5\\.3\\.[1-7]($|\\..*)">>, [{restart_application, ssl}]}, - {<<"5\\.[0-2]($|\\..*)">>, [{restart_application, ssl}]}, + {<<"6\\..*">>, [{restart_application, ssl}]}, + {<<"5\\..*">>, [{restart_application, ssl}]}, {<<"4\\..*">>, [{restart_application, ssl}]}, {<<"3\\..*">>, [{restart_application, ssl}]} ], [ - {<<"5\\.3\\.[1-7]($|\\..*)">>, [{restart_application, ssl}]}, - {<<"5\\.[0-2]($|\\..*)">>, [{restart_application, ssl}]}, + {<<"6\\..*">>, [{restart_application, ssl}]}, + {<<"5\\..*">>, [{restart_application, ssl}]}, {<<"4\\..*">>, [{restart_application, ssl}]}, {<<"3\\..*">>, [{restart_application, ssl}]} ] diff --git a/lib/ssl/src/ssl_certificate.erl b/lib/ssl/src/ssl_certificate.erl index 9c0ed181fe..30d224fee2 100644 --- a/lib/ssl/src/ssl_certificate.erl +++ b/lib/ssl/src/ssl_certificate.erl @@ -282,7 +282,7 @@ other_issuer(OtpCert, CertDbHandle) -> handle_path({BinCert, OTPCert}, Path, PartialChainHandler) -> case public_key:pkix_is_self_signed(OTPCert) of true -> - {BinCert, Path}; + {BinCert, lists:delete(BinCert, Path)}; false -> handle_incomplete_chain(Path, PartialChainHandler) end. diff --git a/lib/ssl/src/ssl_manager.erl b/lib/ssl/src/ssl_manager.erl index 5553fc9220..bf0333ba8d 100644 --- a/lib/ssl/src/ssl_manager.erl +++ b/lib/ssl/src/ssl_manager.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2014. All Rights Reserved. +%% Copyright Ericsson AB 2007-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -30,10 +30,10 @@ lookup_trusted_cert/4, new_session_id/1, clean_cert_db/2, register_session/2, register_session/3, invalidate_session/2, - invalidate_session/3, clear_pem_cache/0, manager_name/1]). + invalidate_session/3, invalidate_pem/1, clear_pem_cache/0, manager_name/1]). % Spawn export --export([init_session_validator/1]). +-export([init_session_validator/1, init_pem_cache_validator/1]). %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, @@ -50,7 +50,9 @@ session_lifetime, certificate_db, session_validation_timer, - last_delay_timer = {undefined, undefined}%% Keep for testing purposes + last_delay_timer = {undefined, undefined},%% Keep for testing purposes + last_pem_check, + clear_pem_cache }). -define('24H_in_msec', 86400000). @@ -118,14 +120,13 @@ connection_init(Trustedcerts, Role) -> %% Description: Cache a pem file and return its content. %%-------------------------------------------------------------------- cache_pem_file(File, DbHandle) -> - MD5 = crypto:hash(md5, File), - case ssl_pkix_db:lookup_cached_pem(DbHandle, MD5) of + case ssl_pkix_db:lookup_cached_pem(DbHandle, File) of [{Content,_}] -> {ok, Content}; [Content] -> {ok, Content}; undefined -> - call({cache_pem, {MD5, File}}) + call({cache_pem, File}) end. %%-------------------------------------------------------------------- @@ -192,6 +193,11 @@ invalidate_session(Host, Port, Session) -> invalidate_session(Port, Session) -> cast({invalidate_session, Port, Session}). + +-spec invalidate_pem(File::binary()) -> ok. +invalidate_pem(File) -> + cast({invalidate_pem, File}). + %%==================================================================== %% gen_server callbacks %%==================================================================== @@ -216,13 +222,17 @@ init([Name, Opts]) -> proplists:get_value(session_cb_init_args, Opts, [])]), Timer = erlang:send_after(SessionLifeTime * 1000 + 5000, self(), validate_sessions), - erlang:send_after(?CLEAR_PEM_CACHE, self(), clear_pem_cache), + Interval = pem_check_interval(), + erlang:send_after(Interval, self(), clear_pem_cache), {ok, #state{certificate_db = CertDb, session_cache_client = ClientSessionCache, session_cache_server = ServerSessionCache, session_cache_cb = CacheCb, session_lifetime = SessionLifeTime, - session_validation_timer = Timer}}. + session_validation_timer = Timer, + last_pem_check = os:timestamp(), + clear_pem_cache = Interval + }}. %%-------------------------------------------------------------------- -spec handle_call(msg(), from(), #state{}) -> {reply, reply(), #state{}}. @@ -279,7 +289,7 @@ handle_call({{new_session_id,Port}, _}, {reply, Id, State}; -handle_call({{cache_pem, File}, _Pid}, _, +handle_call({{cache_pem,File}, _Pid}, _, #state{certificate_db = Db} = State) -> try ssl_pkix_db:cache_pem_file(File, Db) of Result -> @@ -332,7 +342,12 @@ handle_cast({invalidate_session, Host, Port, handle_cast({invalidate_session, Port, #session{session_id = ID} = Session}, #state{session_cache_server = Cache, session_cache_cb = CacheCb} = State) -> - invalidate_session(Cache, CacheCb, {Port, ID}, Session, State). + invalidate_session(Cache, CacheCb, {Port, ID}, Session, State); + +handle_cast({invalidate_pem, File}, + #state{certificate_db = [_, _, PemCache]} = State) -> + ssl_pkix_db:remove(File, PemCache), + {noreply, State}. %%-------------------------------------------------------------------- -spec handle_info(msg(), #state{}) -> {noreply, #state{}}. @@ -353,20 +368,19 @@ handle_info(validate_sessions, #state{session_cache_cb = CacheCb, start_session_validator(ServerCache, CacheCb, LifeTime), {noreply, State#state{session_validation_timer = Timer}}; + handle_info({delayed_clean_session, Key, Cache}, #state{session_cache_cb = CacheCb } = State) -> CacheCb:delete(Cache, Key), {noreply, State}; -handle_info(clear_pem_cache, #state{certificate_db = [_,_,PemChace]} = State) -> - case ssl_pkix_db:db_size(PemChace) of - N when N < ?NOT_TO_BIG -> - ok; - _ -> - ssl_pkix_db:clear(PemChace) - end, - erlang:send_after(?CLEAR_PEM_CACHE, self(), clear_pem_cache), - {noreply, State}; +handle_info(clear_pem_cache, #state{certificate_db = [_,_,PemChace], + clear_pem_cache = Interval, + last_pem_check = CheckPoint} = State) -> + NewCheckPoint = os:timestamp(), + start_pem_cache_validator(PemChace, CheckPoint), + erlang:send_after(Interval, self(), clear_pem_cache), + {noreply, State#state{last_pem_check = NewCheckPoint}}; handle_info({clean_cert_db, Ref, File}, @@ -514,10 +528,9 @@ new_id(Port, Tries, Cache, CacheCb) -> clean_cert_db(Ref, CertDb, RefDb, PemCache, File) -> case ssl_pkix_db:ref_count(Ref, RefDb, 0) of 0 -> - MD5 = crypto:hash(md5, File), - case ssl_pkix_db:lookup_cached_pem(PemCache, MD5) of + case ssl_pkix_db:lookup_cached_pem(PemCache, File) of [{Content, Ref}] -> - ssl_pkix_db:insert(MD5, Content, PemCache); + ssl_pkix_db:insert(File, Content, PemCache); _ -> ok end, @@ -557,3 +570,39 @@ exists_equivalent(#session{ true; exists_equivalent(Session, [ _ | Rest]) -> exists_equivalent(Session, Rest). + +start_pem_cache_validator(PemCache, CheckPoint) -> + spawn_link(?MODULE, init_pem_cache_validator, + [[get(ssl_manager), PemCache, CheckPoint]]). + +init_pem_cache_validator([SslManagerName, PemCache, CheckPoint]) -> + put(ssl_manager, SslManagerName), + ssl_pkix_db:foldl(fun pem_cache_validate/2, + CheckPoint, PemCache). + +pem_cache_validate({File, _}, CheckPoint) -> + case file:read_file_info(File, []) of + {ok, #file_info{mtime = Time}} -> + case is_before_checkpoint(Time, CheckPoint) of + true -> + ok; + false -> + invalidate_pem(File) + end; + _ -> + invalidate_pem(File) + end, + CheckPoint. + +pem_check_interval() -> + case application:get_env(ssl, ssl_pem_cache_clean) of + {ok, Interval} when is_integer(Interval) -> + Interval; + _ -> + ?CLEAR_PEM_CACHE + end. + +is_before_checkpoint(Time, CheckPoint) -> + calendar:datetime_to_gregorian_seconds(calendar:now_to_datetime(CheckPoint)) - + calendar:datetime_to_gregorian_seconds(Time) > 0. + diff --git a/lib/ssl/src/ssl_pkix_db.erl b/lib/ssl/src/ssl_pkix_db.erl index e59aba0618..8531445ba4 100644 --- a/lib/ssl/src/ssl_pkix_db.erl +++ b/lib/ssl/src/ssl_pkix_db.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2013. All Rights Reserved. +%% Copyright Ericsson AB 2007-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -81,10 +81,10 @@ lookup_trusted_cert(DbHandle, Ref, SerialNumber, Issuer) -> {ok, Certs} end. -lookup_cached_pem([_, _, PemChache], MD5) -> - lookup_cached_pem(PemChache, MD5); -lookup_cached_pem(PemChache, MD5) -> - lookup(MD5, PemChache). +lookup_cached_pem([_, _, PemChache], File) -> + lookup_cached_pem(PemChache, File); +lookup_cached_pem(PemChache, File) -> + lookup(File, PemChache). %%-------------------------------------------------------------------- -spec add_trusted_certs(pid(), {erlang:timestamp(), string()} | @@ -100,36 +100,35 @@ add_trusted_certs(_Pid, {der, DerList}, [CerDb, _,_]) -> {ok, NewRef}; add_trusted_certs(_Pid, File, [CertsDb, RefDb, PemChache] = Db) -> - MD5 = crypto:hash(md5, File), - case lookup_cached_pem(Db, MD5) of + case lookup_cached_pem(Db, File) of [{_Content, Ref}] -> ref_count(Ref, RefDb, 1), {ok, Ref}; [Content] -> Ref = make_ref(), update_counter(Ref, 1, RefDb), - insert(MD5, {Content, Ref}, PemChache), + insert(File, {Content, Ref}, PemChache), add_certs_from_pem(Content, Ref, CertsDb), {ok, Ref}; undefined -> - new_trusted_cert_entry({MD5, File}, Db) + new_trusted_cert_entry(File, Db) end. %%-------------------------------------------------------------------- %% %% Description: Cache file as binary in DB %%-------------------------------------------------------------------- --spec cache_pem_file({binary(), binary()}, [db_handle()]) -> {ok, term()}. -cache_pem_file({MD5, File}, [_CertsDb, _RefDb, PemChache]) -> +-spec cache_pem_file(binary(), [db_handle()]) -> {ok, term()}. +cache_pem_file(File, [_CertsDb, _RefDb, PemChache]) -> {ok, PemBin} = file:read_file(File), Content = public_key:pem_decode(PemBin), - insert(MD5, Content, PemChache), + insert(File, Content, PemChache), {ok, Content}. --spec cache_pem_file(reference(), {binary(), binary()}, [db_handle()]) -> {ok, term()}. -cache_pem_file(Ref, {MD5, File}, [_CertsDb, _RefDb, PemChache]) -> +-spec cache_pem_file(reference(), binary(), [db_handle()]) -> {ok, term()}. +cache_pem_file(Ref, File, [_CertsDb, _RefDb, PemChache]) -> {ok, PemBin} = file:read_file(File), Content = public_key:pem_decode(PemBin), - insert(MD5, {Content, Ref}, PemChache), + insert(File, {Content, Ref}, PemChache), {ok, Content}. %%-------------------------------------------------------------------- @@ -245,9 +244,9 @@ add_certs(Cert, Ref, CertsDb) -> error_logger:info_report(Report) end. -new_trusted_cert_entry(FileRef, [CertsDb, RefDb, _] = Db) -> +new_trusted_cert_entry(File, [CertsDb, RefDb, _] = Db) -> Ref = make_ref(), update_counter(Ref, 1, RefDb), - {ok, Content} = cache_pem_file(Ref, FileRef, Db), + {ok, Content} = cache_pem_file(Ref, File, Db), add_certs_from_pem(Content, Ref, CertsDb), {ok, Ref}. diff --git a/lib/ssl/test/Makefile b/lib/ssl/test/Makefile index 3639c2b2da..09cc5981e7 100644 --- a/lib/ssl/test/Makefile +++ b/lib/ssl/test/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1999-2013. All Rights Reserved. +# Copyright Ericsson AB 1999-2015. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -47,9 +47,11 @@ MODULES = \ ssl_npn_handshake_SUITE \ ssl_packet_SUITE \ ssl_payload_SUITE \ + ssl_pem_cache_SUITE \ ssl_session_cache_SUITE \ ssl_to_openssl_SUITE \ ssl_ECC_SUITE \ + ssl_upgrade_SUITE\ make_certs\ erl_make_certs diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index 723562c459..77ef8088b4 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -259,11 +259,6 @@ init_per_testcase(Case, Config) when Case == unordered_protocol_versions_client _ -> {skip, "TLS 1.2 need but not supported on this platform"} end; -init_per_testcase(no_authority_key_identifier, Config) -> - %% Clear cach so that root cert will not - %% be found. - ssl:clear_pem_cache(), - Config; init_per_testcase(protocol_versions, Config) -> ssl:stop(), diff --git a/lib/ssl/test/ssl_certificate_verify_SUITE.erl b/lib/ssl/test/ssl_certificate_verify_SUITE.erl index b7864ba6e7..dab7a941db 100644 --- a/lib/ssl/test/ssl_certificate_verify_SUITE.erl +++ b/lib/ssl/test/ssl_certificate_verify_SUITE.erl @@ -443,7 +443,7 @@ verify_fun_always_run_client(Config) when is_list(Config) -> {unknown, UserState}; (_, valid, [ChainLen]) -> {valid, [ChainLen + 1]}; - (_, valid_peer, [2]) -> + (_, valid_peer, [1]) -> {fail, "verify_fun_was_always_run"}; (_, valid_peer, UserState) -> {valid, UserState} @@ -482,7 +482,7 @@ verify_fun_always_run_server(Config) when is_list(Config) -> {unknown, UserState}; (_, valid, [ChainLen]) -> {valid, [ChainLen + 1]}; - (_, valid_peer, [2]) -> + (_, valid_peer, [1]) -> {fail, "verify_fun_was_always_run"}; (_, valid_peer, UserState) -> {valid, UserState} diff --git a/lib/ssl/test/ssl_pem_cache_SUITE.erl b/lib/ssl/test/ssl_pem_cache_SUITE.erl new file mode 100644 index 0000000000..843079e2fe --- /dev/null +++ b/lib/ssl/test/ssl_pem_cache_SUITE.erl @@ -0,0 +1,127 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2015-2015. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/.2 +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%% + +-module(ssl_pem_cache_SUITE). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("kernel/include/file.hrl"). + +-define(CLEANUP_INTERVAL, 5000). + +%%-------------------------------------------------------------------- +%% Common Test interface functions ----------------------------------- +%%-------------------------------------------------------------------- +all() -> + [pem_cleanup]. + +groups() -> + []. + +init_per_suite(Config0) -> + catch crypto:stop(), + try crypto:start() of + ok -> + ssl:start(), + %% make rsa certs using oppenssl + Result = + (catch make_certs:all(?config(data_dir, Config0), + ?config(priv_dir, Config0))), + ct:log("Make certs ~p~n", [Result]), + + Config1 = ssl_test_lib:make_dsa_cert(Config0), + ssl_test_lib:cert_options(Config1) + catch _:_ -> + {skip, "Crypto did not start"} + end. + +end_per_suite(_Config) -> + application:stop(crypto). + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + +init_per_testcase(pem_cleanup, Config) -> + ssl:stop(), + application:load(ssl), + application:set_env(ssl, ssl_pem_cache_clean, ?CLEANUP_INTERVAL), + ssl:start(), + Config. + +end_per_testcase(_TestCase, Config) -> + %%ssl:stop(), + Config. + +%%-------------------------------------------------------------------- +%% Test Cases -------------------------------------------------------- +%%-------------------------------------------------------------------- +pem_cleanup() -> + [{doc, "Test pem cache invalidate mechanism"}]. +pem_cleanup(Config)when is_list(Config) -> + process_flag(trap_exit, true), + ClientOpts = ?config(client_opts, Config), + ServerOpts = ?config(server_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Server = + ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, no_result, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + Client = + ssl_test_lib:start_client([{node, ClientNode}, + {port, Port}, {host, Hostname}, + {mfa, {ssl_test_lib, no_result, []}}, + {from, self()}, {options, ClientOpts}]), + + Size = ssl_pkix_db:db_size(get_pem_cache()), + Certfile = proplists:get_value(certfile, ServerOpts), + {ok, FileInfo} = file:read_file_info(Certfile), + Time = later(), + ok = file:write_file_info(Certfile, FileInfo#file_info{mtime = Time}), + ct:sleep(2 * ?CLEANUP_INTERVAL), + Size1 = ssl_pkix_db:db_size(get_pem_cache()), + ssl_test_lib:close(Server), + ssl_test_lib:close(Client), + false = Size == Size1. + +get_pem_cache() -> + {status, _, _, StatusInfo} = sys:get_status(whereis(ssl_manager)), + [_, _,_, _, Prop] = StatusInfo, + State = ssl_test_lib:state(Prop), + case element(5, State) of + [_CertDb, _FileRefDb, PemChace] -> + PemChace; + _ -> + undefined + end. + +later()-> + DateTime = calendar:now_to_local_time(os:timestamp()), + Gregorian = calendar:datetime_to_gregorian_seconds(DateTime), + calendar:gregorian_seconds_to_datetime(Gregorian + (2 * ?CLEANUP_INTERVAL)). + diff --git a/lib/ssl/test/ssl_upgrade_SUITE.erl b/lib/ssl/test/ssl_upgrade_SUITE.erl new file mode 100644 index 0000000000..6a6a1b4a7a --- /dev/null +++ b/lib/ssl/test/ssl_upgrade_SUITE.erl @@ -0,0 +1,162 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2014-2015. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/.2 +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(ssl_upgrade_SUITE). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +-record(state, { + config, + server, + client, + soft + }). + +all() -> + [ + minor_upgrade, + major_upgrade + ]. + +init_per_suite(Config0) -> + catch crypto:stop(), + try crypto:start() of + ok -> + case ct_release_test:init(Config0) of + {skip, Reason} -> + {skip, Reason}; + Config -> + Result = + (catch make_certs:all(?config(data_dir, Config), + ?config(priv_dir, Config))), + ct:log("Make certs ~p~n", [Result]), + ssl_test_lib:cert_options(Config) + end + catch _:_ -> + {skip, "Crypto did not start"} + end. + +end_per_suite(Config) -> + ct_release_test:cleanup(Config), + crypto:stop(). + +init_per_testcase(_TestCase, Config) -> + Config. +end_per_testcase(_TestCase, Config) -> + Config. + +major_upgrade(Config) when is_list(Config) -> + ct_release_test:upgrade(ssl, major,{?MODULE, #state{config = Config}}, Config). + +minor_upgrade(Config) when is_list(Config) -> + ct_release_test:upgrade(ssl, minor,{?MODULE, #state{config = Config}}, Config). + +upgrade_init(CTData, #state{config = Config} = State) -> + {ok, {_, _, Up, _Down}} = ct_release_test:get_appup(CTData, ssl), + ct:pal("Up: ~p", [Up]), + Soft = is_soft(Up), %% It is symmetrical, if upgrade is soft so is downgrade + case Soft of + true -> + {Server, Client} = soft_start_connection(Config), + State#state{server = Server, client = Client, + soft = Soft}; + false -> + State#state{soft = Soft} + end. + +upgrade_upgraded(_, #state{soft = false, config = Config} = State) -> + {Server, Client} = restart_start_connection(Config), + ssl_test_lib:check_result(Server, ok, Client, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close(Client), + State; + +upgrade_upgraded(_, #state{server = Server0, client = Client0, + config = Config, soft = true} = State) -> + Server0 ! changed_version, + Client0 ! changed_version, + ssl_test_lib:check_result(Server0, ok, Client0, ok), + ssl_test_lib:close(Server0), + ssl_test_lib:close(Client0), + {Server, Client} = soft_start_connection(Config), + State#state{server = Server, client = Client}. + +upgrade_downgraded(_, #state{soft = false, config = Config} = State) -> + {Server, Client} = restart_start_connection(Config), + ssl_test_lib:check_result(Server, ok, Client, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close(Client), + State; + +upgrade_downgraded(_, #state{server = Server, client = Client, soft = true} = State) -> + Server ! changed_version, + Client ! changed_version, + ssl_test_lib:check_result(Server, ok, Client, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close(Client), + State. + +use_connection(Socket) -> + ssl_test_lib:send_recv_result_active(Socket), + receive + changed_version -> + ssl_test_lib:send_recv_result_active(Socket) + end. + +soft_start_connection(Config) -> + ClientOpts = ?config(client_verification_opts, Config), + ServerOpts = ?config(server_verification_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, use_connection, []}}, + {options, ServerOpts}]), + + Port = ssl_test_lib:inet_port(Server), + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, use_connection, []}}, + {options, ClientOpts}]), + {Server, Client}. + +restart_start_connection(Config) -> + ClientOpts = ?config(client_verification_opts, Config), + ServerOpts = ?config(server_verification_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + + Port = ssl_test_lib:inet_port(Server), + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ClientOpts}]), + {Server, Client}. + +is_soft([{restart_application, ssl}]) -> + false; +is_soft(_) -> + true. + diff --git a/lib/ssl/vsn.mk b/lib/ssl/vsn.mk index bda974da0e..171147adf2 100644 --- a/lib/ssl/vsn.mk +++ b/lib/ssl/vsn.mk @@ -1 +1 @@ -SSL_VSN = 5.3.8 +SSL_VSN = 7.0 diff --git a/lib/stdlib/src/otp_internal.erl b/lib/stdlib/src/otp_internal.erl index 0ace87ef5c..4a338798d0 100644 --- a/lib/stdlib/src/otp_internal.erl +++ b/lib/stdlib/src/otp_internal.erl @@ -578,6 +578,19 @@ obsolete_1(asn1rt, utf8_binary_to_list, 1) -> obsolete_1(asn1rt, utf8_list_to_binary, 1) -> {deprecated,{unicode,characters_to_binary,1}}; +%% Added in OTP 18. +obsolete_1(core_lib, get_anno, 1) -> + {deprecated,{cerl,get_ann,1}}; +obsolete_1(core_lib, set_anno, 2) -> + {deprecated,{cerl,set_ann,2}}; +obsolete_1(core_lib, is_literal, 1) -> + {deprecated,{cerl,is_literal,1}}; +obsolete_1(core_lib, is_literal_list, 1) -> + {deprecated,"deprecated; use lists:all(fun cerl:is_literal/1, L)" + " instead"}; +obsolete_1(core_lib, literal_value, 1) -> + {deprecated,{core_lib,concrete,1}}; + obsolete_1(_, _, _) -> no. |