diff options
Diffstat (limited to 'lib')
257 files changed, 10304 insertions, 3158 deletions
diff --git a/lib/asn1/doc/src/notes.xml b/lib/asn1/doc/src/notes.xml index ae6660c143..1abe983221 100644 --- a/lib/asn1/doc/src/notes.xml +++ b/lib/asn1/doc/src/notes.xml @@ -32,6 +32,31 @@ <p>This document describes the changes made to the asn1 application.</p> +<section><title>Asn1 5.0.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + There was a issue with BER encoding and the + <c>undec_rest</c> option in generated decoders. An + exception could be thrown instead of returning an error + tuple.</p> + <p> + Own Id: OTP-14786 Aux Id: ERL-518 </p> + </item> + <item> + <p> + The asn1ct:test functions crashed on decoders generated + with options <c>no_ok_wrapper</c>, <c>undec_rest</c>.</p> + <p> + Own Id: OTP-14787 Aux Id: ERL-518 </p> + </item> + </list> + </section> + +</section> + <section><title>Asn1 5.0.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/asn1/vsn.mk b/lib/asn1/vsn.mk index ef83b9e3dc..4cd89089e9 100644 --- a/lib/asn1/vsn.mk +++ b/lib/asn1/vsn.mk @@ -1 +1 @@ -ASN1_VSN = 5.0.3 +ASN1_VSN = 5.0.4 diff --git a/lib/common_test/doc/src/ct.xml b/lib/common_test/doc/src/ct.xml index 1a3cfdb0c5..afd8741cd1 100644 --- a/lib/common_test/doc/src/ct.xml +++ b/lib/common_test/doc/src/ct.xml @@ -1060,6 +1060,42 @@ </desc> </func> + <func> + <name>remaining_test_procs() -> {TestProcs,SharedGL,OtherGLs}</name> + <fsummary>>This function will return the identity of test- and group + leader processes that are still running at the time of this call.</fsummary> + <type> + <v>TestProcs = [{pid(),GL}]</v> + <v>GL = pid()</v> + <v>SharedGL = pid()</v> + <v>OtherGLs = [pid()]</v> + </type> + <desc><marker id="remaining_test_procs-0"/> + <p>This function will return the identity of test- and group + leader processes that are still running at the time of this call. + <c>TestProcs</c> are processes in the system that have a Common Test IO + process as group leader. <c>SharedGL</c> is the central Common Test + IO process, responsible for printing to log files for configuration + functions and sequentially executing test cases. <c>OtherGLs</c> are + Common Test IO processes that print to log files for test cases + in parallel test case groups.</p> + <p>The process information returned by this function may be + used to locate and terminate remaining processes after tests have + finished executing. The function would typically by called from + Common Test Hook functions.</p> + <p>Note that processes that execute configuration functions or + test cases are never included in <c>TestProcs</c>. It is therefore safe + to use post configuration hook functions (such as post_end_per_suite, + post_end_per_group, post_end_per_testcase) to terminate all processes + in <c>TestProcs</c> that have the current group leader process as its group + leader.</p> + <p>Note also that the shared group leader (<c>SharedGL</c>) must never be + terminated by the user, only by Common Test. Group leader processes + for parallel test case groups (<c>OtherGLs</c>) may however be terminated + in post_end_per_group hook functions.</p> + </desc> + </func> + <func> <name>remove_config(Callback, Config) -> ok</name> <fsummary>Removes configuration variables (together with diff --git a/lib/common_test/doc/src/notes.xml b/lib/common_test/doc/src/notes.xml index b039023e0f..c6b928bb5d 100644 --- a/lib/common_test/doc/src/notes.xml +++ b/lib/common_test/doc/src/notes.xml @@ -33,6 +33,33 @@ <file>notes.xml</file> </header> +<section><title>Common_Test 1.15.3</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + A new function, <c>ct:remaining_test_procs/0</c>, returns + the identity of test- and group leader processes that are + still running at the time of the call.</p> + <p> + Own Id: OTP-13832</p> + </item> + <item> + <p> + A "latest test result" link is now displayed in the + footer of each test index page, which performs a jump to + the most recently generated test index. This is useful + for making quick comparisons of results between test runs + without having to traverse the log file tree.</p> + <p> + Own Id: OTP-14281</p> + </item> + </list> + </section> + +</section> + <section><title>Common_Test 1.15.2</title> <section><title>Improvements and New Features</title> diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 875301a8b2..69e371a30f 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -89,6 +89,8 @@ -export([get_target_name/1]). -export([parse_table/1, listenv/1]). +-export([remaining_test_procs/0]). + %%---------------------------------------------------------------------- %% Exported types %%---------------------------------------------------------------------- @@ -1474,3 +1476,36 @@ continue() -> %%% in order to let the test case proceed.</p> continue(TestCase) -> test_server:continue(TestCase). + + +%%%----------------------------------------------------------------- +%%% @spec remaining_test_procs() -> {TestProcs,SharedGL,OtherGLs} +%%% TestProcs = [{pid(),GL}] +%%% GL = SharedGL = pid() +%%% OtherGLs = [pid()] +%%% +%%% @doc <p>This function will return the identity of test- and group +%%% leader processes that are still running at the time of this call. +%%% TestProcs are processes in the system that have a Common Test IO +%%% process as group leader. SharedGL is the central Common Test +%%% IO process, responsible for printing to log files for configuration +%%% functions and sequentially executing test cases. OtherGLs are +%%% Common Test IO processes that print to log files for test cases +%%% in parallel test case groups.</p> +%%% <p>The process information returned by this function may be +%%% used to locate and terminate remaining processes after tests have +%%% finished executing. The function would typically by called from +%%% Common Test Hook functions.</p> +%%% <p>Note that processes that execute configuration functions or +%%% test cases are never included in TestProcs. It is therefore safe +%%% to use post configuration hook functions (such as post_end_per_suite, +%%% post_end_per_group, post_end_per_testcase) to terminate all processes +%%% in TestProcs that have the current group leader process as its group +%%% leader.</p> +%%% <p>Note also that the shared group leader (SharedGL) must never be +%%% terminated by the user, only by Common Test. Group leader processes +%%% for parallel test case groups (OtherGLs) may however be terminated +%%% in post_end_per_group hook functions.</p> +%%% +remaining_test_procs() -> + ct_util:remaining_test_procs(). diff --git a/lib/common_test/src/ct_config.erl b/lib/common_test/src/ct_config.erl index b3f983dd46..6c87b11f8d 100644 --- a/lib/common_test/src/ct_config.erl +++ b/lib/common_test/src/ct_config.erl @@ -81,6 +81,7 @@ start(Mode) -> do_start(Parent) -> process_flag(trap_exit,true), + ct_util:mark_process(), register(ct_config_server,self()), ct_util:create_table(?attr_table,bag,#ct_conf.key), {ok,StartDir} = file:get_cwd(), diff --git a/lib/common_test/src/ct_default_gl.erl b/lib/common_test/src/ct_default_gl.erl index d1b52e5f4f..9ae430c546 100644 --- a/lib/common_test/src/ct_default_gl.erl +++ b/lib/common_test/src/ct_default_gl.erl @@ -55,6 +55,7 @@ stop() -> init([ParentGL]) -> register(?MODULE, self()), + ct_util:mark_process(), {ok,#{parent_gl_pid => ParentGL, parent_gl_monitor => erlang:monitor(process,ParentGL)}}. diff --git a/lib/common_test/src/ct_event.erl b/lib/common_test/src/ct_event.erl index 1a0ee4f3cd..8b5bba7600 100644 --- a/lib/common_test/src/ct_event.erl +++ b/lib/common_test/src/ct_event.erl @@ -137,6 +137,7 @@ is_alive() -> %% this function is called to initialize the event handler. %%-------------------------------------------------------------------- init(RecvPids) -> + ct_util:mark_process(), %% RecvPids = [{RecvTag,Pid}] {ok,#state{receivers=RecvPids}}. diff --git a/lib/common_test/src/ct_gen_conn.erl b/lib/common_test/src/ct_gen_conn.erl index badb7c52ae..456bfd8bd1 100644 --- a/lib/common_test/src/ct_gen_conn.erl +++ b/lib/common_test/src/ct_gen_conn.erl @@ -186,9 +186,11 @@ end_log() -> do_within_time(Fun,Timeout) -> Self = self(), Silent = get(silent), - TmpPid = spawn_link(fun() -> put(silent,Silent), - R = Fun(), - Self ! {self(),R} + TmpPid = spawn_link(fun() -> + ct_util:mark_process(), + put(silent,Silent), + R = Fun(), + Self ! {self(),R} end), ConnPid = get(conn_pid), receive @@ -301,6 +303,7 @@ return({To,Ref},Result) -> init_gen(Parent,Opts) -> process_flag(trap_exit,true), + ct_util:mark_process(), put(silent,false), try (Opts#gen_opts.callback):init(Opts#gen_opts.name, Opts#gen_opts.address, diff --git a/lib/common_test/src/ct_hooks_lock.erl b/lib/common_test/src/ct_hooks_lock.erl index fea298e535..a82be288e1 100644 --- a/lib/common_test/src/ct_hooks_lock.erl +++ b/lib/common_test/src/ct_hooks_lock.erl @@ -78,6 +78,7 @@ release() -> %% @doc Initiates the server init(Id) -> + ct_util:mark_process(), {ok, #state{ id = Id }}. %% @doc Handling call messages diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl index 028c265420..9861b1e521 100644 --- a/lib/common_test/src/ct_logs.erl +++ b/lib/common_test/src/ct_logs.erl @@ -666,6 +666,7 @@ log_timestamp({MS,S,US}) -> logger(Parent, Mode, Verbosity) -> register(?MODULE,self()), + ct_util:mark_process(), %%! Below is a temporary workaround for the limitation of %%! max one test run per second. %%! ---> @@ -1004,6 +1005,7 @@ print_to_log(async, FromPid, Category, TCGL, Content, EscChars, State) -> if FromPid /= TCGL -> IoFun = create_io_fun(FromPid, CtLogFd, EscChars), fun() -> + ct_util:mark_process(), test_server:permit_io(TCGL, self()), %% Since asynchronous io gets can get buffered if @@ -1035,6 +1037,7 @@ print_to_log(async, FromPid, Category, TCGL, Content, EscChars, State) -> end; true -> fun() -> + ct_util:mark_process(), unexpected_io(FromPid, Category, ?MAX_IMPORTANCE, Content, CtLogFd, EscChars) end @@ -3002,6 +3005,7 @@ simulate() -> S = self(), Pid = spawn(fun() -> register(?MODULE,self()), + ct_util:mark_process(), S ! {self(),started}, simulate_logger_loop() end), @@ -3129,8 +3133,8 @@ locate_priv_file(FileName) -> filename:join(get(ct_run_dir), FileName); _ -> %% executed on other process than ct_logs - {ok,RunDir} = get_log_dir(true), - filename:join(RunDir, FileName) + {ok,LogDir} = get_log_dir(true), + filename:join(LogDir, FileName) end, case filelib:is_file(PrivResultFile) of true -> @@ -3212,6 +3216,10 @@ get_ts_html_wrapper(TestName, Logdir, PrintLabel, Cwd, TableCols, Encoding) -> ?all_runs_name), Cwd), TestIndex = make_relative(filename:join(filename:dirname(CtLogdir), ?index_name), Cwd), + LatestTest = make_relative(filename:join(filename:dirname(CtLogdir), + ?suitelog_name++".latest.html"), + Cwd), + case Basic of true -> TileFile = filename:join(filename:join(CTPath,"priv"),"tile1.jpg"), @@ -3238,7 +3246,9 @@ get_ts_html_wrapper(TestName, Logdir, PrintLabel, Cwd, TableCols, Encoding) -> "<a href=\"", uri(AllRuns), "\">Test run history\n</a> | ", "<a href=\"", uri(TestIndex), - "\">Top level test index\n</a>\n</p>\n", + "\">Top level test index\n</a> | ", + "<a href=\"", uri(LatestTest), + "\">Latest test result</a>\n</p>\n", Copyright,"</center>\n</body>\n</html>\n"]}; _ -> Copyright = @@ -3285,7 +3295,9 @@ get_ts_html_wrapper(TestName, Logdir, PrintLabel, Cwd, TableCols, Encoding) -> "<a href=\"", uri(AllRuns), "\">Test run history\n</a> | ", "<a href=\"", uri(TestIndex), - "\">Top level test index\n</a>\n</p>\n", + "\">Top level test index\n</a> | ", + "<a href=\"", uri(LatestTest), + "\">Latest test result</a>\n</p>\n", Copyright,"</center>\n</body>\n</html>\n"]} end. diff --git a/lib/common_test/src/ct_master.erl b/lib/common_test/src/ct_master.erl index 44d3fb8f64..e2ea525cdd 100644 --- a/lib/common_test/src/ct_master.erl +++ b/lib/common_test/src/ct_master.erl @@ -346,6 +346,7 @@ init_master(Parent,NodeOptsList,EvHandlers,MasterLogDir,LogDirs, case whereis(ct_master) of undefined -> register(ct_master,self()), + ct_util:mark_process(), ok; _Pid -> io:format("~nWarning: ct_master already running!~n"), @@ -690,6 +691,7 @@ refresh_logs([],Refreshed) -> init_node_ctrl(MasterPid,Cookie,Opts) -> %% make sure tests proceed even if connection to master is lost process_flag(trap_exit, true), + ct_util:mark_process(), MasterNode = node(MasterPid), group_leader(whereis(user),self()), io:format("~n********** node_ctrl process ~w started on ~w **********~n", diff --git a/lib/common_test/src/ct_master_event.erl b/lib/common_test/src/ct_master_event.erl index d535d1274e..bd4d1efc92 100644 --- a/lib/common_test/src/ct_master_event.erl +++ b/lib/common_test/src/ct_master_event.erl @@ -116,6 +116,7 @@ sync_notify(Event) -> %% this function is called to initialize the event handler. %%-------------------------------------------------------------------- init(_) -> + ct_util:mark_process(), ct_master_logs:log("CT Master Event Handler started","",[]), {ok,#state{}}. diff --git a/lib/common_test/src/ct_master_logs.erl b/lib/common_test/src/ct_master_logs.erl index fd92f73f63..1308720823 100644 --- a/lib/common_test/src/ct_master_logs.erl +++ b/lib/common_test/src/ct_master_logs.erl @@ -88,6 +88,7 @@ stop() -> init(Parent,LogDir,Nodes) -> register(?MODULE,self()), + ct_util:mark_process(), Time = calendar:local_time(), RunDir = make_dirname(Time), RunDirAbs = filename:join(LogDir,RunDir), diff --git a/lib/common_test/src/ct_repeat.erl b/lib/common_test/src/ct_repeat.erl index c043c9846c..177ef37d1f 100644 --- a/lib/common_test/src/ct_repeat.erl +++ b/lib/common_test/src/ct_repeat.erl @@ -70,6 +70,7 @@ loop_test(If,Args) when is_list(Args) -> CtrlPid = self(), spawn( fun() -> + ct_util:mark_process(), stop_after(CtrlPid,Secs,ForceStop) end) end, @@ -134,6 +135,7 @@ spawn_tester(script,Ctrl,Args) -> spawn_tester(func,Ctrl,Opts) -> Tester = fun() -> + ct_util:mark_process(), case catch ct_run:run_test2(Opts) of {'EXIT',Reason} -> exit(Reason); diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 9436236719..8c401cf3f5 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -250,6 +250,8 @@ finish(Tracing, ExitStatus, Args) -> end. script_start1(Parent, Args) -> + %% tag this process + ct_util:mark_process(), %% read general start flags Label = get_start_opt(label, fun([Lbl]) -> Lbl end, Args), Profile = get_start_opt(profile, fun([Prof]) -> Prof end, Args), @@ -956,7 +958,10 @@ run_test(StartOpts) when is_list(StartOpts) -> -spec run_test1_fun(_) -> fun(() -> no_return()). run_test1_fun(StartOpts) -> - fun() -> run_test1(StartOpts) end. + fun() -> + ct_util:mark_process(), + run_test1(StartOpts) + end. run_test1(StartOpts) when is_list(StartOpts) -> case proplists:get_value(refresh_logs, StartOpts) of @@ -1447,7 +1452,10 @@ run_testspec(TestSpec) -> -spec run_testspec1_fun(_) -> fun(() -> no_return()). run_testspec1_fun(TestSpec) -> - fun() -> run_testspec1(TestSpec) end. + fun() -> + ct_util:mark_process(), + run_testspec1(TestSpec) + end. run_testspec1(TestSpec) -> {ok,Cwd} = file:get_cwd(), @@ -1906,10 +1914,12 @@ possibly_spawn(true, Tests, Skip, Opts) -> CTUtilSrv = whereis(ct_util_server), Supervisor = fun() -> + ct_util:mark_process(), process_flag(trap_exit, true), link(CTUtilSrv), TestRun = fun() -> + ct_util:mark_process(), TestResult = (catch do_run_test(Tests, Skip, Opts)), case TestResult of {EType,_} = Error when EType == user_error; diff --git a/lib/common_test/src/ct_slave.erl b/lib/common_test/src/ct_slave.erl index 61e6446df8..0c16ad5980 100644 --- a/lib/common_test/src/ct_slave.erl +++ b/lib/common_test/src/ct_slave.erl @@ -318,6 +318,7 @@ monitor_master(MasterNode) -> % code of the masterdeath-waiter process monitor_master_int(MasterNode) -> + ct_util:mark_process(), erlang:monitor_node(MasterNode, true), receive {nodedown, MasterNode}-> diff --git a/lib/common_test/src/ct_telnet_client.erl b/lib/common_test/src/ct_telnet_client.erl index c8d217cd2a..76e4b9ea70 100644 --- a/lib/common_test/src/ct_telnet_client.erl +++ b/lib/common_test/src/ct_telnet_client.erl @@ -118,6 +118,7 @@ get_data(Pid) -> %%%----------------------------------------------------------------- %%% Internal functions init(Parent, Server, Port, Timeout, KeepAlive, NoDelay, ConnName) -> + ct_util:mark_process(), case gen_tcp:connect(Server, Port, [list,{packet,0},{nodelay,NoDelay}], Timeout) of {ok,Sock} -> dbg("~tp connected to: ~tp (port: ~w, keep_alive: ~w)\n", diff --git a/lib/common_test/src/ct_util.erl b/lib/common_test/src/ct_util.erl index 3c0fead5b2..10a06d5c88 100644 --- a/lib/common_test/src/ct_util.erl +++ b/lib/common_test/src/ct_util.erl @@ -65,6 +65,9 @@ -export([warn_duplicates/1]). +-export([mark_process/0, mark_process/1, is_marked/1, is_marked/2, + remaining_test_procs/0]). + -export([get_profile_data/0, get_profile_data/1, get_profile_data/2, open_url/3]). @@ -126,6 +129,7 @@ start(Mode, LogDir, Verbosity) -> do_start(Parent, Mode, LogDir, Verbosity) -> process_flag(trap_exit,true), register(ct_util_server,self()), + mark_process(), create_table(?conn_table,#conn.handle), create_table(?board_table,2), create_table(?suite_table,#suite_data.key), @@ -934,6 +938,70 @@ warn_duplicates(Suites) -> %%% @spec %%% %%% @doc +mark_process() -> + mark_process(system). + +mark_process(Type) -> + put(ct_process_type, Type). + +is_marked(Pid) -> + is_marked(Pid, system). + +is_marked(Pid, Type) -> + case process_info(Pid, dictionary) of + {dictionary,List} -> + Type == proplists:get_value(ct_process_type, List); + undefined -> + false + end. + +remaining_test_procs() -> + Procs = processes(), + {SharedGL,OtherGLs,Procs2} = + lists:foldl( + fun(Pid, ProcTypes = {Shared,Other,Procs1}) -> + case is_marked(Pid, group_leader) of + true -> + if not is_pid(Shared) -> + case test_server_io:get_gl(true) of + Pid -> + {Pid,Other, + lists:delete(Pid,Procs1)}; + _ -> + {Shared,[Pid|Other],Procs1} + end; + true -> % SharedGL already found + {Shared,[Pid|Other],Procs1} + end; + false -> + case is_marked(Pid) of + true -> + {Shared,Other,lists:delete(Pid,Procs1)}; + false -> + ProcTypes + end + end + end, {undefined,[],Procs}, Procs), + + AllGLs = [SharedGL | OtherGLs], + TestProcs = + lists:flatmap(fun(Pid) -> + case process_info(Pid, group_leader) of + {group_leader,GL} -> + case lists:member(GL, AllGLs) of + true -> [{Pid,GL}]; + false -> [] + end; + undefined -> + [] + end + end, Procs2), + {TestProcs, SharedGL, OtherGLs}. + +%%%----------------------------------------------------------------- +%%% @spec +%%% +%%% @doc get_profile_data() -> get_profile_data(all). diff --git a/lib/common_test/src/ct_webtool.erl b/lib/common_test/src/ct_webtool.erl index 9016aca899..82aa78fc4b 100644 --- a/lib/common_test/src/ct_webtool.erl +++ b/lib/common_test/src/ct_webtool.erl @@ -343,6 +343,7 @@ code_change(_,State,_)-> % Start the gen_server %---------------------------------------------------------------------- init({Path,Config})-> + ct_util:mark_process(), case filelib:is_dir(Path) of true -> {ok, Table} = get_tool_files_data(), diff --git a/lib/common_test/src/ct_webtool_sup.erl b/lib/common_test/src/ct_webtool_sup.erl index c02ec69d04..6c6dbde0a6 100644 --- a/lib/common_test/src/ct_webtool_sup.erl +++ b/lib/common_test/src/ct_webtool_sup.erl @@ -46,6 +46,7 @@ stop(Pid)-> %% {error, Reason} %%---------------------------------------------------------------------- init(_StartArgs) -> + ct_util:mark_process(), %%Child1 = %%Child2 ={webcover_backend,{webcover_backend,start_link,[]},permanent,2000,worker,[webcover_backend]}, %%{ok,{{simple_one_for_one,5,10},[Child1]}}. diff --git a/lib/common_test/src/cth_log_redirect.erl b/lib/common_test/src/cth_log_redirect.erl index 1c55e17686..b05f0bd28b 100644 --- a/lib/common_test/src/cth_log_redirect.erl +++ b/lib/common_test/src/cth_log_redirect.erl @@ -56,6 +56,7 @@ id(_Opts) -> ?MODULE. init(?MODULE, _Opts) -> + ct_util:mark_process(), error_logger:add_report_handler(?MODULE), tc_log_async. diff --git a/lib/common_test/src/test_server.erl b/lib/common_test/src/test_server.erl index 35a73e6d2e..7250041e13 100644 --- a/lib/common_test/src/test_server.erl +++ b/lib/common_test/src/test_server.erl @@ -415,6 +415,7 @@ run_test_case_apply(Mod, Func, Args, Name, RunInit, TimetrapData) -> St = #st{ref=Ref,pid=Pid,mf={Mod,Func},last_known_loc=unknown, status=starting,ret_val=[],comment="",timeout=infinity, config=hd(Args)}, + ct_util:mark_process(), run_test_case_msgloop(St). %% Ugly bug (pre R5A): @@ -785,6 +786,7 @@ spawn_fw_call(Mod,IPTC={init_per_testcase,Func},CurrConf,Pid, Why,Loc,SendTo) -> FwCall = fun() -> + ct_util:mark_process(), Skip = {skip,{failed,{Mod,init_per_testcase,Why}}}, %% if init_per_testcase fails, the test case %% should be skipped @@ -815,6 +817,7 @@ spawn_fw_call(Mod,EPTC={end_per_testcase,Func},EndConf,Pid, Why,_Loc,SendTo) -> FwCall = fun() -> + ct_util:mark_process(), {RetVal,Report} = case proplists:get_value(tc_status, EndConf) of undefined -> @@ -864,6 +867,7 @@ spawn_fw_call(Mod,EPTC={end_per_testcase,Func},EndConf,Pid, spawn_fw_call(FwMod,FwFunc,_,_Pid,{framework_error,FwError},_,SendTo) -> FwCall = fun() -> + ct_util:mark_process(), test_server_sup:framework_call(report, [framework_error, {{FwMod,FwFunc}, FwError}]), @@ -880,6 +884,7 @@ spawn_fw_call(FwMod,FwFunc,_,_Pid,{framework_error,FwError},_,SendTo) -> spawn_link(FwCall); spawn_fw_call(Mod,Func,CurrConf,Pid,Error,Loc,SendTo) -> + ct_util:mark_process(), {Func1,EndTCFunc} = case Func of CF when CF == init_per_suite; CF == end_per_suite; CF == init_per_group; CF == end_per_group -> @@ -918,6 +923,7 @@ start_job_proxy() -> %% The io_reply_proxy is not the most satisfying solution but it works... io_reply_proxy(ReplyTo) -> + ct_util:mark_process(), receive IoReply when is_tuple(IoReply), element(1, IoReply) == io_reply -> @@ -927,6 +933,7 @@ io_reply_proxy(ReplyTo) -> end. job_proxy_msgloop() -> + ct_util:mark_process(), receive %% @@ -1804,6 +1811,7 @@ break(CBM, TestCase, Comment) -> spawn_break_process(Pid, PName) -> spawn(fun() -> register(PName, self()), + ct_util:mark_process(), receive continue -> continue(Pid); cancel -> ok @@ -2001,6 +2009,7 @@ time_ms_apply(Func, TCPid, MultAndScale) -> user_timetrap_supervisor(Func, Spawner, TCPid, GL, T0, MultAndScale) -> process_flag(trap_exit, true), + ct_util:mark_process(), Spawner ! {self(),infinity}, MonRef = monitor(process, TCPid), UserTTSup = self(), @@ -2571,6 +2580,7 @@ run_on_shielded_node(Fun, CArgs) when is_function(Fun), is_list(CArgs) -> -spec start_job_proxy_fun(_, _) -> fun(() -> no_return()). start_job_proxy_fun(Master, Fun) -> fun () -> + ct_util:mark_process(), _ = start_job_proxy(), receive Ref -> diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl index c70ea4ef9d..3a454a1e84 100644 --- a/lib/common_test/src/test_server_ctrl.erl +++ b/lib/common_test/src/test_server_ctrl.erl @@ -89,6 +89,7 @@ -define(logdir_ext, ".logs"). -define(data_dir_suffix, "_data/"). -define(suitelog_name, "suite.log"). +-define(suitelog_latest_name, "suite.log.latest"). -define(coverlog_name, "cover.html"). -define(raw_coverlog_name, "cover.log"). -define(cross_coverlog_name, "cross_cover.html"). @@ -1126,6 +1127,7 @@ init_tester(Mod, Func, Args, Dir, Name, {_,_,MinLev}=Levels, RejectIoReqs, CreatePrivDir, TCCallback, ExtraTools) -> process_flag(trap_exit, true), _ = test_server_io:start_link(), + put(app, common_test), put(test_server_name, Name), put(test_server_dir, Dir), put(test_server_total_time, 0), @@ -1150,6 +1152,12 @@ init_tester(Mod, Func, Args, Dir, Name, {_,_,MinLev}=Levels, end, %% before first print, read and set logging options + FWLogDir = + case test_server_sup:framework_call(get_log_dir, [], []) of + {ok,FwDir} -> FwDir; + _ -> filename:dirname(Dir) + end, + put(test_server_framework_logdir, FWLogDir), LogOpts = test_server_sup:framework_call(get_logopts, [], []), put(test_server_logopts, LogOpts), @@ -1712,6 +1720,12 @@ start_log_file() -> test_server_io:set_fd(html, Html), test_server_io:set_fd(unexpected_io, Unexpected), + %% we must assume the redirection file (to the latest suite index) can + %% be stored on the level above the log directory of the current test + TopDir = filename:dirname(get(test_server_framework_logdir)), + RedirectLink = filename:join(TopDir, ?suitelog_latest_name ++ ?html_ext), + make_html_link(RedirectLink, HtmlName, redirect), + make_html_link(filename:absname(?last_test ++ ?html_ext), HtmlName, filename:basename(Dir)), LinkName = filename:join(Dir, ?last_link), @@ -1740,11 +1754,18 @@ make_html_link(LinkName, Target, Explanation) -> false -> "file:" ++ uri_encode(Target) end, - H = [html_header(Explanation), - "<h1>Last test</h1>\n" - "<a href=\"",Href,"\">",Explanation,"</a>\n" - "</body>\n</html>\n"], + H = if Explanation == redirect -> + Meta = ["<meta http-equiv=\"refresh\" " + "content=\"0; url=", Href, "\" />\n"], + [html_header("redirect", Meta), "</html>\n"]; + true -> + [html_header(Explanation), + "<h1>Last test</h1>\n" + "<a href=\"",Href,"\">",Explanation,"</a>\n" + "</body>\n</html>\n"] + end, ok = write_html_file(LinkName, H). + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% start_minor_log_file(Mod, Func, ParallelTC) -> AbsName @@ -3705,6 +3726,7 @@ run_test_case(Ref, Num, Mod, Func, Args, RunInit, TimetrapData, Mode) -> spawn_link( fun() -> process_flag(trap_exit, true), + ct_util:mark_process(), _ = [put(Key, Val) || {Key,Val} <- Dictionary], set_io_buffering({tc,Main}), run_test_case1(Ref, Num, Mod, Func, Args, RunInit, @@ -5658,6 +5680,13 @@ html_header(Title) -> "<body bgcolor=\"white\" text=\"black\" " "link=\"blue\" vlink=\"purple\" alink=\"red\">\n"]. +html_header(Title, Meta) -> + ["<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 3.2 Final//EN\">\n" + "<!-- autogenerated by '", atom_to_list(?MODULE), "'. -->\n" + "<html>\n" + "<head>\n" + "<title>", Title, "</title>\n"] ++ Meta ++ ["</head>\n"]. + open_html_file(File) -> open_utf8_file(File). diff --git a/lib/common_test/src/test_server_gl.erl b/lib/common_test/src/test_server_gl.erl index ce7682d101..24dd5cd54c 100644 --- a/lib/common_test/src/test_server_gl.erl +++ b/lib/common_test/src/test_server_gl.erl @@ -132,6 +132,7 @@ set_props(GL, PropList) -> %%% Internal functions. init([TSIO]) -> + ct_util:mark_process(group_leader), EscChars = case application:get_env(test_server, esc_chars) of {ok,ECBool} -> ECBool; _ -> true diff --git a/lib/common_test/src/test_server_io.erl b/lib/common_test/src/test_server_io.erl index 062e3bd8ff..ef31521950 100644 --- a/lib/common_test/src/test_server_io.erl +++ b/lib/common_test/src/test_server_io.erl @@ -184,6 +184,7 @@ reset_state() -> init([]) -> process_flag(trap_exit, true), + ct_util:mark_process(), Empty = gb_trees:empty(), {ok,Shared} = test_server_gl:start_link(self()), {ok,#st{fds=Empty,shared_gl=Shared,gls=gb_sets:empty(), @@ -262,7 +263,7 @@ handle_call(reset_state, From, #st{phase=stopping,pending_ops=Ops}=St) -> {Result,NewSt1} end, {noreply,St#st{pending_ops=[{From,Op}|Ops]}}; -handle_call(reset_state, _From, #st{fds=Fds,tags=Tags,gls=Gls, +handle_call(reset_state, _From, #st{fds=Fds,tags=Tags,shared_gl=Shared0,gls=Gls, offline_buffer=OfflineBuff}) -> %% close open log files lists:foreach(fun(Tag) -> @@ -273,6 +274,7 @@ handle_call(reset_state, _From, #st{fds=Fds,tags=Tags,gls=Gls, file:close(Fd) end end, Tags), + test_server_gl:stop(Shared0), GlList = gb_sets:to_list(Gls), _ = [test_server_gl:stop(GL) || GL <- GlList], timer:sleep(100), @@ -320,7 +322,7 @@ handle_call(finish, From, St) -> handle_info({'EXIT',Pid,normal}, #st{gls=Gls0,stopping=From}=St) -> Gls = gb_sets:delete_any(Pid, Gls0), - case gb_sets:is_empty(Gls) andalso stopping =/= undefined of + case gb_sets:is_empty(Gls) andalso From =/= undefined of true -> %% No more group leaders left. gen_server:reply(From, ok), @@ -329,6 +331,9 @@ handle_info({'EXIT',Pid,normal}, #st{gls=Gls0,stopping=From}=St) -> %% Wait for more group leaders to finish. {noreply,St#st{gls=Gls,phase=stopping}} end; +handle_info({'EXIT',Pid,killed}, #st{gls=Gls0}=St) -> + %% forced termination of group leader + {noreply,St#st{gls=gb_sets:delete_any(Pid, Gls0)}}; handle_info({'EXIT',_Pid,Reason}, _St) -> exit(Reason); handle_info(stop_group_leaders, #st{gls=Gls}=St) -> diff --git a/lib/common_test/src/test_server_node.erl b/lib/common_test/src/test_server_node.erl index f0f9cea6e0..b2d4f199c3 100644 --- a/lib/common_test/src/test_server_node.erl +++ b/lib/common_test/src/test_server_node.erl @@ -749,6 +749,7 @@ unpack(Bin) -> id(I) -> I. print_data(Port) -> + ct_util:mark_process(), receive {Port, {data, Bytes}} -> io:put_chars(Bytes), diff --git a/lib/common_test/src/test_server_sup.erl b/lib/common_test/src/test_server_sup.erl index 21f4be22fe..6ddbf1ad27 100644 --- a/lib/common_test/src/test_server_sup.erl +++ b/lib/common_test/src/test_server_sup.erl @@ -56,6 +56,7 @@ timetrap(Timeout0, Scale, Pid) -> timetrap(Timeout0, ReportTVal, Scale, Pid) -> process_flag(priority, max), + ct_util:mark_process(), Timeout = if not Scale -> Timeout0; true -> test_server:timetrap_scale_factor() * Timeout0 end, @@ -773,6 +774,7 @@ framework_call(Callback,Func,Args,DefaultReturn) -> false -> ok end, + ct_util:mark_process(), try apply(Mod,Func,Args) of Result -> Result @@ -850,6 +852,7 @@ util_start() -> undefined -> spawn_link(fun() -> register(?MODULE, self()), + put(app, common_test), util_loop(#util_state{starter=Starter}) end), ok; diff --git a/lib/common_test/src/vts.erl b/lib/common_test/src/vts.erl index 99a109cfe8..83fcde2f48 100644 --- a/lib/common_test/src/vts.erl +++ b/lib/common_test/src/vts.erl @@ -157,6 +157,7 @@ test_info(_VtsPid,Type,Data) -> init(Parent) -> register(?MODULE,self()), process_flag(trap_exit,true), + ct_util:mark_process(), Parent ! {self(),started}, {ok,Cwd} = file:get_cwd(), InitState = #state{start_dir=Cwd}, @@ -284,6 +285,7 @@ run_test1(State=#state{tests=Tests,current_log_dir=LogDir, logopts=LogOpts}) -> Self=self(), RunTest = fun() -> + ct_util:mark_process(), case ct_run:do_run(Tests,[],LogDir,LogOpts) of {error,_Reason} -> aborted(); diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile index 0d9149f489..ecd1f727a2 100644 --- a/lib/common_test/test/Makefile +++ b/lib/common_test/test/Makefile @@ -73,7 +73,8 @@ MODULES= \ ct_log_SUITE \ ct_SUITE \ ct_keep_logs_SUITE \ - ct_unicode_SUITE + ct_unicode_SUITE \ + ct_auto_clean_SUITE ERL_FILES= $(MODULES:%=%.erl) HRL_FILES= test_server_test_lib.hrl diff --git a/lib/common_test/test/ct_auto_clean_SUITE.erl b/lib/common_test/test/ct_auto_clean_SUITE.erl new file mode 100644 index 0000000000..fd81430d0d --- /dev/null +++ b/lib/common_test/test/ct_auto_clean_SUITE.erl @@ -0,0 +1,262 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ct_auto_clean_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> Config1 | {skip,Reason} +%% +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Reason = term() +%% The reason for skipping the suite. +%% +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + DataDir = ?config(data_dir, Config), + CTHs = filelib:wildcard(filename:join(DataDir,"cth_*.erl")), + ct:pal("CTHs: ~p",[CTHs]), + [ct:pal("Compiling ~p: ~p", + [FileName,compile:file(FileName,[{outdir,DataDir},debug_info])]) || + FileName <- CTHs], + ct_test_support:init_per_suite([{path_dirs,[DataDir]} | Config]). + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config) -> void() +%% +%% Config = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% +%% Description: Cleanup after the suite. +%%-------------------------------------------------------------------- +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> Config1 | +%% {skip,Reason} +%% TestCase = atom() +%% Name of the test case that is about to run. +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Reason = term() +%% The reason for skipping the test case. +%% +%% Description: Initialization before each test case. +%% +%% Note: This function is free to add any key/value pairs to the Config +%% variable, but should NOT alter/remove any existing entries. +%%-------------------------------------------------------------------- +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config) -> void() +%% +%% TestCase = atom() +%% Name of the test case that is finished. +%% Config = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% +%% Description: Cleanup after each test case. +%%-------------------------------------------------------------------- +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + +%%-------------------------------------------------------------------- +%% Function: all(Clause) -> Descr | TestCases | {skip,Reason} +%% +%% Clause = doc | suite +%% Indicates expected return value. +%% Descr = [string()] | [] +%% String that describes the test suite. +%% TestCases = [TestCase] +%% TestCase = atom() +%% Name of a test case. +%% Reason = term() +%% The reason for skipping the test suite. +%% +%% Description: Returns a description of the test suite (doc) and a +%% list of all test cases in the suite (suite). +%%-------------------------------------------------------------------- +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [clean]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Function: TestCase(Arg) -> Descr | Spec | ok | exit() | {skip,Reason} +%% +%% Arg = doc | suite | Config +%% Indicates expected behaviour and return value. +%% Config = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Descr = [string()] | [] +%% String that describes the test case. +%% Spec = [tuple()] | [] +%% A test specification. +%% Reason = term() +%% The reason for skipping the test case. +%% +%% Description: Test case function. Returns a description of the test +%% case (doc), then returns a test specification (suite), +%% or performs the actual test (Config). +%%-------------------------------------------------------------------- + +%%%----------------------------------------------------------------- +%%% + +clean(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + ACSuite = filename:join(DataDir, "ac_SUITE"), + Opts0 = ct_test_support:get_opts(Config), + Opts = eh_opts(Config) ++ Opts0 ++ [{suite,ACSuite}, + {ct_hooks,[cth_auto_clean]}], + + ERPid = ct_test_support:start_event_receiver(Config), + + ok = ct_test_support:run(Opts, Config), + + Events = ct_test_support:get_events(ERPid, Config), + ct_test_support:log_events(?FUNCTION_NAME, + ct_test_support:reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + TestEvents = events_to_check(?FUNCTION_NAME), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- + +eh_opts(Config) -> + Level = ?config(trace_level, Config), + [{event_handler,{?eh,[{cbm,ct_test_support},{trace_level,Level}]}}]. + +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + events(Test) ++ events_to_check(Test, N-1). + +events(clean) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,9}}, + + {?eh,tc_start,{ac_SUITE,init_per_suite}}, + {?eh,tc_done,{ac_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ac_SUITE,tc1}}, + {?eh,tc_done,{ac_SUITE,tc1,ok}}, + + {?eh,test_stats,{1,0,{0,0}}}, + + {?eh,tc_start,{ac_SUITE,tc2}}, + {?eh,tc_done,{ac_SUITE,tc2,ok}}, + + {?eh,test_stats,{2,0,{0,0}}}, + + [{?eh,tc_start,{ac_SUITE,{init_per_group,s1,[]}}}, + {?eh,tc_done,{ac_SUITE,{init_per_group,s1,[]},ok}}, + + {?eh,tc_start,{ac_SUITE,stc1}}, + {?eh,tc_done,{ac_SUITE,stc1,ok}}, + + {?eh,test_stats,{3,0,{0,0}}}, + + {?eh,tc_start,{ac_SUITE,stc2}}, + {?eh,tc_done,{ac_SUITE,stc2,ok}}, + + {?eh,test_stats,{4,0,{0,0}}}, + + {?eh,tc_start,{ac_SUITE,{end_per_group,s1,[]}}}, + {?eh,tc_done,{ac_SUITE,{end_per_group,s1,[]},ok}}], + + {parallel, + [{?eh,tc_start,{ac_SUITE,{init_per_group,p1,[parallel]}}}, + {?eh,tc_done,{ac_SUITE,{init_per_group,p1,[parallel]},ok}}, + + {?eh,tc_start,{ac_SUITE,ptc1}}, + {?eh,tc_start,{ac_SUITE,ptc2}}, + {?eh,tc_done,{ac_SUITE,ptc1,ok}}, + {?eh,test_stats,{5,0,{0,0}}}, + {?eh,tc_done,{ac_SUITE,ptc2,ok}}, + {?eh,test_stats,{6,0,{0,0}}}, + + {?eh,tc_start,{ac_SUITE,{end_per_group,p1,[parallel]}}}, + {?eh,tc_done,{ac_SUITE,{end_per_group,p1,[parallel]},ok}}]}, + + [{?eh,tc_start,{ac_SUITE,{init_per_group,s2,[]}}}, + {?eh,tc_done,{ac_SUITE,{init_per_group,s2,[]},ok}}, + + {?eh,tc_start,{ac_SUITE,stc1}}, + {?eh,tc_done,{ac_SUITE,stc1,ok}}, + + {?eh,test_stats,{7,0,{0,0}}}, + + {?eh,tc_start,{ac_SUITE,stc2}}, + {?eh,tc_done,{ac_SUITE,stc2,ok}}, + + {?eh,test_stats,{8,0,{0,0}}}, + + {?eh,tc_start,{ac_SUITE,{end_per_group,s2,[]}}}, + {?eh,tc_done,{ac_SUITE,{end_per_group,s2,[]},ok}}], + + {?eh,tc_start,{ac_SUITE,tc1}}, + {?eh,tc_done,{ac_SUITE,tc1,ok}}, + + {?eh,test_stats,{9,0,{0,0}}}, + + {?eh,tc_start,{ac_SUITE,end_per_suite}}, + {?eh,tc_done,{ac_SUITE,end_per_suite,ok}}, + + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]. diff --git a/lib/common_test/test/ct_auto_clean_SUITE_data/ac_SUITE.erl b/lib/common_test/test/ct_auto_clean_SUITE_data/ac_SUITE.erl new file mode 100644 index 0000000000..dae7c1e22c --- /dev/null +++ b/lib/common_test/test/ct_auto_clean_SUITE_data/ac_SUITE.erl @@ -0,0 +1,181 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ac_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% @spec suite() -> Info +%% Info = [tuple()] +%% @end +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{seconds,30}}]. + +%%-------------------------------------------------------------------- +%% @spec init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + start_processes(), + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_suite(Config0) -> term() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + start_processes(), + ok. + +%%-------------------------------------------------------------------- +%% @spec init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + start_processes(), + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_group(GroupName, Config0) -> +%% term() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + start_processes(), + ok. + +%%-------------------------------------------------------------------- +%% @spec init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_testcase(_TestCase, Config) -> + start_processes(), + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_testcase(TestCase, Config0) -> +%% term() | {save_config,Config1} | {fail,Reason} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +end_per_testcase(_TestCase, _Config) -> + start_processes(), + ok. + +%%-------------------------------------------------------------------- +%% @spec groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%% @end +%%-------------------------------------------------------------------- +groups() -> + [{s1,[],[stc1,stc2]}, + {p1,[parallel],[ptc1,ptc2]}, + {s2,[],[stc1,stc2]}]. + +%%! What about nested groups?? + +%%-------------------------------------------------------------------- +%% @spec all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +all() -> + [ + [tc1,tc2], + {group,s1}, + {group,p1}, + {group,s2}, + tc1 + ]. + +tc1(_Config) -> + start_processes(), + ok. + +tc2(_Config) -> + start_processes(), + ok. + +stc1(_Config) -> + start_processes(), + ok. + +stc2(_Config) -> + start_processes(), + ok. + +ptc1(_Config) -> + start_processes(), + ok. + +ptc2(_Config) -> + start_processes(), + ok. + + +%%%----------------------------------------------------------------- +%%% + +start_processes() -> + Init = fun() -> + process_flag(trap_exit, true), + do_spawn(fun() -> receive _ -> ok end end), + receive _ -> + ok + end + end, + do_spawn(Init). + +do_spawn(Fun) -> + Pid = spawn(Fun), + ct:log("Process ~w started with group leader ~w", + [Pid,element(2, process_info(Pid, group_leader))]), + Pid. diff --git a/lib/common_test/test/ct_auto_clean_SUITE_data/cth_auto_clean.erl b/lib/common_test/test/ct_auto_clean_SUITE_data/cth_auto_clean.erl new file mode 100644 index 0000000000..137c81969d --- /dev/null +++ b/lib/common_test/test/ct_auto_clean_SUITE_data/cth_auto_clean.erl @@ -0,0 +1,214 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(cth_auto_clean). + +%% CTH Callbacks +-export([id/1, init/2, + pre_init_per_suite/3, post_init_per_suite/4, + pre_end_per_suite/3, post_end_per_suite/4, + pre_init_per_group/4, post_init_per_group/5, + pre_end_per_group/4, post_end_per_group/5, + pre_init_per_testcase/4, post_init_per_testcase/5, + pre_end_per_testcase/4, post_end_per_testcase/5]). + +id(_Opts) -> + ?MODULE. + +init(?MODULE, _Opts) -> + ok. + +pre_init_per_suite(_Suite, Config, State) -> + identify(?FUNCTION_NAME), + SharedGL = test_server_io:get_gl(true), + SharedGL = find_and_kill(), + do_until(fun() -> ct:remaining_test_procs() end, {[],SharedGL,[]}), + %% get status of processes at startup, to be compared with end result + {Config, [{all_procs,processes()} | State]}. + +post_init_per_suite(_Suite, _Config, Return, State) -> + identify(?FUNCTION_NAME), + SharedGL = find_and_kill(), + do_until(fun() -> ct:remaining_test_procs() end, {[],SharedGL,[]}), + {Return, State}. + +pre_end_per_suite(_Suite, Config, State) -> + identify(?FUNCTION_NAME), + SharedGL = find_and_kill(), + do_until(fun() -> ct:remaining_test_procs() end, {[],SharedGL,[]}), + {Config, State}. + +post_end_per_suite(_Suite, _Config, Return, State) -> + identify(?FUNCTION_NAME), + SharedGL = find_and_kill(), + do_until(fun() -> ct:remaining_test_procs() end, {[],SharedGL,[]}), + AllProcs = processes(), + Remaining = AllProcs--proplists:get_value(all_procs, State), + ct:pal("Final remaining processes = ~p", [Remaining]), + %% only the end_per_suite process shoud remain at this point! + Remaining = [self()], + {Return, State}. + +pre_init_per_group(_Suite, _Group, Config, State) -> + identify(?FUNCTION_NAME), + SharedGL = find_and_kill(procs_and_gls), + do_until(fun() -> ct:remaining_test_procs() end, {[],SharedGL,[]}), + {Config, State}. + +post_init_per_group(_Suite, _Group, _Config, Result, State) -> + identify(?FUNCTION_NAME), + SharedGL = find_and_kill(procs_and_gls), + do_until(fun() -> ct:remaining_test_procs() end, {[],SharedGL,[]}), + {Result, State}. + +pre_init_per_testcase(_Suite, _TC, Config, State) -> + identify(?FUNCTION_NAME), + ThisGL = group_leader(), + find_and_kill(proc, ThisGL), + case proplists:get_value(tc_group_properties, Config) of + [{name,_},parallel] -> + timer:sleep(1000); + _ -> + do_until(fun() -> element(1,ct:remaining_test_procs()) end, []) + end, + {Config, State}. + +post_init_per_testcase(_Suite, _TC, Config, Return, State) -> + identify(?FUNCTION_NAME), + ThisGL = group_leader(), + find_and_kill(proc, ThisGL), + case proplists:get_value(tc_group_properties, Config) of + [{name,_},parallel] -> + timer:sleep(1000); + _ -> + do_until(fun() -> element(1,ct:remaining_test_procs()) end, []) + end, + {Return, State}. + +pre_end_per_testcase(_Suite, _TC, Config, State) -> + identify(?FUNCTION_NAME), + ThisGL = group_leader(), + find_and_kill(proc, ThisGL), + case proplists:get_value(tc_group_properties, Config) of + [{name,_},parallel] -> + timer:sleep(1000); + _ -> + do_until(fun() -> element(1,ct:remaining_test_procs()) end, []) + end, + {Config, State}. + +post_end_per_testcase(_Suite, _TC, Config, Result, State) -> + identify(?FUNCTION_NAME), + ThisGL = group_leader(), + find_and_kill(proc, ThisGL), + case proplists:get_value(tc_group_properties, Config) of + [{name,_},parallel] -> + timer:sleep(1000); + _ -> + do_until(fun() -> element(1,ct:remaining_test_procs()) end, []) + end, + {Result, State}. + +pre_end_per_group(_Suite, _Group, Config, State) -> + identify(?FUNCTION_NAME), + SharedGL = find_and_kill(procs_and_gls), + do_until(fun() -> ct:remaining_test_procs() end, {[],SharedGL,[]}), + {Config, State}. + +post_end_per_group(_Suite, _Group, _Config, Return, State) -> + identify(?FUNCTION_NAME), + SharedGL = find_and_kill(procs_and_gls), + do_until(fun() -> ct:remaining_test_procs() end, {[],SharedGL,[]}), + {Return, State}. + + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- + +identify(Func) -> + ct:pal("********** THIS IS ~w on ~w", [Func, self()]), + ok. + +find_and_kill() -> + find_and_kill(procs). + +find_and_kill(procs) -> + {Procs,SharedGL,_ParallelGLs} = ct:remaining_test_procs(), + ct:pal("Remaining test processes = ~p", [pi(Procs)]), + [pkill(P, kill) || {P,_GL} <- Procs], + SharedGL; + +find_and_kill(procs_and_gls) -> + {Procs,SharedGL,GLs} = ct:remaining_test_procs(), + ct:pal("Remaining test processes = ~p", [pi(Procs)]), + [pkill(P, kill) || {P,_GL} <- Procs], + ct:pal("Remaining group leaders = ~p", [pi(GLs)]), + [pkill(GL, kill) || GL <- GLs, GL /= SharedGL], + SharedGL. + +find_and_kill(proc, ProcGL) -> + {Procs,SharedGL,GLs} = ct:remaining_test_procs(), + ct:pal("Remaining test processes = ~p", [pi(Procs++GLs)]), + [pkill(P, kill) || {P,GL} <- Procs, GL == ProcGL], + SharedGL. + +pi([{P,_GL}|Ps]) -> + pi([P|Ps]); +pi([P|Ps]) -> + case node() == node(P) of + true -> + {_,GL} = process_info(P,group_leader), + {_,CF} = process_info(P,current_function), + {_,IC} = process_info(P,initial_call), + {_,D} = process_info(P,dictionary), + Shared = test_server_io:get_gl(true), + User = whereis(user), + if (GL /= P) and (GL /= Shared) and (GL /= User) -> + [{P,GL,CF,IC,D} | pi([GL|Ps])]; + true -> + [{P,GL,CF,IC,D} | pi(Ps)] + end; + false -> + pi(Ps) + end; +pi([]) -> + []. + +do_until(Fun, Until) -> + io:format("Will do until ~p~n", [Until]), + do_until(Fun, Until, 1000). + +do_until(_, Until, 0) -> + io:format("Couldn't get ~p~n", [Until]), + exit({not_reached,Until}); + +do_until(Fun, Until, N) -> + case Fun() of + Until -> + ok; + _Tmp -> + do_until(Fun, Until, N-1) + end. + +pkill(P, How) -> + ct:pal("KILLING ~w NOW!", [P]), + exit(P, How). + diff --git a/lib/common_test/test/test_server_test_lib.erl b/lib/common_test/test/test_server_test_lib.erl index e3d987a2ea..9ee946af0b 100644 --- a/lib/common_test/test/test_server_test_lib.erl +++ b/lib/common_test/test/test_server_test_lib.erl @@ -121,7 +121,7 @@ parse_suite(FileName) -> end. fline(Fd) -> - case prim_file:read_line(Fd) of + case file:read_line(Fd) of eof -> eof; {ok, Line} -> Line end. diff --git a/lib/common_test/vsn.mk b/lib/common_test/vsn.mk index 7b959ebfe3..96fdc89853 100644 --- a/lib/common_test/vsn.mk +++ b/lib/common_test/vsn.mk @@ -1 +1 @@ -COMMON_TEST_VSN = 1.15.2 +COMMON_TEST_VSN = 1.15.3 diff --git a/lib/compiler/doc/src/compile.xml b/lib/compiler/doc/src/compile.xml index b398871ddf..06afc55c07 100644 --- a/lib/compiler/doc/src/compile.xml +++ b/lib/compiler/doc/src/compile.xml @@ -233,6 +233,15 @@ module.beam: module.erl \ header.hrl</code> </item> + <tag><c>makedep_side_effect</c></tag> + <item> + <p>The dependecies are created as a side effect to the + normal compilation process. This means that the object + file will also be produced. This option override the + <c>makedep</c> option. + </p> + </item> + <tag><c>{makedep_output, Output}</c></tag> <item> <p>Writes generated rules to <c>Output</c> instead of the diff --git a/lib/compiler/doc/src/notes.xml b/lib/compiler/doc/src/notes.xml index 2aec75a2aa..f4a3f9875b 100644 --- a/lib/compiler/doc/src/notes.xml +++ b/lib/compiler/doc/src/notes.xml @@ -32,6 +32,22 @@ <p>This document describes the changes made to the Compiler application.</p> +<section><title>Compiler 7.1.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>The '<c>deterministic</c>' option was not recognized + when given in a <c>-compile()</c> attribute in the source + code.</p> + <p> + Own Id: OTP-14773 Aux Id: ERL-498 </p> + </item> + </list> + </section> + +</section> + <section><title>Compiler 7.1.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/compiler/src/beam_a.erl b/lib/compiler/src/beam_a.erl index cdb32d5d55..6f09dc4be4 100644 --- a/lib/compiler/src/beam_a.erl +++ b/lib/compiler/src/beam_a.erl @@ -42,8 +42,7 @@ function({function,Name,Arity,CLabel,Is0}) -> Is = beam_jump:remove_unused_labels(Is1), {function,Name,Arity,CLabel,Is} catch - Class:Error -> - Stack = erlang:get_stacktrace(), + Class:Error:Stack -> io:fwrite("Function: ~w/~w\n", [Name,Arity]), erlang:raise(Class, Error, Stack) end. diff --git a/lib/compiler/src/beam_asm.erl b/lib/compiler/src/beam_asm.erl index 9ecbb7884c..453e00fce3 100644 --- a/lib/compiler/src/beam_asm.erl +++ b/lib/compiler/src/beam_asm.erl @@ -24,7 +24,7 @@ -export([module/4]). -export([encode/2]). --export_type([fail/0,label/0,reg/0,src/0,module_code/0,function_name/0]). +-export_type([fail/0,label/0,reg/0,reg_num/0,src/0,module_code/0,function_name/0]). -import(lists, [map/2,member/2,keymember/3,duplicate/2,splitwith/2]). -include("beam_opcodes.hrl"). @@ -240,9 +240,7 @@ build_form(Id, Chunks0) when byte_size(Id) =:= 4, is_list(Chunks0) -> chunk(Id, Contents) when byte_size(Id) =:= 4, is_binary(Contents) -> Size = byte_size(Contents), - [<<Id/binary,Size:32>>,Contents|pad(Size)]; -chunk(Id, Contents) when is_list(Contents) -> - chunk(Id, list_to_binary(Contents)). + [<<Id/binary,Size:32>>,Contents|pad(Size)]. %% Build a correctly padded chunk (with a sub-header). diff --git a/lib/compiler/src/beam_block.erl b/lib/compiler/src/beam_block.erl index c640af224d..fe1ce6f60b 100644 --- a/lib/compiler/src/beam_block.erl +++ b/lib/compiler/src/beam_block.erl @@ -37,16 +37,16 @@ function({function,Name,Arity,CLabel,Is0}) -> %% Collect basic blocks and optimize them. Is1 = blockify(Is0), Is2 = embed_lines(Is1), - Is3 = move_allocates(Is2), - Is4 = beam_utils:live_opt(Is3), - Is5 = opt_blocks(Is4), - Is6 = beam_utils:delete_live_annos(Is5), - - %% Done. - {function,Name,Arity,CLabel,Is6} + Is3 = beam_utils:anno_defs(Is2), + Is4 = move_allocates(Is3), + Is5 = beam_utils:live_opt(Is4), + Is6 = opt_blocks(Is5), + Is = beam_utils:delete_live_annos(Is6), + + %% Done. + {function,Name,Arity,CLabel,Is} catch - Class:Error -> - Stack = erlang:get_stacktrace(), + Class:Error:Stack -> io:fwrite("Function: ~w/~w\n", [Name,Arity]), erlang:raise(Class, Error, Stack) end. @@ -173,7 +173,7 @@ find_fixpoint(OptFun, Is0) -> %% safe to assume that if x(N) is initialized, then all lower-numbered %% x registers are also initialized. %% -%% For example, in general it is not safe to transform the following +%% For example, we must be careful when transforming the following %% instructions: %% %% get_tuple_element x(0) Element => x(1) @@ -185,13 +185,9 @@ find_fixpoint(OptFun, Is0) -> %% get_tuple_element x(0) Element => x(1) %% %% The transformation is safe if and only if x(1) has been -%% initialized previously. Unfortunately, beam_reorder may have moved -%% a get_tuple_element instruction so that x(1) is not always -%% initialized when this code is reached. To find whether or not x(1) -%% is initialized, we would need to analyze all code preceding these -%% two instructions (across branches). Since we currently don't have -%% any practical mechanism for doing that, we will have to -%% conservatively assume that the transformation is unsafe. +%% initialized previously. We will use the annotations added by +%% beam_utils:anno_defs/1 to determine whether x(a) has been +%% initialized. move_allocates([{block,Bl0}|Is]) -> Bl = move_allocates_1(reverse(Bl0), []), @@ -200,15 +196,20 @@ move_allocates([I|Is]) -> [I|move_allocates(Is)]; move_allocates([]) -> []. +move_allocates_1([{'%def',_}|Is], Acc) -> + move_allocates_1(Is, Acc); move_allocates_1([I|Is], [{set,[],[],{alloc,Live0,Info}}|Acc]=Acc0) -> - case {alloc_may_pass(I),alloc_live_regs(I, Live0)} of - {false,_} -> - move_allocates_1(Is, [I|Acc0]); - {true,not_possible} -> - move_allocates_1(Is, [I|Acc0]); - {true,Live} when is_integer(Live) -> - A = {set,[],[],{alloc,Live,Info}}, - move_allocates_1(Is, [A,I|Acc]) + case alloc_may_pass(I) of + false -> + move_allocates_1(Is, [I|Acc0]); + true -> + case alloc_live_regs(I, Is, Live0) of + not_possible -> + move_allocates_1(Is, [I|Acc0]); + Live when is_integer(Live) -> + A = {set,[],[],{alloc,Live,Info}}, + move_allocates_1(Is, [A,I|Acc]) + end end; move_allocates_1([I|Is], Acc) -> move_allocates_1(Is, [I|Acc]); @@ -472,16 +473,34 @@ count_ones(Bits, Acc) -> %% Calculate the new number of live registers when we move an allocate %% instruction upwards, passing a 'set' instruction. -alloc_live_regs({set,Ds,Ss,_}, Regs0) -> +alloc_live_regs({set,Ds,Ss,_}, Is, Regs0) -> Rset = x_live(Ss, x_dead(Ds, (1 bsl Regs0)-1)), - live_regs(0, Rset). + Live = live_regs(0, Rset), + case ensure_contiguous(Rset, Live) of + not_possible -> + %% Liveness information (looking forward in the + %% instruction stream) can't prove that moving this + %% allocation instruction is safe. Now use the annotation + %% of defined registers at the beginning of the current + %% block to see whether moving would be safe. + Def0 = defined_regs(Is, 0), + Def = Def0 band ((1 bsl Live) - 1), + ensure_contiguous(Rset bor Def, Live); + Live -> + %% Safe based on liveness information. + Live + end. live_regs(N, 0) -> N; -live_regs(N, Regs) when Regs band 1 =:= 1 -> - live_regs(N+1, Regs bsr 1); -live_regs(_, _) -> - not_possible. +live_regs(N, Regs) -> + live_regs(N+1, Regs bsr 1). + +ensure_contiguous(Regs, Live) -> + case (1 bsl Live) - 1 of + Regs -> Live; + _ -> not_possible + end. x_dead([{x,N}|Rs], Regs) -> x_dead(Rs, Regs band (bnot (1 bsl N))); x_dead([_|Rs], Regs) -> x_dead(Rs, Regs); @@ -490,3 +509,14 @@ x_dead([], Regs) -> Regs. x_live([{x,N}|Rs], Regs) -> x_live(Rs, Regs bor (1 bsl N)); x_live([_|Rs], Regs) -> x_live(Rs, Regs); x_live([], Regs) -> Regs. + +%% defined_regs(ReversedInstructions) -> RegBitmap. +%% Given a reversed instruction stream, determine the +%% the registers that are defined. + +defined_regs([{'%def',Def}|_], Regs) -> + Def bor Regs; +defined_regs([{set,Ds,_,{alloc,Live,_}}|_], Regs) -> + x_live(Ds, Regs bor ((1 bsl Live) - 1)); +defined_regs([{set,Ds,_,_}|Is], Regs) -> + defined_regs(Is, x_live(Ds, Regs)). diff --git a/lib/compiler/src/beam_bs.erl b/lib/compiler/src/beam_bs.erl index beb055b23d..14cedbb582 100644 --- a/lib/compiler/src/beam_bs.erl +++ b/lib/compiler/src/beam_bs.erl @@ -38,8 +38,7 @@ function({function,Name,Arity,CLabel,Is0}, Lc0) -> {Is,Lc} = bsm_opt(Is1, Lc0), {{function,Name,Arity,CLabel,Is},Lc} catch - Class:Error -> - Stack = erlang:get_stacktrace(), + Class:Error:Stack -> io:fwrite("Function: ~w/~w\n", [Name,Arity]), erlang:raise(Class, Error, Stack) end. diff --git a/lib/compiler/src/beam_bsm.erl b/lib/compiler/src/beam_bsm.erl index 9a4e7fb133..9ea5a3eb92 100644 --- a/lib/compiler/src/beam_bsm.erl +++ b/lib/compiler/src/beam_bsm.erl @@ -105,8 +105,7 @@ function({function,Name,Arity,Entry,Is}, FIndex) -> D = #btb{f=FIndex,index=Index}, {function,Name,Arity,Entry,btb_opt_1(Is, D, [])} catch - Class:Error -> - Stack = erlang:get_stacktrace(), + Class:Error:Stack -> io:fwrite("Function: ~w/~w\n", [Name,Arity]), erlang:raise(Class, Error, Stack) end. diff --git a/lib/compiler/src/beam_dead.erl b/lib/compiler/src/beam_dead.erl index d379fdc4eb..da944f3ce6 100644 --- a/lib/compiler/src/beam_dead.erl +++ b/lib/compiler/src/beam_dead.erl @@ -56,8 +56,7 @@ function({function,Name,Arity,CLabel,Is0}, Lc0) -> Is = move_move_into_block(Is3, []), {{function,Name,Arity,CLabel,Is},Lc} catch - Class:Error -> - Stack = erlang:get_stacktrace(), + Class:Error:Stack -> io:fwrite("Function: ~w/~w\n", [Name,Arity]), erlang:raise(Class, Error, Stack) end. @@ -272,7 +271,8 @@ backward([{jump,{f,To0}},{move,Src,Reg}=Move|Is], D, Acc) -> end; backward([{jump,{f,To}}=J|[{bif,Op,{f,BifFail},Ops,Reg}|Is]=Is0], D, Acc) -> try replace_comp_op(To, Reg, Op, Ops, D) of - I -> backward(Is, D, I++Acc) + {Test,Jump} -> + backward([Jump,Test|Is], D, Acc) catch throw:not_possible -> case To =:= BifFail of @@ -446,7 +446,7 @@ prune_redundant([], _) -> []. replace_comp_op(To, Reg, Op, Ops, D) -> False = comp_op_find_shortcut(To, Reg, {atom,false}, D), True = comp_op_find_shortcut(To, Reg, {atom,true}, D), - [bif_to_test(Op, Ops, False),{jump,{f,True}}]. + {bif_to_test(Op, Ops, False),{jump,{f,True}}}. comp_op_find_shortcut(To0, Reg, Val, D) -> case shortcut_select_label(To0, Reg, Val, D) of @@ -483,15 +483,22 @@ not_possible() -> throw(not_possible). %% F1: is_eq_exact F2 Reg Lit2 F1: is_eq_exact F2 Reg Lit2 %% L2: .... L2: %% -combine_eqs(To, [Reg,{Type,_}=Lit1]=Ops, D, [{label,L1}|_]) - when Type =:= atom; Type =:= integer -> +combine_eqs(To, [Reg,{Type,_}=Lit1]=Ops, D, Acc) + when Type =:= atom; Type =:= integer -> + Next = case Acc of + [{label,Lbl}|_] -> Lbl; + [{jump,{f,Lbl}}|_] -> Lbl + end, case beam_utils:code_at(To, D) of [{test,is_eq_exact,{f,F2},[Reg,{Type,_}=Lit2]}, {label,L2}|_] when Lit1 =/= Lit2 -> - {select,select_val,Reg,{f,F2},[Lit1,{f,L1},Lit2,{f,L2}]}; + {select,select_val,Reg,{f,F2},[Lit1,{f,Next},Lit2,{f,L2}]}; + [{test,is_eq_exact,{f,F2},[Reg,{Type,_}=Lit2]}, + {jump,{f,L2}}|_] when Lit1 =/= Lit2 -> + {select,select_val,Reg,{f,F2},[Lit1,{f,Next},Lit2,{f,L2}]}; [{select,select_val,Reg,{f,F2},[{Type,_}|_]=List0}|_] -> List = remove_from_list(Lit1, List0), - {select,select_val,Reg,{f,F2},[Lit1,{f,L1}|List]}; + {select,select_val,Reg,{f,F2},[Lit1,{f,Next}|List]}; _Is -> {test,is_eq_exact,{f,To},Ops} end; diff --git a/lib/compiler/src/beam_disasm.erl b/lib/compiler/src/beam_disasm.erl index 8fd0b36d05..22ba86fa38 100644 --- a/lib/compiler/src/beam_disasm.erl +++ b/lib/compiler/src/beam_disasm.erl @@ -163,8 +163,8 @@ pp_instr(I) -> file(File) -> try process_chunks(File) - catch error:Reason -> - {error,?MODULE,{internal,{Reason,erlang:get_stacktrace()}}} + catch error:Reason:Stack -> + {error,?MODULE,{internal,{Reason,Stack}}} end. %%----------------------------------------------------------------------- @@ -719,42 +719,6 @@ resolve_inst({wait,[Lbl]},_,_,_) -> {wait,Lbl}; resolve_inst({wait_timeout,[Lbl,Int]},_,_,_) -> {wait_timeout,Lbl,resolve_arg(Int)}; -resolve_inst({m_plus,Args},_,_,_) -> - [W,SrcR1,SrcR2,DstR] = resolve_args(Args), - {arithbif,'+',W,[SrcR1,SrcR2],DstR}; -resolve_inst({m_minus,Args},_,_,_) -> - [W,SrcR1,SrcR2,DstR] = resolve_args(Args), - {arithbif,'-',W,[SrcR1,SrcR2],DstR}; -resolve_inst({m_times,Args},_,_,_) -> - [W,SrcR1,SrcR2,DstR] = resolve_args(Args), - {arithbif,'*',W,[SrcR1,SrcR2],DstR}; -resolve_inst({m_div,Args},_,_,_) -> - [W,SrcR1,SrcR2,DstR] = resolve_args(Args), - {arithbif,'/',W,[SrcR1,SrcR2],DstR}; -resolve_inst({int_div,Args},_,_,_) -> - [W,SrcR1,SrcR2,DstR] = resolve_args(Args), - {arithbif,'div',W,[SrcR1,SrcR2],DstR}; -resolve_inst({int_rem,Args},_,_,_) -> - [W,SrcR1,SrcR2,DstR] = resolve_args(Args), - {arithbif,'rem',W,[SrcR1,SrcR2],DstR}; -resolve_inst({int_band,Args},_,_,_) -> - [W,SrcR1,SrcR2,DstR] = resolve_args(Args), - {arithbif,'band',W,[SrcR1,SrcR2],DstR}; -resolve_inst({int_bor,Args},_,_,_) -> - [W,SrcR1,SrcR2,DstR] = resolve_args(Args), - {arithbif,'bor',W,[SrcR1,SrcR2],DstR}; -resolve_inst({int_bxor,Args},_,_,_) -> - [W,SrcR1,SrcR2,DstR] = resolve_args(Args), - {arithbif,'bxor',W,[SrcR1,SrcR2],DstR}; -resolve_inst({int_bsl,Args},_,_,_) -> - [W,SrcR1,SrcR2,DstR] = resolve_args(Args), - {arithbif,'bsl',W,[SrcR1,SrcR2],DstR}; -resolve_inst({int_bsr,Args},_,_,_) -> - [W,SrcR1,SrcR2,DstR] = resolve_args(Args), - {arithbif,'bsr',W,[SrcR1,SrcR2],DstR}; -resolve_inst({int_bnot,Args},_,_,_) -> - [W,SrcR,DstR] = resolve_args(Args), - {arithbif,'bnot',W,[SrcR],DstR}; resolve_inst({is_lt=I,Args0},_,_,_) -> [L|Args] = resolve_args(Args0), {test,I,L,Args}; @@ -800,9 +764,6 @@ resolve_inst({is_nil=I,Args0},_,_,_) -> resolve_inst({is_binary=I,Args0},_,_,_) -> [L|Args] = resolve_args(Args0), {test,I,L,Args}; -resolve_inst({is_constant=I,Args0},_,_,_) -> - [L|Args] = resolve_args(Args0), - {test,I,L,Args}; resolve_inst({is_list=I,Args0},_,_,_) -> [L|Args] = resolve_args(Args0), {test,I,L,Args}; @@ -840,11 +801,6 @@ resolve_inst({get_tuple_element,[Src,{u,Off},Dst]},_,_,_) -> {get_tuple_element,resolve_arg(Src),Off,resolve_arg(Dst)}; resolve_inst({set_tuple_element,[Src,Dst,{u,Off}]},_,_,_) -> {set_tuple_element,resolve_arg(Src),resolve_arg(Dst),Off}; -resolve_inst({put_string,[{u,Len},{u,Off},Dst]},_,Strings,_) -> - String = if Len > 0 -> binary_to_list(Strings, Off+1, Off+Len); - true -> "" - end, - {put_string,Len,{string,String},Dst}; resolve_inst({put_list,[Src1,Src2,Dst]},_,_,_) -> {put_list,resolve_arg(Src1),resolve_arg(Src2),Dst}; resolve_inst({put_tuple,[{u,Arity},Dst]},_,_,_) -> @@ -859,9 +815,6 @@ resolve_inst({case_end,[X]},_,_,_) -> {case_end,resolve_arg(X)}; resolve_inst({call_fun,[{u,N}]},_,_,_) -> {call_fun,N}; -resolve_inst({make_fun,Args},_,_,Lbls) -> - [{f,L},Magic,FreeVars] = resolve_args(Args), - {make_fun,lookup(L,Lbls),Magic,FreeVars}; resolve_inst({is_function=I,Args0},_,_,_) -> [L|Args] = resolve_args(Args0), {test,I,L,Args}; @@ -870,30 +823,6 @@ resolve_inst({call_ext_only,[{u,N},{u,MFAix}]},Imports,_,_) -> %% %% Instructions for handling binaries added in R7A & R7B %% -resolve_inst({bs_start_match,[F,Reg]},_,_,_) -> - {bs_start_match,F,Reg}; -resolve_inst({bs_get_integer=I,[Lbl,Arg2,{u,N},{u,U},Arg5]},_,_,_) -> - [A2,A5] = resolve_args([Arg2,Arg5]), - {test,I,Lbl,[A2,N,decode_field_flags(U),A5]}; -resolve_inst({bs_get_float=I,[Lbl,Arg2,{u,N},{u,U},Arg5]},_,_,_) -> - [A2,A5] = resolve_args([Arg2,Arg5]), - {test,I,Lbl,[A2,N,decode_field_flags(U),A5]}; -resolve_inst({bs_get_binary=I,[Lbl,Arg2,{u,N},{u,U},Arg5]},_,_,_) -> - [A2,A5] = resolve_args([Arg2,Arg5]), - {test,I,Lbl,[A2,N,decode_field_flags(U),A5]}; -resolve_inst({bs_skip_bits,[Lbl,Arg2,{u,N},{u,U}]},_,_,_) -> - A2 = resolve_arg(Arg2), - {test,bs_skip_bits,Lbl,[A2,N,decode_field_flags(U)]}; -resolve_inst({bs_test_tail,[F,{u,N}]},_,_,_) -> - {test,bs_test_tail,F,[N]}; -resolve_inst({bs_save,[{u,N}]},_,_,_) -> - {bs_save,N}; -resolve_inst({bs_restore,[{u,N}]},_,_,_) -> - {bs_restore,N}; -resolve_inst({bs_init,[{u,N},{u,U}]},_,_,_) -> - {bs_init,N,decode_field_flags(U)}; -resolve_inst({bs_final,[F,X]},_,_,_) -> - {bs_final,F,X}; resolve_inst({bs_put_integer,[Lbl,Arg2,{u,N},{u,U},Arg5]},_,_,_) -> [A2,A5] = resolve_args([Arg2,Arg5]), {bs_put_integer,Lbl,A2,N,decode_field_flags(U),A5}; @@ -908,8 +837,6 @@ resolve_inst({bs_put_string,[{u,Len},{u,Off}]},_,Strings,_) -> true -> "" end, {bs_put_string,Len,{string,String}}; -resolve_inst({bs_need_buf,[{u,N}]},_,_,_) -> - {bs_need_buf,N}; %% %% Instructions for handling floating point numbers added in June 2001 (R8). @@ -961,9 +888,6 @@ resolve_inst({raise,[_Reg1,_Reg2]=Regs},_,_,_) -> resolve_inst({bs_init2,[Lbl,Arg2,{u,W},{u,R},{u,F},Arg6]},_,_,_) -> [A2,A6] = resolve_args([Arg2,Arg6]), {bs_init2,Lbl,A2,W,R,decode_field_flags(F),A6}; -resolve_inst({bs_bits_to_bytes,[Lbl,Arg2,Arg3]},_,_,_) -> - [A2,A3] = resolve_args([Arg2,Arg3]), - {bs_bits_to_bytes,Lbl,A2,A3}; resolve_inst({bs_add=I,[Lbl,Arg2,Arg3,Arg4,Arg5]},_,_,_) -> [A2,A3,A4,A5] = resolve_args([Arg2,Arg3,Arg4,Arg5]), {I,Lbl,[A2,A3,A4],A5}; @@ -1041,12 +965,6 @@ resolve_inst({gc_bif3,Args},Imports,_,_) -> {gc_bif,BifName,F,Live,[A1,A2,A3],Reg}; %% -%% New instructions for creating non-byte aligned binaries. -%% -resolve_inst({bs_final2,[X,Y]},_,_,_) -> - {bs_final2,X,Y}; - -%% %% R11B-5. %% resolve_inst({is_bitstr=I,Args0},_,_,_) -> @@ -1165,6 +1083,13 @@ resolve_inst({get_map_elements,Args0},_,_,_) -> {get_map_elements,FLbl,Src,{list,List}}; %% +%% OTP 21. +%% + +resolve_inst({build_stacktrace,[]},_,_,_) -> + build_stacktrace; + +%% %% Catches instructions that are not yet handled. %% resolve_inst(X,_,_,_) -> ?exit({resolve_inst,X}). diff --git a/lib/compiler/src/beam_disasm.hrl b/lib/compiler/src/beam_disasm.hrl index d968cd9587..8cc0bcf99b 100644 --- a/lib/compiler/src/beam_disasm.hrl +++ b/lib/compiler/src/beam_disasm.hrl @@ -26,7 +26,8 @@ %% IT SHOULD BE MOVED TO A FILE THAT DEFINES (AND EXPORTS) %% PROPER TYPES FOR THE SET OF BEAM INSTRUCTIONS. %% --type beam_instr() :: 'bs_init_writable' | 'fclearerror' | 'if_end' +-type beam_instr() :: 'bs_init_writable' | 'build_stacktrace' + | 'fclearerror' | 'if_end' | 'remove_message' | 'return' | 'send' | 'timeout' | tuple(). %% XXX: Very underspecified - FIX THIS diff --git a/lib/compiler/src/beam_except.erl b/lib/compiler/src/beam_except.erl index 9801c68ee2..abd39c661d 100644 --- a/lib/compiler/src/beam_except.erl +++ b/lib/compiler/src/beam_except.erl @@ -45,8 +45,7 @@ function({function,Name,Arity,CLabel,Is0}) -> Is = function_1(Is0), {function,Name,Arity,CLabel,Is} catch - Class:Error -> - Stack = erlang:get_stacktrace(), + Class:Error:Stack -> io:fwrite("Function: ~w/~w\n", [Name,Arity]), erlang:raise(Class, Error, Stack) end. diff --git a/lib/compiler/src/beam_jump.erl b/lib/compiler/src/beam_jump.erl index 0bcec9ce19..c33de217bd 100644 --- a/lib/compiler/src/beam_jump.erl +++ b/lib/compiler/src/beam_jump.erl @@ -336,17 +336,6 @@ opt([{test,_,{f,L}=Lbl,_}=I|[{label,L}|_]=Is], Acc0, St0) -> {Acc,St} = opt_useless_loads(Acc0, L, St0), opt(Is, Acc, St) end; -opt([{test,_,{f,L}=Lbl,_}=I|[{label,L}|_]=Is], Acc0, St0) -> - %% Similar to the above, except we have a fall-through rather than jump - %% Test Label Ops - %% label Label - case beam_utils:is_pure_test(I) of - false -> - opt(Is, [I|Acc0], label_used(Lbl, St0)); - true -> - {Acc,St} = opt_useless_loads(Acc0, L, St0), - opt(Is, Acc, St) - end; opt([{test,Test0,{f,L}=Lbl,Ops}=I|[{jump,To}|Is]=Is0], Acc, St) -> case is_label_defined(Is, L) of false -> diff --git a/lib/compiler/src/beam_peep.erl b/lib/compiler/src/beam_peep.erl index 9436c20b36..eb3192fe8f 100644 --- a/lib/compiler/src/beam_peep.erl +++ b/lib/compiler/src/beam_peep.erl @@ -41,8 +41,7 @@ function({function,Name,Arity,CLabel,Is0}) -> Is = beam_jump:remove_unused_labels(Is1), {function,Name,Arity,CLabel,Is} catch - Class:Error -> - Stack = erlang:get_stacktrace(), + Class:Error:Stack -> io:fwrite("Function: ~w/~w\n", [Name,Arity]), erlang:raise(Class, Error, Stack) end. diff --git a/lib/compiler/src/beam_receive.erl b/lib/compiler/src/beam_receive.erl index 468460eedf..3c8efa577c 100644 --- a/lib/compiler/src/beam_receive.erl +++ b/lib/compiler/src/beam_receive.erl @@ -82,8 +82,7 @@ function({function,Name,Arity,Entry,Is}) -> D = beam_utils:index_labels(Is), {function,Name,Arity,Entry,opt(Is, D, [])} catch - Class:Error -> - Stack = erlang:get_stacktrace(), + Class:Error:Stack -> io:fwrite("Function: ~w/~w\n", [Name,Arity]), erlang:raise(Class, Error, Stack) end. diff --git a/lib/compiler/src/beam_record.erl b/lib/compiler/src/beam_record.erl index 419089b1bc..db1053e48c 100644 --- a/lib/compiler/src/beam_record.erl +++ b/lib/compiler/src/beam_record.erl @@ -15,19 +15,12 @@ %% %% %CopyrightEnd% %% -%% File: beam_record.erl -%% Author: Björn-Egil Dahlberg -%% Created: 2014-09-03 -%% - --module(beam_record). --export([module/2]). %% Rewrite the instruction stream on tagged tuple tests. -%% Tagged tuples means a tuple of any arity with an atom as its first element. -%% Typically records, ok-tuples and error-tuples. -%% -%% from: +%% Tagged tuples means a tuple of any arity with an atom as its +%% first element, such as records and error tuples. +%% +%% From: %% ... %% {test,is_tuple,Fail,[Src]}. %% {test,test_arity,Fail,[Src,Sz]}. @@ -36,13 +29,16 @@ %% ... %% {test,is_eq_exact,Fail,[Dst,Atom]}. %% ... -%% to: +%% To: %% ... %% {test,is_tagged_tuple,Fail,[Src,Sz,Atom]}. %% ... +%% +-module(beam_record). +-export([module/2]). --import(lists, [reverse/1]). +-import(lists, [reverse/1,reverse/2]). -spec module(beam_utils:module_code(), [compile:option()]) -> {'ok',beam_utils:module_code()}. @@ -51,56 +47,85 @@ module({Mod,Exp,Attr,Fs0,Lc}, _Opt) -> Fs = [function(F) || F <- Fs0], {ok,{Mod,Exp,Attr,Fs,Lc}}. -function({function,Name,Arity,CLabel,Is}) -> +function({function,Name,Arity,CLabel,Is0}) -> try - Idx = beam_utils:index_labels(Is), - {function,Name,Arity,CLabel,rewrite(Is,Idx)} + Is1 = beam_utils:anno_defs(Is0), + Idx = beam_utils:index_labels(Is1), + Is = rewrite(reverse(Is1), Idx), + {function,Name,Arity,CLabel,Is} catch - Class:Error -> - Stack = erlang:get_stacktrace(), + Class:Error:Stack -> io:fwrite("Function: ~w/~w\n", [Name,Arity]), erlang:raise(Class, Error, Stack) end. -rewrite(Is,Idx) -> - rewrite(Is,Idx,[]). +rewrite(Is, Idx) -> + rewrite(Is, Idx, 0, []). -rewrite([{test,is_tuple,Fail,[Src]}=I1, - {test,test_arity,Fail,[Src,N]}=I2|Is],Idx,Acc) -> - case is_tagged_tuple(Is,Fail,Src,Idx) of +rewrite([{test,test_arity,Fail,[Src,N]}=TA, + {test,is_tuple,Fail,[Src]}=TT|Is], Idx, Def, Acc0) -> + case is_tagged_tuple(Acc0, Def, Fail, Src, Idx) of no -> - rewrite(Is,Idx,[I2,I1|Acc]); - {Atom,[{block,[]}|Is1]} -> - rewrite(Is1,Idx,[{test,is_tagged_tuple,Fail,[Src,N,Atom]}|Acc]); - {Atom,Is1} -> - rewrite(Is1,Idx,[{test,is_tagged_tuple,Fail,[Src,N,Atom]}|Acc]) + rewrite(Is, Idx, 0, [TT,TA|Acc0]); + {yes,Atom,Acc} -> + I = {test,is_tagged_tuple,Fail,[Src,N,Atom]}, + rewrite(Is, Idx, Def, [I|Acc]) end; -rewrite([I|Is],Idx,Acc) -> - rewrite(Is,Idx,[I|Acc]); -rewrite([],_,Acc) -> reverse(Acc). - -is_tagged_tuple([{block,[{set,[Dst],[Src],{get_tuple_element,0}}=B|Bs]}, - {test,is_eq_exact,Fail,[Dst,{atom,_}=Atom]}|Is],Fail,Src,Idx) -> +rewrite([{block,[{'%def',Def}|Bl]}|Is], Idx, _Def, Acc) -> + rewrite(Is, Idx, Def, [{block,Bl}|Acc]); +rewrite([{label,L}=I|Is], Idx0, Def, Acc) -> + Idx = beam_utils:index_label(L, Acc, Idx0), + rewrite(Is, Idx, Def, [I|Acc]); +rewrite([I|Is], Idx, Def, Acc) -> + rewrite(Is, Idx, Def, [I|Acc]); +rewrite([], _, _, Acc) -> Acc. - %% if Dst is killed in the instruction stream and at fail label, - %% we can safely remove get_tuple_element. - %% - %% if Dst is not killed in the stream, we cannot remove get_tuple_element - %% since it is referenced. - - case is_killed(Dst,Is,Fail,Idx) of - true -> {Atom,[{block,Bs}|Is]}; - false -> {Atom,[{block,[B|Bs]}|Is]} +is_tagged_tuple([{block,Bl}, + {test,is_eq_exact,Fail,[Dst,{atom,_}=Atom]}|Is], + Def, Fail, Src, Idx) -> + case is_tagged_tuple_1(Bl, Is, Fail, Src, Dst, Idx, Def, []) of + no -> + no; + {yes,[]} -> + {yes,Atom,Is}; + {yes,[_|_]=Block} -> + {yes,Atom,[{block,Block}|Is]} end; -is_tagged_tuple([{block,[{set,_,_,_}=B|Bs]}, - {test,is_eq_exact,_,_}=I|Is],Fail,Src,Idx) -> - case is_tagged_tuple([{block,Bs},I|Is],Fail,Src,Idx) of - {Atom,[{block,Bsr}|Isr]} -> {Atom,[{block,[B|Bsr]}|Isr]}; - no -> no +is_tagged_tuple(_, _, _, _, _) -> + no. + +is_tagged_tuple_1([{set,[Dst],[Src],{get_tuple_element,0}}=I|Bl], + Is, Fail, Src, Dst, Idx, Def, Acc) -> + %% Check usage of Dst to find out whether the get_tuple_element + %% is needed. + case usage(Dst, Is, Fail, Idx) of + killed -> + %% Safe to remove the get_tuple_element instruction. + {yes,reverse(Acc, Bl)}; + used -> + %% Actively used. Must keep instruction. + {yes,reverse(Acc, [I|Bl])}; + not_used -> + %% Not actually used (but must be initialized). + case is_defined(Dst, Def) of + false -> + %% Dst must be initialized, but the + %% actual value does not matter. + Kill = {set,[Dst],[nil],move}, + {yes,reverse(Acc, [Kill|Bl])}; + true -> + %% The register is previously initialized. + %% We can remove the instruction. + {yes,reverse(Acc, Bl)} + end end; -is_tagged_tuple(_Is,_Fail,_Src,_Idx) -> +is_tagged_tuple_1([I|Bl], Is, Fail, Src, Dst, Idx, Def, Acc) -> + is_tagged_tuple_1(Bl, Is, Fail, Src, Dst, Idx, Def, [I|Acc]); +is_tagged_tuple_1(_, _, _, _, _, _, _, _) -> no. -is_killed(Dst,Is,{_,Lbl},Idx) -> - beam_utils:is_killed(Dst,Is,Idx) andalso - beam_utils:is_killed_at(Dst,Lbl,Idx). +usage(Dst, Is, Fail, Idx) -> + beam_utils:usage(Dst, [{test,is_number,Fail,[nil]}|Is], Idx). + +is_defined({x,X}, Def) -> + (Def bsr X) band 1 =:= 1. diff --git a/lib/compiler/src/beam_reorder.erl b/lib/compiler/src/beam_reorder.erl index 910b7f6b0a..63bb57a1ac 100644 --- a/lib/compiler/src/beam_reorder.erl +++ b/lib/compiler/src/beam_reorder.erl @@ -35,8 +35,7 @@ function({function,Name,Arity,CLabel,Is0}) -> Is = reorder(Is0), {function,Name,Arity,CLabel,Is} catch - Class:Error -> - Stack = erlang:get_stacktrace(), + Class:Error:Stack -> io:fwrite("Function: ~w/~w\n", [Name,Arity]), erlang:raise(Class, Error, Stack) end. diff --git a/lib/compiler/src/beam_type.erl b/lib/compiler/src/beam_type.erl index 3d842a6fd3..16d5a36447 100644 --- a/lib/compiler/src/beam_type.erl +++ b/lib/compiler/src/beam_type.erl @@ -43,8 +43,7 @@ function({function,Name,Arity,CLabel,Asm0}) -> Asm = beam_utils:delete_live_annos(Asm3), {function,Name,Arity,CLabel,Asm} catch - Class:Error -> - Stack = erlang:get_stacktrace(), + Class:Error:Stack -> io:fwrite("Function: ~w/~w\n", [Name,Arity]), erlang:raise(Class, Error, Stack) end. diff --git a/lib/compiler/src/beam_utils.erl b/lib/compiler/src/beam_utils.erl index 00f396c246..e61d6a43b4 100644 --- a/lib/compiler/src/beam_utils.erl +++ b/lib/compiler/src/beam_utils.erl @@ -22,17 +22,20 @@ -module(beam_utils). -export([is_killed_block/2,is_killed/3,is_killed_at/3, - is_not_used/3, + is_not_used/3,usage/3, empty_label_index/0,index_label/3,index_labels/1,replace_labels/4, code_at/2,bif_to_test/3,is_pure_test/1, live_opt/1,delete_live_annos/1,combine_heap_needs/2, - split_even/1]). + anno_defs/1, + split_even/1 + ]). -export_type([code_index/0,module_code/0,instruction/0]). --import(lists, [map/2,member/2,sort/1,reverse/1,splitwith/2]). +-import(lists, [flatmap/2,map/2,member/2,sort/1,reverse/1,splitwith/2]). --define(is_const(Val), (element(1, Val) =:= integer orelse +-define(is_const(Val), (Val =:= nil orelse + element(1, Val) =:= integer orelse element(1, Val) =:= float orelse element(1, Val) =:= atom orelse element(1, Val) =:= literal)). @@ -59,6 +62,23 @@ {lbl :: code_index(), %Label to code index. res :: result_cache()}). %Result cache for each label. +%% usage(Register, [Instruction], State) -> killed|not_used|used. +%% Determine the usage of Register in the instruction sequence. +%% The return value is one of: +%% +%% killed - The register is not used in any way. +%% not_used - The register is referenced only by an allocating instruction +%% (the actual value does not matter). +%% used - The register is used (its value do matter). + +-spec usage(beam_asm:reg(), [instruction()], code_index()) -> + 'killed' | 'not_used' | 'used'. + +usage(R, Is, D) -> + St = #live{lbl=D,res=gb_trees:empty()}, + {Usage,_} = check_liveness(R, Is, St), + Usage. + %% is_killed_block(Register, [Instruction]) -> true|false %% Determine whether a register is killed by the instruction sequence inside @@ -281,16 +301,33 @@ delete_live_annos([]) -> []. %% combine_heap_needs(HeapNeed1, HeapNeed2) -> HeapNeed %% Combine the heap need for two allocation instructions. --spec combine_heap_needs(term(), term()) -> term(). +-type heap_need_tag() :: 'floats' | 'words'. +-type heap_need() :: non_neg_integer() | + {'alloc',[{heap_need_tag(),non_neg_integer()}]}. +-spec combine_heap_needs(heap_need(), heap_need()) -> heap_need(). -combine_heap_needs({alloc,Alloc1}, {alloc,Alloc2}) -> - {alloc,combine_alloc_lists(Alloc1, Alloc2)}; -combine_heap_needs({alloc,Alloc}, Words) when is_integer(Words) -> - {alloc,combine_alloc_lists(Alloc, [{words,Words}])}; -combine_heap_needs(Words, {alloc,Alloc}) when is_integer(Words) -> - {alloc,combine_alloc_lists(Alloc, [{words,Words}])}; combine_heap_needs(H1, H2) when is_integer(H1), is_integer(H2) -> - H1+H2. + H1 + H2; +combine_heap_needs(H1, H2) -> + combine_alloc_lists([H1,H2]). + + +%% anno_defs(Instructions) -> Instructions' +%% Add {'%def',RegisterBitmap} annotations to the beginning of +%% each block. Iff bit X is set in the the bitmap, it means +%% that {x,X} is defined when the block is entered. + +-spec anno_defs([instruction()]) -> [instruction()]. + +anno_defs(Is0) -> + {Bef,[Fi|Is1]} = + splitwith(fun({func_info,_,_,_}) -> false; + (_) -> true + end, Is0), + {func_info,_,_,Arity} = Fi, + Regs = init_def_regs(Arity), + Is = defs(Is1, Regs, #{}), + Bef ++ [Fi|Is]. %% split_even/1 %% [1,2,3,4,5,6] -> {[1,3,5],[2,4,6]} @@ -299,7 +336,6 @@ combine_heap_needs(H1, H2) when is_integer(H1), is_integer(H2) -> split_even(Rs) -> split_even(Rs, [], []). - %%% %%% Local functions. %%% @@ -317,6 +353,10 @@ check_liveness(R, [{block,Blk}|Is], St0) -> case check_liveness_block(R, Blk, St0) of {transparent,St1} -> check_liveness(R, Is, St1); + {alloc_used,St1} -> + %% Used by an allocating instruction, but value not referenced. + %% Must check the rest of the instructions. + not_used(check_liveness(R, Is, St1)); {Other,_}=Res when is_atom(Other) -> Res end; @@ -375,9 +415,16 @@ check_liveness(R, [{bs_init,_,_,none,Ss,Dst}|Is], St) -> check_liveness(R, [{bs_init,_,_,Live,Ss,Dst}|Is], St) -> case R of {x,X} -> - case X < Live orelse member(R, Ss) of - true -> {used,St}; - false -> {killed,St} + case member(R, Ss) of + true -> + {used,St}; + false -> + if + X < Live -> + not_used(check_liveness(R, Is, St)); + true -> + {killed,St} + end end; {y,_} -> case member(R, Ss) of @@ -587,7 +634,7 @@ check_liveness_ret(R, R, St) -> {used,St}; check_liveness_ret(_, _, St) -> {killed,St}. %% check_liveness_block(Reg, [Instruction], State) -> -%% {killed | not_used | used | transparent,State'} +%% {killed | not_used | used | alloc_used | transparent,State'} %% Finds out how Reg is used in the instruction sequence inside a block. %% Returns one of: %% killed - Reg is assigned a new value or killed by an @@ -595,6 +642,7 @@ check_liveness_ret(_, _, St) -> {killed,St}. %% not_used - The value is not used, but the register is referenced %% e.g. by an allocation instruction %% transparent - Reg is neither used nor killed +%% alloc_used - Used only in an allocate instruction %% used - Reg is explicitly used by an instruction %% %% '%live' annotations are not allowed. @@ -608,7 +656,7 @@ check_liveness_block({x,X}=R, [{set,Ds,Ss,{alloc,Live,Op}}|Is], St0) -> true -> case check_liveness_block_1(R, Ss, Ds, Op, Is, St0) of {killed,St} -> {not_used,St}; - {transparent,St} -> {not_used,St}; + {transparent,St} -> {alloc_used,St}; {_,_}=Res -> Res end end; @@ -722,22 +770,18 @@ label(Old, D, Fb) -> _ -> Fb(Old) end. -%% Help functions for combine_heap_needs. +%% Help function for combine_heap_needs. -combine_alloc_lists(Al1, Al2) -> - combine_alloc_lists_1(sort(Al1++Al2)). - -combine_alloc_lists_1([{words,W1},{words,W2}|T]) - when is_integer(W1), is_integer(W2) -> - [{words,W1+W2}|combine_alloc_lists_1(T)]; -combine_alloc_lists_1([{floats,F1},{floats,F2}|T]) - when is_integer(F1), is_integer(F2) -> - [{floats,F1+F2}|combine_alloc_lists_1(T)]; -combine_alloc_lists_1([{words,_}=W|T]) -> - [W|combine_alloc_lists_1(T)]; -combine_alloc_lists_1([{floats,_}=F|T]) -> - [F|combine_alloc_lists_1(T)]; -combine_alloc_lists_1([]) -> []. +combine_alloc_lists(Al0) -> + Al1 = flatmap(fun(Words) when is_integer(Words) -> + [{words,Words}]; + ({alloc,List}) -> + List + end, Al0), + Al2 = sofs:relation(Al1), + Al3 = sofs:relation_to_family(Al2), + Al4 = sofs:to_external(Al3), + [{Tag,lists:sum(L)} || {Tag,L} <- Al4]. %% live_opt/4. @@ -780,6 +824,8 @@ live_opt([{block,Bl0}|Is], Regs0, D, Acc) -> {Bl,Regs} = live_opt_block(reverse(Bl0), Regs0, D, [Live0]), Live = {'%live',live_regs(Regs),Regs}, live_opt(Is, Regs, D, [{block,[Live|Bl]}|Acc]); +live_opt([build_stacktrace=I|Is], _, D, Acc) -> + live_opt(Is, live_call(1), D, [I|Acc]); live_opt([{label,L}=I|Is], Regs, D0, Acc) -> D = gb_trees:insert(L, Regs, D0), live_opt(Is, Regs, D, [I|Acc]); @@ -950,3 +996,208 @@ split_even([], Ss, Ds) -> {reverse(Ss),reverse(Ds)}; split_even([S,D|Rs], Ss, Ds) -> split_even(Rs, [S|Ss], [D|Ds]). + +%%% +%%% Add annotations for defined registers. +%%% +%%% This analysis is done by scanning the instructions in +%%% execution order. +%%% + +defs([{apply,_}=I|Is], _Regs, D) -> + [I|defs(Is, 1, D)]; +defs([{bif,_,{f,Fail},_Src,Dst}=I|Is], Regs0, D) -> + Regs = def_regs([Dst], Regs0), + [I|defs(Is, Regs, update_regs(Fail, Regs0, D))]; +defs([{block,Block0}|Is], Regs0, D0) -> + {Block,Regs,D} = defs_list(Block0, Regs0, D0), + [{block,[{'%def',Regs0}|Block]}|defs(Is, Regs, D)]; +defs([{bs_init,{f,L},_,_,_,Dst}=I|Is], Regs0, D) -> + Regs = def_regs([Dst], Regs0), + [I|defs(Is, Regs, update_regs(L, Regs, D))]; +defs([{bs_put,{f,L},_,_}=I|Is], Regs, D) -> + [I|defs(Is, Regs, update_regs(L, Regs, D))]; +defs([build_stacktrace=I|Is], _Regs, D) -> + [I|defs(Is, 1, D)]; +defs([{call,_,_}=I|Is], _Regs, D) -> + [I|defs(Is, 1, D)]; +defs([{call_ext,_,{extfunc,M,F,A}}=I|Is], _Regs, D) -> + case erl_bifs:is_exit_bif(M, F, A) of + false -> + [I|defs(Is, 1, D)]; + true -> + [I|defs_unreachable(Is, D)] + end; +defs([{call_ext,_,_}=I|Is], _Regs, D) -> + [I|defs(Is, 1, D)]; +defs([{call_fun,_}=I|Is], _Regs, D) -> + [I|defs(Is, 1, D)]; +defs([{'catch',_,{f,L}}=I|Is], Regs, D) -> + RegsAtLabel = init_def_regs(1), + [I|defs(Is, Regs, update_regs(L, RegsAtLabel, D))]; +defs([{catch_end,_}=I|Is], _Regs, D) -> + Regs = init_def_regs(1), + [I|defs(Is, Regs, D)]; +defs([{gc_bif,_,{f,Fail},Live,_Src,Dst}=I|Is], Regs0, D) -> + true = all_defined(Live, Regs0), %Assertion. + Regs = def_regs([Dst], init_def_regs(Live)), + [I|defs(Is, Regs, update_regs(Fail, Regs0, D))]; +defs([{get_map_elements,{f,L},_Src,{list,DstList}}=I|Is], Regs0, D) -> + {_,Ds} = beam_utils:split_even(DstList), + Regs = def_regs(Ds, Regs0), + [I|defs(Is, Regs, update_regs(L, Regs0, D))]; +defs([{get_tuple_element,_,_,Dst}=I|Is], Regs0, D) -> + Regs = def_regs([Dst], Regs0), + [I|defs(Is, Regs, D)]; +defs([{jump,{f,L}}=I|Is], Regs, D) -> + [I|defs_unreachable(Is, update_regs(L, Regs, D))]; +defs([{label,L}=I|Is], Regs0, D) -> + case D of + #{L:=Regs1} -> + Regs = Regs0 band Regs1, + [I|defs(Is, Regs, D)]; + #{} -> + [I|defs(Is, Regs0, D)] + end; +defs([{loop_rec,{f,L},{x,0}}=I|Is], _Regs, D0) -> + RegsAtLabel = init_def_regs(0), + D = update_regs(L, RegsAtLabel, D0), + [I|defs(Is, init_def_regs(1), D)]; +defs([{loop_rec_end,_}=I|Is], _Regs, D) -> + [I|defs(Is, 0, D)]; +defs([{make_fun2,_,_,_,_}=I|Is], _Regs, D) -> + [I|defs(Is, 1, D)]; +defs([{move,_,Dst}=I|Is], Regs0, D) -> + Regs = def_regs([Dst], Regs0), + [I|defs(Is, Regs, D)]; +defs([{put_map,{f,Fail},_,_,Dst,_,_}=I|Is], Regs0, D) -> + Regs = def_regs([Dst], Regs0), + [I|defs(Is, Regs, update_regs(Fail, Regs0, D))]; +defs([return=I|Is], _Regs, D) -> + [I|defs_unreachable(Is, D)]; +defs([{select,_,_Src,Fail,List}=I|Is], Regs, D0) -> + D = update_list([Fail|List], Regs, D0), + [I|defs_unreachable(Is, D)]; +defs([{test,_,{f,L},_}=I|Is], Regs, D) -> + [I|defs(Is, Regs, update_regs(L, Regs, D))]; +defs([{test,_,{f,L},Live,_,Dst}=I|Is], Regs0, D) -> + true = all_defined(Live, Regs0), %Assertion. + Regs = def_regs([Dst], init_def_regs(Live)), + [I|defs(Is, Regs, update_regs(L, Regs0, D))]; +defs([{'try',_,{f,L}}=I|Is], Regs, D) -> + RegsAtLabel = init_def_regs(3), + [I|defs(Is, Regs, update_regs(L, RegsAtLabel, D))]; +defs([{try_case,_}=I|Is], _Regs, D) -> + [I|defs(Is, init_def_regs(3), D)]; +defs([{wait,_}=I|Is], _Regs, D) -> + [I|defs_unreachable(Is, D)]; +defs([{wait_timeout,_,_}=I|Is], _Regs, D) -> + [I|defs(Is, 0, D)]; + +%% Exceptions. +defs([{badmatch,_}=I|Is], _Regs, D) -> + [I|defs_unreachable(Is, D)]; +defs([{case_end,_}=I|Is], _Regs, D) -> + [I|defs_unreachable(Is, D)]; +defs([if_end=I|Is], _Regs, D) -> + [I|defs_unreachable(Is, D)]; +defs([{try_case_end,_}=I|Is], _Regs, D) -> + [I|defs_unreachable(Is, D)]; + +%% Neutral instructions +defs([{bs_context_to_binary,_}=I|Is], Regs, D) -> + [I|defs(Is, Regs, D)]; +defs([{bs_restore2,_,_}=I|Is], Regs, D) -> + [I|defs(Is, Regs, D)]; +defs([{bs_save2,_,_}=I|Is], Regs, D) -> + [I|defs(Is, Regs, D)]; +defs([{deallocate,_}=I|Is], Regs, D) -> + [I|defs(Is, Regs, D)]; +defs([{kill,_}=I|Is], Regs, D) -> + [I|defs(Is, Regs, D)]; +defs([{line,_}=I|Is], Regs, D) -> + [I|defs(Is, Regs, D)]; +defs([{recv_mark,_}=I|Is], Regs, D) -> + [I|defs(Is, Regs, D)]; +defs([{recv_set,_}=I|Is], Regs, D) -> + [I|defs(Is, Regs, D)]; +defs([timeout=I|Is], Regs, D) -> + [I|defs(Is, Regs, D)]; +defs([{trim,_,_}=I|Is], Regs, D) -> + [I|defs(Is, Regs, D)]; +defs([{try_end,_}=I|Is], Regs, D) -> + [I|defs(Is, Regs, D)]; +defs([{'%',_}=I|Is], Regs, D) -> + [I|defs(Is, Regs, D)]; +defs([], _, _) -> []. + +defs_unreachable([{label,L}=I|Is], D) -> + case D of + #{L:=Regs} -> + [I|defs(Is, Regs, D)]; + #{} -> + defs_unreachable(Is, D) + end; +defs_unreachable([_|Is], D) -> + defs_unreachable(Is, D); +defs_unreachable([], _D) -> []. + +defs_list(Is, Regs, D) -> + defs_list(Is, Regs, D, []). + +defs_list([{set,Ds,_,{alloc,Live,Info}}=I|Is], Regs0, D0, Acc) -> + true = all_defined(Live, Regs0), %Assertion. + D = case Info of + {gc_bif,_,{f,Fail}} -> + update_regs(Fail, Regs0, D0); + {put_map,_,{f,Fail}} -> + update_regs(Fail, Regs0, D0); + _ -> + D0 + end, + Regs = def_regs(Ds, init_def_regs(Live)), + defs_list(Is, Regs, D, [I|Acc]); +defs_list([{set,Ds,_,Info}=I|Is], Regs0, D0, Acc) -> + D = case Info of + {bif,_,{f,Fail}} -> + update_regs(Fail, Regs0, D0); + {try_catch,'catch',{f,Fail}} -> + update_regs(Fail, init_def_regs(1), D0); + {try_catch,'try',{f,Fail}} -> + update_regs(Fail, init_def_regs(3), D0); + _ -> + D0 + end, + Regs = def_regs(Ds, Regs0), + defs_list(Is, Regs, D, [I|Acc]); +defs_list([], Regs, D, Acc) -> + {reverse(Acc),Regs,D}. + +init_def_regs(Arity) -> + (1 bsl Arity) - 1. + +def_regs([{x,X}|T], Regs) -> + def_regs(T, Regs bor (1 bsl X)); +def_regs([_|T], Regs) -> + def_regs(T, Regs); +def_regs([], Regs) -> Regs. + +update_list([{f,L}|T], Regs, D0) -> + D = update_regs(L, Regs, D0), + update_list(T, Regs, D); +update_list([_|T], Regs, D) -> + update_list(T, Regs, D); +update_list([], _Regs, D) -> D. + +update_regs(L, Regs0, D) -> + case D of + #{L:=Regs1} -> + Regs = Regs0 band Regs1, + D#{L:=Regs}; + #{} -> + D#{L=>Regs0} + end. + +all_defined(Live, Regs) -> + All = (1 bsl Live) - 1, + Regs band All =:= All. diff --git a/lib/compiler/src/beam_validator.erl b/lib/compiler/src/beam_validator.erl index be8908dd6b..2ad9747940 100644 --- a/lib/compiler/src/beam_validator.erl +++ b/lib/compiler/src/beam_validator.erl @@ -130,9 +130,8 @@ validate_0(Module, [{function,Name,Ar,Entry,Code}|Fs], Ft) -> throw:Error -> %% Controlled error. [Error|validate_0(Module, Fs, Ft)]; - Class:Error -> + Class:Error:Stack -> %% Crash. - Stack = erlang:get_stacktrace(), io:fwrite("Function: ~w/~w\n", [Name,Ar]), erlang:raise(Class, Error, Stack) end. @@ -294,6 +293,8 @@ valfun_1({bs_context_to_binary,Ctx}, #vst{current=#st{x=Xs}}=Vst) -> end; valfun_1(bs_init_writable=I, Vst) -> call(I, 1, Vst); +valfun_1(build_stacktrace=I, Vst) -> + call(I, 1, Vst); valfun_1({move,{y,_}=Src,{y,_}=Dst}, Vst) -> %% The stack trimming optimization may generate a move from an initialized %% but unassigned Y register to another Y register. diff --git a/lib/compiler/src/beam_z.erl b/lib/compiler/src/beam_z.erl index 787e33c142..1c56b95a9e 100644 --- a/lib/compiler/src/beam_z.erl +++ b/lib/compiler/src/beam_z.erl @@ -38,8 +38,7 @@ function({function,Name,Arity,CLabel,Is0}) -> Is = undo_renames(Is0), {function,Name,Arity,CLabel,Is} catch - Class:Error -> - Stack = erlang:get_stacktrace(), + Class:Error:Stack -> io:fwrite("Function: ~w/~w\n", [Name,Arity]), erlang:raise(Class, Error, Stack) end. diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index 327fecf0e6..770aa2c6c1 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -203,7 +203,12 @@ expand_opts(Opts0) -> {_,_,undefined} -> [debug_info|Opts0]; {_,_,_} -> Opts0 end, - foldr(fun expand_opt/2, [], Opts). + %% iff,unless processing is to complex... + Opts1 = case proplists:is_defined(makedep_side_effect,Opts) of + true -> proplists:delete(makedep,Opts); + false -> Opts + end, + foldr(fun expand_opt/2, [], Opts1). expand_opt(basic_validation, Os) -> [no_code_generation,to_pp,binary|Os]; @@ -674,6 +679,7 @@ select_list_passes_1([], _, Acc) -> standard_passes() -> [?pass(transform_module), + {iff,makedep_side_effect,?pass(makedep_and_output)}, {iff,makedep,[ ?pass(makedep), {unless,binary,?pass(makedep_output)} @@ -1128,6 +1134,16 @@ core_lint_module(Code, St) -> errors=St#compile.errors ++ Es}} end. +%% makedep + output and continue +makedep_and_output(Code0, St) -> + {ok,DepCode,St1} = makedep(Code0,St), + case makedep_output(DepCode, St1) of + {ok,_IgnoreCode,St2} -> + {ok,Code0,St2}; + {error,St2} -> + {error,St2} + end. + makedep(Code0, #compile{ifile=Ifile,ofile=Ofile,options=Opts}=St) -> %% Get the target of the Makefile rule. @@ -1530,15 +1546,14 @@ native_compile_1(Code, St) -> {error,St#compile{errors=St#compile.errors ++ Es}} end catch - Class:R -> - Stk = erlang:get_stacktrace(), + Class:R:Stack -> case IgnoreErrors of true -> Ws = [{St#compile.ifile, - [{none,?MODULE,{native_crash,R,Stk}}]}], + [{none,?MODULE,{native_crash,R,Stack}}]}], {ok,St#compile{warnings=St#compile.warnings ++ Ws}}; false -> - erlang:raise(Class, R, Stk) + erlang:raise(Class, R, Stack) end end. diff --git a/lib/compiler/src/core_lint.erl b/lib/compiler/src/core_lint.erl index 7d3513c0ba..6e2114be56 100644 --- a/lib/compiler/src/core_lint.erl +++ b/lib/compiler/src/core_lint.erl @@ -353,12 +353,6 @@ expr(#c_case{arg=Arg,clauses=Cs}, Def, Rt, St0) -> Pc = case_patcount(Cs), St1 = body(Arg, Def, Pc, St0), clauses(Cs, Def, Pc, Rt, St1); -expr(#c_receive{clauses=Cs,timeout=#c_literal{val=infinity}, - action=#c_literal{}}, - Def, Rt, St) -> - %% If the timeout is 'infinity', the after code can never - %% be reached. We don't care if the return count is wrong. - clauses(Cs, Def, 1, Rt, St); expr(#c_receive{clauses=Cs,timeout=T,action=A}, Def, Rt, St0) -> St1 = expr(T, Def, 1, St0), St2 = body(A, Def, Rt, St1), diff --git a/lib/compiler/src/genop.tab b/lib/compiler/src/genop.tab index b5688de339..397e478e1e 100755 --- a/lib/compiler/src/genop.tab +++ b/lib/compiler/src/genop.tab @@ -545,3 +545,12 @@ BEAM_FORMAT_NUMBER=0 ## Test the arity of Reg and jumps to Lbl if it is not N. ## Test the first element of the tuple and jumps to Lbl if it is not Atom. 159: is_tagged_tuple/4 + +# OTP 21 + +## @spec build_stacktrace +## @doc Given the raw stacktrace in x(0), build a cooked stacktrace suitable +## for human consumption. Store it in x(0). Destroys all other registers. +## Do a garbage collection if necessary to allocate space on the heap +## for the result. +160: build_stacktrace/0 diff --git a/lib/compiler/src/sys_core_alias.erl b/lib/compiler/src/sys_core_alias.erl index 63e2f7488e..1bce1577d1 100644 --- a/lib/compiler/src/sys_core_alias.erl +++ b/lib/compiler/src/sys_core_alias.erl @@ -64,8 +64,7 @@ def({#c_var{name={F,Arity}}=Name,B0}) -> erase(new_var_num), {Name,B1} catch - Class:Error -> - Stack = erlang:get_stacktrace(), + Class:Error:Stack -> io:fwrite("Function: ~w/~w\n", [F,Arity]), erlang:raise(Class, Error, Stack) end. diff --git a/lib/compiler/src/sys_core_bsm.erl b/lib/compiler/src/sys_core_bsm.erl index 3e04cc33df..37e071fafa 100644 --- a/lib/compiler/src/sys_core_bsm.erl +++ b/lib/compiler/src/sys_core_bsm.erl @@ -44,8 +44,7 @@ function([{#c_var{name={F,Arity}}=Name,B0}|Fs], FsAcc, Ws0) -> {B,Ws} -> function(Fs, [{Name,B}|FsAcc], Ws) catch - Class:Error -> - Stack = erlang:get_stacktrace(), + Class:Error:Stack -> io:fwrite("Function: ~w/~w\n", [F,Arity]), erlang:raise(Class, Error, Stack) end; diff --git a/lib/compiler/src/sys_core_dsetel.erl b/lib/compiler/src/sys_core_dsetel.erl index bd3eeae238..9e2df69b33 100644 --- a/lib/compiler/src/sys_core_dsetel.erl +++ b/lib/compiler/src/sys_core_dsetel.erl @@ -81,8 +81,7 @@ visit_module_1([{Name,F0}|Fs], Env, Acc) -> {F,_} -> visit_module_1(Fs, Env, [{Name,F}|Acc]) catch - Class:Error -> - Stack = erlang:get_stacktrace(), + Class:Error:Stack -> #c_var{name={Func,Arity}} = Name, io:fwrite("Function: ~w/~w\n", [Func,Arity]), erlang:raise(Class, Error, Stack) diff --git a/lib/compiler/src/sys_core_fold.erl b/lib/compiler/src/sys_core_fold.erl index df880ff784..e28d48acf5 100644 --- a/lib/compiler/src/sys_core_fold.erl +++ b/lib/compiler/src/sys_core_fold.erl @@ -125,8 +125,7 @@ function_1({#c_var{name={F,Arity}}=Name,B0}) -> end, B0, 20), {Name,B} catch - Class:Error -> - Stack = erlang:get_stacktrace(), + Class:Error:Stack -> io:fwrite("Function: ~w/~w\n", [F,Arity]), erlang:raise(Class, Error, Stack) end. @@ -415,6 +414,8 @@ expr(#c_call{module=M0,name=N0}=Call0, Ctxt, Sub) -> no -> call(Call, M1, N1, Sub); {yes,Seq} -> expr(Seq, Ctxt, Sub) end; +expr(#c_primop{name=#c_literal{val=build_stacktrace}}, effect, _Sub) -> + void(); expr(#c_primop{args=As0}=Prim, _, Sub) -> As1 = expr_list(As0, value, Sub), Prim#c_primop{args=As1}; @@ -2622,9 +2623,13 @@ delay_build_expr_1(#c_receive{clauses=Cs0, timeout=Timeout, action=A0}=Rec, TypeSig) -> Cs = delay_build_cs(Cs0, TypeSig), - A = case Timeout of - #c_literal{val=infinity} -> A0; - _ -> delay_build_expr(A0, TypeSig) + A = case {Timeout,A0} of + {#c_literal{val=infinity},#c_literal{}} -> + {_Type,Arity} = TypeSig, + Es = lists:duplicate(Arity, A0), + core_lib:make_values(Es); + _ -> + delay_build_expr(A0, TypeSig) end, Rec#c_receive{clauses=Cs,action=A}; delay_build_expr_1(#c_seq{body=B0}=Seq, TypeSig) -> diff --git a/lib/compiler/src/v3_codegen.erl b/lib/compiler/src/v3_codegen.erl index 006a6a82d2..8f3399d133 100644 --- a/lib/compiler/src/v3_codegen.erl +++ b/lib/compiler/src/v3_codegen.erl @@ -51,7 +51,7 @@ set_kanno(Kthing, Anno) -> setelement(2, Kthing, Anno). %% Stack/register state record. -record(sr, {reg=[], %Register table stk=[], %Stack table - res=[]}). %Reserved regs: [{reserved,I,V}] + res=[]}). %Registers to reserve %% Internal records. -record(cg_need_heap, {anno=[] :: term(), @@ -77,10 +77,15 @@ functions(Forms, AtomMod) -> function(#k_fdef{anno=#k{a=Anno},func=Name,arity=Arity, vars=As,body=Kb}, AtomMod, St0) -> try - %% Annotate kernel records with variable usage. #k_match{} = Kb, %Assertion. + + %% Try to suppress the stack frame unless it is + %% really needed. + Body0 = avoid_stack_frame(Kb), + + %% Annotate kernel records with variable usage. Vdb0 = init_vars(As), - {Body,_,Vdb} = body(Kb, 1, Vdb0), + {Body,_,Vdb} = body(Body0, 1, Vdb0), %% Generate the BEAM assembly code. {Asm,EntryLabel,St} = cg_fun(Body, As, Vdb, AtomMod, @@ -88,12 +93,141 @@ function(#k_fdef{anno=#k{a=Anno},func=Name,arity=Arity, Func = {function,Name,Arity,EntryLabel,Asm}, {Func,St} catch - Class:Error -> - Stack = erlang:get_stacktrace(), + Class:Error:Stack -> io:fwrite("Function: ~w/~w\n", [Name,Arity]), erlang:raise(Class, Error, Stack) end. + +%% avoid_stack_frame(Kernel) -> Kernel' +%% If possible, avoid setting up a stack frame. Functions +%% that only do matching, calls to guard BIFs, and tail-recursive +%% calls don't need a stack frame. + +avoid_stack_frame(#k_match{body=Body}=M) -> + try + M#k_match{body=avoid_stack_frame_1(Body)} + catch + impossible -> + M + end. + +avoid_stack_frame_1(#k_alt{first=First0,then=Then0}=Alt) -> + First = avoid_stack_frame_1(First0), + Then = avoid_stack_frame_1(Then0), + Alt#k_alt{first=First,then=Then}; +avoid_stack_frame_1(#k_bif{op=Op}=Bif) -> + case Op of + #k_internal{} -> + %% Most internal BIFs clobber the X registers. + throw(impossible); + _ -> + Bif + end; +avoid_stack_frame_1(#k_break{anno=Anno,args=Args}) -> + #k_guard_break{anno=Anno,args=Args}; +avoid_stack_frame_1(#k_guard_break{}=Break) -> + Break; +avoid_stack_frame_1(#k_enter{}=Enter) -> + %% Tail-recursive calls don't need a stack frame. + Enter; +avoid_stack_frame_1(#k_guard{clauses=Cs0}=Guard) -> + Cs = avoid_stack_frame_list(Cs0), + Guard#k_guard{clauses=Cs}; +avoid_stack_frame_1(#k_guard_clause{guard=G0,body=B0}=C) -> + G = avoid_stack_frame_1(G0), + B = avoid_stack_frame_1(B0), + C#k_guard_clause{guard=G,body=B}; +avoid_stack_frame_1(#k_match{anno=A,vars=Vs,body=B0,ret=Ret}) -> + %% Use #k_guard_match{} instead to avoid saving the X registers + %% to the stack before matching. + B = avoid_stack_frame_1(B0), + #k_guard_match{anno=A,vars=Vs,body=B,ret=Ret}; +avoid_stack_frame_1(#k_guard_match{body=B0}=M) -> + B = avoid_stack_frame_1(B0), + M#k_guard_match{body=B}; +avoid_stack_frame_1(#k_protected{arg=Arg0}=Prot) -> + Arg = avoid_stack_frame_1(Arg0), + Prot#k_protected{arg=Arg}; +avoid_stack_frame_1(#k_put{}=Put) -> + Put; +avoid_stack_frame_1(#k_return{}=Ret) -> + Ret; +avoid_stack_frame_1(#k_select{var=#k_var{anno=Vanno},types=Types0}=Select) -> + case member(reuse_for_context, Vanno) of + false -> + Types = avoid_stack_frame_list(Types0), + Select#k_select{types=Types}; + true -> + %% Including binary patterns that overwrite the register containing + %% the binary with the match context may not be safe. For example, + %% bs_match_SUITE:bin_tail_e/1 with inlining will be rejected by + %% beam_validator. + %% + %% Essentially the following code is produced: + %% + %% bs_match {x,0} => {x,0} + %% ... + %% bs_match {x,0} => {x,1} %% ILLEGAL + %% + %% A bs_match instruction will only accept a match context as the + %% source operand if the source and destination registers are the + %% the same (as in the first bs_match instruction above). + %% The second bs_match instruction is therefore illegal. + %% + %% This situation is avoided if there is a stack frame: + %% + %% move {x,0} => {y,0} + %% bs_match {x,0} => {x,0} + %% ... + %% bs_match {y,0} => {x,1} %% LEGAL + %% + throw(impossible) + end; +avoid_stack_frame_1(#k_seq{arg=#k_call{anno=Anno,op=Op}=Call, + body=#k_break{args=BrArgs0}}=Seq) -> + case Op of + #k_remote{mod=#k_atom{val=Mod}, + name=#k_atom{val=Name}, + arity=Arity} -> + case erl_bifs:is_exit_bif(Mod, Name, Arity) of + false -> + %% Will clobber X registers. Must have a stack frame. + throw(impossible); + true -> + %% The call to this BIF will never return. It is safe + %% to suppress the stack frame. + Bif = #k_bif{anno=Anno, + op=#k_internal{name=guard_error,arity=1}, + args=[Call],ret=[]}, + BrArgs = lists:duplicate(length(BrArgs0), #k_nil{}), + GB = #k_guard_break{anno=#k{us=[],ns=[],a=[]},args=BrArgs}, + Seq#k_seq{arg=Bif,body=GB} + end; + _ -> + %% Will clobber X registers. Must have a stack frame. + throw(impossible) + end; +avoid_stack_frame_1(#k_seq{arg=A0,body=B0}=Seq) -> + A = avoid_stack_frame_1(A0), + B = avoid_stack_frame_1(B0), + Seq#k_seq{arg=A,body=B}; +avoid_stack_frame_1(#k_test{}=Test) -> + Test; +avoid_stack_frame_1(#k_type_clause{values=Values0}=TC) -> + Values = avoid_stack_frame_list(Values0), + TC#k_type_clause{values=Values}; +avoid_stack_frame_1(#k_val_clause{body=B0}=VC) -> + B = avoid_stack_frame_1(B0), + VC#k_val_clause{body=B}; +avoid_stack_frame_1(_Body) -> + throw(impossible). + +avoid_stack_frame_list([H|T]) -> + [avoid_stack_frame_1(H)|avoid_stack_frame_list(T)]; +avoid_stack_frame_list([]) -> []. + + %% This pass creates beam format annotated with variable lifetime %% information. Each thing is given an index and for each variable we %% store the first and last index for its occurrence. The variable @@ -219,10 +353,8 @@ expr(#k_put{anno=A}=Put, I, _Vdb) -> Put#k_put{anno=#l{i=I,a=A#k.a}}; expr(#k_break{anno=A}=Break, I, _Vdb) -> Break#k_break{anno=#l{i=I,a=A#k.a}}; -expr(#k_guard_break{anno=A}=Break, I, Vdb) -> - Locked = [V || {V,_,_} <- Vdb], - L = #l{i=I,a=A#k.a}, - Break#k_guard_break{anno=L,locked=Locked}; +expr(#k_guard_break{anno=A}=Break, I, _Vdb) -> + Break#k_guard_break{anno=#l{i=I,a=A#k.a}}; expr(#k_return{anno=A}=Ret, I, _Vdb) -> Ret#k_return{anno=#l{i=I,a=A#k.a}}. @@ -246,14 +378,9 @@ match(#k_alt{anno=A,first=Kf,then=Kt}, Ls, I, Vdb0) -> F = match(Kf, Ls, I+1, Vdb1), T = match(Kt, Ls, I+1, Vdb1), #k_alt{anno=[],first=F,then=T}; -match(#k_select{anno=A,var=V,types=Kts}=Select, Ls0, I, Vdb0) -> - Vanno = get_kanno(V), - Ls1 = case member(no_usage, Vanno) of - false -> add_element(V#k_var.name, Ls0); - true -> Ls0 - end, - Vdb1 = use_vars(union(A#k.us, Ls1), I, Vdb0), - Ts = [type_clause(Tc, Ls1, I+1, Vdb1) || Tc <- Kts], +match(#k_select{anno=A,types=Kts}=Select, Ls, I, Vdb0) -> + Vdb1 = use_vars(union(A#k.us, Ls), I, Vdb0), + Ts = [type_clause(Tc, Ls, I+1, Vdb1) || Tc <- Kts], Select#k_select{anno=[],types=Ts}; match(#k_guard{anno=A,clauses=Kcs}, Ls, I, Vdb0) -> Vdb1 = use_vars(union(A#k.us, Ls), I, Vdb0), @@ -343,7 +470,7 @@ cg_fun(Les, Hvs, Vdb, AtomMod, NameArity, Anno, St0) -> put_reg(V, Reg) end, [], Hvs), stk=[]}, 0, Vdb), - {B,_Aft,St} = cg_list(Les, 0, Vdb, Bef, + {B,_Aft,St} = cg_list(Les, Vdb, Bef, St3#cg{bfail=0, ultimate_failure=UltimateMatchFail, is_top_block=true}), @@ -388,26 +515,26 @@ cg(#k_return{anno=Le,args=Rs}, Vdb, Bef, St) -> return_cg(Rs, Le, Vdb, Bef, St); cg(#k_break{anno=Le,args=Bs}, Vdb, Bef, St) -> break_cg(Bs, Le, Vdb, Bef, St); -cg(#k_guard_break{anno=Le,args=Bs,locked=N}, Vdb, Bef, St) -> - guard_break_cg(Bs, N, Le, Vdb, Bef, St); +cg(#k_guard_break{anno=Le,args=Bs}, Vdb, Bef, St) -> + guard_break_cg(Bs, Le, Vdb, Bef, St); cg(#cg_need_heap{h=H}, _Vdb, Bef, St) -> {[{test_heap,H,max_reg(Bef#sr.reg)}],Bef,St}. %% cg_list([Kexpr], FirstI, Vdb, StackReg, St) -> {[Ainstr],StackReg,St}. -cg_list(Kes, I, Vdb, Bef, St0) -> +cg_list(Kes, Vdb, Bef, St0) -> {Keis,{Aft,St1}} = flatmapfoldl(fun (Ke, {Inta,Sta}) -> {Keis,Intb,Stb} = cg(Ke, Vdb, Inta, Sta), {Keis,{Intb,Stb}} - end, {Bef,St0}, need_heap(Kes, I)), + end, {Bef,St0}, need_heap(Kes)), {Keis,Aft,St1}. %% need_heap([Lkexpr], I, St) -> [Lkexpr]. %% Insert need_heap instructions in Kexpr list. Try to be smart and %% collect them together as much as possible. -need_heap(Kes0, _I) -> +need_heap(Kes0) -> {Kes,H} = need_heap_0(reverse(Kes0), 0, []), %% Prepend need_heap if necessary. @@ -487,7 +614,10 @@ match_cg(M, Rs, Le, Vdb, Bef, St0) -> guard_match_cg(M, Rs, Le, Vdb, Bef, St0) -> I = Le#l.i, {B,St1} = new_label(St0), - #cg{bfail=Fail} = St1, + Fail = case St0 of + #cg{bfail=0,ultimate_failure=Fail0} -> Fail0; + #cg{bfail=Fail0} -> Fail0 + end, {Mis,Aft,St2} = match_cg(M, Fail, Bef, St1#cg{break=B}), %% Update the register descriptors for the return registers. Reg = guard_match_regs(Aft#sr.reg, Rs), @@ -593,9 +723,6 @@ bsm_rename_ctx(#k_protected{arg=Ts0}=Prot, Old, New, _InProt) -> InProt = true, Ts = bsm_rename_ctx_list(Ts0, Old, New, InProt), bsm_forget_var(Prot#k_protected{arg=Ts}, Old); -bsm_rename_ctx(#k_match{body=Ms0}=Match, Old, New, InProt) -> - Ms = bsm_rename_ctx(Ms0, Old, New, InProt), - Match#k_match{body=Ms}; bsm_rename_ctx(#k_guard_match{body=Ms0}=Match, Old, New, InProt) -> Ms = bsm_rename_ctx(Ms0, Old, New, InProt), Match#k_guard_match{body=Ms}; @@ -612,9 +739,8 @@ bsm_rename_ctx(#cg_block{es=Es0}=Block, Old, New, true) -> %% inside the block. Es = bsm_rename_ctx_list(Es0, Old, New, true), bsm_forget_var(Block#cg_block{es=Es}, Old); -bsm_rename_ctx(#k_guard_break{locked=Locked0}=Break, Old, _New, _InProt) -> - Locked = Locked0 -- [Old], - bsm_forget_var(Break#k_guard_break{locked=Locked}, Old). +bsm_rename_ctx(#k_guard_break{}=Break, Old, _New, _InProt) -> + bsm_forget_var(Break, Old). bsm_rename_ctx_list([C|Cs], Old, New, InProt) -> [bsm_rename_ctx(C, Old, New, InProt)| @@ -639,26 +765,55 @@ block_cg(Es, Le, _Vdb, Bef, St) -> block_cg(Es, Le, Bef, St). block_cg(Es, Le, Bef, #cg{is_top_block=false}=St) -> - cg_block(Es, Le#l.i, Le#l.vdb, Bef, St); -block_cg(Es, Le, Bef, St0) -> - {Is0,Aft,St} = cg_block(Es, Le#l.i, Le#l.vdb, Bef, - St0#cg{is_top_block=false,need_frame=false}), - Is = top_level_block(Is0, Aft, max_reg(Bef#sr.reg), St), - {Is,Aft,St#cg{is_top_block=true}}. - -cg_block([], _I, _Vdb, Bef, St0) -> + cg_block(Es, Le#l.vdb, Bef, St); +block_cg(Es, Le, Bef, #cg{is_top_block=true}=St0) -> + %% No stack frame has been established yet. Do we need one? + case need_stackframe(Es) of + true -> + %% We need a stack frame. Generate the code and add the + %% code for creating and deallocating the stack frame. + {Is0,Aft,St} = cg_block(Es, Le#l.vdb, Bef, + St0#cg{is_top_block=false,need_frame=false}), + Is = top_level_block(Is0, Aft, max_reg(Bef#sr.reg), St), + {Is,Aft,St#cg{is_top_block=true}}; + false -> + %% This sequence of instructions ending in a #k_match{} (a + %% 'case' or 'if') in the Erlang code does not need a + %% stack frame yet. Delay the creation (if a stack frame + %% is needed at all, it will be created inside the + %% #k_match{}). + cg_list(Es, Le#l.vdb, Bef, St0) + end. + +%% need_stackframe([Kexpr]) -> true|false. +%% Does this list of instructions need a stack frame? +%% +%% A sequence of instructions that don't clobber the X registers +%% followed by a single #k_match{} doesn't need a stack frame. + +need_stackframe([H|T]) -> + case H of + #k_bif{op=#k_internal{}} -> true; + #k_put{arg=#k_binary{}} -> true; + #k_bif{} -> need_stackframe(T); + #k_put{} -> need_stackframe(T); + #k_guard_match{} -> need_stackframe(T); + #k_match{} when T =:= [] -> false; + _ -> true + end; +need_stackframe([]) -> false. + +cg_block([], _Vdb, Bef, St0) -> {[],Bef,St0}; -cg_block(Kes0, I, Vdb, Bef, St0) -> +cg_block(Kes0, Vdb, Bef, St0) -> {Kes2,Int1,St1} = case basic_block(Kes0) of {Kes1,LastI,Args,Rest} -> - Ke = hd(Kes1), - #l{i=Fb} = get_kanno(Ke), - cg_basic_block(Kes1, Fb, LastI, Args, Vdb, Bef, St0); + cg_basic_block(Kes1, LastI, Args, Vdb, Bef, St0); {Kes1,Rest} -> - cg_list(Kes1, I, Vdb, Bef, St0) + cg_list(Kes1, Vdb, Bef, St0) end, - {Kes3,Int2,St2} = cg_block(Rest, I, Vdb, Int1, St1), + {Kes3,Int2,St2} = cg_block(Rest, Vdb, Int1, St1), {Kes2 ++ Kes3,Int2,St2}. basic_block(Kes) -> basic_block(Kes, []). @@ -679,129 +834,158 @@ basic_block([Ke|Kes], Acc) -> no_block -> {reverse(Acc, [Ke]),Kes} end. -%% #k_put{} instructions that may garbage collect are not allowed in basic blocks. - -collect_block(#k_put{arg=#k_binary{}}) -> - no_block; -collect_block(#k_put{arg=#k_map{}}) -> - no_block; -collect_block(#k_put{}) -> - include; -collect_block(#k_call{op=#k_var{}=Var,args=As}) -> - {block_end,As++[Var]}; +collect_block(#k_put{arg=Arg}) -> + %% #k_put{} instructions that may garbage collect are not allowed + %% in basic blocks. + case Arg of + #k_binary{} -> no_block; + #k_map{} -> no_block; + _ -> include + end; collect_block(#k_call{op=Func,args=As}) -> {block_end,As++func_vars(Func)}; -collect_block(#k_enter{op=#k_var{}=Var,args=As}) -> - {block_end,As++[Var]}; collect_block(#k_enter{op=Func,args=As}) -> {block_end,As++func_vars(Func)}; collect_block(#k_return{args=Rs}) -> {block_end,Rs}; collect_block(#k_break{args=Bs}) -> {block_end,Bs}; -collect_block(_) -> no_block. +collect_block(_) -> no_block. +func_vars(#k_var{}=Var) -> + [Var]; func_vars(#k_remote{mod=M,name=F}) when is_record(M, k_var); is_record(F, k_var) -> [M,F]; func_vars(_) -> []. -%% cg_basic_block([Kexpr], FirstI, LastI, As, Vdb, StackReg, State) -> +%% cg_basic_block([Kexpr], FirstI, LastI, Arguments, Vdb, StackReg, State) -> %% {[Ainstr],StackReg,State}. - -cg_basic_block(Kes, Fb, Lf, As, Vdb, Bef, St0) -> - Res = make_reservation(As, 0), - Regs0 = reserve(Res, Bef#sr.reg, Bef#sr.stk), - Stk = extend_stack(Bef, Lf, Lf+1, Vdb), - Int0 = Bef#sr{reg=Regs0,stk=Stk,res=Res}, - X0_v0 = x0_vars(As, Fb, Lf, Vdb), - {Keis,{Aft,_,St1}} = +%% +%% Do a specialized code generation for a basic block of #put{} +%% instructions (that don't do any garbage collection) followed by a +%% call, break, or return. +%% +%% 'Arguments' is a list of the variables that must be loaded into +%% consecutive X registers before the last instruction in the block. +%% The point of this specialized code generation is to try put the +%% all of the variables in 'Arguments' into the correct X register +%% to begin with, instead of putting them into the first available +%% X register and having to move them to the correct X register +%% later. +%% +%% To achieve that, we attempt to reserve the X registers that the +%% variables in 'Arguments' will need to be in when the block ends. +%% +%% To make it more likely that reservations will be successful, we +%% will try to save variables that need to be saved to the stack as +%% early as possible (if an X register needed by a variable in +%% Arguments is occupied by another variable, the value in the +%% X register can be evicted if it is saved on the stack). +%% +%% We will take care not to increase the size of stack frame compared +%% to what the standard code generator would have done (that is, to +%% save all X registers at the last possible moment). We will do that +%% by extending the stack frame to the minimal size needed to save +%% all that needs to be saved using extend_stack/4, and use +%% save_carefully/4 during code generation to only save the variables +%% that can be saved without growing the stack frame. + +cg_basic_block(Kes, Lf, As, Vdb, Bef, St0) -> + Int0 = reserve_arg_regs(As, Bef), + Int = extend_stack(Int0, Lf, Lf+1, Vdb), + {Keis,{Aft,St1}} = flatmapfoldl(fun(Ke, St) -> cg_basic_block(Ke, St, Lf, Vdb) end, - {Int0,X0_v0,St0}, need_heap(Kes, Fb)), + {Int,St0}, need_heap(Kes)), {Keis,Aft,St1}. -cg_basic_block(#cg_need_heap{}=Ke, {Inta,X0v,Sta}, _Lf, Vdb) -> - {Keis,Intb,Stb} = cg(Ke, Vdb, Inta, Sta), - {Keis, {Intb,X0v,Stb}}; -cg_basic_block(Ke, {Inta,X0_v1,Sta}, Lf, Vdb) -> +cg_basic_block(#cg_need_heap{}=Ke, {Bef,St0}, _Lf, Vdb) -> + {Keis,Aft,St1} = cg(Ke, Vdb, Bef, St0), + {Keis,{Aft,St1}}; +cg_basic_block(Ke, {Bef,St0}, Lf, Vdb) -> #l{i=I} = get_kanno(Ke), - {Sis,Intb} = save_carefully(Inta, I, Lf+1, Vdb), - {X0_v2,Intc} = allocate_x0(X0_v1, I, Intb), - Intd = reserve(Intc), - {Keis,Inte,Stb} = cg(Ke, Vdb, Intd, Sta), - {Sis ++ Keis, {Inte,X0_v2,Stb}}. - -make_reservation([], _) -> []; -make_reservation([#k_var{name=V}|As], I) -> [{I,V}|make_reservation(As, I+1)]; -make_reservation([A|As], I) -> [{I,A}|make_reservation(As, I+1)]. -reserve(Sr) -> Sr#sr{reg=reserve(Sr#sr.res, Sr#sr.reg, Sr#sr.stk)}. - -reserve([{I,V}|Rs], [free|Regs], Stk) -> [{reserved,I,V}|reserve(Rs, Regs, Stk)]; -reserve([{I,V}|Rs], [{I,V}|Regs], Stk) -> [{I,V}|reserve(Rs, Regs, Stk)]; -reserve([{I,V}|Rs], [{I,Var}|Regs], Stk) -> + %% Save all we can to increase the possibility that reserving + %% registers will succeed. + {Sis,Int0} = save_carefully(Bef, I, Lf+1, Vdb), + Int1 = reserve(Int0), + {Keis,Aft,St1} = cg(Ke, Vdb, Int1, St0), + {Sis ++ Keis,{Aft,St1}}. + +%% reserve_arg_regs([Argument], Bef) -> Aft. +%% Try to reserve the X registers for all arguments. All registers +%% that we wish to reserve will be saved in Bef#sr.res. + +reserve_arg_regs(As, Bef) -> + Res = reserve_arg_regs_1(As, 0), + reserve(Bef#sr{res=Res}). + +reserve_arg_regs_1([#k_var{name=V}|As], I) -> + [{I,V}|reserve_arg_regs_1(As, I+1)]; +reserve_arg_regs_1([A|As], I) -> + [{I,A}|reserve_arg_regs_1(As, I+1)]; +reserve_arg_regs_1([], _) -> []. + +%% reserve(Bef) -> Aft. +%% Try to reserve more registers. The registers we wish to reserve +%% are found in Bef#sr.res. + +reserve(#sr{reg=Regs,stk=Stk,res=Res}=Sr) -> + Sr#sr{reg=reserve_1(Res, Regs, Stk)}. + +reserve_1([{I,V}|Rs], [free|Regs], Stk) -> + [{reserved,I,V}|reserve_1(Rs, Regs, Stk)]; +reserve_1([{I,V}|Rs], [{I,V}|Regs], Stk) -> + [{I,V}|reserve_1(Rs, Regs, Stk)]; +reserve_1([{I,V}|Rs], [{I,Var}|Regs], Stk) -> case on_stack(Var, Stk) of - true -> [{reserved,I,V}|reserve(Rs, Regs, Stk)]; - false -> [{I,Var}|reserve(Rs, Regs, Stk)] + true -> [{reserved,I,V}|reserve_1(Rs, Regs, Stk)]; + false -> [{I,Var}|reserve_1(Rs, Regs, Stk)] end; -reserve([{I,V}|Rs], [{reserved,I,_}|Regs], Stk) -> - [{reserved,I,V}|reserve(Rs, Regs, Stk)]; -%reserve([{I,V}|Rs], [Other|Regs], Stk) -> [Other|reserve(Rs, Regs, Stk)]; -reserve([{I,V}|Rs], [], Stk) -> [{reserved,I,V}|reserve(Rs, [], Stk)]; -reserve([], Regs, _) -> Regs. - -extend_stack(Bef, Fb, Lf, Vdb) -> - Stk0 = clear_dead_stk(Bef#sr.stk, Fb, Vdb), - Saves = [V || {V,F,L} <- Vdb, - F < Fb, - L >= Lf, - not on_stack(V, Stk0)], - Stk1 = foldl(fun (V, Stk) -> put_stack(V, Stk) end, Stk0, Saves), - Bef#sr.stk ++ lists:duplicate(length(Stk1) - length(Bef#sr.stk), free). - -save_carefully(Bef, Fb, Lf, Vdb) -> - Stk = Bef#sr.stk, - %% New variables that are in use but not on stack. - New = [VFL || {V,F,L} = VFL <- Vdb, - F < Fb, - L >= Lf, - not on_stack(V, Stk)], - Saves = [V || {V,_,_} <- keysort(2, New)], - save_carefully(Saves, Bef, []). - -save_carefully([], Bef, Acc) -> {reverse(Acc),Bef}; -save_carefully([V|Vs], Bef, Acc) -> - case put_stack_carefully(V, Bef#sr.stk) of - error -> {reverse(Acc),Bef}; +reserve_1([{I,V}|Rs], [{reserved,I,_}|Regs], Stk) -> + [{reserved,I,V}|reserve_1(Rs, Regs, Stk)]; +reserve_1([{I,V}|Rs], [], Stk) -> + [{reserved,I,V}|reserve_1(Rs, [], Stk)]; +reserve_1([], Regs, _) -> Regs. + +%% extend_stack(Bef, FirstBefore, LastFrom, Vdb) -> Aft. +%% Extend the stack enough to fit all variables alive past LastFrom +%% and not already on the stack. + +extend_stack(#sr{stk=Stk0}=Bef, Fb, Lf, Vdb) -> + Stk1 = clear_dead_stk(Stk0, Fb, Vdb), + New = new_not_on_stack(Stk1, Fb, Lf, Vdb), + Stk2 = foldl(fun ({V,_,_}, Stk) -> put_stack(V, Stk) end, Stk1, New), + Stk = Stk0 ++ lists:duplicate(length(Stk2) - length(Stk0), free), + Bef#sr{stk=Stk}. + +%% save_carefully(Bef, FirstBefore, LastFrom, Vdb) -> {[SaveVar],Aft}. +%% Save variables which are used past current point and which are not +%% already on the stack, but only if the variables can be saved without +%% growing the stack frame. + +save_carefully(#sr{stk=Stk}=Bef, Fb, Lf, Vdb) -> + New0 = new_not_on_stack(Stk, Fb, Lf, Vdb), + New = keysort(2, New0), + save_carefully_1(New, Bef, []). + +save_carefully_1([{V,_,_}|Vs], #sr{reg=Regs,stk=Stk0}=Bef, Acc) -> + case put_stack_carefully(V, Stk0) of + error -> + {reverse(Acc),Bef}; Stk1 -> - SrcReg = fetch_reg(V, Bef#sr.reg), + SrcReg = fetch_reg(V, Regs), Move = {move,SrcReg,fetch_stack(V, Stk1)}, {x,_} = SrcReg, %Assertion - must be X register. - save_carefully(Vs, Bef#sr{stk=Stk1}, [Move|Acc]) - end. + save_carefully_1(Vs, Bef#sr{stk=Stk1}, [Move|Acc]) + end; +save_carefully_1([], Bef, Acc) -> + {reverse(Acc),Bef}. -x0_vars([], _Fb, _Lf, _Vdb) -> []; -x0_vars([#k_var{name=V}|_], Fb, _Lf, Vdb) -> - {V,F,_L} = VFL = vdb_find(V, Vdb), - x0_vars1([VFL], Fb, F, Vdb); -x0_vars([X0|_], Fb, Lf, Vdb) -> - x0_vars1([{X0,Lf,Lf}], Fb, Lf, Vdb). - -x0_vars1(X0, Fb, Xf, Vdb) -> - Vs0 = [VFL || {_V,F,L}=VFL <- Vdb, - F >= Fb, - L < Xf], - Vs1 = keysort(3, Vs0), - keysort(2, X0++Vs1). - -allocate_x0([], _, Bef) -> {[],Bef#sr{res=[]}}; -allocate_x0([{_,_,L}|Vs], I, Bef) when L =< I -> - allocate_x0(Vs, I, Bef); -allocate_x0([{V,_F,_L}=VFL|Vs], _, Bef) -> - {[VFL|Vs],Bef#sr{res=reserve_x0(V, Bef#sr.res)}}. - -reserve_x0(V, [_|Res]) -> [{0,V}|Res]; -reserve_x0(V, []) -> [{0,V}]. +%% top_level_block([Instruction], Bef, MaxRegs, St) -> [Instruction]. +%% For the top-level block, allocate a stack frame a necessary, +%% adjust Y register numbering and instructions that return +%% from the function. top_level_block(Keis, #sr{stk=[]}, _MaxRegs, #cg{need_frame=false}) -> Keis; @@ -1347,10 +1531,8 @@ guard_clause_cg(#k_guard_clause{anno=#l{vdb=Vdb},guard=G,body=B}, Fail, Bef, St0 %% the correct exit point. Primops and tests all go to the next %% instruction on success or jump to a failure label. -guard_cg(#k_protected{arg=Ts,ret=Rs,anno=#l{i=I,vdb=Pdb}}, Fail, _Vdb, Bef, St) -> - protected_cg(Ts, Rs, Fail, I, Pdb, Bef, St); -guard_cg(#cg_block{es=Ts,anno=#l{i=I,vdb=Bdb}}, Fail, _Vdb, Bef, St) -> - guard_cg_list(Ts, Fail, I, Bdb, Bef, St); +guard_cg(#k_protected{arg=Ts,ret=Rs,anno=#l{vdb=Pdb}}, Fail, _Vdb, Bef, St) -> + protected_cg(Ts, Rs, Fail, Pdb, Bef, St); guard_cg(#k_test{anno=#l{i=I},op=Test0,args=As,inverted=Inverted}, Fail, Vdb, Bef, St0) -> #k_remote{mod=#k_atom{val=erlang},name=#k_atom{val=Test}} = Test0, @@ -1368,6 +1550,18 @@ guard_cg(G, _Fail, Vdb, Bef, St) -> %%ok = io:fwrite("cg ~w: ~p~n", [?LINE,{Aft}]), {Gis,Aft,St1}. +%% guard_cg_list([Kexpr], Fail, I, Vdb, StackReg, St) -> +%% {[Ainstr],StackReg,St}. + +guard_cg_list(Kes, Fail, Vdb, Bef, St0) -> + {Keis,{Aft,St1}} = + flatmapfoldl(fun (Ke, {Inta,Sta}) -> + {Keis,Intb,Stb} = + guard_cg(Ke, Fail, Vdb, Inta, Sta), + {Keis,{Intb,Stb}} + end, {Bef,St0}, need_heap(Kes)), + {Keis,Aft,St1}. + %% protected_cg([Kexpr], [Ret], Fail, I, Vdb, Bef, St) -> {[Ainstr],Aft,St}. %% Do a protected. Protecteds without return values are just done %% for effect, the return value is not checked, success passes on to @@ -1375,15 +1569,14 @@ guard_cg(G, _Fail, Vdb, Bef, St) -> %% return values then these must be set to 'false' on failure, %% control always passes to the next instruction. -protected_cg(Ts, [], Fail, I, Vdb, Bef, St0) -> +protected_cg(Ts, [], Fail, Vdb, Bef, St0) -> %% Protect these calls, revert when done. - {Tis,Aft,St1} = guard_cg_list(Ts, Fail, I, Vdb, Bef, - St0#cg{bfail=Fail}), + {Tis,Aft,St1} = guard_cg_list(Ts, Fail, Vdb, Bef, St0#cg{bfail=Fail}), {Tis,Aft,St1#cg{bfail=St0#cg.bfail}}; -protected_cg(Ts, Rs, _Fail, I, Vdb, Bef, St0) -> +protected_cg(Ts, Rs, _Fail, Vdb, Bef, St0) -> {Pfail,St1} = new_label(St0), {Psucc,St2} = new_label(St1), - {Tis,Aft,St3} = guard_cg_list(Ts, Pfail, I, Vdb, Bef, + {Tis,Aft,St3} = guard_cg_list(Ts, Pfail, Vdb, Bef, St2#cg{bfail=Pfail}), %%ok = io:fwrite("cg ~w: ~p~n", [?LINE,{Rs,I,Vdb,Aft}]), %% Set return values to false. @@ -1424,18 +1617,6 @@ test_cg(Test, As, Fail, I, Vdb, Bef, St) -> Aft = clear_dead(Bef, I, Vdb), {[beam_utils:bif_to_test(Test, Args, {f,Fail})],Aft,St}. -%% guard_cg_list([Kexpr], Fail, I, Vdb, StackReg, St) -> -%% {[Ainstr],StackReg,St}. - -guard_cg_list(Kes, Fail, I, Vdb, Bef, St0) -> - {Keis,{Aft,St1}} = - flatmapfoldl(fun (Ke, {Inta,Sta}) -> - {Keis,Intb,Stb} = - guard_cg(Ke, Fail, Vdb, Inta, Sta), - {Keis,{Intb,Stb}} - end, {Bef,St0}, need_heap(Kes, I)), - {Keis,Aft,St1}. - %% match_fmf(Fun, LastFail, State, [Clause]) -> {Is,Aft,State}. %% This is a special flatmapfoldl for match code gen where we %% generate a "failure" label for each clause. The last clause uses @@ -1634,12 +1815,7 @@ bif_cg(#k_bif{op=#k_remote{mod=#k_atom{val=erlang},name=#k_atom{val=Name}}, internal_cg(bs_context_to_binary=Instr, [Src0], [], Le, Vdb, Bef, St0) -> [Src] = cg_reg_args([Src0], Bef), - case is_register(Src) of - false -> - {[],clear_dead(Bef, Le#l.i, Vdb), St0}; - true -> - {[{Instr,Src}],clear_dead(Bef, Le#l.i, Vdb), St0} - end; + {[{Instr,Src}],clear_dead(Bef, Le#l.i, Vdb), St0}; internal_cg(dsetelement, [Index0,Tuple0,New0], _Rs, Le, Vdb, Bef, St0) -> [New,Tuple,{integer,Index1}] = cg_reg_args([New0,Tuple0,Index0], Bef), Index = Index1-1, @@ -1661,9 +1837,25 @@ internal_cg(bs_init_writable=I, As, Rs, Le, Vdb, Bef, St) -> {Sis,Int} = cg_setup_call(As, Bef, Le#l.i, Vdb), Reg = load_vars(Rs, clear_regs(Int#sr.reg)), {Sis++[I],clear_dead(Int#sr{reg=Reg}, Le#l.i, Vdb),St}; +internal_cg(build_stacktrace=I, As, Rs, Le, Vdb, Bef, St) -> + %% This behaves like a function call. + {Sis,Int} = cg_setup_call(As, Bef, Le#l.i, Vdb), + Reg = load_vars(Rs, clear_regs(Int#sr.reg)), + {Sis++[I],clear_dead(Int#sr{reg=Reg}, Le#l.i, Vdb),St}; internal_cg(raise, As, Rs, Le, Vdb, Bef, St) -> %% raise can be treated like a guard BIF. - bif_cg(raise, As, Rs, Le, Vdb, Bef, St). + bif_cg(raise, As, Rs, Le, Vdb, Bef, St); +internal_cg(guard_error, [ExitCall], _Rs, Le, Vdb, Bef, St) -> + %% A call an exit BIF from inside a #k_guard_match{}. + %% Generate a standard call, but leave the register descriptors + %% alone, effectively pretending that there was no call. + #k_call{op=#k_remote{mod=#k_atom{val=Mod},name=#k_atom{val=Name}}, + args=As} = ExitCall, + Arity = length(As), + {Ms,_} = cg_call_args(As, Bef, Le#l.i, Vdb), + Call = {call_ext,Arity,{extfunc,Mod,Name,Arity}}, + Is = Ms++[line(Le),Call], + {Is,Bef,St}. %% bif_cg(Bif, [Arg], [Ret], Le, Vdb, StackReg, State) -> %% {[Ainstr],StackReg,State}. @@ -1767,17 +1959,17 @@ cg_recv_wait(#k_atom{val=infinity}, #cg_block{anno=Le,es=Tes}, I, Bef, St0) -> %% But to keep the stack and register information up to date, %% we will generate the code for the 'after' body, and then discard it. Int1 = clear_dead(Bef, I, Le#l.vdb), - {_,Int2,St1} = cg_block(Tes, Le#l.i, Le#l.vdb, + {_,Int2,St1} = cg_block(Tes, Le#l.vdb, Int1#sr{reg=clear_regs(Int1#sr.reg)}, St0), {[{wait,{f,St1#cg.recv}}],Int2,St1}; cg_recv_wait(#k_int{val=0}, #cg_block{anno=Le,es=Tes}, _I, Bef, St0) -> - {Tis,Int,St1} = cg_block(Tes, Le#l.i, Le#l.vdb, Bef, St0), + {Tis,Int,St1} = cg_block(Tes, Le#l.vdb, Bef, St0), {[timeout|Tis],Int,St1}; cg_recv_wait(Te, #cg_block{anno=Le,es=Tes}, I, Bef, St0) -> Reg = cg_reg_arg(Te, Bef), %% Must have empty registers here! Bug if anything in registers. Int0 = clear_dead(Bef, I, Le#l.vdb), - {Tis,Int,St1} = cg_block(Tes, Le#l.i, Le#l.vdb, + {Tis,Int,St1} = cg_block(Tes, Le#l.vdb, Int0#sr{reg=clear_regs(Int0#sr.reg)}, St0), {[{wait_timeout,{f,St1#cg.recv},Reg},timeout] ++ Tis,Int,St1}. @@ -1838,7 +2030,7 @@ catch_cg(#cg_block{es=C}, #k_var{name=R}, Le, Vdb, Bef, St0) -> CatchTag = Le#l.i, Int1 = Bef#sr{stk=put_catch(CatchTag, Bef#sr.stk)}, CatchReg = fetch_stack({catch_tag,CatchTag}, Int1#sr.stk), - {Cis,Int2,St2} = cg_block(C, Le#l.i, Le#l.vdb, Int1, + {Cis,Int2,St2} = cg_block(C, Le#l.vdb, Int1, St1#cg{break=B,in_catch=true}), [] = Int2#sr.reg, %Assertion. Aft = Int2#sr{reg=[{0,R}],stk=drop_catch(CatchTag, Int2#sr.stk)}, @@ -2202,13 +2394,12 @@ break_cg(Bs, Le, Vdb, Bef, St) -> {Ms ++ [{jump,{f,St#cg.break}}], Int#sr{reg=clear_regs(Int#sr.reg)},St}. -guard_break_cg(Bs, Locked, #l{i=I}, Vdb, #sr{reg=Reg0}=Bef, St) -> - RegLocked = get_locked_regs(Reg0, Locked), - #sr{reg=Reg1} = Int = clear_dead(Bef#sr{reg=RegLocked}, I, Vdb), +guard_break_cg(Bs, #l{i=I}, Vdb, #sr{reg=Reg0}=Bef, St) -> + #sr{reg=Reg1} = Int = clear_dead(Bef, I, Vdb), Reg2 = trim_free(Reg1), NumLocked = length(Reg2), Moves0 = gen_moves(Bs, Bef, NumLocked, []), - Moves = order_moves(Moves0, find_scratch_reg(RegLocked)), + Moves = order_moves(Moves0, find_scratch_reg(Reg0)), {BreakVars,_} = mapfoldl(fun(_, RegNum) -> {{RegNum,gbreakvar},RegNum+1} end, length(Reg2), Bs), @@ -2216,20 +2407,6 @@ guard_break_cg(Bs, Locked, #l{i=I}, Vdb, #sr{reg=Reg0}=Bef, St) -> Aft = Int#sr{reg=Reg}, {Moves ++ [{jump,{f,St#cg.break}}],Aft,St}. -get_locked_regs([R|Rs0], Preserve) -> - case {get_locked_regs(Rs0, Preserve),R} of - {[],{_,V}} -> - case lists:member(V, Preserve) of - true -> [R]; - false -> [] - end; - {[],_} -> - []; - {Rs,_} -> - [R|Rs] - end; -get_locked_regs([], _) -> []. - %% cg_reg_arg(Arg0, Info) -> Arg %% cg_reg_args([Arg0], Info) -> [Arg] %% Convert argument[s] into registers. Literal values are returned unchanged. @@ -2370,21 +2547,21 @@ break_up_cycle1(Dst, [M|Path], LastMove) -> %% clear_dead(Sr, Until, Vdb) -> Aft. %% Remove all variables in Sr which have died AT ALL so far. -clear_dead(Sr, Until, Vdb) -> - Sr#sr{reg=clear_dead_reg(Sr, Until, Vdb), - stk=clear_dead_stk(Sr#sr.stk, Until, Vdb)}. +clear_dead(#sr{stk=Stk}=Sr0, Until, Vdb) -> + Sr = Sr0#sr{reg=clear_dead_reg(Sr0, Until, Vdb), + stk=clear_dead_stk(Stk, Until, Vdb)}, + reserve(Sr). clear_dead_reg(Sr, Until, Vdb) -> - Reg = [case R of - {_I,V} = IV -> - case vdb_find(V, Vdb) of - {V,_,L} when L > Until -> IV; - _ -> free %Remove anything else - end; - {reserved,_I,_V} = Reserved -> Reserved; - free -> free - end || R <- Sr#sr.reg], - reserve(Sr#sr.res, Reg, Sr#sr.stk). + [case R of + {_I,V} = IV -> + case vdb_find(V, Vdb) of + {V,_,L} when L > Until -> IV; + _ -> free %Remove anything else + end; + {reserved,_I,_V}=Reserved -> Reserved; + free -> free + end || R <- Sr#sr.reg]. clear_dead_stk(Stk, Until, Vdb) -> [case S of @@ -2456,16 +2633,25 @@ adjust_stack(Bef, Fb, Lf, Vdb) -> save_stack(Stk0, Fb, Lf, Vdb) -> %% New variables that are in use but not on stack. - New = [VFL || {V,F,L} = VFL <- Vdb, - F < Fb, - L >= Lf, - not on_stack(V, Stk0)], + New = new_not_on_stack(Stk0, Fb, Lf, Vdb), + %% Add new variables that are not just dropped immediately. %% N.B. foldr works backwards from the end!! Saves = [V || {V,_,_} <- keysort(3, New)], Stk1 = foldr(fun (V, Stk) -> put_stack(V, Stk) end, Stk0, Saves), {Stk1,Saves}. +%% new_not_on_stack(Stack, FirstBefore, LastFrom, Vdb) -> +%% [{Variable,First,Last}] +%% Return information about all variables that are used past current +%% point and that are not already on the stack. + +new_not_on_stack(Stk, Fb, Lf, Vdb) -> + [VFL || {V,F,L} = VFL <- Vdb, + F < Fb, + L >= Lf, + not on_stack(V, Stk)]. + %% saves([SaveVar], Reg, Stk) -> [{move,Reg,Stk}]. %% Generate move instructions to save variables onto stack. The %% stack/reg info used is that after the new stack has been made. @@ -2571,10 +2757,6 @@ find_stack(_, [], _) -> error. on_stack(V, Stk) -> keymember(V, 1, Stk). -is_register({x,_}) -> true; -is_register({yy,_}) -> true; -is_register(_) -> false. - %% put_catch(CatchTag, Stack) -> Stack' %% drop_catch(CatchTag, Stack) -> Stack' %% Special interface for putting and removing catch tags, to ensure that diff --git a/lib/compiler/src/v3_core.erl b/lib/compiler/src/v3_core.erl index 20cb3343fb..6029b91cdc 100644 --- a/lib/compiler/src/v3_core.erl +++ b/lib/compiler/src/v3_core.erl @@ -920,8 +920,9 @@ try_exception(Ecs0, St0) -> %% Note that Tag is not needed for rethrow - it is already in Info. {Evs,St1} = new_vars(3, St0), % Tag, Value, Info {Ecs1,Ceps,St2} = clauses(Ecs0, St1), + Ecs2 = try_build_stacktrace(Ecs1, hd(Evs)), [_,Value,Info] = Evs, - LA = case Ecs1 of + LA = case Ecs2 of [] -> []; [C|_] -> get_lineno_anno(C) end, @@ -930,7 +931,7 @@ try_exception(Ecs0, St0) -> body=[#iprimop{anno=#a{}, %Must have an #a{} name=#c_literal{val=raise}, args=[Info,Value]}]}, - Hs = [#icase{anno=#a{anno=LA},args=[c_tuple(Evs)],clauses=Ecs1,fc=Ec}], + Hs = [#icase{anno=#a{anno=LA},args=[c_tuple(Evs)],clauses=Ecs2,fc=Ec}], {Evs,Ceps++Hs,St2}. try_after(As, St0) -> @@ -946,6 +947,25 @@ try_after(As, St0) -> Hs = [#icase{anno=#a{},args=[c_tuple(Evs)],clauses=[],fc=Ec}], {Evs,Hs,St1}. +try_build_stacktrace([#iclause{pats=Ps0,body=B0}=C0|Cs], RawStk) -> + [#c_tuple{es=[Class,Exc,Stk]}=Tup] = Ps0, + case Stk of + #c_var{name='_'} -> + %% Stacktrace variable is not used. Nothing to do. + [C0|try_build_stacktrace(Cs, RawStk)]; + _ -> + %% Add code to build the stacktrace. + Ps = [Tup#c_tuple{es=[Class,Exc,RawStk]}], + Call = #iprimop{anno=#a{}, + name=#c_literal{val=build_stacktrace}, + args=[RawStk]}, + Iset = #iset{var=Stk,arg=Call}, + B = [Iset|B0], + C = C0#iclause{pats=Ps,body=B}, + [C|try_build_stacktrace(Cs, RawStk)] + end; +try_build_stacktrace([], _) -> []. + %% expr_bin([ArgExpr], St) -> {[Arg],[PreExpr],St}. %% Flatten the arguments of a bin. Do this straight left to right! %% Note that ibinary needs to have its annotation wrapped in a #a{} @@ -2462,9 +2482,11 @@ cexpr(#icase{anno=A,args=Largs,clauses=Lcs,fc=Lfc}, As, St0) -> cexpr(#ireceive1{anno=A,clauses=Lcs}, As, St0) -> Exp = intersection(A#a.ns, As), %Exports {Ccs,St1} = cclauses(Lcs, Exp, St0), + True = #c_literal{val=true}, + Action = core_lib:make_values(lists:duplicate(1+length(Exp), True)), {#c_receive{anno=A#a.anno, clauses=Ccs, - timeout=#c_literal{val=infinity},action=#c_literal{val=true}}, + timeout=#c_literal{val=infinity},action=Action}, Exp,A#a.us,St1}; cexpr(#ireceive2{anno=A,clauses=Lcs,timeout=Lto,action=Les}, As, St0) -> Exp = intersection(A#a.ns, As), %Exports diff --git a/lib/compiler/src/v3_kernel.erl b/lib/compiler/src/v3_kernel.erl index 3eea058153..fd73e5a7dc 100644 --- a/lib/compiler/src/v3_kernel.erl +++ b/lib/compiler/src/v3_kernel.erl @@ -108,6 +108,7 @@ copy_anno(Kdst, Ksrc) -> -record(iclause, {anno=[],isub,osub,pats,guard,body}). -record(ireceive_accept, {anno=[],arg}). -record(ireceive_next, {anno=[],arg}). +-record(ignored, {anno=[]}). -type warning() :: term(). % XXX: REFINE @@ -162,8 +163,7 @@ function({#c_var{name={F,Arity}=FA},Body}, St0) -> %%B1 = B0, St3 = St2, %Null second pass {make_fdef(#k{us=[],ns=[],a=Ab}, F, Arity, Kvs, B1),St3} catch - Class:Error -> - Stack = erlang:get_stacktrace(), + Class:Error:Stack -> io:fwrite("Function: ~w/~w\n", [F,Arity]), erlang:raise(Class, Error, Stack) end. @@ -489,7 +489,7 @@ make_alt(First0, Then0) -> Then1 = pre_seq(droplast(Then0), last(Then0)), First2 = make_protected(First1), Then2 = make_protected(Then1), - Body = #k_atom{val=ignored}, + Body = #ignored{}, First3 = #k_guard_clause{guard=First2,body=Body}, Then3 = #k_guard_clause{guard=Then2,body=Body}, First = #k_guard{clauses=[First3]}, @@ -2225,7 +2225,9 @@ ubody(E, return, St0) -> {Ea,Pa,St1} = force_atomic(E, St0), ubody(pre_seq(Pa, #ivalues{args=[Ea]}), return, St1) end; -ubody(E, {break,_Rs} = Break, St0) -> +ubody(#ignored{}, {break,_} = Break, St) -> + ubody(#ivalues{args=[]}, Break, St); +ubody(E, {break,[_]} = Break, St0) -> %%ok = io:fwrite("ubody ~w:~p~n", [?LINE,{E,Br}]), %% Exiting expressions need no trailing break. case is_exit_expr(E) of @@ -2233,6 +2235,16 @@ ubody(E, {break,_Rs} = Break, St0) -> false -> {Ea,Pa,St1} = force_atomic(E, St0), ubody(pre_seq(Pa, #ivalues{args=[Ea]}), Break, St1) + end; +ubody(E, {break,Rs}=Break, St0) -> + case is_exit_expr(E) of + true -> + uexpr(E, return, St0); + false -> + {Vs,St1} = new_vars(length(Rs), St0), + Iset = #iset{vars=Vs,arg=E}, + PreSeq = pre_seq([Iset], #ivalues{args=Vs}), + ubody(PreSeq, Break, St1) end. iletrec_funs(#iletrec{defs=Fs}, St0) -> diff --git a/lib/compiler/src/v3_kernel.hrl b/lib/compiler/src/v3_kernel.hrl index 87011b7680..7cd30b25a8 100644 --- a/lib/compiler/src/v3_kernel.hrl +++ b/lib/compiler/src/v3_kernel.hrl @@ -79,7 +79,7 @@ -record(k_guard_clause, {anno=[],guard,body}). -record(k_break, {anno=[],args=[]}). --record(k_guard_break, {anno=[],args=[],locked=[]}). +-record(k_guard_break, {anno=[],args=[]}). -record(k_return, {anno=[],args=[]}). %%k_get_anno(Thing) -> element(2, Thing). diff --git a/lib/compiler/test/beam_utils_SUITE.erl b/lib/compiler/test/beam_utils_SUITE.erl index 710cb050d4..7686e69b63 100644 --- a/lib/compiler/test/beam_utils_SUITE.erl +++ b/lib/compiler/test/beam_utils_SUITE.erl @@ -24,7 +24,8 @@ apply_fun/1,apply_mf/1,bs_init/1,bs_save/1, is_not_killed/1,is_not_used_at/1, select/1,y_catch/1,otp_8949_b/1,liveopt/1,coverage/1, - y_registers/1]). + y_registers/1,user_predef/1,scan_f/1,cafu/1, + receive_label/1,read_size_file_version/1]). -export([id/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -46,7 +47,11 @@ groups() -> otp_8949_b, liveopt, coverage, - y_registers + y_registers, + user_predef, + scan_f, + cafu, + read_size_file_version ]}]. init_per_suite(Config) -> @@ -117,6 +122,15 @@ bs_init(_Config) -> {'EXIT',{badarg,_}} = (catch do_bs_init_2([0.5])), {'EXIT',{badarg,_}} = (catch do_bs_init_2([-1])), {'EXIT',{badarg,_}} = (catch do_bs_init_2([1 bsl 32])), + + <<>> = do_bs_init_3({tag,0}, 0, 0), + <<0>> = do_bs_init_3({tag,0}, 2, 1), + + <<"_build/shared">> = do_bs_init_4([], false), + <<"abc/shared">> = do_bs_init_4(<<"abc">>, false), + <<"foo/foo">> = do_bs_init_4(<<"foo">>, true), + error = do_bs_init_4([], not_boolean), + ok. do_bs_init_1([?MODULE], Sz) -> @@ -134,6 +148,45 @@ do_bs_init_2(SigNos) -> erlang:error(badarg) >>. +do_bs_init_3({tag,Pos}, Offset, Len) -> + N0 = Offset - Pos, + N = if N0 > Len -> Len; + true -> N0 + end, + <<0:N/unit:8>>. + +do_bs_init_4(Arg1, Arg2) -> + Build = + case id(Arg1) of + X when X =:= [] orelse X =:= false -> <<"_build">>; + X -> X + end, + case id(Arg2) of + true -> + id(<<case Build of + Rewrite when is_binary(Rewrite) -> + Rewrite; + Rewrite -> + id(Rewrite) + end/binary, + "/", + case id(<<"foo">>) of + Rewrite when is_binary(Rewrite) -> + Rewrite; + Rewrite -> + id(Rewrite) + end/binary>>); + false -> + id(<<case Build of + Rewrite when is_binary(Rewrite) -> + Rewrite; + Rewrite -> + id(Rewrite) + end/binary, + "/shared">>); + Other -> + error + end. bs_save(_Config) -> {a,30,<<>>} = do_bs_save(<<1:1,30:5>>), @@ -376,5 +429,83 @@ do(A, B) -> {A,B}. appointment(#{"resolution" := Url}) -> do(receive _ -> Url end, #{true => Url}). +%% From epp.erl. +user_predef(_Config) -> + #{key:="value"} = user_predef({key,"value"}, #{}), + #{key:="value"} = user_predef({key,"value"}, #{key=>defined}), + error = user_predef({key,"value"}, #{key=>[defined]}), + ok. + +user_predef({M,Val}, Ms) -> + case Ms of + #{M:=Defs} when is_list(Defs) -> + error; + _ -> + Ms#{M=>Val} + end. + +%% From disk_log_1.erl. +scan_f(_Config) -> + {1,<<>>,[]} = scan_f(<<1:32>>, 1, []), + {1,<<>>,[<<156>>]} = scan_f(<<1:32,156,1:32>>, 1, []), + ok. + +scan_f(<<Size:32,Tail/binary>>, FSz, Acc) when Size =< FSz -> + case Tail of + <<BinTerm:Size/binary,Tail2/binary>> -> + scan_f(Tail2, FSz, [BinTerm | Acc]); + _ -> + {Size,Tail,Acc} + end. + +%% From file_io_server.erl. +cafu(_Config) -> + error = cafu(<<42:32>>, -1, 0, {utf32,big}), + error = cafu(<<42:32>>, 10, 0, {utf32,big}), + error = cafu(<<42:32>>, -1, 0, {utf32,little}), + ok. + +cafu(<<_/big-utf32,Rest/binary>>, N, Count, {utf32,big}) when N < 0 -> + cafu(Rest, -1, Count+1, {utf32,big}); +cafu(<<_/big-utf32,Rest/binary>>, N, Count, {utf32,big}) -> + cafu(Rest, N-1, Count+1, {utf32,big}); +cafu(<<_/little-utf32,Rest/binary>>, N, Count, {utf32,little}) when N < 0 -> + cafu(Rest, -1, Count+1, {utf32,little}); +cafu(_, _, _, _) -> + error. + +-record(rec_label, {bool}). + +receive_label(_Config) -> + Pid = spawn_link(fun() -> do_receive_label(#rec_label{bool=true}) end), + Msg = {a,b,c}, + Pid ! {self(),Msg}, + receive + {ok,Msg} -> + unlink(Pid), + exit(Pid, die), + ok + end. + +do_receive_label(Rec) -> + receive + {From,Message} when Rec#rec_label.bool -> + From ! {ok,Message}, + do_receive_label(Rec) + end. + +read_size_file_version(_Config) -> + ok = do_read_size_file_version({ok,<<42>>}), + {ok,7777} = do_read_size_file_version({ok,<<7777:32>>}), + ok. + +do_read_size_file_version(E) -> + case E of + {ok,<<Version>>} when Version =:= 42 -> + ok; + {ok,<<MaxFiles:32>>} -> + {ok,MaxFiles} + end. + %% The identity function. id(I) -> I. diff --git a/lib/compiler/test/beam_validator_SUITE.erl b/lib/compiler/test/beam_validator_SUITE.erl index c23514b36b..685eb2a72e 100644 --- a/lib/compiler/test/beam_validator_SUITE.erl +++ b/lib/compiler/test/beam_validator_SUITE.erl @@ -421,9 +421,9 @@ try_bin_opt(Mod) -> try do_bin_opt(Mod) catch - Class:Error -> + Class:Error:Stk -> io:format("~p: ~p ~p\n~p\n", - [Mod,Class,Error,erlang:get_stacktrace()]), + [Mod,Class,Error,Stk]), error end. diff --git a/lib/compiler/test/bs_match_SUITE.erl b/lib/compiler/test/bs_match_SUITE.erl index ad48d4c0b7..7e1a432511 100644 --- a/lib/compiler/test/bs_match_SUITE.erl +++ b/lib/compiler/test/bs_match_SUITE.erl @@ -39,7 +39,8 @@ match_string_opt/1,select_on_integer/1, map_and_binary/1,unsafe_branch_caching/1, bad_literals/1,good_literals/1,constant_propagation/1, - parse_xml/1,get_payload/1,escape/1,num_slots_different/1]). + parse_xml/1,get_payload/1,escape/1,num_slots_different/1, + check_bitstring_list/1,guard/1]). -export([coverage_id/1,coverage_external_ignore/2]). @@ -71,7 +72,8 @@ groups() -> match_string_opt,select_on_integer, map_and_binary,unsafe_branch_caching, bad_literals,good_literals,constant_propagation,parse_xml, - get_payload,escape,num_slots_different]}]. + get_payload,escape,num_slots_different, + check_bitstring_list,guard]}]. init_per_suite(Config) -> @@ -1572,6 +1574,32 @@ lgettext(<<"de">>, <<"navigation">>, <<"Results">>) -> lgettext(<<"de">>, <<"navigation">>, <<"Resources">>) -> {ok, <<"Ressourcen">>}. +%% Cover more code in beam_bsm. +check_bitstring_list(_Config) -> + true = check_bitstring_list(<<1:1,0:1,1:1,1:1>>, [1,0,1,1]), + false = check_bitstring_list(<<1:1,0:1,1:1,1:1>>, [0]), + ok. + +check_bitstring_list(<<H:1,T1/bitstring>>, [H|T2]) -> + check_bitstring_list(T1, T2); +check_bitstring_list(<<>>, []) -> + true; +check_bitstring_list(_, _) -> + false. + +guard(_Config) -> + Tuple = id({a,b}), + ok = guard_1(<<1,2,3>>, {1,2,3}), + ok = guard_2(<<42>>, #{}), + ok. + +%% Cover handling of #k_put{} in v3_codegen:bsm_rename_ctx/4. +guard_1(<<A,B,C>>, Tuple) when Tuple =:= {A,B,C} -> + ok. + +%% Cover handling of #k_call{} in v3_codegen:bsm_rename_ctx/4. +guard_2(<<_>>, Healing) when Healing#{[] => Healing} =:= #{[] => #{}} -> + ok. check(F, R) -> R = F(). diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl index daebbe9d9d..35c11d894d 100644 --- a/lib/compiler/test/compile_SUITE.erl +++ b/lib/compiler/test/compile_SUITE.erl @@ -500,9 +500,8 @@ do_kernel_listing({M,A}) -> io:format("*** compilation failure '~p' for module ~s\n", [Error,M]), error; - Class:Error -> - io:format("~p: ~p ~p\n~p\n", - [M,Class,Error,erlang:get_stacktrace()]), + Class:Error:Stk -> + io:format("~p: ~p ~p\n~p\n", [M,Class,Error,Stk]), error end. @@ -902,9 +901,8 @@ do_core_pp({M,A}, Outdir) -> io:format("*** compilation failure '~p' for module ~s\n", [Error,M]), error; - Class:Error -> - io:format("~p: ~p ~p\n~p\n", - [M,Class,Error,erlang:get_stacktrace()]), + Class:Error:Stk -> + io:format("~p: ~p ~p\n~p\n", [M,Class,Error,Stk]), error end. @@ -961,9 +959,8 @@ do_core_roundtrip(Beam, Outdir) -> io:format("*** compilation failure '~p' for file ~s\n", [Error,Beam]), error; - Class:Error -> - io:format("~p: ~p ~p\n~p\n", - [Beam,Class,Error,erlang:get_stacktrace()]), + Class:Error:Stk -> + io:format("~p: ~p ~p\n~p\n", [Beam,Class,Error,Stk]), error end. @@ -1148,9 +1145,8 @@ do_asm(Beam, Outdir) -> [Other,AsmFile]), error end - catch Class:Error -> - io:format("~p: ~p ~p\n~p\n", - [M,Class,Error,erlang:get_stacktrace()]), + catch Class:Error:Stk -> + io:format("~p: ~p ~p\n~p\n", [M,Class,Error,Stk]), error end. @@ -1167,9 +1163,8 @@ do_opt_guards(Beam) -> try {ok,M,Asm} = compile:forms(A, ['S']), do_opt_guards_mod(Asm) - catch Class:Error -> - io:format("~p: ~p ~p\n~p\n", - [M,Class,Error,erlang:get_stacktrace()]), + catch Class:Error:Stk -> + io:format("~p: ~p ~p\n~p\n", [M,Class,Error,Stk]), error end. diff --git a/lib/compiler/test/guard_SUITE.erl b/lib/compiler/test/guard_SUITE.erl index d96cfdb7ac..0d6f8c6f98 100644 --- a/lib/compiler/test/guard_SUITE.erl +++ b/lib/compiler/test/guard_SUITE.erl @@ -35,7 +35,8 @@ basic_andalso_orelse/1,traverse_dcd/1, check_qlc_hrl/1,andalso_semi/1,t_tuple_size/1,binary_part/1, bad_constants/1,bad_guards/1, - guard_in_catch/1,beam_bool_SUITE/1]). + guard_in_catch/1,beam_bool_SUITE/1, + cover_beam_dead/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -54,7 +55,8 @@ groups() -> rel_ops,rel_op_combinations, literal_type_tests,basic_andalso_orelse,traverse_dcd, check_qlc_hrl,andalso_semi,t_tuple_size,binary_part, - bad_constants,bad_guards,guard_in_catch,beam_bool_SUITE]}]. + bad_constants,bad_guards,guard_in_catch,beam_bool_SUITE, + cover_beam_dead]}]. init_per_suite(Config) -> Config. @@ -1619,7 +1621,9 @@ type_tests() -> is_reference, is_port, is_binary, - is_function]. + is_bitstring, + is_function, + is_map]. basic_andalso_orelse(Config) when is_list(Config) -> T = id({type,integers,23,42}), @@ -2202,7 +2206,31 @@ maps() -> evidence(#{0 := Charge}) when 0; #{[] => Charge} == #{[] => 42} -> ok. +cover_beam_dead(_Config) -> + Mod = ?FUNCTION_NAME, + Attr = [], + Fs = [{function,test,1,2, + [{label,1}, + {line,[]}, + {func_info,{atom,Mod},{atom,test},1}, + {label,2}, + %% Cover beam_dead:turn_op/1 using swapped operand order. + {test,is_ne_exact,{f,3},[{integer,1},{x,0}]}, + {test,is_eq_exact,{f,1},[{atom,a},{x,0}]}, + {label,3}, + {move,{atom,ok},{x,0}}, + return]}], + Exp = [{test,1}], + Asm = {Mod,Exp,Attr,Fs,3}, + {ok,Mod,Beam} = compile:forms(Asm, [from_asm,binary,report]), + {module,Mod} = code:load_binary(Mod, Mod, Beam), + ok = Mod:test(1), + ok = Mod:test(a), + {'EXIT',_} = (catch Mod:test(other)), + true = code:delete(Mod), + _ = code:purge(Mod), + ok. %% Call this function to turn off constant propagation. id(I) -> I. diff --git a/lib/compiler/test/map_SUITE.erl b/lib/compiler/test/map_SUITE.erl index 5e90b79aa2..f15917e3cb 100644 --- a/lib/compiler/test/map_SUITE.erl +++ b/lib/compiler/test/map_SUITE.erl @@ -695,8 +695,28 @@ t_is_map(Config) when is_list(Config) -> if is_map(#{b=>1}) -> ok end, if not is_map([1,2,3]) -> ok end, if not is_map(x) -> ok end, + + ok = do_t_is_map(map, #{}), + error = do_t_is_map(map, {a,b,c}), + ok = do_t_is_map(number, 42), + ok = do_t_is_map(number, 42.0), + error = do_t_is_map(number, {a,b,c}), ok. +do_t_is_map(What, X) -> + B = case What of + map -> + %% Cover conversion of is_map/1 BIF to test instruction + %% in beam_utils:bif_to_test/3. + is_map(X); + number -> + is_number(X) + end, + case B of + true -> ok; + false -> error + end. + % test map updates without matching t_update_literals(Config) when is_list(Config) -> Map = #{x=>1,y=>2,z=>3,q=>4}, diff --git a/lib/compiler/test/match_SUITE.erl b/lib/compiler/test/match_SUITE.erl index c31695be24..4b26a8dcdc 100644 --- a/lib/compiler/test/match_SUITE.erl +++ b/lib/compiler/test/match_SUITE.erl @@ -24,7 +24,8 @@ pmatch/1,mixed/1,aliases/1,non_matching_aliases/1, match_in_call/1,untuplify/1,shortcut_boolean/1,letify_guard/1, selectify/1,deselectify/1,underscore/1,match_map/1,map_vars_used/1, - coverage/1,grab_bag/1,literal_binary/1]). + coverage/1,grab_bag/1,literal_binary/1, + unary_op/1]). -include_lib("common_test/include/ct.hrl"). @@ -40,7 +41,7 @@ groups() -> match_in_call,untuplify, shortcut_boolean,letify_guard,selectify,deselectify, underscore,match_map,map_vars_used,coverage, - grab_bag,literal_binary]}]. + grab_bag,literal_binary,unary_op]}]. init_per_suite(Config) -> @@ -617,6 +618,10 @@ grab_bag(_Config) -> {bad,16#555555555555555555555555555555555555555555555555555}], ok = grab_bag_remove_failure(L, unit, 0), + {42,<<43,44>>} = grab_bag_single_valued(<<42,43,44>>), + empty_list = grab_bag_single_valued([]), + empty_tuple = grab_bag_single_valued({}), + ok. grab_bag_remove_failure([], _Unit, _MaxFailure) -> @@ -634,6 +639,12 @@ grab_bag_remove_failure([{stretch,_,Mi}=Stretch | Specs], Unit, _MaxFailure) -> ok end. +%% Cover a line v3_kernel that places binary matching first. +grab_bag_single_valued(<<H,T/bytes>>) -> {H,T}; +grab_bag_single_valued([]) -> empty_list; +grab_bag_single_valued({}) -> empty_tuple. + + %% Regression in 19.0, reported by Alexei Sholik literal_binary(_Config) -> 3 = literal_binary_match(bar, <<"y">>), @@ -652,5 +663,74 @@ literal_binary_match(_, <<"x">>) -> 2; literal_binary_match(_, <<"y">>) -> 3; literal_binary_match(_, _) -> fail. +unary_op(Config) -> + %% ERL-514. This test case only verifies that the code + %% calculates the correct result, not that the generated + %% code is optimial. + + {non_associative,30} = unary_op_1('&'), + {non_associative,300} = unary_op_1('^'), + {non_associative,300} = unary_op_1('not'), + {non_associative,300} = unary_op_1('+'), + {non_associative,300} = unary_op_1('-'), + {non_associative,300} = unary_op_1('~~~'), + {non_associative,300} = unary_op_1('!'), + {non_associative,320} = unary_op_1('@'), + + error = unary_op_1(Config), + error = unary_op_1(abc), + error = unary_op_1(42), + + ok. + +unary_op_1(Vop@1) -> + %% If all optimizations are working as they should, there should + %% be no stack frame and all '=:=' tests should be coalesced into + %% a single select_val instruction. + + case Vop@1 =:= '&' of + true -> + {non_associative,30}; + false -> + case + case Vop@1 =:= '^' of + true -> + true; + false -> + case Vop@1 =:= 'not' of + true -> + true; + false -> + case Vop@1 =:= '+' of + true -> + true; + false -> + case Vop@1 =:= '-' of + true -> + true; + false -> + case Vop@1 =:= '~~~' of + true -> + true; + false -> + Vop@1 =:= '!' + end + end + end + end + end + of + true -> + {non_associative,300}; + false -> + case Vop@1 =:= '@' of + true -> + {non_associative,320}; + false -> + error + end + end + end. + id(I) -> I. diff --git a/lib/compiler/test/misc_SUITE.erl b/lib/compiler/test/misc_SUITE.erl index b12bcbeeab..d93c5dda1e 100644 --- a/lib/compiler/test/misc_SUITE.erl +++ b/lib/compiler/test/misc_SUITE.erl @@ -318,8 +318,7 @@ expect_error(Fun) -> io:format("~p", [Any]), ct:fail(call_was_supposed_to_fail) catch - Class:Reason -> - Stk = erlang:get_stacktrace(), + Class:Reason:Stk -> io:format("~p:~p\n~p\n", [Class,Reason,Stk]), case {Class,Reason} of {error,undef} -> diff --git a/lib/compiler/test/receive_SUITE.erl b/lib/compiler/test/receive_SUITE.erl index 8304672558..5e386790c0 100644 --- a/lib/compiler/test/receive_SUITE.erl +++ b/lib/compiler/test/receive_SUITE.erl @@ -222,9 +222,8 @@ do_ref_opt(Source, PrivDir) -> collect_recv_opt_instrs(Code) end, ok - catch Class:Error -> - io:format("~s: ~p ~p\n~p\n", - [Source,Class,Error,erlang:get_stacktrace()]), + catch Class:Error:Stk -> + io:format("~s: ~p ~p\n~p\n", [Source,Class,Error,Stk]), error end. @@ -265,6 +264,10 @@ export(Config) when is_list(Config) -> self() ! {result,Ref,42}, 42 = export_1(Ref), {error,timeout} = export_1(Ref), + + self() ! {result,Ref}, + {ok,Ref} = export_2(), + ok. export_1(Reference) -> @@ -281,6 +284,10 @@ export_1(Reference) -> id({build,self()}), Result. +export_2() -> + receive {result,Result} -> ok end, + {ok,Result}. + wait(Config) when is_list(Config) -> self() ! <<42>>, <<42>> = wait_1(r, 1, 2), diff --git a/lib/compiler/test/trycatch_SUITE.erl b/lib/compiler/test/trycatch_SUITE.erl index 42dbf7d5f0..8cf7928cc4 100644 --- a/lib/compiler/test/trycatch_SUITE.erl +++ b/lib/compiler/test/trycatch_SUITE.erl @@ -26,7 +26,8 @@ nested_of/1,nested_catch/1,nested_after/1, nested_horrid/1,last_call_optimization/1,bool/1, plain_catch_coverage/1,andalso_orelse/1,get_in_try/1, - hockey/1,handle_info/1,catch_in_catch/1,grab_bag/1]). + hockey/1,handle_info/1,catch_in_catch/1,grab_bag/1, + stacktrace/1,nested_stacktrace/1]). -include_lib("common_test/include/ct.hrl"). @@ -42,7 +43,8 @@ groups() -> after_oops,eclectic,rethrow,nested_of,nested_catch, nested_after,nested_horrid,last_call_optimization, bool,plain_catch_coverage,andalso_orelse,get_in_try, - hockey,handle_info,catch_in_catch,grab_bag]}]. + hockey,handle_info,catch_in_catch,grab_bag, + stacktrace,nested_stacktrace]}]. init_per_suite(Config) -> @@ -1039,5 +1041,123 @@ grab_bag(_Config) -> ok. +stacktrace(_Config) -> + V = [make_ref()|self()], + case ?MODULE:module_info(native) of + false -> + {value2,{caught1,badarg,[{erlang,abs,[V],_}|_]}} = + stacktrace_1({'abs',V}, error, {value,V}), + {caught2,{error,badarith},[{erlang,'+',[0,a],_}, + {?MODULE,my_add,2,_}|_]} = + stacktrace_1({'div',{1,0}}, error, {'add',{0,a}}); + true -> + {value2,{caught1,badarg,[{?MODULE,my_abs,1,_}|_]}} = + stacktrace_1({'abs',V}, error, {value,V}), + {caught2,{error,badarith},[{?MODULE,my_add,2,_}|_]} = + stacktrace_1({'div',{1,0}}, error, {'add',{0,a}}) + end, + {caught2,{error,{try_clause,V}},[{?MODULE,stacktrace_1,3,_}|_]} = + stacktrace_1({value,V}, error, {value,V}), + {caught2,{throw,V},[{?MODULE,foo,1,_}|_]} = + stacktrace_1({value,V}, error, {throw,V}), + + try + stacktrace_2() + catch + error:{badmatch,_}:Stk2 -> + [{?MODULE,stacktrace_2,0,_}, + {?MODULE,stacktrace,1,_}|_] = Stk2, + Stk2 = erlang:get_stacktrace(), + ok + end, + + try + stacktrace_3(a, b) + catch + error:function_clause:Stk3 -> + Stk3 = erlang:get_stacktrace(), + case lists:module_info(native) of + false -> + [{lists,prefix,[a,b],_}|_] = Stk3; + true -> + [{lists,prefix,2,_}|_] = Stk3 + end + end, + + try + throw(x) + catch + throw:x:IntentionallyUnused -> + ok + end. + +stacktrace_1(X, C1, Y) -> + try try foo(X) of + C1 -> value1 + catch + C1:D1:Stk1 -> + Stk1 = erlang:get_stacktrace(), + {caught1,D1,Stk1} + after + foo(Y) + end of + V2 -> {value2,V2} + catch + C2:D2:Stk2 -> {caught2,{C2,D2},Stk2=erlang:get_stacktrace()} + end. + +stacktrace_2() -> + ok = erlang:process_info(self(), current_function), + ok. + +stacktrace_3(A, B) -> + {ok,lists:prefix(A, B)}. + +nested_stacktrace(_Config) -> + V = [{make_ref()}|[self()]], + value1 = nested_stacktrace_1({{value,{V,x1}},void,{V,x1}}, + {void,void,void}), + case ?MODULE:module_info(native) of + false -> + {caught1, + [{erlang,'+',[V,x1],_},{?MODULE,my_add,2,_}|_], + value2} = + nested_stacktrace_1({{'add',{V,x1}},error,badarith}, + {{value,{V,x2}},void,{V,x2}}), + {caught1, + [{erlang,'+',[V,x1],_},{?MODULE,my_add,2,_}|_], + {caught2,[{erlang,abs,[V],_}|_]}} = + nested_stacktrace_1({{'add',{V,x1}},error,badarith}, + {{'abs',V},error,badarg}); + true -> + {caught1, + [{?MODULE,my_add,2,_}|_], + value2} = + nested_stacktrace_1({{'add',{V,x1}},error,badarith}, + {{value,{V,x2}},void,{V,x2}}), + {caught1, + [{?MODULE,my_add,2,_}|_], + {caught2,[{?MODULE,my_abs,1,_}|_]}} = + nested_stacktrace_1({{'add',{V,x1}},error,badarith}, + {{'abs',V},error,badarg}) + end, + ok. + +nested_stacktrace_1({X1,C1,V1}, {X2,C2,V2}) -> + try foo(X1) of + V1 -> value1 + catch + C1:V1:S1 -> + S1 = erlang:get_stacktrace(), + T2 = try foo(X2) of + V2 -> value2 + catch + C2:V2:S2 -> + S2 = erlang:get_stacktrace(), + {caught2,S2} + end, + {caught1,S1,T2} + end. + id(I) -> I. diff --git a/lib/compiler/test/z_SUITE.erl b/lib/compiler/test/z_SUITE.erl index d864184f4c..cd95d0e733 100644 --- a/lib/compiler/test/z_SUITE.erl +++ b/lib/compiler/test/z_SUITE.erl @@ -54,8 +54,7 @@ do_loaded([{M,_}|Ms], E0) -> _ = M:module_info(functions), E0 catch - C:Error -> - Stk = erlang:get_stacktrace(), + C:Error:Stk -> io:format("~p:~p\n~p\n", [C,Error,Stk]), E0 + 1 end, diff --git a/lib/compiler/vsn.mk b/lib/compiler/vsn.mk index 435a57aac2..082786c7d8 100644 --- a/lib/compiler/vsn.mk +++ b/lib/compiler/vsn.mk @@ -1 +1 @@ -COMPILER_VSN = 7.1.3 +COMPILER_VSN = 7.1.4 diff --git a/lib/cosEvent/doc/src/notes.xml b/lib/cosEvent/doc/src/notes.xml index fe94cb64d3..ba0b0d88db 100644 --- a/lib/cosEvent/doc/src/notes.xml +++ b/lib/cosEvent/doc/src/notes.xml @@ -33,7 +33,22 @@ <file>notes.xml</file> </header> - <section><title>cosEvent 2.2.1</title> + <section><title>cosEvent 2.2.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + +<section><title>cosEvent 2.2.1</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/cosEvent/vsn.mk b/lib/cosEvent/vsn.mk index c39bed9fe4..578950294a 100644 --- a/lib/cosEvent/vsn.mk +++ b/lib/cosEvent/vsn.mk @@ -1,2 +1,2 @@ -COSEVENT_VSN = 2.2.1 +COSEVENT_VSN = 2.2.2 diff --git a/lib/cosEventDomain/doc/src/notes.xml b/lib/cosEventDomain/doc/src/notes.xml index 5e5bb2c33e..bd0a119ad2 100644 --- a/lib/cosEventDomain/doc/src/notes.xml +++ b/lib/cosEventDomain/doc/src/notes.xml @@ -32,7 +32,22 @@ <file>notes.xml</file> </header> - <section><title>cosEventDomain 1.2.1</title> + <section><title>cosEventDomain 1.2.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + +<section><title>cosEventDomain 1.2.1</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/cosEventDomain/vsn.mk b/lib/cosEventDomain/vsn.mk index 4e10d6ac60..0c063a00f9 100644 --- a/lib/cosEventDomain/vsn.mk +++ b/lib/cosEventDomain/vsn.mk @@ -1,2 +1,2 @@ -COSEVENTDOMAIN_VSN = 1.2.1 +COSEVENTDOMAIN_VSN = 1.2.2 diff --git a/lib/cosFileTransfer/doc/src/notes.xml b/lib/cosFileTransfer/doc/src/notes.xml index 58ab087014..e0b4bdf64b 100644 --- a/lib/cosFileTransfer/doc/src/notes.xml +++ b/lib/cosFileTransfer/doc/src/notes.xml @@ -31,7 +31,22 @@ <file>notes.xml</file> </header> - <section><title>cosFileTransfer 1.2.1</title> + <section><title>cosFileTransfer 1.2.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + +<section><title>cosFileTransfer 1.2.1</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/cosFileTransfer/vsn.mk b/lib/cosFileTransfer/vsn.mk index e271c05242..561f11fbb2 100644 --- a/lib/cosFileTransfer/vsn.mk +++ b/lib/cosFileTransfer/vsn.mk @@ -1 +1 @@ -COSFILETRANSFER_VSN = 1.2.1 +COSFILETRANSFER_VSN = 1.2.2 diff --git a/lib/cosNotification/doc/src/notes.xml b/lib/cosNotification/doc/src/notes.xml index 1237000153..bf0fc73548 100644 --- a/lib/cosNotification/doc/src/notes.xml +++ b/lib/cosNotification/doc/src/notes.xml @@ -32,7 +32,22 @@ <file>notes.xml</file> </header> - <section><title>cosNotification 1.2.2</title> + <section><title>cosNotification 1.2.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + +<section><title>cosNotification 1.2.2</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/cosNotification/vsn.mk b/lib/cosNotification/vsn.mk index 0d95ab4853..1677057670 100644 --- a/lib/cosNotification/vsn.mk +++ b/lib/cosNotification/vsn.mk @@ -1,2 +1,2 @@ -COSNOTIFICATION_VSN = 1.2.2 +COSNOTIFICATION_VSN = 1.2.3 diff --git a/lib/cosProperty/doc/src/notes.xml b/lib/cosProperty/doc/src/notes.xml index e5d22982c5..4de246de67 100644 --- a/lib/cosProperty/doc/src/notes.xml +++ b/lib/cosProperty/doc/src/notes.xml @@ -33,7 +33,22 @@ </header> - <section><title>cosProperty 1.2.2</title> + <section><title>cosProperty 1.2.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + +<section><title>cosProperty 1.2.2</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/cosProperty/vsn.mk b/lib/cosProperty/vsn.mk index 78ba88445d..a3a952346e 100644 --- a/lib/cosProperty/vsn.mk +++ b/lib/cosProperty/vsn.mk @@ -1,2 +1,2 @@ -COSPROPERTY_VSN = 1.2.2 +COSPROPERTY_VSN = 1.2.3 diff --git a/lib/cosTime/doc/src/notes.xml b/lib/cosTime/doc/src/notes.xml index 686d9e6add..16e02f8b1f 100644 --- a/lib/cosTime/doc/src/notes.xml +++ b/lib/cosTime/doc/src/notes.xml @@ -33,7 +33,22 @@ <file>notes.xml</file> </header> - <section><title>cosTime 1.2.2</title> + <section><title>cosTime 1.2.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + +<section><title>cosTime 1.2.2</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/cosTime/vsn.mk b/lib/cosTime/vsn.mk index 7c9cae2d2f..7d6fcbebcd 100644 --- a/lib/cosTime/vsn.mk +++ b/lib/cosTime/vsn.mk @@ -1,2 +1,2 @@ -COSTIME_VSN = 1.2.2 +COSTIME_VSN = 1.2.3 diff --git a/lib/cosTransactions/doc/src/notes.xml b/lib/cosTransactions/doc/src/notes.xml index 85ace1208b..2401c04c3f 100644 --- a/lib/cosTransactions/doc/src/notes.xml +++ b/lib/cosTransactions/doc/src/notes.xml @@ -33,7 +33,22 @@ <file>notes.xml</file> </header> - <section><title>cosTransactions 1.3.2</title> + <section><title>cosTransactions 1.3.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + +<section><title>cosTransactions 1.3.2</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/cosTransactions/vsn.mk b/lib/cosTransactions/vsn.mk index ab163d83c2..bba2058231 100644 --- a/lib/cosTransactions/vsn.mk +++ b/lib/cosTransactions/vsn.mk @@ -1 +1 @@ -COSTRANSACTIONS_VSN = 1.3.2 +COSTRANSACTIONS_VSN = 1.3.3 diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c index f05bfa10b3..6957d25774 100644 --- a/lib/crypto/c_src/crypto.c +++ b/lib/crypto/c_src/crypto.c @@ -587,7 +587,7 @@ static ErlNifFunc nif_funcs[] = { {"engine_finish_nif", 1, engine_finish_nif}, {"engine_free_nif", 1, engine_free_nif}, {"engine_load_dynamic_nif", 0, engine_load_dynamic_nif}, - {"engine_ctrl_cmd_strings_nif", 2, engine_ctrl_cmd_strings_nif}, + {"engine_ctrl_cmd_strings_nif", 3, engine_ctrl_cmd_strings_nif}, {"engine_register_nif", 2, engine_register_nif}, {"engine_unregister_nif", 2, engine_unregister_nif}, {"engine_add_nif", 1, engine_add_nif}, @@ -4825,9 +4825,10 @@ static ERL_NIF_TERM privkey_to_pubkey_nif(ErlNifEnv* env, int argc, const ERL_NI } else if (argv[0] == atom_ecdsa) { #if defined(HAVE_EC) - EC_KEY *ec = EVP_PKEY_get1_EC_KEY(pkey); - if (ec) { - /* Example of result: + /* not yet implemented + EC_KEY *ec = EVP_PKEY_get1_EC_KEY(pkey); + if (ec) { + / * Example of result: { Curve = {Field, Prime, Point, Order, CoFactor} = { @@ -4841,7 +4842,7 @@ static ERL_NIF_TERM privkey_to_pubkey_nif(ErlNifEnv* env, int argc, const ERL_NI CoFactor = <<1>> }, Key = <<151,...,62>> - } + } or { Curve = @@ -4852,16 +4853,13 @@ static ERL_NIF_TERM privkey_to_pubkey_nif(ErlNifEnv* env, int argc, const ERL_NI }, Key } - */ + * / EVP_PKEY_free(pkey); - return atom_notsup; - } -#else - EVP_PKEY_free(pkey); - return atom_notsup; + return enif_make_list_from_array(env, ..., ...); + */ #endif } - + if (pkey) EVP_PKEY_free(pkey); return enif_make_badarg(env); } @@ -4886,7 +4884,6 @@ static ERL_NIF_TERM engine_by_id_nif(ErlNifEnv* env, int argc, const ERL_NIF_TER #ifdef HAS_ENGINE_SUPPORT ERL_NIF_TERM ret; ErlNifBinary engine_id_bin; - unsigned int engine_id_len = 0; char *engine_id; ENGINE *engine; struct engine_ctx *ctx; @@ -4896,14 +4893,14 @@ static ERL_NIF_TERM engine_by_id_nif(ErlNifEnv* env, int argc, const ERL_NIF_TER PRINTF_ERR0("engine_by_id_nif Leaved: badarg"); return enif_make_badarg(env); } else { - engine_id_len = engine_id_bin.size+1; - engine_id = enif_alloc(engine_id_len); - (void) memcpy(engine_id, engine_id_bin.data, engine_id_len); - engine_id[engine_id_len-1] = '\0'; + engine_id = enif_alloc(engine_id_bin.size+1); + (void) memcpy(engine_id, engine_id_bin.data, engine_id_bin.size); + engine_id[engine_id_bin.size] = '\0'; } engine = ENGINE_by_id(engine_id); if(!engine) { + enif_free(engine_id); PRINTF_ERR0("engine_by_id_nif Leaved: {error, bad_engine_id}"); return enif_make_tuple2(env, atom_error, atom_bad_engine_id); } @@ -4997,7 +4994,7 @@ static ERL_NIF_TERM engine_ctrl_cmd_strings_nif(ErlNifEnv* env, int argc, const unsigned int cmds_len = 0; char **cmds = NULL; struct engine_ctx *ctx; - int i; + int i, optional = 0; // Get Engine if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) { @@ -5021,11 +5018,16 @@ static ERL_NIF_TERM engine_ctrl_cmd_strings_nif(ErlNifEnv* env, int argc, const } } + if(!enif_get_int(env, argv[2], &optional)) { + PRINTF_ERR0("engine_ctrl_cmd_strings_nif Leaved: Parameter optional not an integer"); + return enif_make_badarg(env); + } + for(i = 0; i < cmds_len; i+=2) { PRINTF_ERR2("Cmd: %s:%s\r\n", cmds[i] ? cmds[i] : "(NULL)", cmds[i+1] ? cmds[i+1] : "(NULL)"); - if(!ENGINE_ctrl_cmd_string(ctx->engine, cmds[i], cmds[i+1], 0)) { + if(!ENGINE_ctrl_cmd_string(ctx->engine, cmds[i], cmds[i+1], optional)) { PRINTF_ERR2("Command failed: %s:%s\r\n", cmds[i] ? cmds[i] : "(NULL)", cmds[i+1] ? cmds[i+1] : "(NULL)"); @@ -5034,11 +5036,12 @@ static ERL_NIF_TERM engine_ctrl_cmd_strings_nif(ErlNifEnv* env, int argc, const PRINTF_ERR0("engine_ctrl_cmd_strings_nif Leaved: {error, ctrl_cmd_failed}"); goto error; } -} + } error: for(i = 0; cmds != NULL && cmds[i] != NULL; i++) - enif_free(cmds[i]); + enif_free(cmds[i]); + enif_free(cmds); return ret; #else return atom_notsup; @@ -5377,7 +5380,6 @@ static int get_engine_load_cmd_list(ErlNifEnv* env, const ERL_NIF_TERM term, cha ErlNifBinary tmpbin; int arity; char* tmpstr; - int tmplen = 0; if(!enif_is_empty_list(env, term)) { if(!enif_get_list_cell(env, term, &head, &tail)) { @@ -5392,10 +5394,9 @@ static int get_engine_load_cmd_list(ErlNifEnv* env, const ERL_NIF_TERM term, cha cmds[i] = NULL; return -1; } else { - tmplen = tmpbin.size+1; - tmpstr = enif_alloc(tmplen); - (void) memcpy(tmpstr, tmpbin.data, tmplen); - tmpstr[tmplen-1] = '\0'; + tmpstr = enif_alloc(tmpbin.size+1); + (void) memcpy(tmpstr, tmpbin.data, tmpbin.size); + tmpstr[tmpbin.size] = '\0'; cmds[i++] = tmpstr; } if(!enif_inspect_binary(env, tmp_tuple[1], &tmpbin)) { @@ -5405,10 +5406,9 @@ static int get_engine_load_cmd_list(ErlNifEnv* env, const ERL_NIF_TERM term, cha if(tmpbin.size == 0) cmds[i++] = NULL; else { - tmplen = tmpbin.size+1; - tmpstr = enif_alloc(tmplen); - (void) memcpy(tmpstr, tmpbin.data, tmplen); - tmpstr[tmplen-1] = '\0'; + tmpstr = enif_alloc(tmpbin.size+1); + (void) memcpy(tmpstr, tmpbin.data, tmpbin.size); + tmpstr[tmpbin.size] = '\0'; cmds[i++] = tmpstr; } } diff --git a/lib/crypto/doc/src/crypto.xml b/lib/crypto/doc/src/crypto.xml index c681ead797..3a5efd0bea 100644 --- a/lib/crypto/doc/src/crypto.xml +++ b/lib/crypto/doc/src/crypto.xml @@ -629,10 +629,6 @@ <p>Fetches the corresponding public key from a private key stored in an Engine. The key must be of the type indicated by the Type parameter. </p> - <p> - May throw exception notsup in case there is - no engine support in the underlying OpenSSL implementation. - </p> </desc> </func> @@ -1185,6 +1181,57 @@ _FloatValue = rand:uniform(). % [0.0; 1.0[</pre> </desc> </func> + <func> + <name>engine_ctrl_cmd_string(Engine, CmdName, CmdArg) -> Result</name> + <fsummary>Sends ctrl commands to an OpenSSL engine</fsummary> + <type> + <v>Engine = term()</v> + <v>CmdName = unicode:chardata()</v> + <v>CmdArg = unicode:chardata()</v> + <v>Result = ok | {error, Reason::term()}</v> + </type> + <desc> + <p> + Sends ctrl commands to the OpenSSL engine given by <c>Engine</c>. + This function is the same as calling <c>engine_ctrl_cmd_string/4</c> with + <c>Optional</c> set to <c>false</c>. + </p> + <p> + The function throws a badarg if the parameters are in wrong format. + It may also throw the exception notsup in case there is + no engine support in the underlying OpenSSL implementation. + </p> + </desc> + </func> + + <func> + <name>engine_ctrl_cmd_string(Engine, CmdName, CmdArg, Optional) -> Result</name> + <fsummary>Sends ctrl commands to an OpenSSL engine</fsummary> + <type> + <v>Engine = term()</v> + <v>CmdName = unicode:chardata()</v> + <v>CmdArg = unicode:chardata()</v> + <v>Optional = boolean()</v> + <v>Result = ok | {error, Reason::term()}</v> + </type> + <desc> + <p> + Sends ctrl commands to the OpenSSL engine given by <c>Engine</c>. + <c>Optional</c> is a boolean argument that can relax the semantics of the function. + If set to <c>true</c> it will only return failure if the ENGINE supported the given + command name but failed while executing it, if the ENGINE doesn't support the command + name it will simply return success without doing anything. In this case we assume + the user is only supplying commands specific to the given ENGINE so we set this to + <c>false</c>. + </p> + <p> + The function throws a badarg if the parameters are in wrong format. + It may also throw the exception notsup in case there is + no engine support in the underlying OpenSSL implementation. + </p> + </desc> + </func> + </funcs> <!-- Maybe put this in the users guide --> diff --git a/lib/crypto/doc/src/notes.xml b/lib/crypto/doc/src/notes.xml index 9376e6f649..dbeb886d7b 100644 --- a/lib/crypto/doc/src/notes.xml +++ b/lib/crypto/doc/src/notes.xml @@ -31,6 +31,76 @@ </header> <p>This document describes the changes made to the Crypto application.</p> +<section><title>Crypto 4.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The compatibility function <c>void HMAC_CTX_free</c> in + <c>crypto.c</c> erroneously tried to return a value.</p> + <p> + Own Id: OTP-14720</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Rewrite public and private key encode/decode with EVP + api. New RSA padding options added. This is a modified + half of PR-838.</p> + <p> + Own Id: OTP-14446</p> + </item> + <item> + <p> + The crypto API is extended to use private/public keys + stored in an Engine for sign/verify or encrypt/decrypt + operations.</p> + <p> + The ssl application provides an API to use this new + engine concept in TLS.</p> + <p> + Own Id: OTP-14448</p> + </item> + <item> + <p> Add support to plug in alternative implementations + for some or all of the cryptographic operations supported + by the OpenSSL Engine API. When configured appropriately, + OpenSSL calls the engine's implementation of these + operations instead of its own. </p> + <p> + Own Id: OTP-14567</p> + </item> + <item> + <p> + Replaced a call of the OpenSSL deprecated function + <c>DH_generate_parameters</c> in <c>crypto.c</c>.</p> + <p> + Own Id: OTP-14639</p> + </item> + <item> + <p> + Documentation added about how to use keys stored in an + Engine.</p> + <p> + Own Id: OTP-14735 Aux Id: OTP-14448 </p> + </item> + <item> + <p> Add engine_ ctrl_cmd_string/3,4 the OpenSSL Engine + support in crypto. </p> + <p> + Own Id: OTP-14801</p> + </item> + </list> + </section> + +</section> + <section><title>Crypto 4.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index 91e154280f..df259d5419 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -52,7 +52,9 @@ engine_load/3, engine_load/4, engine_unload/1, - engine_list/0 + engine_list/0, + engine_ctrl_cmd_string/3, + engine_ctrl_cmd_string/4 ]). -export_type([engine_ref/0, @@ -687,7 +689,7 @@ engine_load(EngineId, PreCmds, PostCmds, EngineMethods) when is_list(PreCmds), engine_load_1(Engine, PreCmds, PostCmds, EngineMethods) -> try - ok = engine_nif_wrapper(engine_ctrl_cmd_strings_nif(Engine, ensure_bin_cmds(PreCmds))), + ok = engine_nif_wrapper(engine_ctrl_cmd_strings_nif(Engine, ensure_bin_cmds(PreCmds), 0)), ok = engine_nif_wrapper(engine_add_nif(Engine)), ok = engine_nif_wrapper(engine_init_nif(Engine)), engine_load_2(Engine, PostCmds, EngineMethods), @@ -701,7 +703,7 @@ engine_load_1(Engine, PreCmds, PostCmds, EngineMethods) -> engine_load_2(Engine, PostCmds, EngineMethods) -> try - ok = engine_nif_wrapper(engine_ctrl_cmd_strings_nif(Engine, ensure_bin_cmds(PostCmds))), + ok = engine_nif_wrapper(engine_ctrl_cmd_strings_nif(Engine, ensure_bin_cmds(PostCmds), 0)), [ok = engine_nif_wrapper(engine_register_nif(Engine, engine_method_atom_to_int(Method))) || Method <- EngineMethods], ok @@ -767,6 +769,35 @@ engine_list(Engine0, IdList) -> end end. +%%---------------------------------------------------------------------- +%% Function: engine_ctrl_cmd_string/3 +%%---------------------------------------------------------------------- +-spec engine_ctrl_cmd_string(Engine::term(), + CmdName::unicode:chardata(), + CmdArg::unicode:chardata()) -> + ok | {error, Reason::term()}. +engine_ctrl_cmd_string(Engine, CmdName, CmdArg) -> + engine_ctrl_cmd_string(Engine, CmdName, CmdArg, false). + +%%---------------------------------------------------------------------- +%% Function: engine_ctrl_cmd_string/4 +%%---------------------------------------------------------------------- +-spec engine_ctrl_cmd_string(Engine::term(), + CmdName::unicode:chardata(), + CmdArg::unicode:chardata(), + Optional::boolean()) -> + ok | {error, Reason::term()}. +engine_ctrl_cmd_string(Engine, CmdName, CmdArg, Optional) -> + case engine_ctrl_cmd_strings_nif(Engine, + ensure_bin_cmds([{CmdName, CmdArg}]), + bool_to_int(Optional)) of + ok -> + ok; + notsup -> + erlang:error(notsup); + {error, Error} -> + {error, Error} + end. %%-------------------------------------------------------------------- %%% On load @@ -1100,9 +1131,17 @@ ec_curve(X) -> privkey_to_pubkey(Alg, EngineMap) when Alg == rsa; Alg == dss; Alg == ecdsa -> - case notsup_to_error(privkey_to_pubkey_nif(Alg, format_pkey(Alg,EngineMap))) of + try privkey_to_pubkey_nif(Alg, format_pkey(Alg,EngineMap)) + of [_|_]=L -> map_ensure_bin_as_int(L); X -> X + catch + error:badarg when Alg==ecdsa -> + {error, notsup}; + error:badarg -> + {error, not_found}; + error:notsup -> + {error, notsup} end. privkey_to_pubkey_nif(_Alg, _EngineMap) -> ?nif_stub. @@ -1258,7 +1297,7 @@ engine_init_nif(_Engine) -> ?nif_stub. engine_finish_nif(_Engine) -> ?nif_stub. engine_free_nif(_Engine) -> ?nif_stub. engine_load_dynamic_nif() -> ?nif_stub. -engine_ctrl_cmd_strings_nif(_Engine, _Cmds) -> ?nif_stub. +engine_ctrl_cmd_strings_nif(_Engine, _Cmds, _Optional) -> ?nif_stub. engine_add_nif(_Engine) -> ?nif_stub. engine_remove_nif(_Engine) -> ?nif_stub. engine_register_nif(_Engine, _EngineMethod) -> ?nif_stub. @@ -1301,6 +1340,9 @@ engine_methods_convert_to_bitmask(engine_method_none, _BitMask) -> engine_methods_convert_to_bitmask([M |Ms], BitMask) -> engine_methods_convert_to_bitmask(Ms, BitMask bor engine_method_atom_to_int(M)). +bool_to_int(true) -> 1; +bool_to_int(false) -> 0. + engine_method_atom_to_int(engine_method_rsa) -> 16#0001; engine_method_atom_to_int(engine_method_dsa) -> 16#0002; engine_method_atom_to_int(engine_method_dh) -> 16#0004; diff --git a/lib/crypto/test/engine_SUITE.erl b/lib/crypto/test/engine_SUITE.erl index 5967331d8e..f206f967c7 100644 --- a/lib/crypto/test/engine_SUITE.erl +++ b/lib/crypto/test/engine_SUITE.erl @@ -44,6 +44,8 @@ all() -> pre_command_fail_bad_value, pre_command_fail_bad_key, failed_engine_init, + ctrl_cmd_string, + ctrl_cmd_string_optional, {group, engine_stored_key} ]. @@ -354,6 +356,67 @@ failed_engine_init(Config) when is_list(Config) -> {skip, "Engine not supported on this OpenSSL version"} end. + +ctrl_cmd_string()-> + [{doc, "Test that a not known optional ctrl comand do not fail"}]. +ctrl_cmd_string(Config) when is_list(Config) -> + try + case crypto:get_test_engine() of + {error, notexist} -> + {skip, "OTP Test engine not found"}; + {ok, Engine} -> + case crypto:engine_load(<<"dynamic">>, + [{<<"SO_PATH">>, Engine}, + {<<"ID">>, <<"MD5">>}, + <<"LOAD">>], + []) of + {ok, E} -> + case crypto:engine_ctrl_cmd_string(E, <<"TEST">>, <<"17">>) of + ok -> + ct:fail(fail_ctrl_cmd_should_fail); + {error,ctrl_cmd_failed} -> + ok + end, + ok = crypto:engine_unload(E); + {error, bad_engine_id} -> + {skip, "Dynamic Engine not supported"} + end + end + catch + error:notsup -> + {skip, "Engine not supported on this OpenSSL version"} + end. + +ctrl_cmd_string_optional()-> + [{doc, "Test that a not known optional ctrl comand do not fail"}]. +ctrl_cmd_string_optional(Config) when is_list(Config) -> + try + case crypto:get_test_engine() of + {error, notexist} -> + {skip, "OTP Test engine not found"}; + {ok, Engine} -> + case crypto:engine_load(<<"dynamic">>, + [{<<"SO_PATH">>, Engine}, + {<<"ID">>, <<"MD5">>}, + <<"LOAD">>], + []) of + {ok, E} -> + case crypto:engine_ctrl_cmd_string(E, <<"TEST">>, <<"17">>, true) of + ok -> + ok; + _ -> + ct:fail(fail_ctrl_cmd_string) + end, + ok = crypto:engine_unload(E); + {error, bad_engine_id} -> + {skip, "Dynamic Engine not supported"} + end + end + catch + error:notsup -> + {skip, "Engine not supported on this OpenSSL version"} + end. + %%%---------------------------------------------------------------- %%% Pub/priv key storage tests. Thoose are for testing the crypto.erl %%% support for using priv/pub keys stored in an engine. @@ -432,65 +495,93 @@ pub_encrypt_priv_decrypt_rsa_pwd(Config) -> get_pub_from_priv_key_rsa(Config) -> Priv = #{engine => engine_ref(Config), key_id => key_id(Config, "rsa_private_key.pem")}, - try crypto:privkey_to_pubkey(rsa, Priv) of + case crypto:privkey_to_pubkey(rsa, Priv) of + {error, not_found} -> + {fail, "Key not found"}; + {error, notsup} -> + {skip, "RSA not supported"}; + {error, Error} -> + {fail, {wrong_error,Error}}; Pub -> ct:log("rsa Pub = ~p",[Pub]), sign_verify(rsa, sha, Priv, Pub) - catch - error:notsup -> {skip, "RSA not implemented"} end. get_pub_from_priv_key_rsa_pwd(Config) -> Priv = #{engine => engine_ref(Config), key_id => key_id(Config, "rsa_private_key_pwd.pem"), password => "password"}, - try crypto:privkey_to_pubkey(rsa, Priv) of + case crypto:privkey_to_pubkey(rsa, Priv) of + {error, not_found} -> + {fail, "Key not found"}; + {error, notsup} -> + {skip, "RSA not supported"}; + {error, Error} -> + {fail, {wrong_error,Error}}; Pub -> ct:log("rsa Pub = ~p",[Pub]), sign_verify(rsa, sha, Priv, Pub) - catch - error:notsup -> {skip, "RSA not supported"} end. get_pub_from_priv_key_rsa_pwd_no_pwd(Config) -> Priv = #{engine => engine_ref(Config), key_id => key_id(Config, "rsa_private_key_pwd.pem")}, - try crypto:privkey_to_pubkey(rsa, Priv) of - _ -> {fail, "PWD prot pubkey fetch succeded although no pwd!"} - catch - error:badarg -> ok + case crypto:privkey_to_pubkey(rsa, Priv) of + {error, not_found} -> + ok; + {error, notsup} -> + {skip, "RSA not supported"}; + {error, Error} -> + {fail, {wrong_error,Error}}; + Pub -> + ct:log("rsa Pub = ~p",[Pub]), + {fail, "PWD prot pubkey fetch succeded although no pwd!"} end. get_pub_from_priv_key_rsa_pwd_bad_pwd(Config) -> Priv = #{engine => engine_ref(Config), key_id => key_id(Config, "rsa_private_key_pwd.pem"), password => "Bad password"}, - try crypto:privkey_to_pubkey(rsa, Priv) of - _ -> {fail, "PWD prot pubkey fetch succeded with bad pwd!"} - catch - error:badarg -> ok + case crypto:privkey_to_pubkey(rsa, Priv) of + {error, not_found} -> + ok; + {error, notsup} -> + {skip, "RSA not supported"}; + {error, Error} -> + {fail, {wrong_error,Error}}; + Pub -> + ct:log("rsa Pub = ~p",[Pub]), + {fail, "PWD prot pubkey fetch succeded with bad pwd!"} end. get_pub_from_priv_key_dsa(Config) -> Priv = #{engine => engine_ref(Config), key_id => key_id(Config, "dsa_private_key.pem")}, - try crypto:privkey_to_pubkey(dss, Priv) of + case crypto:privkey_to_pubkey(dss, Priv) of + {error, not_found} -> + {fail, "Key not found"}; + {error, notsup} -> + {skip, "DSA not supported"}; + {error, Error} -> + {fail, {wrong_error,Error}}; Pub -> ct:log("dsa Pub = ~p",[Pub]), sign_verify(dss, sha, Priv, Pub) - catch - error:notsup -> {skip, "DSA not supported"} end. get_pub_from_priv_key_ecdsa(Config) -> Priv = #{engine => engine_ref(Config), key_id => key_id(Config, "ecdsa_private_key.pem")}, - try crypto:privkey_to_pubkey(ecdsa, Priv) of + case crypto:privkey_to_pubkey(ecdsa, Priv) of + {error, not_found} -> + {fail, "Key not found"}; + {error, notsup} -> + {skip, "ECDSA not supported"}; + {error, Error} -> + {fail, {wrong_error,Error}}; Pub -> ct:log("ecdsa Pub = ~p",[Pub]), sign_verify(ecdsa, sha, Priv, Pub) - catch - error:notsup -> {skip, "ECDSA not supported"} end. %%%================================================================ diff --git a/lib/crypto/vsn.mk b/lib/crypto/vsn.mk index 1dceebb4e4..da3915a4fc 100644 --- a/lib/crypto/vsn.mk +++ b/lib/crypto/vsn.mk @@ -1 +1 @@ -CRYPTO_VSN = 4.1 +CRYPTO_VSN = 4.2 diff --git a/lib/debugger/doc/src/notes.xml b/lib/debugger/doc/src/notes.xml index 21fe7d449d..e71746e30d 100644 --- a/lib/debugger/doc/src/notes.xml +++ b/lib/debugger/doc/src/notes.xml @@ -33,6 +33,21 @@ <p>This document describes the changes made to the Debugger application.</p> +<section><title>Debugger 4.2.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Do not quote variables and button names in Debugger + windows. The bug was introduced in Erlang/OTP 20.1. </p> + <p> + Own Id: OTP-14802</p> + </item> + </list> + </section> + +</section> + <section><title>Debugger 4.2.3</title> <section><title>Improvements and New Features</title> diff --git a/lib/debugger/src/dbg_wx_win.erl b/lib/debugger/src/dbg_wx_win.erl index 9f59915476..f1298154ab 100644 --- a/lib/debugger/src/dbg_wx_win.erl +++ b/lib/debugger/src/dbg_wx_win.erl @@ -299,7 +299,7 @@ open_help(_Parent, HelpHtmlFile) -> %%-------------------------------------------------------------------- to_string(Atom) when is_atom(Atom) -> - io_lib:format("~tw", [Atom]); + atom_to_list(Atom); to_string(Integer) when is_integer(Integer) -> integer_to_list(Integer); to_string([]) -> ""; diff --git a/lib/debugger/vsn.mk b/lib/debugger/vsn.mk index 72cedb2240..57da7e5618 100644 --- a/lib/debugger/vsn.mk +++ b/lib/debugger/vsn.mk @@ -1 +1 @@ -DEBUGGER_VSN = 4.2.3 +DEBUGGER_VSN = 4.2.4 diff --git a/lib/dialyzer/doc/src/notes.xml b/lib/dialyzer/doc/src/notes.xml index 6a6e65cb94..a1eecfb3fe 100644 --- a/lib/dialyzer/doc/src/notes.xml +++ b/lib/dialyzer/doc/src/notes.xml @@ -32,6 +32,29 @@ <p>This document describes the changes made to the Dialyzer application.</p> +<section><title>Dialyzer 3.2.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> The error message returned from Dialyzer when, for + example, a modified record field type is not a subtype of + the declared type, no longer includes a call stack. The + bug was introduced in Erlang/OTP 19.3. </p> + <p> + Own Id: OTP-14742</p> + </item> + <item> + <p> A bug relating to maps and never returning functions + has been fixed. </p> + <p> + Own Id: OTP-14743</p> + </item> + </list> + </section> + +</section> + <section><title>Dialyzer 3.2.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/dialyzer/src/dialyzer_dataflow.erl b/lib/dialyzer/src/dialyzer_dataflow.erl index 8367432ac5..93cfea3c5e 100644 --- a/lib/dialyzer/src/dialyzer_dataflow.erl +++ b/lib/dialyzer/src/dialyzer_dataflow.erl @@ -299,6 +299,7 @@ traverse(Tree, Map, State) -> match_fail -> t_none(); raise -> t_none(); bs_init_writable -> t_from_term(<<>>); + build_stacktrace -> t_list(); Other -> erlang:error({'Unsupported primop', Other}) end, {State, Map, Type}; diff --git a/lib/dialyzer/src/dialyzer_typesig.erl b/lib/dialyzer/src/dialyzer_typesig.erl index d03326ec97..b1f6a54503 100644 --- a/lib/dialyzer/src/dialyzer_typesig.erl +++ b/lib/dialyzer/src/dialyzer_typesig.erl @@ -418,6 +418,7 @@ traverse(Tree, DefinedVars, State) -> match_fail -> throw(error); raise -> throw(error); bs_init_writable -> {State, t_from_term(<<>>)}; + build_stacktrace -> {State, t_list()}; Other -> erlang:error({'Unsupported primop', Other}) end; 'receive' -> diff --git a/lib/dialyzer/vsn.mk b/lib/dialyzer/vsn.mk index d130b14fec..1b46f66602 100644 --- a/lib/dialyzer/vsn.mk +++ b/lib/dialyzer/vsn.mk @@ -1 +1 @@ -DIALYZER_VSN = 3.2.2 +DIALYZER_VSN = 3.2.3 diff --git a/lib/diameter/doc/src/notes.xml b/lib/diameter/doc/src/notes.xml index eded788419..ba4525fd20 100644 --- a/lib/diameter/doc/src/notes.xml +++ b/lib/diameter/doc/src/notes.xml @@ -43,6 +43,22 @@ first.</p> <!-- ===================================================================== --> +<section><title>diameter 2.1.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix documentation typo: peer_down/3 was written where + peer_down/3 was intended.</p> + <p> + Own Id: OTP-14805</p> + </item> + </list> + </section> + +</section> + <section><title>diameter 2.1.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/diameter/doc/src/seealso.ent b/lib/diameter/doc/src/seealso.ent index c5a53670d0..72d74c103c 100644 --- a/lib/diameter/doc/src/seealso.ent +++ b/lib/diameter/doc/src/seealso.ent @@ -79,7 +79,7 @@ significant. <!ENTITY app_handle_answer '<seealso marker="diameter_app#Mod:handle_answer-4">handle_answer/4</seealso>'> <!ENTITY app_handle_request '<seealso marker="diameter_app#Mod:handle_request-3">handle_request/3</seealso>'> <!ENTITY app_handle_error '<seealso marker="diameter_app#Mod:handle_error-4">handle_error/4</seealso>'> -<!ENTITY app_peer_down '<seealso marker="diameter_app#Mod:peer_down-3">peer_up/3</seealso>'> +<!ENTITY app_peer_down '<seealso marker="diameter_app#Mod:peer_down-3">peer_down/3</seealso>'> <!ENTITY app_peer_up '<seealso marker="diameter_app#Mod:peer_up-3">peer_up/3</seealso>'> <!ENTITY app_pick_peer '<seealso marker="diameter_app#Mod:pick_peer-4">pick_peer/4</seealso>'> <!ENTITY app_prepare_retransmit '<seealso marker="diameter_app#Mod:prepare_retransmit-3">prepare_retransmit/3</seealso>'> diff --git a/lib/diameter/src/diameter.appup.src b/lib/diameter/src/diameter.appup.src index d0e58e8410..7da59f8b25 100644 --- a/lib/diameter/src/diameter.appup.src +++ b/lib/diameter/src/diameter.appup.src @@ -56,7 +56,8 @@ {"2.0", [{restart_application, diameter}]}, %% 20.0 {"2.1", [{load_module, diameter_gen}, %% 20.1 {update, diameter_reg, {advanced, "2.1"}}]}, - {"2.1.1", [{load_module, diameter_gen}]} + {"2.1.1", [{load_module, diameter_gen}]}, %% 20.1.2 + {"2.1.2", []} %% 20.1.3 ], [ {"0.9", [{restart_application, diameter}]}, @@ -93,6 +94,7 @@ {"1.12.2", [{restart_application, diameter}]}, {"2.0", [{restart_application, diameter}]}, {"2.1", [{restart_application, diameter}]}, - {"2.1.1", [{load_module, diameter_gen}]} + {"2.1.1", [{load_module, diameter_gen}]}, + {"2.1.2", []} ] }. diff --git a/lib/diameter/vsn.mk b/lib/diameter/vsn.mk index bfb260ed8f..0c852d75cd 100644 --- a/lib/diameter/vsn.mk +++ b/lib/diameter/vsn.mk @@ -17,5 +17,5 @@ # %CopyrightEnd% APPLICATION = diameter -DIAMETER_VSN = 2.1.2 +DIAMETER_VSN = 2.1.3 APP_VSN = $(APPLICATION)-$(DIAMETER_VSN)$(PRE_VSN) diff --git a/lib/edoc/doc/src/notes.xml b/lib/edoc/doc/src/notes.xml index 96d7597d83..240789e876 100644 --- a/lib/edoc/doc/src/notes.xml +++ b/lib/edoc/doc/src/notes.xml @@ -32,6 +32,21 @@ <p>This document describes the changes made to the EDoc application.</p> +<section><title>Edoc 0.9.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> The map type is correctly denoted as <c>map()</c> in + function specifications and types. </p> + <p> + Own Id: OTP-14777</p> + </item> + </list> + </section> + +</section> + <section><title>Edoc 0.9.1</title> <section><title>Improvements and New Features</title> diff --git a/lib/edoc/vsn.mk b/lib/edoc/vsn.mk index 065944ccef..2f6d469536 100644 --- a/lib/edoc/vsn.mk +++ b/lib/edoc/vsn.mk @@ -1 +1 @@ -EDOC_VSN = 0.9.1 +EDOC_VSN = 0.9.2 diff --git a/lib/eldap/doc/src/notes.xml b/lib/eldap/doc/src/notes.xml index 7aad745f67..8b066671ee 100644 --- a/lib/eldap/doc/src/notes.xml +++ b/lib/eldap/doc/src/notes.xml @@ -31,6 +31,21 @@ </header> <p>This document describes the changes made to the Eldap application.</p> +<section><title>Eldap 1.2.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + <section><title>Eldap 1.2.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/eldap/vsn.mk b/lib/eldap/vsn.mk index 721387d97d..1636b6bb6d 100644 --- a/lib/eldap/vsn.mk +++ b/lib/eldap/vsn.mk @@ -1 +1 @@ -ELDAP_VSN = 1.2.2 +ELDAP_VSN = 1.2.3 diff --git a/lib/erl_docgen/doc/src/notes.xml b/lib/erl_docgen/doc/src/notes.xml index 59c65665d4..2652b4b0c8 100644 --- a/lib/erl_docgen/doc/src/notes.xml +++ b/lib/erl_docgen/doc/src/notes.xml @@ -31,7 +31,26 @@ </header> <p>This document describes the changes made to the <em>erl_docgen</em> application.</p> - <section><title>Erl_Docgen 0.7.1</title> + <section><title>Erl_Docgen 0.7.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> The style for code, warning and note tags in the pdf + have been changed so they look like the html version. + <br/> The spacing around code blocks have been changed + for both html and pdf so it's the same regardless if the + user have a newline after the start tag (or before the + end tag) or not. </p> + <p> + Own Id: OTP-14674</p> + </item> + </list> + </section> + +</section> + +<section><title>Erl_Docgen 0.7.1</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/erl_docgen/vsn.mk b/lib/erl_docgen/vsn.mk index 17a7c483f4..95b2329ac5 100644 --- a/lib/erl_docgen/vsn.mk +++ b/lib/erl_docgen/vsn.mk @@ -1 +1 @@ -ERL_DOCGEN_VSN = 0.7.1 +ERL_DOCGEN_VSN = 0.7.2 diff --git a/lib/erl_interface/doc/src/notes.xml b/lib/erl_interface/doc/src/notes.xml index ec20f3c67f..641a3de13f 100644 --- a/lib/erl_interface/doc/src/notes.xml +++ b/lib/erl_interface/doc/src/notes.xml @@ -31,6 +31,21 @@ </header> <p>This document describes the changes made to the Erl_interface application.</p> +<section><title>Erl_Interface 3.10.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + <section><title>Erl_Interface 3.10</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/erl_interface/vsn.mk b/lib/erl_interface/vsn.mk index 01fcee86dd..d76d110afd 100644 --- a/lib/erl_interface/vsn.mk +++ b/lib/erl_interface/vsn.mk @@ -1,2 +1,2 @@ -EI_VSN = 3.10 +EI_VSN = 3.10.1 ERL_INTERFACE_VSN = $(EI_VSN) diff --git a/lib/eunit/doc/src/notes.xml b/lib/eunit/doc/src/notes.xml index 7133befe37..b38cb2e70e 100644 --- a/lib/eunit/doc/src/notes.xml +++ b/lib/eunit/doc/src/notes.xml @@ -33,6 +33,21 @@ </header> <p>This document describes the changes made to the EUnit application.</p> +<section><title>Eunit 2.3.5</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + <section><title>Eunit 2.3.4</title> <section><title>Improvements and New Features</title> diff --git a/lib/eunit/vsn.mk b/lib/eunit/vsn.mk index 25bb0dec17..2ed9eaac16 100644 --- a/lib/eunit/vsn.mk +++ b/lib/eunit/vsn.mk @@ -1 +1 @@ -EUNIT_VSN = 2.3.4 +EUNIT_VSN = 2.3.5 diff --git a/lib/hipe/doc/src/notes.xml b/lib/hipe/doc/src/notes.xml index eadaee50e2..bad0c254ce 100644 --- a/lib/hipe/doc/src/notes.xml +++ b/lib/hipe/doc/src/notes.xml @@ -31,6 +31,39 @@ </header> <p>This document describes the changes made to HiPE.</p> +<section><title>Hipe 3.17</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix bug for hipe compiled code using + <c><<X/utf32>></c> binary construction that + could cause faulty result or even VM crash.</p> + <p> + On architectures other than x86_64, code need to be + recompiled to benefit from this fix.</p> + <p> + Own Id: OTP-14740</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Added documentation about limitations of hipe compared to + beam compiled code.</p> + <p> + Own Id: OTP-14767</p> + </item> + </list> + </section> + +</section> + <section><title>Hipe 3.16.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/hipe/icode/hipe_beam_to_icode.erl b/lib/hipe/icode/hipe_beam_to_icode.erl index 4c7fbad711..7ff9fd83eb 100644 --- a/lib/hipe/icode/hipe_beam_to_icode.erl +++ b/lib/hipe/icode/hipe_beam_to_icode.erl @@ -1159,6 +1159,11 @@ trans_fun([{put_map_exact,{f,Lbl},Map,Dst,_N,{list,Pairs}}|Instructions], Env) - gen_put_map_instrs(new, exact, TempMapVar, Dst, new, Pairs, Env1) end, [MapMove, TempMapMove, PutInstructions | trans_fun(Instructions, Env2)]; +%%--- build_stacktrace --- +trans_fun([build_stacktrace|Instructions], Env) -> + Vars = [mk_var({x,0})], %{x,0} is implict arg and dst + [hipe_icode:mk_primop(Vars,build_stacktrace,Vars), + trans_fun(Instructions, Env)]; %%-------------------------------------------------------------------- %%--- ERROR HANDLING --- %%-------------------------------------------------------------------- diff --git a/lib/hipe/icode/hipe_icode_primops.erl b/lib/hipe/icode/hipe_icode_primops.erl index ec9e3c8608..941516e8b1 100644 --- a/lib/hipe/icode/hipe_icode_primops.erl +++ b/lib/hipe/icode/hipe_icode_primops.erl @@ -132,6 +132,7 @@ is_safe({hipe_bs_primop, {bs_match_string, _, _}}) -> false; is_safe({hipe_bs_primop, {bs_append, _, _, _, _}}) -> false; is_safe({hipe_bs_primop, {bs_private_append, _, _}}) -> false; is_safe({hipe_bs_primop, bs_init_writable}) -> true; +is_safe(build_stacktrace) -> true; is_safe(#mkfun{}) -> true; is_safe(#unsafe_element{}) -> true; is_safe(#unsafe_update_element{}) -> true; @@ -234,6 +235,7 @@ fails({hipe_bs_primop, bs_final}) -> false; fails({hipe_bs_primop, {bs_append, _, _, _, _}}) -> true; fails({hipe_bs_primop, {bs_private_append, _, _}}) -> true; fails({hipe_bs_primop, bs_init_writable}) -> true; +fails(build_stacktrace) -> false; fails(#mkfun{}) -> false; fails(#unsafe_element{}) -> false; fails(#unsafe_update_element{}) -> false; @@ -731,6 +733,8 @@ type(Primop, Args) -> erl_types:t_any(); debug_native_called -> erl_types:t_any(); + build_stacktrace -> + erl_types:t_list(); {M, F, A} -> erl_bif_types:type(M, F, A, Args) end. @@ -903,6 +907,8 @@ type(Primop) -> erl_types:t_any(); %%% ----------------------------------------------------- %%% Other + build_stacktrace -> + erl_types:t_any(); #closure_element{} -> erl_types:t_any(); redtest -> diff --git a/lib/hipe/icode/hipe_icode_range.erl b/lib/hipe/icode/hipe_icode_range.erl index 37360cee73..cf74c1eb5b 100644 --- a/lib/hipe/icode/hipe_icode_range.erl +++ b/lib/hipe/icode/hipe_icode_range.erl @@ -1186,7 +1186,8 @@ basic_type(unsafe_hd) -> not_analysed; basic_type(unsafe_tl) -> not_int; basic_type(#element{}) -> not_analysed; basic_type(#unsafe_element{}) -> not_analysed; -basic_type(#unsafe_update_element{}) -> not_analysed. +basic_type(#unsafe_update_element{}) -> not_analysed; +basic_type(build_stacktrace) -> not_int. -spec analyse_bs_get_integer(integer(), integer(), boolean()) -> range_tuple(). diff --git a/lib/hipe/rtl/hipe_rtl_primops.erl b/lib/hipe/rtl/hipe_rtl_primops.erl index 35ae2da895..d646b78a3d 100644 --- a/lib/hipe/rtl/hipe_rtl_primops.erl +++ b/lib/hipe/rtl/hipe_rtl_primops.erl @@ -394,6 +394,8 @@ gen_primop({Op,Dst,Args,Cont,Fail}, IsGuard, ConstTab) -> end; debug_native_called -> [hipe_rtl:mk_call(Dst, Op, Args, Cont, Fail, not_remote)]; + build_stacktrace -> + [hipe_rtl:mk_call(Dst, Op, Args, Cont, Fail, not_remote)]; %% Only names listed above are accepted! MFA:s are not primops! _ -> diff --git a/lib/hipe/vsn.mk b/lib/hipe/vsn.mk index f88d9147b1..508ec00548 100644 --- a/lib/hipe/vsn.mk +++ b/lib/hipe/vsn.mk @@ -1 +1 @@ -HIPE_VSN = 3.16.1 +HIPE_VSN = 3.17 diff --git a/lib/ic/doc/src/notes.xml b/lib/ic/doc/src/notes.xml index ea8bf758cf..fc68ec386c 100644 --- a/lib/ic/doc/src/notes.xml +++ b/lib/ic/doc/src/notes.xml @@ -31,7 +31,22 @@ <file>notes.xml</file> </header> - <section><title>IC 4.4.2</title> + <section><title>IC 4.4.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + +<section><title>IC 4.4.2</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/ic/vsn.mk b/lib/ic/vsn.mk index f0e5e7c266..b9f1ef7f20 100644 --- a/lib/ic/vsn.mk +++ b/lib/ic/vsn.mk @@ -1 +1 @@ -IC_VSN = 4.4.2 +IC_VSN = 4.4.3 diff --git a/lib/inets/doc/src/notes.xml b/lib/inets/doc/src/notes.xml index 07e29b5542..70b2811c0e 100644 --- a/lib/inets/doc/src/notes.xml +++ b/lib/inets/doc/src/notes.xml @@ -33,7 +33,37 @@ <file>notes.xml</file> </header> - <section><title>Inets 6.4.4</title> + <section><title>Inets 6.4.5</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + CGI environment variable CONTENT_LENGTH shall be a string</p> + <p> + Own Id: OTP-14679</p> + </item> + <item> + <p> + In relaxed mode disregard Content-Length header if there + is also a Transfer-Encoding header.</p> + <p> + Own Id: OTP-14727</p> + </item> + <item> + <p> + Eliminated race condition, that could cause http request + to sporadically fail to complete successfully, when + keep-alive connections are used.</p> + <p> + Own Id: OTP-14783</p> + </item> + </list> + </section> + +</section> + +<section><title>Inets 6.4.4</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl index 2efe2c2858..821eb7f02f 100644 --- a/lib/inets/src/http_client/httpc.erl +++ b/lib/inets/src/http_client/httpc.erl @@ -319,7 +319,7 @@ store_cookies(SetCookieHeaders, Url, Profile) Scheme = scheme_to_atom(maps:get(scheme, URI, '')), Host = maps:get(host, URI, ""), Port = maps:get(port, URI, default_port(Scheme)), - Path = maps:get(path, URI, ""), + Path = uri_string:recompose(#{path => maps:get(path, URI, "")}), %% Since the Address part is not actually used %% by the manager when storing cookies, we dont %% care about ipv6-host-with-brackets. @@ -539,7 +539,7 @@ handle_request(Method, Url, Host = http_util:maybe_add_brackets(maps:get(host, URI, ""), BracketedHost), Port = maps:get(port, URI, default_port(Scheme)), Host2 = http_request:normalize_host(Scheme, Host, Port), - Path = maps:get(path, URI, ""), + Path = uri_string:recompose(#{path => maps:get(path, URI, "")}), Query = add_question_mark(maps:get(query, URI, "")), HeadersRecord = header_record(NewHeaders, Host2, HTTPOptions), diff --git a/lib/inets/test/Makefile b/lib/inets/test/Makefile index ffc512050a..95e0559470 100644 --- a/lib/inets/test/Makefile +++ b/lib/inets/test/Makefile @@ -150,6 +150,7 @@ INETS_ROOT = ../../inets MODULES = \ inets_test_lib \ erl_make_certs \ + make_certs \ ftp_SUITE \ ftp_format_SUITE \ http_format_SUITE \ @@ -157,6 +158,8 @@ MODULES = \ httpc_cookie_SUITE \ httpc_proxy_SUITE \ httpd_SUITE \ + httpd_bench_SUITE \ + http_test_lib \ old_httpd_SUITE \ httpd_basic_SUITE \ httpd_mod \ @@ -189,7 +192,7 @@ SOURCE = $(ERL_FILES) $(HRL_FILES) TARGET_FILES = $(MODULES:%=$(EBIN)/%.$(EMULATOR)) -INETS_SPECS = inets.spec +INETS_SPECS = inets.spec inets_bench.spec COVER_FILE = inets.cover INETS_FILES = inets.config $(INETS_SPECS) @@ -200,8 +203,10 @@ INETS_FILES = inets.config $(INETS_SPECS) # inets_ftp_suite \ # inets_tftp_suite + INETS_DATADIRS = inets_SUITE_data inets_socketwrap_SUITE_data -HTTPD_DATADIRS = httpd_test_data httpd_SUITE_data httpd_basic_SUITE_data old_httpd_SUITE_data +HTTPD_DATADIRS = httpd_test_data httpd_SUITE_data httpd_basic_SUITE_data old_httpd_SUITE_data httpd_bench_SUITE_data + HTTPC_DATADIRS = httpc_SUITE_data httpc_proxy_SUITE_data FTP_DATADIRS = ftp_SUITE_data diff --git a/lib/inets/test/http_test_lib.erl b/lib/inets/test/http_test_lib.erl new file mode 100644 index 0000000000..38e9e4976e --- /dev/null +++ b/lib/inets/test/http_test_lib.erl @@ -0,0 +1,180 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2015-2015. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% +-module(http_test_lib). + +-include_lib("common_test/include/ct.hrl"). +-include("inets_test_lib.hrl"). +-include("http_internal.hrl"). +-include("httpc_internal.hrl"). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +dummy_server(SocketType, Inet, Extra) -> + dummy_server(self(), SocketType, Inet, Extra). + +dummy_server(Caller, SocketType, Inet, Extra) -> + Args = [Caller, SocketType, Inet, Extra], + Pid = spawn(?MODULE, dummy_server_init, Args), + receive + {port, Port} -> + {Pid, Port} + end. + +dummy_server_init(Caller, ip_comm, Inet, Extra) -> + ContentCb = proplists:get_value(content_cb, Extra), + BaseOpts = [binary, {packet, 0}, {reuseaddr,true}, {active, false}, {nodelay, true}], + Conf = proplists:get_value(conf, Extra), + {ok, ListenSocket} = gen_tcp:listen(0, [Inet | BaseOpts]), + {ok, Port} = inet:port(ListenSocket), + Caller ! {port, Port}, + dummy_ipcomm_server_loop({httpd_request, parse, [[{max_uri, ?HTTP_MAX_URI_SIZE}, + {max_header, ?HTTP_MAX_HEADER_SIZE}, + {max_version,?HTTP_MAX_VERSION_STRING}, + {max_method, ?HTTP_MAX_METHOD_STRING}, + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH}, + {customize, httpd_custom} + ]]}, + [], ContentCb, Conf, ListenSocket); + +dummy_server_init(Caller, ssl, Inet, Extra) -> + ContentCb = proplists:get_value(content_cb, Extra), + SSLOptions = proplists:get_value(ssl, Extra), + Conf = proplists:get_value(conf, Extra), + BaseOpts = [binary, {reuseaddr,true}, {active, false}, {nodelay, true} | + SSLOptions], + dummy_ssl_server_init(Caller, BaseOpts, Inet, ContentCb, Conf). + +dummy_ssl_server_init(Caller, BaseOpts, Inet, ContentCb, Conf) -> + {ok, ListenSocket} = ssl:listen(0, [Inet | BaseOpts]), + {ok, {_, Port}} = ssl:sockname(ListenSocket), + Caller ! {port, Port}, + dummy_ssl_server_loop({httpd_request, parse, [[{max_uri, ?HTTP_MAX_URI_SIZE}, + {max_method, ?HTTP_MAX_METHOD_STRING}, + {max_version,?HTTP_MAX_VERSION_STRING}, + {max_method, ?HTTP_MAX_METHOD_STRING}, + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH}, + {customize, httpd_custom} + ]]}, + [], ContentCb, Conf, ListenSocket). + +dummy_ipcomm_server_loop(MFA, Handlers, ContentCb, Conf, ListenSocket) -> + receive + stop -> + lists:foreach(fun(Handler) -> Handler ! stop end, Handlers); + {stop, From} -> + Stopper = fun(Handler) -> Handler ! stop end, + lists:foreach(Stopper, Handlers), + From ! {stopped, self()} + after 0 -> + {ok, Socket} = gen_tcp:accept(ListenSocket), + HandlerPid = dummy_request_handler(MFA, Socket, ContentCb, Conf), + gen_tcp:controlling_process(Socket, HandlerPid), + HandlerPid ! ipcomm_controller, + dummy_ipcomm_server_loop(MFA, [HandlerPid | Handlers], + ContentCb, Conf, ListenSocket) + end. + +dummy_ssl_server_loop(MFA, Handlers, ContentCb, Conf, ListenSocket) -> + receive + stop -> + lists:foreach(fun(Handler) -> Handler ! stop end, Handlers); + {stop, From} -> + Stopper = fun(Handler) -> Handler ! stop end, + lists:foreach(Stopper, Handlers), + From ! {stopped, self()} + after 0 -> + {ok, Socket} = ssl:transport_accept(ListenSocket), + HandlerPid = dummy_request_handler(MFA, Socket, ContentCb, Conf), + ssl:controlling_process(Socket, HandlerPid), + HandlerPid ! ssl_controller, + dummy_ssl_server_loop(MFA, [HandlerPid | Handlers], + ContentCb, Conf, ListenSocket) + end. + +dummy_request_handler(MFA, Socket, ContentCb, Conf) -> + spawn(?MODULE, dummy_request_handler_init, [MFA, Socket, ContentCb, Conf]). + +dummy_request_handler_init(MFA, Socket, ContentCb, Conf) -> + SockType = + receive + ipcomm_controller -> + inet:setopts(Socket, [{active, true}]), + ip_comm; + ssl_controller -> + ok = ssl:ssl_accept(Socket, infinity), + ssl:setopts(Socket, [{active, true}]), + ssl + end, + dummy_request_handler_loop(MFA, SockType, Socket, ContentCb, Conf). + +dummy_request_handler_loop({Module, Function, Args}, SockType, Socket, ContentCb, Conf) -> + receive + {Proto, _, Data} when (Proto =:= tcp) orelse (Proto =:= ssl) -> + case handle_request(Module, Function, [Data | Args], Socket, ContentCb, Conf) of + stop when Proto =:= tcp -> + gen_tcp:close(Socket); + stop when Proto =:= ssl -> + ssl:close(Socket); + NewMFA -> + dummy_request_handler_loop(NewMFA, SockType, Socket, ContentCb, Conf) + end; + stop when SockType =:= ip_comm -> + gen_tcp:close(Socket); + stop when SockType =:= ssl -> + ssl:close(Socket) + end. + +handle_request(Module, Function, Args, Socket, ContentCb, Conf) -> + case Module:Function(Args) of + {ok, Result} -> + case ContentCb:handle_http_msg(Result, Socket, Conf) of + stop -> + stop; + <<>> -> + {httpd_request, parse, [[{max_uri,?HTTP_MAX_URI_SIZE}, + {max_header, ?HTTP_MAX_HEADER_SIZE}, + {max_version,?HTTP_MAX_VERSION_STRING}, + {max_method, ?HTTP_MAX_METHOD_STRING}, + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH}, + {customize, httpd_custom} + ]]}; + Data -> + handle_request(httpd_request, parse, + [Data, [{max_uri, ?HTTP_MAX_URI_SIZE}, + {max_header, ?HTTP_MAX_HEADER_SIZE}, + {max_version,?HTTP_MAX_VERSION_STRING}, + {max_method, ?HTTP_MAX_METHOD_STRING}, + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH}, + {customize, httpd_custom} + ]], Socket, ContentCb, Conf) + end; + NewMFA -> + NewMFA + end. + +%% Perform a synchronous stop +dummy_server_stop(Pid) -> + Pid ! {stop, self()}, + receive + {stopped, Pid} -> + ok + end. diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index e0e0c6b6e5..0533b9ab70 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -58,6 +58,9 @@ all() -> groups() -> [ {http, [], real_requests()}, + %% process_leak_on_keepalive is depending on stream_fun_server_close + %% and it shall be the last test case in the suite otherwise cookie + %% will fail. {sim_http, [], only_simulated() ++ [process_leak_on_keepalive]}, {https, [], real_requests()}, {sim_https, [], only_simulated()}, @@ -133,6 +136,7 @@ only_simulated() -> redirect_port_in_host_header, relaxed, multipart_chunks, + get_space, stream_fun_server_close ]. @@ -254,6 +258,16 @@ get_query_string(Config) when is_list(Config) -> {ok, {{_,200,_}, [_ | _], Body = [_ | _]}} = httpc:request(get, Request, [], []), inets_test_lib:check_body(Body). + +%%-------------------------------------------------------------------- +get_space() -> + [{"Test http get request with '%20' in the path of the URL."}]. +get_space(Config) when is_list(Config) -> + Request = {url(group_name(Config), "/space%20.html", Config), []}, + {ok, {{_,200,_}, [_ | _], Body = [_ | _]}} = httpc:request(get, Request, [], []), + + inets_test_lib:check_body(Body). + %%-------------------------------------------------------------------- post() -> [{"Test http post request against local server. We do in this case " @@ -1084,8 +1098,6 @@ remote_socket_close_async(Config) when is_list(Config) -> %%------------------------------------------------------------------------- process_leak_on_keepalive(Config) -> - {ok, ClosedSocket} = gen_tcp:listen(6666, [{active, false}]), - ok = gen_tcp:close(ClosedSocket), Request = {url(group_name(Config), "/dummy.html", Config), []}, HttpcHandlers0 = supervisor:which_children(httpc_handler_sup), {ok, {{_, 200, _}, _, Body}} = httpc:request(get, Request, [], []), @@ -1097,11 +1109,10 @@ process_leak_on_keepalive(Config) -> ordsets:to_list( ordsets:subtract(ordsets:from_list(HttpcHandlers1), ordsets:from_list(HttpcHandlers0))), - sys:replace_state( - Pid, fun (State) -> - Session = element(3, State), - setelement(3, State, Session#session{socket=ClosedSocket}) - end), + State = sys:get_state(Pid), + #session{socket=Socket} = element(3, State), + gen_tcp:close(Socket), + {ok, {{_, 200, _}, _, Body}} = httpc:request(get, Request, [], []), %% bad handler with the closed socket should get replaced by %% the new one, so children count should stay the same @@ -1348,13 +1359,13 @@ group_name(Config) -> server_start(sim_http, _) -> Inet = inet_version(), ok = httpc:set_options([{ipfamily, Inet}]), - {_Pid, Port} = dummy_server(Inet), + {_Pid, Port} = http_test_lib:dummy_server(ip_comm, Inet, [{content_cb, ?MODULE}]), Port; server_start(sim_https, SslConfig) -> Inet = inet_version(), ok = httpc:set_options([{ipfamily, Inet}]), - {_Pid, Port} = dummy_server(ssl, Inet, SslConfig), + {_Pid, Port} = http_test_lib:dummy_server(ssl, Inet, [{ssl, SslConfig}, {content_cb, ?MODULE}]), Port; server_start(_, HttpdConfig) -> @@ -1458,13 +1469,7 @@ receive_replys([ID|IDs]) -> ct:pal({recived_canceld_id, Other}) end. -%% Perform a synchronous stop -dummy_server_stop(Pid) -> - Pid ! {stop, self()}, - receive - {stopped, Pid} -> - ok - end. + inet_version() -> inet. %% Just run inet for now @@ -1592,7 +1597,7 @@ dummy_request_handler_loop({Module, Function, Args}, SockType, Socket) -> handle_request(Module, Function, Args, Socket) -> case Module:Function(Args) of {ok, Result} -> - case handle_http_msg(Result, Socket) of + case handle_http_msg(Result, Socket, []) of stop -> stop; <<>> -> @@ -1617,8 +1622,7 @@ handle_request(Module, Function, Args, Socket) -> NewMFA end. -handle_http_msg({Method, RelUri, _, {_, Headers}, Body}, Socket) -> - +handle_http_msg({Method, RelUri, _, {_, Headers}, Body}, Socket, _) -> ct:print("Request: ~p ~p", [Method, RelUri]), NextRequest = @@ -1745,6 +1749,12 @@ content_length([_Head | Tail]) -> handle_uri("GET","/dummy.html?foo=bar",_,_,_,_) -> "HTTP/1.0 200 OK\r\n\r\nTEST"; +handle_uri("GET","/space%20.html",_,_,_,_) -> + Body = "<HTML><BODY>foobar</BODY></HTML>", + "HTTP/1.1 200 OK\r\n" ++ + "Content-Length:" ++ integer_to_list(length(Body)) ++ "\r\n\r\n" ++ + Body; + handle_uri(_,"/just_close.html",_,_,_,_) -> close; handle_uri(_,"/no_content.html",_,_,_,_) -> diff --git a/lib/inets/test/httpd_bench_SUITE.erl b/lib/inets/test/httpd_bench_SUITE.erl new file mode 100644 index 0000000000..9d8cbf9ae2 --- /dev/null +++ b/lib/inets/test/httpd_bench_SUITE.erl @@ -0,0 +1,846 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +%% +-module(httpd_bench_SUITE). +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). +-include_lib("public_key/include/public_key.hrl"). +-include_lib("kernel/include/file.hrl"). + +-define(remote_host, "NETMARKS_REMOTE_HOST"). +-define(LF, [10]). +-define(CR, [13]). +-define(CRLF, ?CR ++ ?LF). + +%%-------------------------------------------------------------------- +%% Common Test interface functions ----------------------------------- +%%-------------------------------------------------------------------- +suite() -> + [{ct_hooks,[{ts_install_cth,[{nodenames,2}]}]}]. + +all() -> + [ + {group, http_dummy}, + {group, http_inets}, + {group, http_nginx}, + {group, https_inets}, + {group, https_dummy}, + {group, https_nginx}, + {group, http_dummy_keep_alive}, + {group, http_inets_keep_alive}, + {group, http_nginx_keep_alive}, + {group, https_inets_keep_alive}, + {group, https_dummy_keep_alive}, + {group, https_nginx_keep_alive} + ]. + +groups() -> + [ + {http_dummy, [], client_tests()}, + {http_inets, [], client_tests()}, + {http_nginx, [], client_tests()}, + {https_dummy, [], client_tests()}, + {https_inets, [], client_tests()}, + {https_nginx, [], client_tests()}, + {http_dummy_keep_alive, [], client_tests()}, + {http_inets_keep_alive, [], client_tests()}, + {http_nginx_keep_alive, [], client_tests()}, + {https_dummy_keep_alive, [], client_tests()}, + {https_inets_keep_alive, [], client_tests()}, + {https_nginx_keep_alive, [], client_tests()} + ]. + + +client_tests() -> + [wget_small, + erl_dummy_small, + httpc_small, + wget_big, + erl_dummy_big, + httpc_big + ]. + +init_per_suite(Config) -> + try + {Node, Host} = setup(Config, node()), + init_ssl(Config), + [{iter, 10}, {server_node, Node}, {server_host, Host} | Config] + catch _:_ -> + {skipped, "Benchmark machines only"} + end. + +end_per_suite(_Config) -> + [application:stop(App) || App <- [asn1, crypto, public_key, ssl, inets]]. + +init_per_group(Group, Config) when Group == http_dummy_keep_alive; + Group == https_dummy_keep_alive; + Group == http_inets_keep_alive; + Group == https_inets_keep_alive; + Group == http_nginx_keep_alive; + Group == https_nginx_keep_alive -> + Version = http_version(Group), + start_web_server(Group, + [{keep_alive, true}, + {reuse_sessions, false}, + {http_version, Version}, + {http_opts,[{version, Version}]}, + {http_headers, [{"connection", "keep-alive"}]}, + {httpc_opts, [{keep_alive_timeout, 1500}, + {max_keep_alive_length, ?config(iter, Config)}]} + | Config]); +init_per_group(Group, Config) when Group == http_dummy; + Group == https_dummy; + Group == http_inets; + Group == https_inets; + Group == http_nginx; + Group == https_nginx -> + Version = http_version(Group), + start_web_server(Group, + [{keep_alive, false}, + {reuse_sessions, false}, + {http_version, Version}, + {http_headers, [{"connection", "close"}]}, + {http_opts,[{version, Version}]}, + {httpc_opts, [{keep_alive_timeout, 0}, {max_keep_alive_length, 0}]} + | Config]); + + +init_per_group(_, Config) -> + Config. + +end_per_group(Group, Config) -> + stop_web_server(Group, Config). + +init_per_testcase(TestCase, Config) when TestCase == httpc_small; + TestCase == httpc_big + -> + Opts = ?config(httpc_opts, Config), + inets:start(httpc, [{profile, TestCase}, {socket_opts, [{nodelay, true}]}]), + httpc:set_options(Opts, TestCase), + [{profile, TestCase} | proplists:delete(profile, Config)]; + +init_per_testcase(_, Config) -> + Config. +end_per_testcase(TestCase, _Config) when TestCase == httpc_small; + TestCase == httpc_big -> + ok = inets:stop(httpc, TestCase); +end_per_testcase(_TestCase, Config) -> + Config. +%%-------------------------------------------------------------------- +%% Test Cases -------------------------------------------------------- +%%-------------------------------------------------------------------- + +erl_dummy_small(Config) when is_list(Config) -> + {ok, Result} = run_test(httpd_lib_client, "1k_file", Config), + notify(Result, Config, "erl_1k_file"). + +erl_dummy_big(Config) when is_list(Config) -> + {ok, Result} = run_test(httpd_lib_client, "1M_file", Config), + notify(Result, Config, "erl_1M_file"). + +wget_small(Config) when is_list(Config) -> + {ok, Result} = run_test(wget_client, "1k_file", Config), + notify(Result, Config, "wget_1k_file"). + +wget_big(Config) when is_list(Config) -> + {ok, Result} = run_test(wget_client, "1M_file", Config), + notify(Result, Config, "wget_1M_file"). + +httpc_small(Config) when is_list(Config) -> + {ok, Result} = run_test(httpc_client, "1k_file", Config), + notify(Result, Config, "httpc_1k_file"). + +httpc_big(Config) when is_list(Config) -> + {ok, Result} = run_test(httpc_client, "1M_file", Config), + notify(Result, Config, "httpc_1M_file"). + +%%-------------------------------------------------------------------- +%% Internal functions ------------------------------------------------ +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Report benchmark results ------------------------------------------------ +%%-------------------------------------------------------------------- + +notify({TestPerSec, _MBps}, Config, Suffix) -> + Name = lists:concat([?config(protocol,Config), " ", + server_name(Config, [dummy_pid, httpd_pid, nginx_port]), + "", Suffix]), + ct:comment("~p tps", [TestPerSec]), + ct_event:notify(#event{name = benchmark_data, + data=[{value, TestPerSec}, + {suite, ?MODULE}, + {name, Name}]}), + ok. +%%-------------------------------------------------------------------- +%% Setup erlang nodes ------------------------------------------------ +%%-------------------------------------------------------------------- + +server_name(Config, [Server | Rest]) -> + case proplists:get_value(Server, Config) of + undefined -> + server_name(Config, Rest); + _ -> + server_name(Server) + end. + +server_name(httpd_pid) -> + "inets"; +server_name(nginx_port) -> + "nginx"; +server_name(dummy_pid) -> + "erlang". + +setup(_Config, nonode@nohost) -> + exit(dist_not_enabled); +setup(_Config, _LocalNode) -> + Host = case os:getenv(?remote_host) of + false -> + {ok, This} = inet:gethostname(), + This; + RemHost -> + RemHost + end, + Node = list_to_atom("inets_perf_server@" ++ Host), + SlaveArgs = case init:get_argument(pa) of + {ok, PaPaths} -> + lists:append([" -pa " ++ P || [P] <- PaPaths]); + _ -> [] + end, + Prog = + case os:find_executable("erl") of + false -> "erl"; + P -> P + end, + case net_adm:ping(Node) of + pong -> ok; + pang -> + {ok, Node} = slave:start(Host, inets_perf_server, SlaveArgs, no_link, Prog) + end, + Path = code:get_path(), + true = rpc:call(Node, code, set_path, [Path]), + [ensure_started(Node, App) || App <- [asn1, crypto, public_key, ssl, inets]], + [ensure_started(node(), App) || App <- [asn1, crypto, public_key, ssl, inets]], + (Node =:= node()) andalso restrict_schedulers(client), + {Node, Host}. + +ensure_started(Node, App) -> + ok = rpc:call(Node, application, ensure_started, [App]). + + +restrict_schedulers(Type) -> + %% We expect this to run on 8 core machine + Extra0 = 1, + Extra = if (Type =:= server) -> -Extra0; true -> Extra0 end, + Scheds = erlang:system_info(schedulers), + erlang:system_flag(schedulers_online, (Scheds div 2) + Extra). + +%%-------------------------------------------------------------------- +%% Setup TLS input files ------------------------------------------------ +%%-------------------------------------------------------------------- + +init_ssl(Config) -> + DDir = ?config(data_dir, Config), + PDir = ?config(priv_dir, Config), + {ok, _} = make_certs:all(DDir, + PDir). +cert_opts(Config) -> + ClientCaCertFile = filename:join([?config(priv_dir, Config), + "client", "cacerts.pem"]), + ClientCertFile = filename:join([?config(priv_dir, Config), + "client", "cert.pem"]), + ServerCaCertFile = filename:join([?config(priv_dir, Config), + "server", "cacerts.pem"]), + ServerCertFile = filename:join([?config(priv_dir, Config), + "server", "cert.pem"]), + ServerKeyFile = filename:join([?config(priv_dir, Config), + "server", "key.pem"]), + ClientKeyFile = filename:join([?config(priv_dir, Config), + "client", "key.pem"]), + [{server_verification_opts, [{reuseaddr, true}, + {cacertfile, ServerCaCertFile}, + {ciphers, ["ECDHE-RSA-AES256-GCM-SHA384"]}, + {certfile, ServerCertFile}, {keyfile, ServerKeyFile}]}, + {client_verification_opts, [ + %%{verify, verify_peer}, + {cacertfile, ClientCaCertFile}, + {certfile, ClientCertFile}, + {keyfile, ClientKeyFile}]}]. + +%%-------------------------------------------------------------------- +%% Run clients ------------------------------------------------ +%%-------------------------------------------------------------------- + +run_test(Client, File, Config) -> + Parent = self(), + Pid = spawn(fun() -> + receive + go -> + Parent ! {self(), + do_runs(Client, [{file, File} | Config])} + end + end), + Pid ! go, + receive + {Pid,{{tps, Tps}, {mbps, MBps}}} -> + ct:pal("Tps: ~p Bps~p", [Tps, MBps]), + {ok, {Tps, MBps}} + end. + +do_runs(Client, Config) -> + N = ?config(iter, Config), + DataDir = ?config(data_dir, Config), + File = ?config(file, Config), + Name = filename:join(DataDir, File), + Args = ?MODULE:Client(Config), + ?MODULE:Client({init, Args}), + Run = + fun() -> + ok = ?MODULE:Client(Args, N) + end, + {ok, Info} = file:read_file_info(Name, []), + Length = Info#file_info.size, + {TimeInMicro, _} = timer:tc(Run), + ReqPerSecond = (1000000 * N) div TimeInMicro, + BytesPerSecond = (1000000 * N * Length) div TimeInMicro, + {{tps, ReqPerSecond}, {mbps, BytesPerSecond}}. + + +httpc_client({init, [_, Profile, URL, Headers, HTTPOpts]}) -> + %% Make sure pipelining feature will kick in when appropriate. + {ok, {{_ ,200, "OK"}, _,_}} = httpc:request(get,{URL, Headers}, HTTPOpts, + [{body_format, binary}, + {socket_opts, [{nodelay, true}]}], Profile), + ct:sleep(1000); +httpc_client(Config) -> + File = ?config(file, Config), + Protocol = ?config(protocol, Config), + Profile = ?config(profile, Config), + URL = (?config(urlfun,Config))(File), + Headers = ?config(http_headers, Config), + HTTPOpts = ?config(http_opts, Config), + [Protocol, Profile, URL, Headers, HTTPOpts]. +httpc_client(_,0) -> + ok; +httpc_client([Protocol, Profile, URL, Headers, HTTPOpts], N) -> + {ok, {{_ ,200,"OK"}, _,_}} = httpc:request(get,{URL, Headers}, HTTPOpts, [{body_format, binary}, + {socket_opts, [{nodelay, true}]}], Profile), + httpc_client([Protocol, Profile, URL, Headers, HTTPOpts], N-1). + +httpd_lib_client({init, [_, Type, Version, Request, Host, Port, Opts]}) -> + ok = httpd_test_lib:verify_request(Type, Host, + Port, + Opts, node(), + Request, + [{statuscode, 200}, + {version, Version}], infinity), + ct:sleep(1000); +httpd_lib_client(Config) -> + File = ?config(file, Config), + KeepAlive = ?config(keep_alive, Config), + Host = ?config(server_host, Config), + Port = ?config(port, Config), + ReuseSession = ?config(reuse_sessions, Config), + {Type, Opts} = + case ?config(protocol, Config) of + "http" -> + {ip_comm, [{active, true}, {mode, binary},{nodelay, true}]}; + "https" -> + SSLOpts = proplists:get_value(client_verification_opts, cert_opts(Config)), + {ssl, [{active, true}, {mode, binary}, {nodelay, true}, + {reuse_sessions, ReuseSession} | SSLOpts]} + + end, + Version = ?config(http_version, Config), + Request = case KeepAlive of + true -> + http_request("GET /" ++ File ++ " ", Version, Host, {"connection:keep-alive\r\n", ""}); + false -> + http_request("GET /" ++ File ++ " ", Version, Host) + end, + + Args = [KeepAlive, Type, Version, Request, Host, Port, Opts], + httpd_lib_client(Args, 1), + Args. + +httpd_lib_client(_, 0) -> + ok; +httpd_lib_client([true, Type, Version, Request, Host, Port, Opts], N) -> + ok = httpd_test_lib:verify_request_N(Type, Host, + Port, + Opts, node(), + Request, + [{statuscode, 200}, + {version, Version}], infinity, N); +httpd_lib_client([false, Type, Version, Request, Host, Port, Opts] = List, N) -> + ok = httpd_test_lib:verify_request(Type, Host, + Port, + Opts, node(), + Request, + [{statuscode, 200}, + {version, Version}], infinity), + httpd_lib_client(List, N-1). + +wget_client({init,_}) -> + ok; +wget_client(Config) -> + File = ?config(file, Config), + URL = (?config(urlfun,Config))(File), + KeepAlive = ?config(keep_alive, Config), + PrivDir = ?config(priv_dir, Config), + Protocol = ?config(protocol, Config), + Iter = ?config(iter, Config), + FileName = filename:join(PrivDir, "wget_req"), + ProtocolOpts = case Protocol of + "http" -> + []; + "https" -> + proplists:get_value(client_verification_opts, cert_opts(Config)) + end, + wget_req_file(FileName,URL,Iter), + [KeepAlive, FileName, URL, Protocol, ProtocolOpts, Iter]. +wget_client([KeepAlive, WgetFile, _URL, Protocol, ProtocolOpts, _], _) -> + process_flag(trap_exit, true), + Cmd = wget_N(KeepAlive, WgetFile, Protocol, ProtocolOpts), + %%ct:pal("Wget cmd: ~p", [Cmd]), + Port = open_port({spawn, Cmd}, [stderr_to_stdout]), + wait_for_wget(Port). + + +%%-------------------------------------------------------------------- +%% Start/stop servers ------------------------------------------------ +%%-------------------------------------------------------------------- +start_web_server(Group, Config) when Group == http_dummy; + Group == http_dummy_keep_alive -> + start_dummy("http", Config); + +start_web_server(Group, Config) when Group == https_dummy; + Group == https_dummy_keep_alive -> + start_dummy("https", Config); + +start_web_server(Group, Config) when Group == http_inets; + Group == http_inets_keep_alive -> + start_inets("http", [], Config); + +start_web_server(Group, Config) when Group == https_inets; + Group == https_inets_keep_alive -> + Opts = proplists:get_value(server_verification_opts, cert_opts(Config)), + ReuseSessions = ?config(reuse_sessions, Config), + SSLConfHttpd = [{socket_type, {essl, + [{nodelay, true}, {reuse_sessions, ReuseSessions} | Opts]}}], + start_inets("https", SSLConfHttpd, Config); + +start_web_server(Group, Config) when Group == http_nginx; + Group == http_nginx_keep_alive -> + case os:find_executable("nginx") of + false -> + {skip, "nginx not found"}; + _ -> + start_nginx("http", Config) + end; + +start_web_server(Group, Config) when Group == https_nginx; + Group == https_nginx_keep_alive -> + case os:find_executable("nginx") of + false -> + {skip, "nginx not found"}; + _ -> + start_nginx("https", cert_opts(Config) ++ Config) + end. + +start_inets(Protocol, ConfHttpd, Config) -> + PrivDir = ?config(priv_dir, Config), + DataDir = ?config(data_dir, Config), + Node = ?config(server_node, Config), + Host = ?config(server_host, Config), + HTTPVersion = ?config(http_version, Config), + Conf = [httpd, [{port,0}, + {http_version, HTTPVersion}, + {ipfamily, inet}, + {server_name, "inets_test"}, + {server_root, PrivDir}, + {document_root, DataDir}, + {keep_alive, ?config(keep_alive, Config)}, + {keep_alive_timeout, 360} + | ConfHttpd]], + {ok, Pid} = rpc:call(Node, inets, start, Conf), + Port = proplists:get_value(port, rpc:call(Node, httpd, info, [Pid])), + F = fun(File) -> + lists:concat([Protocol,"://",Host,":",Port,"/",File]) + end, + [{httpd_pid,Pid},{urlfun,F},{protocol,Protocol},{port,Port} | Config]. + +start_dummy("http"= Protocol, Config) -> + HTTPVersion = ?config(http_version, Config), + Node = ?config(server_node, Config), + %%DataDir= ?config(data_dir, Config), + Host = ?config(server_host, Config), + Conf = [ + %%{big, filename:join(DataDir, "1M_file")}, + %%{small, filename:join(DataDir, "1k_file")}, + {big, {gen, crypto:rand_bytes(1000000)}}, + {small, {gen, crypto:rand_bytes(1000)}}, + {http_version, HTTPVersion}, + {keep_alive, ?config(keep_alive, Config)} + ], + {Pid, Port} = rpc:call(Node, http_test_lib, dummy_server, [ip_comm, inet, [{content_cb, ?MODULE}, {conf, Conf}]]), + F = fun(File) -> + lists:concat([Protocol,"://",Host,":",Port,"/",File]) + end, + [{dummy_pid,Pid},{urlfun,F},{protocol, Protocol},{port,Port} | Config]; + +start_dummy("https" = Protocol, Config) -> + HTTPVersion = ?config(http_version, Config), + Node = ?config(server_node, Config), + %% DataDir= ?config(data_dir, Config), + Host = ?config(server_host, Config), + SSLOpts = proplists:get_value(server_verification_opts, cert_opts(Config)), + Opts = [{active, true}, {nodelay, true}, {reuseaddr, true} | SSLOpts], + Conf = [%%{big, filename:join(DataDir, "1M_file")}, + %%{small, filename:join(DataDir, "1k_file")}, + {big, {gen, crypto:rand_bytes(1000000)}}, + {small, {gen, crypto:rand_bytes(1000)}}, + {http_version, HTTPVersion}, + {keep_alive, ?config(keep_alive, Config)} + ], + {Pid, Port} = rpc:call(Node, http_test_lib, dummy_server, + [ssl, inet, [{ssl, Opts}, {content_cb, ?MODULE}, {conf, Conf}]]), + F = fun(File) -> + lists:concat([Protocol,"://",Host,":",Port,"/",File]) + end, + [{dummy_pid,Pid},{urlfun,F},{protocol,Protocol},{port,Port} | Config]. + +start_nginx(Protocol, Config) -> + PrivDir = ?config(priv_dir, Config), + DataDir = ?config(data_dir, Config), + Host = ?config(server_host, Config), + Port = inet_port(node()), + + ConfFile = filename:join(PrivDir, "nginx.conf"), + nginx_conf(ConfFile, [{port, Port}, {protocol, Protocol} | Config]), + Cmd = "nginx -c " ++ ConfFile, + NginxPort = open_port({spawn, Cmd}, [{cd, DataDir}, stderr_to_stdout]), + + F = fun(File) -> + lists:concat([Protocol,"://",Host,":",Port,"/",File]) + end, + + wait_for_nginx_up(Host, Port), + + [{port, Port},{nginx_port, NginxPort},{urlfun,F},{protocol, Protocol} | Config ]. + +stop_nginx(Config)-> + PrivDir = ?config(priv_dir, Config), + {ok, Bin} = file:read_file(filename:join(PrivDir, "nginx.pid")), + Pid = string:strip(binary_to_list(Bin), right, $\n), + Cmd = "kill " ++ Pid, + os:cmd(Cmd). + +stop_web_server(Group, Config) when Group == http_inets; + Group == http_inets_keep_alive; + Group == https_inets; + Group == https_inets_keep_alive -> + ServerNode = ?config(server_node, Config), + rpc:call(ServerNode, inets, stop, [httpd, ?config(httpd_pid, Config)]); +stop_web_server(Group, Config) when Group == http_dummy; + Group == http_dummy_keep_alive; + Group == https_dummy; + Group == https_dummy_keep_alive -> + stop_dummy_server(Config); +stop_web_server(Group, Config) when Group == http_nginx; + Group == http_nginx_keep_alive; + Group == https_nginx; + Group == https_nginx_keep_alive -> + stop_nginx(Config). + +stop_dummy_server(Config) -> + case ?config(dummy_pid, Config) of + Pid when is_pid(Pid) -> + exit(Pid, kill); + _ -> + ok + end. + +%%-------------------------------------------------------------------- +%% Misc ------------------------------------------------ +%%-------------------------------------------------------------------- +http_request(Request, "HTTP/1.1" = Version, Host, {Headers, Body}) -> + Request ++ Version ++ "\r\nhost:" ++ Host ++ "\r\n" ++ Headers ++ "\r\n" ++ Body; +http_request(Request, Version, _, {Headers, Body}) -> + Request ++ Version ++ "\r\n" ++ Headers ++ "\r\n" ++ Body. + +http_request(Request, "HTTP/1.1" = Version, Host) -> + Request ++ Version ++ "\r\nhost:" ++ Host ++ "\r\n\r\n"; +http_request(Request, Version, _) -> + Request ++ Version ++ "\r\n\r\n". + +http_version(_) -> + "HTTP/1.1". + +inet_port(Node) -> + {Port, Socket} = do_inet_port(Node), + rpc:call(Node, gen_tcp, close, [Socket]), + Port. + +do_inet_port(Node) -> + {ok, Socket} = rpc:call(Node, gen_tcp, listen, [0, [{reuseaddr, true}]]), + {ok, Port} = rpc:call(Node, inet, port, [Socket]), + {Port, Socket}. + +%%-------------------------------------------------------------------- +%% Dummy server callbacks ------------------------------------------------ +%%-------------------------------------------------------------------- + +handle_request(CB, S, "/1M_file" ++ _, Opts) -> + Name = proplists:get_value(big, Opts), + KeepAlive = proplists:get_value(keep_alive, Opts), + do_handle_request(CB, S, Name, Opts, KeepAlive); +handle_request(CB, S, "/1k_file" ++ _, Opts) -> + Name = proplists:get_value(small, Opts), + KeepAlive = proplists:get_value(keep_alive, Opts), + do_handle_request(CB, S, Name, Opts, KeepAlive). + +do_handle_request(CB, S, Name, Opts, KeepAlive) when is_list(Name) -> + Version = proplists:get_value(http_version, Opts), + {ok, Fdesc} = file:open(Name, [read, binary]), + {ok, Info} = file:read_file_info(Name, []), + Length = Info#file_info.size, + Response = response_status_line_and_headers(Version, "Content-Length:" + ++ integer_to_list(Length) ++ ?CRLF, keep_alive(KeepAlive)), + CB:send(S, Response), + send_file(CB, S, Fdesc); +do_handle_request(CB, S, {gen, Data}, Opts, KeepAlive) -> + Version = proplists:get_value(http_version, Opts), + Length = size(Data), + Response = response_status_line_and_headers(Version, "Content-Length:" + ++ integer_to_list(Length) ++ ?CRLF, keep_alive(KeepAlive)), + CB:send(S, Response), + send_file(CB, S, {gen, Data}). + +send_file(CB, S, {gen, Data}) -> + CB:send(S, Data); + %% ChunkSize = 64*1024, + %% case size(Data) of + %% N when N > ChunkSize -> + %% <<Chunk:N/binary, Rest/binary>> = Data, + %% %%{Chunk, Rest} = lists:split(N, Data), + %% CB:send(S, Chunk), + %% send_file(CB, S, {gen, Rest}); + %% _ -> + %% CB:send(S, Data) + %% end; + +send_file(CB, S, FileDesc) -> + case file:read(FileDesc, 64*1024) of + {ok, Chunk} -> + CB:send(S, Chunk), + send_file(CB, S, FileDesc); + eof -> + file:close(FileDesc), + ok + end. + +response_status_line_and_headers(Version, Headers, ConnectionHeader) -> + StatusLine = [Version, " ", "200 OK", ?CRLF], + [StatusLine, Headers, ConnectionHeader, ?CRLF]. + +keep_alive(true)-> + "Connection:keep-alive\r\n"; +keep_alive(false) -> + "Connection:close\r\n". + +handle_http_msg({_Method, RelUri, _, {_, _Headers}, _Body}, Socket, Conf) -> + handle_request(connect_cb(Socket), Socket, RelUri, Conf), + case proplists:get_value(keep_alive, Conf) of + true -> + <<>>; + false -> + stop + end. + +connect_cb({sslsocket, _, _}) -> + ssl; +connect_cb(_) -> + gen_tcp. + +%%-------------------------------------------------------------------- +%% Setup wget ------------------------------------------------ +%%-------------------------------------------------------------------- +wget_req_file(FileName, Url, Iter) -> + {ok, File} = file:open(FileName, [write]), + write_urls(File, Url, Iter). + +write_urls(File, Url, 1) -> + file:write(File, Url), + file:close(File); +write_urls(File, Url, N) -> + file:write(File, Url), + file:write(File, "\n"), + write_urls(File, Url, N-1). + +wait_for_wget(Port) -> + receive + {Port, {data, _Data}} when is_port(Port) -> + wait_for_wget(Port); + {Port, closed} -> + ok; + {'EXIT', Port, _Reason} -> + ok + end. + +wget_N(KeepAlive, WegetFile, "http", _ProtocolOpts) -> + "wget -i " ++ WegetFile ++ " " ++ wget_keep_alive(KeepAlive) ++ + " --no-cache --timeout=120" ; +wget_N(KeepAlive, WegetFile, "https", ProtocolOpts) -> + + "wget -i " ++ WegetFile ++ " " ++ wget_keep_alive(KeepAlive) + ++ wget_cert(ProtocolOpts) ++ wget_key(ProtocolOpts) + ++ wget_cacert(ProtocolOpts) ++ + " --no-cache --timeout=120". + +wget(KeepAlive, URL, "http", _ProtocolOpts) -> + "wget " ++ URL ++ " " ++ wget_keep_alive(KeepAlive) ++ + " --no-cache --timeout=120" ; +wget(KeepAlive, URL, "https", ProtocolOpts) -> + + "wget " ++ URL ++ " " ++ wget_keep_alive(KeepAlive) + ++ wget_cert(ProtocolOpts) ++ wget_key(ProtocolOpts) + ++ wget_cacert(ProtocolOpts) ++ + " --no-cache --timeout=120". + +wget_keep_alive(true)-> + ""; +wget_keep_alive(false) -> + "--no-http-keep-alive ". + +wget_cacert(ProtocolOpts) -> + "--ca-certificate=" ++ proplists:get_value(cacertfile, ProtocolOpts) ++ " ". + +wget_cert(ProtocolOpts) -> + "--certificate=" ++ proplists:get_value(certfile, ProtocolOpts) ++ " ". + +wget_key(ProtocolOpts) -> + "--private-key=" ++ proplists:get_value(keyfile, ProtocolOpts) ++ " ". + +%%-------------------------------------------------------------------- +%% Setup nginx ------------------------------------------------ +%%-------------------------------------------------------------------- +nginx_conf(ConfFile, Config)-> + Protocol = ?config(protocol, Config), + file:write_file(ConfFile, + [format_nginx_conf(nginx_global(Config)), + format_nginx_conf(nginx_events(Config)), + format_nginx_conf(nginx_http(Protocol, Config))]). + +format_nginx_conf(Directives) -> + lists:map(fun({Key, Value}) -> + io_lib:format("~s ~s;\n", [Key, Value]); + (Str) -> + Str + end, Directives). + + +nginx_global(Config) -> + PrivDir = ?config(priv_dir, Config), + [{"pid", filename:join(PrivDir, "nginx.pid")}, + {"error_log", filename:join(PrivDir, "nginx.pid")}, + {"worker_processes", "1"}]. + +nginx_events(_Config) -> + ["events {\n", + {"worker_connections", "1024"}, + "\n}" + ]. + +nginx_http("http", Config) -> + PrivDir = ?config(priv_dir, Config), + DataDir = ?config(data_dir, Config), + Port = ?config(port, Config), + ["http {\n" | + nginx_defaults(PrivDir) ++ + [" server {", + {root, DataDir}, + {listen, integer_to_list(Port)}, + " location / {\n try_files $uri $uri/ /index.html;\n}" + "}\n", "}\n" + ] + ]; + +nginx_http("https", Config) -> + PrivDir = ?config(priv_dir, Config), + DataDir = ?config(data_dir, Config), + Port = ?config(port, Config), + SSLOpts = ?config(server_verification_opts, Config), + Ciphers = proplists:get_value(ciphers, SSLOpts), + ReuseSession = ?config(reuse_sessions, Config), + ["http {" | + nginx_defaults(PrivDir) ++ + [" server {", + {"root", DataDir}, + {"listen", integer_to_list(Port) ++ " ssl"}, + {"ssl_certificate", ?config(certfile, SSLOpts)}, + {"ssl_certificate_key", ?config(keyfile, SSLOpts)}, + {"ssl_protocols", "TLSv1 TLSv1.1 TLSv1.2"}, + {"ssl_ciphers", Ciphers}, + {"ssl_session_cache", nginx_reuse_session(ReuseSession)}, + " location / {\n try_files $uri $uri/ /index.html;\n}" + "}\n", "}\n" + ] + ]. + +nginx_defaults(PrivDir) -> + [ + %% Set temp and cache file options that will otherwise default to + %% restricted locations accessible only to root. + {"client_body_temp_path", filename:join(PrivDir, "client_body")}, + {"fastcgi_temp_path", filename:join(PrivDir, "fastcgi_temp")}, + {"proxy_temp_path", filename:join(PrivDir, "proxy_temp")}, + {"scgi_temp_path", filename:join(PrivDir, "scgi_temp")}, + {"uwsgi_temp_path", filename:join(PrivDir, "uwsgi_temp_path")}, + {"access_log", filename:join(PrivDir, "access.log")}, + {"error_log", filename:join(PrivDir, "error.log")}, + %% Standard options + {"sendfile", "on"}, + {"tcp_nopush", "on"}, + {"tcp_nodelay", "on"}, + {"keepalive_timeout", "360"}, + {"types_hash_max_size", "2048"}, + {"include", "/etc/nginx/mime.types"}, + {"default_type", "application/octet-stream"} + ]. + +nginx_reuse_session(true) -> + "on"; +nginx_reuse_session(false) -> + "off". + +wait_for_nginx_up(Host, Port) -> + case gen_tcp:connect(Host, Port, []) of + {ok, Socket} -> + gen_tcp:close(Socket); + _ -> + ct:sleep(100), + wait_for_nginx_up(Host, Port) + end. + diff --git a/lib/inets/test/httpd_bench_SUITE_data/1M_file b/lib/inets/test/httpd_bench_SUITE_data/1M_file Binary files differnew file mode 100644 index 0000000000..557989144e --- /dev/null +++ b/lib/inets/test/httpd_bench_SUITE_data/1M_file diff --git a/lib/inets/test/httpd_bench_SUITE_data/1k_file b/lib/inets/test/httpd_bench_SUITE_data/1k_file Binary files differnew file mode 100644 index 0000000000..cade172d80 --- /dev/null +++ b/lib/inets/test/httpd_bench_SUITE_data/1k_file diff --git a/lib/inets/test/httpd_test_lib.erl b/lib/inets/test/httpd_test_lib.erl index 1cecd2642c..b6525037b2 100644 --- a/lib/inets/test/httpd_test_lib.erl +++ b/lib/inets/test/httpd_test_lib.erl @@ -23,7 +23,8 @@ -include("inets_test_lib.hrl"). %% Poll functions --export([verify_request/6, verify_request/7, verify_request/8, is_expect/1]). +-export([verify_request/6, verify_request/7, verify_request/8, is_expect/1, + verify_request_N/9]). -record(state, {request, % string() socket, % socket() @@ -109,9 +110,9 @@ verify_request(SocketType, Host, Port, TranspOpts, Node, RequestStr, Options, Ti {error, Reason}; NewState -> ValidateResult = - validate(RequestStr, NewState, Options, Node, Port), + validate(RequestStr, NewState, Options, Node, Port), inets_test_lib:close(SocketType, Socket), - ValidateResult + ValidateResult end; ConnectError -> @@ -126,6 +127,46 @@ verify_request(SocketType, Host, Port, TranspOpts, Node, RequestStr, Options, Ti {args, [SocketType, Host, Port, TranspOpts]}]}) end. +verify_request_N(SocketType, Host, Port, TranspOpts, Node, RequestStr, Options, TimeOut, N) -> + State = #state{}, + try inets_test_lib:connect_bin(SocketType, Host, Port, TranspOpts) of + {ok, Socket} -> + request_N(SocketType, Socket, RequestStr, Options, TimeOut, Node, Port, State, N); + ConnectError -> + ct:fail({connect_error, ConnectError, + [SocketType, Host, Port, TranspOpts]}) + catch + T:E -> + ct:fail({connect_failure, + [{type, T}, + {error, E}, + {stacktrace, erlang:get_stacktrace()}, + {args, [SocketType, Host, Port, TranspOpts]}]}) + end. + +request_N(SocketType, Socket, RequestStr, Options, TimeOut, Node, Port, State, 0) -> + ok = inets_test_lib:send(SocketType, Socket, RequestStr), + case request(State#state{request = RequestStr, + socket = Socket}, TimeOut) of + {error, Reason} -> + {error, Reason}; + NewState -> + ValidateResult = + validate(RequestStr, NewState, Options, Node, Port), + inets_test_lib:close(SocketType, Socket), + ValidateResult + end; +request_N(SocketType, Socket, RequestStr, Options, TimeOut, Node, Port, State, N) -> + ok = inets_test_lib:send(SocketType, Socket, RequestStr), + case request(State#state{request = RequestStr, + socket = Socket}, TimeOut) of + {error, Reason} -> + {error, Reason}; + _NewState -> + request_N(SocketType, Socket, RequestStr, Options, TimeOut, Node, Port, + #state{}, N-1) + end. + request(#state{mfa = {Module, Function, Args}, request = RequestStr, socket = Socket} = State, TimeOut) -> @@ -160,13 +201,35 @@ request(#state{mfa = {Module, Function, Args}, {ssl_closed, Socket} -> exit({test_failed, connection_closed}); {ssl_error, Socket, Reason} -> - ct:fail({ssl_error, Reason}) + ct:fail({ssl_error, Reason}); + {Socket, {data, Data}} when is_port(Socket) -> + case Module:Function([list_to_binary(Data) | Args]) of + {ok, Parsed} -> + port_handle_http_msg(Parsed, State); + {_, whole_body, _} when HeadRequest =:= "HEAD" -> + State#state{body = <<>>}; + NewMFA -> + request(State#state{mfa = NewMFA}, TimeOut) + end; + {Socket, closed} when Function =:= whole_body -> + State#state{body = hd(Args)}; + {Socket, closed} -> + exit({test_failed, connection_closed}) after TimeOut -> ct:pal("~p ~w[~w]request -> timeout" - "~n", [self(), ?MODULE, ?LINE]), + "~p~n", [self(), ?MODULE, ?LINE, Args]), ct:fail(connection_timed_out) end. + +port_handle_http_msg({Version, StatusCode, ReasonPharse, Headers, Body}, State) -> + State#state{status_line = {Version, + StatusCode, + ReasonPharse}, + headers = Headers, + body = Body}. + + handle_http_msg({Version, StatusCode, ReasonPharse, Headers, Body}, State = #state{request = RequestStr}) -> case is_expect(RequestStr) of diff --git a/lib/inets/test/inets.spec b/lib/inets/test/inets.spec index ed102f8219..6cb3d6526c 100644 --- a/lib/inets/test/inets.spec +++ b/lib/inets/test/inets.spec @@ -1 +1,3 @@ -{suites,"../inets_test",all}. +{suites,"../inets_test", all}. +{skip_suites, "../inets_test", [httpd_bench_SUITE], + "Benchmarks run separately"}. diff --git a/lib/inets/test/inets_bench.spec b/lib/inets/test/inets_bench.spec new file mode 100644 index 0000000000..19136e691b --- /dev/null +++ b/lib/inets/test/inets_bench.spec @@ -0,0 +1 @@ +{suites,"../inets_test",[httpd_bench_SUITE]}. diff --git a/lib/inets/test/inets_test_lib.erl b/lib/inets/test/inets_test_lib.erl index f1185f7574..2529cc5f9b 100644 --- a/lib/inets/test/inets_test_lib.erl +++ b/lib/inets/test/inets_test_lib.erl @@ -463,8 +463,9 @@ connect_bin(essl, Host, Port, Opts0) -> connect(ssl, Host, Port, Opts); connect_bin(ip_comm, Host, Port, Opts0) -> Opts = [binary, {packet, 0} | Opts0], - connect(ip_comm, Host, Port, Opts). - + connect(ip_comm, Host, Port, Opts); +connect_bin(Type, Host, Port, Opts) -> + connect(Type, Host, Port, Opts). connect_byte(SockType, Host, Port) -> connect_byte(SockType, Host, Port, []). @@ -477,27 +478,40 @@ connect_byte(essl, Host, Port, Opts0) -> connect(ssl, Host, Port, Opts); connect_byte(ip_comm, Host, Port, Opts0) -> Opts = [{packet,0} | Opts0], - connect(ip_comm, Host, Port, Opts). + connect(ip_comm, Host, Port, Opts); +connect_byte(Type, Host, Port, Opts) -> + connect(Type, Host, Port, Opts). connect(ip_comm, Host, Port, Opts) -> gen_tcp:connect(Host, Port, Opts); connect(ssl, Host, Port, Opts) -> - ssl:connect(Host, Port, Opts). + ssl:connect(Host, Port, Opts); +connect(openssl_port, Host, Port, Opts) -> + CaCertFile = proplists:get_value(cacertfile, Opts), + Cmd = "openssl s_client -quiet -port " ++ integer_to_list(Port) ++ " -host " ++ Host + ++ " -CAfile " ++ CaCertFile, + ct:log("openssl cmd: ~p~n", [Cmd]), + OpensslPort = open_port({spawn, Cmd}, [stderr_to_stdout]), + read_junk(OpensslPort), + {ok, OpensslPort}. send(ssl, Socket, Data) -> ssl:send(Socket, Data); send(essl, Socket, Data) -> ssl:send(Socket, Data); send(ip_comm,Socket,Data) -> - gen_tcp:send(Socket,Data). - - + gen_tcp:send(Socket,Data); +send(openssl_port, Port, Data) -> + true = port_command(Port, Data), + ok. close(ssl,Socket) -> catch ssl:close(Socket); close(essl,Socket) -> catch ssl:close(Socket); close(ip_comm,Socket) -> - catch gen_tcp:close(Socket). + catch gen_tcp:close(Socket); +close(openssl_port, Port) -> + exit(Port, normal). hours(N) -> trunc(N * 1000 * 60 * 60). @@ -572,3 +586,11 @@ do_inet_port(Node) -> {ok, Socket} = rpc:call(Node, gen_tcp, listen, [0, [{reuseaddr, true}]]), {ok, Port} = rpc:call(Node, inet, port, [Socket]), {Port, Socket}. + +read_junk(OpensslPort) -> + receive + {OpensslPort, _} -> + read_junk(OpensslPort) + after 500 -> + ok + end. diff --git a/lib/inets/test/make_certs.erl b/lib/inets/test/make_certs.erl new file mode 100644 index 0000000000..7215a59823 --- /dev/null +++ b/lib/inets/test/make_certs.erl @@ -0,0 +1,530 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2007-2015. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(make_certs). +-compile([export_all]). + +%-export([all/1, all/2, rootCA/2, intermediateCA/3, endusers/3, enduser/3, revoke/3, gencrl/2, verify/3]). + +-record(config, {commonName, + organizationalUnitName = "Erlang OTP", + organizationName = "Ericsson AB", + localityName = "Stockholm", + countryName = "SE", + emailAddress = "[email protected]", + default_bits = 2048, + v2_crls = true, + ecc_certs = false, + issuing_distribution_point = false, + crl_port = 8000, + openssl_cmd = "openssl"}). + + +default_config() -> + #config{}. + +make_config(Args) -> + make_config(Args, #config{}). + +make_config([], C) -> + C; +make_config([{organizationalUnitName, Name}|T], C) when is_list(Name) -> + make_config(T, C#config{organizationalUnitName = Name}); +make_config([{organizationName, Name}|T], C) when is_list(Name) -> + make_config(T, C#config{organizationName = Name}); +make_config([{localityName, Name}|T], C) when is_list(Name) -> + make_config(T, C#config{localityName = Name}); +make_config([{countryName, Name}|T], C) when is_list(Name) -> + make_config(T, C#config{countryName = Name}); +make_config([{emailAddress, Name}|T], C) when is_list(Name) -> + make_config(T, C#config{emailAddress = Name}); +make_config([{default_bits, Bits}|T], C) when is_integer(Bits) -> + make_config(T, C#config{default_bits = Bits}); +make_config([{v2_crls, Bool}|T], C) when is_boolean(Bool) -> + make_config(T, C#config{v2_crls = Bool}); +make_config([{crl_port, Port}|T], C) when is_integer(Port) -> + make_config(T, C#config{crl_port = Port}); +make_config([{ecc_certs, Bool}|T], C) when is_boolean(Bool) -> + make_config(T, C#config{ecc_certs = Bool}); +make_config([{issuing_distribution_point, Bool}|T], C) when is_boolean(Bool) -> + make_config(T, C#config{issuing_distribution_point = Bool}); +make_config([{openssl_cmd, Cmd}|T], C) when is_list(Cmd) -> + make_config(T, C#config{openssl_cmd = Cmd}). + + +all([DataDir, PrivDir]) -> + all(DataDir, PrivDir). + +all(DataDir, PrivDir) -> + all(DataDir, PrivDir, #config{}). + +all(DataDir, PrivDir, C) when is_list(C) -> + all(DataDir, PrivDir, make_config(C)); +all(DataDir, PrivDir, C = #config{}) -> + ok = filelib:ensure_dir(filename:join(PrivDir, "erlangCA")), + create_rnd(DataDir, PrivDir), % For all requests + rootCA(PrivDir, "erlangCA", C), + intermediateCA(PrivDir, "otpCA", "erlangCA", C), + endusers(PrivDir, "otpCA", ["client", "server", "revoked", "a.server", "b.server"], C), + endusers(PrivDir, "erlangCA", ["localhost"], C), + %% Create keycert files + SDir = filename:join([PrivDir, "server"]), + SC = filename:join([SDir, "cert.pem"]), + SK = filename:join([SDir, "key.pem"]), + SKC = filename:join([SDir, "keycert.pem"]), + append_files([SK, SC], SKC), + CDir = filename:join([PrivDir, "client"]), + CC = filename:join([CDir, "cert.pem"]), + CK = filename:join([CDir, "key.pem"]), + CKC = filename:join([CDir, "keycert.pem"]), + append_files([CK, CC], CKC), + RDir = filename:join([PrivDir, "revoked"]), + RC = filename:join([RDir, "cert.pem"]), + RK = filename:join([RDir, "key.pem"]), + RKC = filename:join([RDir, "keycert.pem"]), + revoke(PrivDir, "otpCA", "revoked", C), + append_files([RK, RC], RKC), + remove_rnd(PrivDir), + {ok, C}. + +append_files(FileNames, ResultFileName) -> + {ok, ResultFile} = file:open(ResultFileName, [write]), + do_append_files(FileNames, ResultFile). + +do_append_files([], RF) -> + ok = file:close(RF); +do_append_files([F|Fs], RF) -> + {ok, Data} = file:read_file(F), + ok = file:write(RF, Data), + do_append_files(Fs, RF). + +rootCA(Root, Name, C) -> + create_ca_dir(Root, Name, ca_cnf(C#config{commonName = Name})), + create_self_signed_cert(Root, Name, req_cnf(C#config{commonName = Name}), C), + file:copy(filename:join([Root, Name, "cert.pem"]), filename:join([Root, Name, "cacerts.pem"])), + gencrl(Root, Name, C). + +intermediateCA(Root, CA, ParentCA, C) -> + create_ca_dir(Root, CA, ca_cnf(C#config{commonName = CA})), + CARoot = filename:join([Root, CA]), + CnfFile = filename:join([CARoot, "req.cnf"]), + file:write_file(CnfFile, req_cnf(C#config{commonName = CA})), + KeyFile = filename:join([CARoot, "private", "key.pem"]), + ReqFile = filename:join([CARoot, "req.pem"]), + create_req(Root, CnfFile, KeyFile, ReqFile, C), + CertFile = filename:join([CARoot, "cert.pem"]), + sign_req(Root, ParentCA, "ca_cert", ReqFile, CertFile, C), + CACertsFile = filename:join(CARoot, "cacerts.pem"), + file:copy(filename:join([Root, ParentCA, "cacerts.pem"]), CACertsFile), + %% append this CA's cert to the cacerts file + {ok, Bin} = file:read_file(CertFile), + {ok, FD} = file:open(CACertsFile, [append]), + file:write(FD, ["\n", Bin]), + file:close(FD), + gencrl(Root, CA, C). + +endusers(Root, CA, Users, C) -> + [enduser(Root, CA, User, C) || User <- Users]. + +enduser(Root, CA, User, C) -> + UsrRoot = filename:join([Root, User]), + file:make_dir(UsrRoot), + CnfFile = filename:join([UsrRoot, "req.cnf"]), + file:write_file(CnfFile, req_cnf(C#config{commonName = User})), + KeyFile = filename:join([UsrRoot, "key.pem"]), + ReqFile = filename:join([UsrRoot, "req.pem"]), + create_req(Root, CnfFile, KeyFile, ReqFile, C), + %create_req(Root, CnfFile, KeyFile, ReqFile), + CertFileAllUsage = filename:join([UsrRoot, "cert.pem"]), + sign_req(Root, CA, "user_cert", ReqFile, CertFileAllUsage, C), + CertFileDigitalSigOnly = filename:join([UsrRoot, "digital_signature_only_cert.pem"]), + sign_req(Root, CA, "user_cert_digital_signature_only", ReqFile, CertFileDigitalSigOnly, C), + CACertsFile = filename:join(UsrRoot, "cacerts.pem"), + file:copy(filename:join([Root, CA, "cacerts.pem"]), CACertsFile), + ok. + +revoke(Root, CA, User, C) -> + UsrCert = filename:join([Root, User, "cert.pem"]), + CACnfFile = filename:join([Root, CA, "ca.cnf"]), + Cmd = [C#config.openssl_cmd, " ca" + " -revoke ", UsrCert, + [" -crl_reason keyCompromise" || C#config.v2_crls ], + " -config ", CACnfFile], + Env = [{"ROOTDIR", filename:absname(Root)}], + cmd(Cmd, Env), + gencrl(Root, CA, C). + +gencrl(Root, CA, C) -> + CACnfFile = filename:join([Root, CA, "ca.cnf"]), + CACRLFile = filename:join([Root, CA, "crl.pem"]), + Cmd = [C#config.openssl_cmd, " ca" + " -gencrl ", + " -crlhours 24", + " -out ", CACRLFile, + " -config ", CACnfFile], + Env = [{"ROOTDIR", filename:absname(Root)}], + cmd(Cmd, Env). + +verify(Root, CA, User, C) -> + CAFile = filename:join([Root, User, "cacerts.pem"]), + CACRLFile = filename:join([Root, CA, "crl.pem"]), + CertFile = filename:join([Root, User, "cert.pem"]), + Cmd = [C#config.openssl_cmd, " verify" + " -CAfile ", CAFile, + " -CRLfile ", CACRLFile, %% this is undocumented, but seems to work + " -crl_check ", + CertFile], + Env = [{"ROOTDIR", filename:absname(Root)}], + try cmd(Cmd, Env) catch + exit:{eval_cmd, _, _} -> + invalid + end. + +create_self_signed_cert(Root, CAName, Cnf, C = #config{ecc_certs = true}) -> + CARoot = filename:join([Root, CAName]), + CnfFile = filename:join([CARoot, "req.cnf"]), + file:write_file(CnfFile, Cnf), + KeyFile = filename:join([CARoot, "private", "key.pem"]), + CertFile = filename:join([CARoot, "cert.pem"]), + Cmd = [C#config.openssl_cmd, " ecparam" + " -out ", KeyFile, + " -name secp521r1 ", + %" -name sect283k1 ", + " -genkey "], + Env = [{"ROOTDIR", filename:absname(Root)}], + cmd(Cmd, Env), + + Cmd2 = [C#config.openssl_cmd, " req" + " -new" + " -x509" + " -config ", CnfFile, + " -key ", KeyFile, + " -outform PEM ", + " -out ", CertFile], + cmd(Cmd2, Env); +create_self_signed_cert(Root, CAName, Cnf, C) -> + CARoot = filename:join([Root, CAName]), + CnfFile = filename:join([CARoot, "req.cnf"]), + file:write_file(CnfFile, Cnf), + KeyFile = filename:join([CARoot, "private", "key.pem"]), + CertFile = filename:join([CARoot, "cert.pem"]), + Cmd = [C#config.openssl_cmd, " req" + " -new" + " -x509" + " -config ", CnfFile, + " -keyout ", KeyFile, + " -outform PEM", + " -out ", CertFile], + Env = [{"ROOTDIR", filename:absname(Root)}], + cmd(Cmd, Env). + + +create_ca_dir(Root, CAName, Cnf) -> + CARoot = filename:join([Root, CAName]), + ok = filelib:ensure_dir(CARoot), + file:make_dir(CARoot), + create_dirs(CARoot, ["certs", "crl", "newcerts", "private"]), + create_rnd(Root, filename:join([CAName, "private"])), + create_files(CARoot, [{"serial", "01\n"}, + {"crlnumber", "01"}, + {"index.txt", ""}, + {"ca.cnf", Cnf}]). + +create_req(Root, CnfFile, KeyFile, ReqFile, C = #config{ecc_certs = true}) -> + Cmd = [C#config.openssl_cmd, " ecparam" + " -out ", KeyFile, + " -name secp521r1 ", + %" -name sect283k1 ", + " -genkey "], + Env = [{"ROOTDIR", filename:absname(Root)}], + cmd(Cmd, Env), + Cmd2 = [C#config.openssl_cmd, " req" + " -new ", + " -key ", KeyFile, + " -outform PEM ", + " -out ", ReqFile, + " -config ", CnfFile], + cmd(Cmd2, Env); + %fix_key_file(KeyFile). +create_req(Root, CnfFile, KeyFile, ReqFile, C) -> + Cmd = [C#config.openssl_cmd, " req" + " -new" + " -config ", CnfFile, + " -outform PEM ", + " -keyout ", KeyFile, + " -out ", ReqFile], + Env = [{"ROOTDIR", filename:absname(Root)}], + cmd(Cmd, Env). + %fix_key_file(KeyFile). + + +sign_req(Root, CA, CertType, ReqFile, CertFile, C) -> + CACnfFile = filename:join([Root, CA, "ca.cnf"]), + Cmd = [C#config.openssl_cmd, " ca" + " -batch" + " -notext" + " -config ", CACnfFile, + " -extensions ", CertType, + " -in ", ReqFile, + " -out ", CertFile], + Env = [{"ROOTDIR", filename:absname(Root)}], + cmd(Cmd, Env). + +%% +%% Misc +%% + +create_dirs(Root, Dirs) -> + lists:foreach(fun(Dir) -> + file:make_dir(filename:join([Root, Dir])) end, + Dirs). + +create_files(Root, NameContents) -> + lists:foreach( + fun({Name, Contents}) -> + file:write_file(filename:join([Root, Name]), Contents) end, + NameContents). + +create_rnd(FromDir, ToDir) -> + From = filename:join([FromDir, "RAND"]), + To = filename:join([ToDir, "RAND"]), + file:copy(From, To). + +remove_rnd(Dir) -> + File = filename:join([Dir, "RAND"]), + file:delete(File). + +cmd(Cmd, Env) -> + FCmd = lists:flatten(Cmd), + Port = open_port({spawn, FCmd}, [stream, eof, exit_status, stderr_to_stdout, + {env, Env}]), + eval_cmd(Port, FCmd). + +eval_cmd(Port, Cmd) -> + receive + {Port, {data, _}} -> + eval_cmd(Port, Cmd); + {Port, eof} -> + ok + end, + receive + {Port, {exit_status, 0}} -> + ok; + {Port, {exit_status, Status}} -> + exit({eval_cmd, Cmd, Status}) + after 0 -> + ok + end. + +%% +%% Contents of configuration files +%% + +req_cnf(C) -> + ["# Purpose: Configuration for requests (end users and CAs)." + "\n" + "ROOTDIR = $ENV::ROOTDIR\n" + "\n" + + "[req]\n" + "input_password = secret\n" + "output_password = secret\n" + "default_bits = ", integer_to_list(C#config.default_bits), "\n" + "RANDFILE = $ROOTDIR/RAND\n" + "encrypt_key = no\n" + "default_md = md5\n" + "#string_mask = pkix\n" + "x509_extensions = ca_ext\n" + "prompt = no\n" + "distinguished_name= name\n" + "\n" + + "[name]\n" + "commonName = ", C#config.commonName, "\n" + "organizationalUnitName = ", C#config.organizationalUnitName, "\n" + "organizationName = ", C#config.organizationName, "\n" + "localityName = ", C#config.localityName, "\n" + "countryName = ", C#config.countryName, "\n" + "emailAddress = ", C#config.emailAddress, "\n" + "\n" + + "[ca_ext]\n" + "basicConstraints = critical, CA:true\n" + "keyUsage = cRLSign, keyCertSign\n" + "subjectKeyIdentifier = hash\n" + "subjectAltName = email:copy\n"]. + +ca_cnf(C = #config{issuing_distribution_point = true}) -> + ["# Purpose: Configuration for CAs.\n" + "\n" + "ROOTDIR = $ENV::ROOTDIR\n" + "default_ca = ca\n" + "\n" + + "[ca]\n" + "dir = $ROOTDIR/", C#config.commonName, "\n" + "certs = $dir/certs\n" + "crl_dir = $dir/crl\n" + "database = $dir/index.txt\n" + "new_certs_dir = $dir/newcerts\n" + "certificate = $dir/cert.pem\n" + "serial = $dir/serial\n" + "crl = $dir/crl.pem\n", + ["crlnumber = $dir/crlnumber\n" || C#config.v2_crls], + "private_key = $dir/private/key.pem\n" + "RANDFILE = $dir/private/RAND\n" + "\n" + "x509_extensions = user_cert\n", + ["crl_extensions = crl_ext\n" || C#config.v2_crls], + "unique_subject = no\n" + "default_days = 3600\n" + "default_md = md5\n" + "preserve = no\n" + "policy = policy_match\n" + "\n" + + "[policy_match]\n" + "commonName = supplied\n" + "organizationalUnitName = optional\n" + "organizationName = match\n" + "countryName = match\n" + "localityName = match\n" + "emailAddress = supplied\n" + "\n" + + "[crl_ext]\n" + "authorityKeyIdentifier=keyid:always,issuer:always\n", + ["issuingDistributionPoint=critical, @idpsec\n" || C#config.issuing_distribution_point], + + "[idpsec]\n" + "fullname=URI:http://localhost:8000/",C#config.commonName,"/crl.pem\n" + + "[user_cert]\n" + "basicConstraints = CA:false\n" + "keyUsage = nonRepudiation, digitalSignature, keyEncipherment\n" + "subjectKeyIdentifier = hash\n" + "authorityKeyIdentifier = keyid,issuer:always\n" + "subjectAltName = email:copy\n" + "issuerAltName = issuer:copy\n" + "crlDistributionPoints=@crl_section\n" + + "[crl_section]\n" + %% intentionally invalid + "URI.1=http://localhost/",C#config.commonName,"/crl.pem\n" + "URI.2=http://localhost:",integer_to_list(C#config.crl_port),"/",C#config.commonName,"/crl.pem\n" + "\n" + + "[user_cert_digital_signature_only]\n" + "basicConstraints = CA:false\n" + "keyUsage = digitalSignature\n" + "subjectKeyIdentifier = hash\n" + "authorityKeyIdentifier = keyid,issuer:always\n" + "subjectAltName = email:copy\n" + "issuerAltName = issuer:copy\n" + "\n" + + "[ca_cert]\n" + "basicConstraints = critical,CA:true\n" + "keyUsage = cRLSign, keyCertSign\n" + "subjectKeyIdentifier = hash\n" + "authorityKeyIdentifier = keyid:always,issuer:always\n" + "subjectAltName = email:copy\n" + "issuerAltName = issuer:copy\n" + "crlDistributionPoints=@crl_section\n" + ]; + +ca_cnf(C = #config{issuing_distribution_point = false}) -> + ["# Purpose: Configuration for CAs.\n" + "\n" + "ROOTDIR = $ENV::ROOTDIR\n" + "default_ca = ca\n" + "\n" + + "[ca]\n" + "dir = $ROOTDIR/", C#config.commonName, "\n" + "certs = $dir/certs\n" + "crl_dir = $dir/crl\n" + "database = $dir/index.txt\n" + "new_certs_dir = $dir/newcerts\n" + "certificate = $dir/cert.pem\n" + "serial = $dir/serial\n" + "crl = $dir/crl.pem\n", + ["crlnumber = $dir/crlnumber\n" || C#config.v2_crls], + "private_key = $dir/private/key.pem\n" + "RANDFILE = $dir/private/RAND\n" + "\n" + "x509_extensions = user_cert\n", + ["crl_extensions = crl_ext\n" || C#config.v2_crls], + "unique_subject = no\n" + "default_days = 3600\n" + "default_md = md5\n" + "preserve = no\n" + "policy = policy_match\n" + "\n" + + "[policy_match]\n" + "commonName = supplied\n" + "organizationalUnitName = optional\n" + "organizationName = match\n" + "countryName = match\n" + "localityName = match\n" + "emailAddress = supplied\n" + "\n" + + "[crl_ext]\n" + "authorityKeyIdentifier=keyid:always,issuer:always\n", + %["issuingDistributionPoint=critical, @idpsec\n" || C#config.issuing_distribution_point], + + %"[idpsec]\n" + %"fullname=URI:http://localhost:8000/",C#config.commonName,"/crl.pem\n" + + "[user_cert]\n" + "basicConstraints = CA:false\n" + "keyUsage = nonRepudiation, digitalSignature, keyEncipherment\n" + "subjectKeyIdentifier = hash\n" + "authorityKeyIdentifier = keyid,issuer:always\n" + "subjectAltName = email:copy\n" + "issuerAltName = issuer:copy\n" + %"crlDistributionPoints=@crl_section\n" + + %%"[crl_section]\n" + %% intentionally invalid + %%"URI.1=http://localhost/",C#config.commonName,"/crl.pem\n" + %%"URI.2=http://localhost:",integer_to_list(C#config.crl_port),"/",C#config.commonName,"/crl.pem\n" + %%"\n" + + "[user_cert_digital_signature_only]\n" + "basicConstraints = CA:false\n" + "keyUsage = digitalSignature\n" + "subjectKeyIdentifier = hash\n" + "authorityKeyIdentifier = keyid,issuer:always\n" + "subjectAltName = email:copy\n" + "issuerAltName = issuer:copy\n" + "\n" + + "[ca_cert]\n" + "basicConstraints = critical,CA:true\n" + "keyUsage = cRLSign, keyCertSign\n" + "subjectKeyIdentifier = hash\n" + "authorityKeyIdentifier = keyid:always,issuer:always\n" + "subjectAltName = email:copy\n" + "issuerAltName = issuer:copy\n" + %"crlDistributionPoints=@crl_section\n" + ]. diff --git a/lib/inets/vsn.mk b/lib/inets/vsn.mk index 560d524bac..05cf4f6cc3 100644 --- a/lib/inets/vsn.mk +++ b/lib/inets/vsn.mk @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = inets -INETS_VSN = 6.4.4 +INETS_VSN = 6.4.5 PRE_VSN = APP_VSN = "$(APPLICATION)-$(INETS_VSN)$(PRE_VSN)" diff --git a/lib/jinterface/doc/src/notes.xml b/lib/jinterface/doc/src/notes.xml index b44a04d7cd..346d467c2d 100644 --- a/lib/jinterface/doc/src/notes.xml +++ b/lib/jinterface/doc/src/notes.xml @@ -31,6 +31,21 @@ </header> <p>This document describes the changes made to the Jinterface application.</p> +<section><title>Jinterface 1.8.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + <section><title>Jinterface 1.8</title> <section><title>Improvements and New Features</title> diff --git a/lib/jinterface/vsn.mk b/lib/jinterface/vsn.mk index 373e2dab22..0a8a1190ec 100644 --- a/lib/jinterface/vsn.mk +++ b/lib/jinterface/vsn.mk @@ -1 +1 @@ -JINTERFACE_VSN = 1.8 +JINTERFACE_VSN = 1.8.1 diff --git a/lib/kernel/doc/src/file.xml b/lib/kernel/doc/src/file.xml index 2ab35b9b05..58abb35428 100644 --- a/lib/kernel/doc/src/file.xml +++ b/lib/kernel/doc/src/file.xml @@ -33,11 +33,14 @@ <description> <p>This module provides an interface to the file system.</p> - <p>On operating systems with thread support, - file operations can be performed in threads of their own, allowing - other Erlang processes to continue executing in parallel with - the file operations. See command-line flag - <c>+A</c> in <seealso marker="erts:erl"><c>erl(1)</c></seealso>.</p> + <warning> + <p>File operations are only guaranteed to appear atomic when going + through the same file server. A NIF or other OS process may observe + intermediate steps on certain operations on some operating systems, + eg. renaming an existing file on Windows, or + <seealso marker="#write_file_info/2"><c>write_file_info/2</c> + </seealso> on any OS at the time of writing.</p> + </warning> <p>Regarding filename encoding, the Erlang VM can operate in two modes. The current mode can be queried using function @@ -1438,8 +1441,12 @@ f.txt: {person, "kalle", 25}. which is 1970-01-01 00:00 UTC.</p></item> </taglist> <p>Default is <c>{time, local}</c>.</p> - <p>If the option <c>raw</c> is set, the file server is not called - and only information about local files is returned.</p> + <p>If the option <c>raw</c> is set, the file server is not called and + only information about local files is returned. Note that this will + break this module's atomicity guarantees as it can race with a + concurrent call to + <seealso marker="#write_file_info/2"><c>write_file_info/1,2</c> + </seealso></p> <note> <p>As file times are stored in POSIX time on most OS, it is faster to query file information with option <c>posix</c>.</p> @@ -1687,8 +1694,12 @@ f.txt: {person, "kalle", 25}. except that if <c><anno>Name</anno></c> is a symbolic link, information about the link is returned in the <c>file_info</c> record and the <c>type</c> field of the record is set to <c>symlink</c>.</p> - <p>If the option <c>raw</c> is set, the file server is not called - and only information about local files is returned.</p> + <p>If the option <c>raw</c> is set, the file server is not called and + only information about local files is returned. Note that this will + break this module's atomicity guarantees as it can race with a + concurrent call to + <seealso marker="#write_file_info/2"><c>write_file_info/1,2</c> + </seealso></p> <p>If <c><anno>Name</anno></c> is not a symbolic link, this function returns the same result as <c>read_file_info/1</c>. On platforms that do not support symbolic links, this function @@ -1826,24 +1837,16 @@ f.txt: {person, "kalle", 25}. <p>The file used must be opened using the <c>raw</c> flag, and the process calling <c>sendfile</c> must be the controlling process of the socket. See <seealso marker="gen_tcp#controlling_process-2"><c>gen_tcp:controlling_process/2</c></seealso>.</p> - <p>If the OS used does not support <c>sendfile</c>, an Erlang fallback - using - <seealso marker="#read/2"><c>read/2</c></seealso> and - <seealso marker="gen_tcp#send/2"><c>gen_tcp:send/2</c></seealso> is used.</p> + <p>If the OS used does not support non-blocking <c>sendfile</c>, an + Erlang fallback using <seealso marker="#read/2"><c>read/2</c></seealso> + and <seealso marker="gen_tcp#send/2"><c>gen_tcp:send/2</c></seealso> is + used.</p> <p>The option list can contain the following options:</p> <taglist> <tag><c>chunk_size</c></tag> <item><p>The chunk size used by the Erlang fallback to send data. If using the fallback, set this to a value that comfortably fits in the systems memory. Default is 20 MB.</p></item> - <tag><c>use_threads</c></tag> - <item><p>Instructs the emulator to use the <c>async</c> thread pool for the - <c>sendfile</c> system call. This can be useful if the OS you are running - on does not properly support non-blocking <c>sendfile</c> calls. Notice that - using <c>async</c> threads potentially makes your system vulnerable to slow - client attacks. If set to <c>true</c> and no <c>async</c> threads are available, - the <c>sendfile</c> call returns <c>{error,einval}</c>. - Introduced in Erlang/OTP 17.0. Default is <c>false</c>.</p></item> </taglist> </desc> </func> @@ -2148,144 +2151,77 @@ f.txt: {person, "kalle", 25}. <section> <title>Performance</title> - <p>Some operating system file operations, for example, a - <c>sync/1</c> or <c>close/1</c> on a huge file, can block their - calling thread for seconds. If this affects the emulator main - thread, the response time is no longer in the order of - milliseconds, depending on the definition of "soft" in soft - real-time system.</p> - <p>If the device driver thread pool is active, file operations are - done through those threads instead, so the emulator can go on - executing Erlang processes. Unfortunately, the time for serving a - file operation increases because of the extra scheduling required - from the operating system.</p> - <p>If the device driver thread pool is disabled or of size 0, large - file reads and writes are segmented into many smaller, which - enable the emulator to serve other processes during the file - operation. This has the same effect as when using the thread - pool, but with larger overhead. Other file operations, for - example, <c>sync/1</c> or <c>close/1</c> on a huge file, still are - a problem.</p> - <p>For increased performance, raw files are recommended. Raw files - use the file system of the host machine of the node.</p> + <p>For increased performance, raw files are recommended.</p> + <p>A normal file is really a process so it can be used as an I/O + device (see <seealso marker="stdlib:io"><c>io</c></seealso>). + Therefore, when data is written to a normal file, the sending of the + data to the file process, copies all data that are not binaries. Opening + the file in binary mode and writing binaries is therefore recommended. + If the file is opened on another node, or if the file server runs as + slave to the file server of another node, also binaries are copied.</p> <note> - <p> - For normal files (non-raw), the file server is used to find the files, - and if the node is running its file server as slave to the file server - of another node, and the other node runs on some other host machine, - they can have different file systems. - However, this is seldom a problem.</p> + <p>Raw files use the file system of the host machine of the node. + For normal files (non-raw), the file server is used to find the files, + and if the node is running its file server as slave to the file server + of another node, and the other node runs on some other host machine, + they can have different file systems. + However, this is seldom a problem.</p> </note> - <p>A normal file is really a process so it can be used as an I/O - device (see - <seealso marker="stdlib:io"><c>io</c></seealso>). - Therefore, when data is written to a - normal file, the sending of the data to the file process, copies - all data that are not binaries. Opening the file in binary mode - and writing binaries is therefore recommended. If the file is - opened on another node, or if the file server runs as slave to - the file server of another node, also binaries are copied.</p> - <p>Caching data to reduce the number of file operations, or rather - the number of calls to the file driver, generally increases - performance. The following function writes 4 MBytes in 23 - seconds when tested:</p> + <p><seealso marker="#open/2"><c>open/2</c></seealso> can be given the + options <c>delayed_write</c> and <c>read_ahead</c> to turn on caching, + which will reduce the number of operating system calls and greatly + improve performance for small reads and writes. However, the overhead + won't disappear completely and it's best to keep the number of file + operations to a minimum. As a contrived example, the following function + writes 4MB in 2.5 seconds when tested:</p> + <code type="none"><![CDATA[ -create_file_slow(Name, N) when integer(N), N >= 0 -> - {ok, FD} = file:open(Name, [raw, write, delayed_write, binary]), - ok = create_file_slow(FD, 0, N), - ok = ?FILE_MODULE:close(FD), - ok. - -create_file_slow(FD, M, M) -> +create_file_slow(Name) -> + {ok, Fd} = file:open(Name, [raw, write, delayed_write, binary]), + create_file_slow_1(Fd, 4 bsl 20), + file:close(Fd). + +create_file_slow_1(_Fd, 0) -> ok; -create_file_slow(FD, M, N) -> - ok = file:write(FD, <<M:32/unsigned>>), - create_file_slow(FD, M+1, N).]]></code> +create_file_slow_1(Fd, M) -> + ok = file:write(Fd, <<0>>), + create_file_slow_1(Fd, M - 1).]]></code> + + <p>The following functionally equivalent code writes 128 bytes per call + to <seealso marker="#write/2"><c>write/2</c></seealso> and so does the + same work in 0.08 seconds, which is roughly 30 times faster:</p> - <p>The following, functionally equivalent, function collects 1024 - entries into a list of 128 32-byte binaries before each call to - <seealso marker="#write/2"><c>write/2</c></seealso> and so - does the same work in 0.52 seconds, - which is 44 times faster:</p> <code type="none"><![CDATA[ -create_file(Name, N) when integer(N), N >= 0 -> - {ok, FD} = file:open(Name, [raw, write, delayed_write, binary]), - ok = create_file(FD, 0, N), - ok = ?FILE_MODULE:close(FD), +create_file(Name) -> + {ok, Fd} = file:open(Name, [raw, write, delayed_write, binary]), + create_file_1(Fd, 4 bsl 20), + file:close(Fd), ok. - -create_file(FD, M, M) -> + +create_file_1(_Fd, 0) -> ok; -create_file(FD, M, N) when M + 1024 =< N -> - create_file(FD, M, M + 1024, []), - create_file(FD, M + 1024, N); -create_file(FD, M, N) -> - create_file(FD, M, N, []). - -create_file(FD, M, M, R) -> - ok = file:write(FD, R); -create_file(FD, M, N0, R) when M + 8 =< N0 -> - N1 = N0-1, N2 = N0-2, N3 = N0-3, N4 = N0-4, - N5 = N0-5, N6 = N0-6, N7 = N0-7, N8 = N0-8, - create_file(FD, M, N8, - [<<N8:32/unsigned, N7:32/unsigned, - N6:32/unsigned, N5:32/unsigned, - N4:32/unsigned, N3:32/unsigned, - N2:32/unsigned, N1:32/unsigned>> | R]); -create_file(FD, M, N0, R) -> - N1 = N0-1, - create_file(FD, M, N1, [<<N1:32/unsigned>> | R]).]]></code> +create_file_1(Fd, M) when M >= 128 -> + ok = file:write(Fd, <<0:(128)/unit:8>>), + create_file_1(Fd, M - 128); +create_file_1(Fd, M) -> + ok = file:write(Fd, <<0:(M)/unit:8>>), + create_file_1(Fd, M - 1).]]></code> - <note> - <p>Trust only your own benchmarks. If the list length in - <c>create_file/2</c> above is increased, it runs slightly - faster, but consumes more memory and causes more memory - fragmentation. How much this affects your application is - something that this simple benchmark cannot predict.</p> - <p>If the size of each binary is increased to 64 bytes, it - also runs slightly faster, but the code is then twice as clumsy. - In the current implementation, binaries larger than 64 bytes are - stored in memory common to all processes and not copied when - sent between processes, while these smaller binaries are stored - on the process heap and copied when sent like any other term.</p> - <p>So, with a binary size of 68 bytes, <c>create_file/2</c> runs - 30 percent slower than with 64 bytes, and causes much more - memory fragmentation. Notice that if the binaries were to be sent - between processes (for example, a non-raw file), the results - would probably be completely different.</p> - </note> - <p>A raw file is really a port. When writing data to a port, it is - efficient to write a list of binaries. It is not needed to - flatten a deep list before writing. On Unix hosts, scatter output, - which writes a set of buffers in one operation, is used when - possible. In this way <c>write(FD, [Bin1, Bin2 | Bin3])</c> - writes the contents of the binaries without copying the data - at all, except for perhaps deep down in the operating system - kernel.</p> - <p>For raw files, <c>pwrite/2</c> and <c>pread/2</c> are - efficiently implemented. The file driver is called only once for - the whole operation, and the list iteration is done in the file - driver.</p> - <p>The options <c>delayed_write</c> and <c>read_ahead</c> to - <seealso marker="#open/2"><c>open/2</c></seealso> - make the file driver cache data to reduce - the number of operating system calls. The function - <c>create_file/2</c> in the recent example takes 60 seconds - without option <c>delayed_write</c>, which is 2.6 - times slower.</p> - <p>As a bad example, <c>create_file_slow/2</c> - without options <c>raw</c>, <c>binary</c>, and <c>delayed_write</c>, - meaning it calls <c>open(Name, [write])</c>, needs - 1 min 20 seconds for the job, which is 3.5 times slower than - the first example, and 150 times slower than the optimized - <c>create_file/2</c>.</p> - <warning> - <p>If an error occurs when accessing an open file with module - <seealso marker="stdlib:io"><c>io</c></seealso>, - the process handling the file exits. The dead - file process can hang if a process tries to access it later. - This will be fixed in a future release.</p> - </warning> + <p>When writing data it's generally more efficient to write a list of + binaries rather than a list of integers. It is not needed to + flatten a deep list before writing. On Unix hosts, scatter output, + which writes a set of buffers in one operation, is used when + possible. In this way <c>write(FD, [Bin1, Bin2 | Bin3])</c> + writes the contents of the binaries without copying the data + at all, except for perhaps deep down in the operating system + kernel.</p> + <warning> + <p>If an error occurs when accessing an open file with module + <seealso marker="stdlib:io"><c>io</c></seealso>, the process + handling the file exits. The dead file process can hang if a process + tries to access it later. This will be fixed in a future release. + </p> + </warning> </section> <section> diff --git a/lib/kernel/doc/src/notes.xml b/lib/kernel/doc/src/notes.xml index a5316dd476..d7f224c38e 100644 --- a/lib/kernel/doc/src/notes.xml +++ b/lib/kernel/doc/src/notes.xml @@ -31,6 +31,21 @@ </header> <p>This document describes the changes made to the Kernel application.</p> +<section><title>Kernel 5.4.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Refactored an internal API.</p> + <p> + Own Id: OTP-14784</p> + </item> + </list> + </section> + +</section> + <section><title>Kernel 5.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/kernel/src/Makefile b/lib/kernel/src/Makefile index 5946620f0f..0bc9f121a0 100644 --- a/lib/kernel/src/Makefile +++ b/lib/kernel/src/Makefile @@ -106,6 +106,7 @@ MODULES = \ inet_sctp \ kernel \ kernel_config \ + kernel_refc \ local_udp \ local_tcp \ net \ @@ -120,6 +121,13 @@ MODULES = \ user \ user_drv \ user_sup \ + raw_file_io \ + raw_file_io_compressed \ + raw_file_io_inflate \ + raw_file_io_deflate \ + raw_file_io_delayed \ + raw_file_io_list \ + raw_file_io_raw \ wrap_log_reader HRL_FILES= ../include/file.hrl ../include/inet.hrl ../include/inet_sctp.hrl \ @@ -226,7 +234,8 @@ $(EBIN)/disk_log_server.beam: disk_log.hrl $(EBIN)/dist_util.beam: ../include/dist_util.hrl ../include/dist.hrl $(EBIN)/erl_boot_server.beam: inet_boot.hrl $(EBIN)/erl_epmd.beam: inet_int.hrl erl_epmd.hrl -$(EBIN)/file.beam: ../include/file.hrl +$(EBIN)/file.beam: ../include/file.hrl file_int.hrl +$(EBIN)/file_io_server.beam: ../include/file.hrl file_int.hrl $(EBIN)/gen_tcp.beam: inet_int.hrl $(EBIN)/gen_udp.beam: inet_int.hrl $(EBIN)/gen_sctp.beam: ../include/inet_sctp.hrl @@ -254,3 +263,10 @@ $(EBIN)/net_kernel.beam: ../include/net_address.hrl $(EBIN)/os.beam: ../include/file.hrl $(EBIN)/ram_file.beam: ../include/file.hrl $(EBIN)/wrap_log_reader.beam: disk_log.hrl ../include/file.hrl +$(EBIN)/raw_file_io.beam: ../include/file.hrl file_int.hrl +$(EBIN)/raw_file_io_compressed.beam: ../include/file.hrl file_int.hrl +$(EBIN)/raw_file_io_inflate.beam: ../include/file.hrl file_int.hrl +$(EBIN)/raw_file_io_deflate.beam: ../include/file.hrl file_int.hrl +$(EBIN)/raw_file_io_delayed.beam: ../include/file.hrl file_int.hrl +$(EBIN)/raw_file_io_list.beam: ../include/file.hrl file_int.hrl +$(EBIN)/raw_file_io_raw.beam: ../include/file.hrl file_int.hrl diff --git a/lib/kernel/src/file.erl b/lib/kernel/src/file.erl index 933f2d5f65..d05199897f 100644 --- a/lib/kernel/src/file.erl +++ b/lib/kernel/src/file.erl @@ -72,7 +72,7 @@ io_device/0, name/0, name_all/0, posix/0]). %%% Includes and defines --include("file.hrl"). +-include("file_int.hrl"). -define(FILE_IO_SERVER_TABLE, file_io_servers). @@ -454,41 +454,23 @@ raw_write_file_info(Name, #file_info{} = Info) -> Reason :: posix() | badarg | system_limit. open(Item, ModeList) when is_list(ModeList) -> - case lists:member(raw, ModeList) of - %% Raw file, use ?PRIM_FILE to handle this file - true -> + case {lists:member(raw, ModeList), lists:member(ram, ModeList)} of + {false, false} -> + %% File server file Args = [file_name(Item) | ModeList], case check_args(Args) of ok -> [FileName | _] = Args, - %% We rely on the returned Handle (in {ok, Handle}) - %% being a pid() or a #file_descriptor{} - ?PRIM_FILE:open(FileName, ModeList); + call(open, [FileName, ModeList]); Error -> Error - end; - false -> - case lists:member(ram, ModeList) of - %% RAM file, use ?RAM_FILE to handle this file - true -> - case check_args(ModeList) of - ok -> - ?RAM_FILE:open(Item, ModeList); - Error -> - Error - end; - %% File server file - false -> - Args = [file_name(Item) | ModeList], - case check_args(Args) of - ok -> - [FileName | _] = Args, - call(open, [FileName, ModeList]); - Error -> - Error - end - end + end; + {true, _Either} -> + raw_file_io:open(file_name(Item), ModeList); + {false, true} -> + ram_file:open(Item, ModeList) end; + %% Old obsolete mode specification in atom or 2-tuple format open(Item, Mode) -> open(Item, mode_list(Mode)). @@ -1254,15 +1236,18 @@ sendfile(File, _Sock, _Offet, _Bytes, _Opts) when is_pid(File) -> sendfile(File, Sock, Offset, Bytes, []) -> sendfile(File, Sock, Offset, Bytes, ?MAX_CHUNK_SIZE, [], [], []); sendfile(File, Sock, Offset, Bytes, Opts) -> - ChunkSize0 = proplists:get_value(chunk_size, Opts, ?MAX_CHUNK_SIZE), - ChunkSize = if ChunkSize0 > ?MAX_CHUNK_SIZE -> - ?MAX_CHUNK_SIZE; - true -> ChunkSize0 - end, - %% Support for headers, trailers and options has been removed because the - %% Darwin and BSD API for using it does not play nice with - %% non-blocking sockets. See unix_efile.c for more info. - sendfile(File, Sock, Offset, Bytes, ChunkSize, [], [], Opts). + try proplists:get_value(chunk_size, Opts, ?MAX_CHUNK_SIZE) of + ChunkSize0 when is_integer(ChunkSize0) -> + ChunkSize = erlang:min(ChunkSize0, ?MAX_CHUNK_SIZE), + %% Support for headers, trailers and options has been removed + %% because the Darwin and BSD API for using it does not play nice + %% with non-blocking sockets. See unix_efile.c for more info. + sendfile(File, Sock, Offset, Bytes, ChunkSize, [], [], Opts); + _Other -> + {error, badarg} + catch + error:_ -> {error, badarg} + end. %% sendfile/2 -spec sendfile(Filename, Socket) -> diff --git a/lib/kernel/src/file_int.hrl b/lib/kernel/src/file_int.hrl new file mode 100644 index 0000000000..bafc330c04 --- /dev/null +++ b/lib/kernel/src/file_int.hrl @@ -0,0 +1,33 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +%% Internal definitions for the 'file' module and friends. +%% + +-ifndef(FILE_INTERNAL_HRL_). +-define(FILE_INTERNAL_HRL_, 1). + +-include("file.hrl"). + +-define(CALL_FD(Fd, Method, Args), + apply(Fd#file_descriptor.module, Method, [Fd | Args])). + +-endif. diff --git a/lib/kernel/src/file_io_server.erl b/lib/kernel/src/file_io_server.erl index deb7b315b1..34d5497a4a 100644 --- a/lib/kernel/src/file_io_server.erl +++ b/lib/kernel/src/file_io_server.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2015. All Rights Reserved. +%% Copyright Ericsson AB 2000-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -28,7 +28,8 @@ -record(state, {handle,owner,mref,buf,read_mode,unic}). --define(PRIM_FILE, prim_file). +-include("file_int.hrl"). + -define(READ_SIZE_LIST, 128). -define(READ_SIZE_BINARY, (8*1024)). @@ -67,8 +68,9 @@ do_start(Spawn, Owner, FileName, ModeList) -> erlang:dt_restore_tag(Utag), %% process_flag(trap_exit, true), case parse_options(ModeList) of - {ReadMode, UnicodeMode, Opts} -> - case ?PRIM_FILE:open(FileName, Opts) of + {ReadMode, UnicodeMode, Opts0} -> + Opts = maybe_add_read_ahead(ReadMode, Opts0), + case raw_file_io:open(FileName, [raw | Opts]) of {error, Reason} = Error -> Self ! {Ref, Error}, exit(Reason); @@ -157,6 +159,24 @@ valid_enc({utf32,little}) -> valid_enc(_Other) -> {error,badarg}. +%% Add a small read_ahead buffer if the file is opened for reading +%% only in list mode and no read_ahead is already given. +maybe_add_read_ahead(binary, Opts) -> + Opts; +maybe_add_read_ahead(list, Opts) -> + P = fun(read_ahead) -> true; + ({read_ahead,_}) -> true; + (append) -> true; + (exclusive) -> true; + (write) -> true; + (_) -> false + end, + case lists:any(P, Opts) of + false -> + [{read_ahead, 4096}|Opts]; + true -> + Opts + end. server_loop(#state{mref = Mref} = State) -> receive @@ -205,7 +225,7 @@ io_reply(From, ReplyAs, Reply) -> file_request({advise,Offset,Length,Advise}, #state{handle=Handle}=State) -> - case ?PRIM_FILE:advise(Handle, Offset, Length, Advise) of + case ?CALL_FD(Handle, advise, [Offset, Length, Advise]) of {error,Reason}=Reply -> {stop,Reason,Reply,State}; Reply -> @@ -213,7 +233,7 @@ file_request({advise,Offset,Length,Advise}, end; file_request({allocate, Offset, Length}, #state{handle = Handle} = State) -> - Reply = ?PRIM_FILE:allocate(Handle, Offset, Length), + Reply = ?CALL_FD(Handle, allocate, [Offset, Length]), {reply, Reply, State}; file_request({pread,At,Sz}, State) when At =:= cur; @@ -256,7 +276,7 @@ file_request({pwrite,At,Data}, end; file_request(datasync, #state{handle=Handle}=State) -> - case ?PRIM_FILE:datasync(Handle) of + case ?CALL_FD(Handle, datasync, []) of {error,Reason}=Reply -> {stop,Reason,Reply,State}; Reply -> @@ -264,7 +284,7 @@ file_request(datasync, end; file_request(sync, #state{handle=Handle}=State) -> - case ?PRIM_FILE:sync(Handle) of + case ?CALL_FD(Handle, sync, []) of {error,Reason}=Reply -> {stop,Reason,Reply,State}; Reply -> @@ -272,7 +292,7 @@ file_request(sync, end; file_request(close, #state{handle=Handle}=State) -> - case ?PRIM_FILE:close(Handle) of + case ?CALL_FD(Handle, close, []) of {error,Reason}=Reply -> {stop,Reason,Reply,State#state{buf= <<>>}}; Reply -> @@ -288,7 +308,7 @@ file_request({position,At}, end; file_request(truncate, #state{handle=Handle}=State) -> - case ?PRIM_FILE:truncate(Handle) of + case ?CALL_FD(Handle, truncate, []) of {error,Reason}=Reply -> {stop,Reason,Reply,State#state{buf= <<>>}}; Reply -> @@ -398,7 +418,7 @@ io_request_loop([Request|Tail], %% put_chars(Chars, latin1, #state{handle=Handle, unic=latin1}=State) -> NewState = State#state{buf = <<>>}, - case ?PRIM_FILE:write(Handle, Chars) of + case ?CALL_FD(Handle, write, [Chars]) of {error,Reason}=Reply -> {stop,Reason,Reply,NewState}; Reply -> @@ -408,7 +428,7 @@ put_chars(Chars, InEncoding, #state{handle=Handle, unic=OutEncoding}=State) -> NewState = State#state{buf = <<>>}, case unicode:characters_to_binary(Chars,InEncoding,OutEncoding) of Bin when is_binary(Bin) -> - case ?PRIM_FILE:write(Handle, Bin) of + case ?CALL_FD(Handle, write, [Bin]) of {error,Reason}=Reply -> {stop,Reason,Reply,NewState}; Reply -> @@ -422,7 +442,7 @@ put_chars(Chars, InEncoding, #state{handle=Handle, unic=OutEncoding}=State) -> get_line(S, {<<>>, Cont}, OutEnc, #state{handle=Handle, read_mode=Mode, unic=InEnc}=State) -> - case ?PRIM_FILE:read(Handle, read_size(Mode)) of + case ?CALL_FD(Handle, read, [read_size(Mode)]) of {ok,Bin} -> get_line(S, convert_enc([Cont, Bin], InEnc, OutEnc), OutEnc, State); eof -> @@ -472,7 +492,7 @@ get_chars(N, OutEnc,#state{handle=Handle,buf=Buf,read_mode=ReadMode,unic=latin1} BufSize = byte_size(Buf), NeedSize = N-BufSize, Size = erlang:max(NeedSize, ?READ_SIZE_BINARY), - case ?PRIM_FILE:read(Handle, Size) of + case ?CALL_FD(Handle, read, [Size]) of {ok, B} -> if BufSize+byte_size(B) < N -> std_reply(cat(Buf, B, ReadMode,latin1,OutEnc), State); @@ -504,7 +524,7 @@ get_chars(N, OutEnc,#state{handle=Handle,buf=Buf,read_mode=ReadMode,unic=InEncod %% Need more, Try to read 4*needed in bytes... NeedSize = (N - BufCount) * 4, Size = erlang:max(NeedSize, ?READ_SIZE_BINARY), - case ?PRIM_FILE:read(Handle, Size) of + case ?CALL_FD(Handle, read, [Size]) of {ok, B} -> NewBuf = list_to_binary([Buf,B]), {NewCount,NewSplit} = count_and_find(NewBuf,N,InEncoding), @@ -544,7 +564,7 @@ get_chars(Mod, Func, XtraArg, OutEnc, #state{buf=Buf}=State) -> get_chars_empty(Mod, Func, XtraArg, S, latin1, #state{handle=Handle,read_mode=ReadMode, unic=latin1}=State) -> - case ?PRIM_FILE:read(Handle, read_size(ReadMode)) of + case ?CALL_FD(Handle, read, [read_size(ReadMode)]) of {ok,Bin} -> get_chars_apply(Mod, Func, XtraArg, S, latin1, State, Bin); eof -> @@ -554,7 +574,7 @@ get_chars_empty(Mod, Func, XtraArg, S, latin1, end; get_chars_empty(Mod, Func, XtraArg, S, OutEnc, #state{handle=Handle,read_mode=ReadMode}=State) -> - case ?PRIM_FILE:read(Handle, read_size(ReadMode)) of + case ?CALL_FD(Handle, read, [read_size(ReadMode)]) of {ok,Bin} -> get_chars_apply(Mod, Func, XtraArg, S, OutEnc, State, Bin); eof -> @@ -564,7 +584,7 @@ get_chars_empty(Mod, Func, XtraArg, S, OutEnc, end. get_chars_notempty(Mod, Func, XtraArg, S, OutEnc, #state{handle=Handle,read_mode=ReadMode,buf = B}=State) -> - case ?PRIM_FILE:read(Handle, read_size(ReadMode)) of + case ?CALL_FD(Handle, read, [read_size(ReadMode)]) of {ok,Bin} -> get_chars_apply(Mod, Func, XtraArg, S, OutEnc, State, list_to_binary([B,Bin])); eof -> @@ -918,13 +938,10 @@ cbv({utf32,little},_) -> %% Compensates ?PRIM_FILE:position/2 for the number of bytes %% we have buffered position(Handle, At, Buf) -> - ?PRIM_FILE:position( - Handle, - case At of - cur -> - {cur, -byte_size(Buf)}; - {cur, Offs} -> - {cur, Offs-byte_size(Buf)}; - _ -> - At - end). + SeekTo = + case At of + {cur, Offs} -> {cur, Offs-byte_size(Buf)}; + cur -> {cur, -byte_size(Buf)}; + _ -> At + end, + ?CALL_FD(Handle, position, [SeekTo]). diff --git a/lib/kernel/src/file_server.erl b/lib/kernel/src/file_server.erl index 6e8f64d932..ecc1ffbdd6 100644 --- a/lib/kernel/src/file_server.erl +++ b/lib/kernel/src/file_server.erl @@ -63,7 +63,7 @@ stop() -> %%% Callback functions from gen_server %%%---------------------------------------------------------------------- --type state() :: port(). % Internal type +-type state() :: term(). % Internal type %%---------------------------------------------------------------------- %% Func: init/1 @@ -77,14 +77,8 @@ stop() -> init([]) -> process_flag(trap_exit, true), - case ?PRIM_FILE:start() of - {ok, Handle} -> - ?FILE_IO_SERVER_TABLE = - ets:new(?FILE_IO_SERVER_TABLE, [named_table]), - {ok, Handle}; - {error, Reason} -> - {stop, Reason} - end. + ?FILE_IO_SERVER_TABLE = ets:new(?FILE_IO_SERVER_TABLE, [named_table]), + {ok, undefined}. %%---------------------------------------------------------------------- %% Func: handle_call/3 @@ -101,7 +95,7 @@ init([]) -> {'reply', 'eof' | 'ok' | {'error', term()} | {'ok', term()}, state()} | {'stop', 'normal', 'stopped', state()}. -handle_call({open, Name, ModeList}, {Pid, _Tag} = _From, Handle) +handle_call({open, Name, ModeList}, {Pid, _Tag} = _From, State) when is_list(ModeList) -> Child = ?FILE_IO_SERVER:start_link(Pid, Name, ModeList), case Child of @@ -110,78 +104,78 @@ handle_call({open, Name, ModeList}, {Pid, _Tag} = _From, Handle) _ -> ok end, - {reply, Child, Handle}; + {reply, Child, State}; -handle_call({open, _Name, _Mode}, _From, Handle) -> - {reply, {error, einval}, Handle}; +handle_call({open, _Name, _Mode}, _From, State) -> + {reply, {error, einval}, State}; -handle_call({read_file, Name}, _From, Handle) -> - {reply, ?PRIM_FILE:read_file(Name), Handle}; +handle_call({read_file, Name}, _From, State) -> + {reply, ?PRIM_FILE:read_file(Name), State}; -handle_call({write_file, Name, Bin}, _From, Handle) -> - {reply, ?PRIM_FILE:write_file(Name, Bin), Handle}; +handle_call({write_file, Name, Bin}, _From, State) -> + {reply, ?PRIM_FILE:write_file(Name, Bin), State}; -handle_call({set_cwd, Name}, _From, Handle) -> - {reply, ?PRIM_FILE:set_cwd(Handle, Name), Handle}; +handle_call({set_cwd, Name}, _From, State) -> + {reply, ?PRIM_FILE:set_cwd(Name), State}; -handle_call({delete, Name}, _From, Handle) -> - {reply, ?PRIM_FILE:delete(Handle, Name), Handle}; +handle_call({delete, Name}, _From, State) -> + {reply, ?PRIM_FILE:delete(Name), State}; -handle_call({rename, Fr, To}, _From, Handle) -> - {reply, ?PRIM_FILE:rename(Handle, Fr, To), Handle}; +handle_call({rename, Fr, To}, _From, State) -> + {reply, ?PRIM_FILE:rename(Fr, To), State}; -handle_call({make_dir, Name}, _From, Handle) -> - {reply, ?PRIM_FILE:make_dir(Handle, Name), Handle}; +handle_call({make_dir, Name}, _From, State) -> + {reply, ?PRIM_FILE:make_dir(Name), State}; -handle_call({del_dir, Name}, _From, Handle) -> - {reply, ?PRIM_FILE:del_dir(Handle, Name), Handle}; +handle_call({del_dir, Name}, _From, State) -> + {reply, ?PRIM_FILE:del_dir(Name), State}; -handle_call({list_dir, Name}, _From, Handle) -> - {reply, ?PRIM_FILE:list_dir(Handle, Name), Handle}; -handle_call({list_dir_all, Name}, _From, Handle) -> - {reply, ?PRIM_FILE:list_dir_all(Handle, Name), Handle}; +handle_call({list_dir, Name}, _From, State) -> + {reply, ?PRIM_FILE:list_dir(Name), State}; +handle_call({list_dir_all, Name}, _From, State) -> + {reply, ?PRIM_FILE:list_dir_all(Name), State}; -handle_call(get_cwd, _From, Handle) -> - {reply, ?PRIM_FILE:get_cwd(Handle), Handle}; -handle_call({get_cwd}, _From, Handle) -> - {reply, ?PRIM_FILE:get_cwd(Handle), Handle}; -handle_call({get_cwd, Name}, _From, Handle) -> - {reply, ?PRIM_FILE:get_cwd(Handle, Name), Handle}; +handle_call(get_cwd, _From, State) -> + {reply, ?PRIM_FILE:get_cwd(), State}; +handle_call({get_cwd}, _From, State) -> + {reply, ?PRIM_FILE:get_cwd(), State}; +handle_call({get_cwd, Name}, _From, State) -> + {reply, ?PRIM_FILE:get_cwd(Name), State}; -handle_call({read_file_info, Name}, _From, Handle) -> - {reply, ?PRIM_FILE:read_file_info(Handle, Name), Handle}; +handle_call({read_file_info, Name}, _From, State) -> + {reply, ?PRIM_FILE:read_file_info(Name), State}; -handle_call({read_file_info, Name, Opts}, _From, Handle) -> - {reply, ?PRIM_FILE:read_file_info(Handle, Name, Opts), Handle}; +handle_call({read_file_info, Name, Opts}, _From, State) -> + {reply, ?PRIM_FILE:read_file_info(Name, Opts), State}; -handle_call({altname, Name}, _From, Handle) -> - {reply, ?PRIM_FILE:altname(Handle, Name), Handle}; +handle_call({altname, Name}, _From, State) -> + {reply, ?PRIM_FILE:altname(Name), State}; -handle_call({write_file_info, Name, Info}, _From, Handle) -> - {reply, ?PRIM_FILE:write_file_info(Handle, Name, Info), Handle}; +handle_call({write_file_info, Name, Info}, _From, State) -> + {reply, ?PRIM_FILE:write_file_info(Name, Info), State}; -handle_call({write_file_info, Name, Info, Opts}, _From, Handle) -> - {reply, ?PRIM_FILE:write_file_info(Handle, Name, Info, Opts), Handle}; +handle_call({write_file_info, Name, Info, Opts}, _From, State) -> + {reply, ?PRIM_FILE:write_file_info(Name, Info, Opts), State}; -handle_call({read_link_info, Name}, _From, Handle) -> - {reply, ?PRIM_FILE:read_link_info(Handle, Name), Handle}; +handle_call({read_link_info, Name}, _From, State) -> + {reply, ?PRIM_FILE:read_link_info(Name), State}; -handle_call({read_link_info, Name, Opts}, _From, Handle) -> - {reply, ?PRIM_FILE:read_link_info(Handle, Name, Opts), Handle}; +handle_call({read_link_info, Name, Opts}, _From, State) -> + {reply, ?PRIM_FILE:read_link_info(Name, Opts), State}; -handle_call({read_link, Name}, _From, Handle) -> - {reply, ?PRIM_FILE:read_link(Handle, Name), Handle}; -handle_call({read_link_all, Name}, _From, Handle) -> - {reply, ?PRIM_FILE:read_link_all(Handle, Name), Handle}; +handle_call({read_link, Name}, _From, State) -> + {reply, ?PRIM_FILE:read_link(Name), State}; +handle_call({read_link_all, Name}, _From, State) -> + {reply, ?PRIM_FILE:read_link_all(Name), State}; -handle_call({make_link, Old, New}, _From, Handle) -> - {reply, ?PRIM_FILE:make_link(Handle, Old, New), Handle}; +handle_call({make_link, Old, New}, _From, State) -> + {reply, ?PRIM_FILE:make_link(Old, New), State}; -handle_call({make_symlink, Old, New}, _From, Handle) -> - {reply, ?PRIM_FILE:make_symlink(Handle, Old, New), Handle}; +handle_call({make_symlink, Old, New}, _From, State) -> + {reply, ?PRIM_FILE:make_symlink(Old, New), State}; handle_call({copy, SourceName, SourceOpts, DestName, DestOpts, Length}, - _From, Handle) -> + _From, State) -> Reply = case ?PRIM_FILE:open(SourceName, [read, binary | SourceOpts]) of {ok, Source} -> @@ -201,14 +195,14 @@ handle_call({copy, SourceName, SourceOpts, DestName, DestOpts, Length}, {error, _} = Error -> Error end, - {reply, Reply, Handle}; + {reply, Reply, State}; -handle_call(stop, _From, Handle) -> - {stop, normal, stopped, Handle}; +handle_call(stop, _From, State) -> + {stop, normal, stopped, State}; -handle_call(Request, From, Handle) -> +handle_call(Request, From, State) -> error_logger:error_msg("handle_call(~tp, ~tp, _)", [Request, From]), - {noreply, Handle}. + {noreply, State}. %%---------------------------------------------------------------------- %% Func: handle_cast/2 @@ -233,14 +227,9 @@ handle_cast(Msg, State) -> -spec handle_info(term(), state()) -> {'noreply', state()} | {'stop', 'normal', state()}. -handle_info({'EXIT', Pid, _Reason}, Handle) when is_pid(Pid) -> +handle_info({'EXIT', Pid, _Reason}, State) when is_pid(Pid) -> ets:delete(?FILE_IO_SERVER_TABLE, Pid), - {noreply, Handle}; - -handle_info({'EXIT', Handle, _Reason}, Handle) -> - error_logger:error_msg("Port controlling ~w terminated in ~w", - [?FILE_SERVER, ?MODULE]), - {stop, normal, Handle}; + {noreply, State}; handle_info(Info, State) -> error_logger:error_msg("handle_Info(~tp, _)", [Info]), @@ -254,8 +243,8 @@ handle_info(Info, State) -> -spec terminate(term(), state()) -> 'ok'. -terminate(_Reason, Handle) -> - ?PRIM_FILE:stop(Handle). +terminate(_Reason, _State) -> + ok. %%---------------------------------------------------------------------- %% Func: code_change/3 diff --git a/lib/kernel/src/inet_int.hrl b/lib/kernel/src/inet_int.hrl index bc5b67f7bf..357e27826c 100644 --- a/lib/kernel/src/inet_int.hrl +++ b/lib/kernel/src/inet_int.hrl @@ -100,6 +100,8 @@ -define(TCP_REQ_RECV, 42). -define(TCP_REQ_UNRECV, 43). -define(TCP_REQ_SHUTDOWN, 44). +-define(TCP_REQ_SENDFILE, 45). + %% UDP and SCTP requests -define(PACKET_REQ_RECV, 60). %%-define(SCTP_REQ_LISTEN, 61). MERGED @@ -319,6 +321,12 @@ [((X) bsr 24) band 16#ff, ((X) bsr 16) band 16#ff, ((X) bsr 8) band 16#ff, (X) band 16#ff]). +-define(int64(X), + [((X) bsr 56) band 16#ff, ((X) bsr 48) band 16#ff, + ((X) bsr 40) band 16#ff, ((X) bsr 32) band 16#ff, + ((X) bsr 24) band 16#ff, ((X) bsr 16) band 16#ff, + ((X) bsr 8) band 16#ff, (X) band 16#ff]). + -define(intAID(X), % For SCTP AssocID ?int32(X)). diff --git a/lib/kernel/src/kernel.app.src b/lib/kernel/src/kernel.app.src index 080b11fc4d..82a3571da9 100644 --- a/lib/kernel/src/kernel.app.src +++ b/lib/kernel/src/kernel.app.src @@ -57,6 +57,7 @@ inet_tcp_dist, kernel, kernel_config, + kernel_refc, local_tcp, local_udp, net, @@ -88,6 +89,13 @@ inet_udp, inet_sctp, pg2, + raw_file_io, + raw_file_io_compressed, + raw_file_io_deflate, + raw_file_io_delayed, + raw_file_io_inflate, + raw_file_io_list, + raw_file_io_raw, seq_trace, standard_error, wrap_log_reader]}, @@ -107,6 +115,7 @@ heart, init, kernel_config, + kernel_refc, kernel_sup, net_kernel, net_sup, diff --git a/lib/kernel/src/kernel.erl b/lib/kernel/src/kernel.erl index cba57088ec..0382764b39 100644 --- a/lib/kernel/src/kernel.erl +++ b/lib/kernel/src/kernel.erl @@ -111,6 +111,13 @@ init([]) -> type => worker, modules => [kernel_config]}, + RefC = #{id => kernel_refc, + start => {kernel_refc, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [kernel_refc]}, + Code = #{id => code_server, start => {code, start_link, []}, restart => permanent, @@ -148,7 +155,7 @@ init([]) -> case init:get_argument(mode) of {ok, [["minimal"]]} -> - {ok, {SupFlags, [Code, File, StdError, User, Config, SafeSup]}}; + {ok, {SupFlags, [Code, File, StdError, User, Config, RefC, SafeSup]}}; _ -> Rpc = #{id => rex, start => {rpc, start_link, []}, @@ -199,7 +206,7 @@ init([]) -> {ok, {SupFlags, [Code, Rpc, Global, InetDb | DistAC] ++ [NetSup, GlGroup, File, SigSrv, - StdError, User, Config, SafeSup] ++ Timer}} + StdError, User, Config, RefC, SafeSup] ++ Timer}} end; init(safe) -> SupFlags = #{strategy => one_for_one, diff --git a/lib/kernel/src/kernel_refc.erl b/lib/kernel/src/kernel_refc.erl new file mode 100644 index 0000000000..05076dc885 --- /dev/null +++ b/lib/kernel/src/kernel_refc.erl @@ -0,0 +1,139 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(kernel_refc). + +-behaviour(gen_server). + +%% External exports +-export([start_link/0, scheduler_wall_time/1]). +%% Internal exports +-export([init/1, handle_info/2, terminate/2]). +-export([handle_call/3, handle_cast/2, code_change/3]). + +%%%----------------------------------------------------------------- +%%% This module implements a process that handles reference counters for +%%% various erts or other kernel resources which needs reference counting. +%%% +%%% Should not be documented nor used directly by user applications. +%%%----------------------------------------------------------------- +start_link() -> + gen_server:start_link({local,kernel_refc}, kernel_refc, [], []). + +-spec scheduler_wall_time(boolean()) -> boolean(). +scheduler_wall_time(Bool) -> + gen_server:call(kernel_refc, {scheduler_wall_time, self(), Bool}, infinity). + +%%----------------------------------------------------------------- +%% Callback functions from gen_server +%%----------------------------------------------------------------- + +-spec init([]) -> {'ok', map()} | {'stop', term()}. + +init([]) -> + resource(scheduler_wall_time, false), + {ok, #{scheduler_wall_time=>#{}}}. + +-spec handle_call(term(), term(), State) -> {'reply', term(), State}. +handle_call({What, Who, false}, _From, State) -> + {Reply, Cnt} = do_stop(What, maps:get(What, State), Who), + {reply, Reply, State#{What:=Cnt}}; +handle_call({What, Who, true}, _From, State) -> + {Reply, Cnt} = do_start(What, maps:get(What, State), Who), + {reply, Reply, State#{What:=Cnt}}; +handle_call(_, _From, State) -> + {reply, badarg, State}. + +-spec handle_cast(term(), State) -> {'noreply', State}. +handle_cast(_, State) -> + {noreply, State}. + +-spec handle_info(term(), State) -> {'noreply', State}. +handle_info({'DOWN', _Ref, process, Pid, _}, State) -> + Cleanup = fun(Resource, Cnts) -> + cleanup(Resource, Cnts, Pid) + end, + {noreply, maps:map(Cleanup, State)}; +handle_info(_, State) -> + {noreply, State}. + +-spec terminate(term(), term()) -> 'ok'. +terminate(_Reason, _State) -> + ok. + +-spec code_change(term(), State, term()) -> {'ok', State}. +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%----------------------------------------------------------------- +%% Internal functions +%%----------------------------------------------------------------- + +do_start(Resource, Cnt, Pid) -> + case maps:get(Pid, Cnt, undefined) of + undefined -> + Ref = erlang:monitor(process, Pid), + case any(Cnt) of + true -> + {true, Cnt#{Pid=>{1, Ref}}}; + false -> + resource(Resource, true), + {false, Cnt#{Pid=>{1, Ref}}} + end; + {N, Ref} -> + {true, Cnt#{Pid=>{N+1, Ref}}} + end. + +do_stop(Resource, Cnt0, Pid) -> + case maps:get(Pid, Cnt0, undefined) of + undefined -> + {any(Cnt0), Cnt0}; + {1, Ref} -> + erlang:demonitor(Ref, [flush]), + Cnt = maps:remove(Pid, Cnt0), + case any(Cnt) of + true -> + {true, Cnt}; + false -> + resource(Resource, false), + {true, Cnt} + end; + {N, Ref} -> + {true, Cnt0#{Pid=>{N-1, Ref}}} + end. + +cleanup(Resource, Cnt0, Pid) -> + case maps:is_key(Pid, Cnt0) of + true -> + Cnt = maps:remove(Pid, Cnt0), + case any(Cnt) of + true -> + Cnt; + false -> + resource(Resource, false), + Cnt + end; + false -> + Cnt0 + end. + +any(Cnt) -> maps:size(Cnt) > 0. + +resource(scheduler_wall_time, Enable) -> + _ = erts_internal:scheduler_wall_time(Enable). diff --git a/lib/kernel/src/raw_file_io.erl b/lib/kernel/src/raw_file_io.erl new file mode 100644 index 0000000000..e3c07c8f78 --- /dev/null +++ b/lib/kernel/src/raw_file_io.erl @@ -0,0 +1,75 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(raw_file_io). + +-export([open/2]). + +open(Filename, Modes) -> + %% Layers are applied in this order, and the listed modules will call this + %% function again as necessary. eg. a raw compressed delayed file in list + %% mode will walk through [_list -> _compressed -> _delayed -> _raw]. + ModuleOrder = [{raw_file_io_list, fun match_list/1}, + {raw_file_io_compressed, fun match_compressed/1}, + {raw_file_io_delayed, fun match_delayed/1}, + {raw_file_io_raw, fun match_raw/1}], + open_1(ModuleOrder, Filename, add_implicit_modes(Modes)). +open_1([], _Filename, _Modes) -> + error(badarg); +open_1([{Module, Match} | Rest], Filename, Modes) -> + case lists:any(Match, Modes) of + true -> + {Options, ChildModes} = + lists:partition(fun(Mode) -> Match(Mode) end, Modes), + Module:open_layer(Filename, ChildModes, Options); + false -> + open_1(Rest, Filename, Modes) + end. + +%% 'read' and 'list' mode are enabled unless disabled by another option, so +%% we'll explicitly add them to avoid duplicating this logic in child layers. +add_implicit_modes(Modes0) -> + Modes1 = add_unless_matched(Modes0, fun match_writable/1, read), + add_unless_matched(Modes1, fun match_binary/1, list). +add_unless_matched(Modes, Match, Default) -> + case lists:any(Match, Modes) of + false -> [Default | Modes]; + true -> Modes + end. + +match_list(list) -> true; +match_list(_Other) -> false. + +match_compressed(compressed) -> true; +match_compressed(_Other) -> false. + +match_delayed({delayed_write, _Size, _Timeout}) -> true; +match_delayed(delayed_write) -> true; +match_delayed(_Other) -> false. + +match_raw(raw) -> true; +match_raw(_Other) -> false. + +match_writable(write) -> true; +match_writable(append) -> true; +match_writable(exclusive) -> true; +match_writable(_Other) -> false. + +match_binary(binary) -> true; +match_binary(_Other) -> false. diff --git a/lib/kernel/src/raw_file_io_compressed.erl b/lib/kernel/src/raw_file_io_compressed.erl new file mode 100644 index 0000000000..d5ab042d25 --- /dev/null +++ b/lib/kernel/src/raw_file_io_compressed.erl @@ -0,0 +1,134 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(raw_file_io_compressed). + +-export([close/1, sync/1, datasync/1, truncate/1, advise/4, allocate/3, + position/2, write/2, pwrite/2, pwrite/3, + read_line/1, read/2, pread/2, pread/3]). + +%% OTP internal. +-export([ipread_s32bu_p32bu/3, sendfile/8]). + +-export([open_layer/3]). + +-include("file_int.hrl"). + +open_layer(Filename, Modes, Options) -> + IsAppend = lists:member(append, Modes), + IsDeflate = lists:member(write, Modes), + IsInflate = lists:member(read, Modes), + if + IsDeflate, IsInflate; IsAppend -> + {error, einval}; + IsDeflate, not IsInflate -> + start_server_module(raw_file_io_deflate, Filename, Modes, Options); + IsInflate -> + start_server_module(raw_file_io_inflate, Filename, Modes, Options) + end. + +start_server_module(Module, Filename, Modes, Options) -> + Secret = make_ref(), + case gen_statem:start(Module, {self(), Secret, Options}, []) of + {ok, Pid} -> open_next_layer(Pid, Secret, Filename, Modes); + Other -> Other + end. + +open_next_layer(Pid, Secret, Filename, Modes) -> + case gen_statem:call(Pid, {'$open', Secret, Filename, Modes}, infinity) of + ok -> + PublicFd = #file_descriptor{ + module = raw_file_io_compressed, data = {self(), Pid} }, + {ok, PublicFd}; + Other -> Other + end. + +close(Fd) -> + wrap_call(Fd, [close]). + +sync(Fd) -> + wrap_call(Fd, [sync]). +datasync(Fd) -> + wrap_call(Fd, [datasync]). + +truncate(Fd) -> + wrap_call(Fd, [truncate]). + +advise(Fd, Offset, Length, Advise) -> + wrap_call(Fd, [advise, Offset, Length, Advise]). +allocate(Fd, Offset, Length) -> + wrap_call(Fd, [allocate, Offset, Length]). + +position(Fd, Mark) -> + wrap_call(Fd, [position, Mark]). + +write(Fd, IOData) -> + try + CompactedData = erlang:iolist_to_iovec(IOData), + wrap_call(Fd, [write, CompactedData]) + catch + error:badarg -> {error, badarg} + end. + +pwrite(Fd, Offset, IOData) -> + try + CompactedData = erlang:iolist_to_iovec(IOData), + wrap_call(Fd, [pwrite, Offset, CompactedData]) + catch + error:badarg -> {error, badarg} + end. +pwrite(Fd, LocBytes) -> + try + CompactedLocBytes = + [ {Offset, erlang:iolist_to_iovec(IOData)} || + {Offset, IOData} <- LocBytes ], + wrap_call(Fd, [pwrite, CompactedLocBytes]) + catch + error:badarg -> {error, badarg} + end. + +read_line(Fd) -> + wrap_call(Fd, [read_line]). +read(Fd, Size) -> + wrap_call(Fd, [read, Size]). +pread(Fd, Offset, Size) -> + wrap_call(Fd, [pread, Offset, Size]). +pread(Fd, LocNums) -> + wrap_call(Fd, [pread, LocNums]). + +ipread_s32bu_p32bu(Fd, Offset, MaxSize) -> + wrap_call(Fd, [ipread_s32bu_p32bu, Offset, MaxSize]). + +sendfile(_,_,_,_,_,_,_,_) -> + {error, enotsup}. + +wrap_call(Fd, Command) -> + {_Owner, Pid} = get_fd_data(Fd), + try gen_statem:call(Pid, Command, infinity) of + Result -> Result + catch + exit:{noproc, _StackTrace} -> {error, einval} + end. + +get_fd_data(#file_descriptor{ data = Data }) -> + {Owner, _ServerPid} = Data, + case self() of + Owner -> Data; + _ -> error(not_on_controlling_process) + end. diff --git a/lib/kernel/src/raw_file_io_deflate.erl b/lib/kernel/src/raw_file_io_deflate.erl new file mode 100644 index 0000000000..acfc546743 --- /dev/null +++ b/lib/kernel/src/raw_file_io_deflate.erl @@ -0,0 +1,159 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(raw_file_io_deflate). + +-behavior(gen_statem). + +-export([init/1, callback_mode/0, terminate/3]). +-export([opening/3, opened/3]). + +-include("file_int.hrl"). + +-define(GZIP_WBITS, 16 + 15). + +callback_mode() -> state_functions. + +init({Owner, Secret, [compressed]}) -> + Monitor = monitor(process, Owner), + Z = zlib:open(), + ok = zlib:deflateInit(Z, default, deflated, ?GZIP_WBITS, 8, default), + Data = + #{ owner => Owner, + monitor => Monitor, + secret => Secret, + position => 0, + zlib => Z }, + {ok, opening, Data}. + +opening({call, From}, {'$open', Secret, Filename, Modes}, #{ secret := Secret } = Data) -> + case raw_file_io:open(Filename, Modes) of + {ok, PrivateFd} -> + NewData = Data#{ handle => PrivateFd }, + {next_state, opened, NewData, [{reply, From, ok}]}; + Other -> + {stop_and_reply, normal, [{reply, From, Other}]} + end; +opening(_Event, _Contents, _Data) -> + {keep_state_and_data, [postpone]}. + +%% + +opened(info, {'DOWN', Monitor, process, _Owner, Reason}, #{ monitor := Monitor } = Data) -> + if + Reason =/= kill -> flush_deflate_state(Data); + Reason =:= kill -> ignored + end, + {stop, shutdown}; + +opened(info, _Message, _Data) -> + keep_state_and_data; + +opened({call, {Owner, _Tag} = From}, [close], #{ owner := Owner } = Data) -> + #{ handle := PrivateFd } = Data, + Response = + case flush_deflate_state(Data) of + ok -> ?CALL_FD(PrivateFd, close, []); + Other -> Other + end, + {stop_and_reply, normal, [{reply, From, Response}]}; + +opened({call, {Owner, _Tag} = From}, [position, Mark], #{ owner := Owner } = Data) -> + case position(Data, Mark) of + {ok, NewData, Result} -> + Response = {ok, Result}, + {keep_state, NewData, [{reply, From, Response}]}; + Other -> + {keep_state_and_data, [{reply, From, Other}]} + end; + +opened({call, {Owner, _Tag} = From}, [write, IOVec], #{ owner := Owner } = Data) -> + case write(Data, IOVec) of + {ok, NewData} -> {keep_state, NewData, [{reply, From, ok}]}; + Other -> {keep_state_and_data, [{reply, From, Other}]} + end; + +opened({call, {Owner, _Tag} = From}, [read, _Size], #{ owner := Owner }) -> + Response = {error, ebadf}, + {keep_state_and_data, [{reply, From, Response}]}; + +opened({call, {Owner, _Tag} = From}, [read_line], #{ owner := Owner }) -> + Response = {error, ebadf}, + {keep_state_and_data, [{reply, From, Response}]}; + +opened({call, {Owner, _Tag} = From}, _Command, #{ owner := Owner }) -> + Response = {error, enotsup}, + {keep_state_and_data, [{reply, From, Response}]}; + +opened({call, _From}, _Command, _Data) -> + %% The client functions filter this out, so we'll crash if the user does + %% anything stupid on purpose. + {shutdown, protocol_violation}; + +opened(_Event, _Request, _Data) -> + keep_state_and_data. + +write(Data, IOVec) -> + #{ handle := PrivateFd, position := Position, zlib := Z } = Data, + UncompressedSize = iolist_size(IOVec), + case ?CALL_FD(PrivateFd, write, [zlib:deflate(Z, IOVec)]) of + ok -> {ok, Data#{ position := (Position + UncompressedSize) }}; + Other -> Other + end. + +%% +%% We support "seeking" forward as long as it isn't relative to EOF. +%% +%% Seeking is a bit of a misnomer as it's really just compressing zeroes until +%% we reach the desired point, but it has always behaved like this. +%% + +position(Data, Mark) when is_atom(Mark) -> + position(Data, {Mark, 0}); +position(Data, Offset) when is_integer(Offset) -> + position(Data, {bof, Offset}); +position(Data, {bof, Offset}) when is_integer(Offset) -> + position_1(Data, Offset); +position(Data, {cur, Offset}) when is_integer(Offset) -> + #{ position := Position } = Data, + position_1(Data, Position + Offset); +position(_Data, {eof, Offset}) when is_integer(Offset) -> + {error, einval}; +position(_Data, _Any) -> + {error, badarg}. + +position_1(#{ position := Desired } = Data, Desired) -> + {ok, Data, Desired}; +position_1(#{ position := Current } = Data, Desired) when Current < Desired -> + BytesToWrite = min(Desired - Current, 4 bsl 20), + case write(Data, <<0:(BytesToWrite)/unit:8>>) of + {ok, NewData} -> position_1(NewData, Desired); + Other -> Other + end; +position_1(#{ position := Current }, Desired) when Current > Desired -> + {error, einval}. + +flush_deflate_state(#{ handle := PrivateFd, zlib := Z }) -> + case ?CALL_FD(PrivateFd, write, [zlib:deflate(Z, [], finish)]) of + ok -> ok; + Other -> Other + end. + +terminate(_Reason, _State, _Data) -> + ok. diff --git a/lib/kernel/src/raw_file_io_delayed.erl b/lib/kernel/src/raw_file_io_delayed.erl new file mode 100644 index 0000000000..d2ad7550a1 --- /dev/null +++ b/lib/kernel/src/raw_file_io_delayed.erl @@ -0,0 +1,320 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(raw_file_io_delayed). + +-behavior(gen_statem). + +-export([close/1, sync/1, datasync/1, truncate/1, advise/4, allocate/3, + position/2, write/2, pwrite/2, pwrite/3, + read_line/1, read/2, pread/2, pread/3]). + +%% OTP internal. +-export([ipread_s32bu_p32bu/3, sendfile/8]). + +-export([open_layer/3]). + +-export([init/1, callback_mode/0, terminate/3]). +-export([opening/3, opened/3]). + +-include("file_int.hrl"). + +open_layer(Filename, Modes, Options) -> + Secret = make_ref(), + case gen_statem:start(?MODULE, {self(), Secret, Options}, []) of + {ok, Pid} -> + gen_statem:call(Pid, {'$open', Secret, Filename, Modes}, infinity); + Other -> + Other + end. + +callback_mode() -> state_functions. + +init({Owner, Secret, Options}) -> + Monitor = monitor(process, Owner), + Defaults = + #{ owner => Owner, + monitor => Monitor, + secret => Secret, + timer => none, + pid => self(), + buffer => prim_buffer:new(), + delay_size => 64 bsl 10, + delay_time => 2000 }, + Data = fill_delay_values(Defaults, Options), + {ok, opening, Data}. + +fill_delay_values(Data, []) -> + Data; +fill_delay_values(Data, [{delayed_write, Size, Time} | Options]) -> + fill_delay_values(Data#{ delay_size => Size, delay_time => Time }, Options); +fill_delay_values(Data, [_ | Options]) -> + fill_delay_values(Data, Options). + +opening({call, From}, {'$open', Secret, Filename, Modes}, #{ secret := Secret } = Data) -> + case raw_file_io:open(Filename, Modes) of + {ok, PrivateFd} -> + PublicData = maps:with([owner, buffer, delay_size, pid], Data), + PublicFd = #file_descriptor{ module = ?MODULE, data = PublicData }, + + NewData = Data#{ handle => PrivateFd }, + Response = {ok, PublicFd}, + {next_state, opened, NewData, [{reply, From, Response}]}; + Other -> + {stop_and_reply, normal, [{reply, From, Other}]} + end; +opening(_Event, _Contents, _Data) -> + {keep_state_and_data, [postpone]}. + +%% + +opened(info, {'$timed_out', Secret}, #{ secret := Secret } = Data) -> + %% If the user writes something at this exact moment, the flush will fail + %% and the timer won't reset on the next write since the buffer won't be + %% empty (Unless we collided on a flush). We therefore reset the timeout to + %% ensure that data won't sit idle for extended periods of time. + case try_flush_write_buffer(Data) of + busy -> gen_statem:cast(self(), '$reset_timeout'); + ok -> ok + end, + {keep_state, Data#{ timer => none }, []}; + +opened(info, {'DOWN', Monitor, process, _Owner, Reason}, #{ monitor := Monitor } = Data) -> + if + Reason =/= kill -> try_flush_write_buffer(Data); + Reason =:= kill -> ignored + end, + {stop, shutdown}; + +opened(info, _Message, _Data) -> + keep_state_and_data; + +opened({call, {Owner, _Tag} = From}, [close], #{ owner := Owner } = Data) -> + case flush_write_buffer(Data) of + ok -> + #{ handle := PrivateFd } = Data, + Response = ?CALL_FD(PrivateFd, close, []), + {stop_and_reply, normal, [{reply, From, Response}]}; + Other -> + {stop_and_reply, normal, [{reply, From, Other}]} + end; + +opened({call, {Owner, _Tag} = From}, '$wait', #{ owner := Owner }) -> + %% Used in write/2 to synchronize writes on lock conflicts. + {keep_state_and_data, [{reply, From, ok}]}; + +opened({call, {Owner, _Tag} = From}, '$synchronous_flush', #{ owner := Owner } = Data) -> + cancel_flush_timeout(Data), + Response = flush_write_buffer(Data), + {keep_state_and_data, [{reply, From, Response}]}; + +opened({call, {Owner, _Tag} = From}, Command, #{ owner := Owner } = Data) -> + Response = + case flush_write_buffer(Data) of + ok -> dispatch_command(Data, Command); + Other -> Other + end, + {keep_state_and_data, [{reply, From, Response}]}; + +opened({call, _From}, _Command, _Data) -> + %% The client functions filter this out, so we'll crash if the user does + %% anything stupid on purpose. + {shutdown, protocol_violation}; + +opened(cast, '$reset_timeout', #{ delay_time := Timeout, secret := Secret } = Data) -> + cancel_flush_timeout(Data), + Timer = erlang:send_after(Timeout, self(), {'$timed_out', Secret}), + {keep_state, Data#{ timer => Timer }, []}; + +opened(cast, _Message, _Data) -> + {keep_state_and_data, []}. + +dispatch_command(Data, [Function | Args]) -> + #{ handle := Handle } = Data, + Module = Handle#file_descriptor.module, + apply(Module, Function, [Handle | Args]). + +cancel_flush_timeout(#{ timer := none }) -> + ok; +cancel_flush_timeout(#{ timer := Timer }) -> + _ = erlang:cancel_timer(Timer, [{async, true}]), + ok. + +try_flush_write_buffer(#{ buffer := Buffer, handle := PrivateFd }) -> + case prim_buffer:try_lock(Buffer) of + acquired -> + flush_write_buffer_1(Buffer, PrivateFd), + prim_buffer:unlock(Buffer), + ok; + busy -> + busy + end. + +%% This is only safe to use when there is no chance of conflict with the owner +%% process, or in other words, "during synchronous calls outside of the locked +%% section of write/2" +flush_write_buffer(#{ buffer := Buffer, handle := PrivateFd }) -> + acquired = prim_buffer:try_lock(Buffer), + Result = flush_write_buffer_1(Buffer, PrivateFd), + prim_buffer:unlock(Buffer), + Result. + +flush_write_buffer_1(Buffer, PrivateFd) -> + case prim_buffer:size(Buffer) of + Size when Size > 0 -> + ?CALL_FD(PrivateFd, write, [prim_buffer:read_iovec(Buffer, Size)]); + 0 -> + ok + end. + +terminate(_Reason, _State, _Data) -> + ok. + +%% Client functions + +write(Fd, IOData) -> + try + enqueue_write(Fd, erlang:iolist_to_iovec(IOData)) + catch + error:badarg -> {error, badarg} + end. +enqueue_write(_Fd, []) -> + ok; +enqueue_write(Fd, IOVec) -> + %% get_fd_data will reject everyone except the process that opened the Fd, + %% so we can't race with anyone except the wrapper process. + #{ delay_size := DelaySize, + buffer := Buffer, + pid := Pid } = get_fd_data(Fd), + case prim_buffer:try_lock(Buffer) of + acquired -> + %% (The wrapper process will exit without flushing if we're killed + %% while holding the lock). + enqueue_write_locked(Pid, Buffer, DelaySize, IOVec); + busy -> + %% This can only happen while we're processing a timeout in the + %% wrapper process, so we perform a bogus call to get a completion + %% notification before trying again. + gen_statem:call(Pid, '$wait'), + enqueue_write(Fd, IOVec) + end. +enqueue_write_locked(Pid, Buffer, DelaySize, IOVec) -> + %% The synchronous operations (write, forced flush) are safe since we're + %% running on the only process that can fill the buffer; a timeout being + %% processed just before $synchronous_flush will cause the flush to nop, + %% and a timeout sneaking in just before a synchronous write won't do + %% anything since the buffer is guaranteed to be empty at that point. + BufSize = prim_buffer:size(Buffer), + case is_iovec_smaller_than(IOVec, DelaySize - BufSize) of + true when BufSize > 0 -> + prim_buffer:write(Buffer, IOVec), + prim_buffer:unlock(Buffer); + true -> + prim_buffer:write(Buffer, IOVec), + prim_buffer:unlock(Buffer), + gen_statem:cast(Pid, '$reset_timeout'); + false when BufSize > 0 -> + prim_buffer:write(Buffer, IOVec), + prim_buffer:unlock(Buffer), + gen_statem:call(Pid, '$synchronous_flush'); + false -> + prim_buffer:unlock(Buffer), + gen_statem:call(Pid, [write, IOVec]) + end. + +%% iolist_size/1 will always look through the entire list to get a precise +%% amount, which is pretty inefficient since we only need to know whether we've +%% hit the buffer threshold or not. +%% +%% We only handle the binary case since write/2 forcibly translates input to +%% erlang:iovec(). +is_iovec_smaller_than(IOVec, Max) -> + is_iovec_smaller_than_1(IOVec, Max, 0). +is_iovec_smaller_than_1(_IOVec, Max, Acc) when Acc >= Max -> + false; +is_iovec_smaller_than_1([], _Max, _Acc) -> + true; +is_iovec_smaller_than_1([Binary | Rest], Max, Acc) when is_binary(Binary) -> + is_iovec_smaller_than_1(Rest, Max, Acc + byte_size(Binary)). + +close(Fd) -> + wrap_call(Fd, [close]). + +sync(Fd) -> + wrap_call(Fd, [sync]). +datasync(Fd) -> + wrap_call(Fd, [datasync]). + +truncate(Fd) -> + wrap_call(Fd, [truncate]). + +advise(Fd, Offset, Length, Advise) -> + wrap_call(Fd, [advise, Offset, Length, Advise]). +allocate(Fd, Offset, Length) -> + wrap_call(Fd, [allocate, Offset, Length]). + +position(Fd, Mark) -> + wrap_call(Fd, [position, Mark]). + +pwrite(Fd, Offset, IOData) -> + try + CompactedData = erlang:iolist_to_iovec(IOData), + wrap_call(Fd, [pwrite, Offset, CompactedData]) + catch + error:badarg -> {error, badarg} + end. +pwrite(Fd, LocBytes) -> + try + CompactedLocBytes = + [ {Offset, erlang:iolist_to_iovec(IOData)} || + {Offset, IOData} <- LocBytes ], + wrap_call(Fd, [pwrite, CompactedLocBytes]) + catch + error:badarg -> {error, badarg} + end. + +read_line(Fd) -> + wrap_call(Fd, [read_line]). +read(Fd, Size) -> + wrap_call(Fd, [read, Size]). +pread(Fd, Offset, Size) -> + wrap_call(Fd, [pread, Offset, Size]). +pread(Fd, LocNums) -> + wrap_call(Fd, [pread, LocNums]). + +ipread_s32bu_p32bu(Fd, Offset, MaxSize) -> + wrap_call(Fd, [ipread_s32bu_p32bu, Offset, MaxSize]). + +sendfile(_,_,_,_,_,_,_,_) -> + {error, enotsup}. + +wrap_call(Fd, Command) -> + #{ pid := Pid } = get_fd_data(Fd), + try gen_statem:call(Pid, Command, infinity) of + Result -> Result + catch + exit:{noproc, _StackTrace} -> {error, einval} + end. + +get_fd_data(#file_descriptor{ data = Data }) -> + #{ owner := Owner } = Data, + case self() of + Owner -> Data; + _ -> error(not_on_controlling_process) + end. diff --git a/lib/kernel/src/raw_file_io_inflate.erl b/lib/kernel/src/raw_file_io_inflate.erl new file mode 100644 index 0000000000..7e9780310c --- /dev/null +++ b/lib/kernel/src/raw_file_io_inflate.erl @@ -0,0 +1,261 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(raw_file_io_inflate). + +-behavior(gen_statem). + +-export([init/1, callback_mode/0, terminate/3]). +-export([opening/3, opened_gzip/3, opened_passthrough/3]). + +-include("file_int.hrl"). + +-define(INFLATE_CHUNK_SIZE, (1 bsl 10)). +-define(GZIP_WBITS, (16 + 15)). + +callback_mode() -> state_functions. + +init({Owner, Secret, [compressed]}) -> + Monitor = monitor(process, Owner), + %% We're using the undocumented inflateInit/3 to open the stream in + %% 'reset mode', which resets the inflate state at the end of every stream, + %% allowing us to read concatenated gzip files. + Z = zlib:open(), + ok = zlib:inflateInit(Z, ?GZIP_WBITS, reset), + Data = + #{ owner => Owner, + monitor => Monitor, + secret => Secret, + position => 0, + buffer => prim_buffer:new(), + zlib => Z }, + {ok, opening, Data}. + +%% The old driver fell back to plain reads if the file didn't start with the +%% magic gzip bytes. +choose_decompression_state(PrivateFd) -> + State = + case ?CALL_FD(PrivateFd, read, [2]) of + {ok, <<16#1F, 16#8B>>} -> opened_gzip; + _Other -> opened_passthrough + end, + {ok, 0} = ?CALL_FD(PrivateFd, position, [0]), + State. + +opening({call, From}, {'$open', Secret, Filename, Modes}, #{ secret := Secret } = Data) -> + case raw_file_io:open(Filename, Modes) of + {ok, PrivateFd} -> + NextState = choose_decompression_state(PrivateFd), + NewData = Data#{ handle => PrivateFd }, + {next_state, NextState, NewData, [{reply, From, ok}]}; + Other -> + {stop_and_reply, normal, [{reply, From, Other}]} + end; +opening(_Event, _Contents, _Data) -> + {keep_state_and_data, [postpone]}. + +internal_close(From, Data) -> + #{ handle := PrivateFd } = Data, + Response = ?CALL_FD(PrivateFd, close, []), + {stop_and_reply, normal, [{reply, From, Response}]}. + +opened_passthrough(info, {'DOWN', Monitor, process, _Owner, _Reason}, #{ monitor := Monitor }) -> + {stop, shutdown}; + +opened_passthrough(info, _Message, _Data) -> + keep_state_and_data; + +opened_passthrough({call, {Owner, _Tag} = From}, [close], #{ owner := Owner } = Data) -> + internal_close(From, Data); + +opened_passthrough({call, {Owner, _Tag} = From}, [Method | Args], #{ owner := Owner } = Data) -> + #{ handle := PrivateFd } = Data, + Response = ?CALL_FD(PrivateFd, Method, Args), + {keep_state_and_data, [{reply, From, Response}]}; + +opened_passthrough({call, _From}, _Command, _Data) -> + %% The client functions filter this out, so we'll crash if the user does + %% anything stupid on purpose. + {shutdown, protocol_violation}; + +opened_passthrough(_Event, _Request, _Data) -> + keep_state_and_data. + +%% + +opened_gzip(info, {'DOWN', Monitor, process, _Owner, _Reason}, #{ monitor := Monitor }) -> + {stop, shutdown}; + +opened_gzip(info, _Message, _Data) -> + keep_state_and_data; + +opened_gzip({call, {Owner, _Tag} = From}, [close], #{ owner := Owner } = Data) -> + internal_close(From, Data); + +opened_gzip({call, {Owner, _Tag} = From}, [position, Mark], #{ owner := Owner } = Data) -> + case position(Data, Mark) of + {ok, NewData, Result} -> + Response = {ok, Result}, + {keep_state, NewData, [{reply, From, Response}]}; + Other -> + {keep_state_and_data, [{reply, From, Other}]} + end; + +opened_gzip({call, {Owner, _Tag} = From}, [read, Size], #{ owner := Owner } = Data) -> + case read(Data, Size) of + {ok, NewData, Result} -> + Response = {ok, Result}, + {keep_state, NewData, [{reply, From, Response}]}; + Other -> + {keep_state_and_data, [{reply, From, Other}]} + end; + +opened_gzip({call, {Owner, _Tag} = From}, [read_line], #{ owner := Owner } = Data) -> + case read_line(Data) of + {ok, NewData, Result} -> + Response = {ok, Result}, + {keep_state, NewData, [{reply, From, Response}]}; + Other -> + {keep_state_and_data, [{reply, From, Other}]} + end; + +opened_gzip({call, {Owner, _Tag} = From}, [write, _IOData], #{ owner := Owner }) -> + Response = {error, ebadf}, + {keep_state_and_data, [{reply, From, Response}]}; + +opened_gzip({call, {Owner, _Tag} = From}, _Request, #{ owner := Owner }) -> + Response = {error, enotsup}, + {keep_state_and_data, [{reply, From, Response}]}; + +opened_gzip({call, _From}, _Request, _Data) -> + %% The client functions filter this out, so we'll crash if the user does + %% anything stupid on purpose. + {shutdown, protocol_violation}; + +opened_gzip(_Event, _Request, _Data) -> + keep_state_and_data. + +%% + +read(#{ buffer := Buffer } = Data, Size) -> + try read_1(Data, Buffer, prim_buffer:size(Buffer), Size) of + Result -> Result + catch + error:badarg -> {error, badarg}; + error:_ -> {error, eio} + end. +read_1(Data, Buffer, BufferSize, ReadSize) when BufferSize >= ReadSize -> + #{ position := Position } = Data, + Decompressed = prim_buffer:read(Buffer, ReadSize), + {ok, Data#{ position => (Position + ReadSize) }, Decompressed}; +read_1(Data, Buffer, BufferSize, ReadSize) when BufferSize < ReadSize -> + #{ handle := PrivateFd } = Data, + case ?CALL_FD(PrivateFd, read, [?INFLATE_CHUNK_SIZE]) of + {ok, Compressed} -> + #{ zlib := Z } = Data, + Uncompressed = erlang:iolist_to_iovec(zlib:inflate(Z, Compressed)), + prim_buffer:write(Buffer, Uncompressed), + read_1(Data, Buffer, prim_buffer:size(Buffer), ReadSize); + eof when BufferSize > 0 -> + read_1(Data, Buffer, BufferSize, BufferSize); + Other -> + Other + end. + +read_line(#{ buffer := Buffer } = Data) -> + try read_line_1(Data, Buffer, prim_buffer:find_byte_index(Buffer, $\n)) of + {ok, NewData, Decompressed} -> {ok, NewData, Decompressed}; + Other -> Other + catch + error:badarg -> {error, badarg}; + error:_ -> {error, eio} + end. + +read_line_1(Data, Buffer, not_found) -> + #{ handle := PrivateFd, zlib := Z } = Data, + case ?CALL_FD(PrivateFd, read, [?INFLATE_CHUNK_SIZE]) of + {ok, Compressed} -> + Uncompressed = erlang:iolist_to_iovec(zlib:inflate(Z, Compressed)), + prim_buffer:write(Buffer, Uncompressed), + read_line_1(Data, Buffer, prim_buffer:find_byte_index(Buffer, $\n)); + eof -> + case prim_buffer:size(Buffer) of + Size when Size > 0 -> {ok, prim_buffer:read(Buffer, Size)}; + Size when Size =:= 0 -> eof + end; + Error -> + Error + end; +read_line_1(Data, Buffer, {ok, LFIndex}) -> + %% Translate CRLF into just LF, completely ignoring which encoding is used, + %% but treat the file position as including CR. + #{ position := Position } = Data, + NewData = Data#{ position => (Position + LFIndex + 1) }, + CRIndex = (LFIndex - 1), + TranslatedLine = + case prim_buffer:read(Buffer, LFIndex + 1) of + <<Line:CRIndex/binary, "\r\n">> -> <<Line/binary, "\n">>; + Line -> Line + end, + {ok, NewData, TranslatedLine}. + +%% +%% We support seeking in both directions as long as it isn't relative to EOF. +%% +%% Seeking backwards is extremely inefficient since we have to seek to the very +%% beginning and then decompress up to the desired point. +%% + +position(Data, Mark) when is_atom(Mark) -> + position(Data, {Mark, 0}); +position(Data, Offset) when is_integer(Offset) -> + position(Data, {bof, Offset}); +position(Data, {bof, Offset}) when is_integer(Offset) -> + position_1(Data, Offset); +position(Data, {cur, Offset}) when is_integer(Offset) -> + #{ position := Position } = Data, + position_1(Data, Position + Offset); +position(_Data, {eof, Offset}) when is_integer(Offset) -> + {error, einval}; +position(_Data, _Other) -> + {error, badarg}. + +position_1(_Data, Desired) when Desired < 0 -> + {error, einval}; +position_1(#{ position := Desired } = Data, Desired) -> + {ok, Data, Desired}; +position_1(#{ position := Current } = Data, Desired) when Current < Desired -> + case read(Data, min(Desired - Current, ?INFLATE_CHUNK_SIZE)) of + {ok, NewData, _Data} -> position_1(NewData, Desired); + eof -> {ok, Data, Current}; + Other -> Other + end; +position_1(#{ position := Current } = Data, Desired) when Current > Desired -> + #{ handle := PrivateFd, buffer := Buffer, zlib := Z } = Data, + case ?CALL_FD(PrivateFd, position, [bof]) of + {ok, 0} -> + ok = zlib:inflateReset(Z), + prim_buffer:wipe(Buffer), + position_1(Data#{ position => 0 }, Desired); + Other -> + Other + end. + +terminate(_Reason, _State, _Data) -> + ok. diff --git a/lib/kernel/src/raw_file_io_list.erl b/lib/kernel/src/raw_file_io_list.erl new file mode 100644 index 0000000000..2e16e63f0e --- /dev/null +++ b/lib/kernel/src/raw_file_io_list.erl @@ -0,0 +1,128 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(raw_file_io_list). + +-export([close/1, sync/1, datasync/1, truncate/1, advise/4, allocate/3, + position/2, write/2, pwrite/2, pwrite/3, + read_line/1, read/2, pread/2, pread/3]). + +%% OTP internal. +-export([ipread_s32bu_p32bu/3, sendfile/8]). + +-export([open_layer/3]). + +-include("file_int.hrl"). + +open_layer(Filename, Modes, [list]) -> + case raw_file_io:open(Filename, [binary | Modes]) of + {ok, PrivateFd} -> {ok, make_public_fd(PrivateFd, Modes)}; + Other -> Other + end. + +%% We can skip wrapping the file if it's write-only since only read operations +%% are affected by list mode. Since raw_file_io fills in all implicit options +%% for us, all we need to do is check whether 'read' is among them. +make_public_fd(PrivateFd, Modes) -> + case lists:member(read, Modes) of + true -> #file_descriptor{ module = ?MODULE, data = PrivateFd }; + false -> PrivateFd + end. + +close(Fd) -> + PrivateFd = Fd#file_descriptor.data, + ?CALL_FD(PrivateFd, close, []). + +sync(Fd) -> + PrivateFd = Fd#file_descriptor.data, + ?CALL_FD(PrivateFd, sync, []). +datasync(Fd) -> + PrivateFd = Fd#file_descriptor.data, + ?CALL_FD(PrivateFd, datasync, []). + +truncate(Fd) -> + PrivateFd = Fd#file_descriptor.data, + ?CALL_FD(PrivateFd, truncate, []). + +advise(Fd, Offset, Length, Advise) -> + PrivateFd = Fd#file_descriptor.data, + ?CALL_FD(PrivateFd, advise, [Offset, Length, Advise]). +allocate(Fd, Offset, Length) -> + PrivateFd = Fd#file_descriptor.data, + ?CALL_FD(PrivateFd, allocate, [Offset, Length]). + +position(Fd, Mark) -> + PrivateFd = Fd#file_descriptor.data, + ?CALL_FD(PrivateFd, position, [Mark]). + +write(Fd, IOData) -> + PrivateFd = Fd#file_descriptor.data, + ?CALL_FD(PrivateFd, write, [IOData]). + +pwrite(Fd, Offset, IOData) -> + PrivateFd = Fd#file_descriptor.data, + ?CALL_FD(PrivateFd, pwrite, [Offset, IOData]). +pwrite(Fd, LocBytes) -> + PrivateFd = Fd#file_descriptor.data, + ?CALL_FD(PrivateFd, pwrite, [LocBytes]). + +read_line(Fd) -> + PrivateFd = Fd#file_descriptor.data, + case ?CALL_FD(PrivateFd, read_line, []) of + {ok, Binary} -> {ok, binary_to_list(Binary)}; + Other -> Other + end. +read(Fd, Size) -> + PrivateFd = Fd#file_descriptor.data, + case ?CALL_FD(PrivateFd, read, [Size]) of + {ok, Binary} -> {ok, binary_to_list(Binary)}; + Other -> Other + end. +pread(Fd, Offset, Size) -> + PrivateFd = Fd#file_descriptor.data, + case ?CALL_FD(PrivateFd, pread, [Offset, Size]) of + {ok, Binary} -> {ok, binary_to_list(Binary)}; + Other -> Other + end. +pread(Fd, LocNums) -> + PrivateFd = Fd#file_descriptor.data, + case ?CALL_FD(PrivateFd, pread, [LocNums]) of + {ok, LocResults} -> + TranslatedResults = + [ case Result of + Result when is_binary(Result) -> binary_to_list(Result); + eof -> eof + end || Result <- LocResults ], + {ok, TranslatedResults}; + Other -> Other + end. + +ipread_s32bu_p32bu(Fd, Offset, MaxSize) -> + PrivateFd = Fd#file_descriptor.data, + case ?CALL_FD(PrivateFd, ipread_s32bu_p32bu, [Offset, MaxSize]) of + {ok, {Size, Pointer, Binary}} when is_binary(Binary) -> + {ok, {Size, Pointer, binary_to_list(Binary)}}; + Other -> + Other + end. + +sendfile(Fd, Dest, Offset, Bytes, ChunkSize, Headers, Trailers, Flags) -> + Args = [Dest, Offset, Bytes, ChunkSize, Headers, Trailers, Flags], + PrivateFd = Fd#file_descriptor.data, + ?CALL_FD(PrivateFd, sendfile, Args). diff --git a/lib/kernel/src/raw_file_io_raw.erl b/lib/kernel/src/raw_file_io_raw.erl new file mode 100644 index 0000000000..9a9fe78eb1 --- /dev/null +++ b/lib/kernel/src/raw_file_io_raw.erl @@ -0,0 +1,25 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(raw_file_io_raw). + +-export([open_layer/3]). + +open_layer(Filename, Modes, [raw]) -> + prim_file:open(Filename, [raw | Modes]). diff --git a/lib/kernel/test/disk_log_SUITE.erl b/lib/kernel/test/disk_log_SUITE.erl index fe2fc778f2..7beaadb1d6 100644 --- a/lib/kernel/test/disk_log_SUITE.erl +++ b/lib/kernel/test/disk_log_SUITE.erl @@ -89,8 +89,6 @@ dist_terminate/1, dist_accessible/1, dist_deadlock/1, dist_open2/1, other_groups/1, - evil/1, - otp_6278/1, otp_10131/1]). -export([head_fun/1, hf/0, lserv/1, @@ -123,7 +121,7 @@ [halt_int, wrap_int, halt_ext, wrap_ext, read_mode, head, notif, new_idx_vsn, reopen, block, unblock, open, close, error, chunk, truncate, many_users, info, change_size, - change_attribute, distribution, evil, otp_6278, otp_10131]). + change_attribute, distribution, otp_6278, otp_10131]). %% These test cases should be skipped if the VxWorks card is %% configured without NFS cache. @@ -149,7 +147,7 @@ all() -> {group, open}, {group, close}, {group, error}, chunk, truncate, many_users, {group, info}, {group, change_size}, change_attribute, - {group, distribution}, evil, otp_6278, otp_10131]. + {group, distribution}, otp_6278, otp_10131]. groups() -> [{halt_int, [], [halt_int_inf, {group, halt_int_sz}]}, @@ -4676,119 +4674,6 @@ other_groups(Conf) when is_list(Conf) -> ok. --define(MAX, ?MAX_FWRITE_CACHE). % as in disk_log_1.erl -%% Evil cases such as closed file descriptor port. -evil(Conf) when is_list(Conf) -> - Dir = ?privdir(Conf), - File = filename:join(Dir, "n.LOG"), - Log = n, - - %% Not a very thorough test. - - ok = setup_evil_filled_cache_wrap(Log, Dir), - {error, {file_error,_,einval}} = disk_log:log(Log, apa), - ok = disk_log:close(Log), - - ok = setup_evil_filled_cache_halt(Log, Dir), - {error, {file_error,_,einval}} = disk_log:truncate(Log, apa), - ok = stop_evil(Log), - - %% White box test. - file:delete(File), - Ports0 = erlang:ports(), - {ok, Log} = disk_log:open([{name,Log},{file,File},{type,halt}, - {size,?MAX+50},{format,external}]), - [Fd] = erlang:ports() -- Ports0, - {B,_} = x_mk_bytes(30), - ok = disk_log:blog(Log, <<0:(?MAX-1)/unit:8>>), - exit(Fd, kill), - {error, {file_error,_,einval}} = disk_log:blog_terms(Log, [B,B]), - ok= disk_log:close(Log), - file:delete(File), - - ok = setup_evil_wrap(Log, Dir), - {error, {file_error,_,einval}} = disk_log:close(Log), - - ok = setup_evil_wrap(Log, Dir), - {error, {file_error,_,einval}} = disk_log:log(Log, apa), - ok = stop_evil(Log), - - ok = setup_evil_halt(Log, Dir), - {error, {file_error,_,einval}} = disk_log:log(Log, apa), - ok = stop_evil(Log), - - ok = setup_evil_wrap(Log, Dir), - {error, {file_error,_,einval}} = disk_log:reopen(Log, apa), - {error, {file_error,_,einval}} = disk_log:reopen(Log, apa), - ok = stop_evil(Log), - - ok = setup_evil_wrap(Log, Dir), - {error, {file_error,_,einval}} = disk_log:reopen(Log, apa), - ok = stop_evil(Log), - - ok = setup_evil_wrap(Log, Dir), - {error, {file_error,_,einval}} = disk_log:inc_wrap_file(Log), - ok = stop_evil(Log), - - ok = setup_evil_wrap(Log, Dir), - {error, {file_error,_,einval}} = disk_log:chunk(Log, start), - ok = stop_evil(Log), - - ok = setup_evil_wrap(Log, Dir), - {error, {file_error,_,einval}} = disk_log:truncate(Log), - ok = stop_evil(Log), - - ok = setup_evil_wrap(Log, Dir), - {error, {file_error,_,einval}} = disk_log:chunk_step(Log, start, 1), - ok = stop_evil(Log), - - io:format("messages: ~p~n", [erlang:process_info(self(), messages)]), - del(File, 2), - file:delete(File), - ok. - -setup_evil_wrap(Log, Dir) -> - setup_evil(Log, [{type,wrap},{size,{100,2}}], Dir). - -setup_evil_halt(Log, Dir) -> - setup_evil(Log, [{type,halt},{size,10000}], Dir). - -setup_evil(Log, Args, Dir) -> - File = filename:join(Dir, lists:concat([Log, ".LOG"])), - file:delete(File), - del(File, 2), - ok = disk_log:start(), - Ports0 = erlang:ports(), - {ok, Log} = disk_log:open([{name,Log},{file,File} | Args]), - [Fd] = erlang:ports() -- Ports0, - exit(Fd, kill), - ok = disk_log:log_terms(n, [<<0:10/unit:8>>]), - timer:sleep(2500), % TIMEOUT in disk_log_1.erl is 2000 - ok. - -stop_evil(Log) -> - {error, _} = disk_log:close(Log), - ok. - -setup_evil_filled_cache_wrap(Log, Dir) -> - setup_evil_filled_cache(Log, [{type,wrap},{size,{?MAX,2}}], Dir). - -setup_evil_filled_cache_halt(Log, Dir) -> - setup_evil_filled_cache(Log, [{type,halt},{size,infinity}], Dir). - -%% The cache is filled, and the file descriptor port gone. -setup_evil_filled_cache(Log, Args, Dir) -> - File = filename:join(Dir, lists:concat([Log, ".LOG"])), - file:delete(File), - del(File, 2), - ok = disk_log:start(), - Ports0 = erlang:ports(), - {ok, Log} = disk_log:open([{name,Log},{file,File} | Args]), - [Fd] = erlang:ports() -- Ports0, - ok = disk_log:log_terms(n, [<<0:?MAX/unit:8>>]), - exit(Fd, kill), - ok. - %% OTP-6278. open/1 creates no status or crash report. otp_6278(Conf) when is_list(Conf) -> Dir = ?privdir(Conf), diff --git a/lib/kernel/test/erl_prim_loader_SUITE.erl b/lib/kernel/test/erl_prim_loader_SUITE.erl index b6417210b9..3502a4ad08 100644 --- a/lib/kernel/test/erl_prim_loader_SUITE.erl +++ b/lib/kernel/test/erl_prim_loader_SUITE.erl @@ -33,6 +33,7 @@ primary_archive/1, virtual_dir_in_archive/1, get_modules/1]). +-define(PRIM_FILE, prim_file). %%----------------------------------------------------------------- %% Test suite for erl_prim_loader. (Most code is run during system start/stop.) @@ -461,7 +462,7 @@ primary_archive(Config) when is_list(Config) -> %% Set primary archive ExpectedEbins = [Archive, DictDir ++ "/ebin", DummyDir ++ "/ebin"], io:format("ExpectedEbins: ~p\n", [ExpectedEbins]), - {ok, FileInfo} = prim_file:read_file_info(Archive), + {ok, FileInfo} = ?PRIM_FILE:read_file_info(Archive), {ok, Ebins} = rpc:call(Node, erl_prim_loader, set_primary_archive, [Archive, ArchiveBin, FileInfo, fun escript:parse_file/1]), diff --git a/lib/kernel/test/file_SUITE.erl b/lib/kernel/test/file_SUITE.erl index 119e1f24bb..0cb8087a76 100644 --- a/lib/kernel/test/file_SUITE.erl +++ b/lib/kernel/test/file_SUITE.erl @@ -39,6 +39,8 @@ -define(FILE_FIN_PER_TESTCASE(Config), Config). -endif. +-define(PRIM_FILE, prim_file). + -module(?FILE_SUITE). -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, @@ -97,6 +99,12 @@ -export([unicode_mode/1]). +-export([volume_relative_paths/1]). + +-export([tiny_writes/1, tiny_writes_delayed/1, + large_writes/1, large_writes_delayed/1, + tiny_reads/1, tiny_reads_ahead/1]). + %% Debug exports -export([create_file_slow/2, create_file/2, create_bin/2]). -export([verify_file/2, verify_bin/3]). @@ -107,6 +115,8 @@ -export([disc_free/1, memsize/0]). -include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + -include_lib("kernel/include/file.hrl"). -define(THROW_ERROR(RES), throw({fail, ?LINE, RES})). @@ -118,13 +128,13 @@ suite() -> all() -> [unicode, altname, read_write_file, {group, dirs}, - {group, files}, delete, rename, names, {group, errors}, - {group, compression}, {group, links}, copy, + {group, files}, delete, rename, names, volume_relative_paths, + {group, errors}, {group, compression}, {group, links}, copy, delayed_write, read_ahead, segment_read, segment_write, ipread, pid2name, interleaved_read_write, otp_5814, otp_10852, large_file, large_write, read_line_1, read_line_2, read_line_3, read_line_4, standard_io, old_io_protocol, - unicode_mode + unicode_mode, {group, bench} ]. groups() -> @@ -154,11 +164,19 @@ groups() -> write_compressed, compress_errors, catenated_gzips, compress_async_crash]}, {links, [], - [make_link, read_link_info_for_non_link, symlinks]}]. + [make_link, read_link_info_for_non_link, symlinks]}, + {bench, [], + [tiny_writes, tiny_writes_delayed, + large_writes, large_writes_delayed, + tiny_reads, tiny_reads_ahead]}]. init_per_group(_GroupName, Config) -> Config. +end_per_group(bench, Config) -> + ScratchDir = proplists:get_value(priv_dir, Config), + file:delete(filename:join(ScratchDir, "benchmark_scratch_file")), + Config; end_per_group(_GroupName, Config) -> Config. @@ -473,7 +491,7 @@ um_check_unicode(_Utf8Bin, {ok, _ListOrBin}, _, _UTF8_) -> um_filename(Bin, Dir, Options) when is_binary(Bin) -> um_filename(binary_to_list(Bin), Dir, Options); um_filename(Str = [_|_], Dir, Options) -> - Name = hd(string:tokens(Str, ":")), + Name = hd(string:lexemes(Str, ":")), Enc = atom_to_list(proplists:get_value(encoding, Options, latin1)), File = case lists:member(binary, Options) of true -> @@ -638,6 +656,10 @@ cur_dir_0(Config) when is_list(Config) -> {ok,NewDirFiles} = ?FILE_MODULE:list_dir("."), true = lists:member(UncommonName,NewDirFiles), + %% Ensure that we get the same result with a trailing slash; the + %% APIs used on Windows will choke on them if passed directly. + {ok,NewDirFiles} = ?FILE_MODULE:list_dir("./"), + %% Delete the directory and return to the old current directory %% and check that the created file isn't there (too!) expect({error, einval}, {error, eacces}, @@ -690,10 +712,15 @@ win_cur_dir_1(_Config) -> %% Get the drive letter from the current directory, %% and try to get current directory for that drive. - [Drive,$:|_] = BaseDir, - {ok,BaseDir} = ?FILE_MODULE:get_cwd([Drive,$:]), + [CurDrive,$:|_] = BaseDir, + {ok,BaseDir} = ?FILE_MODULE:get_cwd([CurDrive,$:]), io:format("BaseDir = ~s\n", [BaseDir]), + %% We should error out on non-existent drives. Any reasonable system will + %% have at least one. + CurDirs = [?FILE_MODULE:get_cwd([Drive,$:]) || Drive <- lists:seq($A, $Z)], + lists:member({error,eaccess}, CurDirs), + %% Unfortunately, there is no way to move away from the %% current drive as we can't use the "subst" command from %% a SSH connection. We can't test any more. @@ -831,7 +858,7 @@ no_untranslatable_names() -> end. start_node(Name, Args) -> - [_,Host] = string:tokens(atom_to_list(node()), "@"), + [_,Host] = string:lexemes(atom_to_list(node()), "@"), ct:log("Trying to start ~w@~s~n", [Name,Host]), case test_server:start_node(Name, peer, [{args,Args}]) of {error,Reason} -> @@ -1019,6 +1046,23 @@ close(Config) when is_list(Config) -> Val = ?FILE_MODULE:close(Fd1), io:format("Second close gave: ~p",[Val]), + %% All operations on a closed raw file should EINVAL, even if they're not + %% supported on the current platform. + {ok,Fd2} = ?FILE_MODULE:open(Name, [read, write, raw]), + ok = ?FILE_MODULE:close(Fd2), + + {error, einval} = ?FILE_MODULE:advise(Fd2, 5, 5, normal), + {error, einval} = ?FILE_MODULE:allocate(Fd2, 5, 5), + {error, einval} = ?FILE_MODULE:close(Fd2), + {error, einval} = ?FILE_MODULE:datasync(Fd2), + {error, einval} = ?FILE_MODULE:position(Fd2, 5), + {error, einval} = ?FILE_MODULE:pread(Fd2, 5, 1), + {error, einval} = ?FILE_MODULE:pwrite(Fd2, 5, "einval please"), + {error, einval} = ?FILE_MODULE:read(Fd2, 1), + {error, einval} = ?FILE_MODULE:sync(Fd2), + {error, einval} = ?FILE_MODULE:truncate(Fd2), + {error, einval} = ?FILE_MODULE:write(Fd2, "einval please"), + [] = flush(), ok. @@ -1132,8 +1176,8 @@ pread_write_test(File, Data) -> end, I = Size + 17, ok = ?FILE_MODULE:pwrite(File, 0, Data), - Res = ?FILE_MODULE:pread(File, 0, I), - {ok, Data} = Res, + {ok, Data} = ?FILE_MODULE:pread(File, 0, I), + {ok, [Data]} = ?FILE_MODULE:pread(File, [{0, I}]), eof = ?FILE_MODULE:pread(File, I, 1), ok = ?FILE_MODULE:pwrite(File, [{0, Data}, {I, Data}]), {ok, [Data, eof, Data]} = @@ -2044,13 +2088,22 @@ names(Config) when is_list(Config) -> ok = ?FILE_MODULE:close(Fd2), {ok,Fd3} = ?FILE_MODULE:open(Name3,read), ok = ?FILE_MODULE:close(Fd3), + + %% Now try the same on raw files. + {ok,Fd4} = ?FILE_MODULE:open(Name2, [read, raw]), + ok = ?FILE_MODULE:close(Fd4), + {ok,Fd4f} = ?FILE_MODULE:open(lists:flatten(Name2), [read, raw]), + ok = ?FILE_MODULE:close(Fd4f), + {ok,Fd5} = ?FILE_MODULE:open(Name3, [read, raw]), + ok = ?FILE_MODULE:close(Fd5), + case length(Name1) > 255 of true -> io:format("Path too long for an atom:\n\n~p\n", [Name1]); false -> Name4 = list_to_atom(Name1), - {ok,Fd4} = ?FILE_MODULE:open(Name4,read), - ok = ?FILE_MODULE:close(Fd4) + {ok,Fd6} = ?FILE_MODULE:open(Name4,read), + ok = ?FILE_MODULE:close(Fd6) end, %% Try some path names @@ -2074,6 +2127,22 @@ names(Config) when is_list(Config) -> [] = flush(), ok. +volume_relative_paths(Config) when is_list(Config) -> + case os:type() of + {win32, _} -> + {ok, [Drive, $: | _]} = file:get_cwd(), + %% Relative to current device root. + {ok, RootInfo} = file:read_file_info([Drive, $:, $/]), + {ok, RootInfo} = file:read_file_info("/"), + %% Relative to current device directory. + {ok, DirContents} = file:list_dir([Drive, $:]), + {ok, DirContents} = file:list_dir("."), + [] = flush(), + ok; + _ -> + {skip, "This test is Windows-specific."} + end. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -2641,8 +2710,8 @@ altname(Config) when is_list(Config) -> {skipped, "Altname not supported on this platform"}; {ok, "LONGAL~1"} -> {ok, "A_FILE~1"} = ?FILE_MODULE:altname(Name), - {ok, "C:/"} = ?FILE_MODULE:altname("C:/"), - {ok, "C:\\"} = ?FILE_MODULE:altname("C:\\"), + {ok, "c:/"} = ?FILE_MODULE:altname("C:/"), + {ok, "c:/"} = ?FILE_MODULE:altname("C:\\"), {error,enoent} = ?FILE_MODULE:altname(NonexName), {ok, "short"} = ?FILE_MODULE:altname(ShortName), ok @@ -2923,20 +2992,22 @@ delayed_write(Config) when is_list(Config) -> %% %% Test caching and normal close of non-raw file {ok, Fd1} = - ?FILE_MODULE:open(File, [write, {delayed_write, Size+1, 2000}]), + ?FILE_MODULE:open(File, [write, {delayed_write, Size+1, 400}]), ok = ?FILE_MODULE:write(Fd1, Data1), - timer:sleep(1000), % Just in case the file system is slow + %% Wait for a reasonable amount of time to check whether the write was + %% practically instantaneous or actually delayed. + timer:sleep(100), {ok, Fd2} = ?FILE_MODULE:open(File, [read]), eof = ?FILE_MODULE:read(Fd2, 1), ok = ?FILE_MODULE:write(Fd1, Data1), % Data flush on size - timer:sleep(1000), % Just in case the file system is slow + timer:sleep(100), {ok, Data1Data1} = ?FILE_MODULE:pread(Fd2, bof, 2*Size+1), ok = ?FILE_MODULE:write(Fd1, Data1), - timer:sleep(3000), % Wait until data flush on timeout + timer:sleep(500), % Wait until data flush on timeout {ok, Data1Data1Data1} = ?FILE_MODULE:pread(Fd2, bof, 3*Size+1), ok = ?FILE_MODULE:write(Fd1, Data1), ok = ?FILE_MODULE:close(Fd1), % Data flush on close - timer:sleep(1000), % Just in case the file system is slow + timer:sleep(100), {ok, Data1Data1Data1Data1} = ?FILE_MODULE:pread(Fd2, bof, 4*Size+1), ok = ?FILE_MODULE:close(Fd2), %% @@ -2970,7 +3041,7 @@ delayed_write(Config) when is_list(Config) -> {'DOWN', Mref1, _, _, _} = Down1a -> ct:fail(Down1a) end, - timer:sleep(1000), % Just in case the file system is slow + timer:sleep(100), % Just in case the file system is slow {ok, Fd3} = ?FILE_MODULE:open(File, [read]), eof = ?FILE_MODULE:read(Fd3, 1), Child1 ! {Parent, continue, normal}, @@ -2980,7 +3051,7 @@ delayed_write(Config) when is_list(Config) -> {'DOWN', Mref1, _, _, _} = Down1b -> ct:fail(Down1b) end, - timer:sleep(1000), % Just in case the file system is slow + timer:sleep(100), % Just in case the file system is slow {ok, Data1} = ?FILE_MODULE:pread(Fd3, bof, Size+1), ok = ?FILE_MODULE:close(Fd3), %% @@ -2993,7 +3064,7 @@ delayed_write(Config) when is_list(Config) -> {'DOWN', Mref2, _, _, _} = Down2a -> ct:fail(Down2a) end, - timer:sleep(1000), % Just in case the file system is slow + timer:sleep(100), % Just in case the file system is slow {ok, Fd4} = ?FILE_MODULE:open(File, [read]), eof = ?FILE_MODULE:read(Fd4, 1), Child2 ! {Parent, continue, kill}, @@ -3003,7 +3074,7 @@ delayed_write(Config) when is_list(Config) -> {'DOWN', Mref2, _, _, _} = Down2b -> ct:fail(Down2b) end, - timer:sleep(1000), % Just in case the file system is slow + timer:sleep(100), % Just in case the file system is slow eof = ?FILE_MODULE:pread(Fd4, bof, 1), ok = ?FILE_MODULE:close(Fd4), %% @@ -3095,6 +3166,16 @@ read_ahead(Config) when is_list(Config) -> Data1Data2Data3 = Data1++Data2++Data3, {ok, Data1Data2Data3} = ?FILE_MODULE:read(Fd5, 3*Size+1), ok = ?FILE_MODULE:close(Fd5), + + %% Ensure that a read that draws from both the buffer and the file won't + %% return anything wonky. + SplitData = << <<(I rem 256)>> || I <- lists:seq(1, 1024) >>, + file:write_file(File, SplitData), + {ok, Fd6} = ?FILE_MODULE:open(File, [raw, read, binary, {read_ahead, 256}]), + {ok, <<1>>} = file:read(Fd6, 1), + <<1, Shifted:512/binary, _Rest/binary>> = SplitData, + {ok, Shifted} = file:read(Fd6, 512), + %% [] = flush(), ok. @@ -3699,6 +3780,83 @@ do_large_write(Name) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Benchmarks +%% +%% Note that we only measure the time it takes to run the isolated file +%% operations and that the actual test runtime can differ significantly, +%% especially on the write side as the files need to be truncated before +%% writing. + +large_writes(Config) when is_list(Config) -> + Modes = [raw, binary], + OpCount = 4096, + Data = <<0:(64 bsl 10)/unit:8>>, + run_write_benchmark(Config, Modes, OpCount, Data). + +large_writes_delayed(Config) when is_list(Config) -> + %% Each write is exactly as large as the delay buffer, causing the writes + %% to pass through each time, giving us a decent idea of how much overhead + %% delayed_write adds. + Modes = [raw, binary, {delayed_write, 64 bsl 10, 2000}], + OpCount = 4096, + Data = <<0:(64 bsl 10)/unit:8>>, + run_write_benchmark(Config, Modes, OpCount, Data). + +tiny_writes(Config) when is_list(Config) -> + Modes = [raw, binary], + OpCount = 512 bsl 10, + Data = <<0>>, + run_write_benchmark(Config, Modes, OpCount, Data). + +tiny_writes_delayed(Config) when is_list(Config) -> + Modes = [raw, binary, {delayed_write, 512 bsl 10, 2000}], + OpCount = 512 bsl 10, + Data = <<0>>, + run_write_benchmark(Config, Modes, OpCount, Data). + +%% The read benchmarks assume that "benchmark_scratch_file" has been filled by +%% the write benchmarks. + +tiny_reads(Config) when is_list(Config) -> + Modes = [raw, binary], + OpCount = 512 bsl 10, + run_read_benchmark(Config, Modes, OpCount, 1). + +tiny_reads_ahead(Config) when is_list(Config) -> + Modes = [raw, binary, {read_ahead, 512 bsl 10}], + OpCount = 512 bsl 10, + run_read_benchmark(Config, Modes, OpCount, 1). + +run_write_benchmark(Config, Modes, OpCount, Data) -> + run_benchmark(Config, [write | Modes], OpCount, fun file:write/2, Data). + +run_read_benchmark(Config, Modes, OpCount, OpSize) -> + run_benchmark(Config, [read | Modes], OpCount, fun file:read/2, OpSize). + +run_benchmark(Config, Modes, OpCount, Fun, Arg) -> + ScratchDir = proplists:get_value(priv_dir, Config), + Path = filename:join(ScratchDir, "benchmark_scratch_file"), + {ok, Fd} = file:open(Path, Modes), + submit_throughput_results(Fun, [Fd, Arg], OpCount). + +submit_throughput_results(Fun, Args, Times) -> + MSecs = measure_repeated_file_op(Fun, Args, Times, millisecond), + IOPS = trunc(Times * (1000 / MSecs)), + ct_event:notify(#event{ name = benchmark_data, data = [{value,IOPS}] }), + {comment, io_lib:format("~p IOPS, ~p ms", [IOPS, trunc(MSecs)])}. + +measure_repeated_file_op(Fun, Args, Times, Unit) -> + Start = os:perf_counter(Unit), + repeated_apply(Fun, Args, Times), + os:perf_counter(Unit) - Start. + +repeated_apply(_F, _Args, Times) when Times =< 0 -> + ok; +repeated_apply(F, Args, Times) -> + erlang:apply(F, Args), + repeated_apply(F, Args, Times - 1). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% response_analysis(Module, Function, Arguments) -> @@ -3934,7 +4092,7 @@ read_line_create_files(TestData) -> read_line_remove_files(TestData) -> [ file:delete(File) || {_Function,File,_,_} <- TestData ]. -%% read_line with prim_file. +%% read_line with ?PRIM_FILE. read_line_1(Config) when is_list(Config) -> PrivDir = proplists:get_value(priv_dir, Config), All = read_line_testdata(PrivDir), @@ -4103,9 +4261,9 @@ read_line_create7(Filename) -> file:close(F). read_line_all(Filename) -> - {ok,F} = prim_file:open(Filename,[read,binary]), + {ok,F} = ?PRIM_FILE:open(Filename,[read,binary]), X=read_rl_lines(F), - prim_file:close(F), + ?PRIM_FILE:close(F), Bin = list_to_binary([B || {ok,B} <- X]), Bin = re:replace(list_to_binary([element(2,file:read_file(Filename))]), "\r\n","\n",[global,{return,binary}]), @@ -4138,7 +4296,7 @@ read_line_all4(Filename) -> {length(X),Bin}. read_rl_lines(F) -> - case prim_file:read_line(F) of + case ?PRIM_FILE:read_line(F) of eof -> []; {error,X} -> @@ -4158,9 +4316,9 @@ read_rl_lines2(F) -> end. read_line_all_alternating(Filename) -> - {ok,F} = prim_file:open(Filename,[read,binary]), + {ok,F} = ?PRIM_FILE:open(Filename,[read,binary]), X=read_rl_lines(F,true), - prim_file:close(F), + ?PRIM_FILE:close(F), Bin = list_to_binary([B || {ok,B} <- X]), Bin = re:replace(list_to_binary([element(2,file:read_file(Filename))]), "\r\n","\n",[global,{return,binary}]), @@ -4194,8 +4352,8 @@ read_line_all_alternating4(Filename) -> read_rl_lines(F,Alternate) -> case begin case Alternate of - true -> prim_file:read(F,1); - false -> prim_file:read_line(F) + true -> ?PRIM_FILE:read(F,1); + false -> ?PRIM_FILE:read_line(F) end end of eof -> diff --git a/lib/kernel/test/file_name_SUITE.erl b/lib/kernel/test/file_name_SUITE.erl index f23529fec9..3afc647081 100644 --- a/lib/kernel/test/file_name_SUITE.erl +++ b/lib/kernel/test/file_name_SUITE.erl @@ -77,6 +77,7 @@ init_per_testcase/2, end_per_testcase/2]). -export([normal/1,icky/1,very_icky/1,normalize/1,home_dir/1]). +-define(PRIM_FILE, prim_file). init_per_testcase(_Func, Config) -> Config. @@ -131,7 +132,7 @@ home_dir(Config) when is_list(Config) -> os:putenv("HOME",NewHome), {"HOME",Save}; _ -> - rm_rf(prim_file,NewHome), + rm_rf(?PRIM_FILE,NewHome), throw(unsupported_os) end, try @@ -145,7 +146,7 @@ home_dir(Config) when is_list(Config) -> _ -> os:putenv(SaveOldName,SaveOldValue) end, - rm_rf(prim_file,NewHome) + rm_rf(?PRIM_FILE,NewHome) end catch throw:need_unicode_mode -> @@ -190,7 +191,7 @@ normal(Config) when is_list(Config) -> try Priv = proplists:get_value(priv_dir, Config), file:set_cwd(Priv), - ok = check_normal(prim_file), + ok = check_normal(?PRIM_FILE), ok = check_normal(file), %% If all is good, delete dir again (avoid hanging dir on windows) rm_rf(file,"normal_dir"), @@ -210,7 +211,7 @@ icky(Config) when is_list(Config) -> try Priv = proplists:get_value(priv_dir, Config), file:set_cwd(Priv), - ok = check_icky(prim_file), + ok = check_icky(?PRIM_FILE), ok = check_icky(file), %% If all is good, delete dir again (avoid hanging dir on windows) rm_rf(file,"icky_dir"), @@ -229,7 +230,7 @@ very_icky(Config) when is_list(Config) -> try Priv = proplists:get_value(priv_dir, Config), file:set_cwd(Priv), - case check_very_icky(prim_file) of + case check_very_icky(?PRIM_FILE) of need_unicode_mode -> {skipped,"VM needs to be started in Unicode filename mode"}; ok -> @@ -292,11 +293,6 @@ check_normal(Mod) -> ok end, [ begin - {ok, FD} = Mod:open(Name,[read]), - {ok, Content} = Mod:read(FD,1024), - ok = file:close(FD) - end || {regular,Name,Content} <- NormalDir ], - [ begin {ok, FD} = Mod:open(Name,[read,binary]), BC = list_to_binary(Content), {ok, BC} = Mod:read(FD,1024), @@ -412,11 +408,6 @@ check_icky(Mod) -> ok end, [ begin - {ok, FD} = Mod:open(Name,[read]), - {ok, Content} = Mod:read(FD,1024), - ok = file:close(FD) - end || {regular,Name,Content} <- IckyDir ], - [ begin {ok, FD} = Mod:open(Name,[read,binary]), BC = list_to_binary([Content]), {ok, BC} = Mod:read(FD,1024), @@ -521,11 +512,6 @@ check_very_icky(Mod) -> ok end, [ begin - {ok, FD} = Mod:open(Name,[read]), - {ok, Content} = Mod:read(FD,1024), - ok = file:close(FD) - end || {regular,Name,Content} <- VeryIckyDir ], - [ begin {ok, FD} = Mod:open(Name,[read,binary]), BC = list_to_binary([Content]), {ok, BC} = Mod:read(FD,1024), diff --git a/lib/kernel/test/kernel_SUITE.erl b/lib/kernel/test/kernel_SUITE.erl index da56359294..7898988dbe 100644 --- a/lib/kernel/test/kernel_SUITE.erl +++ b/lib/kernel/test/kernel_SUITE.erl @@ -30,14 +30,14 @@ -export([init_per_testcase/2, end_per_testcase/2]). %% Test cases must be exported. --export([app_test/1, appup_test/1]). +-export([app_test/1, appup_test/1, refc/1]). suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap,{minutes,2}}]. all() -> - [app_test, appup_test]. + [app_test, appup_test, refc]. groups() -> []. @@ -163,3 +163,68 @@ check_appup([Vsn|Vsns],Instrs,Expected) -> end; check_appup([],_,_) -> ok. + +%%% Check that refc module handles the counters as expected +refc(_Config) -> + Enable = fun(Enable) -> erlang:system_flag(scheduler_wall_time, Enable) end, + IsOn = fun() -> undefined /= erlang:statistics(scheduler_wall_time) end, + Tester = self(), + Loop = fun Loop() -> + receive + die -> normal; + {apply, Bool} -> + Res = Enable(Bool), + Tester ! {self(), Res}, + Loop() + end + end, + + %% Counter should be 0 + false = Enable(false), + + false = Enable(true), + true = Enable(true), + true = Enable(false), + true = Enable(false), + + %% Counter should be 0 + false = IsOn(), + + P1 = spawn_link(Loop), + P1 ! {apply, true}, + receive {P1, R1} -> false = R1 end, + + %% P1 has turned it on counter should be one + true = IsOn(), + true = Enable(true), + true = Enable(false), + true = IsOn(), + + P1 ! {apply, false}, + receive {P1, R2} -> true = R2 end, + false = IsOn(), + + P1 ! {apply, true}, + receive {P1, R3} -> false = R3 end, + true = IsOn(), + true = Enable(false), + + + P1 ! die, + timer:sleep(100), + false = IsOn(), + false = Enable(false), + + P2 = spawn_link(Loop), + P2 ! {apply, true}, + receive {P2, R4} -> false = R4 end, + true = IsOn(), + P2 ! {apply, true}, + receive {P2, R5} -> true = R5 end, + true = IsOn(), + + P2 ! die, + timer:sleep(100), + false = IsOn(), + + ok. diff --git a/lib/kernel/test/kernel_bench.spec b/lib/kernel/test/kernel_bench.spec index 8de60dae31..4de133f21b 100644 --- a/lib/kernel/test/kernel_bench.spec +++ b/lib/kernel/test/kernel_bench.spec @@ -1 +1,2 @@ {groups,"../kernel_test",zlib_SUITE,[bench]}. +{groups,"../kernel_test",file_SUITE,[bench]}. diff --git a/lib/kernel/test/prim_file_SUITE.erl b/lib/kernel/test/prim_file_SUITE.erl index 2f4330c217..db753679ea 100644 --- a/lib/kernel/test/prim_file_SUITE.erl +++ b/lib/kernel/test/prim_file_SUITE.erl @@ -21,38 +21,23 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2, end_per_testcase/2, read_write_file/1, free_memory/0]). --export([cur_dir_0a/1, cur_dir_0b/1, - cur_dir_1a/1, cur_dir_1b/1, - make_del_dir_a/1, make_del_dir_b/1, - pos1/1, pos2/1]). --export([close/1, - delete_a/1, delete_b/1]). --export([ open1/1, modes/1]). --export([ - file_info_basic_file_a/1, file_info_basic_file_b/1, - file_info_basic_directory_a/1, file_info_basic_directory_b/1, - file_info_bad_a/1, file_info_bad_b/1, - file_info_times_a/1, file_info_times_b/1, - file_write_file_info_a/1, file_write_file_info_b/1, - file_read_file_info_opts/1, file_write_file_info_opts/1, - file_write_read_file_info_opts/1 - ]). --export([rename_a/1, rename_b/1, - access/1, truncate/1, datasync/1, sync/1, +-export([cur_dir_0/1, cur_dir_1/1, + make_del_dir/1, pos1/1, pos2/1]). +-export([close/1, delete/1]). +-export([open1/1, modes/1]). +-export([file_info_basic_file/1, file_info_basic_directory/1, file_info_bad/1, + file_info_times/1, file_write_file_info/1, + file_read_file_info_opts/1, file_write_file_info_opts/1, + file_write_read_file_info_opts/1]). +-export([rename/1, access/1, truncate/1, datasync/1, sync/1, read_write/1, pread_write/1, append/1, exclusive/1]). --export([ e_delete/1, e_rename/1, e_make_dir/1, e_del_dir/1]). +-export([e_delete/1, e_rename/1, e_make_dir/1, e_del_dir/1]). --export([ read_not_really_compressed/1, - read_compressed/1, write_compressed/1, - compress_errors/1]). - --export([ - make_link_a/1, make_link_b/1, - read_link_info_for_non_link/1, - symlinks_a/1, symlinks_b/1, - list_dir_limit/1, - list_dir_error/1, - list_dir/1]). +-export([make_link/1, read_link_info_for_non_link/1, + symlinks/1, + list_dir_limit/1, + list_dir_error/1, + list_dir/1]). -export([advise/1]). -export([large_write/1]). @@ -67,29 +52,16 @@ -define(PRIM_FILE, prim_file). -%% Calls ?PRIM_FILE:F with arguments A and an optional handle H -%% as first argument, unless the handle is [], i.e no handle. -%% This is a macro to give the compiler and thereby -%% the cross reference tool the possibility to interprete -%% the call, since M, F, A (or [H | A]) can all be known at -%% compile time. --define(PRIM_FILE_call(F, H, A), - case H of - [] -> apply(?PRIM_FILE, F, A); - _ -> apply(?PRIM_FILE, F, [H | A]) - end). - suite() -> []. all() -> [read_write_file, {group, dirs}, {group, files}, - delete_a, delete_b, rename_a, rename_b, {group, errors}, - {group, compression}, {group, links}, list_dir_limit, list_dir]. + delete, rename, {group, errors}, {group, links}, + list_dir_limit, list_dir]. groups() -> [{dirs, [], - [make_del_dir_a, make_del_dir_b, cur_dir_0a, cur_dir_0b, - cur_dir_1a, cur_dir_1b]}, + [make_del_dir, cur_dir_0, cur_dir_1]}, {files, [], [{group, open}, {group, pos}, {group, file_info}, truncate, sync, datasync, advise, large_write, allocate]}, @@ -98,22 +70,14 @@ groups() -> append, exclusive]}, {pos, [], [pos1, pos2]}, {file_info, [], - [file_info_basic_file_a, file_info_basic_file_b, - file_info_basic_directory_a, - file_info_basic_directory_b, file_info_bad_a, - file_info_bad_b, file_info_times_a, file_info_times_b, - file_write_file_info_a, file_write_file_info_b, - file_read_file_info_opts, file_write_file_info_opts, - file_write_read_file_info_opts + [file_info_basic_file,file_info_basic_directory, file_info_bad, + file_info_times, file_write_file_info, file_read_file_info_opts, + file_write_file_info_opts, file_write_read_file_info_opts ]}, {errors, [], [e_delete, e_rename, e_make_dir, e_del_dir]}, - {compression, [], - [read_compressed, read_not_really_compressed, - write_compressed, compress_errors]}, {links, [], - [make_link_a, make_link_b, read_link_info_for_non_link, - symlinks_a, symlinks_b, list_dir_error]}]. + [make_link, read_link_info_for_non_link, symlinks, list_dir_error]}]. init_per_testcase(large_write, Config) -> {ok, Started} = application:ensure_all_started(os_mon), @@ -246,39 +210,27 @@ read_write_file(Config) when is_list(Config) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -make_del_dir_a(Config) when is_list(Config) -> - make_del_dir(Config, [], "_a"). - -make_del_dir_b(Config) when is_list(Config) -> - {ok, Handle} = ?PRIM_FILE:start(), - Result = make_del_dir(Config, Handle, "_b"), - ok = ?PRIM_FILE:stop(Handle), - %% Just to make sure the state of the server makes a difference - {error, einval} = ?PRIM_FILE_call(get_cwd, Handle, []), - Result. - -make_del_dir(Config, Handle, Suffix) -> +make_del_dir(Config) when is_list(Config) -> RootDir = proplists:get_value(priv_dir,Config), NewDir = filename:join(RootDir, atom_to_list(?MODULE) - ++"_mk-dir"++Suffix), - ok = ?PRIM_FILE_call(make_dir, Handle, [NewDir]), - {error, eexist} = ?PRIM_FILE_call(make_dir, Handle, [NewDir]), - ok = ?PRIM_FILE_call(del_dir, Handle, [NewDir]), - {error, enoent} = ?PRIM_FILE_call(del_dir, Handle, [NewDir]), + ++"_mk-dir"), + ok = ?PRIM_FILE:make_dir(NewDir), + {error, eexist} = ?PRIM_FILE:make_dir(NewDir), + ok = ?PRIM_FILE:del_dir(NewDir), + {error, enoent} = ?PRIM_FILE:del_dir(NewDir), %% Make sure we are not in a directory directly under test_server %% as that would result in eacces errors when trying to delete '..', %% because there are processes having that directory as current. - ok = ?PRIM_FILE_call(make_dir, Handle, [NewDir]), - {ok, CurrentDir} = ?PRIM_FILE_call(get_cwd, Handle, []), + ok = ?PRIM_FILE:make_dir(NewDir), + {ok, CurrentDir} = ?PRIM_FILE:get_cwd(), case {os:type(), length(NewDir) >= 260 } of {{win32,_}, true} -> io:format("Skip set_cwd for windows path longer than 260 (MAX_PATH)\n", []), io:format("\nNewDir = ~p\n", [NewDir]); _ -> - ok = ?PRIM_FILE_call(set_cwd, Handle, [NewDir]) + ok = ?PRIM_FILE:set_cwd(NewDir) end, try %% Check that we get an error when trying to create... @@ -286,14 +238,14 @@ make_del_dir(Config, Handle, Suffix) -> NewDir2 = filename:join(RootDir, atom_to_list(?MODULE) ++"_mk-dir-noexist/foo"), - {error, enoent} = ?PRIM_FILE_call(make_dir, Handle, [NewDir2]), + {error, enoent} = ?PRIM_FILE:make_dir(NewDir2), %% a nameless directory - {error, enoent} = ?PRIM_FILE_call(make_dir, Handle, [""]), + {error, enoent} = ?PRIM_FILE:make_dir(""), %% a directory with illegal name - {error, badarg} = ?PRIM_FILE_call(make_dir, Handle, ['mk-dir']), + {error, badarg} = ?PRIM_FILE:make_dir('mk-dir'), %% a directory with illegal name, even if it's a (bad) list - {error, badarg} = ?PRIM_FILE_call(make_dir, Handle, [[1,2,3,{}]]), + {error, badarg} = ?PRIM_FILE:make_dir([1,2,3,{}]), %% Maybe this isn't an error, exactly, but worth mentioning anyway: %% ok = ?PRIM_FILE:make_dir([$f,$o,$o,0,$b,$a,$r])), @@ -306,125 +258,101 @@ make_del_dir(Config, Handle, Suffix) -> %% Try deleting some bad directories %% Deleting the parent directory to the current, sounds dangerous, huh? %% Don't worry ;-) the parent directory should never be empty, right? - case ?PRIM_FILE_call(del_dir, Handle, [".."]) of + case ?PRIM_FILE:del_dir("..") of {error, eexist} -> ok; {error, eacces} -> ok; %OpenBSD {error, einval} -> ok %FreeBSD end, - {error, enoent} = ?PRIM_FILE_call(del_dir, Handle, [""]), - {error, badarg} = ?PRIM_FILE_call(del_dir, Handle, [[3,2,1,{}]]) + {error, enoent} = ?PRIM_FILE:del_dir(""), + {error, badarg} = ?PRIM_FILE:del_dir([3,2,1,{}]) after - ok = ?PRIM_FILE_call(set_cwd, Handle, [CurrentDir]) + ok = ?PRIM_FILE:set_cwd(CurrentDir) end, ok. -cur_dir_0a(Config) when is_list(Config) -> - cur_dir_0(Config, []). - -cur_dir_0b(Config) when is_list(Config) -> - {ok, Handle} = ?PRIM_FILE:start(), - Result = cur_dir_0(Config, Handle), - ok = ?PRIM_FILE:stop(Handle), - Result. - -cur_dir_0(Config, Handle) -> +cur_dir_0(Config) when is_list(Config) -> %% Find out the current dir, and cd to it ;-) - {ok,BaseDir} = ?PRIM_FILE_call(get_cwd, Handle, []), + {ok,BaseDir} = ?PRIM_FILE:get_cwd(), Dir1 = BaseDir ++ "", %% Check that it's a string - ok = ?PRIM_FILE_call(set_cwd, Handle, [Dir1]), - DirName = atom_to_list(?MODULE) ++ - case Handle of - [] -> - "_curdir"; - _ -> - "_curdir_h" - end, + ok = ?PRIM_FILE:set_cwd(Dir1), + DirName = atom_to_list(?MODULE) ++ "_curdir", %% Make a new dir, and cd to that RootDir = proplists:get_value(priv_dir,Config), NewDir = filename:join(RootDir, DirName), - ok = ?PRIM_FILE_call(make_dir, Handle, [NewDir]), + ok = ?PRIM_FILE:make_dir(NewDir), case {os:type(), length(NewDir) >= 260} of {{win32,_}, true} -> io:format("Skip set_cwd for windows path longer than 260 (MAX_PATH):\n"), io:format("\nNewDir = ~p\n", [NewDir]); _ -> io:format("cd to ~s",[NewDir]), - ok = ?PRIM_FILE_call(set_cwd, Handle, [NewDir]), + ok = ?PRIM_FILE:set_cwd(NewDir), %% Create a file in the new current directory, and check that it %% really is created there UncommonName = "uncommon.fil", {ok,Fd} = ?PRIM_FILE:open(UncommonName, [read, write]), ok = ?PRIM_FILE:close(Fd), - {ok,NewDirFiles} = ?PRIM_FILE_call(list_dir, Handle, ["."]), + {ok,NewDirFiles} = ?PRIM_FILE:list_dir("."), true = lists:member(UncommonName,NewDirFiles), %% Delete the directory and return to the old current directory %% and check that the created file isn't there (too!) expect({error, einval}, {error, eacces}, {error, eexist}, - ?PRIM_FILE_call(del_dir, Handle, [NewDir])), - ?PRIM_FILE_call(delete, Handle, [UncommonName]), - {ok,[]} = ?PRIM_FILE_call(list_dir, Handle, ["."]), - ok = ?PRIM_FILE_call(set_cwd, Handle, [Dir1]), + ?PRIM_FILE:del_dir(NewDir)), + ?PRIM_FILE:delete(UncommonName), + {ok,[]} = ?PRIM_FILE:list_dir("."), + ok = ?PRIM_FILE:set_cwd(Dir1), io:format("cd back to ~s",[Dir1]), - ok = ?PRIM_FILE_call(del_dir, Handle, [NewDir]), - {error, enoent} = ?PRIM_FILE_call(set_cwd, Handle, [NewDir]), - ok = ?PRIM_FILE_call(set_cwd, Handle, [Dir1]), + ok = ?PRIM_FILE:del_dir(NewDir), + {error, enoent} = ?PRIM_FILE:set_cwd(NewDir), + ok = ?PRIM_FILE:set_cwd(Dir1), io:format("cd back to ~s",[Dir1]), - {ok,OldDirFiles} = ?PRIM_FILE_call(list_dir, Handle, ["."]), + {ok,OldDirFiles} = ?PRIM_FILE:list_dir("."), false = lists:member(UncommonName,OldDirFiles) end, %% Try doing some bad things {error, badarg} = - ?PRIM_FILE_call(set_cwd, Handle, [{foo,bar}]), + ?PRIM_FILE:set_cwd({foo,bar}), {error, enoent} = - ?PRIM_FILE_call(set_cwd, Handle, [""]), + ?PRIM_FILE:set_cwd(""), {error, enoent} = - ?PRIM_FILE_call(set_cwd, Handle, [".......a......"]), + ?PRIM_FILE:set_cwd(".......a......"), {ok,BaseDir} = - ?PRIM_FILE_call(get_cwd, Handle, []), %% Still there? + ?PRIM_FILE:get_cwd(), %% Still there? %% On Windows, there should only be slashes, no backslashes, %% in the return value of get_cwd(). %% (The test is harmless on Unix, because filenames usually %% don't contain backslashes.) - {ok, BaseDir} = ?PRIM_FILE_call(get_cwd, Handle, []), + {ok, BaseDir} = ?PRIM_FILE:get_cwd(), false = lists:member($\\, BaseDir), ok. %% Tests ?PRIM_FILE:get_cwd/1. -cur_dir_1a(Config) when is_list(Config) -> - cur_dir_1(Config, []). - -cur_dir_1b(Config) when is_list(Config) -> - {ok, Handle} = ?PRIM_FILE:start(), - Result = cur_dir_1(Config, Handle), - ok = ?PRIM_FILE:stop(Handle), - Result. - -cur_dir_1(Config, Handle) -> +cur_dir_1(Config) when is_list(Config) -> case os:type() of {win32, _} -> - win_cur_dir_1(Config, Handle); + win_cur_dir_1(Config); _ -> {error, enotsup} = - ?PRIM_FILE_call(get_cwd, Handle, ["d:"]) + ?PRIM_FILE:get_cwd("d:") end, ok. -win_cur_dir_1(_Config, Handle) -> - {ok, BaseDir} = ?PRIM_FILE_call(get_cwd, Handle, []), +win_cur_dir_1(_Config) -> + {ok, BaseDir} = ?PRIM_FILE:get_cwd(), %% Get the drive letter from the current directory, %% and try to get current directory for that drive. [Drive, $:|_] = BaseDir, - {ok, BaseDir} = ?PRIM_FILE_call(get_cwd, Handle, [[Drive, $:]]), + {ok, BaseDir} = ?PRIM_FILE:get_cwd([Drive, $:]), io:format("BaseDir = ~s\n", [BaseDir]), %% Unfortunately, there is no way to move away from the @@ -446,12 +374,12 @@ open1(Config) when is_list(Config) -> Name = filename:join(NewDir, "foo1.fil"), {ok,Fd1} = ?PRIM_FILE:open(Name, [read, write]), {ok,Fd2} = ?PRIM_FILE:open(Name, [read]), - Str = "{a,tuple}.\n", - Length = length(Str), - ?PRIM_FILE:write(Fd1,Str), + Bin = list_to_binary("{a,tuple}.\n"), + Length = byte_size(Bin), + ?PRIM_FILE:write(Fd1,Bin), {ok,0} = ?PRIM_FILE:position(Fd1,bof), - {ok, Str} = ?PRIM_FILE:read(Fd1,Length), - {ok, Str} = ?PRIM_FILE:read(Fd2,Length), + {ok, Bin} = ?PRIM_FILE:read(Fd1,Length), + {ok, Bin} = ?PRIM_FILE:read(Fd2,Length), ok = ?PRIM_FILE:close(Fd2), {ok,0} = ?PRIM_FILE:position(Fd1,bof), ok = ?PRIM_FILE:truncate(Fd1), @@ -471,13 +399,13 @@ modes(Config) when is_list(Config) -> ++"_open_modes"), ok = ?PRIM_FILE:make_dir(NewDir), Name1 = filename:join(NewDir, "foo1.fil"), - Marker = "hello, world", - Length = length(Marker), + Marker = <<"hello, world">>, + Length = byte_size(Marker), %% write {ok, Fd1} = ?PRIM_FILE:open(Name1, [write]), ok = ?PRIM_FILE:write(Fd1, Marker), - ok = ?PRIM_FILE:write(Fd1, ".\n"), + ok = ?PRIM_FILE:write(Fd1, <<".\n">>), ok = ?PRIM_FILE:close(Fd1), %% read @@ -496,12 +424,6 @@ modes(Config) when is_list(Config) -> {ok, Marker} = ?PRIM_FILE:read(Fd4, Length), ok = ?PRIM_FILE:close(Fd4), - %% read and binary - BinaryMarker = list_to_binary(Marker), - {ok, Fd5} = ?PRIM_FILE:open(Name1, [read, binary]), - {ok, BinaryMarker} = ?PRIM_FILE:read(Fd5, Length), - ok = ?PRIM_FILE:close(Fd5), - ok. close(Config) when is_list(Config) -> @@ -528,9 +450,9 @@ access(Config) when is_list(Config) -> Name = filename:join(RootDir, atom_to_list(?MODULE) ++"_access.fil"), - Str = "ABCDEFGH", + Bin = <<"ABCDEFGH">>, {ok,Fd1} = ?PRIM_FILE:open(Name, [write]), - ?PRIM_FILE:write(Fd1,Str), + ?PRIM_FILE:write(Fd1,Bin), ok = ?PRIM_FILE:close(Fd1), %% Check that we can't write when in read only mode {ok,Fd2} = ?PRIM_FILE:open(Name, [read]), @@ -542,7 +464,7 @@ access(Config) when is_list(Config) -> end, ok = ?PRIM_FILE:close(Fd2), {ok, Fd3} = ?PRIM_FILE:open(Name, [read]), - {ok, Str} = ?PRIM_FILE:read(Fd3,length(Str)), + {ok, Bin} = ?PRIM_FILE:read(Fd3,byte_size(Bin)), ok = ?PRIM_FILE:close(Fd3), ok. @@ -564,7 +486,7 @@ read_write(Config) when is_list(Config) -> ok. read_write_test(File) -> - Marker = "hello, world", + Marker = <<"hello, world">>, ok = ?PRIM_FILE:write(File, Marker), {ok, 0} = ?PRIM_FILE:position(File, 0), {ok, Marker} = ?PRIM_FILE:read(File, 100), @@ -590,15 +512,15 @@ pread_write(Config) when is_list(Config) -> ok. pread_write_test(File) -> - Marker = "hello, world", - Len = length(Marker), + Marker = <<"hello, world">>, + Len = byte_size(Marker), ok = ?PRIM_FILE:write(File, Marker), {ok, Marker} = ?PRIM_FILE:pread(File, 0, 100), eof = ?PRIM_FILE:pread(File, 100, 1), ok = ?PRIM_FILE:pwrite(File, Len, Marker), {ok, Marker} = ?PRIM_FILE:pread(File, Len, 100), eof = ?PRIM_FILE:pread(File, 100, 1), - MM = Marker ++ Marker, + MM = <<Marker/binary,Marker/binary>>, {ok, MM} = ?PRIM_FILE:pread(File, 0, 100), ok = ?PRIM_FILE:close(File), ok. @@ -655,24 +577,24 @@ pos1(Config) when is_list(Config) -> atom_to_list(?MODULE) ++"_pos1.fil"), {ok, Fd1} = ?PRIM_FILE:open(Name, [write]), - ?PRIM_FILE:write(Fd1,"ABCDEFGH"), + ?PRIM_FILE:write(Fd1,<<"ABCDEFGH">>), ok = ?PRIM_FILE:close(Fd1), {ok, Fd2} = ?PRIM_FILE:open(Name, [read]), %% Start pos is first char io:format("Relative positions"), - {ok, "A"} = ?PRIM_FILE:read(Fd2,1), + {ok, <<"A">>} = ?PRIM_FILE:read(Fd2,1), {ok, 2} = ?PRIM_FILE:position(Fd2,{cur,1}), - {ok, "C"} = ?PRIM_FILE:read(Fd2,1), + {ok, <<"C">>} = ?PRIM_FILE:read(Fd2,1), {ok, 0} = ?PRIM_FILE:position(Fd2,{cur,-3}), - {ok, "A"} = ?PRIM_FILE:read(Fd2,1), + {ok, <<"A">>} = ?PRIM_FILE:read(Fd2,1), %% Backwards from first char should be an error {ok,0} = ?PRIM_FILE:position(Fd2,{cur,-1}), {error, einval} = ?PRIM_FILE:position(Fd2,{cur,-1}), %% Reset position and move again {ok, 0} = ?PRIM_FILE:position(Fd2,0), {ok, 2} = ?PRIM_FILE:position(Fd2,{cur,2}), - {ok, "C"} = ?PRIM_FILE:read(Fd2,1), + {ok, <<"C">>} = ?PRIM_FILE:read(Fd2,1), %% Go a lot forwards {ok, 13} = ?PRIM_FILE:position(Fd2,{cur,10}), eof = ?PRIM_FILE:read(Fd2,1), @@ -684,27 +606,27 @@ pos1(Config) when is_list(Config) -> {ok, 8} = ?PRIM_FILE:position(Fd2,cur), eof = ?PRIM_FILE:read(Fd2,1), {ok, 7} = ?PRIM_FILE:position(Fd2,7), - {ok, "H"} = ?PRIM_FILE:read(Fd2,1), + {ok, <<"H">>} = ?PRIM_FILE:read(Fd2,1), {ok, 0} = ?PRIM_FILE:position(Fd2,0), - {ok, "A"} = ?PRIM_FILE:read(Fd2,1), + {ok, <<"A">>} = ?PRIM_FILE:read(Fd2,1), {ok, 3} = ?PRIM_FILE:position(Fd2,3), - {ok, "D"} = ?PRIM_FILE:read(Fd2,1), + {ok, <<"D">>} = ?PRIM_FILE:read(Fd2,1), {ok, 12} = ?PRIM_FILE:position(Fd2,12), eof = ?PRIM_FILE:read(Fd2,1), {ok, 3} = ?PRIM_FILE:position(Fd2,3), - {ok, "D"} = ?PRIM_FILE:read(Fd2,1), + {ok, <<"D">>} = ?PRIM_FILE:read(Fd2,1), %% Try the {bof,X} notation {ok, 3} = ?PRIM_FILE:position(Fd2,{bof,3}), - {ok, "D"} = ?PRIM_FILE:read(Fd2,1), + {ok, <<"D">>} = ?PRIM_FILE:read(Fd2,1), %% Try eof positions io:format("EOF positions"), {ok, 8} = ?PRIM_FILE:position(Fd2,{eof,0}), eof = ?PRIM_FILE:read(Fd2,1), {ok, 7} = ?PRIM_FILE:position(Fd2,{eof,-1}), - {ok, "H"} = ?PRIM_FILE:read(Fd2,1), + {ok, <<"H">>} = ?PRIM_FILE:read(Fd2,1), {ok, 0} = ?PRIM_FILE:position(Fd2,{eof,-8}), - {ok, "A"} = ?PRIM_FILE:read(Fd2,1), + {ok, <<"A">>} = ?PRIM_FILE:read(Fd2,1), {error, einval} = ?PRIM_FILE:position(Fd2,{eof,-9}), ok. @@ -714,7 +636,7 @@ pos2(Config) when is_list(Config) -> atom_to_list(?MODULE) ++"_pos2.fil"), {ok,Fd1} = ?PRIM_FILE:open(Name, [write]), - ?PRIM_FILE:write(Fd1,"ABCDEFGH"), + ?PRIM_FILE:write(Fd1,<<"ABCDEFGH">>), ok = ?PRIM_FILE:close(Fd1), {ok, Fd2} = ?PRIM_FILE:open(Name, [read]), {error, einval} = ?PRIM_FILE:position(Fd2,-1), @@ -722,35 +644,25 @@ pos2(Config) when is_list(Config) -> %% Make sure that we still can search after an error. {ok, 0} = ?PRIM_FILE:position(Fd2, 0), {ok, 3} = ?PRIM_FILE:position(Fd2, {bof,3}), - {ok, "D"} = ?PRIM_FILE:read(Fd2,1), + {ok, <<"D">>} = ?PRIM_FILE:read(Fd2,1), io:format("DONE"), ok. - -file_info_basic_file_a(Config) when is_list(Config) -> - file_info_basic_file(Config, [], "_a"). - -file_info_basic_file_b(Config) when is_list(Config) -> - {ok, Handle} = ?PRIM_FILE:start(), - Result = file_info_basic_file(Config, Handle, "_b"), - ok = ?PRIM_FILE:stop(Handle), - Result. - -file_info_basic_file(Config, Handle, Suffix) -> +file_info_basic_file(Config) when is_list(Config) -> RootDir = proplists:get_value(priv_dir, Config), %% Create a short file. Name = filename:join(RootDir, atom_to_list(?MODULE) - ++"_basic_test"++Suffix++".fil"), + ++"_basic_test"".fil"), {ok,Fd1} = ?PRIM_FILE:open(Name, [write]), ?PRIM_FILE:write(Fd1, "foo bar"), ok = ?PRIM_FILE:close(Fd1), %% Test that the file has the expected attributes. %% The times are tricky, so we will save them to a separate test case. - {ok, FileInfo} = ?PRIM_FILE_call(read_file_info, Handle, [Name]), + {ok, FileInfo} = ?PRIM_FILE:read_file_info(Name), #file_info{size = Size, type = Type, access = Access, atime = AccessTime, mtime = ModifyTime} = FileInfo, @@ -768,39 +680,30 @@ file_info_basic_file(Config, Handle, Suffix) -> ok. -file_info_basic_directory_a(Config) when is_list(Config) -> - file_info_basic_directory(Config, []). - -file_info_basic_directory_b(Config) when is_list(Config) -> - {ok, Handle} = ?PRIM_FILE:start(), - Result = file_info_basic_directory(Config, Handle), - ok = ?PRIM_FILE:stop(Handle), - Result. - -file_info_basic_directory(Config, Handle) -> +file_info_basic_directory(Config) when is_list(Config) -> %% Note: filename:join/1 removes any trailing slash, %% which is essential for ?PRIM_FILE:read_file_info/1 to work on %% platforms such as Windows95. RootDir = filename:join([proplists:get_value(priv_dir, Config)]), %% Test that the RootDir directory has the expected attributes. - test_directory(RootDir, read_write, Handle), + test_directory(RootDir, read_write), %% Note that on Windows file systems, "/" or "c:/" are *NOT* directories. %% Therefore, test that ?PRIM_FILE:read_file_info/1 behaves %% as if they were directories. case os:type() of {win32, _} -> - test_directory("/", read_write, Handle), - test_directory("c:/", read_write, Handle), - test_directory("c:\\", read_write, Handle); + test_directory("/", read_write), + test_directory("c:/", read_write), + test_directory("c:\\", read_write); _ -> - test_directory("/", read, Handle) + test_directory("/", read) end, ok. -test_directory(Name, ExpectedAccess, Handle) -> - {ok, FileInfo} = ?PRIM_FILE_call(read_file_info, Handle, [Name]), +test_directory(Name, ExpectedAccess) -> + {ok, FileInfo} = ?PRIM_FILE:read_file_info(Name), #file_info{size = Size, type = Type, access = Access, atime = AccessTime, mtime = ModifyTime} = FileInfo, @@ -824,45 +727,24 @@ all_integers([]) -> %% Try something nonexistent. -file_info_bad_a(Config) when is_list(Config) -> - file_info_bad(Config, []). - -file_info_bad_b(Config) when is_list(Config) -> - {ok, Handle} = ?PRIM_FILE:start(), - Result = file_info_bad(Config, Handle), - ok = ?PRIM_FILE:stop(Handle), - Result. - -file_info_bad(Config, Handle) -> +file_info_bad(Config) when is_list(Config) -> RootDir = filename:join([proplists:get_value(priv_dir, Config)]), - {error, enoent} = - ?PRIM_FILE_call( - read_file_info, Handle, - [filename:join(RootDir, - atom_to_list(?MODULE)++"_nonexistent")]), + NonExistent = filename:join(RootDir, atom_to_list(?MODULE)++"_nonexistent"), + {error, enoent} = ?PRIM_FILE:read_file_info(NonExistent), ok. %% Test that the file times behave as they should. -file_info_times_a(Config) when is_list(Config) -> - file_info_times(Config, [], "_a"). - -file_info_times_b(Config) when is_list(Config) -> - {ok, Handle} = ?PRIM_FILE:start(), - Result = file_info_times(Config, Handle, "_b"), - ok = ?PRIM_FILE:stop(Handle), - Result. - -file_info_times(Config, Handle, Suffix) -> +file_info_times(Config) when is_list(Config) -> %% We have to try this twice, since if the test runs across the change %% of a month the time diff calculations will fail. But it won't happen %% if you run it twice in succession. test_server:m_out_of_n( 1,2, - fun() -> file_info_int(Config, Handle, Suffix) end), + fun() -> file_info_int(Config) end), ok. -file_info_int(Config, Handle, Suffix) -> +file_info_int(Config) -> %% Note: filename:join/1 removes any trailing slash, %% which is essential for ?PRIM_FILE:read_file_info/1 to work on %% platforms such as Windows95. @@ -872,14 +754,14 @@ file_info_int(Config, Handle, Suffix) -> Name = filename:join(RootDir, atom_to_list(?MODULE) - ++"_file_info"++Suffix++".fil"), + ++"_file_info.fil"), {ok,Fd1} = ?PRIM_FILE:open(Name, [write]), ?PRIM_FILE:write(Fd1,"foo"), %% check that the file got a modify date max a few seconds away from now {ok, #file_info{type = regular, atime = AccTime1, mtime = ModTime1}} = - ?PRIM_FILE_call(read_file_info, Handle, [Name]), + ?PRIM_FILE:read_file_info(Name), Now = erlang:localtime(), io:format("Now ~p",[Now]), io:format("Open file Acc ~p Mod ~p",[AccTime1,ModTime1]), @@ -897,7 +779,7 @@ file_info_int(Config, Handle, Suffix) -> ok = ?PRIM_FILE:close(Fd1), {ok, #file_info{size = Size, type = regular, access = Access, atime = AccTime2, mtime = ModTime2}} = - ?PRIM_FILE_call(read_file_info, Handle, [Name]), + ?PRIM_FILE:read_file_info(Name), io:format("Closed file Acc ~p Mod ~p",[AccTime2,ModTime2]), true = time_dist(ModTime1, ModTime2) >= 0, @@ -909,7 +791,7 @@ file_info_int(Config, Handle, Suffix) -> {ok, #file_info{size = DSize, type = directory, access = DAccess, atime = AccTime3, mtime = ModTime3}} = - ?PRIM_FILE_call(read_file_info, Handle, [RootDir]), + ?PRIM_FILE:read_file_info(RootDir), %% this dir was modified only a few secs ago io:format("Dir Acc ~p; Mod ~p; Now ~p", [AccTime3, ModTime3, Now]), @@ -936,16 +818,7 @@ filter_atime(Atime, Config) -> %% Test the write_file_info/2 function. -file_write_file_info_a(Config) when is_list(Config) -> - file_write_file_info(Config, [], "_a"). - -file_write_file_info_b(Config) when is_list(Config) -> - {ok, Handle} = ?PRIM_FILE:start(), - Result = file_write_file_info(Config, Handle, "_b"), - ok = ?PRIM_FILE:stop(Handle), - Result. - -file_write_file_info(Config, Handle, Suffix) -> +file_write_file_info(Config) when is_list(Config) -> RootDir = get_good_directory(Config), io:format("RootDir = ~p", [RootDir]), @@ -955,16 +828,16 @@ file_write_file_info(Config, Handle, Suffix) -> Name = filename:join(RootDir, atom_to_list(?MODULE) - ++"_write_file_info_ro"++Suffix), + ++"_write_file_info_ro"), ok = ?PRIM_FILE:write_file(Name, "hello"), Time = {{1997, 01, 02}, {12, 35, 42}}, Info = #file_info{mode=8#400, atime=Time, mtime=Time, ctime=Time}, - ok = ?PRIM_FILE_call(write_file_info, Handle, [Name, Info]), + ok = ?PRIM_FILE:write_file_info(Name, Info), %% Read back the times. {ok, ActualInfo} = - ?PRIM_FILE_call(read_file_info, Handle, [Name]), + ?PRIM_FILE:read_file_info(Name), #file_info{mode=_Mode, atime=ActAtime, mtime=Time, ctime=ActCtime} = ActualInfo, FilteredAtime = filter_atime(Time, Config), @@ -980,14 +853,11 @@ file_write_file_info(Config, Handle, Suffix) -> {error, eacces} = ?PRIM_FILE:write_file(Name, "hello again"), %% Make the file writable again. - - ?PRIM_FILE_call(write_file_info, Handle, - [Name, #file_info{mode=8#600}]), + ?PRIM_FILE:write_file_info(Name, #file_info{mode=8#600}), ok = ?PRIM_FILE:write_file(Name, "hello again"), %% And unwritable. - ?PRIM_FILE_call(write_file_info, Handle, - [Name, #file_info{mode=8#400}]), + ?PRIM_FILE:write_file_info(Name, #file_info{mode=8#400}), {error, eacces} = ?PRIM_FILE:write_file(Name, "hello again"), %% Write the times again. @@ -995,9 +865,9 @@ file_write_file_info(Config, Handle, Suffix) -> NewTime = {{1997, 02, 15}, {13, 18, 20}}, NewInfo = #file_info{atime=NewTime, mtime=NewTime, ctime=NewTime}, - ok = ?PRIM_FILE_call(write_file_info, Handle, [Name, NewInfo]), + ok = ?PRIM_FILE:write_file_info(Name, NewInfo), {ok, ActualInfo2} = - ?PRIM_FILE_call(read_file_info, Handle, [Name]), + ?PRIM_FILE:read_file_info(Name), #file_info{atime=NewActAtime, mtime=NewTime, ctime=NewActCtime} = ActualInfo2, NewFilteredAtime = filter_atime(NewTime, Config), @@ -1012,14 +882,12 @@ file_write_file_info(Config, Handle, Suffix) -> %% Make the file writeable again, so that we can remove the %% test suites ... :-) - ?PRIM_FILE_call(write_file_info, Handle, - [Name, #file_info{mode=8#600}]), + ?PRIM_FILE:write_file_info(Name, #file_info{mode=8#600}), ok. %% Test the write_file_info/3 function. file_write_file_info_opts(Config) when is_list(Config) -> - {ok, Handle} = ?PRIM_FILE:start(), RootDir = get_good_directory(Config), io:format("RootDir = ~p", [RootDir]), @@ -1028,7 +896,7 @@ file_write_file_info_opts(Config) when is_list(Config) -> lists:foreach(fun ({FI, Opts}) -> - ok = ?PRIM_FILE_call(write_file_info, Handle, [Name, FI, Opts]) + ok = ?PRIM_FILE:write_file_info(Name, FI, Opts) end, [ {#file_info{ mode=8#600, atime = Time, mtime = Time, ctime = Time}, Opts} || Opts <- [[{time, posix}]], @@ -1038,7 +906,7 @@ file_write_file_info_opts(Config) when is_list(Config) -> %% REM: determine date range dependent on time_t = Uint32 | Sint32 | Sint64 | Uint64 %% Determine time_t on os:type()? lists:foreach(fun ({FI, Opts}) -> - ok = ?PRIM_FILE_call(write_file_info, Handle, [Name, FI, Opts]) + ok = ?PRIM_FILE:write_file_info(Name, FI, Opts) end, [ {#file_info{ mode=8#400, atime = Time, mtime = Time, ctime = Time}, Opts} || Opts <- [[{time, universal}],[{time, local}]], Time <- [ @@ -1050,11 +918,9 @@ file_write_file_info_opts(Config) when is_list(Config) -> {{2037,2,3},{23,59,59}}, erlang:localtime() ]]), - ok = ?PRIM_FILE:stop(Handle), ok. file_read_file_info_opts(Config) when is_list(Config) -> - {ok, Handle} = ?PRIM_FILE:start(), RootDir = get_good_directory(Config), io:format("RootDir = ~p", [RootDir]), @@ -1063,41 +929,38 @@ file_read_file_info_opts(Config) when is_list(Config) -> lists:foreach(fun (Opts) -> - {ok,_} = ?PRIM_FILE_call(read_file_info, Handle, [Name, Opts]) + {ok,_} = ?PRIM_FILE:read_file_info(Name, Opts) end, [[{time, Type}] || Type <- [local, universal, posix]]), - ok = ?PRIM_FILE:stop(Handle), ok. %% Test the write and read back *_file_info/3 functions. file_write_read_file_info_opts(Config) when is_list(Config) -> - {ok, Handle} = ?PRIM_FILE:start(), RootDir = get_good_directory(Config), io:format("RootDir = ~p", [RootDir]), Name = filename:join(RootDir, atom_to_list(?MODULE) ++"_read_write_file_info_opts"), ok = ?PRIM_FILE:write_file(Name, "hello_opts2"), - ok = file_write_read_file_info_opts(Handle, Name, {{1989, 04, 28}, {19,30,22}}, [{time, local}]), - ok = file_write_read_file_info_opts(Handle, Name, {{1989, 04, 28}, {19,30,22}}, [{time, universal}]), + ok = file_write_read_file_info_opts(Name, {{1989, 04, 28}, {19,30,22}}, [{time, local}]), + ok = file_write_read_file_info_opts(Name, {{1989, 04, 28}, {19,30,22}}, [{time, universal}]), %% will not work on platforms with unsigned time_t - %ok = file_write_read_file_info_opts(Handle, Name, {{1930, 04, 28}, {19,30,22}}, [{time, local}]), - %ok = file_write_read_file_info_opts(Handle, Name, {{1930, 04, 28}, {19,30,22}}, [{time, universal}]), - ok = file_write_read_file_info_opts(Handle, Name, 1, [{time, posix}]), + %ok = file_write_read_file_info_opts(Name, {{1930, 04, 28}, {19,30,22}}, [{time, local}]), + %ok = file_write_read_file_info_opts(Name, {{1930, 04, 28}, {19,30,22}}, [{time, universal}]), + ok = file_write_read_file_info_opts(Name, 1, [{time, posix}]), %% will not work on platforms with unsigned time_t - %ok = file_write_read_file_info_opts(Handle, Name, -1, [{time, posix}]), - %ok = file_write_read_file_info_opts(Handle, Name, -300000, [{time, posix}]), - ok = file_write_read_file_info_opts(Handle, Name, 300000, [{time, posix}]), - ok = file_write_read_file_info_opts(Handle, Name, 0, [{time, posix}]), + %ok = file_write_read_file_info_opts(Name, -1, [{time, posix}]), + %ok = file_write_read_file_info_opts(Name, -300000, [{time, posix}]), + ok = file_write_read_file_info_opts(Name, 300000, [{time, posix}]), + ok = file_write_read_file_info_opts(Name, 0, [{time, posix}]), - ok = ?PRIM_FILE:stop(Handle), ok. -file_write_read_file_info_opts(Handle, Name, Mtime, Opts) -> - {ok, FI} = ?PRIM_FILE_call(read_file_info, Handle, [Name, Opts]), +file_write_read_file_info_opts(Name, Mtime, Opts) -> + {ok, FI} = ?PRIM_FILE:read_file_info(Name, Opts), FI2 = FI#file_info{ mtime = Mtime }, - ok = ?PRIM_FILE_call(write_file_info, Handle, [Name, FI2, Opts]), - {ok, FI3} = ?PRIM_FILE_call(read_file_info, Handle, [Name, Opts]), + ok = ?PRIM_FILE:write_file_info(Name, FI2, Opts), + {ok, FI3} = ?PRIM_FILE:read_file_info(Name, Opts), io:format("Expecting mtime = ~p, got ~p~n", [FI2#file_info.mtime, FI3#file_info.mtime]), FI2 = FI3, ok. @@ -1175,8 +1038,8 @@ advise(Config) when is_list(Config) -> atom_to_list(?MODULE) ++"_advise.fil"), - Line1 = "Hello\n", - Line2 = "World!\n", + Line1 = <<"Hello\n">>, + Line2 = <<"World!\n">>, {ok, Fd} = ?PRIM_FILE:open(Advise, [write]), ok = ?PRIM_FILE:advise(Fd, 0, 0, normal), @@ -1226,7 +1089,7 @@ advise(Config) when is_list(Config) -> {ok, Fd9} = ?PRIM_FILE:open(Advise, [read]), Offset = 0, %% same as a 0 length in some implementations - Length = length(Line1) + length(Line2), + Length = byte_size(Line1) + byte_size(Line2), ok = ?PRIM_FILE:advise(Fd9, Offset, Length, sequential), {ok, Line1} = ?PRIM_FILE:read_line(Fd9), {ok, Line2} = ?PRIM_FILE:read_line(Fd9), @@ -1250,23 +1113,18 @@ do_large_write(Name) -> Chunk = <<0:ChunkSize/unit:8>>, Data = zip_data(lists:duplicate(Chunks, Chunk), Interleave), Size = Chunks * ChunkSize + Chunks, % 4 G + 32 - Wordsize = erlang:system_info(wordsize), - case prim_file:write_file(Name, Data) of - ok when Wordsize =:= 8 -> - {ok,#file_info{size=Size}} = file:read_file_info(Name), - {ok,Fd} = prim_file:open(Name, [read]), - check_large_write(Fd, ChunkSize, 0, Interleave); - {error,einval} when Wordsize =:= 4 -> - ok - end. + ok = ?PRIM_FILE:write_file(Name, Data), + {ok,#file_info{size=Size}} = file:read_file_info(Name), + {ok,Fd} = ?PRIM_FILE:open(Name, [read]), + check_large_write(Fd, ChunkSize, 0, Interleave). check_large_write(Fd, ChunkSize, Pos, [X|Interleave]) -> Pos1 = Pos + ChunkSize, - {ok,Pos1} = prim_file:position(Fd, {cur,ChunkSize}), - {ok,[X]} = prim_file:read(Fd, 1), + {ok,Pos1} = ?PRIM_FILE:position(Fd, {cur,ChunkSize}), + {ok,<<X>>} = ?PRIM_FILE:read(Fd, 1), check_large_write(Fd, ChunkSize, Pos1+1, Interleave); check_large_write(Fd, _, _, []) -> - eof = prim_file:read(Fd, 1), + eof = ?PRIM_FILE:read(Fd, 1), ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1338,71 +1196,53 @@ allocate_and_assert(Fd, Offset, Length) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -delete_a(Config) when is_list(Config) -> - delete(Config, [], "_a"). - -delete_b(Config) when is_list(Config) -> - {ok, Handle} = ?PRIM_FILE:start(), - Result = delete(Config, Handle, "_b"), - ok = ?PRIM_FILE:stop(Handle), - Result. - -delete(Config, Handle, Suffix) -> +delete(Config) when is_list(Config) -> RootDir = proplists:get_value(priv_dir,Config), Name = filename:join(RootDir, atom_to_list(?MODULE) - ++"_delete"++Suffix++".fil"), + ++"_delete.fil"), {ok, Fd1} = ?PRIM_FILE:open(Name, [write]), ?PRIM_FILE:write(Fd1,"ok.\n"), ok = ?PRIM_FILE:close(Fd1), %% Check that the file is readable {ok, Fd2} = ?PRIM_FILE:open(Name, [read]), ok = ?PRIM_FILE:close(Fd2), - ok = ?PRIM_FILE_call(delete, Handle, [Name]), + ok = ?PRIM_FILE:delete(Name), %% Check that the file is not readable anymore {error, _} = ?PRIM_FILE:open(Name, [read]), %% Try deleting a nonexistent file - {error, enoent} = ?PRIM_FILE_call(delete, Handle, [Name]), + {error, enoent} = ?PRIM_FILE:delete(Name), ok. -rename_a(Config) when is_list(Config) -> - rename(Config, [], "_a"). - -rename_b(Config) when is_list(Config) -> - {ok, Handle} = ?PRIM_FILE:start(), - Result = rename(Config, Handle, "_b"), - ok = ?PRIM_FILE:stop(Handle), - Result. - -rename(Config, Handle, Suffix) -> +rename(Config) when is_list(Config) -> RootDir = proplists:get_value(priv_dir,Config), - FileName1 = atom_to_list(?MODULE)++"_rename"++Suffix++".fil", - FileName2 = atom_to_list(?MODULE)++"_rename"++Suffix++".ful", + FileName1 = atom_to_list(?MODULE)++"_rename.fil", + FileName2 = atom_to_list(?MODULE)++"_rename.ful", Name1 = filename:join(RootDir, FileName1), Name2 = filename:join(RootDir, FileName2), {ok,Fd1} = ?PRIM_FILE:open(Name1, [write]), ok = ?PRIM_FILE:close(Fd1), %% Rename, and check that it really changed name - ok = ?PRIM_FILE_call(rename, Handle, [Name1, Name2]), + ok = ?PRIM_FILE:rename(Name1, Name2), {error, _} = ?PRIM_FILE:open(Name1, [read]), {ok, Fd2} = ?PRIM_FILE:open(Name2, [read]), ok = ?PRIM_FILE:close(Fd2), %% Try renaming something to itself - ok = ?PRIM_FILE_call(rename, Handle, [Name2, Name2]), + ok = ?PRIM_FILE:rename(Name2, Name2), %% Try renaming something that doesn't exist {error, enoent} = - ?PRIM_FILE_call(rename, Handle, [Name1, Name2]), + ?PRIM_FILE:rename(Name1, Name2), %% Try renaming to something else than a string {error, badarg} = - ?PRIM_FILE_call(rename, Handle, [Name1, foobar]), + ?PRIM_FILE:rename(Name1, foobar), %% Move between directories DirName1 = filename:join(RootDir, atom_to_list(?MODULE) - ++"_rename_dir"++Suffix), + ++"_rename_dir"), DirName2 = filename:join(RootDir, atom_to_list(?MODULE) - ++"_second_rename_dir"++Suffix), + ++"_second_rename_dir"), Name1foo = filename:join(DirName1, "foo.fil"), Name2foo = filename:join(DirName2, "foo.fil"), Name2bar = filename:join(DirName2, "bar.dir"), @@ -1410,21 +1250,21 @@ rename(Config, Handle, Suffix) -> %% The name has to include the full file name, path is not enough expect( {error, eexist}, {error, eisdir}, - ?PRIM_FILE_call(rename, Handle, [Name2, DirName1])), + ?PRIM_FILE:rename(Name2, DirName1)), ok = - ?PRIM_FILE_call(rename, Handle, [Name2, Name1foo]), + ?PRIM_FILE:rename(Name2, Name1foo), %% Now rename the directory - ok = ?PRIM_FILE_call(rename, Handle, [DirName1, DirName2]), + ok = ?PRIM_FILE:rename(DirName1, DirName2), %% And check that the file is there now {ok,Fd3} = ?PRIM_FILE:open(Name2foo, [read]), ok = ?PRIM_FILE:close(Fd3), %% Try some dirty things now: move the directory into itself {error, Msg1} = - ?PRIM_FILE_call(rename, Handle, [DirName2, Name2bar]), + ?PRIM_FILE:rename(DirName2, Name2bar), io:format("Errmsg1: ~p",[Msg1]), %% move dir into a file in itself {error, Msg2} = - ?PRIM_FILE_call(rename, Handle, [DirName2, Name2foo]), + ?PRIM_FILE:rename(DirName2, Name2foo), io:format("Errmsg2: ~p",[Msg2]), ok. @@ -1657,165 +1497,19 @@ e_del_dir(Config) when is_list(Config) -> ok. -%% Trying reading and positioning from a compressed file. - -read_compressed(Config) when is_list(Config) -> - Data = proplists:get_value(data_dir, Config), - Real = filename:join(Data, "realmen.html.gz"), - {ok, Fd} = ?PRIM_FILE:open(Real, [read, compressed]), - try_read_file(Fd). - -%% Trying reading and positioning from an uncompressed file, -%% but with the compressed flag given. - -read_not_really_compressed(Config) when is_list(Config) -> - Data = proplists:get_value(data_dir, Config), - Priv = proplists:get_value(priv_dir, Config), - - %% The file realmen.html might have got CRs added (by WinZip). - %% Remove them, or the file positions will not be correct. - - Real = filename:join(Data, "realmen.html"), - RealPriv = filename:join(Priv, - atom_to_list(?MODULE)++"_realmen.html"), - {ok, RealDataBin} = ?PRIM_FILE:read_file(Real), - RealData = remove_crs(binary_to_list(RealDataBin), []), - ok = ?PRIM_FILE:write_file(RealPriv, RealData), - {ok, Fd} = ?PRIM_FILE:open(RealPriv, [read, compressed]), - try_read_file(Fd). - -remove_crs([$\r|Rest], Result) -> - remove_crs(Rest, Result); -remove_crs([C|Rest], Result) -> - remove_crs(Rest, [C|Result]); -remove_crs([], Result) -> - lists:reverse(Result). - -try_read_file(Fd) -> - %% Seek to the current position (nothing should happen). - - {ok, 0} = ?PRIM_FILE:position(Fd, 0), - {ok, 0} = ?PRIM_FILE:position(Fd, {cur, 0}), - - %% Read a few lines from a compressed file. - - ShouldBe = "<TITLE>Real Programmers Don't Use PASCAL</TITLE>\n", - {ok, ShouldBe} = ?PRIM_FILE:read(Fd, length(ShouldBe)), - - %% Now seek forward. - - {ok, 381} = ?PRIM_FILE:position(Fd, 381), - Back = "Back in the good old days -- the \"Golden Era\" " ++ - "of computers, it was\n", - {ok, Back} = ?PRIM_FILE:read(Fd, length(Back)), - - %% Try to search forward relative to the current position. - - {ok, CurPos} = ?PRIM_FILE:position(Fd, {cur, 0}), - RealPos = 4273, - {ok, RealPos} = ?PRIM_FILE:position(Fd, {cur, RealPos-CurPos}), - RealProg = "<LI> Real Programmers aren't afraid to use GOTOs.\n", - {ok, RealProg} = ?PRIM_FILE:read(Fd, length(RealProg)), - - %% Seek backward. - - AfterTitle = length("<TITLE>"), - {ok, AfterTitle} = ?PRIM_FILE:position(Fd, AfterTitle), - Title = "Real Programmers Don't Use PASCAL</TITLE>\n", - {ok, Title} = ?PRIM_FILE:read(Fd, length(Title)), - - %% Done. - - ?PRIM_FILE:close(Fd), - ok. - -write_compressed(Config) when is_list(Config) -> - Priv = proplists:get_value(priv_dir, Config), - MyFile = filename:join(Priv, - atom_to_list(?MODULE)++"_test.gz"), - - %% Write a file. - - {ok, Fd} = ?PRIM_FILE:open(MyFile, [write, compressed]), - {ok, 0} = ?PRIM_FILE:position(Fd, 0), - Prefix = "hello\n", - End = "end\n", - ok = ?PRIM_FILE:write(Fd, Prefix), - {ok, 143} = ?PRIM_FILE:position(Fd, 143), - ok = ?PRIM_FILE:write(Fd, End), - ok = ?PRIM_FILE:close(Fd), - - %% Read the file and verify the contents. - - {ok, Fd1} = ?PRIM_FILE:open(MyFile, [read, compressed]), - {ok, Prefix} = ?PRIM_FILE:read(Fd1, length(Prefix)), - Second = lists:duplicate(143-length(Prefix), 0) ++ End, - {ok, Second} = ?PRIM_FILE:read(Fd1, length(Second)), - ok = ?PRIM_FILE:close(Fd1), - - %% Ensure that the file is compressed. - - TotalSize = 143 + length(End), - case ?PRIM_FILE:read_file_info(MyFile) of - {ok, #file_info{size=Size}} when Size < TotalSize -> - ok; - {ok, #file_info{size=Size}} when Size == TotalSize -> - ct:fail(file_not_compressed) - end, - - %% Write again to ensure that the file is truncated. - - {ok, Fd2} = ?PRIM_FILE:open(MyFile, [write, compressed]), - NewString = "aaaaaaaaaaa", - ok = ?PRIM_FILE:write(Fd2, NewString), - ok = ?PRIM_FILE:close(Fd2), - {ok, Fd3} = ?PRIM_FILE:open(MyFile, [read, compressed]), - {ok, NewString} = ?PRIM_FILE:read(Fd3, 1024), - ok = ?PRIM_FILE:close(Fd3), - - ok. - -compress_errors(Config) when is_list(Config) -> - Data = proplists:get_value(data_dir, Config), - {error, enoent} = ?PRIM_FILE:open("non_existing__", - [compressed, read]), - {error, einval} = ?PRIM_FILE:open("non_existing__", - [compressed, read, write]), - - %% Read a corrupted .gz file. - - Corrupted = filename:join(Data, "corrupted.gz"), - {ok, Fd} = ?PRIM_FILE:open(Corrupted, [read, compressed]), - {error, eio} = ?PRIM_FILE:read(Fd, 100), - ?PRIM_FILE:close(Fd), - - ok. - - -%% Test creating a hard link. -make_link_a(Config) when is_list(Config) -> - make_link(Config, [], "_a"). - -%% Test creating a hard link. -make_link_b(Config) when is_list(Config) -> - {ok, Handle} = ?PRIM_FILE:start(), - Result = make_link(Config, Handle, "_b"), - ok = ?PRIM_FILE:stop(Handle), - Result. - -make_link(Config, Handle, Suffix) -> +make_link(Config) when is_list(Config) -> RootDir = proplists:get_value(priv_dir, Config), NewDir = filename:join(RootDir, atom_to_list(?MODULE) - ++"_make_link"++Suffix), - ok = ?PRIM_FILE_call(make_dir, Handle, [NewDir]), + ++"_make_link"), + ok = ?PRIM_FILE:make_dir(NewDir), Name = filename:join(NewDir, "a_file"), ok = ?PRIM_FILE:write_file(Name, "some contents\n"), Alias = filename:join(NewDir, "an_alias"), Result = - case ?PRIM_FILE_call(make_link, Handle, [Name, Alias]) of + case ?PRIM_FILE:make_link(Name, Alias) of {error, enotsup} -> {skipped, "Links not supported on this platform"}; ok -> @@ -1826,12 +1520,12 @@ make_link(Config, Handle, Suffix) -> %% since they are not used on symbolic links. {ok, Info} = - ?PRIM_FILE_call(read_link_info, Handle, [Name]), + ?PRIM_FILE:read_link_info(Name), {ok, Info} = - ?PRIM_FILE_call(read_link_info, Handle, [Alias]), + ?PRIM_FILE:read_link_info(Alias), #file_info{links = 2, type = regular} = Info, {error, eexist} = - ?PRIM_FILE_call(make_link, Handle, [Name, Alias]), + ?PRIM_FILE:make_link(Name, Alias), ok end, @@ -1843,30 +1537,19 @@ read_link_info_for_non_link(Config) when is_list(Config) -> {ok, #file_info{type=directory}} = ?PRIM_FILE:read_link_info("."), ok. -%% Test operations on symbolic links (for Unix). -symlinks_a(Config) when is_list(Config) -> - symlinks(Config, [], "_a"). - -%% Test operations on symbolic links (for Unix). -symlinks_b(Config) when is_list(Config) -> - {ok, Handle} = ?PRIM_FILE:start(), - Result = symlinks(Config, Handle, "_b"), - ok = ?PRIM_FILE:stop(Handle), - Result. - -symlinks(Config, Handle, Suffix) -> +symlinks(Config) when is_list(Config) -> RootDir = proplists:get_value(priv_dir, Config), NewDir = filename:join(RootDir, atom_to_list(?MODULE) - ++"_make_symlink"++Suffix), - ok = ?PRIM_FILE_call(make_dir, Handle, [NewDir]), + ++"_make_symlink"), + ok = ?PRIM_FILE:make_dir(NewDir), Name = filename:join(NewDir, "a_plain_file"), ok = ?PRIM_FILE:write_file(Name, "some stupid content\n"), Alias = filename:join(NewDir, "a_symlink_alias"), Result = - case ?PRIM_FILE_call(make_symlink, Handle, [Name, Alias]) of + case ?PRIM_FILE:make_symlink(Name, Alias) of {error, enotsup} -> {skipped, "Links not supported on this platform"}; {error, eperm} -> @@ -1874,20 +1557,20 @@ symlinks(Config, Handle, Suffix) -> {skipped, "Windows user not privileged to create links"}; ok -> {ok, Info1} = - ?PRIM_FILE_call(read_file_info, Handle, [Name]), + ?PRIM_FILE:read_file_info(Name), {ok, Info1} = - ?PRIM_FILE_call(read_file_info, Handle, [Alias]), + ?PRIM_FILE:read_file_info(Alias), {ok, Info1} = - ?PRIM_FILE_call(read_link_info, Handle, [Name]), + ?PRIM_FILE:read_link_info(Name), #file_info{links = 1, type = regular} = Info1, {ok, Info2} = - ?PRIM_FILE_call(read_link_info, Handle, [Alias]), + ?PRIM_FILE:read_link_info(Alias), #file_info{links=1, type=symlink} = Info2, {ok, Name} = - ?PRIM_FILE_call(read_link, Handle, [Alias]), + ?PRIM_FILE:read_link(Alias), {ok, Name} = - ?PRIM_FILE_call(read_link_all, Handle, [Alias]), + ?PRIM_FILE:read_link_all(Alias), %% If all is good, delete dir again (avoid hanging dir on windows) rm_rf(?PRIM_FILE,NewDir), ok @@ -1907,10 +1590,9 @@ list_dir_limit(Config) when is_list(Config) -> RootDir = proplists:get_value(priv_dir, Config), NewDir = filename:join(RootDir, atom_to_list(?MODULE)++"_list_dir_limit"), - {ok, Handle1} = ?PRIM_FILE:start(), - ok = ?PRIM_FILE_call(make_dir, Handle1, [NewDir]), + ok = ?PRIM_FILE:make_dir(NewDir), Ref = erlang:start_timer(MaxTime*1000, self(), []), - Result = list_dir_limit_loop(NewDir, Handle1, Ref, MaxNumber, 0), + Result = list_dir_limit_loop(NewDir, Ref, MaxNumber, 0), Time = case erlang:cancel_timer(Ref) of false -> MaxTime; T -> MaxTime - (T div 1000) @@ -1920,21 +1602,18 @@ list_dir_limit(Config) when is_list(Config) -> {error, _Reason, N} -> N; _ -> 0 end, - {ok, Handle2} = ?PRIM_FILE:start(), - list_dir_limit_cleanup(NewDir, Handle2, Number, 0), - ok = ?PRIM_FILE:stop(Handle1), - ok = ?PRIM_FILE:stop(Handle2), + list_dir_limit_cleanup(NewDir, Number, 0), {ok, Number} = Result, {comment, "Created " ++ integer_to_list(Number) ++ " files in " ++ integer_to_list(Time) ++ " seconds."}. -list_dir_limit_loop(Dir, Handle, _Ref, N, Cnt) when Cnt >= N -> - list_dir_check(Dir, Handle, Cnt); -list_dir_limit_loop(Dir, Handle, Ref, N, Cnt) -> +list_dir_limit_loop(Dir, _Ref, N, Cnt) when Cnt >= N -> + list_dir_check(Dir, Cnt); +list_dir_limit_loop(Dir, Ref, N, Cnt) -> receive {timeout, Ref, []} -> - list_dir_check(Dir, Handle, Cnt) + list_dir_check(Dir, Cnt) after 0 -> Name = integer_to_list(Cnt), case ?PRIM_FILE:write_file(filename:join(Dir, Name), Name) of @@ -1942,23 +1621,23 @@ list_dir_limit_loop(Dir, Handle, Ref, N, Cnt) -> Next = Cnt + 1, case Cnt rem 100 of 0 -> - case list_dir_check(Dir, Handle, Next) of + case list_dir_check(Dir, Next) of {ok, Next} -> list_dir_limit_loop( - Dir, Handle, Ref, N, Next); + Dir, Ref, N, Next); Other -> Other end; _ -> - list_dir_limit_loop(Dir, Handle, Ref, N, Next) + list_dir_limit_loop(Dir, Ref, N, Next) end; {error, Reason} -> {error, Reason, Cnt} end end. -list_dir_check(Dir, Handle, Cnt) -> - case ?PRIM_FILE:list_dir(Handle, Dir) of +list_dir_check(Dir, Cnt) -> + case ?PRIM_FILE:list_dir(Dir) of {ok, ListDir} -> case length(ListDir) of Cnt -> @@ -1975,18 +1654,18 @@ list_dir_check(Dir, Handle, Cnt) -> %% Deletes N files while ignoring errors, then continues deleting %% as long as they exist. -list_dir_limit_cleanup(Dir, Handle, N, Cnt) when Cnt >= N -> +list_dir_limit_cleanup(Dir, N, Cnt) when Cnt >= N -> Name = integer_to_list(Cnt), - case ?PRIM_FILE:delete(Handle, filename:join(Dir, Name)) of + case ?PRIM_FILE:delete(filename:join(Dir, Name)) of ok -> - list_dir_limit_cleanup(Dir, Handle, N, Cnt+1); + list_dir_limit_cleanup(Dir, N, Cnt+1); _ -> ok end; -list_dir_limit_cleanup(Dir, Handle, N, Cnt) -> +list_dir_limit_cleanup(Dir, N, Cnt) -> Name = integer_to_list(Cnt), - ?PRIM_FILE:delete(Handle, filename:join(Dir, Name)), - list_dir_limit_cleanup(Dir, Handle, N, Cnt+1). + ?PRIM_FILE:delete(filename:join(Dir, Name)), + list_dir_limit_cleanup(Dir, N, Cnt+1). %%% %%% Test list_dir() on a non-existing pathname. @@ -1995,7 +1674,7 @@ list_dir_limit_cleanup(Dir, Handle, N, Cnt) -> list_dir_error(Config) -> Priv = proplists:get_value(priv_dir, Config), NonExisting = filename:join(Priv, "non-existing-dir"), - {error,enoent} = prim_file:list_dir(NonExisting), + {error,enoent} = ?PRIM_FILE:list_dir(NonExisting), ok. %%% @@ -2063,7 +1742,7 @@ do_run_large_file_test(Config, Run, Name0) -> {'DOWN',Mref,_,_,_} -> ok; {Tester,done} -> ok end, - prim_file:delete(Name) + ?PRIM_FILE:delete(Name) end), %% Run the test case. diff --git a/lib/kernel/test/sendfile_SUITE.erl b/lib/kernel/test/sendfile_SUITE.erl index bfa564c32c..0c0b1cbcb6 100644 --- a/lib/kernel/test/sendfile_SUITE.erl +++ b/lib/kernel/test/sendfile_SUITE.erl @@ -23,30 +23,41 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("kernel/include/file.hrl"). --compile(export_all). - -all() -> [{group,async_threads}, - {group,no_async_threads}]. - -groups() -> - [{async_threads,[],tcs()}, - {no_async_threads,[],tcs()}]. - -tcs() -> - [t_sendfile_small - ,t_sendfile_big_all - ,t_sendfile_big_size - ,t_sendfile_many_small - ,t_sendfile_partial - ,t_sendfile_offset - ,t_sendfile_sendafter - ,t_sendfile_recvafter - ,t_sendfile_recvafter_remoteclose - ,t_sendfile_sendduring - ,t_sendfile_recvduring - ,t_sendfile_closeduring - ,t_sendfile_crashduring - ]. +-export([all/0, init_per_suite/1, end_per_suite/1, init_per_testcase/2]). + +-export([sendfile_server/2, sendfile_do_recv/2, init/1, handle_event/2]). + +-export( + [t_sendfile_small/1, + t_sendfile_big_all/1, + t_sendfile_big_size/1, + t_sendfile_many_small/1, + t_sendfile_partial/1, + t_sendfile_offset/1, + t_sendfile_sendafter/1, + t_sendfile_recvafter/1, + t_sendfile_recvafter_remoteclose/1, + t_sendfile_sendduring/1, + t_sendfile_recvduring/1, + t_sendfile_closeduring/1, + t_sendfile_crashduring/1, + t_sendfile_arguments/1]). + +all() -> + [t_sendfile_small, + t_sendfile_big_all, + t_sendfile_big_size, + t_sendfile_many_small, + t_sendfile_partial, + t_sendfile_offset, + t_sendfile_sendafter, + t_sendfile_recvafter, + t_sendfile_recvafter_remoteclose, + t_sendfile_sendduring, + t_sendfile_recvduring, + t_sendfile_closeduring, + t_sendfile_crashduring, + t_sendfile_arguments]. init_per_suite(Config) -> case {os:type(),os:version()} of @@ -72,28 +83,18 @@ init_per_suite(Config) -> end_per_suite(Config) -> file:delete(proplists:get_value(big_file, Config)). -init_per_group(async_threads,Config) -> - case erlang:system_info(thread_pool_size) of - 0 -> - {skip,"No async threads"}; - _ -> - [{sendfile_opts,[{use_threads,true}]}|Config] - end; -init_per_group(no_async_threads,Config) -> - [{sendfile_opts,[{use_threads,false}]}|Config]. - -end_per_group(_,_Config) -> - ok. - init_per_testcase(TC,Config) when TC == t_sendfile_recvduring; TC == t_sendfile_sendduring -> Filename = proplists:get_value(small_file, Config), Send = fun(Sock) -> {_Size, Data} = sendfile_file_info(Filename), - {ok,D} = file:open(Filename, [raw,binary,read]), - prim_file:sendfile(D, Sock, 0, 0, 0, - [],[],[]), + {ok,Fd} = file:open(Filename, [raw,binary,read]), + %% Determine whether the driver has native support by + %% hitting the raw module directly; file:sendfile/5 will + %% land in the fallback if it doesn't. + RawModule = Fd#file_descriptor.module, + {ok, _Ignored} = RawModule:sendfile(Fd,Sock,0,0,0,[],[],[]), Data end, @@ -105,9 +106,8 @@ init_per_testcase(TC,Config) when TC == t_sendfile_recvduring; ct:log("Error: ~p",[Error]), {skip,"Not supported"} end; -init_per_testcase(_Tc,Config) -> - Config ++ [{sendfile_opts,[{use_threads,false}]}]. - +init_per_testcase(_TC,Config) -> + Config. t_sendfile_small(Config) when is_list(Config) -> Filename = proplists:get_value(small_file, Config), @@ -124,7 +124,7 @@ t_sendfile_small(Config) when is_list(Config) -> t_sendfile_many_small(Config) when is_list(Config) -> Filename = proplists:get_value(small_file, Config), FileOpts = proplists:get_value(file_opts, Config, []), - SendfileOpts = proplists:get_value(sendfile_opts, Config), + SendfileOpts = proplists:get_value(sendfile_opts, Config, []), error_logger:add_report_handler(?MODULE,[self()]), @@ -151,7 +151,7 @@ t_sendfile_many_small(Config) when is_list(Config) -> t_sendfile_big_all(Config) when is_list(Config) -> Filename = proplists:get_value(big_file, Config), - SendfileOpts = proplists:get_value(sendfile_opts, Config), + SendfileOpts = proplists:get_value(sendfile_opts, Config, []), Send = fun(Sock) -> {ok, #file_info{size = Size}} = @@ -165,7 +165,7 @@ t_sendfile_big_all(Config) when is_list(Config) -> t_sendfile_big_size(Config) -> Filename = proplists:get_value(big_file, Config), FileOpts = proplists:get_value(file_opts, Config, []), - SendfileOpts = proplists:get_value(sendfile_opts, Config), + SendfileOpts = proplists:get_value(sendfile_opts, Config, []), SendAll = fun(Sock) -> {ok, #file_info{size = Size}} = @@ -180,7 +180,7 @@ t_sendfile_big_size(Config) -> t_sendfile_partial(Config) -> Filename = proplists:get_value(small_file, Config), FileOpts = proplists:get_value(file_opts, Config, []), - SendfileOpts = proplists:get_value(sendfile_opts, Config), + SendfileOpts = proplists:get_value(sendfile_opts, Config, []), SendSingle = fun(Sock) -> {_Size, <<Data:5/binary,_/binary>>} = @@ -217,7 +217,7 @@ t_sendfile_partial(Config) -> t_sendfile_offset(Config) -> Filename = proplists:get_value(small_file, Config), FileOpts = proplists:get_value(file_opts, Config, []), - SendfileOpts = proplists:get_value(sendfile_opts, Config), + SendfileOpts = proplists:get_value(sendfile_opts, Config, []), Send = fun(Sock) -> {_Size, <<_:5/binary,Data:3/binary,_/binary>> = AllData} = @@ -233,7 +233,7 @@ t_sendfile_offset(Config) -> t_sendfile_sendafter(Config) -> Filename = proplists:get_value(small_file, Config), - SendfileOpts = proplists:get_value(sendfile_opts, Config), + SendfileOpts = proplists:get_value(sendfile_opts, Config, []), Send = fun(Sock) -> {Size, Data} = sendfile_file_info(Filename), @@ -246,7 +246,7 @@ t_sendfile_sendafter(Config) -> t_sendfile_recvafter(Config) -> Filename = proplists:get_value(small_file, Config), - SendfileOpts = proplists:get_value(sendfile_opts, Config), + SendfileOpts = proplists:get_value(sendfile_opts, Config, []), Send = fun(Sock) -> {Size, Data} = sendfile_file_info(Filename), @@ -279,7 +279,7 @@ t_sendfile_recvafter_remoteclose(Config) -> t_sendfile_sendduring(Config) -> Filename = proplists:get_value(big_file, Config), - SendfileOpts = proplists:get_value(sendfile_opts, Config), + SendfileOpts = proplists:get_value(sendfile_opts, Config, []), Send = fun(Sock) -> {ok, #file_info{size = Size}} = @@ -296,7 +296,7 @@ t_sendfile_sendduring(Config) -> t_sendfile_recvduring(Config) -> Filename = proplists:get_value(big_file, Config), - SendfileOpts = proplists:get_value(sendfile_opts, Config), + SendfileOpts = proplists:get_value(sendfile_opts, Config, []), Send = fun(Sock) -> {ok, #file_info{size = Size}} = @@ -315,7 +315,7 @@ t_sendfile_recvduring(Config) -> t_sendfile_closeduring(Config) -> Filename = proplists:get_value(big_file, Config), - SendfileOpts = proplists:get_value(sendfile_opts, Config), + SendfileOpts = proplists:get_value(sendfile_opts, Config, []), Send = fun(Sock,SFServPid) -> spawn_link(fun() -> @@ -345,7 +345,7 @@ t_sendfile_closeduring(Config) -> t_sendfile_crashduring(Config) -> Filename = proplists:get_value(big_file, Config), - SendfileOpts = proplists:get_value(sendfile_opts, Config), + SendfileOpts = proplists:get_value(sendfile_opts, Config, []), error_logger:add_report_handler(?MODULE,[self()]), @@ -373,6 +373,36 @@ t_sendfile_crashduring(Config) -> end end. +t_sendfile_arguments(Config) -> + Filename = proplists:get_value(small_file, Config), + + {ok, Listener} = gen_tcp:listen(0, + [{packet, 0}, {active, false}, {reuseaddr, true}]), + {ok, Port} = inet:port(Listener), + + ErrorCheck = + fun(Reason, Offset, Length, Opts) -> + {ok, Sender} = gen_tcp:connect({127, 0, 0, 1}, Port, + [{packet, 0}, {active, false}]), + {ok, Receiver} = gen_tcp:accept(Listener), + {ok, Fd} = file:open(Filename, [read, raw]), + {error, Reason} = file:sendfile(Fd, Sender, Offset, Length, Opts), + gen_tcp:close(Receiver), + gen_tcp:close(Sender), + file:close(Fd) + end, + + ErrorCheck(einval, -1, 0, []), + ErrorCheck(einval, 0, -1, []), + ErrorCheck(badarg, gurka, 0, []), + ErrorCheck(badarg, 0, gurka, []), + ErrorCheck(badarg, 0, 0, gurka), + ErrorCheck(badarg, 0, 0, [{chunk_size, gurka}]), + + gen_tcp:close(Listener), + + ok. + %% Generic sendfile server code sendfile_send(Send) -> sendfile_send({127,0,0,1},Send). diff --git a/lib/kernel/vsn.mk b/lib/kernel/vsn.mk index cef54dd41a..106bda01ca 100644 --- a/lib/kernel/vsn.mk +++ b/lib/kernel/vsn.mk @@ -1 +1 @@ -KERNEL_VSN = 5.4 +KERNEL_VSN = 5.4.1 diff --git a/lib/megaco/doc/src/notes.xml b/lib/megaco/doc/src/notes.xml index deb2bfcff3..54e048a172 100644 --- a/lib/megaco/doc/src/notes.xml +++ b/lib/megaco/doc/src/notes.xml @@ -37,7 +37,22 @@ section is the version number of Megaco.</p> - <section><title>Megaco 3.18.2</title> + <section><title>Megaco 3.18.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + +<section><title>Megaco 3.18.2</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/megaco/vsn.mk b/lib/megaco/vsn.mk index 9c6ba5bba0..a4f7de7f07 100644 --- a/lib/megaco/vsn.mk +++ b/lib/megaco/vsn.mk @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = megaco -MEGACO_VSN = 3.18.2 +MEGACO_VSN = 3.18.3 PRE_VSN = APP_VSN = "$(APPLICATION)-$(MEGACO_VSN)$(PRE_VSN)" diff --git a/lib/mnesia/doc/src/notes.xml b/lib/mnesia/doc/src/notes.xml index 026c6a89d7..ba94e913f5 100644 --- a/lib/mnesia/doc/src/notes.xml +++ b/lib/mnesia/doc/src/notes.xml @@ -39,7 +39,49 @@ thus constitutes one section in this document. The title of each section is the version number of Mnesia.</p> - <section><title>Mnesia 4.15.1</title> + <section><title>Mnesia 4.15.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Removed a quadratic behavior in startup. This change + implies that backend plugins (if used) must be set when + the schema is created or via configuration parameters + before mnesia is started.</p> + <p> + Own Id: OTP-14829 Aux Id: ERIERL-84 </p> + </item> + <item> + <p> + Bad timing could crash mnesia after a checkpoint was + deactivated and reactivated with the same checkpoint name + on different tables.</p> + <p> + Own Id: OTP-14841 Aux Id: ERIERL-113 </p> + </item> + </list> + </section> + +</section> + +<section><title>Mnesia 4.15.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix backup error handling, the real failure reason was + not returned.</p> + <p> + Own Id: OTP-14776 Aux Id: ERIERL-103 </p> + </item> + </list> + </section> + +</section> + +<section><title>Mnesia 4.15.1</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/mnesia/src/mnesia.erl b/lib/mnesia/src/mnesia.erl index 190fb2b56d..95f7d4afc1 100644 --- a/lib/mnesia/src/mnesia.erl +++ b/lib/mnesia/src/mnesia.erl @@ -151,7 +151,8 @@ {'snmp', SnmpStruct::term()} | {'storage_properties', [{Backend::module(), [BackendProp::_]}]} | {'type', 'set' | 'ordered_set' | 'bag'} | - {'local_content', boolean()}. + {'local_content', boolean()} | + {'user_properties', proplists:proplist()}. -type t_result(Res) :: {'atomic', Res} | {'aborted', Reason::term()}. -type activity() :: 'ets' | 'async_dirty' | 'sync_dirty' | 'transaction' | 'sync_transaction' | diff --git a/lib/mnesia/src/mnesia_checkpoint.erl b/lib/mnesia/src/mnesia_checkpoint.erl index 2ff77326a9..8112378ffd 100644 --- a/lib/mnesia/src/mnesia_checkpoint.erl +++ b/lib/mnesia/src/mnesia_checkpoint.erl @@ -857,9 +857,9 @@ retainer_loop(Cp = #checkpoint_args{is_activated=false, name=Name}) -> retainer_loop(Cp = #checkpoint_args{name=Name}) -> receive {_From, {retain, Tid, Tab, Key, OldRecs}} -> - R = val({Tab, {retainer, Name}}), + R = ?catch_val({Tab, {retainer, Name}}), PendingTab = Cp#checkpoint_args.pending_tab, - case R#retainer.really_retain of + case is_record(R, retainer) andalso R#retainer.really_retain of true -> Store = R#retainer.store, try true = ets:member(PendingTab, Tid), diff --git a/lib/mnesia/src/mnesia_log.erl b/lib/mnesia/src/mnesia_log.erl index 55b1d6e419..a2de23a2a3 100644 --- a/lib/mnesia/src/mnesia_log.erl +++ b/lib/mnesia/src/mnesia_log.erl @@ -752,8 +752,8 @@ abort_write(B, What, Args, Reason) -> Opaque = B#backup_args.opaque, dbg_out("Failed to perform backup. M=~p:F=~tp:A=~tp -> ~tp~n", [Mod, What, Args, Reason]), - try apply(Mod, abort_write, [Opaque]) of - {ok, _Res} -> throw({error, Reason}) + try {ok, _Res} = apply(Mod, abort_write, [Opaque]) of + _ -> throw({error, Reason}) catch _:Other -> error("Failed to abort backup. ~p:~tp~tp -> ~tp~n", [Mod, abort_write, [Opaque], Other]), diff --git a/lib/mnesia/src/mnesia_schema.erl b/lib/mnesia/src/mnesia_schema.erl index 83cc19c678..71952af31c 100644 --- a/lib/mnesia/src/mnesia_schema.erl +++ b/lib/mnesia/src/mnesia_schema.erl @@ -952,19 +952,9 @@ get_index_plugins() -> get_schema_user_property(mnesia_index_plugins). get_schema_user_property(Key) -> - Tab = schema, - %% Must work reliably both within transactions and outside of transactions - Res = case get(mnesia_activity_state) of - undefined -> - dirty_read_table_property(Tab, Key); - _ -> - do_read_table_property(Tab, Key) - end, - case Res of - undefined -> - []; - {_, Types} -> - Types + case dirty_read_table_property(schema, Key) of + undefined -> []; + {_, Types} -> Types end. get_ext_types_disc() -> diff --git a/lib/mnesia/vsn.mk b/lib/mnesia/vsn.mk index a95f468ba2..45f811846d 100644 --- a/lib/mnesia/vsn.mk +++ b/lib/mnesia/vsn.mk @@ -1 +1 @@ -MNESIA_VSN = 4.15.1 +MNESIA_VSN = 4.15.3 diff --git a/lib/observer/doc/src/notes.xml b/lib/observer/doc/src/notes.xml index 05ea550964..96cd89b375 100644 --- a/lib/observer/doc/src/notes.xml +++ b/lib/observer/doc/src/notes.xml @@ -32,6 +32,73 @@ <p>This document describes the changes made to the Observer application.</p> +<section><title>Observer 2.6</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + A bug introduced in OTP-20 would make Crashdump Viewer + crash when trying to expand an empty binary. This is now + corrected.</p> + <p> + Own Id: OTP-14642</p> + </item> + <item> + <p> + If a match spec in the config file contained more than + one clause, observer would earlier crash when trying to + display it in the GUI. This is now corrected.</p> + <p> + Own Id: OTP-14643 Aux Id: ERL-489 </p> + </item> + <item> + <p>Writing of crash dumps is significantly faster.</p> + <p>Maps are now included in crash dumps.</p> + <p>Constants terms would only be shown in one process, + while other processes referencing the same constant term + would show a marker for incomplete heap. </p> + <p> + Own Id: OTP-14685 Aux Id: OTP-14611, OTP-14603, OTP-14595 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p>Binaries and some other data in crash dumps are now + encoded in base64 (instead of in hex), which will reduce + the size of crash dumps.</p> + <p>A few bugs in the handling of sub binaries in + <c>crashdump_viewer</c> have been fixed.</p> + <p> + Own Id: OTP-14686</p> + </item> + <item> + <p> + In order to allow future improvements, Crashdump Viewer + now checks the version tag of the crashdump to see that + it is a known format. If the crashdump version is newer + than Crashdump Viewer is prepared to read, then an + information dialog is displayed before Crashdump Viewer + terminates.</p> + <p> + If an incomplete process heap is discovered in a + crashdump, Crashdump Viewer will now display a warning + for this, similar to the warning displayed when a + crashdump is truncated. Incomplete heaps can occur if for + instance the literals are not included, which is the case + for all dumps prior to OTP-20.2.</p> + <p> + Own Id: OTP-14755</p> + </item> + </list> + </section> + +</section> + <section><title>Observer 2.5</title> <section><title>Improvements and New Features</title> diff --git a/lib/observer/vsn.mk b/lib/observer/vsn.mk index 5f43198f85..fc1fca0925 100644 --- a/lib/observer/vsn.mk +++ b/lib/observer/vsn.mk @@ -1 +1 @@ -OBSERVER_VSN = 2.5 +OBSERVER_VSN = 2.6 diff --git a/lib/odbc/doc/src/notes.xml b/lib/odbc/doc/src/notes.xml index 6a8b0485eb..2aa55ca99c 100644 --- a/lib/odbc/doc/src/notes.xml +++ b/lib/odbc/doc/src/notes.xml @@ -32,7 +32,22 @@ <p>This document describes the changes made to the odbc application. </p> - <section><title>ODBC 2.12</title> + <section><title>ODBC 2.12.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + +<section><title>ODBC 2.12</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/odbc/vsn.mk b/lib/odbc/vsn.mk index 2e313570e1..3f7677a71d 100644 --- a/lib/odbc/vsn.mk +++ b/lib/odbc/vsn.mk @@ -1 +1 @@ -ODBC_VSN = 2.12 +ODBC_VSN = 2.12.1 diff --git a/lib/orber/doc/src/notes.xml b/lib/orber/doc/src/notes.xml index 5a82270b28..35da4f73da 100644 --- a/lib/orber/doc/src/notes.xml +++ b/lib/orber/doc/src/notes.xml @@ -33,7 +33,28 @@ <file>notes.xml</file> </header> - <section><title>Orber 3.8.3</title> + <section><title>Orber 3.8.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + <item> + <p> Removed the man warnings by using the code tag + instead of c tag. </p> + <p> + Own Id: OTP-14673</p> + </item> + </list> + </section> + +</section> + +<section><title>Orber 3.8.3</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/orber/vsn.mk b/lib/orber/vsn.mk index 595e686cb7..bfd3f283b5 100644 --- a/lib/orber/vsn.mk +++ b/lib/orber/vsn.mk @@ -1 +1 @@ -ORBER_VSN = 3.8.3 +ORBER_VSN = 3.8.4 diff --git a/lib/os_mon/doc/src/notes.xml b/lib/os_mon/doc/src/notes.xml index b29a64155e..cec0856a8b 100644 --- a/lib/os_mon/doc/src/notes.xml +++ b/lib/os_mon/doc/src/notes.xml @@ -31,6 +31,21 @@ </header> <p>This document describes the changes made to the OS_Mon application.</p> +<section><title>Os_Mon 2.4.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + <section><title>Os_Mon 2.4.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/os_mon/vsn.mk b/lib/os_mon/vsn.mk index e4250f577b..eb4f13ea9e 100644 --- a/lib/os_mon/vsn.mk +++ b/lib/os_mon/vsn.mk @@ -1 +1 @@ -OS_MON_VSN = 2.4.3 +OS_MON_VSN = 2.4.4 diff --git a/lib/otp_mibs/doc/src/notes.xml b/lib/otp_mibs/doc/src/notes.xml index dbd2f47ffb..c99148a904 100644 --- a/lib/otp_mibs/doc/src/notes.xml +++ b/lib/otp_mibs/doc/src/notes.xml @@ -32,6 +32,21 @@ <p>This document describes the changes made to the OTP_Mibs application.</p> +<section><title>Otp_Mibs 1.1.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + <section><title>Otp_Mibs 1.1.1</title> <section><title>Improvements and New Features</title> diff --git a/lib/otp_mibs/vsn.mk b/lib/otp_mibs/vsn.mk index 7a793007ee..13406cb5b1 100644 --- a/lib/otp_mibs/vsn.mk +++ b/lib/otp_mibs/vsn.mk @@ -1,4 +1,4 @@ -OTP_MIBS_VSN = 1.1.1 +OTP_MIBS_VSN = 1.1.2 # Note: The branch 'otp_mibs' is defunct as of otp_mibs-1.0.4 and # should NOT be used again. diff --git a/lib/parsetools/doc/src/notes.xml b/lib/parsetools/doc/src/notes.xml index 3fa7169f50..b3370a06ab 100644 --- a/lib/parsetools/doc/src/notes.xml +++ b/lib/parsetools/doc/src/notes.xml @@ -31,6 +31,21 @@ </header> <p>This document describes the changes made to the Parsetools application.</p> +<section><title>Parsetools 2.1.6</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Warnings about unused functions in <c>leexinc.hrl</c> + are suppressed. </p> + <p> + Own Id: OTP-14697</p> + </item> + </list> + </section> + +</section> + <section><title>Parsetools 2.1.5</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/parsetools/vsn.mk b/lib/parsetools/vsn.mk index 502ca00a47..b6d2ce0cd4 100644 --- a/lib/parsetools/vsn.mk +++ b/lib/parsetools/vsn.mk @@ -1 +1 @@ -PARSETOOLS_VSN = 2.1.5 +PARSETOOLS_VSN = 2.1.6 diff --git a/lib/public_key/doc/src/notes.xml b/lib/public_key/doc/src/notes.xml index a4c0194328..11012ee9e5 100644 --- a/lib/public_key/doc/src/notes.xml +++ b/lib/public_key/doc/src/notes.xml @@ -35,6 +35,31 @@ <file>notes.xml</file> </header> +<section><title>Public_Key 1.5.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed a bug in <c>public_key:ssh_encode/2</c> that made + it possible to erroneously encode e.g. an RSA key with + another type e.g. ECDSA in the resulting binary.</p> + <p> + Own Id: OTP-14570 Aux Id: ERIERL-52, OTP-14676 </p> + </item> + <item> + <p> + Corrected handling of parameterized EC keys in + public_key:generate_key/1 so that it will work as + expected instead of causing a runtime error in crypto.</p> + <p> + Own Id: OTP-14620</p> + </item> + </list> + </section> + +</section> + <section><title>Public_Key 1.5.1</title> <section><title>Improvements and New Features</title> diff --git a/lib/public_key/test/public_key_SUITE.erl b/lib/public_key/test/public_key_SUITE.erl index 9e5e288a1a..449d1fc040 100644 --- a/lib/public_key/test/public_key_SUITE.erl +++ b/lib/public_key/test/public_key_SUITE.erl @@ -64,7 +64,9 @@ all() -> groups() -> [{pem_decode_encode, [], [dsa_pem, rsa_pem, ec_pem, encrypted_pem, dh_pem, cert_pem, pkcs7_pem, pkcs10_pem, ec_pem2, - ec_pem_encode_generated, gen_ec_param]}, + ec_pem_encode_generated, + gen_ec_param_prime_field, gen_ec_param_char_2_field + ]}, {ssh_public_key_decode_encode, [], [ssh_rsa_public_key, ssh_dsa_public_key, ssh_ecdsa_public_key, ssh_rfc4716_rsa_comment, ssh_rfc4716_dsa_comment, @@ -105,18 +107,11 @@ init_per_testcase(pkix_test_data_all_default, Config) -> init_common_per_testcase(Config) end; -init_per_testcase(gen_ec_param, Config) -> - case crypto:ec_curves() of - [] -> - {skip, missing_ecc_support}; - Curves -> - case lists:member(secp521r1, Curves) of - true -> - init_common_per_testcase(Config); - false -> - {skip, missing_ecc_secp52r1_support} - end - end; +init_per_testcase(gen_ec_param_prime_field=TC, Config) -> + init_per_testcase_gen_ec_param(TC, secp521r1, Config); + +init_per_testcase(gen_ec_param_char_2_field=TC, Config) -> + init_per_testcase_gen_ec_param(TC, sect571r1, Config); init_per_testcase(TestCase, Config) -> case TestCase of @@ -1218,12 +1213,19 @@ short_crl_issuer_hash(Config) when is_list(Config) -> Issuer = public_key:pkix_crl_issuer(CrlDER), CrlIssuerHash = public_key:short_name_hash(Issuer). + +%%-------------------------------------------------------------------- +gen_ec_param_prime_field() -> + [{doc, "Generate key with EC prime_field parameters"}]. +gen_ec_param_prime_field(Config) when is_list(Config) -> + Datadir = proplists:get_value(data_dir, Config), + do_gen_ec_param(filename:join(Datadir, "ec_key_param0.pem")). + %%-------------------------------------------------------------------- -gen_ec_param() -> - [{doc, "Generate key with EC parameters"}]. -gen_ec_param(Config) when is_list(Config) -> +gen_ec_param_char_2_field() -> + [{doc, "Generate key with EC characteristic_two_field parameters"}]. +gen_ec_param_char_2_field(Config) when is_list(Config) -> Datadir = proplists:get_value(data_dir, Config), - do_gen_ec_param(filename:join(Datadir, "ec_key_param0.pem")), do_gen_ec_param(filename:join(Datadir, "ec_key_param1.pem")). %%-------------------------------------------------------------------- @@ -1310,6 +1312,30 @@ do_gen_ec_param(File) -> ct:fail({key_gen_fail, File}) end. +init_per_testcase_gen_ec_param(TC, Curve, Config) -> + case crypto:ec_curves() of + [] -> + {skip, missing_ec_support}; + Curves -> + case lists:member(Curve, Curves) + andalso crypto_supported_curve(Curve, Curves) + of + true -> + init_common_per_testcase(Config); + false -> + {skip, {missing_ec_support, Curve}} + end + end. + + +crypto_supported_curve(Curve, Curves) -> + try crypto:generate_key(ecdh, Curve) of + {error,_} -> false; % Just in case crypto is changed in the future... + _-> true + catch + _:_-> false + end. + incorrect_countryname_pkix_cert() -> <<48,130,5,186,48,130,4,162,160,3,2,1,2,2,7,7,250,61,63,6,140,137,48,13,6,9,42, 134,72,134,247,13,1,1,5,5,0,48,129,220,49,11,48,9,6,3,85,4,6,19,2,85,83,49, 16,48,14,6,3,85,4,8,19,7,65,114,105,122,111,110,97,49,19,48,17,6,3,85,4,7,19, 10,83,99,111,116,116,115,100,97,108,101,49,37,48,35,6,3,85,4,10,19,28,83,116, 97,114,102,105,101,108,100,32,84,101,99,104,110,111,108,111,103,105,101,115, 44,32,73,110,99,46,49,57,48,55,6,3,85,4,11,19,48,104,116,116,112,58,47,47,99, 101,114,116,105,102,105,99,97,116,101,115,46,115,116,97,114,102,105,101,108, 100,116,101,99,104,46,99,111,109,47,114,101,112,111,115,105,116,111,114,121, 49,49,48,47,6,3,85,4,3,19,40,83,116,97,114,102,105,101,108,100,32,83,101,99, 117,114,101,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117, 116,104,111,114,105,116,121,49,17,48,15,6,3,85,4,5,19,8,49,48,54,56,56,52,51, 53,48,30,23,13,49,48,49,48,50,51,48,49,51,50,48,53,90,23,13,49,50,49,48,50, 51,48,49,51,50,48,53,90,48,122,49,11,48,9,6,3,85,4,6,12,2,85,83,49,11,48,9,6, 3,85,4,8,12,2,65,90,49,19,48,17,6,3,85,4,7,12,10,83,99,111,116,116,115,100, 97,108,101,49,38,48,36,6,3,85,4,10,12,29,83,112,101,99,105,97,108,32,68,111, 109,97,105,110,32,83,101,114,118,105,99,101,115,44,32,73,110,99,46,49,33,48, 31,6,3,85,4,3,12,24,42,46,108,111,103,105,110,46,115,101,99,117,114,101,115, 101,114,118,101,114,46,110,101,116,48,130,1,34,48,13,6,9,42,134,72,134,247, 13,1,1,1,5,0,3,130,1,15,0,48,130,1,10,2,130,1,1,0,185,136,240,80,141,36,124, 245,182,130,73,19,188,74,166,117,72,228,185,209,43,129,244,40,44,193,231,11, 209,12,234,88,43,142,1,162,48,122,17,95,230,105,171,131,12,147,46,204,36,80, 250,171,33,253,35,62,83,22,71,212,186,141,14,198,89,89,121,204,224,122,246, 127,110,188,229,162,67,95,6,74,231,127,99,131,7,240,85,102,203,251,50,58,58, 104,245,103,181,183,134,32,203,121,232,54,32,188,139,136,112,166,126,14,91, 223,153,172,164,14,61,38,163,208,215,186,210,136,213,143,70,147,173,109,217, 250,169,108,31,211,104,238,103,93,182,59,165,43,196,189,218,241,30,148,240, 109,90,69,176,194,52,116,173,151,135,239,10,209,179,129,192,102,75,11,25,168, 223,32,174,84,223,134,70,167,55,172,143,27,130,123,226,226,7,34,142,166,39, 48,246,96,231,150,84,220,106,133,193,55,95,159,227,24,249,64,36,1,142,171,16, 202,55,126,7,156,15,194,22,116,53,113,174,104,239,203,120,45,131,57,87,84, 163,184,27,83,57,199,91,200,34,43,98,61,180,144,76,65,170,177,2,3,1,0,1,163, 130,1,224,48,130,1,220,48,15,6,3,85,29,19,1,1,255,4,5,48,3,1,1,0,48,29,6,3, 85,29,37,4,22,48,20,6,8,43,6,1,5,5,7,3,1,6,8,43,6,1,5,5,7,3,2,48,14,6,3,85, 29,15,1,1,255,4,4,3,2,5,160,48,56,6,3,85,29,31,4,49,48,47,48,45,160,43,160, 41,134,39,104,116,116,112,58,47,47,99,114,108,46,115,116,97,114,102,105,101, 108,100,116,101,99,104,46,99,111,109,47,115,102,115,50,45,48,46,99,114,108, 48,83,6,3,85,29,32,4,76,48,74,48,72,6,11,96,134,72,1,134,253,110,1,7,23,2,48, 57,48,55,6,8,43,6,1,5,5,7,2,1,22,43,104,116,116,112,115,58,47,47,99,101,114, 116,115,46,115,116,97,114,102,105,101,108,100,116,101,99,104,46,99,111,109, 47,114,101,112,111,115,105,116,111,114,121,47,48,129,141,6,8,43,6,1,5,5,7,1, 1,4,129,128,48,126,48,42,6,8,43,6,1,5,5,7,48,1,134,30,104,116,116,112,58,47, 47,111,99,115,112,46,115,116,97,114,102,105,101,108,100,116,101,99,104,46,99, 111,109,47,48,80,6,8,43,6,1,5,5,7,48,2,134,68,104,116,116,112,58,47,47,99, 101,114,116,105,102,105,99,97,116,101,115,46,115,116,97,114,102,105,101,108, 100,116,101,99,104,46,99,111,109,47,114,101,112,111,115,105,116,111,114,121, 47,115,102,95,105,110,116,101,114,109,101,100,105,97,116,101,46,99,114,116, 48,31,6,3,85,29,35,4,24,48,22,128,20,73,75,82,39,209,27,188,242,161,33,106, 98,123,81,66,122,138,215,213,86,48,59,6,3,85,29,17,4,52,48,50,130,24,42,46, 108,111,103,105,110,46,115,101,99,117,114,101,115,101,114,118,101,114,46,110, 101,116,130,22,108,111,103,105,110,46,115,101,99,117,114,101,115,101,114,118, 101,114,46,110,101,116,48,29,6,3,85,29,14,4,22,4,20,138,233,191,208,157,203, 249,85,242,239,20,195,48,10,148,49,144,101,255,116,48,13,6,9,42,134,72,134, 247,13,1,1,5,5,0,3,130,1,1,0,82,31,121,162,49,50,143,26,167,202,143,61,71, 189,201,199,57,81,122,116,90,192,88,24,102,194,174,48,157,74,27,87,210,223, 253,93,3,91,150,109,120,1,110,27,11,200,198,141,222,246,14,200,71,105,41,138, 13,114,122,106,63,17,197,181,234,121,61,89,74,65,41,231,248,219,129,83,176, 219,55,107,55,211,112,98,38,49,69,77,96,221,108,123,152,12,210,159,157,141, 43,226,55,187,129,3,82,49,136,66,81,196,91,234,196,10,82,48,6,80,163,83,71, 127,102,177,93,209,129,26,104,2,84,24,255,248,161,3,244,169,234,92,122,110, 43,4,17,113,185,235,108,219,210,236,132,216,177,227,17,169,58,162,159,182, 162,93,160,229,200,9,163,229,110,121,240,168,232,14,91,214,188,196,109,210, 164,222,0,109,139,132,113,91,16,118,173,178,176,80,132,34,41,199,51,206,250, 224,132,60,115,192,94,107,163,219,212,226,225,65,169,148,108,213,46,174,173, 103,110,189,229,166,149,254,31,51,44,144,108,187,182,11,251,201,206,86,138, 208,59,51,86,132,235,81,225,88,34,190,8,184>>. diff --git a/lib/public_key/vsn.mk b/lib/public_key/vsn.mk index c01d8820f2..99a0cc087e 100644 --- a/lib/public_key/vsn.mk +++ b/lib/public_key/vsn.mk @@ -1 +1 @@ -PUBLIC_KEY_VSN = 1.5.1 +PUBLIC_KEY_VSN = 1.5.2 diff --git a/lib/runtime_tools/doc/src/LTTng.xml b/lib/runtime_tools/doc/src/LTTng.xml index 392d54857c..93937b3fdc 100644 --- a/lib/runtime_tools/doc/src/LTTng.xml +++ b/lib/runtime_tools/doc/src/LTTng.xml @@ -134,7 +134,7 @@ $ make </code> <p><em>port_open</em></p> <list type="bulleted"> <item><c>pid : string</c> :: Process ID. Ex. <c>"<0.131.0>"</c></item> - <item><c>driver : string</c> :: Driver name. Ex. <c>"efile"</c></item> + <item><c>driver : string</c> :: Driver name. Ex. <c>"tcp_inet"</c></item> <item><c>port : string</c> :: Port ID. Ex. <c>"#Port<0.1031>"</c></item> </list> <p> @@ -323,7 +323,7 @@ $ make </code> <p><em>driver_init</em></p> <list type="bulleted"> - <item><c>driver : string</c> :: Driver name. Ex. <c>"efile"</c></item> + <item><c>driver : string</c> :: Driver name. Ex. <c>"tcp_inet"</c></item> <item><c>major : integer</c> :: Major version. Ex. <c>3</c></item> <item><c>minor : integer</c> :: Minor version. Ex. <c>1</c></item> <item><c>flags : integer</c> :: Flags. Ex. <c>1</c></item> @@ -334,7 +334,7 @@ $ make </code> <p><em>driver_start</em></p> <list type="bulleted"> <item><c>pid : string</c> :: Process ID. Ex. <c>"<0.131.0>"</c></item> - <item><c>driver : string</c> :: Driver name. Ex. <c>"efile"</c></item> + <item><c>driver : string</c> :: Driver name. Ex. <c>"tcp_inet"</c></item> <item><c>port : string</c> :: Port ID. Ex. <c>"#Port<0.1031>"</c></item> </list> <p>Example:</p> @@ -344,7 +344,7 @@ $ make </code> <list type="bulleted"> <item><c>pid : string</c> :: Process ID. Ex. <c>"<0.131.0>"</c></item> <item><c>port : string</c> :: Port ID. Ex. <c>"#Port<0.1031>"</c></item> - <item><c>driver : string</c> :: Driver name. Ex. <c>"efile"</c></item> + <item><c>driver : string</c> :: Driver name. Ex. <c>"tcp_inet"</c></item> <item><c>bytes : integer</c> :: Size of data returned. Ex. <c>82</c></item> </list> <p>Example:</p> @@ -354,7 +354,7 @@ $ make </code> <list type="bulleted"> <item><c>pid : string</c> :: Process ID. Ex. <c>"<0.131.0>"</c></item> <item><c>port : string</c> :: Port ID. Ex. <c>"#Port<0.1031>"</c></item> - <item><c>driver : string</c> :: Driver name. Ex. <c>"efile"</c></item> + <item><c>driver : string</c> :: Driver name. Ex. <c>"tcp_inet"</c></item> <item><c>bytes : integer</c> :: Size of data returned. Ex. <c>82</c></item> </list> <p>Example:</p> @@ -364,7 +364,7 @@ $ make </code> <list type="bulleted"> <item><c>pid : string</c> :: Process ID. Ex. <c>"<0.131.0>"</c></item> <item><c>port : string</c> :: Port ID. Ex. <c>"#Port<0.1031>"</c></item> - <item><c>driver : string</c> :: Driver name. Ex. <c>"efile"</c></item> + <item><c>driver : string</c> :: Driver name. Ex. <c>"tcp_inet"</c></item> </list> <p>Example:</p> <code type="none">driver_ready_input: { cpu_id = 5 }, { pid = "<0.189.0>", port = "#Port<0.3637>", driver = "inet_gethost 4 " }</code> @@ -373,7 +373,7 @@ $ make </code> <list type="bulleted"> <item><c>pid : string</c> :: Process ID. Ex. <c>"<0.131.0>"</c></item> <item><c>port : string</c> :: Port ID. Ex. <c>"#Port<0.1031>"</c></item> - <item><c>driver : string</c> :: Driver name. Ex. <c>"efile"</c></item> + <item><c>driver : string</c> :: Driver name. Ex. <c>"tcp_inet"</c></item> </list> <p>Example:</p> <code type="none">driver_ready_output: { cpu_id = 5 }, { pid = "<0.194.0>", port = "#Port<0.3663>", driver = "tcp_inet" }</code> @@ -382,14 +382,14 @@ $ make </code> <list type="bulleted"> <item><c>pid : string</c> :: Process ID. Ex. <c>"<0.131.0>"</c></item> <item><c>port : string</c> :: Port ID. Ex. <c>"#Port<0.1031>"</c></item> - <item><c>driver : string</c> :: Driver name. Ex. <c>"efile"</c></item> + <item><c>driver : string</c> :: Driver name. Ex. <c>"tcp_inet"</c></item> </list> <p>Example:</p> <code type="none">driver_timeout: { cpu_id = 5 }, { pid = "<0.196.0>", port = "#Port<0.3664>", driver = "tcp_inet" }</code> <p><em>driver_stop_select</em></p> <list type="bulleted"> - <item><c>driver : string</c> :: Driver name. Ex. <c>"efile"</c></item> + <item><c>driver : string</c> :: Driver name. Ex. <c>"tcp_inet"</c></item> </list> <p>Example:</p> <code type="none">driver_stop_select: { cpu_id = 5 }, { driver = "unknown" }</code> @@ -398,7 +398,7 @@ $ make </code> <list type="bulleted"> <item><c>pid : string</c> :: Process ID. Ex. <c>"<0.131.0>"</c></item> <item><c>port : string</c> :: Port ID. Ex. <c>"#Port<0.1031>"</c></item> - <item><c>driver : string</c> :: Driver name. Ex. <c>"efile"</c></item> + <item><c>driver : string</c> :: Driver name. Ex. <c>"tcp_inet"</c></item> </list> <p>Example:</p> <code type="none">driver_flush: { cpu_id = 7 }, { pid = "<0.204.0>", port = "#Port<0.3686>", driver = "tcp_inet" }</code> @@ -407,32 +407,32 @@ $ make </code> <list type="bulleted"> <item><c>pid : string</c> :: Process ID. Ex. <c>"<0.131.0>"</c></item> <item><c>port : string</c> :: Port ID. Ex. <c>"#Port<0.1031>"</c></item> - <item><c>driver : string</c> :: Driver name. Ex. <c>"efile"</c></item> + <item><c>driver : string</c> :: Driver name. Ex. <c>"tcp_inet"</c></item> </list> <p>Example:</p> - <code type="none">driver_stop: { cpu_id = 5 }, { pid = "[]", port = "#Port<0.3673>", driver = "efile" }</code> + <code type="none">driver_stop: { cpu_id = 5 }, { pid = "[]", port = "#Port<0.3673>", driver = "tcp_inet" }</code> <p><em>driver_process_exit</em></p> <list type="bulleted"> <item><c>pid : string</c> :: Process ID. Ex. <c>"<0.131.0>"</c></item> <item><c>port : string</c> :: Port ID. Ex. <c>"#Port<0.1031>"</c></item> - <item><c>driver : string</c> :: Driver name. Ex. <c>"efile"</c></item> + <item><c>driver : string</c> :: Driver name. Ex. <c>"tcp_inet"</c></item> </list> <p><em>driver_ready_async</em></p> <list type="bulleted"> <item><c>pid : string</c> :: Process ID. Ex. <c>"<0.131.0>"</c></item> <item><c>port : string</c> :: Port ID. Ex. <c>"#Port<0.1031>"</c></item> - <item><c>driver : string</c> :: Driver name. Ex. <c>"efile"</c></item> + <item><c>driver : string</c> :: Driver name. Ex. <c>"tcp_inet"</c></item> </list> <p>Example:</p> - <code type="none">driver_ready_async: { cpu_id = 3 }, { pid = "<0.181.0>", port = "#Port<0.3622>", driver = "efile" }</code> + <code type="none">driver_ready_async: { cpu_id = 3 }, { pid = "<0.181.0>", port = "#Port<0.3622>", driver = "tcp_inet" }</code> <p><em>driver_call</em></p> <list type="bulleted"> <item><c>pid : string</c> :: Process ID. Ex. <c>"<0.131.0>"</c></item> <item><c>port : string</c> :: Port ID. Ex. <c>"#Port<0.1031>"</c></item> - <item><c>driver : string</c> :: Driver name. Ex. <c>"efile"</c></item> + <item><c>driver : string</c> :: Driver name. Ex. <c>"tcp_inet"</c></item> <item><c>command : integer</c> :: Command integer. Ex. <c>1</c></item> <item><c>bytes : integer</c> :: Size of data returned. Ex. <c>82</c></item> </list> @@ -443,7 +443,7 @@ $ make </code> <list type="bulleted"> <item><c>pid : string</c> :: Process ID. Ex. <c>"<0.131.0>"</c></item> <item><c>port : string</c> :: Port ID. Ex. <c>"#Port<0.1031>"</c></item> - <item><c>driver : string</c> :: Driver name. Ex. <c>"efile"</c></item> + <item><c>driver : string</c> :: Driver name. Ex. <c>"tcp_inet"</c></item> <item><c>command : integer</c> :: Command integer. Ex. <c>1</c></item> <item><c>bytes : integer</c> :: Size of data returned. Ex. <c>82</c></item> </list> diff --git a/lib/runtime_tools/doc/src/notes.xml b/lib/runtime_tools/doc/src/notes.xml index 8b4d437c26..93e3e26fda 100644 --- a/lib/runtime_tools/doc/src/notes.xml +++ b/lib/runtime_tools/doc/src/notes.xml @@ -32,6 +32,21 @@ <p>This document describes the changes made to the Runtime_Tools application.</p> +<section><title>Runtime_Tools 1.12.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + <section><title>Runtime_Tools 1.12.2</title> <section><title>Improvements and New Features</title> diff --git a/lib/runtime_tools/examples/efile_drv.d b/lib/runtime_tools/examples/efile_drv.d deleted file mode 100644 index a470518dd9..0000000000 --- a/lib/runtime_tools/examples/efile_drv.d +++ /dev/null @@ -1,105 +0,0 @@ -/* example usage: dtrace -q -s /path/to/efile_drv.d */ -/* - * %CopyrightBegin% - * - * Copyright Scott Lystig Fritchie 2011-2016. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -BEGIN -{ - op_map[1] = "OPEN"; - op_map[2] = "READ"; - op_map[3] = "LSEEK"; - op_map[4] = "WRITE"; - op_map[5] = "FSTAT"; - op_map[6] = "PWD"; - op_map[7] = "READDIR"; - op_map[8] = "CHDIR"; - op_map[9] = "FSYNC"; - op_map[10] = "MKDIR"; - op_map[11] = "DELETE"; - op_map[12] = "RENAME"; - op_map[13] = "RMDIR"; - op_map[14] = "TRUNCATE"; - op_map[15] = "READ_FILE"; - op_map[16] = "WRITE_INFO"; - op_map[19] = "LSTAT"; - op_map[20] = "READLINK"; - op_map[21] = "LINK"; - op_map[22] = "SYMLINK"; - op_map[23] = "CLOSE"; - op_map[24] = "PWRITEV"; - op_map[25] = "PREADV"; - op_map[26] = "SETOPT"; - op_map[27] = "IPREAD"; - op_map[28] = "ALTNAME"; - op_map[29] = "READ_LINE"; - op_map[30] = "FDATASYNC"; - op_map[31] = "FADVISE"; -} - -erlang*:::aio_pool-add -{ - printf("async I/O pool port %s queue len %d\n", copyinstr(arg0), arg1); -} - -erlang*:::aio_pool-get -{ - printf("async I/O pool port %s queue len %d\n", copyinstr(arg0), arg1); -} - -erlang*:::efile_drv-entry -{ - printf("efile_drv enter tag={%d,%d} %s%s | %s (%d) | args: %s %s , %d %d (port %s)\n", - arg0, arg1, - arg2 == NULL ? "" : "user tag ", - arg2 == NULL ? "" : copyinstr(arg2), - op_map[arg3], arg3, - arg4 == NULL ? "" : copyinstr(arg4), - arg5 == NULL ? "" : copyinstr(arg5), arg6, arg7, - /* NOTE: port name in args[10] is experimental */ - (args[10] == NULL) ? - "?" : copyinstr((user_addr_t) args[10])); -} - -erlang*:::efile_drv-int* -{ - printf("async I/O worker tag={%d,%d} | %s (%d) | %s\n", - arg0, arg1, op_map[arg2], arg2, probename); -} - -/* efile_drv-return error case */ -erlang*:::efile_drv-return -/arg4 == 0/ -{ - printf("efile_drv return tag={%d,%d} %s%s | %s (%d) | errno %d\n", - arg0, arg1, - arg2 == NULL ? "" : "user tag ", - arg2 == NULL ? "" : copyinstr(arg2), - op_map[arg3], arg3, - arg5); -} - -/* efile_drv-return success case */ -erlang*:::efile_drv-return -/arg4 != 0/ -{ - printf("efile_drv return tag={%d,%d} %s | %s (%d) ok\n", - arg0, arg1, - arg2 == NULL ? "" : copyinstr(arg2), - op_map[arg3], arg3); -} diff --git a/lib/runtime_tools/examples/efile_drv.systemtap b/lib/runtime_tools/examples/efile_drv.systemtap deleted file mode 100644 index 29c3637e10..0000000000 --- a/lib/runtime_tools/examples/efile_drv.systemtap +++ /dev/null @@ -1,113 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Scott Lystig Fritchie and Andreas Schultz, 2011-2016. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ -/* - * Note: This file assumes that you're using the non-SMP-enabled Erlang - * virtual machine, "beam". The SMP-enabled VM is called "beam.smp". - * Note that other variations of the virtual machine also have - * different names, e.g. the debug build of the SMP-enabled VM - * is "beam.debug.smp". - * - * To use a different virtual machine, replace each instance of - * "beam" with "beam.smp" or the VM name appropriate to your - * environment. - */ - -probe begin -{ - op_map[1] = "OPEN"; - op_map[2] = "READ"; - op_map[3] = "LSEEK"; - op_map[4] = "WRITE"; - op_map[5] = "FSTAT"; - op_map[6] = "PWD"; - op_map[7] = "READDIR"; - op_map[8] = "CHDIR"; - op_map[9] = "FSYNC"; - op_map[10] = "MKDIR"; - op_map[11] = "DELETE"; - op_map[12] = "RENAME"; - op_map[13] = "RMDIR"; - op_map[14] = "TRUNCATE"; - op_map[15] = "READ_FILE"; - op_map[16] = "WRITE_INFO"; - op_map[19] = "LSTAT"; - op_map[20] = "READLINK"; - op_map[21] = "LINK"; - op_map[22] = "SYMLINK"; - op_map[23] = "CLOSE"; - op_map[24] = "PWRITEV"; - op_map[25] = "PREADV"; - op_map[26] = "SETOPT"; - op_map[27] = "IPREAD"; - op_map[28] = "ALTNAME"; - op_map[29] = "READ_LINE"; - op_map[30] = "FDATASYNC"; - op_map[31] = "FADVISE"; -} - -probe process("beam").mark("aio_pool-add") -{ - printf("async I/O pool port %s queue len %d\n", user_string($arg1), $arg2); -} - -probe process("beam").mark("aio_pool-get") -{ - printf("async I/O pool port %s queue len %d\n", user_string($arg1), $arg2); -} - -probe process("beam").mark("efile_drv-entry") -{ - printf("efile_drv enter tag={%d,%d} %s%s | %s (%d) | args: %s %s , %d %d (port %s)\n", - $arg1, $arg2, - $arg3 == NULL ? "" : "user tag ", - $arg3 == NULL ? "" : user_string($arg3), - op_map[$arg4], $arg4, - $arg5 == NULL ? "" : user_string($arg5), - $arg6 == NULL ? "" : user_string($arg6), $arg7, $arg8, - /* NOTE: port name in $arg[11] is experimental */ - user_string($arg11)) -} - -probe process("beam").mark("efile_drv-int*") -{ - printf("async I/O worker tag={%d,%d} | %s (%d) | %s\n", - $arg1, $arg2, op_map[$arg3], $arg3, probefunc()); -} - -probe process("beam").mark("efile_drv-return") -{ - if ($arg5 == 0) { - /* efile_drv-return error case */ - printf("efile_drv return tag={%d,%d} %s%s | %s (%d) | errno %d\n", - $arg1, $arg2, - $arg3 == NULL ? "" : "user tag ", - $arg3 == NULL ? "" : user_string($arg3), - op_map[$arg4], $arg4, - $arg6); - } else { - /* efile_drv-return success case */ - printf("efile_drv return tag={%d,%d} %s | %s (%d) ok\n", - $arg1, $arg2, - $arg3 == NULL ? "" : user_string($arg3), - op_map[$arg4], $arg4); - } -} - -global op_map; diff --git a/lib/runtime_tools/src/observer_backend.erl b/lib/runtime_tools/src/observer_backend.erl index 1b075a507d..a1edde8516 100644 --- a/lib/runtime_tools/src/observer_backend.erl +++ b/lib/runtime_tools/src/observer_backend.erl @@ -293,7 +293,7 @@ fetch_stats_loop(Parent, Time) -> erlang:system_flag(scheduler_wall_time, true), receive _Msg -> - %% erlang:system_flag(scheduler_wall_time, false) + erlang:system_flag(scheduler_wall_time, false), ok after Time -> _M = Parent ! {stats, 1, @@ -340,7 +340,6 @@ etop_collect(Collector) -> case SchedulerWallTime of undefined -> - erlang:system_flag(scheduler_wall_time,true), spawn(fun() -> flag_holder_proc(Collector) end), ok; _ -> @@ -348,10 +347,11 @@ etop_collect(Collector) -> end. flag_holder_proc(Collector) -> + erlang:system_flag(scheduler_wall_time,true), Ref = erlang:monitor(process,Collector), receive {'DOWN',Ref,_,_,_} -> - %% erlang:system_flag(scheduler_wall_time,false) + erlang:system_flag(scheduler_wall_time,false), ok end. diff --git a/lib/runtime_tools/vsn.mk b/lib/runtime_tools/vsn.mk index d8a4ede136..872bd5db1d 100644 --- a/lib/runtime_tools/vsn.mk +++ b/lib/runtime_tools/vsn.mk @@ -1 +1 @@ -RUNTIME_TOOLS_VSN = 1.12.2 +RUNTIME_TOOLS_VSN = 1.12.3 diff --git a/lib/sasl/doc/src/notes.xml b/lib/sasl/doc/src/notes.xml index b144122c4b..e532c3cd6f 100644 --- a/lib/sasl/doc/src/notes.xml +++ b/lib/sasl/doc/src/notes.xml @@ -31,6 +31,26 @@ </header> <p>This document describes the changes made to the SASL application.</p> +<section><title>SASL 3.1.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The Report Browser, rb, could earlier not handle reports + that were not lists, for example generated by + <c>error_logger:info_report({some, tuple})</c>. This term + is allowed as input to error_logger, but rb would state + that "A report on bad form was encountered". This is now + corrected.</p> + <p> + Own Id: OTP-13906 Aux Id: ERL-261 </p> + </item> + </list> + </section> + +</section> + <section><title>SASL 3.1</title> <section><title>Improvements and New Features</title> diff --git a/lib/sasl/src/systools_make.erl b/lib/sasl/src/systools_make.erl index 391b1fb5cc..9d960b7361 100644 --- a/lib/sasl/src/systools_make.erl +++ b/lib/sasl/src/systools_make.erl @@ -1501,7 +1501,7 @@ preloaded() -> [erl_prim_loader,erl_tracer,erlang, erts_code_purger,erts_dirty_process_code_checker, erts_internal,erts_literal_area_collector, - init,otp_ring0,prim_eval,prim_file, + init,otp_ring0,prim_buffer,prim_eval,prim_file, prim_inet,prim_zip,zlib]. %%______________________________________________________________________ diff --git a/lib/sasl/test/release_handler_SUITE.erl b/lib/sasl/test/release_handler_SUITE.erl index 4935782cf2..824820c214 100644 --- a/lib/sasl/test/release_handler_SUITE.erl +++ b/lib/sasl/test/release_handler_SUITE.erl @@ -22,7 +22,8 @@ -include_lib("common_test/include/ct.hrl"). -include("test_lib.hrl"). --compile(export_all). +-compile([export_all, nowarn_export_all]). +-export([scheduler_wall_time/0, garbage_collect/0]). %% rpc'ed % Default timetrap timeout (set in init_per_testcase). %-define(default_timeout, ?t:minutes(40)). @@ -1085,8 +1086,9 @@ otp_9395_update_many_mods(Conf) when is_list(Conf) -> Rel2Dir = filename:dirname(Rel2), %% Start a slave node + PA = filename:dirname(code:which(?MODULE)), {ok, Node} = t_start_node(otp_9395_update_many_mods, Rel1, - filename:join(Rel1Dir,"sys.config")), + filename:join(Rel1Dir,"sys.config"), "-pa " ++ PA), %% Start a lot of processes on the new node, all with refs to each %% module that will be updated @@ -1109,8 +1111,8 @@ otp_9395_update_many_mods(Conf) when is_list(Conf) -> [RelVsn2, filename:join(Rel2Dir, "sys.config")]), %% First, install release directly and check how much time it takes - rpc:call(Node,erlang,garbage_collect,[]), - rpc:call(Node,erlang,system_flag,[scheduler_wall_time,true]), + rpc:call(Node,?MODULE,garbage_collect,[]), + SWTFlag0 = spawn_link(Node, ?MODULE, scheduler_wall_time, []), {TInst0,{ok, _, []}} = timer:tc(rpc,call,[Node, release_handler, install_release, [RelVsn2]]), SWT0 = rpc:call(Node,erlang,statistics,[scheduler_wall_time]), @@ -1135,9 +1137,9 @@ otp_9395_update_many_mods(Conf) when is_list(Conf) -> %% Finally install release after check and purge, and check that %% this install was faster than the first. - rpc:call(Node,erlang,system_flag,[scheduler_wall_time,false]), - rpc:call(Node,erlang,garbage_collect,[]), - rpc:call(Node,erlang,system_flag,[scheduler_wall_time,true]), + SWTFlag0 ! die, + rpc:call(Node,?MODULE,garbage_collect,[]), + _SWTFlag1 = spawn_link(Node, ?MODULE, scheduler_wall_time, []), {TInst2,{ok, _RelVsn1, []}} = timer:tc(rpc,call,[Node, release_handler, install_release, [RelVsn2]]), SWT2 = rpc:call(Node,erlang,statistics,[scheduler_wall_time]), @@ -1161,6 +1163,15 @@ otp_9395_update_many_mods(Conf) when is_list(Conf) -> ok. +scheduler_wall_time() -> + erlang:system_flag(scheduler_wall_time,true), + receive _Msg -> normal end. + +garbage_collect() -> + Pids = processes(), + [erlang:garbage_collect(Pid) || Pid <- Pids]. + + otp_9395_update_many_mods(cleanup,_Conf) -> stop_node(node_name(otp_9395_update_many_mods)). @@ -1190,8 +1201,9 @@ otp_9395_rm_many_mods(Conf) when is_list(Conf) -> Rel2Dir = filename:dirname(Rel2), %% Start a slave node + PA = filename:dirname(code:which(?MODULE)), {ok, Node} = t_start_node(otp_9395_rm_many_mods, Rel1, - filename:join(Rel1Dir,"sys.config")), + filename:join(Rel1Dir,"sys.config"), "-pa " ++ PA), %% Start a lot of processes on the new node, all with refs to each %% module that will be updated @@ -1214,8 +1226,8 @@ otp_9395_rm_many_mods(Conf) when is_list(Conf) -> [RelVsn2, filename:join(Rel2Dir, "sys.config")]), %% First, install release directly and check how much time it takes - rpc:call(Node,erlang,garbage_collect,[]), - rpc:call(Node,erlang,system_flag,[scheduler_wall_time,true]), + rpc:call(Node,?MODULE,garbage_collect,[]), + SWTFlag0 = spawn_link(Node, ?MODULE, scheduler_wall_time, []), {TInst0,{ok, _, []}} = timer:tc(rpc,call,[Node, release_handler, install_release, [RelVsn2]]), SWT0 = rpc:call(Node,erlang,statistics,[scheduler_wall_time]), @@ -1240,9 +1252,9 @@ otp_9395_rm_many_mods(Conf) when is_list(Conf) -> %% Finally install release after check and purge, and check that %% this install was faster than the first. - rpc:call(Node,erlang,system_flag,[scheduler_wall_time,false]), - rpc:call(Node,erlang,garbage_collect,[]), - rpc:call(Node,erlang,system_flag,[scheduler_wall_time,true]), + SWTFlag0 ! die, + rpc:call(Node,?MODULE,garbage_collect,[]), + _SWTFlag1 = spawn_link(Node, ?MODULE, scheduler_wall_time, []), {TInst2,{ok, _RelVsn1, []}} = timer:tc(rpc,call,[Node, release_handler, install_release, [RelVsn2]]), SWT2 = rpc:call(Node,erlang,statistics,[scheduler_wall_time]), diff --git a/lib/sasl/vsn.mk b/lib/sasl/vsn.mk index e980a42688..2488197ec5 100644 --- a/lib/sasl/vsn.mk +++ b/lib/sasl/vsn.mk @@ -1 +1 @@ -SASL_VSN = 3.1 +SASL_VSN = 3.1.1 diff --git a/lib/snmp/doc/src/notes.xml b/lib/snmp/doc/src/notes.xml index 6bdcae5dd7..1b5f94ed07 100644 --- a/lib/snmp/doc/src/notes.xml +++ b/lib/snmp/doc/src/notes.xml @@ -34,7 +34,22 @@ </header> - <section><title>SNMP 5.2.8</title> + <section><title>SNMP 5.2.9</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + +<section><title>SNMP 5.2.8</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/snmp/vsn.mk b/lib/snmp/vsn.mk index ef48608bda..c195f9f5d9 100644 --- a/lib/snmp/vsn.mk +++ b/lib/snmp/vsn.mk @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = snmp -SNMP_VSN = 5.2.8 +SNMP_VSN = 5.2.9 PRE_VSN = APP_VSN = "$(APPLICATION)-$(SNMP_VSN)$(PRE_VSN)" diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index c9e153f30c..3a2f55a487 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -30,6 +30,65 @@ <file>notes.xml</file> </header> +<section><title>Ssh 4.6.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix problem with OpenSSH 7.2 (and later) clients that has + used sha1 instead of sha2 for rsa-sha-256/512 user's + public keys.</p> + <p> + Own Id: OTP-14827 Aux Id: ERL-531 </p> + </item> + </list> + </section> + +</section> + +<section><title>Ssh 4.6.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Passphrase option for ecdsa public keys was missing.</p> + <p> + Own Id: OTP-14602</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + The host and user public key handling is hardened so that + a faulty plugin can't deliver a key of wrong type.</p> + <p> + Better checks in the server of the available hostkey's + types at start and at each accept.</p> + <p> + Better checks in the client of the available user public + key types at connect.</p> + <p> + Own Id: OTP-14676 Aux Id: ERIERL-52, OTP-14570 </p> + </item> + <item> + <p> + SSH can now fetch the host key from the private keys + stored in an Engine. See the crypto application for + details about Engines.</p> + <p> + Own Id: OTP-14757</p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.6.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssh/src/ssh.app.src b/lib/ssh/src/ssh.app.src index 974292fde1..4a22322333 100644 --- a/lib/ssh/src/ssh.app.src +++ b/lib/ssh/src/ssh.app.src @@ -42,10 +42,10 @@ {env, []}, {mod, {ssh_app, []}}, {runtime_dependencies, [ - "crypto-3.7.3", + "crypto-4.2", "erts-6.0", "kernel-3.0", - "public_key-1.4", + "public_key-1.5.2", "stdlib-3.3" ]}]}. diff --git a/lib/ssh/src/ssh_auth.erl b/lib/ssh/src/ssh_auth.erl index 894877f8bf..03d264745b 100644 --- a/lib/ssh/src/ssh_auth.erl +++ b/lib/ssh/src/ssh_auth.erl @@ -304,11 +304,10 @@ handle_userauth_request(#ssh_msg_userauth_request{user = User, SigWLen/binary>> }, SessionId, - #ssh{opts = Opts, - userauth_supported_methods = Methods} = Ssh) -> + #ssh{userauth_supported_methods = Methods} = Ssh) -> case verify_sig(SessionId, User, "ssh-connection", - BAlg, KeyBlob, SigWLen, Opts) of + BAlg, KeyBlob, SigWLen, Ssh) of true -> {authorized, User, ssh_transport:ssh_packet( @@ -518,7 +517,7 @@ pre_verify_sig(User, KeyBlob, Opts) -> false end. -verify_sig(SessionId, User, Service, AlgBin, KeyBlob, SigWLen, Opts) -> +verify_sig(SessionId, User, Service, AlgBin, KeyBlob, SigWLen, #ssh{opts = Opts} = Ssh) -> try Alg = binary_to_list(AlgBin), {KeyCb,KeyCbOpts} = ?GET_OPT(key_cb, Opts), @@ -529,7 +528,7 @@ verify_sig(SessionId, User, Service, AlgBin, KeyBlob, SigWLen, Opts) -> <<?UINT32(AlgSigLen), AlgSig:AlgSigLen/binary>> = SigWLen, <<?UINT32(AlgLen), _Alg:AlgLen/binary, ?UINT32(SigLen), Sig:SigLen/binary>> = AlgSig, - ssh_transport:verify(PlainText, ssh_transport:sha(Alg), Sig, Key) + ssh_transport:verify(PlainText, ssh_transport:sha(Alg), Sig, Key, Ssh) catch _:_ -> false diff --git a/lib/ssh/src/ssh_transport.erl b/lib/ssh/src/ssh_transport.erl index 90a94a7e86..154894cda8 100644 --- a/lib/ssh/src/ssh_transport.erl +++ b/lib/ssh/src/ssh_transport.erl @@ -51,7 +51,7 @@ extract_public_key/1, ssh_packet/2, pack/2, valid_key_sha_alg/2, - sha/1, sign/3, verify/4]). + sha/1, sign/3, verify/5]). %%% For test suites -export([pack/3]). @@ -825,7 +825,7 @@ extract_public_key(#{engine:=_, key_id:=_, algorithm:=Alg} = M) -> verify_host_key(#ssh{algorithms=Alg}=SSH, PublicKey, Digest, {AlgStr,Signature}) -> case atom_to_list(Alg#alg.hkey) of AlgStr -> - case verify(Digest, sha(Alg#alg.hkey), Signature, PublicKey) of + case verify(Digest, sha(Alg#alg.hkey), Signature, PublicKey, SSH) of false -> {error, bad_signature}; true -> @@ -1288,7 +1288,7 @@ mk_dss_sig(DerSignature) -> <<R:160/big-unsigned-integer, S:160/big-unsigned-integer>>. -verify(PlainText, HashAlg, Sig, {_, #'Dss-Parms'{}} = Key) -> +verify(PlainText, HashAlg, Sig, {_, #'Dss-Parms'{}} = Key, _) -> case Sig of <<R:160/big-unsigned-integer, S:160/big-unsigned-integer>> -> Signature = public_key:der_encode('Dss-Sig-Value', #'Dss-Sig-Value'{r = R, s = S}), @@ -1296,7 +1296,7 @@ verify(PlainText, HashAlg, Sig, {_, #'Dss-Parms'{}} = Key) -> _ -> false end; -verify(PlainText, HashAlg, Sig, {#'ECPoint'{},_} = Key) -> +verify(PlainText, HashAlg, Sig, {#'ECPoint'{},_} = Key, _) -> case Sig of <<?UINT32(Rlen),R:Rlen/big-signed-integer-unit:8, ?UINT32(Slen),S:Slen/big-signed-integer-unit:8>> -> @@ -1306,7 +1306,15 @@ verify(PlainText, HashAlg, Sig, {#'ECPoint'{},_} = Key) -> _ -> false end; -verify(PlainText, HashAlg, Sig, Key) -> + +verify(PlainText, HashAlg, Sig, #'RSAPublicKey'{}=Key, #ssh{role = server, + c_version = "SSH-2.0-OpenSSH_7."++_}) + when HashAlg == sha256; HashAlg == sha512 -> + %% Public key signing bug in in OpenSSH >= 7.2 + public_key:verify(PlainText, HashAlg, Sig, Key) + orelse public_key:verify(PlainText, sha, Sig, Key); + +verify(PlainText, HashAlg, Sig, Key, _) -> public_key:verify(PlainText, HashAlg, Sig, Key). diff --git a/lib/ssh/vsn.mk b/lib/ssh/vsn.mk index 59775d2d7f..004db6e3a2 100644 --- a/lib/ssh/vsn.mk +++ b/lib/ssh/vsn.mk @@ -1,5 +1,5 @@ #-*-makefile-*- ; force emacs to enter makefile-mode -SSH_VSN = 4.6.2 +SSH_VSN = 4.6.4 APP_VSN = "ssh-$(SSH_VSN)" diff --git a/lib/ssl/doc/src/notes.xml b/lib/ssl/doc/src/notes.xml index 37c916e585..79176f5edf 100644 --- a/lib/ssl/doc/src/notes.xml +++ b/lib/ssl/doc/src/notes.xml @@ -27,6 +27,61 @@ </header> <p>This document describes the changes made to the SSL application.</p> +<section><title>SSL 8.2.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Packet options cannot be supported for unreliable + transports, that is, packet option for DTLS over udp will + not be supported.</p> + <p> + Own Id: OTP-14664</p> + </item> + <item> + <p> + Ensure data delivery before close if possible. This fix + is related to fix in PR-1479.</p> + <p> + Own Id: OTP-14794</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + The crypto API is extended to use private/public keys + stored in an Engine for sign/verify or encrypt/decrypt + operations.</p> + <p> + The ssl application provides an API to use this new + engine concept in TLS.</p> + <p> + Own Id: OTP-14448</p> + </item> + <item> + <p> + Implemented renegotiation for DTLS</p> + <p> + Own Id: OTP-14563</p> + </item> + <item> + <p> + A new command line option <c>-ssl_dist_optfile</c> has + been added to facilitate specifying the many options + needed when using SSL as the distribution protocol.</p> + <p> + Own Id: OTP-14657</p> + </item> + </list> + </section> + +</section> + <section><title>SSL 8.2.2</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/ssl/doc/src/ssl_introduction.xml b/lib/ssl/doc/src/ssl_introduction.xml index d3e39dbb01..25b05a769d 100644 --- a/lib/ssl/doc/src/ssl_introduction.xml +++ b/lib/ssl/doc/src/ssl_introduction.xml @@ -36,7 +36,7 @@ <title>Purpose</title> <p>Transport Layer Security (TLS) and its predecessor, the Secure Sockets Layer (SSL), are cryptographic protocols designed to - provide communications security over a computer network. The protocols use + provide communications security over a computer network. The protocols use X.509 certificates and hence public key (asymmetric) cryptography to authenticate the counterpart with whom they communicate, and to exchange a symmetric key for payload encryption. The protocol provides diff --git a/lib/ssl/doc/src/using_ssl.xml b/lib/ssl/doc/src/using_ssl.xml index f84cd6e391..61918a346d 100644 --- a/lib/ssl/doc/src/using_ssl.xml +++ b/lib/ssl/doc/src/using_ssl.xml @@ -152,4 +152,41 @@ Shell got {ssl,{sslsocket,[...]},"foo"} ok</code> </section> </section> + + <section> + <title>Using an Engine Stored Key</title> + + <p>Erlang ssl application is able to use private keys provided + by OpenSSL engines using the following mechanism:</p> + + <code type="erl">1> ssl:start(). +ok</code> + + <p>Load a crypto engine, should be done once per engine used. For example + dynamically load the engine called <c>MyEngine</c>: + </p> + <code type="erl">2> {ok, EngineRef} = +crypto:engine_load(<<"dynamic">>, + [{<<"SO_PATH">>, "/tmp/user/engines/MyEngine"},<<"LOAD">>],[]). +{ok,#Ref<0.2399045421.3028942852.173962>} + </code> + + <p>Create a map with the engine information and the algorithm used by the engine:</p> + <code type="erl">3> PrivKey = + #{algorithm => rsa, + engine => EngineRef, + key_id => "id of the private key in Engine"}. + </code> + <p>Use the map in the ssl key option:</p> + <code type="erl">4> {ok, SSLSocket} = +ssl:connect("localhost", 9999, + [{cacertfile, "cacerts.pem"}, + {certfile, "cert.pem"}, + {key, PrivKey}], infinity). + </code> + + <p>See also <seealso marker="crypto:engine_load#engine_load"> crypto documentation</seealso> </p> + + </section> + </chapter> diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl index e5760e7951..3b5a548f72 100644 --- a/lib/ssl/src/dtls_connection.erl +++ b/lib/ssl/src/dtls_connection.erl @@ -143,10 +143,16 @@ next_record(#state{role = server, dtls_udp_listener:active_once(Listener, Client, self()), {no_record, State}; next_record(#state{role = client, - socket = {_Server, Socket}, + socket = {_Server, Socket} = DTLSSocket, + close_tag = CloseTag, transport_cb = Transport} = State) -> - dtls_socket:setopts(Transport, Socket, [{active,once}]), - {no_record, State}; + case dtls_socket:setopts(Transport, Socket, [{active,once}]) of + ok -> + {no_record, State}; + _ -> + self() ! {CloseTag, DTLSSocket}, + {no_record, State} + end; next_record(State) -> {no_record, State}. @@ -218,12 +224,12 @@ next_event(StateName, Record, #state{connection_states = #{current_read := #{epoch := CurrentEpoch}}} = State0, Actions) -> case Record of no_record -> - {next_state, StateName, State0, Actions}; + {next_state, StateName, State0, Actions}; #ssl_tls{epoch = CurrentEpoch, version = Version} = Record -> State = dtls_version(StateName, Version, State0), - {next_state, StateName, State, - [{next_event, internal, {protocol_record, Record}} | Actions]}; + {next_state, StateName, State, + [{next_event, internal, {protocol_record, Record}} | Actions]}; #ssl_tls{epoch = _Epoch, version = _Version} = _Record -> %% TODO maybe buffer later epoch @@ -605,6 +611,12 @@ certify(info, Event, State) -> gen_info(Event, ?FUNCTION_NAME, State); certify(internal = Type, #server_hello_done{} = Event, State) -> ssl_connection:certify(Type, Event, prepare_flight(State), ?MODULE); +certify(internal, #change_cipher_spec{type = <<1>>}, State0) -> + {State1, Actions0} = send_handshake_flight(State0, retransmit_epoch(?FUNCTION_NAME, State0)), + {Record, State2} = next_record(State1), + {next_state, ?FUNCTION_NAME, State, Actions} = next_event(?FUNCTION_NAME, Record, State2, Actions0), + %% This will reset the retransmission timer by repeating the enter state event + {repeat_state, State, Actions}; certify(state_timeout, Event, State) -> handle_state_timeout(Event, ?FUNCTION_NAME, State); certify(Type, Event, State) -> @@ -864,12 +876,14 @@ handle_info(new_cookie_secret, StateName, handle_info(Msg, StateName, State) -> ssl_connection:StateName(info, Msg, State, ?MODULE). -handle_state_timeout(flight_retransmission_timeout, StateName, - #state{flight_state = {retransmit, NextTimeout}} = State0) -> - {State1, Actions} = send_handshake_flight(State0#state{flight_state = {retransmit, NextTimeout}}, +handle_state_timeout(flight_retransmission_timeout, StateName, + #state{flight_state = {retransmit, NextTimeout}} = State0) -> + {State1, Actions0} = send_handshake_flight(State0#state{flight_state = {retransmit, NextTimeout}}, retransmit_epoch(StateName, State0)), - {Record, State} = next_record(State1), - next_event(StateName, Record, State, Actions). + {Record, State2} = next_record(State1), + {next_state, StateName, State, Actions} = next_event(StateName, Record, State2, Actions0), + %% This will reset the retransmission timer by repeating the enter state event + {repeat_state, State, Actions}. handle_alerts([], Result) -> Result; diff --git a/lib/ssl/src/dtls_handshake.erl b/lib/ssl/src/dtls_handshake.erl index 1d6f0a42c8..6071eece13 100644 --- a/lib/ssl/src/dtls_handshake.erl +++ b/lib/ssl/src/dtls_handshake.erl @@ -67,7 +67,8 @@ client_hello(Host, Port, ConnectionStates, SslOpts, %%-------------------------------------------------------------------- client_hello(Host, Port, Cookie, ConnectionStates, #ssl_options{versions = Versions, - ciphers = UserSuites + ciphers = UserSuites, + fallback = Fallback } = SslOpts, Cache, CacheCb, Renegotiation, OwnCert) -> Version = dtls_record:highest_protocol_version(Versions), @@ -83,7 +84,9 @@ client_hello(Host, Port, Cookie, ConnectionStates, #client_hello{session_id = Id, client_version = Version, - cipher_suites = ssl_handshake:cipher_suites(CipherSuites, Renegotiation), + cipher_suites = + ssl_handshake:cipher_suites(CipherSuites, + Renegotiation, Fallback), compression_methods = ssl_record:compressions(), random = SecParams#security_parameters.client_random, cookie = Cookie, @@ -189,7 +192,7 @@ handle_client_hello(Version, no_suite -> ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY); _ -> - {KeyExAlg,_,_,_} = ssl_cipher:suite_definition(CipherSuite), + #{key_exchange := KeyExAlg} = ssl_cipher:suite_definition(CipherSuite), case ssl_handshake:select_hashsign(ClientHashSigns, Cert, KeyExAlg, SupportedHashSigns, TLSVersion) of #alert{} = Alert -> diff --git a/lib/ssl/src/dtls_udp_listener.erl b/lib/ssl/src/dtls_udp_listener.erl index c9e04767aa..0608c6bd2b 100644 --- a/lib/ssl/src/dtls_udp_listener.erl +++ b/lib/ssl/src/dtls_udp_listener.erl @@ -84,7 +84,7 @@ init([Port, EmOpts, InetOptions, DTLSOptions]) -> listener = Socket, close = false}} catch _:_ -> - {error, closed} + {stop, {shutdown, {error, closed}}} end. handle_call({accept, _}, _, #state{close = true} = State) -> {reply, {error, closed}, State}; @@ -153,15 +153,18 @@ handle_info({udp_error, Socket, Error}, #state{listener = Socket} = State) -> handle_info({'DOWN', _, process, Pid, _}, #state{clients = Clients, dtls_processes = Processes0, + dtls_msq_queues = MsgQueues0, close = ListenClosed} = State) -> Client = kv_get(Pid, Processes0), Processes = kv_delete(Pid, Processes0), + MsgQueues = kv_delete(Client, MsgQueues0), case ListenClosed andalso kv_empty(Processes) of true -> {stop, normal, State}; false -> {noreply, State#state{clients = set_delete(Client, Clients), - dtls_processes = Processes}} + dtls_processes = Processes, + dtls_msq_queues = MsgQueues}} end. terminate(_Reason, _State) -> diff --git a/lib/ssl/src/ssl.app.src b/lib/ssl/src/ssl.app.src index c5b55641a1..3c6cd254c1 100644 --- a/lib/ssl/src/ssl.app.src +++ b/lib/ssl/src/ssl.app.src @@ -63,4 +63,4 @@ {env, []}, {mod, {ssl_app, []}}, {runtime_dependencies, ["stdlib-3.2","public_key-1.5","kernel-6.0", - "erts-10.0","crypto-3.3", "inets-5.10.7"]}]}. + "erts-10.0","crypto-4.2", "inets-5.10.7"]}]}. diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index 4bff9fdf39..656ed94ea5 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -374,13 +374,12 @@ negotiated_protocol(#sslsocket{pid = Pid}) -> ssl_connection:negotiated_protocol(Pid). %%-------------------------------------------------------------------- --spec cipher_suites() -> [ssl_cipher:erl_cipher_suite()] | [string()]. +-spec cipher_suites() -> [ssl_cipher:old_erl_cipher_suite()] | [string()]. %%-------------------------------------------------------------------- cipher_suites() -> cipher_suites(erlang). %%-------------------------------------------------------------------- --spec cipher_suites(erlang | openssl | all) -> [ssl_cipher:erl_cipher_suite()] | - [string()]. +-spec cipher_suites(erlang | openssl | all) -> [ssl_cipher:old_erl_cipher_suite() | string()]. %% Description: Returns all supported cipher suites. %%-------------------------------------------------------------------- cipher_suites(erlang) -> @@ -1153,9 +1152,8 @@ binary_cipher_suites(Version, []) -> %% not require explicit configuration ssl_cipher:filter_suites(ssl_cipher:suites(tls_version(Version))); binary_cipher_suites(Version, [Tuple|_] = Ciphers0) when is_tuple(Tuple) -> - Ciphers = [ssl_cipher:suite(C) || C <- Ciphers0], + Ciphers = [ssl_cipher:suite(tuple_to_map(C)) || C <- Ciphers0], binary_cipher_suites(Version, Ciphers); - binary_cipher_suites(Version, [Cipher0 | _] = Ciphers0) when is_binary(Cipher0) -> All = ssl_cipher:all_suites(tls_version(Version)), case [Cipher || Cipher <- Ciphers0, lists:member(Cipher, All)] of @@ -1175,6 +1173,17 @@ binary_cipher_suites(Version, Ciphers0) -> Ciphers = [ssl_cipher:openssl_suite(C) || C <- string:lexemes(Ciphers0, ":")], binary_cipher_suites(Version, Ciphers). +tuple_to_map({Kex, Cipher, Mac}) -> + #{key_exchange => Kex, + cipher => Cipher, + mac => Mac, + prf => default_prf}; +tuple_to_map({Kex, Cipher, Mac, Prf}) -> + #{key_exchange => Kex, + cipher => Cipher, + mac => Mac, + prf => Prf}. + handle_eccs_option(Value, Version) when is_list(Value) -> {_Major, Minor} = tls_version(Version), try tls_v1:ecc_curves(Minor, Value) of diff --git a/lib/ssl/src/ssl_cipher.erl b/lib/ssl/src/ssl_cipher.erl index b6cd22dd13..b0e38fb9ad 100644 --- a/lib/ssl/src/ssl_cipher.erl +++ b/lib/ssl/src/ssl_cipher.erl @@ -44,20 +44,21 @@ is_stream_ciphersuite/1]). -export_type([cipher_suite/0, - erl_cipher_suite/0, openssl_cipher_suite/0, + erl_cipher_suite/0, old_erl_cipher_suite/0, openssl_cipher_suite/0, hash/0, key_algo/0, sign_algo/0]). --type cipher() :: null |rc4_128 | des_cbc | '3des_ede_cbc' - | aes_128_cbc | aes_256_cbc | aes_128_gcm | aes_256_gcm | chacha20_poly1305. +-type cipher() :: null |rc4_128 | des_cbc | '3des_ede_cbc' | aes_128_cbc | aes_256_cbc | aes_128_gcm | aes_256_gcm | chacha20_poly1305. -type hash() :: null | md5 | sha | sha224 | sha256 | sha384 | sha512. -type sign_algo() :: rsa | dsa | ecdsa. --type key_algo() :: null | rsa | dhe_rsa | dhe_dss | ecdhe_ecdsa| ecdh_ecdsa | ecdh_rsa| srp_rsa| srp_dss | - psk | dhe_psk | rsa_psk | dh_anon | ecdh_anon | srp_anon. --type erl_cipher_suite() :: {key_algo(), cipher(), hash()} % Pre TLS 1.2 - %% TLS 1.2, internally PRE TLS 1.2 will use default_prf - | {key_algo(), cipher(), hash(), hash() | default_prf}. - - +-type key_algo() :: null | rsa | dhe_rsa | dhe_dss | ecdhe_ecdsa| ecdh_ecdsa | ecdh_rsa| srp_rsa| srp_dss | psk | dhe_psk | rsa_psk | dh_anon | ecdh_anon | srp_anon. +-type erl_cipher_suite() :: #{key_exchange := key_algo(), + cipher := cipher(), + mac := hash(), + prf := hash() | default_prf %% Old cipher suites, version dependent + }. +-type old_erl_cipher_suite() :: {key_algo(), cipher(), hash()} % Pre TLS 1.2 + %% TLS 1.2, internally PRE TLS 1.2 will use default_prf + | {key_algo(), cipher(), hash(), hash() | default_prf}. -type cipher_suite() :: binary(). -type cipher_enum() :: integer(). -type openssl_cipher_suite() :: string(). @@ -83,7 +84,8 @@ security_parameters(?TLS_NULL_WITH_NULL_NULL = CipherSuite, SecParams) -> %% cipher values has been updated according to <CipherSuite> %%------------------------------------------------------------------- security_parameters(Version, CipherSuite, SecParams) -> - { _, Cipher, Hash, PrfHashAlg} = suite_definition(CipherSuite), + #{cipher := Cipher, mac := Hash, + prf := PrfHashAlg} = suite_definition(CipherSuite), SecParams#security_parameters{ cipher_suite = CipherSuite, bulk_cipher_algorithm = bulk_cipher_algorithm(Cipher), @@ -465,353 +467,740 @@ des_suites(_)-> %%------------------------------------------------------------------- %% TLS v1.1 suites suite_definition(?TLS_NULL_WITH_NULL_NULL) -> - {null, null, null, null}; + #{key_exchange => null, + cipher => null, + mac => null, + prf => null}; %% RFC 5746 - Not a real cipher suite used to signal empty "renegotiation_info" extension %% to avoid handshake failure from old servers that do not ignore %% hello extension data as they should. suite_definition(?TLS_EMPTY_RENEGOTIATION_INFO_SCSV) -> - {null, null, null, null}; -%% suite_definition(?TLS_RSA_WITH_NULL_MD5) -> -%% {rsa, null, md5, default_prf}; -%% suite_definition(?TLS_RSA_WITH_NULL_SHA) -> -%% {rsa, null, sha, default_prf}; + #{key_exchange => null, + cipher => null, + mac => null, + prf => null}; suite_definition(?TLS_RSA_WITH_RC4_128_MD5) -> - {rsa, rc4_128, md5, default_prf}; + #{key_exchange => rsa, + cipher => rc4_128, + mac => md5, + prf => default_prf}; suite_definition(?TLS_RSA_WITH_RC4_128_SHA) -> - {rsa, rc4_128, sha, default_prf}; + #{key_exchange => rsa, + cipher => rc4_128, + mac => sha, + prf => default_prf}; suite_definition(?TLS_RSA_WITH_DES_CBC_SHA) -> - {rsa, des_cbc, sha, default_prf}; + #{key_exchange => rsa, + cipher => des_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_RSA_WITH_3DES_EDE_CBC_SHA) -> - {rsa, '3des_ede_cbc', sha, default_prf}; + #{key_exchange => rsa, + cipher => '3des_ede_cbc', + mac => sha, + prf => default_prf}; suite_definition(?TLS_DHE_DSS_WITH_DES_CBC_SHA) -> - {dhe_dss, des_cbc, sha, default_prf}; + #{key_exchange => dhe_dss, + cipher => des_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA) -> - {dhe_dss, '3des_ede_cbc', sha, default_prf}; + #{key_exchange => dhe_dss, + cipher => '3des_ede_cbc', + mac => sha, + prf => default_prf}; suite_definition(?TLS_DHE_RSA_WITH_DES_CBC_SHA) -> - {dhe_rsa, des_cbc, sha, default_prf}; + #{key_exchange => dhe_rsa, + cipher => des_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA) -> - {dhe_rsa, '3des_ede_cbc', sha, default_prf}; - + #{key_exchange => dhe_rsa, + cipher => '3des_ede_cbc', + mac => sha, + prf => default_prf}; %%% TSL V1.1 AES suites suite_definition(?TLS_RSA_WITH_AES_128_CBC_SHA) -> - {rsa, aes_128_cbc, sha, default_prf}; + #{key_exchange => rsa, + cipher => aes_128_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_DHE_DSS_WITH_AES_128_CBC_SHA) -> - {dhe_dss, aes_128_cbc, sha, default_prf}; + #{key_exchange => dhe_dss, + cipher => aes_128_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_DHE_RSA_WITH_AES_128_CBC_SHA) -> - {dhe_rsa, aes_128_cbc, sha, default_prf}; + #{key_exchange => dhe_rsa, + cipher => aes_128_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_RSA_WITH_AES_256_CBC_SHA) -> - {rsa, aes_256_cbc, sha, default_prf}; + #{key_exchange => rsa, + cipher => aes_256_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_DHE_DSS_WITH_AES_256_CBC_SHA) -> - {dhe_dss, aes_256_cbc, sha, default_prf}; + #{key_exchange => dhe_dss, + cipher => aes_256_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_DHE_RSA_WITH_AES_256_CBC_SHA) -> - {dhe_rsa, aes_256_cbc, sha, default_prf}; - + #{key_exchange => dhe_rsa, + cipher => aes_256_cbc, + mac => sha, + prf => default_prf}; %% TLS v1.2 suites - %% suite_definition(?TLS_RSA_WITH_NULL_SHA) -> %% {rsa, null, sha, default_prf}; suite_definition(?TLS_RSA_WITH_AES_128_CBC_SHA256) -> - {rsa, aes_128_cbc, sha256, default_prf}; + #{key_exchange => rsa, + cipher => aes_128_cbc, + mac => sha256, + prf => default_prf}; suite_definition(?TLS_RSA_WITH_AES_256_CBC_SHA256) -> - {rsa, aes_256_cbc, sha256, default_prf}; + #{key_exchange => rsa, + cipher => aes_256_cbc, + mac => sha256, + prf => default_prf}; suite_definition(?TLS_DHE_DSS_WITH_AES_128_CBC_SHA256) -> - {dhe_dss, aes_128_cbc, sha256, default_prf}; + #{key_exchange => dhe_dss, + cipher => aes_128_cbc, + mac => sha256, + prf => default_prf}; suite_definition(?TLS_DHE_RSA_WITH_AES_128_CBC_SHA256) -> - {dhe_rsa, aes_128_cbc, sha256, default_prf}; + #{key_exchange => dhe_rsa, + cipher => aes_128_cbc, + mac => sha256, + prf => default_prf}; suite_definition(?TLS_DHE_DSS_WITH_AES_256_CBC_SHA256) -> - {dhe_dss, aes_256_cbc, sha256, default_prf}; + #{key_exchange => dhe_dss, + cipher => aes_256_cbc, + mac => sha256, + prf => default_prf}; suite_definition(?TLS_DHE_RSA_WITH_AES_256_CBC_SHA256) -> - {dhe_rsa, aes_256_cbc, sha256, default_prf}; - + #{key_exchange => dhe_rsa, + cipher => aes_256_cbc, + mac => sha256, + prf => default_prf}; %% not defined YET: %% TLS_DH_DSS_WITH_AES_128_CBC_SHA256 DH_DSS AES_128_CBC SHA256 %% TLS_DH_RSA_WITH_AES_128_CBC_SHA256 DH_RSA AES_128_CBC SHA256 %% TLS_DH_DSS_WITH_AES_256_CBC_SHA256 DH_DSS AES_256_CBC SHA256 %% TLS_DH_RSA_WITH_AES_256_CBC_SHA256 DH_RSA AES_256_CBC SHA256 - %%% DH-ANON deprecated by TLS spec and not available %%% by default, but good for testing purposes. suite_definition(?TLS_DH_anon_WITH_RC4_128_MD5) -> - {dh_anon, rc4_128, md5, default_prf}; + #{key_exchange => dh_anon, + cipher => rc4_128, + mac => md5, + prf => default_prf}; suite_definition(?TLS_DH_anon_WITH_DES_CBC_SHA) -> - {dh_anon, des_cbc, sha, default_prf}; + #{key_exchange => dh_anon, + cipher => des_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_DH_anon_WITH_3DES_EDE_CBC_SHA) -> - {dh_anon, '3des_ede_cbc', sha, default_prf}; + #{key_exchange => dh_anon, + cipher => '3des_ede_cbc', + mac => sha, + prf => default_prf}; suite_definition(?TLS_DH_anon_WITH_AES_128_CBC_SHA) -> - {dh_anon, aes_128_cbc, sha, default_prf}; + #{key_exchange => dh_anon, + cipher => aes_128_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_DH_anon_WITH_AES_256_CBC_SHA) -> - {dh_anon, aes_256_cbc, sha, default_prf}; + #{key_exchange => dh_anon, + cipher => aes_256_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_DH_anon_WITH_AES_128_CBC_SHA256) -> - {dh_anon, aes_128_cbc, sha256, default_prf}; + #{key_exchange => dh_anon, + cipher => aes_128_cbc, + mac => sha256, + prf => default_prf}; suite_definition(?TLS_DH_anon_WITH_AES_256_CBC_SHA256) -> - {dh_anon, aes_256_cbc, sha256, default_prf}; - + #{key_exchange => dh_anon, + cipher => aes_256_cbc, + mac => sha256, + prf => default_prf}; %%% PSK Cipher Suites RFC 4279 - suite_definition(?TLS_PSK_WITH_RC4_128_SHA) -> - {psk, rc4_128, sha, default_prf}; + #{key_exchange => psk, + cipher => rc4_128, + mac => sha, + prf => default_prf}; suite_definition(?TLS_PSK_WITH_3DES_EDE_CBC_SHA) -> - {psk, '3des_ede_cbc', sha, default_prf}; + #{key_exchange => psk, + cipher => '3des_ede_cbc', + mac => sha, + prf => default_prf}; suite_definition(?TLS_PSK_WITH_AES_128_CBC_SHA) -> - {psk, aes_128_cbc, sha, default_prf}; + #{key_exchange => psk, + cipher => aes_128_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_PSK_WITH_AES_256_CBC_SHA) -> - {psk, aes_256_cbc, sha, default_prf}; + #{key_exchange => psk, + cipher => aes_256_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_DHE_PSK_WITH_RC4_128_SHA) -> - {dhe_psk, rc4_128, sha, default_prf}; + #{key_exchange => dhe_psk, + cipher => rc4_128, + mac => sha, + prf => default_prf}; suite_definition(?TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA) -> - {dhe_psk, '3des_ede_cbc', sha, default_prf}; + #{key_exchange => dhe_psk, + cipher => '3des_ede_cbc', + mac => sha, + prf => default_prf}; suite_definition(?TLS_DHE_PSK_WITH_AES_128_CBC_SHA) -> - {dhe_psk, aes_128_cbc, sha, default_prf}; + #{key_exchange => dhe_psk, + cipher => aes_128_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_DHE_PSK_WITH_AES_256_CBC_SHA) -> - {dhe_psk, aes_256_cbc, sha, default_prf}; + #{key_exchange => dhe_psk, + cipher => aes_256_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_RSA_PSK_WITH_RC4_128_SHA) -> - {rsa_psk, rc4_128, sha, default_prf}; + #{key_exchange => rsa_psk, + cipher => rc4_128, + mac => sha, + prf => default_prf}; suite_definition(?TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA) -> - {rsa_psk, '3des_ede_cbc', sha, default_prf}; + #{key_exchange => rsa_psk, + cipher => '3des_ede_cbc', + mac => sha, + prf => default_prf}; suite_definition(?TLS_RSA_PSK_WITH_AES_128_CBC_SHA) -> - {rsa_psk, aes_128_cbc, sha, default_prf}; + #{key_exchange => rsa_psk, + cipher => aes_128_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_RSA_PSK_WITH_AES_256_CBC_SHA) -> - {rsa_psk, aes_256_cbc, sha, default_prf}; - + #{key_exchange => rsa_psk, + cipher => aes_256_cbc, + mac => sha, + prf => default_prf}; %%% PSK NULL Cipher Suites RFC 4785 - suite_definition(?TLS_PSK_WITH_NULL_SHA) -> - {psk, null, sha, default_prf}; + #{key_exchange => psk, + cipher => null, + mac => sha, + prf => default_prf}; suite_definition(?TLS_DHE_PSK_WITH_NULL_SHA) -> - {dhe_psk, null, sha, default_prf}; + #{key_exchange => dhe_psk, + cipher => null, + mac => sha, + prf => default_prf}; suite_definition(?TLS_RSA_PSK_WITH_NULL_SHA) -> - {rsa_psk, null, sha, default_prf}; - + #{key_exchange => rsa_psk, + cipher => null, + mac => sha, + prf => default_prf}; %%% TLS 1.2 PSK Cipher Suites RFC 5487 - suite_definition(?TLS_PSK_WITH_AES_128_GCM_SHA256) -> - {psk, aes_128_gcm, null, sha256}; + #{key_exchange => psk, + cipher => aes_128_gcm, + mac => null, + prf => sha256}; suite_definition(?TLS_PSK_WITH_AES_256_GCM_SHA384) -> - {psk, aes_256_gcm, null, sha384}; + #{key_exchange => psk, + cipher => aes_256_gcm, + mac => null, + prf => sha384}; suite_definition(?TLS_DHE_PSK_WITH_AES_128_GCM_SHA256) -> - {dhe_psk, aes_128_gcm, null, sha256}; + #{key_exchange => dhe_psk, + cipher => aes_128_gcm, + mac => null, + prf => sha256}; suite_definition(?TLS_DHE_PSK_WITH_AES_256_GCM_SHA384) -> - {dhe_psk, aes_256_gcm, null, sha384}; + #{key_exchange => dhe_psk, + cipher => aes_256_gcm, + mac => null, + prf => sha384}; suite_definition(?TLS_RSA_PSK_WITH_AES_128_GCM_SHA256) -> - {rsa_psk, aes_128_gcm, null, sha256}; + #{key_exchange => rsa_psk, + cipher => aes_128_gcm, + mac => null, + prf => sha256}; suite_definition(?TLS_RSA_PSK_WITH_AES_256_GCM_SHA384) -> - {rsa_psk, aes_256_gcm, null, sha384}; - + #{key_exchange => rsa_psk, + cipher => aes_256_gcm, + mac => null, + prf => sha384}; suite_definition(?TLS_PSK_WITH_AES_128_CBC_SHA256) -> - {psk, aes_128_cbc, sha256, default_prf}; + #{key_exchange => psk, + cipher => aes_128_cbc, + mac => sha256, + prf => default_prf}; suite_definition(?TLS_PSK_WITH_AES_256_CBC_SHA384) -> - {psk, aes_256_cbc, sha384, default_prf}; + #{key_exchange => psk, + cipher => aes_256_cbc, + mac => sha384, + prf => default_prf}; suite_definition(?TLS_DHE_PSK_WITH_AES_128_CBC_SHA256) -> - {dhe_psk, aes_128_cbc, sha256, default_prf}; + #{key_exchange => dhe_psk, + cipher => aes_128_cbc, + mac => sha256, + prf => default_prf}; suite_definition(?TLS_DHE_PSK_WITH_AES_256_CBC_SHA384) -> - {dhe_psk, aes_256_cbc, sha384, default_prf}; + #{key_exchange => dhe_psk, + cipher => aes_256_cbc, + mac => sha384, + prf => default_prf}; suite_definition(?TLS_RSA_PSK_WITH_AES_128_CBC_SHA256) -> - {rsa_psk, aes_128_cbc, sha256, default_prf}; + #{key_exchange => rsa_psk, + cipher => aes_128_cbc, + mac => sha256, + prf => default_prf}; suite_definition(?TLS_RSA_PSK_WITH_AES_256_CBC_SHA384) -> - {rsa_psk, aes_256_cbc, sha384, default_prf}; - + #{key_exchange => rsa_psk, + cipher => aes_256_cbc, + mac => sha384, + prf => default_prf}; suite_definition(?TLS_PSK_WITH_NULL_SHA256) -> - {psk, null, sha256, default_prf}; + #{key_exchange => psk, + cipher => null, + mac => sha256, + prf => default_prf}; suite_definition(?TLS_PSK_WITH_NULL_SHA384) -> - {psk, null, sha384, default_prf}; + #{key_exchange => psk, + cipher => null, + mac => sha384, + prf => default_prf}; suite_definition(?TLS_DHE_PSK_WITH_NULL_SHA256) -> - {dhe_psk, null, sha256, default_prf}; + #{key_exchange => dhe_psk, + cipher => null, + mac => sha256, + prf => default_prf}; suite_definition(?TLS_DHE_PSK_WITH_NULL_SHA384) -> - {dhe_psk, null, sha384, default_prf}; + #{key_exchange => dhe_psk, + cipher => null, + mac => sha384, + prf => default_prf}; suite_definition(?TLS_RSA_PSK_WITH_NULL_SHA256) -> - {rsa_psk, null, sha256, default_prf}; + #{key_exchange => rsa_psk, + cipher => null, + mac => sha256, + prf => default_prf}; suite_definition(?TLS_RSA_PSK_WITH_NULL_SHA384) -> - {rsa_psk, null, sha384, default_prf}; - + #{key_exchange => rsa_psk, + cipher => null, + mac => sha384, + prf => default_prf}; %%% ECDHE PSK Cipher Suites RFC 5489 - suite_definition(?TLS_ECDHE_PSK_WITH_RC4_128_SHA) -> - {ecdhe_psk, rc4_128, sha, default_prf}; + #{key_exchange => ecdhe_psk, + cipher => rc4_128, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA) -> - {ecdhe_psk, '3des_ede_cbc', sha, default_prf}; + #{key_exchange => ecdhe_psk, + cipher => '3des_ede_cbc', + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA) -> - {ecdhe_psk, aes_128_cbc, sha, default_prf}; + #{key_exchange => ecdhe_psk, + cipher => aes_128_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA) -> - {ecdhe_psk, aes_256_cbc, sha, default_prf}; + #{key_exchange => ecdhe_psk, + cipher => aes_256_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256) -> - {ecdhe_psk, aes_128_cbc, sha256, default_prf}; + #{key_exchange => ecdhe_psk, + cipher => aes_128_cbc, + mac => sha256, + prf => default_prf}; suite_definition(?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384) -> - {ecdhe_psk, aes_256_cbc, sha384, default_prf}; + #{key_exchange => ecdhe_psk, + cipher => aes_256_cbc, + mac => sha384, + prf => default_prf}; suite_definition(?TLS_ECDHE_PSK_WITH_NULL_SHA256) -> - {ecdhe_psk, null, sha256, default_prf}; + #{key_exchange => ecdhe_psk, + cipher => null, + mac => sha256, + prf => default_prf}; suite_definition(?TLS_ECDHE_PSK_WITH_NULL_SHA384) -> - {ecdhe_psk, null, sha384, default_prf}; - + #{key_exchange => ecdhe_psk, + cipher => null, mac => sha384, + prf => default_prf}; %%% ECDHE_PSK with AES-GCM and AES-CCM Cipher Suites, draft-ietf-tls-ecdhe-psk-aead-05 - suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256) -> - {ecdhe_psk, aes_128_gcm, null, sha256}; + #{key_exchange => ecdhe_psk, + cipher => aes_128_gcm, + mac => null, + prf => sha256}; suite_definition(?TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384) -> - {ecdhe_psk, aes_256_gcm, null, sha384}; + #{key_exchange => ecdhe_psk, + cipher => aes_256_gcm, + mac => null, + prf => sha384}; %% suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256) -> -%% {ecdhe_psk, aes_128_ccm, null, sha256}; +%% #{key_exchange => ecdhe_psk, +%% cipher => aes_128_ccm, +%% mac => null, +%% prf =>sha256}; %% suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256) -> -%% {ecdhe_psk, aes_256_ccm, null, sha256}; - +%% #{key_exchange => ecdhe_psk, +%% cipher => aes_256_ccm, +%% mac => null, +%% prf => sha256}; %%% SRP Cipher Suites RFC 5054 - suite_definition(?TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA) -> - {srp_anon, '3des_ede_cbc', sha, default_prf}; + #{key_exchange => srp_anon, + cipher => '3des_ede_cbc', + mac => sha, + prf => default_prf}; suite_definition(?TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA) -> - {srp_rsa, '3des_ede_cbc', sha, default_prf}; + #{key_exchange => srp_rsa, + cipher => '3des_ede_cbc', + mac => sha, + prf => default_prf}; suite_definition(?TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA) -> - {srp_dss, '3des_ede_cbc', sha, default_prf}; + #{key_exchange => srp_dss, + cipher => '3des_ede_cbc', + mac => sha, + prf => default_prf}; suite_definition(?TLS_SRP_SHA_WITH_AES_128_CBC_SHA) -> - {srp_anon, aes_128_cbc, sha, default_prf}; + #{key_exchange => srp_anon, + cipher => aes_128_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA) -> - {srp_rsa, aes_128_cbc, sha, default_prf}; + #{key_exchange => srp_rsa, + cipher => aes_128_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA) -> - {srp_dss, aes_128_cbc, sha, default_prf}; + #{key_exchange => srp_dss, + cipher => aes_128_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_SRP_SHA_WITH_AES_256_CBC_SHA) -> - {srp_anon, aes_256_cbc, sha, default_prf}; + #{key_exchange => srp_anon, + cipher => aes_256_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA) -> - {srp_rsa, aes_256_cbc, sha, default_prf}; + #{key_exchange => srp_rsa, + cipher => aes_256_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA) -> - {srp_dss, aes_256_cbc, sha, default_prf}; - + #{key_exchange => srp_dss, + cipher => aes_256_cbc, + mac => sha, + prf => default_prf}; %% RFC 4492 EC TLS suites suite_definition(?TLS_ECDH_ECDSA_WITH_NULL_SHA) -> - {ecdh_ecdsa, null, sha, default_prf}; + #{key_exchange => ecdh_ecdsa, + cipher => null, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDH_ECDSA_WITH_RC4_128_SHA) -> - {ecdh_ecdsa, rc4_128, sha, default_prf}; + #{key_exchange => ecdh_ecdsa, + cipher => rc4_128, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA) -> - {ecdh_ecdsa, '3des_ede_cbc', sha, default_prf}; + #{key_exchange => ecdh_ecdsa, + cipher => '3des_ede_cbc', + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA) -> - {ecdh_ecdsa, aes_128_cbc, sha, default_prf}; + #{key_exchange => ecdh_ecdsa, + cipher => aes_128_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA) -> - {ecdh_ecdsa, aes_256_cbc, sha, default_prf}; - + #{key_exchange => ecdh_ecdsa, + cipher => aes_256_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDHE_ECDSA_WITH_NULL_SHA) -> - {ecdhe_ecdsa, null, sha, default_prf}; + #{key_exchange => ecdhe_ecdsa, + cipher => null, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDHE_ECDSA_WITH_RC4_128_SHA) -> - {ecdhe_ecdsa, rc4_128, sha, default_prf}; + #{key_exchange => ecdhe_ecdsa, + cipher => rc4_128, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA) -> - {ecdhe_ecdsa, '3des_ede_cbc', sha, default_prf}; + #{key_exchange => ecdhe_ecdsa, + cipher => '3des_ede_cbc', + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA) -> - {ecdhe_ecdsa, aes_128_cbc, sha, default_prf}; + #{key_exchange => ecdhe_ecdsa, + cipher => aes_128_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA) -> - {ecdhe_ecdsa, aes_256_cbc, sha, default_prf}; - + #{key_exchange => ecdhe_ecdsa, + cipher => aes_256_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDH_RSA_WITH_NULL_SHA) -> - {ecdh_rsa, null, sha, default_prf}; + #{key_exchange => ecdh_rsa, + cipher => null, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDH_RSA_WITH_RC4_128_SHA) -> - {ecdh_rsa, rc4_128, sha, default_prf}; + #{key_exchange => ecdh_rsa, + cipher => rc4_128, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA) -> - {ecdh_rsa, '3des_ede_cbc', sha, default_prf}; + #{key_exchange => ecdh_rsa, + cipher => '3des_ede_cbc', + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDH_RSA_WITH_AES_128_CBC_SHA) -> - {ecdh_rsa, aes_128_cbc, sha, default_prf}; + #{key_exchange => ecdh_rsa, + cipher => aes_128_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDH_RSA_WITH_AES_256_CBC_SHA) -> - {ecdh_rsa, aes_256_cbc, sha, default_prf}; - + #{key_exchange => ecdh_rsa, + cipher => aes_256_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDHE_RSA_WITH_NULL_SHA) -> - {ecdhe_rsa, null, sha, default_prf}; + #{key_exchange => ecdhe_rsa, + cipher => null, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDHE_RSA_WITH_RC4_128_SHA) -> - {ecdhe_rsa, rc4_128, sha, default_prf}; + #{key_exchange => ecdhe_rsa, + cipher => rc4_128, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA) -> - {ecdhe_rsa, '3des_ede_cbc', sha, default_prf}; + #{key_exchange => ecdhe_rsa, + cipher => '3des_ede_cbc', + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA) -> - {ecdhe_rsa, aes_128_cbc, sha, default_prf}; + #{key_exchange => ecdhe_rsa, + cipher => aes_128_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA) -> - {ecdhe_rsa, aes_256_cbc, sha, default_prf}; - + #{key_exchange => ecdhe_rsa, + cipher => aes_256_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDH_anon_WITH_NULL_SHA) -> - {ecdh_anon, null, sha, default_prf}; + #{key_exchange => ecdh_anon, + cipher => null, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDH_anon_WITH_RC4_128_SHA) -> - {ecdh_anon, rc4_128, sha, default_prf}; + #{key_exchange => ecdh_anon, + cipher => rc4_128, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA) -> - {ecdh_anon, '3des_ede_cbc', sha, default_prf}; + #{key_exchange => ecdh_anon, + cipher => '3des_ede_cbc', + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDH_anon_WITH_AES_128_CBC_SHA) -> - {ecdh_anon, aes_128_cbc, sha, default_prf}; + #{key_exchange => ecdh_anon, + cipher => aes_128_cbc, + mac => sha, + prf => default_prf}; suite_definition(?TLS_ECDH_anon_WITH_AES_256_CBC_SHA) -> - {ecdh_anon, aes_256_cbc, sha, default_prf}; - + #{key_exchange => ecdh_anon, + cipher => aes_256_cbc, + mac => sha, + prf => default_prf}; %% RFC 5289 EC TLS suites suite_definition(?TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256) -> - {ecdhe_ecdsa, aes_128_cbc, sha256, sha256}; + #{key_exchange => ecdhe_ecdsa, + cipher => aes_128_cbc, + mac => sha256, + prf => sha256}; suite_definition(?TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384) -> - {ecdhe_ecdsa, aes_256_cbc, sha384, sha384}; + #{key_exchange => ecdhe_ecdsa, + cipher => aes_256_cbc, + mac => sha384, + prf => sha384}; suite_definition(?TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256) -> - {ecdh_ecdsa, aes_128_cbc, sha256, sha256}; + #{key_exchange => ecdh_ecdsa, + cipher => aes_128_cbc, + mac => sha256, + prf => sha256}; suite_definition(?TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384) -> - {ecdh_ecdsa, aes_256_cbc, sha384, sha384}; + #{key_exchange => ecdh_ecdsa, + cipher => aes_256_cbc, + mac => sha384, + prf => sha384}; suite_definition(?TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256) -> - {ecdhe_rsa, aes_128_cbc, sha256, sha256}; + #{key_exchange => ecdhe_rsa, + cipher => aes_128_cbc, + mac => sha256, + prf => sha256}; suite_definition(?TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384) -> - {ecdhe_rsa, aes_256_cbc, sha384, sha384}; + #{key_exchange => ecdhe_rsa, + cipher => aes_256_cbc, + mac => sha384, + prf => sha384}; suite_definition(?TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256) -> - {ecdh_rsa, aes_128_cbc, sha256, sha256}; + #{key_exchange => ecdh_rsa, + cipher => aes_128_cbc, + mac => sha256, + prf => sha256}; suite_definition(?TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384) -> - {ecdh_rsa, aes_256_cbc, sha384, sha384}; - + #{key_exchange => ecdh_rsa, + cipher => aes_256_cbc, + mac => sha384, + prf => sha384}; %% RFC 5288 AES-GCM Cipher Suites suite_definition(?TLS_RSA_WITH_AES_128_GCM_SHA256) -> - {rsa, aes_128_gcm, null, sha256}; + #{key_exchange => rsa, + cipher => aes_128_gcm, + mac => null, + prf => sha256}; suite_definition(?TLS_RSA_WITH_AES_256_GCM_SHA384) -> - {rsa, aes_256_gcm, null, sha384}; + #{key_exchange => rsa, + cipher => aes_256_gcm, + mac => null, + prf => sha384}; suite_definition(?TLS_DHE_RSA_WITH_AES_128_GCM_SHA256) -> - {dhe_rsa, aes_128_gcm, null, sha256}; + #{key_exchange => dhe_rsa, + cipher => aes_128_gcm, + mac => null, + prf => sha256}; suite_definition(?TLS_DHE_RSA_WITH_AES_256_GCM_SHA384) -> - {dhe_rsa, aes_256_gcm, null, sha384}; + #{key_exchange => dhe_rsa, + cipher => aes_256_gcm, + mac => null, + prf => sha384}; suite_definition(?TLS_DH_RSA_WITH_AES_128_GCM_SHA256) -> - {dh_rsa, aes_128_gcm, null, sha256}; + #{key_exchange => dh_rsa, + cipher => aes_128_gcm, + mac => null, + prf => sha256}; suite_definition(?TLS_DH_RSA_WITH_AES_256_GCM_SHA384) -> - {dh_rsa, aes_256_gcm, null, sha384}; + #{key_exchange => dh_rsa, + cipher => aes_256_gcm, + mac => null, + prf => sha384}; suite_definition(?TLS_DHE_DSS_WITH_AES_128_GCM_SHA256) -> - {dhe_dss, aes_128_gcm, null, sha256}; + #{key_exchange => dhe_dss, + cipher => aes_128_gcm, + mac => null, + prf => sha256}; suite_definition(?TLS_DHE_DSS_WITH_AES_256_GCM_SHA384) -> - {dhe_dss, aes_256_gcm, null, sha384}; + #{key_exchange => dhe_dss, + cipher => aes_256_gcm, + mac => null, + prf => sha384}; suite_definition(?TLS_DH_DSS_WITH_AES_128_GCM_SHA256) -> - {dh_dss, aes_128_gcm, null, sha256}; + #{key_exchange => dh_dss, + cipher => aes_128_gcm, + mac => null, + prf => sha256}; suite_definition(?TLS_DH_DSS_WITH_AES_256_GCM_SHA384) -> - {dh_dss, aes_256_gcm, null, sha384}; + #{key_exchange => dh_dss, + cipher => aes_256_gcm, + mac => null, + prf => sha384}; suite_definition(?TLS_DH_anon_WITH_AES_128_GCM_SHA256) -> - {dh_anon, aes_128_gcm, null, sha256}; + #{key_exchange => dh_anon, + cipher => aes_128_gcm, + mac => null, + prf => sha256}; suite_definition(?TLS_DH_anon_WITH_AES_256_GCM_SHA384) -> - {dh_anon, aes_256_gcm, null, sha384}; - + #{key_exchange => dh_anon, + cipher => aes_256_gcm, + mac => null, + prf => sha384}; %% RFC 5289 ECC AES-GCM Cipher Suites suite_definition(?TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256) -> - {ecdhe_ecdsa, aes_128_gcm, null, sha256}; + #{key_exchange => ecdhe_ecdsa, + cipher => aes_128_gcm, + mac => null, + prf => sha256}; suite_definition(?TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384) -> - {ecdhe_ecdsa, aes_256_gcm, null, sha384}; + #{key_exchange => ecdhe_ecdsa, + cipher => aes_256_gcm, + mac => null, + prf => sha384}; suite_definition(?TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256) -> - {ecdh_ecdsa, aes_128_gcm, null, sha256}; + #{key_exchange => ecdh_ecdsa, + cipher => aes_128_gcm, + mac => null, + prf => sha256}; suite_definition(?TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384) -> - {ecdh_ecdsa, aes_256_gcm, null, sha384}; + #{key_exchange => ecdh_ecdsa, + cipher => aes_256_gcm, + mac => null, + prf => sha384}; suite_definition(?TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) -> - {ecdhe_rsa, aes_128_gcm, null, sha256}; + #{key_exchange => ecdhe_rsa, + cipher => aes_128_gcm, + mac => null, + prf => sha256}; suite_definition(?TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) -> - {ecdhe_rsa, aes_256_gcm, null, sha384}; + #{key_exchange => ecdhe_rsa, + cipher => aes_256_gcm, + mac => null, + prf => sha384}; suite_definition(?TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256) -> - {ecdh_rsa, aes_128_gcm, null, sha256}; + #{key_exchange => ecdh_rsa, + cipher => aes_128_gcm, + mac => null, + prf => sha256}; suite_definition(?TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384) -> - {ecdh_rsa, aes_256_gcm, null, sha384}; - + #{key_exchange => ecdh_rsa, + cipher => aes_256_gcm, + mac => null, + prf => sha384}; %% draft-agl-tls-chacha20poly1305-04 Chacha20/Poly1305 Suites suite_definition(?TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256) -> - {ecdhe_rsa, chacha20_poly1305, null, sha256}; + #{key_exchange => ecdhe_rsa, + cipher => chacha20_poly1305, + mac => null, + prf => sha256}; suite_definition(?TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256) -> - {ecdhe_ecdsa, chacha20_poly1305, null, sha256}; + #{key_exchange => ecdhe_ecdsa, + cipher => chacha20_poly1305, + mac => null, + prf => sha256}; suite_definition(?TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256) -> - {dhe_rsa, chacha20_poly1305, null, sha256}. + #{key_exchange => dhe_rsa, + cipher => chacha20_poly1305, + mac => null, + prf => sha256}. %%-------------------------------------------------------------------- --spec erl_suite_definition(cipher_suite()) -> erl_cipher_suite(). +-spec erl_suite_definition(cipher_suite() | erl_cipher_suite()) -> old_erl_cipher_suite(). %% %% Description: Return erlang cipher suite definition. Filters last value %% for now (compatibility reasons). %%-------------------------------------------------------------------- -erl_suite_definition(S) -> - case suite_definition(S) of - {KeyExchange, Cipher, Hash, default_prf} -> +erl_suite_definition(Bin) when is_binary(Bin) -> + erl_suite_definition(suite_definition(Bin)); +erl_suite_definition(#{key_exchange := KeyExchange, cipher := Cipher, + mac := Hash, prf := Prf}) -> + case Prf of + default_prf -> {KeyExchange, Cipher, Hash}; - Suite -> - Suite + _ -> + {KeyExchange, Cipher, Hash, Prf} end. %%-------------------------------------------------------------------- @@ -819,327 +1208,607 @@ erl_suite_definition(S) -> %% %% Description: Return TLS cipher suite definition. %%-------------------------------------------------------------------- - %% TLS v1.1 suites -%%suite({rsa, null, md5}) -> -%% ?TLS_RSA_WITH_NULL_MD5; -%%suite({rsa, null, sha}) -> -%% ?TLS_RSA_WITH_NULL_SHA; -suite({rsa, rc4_128, md5}) -> +suite(#{key_exchange := rsa, + cipher := rc4_128, + mac := md5}) -> ?TLS_RSA_WITH_RC4_128_MD5; -suite({rsa, rc4_128, sha}) -> +suite(#{key_exchange := rsa, + cipher := rc4_128, + mac := sha}) -> ?TLS_RSA_WITH_RC4_128_SHA; -suite({rsa, des_cbc, sha}) -> +suite(#{key_exchange := rsa, + cipher := des_cbc, + mac := sha}) -> ?TLS_RSA_WITH_DES_CBC_SHA; -suite({rsa, '3des_ede_cbc', sha}) -> +suite(#{key_exchange := rsa, + cipher :='3des_ede_cbc', + mac := sha}) -> ?TLS_RSA_WITH_3DES_EDE_CBC_SHA; -suite({dhe_dss, des_cbc, sha}) -> +suite(#{key_exchange := dhe_dss, + cipher:= des_cbc, + mac := sha}) -> ?TLS_DHE_DSS_WITH_DES_CBC_SHA; -suite({dhe_dss, '3des_ede_cbc', sha}) -> +suite(#{key_exchange := dhe_dss, + cipher:= '3des_ede_cbc', + mac := sha}) -> ?TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA; -suite({dhe_rsa, des_cbc, sha}) -> +suite(#{key_exchange := dhe_rsa, + cipher:= des_cbc, + mac := sha}) -> ?TLS_DHE_RSA_WITH_DES_CBC_SHA; -suite({dhe_rsa, '3des_ede_cbc', sha}) -> +suite(#{key_exchange := dhe_rsa, + cipher:= '3des_ede_cbc', + mac := sha}) -> ?TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA; -suite({dh_anon, rc4_128, md5}) -> +suite(#{key_exchange := dh_anon, + cipher:= rc4_128, + mac := md5}) -> ?TLS_DH_anon_WITH_RC4_128_MD5; -suite({dh_anon, des_cbc, sha}) -> +suite(#{key_exchange := dh_anon, + cipher:= des_cbc, + mac := sha}) -> ?TLS_DH_anon_WITH_DES_CBC_SHA; -suite({dh_anon, '3des_ede_cbc', sha}) -> +suite(#{key_exchange := dh_anon, + cipher:= '3des_ede_cbc', + mac := sha}) -> ?TLS_DH_anon_WITH_3DES_EDE_CBC_SHA; - %%% TSL V1.1 AES suites -suite({rsa, aes_128_cbc, sha}) -> +suite(#{key_exchange := rsa, + cipher := aes_128_cbc, + mac := sha}) -> ?TLS_RSA_WITH_AES_128_CBC_SHA; -suite({dhe_dss, aes_128_cbc, sha}) -> +suite(#{key_exchange := dhe_dss, + cipher := aes_128_cbc, + mac := sha}) -> ?TLS_DHE_DSS_WITH_AES_128_CBC_SHA; -suite({dhe_rsa, aes_128_cbc, sha}) -> +suite(#{key_exchange := dhe_rsa, + cipher := aes_128_cbc, + mac := sha}) -> ?TLS_DHE_RSA_WITH_AES_128_CBC_SHA; -suite({dh_anon, aes_128_cbc, sha}) -> +suite(#{key_exchange := dh_anon, + cipher := aes_128_cbc, + mac := sha}) -> ?TLS_DH_anon_WITH_AES_128_CBC_SHA; -suite({rsa, aes_256_cbc, sha}) -> +suite(#{key_exchange := rsa, + cipher := aes_256_cbc, + mac := sha}) -> ?TLS_RSA_WITH_AES_256_CBC_SHA; -suite({dhe_dss, aes_256_cbc, sha}) -> +suite(#{key_exchange := dhe_dss, + cipher := aes_256_cbc, + mac := sha}) -> ?TLS_DHE_DSS_WITH_AES_256_CBC_SHA; -suite({dhe_rsa, aes_256_cbc, sha}) -> +suite(#{key_exchange := dhe_rsa, + cipher := aes_256_cbc, + mac := sha}) -> ?TLS_DHE_RSA_WITH_AES_256_CBC_SHA; -suite({dh_anon, aes_256_cbc, sha}) -> +suite(#{key_exchange := dh_anon, + cipher := aes_256_cbc, + mac := sha}) -> ?TLS_DH_anon_WITH_AES_256_CBC_SHA; - %% TLS v1.2 suites - -%% suite_definition(?TLS_RSA_WITH_NULL_SHA) -> -%% {rsa, null, sha, sha256}; -suite({rsa, aes_128_cbc, sha256}) -> +suite(#{key_exchange := rsa, + cipher := aes_128_cbc, + mac := sha256}) -> ?TLS_RSA_WITH_AES_128_CBC_SHA256; -suite({rsa, aes_256_cbc, sha256}) -> +suite(#{key_exchange := rsa, + cipher := aes_256_cbc, + mac := sha256}) -> ?TLS_RSA_WITH_AES_256_CBC_SHA256; -suite({dhe_dss, aes_128_cbc, sha256}) -> +suite(#{key_exchange := dhe_dss, + cipher := aes_128_cbc, + mac := sha256}) -> ?TLS_DHE_DSS_WITH_AES_128_CBC_SHA256; -suite({dhe_rsa, aes_128_cbc, sha256}) -> +suite(#{key_exchange := dhe_rsa, + cipher := aes_128_cbc, + mac := sha256}) -> ?TLS_DHE_RSA_WITH_AES_128_CBC_SHA256; -suite({dhe_dss, aes_256_cbc, sha256}) -> +suite(#{key_exchange := dhe_dss, + cipher := aes_256_cbc, + mac := sha256}) -> ?TLS_DHE_DSS_WITH_AES_256_CBC_SHA256; -suite({dhe_rsa, aes_256_cbc, sha256}) -> +suite(#{key_exchange := dhe_rsa, + cipher := aes_256_cbc, + mac := sha256}) -> ?TLS_DHE_RSA_WITH_AES_256_CBC_SHA256; -suite({dh_anon, aes_128_cbc, sha256}) -> +suite(#{key_exchange := dh_anon, + cipher := aes_128_cbc, + mac := sha256}) -> ?TLS_DH_anon_WITH_AES_128_CBC_SHA256; -suite({dh_anon, aes_256_cbc, sha256}) -> +suite(#{key_exchange := dh_anon, + cipher := aes_256_cbc, + mac := sha256}) -> ?TLS_DH_anon_WITH_AES_256_CBC_SHA256; - %%% PSK Cipher Suites RFC 4279 - -suite({psk, rc4_128,sha}) -> +suite(#{key_exchange := psk, + cipher := rc4_128, + mac := sha}) -> ?TLS_PSK_WITH_RC4_128_SHA; -suite({psk, '3des_ede_cbc',sha}) -> +suite(#{key_exchange := psk, + cipher := '3des_ede_cbc', + mac := sha}) -> ?TLS_PSK_WITH_3DES_EDE_CBC_SHA; -suite({psk, aes_128_cbc,sha}) -> +suite(#{key_exchange := psk, + cipher := aes_128_cbc, + mac := sha}) -> ?TLS_PSK_WITH_AES_128_CBC_SHA; -suite({psk, aes_256_cbc,sha}) -> +suite(#{key_exchange := psk, + cipher := aes_256_cbc, + mac := sha}) -> ?TLS_PSK_WITH_AES_256_CBC_SHA; -suite({dhe_psk, rc4_128,sha}) -> +suite(#{key_exchange := dhe_psk, + cipher := rc4_128, + mac := sha}) -> ?TLS_DHE_PSK_WITH_RC4_128_SHA; -suite({dhe_psk, '3des_ede_cbc',sha}) -> +suite(#{key_exchange := dhe_psk, + cipher := '3des_ede_cbc', + mac := sha}) -> ?TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA; -suite({dhe_psk, aes_128_cbc,sha}) -> +suite(#{key_exchange := dhe_psk, + cipher := aes_128_cbc, + mac := sha}) -> ?TLS_DHE_PSK_WITH_AES_128_CBC_SHA; -suite({dhe_psk, aes_256_cbc,sha}) -> +suite(#{key_exchange := dhe_psk, + cipher := aes_256_cbc, + mac := sha}) -> ?TLS_DHE_PSK_WITH_AES_256_CBC_SHA; -suite({rsa_psk, rc4_128,sha}) -> +suite(#{key_exchange := rsa_psk, + cipher := rc4_128, + mac := sha}) -> ?TLS_RSA_PSK_WITH_RC4_128_SHA; -suite({rsa_psk, '3des_ede_cbc',sha}) -> +suite(#{key_exchange := rsa_psk, + cipher := '3des_ede_cbc', + mac := sha}) -> ?TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA; -suite({rsa_psk, aes_128_cbc,sha}) -> +suite(#{key_exchange := rsa_psk, + cipher := aes_128_cbc, + mac := sha}) -> ?TLS_RSA_PSK_WITH_AES_128_CBC_SHA; -suite({rsa_psk, aes_256_cbc,sha}) -> +suite(#{key_exchange := rsa_psk, + cipher := aes_256_cbc, + mac := sha}) -> ?TLS_RSA_PSK_WITH_AES_256_CBC_SHA; - %%% PSK NULL Cipher Suites RFC 4785 - -suite({psk, null, sha}) -> +suite(#{key_exchange := psk, + cipher := null, + mac := sha}) -> ?TLS_PSK_WITH_NULL_SHA; -suite({dhe_psk, null, sha}) -> +suite(#{key_exchange := dhe_psk, + cipher := null, + mac := sha}) -> ?TLS_DHE_PSK_WITH_NULL_SHA; -suite({rsa_psk, null, sha}) -> +suite(#{key_exchange := rsa_psk, + cipher := null, + mac := sha}) -> ?TLS_RSA_PSK_WITH_NULL_SHA; - %%% TLS 1.2 PSK Cipher Suites RFC 5487 - -suite({psk, aes_128_gcm, null, sha256}) -> +suite(#{key_exchange := psk, + cipher := aes_128_gcm, + mac := null, + prf := sha256}) -> ?TLS_PSK_WITH_AES_128_GCM_SHA256; -suite({psk, aes_256_gcm, null, sha384}) -> +suite(#{key_exchange := psk, + cipher := aes_256_gcm, + mac := null, + prf := sha384}) -> ?TLS_PSK_WITH_AES_256_GCM_SHA384; -suite({dhe_psk, aes_128_gcm, null, sha256}) -> +suite(#{key_exchange := dhe_psk, + cipher := aes_128_gcm, + mac := null, + prf := sha256}) -> ?TLS_DHE_PSK_WITH_AES_128_GCM_SHA256; -suite({dhe_psk, aes_256_gcm, null, sha384}) -> +suite(#{key_exchange := dhe_psk, + cipher := aes_256_gcm, + mac := null, + prf := sha384}) -> ?TLS_DHE_PSK_WITH_AES_256_GCM_SHA384; -suite({rsa_psk, aes_128_gcm, null, sha256}) -> +suite(#{key_exchange := rsa_psk, + cipher := aes_128_gcm, + mac := null, + prf := sha256}) -> ?TLS_RSA_PSK_WITH_AES_128_GCM_SHA256; -suite({rsa_psk, aes_256_gcm, null, sha384}) -> +suite(#{key_exchange := rsa_psk, + cipher := aes_256_gcm, + mac := null, + prf := sha384}) -> ?TLS_RSA_PSK_WITH_AES_256_GCM_SHA384; - -suite({psk, aes_128_cbc, sha256}) -> +suite(#{key_exchange := psk, + cipher := aes_128_cbc, + mac := sha256}) -> ?TLS_PSK_WITH_AES_128_CBC_SHA256; -suite({psk, aes_256_cbc, sha384}) -> +suite(#{key_exchange := psk, + cipher := aes_256_cbc, + mac := sha384}) -> ?TLS_PSK_WITH_AES_256_CBC_SHA384; -suite({dhe_psk, aes_128_cbc, sha256}) -> +suite(#{key_exchange := dhe_psk, + cipher := aes_128_cbc, + mac := sha256}) -> ?TLS_DHE_PSK_WITH_AES_128_CBC_SHA256; -suite({dhe_psk, aes_256_cbc, sha384}) -> +suite(#{key_exchange := dhe_psk, + cipher := aes_256_cbc, + mac := sha384}) -> ?TLS_DHE_PSK_WITH_AES_256_CBC_SHA384; -suite({rsa_psk, aes_128_cbc, sha256}) -> +suite(#{key_exchange := rsa_psk, + cipher := aes_128_cbc, + mac := sha256}) -> ?TLS_RSA_PSK_WITH_AES_128_CBC_SHA256; -suite({rsa_psk, aes_256_cbc, sha384}) -> +suite(#{key_exchange := rsa_psk, + cipher := aes_256_cbc, + mac := sha384}) -> ?TLS_RSA_PSK_WITH_AES_256_CBC_SHA384; - -suite({psk, null, sha256}) -> +suite(#{key_exchange := psk, + cipher := null, + mac := sha256}) -> ?TLS_PSK_WITH_NULL_SHA256; -suite({psk, null, sha384}) -> +suite(#{key_exchange := psk, + cipher := null, + mac := sha384}) -> ?TLS_PSK_WITH_NULL_SHA384; -suite({dhe_psk, null, sha256}) -> +suite(#{key_exchange := dhe_psk, + cipher := null, + mac := sha256}) -> ?TLS_DHE_PSK_WITH_NULL_SHA256; -suite({dhe_psk, null, sha384}) -> +suite(#{key_exchange := dhe_psk, + cipher := null, + mac := sha384}) -> ?TLS_DHE_PSK_WITH_NULL_SHA384; -suite({rsa_psk, null, sha256}) -> +suite(#{key_exchange := rsa_psk, + cipher := null, + mac := sha256}) -> ?TLS_RSA_PSK_WITH_NULL_SHA256; -suite({rsa_psk, null, sha384}) -> +suite(#{key_exchange := rsa_psk, + cipher := null, + mac := sha384}) -> ?TLS_RSA_PSK_WITH_NULL_SHA384; - %%% ECDHE PSK Cipher Suites RFC 5489 - -suite({ecdhe_psk, rc4_128,sha}) -> +suite(#{key_exchange := ecdhe_psk, + cipher := rc4_128, + mac := sha}) -> ?TLS_ECDHE_PSK_WITH_RC4_128_SHA; -suite({ecdhe_psk, '3des_ede_cbc',sha}) -> +suite(#{key_exchange := ecdhe_psk, + cipher :='3des_ede_cbc', + mac := sha}) -> ?TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA; -suite({ecdhe_psk, aes_128_cbc,sha}) -> +suite(#{key_exchange := ecdhe_psk, + cipher := aes_128_cbc, + mac := sha}) -> ?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA; -suite({ecdhe_psk, aes_256_cbc,sha}) -> +suite(#{key_exchange := ecdhe_psk, + cipher := aes_256_cbc, + mac := sha}) -> ?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA; -suite({ecdhe_psk, aes_128_cbc, sha256}) -> +suite(#{key_exchange := ecdhe_psk, + cipher := aes_128_cbc, + mac := sha256}) -> ?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256; -suite({ecdhe_psk, aes_256_cbc, sha384}) -> +suite(#{key_exchange := ecdhe_psk, + cipher := aes_256_cbc, + mac := sha384}) -> ?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384; -suite({ecdhe_psk, null, sha256}) -> +suite(#{key_exchange := ecdhe_psk, + cipher := null, + mac := sha256}) -> ?TLS_ECDHE_PSK_WITH_NULL_SHA256; -suite({ecdhe_psk, null, sha384}) -> +suite(#{key_exchange := ecdhe_psk, + cipher := null, + mac := sha384}) -> ?TLS_ECDHE_PSK_WITH_NULL_SHA384; - %%% ECDHE_PSK with AES-GCM and AES-CCM Cipher Suites, draft-ietf-tls-ecdhe-psk-aead-05 - -suite({ecdhe_psk, aes_128_gcm, null, sha256}) -> +suite(#{key_exchange := ecdhe_psk, + cipher := aes_128_gcm, + mac := null, + prf := sha256}) -> ?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256; -suite({ecdhe_psk, aes_256_gcm, null, sha384}) -> +suite(#{key_exchange := ecdhe_psk, + cipher := aes_256_gcm, + mac := null, + prf := sha384}) -> ?TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384; -%% suite({ecdhe_psk, aes_128_ccm, null, sha256}) -> -%% ?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256; -%% suite({ecdhe_psk, aes_256_ccm, null, sha256}) -> -%% ?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256; - + %% suite(#{key_exchange := ecdhe_psk, + %% cipher := aes_128_ccm, + %% mac := null, + %% prf := sha256}) -> + %% ?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256; + %% suite(#{key_exchange := ecdhe_psk, + %% cipher := aes_256_ccm, + %% mac := null, + %% prf := sha256}) -> + %% ?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256; %%% SRP Cipher Suites RFC 5054 - -suite({srp_anon, '3des_ede_cbc', sha}) -> +suite(#{key_exchange := srp_anon, + cipher := '3des_ede_cbc', + mac := sha}) -> ?TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA; -suite({srp_rsa, '3des_ede_cbc', sha}) -> +suite(#{key_exchange := srp_rsa, + cipher := '3des_ede_cbc', + mac := sha}) -> ?TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA; -suite({srp_dss, '3des_ede_cbc', sha}) -> +suite(#{key_exchange := srp_dss, + cipher := '3des_ede_cbc', + mac := sha}) -> ?TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA; -suite({srp_anon, aes_128_cbc, sha}) -> +suite(#{key_exchange := srp_anon, + cipher := aes_128_cbc, + mac := sha}) -> ?TLS_SRP_SHA_WITH_AES_128_CBC_SHA; -suite({srp_rsa, aes_128_cbc, sha}) -> +suite(#{key_exchange := srp_rsa, + cipher := aes_128_cbc, + mac := sha}) -> ?TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA; -suite({srp_dss, aes_128_cbc, sha}) -> +suite(#{key_exchange := srp_dss, + cipher := aes_128_cbc, + mac := sha}) -> ?TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA; -suite({srp_anon, aes_256_cbc, sha}) -> +suite(#{key_exchange := srp_anon, + cipher := aes_256_cbc, + mac := sha}) -> ?TLS_SRP_SHA_WITH_AES_256_CBC_SHA; -suite({srp_rsa, aes_256_cbc, sha}) -> +suite(#{key_exchange := srp_rsa, + cipher := aes_256_cbc, + mac := sha}) -> ?TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA; -suite({srp_dss, aes_256_cbc, sha}) -> +suite(#{key_exchange := srp_dss, + cipher := aes_256_cbc, + mac := sha}) -> ?TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA; - %%% RFC 4492 EC TLS suites -suite({ecdh_ecdsa, null, sha}) -> +suite(#{key_exchange := ecdh_ecdsa, + cipher := null, + mac := sha}) -> ?TLS_ECDH_ECDSA_WITH_NULL_SHA; -suite({ecdh_ecdsa, rc4_128, sha}) -> +suite(#{key_exchange := ecdh_ecdsa, + cipher := rc4_128, + mac := sha}) -> ?TLS_ECDH_ECDSA_WITH_RC4_128_SHA; -suite({ecdh_ecdsa, '3des_ede_cbc', sha}) -> +suite(#{key_exchange := ecdh_ecdsa, + cipher := '3des_ede_cbc', + mac := sha}) -> ?TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA; -suite({ecdh_ecdsa, aes_128_cbc, sha}) -> +suite(#{key_exchange := ecdh_ecdsa, + cipher := aes_128_cbc, + mac := sha}) -> ?TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA; -suite({ecdh_ecdsa, aes_256_cbc, sha}) -> +suite(#{key_exchange := ecdh_ecdsa, + cipher := aes_256_cbc, + mac := sha}) -> ?TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA; - -suite({ecdhe_ecdsa, null, sha}) -> +suite(#{key_exchange := ecdhe_ecdsa, + cipher := null, + mac := sha}) -> ?TLS_ECDHE_ECDSA_WITH_NULL_SHA; -suite({ecdhe_ecdsa, rc4_128, sha}) -> +suite(#{key_exchange := ecdhe_ecdsa, + cipher := rc4_128, + mac := sha}) -> ?TLS_ECDHE_ECDSA_WITH_RC4_128_SHA; -suite({ecdhe_ecdsa, '3des_ede_cbc', sha}) -> +suite(#{key_exchange := ecdhe_ecdsa, + cipher := '3des_ede_cbc', + mac := sha}) -> ?TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA; -suite({ecdhe_ecdsa, aes_128_cbc, sha}) -> +suite(#{key_exchange := ecdhe_ecdsa, + cipher := aes_128_cbc, + mac := sha}) -> ?TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA; -suite({ecdhe_ecdsa, aes_256_cbc, sha}) -> +suite(#{key_exchange := ecdhe_ecdsa, + cipher := aes_256_cbc, + mac := sha}) -> ?TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA; - -suite({ecdh_rsa, null, sha}) -> +suite(#{key_exchange := ecdh_rsa, + cipher := null, + mac := sha}) -> ?TLS_ECDH_RSA_WITH_NULL_SHA; -suite({ecdh_rsa, rc4_128, sha}) -> +suite(#{key_exchange := ecdh_rsa, + cipher := rc4_128, + mac := sha}) -> ?TLS_ECDH_RSA_WITH_RC4_128_SHA; -suite({ecdh_rsa, '3des_ede_cbc', sha}) -> +suite(#{key_exchange := ecdh_rsa, + cipher := '3des_ede_cbc', mac := sha}) -> ?TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA; -suite({ecdh_rsa, aes_128_cbc, sha}) -> +suite(#{key_exchange := ecdh_rsa, + cipher := aes_128_cbc, + mac := sha}) -> ?TLS_ECDH_RSA_WITH_AES_128_CBC_SHA; -suite({ecdh_rsa, aes_256_cbc, sha}) -> +suite(#{key_exchange := ecdh_rsa, + cipher := aes_256_cbc, + mac := sha}) -> ?TLS_ECDH_RSA_WITH_AES_256_CBC_SHA; - -suite({ecdhe_rsa, null, sha}) -> +suite(#{key_exchange := ecdhe_rsa, + cipher := null, + mac := sha}) -> ?TLS_ECDHE_RSA_WITH_NULL_SHA; -suite({ecdhe_rsa, rc4_128, sha}) -> +suite(#{key_exchange := ecdhe_rsa, + cipher := rc4_128, + mac := sha}) -> ?TLS_ECDHE_RSA_WITH_RC4_128_SHA; -suite({ecdhe_rsa, '3des_ede_cbc', sha}) -> +suite(#{key_exchange := ecdhe_rsa, + cipher := '3des_ede_cbc', + mac := sha}) -> ?TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA; -suite({ecdhe_rsa, aes_128_cbc, sha}) -> +suite(#{key_exchange := ecdhe_rsa, + cipher := aes_128_cbc, + mac := sha}) -> ?TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA; -suite({ecdhe_rsa, aes_256_cbc, sha}) -> +suite(#{key_exchange := ecdhe_rsa, + cipher := aes_256_cbc, + mac := sha}) -> ?TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA; - -suite({ecdh_anon, null, sha}) -> +suite(#{key_exchange := ecdh_anon, + cipher := null, + mac := sha}) -> ?TLS_ECDH_anon_WITH_NULL_SHA; -suite({ecdh_anon, rc4_128, sha}) -> +suite(#{key_exchange := ecdh_anon, + cipher := rc4_128, + mac := sha}) -> ?TLS_ECDH_anon_WITH_RC4_128_SHA; -suite({ecdh_anon, '3des_ede_cbc', sha}) -> +suite(#{key_exchange := ecdh_anon, + cipher := '3des_ede_cbc', + mac := sha}) -> ?TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA; -suite({ecdh_anon, aes_128_cbc, sha}) -> +suite(#{key_exchange := ecdh_anon, + cipher := aes_128_cbc, + mac := sha}) -> ?TLS_ECDH_anon_WITH_AES_128_CBC_SHA; -suite({ecdh_anon, aes_256_cbc, sha}) -> +suite(#{key_exchange := ecdh_anon, + cipher := aes_256_cbc, + mac := sha}) -> ?TLS_ECDH_anon_WITH_AES_256_CBC_SHA; - %%% RFC 5289 EC TLS suites -suite({ecdhe_ecdsa, aes_128_cbc, sha256, sha256}) -> +suite(#{key_exchange := ecdhe_ecdsa, + cipher := aes_128_cbc, + mac:= sha256, + prf := sha256}) -> ?TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256; -suite({ecdhe_ecdsa, aes_256_cbc, sha384, sha384}) -> +suite(#{key_exchange := ecdhe_ecdsa, + cipher := aes_256_cbc, + mac := sha384, + prf := sha384}) -> ?TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384; -suite({ecdh_ecdsa, aes_128_cbc, sha256, sha256}) -> +suite(#{key_exchange := ecdh_ecdsa, + cipher := aes_128_cbc, + mac := sha256, + prf := sha256}) -> ?TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256; -suite({ecdh_ecdsa, aes_256_cbc, sha384, sha384}) -> +suite(#{key_exchange := ecdh_ecdsa, + cipher := aes_256_cbc, + mac := sha384, + prf := sha384}) -> ?TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384; -suite({ecdhe_rsa, aes_128_cbc, sha256, sha256}) -> +suite(#{key_exchange := ecdhe_rsa, + cipher := aes_128_cbc, + mac := sha256, + prf := sha256}) -> ?TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256; -suite({ecdhe_rsa, aes_256_cbc, sha384, sha384}) -> +suite(#{key_exchange := ecdhe_rsa, + cipher := aes_256_cbc, + mac := sha384, + prf := sha384}) -> ?TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384; -suite({ecdh_rsa, aes_128_cbc, sha256, sha256}) -> +suite(#{key_exchange := ecdh_rsa, + cipher := aes_128_cbc, + mac := sha256, + prf := sha256}) -> ?TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256; -suite({ecdh_rsa, aes_256_cbc, sha384, sha384}) -> +suite(#{key_exchange := ecdh_rsa, + cipher := aes_256_cbc, + mac := sha384, + prf := sha384}) -> ?TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384; - %% RFC 5288 AES-GCM Cipher Suites -suite({rsa, aes_128_gcm, null, sha256}) -> +suite(#{key_exchange := rsa, + cipher := aes_128_gcm, + mac := null, + prf := sha256}) -> ?TLS_RSA_WITH_AES_128_GCM_SHA256; -suite({rsa, aes_256_gcm, null, sha384}) -> +suite(#{key_exchange := rsa, + cipher := aes_256_gcm, + mac := null, + prf := sha384}) -> ?TLS_RSA_WITH_AES_256_GCM_SHA384; -suite({dhe_rsa, aes_128_gcm, null, sha256}) -> +suite(#{key_exchange := dhe_rsa, + cipher := aes_128_gcm, + mac := null, + prf := sha256}) -> ?TLS_DHE_RSA_WITH_AES_128_GCM_SHA256; -suite({dhe_rsa, aes_256_gcm, null, sha384}) -> +suite(#{key_exchange := dhe_rsa, + cipher := aes_256_gcm, + mac := null, + prf := sha384}) -> ?TLS_DHE_RSA_WITH_AES_256_GCM_SHA384; -suite({dh_rsa, aes_128_gcm, null, sha256}) -> +suite(#{key_exchange := dh_rsa, + cipher := aes_128_gcm, + mac := null, + prf := sha256}) -> ?TLS_DH_RSA_WITH_AES_128_GCM_SHA256; -suite({dh_rsa, aes_256_gcm, null, sha384}) -> +suite(#{key_exchange := dh_rsa, + cipher := aes_256_gcm, + mac := null, + prf := sha384}) -> ?TLS_DH_RSA_WITH_AES_256_GCM_SHA384; -suite({dhe_dss, aes_128_gcm, null, sha256}) -> +suite(#{key_exchange := dhe_dss, + cipher := aes_128_gcm, + mac := null, + prf := sha256}) -> ?TLS_DHE_DSS_WITH_AES_128_GCM_SHA256; -suite({dhe_dss, aes_256_gcm, null, sha384}) -> +suite(#{key_exchange := dhe_dss, + cipher := aes_256_gcm, + mac := null, + prf := sha384}) -> ?TLS_DHE_DSS_WITH_AES_256_GCM_SHA384; -suite({dh_dss, aes_128_gcm, null, sha256}) -> +suite(#{key_exchange := dh_dss, + cipher := aes_128_gcm, + mac := null, + prf := sha256}) -> ?TLS_DH_DSS_WITH_AES_128_GCM_SHA256; -suite({dh_dss, aes_256_gcm, null, sha384}) -> +suite(#{key_exchange := dh_dss, + cipher := aes_256_gcm, + mac := null, + prf := sha384}) -> ?TLS_DH_DSS_WITH_AES_256_GCM_SHA384; -suite({dh_anon, aes_128_gcm, null, sha256}) -> +suite(#{key_exchange := dh_anon, + cipher := aes_128_gcm, + mac := null, + prf := sha256}) -> ?TLS_DH_anon_WITH_AES_128_GCM_SHA256; -suite({dh_anon, aes_256_gcm, null, sha384}) -> +suite(#{key_exchange := dh_anon, + cipher := aes_256_gcm, + mac := null, + prf := sha384}) -> ?TLS_DH_anon_WITH_AES_256_GCM_SHA384; - %% RFC 5289 ECC AES-GCM Cipher Suites -suite({ecdhe_ecdsa, aes_128_gcm, null, sha256}) -> +suite(#{key_exchange := ecdhe_ecdsa, + cipher := aes_128_gcm, + mac := null, + prf := sha256}) -> ?TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256; -suite({ecdhe_ecdsa, aes_256_gcm, null, sha384}) -> +suite(#{key_exchange := ecdhe_ecdsa, + cipher := aes_256_gcm, + mac := null, + prf := sha384}) -> ?TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384; -suite({ecdh_ecdsa, aes_128_gcm, null, sha256}) -> +suite(#{key_exchange := ecdh_ecdsa, + cipher := aes_128_gcm, + mac := null, + prf := sha256}) -> ?TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256; -suite({ecdh_ecdsa, aes_256_gcm, null, sha384}) -> +suite(#{key_exchange := ecdh_ecdsa, + cipher := aes_256_gcm, + mac := null, + prf := sha384}) -> ?TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384; -suite({ecdhe_rsa, aes_128_gcm, null, sha256}) -> +suite(#{key_exchange := ecdhe_rsa, + cipher := aes_128_gcm, + mac := null, + prf := sha256}) -> ?TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256; -suite({ecdhe_rsa, aes_256_gcm, null, sha384}) -> +suite(#{key_exchange := ecdhe_rsa, + cipher := aes_256_gcm, + mac := null, + prf := sha384}) -> ?TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384; -suite({ecdh_rsa, aes_128_gcm, null, sha256}) -> +suite(#{key_exchange := ecdh_rsa, + cipher := aes_128_gcm, + mac := null, + prf := sha256}) -> ?TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256; -suite({ecdh_rsa, aes_256_gcm, null, sha384}) -> +suite(#{key_exchange := ecdh_rsa, + cipher := aes_256_gcm, + mac := null, + prf := sha384}) -> ?TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384; - - %% draft-agl-tls-chacha20poly1305-04 Chacha20/Poly1305 Suites -suite({ecdhe_rsa, chacha20_poly1305, null, sha256}) -> +suite(#{key_exchange := ecdhe_rsa, + cipher := chacha20_poly1305, + mac := null, + prf := sha256}) -> ?TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256; -suite({ecdhe_ecdsa, chacha20_poly1305, null, sha256}) -> +suite(#{key_exchange := ecdhe_ecdsa, + cipher := chacha20_poly1305, + mac := null, + prf := sha256}) -> ?TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256; -suite({dhe_rsa, chacha20_poly1305, null, sha256}) -> +suite(#{key_exchange := dhe_rsa, + cipher := chacha20_poly1305, + mac := null, + prf := sha256}) -> ?TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256. %%-------------------------------------------------------------------- @@ -1516,14 +2185,13 @@ filter(DerCert, Ciphers) -> %% %% Description: Filter suites for algorithms supported by crypto. %%------------------------------------------------------------------- -filter_suites(Suites = [Value|_]) when is_tuple(Value) -> +filter_suites(Suites = [Value|_]) when is_map(Value) -> Algos = crypto:supports(), Hashs = proplists:get_value(hashs, Algos), - lists:filter(fun({KeyExchange, Cipher, Hash}) -> - is_acceptable_keyexchange(KeyExchange, proplists:get_value(public_keys, Algos)) andalso - is_acceptable_cipher(Cipher, proplists:get_value(ciphers, Algos)) andalso - is_acceptable_hash(Hash, proplists:get_value(hashs, Algos)); - ({KeyExchange, Cipher, Hash, Prf}) -> + lists:filter(fun(#{key_exchange := KeyExchange, + cipher := Cipher, + mac := Hash, + prf := Prf}) -> is_acceptable_keyexchange(KeyExchange, proplists:get_value(public_keys, Algos)) andalso is_acceptable_cipher(Cipher, proplists:get_value(ciphers, Algos)) andalso is_acceptable_hash(Hash, Hashs) andalso @@ -1534,9 +2202,12 @@ filter_suites(Suites) -> Algos = crypto:supports(), Hashs = proplists:get_value(hashs, Algos), lists:filter(fun(Suite) -> - {KeyExchange, Cipher, Hash, Prf} = ssl_cipher:suite_definition(Suite), + #{key_exchange := KeyExchange, + cipher := Cipher, + mac := Hash, + prf := Prf} = suite_definition(Suite), is_acceptable_keyexchange(KeyExchange, proplists:get_value(public_keys, Algos)) andalso - is_acceptable_cipher(Cipher, proplists:get_value(ciphers, Algos)) andalso + is_acceptable_cipher(Cipher, proplists:get_value(ciphers, Algos)) andalso is_acceptable_hash(Hash, Hashs) andalso is_acceptable_prf(Prf, Hashs) end, Suites). diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index 9f5f2df3ea..d046145dff 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -495,7 +495,7 @@ handle_session(#server_hello{cipher_suite = CipherSuite, #state{session = #session{session_id = OldId}, negotiated_version = ReqVersion, negotiated_protocol = CurrentProtocol} = State0) -> - {KeyAlgorithm, _, _, _} = + #{key_exchange := KeyAlgorithm} = ssl_cipher:suite_definition(CipherSuite), PremasterSecret = make_premaster_secret(ReqVersion, KeyAlgorithm), @@ -1406,9 +1406,9 @@ connection_info(#state{sni_hostname = SNIHostname, negotiated_version = {_,_} = Version, ssl_options = Opts}) -> RecordCB = record_cb(Connection), - CipherSuiteDef = ssl_cipher:erl_suite_definition(CipherSuite), - IsNamedCurveSuite = lists:member(element(1,CipherSuiteDef), - [ecdh_ecdsa, ecdhe_ecdsa, ecdh_anon]), + CipherSuiteDef = #{key_exchange := KexAlg} = ssl_cipher:suite_definition(CipherSuite), + IsNamedCurveSuite = lists:member(KexAlg, + [ecdh_ecdsa, ecdhe_ecdsa, ecdh_anon]), CurveInfo = case ECCCurve of {namedCurve, Curve} when IsNamedCurveSuite -> [{ecc, {named_curve, pubkey_cert_records:namedCurves(Curve)}}]; @@ -1417,7 +1417,7 @@ connection_info(#state{sni_hostname = SNIHostname, end, [{protocol, RecordCB:protocol_version(Version)}, {session_id, SessionId}, - {cipher_suite, CipherSuiteDef}, + {cipher_suite, ssl_cipher:erl_suite_definition(CipherSuiteDef)}, {sni_hostname, SNIHostname} | CurveInfo] ++ ssl_options_list(Opts). security_info(#state{connection_states = ConnectionStates}) -> @@ -1485,7 +1485,7 @@ resumed_server_hello(#state{session = Session, server_hello(ServerHello, State0, Connection) -> CipherSuite = ServerHello#server_hello.cipher_suite, - {KeyAlgorithm, _, _, _} = ssl_cipher:suite_definition(CipherSuite), + #{key_exchange := KeyAlgorithm} = ssl_cipher:suite_definition(CipherSuite), State = Connection:queue_handshake(ServerHello, State0), State#state{key_algorithm = KeyAlgorithm}. @@ -1499,8 +1499,8 @@ handle_peer_cert(Role, PeerCert, PublicKeyInfo, State1 = State0#state{session = Session#session{peer_certificate = PeerCert}, public_key_info = PublicKeyInfo}, - {KeyAlg,_,_,_} = ssl_cipher:suite_definition(CipherSuite), - State2 = handle_peer_cert_key(Role, PeerCert, PublicKeyInfo, KeyAlg, State1), + #{key_exchange := KeyAlgorithm} = ssl_cipher:suite_definition(CipherSuite), + State2 = handle_peer_cert_key(Role, PeerCert, PublicKeyInfo, KeyAlgorithm, State1), {Record, State} = Connection:next_record(State2), Connection:next_event(certify, Record, State). @@ -1582,24 +1582,24 @@ server_certify_and_key_exchange(State0, Connection) -> certify_client_key_exchange(#encrypted_premaster_secret{premaster_secret= EncPMS}, #state{private_key = Key, client_hello_version = {Major, Minor} = Version} = State, Connection) -> - + FakeSecret = make_premaster_secret(Version, rsa), %% Countermeasure for Bleichenbacher attack always provide some kind of premaster secret %% and fail handshake later.RFC 5246 section 7.4.7.1. PremasterSecret = try ssl_handshake:premaster_secret(EncPMS, Key) of Secret when erlang:byte_size(Secret) == ?NUM_OF_PREMASTERSECRET_BYTES -> case Secret of - <<?BYTE(Major), ?BYTE(Minor), _/binary>> -> %% Correct - Secret; + <<?BYTE(Major), ?BYTE(Minor), Rest/binary>> -> %% Correct + <<?BYTE(Major), ?BYTE(Minor), Rest/binary>>; <<?BYTE(_), ?BYTE(_), Rest/binary>> -> %% Version mismatch <<?BYTE(Major), ?BYTE(Minor), Rest/binary>> end; _ -> %% erlang:byte_size(Secret) =/= ?NUM_OF_PREMASTERSECRET_BYTES - make_premaster_secret(Version, rsa) + FakeSecret catch #alert{description = ?DECRYPT_ERROR} -> - make_premaster_secret(Version, rsa) - end, + FakeSecret + end, calculate_master_secret(PremasterSecret, State, Connection, certify, cipher); certify_client_key_exchange(#client_diffie_hellman_public{dh_public = ClientPublicDhKey}, #state{diffie_hellman_params = #'DHParameter'{} = Params, diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index 24f3a97b9b..0c55af9174 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -67,7 +67,7 @@ %% Cipher suites handling -export([available_suites/2, available_signature_algs/2, available_signature_algs/4, - cipher_suites/2, prf/6, select_session/11, supported_ecc/1, + cipher_suites/3, prf/6, select_session/11, supported_ecc/1, premaster_secret/2, premaster_secret/3, premaster_secret/4]). %% Extensions handling @@ -801,6 +801,11 @@ available_signature_algs(#hash_sign_algos{hash_sign_algos = ClientHashSigns}, Su available_signature_algs(_, _, _, _) -> undefined. +cipher_suites(Suites, Renegotiation, true) -> + %% TLS_FALLBACK_SCSV should be placed last -RFC7507 + cipher_suites(Suites, Renegotiation) ++ [?TLS_FALLBACK_SCSV]; +cipher_suites(Suites, Renegotiation, false) -> + cipher_suites(Suites, Renegotiation). cipher_suites(Suites, false) -> [?TLS_EMPTY_RENEGOTIATION_INFO_SCSV | Suites]; cipher_suites(Suites, true) -> @@ -1060,7 +1065,6 @@ select_hashsign(HashSigns, Cert, KeyExAlgo, select_hashsign(HashSigns, Cert, KeyExAlgo, tls_v1:default_signature_algs(Version), Version); select_hashsign(#hash_sign_algos{hash_sign_algos = HashSigns}, Cert, KeyExAlgo, SupportedHashSigns, {Major, Minor}) when Major >= 3 andalso Minor >= 3 -> - #'OTPCertificate'{tbsCertificate = TBSCert} = public_key:pkix_decode_cert(Cert, otp), #'OTPCertificate'{tbsCertificate = TBSCert, signatureAlgorithm = {_,SignAlgo, _}} = public_key:pkix_decode_cert(Cert, otp), #'OTPSubjectPublicKeyInfo'{algorithm = {_, SubjAlgo, _}} = @@ -1184,19 +1188,22 @@ certificate_types(_, {N, M}) when N >= 3 andalso M >= 3 -> false -> <<?BYTE(?RSA_SIGN), ?BYTE(?DSS_SIGN)>> end; -certificate_types({KeyExchange, _, _, _}, _) when KeyExchange == rsa; - KeyExchange == dh_rsa; - KeyExchange == dhe_rsa; - KeyExchange == ecdhe_rsa -> + +certificate_types(#{key_exchange := KeyExchange}, _) when KeyExchange == rsa; + KeyExchange == dh_rsa; + KeyExchange == dhe_rsa; + KeyExchange == ecdhe_rsa -> <<?BYTE(?RSA_SIGN)>>; -certificate_types({KeyExchange, _, _, _}, _) when KeyExchange == dh_dss; - KeyExchange == dhe_dss; - KeyExchange == srp_dss -> + +certificate_types(#{key_exchange := KeyExchange}, _) when KeyExchange == dh_dss; + KeyExchange == dhe_dss; + KeyExchange == srp_dss -> <<?BYTE(?DSS_SIGN)>>; -certificate_types({KeyExchange, _, _, _}, _) when KeyExchange == dh_ecdsa; - KeyExchange == dhe_ecdsa; - KeyExchange == ecdh_ecdsa; - KeyExchange == ecdhe_ecdsa -> + +certificate_types(#{key_exchange := KeyExchange}, _) when KeyExchange == dh_ecdsa; + KeyExchange == dhe_ecdsa; + KeyExchange == ecdh_ecdsa; + KeyExchange == ecdhe_ecdsa -> <<?BYTE(?ECDSA_SIGN)>>; certificate_types(_, _) -> <<?BYTE(?RSA_SIGN)>>. @@ -2062,23 +2069,23 @@ handle_psk_identity(PSKIdentity, {Fun, UserState}) -> filter_hashsigns([], [], _, Acc) -> lists:reverse(Acc); -filter_hashsigns([Suite | Suites], [{KeyExchange,_,_,_} | Algos], HashSigns, +filter_hashsigns([Suite | Suites], [#{key_exchange := KeyExchange} | Algos], HashSigns, Acc) when KeyExchange == dhe_ecdsa; KeyExchange == ecdhe_ecdsa -> do_filter_hashsigns(ecdsa, Suite, Suites, Algos, HashSigns, Acc); -filter_hashsigns([Suite | Suites], [{KeyExchange,_,_,_} | Algos], HashSigns, +filter_hashsigns([Suite | Suites], [#{key_exchange := KeyExchange} | Algos], HashSigns, Acc) when KeyExchange == rsa; KeyExchange == dhe_rsa; KeyExchange == ecdhe_rsa; KeyExchange == srp_rsa; KeyExchange == rsa_psk -> do_filter_hashsigns(rsa, Suite, Suites, Algos, HashSigns, Acc); -filter_hashsigns([Suite | Suites], [{KeyExchange,_,_,_} | Algos], HashSigns, Acc) when +filter_hashsigns([Suite | Suites], [#{key_exchange := KeyExchange} | Algos], HashSigns, Acc) when KeyExchange == dhe_dss; KeyExchange == srp_dss -> do_filter_hashsigns(dsa, Suite, Suites, Algos, HashSigns, Acc); -filter_hashsigns([Suite | Suites], [{KeyExchange,_,_,_} | Algos], HashSigns, Acc) when +filter_hashsigns([Suite | Suites], [#{key_exchange := KeyExchange} | Algos], HashSigns, Acc) when KeyExchange == dh_dss; KeyExchange == dh_rsa; KeyExchange == dh_ecdsa; @@ -2088,7 +2095,7 @@ filter_hashsigns([Suite | Suites], [{KeyExchange,_,_,_} | Algos], HashSigns, Acc %% algorithm pair appearing in the hash_sign extension. The names %% DH_DSS, DH_RSA, ECDH_ECDSA, and ECDH_RSA are historical. filter_hashsigns(Suites, Algos, HashSigns, [Suite| Acc]); -filter_hashsigns([Suite | Suites], [{KeyExchange,_,_,_} | Algos], HashSigns, Acc) when +filter_hashsigns([Suite | Suites], [#{key_exchange := KeyExchange} | Algos], HashSigns, Acc) when KeyExchange == dh_anon; KeyExchange == ecdh_anon; KeyExchange == srp_anon; @@ -2294,15 +2301,15 @@ handle_ecc_point_fmt_extension(_) -> advertises_ec_ciphers([]) -> false; -advertises_ec_ciphers([{ecdh_ecdsa, _,_,_} | _]) -> +advertises_ec_ciphers([#{key_exchange := ecdh_ecdsa} | _]) -> true; -advertises_ec_ciphers([{ecdhe_ecdsa, _,_,_} | _]) -> +advertises_ec_ciphers([#{key_exchange := ecdhe_ecdsa} | _]) -> true; -advertises_ec_ciphers([{ecdh_rsa, _,_,_} | _]) -> +advertises_ec_ciphers([#{key_exchange := ecdh_rsa} | _]) -> true; -advertises_ec_ciphers([{ecdhe_rsa, _,_,_} | _]) -> +advertises_ec_ciphers([#{key_exchange := ecdhe_rsa} | _]) -> true; -advertises_ec_ciphers([{ecdh_anon, _,_,_} | _]) -> +advertises_ec_ciphers([#{key_exchange := ecdh_anon} | _]) -> true; advertises_ec_ciphers([{ecdhe_psk, _,_,_} | _]) -> true; diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index ddc3d4f61f..406a095d2e 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -141,12 +141,14 @@ next_record(#state{protocol_buffers = end; next_record(#state{protocol_buffers = #protocol_buffers{tls_packets = [], tls_cipher_texts = []}, socket = Socket, + close_tag = CloseTag, transport_cb = Transport} = State) -> case tls_socket:setopts(Transport, Socket, [{active,once}]) of ok -> {no_record, State}; _ -> - {socket_closed, State} + self() ! {CloseTag, Socket}, + {no_record, State} end; next_record(State) -> {no_record, State}. @@ -154,15 +156,10 @@ next_record(State) -> next_event(StateName, Record, State) -> next_event(StateName, Record, State, []). -next_event(StateName, socket_closed, State, _) -> - ssl_connection:handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), StateName, State), - {stop, {shutdown, transport_closed}, State}; next_event(connection = StateName, no_record, State0, Actions) -> case next_record_if_active(State0) of {no_record, State} -> ssl_connection:hibernate_after(StateName, State, Actions); - {socket_closed, State} -> - next_event(StateName, socket_closed, State, Actions); {#ssl_tls{} = Record, State} -> {next_state, StateName, State, [{next_event, internal, {protocol_record, Record}} | Actions]}; {#alert{} = Alert, State} -> diff --git a/lib/ssl/src/tls_handshake.erl b/lib/ssl/src/tls_handshake.erl index a38c5704a6..8817418fb0 100644 --- a/lib/ssl/src/tls_handshake.erl +++ b/lib/ssl/src/tls_handshake.erl @@ -67,14 +67,7 @@ client_hello(Host, Port, ConnectionStates, AvailableCipherSuites, SslOpts, ConnectionStates, Renegotiation), - CipherSuites = - case Fallback of - true -> - [?TLS_FALLBACK_SCSV | - ssl_handshake:cipher_suites(AvailableCipherSuites, Renegotiation)]; - false -> - ssl_handshake:cipher_suites(AvailableCipherSuites, Renegotiation) - end, + CipherSuites = ssl_handshake:cipher_suites(AvailableCipherSuites, Renegotiation, Fallback), Id = ssl_session:client_id({Host, Port, SslOpts}, Cache, CacheCb, OwnCert), #client_hello{session_id = Id, client_version = Version, @@ -203,7 +196,7 @@ handle_client_hello(Version, no_suite -> ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_ciphers); _ -> - {KeyExAlg,_,_,_} = ssl_cipher:suite_definition(CipherSuite), + #{key_exchange := KeyExAlg} = ssl_cipher:suite_definition(CipherSuite), case ssl_handshake:select_hashsign(ClientHashSigns, Cert, KeyExAlg, SupportedHashSigns, Version) of #alert{} = Alert -> diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index 4e7252f469..03c3ed9be3 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -1025,48 +1025,54 @@ string_regex_filter(_Str, _Search) -> false. anonymous_suites(Version) -> - Suites = [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:anonymous_suites(Version)], - ssl_cipher:filter_suites(Suites). + [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:filter_suites(ssl_cipher:anonymous_suites(Version))]. psk_suites(Version) -> - Suites = [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:psk_suites(Version)], - ssl_cipher:filter_suites(Suites). + [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:filter_suites(ssl_cipher:psk_suites(Version))]. psk_anon_suites(Version) -> - Suites = [Suite || Suite <- psk_suites(Version), is_psk_anon_suite(Suite)], - ssl_cipher:filter_suites(Suites). + [Suite || Suite <- psk_suites(Version), is_psk_anon_suite(Suite)]. srp_suites() -> - Suites = - [{srp_anon, '3des_ede_cbc', sha}, - {srp_rsa, '3des_ede_cbc', sha}, - {srp_anon, aes_128_cbc, sha}, - {srp_rsa, aes_128_cbc, sha}, - {srp_anon, aes_256_cbc, sha}, - {srp_rsa, aes_256_cbc, sha}], - ssl_cipher:filter_suites(Suites). - + [ssl_cipher:erl_suite_definition(Suite) || + Suite <- + ssl_cipher:filter_suites([tuple_to_map(S) || + S <- [{srp_anon,'3des_ede_cbc', sha}, + {srp_rsa, '3des_ede_cbc', sha}, + {srp_anon, aes_128_cbc, sha}, + {srp_rsa, aes_128_cbc, sha}, + {srp_anon, aes_256_cbc, sha}, + {srp_rsa, aes_256_cbc, sha}]])]. srp_anon_suites() -> - Suites = - [{srp_anon, '3des_ede_cbc', sha}, - {srp_anon, aes_128_cbc, sha}, - {srp_anon, aes_256_cbc, sha}], - ssl_cipher:filter_suites(Suites). - + [ssl_cipher:erl_suite_definition(Suite) || + Suite <- + ssl_cipher:filter_suites([tuple_to_map(S) || + S <-[{srp_anon, '3des_ede_cbc', sha}, + {srp_anon, aes_128_cbc, sha}, + {srp_anon, aes_256_cbc, sha}]])]. srp_dss_suites() -> - Suites = - [{srp_dss, '3des_ede_cbc', sha}, - {srp_dss, aes_128_cbc, sha}, - {srp_dss, aes_256_cbc, sha}], - ssl_cipher:filter_suites(Suites). - + [ssl_cipher:erl_suite_definition(Suite) || + Suite <- + ssl_cipher:filter_suites([tuple_to_map(S) || + S <- [{srp_dss, '3des_ede_cbc', sha}, + {srp_dss, aes_128_cbc, sha}, + {srp_dss, aes_256_cbc, sha}]])]. rc4_suites(Version) -> - Suites = [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:rc4_suites(Version)], - ssl_cipher:filter_suites(Suites). + [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:filter_suites(ssl_cipher:rc4_suites(Version))]. des_suites(Version) -> - Suites = ssl_cipher:des_suites(Version), - ssl_cipher:filter_suites(Suites). + [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:filter_suites(ssl_cipher:des_suites(Version))]. + +tuple_to_map({Kex, Cipher, Mac}) -> + #{key_exchange => Kex, + cipher => Cipher, + mac => Mac, + prf => default_prf}; +tuple_to_map({Kex, Cipher, Mac, Prf}) -> + #{key_exchange => Kex, + cipher => Cipher, + mac => Mac, + prf => Prf}. pem_to_der(File) -> {ok, PemBin} = file:read_file(File), diff --git a/lib/ssl/vsn.mk b/lib/ssl/vsn.mk index cf6481d14c..2650399eea 100644 --- a/lib/ssl/vsn.mk +++ b/lib/ssl/vsn.mk @@ -1 +1 @@ -SSL_VSN = 8.2.2 +SSL_VSN = 8.2.3 diff --git a/lib/stdlib/doc/src/digraph.xml b/lib/stdlib/doc/src/digraph.xml index 5332d7aba5..db96beed6c 100644 --- a/lib/stdlib/doc/src/digraph.xml +++ b/lib/stdlib/doc/src/digraph.xml @@ -170,6 +170,10 @@ <p>If the edge would create a cycle in an <seealso marker="#acyclic_digraph">acyclic digraph</seealso>, <c>{error, {bad_edge, <anno>Path</anno>}}</c> is returned. + If <c><anno>G</anno></c> already has an edge with value + <c><anno>E</anno></c> connecting a different pair of vertices, + <c>{error, {bad_edge, [<anno>V1</anno>, <anno>V2</anno>]}}</c> + is returned. If either of <c><anno>V1</anno></c> or <c><anno>V2</anno></c> is not a vertex of digraph <c><anno>G</anno></c>, <c>{error, {bad_vertex, </c><anno>V</anno><c>}}</c> is diff --git a/lib/stdlib/doc/src/ets.xml b/lib/stdlib/doc/src/ets.xml index 576959b1c8..a0ec22c515 100644 --- a/lib/stdlib/doc/src/ets.xml +++ b/lib/stdlib/doc/src/ets.xml @@ -1961,7 +1961,7 @@ true</pre> The return value is a list of the new counter values from each update operation in the same order as in the operation list. If an empty list is specified, nothing is updated and an empty list is - returned. If the function fails, no updates is done.</p> + returned. If the function fails, no updates are done.</p> <p>The specified <c><anno>Key</anno></c> is used to identify the object by either <em>matching</em> the key of an object in a <c>set</c> table, or <em>compare equal</em> to the key of an object in an diff --git a/lib/stdlib/doc/src/notes.xml b/lib/stdlib/doc/src/notes.xml index d396f1de8f..b61e5b9b9e 100644 --- a/lib/stdlib/doc/src/notes.xml +++ b/lib/stdlib/doc/src/notes.xml @@ -31,6 +31,49 @@ </header> <p>This document describes the changes made to the STDLIB application.</p> +<section><title>STDLIB 3.4.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Make <c>ets:i/1</c> exit cleaner when ^D is input + while browsing a table. Only the old Erlang shell is + affected (<c>erl(1)</c> flag <c>-oldshell</c>). </p> + <p> + Own Id: OTP-14663</p> + </item> + <item> + <p> + Fixed handling of windows UNC paths in module + <c>filename</c>.</p> + <p> + Own Id: OTP-14693</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Improve performance of the new string functionality when + handling ASCII characters.</p> + <p> + Own Id: OTP-14670</p> + </item> + <item> + <p> + Added a clarification to the documentation of + <c>unicode:characters_to_list/2</c>.</p> + <p> + Own Id: OTP-14798</p> + </item> + </list> + </section> + +</section> + <section><title>STDLIB 3.4.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/stdlib/doc/src/unicode.xml b/lib/stdlib/doc/src/unicode.xml index e86f45431f..d822aca89c 100644 --- a/lib/stdlib/doc/src/unicode.xml +++ b/lib/stdlib/doc/src/unicode.xml @@ -239,8 +239,13 @@ <c><anno>InEncoding</anno></c>.</p> </item> </list> - <p>Only when <c><anno>InEncoding</anno></c> is one of the UTF - encodings, integers in the list are allowed to be > 255.</p> + <p> + Note that integers in the list always represent code points + regardless of <c><anno>InEncoding</anno></c> passed. If + <c><anno>InEncoding</anno> latin1</c> is passed, only code + points < 256 are allowed; otherwise, all valid unicode code + points are allowed. + </p> <p>If <c><anno>InEncoding</anno></c> is <c>latin1</c>, parameter <c><anno>Data</anno></c> corresponds to the <c>iodata()</c> type, but for <c>unicode</c>, parameter <c><anno>Data</anno></c> can diff --git a/lib/stdlib/src/base64.erl b/lib/stdlib/src/base64.erl index c8cf6fdffe..6ea4147abf 100644 --- a/lib/stdlib/src/base64.erl +++ b/lib/stdlib/src/base64.erl @@ -24,22 +24,11 @@ -export([encode/1, decode/1, mime_decode/1, encode_to_string/1, decode_to_string/1, mime_decode_to_string/1]). -%%------------------------------------------------------------------------- %% The following type is a subtype of string() for return values %% of (some) functions of this module. -%%------------------------------------------------------------------------- - -type ascii_string() :: [1..255]. -type ascii_binary() :: binary(). -%%------------------------------------------------------------------------- -%% encode_to_string(ASCII) -> Base64String -%% ASCII - string() | binary() -%% Base64String - string() -%% -%% Description: Encodes a plain ASCII string (or binary) into base64. -%%------------------------------------------------------------------------- - -spec encode_to_string(Data) -> Base64String when Data :: ascii_string() | ascii_binary(), Base64String :: ascii_string(). @@ -47,66 +36,67 @@ encode_to_string(Bin) when is_binary(Bin) -> encode_to_string(binary_to_list(Bin)); encode_to_string(List) when is_list(List) -> - encode_l(List). - -%%------------------------------------------------------------------------- -%% encode(ASCII) -> Base64 -%% ASCII - string() | binary() -%% Base64 - binary() -%% -%% Description: Encodes a plain ASCII string (or binary) into base64. -%%------------------------------------------------------------------------- + encode_list_to_string(List). -spec encode(Data) -> Base64 when Data :: ascii_string() | ascii_binary(), Base64 :: ascii_binary(). encode(Bin) when is_binary(Bin) -> - encode_binary(Bin); + encode_binary(Bin, <<>>); encode(List) when is_list(List) -> - list_to_binary(encode_l(List)). - --spec encode_l(ascii_string()) -> ascii_string(). + encode_list(List, <<>>). -encode_l([]) -> +encode_list_to_string([]) -> []; -encode_l([A]) -> - [b64e(A bsr 2), - b64e((A band 3) bsl 4), $=, $=]; -encode_l([A,B]) -> - [b64e(A bsr 2), - b64e(((A band 3) bsl 4) bor (B bsr 4)), - b64e((B band 15) bsl 2), $=]; -encode_l([A,B,C|Ls]) -> - BB = (A bsl 16) bor (B bsl 8) bor C, +encode_list_to_string([B1]) -> + [b64e(B1 bsr 2), + b64e((B1 band 3) bsl 4), $=, $=]; +encode_list_to_string([B1,B2]) -> + [b64e(B1 bsr 2), + b64e(((B1 band 3) bsl 4) bor (B2 bsr 4)), + b64e((B2 band 15) bsl 2), $=]; +encode_list_to_string([B1,B2,B3|Ls]) -> + BB = (B1 bsl 16) bor (B2 bsl 8) bor B3, [b64e(BB bsr 18), b64e((BB bsr 12) band 63), b64e((BB bsr 6) band 63), - b64e(BB band 63) | encode_l(Ls)]. - -encode_binary(Bin) -> - Split = 3*(byte_size(Bin) div 3), - <<Main0:Split/binary,Rest/binary>> = Bin, - Main = << <<(b64e(C)):8>> || <<C:6>> <= Main0 >>, - case Rest of - <<A:6,B:6,C:4>> -> - <<Main/binary,(b64e(A)):8,(b64e(B)):8,(b64e(C bsl 2)):8,$=:8>>; - <<A:6,B:2>> -> - <<Main/binary,(b64e(A)):8,(b64e(B bsl 4)):8,$=:8,$=:8>>; - <<>> -> - Main - end. + b64e(BB band 63) | encode_list_to_string(Ls)]. -%%------------------------------------------------------------------------- -%% mime_decode(Base64) -> ASCII -%% decode(Base64) -> ASCII -%% Base64 - string() | binary() -%% ASCII - binary() -%% -%% Description: Decodes an base64 encoded string to plain ASCII. -%% mime_decode strips away all characters not Base64 before converting, -%% whereas decode crashes if an illegal character is found -%%------------------------------------------------------------------------- +encode_binary(<<>>, A) -> + A; +encode_binary(<<B1:8>>, A) -> + <<A/bits,(b64e(B1 bsr 2)):8,(b64e((B1 band 3) bsl 4)):8,$=:8,$=:8>>; +encode_binary(<<B1:8, B2:8>>, A) -> + <<A/bits,(b64e(B1 bsr 2)):8, + (b64e(((B1 band 3) bsl 4) bor (B2 bsr 4))):8, + (b64e((B2 band 15) bsl 2)):8, $=:8>>; +encode_binary(<<B1:8, B2:8, B3:8, Ls/bits>>, A) -> + BB = (B1 bsl 16) bor (B2 bsl 8) bor B3, + encode_binary(Ls, + <<A/bits,(b64e(BB bsr 18)):8, + (b64e((BB bsr 12) band 63)):8, + (b64e((BB bsr 6) band 63)):8, + (b64e(BB band 63)):8>>). + +encode_list([], A) -> + A; +encode_list([B1], A) -> + <<A/bits,(b64e(B1 bsr 2)):8,(b64e((B1 band 3) bsl 4)):8,$=:8,$=:8>>; +encode_list([B1,B2], A) -> + <<A/bits,(b64e(B1 bsr 2)):8, + (b64e(((B1 band 3) bsl 4) bor (B2 bsr 4))):8, + (b64e((B2 band 15) bsl 2)):8, $=:8>>; +encode_list([B1,B2,B3|Ls], A) -> + BB = (B1 bsl 16) bor (B2 bsl 8) bor B3, + encode_list(Ls, + <<A/bits,(b64e(BB bsr 18)):8, + (b64e((BB bsr 12) band 63)):8, + (b64e((BB bsr 6) band 63)):8, + (b64e(BB band 63)):8>>). + +%% mime_decode strips away all characters not Base64 before +%% converting, whereas decode crashes if an illegal character is found -spec decode(Base64) -> Data when Base64 :: ascii_string() | ascii_binary(), @@ -122,32 +112,13 @@ decode(List) when is_list(List) -> Data :: ascii_binary(). mime_decode(Bin) when is_binary(Bin) -> - mime_decode_binary(<<>>, Bin); + mime_decode_binary(Bin, <<>>); mime_decode(List) when is_list(List) -> - mime_decode(list_to_binary(List)). + mime_decode_list(List, <<>>). --spec decode_l(ascii_string()) -> ascii_string(). - -decode_l(List) -> - L = strip_spaces(List, []), - decode(L, []). - --spec mime_decode_l(ascii_string()) -> ascii_string(). - -mime_decode_l(List) -> - L = strip_illegal(List, [], 0), - decode(L, []). - -%%------------------------------------------------------------------------- -%% mime_decode_to_string(Base64) -> ASCII -%% decode_to_string(Base64) -> ASCII -%% Base64 - string() | binary() -%% ASCII - binary() -%% -%% Description: Decodes an base64 encoded string to plain ASCII. -%% mime_decode strips away all characters not Base64 before converting, -%% whereas decode crashes if an illegal character is found -%%------------------------------------------------------------------------- +%% mime_decode_to_string strips away all characters not Base64 before +%% converting, whereas decode_to_string crashes if an illegal +%% character is found -spec decode_to_string(Base64) -> DataString when Base64 :: ascii_string() | ascii_binary(), @@ -156,7 +127,7 @@ mime_decode_l(List) -> decode_to_string(Bin) when is_binary(Bin) -> decode_to_string(binary_to_list(Bin)); decode_to_string(List) when is_list(List) -> - decode_l(List). + decode_list_to_string(List). -spec mime_decode_to_string(Base64) -> DataString when Base64 :: ascii_string() | ascii_binary(), @@ -165,115 +136,195 @@ decode_to_string(List) when is_list(List) -> mime_decode_to_string(Bin) when is_binary(Bin) -> mime_decode_to_string(binary_to_list(Bin)); mime_decode_to_string(List) when is_list(List) -> - mime_decode_l(List). - -%% One-based decode map. --define(DECODE_MAP, - {bad,bad,bad,bad,bad,bad,bad,bad,ws,ws,bad,bad,ws,bad,bad, %1-15 - bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, %16-31 - ws,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,62,bad,bad,bad,63, %32-47 - 52,53,54,55,56,57,58,59,60,61,bad,bad,bad,eq,bad,bad, %48-63 - bad,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14, - 15,16,17,18,19,20,21,22,23,24,25,bad,bad,bad,bad,bad, - bad,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40, - 41,42,43,44,45,46,47,48,49,50,51,bad,bad,bad,bad,bad, - bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, - bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, - bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, - bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, - bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, - bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, - bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, - bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad}). + mime_decode_list_to_string(List). -decode_binary(<<C1:8, Cs/bits>>, A) -> - case element(C1, ?DECODE_MAP) of - ws -> decode_binary(Cs, A); - B1 -> decode_binary(Cs, A, B1) +%% Skipping pad character if not at end of string. Also liberal about +%% excess padding and skipping of other illegal (non-base64 alphabet) +%% characters. See section 3.3 of RFC4648 +mime_decode_list([0 | Cs], A) -> + mime_decode_list(Cs, A); +mime_decode_list([C1 | Cs], A) -> + case b64d(C1) of + B1 when is_integer(B1) -> mime_decode_list(Cs, A, B1); + _ -> mime_decode_list(Cs, A) % eq is padding end; -decode_binary(<<>>, A) -> +mime_decode_list([], A) -> A. -decode_binary(<<C2:8, Cs/bits>>, A, B1) -> - case element(C2, ?DECODE_MAP) of - ws -> decode_binary(Cs, A, B1); - B2 -> decode_binary(Cs, A, B1, B2) +mime_decode_list([0 | Cs], A, B1) -> + mime_decode_list(Cs, A, B1); +mime_decode_list([C2 | Cs], A, B1) -> + case b64d(C2) of + B2 when is_integer(B2) -> + mime_decode_list(Cs, A, B1, B2); + _ -> mime_decode_list(Cs, A, B1) % eq is padding end. -decode_binary(<<C3:8, Cs/bits>>, A, B1, B2) -> - case element(C3, ?DECODE_MAP) of - ws -> decode_binary(Cs, A, B1, B2); - B3 -> decode_binary(Cs, A, B1, B2, B3) +mime_decode_list([0 | Cs], A, B1, B2) -> + mime_decode_list(Cs, A, B1, B2); +mime_decode_list([C3 | Cs], A, B1, B2) -> + case b64d(C3) of + B3 when is_integer(B3) -> + mime_decode_list(Cs, A, B1, B2, B3); + eq=B3 -> + mime_decode_list_after_eq(Cs, A, B1, B2, B3); + _ -> mime_decode_list(Cs, A, B1, B2) end. -decode_binary(<<C4:8, Cs/bits>>, A, B1, B2, B3) -> - case element(C4, ?DECODE_MAP) of - ws -> decode_binary(Cs, A, B1, B2, B3); - eq when B3 =:= eq -> only_ws_binary(Cs, <<A/binary,B1:6,(B2 bsr 4):2>>); - eq -> only_ws_binary(Cs, <<A/binary,B1:6,B2:6,(B3 bsr 2):4>>); - B4 -> decode_binary(Cs, <<A/binary,B1:6,B2:6,B3:6,B4:6>>) +mime_decode_list([0 | Cs], A, B1, B2, B3) -> + mime_decode_list(Cs, A, B1, B2, B3); +mime_decode_list([C4 | Cs], A, B1, B2, B3) -> + case b64d(C4) of + B4 when is_integer(B4) -> + mime_decode_list(Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>); + eq -> + mime_decode_list_after_eq(Cs, A, B1, B2, B3); + _ -> mime_decode_list(Cs, A, B1, B2, B3) end. -only_ws_binary(<<>>, A) -> - A; -only_ws_binary(<<C:8, Cs/bits>>, A) -> - case element(C, ?DECODE_MAP) of - ws -> only_ws_binary(Cs, A); - _ -> erlang:error(function_clause) +mime_decode_list_after_eq([0 | Cs], A, B1, B2, B3) -> + mime_decode_list_after_eq(Cs, A, B1, B2, B3); +mime_decode_list_after_eq([C | Cs], A, B1, B2, B3) -> + case b64d(C) of + B when is_integer(B) -> + %% More valid data, skip the eq as invalid + case B3 of + eq -> mime_decode_list(Cs, A, B1, B2, B); + _ -> mime_decode_list(Cs, <<A/bits,B1:6,B2:6,B3:6,B:6>>) + end; + _ -> mime_decode_list_after_eq(Cs, A, B1, B2, B3) + end; +mime_decode_list_after_eq([], A, B1, B2, eq) -> + <<A/bits,B1:6,(B2 bsr 4):2>>; +mime_decode_list_after_eq([], A, B1, B2, B3) -> + <<A/bits,B1:6,B2:6,(B3 bsr 2):4>>. + +mime_decode_binary(<<0:8, Cs/bits>>, A) -> + mime_decode_binary(Cs, A); +mime_decode_binary(<<C1:8, Cs/bits>>, A) -> + case b64d(C1) of + B1 when is_integer(B1) -> mime_decode_binary(Cs, A, B1); + _ -> mime_decode_binary(Cs, A) % eq is padding + end; +mime_decode_binary(<<>>, A) -> + A. + +mime_decode_binary(<<0:8, Cs/bits>>, A, B1) -> + mime_decode_binary(Cs, A, B1); +mime_decode_binary(<<C2:8, Cs/bits>>, A, B1) -> + case b64d(C2) of + B2 when is_integer(B2) -> + mime_decode_binary(Cs, A, B1, B2); + _ -> mime_decode_binary(Cs, A, B1) % eq is padding end. -%% Skipping pad character if not at end of string. Also liberal about -%% excess padding and skipping of other illegal (non-base64 alphabet) -%% characters. See section 3.3 of RFC4648 -mime_decode_binary(Result, <<0:8,T/bits>>) -> - mime_decode_binary(Result, T); -mime_decode_binary(Result0, <<C:8,T/bits>>) -> - case element(C, ?DECODE_MAP) of - Bits when is_integer(Bits) -> - mime_decode_binary(<<Result0/bits,Bits:6>>, T); +mime_decode_binary(<<0:8, Cs/bits>>, A, B1, B2) -> + mime_decode_binary(Cs, A, B1, B2); +mime_decode_binary(<<C3:8, Cs/bits>>, A, B1, B2) -> + case b64d(C3) of + B3 when is_integer(B3) -> + mime_decode_binary(Cs, A, B1, B2, B3); + eq=B3 -> + mime_decode_binary_after_eq(Cs, A, B1, B2, B3); + _ -> mime_decode_binary(Cs, A, B1, B2) + end. + +mime_decode_binary(<<0:8, Cs/bits>>, A, B1, B2, B3) -> + mime_decode_binary(Cs, A, B1, B2, B3); +mime_decode_binary(<<C4:8, Cs/bits>>, A, B1, B2, B3) -> + case b64d(C4) of + B4 when is_integer(B4) -> + mime_decode_binary(Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>); eq -> - mime_decode_binary_after_eq(Result0, T, false); - _ -> - mime_decode_binary(Result0, T) + mime_decode_binary_after_eq(Cs, A, B1, B2, B3); + _ -> mime_decode_binary(Cs, A, B1, B2, B3) + end. + +mime_decode_binary_after_eq(<<0:8, Cs/bits>>, A, B1, B2, B3) -> + mime_decode_binary_after_eq(Cs, A, B1, B2, B3); +mime_decode_binary_after_eq(<<C:8, Cs/bits>>, A, B1, B2, B3) -> + case b64d(C) of + B when is_integer(B) -> + %% More valid data, skip the eq as invalid + case B3 of + eq -> mime_decode_binary(Cs, A, B1, B2, B); + _ -> mime_decode_binary(Cs, <<A/bits,B1:6,B2:6,B3:6,B:6>>) + end; + _ -> mime_decode_binary_after_eq(Cs, A, B1, B2, B3) end; -mime_decode_binary(Result, _) -> - true = is_binary(Result), - Result. - -mime_decode_binary_after_eq(Result, <<0:8,T/bits>>, Eq) -> - mime_decode_binary_after_eq(Result, T, Eq); -mime_decode_binary_after_eq(Result0, <<C:8,T/bits>>, Eq) -> - case element(C, ?DECODE_MAP) of - bad -> - mime_decode_binary_after_eq(Result0, T, Eq); - ws -> - mime_decode_binary_after_eq(Result0, T, Eq); +mime_decode_binary_after_eq(<<>>, A, B1, B2, eq) -> + <<A/bits,B1:6,(B2 bsr 4):2>>; +mime_decode_binary_after_eq(<<>>, A, B1, B2, B3) -> + <<A/bits,B1:6,B2:6,(B3 bsr 2):4>>. + +mime_decode_list_to_string([0 | Cs]) -> + mime_decode_list_to_string(Cs); +mime_decode_list_to_string([C1 | Cs]) -> + case b64d(C1) of + B1 when is_integer(B1) -> mime_decode_list_to_string(Cs, B1); + _ -> mime_decode_list_to_string(Cs) % eq is padding + end; +mime_decode_list_to_string([]) -> + []. + +mime_decode_list_to_string([0 | Cs], B1) -> + mime_decode_list_to_string(Cs, B1); +mime_decode_list_to_string([C2 | Cs], B1) -> + case b64d(C2) of + B2 when is_integer(B2) -> + mime_decode_list_to_string(Cs, B1, B2); + _ -> mime_decode_list_to_string(Cs, B1) % eq is padding + end. + +mime_decode_list_to_string([0 | Cs], B1, B2) -> + mime_decode_list_to_string(Cs, B1, B2); +mime_decode_list_to_string([C3 | Cs], B1, B2) -> + case b64d(C3) of + B3 when is_integer(B3) -> + mime_decode_list_to_string(Cs, B1, B2, B3); + eq=B3 -> mime_decode_list_to_string_after_eq(Cs, B1, B2, B3); + _ -> mime_decode_list_to_string(Cs, B1, B2) + end. + +mime_decode_list_to_string([0 | Cs], B1, B2, B3) -> + mime_decode_list_to_string(Cs, B1, B2, B3); +mime_decode_list_to_string([C4 | Cs], B1, B2, B3) -> + case b64d(C4) of + B4 when is_integer(B4) -> + Bits4x6 = (B1 bsl 18) bor (B2 bsl 12) bor (B3 bsl 6) bor B4, + Octet1 = Bits4x6 bsr 16, + Octet2 = (Bits4x6 bsr 8) band 16#ff, + Octet3 = Bits4x6 band 16#ff, + [Octet1, Octet2, Octet3 | mime_decode_list_to_string(Cs)]; eq -> - mime_decode_binary_after_eq(Result0, T, true); - Bits when is_integer(Bits) -> + mime_decode_list_to_string_after_eq(Cs, B1, B2, B3); + _ -> mime_decode_list_to_string(Cs, B1, B2, B3) + end. + +mime_decode_list_to_string_after_eq([0 | Cs], B1, B2, B3) -> + mime_decode_list_to_string_after_eq(Cs, B1, B2, B3); +mime_decode_list_to_string_after_eq([C | Cs], B1, B2, B3) -> + case b64d(C) of + B when is_integer(B) -> %% More valid data, skip the eq as invalid - mime_decode_binary(<<Result0/bits,Bits:6>>, T) + case B3 of + eq -> mime_decode_list_to_string(Cs, B1, B2, B); + _ -> + Bits4x6 = (B1 bsl 18) bor (B2 bsl 12) bor (B3 bsl 6) bor B, + Octet1 = Bits4x6 bsr 16, + Octet2 = (Bits4x6 bsr 8) band 16#ff, + Octet3 = Bits4x6 band 16#ff, + [Octet1, Octet2, Octet3 | mime_decode_list_to_string(Cs)] + end; + _ -> mime_decode_list_to_string_after_eq(Cs, B1, B2, B3) end; -mime_decode_binary_after_eq(Result0, <<>>, Eq) -> - %% No more valid data. - case bit_size(Result0) rem 8 of - 0 -> - %% '====' is not uncommon. - Result0; - 4 when Eq -> - %% enforce at least one more '=' only ignoring illegals and spacing - Split = byte_size(Result0) - 1, - <<Result:Split/bytes,_:4>> = Result0, - Result; - 2 -> - %% remove 2 bits - Split = byte_size(Result0) - 1, - <<Result:Split/bytes,_:2>> = Result0, - Result - end. +mime_decode_list_to_string_after_eq([], B1, B2, eq) -> + binary_to_list(<<B1:6,(B2 bsr 4):2>>); +mime_decode_list_to_string_after_eq([], B1, B2, B3) -> + binary_to_list(<<B1:6,B2:6,(B3 bsr 2):4>>). decode_list([C1 | Cs], A) -> - case element(C1, ?DECODE_MAP) of + case b64d(C1) of ws -> decode_list(Cs, A); B1 -> decode_list(Cs, A, B1) end; @@ -281,122 +332,130 @@ decode_list([], A) -> A. decode_list([C2 | Cs], A, B1) -> - case element(C2, ?DECODE_MAP) of + case b64d(C2) of ws -> decode_list(Cs, A, B1); B2 -> decode_list(Cs, A, B1, B2) end. decode_list([C3 | Cs], A, B1, B2) -> - case element(C3, ?DECODE_MAP) of + case b64d(C3) of ws -> decode_list(Cs, A, B1, B2); B3 -> decode_list(Cs, A, B1, B2, B3) end. decode_list([C4 | Cs], A, B1, B2, B3) -> - case element(C4, ?DECODE_MAP) of + case b64d(C4) of ws -> decode_list(Cs, A, B1, B2, B3); - eq when B3 =:= eq -> only_ws(Cs, <<A/binary,B1:6,(B2 bsr 4):2>>); - eq -> only_ws(Cs, <<A/binary,B1:6,B2:6,(B3 bsr 2):4>>); - B4 -> decode_list(Cs, <<A/binary,B1:6,B2:6,B3:6,B4:6>>) + eq when B3 =:= eq -> only_ws(Cs, <<A/bits,B1:6,(B2 bsr 4):2>>); + eq -> only_ws(Cs, <<A/bits,B1:6,B2:6,(B3 bsr 2):4>>); + B4 -> decode_list(Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>) end. -only_ws([], A) -> - A; -only_ws([C | Cs], A) -> - case element(C, ?DECODE_MAP) of - ws -> only_ws(Cs, A); - _ -> erlang:error(function_clause) - end. +decode_binary(<<C1:8, Cs/bits>>, A) -> + case b64d(C1) of + ws -> decode_binary(Cs, A); + B1 -> decode_binary(Cs, A, B1) + end; +decode_binary(<<>>, A) -> + A. -decode([], A) -> A; -decode([$=,$=,C2,C1|Cs], A) -> - Bits2x6 = (b64d(C1) bsl 18) bor (b64d(C2) bsl 12), - Octet1 = Bits2x6 bsr 16, - decode(Cs, [Octet1|A]); -decode([$=,C3,C2,C1|Cs], A) -> - Bits3x6 = (b64d(C1) bsl 18) bor (b64d(C2) bsl 12) - bor (b64d(C3) bsl 6), - Octet1 = Bits3x6 bsr 16, - Octet2 = (Bits3x6 bsr 8) band 16#ff, - decode(Cs, [Octet1,Octet2|A]); -decode([C4,C3,C2,C1| Cs], A) -> - Bits4x6 = (b64d(C1) bsl 18) bor (b64d(C2) bsl 12) - bor (b64d(C3) bsl 6) bor b64d(C4), - Octet1 = Bits4x6 bsr 16, - Octet2 = (Bits4x6 bsr 8) band 16#ff, - Octet3 = Bits4x6 band 16#ff, - decode(Cs, [Octet1,Octet2,Octet3|A]). +decode_binary(<<C2:8, Cs/bits>>, A, B1) -> + case b64d(C2) of + ws -> decode_binary(Cs, A, B1); + B2 -> decode_binary(Cs, A, B1, B2) + end. -%%%======================================================================== -%%% Internal functions -%%%======================================================================== +decode_binary(<<C3:8, Cs/bits>>, A, B1, B2) -> + case b64d(C3) of + ws -> decode_binary(Cs, A, B1, B2); + B3 -> decode_binary(Cs, A, B1, B2, B3) + end. -strip_spaces([], A) -> A; -strip_spaces([$\s|Cs], A) -> strip_spaces(Cs, A); -strip_spaces([$\t|Cs], A) -> strip_spaces(Cs, A); -strip_spaces([$\r|Cs], A) -> strip_spaces(Cs, A); -strip_spaces([$\n|Cs], A) -> strip_spaces(Cs, A); -strip_spaces([C|Cs], A) -> strip_spaces(Cs, [C | A]). +decode_binary(<<C4:8, Cs/bits>>, A, B1, B2, B3) -> + case b64d(C4) of + ws -> decode_binary(Cs, A, B1, B2, B3); + eq when B3 =:= eq -> only_ws_binary(Cs, <<A/bits,B1:6,(B2 bsr 4):2>>); + eq -> only_ws_binary(Cs, <<A/bits,B1:6,B2:6,(B3 bsr 2):4>>); + B4 -> decode_binary(Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>) + end. -%% Skipping pad character if not at end of string. Also liberal about -%% excess padding and skipping of other illegal (non-base64 alphabet) -%% characters. See section 3.3 of RFC4648 -strip_illegal([], A, _Cnt) -> +only_ws_binary(<<>>, A) -> A; -strip_illegal([0|Cs], A, Cnt) -> - strip_illegal(Cs, A, Cnt); -strip_illegal([C|Cs], A, Cnt) -> - case element(C, ?DECODE_MAP) of - bad -> - strip_illegal(Cs, A, Cnt); - ws -> - strip_illegal(Cs, A, Cnt); - eq -> - case {tail_contains_more(Cs, false), Cnt rem 4} of - {{[], _}, 0} -> - A; %% Ignore extra = - {{[], true}, 2} -> - [$=|[$=|A]]; %% 'XX==' - {{[], _}, 3} -> - [$=|A]; %% 'XXX=' - {{[H|T], _}, _} -> - %% more data, skip equals - strip_illegal(T, [H|A], Cnt+1) - end; - _ -> - strip_illegal(Cs, [C|A], Cnt+1) +only_ws_binary(<<C:8, Cs/bits>>, A) -> + case b64d(C) of + ws -> only_ws_binary(Cs, A) end. -%% Search the tail for more valid data and remember if we saw -%% another equals along the way. -tail_contains_more([], Eq) -> - {[], Eq}; -tail_contains_more(<<>>, Eq) -> - {<<>>, Eq}; -tail_contains_more([C|T]=More, Eq) -> - case element(C, ?DECODE_MAP) of - bad -> - tail_contains_more(T, Eq); - ws -> - tail_contains_more(T, Eq); - eq -> - tail_contains_more(T, true); - _ -> - {More, Eq} +decode_list_to_string([C1 | Cs]) -> + case b64d(C1) of + ws -> decode_list_to_string(Cs); + B1 -> decode_list_to_string(Cs, B1) end; -tail_contains_more(<<C:8,T/bits>> =More, Eq) -> - case element(C, ?DECODE_MAP) of - bad -> - tail_contains_more(T, Eq); - ws -> - tail_contains_more(T, Eq); - eq -> - tail_contains_more(T, true); - _ -> - {More, Eq} +decode_list_to_string([]) -> + []. + +decode_list_to_string([C2 | Cs], B1) -> + case b64d(C2) of + ws -> decode_list_to_string(Cs, B1); + B2 -> decode_list_to_string(Cs, B1, B2) + end. + +decode_list_to_string([C3 | Cs], B1, B2) -> + case b64d(C3) of + ws -> decode_list_to_string(Cs, B1, B2); + B3 -> decode_list_to_string(Cs, B1, B2, B3) + end. + +decode_list_to_string([C4 | Cs], B1, B2, B3) -> + case b64d(C4) of + ws -> + decode_list_to_string(Cs, B1, B2, B3); + eq when B3 =:= eq -> + only_ws(Cs, binary_to_list(<<B1:6,(B2 bsr 4):2>>)); + eq -> + only_ws(Cs, binary_to_list(<<B1:6,B2:6,(B3 bsr 2):4>>)); + B4 -> + Bits4x6 = (B1 bsl 18) bor (B2 bsl 12) bor (B3 bsl 6) bor B4, + Octet1 = Bits4x6 bsr 16, + Octet2 = (Bits4x6 bsr 8) band 16#ff, + Octet3 = Bits4x6 band 16#ff, + [Octet1, Octet2, Octet3 | decode_list_to_string(Cs)] + end. + +only_ws([], A) -> + A; +only_ws([C | Cs], A) -> + case b64d(C) of + ws -> only_ws(Cs, A) end. - + +%%%======================================================================== +%%% Internal functions +%%%======================================================================== + %% accessors +-compile({inline, [{b64d, 1}]}). +%% One-based decode map. +b64d(X) -> + element(X, + {bad,bad,bad,bad,bad,bad,bad,bad,ws,ws,bad,bad,ws,bad,bad, %1-15 + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, %16-31 + ws,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,62,bad,bad,bad,63, %32-47 + 52,53,54,55,56,57,58,59,60,61,bad,bad,bad,eq,bad,bad, %48-63 + bad,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14, + 15,16,17,18,19,20,21,22,23,24,25,bad,bad,bad,bad,bad, + bad,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40, + 41,42,43,44,45,46,47,48,49,50,51,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad}). + +-compile({inline, [{b64e, 1}]}). b64e(X) -> element(X+1, {$A, $B, $C, $D, $E, $F, $G, $H, $I, $J, $K, $L, $M, $N, @@ -404,9 +463,3 @@ b64e(X) -> $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k, $l, $m, $n, $o, $p, $q, $r, $s, $t, $u, $v, $w, $x, $y, $z, $0, $1, $2, $3, $4, $5, $6, $7, $8, $9, $+, $/}). - - -b64d(X) -> - b64d_ok(element(X, ?DECODE_MAP)). - -b64d_ok(I) when is_integer(I) -> I. diff --git a/lib/stdlib/src/erl_compile.erl b/lib/stdlib/src/erl_compile.erl index 18d7548fdc..f781312ca2 100644 --- a/lib/stdlib/src/erl_compile.erl +++ b/lib/stdlib/src/erl_compile.erl @@ -188,6 +188,8 @@ parse_dep_option("", T) -> {[makedep,{makedep_output,standard_io}],T}; parse_dep_option("D", T) -> {[makedep],T}; +parse_dep_option("MD", T) -> + {[makedep_side_effect],T}; parse_dep_option("F"++Opt, T0) -> {File,T} = get_option("MF", Opt, T0), {[makedep,{makedep_output,File}],T}; @@ -221,6 +223,7 @@ usage() -> "the dependencies"}, {"-MP","add a phony target for each dependency"}, {"-MD","same as -M -MT file (with default 'file')"}, + {"-MMD","generate dependencies as a side-effect"}, {"-o name","name output directory or file"}, {"-pa path","add path to the front of Erlang's code path"}, {"-pz path","add path to the end of Erlang's code path"}, diff --git a/lib/stdlib/src/erl_lint.erl b/lib/stdlib/src/erl_lint.erl index f58cb35cea..1930c462e8 100644 --- a/lib/stdlib/src/erl_lint.erl +++ b/lib/stdlib/src/erl_lint.erl @@ -144,6 +144,7 @@ value_option(Flag, Default, On, OnVal, Off, OffVal, Opts) -> :: dict:dict(ta(), #typeinfo{}), exp_types=gb_sets:empty() %Exported types :: gb_sets:set(ta()), + in_try_head=false :: boolean(), %In a try head. catch_scope = none %Inside/outside try or catch :: catch_scope() }). @@ -312,6 +313,10 @@ format_error({unused_var, V}) -> io_lib:format("variable ~w is unused", [V]); format_error({variable_in_record_def,V}) -> io_lib:format("variable ~w in record definition", [V]); +format_error({stacktrace_guard,V}) -> + io_lib:format("stacktrace variable ~w must not be used in a guard", [V]); +format_error({stacktrace_bound,V}) -> + io_lib:format("stacktrace variable ~w must not be previously bound", [V]); %% --- binaries --- format_error({undefined_bittype,Type}) -> io_lib:format("bit type ~tw undefined", [Type]); @@ -3218,11 +3223,11 @@ is_module_dialyzer_option(Option) -> try_clauses(Scs, Ccs, In, Vt, St0) -> {Csvt0,St1} = icrt_clauses(Scs, Vt, St0), - St2 = St1#lint{catch_scope=try_catch}, + St2 = St1#lint{catch_scope=try_catch,in_try_head=true}, {Csvt1,St3} = icrt_clauses(Ccs, Vt, St2), Csvt = Csvt0 ++ Csvt1, UpdVt = icrt_export(Csvt, Vt, In, St3), - {UpdVt,St3}. + {UpdVt,St3#lint{in_try_head=false}}. %% icrt_clauses(Clauses, In, ImportVarTable, State) -> %% {UpdVt,State}. @@ -3239,12 +3244,29 @@ icrt_clauses(Cs, Vt, St) -> mapfoldl(fun (C, St0) -> icrt_clause(C, Vt, St0) end, St, Cs). icrt_clause({clause,_Line,H,G,B}, Vt0, #lint{catch_scope=Scope}=St0) -> - {Hvt,Binvt,St1} = head(H, Vt0, St0), - Vt1 = vtupdate(Hvt, Binvt), - {Gvt,St2} = guard(G, vtupdate(Vt1, Vt0), St1), - Vt2 = vtupdate(Gvt, Vt1), - {Bvt,St3} = exprs(B, vtupdate(Vt2, Vt0), St2), - {vtupdate(Bvt, Vt2),St3#lint{catch_scope=Scope}}. + Vt1 = taint_stack_var(Vt0, H, St0), + {Hvt,Binvt,St1} = head(H, Vt1, St0), + Vt2 = vtupdate(Hvt, Binvt), + Vt3 = taint_stack_var(Vt2, H, St0), + {Gvt,St2} = guard(G, vtupdate(Vt3, Vt0), St1#lint{in_try_head=false}), + Vt4 = vtupdate(Gvt, Vt2), + {Bvt,St3} = exprs(B, vtupdate(Vt4, Vt0), St2), + {vtupdate(Bvt, Vt4),St3#lint{catch_scope=Scope}}. + +taint_stack_var(Vt, Pat, #lint{in_try_head=true}) -> + [{tuple,_,[_,_,{var,_,Stk}]}] = Pat, + case Stk of + '_' -> + Vt; + _ -> + lists:map(fun({V,{bound,Used,Lines}}) when V =:= Stk -> + {V,{stacktrace,Used,Lines}}; + (B) -> + B + end, Vt) + end; +taint_stack_var(Vt, _Pat, #lint{in_try_head=false}) -> + Vt. icrt_export(Vts, Vt, {Tag,Attrs}, St) -> {_File,Loc} = loc(Attrs, St), @@ -3484,6 +3506,9 @@ pat_var(V, Line, Vt, Bvt, St) -> {[{V,{bound,used,Ls}}],[], %% As this is matching, exported vars are risky. add_warning(Line, {exported_var,V,From}, St)}; + {ok,{stacktrace,_Usage,Ls}} -> + {[{V,{bound,used,Ls}}],[], + add_error(Line, {stacktrace_bound,V}, St)}; error when St#lint.recdef_top -> {[],[{V,{bound,unused,[Line]}}], add_error(Line, {variable_in_record_def,V}, St)}; @@ -3541,6 +3566,9 @@ expr_var(V, Line, Vt, St) -> false -> {[{V,{{export,From},used,Ls}}],St} end; + {ok,{stacktrace,_Usage,Ls}} -> + {[{V,{bound,used,Ls}}], + add_error(Line, {stacktrace_guard,V}, St)}; error -> {[{V,{bound,used,[Line]}}], add_error(Line, {unbound_var,V}, St)} diff --git a/lib/stdlib/src/erl_parse.yrl b/lib/stdlib/src/erl_parse.yrl index 6e72d64acc..14ca24362e 100644 --- a/lib/stdlib/src/erl_parse.yrl +++ b/lib/stdlib/src/erl_parse.yrl @@ -29,6 +29,10 @@ clause_args clause_guard clause_body expr expr_100 expr_150 expr_160 expr_200 expr_300 expr_400 expr_500 expr_600 expr_700 expr_800 expr_max +pat_expr pat_expr_200 pat_expr_300 pat_expr_400 pat_expr_500 +pat_expr_600 pat_expr_700 pat_expr_800 +pat_expr_max map_pat_expr record_pat_expr +pat_argument_list pat_exprs list tail list_comprehension lc_expr lc_exprs binary_comprehension @@ -37,7 +41,7 @@ record_expr record_tuple record_field record_fields map_expr map_tuple map_field map_field_assoc map_field_exact map_fields map_key if_expr if_clause if_clauses case_expr cr_clause cr_clauses receive_expr fun_expr fun_clause fun_clauses atom_or_var integer_or_var -try_expr try_catch try_clause try_clauses +try_expr try_catch try_clause try_clauses try_opt_stacktrace function_call argument_list exprs guard atomic strings @@ -66,7 +70,7 @@ char integer float atom string var 'spec' 'callback' % helper dot. -Expect 2. +Expect 0. Rootsymbol form. @@ -210,7 +214,7 @@ function_clause -> atom clause_args clause_guard clause_body : {clause,?anno('$1'),element(3, '$1'),'$2','$3','$4'}. -clause_args -> argument_list : element(1, '$1'). +clause_args -> pat_argument_list : element(1, '$1'). clause_guard -> 'when' guard : '$2'. clause_guard -> '$empty' : []. @@ -275,6 +279,53 @@ expr_max -> receive_expr : '$1'. expr_max -> fun_expr : '$1'. expr_max -> try_expr : '$1'. +pat_expr -> pat_expr_200 '=' pat_expr : {match,?anno('$2'),'$1','$3'}. +pat_expr -> pat_expr_200 : '$1'. + +pat_expr_200 -> pat_expr_300 comp_op pat_expr_300 : + ?mkop2('$1', '$2', '$3'). +pat_expr_200 -> pat_expr_300 : '$1'. + +pat_expr_300 -> pat_expr_400 list_op pat_expr_300 : + ?mkop2('$1', '$2', '$3'). +pat_expr_300 -> pat_expr_400 : '$1'. + +pat_expr_400 -> pat_expr_400 add_op pat_expr_500 : + ?mkop2('$1', '$2', '$3'). +pat_expr_400 -> pat_expr_500 : '$1'. + +pat_expr_500 -> pat_expr_500 mult_op pat_expr_600 : + ?mkop2('$1', '$2', '$3'). +pat_expr_500 -> pat_expr_600 : '$1'. + +pat_expr_600 -> prefix_op pat_expr_700 : + ?mkop1('$1', '$2'). +pat_expr_600 -> map_pat_expr : '$1'. +pat_expr_600 -> pat_expr_700 : '$1'. + +pat_expr_700 -> record_pat_expr : '$1'. +pat_expr_700 -> pat_expr_800 : '$1'. + +pat_expr_800 -> pat_expr_max : '$1'. + +pat_expr_max -> var : '$1'. +pat_expr_max -> atomic : '$1'. +pat_expr_max -> list : '$1'. +pat_expr_max -> binary : '$1'. +pat_expr_max -> tuple : '$1'. +pat_expr_max -> '(' pat_expr ')' : '$2'. + +map_pat_expr -> '#' map_tuple : + {map, ?anno('$1'),'$2'}. +map_pat_expr -> pat_expr_max '#' map_tuple : + {map, ?anno('$2'),'$1','$3'}. +map_pat_expr -> map_pat_expr '#' map_tuple : + {map, ?anno('$2'),'$1','$3'}. + +record_pat_expr -> '#' atom '.' atom : + {record_index,?anno('$1'),element(3, '$2'),'$4'}. +record_pat_expr -> '#' atom record_tuple : + {record,?anno('$1'),element(3, '$2'),'$3'}. list -> '[' ']' : {nil,?anno('$1')}. list -> '[' expr tail : {cons,?anno('$1'),'$2','$3'}. @@ -397,6 +448,10 @@ case_expr -> 'case' expr 'of' cr_clauses 'end' : cr_clauses -> cr_clause : ['$1']. cr_clauses -> cr_clause ';' cr_clauses : ['$1' | '$3']. +%% FIXME: merl in syntax_tools depends on patterns in a 'case' being +%% full expressions. Therefore, we can't use pat_expr here. There +%% should be a better way. + cr_clause -> expr clause_guard clause_body : {clause,?anno('$1'),['$1'],'$2','$3'}. @@ -424,11 +479,11 @@ integer_or_var -> var : '$1'. fun_clauses -> fun_clause : ['$1']. fun_clauses -> fun_clause ';' fun_clauses : ['$1' | '$3']. -fun_clause -> argument_list clause_guard clause_body : +fun_clause -> pat_argument_list clause_guard clause_body : {Args,Anno} = '$1', {clause,Anno,'fun',Args,'$2','$3'}. -fun_clause -> var argument_list clause_guard clause_body : +fun_clause -> var pat_argument_list clause_guard clause_body : {clause,element(2, '$1'),element(3, '$1'),element(1, '$2'),'$3','$4'}. try_expr -> 'try' exprs 'of' cr_clauses try_catch : @@ -446,24 +501,31 @@ try_catch -> 'after' exprs 'end' : try_clauses -> try_clause : ['$1']. try_clauses -> try_clause ';' try_clauses : ['$1' | '$3']. -try_clause -> expr clause_guard clause_body : +try_clause -> pat_expr clause_guard clause_body : A = ?anno('$1'), {clause,A,[{tuple,A,[{atom,A,throw},'$1',{var,A,'_'}]}],'$2','$3'}. -try_clause -> atom ':' expr clause_guard clause_body : +try_clause -> atom ':' pat_expr try_opt_stacktrace clause_guard clause_body : A = ?anno('$1'), - {clause,A,[{tuple,A,['$1','$3',{var,A,'_'}]}],'$4','$5'}. -try_clause -> var ':' expr clause_guard clause_body : + {clause,A,[{tuple,A,['$1','$3',{var,A,'$4'}]}],'$5','$6'}. +try_clause -> var ':' pat_expr try_opt_stacktrace clause_guard clause_body : A = ?anno('$1'), - {clause,A,[{tuple,A,['$1','$3',{var,A,'_'}]}],'$4','$5'}. + {clause,A,[{tuple,A,['$1','$3',{var,A,'$4'}]}],'$5','$6'}. +try_opt_stacktrace -> ':' var : element(3, '$2'). +try_opt_stacktrace -> '$empty' : '_'. argument_list -> '(' ')' : {[],?anno('$1')}. argument_list -> '(' exprs ')' : {'$2',?anno('$1')}. +pat_argument_list -> '(' ')' : {[],?anno('$1')}. +pat_argument_list -> '(' pat_exprs ')' : {'$2',?anno('$1')}. exprs -> expr : ['$1']. exprs -> expr ',' exprs : ['$1' | '$3']. +pat_exprs -> pat_expr : ['$1']. +pat_exprs -> pat_expr ',' pat_exprs : ['$1' | '$3']. + guard -> exprs : ['$1']. guard -> exprs ';' guard : ['$1'|'$3']. diff --git a/lib/stdlib/src/gen.erl b/lib/stdlib/src/gen.erl index 33af0aed8f..4b1d448487 100644 --- a/lib/stdlib/src/gen.erl +++ b/lib/stdlib/src/gen.erl @@ -49,6 +49,7 @@ | {'logfile', string()}. -type option() :: {'timeout', timeout()} | {'debug', [debug_flag()]} + | {'hibernate_after', timeout()} | {'spawn_opt', [proc_lib:spawn_option()]}. -type options() :: [option()]. diff --git a/lib/stdlib/src/string.erl b/lib/stdlib/src/string.erl index 5a4d2df2a6..e01bb7d85e 100644 --- a/lib/stdlib/src/string.erl +++ b/lib/stdlib/src/string.erl @@ -74,15 +74,16 @@ -export([to_upper/1, to_lower/1]). %% -import(lists,[member/2]). - -compile({no_auto_import,[length/1]}). +-compile({inline, [btoken/2, rev/1, append/2, stack/2, search_compile/1]}). +-define(ASCII_LIST(CP1,CP2), CP1 < 256, CP2 < 256, CP1 =/= $\r). -export_type([grapheme_cluster/0]). -type grapheme_cluster() :: char() | [char()]. -type direction() :: 'leading' | 'trailing'. --dialyzer({no_improper_lists, stack/2}). +-dialyzer({no_improper_lists, [stack/2, length_b/3]}). %%% BIFs internal (not documented) should not to be used outside of this module %%% May be removed -export([list_to_float/1, list_to_integer/1]). @@ -127,8 +128,10 @@ is_empty(_) -> false. %% Count the number of grapheme clusters in chardata -spec length(String::unicode:chardata()) -> non_neg_integer(). +length(<<CP1/utf8, Bin/binary>>) -> + length_b(Bin, CP1, 0); length(CD) -> - length_1(unicode_util:gc(CD), 0). + length_1(CD, 0). %% Convert a string to a list of grapheme clusters -spec to_graphemes(String::unicode:chardata()) -> [grapheme_cluster()]. @@ -176,6 +179,8 @@ equal(A, B, true, Norm) -> %% Reverse grapheme clusters -spec reverse(String::unicode:chardata()) -> [grapheme_cluster()]. +reverse(<<CP1/utf8, Rest/binary>>) -> + reverse_b(Rest, CP1, []); reverse(CD) -> reverse_1(CD, []). @@ -186,7 +191,10 @@ reverse(CD) -> Start :: non_neg_integer(), Slice :: unicode:chardata(). slice(CD, N) when is_integer(N), N >= 0 -> - slice_l(CD, N, is_binary(CD)). + case slice_l0(CD, N) of + [] when is_binary(CD) -> <<>>; + Res -> Res + end. -spec slice(String, Start, Length) -> Slice when String::unicode:chardata(), @@ -195,9 +203,15 @@ slice(CD, N) when is_integer(N), N >= 0 -> Slice :: unicode:chardata(). slice(CD, N, Length) when is_integer(N), N >= 0, is_integer(Length), Length > 0 -> - slice_trail(slice_l(CD, N, is_binary(CD)), Length); + case slice_l0(CD, N) of + [] when is_binary(CD) -> <<>>; + L -> slice_trail(L, Length) + end; slice(CD, N, infinity) -> - slice_l(CD, N, is_binary(CD)); + case slice_l0(CD, N) of + [] when is_binary(CD) -> <<>>; + Res -> Res + end; slice(CD, _, 0) -> case is_binary(CD) of true -> <<>>; @@ -256,18 +270,22 @@ trim(Str, Dir) -> Dir :: direction() | 'both', Characters :: [grapheme_cluster()]. trim(Str, _, []) -> Str; +trim(Str, leading, [Sep]) when is_list(Str), Sep < 256 -> + trim_ls(Str, Sep); trim(Str, leading, Sep) when is_list(Sep) -> - trim_l(Str, search_pattern(Sep)); -trim(Str, trailing, Sep) when is_list(Sep) -> - trim_t(Str, 0, search_pattern(Sep)); -trim(Str, both, Sep0) when is_list(Sep0) -> - Sep = search_pattern(Sep0), - trim_t(trim_l(Str,Sep), 0, Sep). + trim_l(Str, Sep); +trim(Str, trailing, [Sep]) when is_list(Str), Sep < 256 -> + trim_ts(Str, Sep); +trim(Str, trailing, Seps0) when is_list(Seps0) -> + Seps = search_pattern(Seps0), + trim_t(Str, 0, Seps); +trim(Str, both, Sep) when is_list(Sep) -> + trim(trim(Str,leading,Sep), trailing, Sep). %% Delete trailing newlines or \r\n -spec chomp(String::unicode:chardata()) -> unicode:chardata(). chomp(Str) -> - trim_t(Str,0, {[[$\r,$\n],$\n], [$\r,$\n], [<<$\r>>,<<$\n>>]}). + trim(Str, trailing, [[$\r,$\n],$\n]). %% Split String into two parts where the leading part consists of Characters -spec take(String, Characters) -> {Leading, Trailing} when @@ -300,8 +318,7 @@ take(Str, [], Complement, Dir) -> {true, leading} -> {Str, Empty}; {true, trailing} -> {Empty, Str} end; -take(Str, Sep0, false, leading) -> - Sep = search_pattern(Sep0), +take(Str, Sep, false, leading) -> take_l(Str, Sep, []); take(Str, Sep0, true, leading) -> Sep = search_pattern(Sep0), @@ -461,6 +478,7 @@ replace(String, SearchPattern, Replacement, Where) -> SeparatorList::[grapheme_cluster()]) -> [unicode:chardata()]. lexemes([], _) -> []; +lexemes(Str, []) -> [Str]; lexemes(Str, Seps0) when is_list(Seps0) -> Seps = search_pattern(Seps0), lexemes_m(Str, Seps, []). @@ -494,13 +512,13 @@ find(String, SearchPattern, leading) -> find(String, SearchPattern, trailing) -> find_r(String, unicode:characters_to_list(SearchPattern), nomatch). -%% Fetch first codepoint and return rest in tail +%% Fetch first grapheme cluster and return rest in tail -spec next_grapheme(String::unicode:chardata()) -> maybe_improper_list(grapheme_cluster(),unicode:chardata()) | {error,unicode:chardata()}. next_grapheme(CD) -> unicode_util:gc(CD). -%% Fetch first grapheme cluster and return rest in tail +%% Fetch first codepoint and return rest in tail -spec next_codepoint(String::unicode:chardata()) -> maybe_improper_list(char(),unicode:chardata()) | {error,unicode:chardata()}. @@ -508,10 +526,23 @@ next_codepoint(CD) -> unicode_util:cp(CD). %% Internals -length_1([_|Rest], N) -> - length_1(unicode_util:gc(Rest), N+1); -length_1([], N) -> - N. +length_1([CP1|[CP2|_]=Cont], N) when ?ASCII_LIST(CP1,CP2) -> + length_1(Cont, N+1); +length_1(Str, N) -> + case unicode_util:gc(Str) of + [] -> N; + [_|Rest] -> length_1(Rest, N+1) + end. + +length_b(<<CP2/utf8, Rest/binary>>, CP1, N) + when ?ASCII_LIST(CP1,CP2) -> + length_b(Rest, CP2, N+1); +length_b(Bin0, CP1, N) -> + [_|Bin1] = unicode_util:gc([CP1|Bin0]), + case unicode_util:cp(Bin1) of + [] -> N+1; + [CP3|Bin] -> length_b(Bin, CP3, N+1) + end. equal_1([A|AR], [B|BR]) when is_integer(A), is_integer(B) -> A =:= B andalso equal_1(AR, BR); @@ -550,29 +581,66 @@ equal_norm_nocase(A0, B0, Norm) -> {L1,L2} when is_list(L1), is_list(L2) -> false end. +reverse_1([CP1|[CP2|_]=Cont], Acc) when ?ASCII_LIST(CP1,CP2) -> + reverse_1(Cont, [CP1|Acc]); reverse_1(CD, Acc) -> case unicode_util:gc(CD) of [GC|Rest] -> reverse_1(Rest, [GC|Acc]); [] -> Acc end. -slice_l(CD, N, Binary) when N > 0 -> +reverse_b(<<CP2/utf8, Rest/binary>>, CP1, Acc) + when ?ASCII_LIST(CP1,CP2) -> + reverse_b(Rest, CP2, [CP1|Acc]); +reverse_b(Bin0, CP1, Acc) -> + [GC|Bin1] = unicode_util:gc([CP1|Bin0]), + case unicode_util:cp(Bin1) of + [] -> [GC|Acc]; + [CP3|Bin] -> reverse_b(Bin, CP3, [GC|Acc]) + end. + +slice_l0(<<CP1/utf8, Bin/binary>>, N) when N > 0 -> + slice_lb(Bin, CP1, N); +slice_l0(L, N) -> + slice_l(L, N). + +slice_l([CP1|[CP2|_]=Cont], N) when ?ASCII_LIST(CP1,CP2),N > 0 -> + slice_l(Cont, N-1); +slice_l(CD, N) when N > 0 -> case unicode_util:gc(CD) of - [_|Cont] -> slice_l(Cont, N-1, Binary); - [] when Binary -> <<>>; + [_|Cont] -> slice_l(Cont, N-1); [] -> [] end; -slice_l(Cont, 0, Binary) -> - case is_empty(Cont) of - true when Binary -> <<>>; - _ -> Cont +slice_l(Cont, 0) -> + Cont. + +slice_lb(<<CP2/utf8, Bin/binary>>, CP1, N) when ?ASCII_LIST(CP1,CP2), N > 1 -> + slice_lb(Bin, CP2, N-1); +slice_lb(Bin, CP1, N) -> + [_|Rest] = unicode_util:gc([CP1|Bin]), + if N > 1 -> + case unicode_util:cp(Rest) of + [CP2|Cont] -> slice_lb(Cont, CP2, N-1); + [] -> <<>> + end; + N =:= 1 -> + Rest end. +slice_trail(Orig, N) when is_binary(Orig) -> + case Orig of + <<CP1/utf8, Bin/binary>> when N > 0 -> + Length = slice_bin(Bin, CP1, N), + Sz = byte_size(Orig) - Length, + <<Keep:Sz/binary, _/binary>> = Orig, + Keep; + _ -> <<>> + end; slice_trail(CD, N) when is_list(CD) -> - slice_list(CD, N); -slice_trail(CD, N) when is_binary(CD) -> - slice_bin(CD, N, CD). + slice_list(CD, N). +slice_list([CP1|[CP2|_]=Cont], N) when ?ASCII_LIST(CP1,CP2),N > 0 -> + [CP1|slice_list(Cont, N-1)]; slice_list(CD, N) when N > 0 -> case unicode_util:gc(CD) of [GC|Cont] -> append(GC, slice_list(Cont, N-1)); @@ -581,17 +649,16 @@ slice_list(CD, N) when N > 0 -> slice_list(_, 0) -> []. -slice_bin(CD, N, Orig) when N > 0 -> - case unicode_util:gc(CD) of - [_|Cont] -> slice_bin(Cont, N-1, Orig); - [] -> Orig +slice_bin(<<CP2/utf8, Bin/binary>>, CP1, N) when ?ASCII_LIST(CP1,CP2), N > 0 -> + slice_bin(Bin, CP2, N-1); +slice_bin(CD, CP1, N) when N > 0 -> + [_|Bin] = unicode_util:gc([CP1|CD]), + case unicode_util:cp(Bin) of + [CP2|Cont] -> slice_bin(Cont, CP2, N-1); + [] -> 0 end; -slice_bin([], 0, Orig) -> - Orig; -slice_bin(CD, 0, Orig) -> - Sz = byte_size(Orig) - byte_size(CD), - <<Keep:Sz/binary, _/binary>> = Orig, - Keep. +slice_bin(CD, CP1, 0) -> + byte_size(CD)+byte_size(<<CP1/utf8>>). uppercase_list(CPs0) -> case unicode_util:uppercase(CPs0) of @@ -641,16 +708,31 @@ casefold_bin(CPs0, Acc) -> [] -> Acc end. - +%% Fast path for ascii searching for one character in lists +trim_ls([CP1|[CP2|_]=Cont]=Str, Sep) + when ?ASCII_LIST(CP1,CP2) -> + case Sep of + CP1 -> trim_ls(Cont, Sep); + _ -> Str + end; +trim_ls(Str, Sep) -> + trim_l(Str, [Sep]). + +trim_l([CP1|[CP2|_]=Cont]=Str, Sep) + when ?ASCII_LIST(CP1,CP2) -> + case lists:member(CP1, Sep) of + true -> trim_l(Cont, Sep); + false -> Str + end; trim_l([Bin|Cont0], Sep) when is_binary(Bin) -> case bin_search_inv(Bin, Cont0, Sep) of {nomatch, Cont} -> trim_l(Cont, Sep); Keep -> Keep end; -trim_l(Str, {GCs, _, _}=Sep) when is_list(Str) -> +trim_l(Str, Sep) when is_list(Str) -> case unicode_util:gc(Str) of [C|Cs] -> - case lists:member(C, GCs) of + case lists:member(C, Sep) of true -> trim_l(Cs, Sep); false -> Str end; @@ -662,15 +744,51 @@ trim_l(Bin, Sep) when is_binary(Bin) -> [Keep] -> Keep end. -trim_t([Bin|Cont0], N, Sep) when is_binary(Bin) -> +%% Fast path for ascii searching for one character in lists +trim_ts([Sep|Cs1]=Str, Sep) -> + case Cs1 of + [] -> []; + [CP2|_] when ?ASCII_LIST(Sep,CP2) -> + Tail = trim_ts(Cs1, Sep), + case is_empty(Tail) of + true -> []; + false -> [Sep|Tail] + end; + _ -> + trim_t(Str, 0, search_pattern([Sep])) + end; +trim_ts([CP|Cont],Sep) when is_integer(CP) -> + [CP|trim_ts(Cont, Sep)]; +trim_ts(Str, Sep) -> + trim_t(Str, 0, search_pattern([Sep])). + +trim_t([CP1|Cont]=Cs0, _, {GCs,CPs,_}=Seps) when is_integer(CP1) -> + case lists:member(CP1, CPs) of + true -> + [GC|Cs1] = unicode_util:gc(Cs0), + case lists:member(GC, GCs) of + true -> + Tail = trim_t(Cs1, 0, Seps), + case is_empty(Tail) of + true -> []; + false -> append(GC,Tail) + end; + false -> + append(GC,trim_t(Cs1, 0, Seps)) + end; + false -> + [CP1|trim_t(Cont, 0, Seps)] + end; +trim_t([Bin|Cont0], N, {GCs,_,_}=Seps0) when is_binary(Bin) -> <<_:N/binary, Rest/binary>> = Bin, - case bin_search(Rest, Cont0, Sep) of + Seps = search_compile(Seps0), + case bin_search(Rest, Cont0, Seps) of {nomatch,_} -> - stack(Bin, trim_t(Cont0, 0, Sep)); + stack(Bin, trim_t(Cont0, 0, Seps)); [SepStart|Cont1] -> - case bin_search_inv(SepStart, Cont1, Sep) of + case bin_search_inv(SepStart, Cont1, GCs) of {nomatch, Cont} -> - Tail = trim_t(Cont, 0, Sep), + Tail = trim_t(Cont, 0, Seps), case is_empty(Tail) of true -> KeepSz = byte_size(Bin) - byte_size(SepStart), @@ -682,67 +800,69 @@ trim_t([Bin|Cont0], N, Sep) when is_binary(Bin) -> end; [NonSep|Cont] when is_binary(NonSep) -> KeepSz = byte_size(Bin) - byte_size(NonSep), - trim_t([Bin|Cont], KeepSz, Sep) + trim_t([Bin|Cont], KeepSz, Seps) end end; -trim_t(Str, 0, {GCs,CPs,_}=Sep) when is_list(Str) -> - case unicode_util:cp(Str) of - [CP|Cs] -> - case lists:member(CP, CPs) of +trim_t(Str, 0, {GCs,_,_}=Seps) when is_list(Str) -> + case unicode_util:gc(Str) of + [GC|Cs1] -> + case lists:member(GC, GCs) of true -> - [GC|Cs1] = unicode_util:gc(Str), - case lists:member(GC, GCs) of - true -> - Tail = trim_t(Cs1, 0, Sep), - case is_empty(Tail) of - true -> []; - false -> append(GC,Tail) - end; - false -> - append(GC,trim_t(Cs1, 0, Sep)) + Tail = trim_t(Cs1, 0, Seps), + case is_empty(Tail) of + true -> []; + false -> append(GC,Tail) end; false -> - append(CP,trim_t(Cs, 0, Sep)) + append(GC,trim_t(Cs1, 0, Seps)) end; [] -> [] end; -trim_t(Bin, N, Sep) when is_binary(Bin) -> +trim_t(Bin, N, {GCs,_,_}=Seps0) when is_binary(Bin) -> <<_:N/binary, Rest/binary>> = Bin, - case bin_search(Rest, Sep) of + Seps = search_compile(Seps0), + case bin_search(Rest, [], Seps) of {nomatch,_} -> Bin; [SepStart] -> - case bin_search_inv(SepStart, [], Sep) of + case bin_search_inv(SepStart, [], GCs) of {nomatch,_} -> KeepSz = byte_size(Bin) - byte_size(SepStart), <<Keep:KeepSz/binary, _/binary>> = Bin, Keep; [NonSep] -> KeepSz = byte_size(Bin) - byte_size(NonSep), - trim_t(Bin, KeepSz, Sep) + trim_t(Bin, KeepSz, Seps) end end. -take_l([Bin|Cont0], Sep, Acc) when is_binary(Bin) -> - case bin_search_inv(Bin, Cont0, Sep) of + +take_l([CP1|[CP2|_]=Cont]=Str, Seps, Acc) + when ?ASCII_LIST(CP1,CP2) -> + case lists:member(CP1, Seps) of + true -> take_l(Cont, Seps, [CP1|Acc]); + false -> {rev(Acc), Str} + end; +take_l([Bin|Cont0], Seps, Acc) when is_binary(Bin) -> + case bin_search_inv(Bin, Cont0, Seps) of {nomatch, Cont} -> Used = cp_prefix(Cont0, Cont), - take_l(Cont, Sep, [unicode:characters_to_binary([Bin|Used])|Acc]); + take_l(Cont, Seps, [unicode:characters_to_binary([Bin|Used])|Acc]); [Bin1|_]=After when is_binary(Bin1) -> First = byte_size(Bin) - byte_size(Bin1), <<Keep:First/binary, _/binary>> = Bin, {btoken(Keep,Acc), After} end; -take_l(Str, {GCs, _, _}=Sep, Acc) when is_list(Str) -> +take_l(Str, Seps, Acc) when is_list(Str) -> case unicode_util:gc(Str) of [C|Cs] -> - case lists:member(C, GCs) of - true -> take_l(Cs, Sep, append(rev(C),Acc)); + case lists:member(C, Seps) of + true -> take_l(Cs, Seps, append(rev(C),Acc)); false -> {rev(Acc), Str} end; [] -> {rev(Acc), []} end; -take_l(Bin, Sep, Acc) when is_binary(Bin) -> - case bin_search_inv(Bin, [], Sep) of +take_l(Bin, Seps, Acc) when is_binary(Bin) -> + case bin_search_inv(Bin, [], Seps) of {nomatch,_} -> {btoken(Bin, Acc), <<>>}; [After] -> @@ -751,27 +871,41 @@ take_l(Bin, Sep, Acc) when is_binary(Bin) -> {btoken(Keep, Acc), After} end. -take_lc([Bin|Cont0], Sep, Acc) when is_binary(Bin) -> - case bin_search(Bin, Cont0, Sep) of + +take_lc([CP1|Cont]=Str0, {GCs,CPs,_}=Seps, Acc) when is_integer(CP1) -> + case lists:member(CP1, CPs) of + true -> + [GC|Str] = unicode_util:gc(Str0), + case lists:member(GC, GCs) of + false -> take_lc(Str, Seps, append(rev(GC),Acc)); + true -> {rev(Acc), Str0} + end; + false -> + take_lc(Cont, Seps, append(CP1,Acc)) + end; +take_lc([Bin|Cont0], Seps0, Acc) when is_binary(Bin) -> + Seps = search_compile(Seps0), + case bin_search(Bin, Cont0, Seps) of {nomatch, Cont} -> Used = cp_prefix(Cont0, Cont), - take_lc(Cont, Sep, [unicode:characters_to_binary([Bin|Used])|Acc]); + take_lc(Cont, Seps, [unicode:characters_to_binary([Bin|Used])|Acc]); [Bin1|_]=After when is_binary(Bin1) -> First = byte_size(Bin) - byte_size(Bin1), <<Keep:First/binary, _/binary>> = Bin, {btoken(Keep,Acc), After} end; -take_lc(Str, {GCs, _, _}=Sep, Acc) when is_list(Str) -> +take_lc(Str, {GCs,_,_}=Seps, Acc) when is_list(Str) -> case unicode_util:gc(Str) of [C|Cs] -> case lists:member(C, GCs) of - false -> take_lc(Cs, Sep, append(rev(C),Acc)); + false -> take_lc(Cs, Seps, append(rev(C),Acc)); true -> {rev(Acc), Str} end; [] -> {rev(Acc), []} end; -take_lc(Bin, Sep, Acc) when is_binary(Bin) -> - case bin_search(Bin, [], Sep) of +take_lc(Bin, Seps0, Acc) when is_binary(Bin) -> + Seps = search_compile(Seps0), + case bin_search(Bin, [], Seps) of {nomatch,_} -> {btoken(Bin, Acc), <<>>}; [After] -> @@ -780,148 +914,192 @@ take_lc(Bin, Sep, Acc) when is_binary(Bin) -> {btoken(Keep, Acc), After} end. -take_t([Bin|Cont0], N, Sep) when is_binary(Bin) -> + +take_t([CP1|Cont]=Str0, _, {GCs,CPs,_}=Seps) when is_integer(CP1) -> + case lists:member(CP1, CPs) of + true -> + [GC|Str] = unicode_util:gc(Str0), + case lists:member(GC, GCs) of + true -> + {Head, Tail} = take_t(Str, 0, Seps), + case is_empty(Head) of + true -> {Head, append(GC,Tail)}; + false -> {append(GC,Head), Tail} + end; + false -> + {Head, Tail} = take_t(Str, 0, Seps), + {append(GC,Head), Tail} + end; + false -> + {Head, Tail} = take_t(Cont, 0, Seps), + {[CP1|Head], Tail} + end; +take_t([Bin|Cont0], N, {GCs,_,_}=Seps0) when is_binary(Bin) -> <<_:N/binary, Rest/binary>> = Bin, - case bin_search(Rest, Cont0, Sep) of + Seps = search_compile(Seps0), + case bin_search(Rest, Cont0, Seps) of {nomatch,Cont} -> Used = cp_prefix(Cont0, Cont), - {Head, Tail} = take_t(Cont, 0, Sep), + {Head, Tail} = take_t(Cont, 0, Seps), {stack(unicode:characters_to_binary([Bin|Used]), Head), Tail}; [SepStart|Cont1] -> - case bin_search_inv(SepStart, Cont1, Sep) of + case bin_search_inv(SepStart, Cont1, GCs) of {nomatch, Cont} -> - {Head, Tail} = take_t(Cont, 0, Sep), + {Head, Tail} = take_t(Cont, 0, Seps), Used = cp_prefix(Cont0, Cont), - case equal(Tail, Cont) of + case is_empty(Head) of true -> KeepSz = byte_size(Bin) - byte_size(SepStart), <<Keep:KeepSz/binary, End/binary>> = Bin, - {stack(Keep,Head), stack(stack(End,Used),Tail)}; + {Keep, stack(stack(End,Used),Tail)}; false -> {stack(unicode:characters_to_binary([Bin|Used]),Head), Tail} end; [NonSep|Cont] when is_binary(NonSep) -> KeepSz = byte_size(Bin) - byte_size(NonSep), - take_t([Bin|Cont], KeepSz, Sep) + take_t([Bin|Cont], KeepSz, Seps) end end; -take_t(Str, 0, {GCs,CPs,_}=Sep) when is_list(Str) -> - case unicode_util:cp(Str) of - [CP|Cs] -> - case lists:member(CP, CPs) of +take_t(Str, 0, {GCs,_,_}=Seps) when is_list(Str) -> + case unicode_util:gc(Str) of + [GC|Cs1] -> + case lists:member(GC, GCs) of true -> - [GC|Cs1] = unicode_util:gc(Str), - case lists:member(GC, GCs) of - true -> - {Head, Tail} = take_t(Cs1, 0, Sep), - case equal(Tail, Cs1) of - true -> {Head, append(GC,Tail)}; - false -> {append(GC,Head), Tail} - end; - false -> - {Head, Tail} = take_t(Cs, 0, Sep), - {append(CP,Head), Tail} + {Head, Tail} = take_t(Cs1, 0, Seps), + case is_empty(Head) of + true -> {Head, append(GC,Tail)}; + false -> {append(GC,Head), Tail} end; false -> - {Head, Tail} = take_t(Cs, 0, Sep), - {append(CP,Head), Tail} + {Head, Tail} = take_t(Cs1, 0, Seps), + {append(GC,Head), Tail} end; [] -> {[],[]} end; -take_t(Bin, N, Sep) when is_binary(Bin) -> +take_t(Bin, N, {GCs,_,_}=Seps0) when is_binary(Bin) -> <<_:N/binary, Rest/binary>> = Bin, - case bin_search(Rest, Sep) of + Seps = search_compile(Seps0), + case bin_search(Rest, [], Seps) of {nomatch,_} -> {Bin, <<>>}; [SepStart] -> - case bin_search_inv(SepStart, [], Sep) of + case bin_search_inv(SepStart, [], GCs) of {nomatch,_} -> KeepSz = byte_size(Bin) - byte_size(SepStart), <<Before:KeepSz/binary, End/binary>> = Bin, {Before, End}; [NonSep] -> KeepSz = byte_size(Bin) - byte_size(NonSep), - take_t(Bin, KeepSz, Sep) + take_t(Bin, KeepSz, Seps) end end. -take_tc([Bin|Cont0], N, Sep) when is_binary(Bin) -> +take_tc([CP1|[CP2|_]=Cont], _, {GCs,_,_}=Seps) when ?ASCII_LIST(CP1,CP2) -> + case lists:member(CP1, GCs) of + false -> + {Head, Tail} = take_tc(Cont, 0, Seps), + case is_empty(Head) of + true -> {Head, append(CP1,Tail)}; + false -> {append(CP1,Head), Tail} + end; + true -> + {Head, Tail} = take_tc(Cont, 0, Seps), + {append(CP1,Head), Tail} + end; +take_tc([Bin|Cont0], N, {GCs,_,_}=Seps0) when is_binary(Bin) -> <<_:N/binary, Rest/binary>> = Bin, - case bin_search_inv(Rest, Cont0, Sep) of + case bin_search_inv(Rest, Cont0, GCs) of {nomatch,Cont} -> Used = cp_prefix(Cont0, Cont), - {Head, Tail} = take_tc(Cont, 0, Sep), + {Head, Tail} = take_tc(Cont, 0, Seps0), {stack(unicode:characters_to_binary([Bin|Used]), Head), Tail}; [SepStart|Cont1] -> - case bin_search(SepStart, Cont1, Sep) of + Seps = search_compile(Seps0), + case bin_search(SepStart, Cont1, Seps) of {nomatch, Cont} -> - {Head, Tail} = take_tc(Cont, 0, Sep), + {Head, Tail} = take_tc(Cont, 0, Seps), Used = cp_prefix(Cont0, Cont), - case equal(Tail, Cont) of + case is_empty(Head) of true -> KeepSz = byte_size(Bin) - byte_size(SepStart), <<Keep:KeepSz/binary, End/binary>> = Bin, - {stack(Keep,Head), stack(stack(End,Used),Tail)}; + {Keep, stack(stack(End,Used),Tail)}; false -> {stack(unicode:characters_to_binary([Bin|Used]),Head), Tail} end; [NonSep|Cont] when is_binary(NonSep) -> KeepSz = byte_size(Bin) - byte_size(NonSep), - take_tc([Bin|Cont], KeepSz, Sep) + take_tc([Bin|Cont], KeepSz, Seps) end end; -take_tc(Str, 0, {GCs,CPs,_}=Sep) when is_list(Str) -> - case unicode_util:cp(Str) of - [CP|Cs] -> - case lists:member(CP, CPs) of - true -> - [GC|Cs1] = unicode_util:gc(Str), - case lists:member(GC, GCs) of - false -> - {Head, Tail} = take_tc(Cs1, 0, Sep), - case equal(Tail, Cs1) of - true -> {Head, append(GC,Tail)}; - false -> {append(GC,Head), Tail} - end; - true -> - {Head, Tail} = take_tc(Cs1, 0, Sep), - {append(GC,Head), Tail} - end; +take_tc(Str, 0, {GCs,_,_}=Seps) when is_list(Str) -> + case unicode_util:gc(Str) of + [GC|Cs1] -> + case lists:member(GC, GCs) of false -> - {Head, Tail} = take_tc(Cs, 0, Sep), - case equal(Tail, Cs) of - true -> {Head, append(CP,Tail)}; - false -> {append(CP,Head), Tail} - end + {Head, Tail} = take_tc(Cs1, 0, Seps), + case is_empty(Head) of + true -> {Head, append(GC,Tail)}; + false -> {append(GC,Head), Tail} + end; + true -> + {Head, Tail} = take_tc(Cs1, 0, Seps), + {append(GC,Head), Tail} end; [] -> {[],[]} end; -take_tc(Bin, N, Sep) when is_binary(Bin) -> +take_tc(Bin, N, {GCs,_,_}=Seps0) when is_binary(Bin) -> <<_:N/binary, Rest/binary>> = Bin, - case bin_search_inv(Rest, [], Sep) of + case bin_search_inv(Rest, [], GCs) of {nomatch,_} -> {Bin, <<>>}; [SepStart] -> - case bin_search(SepStart, [], Sep) of + Seps = search_compile(Seps0), + case bin_search(SepStart, [], Seps) of {nomatch,_} -> KeepSz = byte_size(Bin) - byte_size(SepStart), <<Before:KeepSz/binary, End/binary>> = Bin, {Before, End}; [NonSep] -> KeepSz = byte_size(Bin) - byte_size(NonSep), - take_tc(Bin, KeepSz, Sep) + take_tc(Bin, KeepSz, Seps) end end. -prefix_1(Cs, []) -> Cs; -prefix_1(Cs, [_]=Pre) -> - prefix_2(unicode_util:gc(Cs), Pre); -prefix_1(Cs, Pre) -> - prefix_2(unicode_util:cp(Cs), Pre). - -prefix_2([C|Cs], [C|Pre]) -> - prefix_1(Cs, Pre); -prefix_2(_, _) -> - nomatch. +prefix_1(Cs0, [GC]) -> + case unicode_util:gc(Cs0) of + [GC|Cs] -> Cs; + _ -> nomatch + end; +prefix_1([CP|Cs], [Pre|PreR]) when is_integer(CP) -> + case CP =:= Pre of + true -> prefix_1(Cs,PreR); + false -> nomatch + end; +prefix_1(<<CP/utf8, Cs/binary>>, [Pre|PreR]) -> + case CP =:= Pre of + true -> prefix_1(Cs,PreR); + false -> nomatch + end; +prefix_1(Cs0, [Pre|PreR]) -> + case unicode_util:cp(Cs0) of + [Pre|Cs] -> prefix_1(Cs,PreR); + _ -> nomatch + end. +split_1([CP1|Cs]=Cs0, [C|_]=Needle, _, Where, Curr, Acc) when is_integer(CP1) -> + case CP1=:=C of + true -> + case prefix_1(Cs0, Needle) of + nomatch -> split_1(Cs, Needle, 0, Where, append(C,Curr), Acc); + Rest when Where =:= leading -> + [rev(Curr), Rest]; + Rest when Where =:= trailing -> + split_1(Cs, Needle, 0, Where, [C|Curr], [rev(Curr), Rest]); + Rest when Where =:= all -> + split_1(Rest, Needle, 0, Where, [], [rev(Curr)|Acc]) + end; + false -> + split_1(Cs, Needle, 0, Where, append(CP1,Curr), Acc) + end; split_1([Bin|Cont0], Needle, Start, Where, Curr0, Acc) when is_binary(Bin) -> case bin_search_str(Bin, Start, Cont0, Needle) of @@ -981,32 +1159,50 @@ split_1(Bin, [_C|_]=Needle, Start, Where, Curr0, Acc) -> end end. -lexemes_m([Bin|Cont0], Seps, Ts) when is_binary(Bin) -> - case bin_search_inv(Bin, Cont0, Seps) of +lexemes_m([CP|_]=Cs0, {GCs,CPs,_}=Seps, Ts) when is_integer(CP) -> + case lists:member(CP, CPs) of + true -> + [GC|Cs2] = unicode_util:gc(Cs0), + case lists:member(GC, GCs) of + true -> + lexemes_m(Cs2, Seps, Ts); + false -> + {Lexeme,Rest} = lexeme_pick(Cs0, Seps, []), + lexemes_m(Rest, Seps, [Lexeme|Ts]) + end; + false -> + {Lexeme,Rest} = lexeme_pick(Cs0, Seps, []), + lexemes_m(Rest, Seps, [Lexeme|Ts]) + end; +lexemes_m([Bin|Cont0], {GCs,_,_}=Seps0, Ts) when is_binary(Bin) -> + case bin_search_inv(Bin, Cont0, GCs) of {nomatch,Cont} -> - lexemes_m(Cont, Seps, Ts); + lexemes_m(Cont, Seps0, Ts); Cs -> + Seps = search_compile(Seps0), {Lexeme,Rest} = lexeme_pick(Cs, Seps, []), lexemes_m(Rest, Seps, [Lexeme|Ts]) end; -lexemes_m(Cs0, {GCs, _, _}=Seps, Ts) when is_list(Cs0) -> +lexemes_m(Cs0, {GCs, _, _}=Seps0, Ts) when is_list(Cs0) -> case unicode_util:gc(Cs0) of [C|Cs] -> case lists:member(C, GCs) of true -> - lexemes_m(Cs, Seps, Ts); + lexemes_m(Cs, Seps0, Ts); false -> + Seps = search_compile(Seps0), {Lexeme,Rest} = lexeme_pick(Cs0, Seps, []), lexemes_m(Rest, Seps, [Lexeme|Ts]) end; [] -> lists:reverse(Ts) end; -lexemes_m(Bin, Seps, Ts) when is_binary(Bin) -> - case bin_search_inv(Bin, [], Seps) of +lexemes_m(Bin, {GCs,_,_}=Seps0, Ts) when is_binary(Bin) -> + case bin_search_inv(Bin, [], GCs) of {nomatch,_} -> lists:reverse(Ts); [Cs] -> + Seps = search_compile(Seps0), {Lexeme,Rest} = lexeme_pick(Cs, Seps, []), lexemes_m(Rest, Seps, add_non_empty(Lexeme,Ts)) end. @@ -1037,7 +1233,7 @@ lexeme_pick(Cs0, {GCs, CPs, _} = Seps, Tkn) when is_list(Cs0) -> true -> [GC|Cs2] = unicode_util:gc(Cs0), case lists:member(GC, GCs) of - true -> {rev(Tkn), Cs0}; + true -> {rev(Tkn), Cs2}; false -> lexeme_pick(Cs2, Seps, append(rev(GC),Tkn)) end; false -> @@ -1047,7 +1243,7 @@ lexeme_pick(Cs0, {GCs, CPs, _} = Seps, Tkn) when is_list(Cs0) -> {rev(Tkn), []} end; lexeme_pick(Bin, Seps, Tkn) when is_binary(Bin) -> - case bin_search(Bin, Seps) of + case bin_search(Bin, [], Seps) of {nomatch,_} -> {btoken(Bin,Tkn), []}; [Left] -> @@ -1056,35 +1252,38 @@ lexeme_pick(Bin, Seps, Tkn) when is_binary(Bin) -> {btoken(Lexeme, Tkn), Left} end. -nth_lexeme_m([Bin|Cont0], Seps, N) when is_binary(Bin) -> - case bin_search_inv(Bin, Cont0, Seps) of +nth_lexeme_m([Bin|Cont0], {GCs,_,_}=Seps0, N) when is_binary(Bin) -> + case bin_search_inv(Bin, Cont0, GCs) of {nomatch,Cont} -> - nth_lexeme_m(Cont, Seps, N); + nth_lexeme_m(Cont, Seps0, N); Cs when N > 1 -> - Rest = lexeme_skip(Cs, Seps), - nth_lexeme_m(Rest, Seps, N-1); + Rest = lexeme_skip(Cs, Seps0), + nth_lexeme_m(Rest, Seps0, N-1); Cs -> + Seps = search_compile(Seps0), {Lexeme,_} = lexeme_pick(Cs, Seps, []), Lexeme end; -nth_lexeme_m(Cs0, {GCs, _, _}=Seps, N) when is_list(Cs0) -> +nth_lexeme_m(Cs0, {GCs, _, _}=Seps0, N) when is_list(Cs0) -> case unicode_util:gc(Cs0) of [C|Cs] -> case lists:member(C, GCs) of true -> - nth_lexeme_m(Cs, Seps, N); + nth_lexeme_m(Cs, Seps0, N); false when N > 1 -> - Cs1 = lexeme_skip(Cs, Seps), - nth_lexeme_m(Cs1, Seps, N-1); + Cs1 = lexeme_skip(Cs, Seps0), + nth_lexeme_m(Cs1, Seps0, N-1); false -> + Seps = search_compile(Seps0), {Lexeme,_} = lexeme_pick(Cs0, Seps, []), Lexeme end; [] -> [] end; -nth_lexeme_m(Bin, Seps, N) when is_binary(Bin) -> - case bin_search_inv(Bin, [], Seps) of +nth_lexeme_m(Bin, {GCs,_,_}=Seps0, N) when is_binary(Bin) -> + Seps = search_compile(Seps0), + case bin_search_inv(Bin, [], GCs) of [Cs] when N > 1 -> Cs1 = lexeme_skip(Cs, Seps), nth_lexeme_m(Cs1, Seps, N-1); @@ -1100,16 +1299,17 @@ lexeme_skip([CP|Cs1]=Cs0, {GCs,CPs,_}=Seps) when is_integer(CP) -> true -> [GC|Cs2] = unicode_util:gc(Cs0), case lists:member(GC, GCs) of - true -> Cs0; + true -> Cs2; false -> lexeme_skip(Cs2, Seps) end; false -> lexeme_skip(Cs1, Seps) end; -lexeme_skip([Bin|Cont0], Seps) when is_binary(Bin) -> +lexeme_skip([Bin|Cont0], Seps0) when is_binary(Bin) -> + Seps = search_compile(Seps0), case bin_search(Bin, Cont0, Seps) of {nomatch,_} -> lexeme_skip(Cont0, Seps); - Cs -> Cs + Cs -> tl(unicode_util:gc(Cs)) end; lexeme_skip(Cs0, {GCs, CPs, _} = Seps) when is_list(Cs0) -> case unicode_util:cp(Cs0) of @@ -1118,7 +1318,7 @@ lexeme_skip(Cs0, {GCs, CPs, _} = Seps) when is_list(Cs0) -> true -> [GC|Cs2] = unicode_util:gc(Cs0), case lists:member(GC, GCs) of - true -> Cs0; + true -> Cs2; false -> lexeme_skip(Cs2, Seps) end; false -> @@ -1127,12 +1327,23 @@ lexeme_skip(Cs0, {GCs, CPs, _} = Seps) when is_list(Cs0) -> [] -> [] end; -lexeme_skip(Bin, Seps) when is_binary(Bin) -> - case bin_search(Bin, Seps) of +lexeme_skip(Bin, Seps0) when is_binary(Bin) -> + Seps = search_compile(Seps0), + case bin_search(Bin, [], Seps) of {nomatch,_} -> <<>>; - [Left] -> Left + [Left] -> tl(unicode_util:gc(Left)) end. +find_l([C1|Cs]=Cs0, [C|_]=Needle) when is_integer(C1) -> + case C1 of + C -> + case prefix_1(Cs0, Needle) of + nomatch -> find_l(Cs, Needle); + _ -> Cs0 + end; + _ -> + find_l(Cs, Needle) + end; find_l([Bin|Cont0], Needle) when is_binary(Bin) -> case bin_search_str(Bin, 0, Cont0, Needle) of {nomatch, _, Cont} -> @@ -1157,6 +1368,16 @@ find_l(Bin, Needle) -> {_Before, [Cs], _After} -> Cs end. +find_r([Cp|Cs]=Cs0, [C|_]=Needle, Res) when is_integer(Cp) -> + case Cp of + C -> + case prefix_1(Cs0, Needle) of + nomatch -> find_r(Cs, Needle, Res); + _ -> find_r(Cs, Needle, Cs0) + end; + _ -> + find_r(Cs, Needle, Res) + end; find_r([Bin|Cont0], Needle, Res) when is_binary(Bin) -> case bin_search_str(Bin, 0, Cont0, Needle) of {nomatch,_,Cont} -> @@ -1227,11 +1448,6 @@ cp_prefix_1(Orig, Until, Cont) -> %% Binary special -bin_search(Bin, Seps) -> - bin_search(Bin, [], Seps). - -bin_search(_Bin, Cont, {[],_,_}) -> - {nomatch, Cont}; bin_search(Bin, Cont, {Seps,_,BP}) -> bin_search_loop(Bin, 0, BP, Cont, Seps). @@ -1239,10 +1455,14 @@ bin_search(Bin, Cont, {Seps,_,BP}) -> %% i.e. å in nfd form $a "COMBINING RING ABOVE" %% and PREPEND characters like "ARABIC NUMBER SIGN" 1536 <<216,128>> %% combined with other characters are currently ignored. +search_pattern({_,_,_}=P) -> P; search_pattern(Seps) -> CPs = search_cp(Seps), - Bin = bin_pattern(CPs), - {Seps, CPs, Bin}. + {Seps, CPs, undefined}. + +search_compile({Sep, CPs, undefined}) -> + {Sep, CPs, binary:compile_pattern(bin_pattern(CPs))}; +search_compile({_,_,_}=Compiled) -> Compiled. search_cp([CP|Seps]) when is_integer(CP) -> [CP|search_cp(Seps)]; @@ -1263,9 +1483,21 @@ bin_search_loop(Bin0, Start, BinSeps, Cont, Seps) -> case binary:match(Bin, BinSeps) of nomatch -> {nomatch,Cont}; + {Where, _CL} when Cont =:= [] -> + <<_:Where/binary, Cont1/binary>> = Bin, + [GC|Cont2] = unicode_util:gc(Cont1), + case lists:member(GC, Seps) of + false when Cont2 =:= [] -> + {nomatch, []}; + false -> + Next = byte_size(Bin0) - byte_size(Cont2), + bin_search_loop(Bin0, Next, BinSeps, Cont, Seps); + true -> + [Cont1] + end; {Where, _CL} -> <<_:Where/binary, Cont0/binary>> = Bin, - Cont1 = stack(Cont0, Cont), + Cont1 = [Cont0|Cont], [GC|Cont2] = unicode_util:gc(Cont1), case lists:member(GC, Seps) of false -> @@ -1273,55 +1505,108 @@ bin_search_loop(Bin0, Start, BinSeps, Cont, Seps) -> [BinR|Cont] when is_binary(BinR) -> Next = byte_size(Bin0) - byte_size(BinR), bin_search_loop(Bin0, Next, BinSeps, Cont, Seps); - BinR when is_binary(BinR), Cont =:= [] -> - Next = byte_size(Bin0) - byte_size(BinR), - bin_search_loop(Bin0, Next, BinSeps, Cont, Seps); _ -> {nomatch, Cont2} end; - true when is_list(Cont1) -> - Cont1; true -> - [Cont1] + Cont1 end end. -bin_search_inv(Bin, Cont, {[], _, _}) -> - [Bin|Cont]; -bin_search_inv(Bin, Cont, {[Sep], _, _}) -> - bin_search_inv_1([Bin|Cont], Sep); -bin_search_inv(Bin, Cont, {Seps, _, _}) -> - bin_search_inv_n([Bin|Cont], Seps). - -bin_search_inv_1([<<>>|CPs], _) -> - {nomatch, CPs}; -bin_search_inv_1(CPs = [Bin0|Cont], Sep) when is_binary(Bin0) -> - case unicode_util:gc(CPs) of - [Sep|Bin] when is_binary(Bin), Cont =:= [] -> - bin_search_inv_1([Bin], Sep); - [Sep|[Bin|Cont]=Cs] when is_binary(Bin) -> - bin_search_inv_1(Cs, Sep); - [Sep|Cs] -> - {nomatch, Cs}; - _ -> CPs - end. +bin_search_inv(<<>>, Cont, _) -> + {nomatch, Cont}; +bin_search_inv(Bin, Cont, [Sep]) -> + bin_search_inv_1(Bin, Cont, Sep); +bin_search_inv(Bin, Cont, Seps) -> + bin_search_inv_n(Bin, Cont, Seps). + +bin_search_inv_1(<<CP1/utf8, BinRest/binary>>=Bin0, Cont, Sep) -> + case BinRest of + <<CP2/utf8, _/binary>> when ?ASCII_LIST(CP1, CP2) -> + case CP1 of + Sep -> bin_search_inv_1(BinRest, Cont, Sep); + _ -> [Bin0|Cont] + end; + _ when Cont =:= [] -> + case unicode_util:gc(Bin0) of + [Sep|Bin] -> bin_search_inv_1(Bin, Cont, Sep); + _ -> [Bin0|Cont] + end; + _ -> + case unicode_util:gc([Bin0|Cont]) of + [Sep|[Bin|Cont]] when is_binary(Bin) -> + bin_search_inv_1(Bin, Cont, Sep); + [Sep|Cs] -> + {nomatch, Cs}; + _ -> [Bin0|Cont] + end + end; +bin_search_inv_1(<<>>, Cont, _Sep) -> + {nomatch, Cont}; +bin_search_inv_1([], Cont, _Sep) -> + {nomatch, Cont}. -bin_search_inv_n([<<>>|CPs], _) -> - {nomatch, CPs}; -bin_search_inv_n([Bin0|Cont]=CPs, Seps) when is_binary(Bin0) -> - [C|Cs0] = unicode_util:gc(CPs), - case {lists:member(C, Seps), Cs0} of - {true, Cs} when is_binary(Cs), Cont =:= [] -> - bin_search_inv_n([Cs], Seps); - {true, [Bin|Cont]=Cs} when is_binary(Bin) -> - bin_search_inv_n(Cs, Seps); - {true, Cs} -> {nomatch, Cs}; - {false, _} -> CPs - end. +bin_search_inv_n(<<CP1/utf8, BinRest/binary>>=Bin0, Cont, Seps) -> + case BinRest of + <<CP2/utf8, _/binary>> when ?ASCII_LIST(CP1, CP2) -> + case lists:member(CP1,Seps) of + true -> bin_search_inv_n(BinRest, Cont, Seps); + false -> [Bin0|Cont] + end; + _ when Cont =:= [] -> + [GC|Bin] = unicode_util:gc(Bin0), + case lists:member(GC, Seps) of + true -> bin_search_inv_n(Bin, Cont, Seps); + false -> [Bin0|Cont] + end; + _ -> + [GC|Cs0] = unicode_util:gc([Bin0|Cont]), + case lists:member(GC, Seps) of + false -> [Bin0|Cont]; + true -> + case Cs0 of + [Bin|Cont] when is_binary(Bin) -> + bin_search_inv_n(Bin, Cont, Seps); + _ -> + {nomatch, Cs0} + end + end + end; +bin_search_inv_n(<<>>, Cont, _Sep) -> + {nomatch, Cont}; +bin_search_inv_n([], Cont, _Sep) -> + {nomatch, Cont}. + +bin_search_str(Bin0, Start, [], SearchCPs) -> + Compiled = binary:compile_pattern(unicode:characters_to_binary(SearchCPs)), + bin_search_str_1(Bin0, Start, Compiled, SearchCPs); bin_search_str(Bin0, Start, Cont, [CP|_]=SearchCPs) -> + First = binary:compile_pattern(<<CP/utf8>>), + bin_search_str_2(Bin0, Start, Cont, First, SearchCPs). + +bin_search_str_1(Bin0, Start, First, SearchCPs) -> + <<_:Start/binary, Bin/binary>> = Bin0, + case binary:match(Bin, First) of + nomatch -> {nomatch, byte_size(Bin0), []}; + {Where0, _} -> + Where = Start+Where0, + <<Keep:Where/binary, Cs0/binary>> = Bin0, + case prefix_1(Cs0, SearchCPs) of + nomatch -> + <<_/utf8, Cs/binary>> = Cs0, + KeepSz = byte_size(Bin0) - byte_size(Cs), + bin_search_str_1(Bin0, KeepSz, First, SearchCPs); + [] -> + {Keep, [Cs0], <<>>}; + Rest -> + {Keep, [Cs0], Rest} + end + end. + +bin_search_str_2(Bin0, Start, Cont, First, SearchCPs) -> <<_:Start/binary, Bin/binary>> = Bin0, - case binary:match(Bin, <<CP/utf8>>) of + case binary:match(Bin, First) of nomatch -> {nomatch, byte_size(Bin0), Cont}; {Where0, _} -> Where = Start+Where0, @@ -1330,7 +1615,7 @@ bin_search_str(Bin0, Start, Cont, [CP|_]=SearchCPs) -> case prefix_1(stack(Cs0,Cont), SearchCPs) of nomatch when is_binary(Cs) -> KeepSz = byte_size(Bin0) - byte_size(Cs), - bin_search_str(Bin0, KeepSz, Cont, SearchCPs); + bin_search_str_2(Bin0, KeepSz, Cont, First, SearchCPs); nomatch -> {nomatch, Where, stack([GC|Cs],Cont)}; [] -> diff --git a/lib/stdlib/test/base64_SUITE.erl b/lib/stdlib/test/base64_SUITE.erl index 48b3f5f959..1fc4c3fc0e 100644 --- a/lib/stdlib/test/base64_SUITE.erl +++ b/lib/stdlib/test/base64_SUITE.erl @@ -97,10 +97,9 @@ base64_otp_5635(Config) when is_list(Config) -> <<"===">> = base64:decode(base64:encode("===")), ok. %%------------------------------------------------------------------------- -%% OTP-6279: Guard needed so that function fails in a correct -%% way for faulty input, i.e. function_clause. +%% OTP-6279: Make sure illegal characters are rejected when decoding. base64_otp_6279(Config) when is_list(Config) -> - {'EXIT',{function_clause, _}} = (catch base64:decode("dGVzda==a")), + {'EXIT',_} = (catch base64:decode("dGVzda==a")), ok. %%------------------------------------------------------------------------- %% Encode and decode big binaries. @@ -115,48 +114,61 @@ big(Config) when is_list(Config) -> %%------------------------------------------------------------------------- %% Make sure illegal characters are rejected when decoding. illegal(Config) when is_list(Config) -> - {'EXIT',{function_clause, _}} = (catch base64:decode("()")), + %% A few samples with different error reasons. Nothing can be + %% assumed about the reason for the crash. + {'EXIT',_} = (catch base64:decode("()")), + {'EXIT',_} = (catch base64:decode(<<19:8,20:8,21:8,22:8>>)), + {'EXIT',_} = (catch base64:decode([19,20,21,22])), + {'EXIT',_} = (catch base64:decode_to_string(<<19:8,20:8,21:8,22:8>>)), + {'EXIT',_} = (catch base64:decode_to_string([19,20,21,22])), ok. %%------------------------------------------------------------------------- %% mime_decode and mime_decode_to_string have different implementations -%% so test both with the same input separately. Both functions have -%% the same implementation for binary/string arguments. +%% so test both with the same input separately. %% %% Test base64:mime_decode/1. mime_decode(Config) when is_list(Config) -> + MimeDecode = fun(In) -> + Out = base64:mime_decode(In), + Out = base64:mime_decode(binary_to_list(In)) + end, %% Test correct padding - <<"one">> = base64:mime_decode(<<"b25l">>), - <<"on">> = base64:mime_decode(<<"b24=">>), - <<"o">> = base64:mime_decode(<<"bw==">>), + <<"one">> = MimeDecode(<<"b25l">>), + <<"on">> = MimeDecode(<<"b24=">>), + <<"o">> = MimeDecode(<<"bw==">>), %% Test 1 extra padding - <<"one">> = base64:mime_decode(<<"b25l= =">>), - <<"on">> = base64:mime_decode(<<"b24== =">>), - <<"o">> = base64:mime_decode(<<"bw=== =">>), + <<"one">> = MimeDecode(<<"b25l= =">>), + <<"on">> = MimeDecode(<<"b24== =">>), + <<"o">> = MimeDecode(<<"bw=== =">>), %% Test 2 extra padding - <<"one">> = base64:mime_decode(<<"b25l===">>), - <<"on">> = base64:mime_decode(<<"b24====">>), - <<"o">> = base64:mime_decode(<<"bw=====">>), + <<"one">> = MimeDecode(<<"b25l===">>), + <<"on">> = MimeDecode(<<"b24====">>), + <<"o">> = MimeDecode(<<"bw=====">>), %% Test misc embedded padding - <<"one">> = base64:mime_decode(<<"b2=5l===">>), - <<"on">> = base64:mime_decode(<<"b=24====">>), - <<"o">> = base64:mime_decode(<<"b=w=====">>), + <<"one">> = MimeDecode(<<"b2=5l===">>), + <<"on">> = MimeDecode(<<"b=24====">>), + <<"o">> = MimeDecode(<<"b=w=====">>), %% Test misc white space and illegals with embedded padding - <<"one">> = base64:mime_decode(<<" b~2=\r\n5()l===">>), - <<"on">> = base64:mime_decode(<<"\tb =2\"¤4=¤= ==">>), - <<"o">> = base64:mime_decode(<<"\nb=w=====">>), + <<"one">> = MimeDecode(<<" b~2=\r\n5()l===">>), + <<"on">> = MimeDecode(<<"\tb =2\"¤4=¤= ==">>), + <<"o">> = MimeDecode(<<"\nb=w=====">>), %% Two pads <<"Aladdin:open sesame">> = - base64:mime_decode("QWxhZGRpbjpvc()GVuIHNlc2FtZQ=="), + MimeDecode(<<"QWxhZGRpbjpvc()GVuIHNlc2FtZQ==">>), %% One pad to ignore, followed by more text - <<"Hello World!!">> = base64:mime_decode(<<"SGVsb)(G8gV29ybGQ=h IQ= =">>), + <<"Hello World!!">> = MimeDecode(<<"SGVsb)(G8gV29ybGQ=h IQ= =">>), %% No pad <<"Aladdin:open sesam">> = - base64:mime_decode("QWxhZGRpbjpvcG¤\")(VuIHNlc2Ft"), + MimeDecode(<<"QWxhZGRpbjpvcG¤\")(VuIHNlc2Ft">>), %% Encoded base 64 strings may be divided by non base 64 chars. %% In this cases whitespaces. <<"0123456789!@#0^&*();:<>,. []{}">> = - base64:mime_decode( + MimeDecode( <<"MDEy MzQ1Njc4 \tOSFAIzBeJ \nio)(oKTs6 PD4sLi \r\nBbXXt9">>), + %% Zeroes + <<"012">> = MimeDecode(<<"\000M\000D\000E\000y=\000">>), + <<"o">> = MimeDecode(<<"bw==\000">>), + <<"o">> = MimeDecode(<<"bw=\000=">>), ok. %%------------------------------------------------------------------------- @@ -165,39 +177,48 @@ mime_decode(Config) when is_list(Config) -> %% Test base64:mime_decode_to_string/1. mime_decode_to_string(Config) when is_list(Config) -> + MimeDecodeToString = + fun(In) -> + Out = base64:mime_decode_to_string(In), + Out = base64:mime_decode_to_string(binary_to_list(In)) + end, %% Test correct padding - "one" = base64:mime_decode_to_string(<<"b25l">>), - "on" = base64:mime_decode_to_string(<<"b24=">>), - "o" = base64:mime_decode_to_string(<<"bw==">>), + "one" = MimeDecodeToString(<<"b25l">>), + "on" = MimeDecodeToString(<<"b24=">>), + "o" = MimeDecodeToString(<<"bw==">>), %% Test 1 extra padding - "one" = base64:mime_decode_to_string(<<"b25l= =">>), - "on" = base64:mime_decode_to_string(<<"b24== =">>), - "o" = base64:mime_decode_to_string(<<"bw=== =">>), + "one" = MimeDecodeToString(<<"b25l= =">>), + "on" = MimeDecodeToString(<<"b24== =">>), + "o" = MimeDecodeToString(<<"bw=== =">>), %% Test 2 extra padding - "one" = base64:mime_decode_to_string(<<"b25l===">>), - "on" = base64:mime_decode_to_string(<<"b24====">>), - "o" = base64:mime_decode_to_string(<<"bw=====">>), + "one" = MimeDecodeToString(<<"b25l===">>), + "on" = MimeDecodeToString(<<"b24====">>), + "o" = MimeDecodeToString(<<"bw=====">>), %% Test misc embedded padding - "one" = base64:mime_decode_to_string(<<"b2=5l===">>), - "on" = base64:mime_decode_to_string(<<"b=24====">>), - "o" = base64:mime_decode_to_string(<<"b=w=====">>), + "one" = MimeDecodeToString(<<"b2=5l===">>), + "on" = MimeDecodeToString(<<"b=24====">>), + "o" = MimeDecodeToString(<<"b=w=====">>), %% Test misc white space and illegals with embedded padding - "one" = base64:mime_decode_to_string(<<" b~2=\r\n5()l===">>), - "on" = base64:mime_decode_to_string(<<"\tb =2\"¤4=¤= ==">>), - "o" = base64:mime_decode_to_string(<<"\nb=w=====">>), + "one" = MimeDecodeToString(<<" b~2=\r\n5()l===">>), + "on" = MimeDecodeToString(<<"\tb =2\"¤4=¤= ==">>), + "o" = MimeDecodeToString(<<"\nb=w=====">>), %% Two pads "Aladdin:open sesame" = - base64:mime_decode_to_string("QWxhZGRpbjpvc()GVuIHNlc2FtZQ=="), + MimeDecodeToString(<<"QWxhZGRpbjpvc()GVuIHNlc2FtZQ==">>), %% One pad to ignore, followed by more text - "Hello World!!" = base64:mime_decode_to_string(<<"SGVsb)(G8gV29ybGQ=h IQ= =">>), + "Hello World!!" = MimeDecodeToString(<<"SGVsb)(G8gV29ybGQ=h IQ= =">>), %% No pad "Aladdin:open sesam" = - base64:mime_decode_to_string("QWxhZGRpbjpvcG¤\")(VuIHNlc2Ft"), + MimeDecodeToString(<<"QWxhZGRpbjpvcG¤\")(VuIHNlc2Ft">>), %% Encoded base 64 strings may be divided by non base 64 chars. %% In this cases whitespaces. "0123456789!@#0^&*();:<>,. []{}" = - base64:mime_decode_to_string( + MimeDecodeToString( <<"MDEy MzQ1Njc4 \tOSFAIzBeJ \nio)(oKTs6 PD4sLi \r\nBbXXt9">>), + %% Zeroes + "012" = MimeDecodeToString(<<"\000M\000D\000E\000y=\000">>), + "o" = MimeDecodeToString(<<"bw==\000">>), + "o" = MimeDecodeToString(<<"bw=\000=">>), ok. %%------------------------------------------------------------------------- diff --git a/lib/stdlib/test/erl_lint_SUITE.erl b/lib/stdlib/test/erl_lint_SUITE.erl index b76bece07f..5efffc6a5c 100644 --- a/lib/stdlib/test/erl_lint_SUITE.erl +++ b/lib/stdlib/test/erl_lint_SUITE.erl @@ -66,7 +66,8 @@ otp_11851/1,otp_11879/1,otp_13230/1, record_errors/1, otp_11879_cont/1, non_latin1_module/1, otp_14323/1, - get_stacktrace/1, otp_14285/1, otp_14378/1]). + get_stacktrace/1, stacktrace_syntax/1, + otp_14285/1, otp_14378/1]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -87,7 +88,7 @@ all() -> maps, maps_type, maps_parallel_match, otp_11851, otp_11879, otp_13230, record_errors, otp_11879_cont, non_latin1_module, otp_14323, - get_stacktrace, otp_14285, otp_14378]. + get_stacktrace, stacktrace_syntax, otp_14285, otp_14378]. groups() -> [{unused_vars_warn, [], @@ -4129,6 +4130,40 @@ get_stacktrace(Config) -> run(Config, Ts), ok. +stacktrace_syntax(Config) -> + Ts = [{guard, + <<"t1() -> + try error(foo) + catch _:_:Stk when is_number(Stk) -> ok + end. + ">>, + [], + {errors,[{3,erl_lint,{stacktrace_guard,'Stk'}}],[]}}, + {bound, + <<"t1() -> + Stk = [], + try error(foo) + catch _:_:Stk -> ok + end. + ">>, + [], + {errors,[{4,erl_lint,{stacktrace_bound,'Stk'}}],[]}}, + {guard_and_bound, + <<"t1() -> + Stk = [], + try error(foo) + catch _:_:Stk when is_integer(Stk) -> ok + end. + ">>, + [], + {errors,[{4,erl_lint,{stacktrace_bound,'Stk'}}, + {4,erl_lint,{stacktrace_guard,'Stk'}}],[]}} + ], + + run(Config, Ts), + ok. + + %% Unicode atoms. otp_14285(Config) -> %% A small sample of all the errors and warnings in module erl_lint. diff --git a/lib/stdlib/test/filelib_SUITE.erl b/lib/stdlib/test/filelib_SUITE.erl index 1236fe45f4..930cea347f 100644 --- a/lib/stdlib/test/filelib_SUITE.erl +++ b/lib/stdlib/test/filelib_SUITE.erl @@ -33,6 +33,8 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("kernel/include/file.hrl"). +-define(PRIM_FILE, prim_file). + init_per_testcase(_Case, Config) -> Config. @@ -446,10 +448,10 @@ wildcard_symlink(Config) when is_list(Config) -> erl_prim_loader)), ["sub","symlink"] = basenames(Dir, filelib:wildcard(filename:join(Dir, "*"), - prim_file)), + ?PRIM_FILE)), ["symlink"] = basenames(Dir, filelib:wildcard(filename:join(Dir, "symlink"), - prim_file)), + ?PRIM_FILE)), ok = file:delete(AFile), %% The symlink should still be visible even when its target %% has been deleted. @@ -465,10 +467,10 @@ wildcard_symlink(Config) when is_list(Config) -> erl_prim_loader)), ["sub","symlink"] = basenames(Dir, filelib:wildcard(filename:join(Dir, "*"), - prim_file)), + ?PRIM_FILE)), ["symlink"] = basenames(Dir, filelib:wildcard(filename:join(Dir, "symlink"), - prim_file)), + ?PRIM_FILE)), ok end. @@ -497,17 +499,17 @@ is_file_symlink(Config) -> ok -> true = filelib:is_dir(DirAlias), true = filelib:is_dir(DirAlias, erl_prim_loader), - true = filelib:is_dir(DirAlias, prim_file), + true = filelib:is_dir(DirAlias, ?PRIM_FILE), true = filelib:is_file(DirAlias), true = filelib:is_file(DirAlias, erl_prim_loader), - true = filelib:is_file(DirAlias, prim_file), + true = filelib:is_file(DirAlias, ?PRIM_FILE), ok = file:make_symlink(AFile,FileAlias), true = filelib:is_file(FileAlias), true = filelib:is_file(FileAlias, erl_prim_loader), - true = filelib:is_file(FileAlias, prim_file), + true = filelib:is_file(FileAlias, ?PRIM_FILE), true = filelib:is_regular(FileAlias), true = filelib:is_regular(FileAlias, erl_prim_loader), - true = filelib:is_regular(FileAlias, prim_file), + true = filelib:is_regular(FileAlias, ?PRIM_FILE), ok end. @@ -528,11 +530,11 @@ file_props_symlink(Config) -> {_,_} = LastMod = filelib:last_modified(AFile), LastMod = filelib:last_modified(Alias), LastMod = filelib:last_modified(Alias, erl_prim_loader), - LastMod = filelib:last_modified(Alias, prim_file), + LastMod = filelib:last_modified(Alias, ?PRIM_FILE), FileSize = filelib:file_size(AFile), FileSize = filelib:file_size(Alias), FileSize = filelib:file_size(Alias, erl_prim_loader), - FileSize = filelib:file_size(Alias, prim_file) + FileSize = filelib:file_size(Alias, ?PRIM_FILE) end. find_source(Config) when is_list(Config) -> diff --git a/lib/stdlib/test/qlc_SUITE.erl b/lib/stdlib/test/qlc_SUITE.erl index 949142ec77..8f8a0f6e73 100644 --- a/lib/stdlib/test/qlc_SUITE.erl +++ b/lib/stdlib/test/qlc_SUITE.erl @@ -1695,28 +1695,7 @@ sort(Config) when is_list(Config) -> [true || I <- lists:seq(1, 50000), not ets:insert(E, {I, I})], H = qlc:q([{X,Y} || X <- [a,b], Y <- qlc:sort(ets:table(E))]), 100000 = length(qlc:e(H)), - ets:delete(E)">>, - - begin - TmpDir = ?privdir, - [<<"TE = process_flag(trap_exit, true), - E = ets:new(foo, []), - [true || I <- lists:seq(1, 50000), not ets:insert(E, {I, I})], - Ports = erlang:ports(), - H = qlc:q([{X,Y} || X <- [a,b], - begin - [P] = erlang:ports() -- Ports, - exit(P, port_exit), - true - end, - Y <- qlc:sort(ets:table(E), - [{tmpdir,\"">>, - TmpDir, <<"\"}])]), - {error, qlc, {file_error, _, _}} = (catch qlc:e(H)), - receive {'EXIT', _, port_exit} -> ok end, - ets:delete(E), - process_flag(trap_exit, TE)">>] - end + ets:delete(E)">> ], run(Config, Ts), diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl index ef4f9faad9..3d3241b33d 100644 --- a/lib/stdlib/test/rand_SUITE.erl +++ b/lib/stdlib/test/rand_SUITE.erl @@ -29,6 +29,9 @@ basic_stats_uniform_1/1, basic_stats_uniform_2/1, basic_stats_standard_normal/1, basic_stats_normal/1, + stats_standard_normal_box_muller/1, + stats_standard_normal_box_muller_2/1, + stats_standard_normal/1, uniform_real_conv/1, plugin/1, measure/1, reference_jump_state/1, reference_jump_procdict/1]). @@ -57,7 +60,10 @@ all() -> groups() -> [{basic_stats, [parallel], [basic_stats_uniform_1, basic_stats_uniform_2, - basic_stats_standard_normal]}, + basic_stats_standard_normal, + stats_standard_normal_box_muller, + stats_standard_normal_box_muller_2, + stats_standard_normal]}, {reference_jump, [parallel], [reference_jump_state, reference_jump_procdict]}]. @@ -410,6 +416,206 @@ normal_s(Mean, Variance, State0) when Mean == 0, Variance == 1 -> normal_s(Mean, Variance, State0) -> rand:normal_s(Mean, Variance, State0). + + +-dialyzer({no_improper_lists, stats_standard_normal_box_muller/1}). +stats_standard_normal_box_muller(Config) when is_list(Config) -> + try math:erfc(1.0) of + _ -> + TwoPi = 2.0 * math:pi(), + NormalS = + fun + ([S0]) -> + {U1, S1} = rand:uniform_real_s(S0), + R = math:sqrt(-2.0 * math:log(U1)), + {U2, S2} = rand:uniform_s(S1), + T = TwoPi * U2, + Z0 = R * math:cos(T), + Z1 = R * math:sin(T), + {Z0, [S2|Z1]}; + ([S|Z]) -> + {Z, [S]} + end, + State = [rand:seed(exrop)], + stats_standard_normal(NormalS, State) + catch error:_ -> + {skip, "math:erfc/1 not supported"} + end. + +-dialyzer({no_improper_lists, stats_standard_normal_box_muller_2/1}). +stats_standard_normal_box_muller_2(Config) when is_list(Config) -> + try math:erfc(1.0) of + _ -> + TwoPi = 2.0 * math:pi(), + NormalS = + fun + ([S0]) -> + {U0, S1} = rand:uniform_s(S0), + U1 = 1.0 - U0, + R = math:sqrt(-2.0 * math:log(U1)), + {U2, S2} = rand:uniform_s(S1), + T = TwoPi * U2, + Z0 = R * math:cos(T), + Z1 = R * math:sin(T), + {Z0, [S2|Z1]}; + ([S|Z]) -> + {Z, [S]} + end, + State = [rand:seed(exrop)], + stats_standard_normal(NormalS, State) + catch error:_ -> + {skip, "math:erfc/1 not supported"} + end. + + +stats_standard_normal(Config) when is_list(Config) -> + try math:erfc(1.0) of + _ -> + stats_standard_normal( + fun rand:normal_s/1, rand:seed_s(exrop)) + catch error:_ -> + {skip, "math:erfc/1 not supported"} + end. +%% +stats_standard_normal(Fun, S) -> +%%% +%%% ct config: +%%% {rand_SUITE, [{stats_standard_normal,[{seconds, 8}, {std_devs, 4.2}]}]}. +%%% + Seconds = ct:get_config({?MODULE, ?FUNCTION_NAME, seconds}, 8), + StdDevs = + ct:get_config( + {?MODULE, ?FUNCTION_NAME, std_devs}, + 4.2), % probability erfc(4.2/sqrt(2)) (1/37465) to fail a bucket +%%% + ct:timetrap({seconds, Seconds + 120}), + %% Buckets is chosen to get a range where the the probability to land + %% in the top catch-all bucket is not vanishingly low, but with + %% these values it is about 1/25 of the probability for the low bucket + %% (closest to 0). + %% + %% Rounds is calculated so the expected value for the low + %% bucket will be at least TargetHits. + %% + InvDelta = 512, + Buckets = 4 * InvDelta, % 4 std devs range + TargetHits = 1024, + Sqrt2 = math:sqrt(2.0), + W = InvDelta * Sqrt2, + P0 = math:erf(1 / W), + Rounds = TargetHits * ceil(1.0 / P0), + Histogram = array:new({default, 0}), + StopTime = erlang:monotonic_time(second) + Seconds, + ct:pal( + "Running standard normal test against ~w std devs for ~w seconds...", + [StdDevs, Seconds]), + {PositiveHistogram, NegativeHistogram, Outlier, TotalRounds} = + stats_standard_normal( + InvDelta, Buckets, Histogram, Histogram, 0.0, + Fun, S, Rounds, StopTime, Rounds, 0), + Precision = math:sqrt(TotalRounds * P0) / StdDevs, + TopP = math:erfc(Buckets / W), + TopPrecision = math:sqrt(TotalRounds * TopP) / StdDevs, + OutlierProbability = math:erfc(Outlier / Sqrt2) * TotalRounds, + InvOP = 1.0 / OutlierProbability, + ct:pal( + "Total rounds: ~w, tolerance: 1/~.2f..1/~.2f, " + "outlier: ~.2f, probability 1/~.2f.", + [TotalRounds, Precision, TopPrecision, Outlier, InvOP]), + {TotalRounds, [], []} = + {TotalRounds, + check_histogram( + W, TotalRounds, StdDevs, PositiveHistogram, Buckets), + check_histogram( + W, TotalRounds, StdDevs, NegativeHistogram, Buckets)}, + %% If the probability for getting this Outlier is lower than 1/50, + %% then this is fishy! + true = (1/50 =< OutlierProbability), + {comment, {tp, TopPrecision, op, InvOP}}. +%% +stats_standard_normal( + InvDelta, Buckets, PositiveHistogram, NegativeHistogram, Outlier, + Fun, S, 0, StopTime, Rounds, TotalRounds) -> + case erlang:monotonic_time(second) of + Now when Now < StopTime -> + stats_standard_normal( + InvDelta, Buckets, + PositiveHistogram, NegativeHistogram, Outlier, + Fun, S, Rounds, StopTime, Rounds, TotalRounds + Rounds); + _ -> + {PositiveHistogram, NegativeHistogram, + Outlier, TotalRounds + Rounds} + end; +stats_standard_normal( + InvDelta, Buckets, PositiveHistogram, NegativeHistogram, Outlier, + Fun, S, Count, StopTime, Rounds, TotalRounds) -> + case Fun(S) of + {X, NewS} when 0.0 =< X -> + Bucket = min(Buckets, floor(X * InvDelta)), + stats_standard_normal( + InvDelta, Buckets, + increment_bucket(Bucket, PositiveHistogram), + NegativeHistogram, max(Outlier, X), + Fun, NewS, Count - 1, StopTime, Rounds, TotalRounds); + {MinusX, NewS} -> + X = -MinusX, + Bucket = min(Buckets, floor(X * InvDelta)), + stats_standard_normal( + InvDelta, Buckets, + PositiveHistogram, + increment_bucket(Bucket, NegativeHistogram), max(Outlier, X), + Fun, NewS, Count - 1, StopTime, Rounds, TotalRounds) + end. + +increment_bucket(Bucket, Array) -> + array:set(Bucket, array:get(Bucket, Array) + 1, Array). + +check_histogram(W, Rounds, StdDevs, Histogram, Buckets) -> + %%PrevBucket = 512, + %%Bucket = PrevBucket - 1, + %%P = 0.5 * math:erfc(PrevBucket / W), + TargetP = 0.5 * math:erfc(Buckets / W), + P = 0.0, + N = 0, + check_histogram( + W, Rounds, StdDevs, Histogram, TargetP, + Buckets, Buckets, P, N). +%% +check_histogram( + _W, _Rounds, _StdDevs, _Histogram, _TargetP, + 0, _PrevBucket, _PrevP, _PrevN) -> + []; +check_histogram( + W, Rounds, StdDevs, Histogram, TargetP, + Bucket, PrevBucket, PrevP, PrevN) -> + N = PrevN + array:get(Bucket, Histogram), + P = 0.5 * math:erfc(Bucket / W), + BucketP = P - PrevP, + if + TargetP =< BucketP -> + check_histogram( + W, Rounds, StdDevs, Histogram, TargetP, + Bucket - 1, PrevBucket, PrevP, N); + true -> + Exp = BucketP * Rounds, + Var = Rounds * BucketP*(1.0 - BucketP), + Threshold = StdDevs * math:sqrt(Var), + LowerLimit = floor(Exp - Threshold), + UpperLimit = ceil(Exp + Threshold), + if + N < LowerLimit; UpperLimit < N -> + [#{bucket => {Bucket, PrevBucket}, n => N, exp => Exp, + lower => LowerLimit, upper => UpperLimit} | + check_histogram( + W, Rounds, StdDevs, Histogram, TargetP, + Bucket - 1, Bucket, P, 0)]; + true -> + check_histogram( + W, Rounds, StdDevs, Histogram, TargetP, + Bucket - 1, Bucket, P, 0) + end + end. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% White box test of the conversion to float diff --git a/lib/stdlib/test/stdlib_bench_SUITE.erl b/lib/stdlib/test/stdlib_bench_SUITE.erl index 8670e7029c..2a9981bb9e 100644 --- a/lib/stdlib/test/stdlib_bench_SUITE.erl +++ b/lib/stdlib/test/stdlib_bench_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2012-2016. All Rights Reserved. +%% Copyright Ericsson AB 2012-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -28,13 +28,20 @@ suite() -> [{ct_hooks,[{ts_install_cth,[{nodenames,2}]}]}]. all() -> - [{group,unicode}]. + [{group,unicode}, {group,base64}]. groups() -> [{unicode,[{repeat,5}], [norm_nfc_list, norm_nfc_deep_l, norm_nfc_binary, string_lexemes_list, string_lexemes_binary - ]}]. + ]}, + {base64,[{repeat,5}], + [decode_binary, decode_binary_to_string, + decode_list, decode_list_to_string, + encode_binary, encode_binary_to_string, + encode_list, encode_list_to_string, + mime_binary_decode, mime_binary_decode_to_string, + mime_list_decode, mime_list_decode_to_string]}]. init_per_group(_GroupName, Config) -> Config. @@ -105,3 +112,97 @@ norm_data(Config) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +decode_binary(_Config) -> + test(decode, encoded_binary()). + +decode_binary_to_string(_Config) -> + test(decode_to_string, encoded_binary()). + +decode_list(_Config) -> + test(decode, encoded_list()). + +decode_list_to_string(_Config) -> + test(decode_to_string, encoded_list()). + +encode_binary(_Config) -> + test(encode, binary()). + +encode_binary_to_string(_Config) -> + test(encode_to_string, binary()). + +encode_list(_Config) -> + test(encode, list()). + +encode_list_to_string(_Config) -> + test(encode_to_string, list()). + +mime_binary_decode(_Config) -> + test(mime_decode, encoded_binary()). + +mime_binary_decode_to_string(_Config) -> + test(mime_decode_to_string, encoded_binary()). + +mime_list_decode(_Config) -> + test(mime_decode, encoded_list()). + +mime_list_decode_to_string(_Config) -> + test(mime_decode_to_string, encoded_list()). + +-define(SIZE, 10000). +-define(N, 1000). + +encoded_binary() -> + list_to_binary(encoded_list()). + +encoded_list() -> + L = random_byte_list(round(?SIZE*0.75)), + base64:encode_to_string(L). + +binary() -> + list_to_binary(list()). + +list() -> + random_byte_list(?SIZE). + +test(Func, Data) -> + F = fun() -> loop(?N, Func, Data) end, + {Time, ok} = timer:tc(fun() -> lspawn(F) end), + report_base64(Time). + +loop(0, _F, _D) -> garbage_collect(), ok; +loop(N, F, D) -> + _ = base64:F(D), + loop(N - 1, F, D). + +lspawn(Fun) -> + {Pid, Ref} = spawn_monitor(fun() -> exit(Fun()) end), + receive + {'DOWN', Ref, process, Pid, Rep} -> Rep + end. + +report_base64(Time) -> + Tps = round((?N*1000000)/Time), + ct_event:notify(#event{name = benchmark_data, + data = [{suite, "stdlib_base64"}, + {value, Tps}]}), + Tps. + +%% Copied from base64_SUITE.erl. + +random_byte_list(N) -> + random_byte_list(N, []). + +random_byte_list(0, Acc) -> + Acc; +random_byte_list(N, Acc) -> + random_byte_list(N-1, [rand:uniform(255)|Acc]). + +make_big_binary(N) -> + list_to_binary(mbb(N, [])). + +mbb(N, Acc) when N > 256 -> + B = list_to_binary(lists:seq(0, 255)), + mbb(N - 256, [B | Acc]); +mbb(N, Acc) -> + B = list_to_binary(lists:seq(0, N-1)), + lists:reverse(Acc, B). diff --git a/lib/stdlib/test/string_SUITE.erl b/lib/stdlib/test/string_SUITE.erl index 05f18ef238..d02a6eac0a 100644 --- a/lib/stdlib/test/string_SUITE.erl +++ b/lib/stdlib/test/string_SUITE.erl @@ -48,6 +48,7 @@ %% Run tests when debugging them -export([debug/0, time_func/4]). +-compile([nowarn_deprecated_function]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -92,14 +93,11 @@ end_per_testcase(_Case, _Config) -> ok. debug() -> - Config = [{data_dir, ?MODULE_STRING++"_data"}], + Config = [{data_dir, "./" ++ ?MODULE_STRING++"_data"}], [io:format("~p:~p~n",[Test,?MODULE:Test(Config)]) || {_,Tests} <- groups(), Test <- Tests]. -define(TEST(B,C,D), test(?LINE,?FUNCTION_NAME,B,C,D, true)). --define(TEST_EQ(B,C,D), - test(?LINE,?FUNCTION_NAME,B,C,D, true), - test(?LINE,?FUNCTION_NAME,hd(C),[B|tl(C),D, true)). -define(TEST_NN(B,C,D), test(?LINE,?FUNCTION_NAME,B,C,D, false), @@ -294,6 +292,7 @@ trim(_) -> ?TEST(["..h", ".e", <<"j..">>], [both, ". "], "h.ej"), ?TEST(["..h", <<".ejsa"/utf8>>, "n.."], [both, ". "], "h.ejsan"), %% Test that it behaves with graphemes (i.e. nfd tests are the hard part) + ?TEST([1013,101,778,101,101], [trailing, [101]], [1013,101,778]), ?TEST("aaåaa", [both, "a"], "å"), ?TEST(["aaa",778,"äöoo"], [both, "ao"], "åäö"), ?TEST([<<"aaa">>,778,"äöoo"], [both, "ao"], "åäö"), @@ -353,6 +352,7 @@ take(_) -> ?TEST([<<>>,<<"..">>, " h.ej", <<" ..">>], [Chars, true, leading], {".. ", "h.ej .."}), ?TEST(["..h", <<".ejsa"/utf8>>, "n.."], [Chars, true, leading], {"..", "h.ejsan.."}), %% Test that it behaves with graphemes (i.e. nfd tests are the hard part) + ?TEST([101,778], [[[101, 779]], true], {[101,778], []}), ?TEST(["aaee",778,"äöoo"], [[[$e,778]], true, leading], {"aae", [$e,778|"äöoo"]}), ?TEST([<<"aae">>,778,"äöoo"], [[[$e,778]],true,leading], {"aa", [$e,778|"äöoo"]}), ?TEST([<<"e">>,778,"åäöe", <<778/utf8>>], [[[$e,778]], true, leading], {[], [$e,778]++"åäöe"++[778]}), @@ -713,29 +713,123 @@ nth_lexeme(_) -> meas(Config) -> + Parent = self(), + Exec = fun() -> + DataDir0 = proplists:get_value(data_dir, Config), + DataDir = filename:join(lists:droplast(filename:split(DataDir0))), + case proplists:get_value(profile, Config, false) of + false -> + do_measure(DataDir); + eprof -> + eprof:profile(fun() -> do_measure(DataDir) end, [set_on_spawn]), + eprof:stop_profiling(), + eprof:analyze(), + eprof:stop() + end, + Parent ! {test_done, self()}, + normal + end, + ct:timetrap({minutes,2}), case ct:get_timetrap_info() of {_,{_,Scale}} when Scale > 1 -> {skip,{will_not_run_in_debug,Scale}}; - _ -> % No scaling - DataDir = proplists:get_value(data_dir, Config), - TestDir = filename:dirname(string:trim(DataDir, trailing, "/")), - do_measure(TestDir) + _ -> % No scaling, run at most 1.5 min + Tester = spawn(Exec), + receive {test_done, Tester} -> ok + after 90000 -> + io:format("Timelimit reached stopping~n",[]), + exit(Tester, die) + end, + ok end. -do_measure(TestDir) -> - File = filename:join(TestDir, ?MODULE_STRING ++ ".erl"), +do_measure(DataDir) -> + File = filename:join([DataDir,"unicode_util_SUITE_data","NormalizationTest.txt"]), io:format("File ~s ",[File]), {ok, Bin} = file:read_file(File), io:format("~p~n",[byte_size(Bin)]), Do = fun(Name, Func, Mode) -> - {N, Mean, Stddev, _} = time_func(Func, Mode, Bin, 50), - io:format("~10w ~6w ~6.2fms ±~4.2fms #~.2w gc included~n", + {N, Mean, Stddev, _} = time_func(Func, Mode, Bin, 20), + io:format("~15w ~6w ~6.2fms ±~5.2fms #~.2w gc included~n", [Name, Mode, Mean/1000, Stddev/1000, N]) end, + Do2 = fun(Name, Func, Mode) -> + {N, Mean, Stddev, _} = time_func(Func, binary, <<>>, 20), + io:format("~15w ~6w ~6.2fms ±~5.2fms #~.2w gc included~n", + [Name, Mode, Mean/1000, Stddev/1000, N]) + end, io:format("----------------------~n"), - Do(tokens, fun(Str) -> string:tokens(Str, [$\n,$\r]) end, list), + + Do(old_tokens, fun(Str) -> string:tokens(Str, [$\n,$\r]) end, list), Tokens = {lexemes, fun(Str) -> string:lexemes(Str, [$\n,$\r]) end}, [Do(Name,Fun,Mode) || {Name,Fun} <- [Tokens], Mode <- [list, binary]], + + S0 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxy.....", + S0B = <<"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxy.....">>, + Do2(old_strip_l, repeat(fun() -> string:strip(S0, left, $x) end), list), + Do2(trim_l, repeat(fun() -> string:trim(S0, leading, [$x]) end), list), + Do2(trim_l, repeat(fun() -> string:trim(S0B, leading, [$x]) end), binary), + Do2(old_strip_r, repeat(fun() -> string:strip(S0, right, $.) end), list), + Do2(trim_t, repeat(fun() -> string:trim(S0, trailing, [$.]) end), list), + Do2(trim_t, repeat(fun() -> string:trim(S0B, trailing, [$.]) end), binary), + + Do2(old_chr_sub, repeat(fun() -> string:sub_string(S0, string:chr(S0, $.)) end), list), + Do2(old_str_sub, repeat(fun() -> string:sub_string(S0, string:str(S0, [$.])) end), list), + Do2(find, repeat(fun() -> string:find(S0, [$.]) end), list), + Do2(find, repeat(fun() -> string:find(S0B, [$.]) end), binary), + Do2(old_str_sub2, repeat(fun() -> N = string:str(S0, "xy.."), + {string:sub_string(S0,1,N), string:sub_string(S0,N+4)} end), list), + Do2(split, repeat(fun() -> string:split(S0, "xy..") end), list), + Do2(split, repeat(fun() -> string:split(S0B, "xy..") end), binary), + + Do2(old_rstr_sub, repeat(fun() -> string:sub_string(S0, string:rstr(S0, [$y])) end), list), + Do2(find_t, repeat(fun() -> string:find(S0, [$y], trailing) end), list), + Do2(find_t, repeat(fun() -> string:find(S0B, [$y], trailing) end), binary), + Do2(old_rstr_sub2, repeat(fun() -> N = string:rstr(S0, "y.."), + {string:sub_string(S0,1,N), string:sub_string(S0,N+3)} end), list), + Do2(split_t, repeat(fun() -> string:split(S0, "y..", trailing) end), list), + Do2(split_t, repeat(fun() -> string:split(S0B, "y..", trailing) end), binary), + + Do2(old_span, repeat(fun() -> N=string:span(S0, [$x, $y]), + {string:sub_string(S0,1,N),string:sub_string(S0,N+1)} + end), list), + Do2(take, repeat(fun() -> string:take(S0, [$x, $y]) end), list), + Do2(take, repeat(fun() -> string:take(S0B, [$x, $y]) end), binary), + + Do2(old_cspan, repeat(fun() -> N=string:cspan(S0, [$.,$y]), + {string:sub_string(S0,1,N),string:sub_string(S0,N+1)} + end), list), + Do2(take_c, repeat(fun() -> string:take(S0, [$.,$y], true) end), list), + Do2(take_c, repeat(fun() -> string:take(S0B, [$.,$y], true) end), binary), + + Do2(old_substr, repeat(fun() -> string:substr(S0, 21, 15) end), list), + Do2(slice, repeat(fun() -> string:slice(S0, 20, 15) end), list), + Do2(slice, repeat(fun() -> string:slice(S0B, 20, 15) end), binary), + + io:format("--~n",[]), + NthTokens = {nth_lexemes, fun(Str) -> string:nth_lexeme(Str, 18000, [$\n,$\r]) end}, + [Do(Name,Fun,Mode) || {Name,Fun} <- [NthTokens], Mode <- [list, binary]], + Do2(take_t, repeat(fun() -> string:take(S0, [$.,$y], false, trailing) end), list), + Do2(take_t, repeat(fun() -> string:take(S0B, [$.,$y], false, trailing) end), binary), + Do2(take_tc, repeat(fun() -> string:take(S0, [$x], true, trailing) end), list), + Do2(take_tc, repeat(fun() -> string:take(S0B, [$x], true, trailing) end), binary), + + Length = {length, fun(Str) -> string:length(Str) end}, + [Do(Name,Fun,Mode) || {Name,Fun} <- [Length], Mode <- [list, binary]], + + Reverse = {reverse, fun(Str) -> string:reverse(Str) end}, + [Do(Name,Fun,Mode) || {Name,Fun} <- [Reverse], Mode <- [list, binary]], + + ok. + +repeat(F) -> + fun(_) -> repeat_1(F,20000) end. + +repeat_1(F, N) when N > 0 -> + F(), + repeat_1(F, N-1); +repeat_1(_, _) -> + erlang:garbage_collect(), ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -865,8 +959,6 @@ check_types_1({list, _},{list, undefined}) -> ok; check_types_1({list, _},{list, codepoints}) -> ok; -check_types_1({list, _},{list, {list, codepoints}}) -> - ok; check_types_1({list, {list, _}},{list, {list, codepoints}}) -> ok; check_types_1(mixed,_) -> diff --git a/lib/stdlib/test/unicode_util_SUITE.erl b/lib/stdlib/test/unicode_util_SUITE.erl index 7dba0a2fd0..632d9ae6e6 100644 --- a/lib/stdlib/test/unicode_util_SUITE.erl +++ b/lib/stdlib/test/unicode_util_SUITE.erl @@ -312,12 +312,23 @@ get(_) -> add_get_tests. count(Config) -> + Parent = self(), + Exec = fun() -> + do_measure(Config), + Parent ! {test_done, self()} + end, ct:timetrap({minutes,5}), case ct:get_timetrap_info() of - {_,{_,Scale}} -> + {_,{_,Scale}} when Scale > 1 -> {skip,{measurments_skipped_debug,Scale}}; - _ -> % No scaling - do_measure(Config) + _ -> % No scaling, run at most 2 min + Tester = spawn(Exec), + receive {test_done, Tester} -> ok + after 120000 -> + io:format("Timelimit reached stopping~n",[]), + exit(Tester, die) + end, + ok end. do_measure(Config) -> diff --git a/lib/stdlib/uc_spec/gen_unicode_mod.escript b/lib/stdlib/uc_spec/gen_unicode_mod.escript index 674e5a0628..fe5a860d45 100755 --- a/lib/stdlib/uc_spec/gen_unicode_mod.escript +++ b/lib/stdlib/uc_spec/gen_unicode_mod.escript @@ -170,7 +170,7 @@ gen_header(Fd) -> io:put_chars(Fd, "-export([spec_version/0, lookup/1, get_case/1]).\n"), io:put_chars(Fd, "-inline([class/1]).\n"), io:put_chars(Fd, "-compile(nowarn_unused_vars).\n"), - io:put_chars(Fd, "-dialyzer({no_improper_lists, [cp/1, gc_prepend/2, gc_e_cont/2]}).\n"), + io:put_chars(Fd, "-dialyzer({no_improper_lists, [cp/1, gc/1, gc_prepend/2, gc_e_cont/2]}).\n"), io:put_chars(Fd, "-type gc() :: char()|[char()].\n\n\n"), ok. @@ -240,7 +240,7 @@ gen_norm(Fd) -> "-spec nfd(unicode:chardata()) -> maybe_improper_list(gc(),unicode:chardata()) | {error, unicode:chardata()}.\n" "nfd(Str0) ->\n" " case gc(Str0) of\n" - " [GC|R] when GC < 127 -> [GC|R];\n" + " [GC|R] when GC < 128 -> [GC|R];\n" " [GC|Str] -> [decompose(GC)|Str];\n" " [] -> [];\n" " {error,_}=Error -> Error\n end.\n\n" @@ -250,7 +250,7 @@ gen_norm(Fd) -> "-spec nfkd(unicode:chardata()) -> maybe_improper_list(gc(),unicode:chardata()) | {error, unicode:chardata()}.\n" "nfkd(Str0) ->\n" " case gc(Str0) of\n" - " [GC|R] when GC < 127 -> [GC|R];\n" + " [GC|R] when GC < 128 -> [GC|R];\n" " [GC|Str] -> [decompose_compat(GC)|Str];\n" " [] -> [];\n" " {error,_}=Error -> Error\n end.\n\n" @@ -260,7 +260,7 @@ gen_norm(Fd) -> "-spec nfc(unicode:chardata()) -> maybe_improper_list(gc(),unicode:chardata()) | {error, unicode:chardata()}.\n" "nfc(Str0) ->\n" " case gc(Str0) of\n" - " [GC|R] when GC < 255 -> [GC|R];\n" + " [GC|R] when GC < 256 -> [GC|R];\n" " [GC|Str] -> [compose(decompose(GC))|Str];\n" " [] -> [];\n" " {error,_}=Error -> Error\n end.\n\n" @@ -270,7 +270,7 @@ gen_norm(Fd) -> "-spec nfkc(unicode:chardata()) -> maybe_improper_list(gc(),unicode:chardata()) | {error, unicode:chardata()}.\n" "nfkc(Str0) ->\n" " case gc(Str0) of\n" - " [GC|R] when GC < 127 -> [GC|R];\n" + " [GC|R] when GC < 128 -> [GC|R];\n" " [GC|Str] -> [compose_compat_0(decompose_compat(GC))|Str];\n" " [] -> [];\n" " {error,_}=Error -> Error\n end.\n\n" @@ -476,13 +476,30 @@ gen_gc(Fd, GBP) -> "-spec gc(String::unicode:chardata()) ->" " maybe_improper_list() | {error, unicode:chardata()}.\n"), io:put_chars(Fd, + "gc([CP1, CP2|_]=T)\n" + " when CP1 < 256, CP2 < 256, CP1 =/= $\r -> %% Ascii Fast path\n" + " T;\n" + "gc(<<CP1/utf8, Rest/binary>>) ->\n" + " if CP1 < 256, CP1 =/= $\r ->\n" + " case Rest of\n" + " <<CP2/utf8, _/binary>> when CP2 < 256 -> %% Ascii Fast path\n" + " [CP1|Rest];\n" + " _ -> gc_1([CP1|Rest])\n" + " end;\n" + " true -> gc_1([CP1|Rest])\n" + " end;\n" "gc(Str) ->\n" " gc_1(cp(Str)).\n\n" "gc_1([$\\r|R0] = R) ->\n" " case cp(R0) of % Don't break CRLF\n" " [$\\n|R1] -> [[$\\r,$\\n]|R1];\n" " _ -> R\n" - " end;\n"), + " end;\n" + %% "gc_1([CP1, CP2|_]=T) when CP1 < 256, CP2 < 256 ->\n" + %% " T; %% Fast path\n" + %% "gc_1([CP1|<<CP2/utf8, _/binary>>]=T) when CP1 < 256, CP2 < 256 ->\n" + %% " T; %% Fast path\n" + ), io:put_chars(Fd, "%% Handle control\n"), GenControl = fun(Range) -> io:format(Fd, "gc_1~s R0;\n", [gen_clause(Range)]) end, @@ -490,7 +507,7 @@ gen_gc(Fd, GBP) -> [R1,R2,R3|Crs] = CRs0, [GenControl(CP) || CP <- merge_ranges([R1,R2,R3], split), CP =/= {$\r, undefined}], %%GenControl(R1),GenControl(R2),GenControl(R3), - io:format(Fd, "gc_1([CP|R]) when CP < 255 -> gc_extend(R,CP);\n", []), + io:format(Fd, "gc_1([CP|R]) when CP < 256 -> gc_extend(R,CP);\n", []), [GenControl(CP) || CP <- Crs], %% One clause per CP %% CRs0 = merge_ranges(maps:get(cr, GBP) ++ maps:get(lf, GBP) ++ maps:get(control, GBP)), diff --git a/lib/stdlib/vsn.mk b/lib/stdlib/vsn.mk index 48db5dc900..69d258c2f0 100644 --- a/lib/stdlib/vsn.mk +++ b/lib/stdlib/vsn.mk @@ -1 +1 @@ -STDLIB_VSN = 3.4.2 +STDLIB_VSN = 3.4.3 diff --git a/lib/syntax_tools/doc/src/notes.xml b/lib/syntax_tools/doc/src/notes.xml index 8c91f01e3b..bd2bcde2c2 100644 --- a/lib/syntax_tools/doc/src/notes.xml +++ b/lib/syntax_tools/doc/src/notes.xml @@ -32,6 +32,21 @@ <p>This document describes the changes made to the Syntax_Tools application.</p> +<section><title>Syntax_Tools 2.1.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + <section><title>Syntax_Tools 2.1.3</title> <section><title>Improvements and New Features</title> diff --git a/lib/syntax_tools/src/erl_prettypr.erl b/lib/syntax_tools/src/erl_prettypr.erl index 40ddd2b22a..f03f326278 100644 --- a/lib/syntax_tools/src/erl_prettypr.erl +++ b/lib/syntax_tools/src/erl_prettypr.erl @@ -774,9 +774,17 @@ lay_2(Node, Ctxt) -> class_qualifier -> Ctxt1 = set_prec(Ctxt, max_prec()), D1 = lay(erl_syntax:class_qualifier_argument(Node), Ctxt1), - D2 = lay(erl_syntax:class_qualifier_body(Node), Ctxt1), - beside(D1, beside(text(":"), D2)); - + D2 = lay(erl_syntax:class_qualifier_body(Node), Ctxt1), + Stacktrace = erl_syntax:class_qualifier_stacktrace(Node), + case erl_syntax:variable_name(Stacktrace) of + '_' -> + beside(D1, beside(text(":"), D2)); + _ -> + D3 = lay(erl_syntax:class_qualifier_stacktrace(Node), + Ctxt1), + beside(D1, beside(beside(text(":"), D2), + beside(text(":"), D3))) + end; comment -> D = stack_comment_lines( erl_syntax:comment_text(Node)), diff --git a/lib/syntax_tools/src/erl_syntax.erl b/lib/syntax_tools/src/erl_syntax.erl index 9b2b503762..ed552b73df 100644 --- a/lib/syntax_tools/src/erl_syntax.erl +++ b/lib/syntax_tools/src/erl_syntax.erl @@ -342,8 +342,10 @@ typed_record_field_body/1, typed_record_field_type/1, class_qualifier/2, + class_qualifier/3, class_qualifier_argument/1, class_qualifier_body/1, + class_qualifier_stacktrace/1, tuple/1, tuple_elements/1, tuple_size/1, @@ -3884,7 +3886,7 @@ fold_try_clause({clause, Pos, [P], Guard, Body}) -> class_qualifier -> {tuple, Pos, [class_qualifier_argument(P), class_qualifier_body(P), - {var, Pos, '_'}]}; + class_qualifier_stacktrace(P)]}; _ -> {tuple, Pos, [{atom, Pos, throw}, P, {var, Pos, '_'}]} end, @@ -3896,9 +3898,9 @@ unfold_try_clauses(Cs) -> unfold_try_clause({clause, Pos, [{tuple, _, [{atom, _, throw}, V, _]}], Guard, Body}) -> {clause, Pos, [V], Guard, Body}; -unfold_try_clause({clause, Pos, [{tuple, _, [C, V, _]}], +unfold_try_clause({clause, Pos, [{tuple, _, [C, V, Stacktrace]}], Guard, Body}) -> - {clause, Pos, [class_qualifier(C, V)], Guard, Body}. + {clause, Pos, [class_qualifier(C, V, Stacktrace)], Guard, Body}. %% ===================================================================== @@ -6725,9 +6727,12 @@ try_expr_after(Node) -> %% %% @see class_qualifier_argument/1 %% @see class_qualifier_body/1 +%% @see class_qualifier_stacktrace/1 %% @see try_expr/4 --record(class_qualifier, {class :: syntaxTree(), body :: syntaxTree()}). +-record(class_qualifier, {class :: syntaxTree(), + body :: syntaxTree(), + stacktrace :: syntaxTree()}). %% type(Node) = class_qualifier %% data(Node) = #class_qualifier{class :: Class, body :: Body} @@ -6737,8 +6742,27 @@ try_expr_after(Node) -> -spec class_qualifier(syntaxTree(), syntaxTree()) -> syntaxTree(). class_qualifier(Class, Body) -> + Underscore = {var, get_pos(Body), '_'}, tree(class_qualifier, - #class_qualifier{class = Class, body = Body}). + #class_qualifier{class = Class, body = Body, + stacktrace = Underscore}). + +%% ===================================================================== +%% @doc Creates an abstract class qualifier. The result represents +%% "<code><em>Class</em>:<em>Body</em>:<em>Stacktrace</em></code>". +%% +%% @see class_qualifier_argument/1 +%% @see class_qualifier_body/1 +%% @see try_expr/4 + +-spec class_qualifier(syntaxTree(), syntaxTree(), syntaxTree()) -> + syntaxTree(). + +class_qualifier(Class, Body, Stacktrace) -> + tree(class_qualifier, + #class_qualifier{class = Class, + body = Body, + stacktrace = Stacktrace}). %% ===================================================================== @@ -6763,6 +6787,16 @@ class_qualifier_argument(Node) -> class_qualifier_body(Node) -> (data(Node))#class_qualifier.body. +%% ===================================================================== +%% @doc Returns the stacktrace subtree of a `class_qualifier' node. +%% +%% @see class_qualifier/2 + +-spec class_qualifier_stacktrace(syntaxTree()) -> syntaxTree(). + +class_qualifier_stacktrace(Node) -> + (data(Node))#class_qualifier.stacktrace. + %% ===================================================================== %% @doc Creates an abstract "implicit fun" expression. If diff --git a/lib/syntax_tools/vsn.mk b/lib/syntax_tools/vsn.mk index e0880d61ee..8d37c40742 100644 --- a/lib/syntax_tools/vsn.mk +++ b/lib/syntax_tools/vsn.mk @@ -1 +1 @@ -SYNTAX_TOOLS_VSN = 2.1.3 +SYNTAX_TOOLS_VSN = 2.1.4 diff --git a/lib/tools/doc/src/notes.xml b/lib/tools/doc/src/notes.xml index 3eaa2058a0..1edc08c9cd 100644 --- a/lib/tools/doc/src/notes.xml +++ b/lib/tools/doc/src/notes.xml @@ -31,6 +31,21 @@ </header> <p>This document describes the changes made to the Tools application.</p> +<section><title>Tools 2.11.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + <section><title>Tools 2.11</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/tools/emacs/erlang.el b/lib/tools/emacs/erlang.el index d9efadf64a..6b93d63182 100644 --- a/lib/tools/emacs/erlang.el +++ b/lib/tools/emacs/erlang.el @@ -885,7 +885,6 @@ resulting regexp is surrounded by \\_< and \\_>." "append" "append_element" "await_proc_exit" - "await_sched_wall_time_modifications" "bump_reductions" "call_on_load_function" "cancel_timer" @@ -923,7 +922,6 @@ resulting regexp is surrounded by \\_< and \\_>." "function_exported" "garbage_collect_message_area" "gather_gc_info_result" - "gather_sched_wall_time_result" "get_cookie" "get_module_info" "get_stacktrace" diff --git a/lib/tools/test/fprof_SUITE.erl b/lib/tools/test/fprof_SUITE.erl index 8fd164a4b3..ae0e7253ad 100644 --- a/lib/tools/test/fprof_SUITE.erl +++ b/lib/tools/test/fprof_SUITE.erl @@ -51,7 +51,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}, - {timetrap,{seconds,60}}]. + {timetrap,{seconds,240}}]. all() -> case test_server:is_native(fprof_SUITE) of @@ -571,7 +571,7 @@ seq_r(Start, Stop, Succ, R) -> create_file_slow(Name, N) when is_integer(N), N >= 0 -> {ok, FD} = - file:open(Name, [raw, write, delayed_write, binary]), + file:open(Name, [raw, write, binary]), if N > 256 -> ok = file:write(FD, lists:map(fun (X) -> <<X:32/unsigned>> end, diff --git a/lib/tools/vsn.mk b/lib/tools/vsn.mk index b9249ae45c..6cafbca6a7 100644 --- a/lib/tools/vsn.mk +++ b/lib/tools/vsn.mk @@ -1 +1 @@ -TOOLS_VSN = 2.11 +TOOLS_VSN = 2.11.1 diff --git a/lib/wx/doc/src/notes.xml b/lib/wx/doc/src/notes.xml index 599b5b64fd..69ea906ec0 100644 --- a/lib/wx/doc/src/notes.xml +++ b/lib/wx/doc/src/notes.xml @@ -32,6 +32,22 @@ <p>This document describes the changes made to the wxErlang application.</p> +<section><title>Wx 1.8.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + wx crashes in otp 20.1 if empty binaries was sent down as + arguments.</p> + <p> + Own Id: OTP-14688</p> + </item> + </list> + </section> + +</section> + <section><title>Wx 1.8.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/wx/vsn.mk b/lib/wx/vsn.mk index 039fae322e..7da4529c98 100644 --- a/lib/wx/vsn.mk +++ b/lib/wx/vsn.mk @@ -1 +1 @@ -WX_VSN = 1.8.2 +WX_VSN = 1.8.3 diff --git a/lib/xmerl/doc/src/notes.xml b/lib/xmerl/doc/src/notes.xml index 1162561225..f62a8dc53d 100644 --- a/lib/xmerl/doc/src/notes.xml +++ b/lib/xmerl/doc/src/notes.xml @@ -32,6 +32,21 @@ <p>This document describes the changes made to the Xmerl application.</p> +<section><title>Xmerl 1.3.16</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Removed all old unused files in the documentation. + </p> + <p> + Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> + </item> + </list> + </section> + +</section> + <section><title>Xmerl 1.3.15</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/xmerl/vsn.mk b/lib/xmerl/vsn.mk index 2e9c9061d9..ddff0c8894 100644 --- a/lib/xmerl/vsn.mk +++ b/lib/xmerl/vsn.mk @@ -1 +1 @@ -XMERL_VSN = 1.3.15 +XMERL_VSN = 1.3.16 |